{ "best_global_step": 350, "best_metric": 0.6084344156610773, "best_model_checkpoint": "/jupyter_kernel/trained_models/lg_nss_tune_whisper_large_v1.0/checkpoint-350", "epoch": 1.8918918918918919, "eval_steps": 50, "global_step": 350, "is_hyper_param_search": false, "is_local_process_zero": true, "is_world_process_zero": true, "log_history": [ { "epoch": 0, "eval_cer": 0.4712086318852988, "eval_loss": 1.8530521392822266, "eval_runtime": 883.0991, "eval_samples_per_second": 0.848, "eval_steps_per_second": 0.053, "eval_wer": 0.8136203891549783, "num_input_tokens_seen": 0, "step": 0 }, { "epoch": 0.02702702702702703, "grad_norm": 6.694199562072754, "learning_rate": 5e-06, "loss": 1.5511, "num_input_tokens_seen": 61440000, "step": 5, "train_runtime": 934.0126, "train_tokens_per_second": 65780.696 }, { "epoch": 0.05405405405405406, "grad_norm": 5.178955078125, "learning_rate": 1.125e-05, "loss": 1.3507, "num_input_tokens_seen": 122880000, "step": 10, "train_runtime": 978.6837, "train_tokens_per_second": 125556.396 }, { "epoch": 0.08108108108108109, "grad_norm": 4.756581783294678, "learning_rate": 1.75e-05, "loss": 1.179, "num_input_tokens_seen": 184320000, "step": 15, "train_runtime": 1023.409, "train_tokens_per_second": 180103.944 }, { "epoch": 0.10810810810810811, "grad_norm": 4.099735260009766, "learning_rate": 2.375e-05, "loss": 1.1463, "num_input_tokens_seen": 245760000, "step": 20, "train_runtime": 1068.6171, "train_tokens_per_second": 229979.476 }, { "epoch": 0.13513513513513514, "grad_norm": 4.195594310760498, "learning_rate": 3e-05, "loss": 1.0876, "num_input_tokens_seen": 307200000, "step": 25, "train_runtime": 1113.4632, "train_tokens_per_second": 275895.959 }, { "epoch": 0.16216216216216217, "grad_norm": 4.167407035827637, "learning_rate": 3.625e-05, "loss": 1.193, "num_input_tokens_seen": 368640000, "step": 30, "train_runtime": 1158.068, "train_tokens_per_second": 318323.272 }, { "epoch": 0.1891891891891892, "grad_norm": 3.8050270080566406, "learning_rate": 4.25e-05, "loss": 1.1076, "num_input_tokens_seen": 430080000, "step": 35, "train_runtime": 1203.1492, "train_tokens_per_second": 357461.888 }, { "epoch": 0.21621621621621623, "grad_norm": 3.8383731842041016, "learning_rate": 4.875e-05, "loss": 1.0734, "num_input_tokens_seen": 491520000, "step": 40, "train_runtime": 1248.4355, "train_tokens_per_second": 393708.774 }, { "epoch": 0.24324324324324326, "grad_norm": 3.918095350265503, "learning_rate": 4.9172026175368326e-05, "loss": 0.9229, "num_input_tokens_seen": 552960000, "step": 45, "train_runtime": 1293.5005, "train_tokens_per_second": 427491.146 }, { "epoch": 0.2702702702702703, "grad_norm": 3.4553701877593994, "learning_rate": 4.815157253826046e-05, "loss": 1.1542, "num_input_tokens_seen": 614400000, "step": 50, "train_runtime": 1338.8054, "train_tokens_per_second": 458916.573 }, { "epoch": 0.2702702702702703, "eval_cer": 0.4179175096256484, "eval_loss": 1.127152442932129, "eval_runtime": 873.5485, "eval_samples_per_second": 0.857, "eval_steps_per_second": 0.054, "eval_wer": 0.7243194877471738, "num_input_tokens_seen": 614400000, "step": 50 }, { "epoch": 0.2972972972972973, "grad_norm": 4.08140230178833, "learning_rate": 4.714708807666149e-05, "loss": 1.0003, "num_input_tokens_seen": 675840000, "step": 55, "train_runtime": 2304.053, "train_tokens_per_second": 293326.581 }, { "epoch": 0.32432432432432434, "grad_norm": 3.4815714359283447, "learning_rate": 4.615840531294598e-05, "loss": 0.9768, "num_input_tokens_seen": 737280000, "step": 60, "train_runtime": 2348.4672, "train_tokens_per_second": 313940.941 }, { "epoch": 0.35135135135135137, "grad_norm": 3.143886089324951, "learning_rate": 4.5185357652343744e-05, "loss": 0.921, "num_input_tokens_seen": 798720000, "step": 65, "train_runtime": 2393.166, "train_tokens_per_second": 333750.358 }, { "epoch": 0.3783783783783784, "grad_norm": 3.148380756378174, "learning_rate": 4.422777938293975e-05, "loss": 0.9031, "num_input_tokens_seen": 860160000, "step": 70, "train_runtime": 2437.7177, "train_tokens_per_second": 352854.636 }, { "epoch": 0.40540540540540543, "grad_norm": 3.1794612407684326, "learning_rate": 4.328550567567407e-05, "loss": 0.9638, "num_input_tokens_seen": 921600000, "step": 75, "train_runtime": 2481.9555, "train_tokens_per_second": 371320.115 }, { "epoch": 0.43243243243243246, "grad_norm": 3.83333158493042, "learning_rate": 4.2358372584342004e-05, "loss": 0.9354, "num_input_tokens_seen": 983040000, "step": 80, "train_runtime": 2526.153, "train_tokens_per_second": 389145.073 }, { "epoch": 0.4594594594594595, "grad_norm": 3.1049253940582275, "learning_rate": 4.144621704559403e-05, "loss": 0.9552, "num_input_tokens_seen": 1044480000, "step": 85, "train_runtime": 2570.4868, "train_tokens_per_second": 406335.477 }, { "epoch": 0.4864864864864865, "grad_norm": 3.3949313163757324, "learning_rate": 4.054887687893573e-05, "loss": 0.8676, "num_input_tokens_seen": 1105920000, "step": 90, "train_runtime": 2614.6645, "train_tokens_per_second": 422968.222 }, { "epoch": 0.5135135135135135, "grad_norm": 3.3787624835968018, "learning_rate": 3.9666190786727904e-05, "loss": 0.8933, "num_input_tokens_seen": 1167360000, "step": 95, "train_runtime": 2658.827, "train_tokens_per_second": 439050.749 }, { "epoch": 0.5405405405405406, "grad_norm": 3.108181953430176, "learning_rate": 3.879799835418654e-05, "loss": 0.854, "num_input_tokens_seen": 1228800000, "step": 100, "train_runtime": 2702.8667, "train_tokens_per_second": 454628.419 }, { "epoch": 0.5405405405405406, "eval_cer": 0.29776486627376264, "eval_loss": 0.9693590402603149, "eval_runtime": 853.632, "eval_samples_per_second": 0.877, "eval_steps_per_second": 0.055, "eval_wer": 0.6468414575552031, "num_input_tokens_seen": 1228800000, "step": 100 }, { "epoch": 0.5675675675675675, "grad_norm": 3.0080583095550537, "learning_rate": 3.794414004938272e-05, "loss": 0.9135, "num_input_tokens_seen": 1290240000, "step": 105, "train_runtime": 3647.5305, "train_tokens_per_second": 353729.737 }, { "epoch": 0.5945945945945946, "grad_norm": 3.331343650817871, "learning_rate": 3.7104457223242756e-05, "loss": 0.9881, "num_input_tokens_seen": 1351680000, "step": 110, "train_runtime": 3692.651, "train_tokens_per_second": 366045.961 }, { "epoch": 0.6216216216216216, "grad_norm": 4.435062885284424, "learning_rate": 3.627879210954812e-05, "loss": 0.9993, "num_input_tokens_seen": 1413120000, "step": 115, "train_runtime": 3737.2167, "train_tokens_per_second": 378120.973 }, { "epoch": 0.6486486486486487, "grad_norm": 3.606947422027588, "learning_rate": 3.546698782493543e-05, "loss": 1.0589, "num_input_tokens_seen": 1474560000, "step": 120, "train_runtime": 3782.5686, "train_tokens_per_second": 389830.343 }, { "epoch": 0.6756756756756757, "grad_norm": 2.7611401081085205, "learning_rate": 3.4668888368896486e-05, "loss": 0.7697, "num_input_tokens_seen": 1536000000, "step": 125, "train_runtime": 3829.5601, "train_tokens_per_second": 401090.456 }, { "epoch": 0.7027027027027027, "grad_norm": 2.7181291580200195, "learning_rate": 3.3884338623778245e-05, "loss": 0.8414, "num_input_tokens_seen": 1597440000, "step": 130, "train_runtime": 3875.7508, "train_tokens_per_second": 412162.72 }, { "epoch": 0.7297297297297297, "grad_norm": 3.344625234603882, "learning_rate": 3.311318435478286e-05, "loss": 0.9034, "num_input_tokens_seen": 1658880000, "step": 135, "train_runtime": 3920.4138, "train_tokens_per_second": 423139.005 }, { "epoch": 0.7567567567567568, "grad_norm": 2.8509151935577393, "learning_rate": 3.235527220996761e-05, "loss": 0.7987, "num_input_tokens_seen": 1720320000, "step": 140, "train_runtime": 3965.1338, "train_tokens_per_second": 433861.779 }, { "epoch": 0.7837837837837838, "grad_norm": 2.9075114727020264, "learning_rate": 3.1610449720244975e-05, "loss": 0.8147, "num_input_tokens_seen": 1781760000, "step": 145, "train_runtime": 4008.9754, "train_tokens_per_second": 444442.733 }, { "epoch": 0.8108108108108109, "grad_norm": 2.9646787643432617, "learning_rate": 3.087856529938261e-05, "loss": 0.855, "num_input_tokens_seen": 1843200000, "step": 150, "train_runtime": 4052.7288, "train_tokens_per_second": 454804.669 }, { "epoch": 0.8108108108108109, "eval_cer": 0.3384151074503347, "eval_loss": 0.9674391150474548, "eval_runtime": 871.3397, "eval_samples_per_second": 0.86, "eval_steps_per_second": 0.054, "eval_wer": 0.6576210232547248, "num_input_tokens_seen": 1843200000, "step": 150 }, { "epoch": 0.8378378378378378, "grad_norm": 3.101228952407837, "learning_rate": 3.015946824400329e-05, "loss": 0.8295, "num_input_tokens_seen": 1904640000, "step": 155, "train_runtime": 5021.2234, "train_tokens_per_second": 379317.914 }, { "epoch": 0.8648648648648649, "grad_norm": 2.8363142013549805, "learning_rate": 2.945300873358501e-05, "loss": 0.664, "num_input_tokens_seen": 1966080000, "step": 160, "train_runtime": 5066.1637, "train_tokens_per_second": 388080.63 }, { "epoch": 0.8918918918918919, "grad_norm": 2.5507912635803223, "learning_rate": 2.8759037830460915e-05, "loss": 0.9147, "num_input_tokens_seen": 2027520000, "step": 165, "train_runtime": 5110.4979, "train_tokens_per_second": 396736.298 }, { "epoch": 0.918918918918919, "grad_norm": 3.1494266986846924, "learning_rate": 2.8077407479819295e-05, "loss": 0.8447, "num_input_tokens_seen": 2088960000, "step": 170, "train_runtime": 5156.8332, "train_tokens_per_second": 405085.818 }, { "epoch": 0.9459459459459459, "grad_norm": 2.359222650527954, "learning_rate": 2.740797050970364e-05, "loss": 0.766, "num_input_tokens_seen": 2150400000, "step": 175, "train_runtime": 5200.7084, "train_tokens_per_second": 413482.134 }, { "epoch": 0.972972972972973, "grad_norm": 2.9903037548065186, "learning_rate": 2.6750580631012613e-05, "loss": 0.7747, "num_input_tokens_seen": 2211840000, "step": 180, "train_runtime": 5244.7543, "train_tokens_per_second": 421724.24 }, { "epoch": 1.0, "grad_norm": 9.7564115524292, "learning_rate": 2.61050924375e-05, "loss": 0.87, "num_input_tokens_seen": 2262912000, "step": 185, "train_runtime": 5275.773, "train_tokens_per_second": 428925.202 }, { "epoch": 1.027027027027027, "grad_norm": 1.9875379800796509, "learning_rate": 2.5471361405774797e-05, "loss": 0.4915, "num_input_tokens_seen": 2324352000, "step": 190, "train_runtime": 5325.1726, "train_tokens_per_second": 436483.88 }, { "epoch": 1.054054054054054, "grad_norm": 2.469710111618042, "learning_rate": 2.4849243895301164e-05, "loss": 0.515, "num_input_tokens_seen": 2385792000, "step": 195, "train_runtime": 5368.58, "train_tokens_per_second": 444399.081 }, { "epoch": 1.0810810810810811, "grad_norm": 1.9504098892211914, "learning_rate": 2.4238597148398397e-05, "loss": 0.4768, "num_input_tokens_seen": 2447232000, "step": 200, "train_runtime": 5411.6359, "train_tokens_per_second": 452216.674 }, { "epoch": 1.0810810810810811, "eval_cer": 0.338837217731344, "eval_loss": 0.943603515625, "eval_runtime": 846.9243, "eval_samples_per_second": 0.884, "eval_steps_per_second": 0.055, "eval_wer": 0.6542248808655325, "num_input_tokens_seen": 2447232000, "step": 200 }, { "epoch": 1.1081081081081081, "grad_norm": 2.1606149673461914, "learning_rate": 2.3639279290241e-05, "loss": 0.5108, "num_input_tokens_seen": 2508672000, "step": 205, "train_runtime": 6355.0036, "train_tokens_per_second": 394755.402 }, { "epoch": 1.135135135135135, "grad_norm": 2.4380221366882324, "learning_rate": 2.305114932885864e-05, "loss": 0.5376, "num_input_tokens_seen": 2570112000, "step": 210, "train_runtime": 6399.9936, "train_tokens_per_second": 401580.404 }, { "epoch": 1.1621621621621623, "grad_norm": 2.1753928661346436, "learning_rate": 2.2474067155136106e-05, "loss": 0.4848, "num_input_tokens_seen": 2631552000, "step": 215, "train_runtime": 6444.5402, "train_tokens_per_second": 408338.21 }, { "epoch": 1.1891891891891893, "grad_norm": 2.556577205657959, "learning_rate": 2.1907893542813412e-05, "loss": 0.5193, "num_input_tokens_seen": 2692992000, "step": 220, "train_runtime": 6489.0434, "train_tokens_per_second": 415006.011 }, { "epoch": 1.2162162162162162, "grad_norm": 2.411965847015381, "learning_rate": 2.1352490148485728e-05, "loss": 0.5359, "num_input_tokens_seen": 2754432000, "step": 225, "train_runtime": 6532.5362, "train_tokens_per_second": 421648.178 }, { "epoch": 1.2432432432432432, "grad_norm": 2.55607271194458, "learning_rate": 2.0807719511603353e-05, "loss": 0.5095, "num_input_tokens_seen": 2815872000, "step": 230, "train_runtime": 6575.6388, "train_tokens_per_second": 428227.898 }, { "epoch": 1.2702702702702702, "grad_norm": 2.326690435409546, "learning_rate": 2.0273445054471802e-05, "loss": 0.5578, "num_input_tokens_seen": 2877312000, "step": 235, "train_runtime": 6618.6865, "train_tokens_per_second": 434725.527 }, { "epoch": 1.2972972972972974, "grad_norm": 2.4669768810272217, "learning_rate": 1.974953108225174e-05, "loss": 0.5108, "num_input_tokens_seen": 2938752000, "step": 240, "train_runtime": 6662.1141, "train_tokens_per_second": 441114.029 }, { "epoch": 1.3243243243243243, "grad_norm": 2.3330671787261963, "learning_rate": 1.9235842782958985e-05, "loss": 0.5165, "num_input_tokens_seen": 3000192000, "step": 245, "train_runtime": 6705.2682, "train_tokens_per_second": 447438.033 }, { "epoch": 1.3513513513513513, "grad_norm": 2.568614959716797, "learning_rate": 1.8732246227464544e-05, "loss": 0.5197, "num_input_tokens_seen": 3061632000, "step": 250, "train_runtime": 6748.5735, "train_tokens_per_second": 453670.985 }, { "epoch": 1.3513513513513513, "eval_cer": 0.329565296789743, "eval_loss": 0.9339933395385742, "eval_runtime": 837.5193, "eval_samples_per_second": 0.894, "eval_steps_per_second": 0.056, "eval_wer": 0.6662925860618296, "num_input_tokens_seen": 3061632000, "step": 250 }, { "epoch": 1.3783783783783785, "grad_norm": 2.659301519393921, "learning_rate": 1.8238608369494596e-05, "loss": 0.5545, "num_input_tokens_seen": 3123072000, "step": 255, "train_runtime": 7674.6612, "train_tokens_per_second": 406932.882 }, { "epoch": 1.4054054054054055, "grad_norm": 2.174906015396118, "learning_rate": 1.7754797045630454e-05, "loss": 0.5572, "num_input_tokens_seen": 3184512000, "step": 260, "train_runtime": 7718.6601, "train_tokens_per_second": 412573.163 }, { "epoch": 1.4324324324324325, "grad_norm": 2.2907702922821045, "learning_rate": 1.728068097530864e-05, "loss": 0.4729, "num_input_tokens_seen": 3245952000, "step": 265, "train_runtime": 7761.7975, "train_tokens_per_second": 418195.915 }, { "epoch": 1.4594594594594594, "grad_norm": 2.1810221672058105, "learning_rate": 1.681612976082083e-05, "loss": 0.4723, "num_input_tokens_seen": 3307392000, "step": 270, "train_runtime": 7804.7904, "train_tokens_per_second": 423764.359 }, { "epoch": 1.4864864864864864, "grad_norm": 2.441494941711426, "learning_rate": 1.6361013887313843e-05, "loss": 0.5911, "num_input_tokens_seen": 3368832000, "step": 275, "train_runtime": 7848.1781, "train_tokens_per_second": 429250.199 }, { "epoch": 1.5135135135135136, "grad_norm": 2.4569575786590576, "learning_rate": 1.5915204722789705e-05, "loss": 0.524, "num_input_tokens_seen": 3430272000, "step": 280, "train_runtime": 7891.6157, "train_tokens_per_second": 434672.968 }, { "epoch": 1.5405405405405406, "grad_norm": 2.1347854137420654, "learning_rate": 1.547857451810559e-05, "loss": 0.4959, "num_input_tokens_seen": 3491712000, "step": 285, "train_runtime": 7934.8481, "train_tokens_per_second": 440047.746 }, { "epoch": 1.5675675675675675, "grad_norm": 2.2399299144744873, "learning_rate": 1.5050996406973835e-05, "loss": 0.5164, "num_input_tokens_seen": 3553152000, "step": 290, "train_runtime": 7979.2749, "train_tokens_per_second": 445297.605 }, { "epoch": 1.5945945945945947, "grad_norm": 2.3631114959716797, "learning_rate": 1.4632344405961962e-05, "loss": 0.5464, "num_input_tokens_seen": 3614592000, "step": 295, "train_runtime": 8023.6934, "train_tokens_per_second": 450489.798 }, { "epoch": 1.6216216216216215, "grad_norm": 1.9993412494659424, "learning_rate": 1.4222493414492655e-05, "loss": 0.5372, "num_input_tokens_seen": 3676032000, "step": 300, "train_runtime": 8067.0957, "train_tokens_per_second": 455682.209 }, { "epoch": 1.6216216216216215, "eval_cer": 0.3121871628496099, "eval_loss": 0.8871492147445679, "eval_runtime": 865.3356, "eval_samples_per_second": 0.866, "eval_steps_per_second": 0.054, "eval_wer": 0.6394179350835714, "num_input_tokens_seen": 3676032000, "step": 300 }, { "epoch": 1.6486486486486487, "grad_norm": 2.2491278648376465, "learning_rate": 1.382131921484375e-05, "loss": 0.4566, "num_input_tokens_seen": 3737472000, "step": 305, "train_runtime": 9022.1425, "train_tokens_per_second": 414255.482 }, { "epoch": 1.6756756756756757, "grad_norm": 2.2252557277679443, "learning_rate": 1.3428698472148269e-05, "loss": 0.4409, "num_input_tokens_seen": 3798912000, "step": 310, "train_runtime": 9065.3762, "train_tokens_per_second": 419057.292 }, { "epoch": 1.7027027027027026, "grad_norm": 2.4980313777923584, "learning_rate": 1.3044508734394403e-05, "loss": 0.4633, "num_input_tokens_seen": 3860352000, "step": 315, "train_runtime": 9108.6721, "train_tokens_per_second": 423810.621 }, { "epoch": 1.7297297297297298, "grad_norm": 2.2038354873657227, "learning_rate": 1.26686284324255e-05, "loss": 0.4169, "num_input_tokens_seen": 3921792000, "step": 320, "train_runtime": 9151.5702, "train_tokens_per_second": 428537.607 }, { "epoch": 1.7567567567567568, "grad_norm": 2.1892924308776855, "learning_rate": 1.2300936879940077e-05, "loss": 0.5566, "num_input_tokens_seen": 3983232000, "step": 325, "train_runtime": 9194.743, "train_tokens_per_second": 433207.539 }, { "epoch": 1.7837837837837838, "grad_norm": 2.390801429748535, "learning_rate": 1.1941314273491836e-05, "loss": 0.5153, "num_input_tokens_seen": 4044672000, "step": 330, "train_runtime": 9237.5451, "train_tokens_per_second": 437851.393 }, { "epoch": 1.810810810810811, "grad_norm": 2.390781879425049, "learning_rate": 1.1589641692489621e-05, "loss": 0.5916, "num_input_tokens_seen": 4106112000, "step": 335, "train_runtime": 9280.5769, "train_tokens_per_second": 442441.459 }, { "epoch": 1.8378378378378377, "grad_norm": 2.5694046020507812, "learning_rate": 1.1245801099197466e-05, "loss": 0.4809, "num_input_tokens_seen": 4167552000, "step": 340, "train_runtime": 9323.404, "train_tokens_per_second": 446998.971 }, { "epoch": 1.864864864864865, "grad_norm": 2.1600019931793213, "learning_rate": 1.0909675338734569e-05, "loss": 0.4057, "num_input_tokens_seen": 4228992000, "step": 345, "train_runtime": 9367.0963, "train_tokens_per_second": 451473.097 }, { "epoch": 1.8918918918918919, "grad_norm": 2.2605528831481934, "learning_rate": 1.0581148139075277e-05, "loss": 0.4403, "num_input_tokens_seen": 4290432000, "step": 350, "train_runtime": 9410.0927, "train_tokens_per_second": 455939.398 }, { "epoch": 1.8918918918918919, "eval_cer": 0.27202566552379415, "eval_loss": 0.8798862099647522, "eval_runtime": 816.6633, "eval_samples_per_second": 0.917, "eval_steps_per_second": 0.058, "eval_wer": 0.6084344156610773, "num_input_tokens_seen": 4290432000, "step": 350 } ], "logging_steps": 5, "max_steps": 1000, "num_input_tokens_seen": 4290432000, "num_train_epochs": 6, "save_steps": 50, "stateful_callbacks": { "TrainerControl": { "args": { "should_epoch_stop": false, "should_evaluate": false, "should_log": false, "should_save": true, "should_training_stop": false }, "attributes": {} } }, "total_flos": 3.796024857919488e+19, "train_batch_size": 32, "trial_name": null, "trial_params": null }