{"Time": "2026-07-23T16:52:44.456541", "Step": 500, "CTC Loss": 1449.5287173996758, "Gradient Norm": 4569.6753521728515, "Data Epoch": 1, "Data Time": 0.31848214752972126, "Compute Time": 719.0323553364724, "Lapse Time": 719.6896149590611, "Total Time": 722.9267744719982, "Wall Time": 722.9267744719982, "Learning Rate": 5.0401e-06, "Batch Size": 9.0, "Elements per Batch": 3033564.0, "Elements per Second": 2108543.1446845746, "Number of Examples": 4850.0, "Number of Elements": 1516782277.0, "Padding": 82082805.0, "Padding Ratio (%)": 0.051338168507203664, "Total Number of Examples": 4850.0, "Total Number of Elements": 1516782277.0, "Peak Active Device Memory": 9918107136, "Peak Active Device Memory (%)": 0.3922754634249707, "Peak Reserved Device Memory": 11184111616, "Peak Reserved Device Memory (%)": 0.4423477692873954}
{"Time": "2026-07-23T17:06:33.329770", "Step": 1000, "CTC Loss": 234.35882826566942, "Gradient Norm": 2123.9837996826172, "Data Epoch": 1, "Data Time": 0.10984984412789345, "Compute Time": 828.4769413042814, "Lapse Time": 828.8483631741256, "Total Time": 1551.8019425626844, "Wall Time": 1551.8019425626844, "Learning Rate": 9.990100000000001e-06, "Batch Size": 9.0, "Elements per Batch": 3032934.0, "Elements per Second": 1830184.9561205276, "Number of Examples": 4865.0, "Number of Elements": 1516467080.0, "Padding": 85051766.0, "Padding Ratio (%)": 0.05310694045994386, "Total Number of Examples": 9715.0, "Total Number of Elements": 3033249357.0, "Peak Active Device Memory": 9912688128, "Peak Active Device Memory (%)": 0.3920611338310921, "Peak Reserved Device Memory": 11205083136, "Peak Reserved Device Memory (%)": 0.44317722319567854}
{"Time": "2026-07-23T17:20:22.814018", "Step": 1500, "CTC Loss": 95.66566673282698, "Gradient Norm": 960.0641820983886, "Data Epoch": 1, "Data Time": 0.10898501984775066, "Compute Time": 829.1109280511737, "Lapse Time": 829.4608175363392, "Total Time": 2381.285374844447, "Wall Time": 2381.285374844447, "Learning Rate": 1e-05, "Batch Size": 9.0, "Elements per Batch": 3033913.0, "Elements per Second": 1829378.1554062546, "Number of Examples": 4905.0, "Number of Elements": 1516956795.0, "Padding": 84447402.0, "Padding Ratio (%)": 0.05273334624587599, "Total Number of Examples": 14620.0, "Total Number of Elements": 4550206152.0, "Peak Active Device Memory": 9925716480, "Peak Active Device Memory (%)": 0.39257642397147713, "Peak Reserved Device Memory": 11205083136, "Peak Reserved Device Memory (%)": 0.44317722319567854}
{"Time": "2026-07-23T17:34:14.201454", "Step": 2000, "CTC Loss": 77.73986439173774, "Gradient Norm": 1117.46915335083, "Data Epoch": 1, "Data Time": 0.10374995693564415, "Compute Time": 830.9837212190032, "Lapse Time": 831.3636290933937, "Total Time": 3212.673914728686, "Wall Time": 3212.673914728686, "Learning Rate": 1e-05, "Batch Size": 9.0, "Elements per Batch": 3037993.0, "Elements per Second": 1827721.838774336, "Number of Examples": 4884.0, "Number of Elements": 1518996721.0, "Padding": 84838104.0, "Padding Ratio (%)": 0.05289703320913985, "Total Number of Examples": 19504.0, "Total Number of Elements": 6069202873.0, "Peak Active Device Memory": 9933433344, "Peak Active Device Memory (%)": 0.3928816370892907, "Peak Reserved Device Memory": 11205083136, "Peak Reserved Device Memory (%)": 0.44317722319567854}
{"Time": "2026-07-23T17:48:03.899604", "Step": 2500, "CTC Loss": 70.41343987482529, "Gradient Norm": 842.2129847412109, "Data Epoch": 1, "Data Time": 0.1045993510633707, "Compute Time": 827.2489951103926, "Lapse Time": 829.6773332357407, "Total Time": 4042.3741542566568, "Wall Time": 4042.3741542566568, "Learning Rate": 1e-05, "Batch Size": 9.0, "Elements per Batch": 3033347.0, "Elements per Second": 1833162.7881392585, "Number of Examples": 4870.0, "Number of Elements": 1516673822.0, "Padding": 81844744.0, "Padding Ratio (%)": 0.05120037123172206, "Total Number of Examples": 24374.0, "Total Number of Elements": 7585876695.0, "Peak Active Device Memory": 9909505536, "Peak Active Device Memory (%)": 0.39193525772040144, "Peak Reserved Device Memory": 11207180288, "Peak Reserved Device Memory (%)": 0.44326016858650685}
{"Time": "2026-07-23T18:01:51.528837", "Step": 3000, "CTC Loss": 64.9089498174335, "Gradient Norm": 995.1210192565918, "Data Epoch": 1, "Data Time": 0.09955471940338612, "Compute Time": 827.1752322241664, "Lapse Time": 827.6068176273257, "Total Time": 4870.000915052369, "Wall Time": 4870.000915052369, "Learning Rate": 1e-05, "Batch Size": 9.0, "Elements per Batch": 3033484.0, "Elements per Second": 1833420.2769598733, "Number of Examples": 4871.0, "Number of Elements": 1516742369.0, "Padding": 83965817.0, "Padding Ratio (%)": 0.05245541800459062, "Total Number of Examples": 29245.0, "Total Number of Elements": 9102619064.0, "Peak Active Device Memory": 9923111424, "Peak Active Device Memory (%)": 0.3924733902438076, "Peak Reserved Device Memory": 11207180288, "Peak Reserved Device Memory (%)": 0.44326016858650685}
{"Time": "2026-07-23T18:15:41.557543", "Step": 3500, "CTC Loss": 60.05711448299467, "Gradient Norm": 713.1735663604736, "Data Epoch": 1, "Data Time": 0.09969352371990681, "Compute Time": 829.5913811922073, "Lapse Time": 830.0033830087632, "Total Time": 5700.0291149728, "Wall Time": 5700.0291149728, "Learning Rate": 1e-05, "Batch Size": 9.0, "Elements per Batch": 3039252.0, "Elements per Second": 1831556.8460469402, "Number of Examples": 4908.0, "Number of Elements": 1519626368.0, "Padding": 82229186.0, "Padding Ratio (%)": 0.051333708457460575, "Total Number of Examples": 34153.0, "Total Number of Elements": 10622245432.0, "Peak Active Device Memory": 9898014720, "Peak Active Device Memory (%)": 0.39148077934970815, "Peak Reserved Device Memory": 11207180288, "Peak Reserved Device Memory (%)": 0.44326016858650685}
{"Time": "2026-07-23T18:29:32.445628", "Step": 4000, "CTC Loss": 56.98146555985957, "Gradient Norm": 647.6810155944825, "Data Epoch": 1, "Data Time": 0.09878464601933956, "Compute Time": 830.4588724579662, "Lapse Time": 830.8638224117458, "Total Time": 6530.9177429154515, "Wall Time": 6530.9177429154515, "Learning Rate": 1e-05, "Batch Size": 9.0, "Elements per Batch": 3031211.0, "Elements per Second": 1824804.8152185618, "Number of Examples": 4881.0, "Number of Elements": 1515605612.0, "Padding": 83202477.0, "Padding Ratio (%)": 0.052040315265129984, "Total Number of Examples": 39034.0, "Total Number of Elements": 12137851044.0, "Peak Active Device Memory": 9926924288, "Peak Active Device Memory (%)": 0.39262419452249375, "Peak Reserved Device Memory": 11207180288, "Peak Reserved Device Memory (%)": 0.44326016858650685}
{"Time": "2026-07-23T18:43:24.037865", "Step": 4500, "CTC Loss": 54.413365180751214, "Gradient Norm": 648.8265440216064, "Data Epoch": 1, "Data Time": 0.10108165256679058, "Compute Time": 831.1870988756418, "Lapse Time": 831.5684912595898, "Total Time": 7362.508957607672, "Wall Time": 7362.508957607672, "Learning Rate": 1e-05, "Batch Size": 9.0, "Elements per Batch": 3034803.0, "Elements per Second": 1825361.7079408348, "Number of Examples": 4864.0, "Number of Elements": 1517401613.0, "Padding": 82004147.0, "Padding Ratio (%)": 0.051271634159927, "Total Number of Examples": 43898.0, "Total Number of Elements": 13655252657.0, "Peak Active Device Memory": 9923602944, "Peak Active Device Memory (%)": 0.392492830569783, "Peak Reserved Device Memory": 11207180288, "Peak Reserved Device Memory (%)": 0.44326016858650685}
{"Time": "2026-07-23T18:57:16.657933", "Step": 5000, "CTC Loss": 50.69734054773194, "Gradient Norm": 865.9657755432129, "Data Epoch": 1, "Data Time": 0.09662867337465286, "Compute Time": 831.1135243978351, "Lapse Time": 832.5977883245796, "Total Time": 8195.13159125112, "Wall Time": 8195.13159125112, "Learning Rate": 1e-05, "Batch Size": 9.0, "Elements per Batch": 3028284.0, "Elements per Second": 1821611.7264754868, "Number of Examples": 4861.0, "Number of Elements": 1514142162.0, "Padding": 84999929.0, "Padding Ratio (%)": 0.053153456142753734, "Total Number of Examples": 48759.0, "Total Number of Elements": 15169394819.0, "Peak Active Device Memory": 9915226624, "Peak Active Device Memory (%)": 0.39216153501461914, "Peak Reserved Device Memory": 11207180288, "Peak Reserved Device Memory (%)": 0.44326016858650685}
{"Time": "2026-07-23T19:11:08.946316", "Step": 5500, "CTC Loss": 48.776611978526574, "Gradient Norm": 714.8557177124023, "Data Epoch": 1, "Data Time": 0.09455043263733387, "Compute Time": 827.9029018618166, "Lapse Time": 828.3181406036019, "Total Time": 9027.417748311535, "Wall Time": 9027.417748311535, "Learning Rate": 7.415786302360811e-06, "Batch Size": 9.0, "Elements per Batch": 3033064.0, "Elements per Second": 1831566.0414142048, "Number of Examples": 4871.0, "Number of Elements": 1516532016.0, "Padding": 83419835.0, "Padding Ratio (%)": 0.05213896589941818, "Total Number of Examples": 53630.0, "Total Number of Elements": 16685926835.0, "Peak Active Device Memory": 9919478784, "Peak Active Device Memory (%)": 0.3923297140846458, "Peak Reserved Device Memory": 12993953792, "Peak Reserved Device Memory (%)": 0.5139296415722299}
{"Time": "2026-07-23T19:25:01.763584", "Step": 6000, "CTC Loss": 44.38405428349157, "Gradient Norm": 559.9273562469482, "Data Epoch": 1, "Data Time": 0.09500757418572903, "Compute Time": 832.363606126979, "Lapse Time": 832.7951816897839, "Total Time": 9860.235788943246, "Wall Time": 9860.235788943246, "Learning Rate": 5.496094695895029e-06, "Batch Size": 9.0, "Elements per Batch": 3034673.0, "Elements per Second": 1822717.6114543662, "Number of Examples": 4901.0, "Number of Elements": 1517336976.0, "Padding": 85398046.0, "Padding Ratio (%)": 0.05328269790563046, "Total Number of Examples": 58531.0, "Total Number of Elements": 18203263811.0, "Peak Active Device Memory": 9925517824, "Peak Active Device Memory (%)": 0.3925685668397288, "Peak Reserved Device Memory": 12996050944, "Peak Reserved Device Memory (%)": 0.5140125869630582}
{"Time": "2026-07-23T19:38:53.263601", "Step": 6500, "CTC Loss": 43.28304841336437, "Gradient Norm": 854.9989039001465, "Data Epoch": 1, "Data Time": 0.09300027042627335, "Compute Time": 831.0761992521584, "Lapse Time": 831.4765512738377, "Total Time": 10691.735167177394, "Wall Time": 10691.735167177394, "Learning Rate": 4.073345114681779e-06, "Batch Size": 9.0, "Elements per Batch": 3034483.0, "Elements per Second": 1825430.681107395, "Number of Examples": 4861.0, "Number of Elements": 1517241758.0, "Padding": 85242269.0, "Padding Ratio (%)": 0.05319383380037897, "Total Number of Examples": 63392.0, "Total Number of Elements": 19720505569.0, "Peak Active Device Memory": 9926475264, "Peak Active Device Memory (%)": 0.39260643497470166, "Peak Reserved Device Memory": 12996050944, "Peak Reserved Device Memory (%)": 0.5140125869630582}
{"Time": "2026-07-23T19:52:44.450628", "Step": 7000, "CTC Loss": 41.31237777129583, "Gradient Norm": 743.6602358856201, "Data Epoch": 1, "Data Time": 0.09563376940786839, "Compute Time": 830.7464129272848, "Lapse Time": 831.1647398229688, "Total Time": 11522.922715427354, "Wall Time": 11522.922715427354, "Learning Rate": 3.0188963875921575e-06, "Batch Size": 9.0, "Elements per Batch": 3034875.0, "Elements per Second": 1826385.5278306059, "Number of Examples": 4877.0, "Number of Elements": 1517437890.0, "Padding": 83335486.0, "Padding Ratio (%)": 0.05205951526270262, "Total Number of Examples": 68269.0, "Total Number of Elements": 21237943459.0, "Peak Active Device Memory": 9932715008, "Peak Active Device Memory (%)": 0.3928532258628913, "Peak Reserved Device Memory": 12996050944, "Peak Reserved Device Memory (%)": 0.5140125869630582}
{"Time": "2026-07-23T20:06:35.712041", "Step": 7500, "CTC Loss": 38.55709586008006, "Gradient Norm": 618.9990160064697, "Data Epoch": 1, "Data Time": 0.09975391998887062, "Compute Time": 829.3391736838967, "Lapse Time": 831.2418416924775, "Total Time": 12354.18627366051, "Wall Time": 12354.18627366051, "Learning Rate": 2.237408111129068e-06, "Batch Size": 9.0, "Elements per Batch": 3033600.0, "Elements per Second": 1828706.3682696805, "Number of Examples": 4892.0, "Number of Elements": 1516800249.0, "Padding": 82523024.0, "Padding Ratio (%)": 0.051598713901789134, "Total Number of Examples": 73161.0, "Total Number of Elements": 22754743708.0, "Peak Active Device Memory": 9932081664, "Peak Active Device Memory (%)": 0.3928281761928584, "Peak Reserved Device Memory": 12996050944, "Peak Reserved Device Memory (%)": 0.5140125869630582}
{"Time": "2026-07-23T20:20:25.482445", "Step": 8000, "CTC Loss": 38.79082885255463, "Gradient Norm": 582.6709223480225, "Data Epoch": 1, "Data Time": 0.09252194128930569, "Compute Time": 825.4439204372466, "Lapse Time": 825.9616165794432, "Total Time": 13183.954418048263, "Wall Time": 13183.954418048263, "Learning Rate": 1.6582202278690584e-06, "Batch Size": 9.0, "Elements per Batch": 3037455.0, "Elements per Second": 1839685.7679889228, "Number of Examples": 4871.0, "Number of Elements": 1518727644.0, "Padding": 82515123.0, "Padding Ratio (%)": 0.05153192551470242, "Total Number of Examples": 78032.0, "Total Number of Elements": 24273471352.0, "Peak Active Device Memory": 9929443328, "Peak Active Device Memory (%)": 0.39272382619311763, "Peak Reserved Device Memory": 12996050944, "Peak Reserved Device Memory (%)": 0.5140125869630582}
{"Time": "2026-07-23T20:34:16.435596", "Step": 8500, "CTC Loss": 38.44083957347357, "Gradient Norm": 700.7633786926269, "Data Epoch": 1, "Data Time": 0.09213746525347233, "Compute Time": 830.5251655187458, "Lapse Time": 830.9303377494216, "Total Time": 14014.907514667138, "Wall Time": 14014.907514667138, "Learning Rate": 1.2289641350797317e-06, "Batch Size": 9.0, "Elements per Batch": 3036372.0, "Elements per Second": 1827780.4778998757, "Number of Examples": 4904.0, "Number of Elements": 1518186091.0, "Padding": 84159004.0, "Padding Ratio (%)": 0.052522396244486895, "Total Number of Examples": 82936.0, "Total Number of Elements": 25791657443.0, "Peak Active Device Memory": 9926634496, "Peak Active Device Memory (%)": 0.3926127328303041, "Peak Reserved Device Memory": 12624855040, "Peak Reserved Device Memory (%)": 0.49933125278644674}
{"Time": "2026-07-23T20:48:04.282809", "Step": 9000, "CTC Loss": 38.10899847619286, "Gradient Norm": 582.2207745666503, "Data Epoch": 1, "Data Time": 0.09311469458043575, "Compute Time": 827.4048399552703, "Lapse Time": 827.8243797663599, "Total Time": 14842.754655143246, "Wall Time": 14842.754655143246, "Learning Rate": 9.108276572245135e-07, "Batch Size": 9.0, "Elements per Batch": 3027178.0, "Elements per Second": 1829115.2364726549, "Number of Examples": 4849.0, "Number of Elements": 1513589117.0, "Padding": 81391247.0, "Padding Ratio (%)": 0.05102962320857763, "Total Number of Examples": 87785.0, "Total Number of Elements": 27305246560.0, "Peak Active Device Memory": 9912251904, "Peak Active Device Memory (%)": 0.3920438805417889, "Peak Reserved Device Memory": 12624855040, "Peak Reserved Device Memory (%)": 0.49933125278644674}
{"Time": "2026-07-23T21:01:56.572071", "Step": 9500, "CTC Loss": 37.19772177612886, "Gradient Norm": 693.4713052368164, "Data Epoch": 1, "Data Time": 0.08969039469957352, "Compute Time": 831.8500354345888, "Lapse Time": 832.2658338267356, "Total Time": 15675.04327236861, "Wall Time": 15675.04327236861, "Learning Rate": 6.750457539684617e-07, "Batch Size": 9.0, "Elements per Batch": 3041599.0, "Elements per Second": 1828016.9894327943, "Number of Examples": 4909.0, "Number of Elements": 1520799953.0, "Padding": 83650121.0, "Padding Ratio (%)": 0.052136319076264384, "Total Number of Examples": 92694.0, "Total Number of Elements": 28826046513.0, "Peak Active Device Memory": 9925305344, "Peak Active Device Memory (%)": 0.39256016294881235, "Peak Reserved Device Memory": 12624855040, "Peak Reserved Device Memory (%)": 0.49933125278644674}
{"Time": "2026-07-23T21:15:46.422974", "Step": 10000, "CTC Loss": 37.40884138611228, "Gradient Norm": 709.7661766967774, "Data Epoch": 1, "Data Time": 0.0912855826318264, "Compute Time": 829.3966929614544, "Lapse Time": 829.8290401771665, "Total Time": 16504.895128153265, "Wall Time": 16504.895128153265, "Learning Rate": 5.002996629894001e-07, "Batch Size": 9.0, "Elements per Batch": 3027522.0, "Elements per Second": 1824934.3488460763, "Number of Examples": 4858.0, "Number of Elements": 1513761104.0, "Padding": 82856359.0, "Padding Ratio (%)": 0.05189493471048174, "Total Number of Examples": 97552.0, "Total Number of Elements": 30339807617.0, "Peak Active Device Memory": 9920528896, "Peak Active Device Memory (%)": 0.3923712475310786, "Peak Reserved Device Memory": 12624855040, "Peak Reserved Device Memory (%)": 0.49933125278644674}