[ { "dataset": "math500", "run": "baseline", "method": "baseline", "num_speculative_tokens": null, "requests": 500, "output_tokens": 63642, "request_throughput_rps": 0.8912881111809688, "output_throughput_tps": 113.44671594355843, "speedup_vs_baseline": 1.0, "latency_mean_ms": 1121.8503840199992, "latency_p50_ms": 1100.7910255029856, "latency_p95_ms": 1314.5946230510162, "latency_p99_ms": 1484.1045933025453, "ttft_mean_ms": 169.94078358601837, "ttft_p50_ms": 146.2457435009128, "ttft_p95_ms": 335.47060909950216, "ttft_p99_ms": 373.49712165247183, "itl_mean_ms": 7.536181297069743, "itl_p50_ms": 7.382753629891218, "itl_p95_ms": 8.68439770515185, "itl_p99_ms": 10.087323529295983, "tpot_mean_ms": 7.537555813759378, "tpot_p50_ms": 7.383992137781786, "tpot_p95_ms": 8.685839803156412, "acceptance_rate": null, "acceptance_pos_1": null, "acceptance_pos_2": null, "acceptance_pos_3": null, "mean_acceptance_length": null }, { "dataset": "math500", "run": "dflash_k3", "method": "dflash", "num_speculative_tokens": 3, "requests": 500, "output_tokens": 63731, "request_throughput_rps": 0.5074555001107937, "output_throughput_tps": 64.68129295512199, "speedup_vs_baseline": 0.5701468959868523, "latency_mean_ms": 1970.4961424658832, "latency_p50_ms": 1974.9981015011144, "latency_p95_ms": 2394.326527546944, "latency_p99_ms": 2618.456352452631, "ttft_mean_ms": 176.97923488401284, "ttft_p50_ms": 151.66605199920014, "ttft_p95_ms": 348.84992924853566, "ttft_p99_ms": 401.735668064066, "itl_mean_ms": 14.179071923655801, "itl_p50_ms": 14.188702799193972, "itl_p95_ms": 17.33207021927915, "itl_p99_ms": 18.87767345473462, "tpot_mean_ms": 14.180381967095736, "tpot_p50_ms": 14.189951897641887, "tpot_p95_ms": 17.33332515668878, "acceptance_rate": 0.06323912607737021, "acceptance_pos_1": 0.16818250150330727, "acceptance_pos_2": 0.01894167167769092, "acceptance_pos_3": 0.0025932050511124475, "mean_acceptance_length": 1.1897173782321107 }, { "dataset": "math500", "run": "dflash_k7", "method": "dflash", "num_speculative_tokens": 7, "requests": 500, "output_tokens": 63668, "request_throughput_rps": 0.5823973894271881, "output_throughput_tps": 74.16015398010043, "speedup_vs_baseline": 0.6537003152827826, "latency_mean_ms": 1716.9249164580651, "latency_p50_ms": 1721.4581165007985, "latency_p95_ms": 2122.7248207490447, "latency_p99_ms": 2282.573311009855, "ttft_mean_ms": 138.95278630002576, "ttft_p50_ms": 141.47450450036558, "ttft_p95_ms": 232.83745254921087, "ttft_p99_ms": 255.96303990992965, "itl_mean_ms": 12.488242557152846, "itl_p50_ms": 12.48854895669593, "itl_p95_ms": 15.423414866535039, "itl_p99_ms": 16.76060266095041, "tpot_mean_ms": 12.489462153205425, "tpot_p50_ms": 12.489741562990455, "tpot_p95_ms": 15.424737617707956, "acceptance_rate": 0.027334900964792862, "acceptance_pos_1": 0.16893291649402992, "acceptance_pos_2": 0.019209763079588684, "acceptance_pos_3": 0.002561301743945158, "mean_acceptance_length": 1.19134430675355 }, { "dataset": "math500", "run": "dflare_k3", "method": "dflare", "num_speculative_tokens": 3, "requests": 500, "output_tokens": 63731, "request_throughput_rps": 0.7354717914893614, "output_throughput_tps": 93.74470548681697, "speedup_vs_baseline": 0.8263324742997098, "latency_mean_ms": 1359.560892554051, "latency_p50_ms": 1336.2468844989053, "latency_p95_ms": 1919.7459352993976, "latency_p99_ms": 2183.3393649194472, "ttft_mean_ms": 150.9933998281267, "ttft_p50_ms": 150.4645309996704, "ttft_p95_ms": 254.30402324909662, "ttft_p99_ms": 281.55977436035755, "itl_mean_ms": 9.55804924402514, "itl_p50_ms": 9.37427223622698, "itl_p95_ms": 13.641124985851773, "itl_p99_ms": 15.580885431589866, "tpot_mean_ms": 9.559225141953267, "tpot_p50_ms": 9.37540769291087, "tpot_p95_ms": 13.642259077954161, "acceptance_rate": 0.39181146804128403, "acceptance_pos_1": 0.6452053391595262, "acceptance_pos_2": 0.35083467039907146, "acceptance_pos_3": 0.1793943945652545, "mean_acceptance_length": 2.175434404123852 }, { "dataset": "math500", "run": "dflare_k7", "method": "dflare", "num_speculative_tokens": 7, "requests": 500, "output_tokens": 63668, "request_throughput_rps": 0.7652758346384362, "output_throughput_tps": 97.4471636795199, "speedup_vs_baseline": 0.8589685727703342, "latency_mean_ms": 1306.6064661140117, "latency_p50_ms": 1288.1172004999826, "latency_p95_ms": 1831.9392699944729, "latency_p99_ms": 2117.9882052937314, "ttft_mean_ms": 149.97463030007202, "ttft_p50_ms": 150.71226700456464, "ttft_p95_ms": 241.5094937001413, "ttft_p99_ms": 269.5821494147821, "itl_mean_ms": 9.164062260873436, "itl_p50_ms": 9.109615248048863, "itl_p95_ms": 13.162089144880586, "itl_p99_ms": 16.025888632277603, "tpot_mean_ms": 9.165267968224493, "tpot_p50_ms": 9.110725452762747, "tpot_p95_ms": 13.163240382294413, "acceptance_rate": 0.18547955359716675, "acceptance_pos_1": 0.6482415681752667, "acceptance_pos_2": 0.3530196021908331, "acceptance_pos_3": 0.17180743730181608, "mean_acceptance_length": 2.2983568751801675 }, { "dataset": "math500", "run": "dspark_k3", "method": "dspark", "num_speculative_tokens": 3, "requests": 500, "output_tokens": 63731, "request_throughput_rps": 1.0278341396966633, "output_throughput_tps": 131.0097951140161, "speedup_vs_baseline": 1.1548134648445494, "latency_mean_ms": 972.8192880541756, "latency_p50_ms": 951.3454184962029, "latency_p95_ms": 1383.5792890022267, "latency_p99_ms": 1658.223661600423, "ttft_mean_ms": 139.16257599797973, "ttft_p50_ms": 140.1768340037961, "ttft_p95_ms": 243.82559089899584, "ttft_p99_ms": 256.6295453281782, "itl_mean_ms": 6.592793092238339, "itl_p50_ms": 6.475244685019803, "itl_p95_ms": 9.666808936191497, "itl_p99_ms": 11.624004104496413, "tpot_mean_ms": 6.593887856670312, "tpot_p50_ms": 6.476409696829716, "tpot_p95_ms": 9.667887905099029, "acceptance_rate": 0.44804447383243945, "acceptance_pos_1": 0.6541032988024392, "acceptance_pos_2": 0.4218646682830064, "acceptance_pos_3": 0.26816545441187273, "mean_acceptance_length": 2.344133421497318 }, { "dataset": "math500", "run": "dspark_k7", "method": "dspark", "num_speculative_tokens": 7, "requests": 500, "output_tokens": 63668, "request_throughput_rps": 1.0718413712400758, "output_throughput_tps": 136.4839928482263, "speedup_vs_baseline": 1.2030669351074852, "latency_mean_ms": 932.8721254997654, "latency_p50_ms": 924.5265510035097, "latency_p95_ms": 1415.9799854001901, "latency_p99_ms": 1651.539169616298, "ttft_mean_ms": 142.46345183774247, "ttft_p50_ms": 146.44532399688615, "ttft_p95_ms": 239.27307030317024, "ttft_p99_ms": 256.7968881978595, "itl_mean_ms": 6.260414953927999, "itl_p50_ms": 6.1309666968776675, "itl_p95_ms": 10.081072708306051, "itl_p99_ms": 12.369931453538051, "tpot_mean_ms": 6.261500149276782, "tpot_p50_ms": 6.13199001179062, "tpot_p95_ms": 10.082133980709683, "acceptance_rate": 0.2283637249725866, "acceptance_pos_1": 0.6400113714819478, "acceptance_pos_2": 0.3961336961377574, "acceptance_pos_3": 0.24038500588880315, "mean_acceptance_length": 2.598546074808106 }, { "dataset": "gsm8k", "run": "baseline", "method": "baseline", "num_speculative_tokens": null, "requests": 1319, "output_tokens": 168500, "request_throughput_rps": 0.9045831802614913, "output_throughput_tps": 115.55895820626328, "speedup_vs_baseline": 1.0, "latency_mean_ms": 1105.3625852046998, "latency_p50_ms": 1089.5358249981655, "latency_p95_ms": 1276.3373936999415, "latency_p99_ms": 1319.7392648432285, "ttft_mean_ms": 164.77733511295938, "ttft_p50_ms": 141.7763389981701, "ttft_p95_ms": 332.90244539603, "ttft_p99_ms": 382.7308631595223, "itl_mean_ms": 7.419506798811708, "itl_p50_ms": 7.384766740153826, "itl_p95_ms": 7.931435276391204, "itl_p99_ms": 8.172192777608757, "tpot_mean_ms": 7.420847337932145, "tpot_p50_ms": 7.386314905524496, "tpot_p95_ms": 7.932668355122615, "acceptance_rate": null, "acceptance_pos_1": null, "acceptance_pos_2": null, "acceptance_pos_3": null, "mean_acceptance_length": null }, { "dataset": "gsm8k", "run": "dflash_k3", "method": "dflash", "num_speculative_tokens": 3, "requests": 1319, "output_tokens": 168418, "request_throughput_rps": 0.5144087930838571, "output_throughput_tps": 65.6828658935535, "speedup_vs_baseline": 0.568392679486734, "latency_mean_ms": 1943.8572523131418, "latency_p50_ms": 1934.8288309993222, "latency_p95_ms": 2365.305382902443, "latency_p99_ms": 2508.7120192195284, "ttft_mean_ms": 178.53159811208383, "ttft_p50_ms": 159.649838002224, "ttft_p95_ms": 355.616971301788, "ttft_p99_ms": 411.2498142791445, "itl_mean_ms": 13.929846219481163, "itl_p50_ms": 13.876137322875465, "itl_p95_ms": 16.931091559855826, "itl_p99_ms": 17.821733611478884, "tpot_mean_ms": 13.931159616121034, "tpot_p50_ms": 13.87738181893644, "tpot_p95_ms": 16.93262100001677, "acceptance_rate": 0.06835341539008165, "acceptance_pos_1": 0.1835041448500868, "acceptance_pos_2": 0.019489077905896417, "acceptance_pos_3": 0.0020670234142617415, "mean_acceptance_length": 1.2050602461702449 }, { "dataset": "gsm8k", "run": "dflash_k7", "method": "dflash", "num_speculative_tokens": 7, "requests": 1319, "output_tokens": 168457, "request_throughput_rps": 0.5735945671017055, "output_throughput_tps": 73.25702804416376, "speedup_vs_baseline": 0.6339363834814603, "latency_mean_ms": 1743.2748062024025, "latency_p50_ms": 1739.7538450004504, "latency_p95_ms": 2099.799393000285, "latency_p99_ms": 2201.536506499797, "ttft_mean_ms": 152.56252339044147, "ttft_p50_ms": 159.50948600038828, "ttft_p95_ms": 246.41776080115957, "ttft_p99_ms": 261.92414222023217, "itl_mean_ms": 12.548829301304638, "itl_p50_ms": 12.529087370075011, "itl_p95_ms": 15.037010089759487, "itl_p99_ms": 15.820388422991211, "tpot_mean_ms": 12.550042645359513, "tpot_p50_ms": 12.530304661416404, "tpot_p95_ms": 15.038574778736802, "acceptance_rate": 0.02749077227835654, "acceptance_pos_1": 0.1718565178355114, "acceptance_pos_2": 0.018334307172683095, "acceptance_pos_3": 0.002037936980717981, "mean_acceptance_length": 1.1924354059484958 }, { "dataset": "gsm8k", "run": "dflare_k3", "method": "dflare", "num_speculative_tokens": 3, "requests": 1319, "output_tokens": 168418, "request_throughput_rps": 0.6339721362775607, "output_throughput_tps": 80.94944597998045, "speedup_vs_baseline": 0.7005034247149607, "latency_mean_ms": 1577.243275963595, "latency_p50_ms": 1558.0307409982197, "latency_p95_ms": 2166.574809700978, "latency_p99_ms": 2443.7027309426035, "ttft_mean_ms": 150.18097911596254, "ttft_p50_ms": 151.24057199864183, "ttft_p95_ms": 254.45422550219513, "ttft_p99_ms": 278.5744088418869, "itl_mean_ms": 11.259462841455447, "itl_p50_ms": 11.12902188976914, "itl_p95_ms": 15.608968344093311, "itl_p99_ms": 17.811628303623607, "tpot_mean_ms": 11.260660320716259, "tpot_p50_ms": 11.130129110223226, "tpot_p95_ms": 15.610202330715275, "acceptance_rate": 0.2788578880838151, "acceptance_pos_1": 0.5339278493601303, "acceptance_pos_2": 0.21577434619629957, "acceptance_pos_3": 0.08687146869501547, "mean_acceptance_length": 1.8365736642514454 }, { "dataset": "gsm8k", "run": "dflare_k7", "method": "dflare", "num_speculative_tokens": 7, "requests": 1319, "output_tokens": 168457, "request_throughput_rps": 0.6414136486780062, "output_throughput_tps": 81.91858909427664, "speedup_vs_baseline": 0.7088899931761125, "latency_mean_ms": 1558.9424188529392, "latency_p50_ms": 1528.7051639970741, "latency_p95_ms": 2191.441884898086, "latency_p99_ms": 2472.421863719936, "ttft_mean_ms": 152.77644680597203, "ttft_p50_ms": 155.94628499820828, "ttft_p95_ms": 255.01179830098403, "ttft_p99_ms": 283.681260881276, "itl_mean_ms": 11.093297977502198, "itl_p50_ms": 10.81332909192518, "itl_p95_ms": 15.859892152744344, "itl_p99_ms": 18.10012607823774, "tpot_mean_ms": 11.094531792060318, "tpot_p50_ms": 10.814917770089934, "tpot_p95_ms": 15.861106548793142, "acceptance_rate": 0.12798578221940327, "acceptance_pos_1": 0.5417164364111695, "acceptance_pos_2": 0.21953528205359357, "acceptance_pos_3": 0.0932478420590025, "mean_acceptance_length": 1.8959004755358229 }, { "dataset": "gsm8k", "run": "dspark_k3", "method": "dspark", "num_speculative_tokens": 3, "requests": 1319, "output_tokens": 168418, "request_throughput_rps": 0.8767773042557421, "output_throughput_tps": 111.95229721618163, "speedup_vs_baseline": 0.9687894296896996, "latency_mean_ms": 1140.4352353411668, "latency_p50_ms": 1110.0252579999506, "latency_p95_ms": 1627.445946199441, "latency_p99_ms": 1797.3359449025884, "ttft_mean_ms": 137.89601687177677, "ttft_p50_ms": 139.47895800083643, "ttft_p95_ms": 232.75966649744078, "ttft_p99_ms": 255.23558505970868, "itl_mean_ms": 7.9109000767037285, "itl_p50_ms": 7.7173600157683415, "itl_p95_ms": 11.63812698740246, "itl_p99_ms": 13.068145703495471, "tpot_mean_ms": 7.912022636927084, "tpot_p50_ms": 7.718454007901284, "tpot_p95_ms": 11.63953219130328, "acceptance_rate": 0.3119374812973321, "acceptance_pos_1": 0.5350344129088691, "acceptance_pos_2": 0.257809083166448, "acceptance_pos_3": 0.14296894781667932, "mean_acceptance_length": 1.9358124438919964 }, { "dataset": "gsm8k", "run": "dspark_k7", "method": "dspark", "num_speculative_tokens": 7, "requests": 1319, "output_tokens": 168457, "request_throughput_rps": 0.8698276010199105, "output_throughput_tps": 111.09063547006147, "speedup_vs_baseline": 0.9613329610654137, "latency_mean_ms": 1149.4913456012362, "latency_p50_ms": 1122.8163880005013, "latency_p95_ms": 1690.7677308932762, "latency_p99_ms": 1912.2929262730754, "ttft_mean_ms": 142.42033671721288, "ttft_p50_ms": 144.59181799611542, "ttft_p95_ms": 238.7138173042331, "ttft_p99_ms": 260.84847668505967, "itl_mean_ms": 7.943716780175663, "itl_p50_ms": 7.752916023642096, "itl_p95_ms": 11.936723438570985, "itl_p99_ms": 13.552413253823115, "tpot_mean_ms": 7.944849474277738, "tpot_p50_ms": 7.753948110208927, "tpot_p95_ms": 11.937846825916434, "acceptance_rate": 0.14634874499637113, "acceptance_pos_1": 0.5336472057746123, "acceptance_pos_2": 0.25006905994403145, "acceptance_pos_3": 0.13272720721586337, "mean_acceptance_length": 2.024441214974598 }, { "dataset": "humaneval", "run": "baseline", "method": "baseline", "num_speculative_tokens": null, "requests": 164, "output_tokens": 11593, "request_throughput_rps": 1.3248947198163699, "output_throughput_tps": 93.65551516360473, "speedup_vs_baseline": 1.0, "latency_mean_ms": 754.6755473965566, "latency_p50_ms": 647.226564502489, "latency_p95_ms": 1367.1402201009187, "latency_p99_ms": 1480.0008050996985, "ttft_mean_ms": 189.28278684156462, "ttft_p50_ms": 179.46013799883076, "ttft_p95_ms": 342.49747914982436, "ttft_p99_ms": 379.2299685735634, "itl_mean_ms": 8.10896273002792, "itl_p50_ms": 7.874335342868083, "itl_p95_ms": 9.866437620078749, "itl_p99_ms": 14.497655275340552, "tpot_mean_ms": 8.11360654911452, "tpot_p50_ms": 7.876261433408698, "tpot_p95_ms": 9.867678297241653, "acceptance_rate": null, "acceptance_pos_1": null, "acceptance_pos_2": null, "acceptance_pos_3": null, "mean_acceptance_length": null }, { "dataset": "humaneval", "run": "dflash_k3", "method": "dflash", "num_speculative_tokens": 3, "requests": 164, "output_tokens": 11403, "request_throughput_rps": 0.8482418297936308, "output_throughput_tps": 58.97866820205349, "speedup_vs_baseline": 0.6297404706922488, "latency_mean_ms": 1178.8075736159687, "latency_p50_ms": 1028.3337114997266, "latency_p95_ms": 2207.2047492016277, "latency_p99_ms": 2550.0725987079936, "ttft_mean_ms": 162.98695540243034, "ttft_p50_ms": 142.10437850124435, "ttft_p95_ms": 323.0461064536938, "ttft_p99_ms": 404.1426902490639, "itl_mean_ms": 15.05167389736459, "itl_p50_ms": 15.00903958145443, "itl_p95_ms": 20.248163725045735, "itl_p99_ms": 21.988641405840404, "tpot_mean_ms": 15.056187308266047, "tpot_p50_ms": 15.011203893487512, "tpot_p95_ms": 20.267178071169052, "acceptance_rate": 0.0599447417200014, "acceptance_pos_1": 0.1643059490084986, "acceptance_pos_2": 0.01447906830343091, "acceptance_pos_3": 0.0010492078480747037, "mean_acceptance_length": 1.1798342251600042 }, { "dataset": "humaneval", "run": "dflash_k7", "method": "dflash", "num_speculative_tokens": 7, "requests": 164, "output_tokens": 10873, "request_throughput_rps": 0.9775628464767565, "output_throughput_tps": 64.81122457159618, "speedup_vs_baseline": 0.6920171701407963, "latency_mean_ms": 1022.8616001402481, "latency_p50_ms": 861.6272605004269, "latency_p95_ms": 2083.6442281001837, "latency_p99_ms": 2187.8588047894846, "ttft_mean_ms": 157.66290578664288, "ttft_p50_ms": 154.99908150104602, "ttft_p95_ms": 238.5054721988126, "ttft_p99_ms": 254.32537110971683, "itl_mean_ms": 13.23754715845204, "itl_p50_ms": 13.002521015674134, "itl_p95_ms": 16.242605144630183, "itl_p99_ms": 17.38454934863588, "tpot_mean_ms": 13.2414967604306, "tpot_p50_ms": 13.004417010839589, "tpot_p95_ms": 16.244994796315652, "acceptance_rate": 0.027065165688917075, "acceptance_pos_1": 0.17458379578246394, "acceptance_pos_2": 0.014095449500554939, "acceptance_pos_3": 0.0007769145394006659, "mean_acceptance_length": 1.1894561598224196 }, { "dataset": "humaneval", "run": "dflare_k3", "method": "dflare", "num_speculative_tokens": 3, "requests": 164, "output_tokens": 11403, "request_throughput_rps": 1.4279144513142106, "output_throughput_tps": 99.28358834351184, "speedup_vs_baseline": 1.060093344957588, "latency_mean_ms": 700.2340810428306, "latency_p50_ms": 654.7297254983278, "latency_p95_ms": 1298.7420633009606, "latency_p99_ms": 1572.5833712578244, "ttft_mean_ms": 163.544509548978, "ttft_p50_ms": 175.85190250065352, "ttft_p95_ms": 255.46592105038144, "ttft_p99_ms": 272.46247330847837, "itl_mean_ms": 8.18464679387673, "itl_p50_ms": 8.052460137035897, "itl_p95_ms": 11.823310047215534, "itl_p99_ms": 14.873981783527022, "tpot_mean_ms": 8.188209899387681, "tpot_p50_ms": 8.053541158448532, "tpot_p95_ms": 11.828740774528667, "acceptance_rate": 0.5862826458384184, "acceptance_pos_1": 0.7759336099585062, "acceptance_pos_2": 0.5643153526970954, "acceptance_pos_3": 0.4185989748596534, "mean_acceptance_length": 2.758847937515255 }, { "dataset": "humaneval", "run": "dflare_k7", "method": "dflare", "num_speculative_tokens": 7, "requests": 164, "output_tokens": 10873, "request_throughput_rps": 1.7259982556670077, "output_throughput_tps": 114.43157947480107, "speedup_vs_baseline": 1.2218349263778336, "latency_mean_ms": 579.2890396647502, "latency_p50_ms": 542.1822514981613, "latency_p95_ms": 1124.121437451322, "latency_p99_ms": 1337.6976848527559, "ttft_mean_ms": 153.88535212816083, "ttft_p50_ms": 160.26985099961166, "ttft_p95_ms": 249.8991386499256, "ttft_p99_ms": 257.3446220774349, "itl_mean_ms": 7.2881835467627765, "itl_p50_ms": 6.791492409145974, "itl_p95_ms": 11.28842372509098, "itl_p99_ms": 14.967644061684823, "tpot_mean_ms": 7.292224562039036, "tpot_p50_ms": 6.792956900041677, "tpot_p95_ms": 11.300806915223802, "acceptance_rate": 0.3260095527572731, "acceptance_pos_1": 0.7689969604863222, "acceptance_pos_2": 0.5525835866261398, "acceptance_pos_3": 0.40091185410334346, "mean_acceptance_length": 3.282066869300912 }, { "dataset": "humaneval", "run": "dspark_k3", "method": "dspark", "num_speculative_tokens": 3, "requests": 164, "output_tokens": 11403, "request_throughput_rps": 1.8838722899407694, "output_throughput_tps": 130.98655928167435, "speedup_vs_baseline": 1.3985995277785492, "latency_mean_ms": 530.7444518413565, "latency_p50_ms": 495.157746001496, "latency_p95_ms": 905.6995584996912, "latency_p99_ms": 1086.2262809240203, "ttft_mean_ms": 146.0434188045882, "ttft_p50_ms": 152.46622500490048, "ttft_p95_ms": 236.91825044916186, "ttft_p99_ms": 255.82899200926477, "itl_mean_ms": 6.104835246645276, "itl_p50_ms": 5.68536281673191, "itl_p95_ms": 8.98480516036281, "itl_p99_ms": 11.694451736160172, "tpot_mean_ms": 6.108340038664795, "tpot_p50_ms": 5.68948486237622, "tpot_p95_ms": 8.989236841355433, "acceptance_rate": 0.6043326693227091, "acceptance_pos_1": 0.7711653386454184, "acceptance_pos_2": 0.5893924302788844, "acceptance_pos_3": 0.4524402390438247, "mean_acceptance_length": 2.812998007968128 }, { "dataset": "humaneval", "run": "dspark_k7", "method": "dspark", "num_speculative_tokens": 7, "requests": 164, "output_tokens": 10873, "request_throughput_rps": 2.193269418244654, "output_throughput_tps": 145.4110877108178, "speedup_vs_baseline": 1.5526163884401514, "latency_mean_ms": 455.86265411590676, "latency_p50_ms": 409.3146830055048, "latency_p95_ms": 852.2122731519629, "latency_p99_ms": 1123.0342373249005, "ttft_mean_ms": 143.6314728172608, "ttft_p50_ms": 148.64660549210384, "ttft_p95_ms": 241.35335180908442, "ttft_p99_ms": 261.36555448989384, "itl_mean_ms": 5.509904282925857, "itl_p50_ms": 5.137632333503879, "itl_p95_ms": 9.656399889036864, "itl_p99_ms": 11.480592018691826, "tpot_mean_ms": 5.513652760595837, "tpot_p50_ms": 5.139975778089672, "tpot_p95_ms": 9.664061358919506, "acceptance_rate": 0.3497128270580728, "acceptance_pos_1": 0.7345245692405871, "acceptance_pos_2": 0.5520102105934908, "acceptance_pos_3": 0.3937460114869177, "mean_acceptance_length": 3.4479897894065092 }, { "dataset": "mbpp", "run": "baseline", "method": "baseline", "num_speculative_tokens": null, "requests": 500, "output_tokens": 61467, "request_throughput_rps": 0.9566788994866338, "output_throughput_tps": 117.60836382948983, "speedup_vs_baseline": 1.0, "latency_mean_ms": 1045.163285173956, "latency_p50_ms": 1051.2518215000455, "latency_p95_ms": 1252.2429954937252, "latency_p99_ms": 1313.4473929634257, "ttft_mean_ms": 170.204570235961, "ttft_p50_ms": 147.96629649936222, "ttft_p95_ms": 343.0072944531275, "ttft_p99_ms": 394.77327711974795, "itl_mean_ms": 7.1850096596128905, "itl_p50_ms": 7.149241377965513, "itl_p95_ms": 7.642400291320152, "itl_p99_ms": 7.887752700210706, "tpot_mean_ms": 7.1880918593224585, "tpot_p50_ms": 7.1504635433104955, "tpot_p95_ms": 7.643652335044196, "acceptance_rate": null, "acceptance_pos_1": null, "acceptance_pos_2": null, "acceptance_pos_3": null, "mean_acceptance_length": null }, { "dataset": "mbpp", "run": "dflash_k3", "method": "dflash", "num_speculative_tokens": 3, "requests": 500, "output_tokens": 61643, "request_throughput_rps": 0.5463233439436157, "output_throughput_tps": 67.3540197814326, "speedup_vs_baseline": 0.5726975326268746, "latency_mean_ms": 1830.2107265740342, "latency_p50_ms": 1857.0996774942614, "latency_p95_ms": 2291.14192195193, "latency_p99_ms": 2455.683664069438, "ttft_mean_ms": 181.87522897598683, "ttft_p50_ms": 156.96376850246452, "ttft_p95_ms": 374.19471385073825, "ttft_p99_ms": 413.4018152903445, "itl_mean_ms": 13.36878509883122, "itl_p50_ms": 13.397268279545893, "itl_p95_ms": 16.285712324037817, "itl_p99_ms": 17.16560508802913, "tpot_mean_ms": 13.37178172838754, "tpot_p50_ms": 13.398526082703144, "tpot_p95_ms": 16.287178615374877, "acceptance_rate": 0.0651825815270455, "acceptance_pos_1": 0.18390939269673892, "acceptance_pos_2": 0.011013473930872876, "acceptance_pos_3": 0.0006248779535247023, "mean_acceptance_length": 1.1955477445811364 }, { "dataset": "mbpp", "run": "dflash_k7", "method": "dflash", "num_speculative_tokens": 7, "requests": 500, "output_tokens": 61199, "request_throughput_rps": 0.6112544218481429, "output_throughput_tps": 74.81631872536899, "speedup_vs_baseline": 0.636147942962957, "latency_mean_ms": 1635.86529272798, "latency_p50_ms": 1672.3464854994745, "latency_p95_ms": 2016.2770743999317, "latency_p99_ms": 2129.984438859737, "ttft_mean_ms": 152.66953481407472, "ttft_p50_ms": 159.74267549972865, "ttft_p95_ms": 244.47033859860304, "ttft_p99_ms": 260.42180475840723, "itl_mean_ms": 12.100660442193382, "itl_p50_ms": 12.01978580708575, "itl_p95_ms": 14.406062782671242, "itl_p99_ms": 15.092129389210932, "tpot_mean_ms": 12.10357721618688, "tpot_p50_ms": 12.02091915748017, "tpot_p95_ms": 14.40738571140975, "acceptance_rate": 0.028364658688842314, "acceptance_pos_1": 0.18660277646316453, "acceptance_pos_2": 0.011358258400378608, "acceptance_pos_3": 0.0005718567597412841, "mean_acceptance_length": 1.1985526108218962 }, { "dataset": "mbpp", "run": "dflare_k3", "method": "dflare", "num_speculative_tokens": 3, "requests": 500, "output_tokens": 61643, "request_throughput_rps": 0.8389172370608934, "output_throughput_tps": 103.4267504882893, "speedup_vs_baseline": 0.8794166258297648, "latency_mean_ms": 1191.9062847682944, "latency_p50_ms": 1208.0225355002767, "latency_p95_ms": 1663.9658735984995, "latency_p99_ms": 1887.62577743968, "ttft_mean_ms": 151.1433905701415, "ttft_p50_ms": 151.9770585000515, "ttft_p95_ms": 261.6679305512662, "ttft_p99_ms": 283.6371392303044, "itl_mean_ms": 8.617537094703247, "itl_p50_ms": 8.440096641743008, "itl_p95_ms": 12.05171109251937, "itl_p99_ms": 14.760888378997743, "tpot_mean_ms": 8.620249662427044, "tpot_p50_ms": 8.441339763779492, "tpot_p95_ms": 12.052845620474532, "acceptance_rate": 0.4696401115367576, "acceptance_pos_1": 0.6867328590415136, "acceptance_pos_2": 0.4438276913454773, "acceptance_pos_3": 0.278359784223282, "mean_acceptance_length": 2.408920334610273 }, { "dataset": "mbpp", "run": "dflare_k7", "method": "dflare", "num_speculative_tokens": 7, "requests": 500, "output_tokens": 61199, "request_throughput_rps": 0.943455812538664, "output_throughput_tps": 115.47710454310739, "speedup_vs_baseline": 0.9818783357153716, "latency_mean_ms": 1059.8270873299334, "latency_p50_ms": 1066.4042809985403, "latency_p95_ms": 1511.5341901015197, "latency_p99_ms": 1678.6425156925543, "ttft_mean_ms": 149.29563258778944, "ttft_p50_ms": 151.24238849966787, "ttft_p95_ms": 254.46768340189007, "ttft_p99_ms": 276.2093989884306, "itl_mean_ms": 7.8246956861011725, "itl_p50_ms": 7.491708066904327, "itl_p95_ms": 11.13285018701698, "itl_p99_ms": 19.920353018048132, "tpot_mean_ms": 7.827419804285574, "tpot_p50_ms": 7.492821476362439, "tpot_p95_ms": 11.13404580474156, "acceptance_rate": 0.24501954356323288, "acceptance_pos_1": 0.6746690203000882, "acceptance_pos_2": 0.42497793468667255, "acceptance_pos_3": 0.2500882612533098, "mean_acceptance_length": 2.71513680494263 }, { "dataset": "mbpp", "run": "dspark_k3", "method": "dspark", "num_speculative_tokens": 3, "requests": 500, "output_tokens": 61643, "request_throughput_rps": 1.1901763850957827, "output_throughput_tps": 146.73208581291865, "speedup_vs_baseline": 1.247633085225578, "latency_mean_ms": 840.1123063238338, "latency_p50_ms": 843.1062914969516, "latency_p95_ms": 1144.961133646211, "latency_p99_ms": 1246.0605967608715, "ttft_mean_ms": 133.81045833000098, "ttft_p50_ms": 131.7662695000763, "ttft_p95_ms": 238.02317654808573, "ttft_p99_ms": 263.0950807044428, "itl_mean_ms": 5.951950751595078, "itl_p50_ms": 5.73346276769729, "itl_p95_ms": 8.067283917319386, "itl_p99_ms": 14.154150555114027, "tpot_mean_ms": 5.954451005112758, "tpot_p50_ms": 5.734595043312457, "tpot_p95_ms": 8.068427130726949, "acceptance_rate": 0.5237361111111111, "acceptance_pos_1": 0.7102916666666667, "acceptance_pos_2": 0.510625, "acceptance_pos_3": 0.35029166666666667, "mean_acceptance_length": 2.5712083333333333 }, { "dataset": "mbpp", "run": "dspark_k7", "method": "dspark", "num_speculative_tokens": 7, "requests": 500, "output_tokens": 61199, "request_throughput_rps": 1.292868654843097, "output_throughput_tps": 158.2445376154854, "speedup_vs_baseline": 1.3455211216517766, "latency_mean_ms": 773.3765653301089, "latency_p50_ms": 777.3538839974208, "latency_p95_ms": 1069.8221424441717, "latency_p99_ms": 1181.5079208668612, "ttft_mean_ms": 144.87902384001063, "ttft_p50_ms": 148.97060450311983, "ttft_p95_ms": 240.50168554604164, "ttft_p99_ms": 261.4171255609835, "itl_mean_ms": 5.4609269159301945, "itl_p50_ms": 5.171082055140708, "itl_p95_ms": 7.696279004271422, "itl_p99_ms": 16.670999059686896, "tpot_mean_ms": 5.463527012393008, "tpot_p50_ms": 5.172086559067843, "tpot_p95_ms": 7.697418791287318, "acceptance_rate": 0.2885384335766015, "acceptance_pos_1": 0.6823742415345468, "acceptance_pos_2": 0.46951458211000197, "acceptance_pos_3": 0.3062732432961441, "mean_acceptance_length": 3.0197690350362105 }, { "dataset": "mt_bench", "run": "baseline", "method": "baseline", "num_speculative_tokens": null, "requests": 160, "output_tokens": 20102, "request_throughput_rps": 0.9303956789559812, "output_throughput_tps": 116.89258711483208, "speedup_vs_baseline": 1.0, "latency_mean_ms": 1074.6847601811623, "latency_p50_ms": 1056.2882739977795, "latency_p95_ms": 1319.7490790018492, "latency_p99_ms": 1393.648768519415, "ttft_mean_ms": 160.69353549364678, "ttft_p50_ms": 136.07312599924626, "ttft_p95_ms": 327.46918089833343, "ttft_p99_ms": 382.2414645081152, "itl_mean_ms": 7.309607731920508, "itl_p50_ms": 7.198581275588367, "itl_p95_ms": 8.339637732306286, "itl_p99_ms": 9.805146286736829, "tpot_mean_ms": 7.3133077895495875, "tpot_p50_ms": 7.1998062007842805, "tpot_p95_ms": 8.34087489329106, "acceptance_rate": null, "acceptance_pos_1": null, "acceptance_pos_2": null, "acceptance_pos_3": null, "mean_acceptance_length": null }, { "dataset": "mt_bench", "run": "dflash_k3", "method": "dflash", "num_speculative_tokens": 3, "requests": 160, "output_tokens": 20102, "request_throughput_rps": 0.5241432036444518, "output_throughput_tps": 65.85204174787981, "speedup_vs_baseline": 0.5633551568431672, "latency_mean_ms": 1907.7480007938448, "latency_p50_ms": 1930.5097770011344, "latency_p95_ms": 2463.4897702966555, "latency_p99_ms": 2646.199334648336, "ttft_mean_ms": 175.24922163756855, "ttft_p50_ms": 140.2629744989099, "ttft_p95_ms": 376.67113924690057, "ttft_p99_ms": 423.97999386870646, "itl_mean_ms": 13.833307178800341, "itl_p50_ms": 13.946147039380268, "itl_p95_ms": 17.445409657068915, "itl_p99_ms": 18.298039104984053, "tpot_mean_ms": 13.837227284098185, "tpot_p50_ms": 13.947568574815339, "tpot_p95_ms": 17.446681594068878, "acceptance_rate": 0.060423199037646175, "acceptance_pos_1": 0.16210140211796722, "acceptance_pos_2": 0.017630006507720522, "acceptance_pos_3": 0.0015381884872507838, "mean_acceptance_length": 1.1812695971129386 }, { "dataset": "mt_bench", "run": "dflash_k7", "method": "dflash", "num_speculative_tokens": 7, "requests": 160, "output_tokens": 20102, "request_throughput_rps": 0.583168838359363, "output_throughput_tps": 73.26787492937446, "speedup_vs_baseline": 0.6267965893970513, "latency_mean_ms": 1714.6447665063306, "latency_p50_ms": 1721.8866950006486, "latency_p95_ms": 2135.267027699956, "latency_p99_ms": 2242.278299470854, "ttft_mean_ms": 139.6615898878281, "ttft_p50_ms": 138.0474095003592, "ttft_p95_ms": 237.62467689830373, "ttft_p99_ms": 245.49128495971672, "itl_mean_ms": 12.686621501440076, "itl_p50_ms": 12.566233303133897, "itl_p95_ms": 15.546194449997866, "itl_p99_ms": 16.16330862709806, "tpot_mean_ms": 12.6899270927419, "tpot_p50_ms": 12.567349094479857, "tpot_p95_ms": 15.547512893692872, "acceptance_rate": 0.02348325183475921, "acceptance_pos_1": 0.14881334188582424, "acceptance_pos_2": 0.01463642194880168, "acceptance_pos_3": 0.0008746865706455186, "mean_acceptance_length": 1.1643827628433145 }, { "dataset": "mt_bench", "run": "dflare_k3", "method": "dflare", "num_speculative_tokens": 3, "requests": 160, "output_tokens": 20102, "request_throughput_rps": 0.6809063012234507, "output_throughput_tps": 85.5473654199613, "speedup_vs_baseline": 0.7318459410597347, "latency_mean_ms": 1468.511645206172, "latency_p50_ms": 1454.6153324990883, "latency_p95_ms": 2127.0678361986943, "latency_p99_ms": 2281.048237952054, "ttft_mean_ms": 159.72944469986032, "ttft_p50_ms": 168.77598499922897, "ttft_p95_ms": 265.74251724632626, "ttft_p99_ms": 281.86643974164326, "itl_mean_ms": 10.654948740516124, "itl_p50_ms": 10.288291291322736, "itl_p95_ms": 15.705922906309695, "itl_p99_ms": 16.95837595336085, "tpot_mean_ms": 10.6584103940507, "tpot_p50_ms": 10.289786913365493, "tpot_p95_ms": 15.707090863808236, "acceptance_rate": 0.3230895107183767, "acceptance_pos_1": 0.5774177712322042, "acceptance_pos_2": 0.27245949926362295, "acceptance_pos_3": 0.1193912616593029, "mean_acceptance_length": 1.9692685321551302 }, { "dataset": "mt_bench", "run": "dflare_k7", "method": "dflare", "num_speculative_tokens": 7, "requests": 160, "output_tokens": 20102, "request_throughput_rps": 0.7093667936140545, "output_throughput_tps": 89.12307053268577, "speedup_vs_baseline": 0.7624356063326214, "latency_mean_ms": 1409.5885398314294, "latency_p50_ms": 1421.8769274993974, "latency_p95_ms": 2124.1017438478584, "latency_p99_ms": 2515.584632036989, "ttft_mean_ms": 146.91454585622523, "ttft_p50_ms": 142.44566849811235, "ttft_p95_ms": 259.6700968468212, "ttft_p99_ms": 287.97775861195987, "itl_mean_ms": 10.234430485377915, "itl_p50_ms": 10.088829999998364, "itl_p95_ms": 15.341923362584064, "itl_p99_ms": 19.70016743978345, "tpot_mean_ms": 10.238094617695932, "tpot_p50_ms": 10.089903972448514, "tpot_p95_ms": 15.343091349977053, "acceptance_rate": 0.14626535131870344, "acceptance_pos_1": 0.5742903160861688, "acceptance_pos_2": 0.2700825447956513, "acceptance_pos_3": 0.1106301590497282, "mean_acceptance_length": 2.023857459230924 }, { "dataset": "mt_bench", "run": "dspark_k3", "method": "dspark", "num_speculative_tokens": 3, "requests": 160, "output_tokens": 20102, "request_throughput_rps": 0.9638367060855711, "output_throughput_tps": 121.09403416082594, "speedup_vs_baseline": 1.035942801418763, "latency_mean_ms": 1037.4081412996475, "latency_p50_ms": 1001.3265734996821, "latency_p95_ms": 1565.5059430991967, "latency_p99_ms": 1781.7731402603386, "ttft_mean_ms": 141.7672610684349, "ttft_p50_ms": 145.46368399896892, "ttft_p95_ms": 240.4056666531687, "ttft_p99_ms": 261.25688084066496, "itl_mean_ms": 7.313367654694055, "itl_p50_ms": 6.893495897653395, "itl_p95_ms": 11.711175241387847, "itl_p99_ms": 16.168680668779416, "tpot_mean_ms": 7.316552068142107, "tpot_p50_ms": 6.894642385820525, "tpot_p95_ms": 11.712829905159229, "acceptance_rate": 0.3706431026018655, "acceptance_pos_1": 0.5900483904902167, "acceptance_pos_2": 0.3288449400378708, "acceptance_pos_3": 0.19303597727750893, "mean_acceptance_length": 2.1119293078055965 }, { "dataset": "mt_bench", "run": "dspark_k7", "method": "dspark", "num_speculative_tokens": 7, "requests": 160, "output_tokens": 20102, "request_throughput_rps": 0.9729636803231215, "output_throughput_tps": 122.24072438659618, "speedup_vs_baseline": 1.045752578531864, "latency_mean_ms": 1027.6755360433526, "latency_p50_ms": 1009.7822754978552, "latency_p95_ms": 1596.3315117551247, "latency_p99_ms": 2005.884019333607, "ttft_mean_ms": 141.24648624401743, "ttft_p50_ms": 139.84237199474592, "ttft_p95_ms": 255.56160285341318, "ttft_p99_ms": 265.59654699100065, "itl_mean_ms": 7.283154806994812, "itl_p50_ms": 6.8463791023933425, "itl_p95_ms": 11.787724323593851, "itl_p99_ms": 15.836048513204252, "tpot_mean_ms": 7.286351986039958, "tpot_p50_ms": 6.847493661481403, "tpot_p95_ms": 11.788784734661597, "acceptance_rate": 0.1742194735272419, "acceptance_pos_1": 0.5641138336446544, "acceptance_pos_2": 0.30238435336776176, "acceptance_pos_3": 0.17075046698165036, "mean_acceptance_length": 2.2195363146906932 }, { "dataset": "ultrachat", "run": "baseline", "method": "baseline", "num_speculative_tokens": null, "requests": 5000, "output_tokens": 636089, "request_throughput_rps": 0.4938431604479032, "output_throughput_tps": 62.825640417229266, "speedup_vs_baseline": 1.0, "latency_mean_ms": 2024.8159037780163, "latency_p50_ms": 1899.8980629985454, "latency_p95_ms": 3256.729267303672, "latency_p99_ms": 3405.7137782224895, "ttft_mean_ms": 166.95593686518114, "ttft_p50_ms": 163.98859200126026, "ttft_p95_ms": 289.8830948040995, "ttft_p99_ms": 336.2988253685762, "itl_mean_ms": 14.712897944777563, "itl_p50_ms": 13.827794673245338, "itl_p95_ms": 23.864679124793426, "itl_p99_ms": 24.76342741740255, "tpot_mean_ms": 14.714261969753975, "tpot_p50_ms": 13.829047236223541, "tpot_p95_ms": 23.866155100384233, "acceptance_rate": null, "acceptance_pos_1": null, "acceptance_pos_2": null, "acceptance_pos_3": null, "mean_acceptance_length": null }, { "dataset": "ultrachat", "run": "dflash_k3", "method": "dflash", "num_speculative_tokens": 3, "requests": 5000, "output_tokens": 635846, "request_throughput_rps": 0.4093959318736204, "output_throughput_tps": 52.06255313962281, "speedup_vs_baseline": 0.82868320631309, "latency_mean_ms": 2442.487533897996, "latency_p50_ms": 2383.8188690006064, "latency_p95_ms": 3377.258205899534, "latency_p99_ms": 3705.6293327211833, "ttft_mean_ms": 168.35537181059328, "ttft_p50_ms": 170.54413749974628, "ttft_p95_ms": 252.9078775001381, "ttft_p99_ms": 299.23500963023804, "itl_mean_ms": 18.0134018722681, "itl_p50_ms": 17.54899170471861, "itl_p95_ms": 24.705050003148795, "itl_p99_ms": 27.191358835904875, "tpot_mean_ms": 18.01468435680956, "tpot_p50_ms": 17.550207216533355, "tpot_p95_ms": 24.706307322046488, "acceptance_rate": 0.07081888392688648, "acceptance_pos_1": 0.18361221407447353, "acceptance_pos_2": 0.02588322896495871, "acceptance_pos_3": 0.002961208741227203, "mean_acceptance_length": 1.2124566517806594 }, { "dataset": "ultrachat", "run": "dflash_k7", "method": "dflash", "num_speculative_tokens": 7, "requests": 5000, "output_tokens": 636040, "request_throughput_rps": 0.4019325528965641, "output_throughput_tps": 51.12903618886613, "speedup_vs_baseline": 0.8138243533900298, "latency_mean_ms": 2487.8614642295834, "latency_p50_ms": 2432.8269440011354, "latency_p95_ms": 3443.123125650527, "latency_p99_ms": 3724.6332480803176, "ttft_mean_ms": 172.7844437341555, "ttft_p50_ms": 173.04360800153518, "ttft_p95_ms": 258.7694342015311, "ttft_p99_ms": 299.8475944703024, "itl_mean_ms": 18.332019165246155, "itl_p50_ms": 17.85771458268332, "itl_p95_ms": 25.186547632286487, "itl_p99_ms": 27.352591655754214, "tpot_mean_ms": 18.33325328096626, "tpot_p50_ms": 17.85883874804062, "tpot_p95_ms": 25.18777078149849, "acceptance_rate": 0.03064899424451109, "acceptance_pos_1": 0.1849223980874338, "acceptance_pos_2": 0.025778901811864367, "acceptance_pos_3": 0.0031517369145264325, "mean_acceptance_length": 1.2145429597115776 }, { "dataset": "ultrachat", "run": "dflare_k3", "method": "dflare", "num_speculative_tokens": 3, "requests": 5000, "output_tokens": 635846, "request_throughput_rps": 0.4906630866625573, "output_throughput_tps": 62.397232200408084, "speedup_vs_baseline": 0.9931809972174402, "latency_mean_ms": 2037.9430946188047, "latency_p50_ms": 2007.9161120011122, "latency_p95_ms": 2756.5027899490815, "latency_p99_ms": 3044.030925800253, "ttft_mean_ms": 185.9597727056098, "ttft_p50_ms": 186.23321399718407, "ttft_p95_ms": 279.4342198045342, "ttft_p99_ms": 323.5009970532701, "itl_mean_ms": 14.673263611031553, "itl_p50_ms": 14.47747526376982, "itl_p95_ms": 19.803011401590926, "itl_p99_ms": 21.86518714472933, "tpot_mean_ms": 14.674543237540032, "tpot_p50_ms": 14.478983830686477, "tpot_p95_ms": 19.804314228371027, "acceptance_rate": 0.3113198824188073, "acceptance_pos_1": 0.572229825914399, "acceptance_pos_2": 0.26063458484348756, "acceptance_pos_3": 0.10109523649853532, "mean_acceptance_length": 1.933959647256422 }, { "dataset": "ultrachat", "run": "dflare_k7", "method": "dflare", "num_speculative_tokens": 7, "requests": 5000, "output_tokens": 636040, "request_throughput_rps": 0.5191022628219358, "output_throughput_tps": 66.03396064905282, "speedup_vs_baseline": 1.0510670517724432, "latency_mean_ms": 1925.9773102255406, "latency_p50_ms": 1903.7692794991017, "latency_p95_ms": 2635.482599895113, "latency_p99_ms": 2962.525904481372, "ttft_mean_ms": 180.70931879136043, "ttft_p50_ms": 182.9802385000221, "ttft_p95_ms": 272.8241580512986, "ttft_p99_ms": 313.5787184005314, "itl_mean_ms": 13.822185940257995, "itl_p50_ms": 13.62802470079417, "itl_p95_ms": 18.901379390565385, "itl_p99_ms": 21.488975903824134, "tpot_mean_ms": 13.823438931487352, "tpot_p50_ms": 13.629216417345067, "tpot_p95_ms": 18.902619697261734, "acceptance_rate": 0.14394009500546442, "acceptance_pos_1": 0.5769995228510126, "acceptance_pos_2": 0.26659988687461095, "acceptance_pos_3": 0.10386364282712356, "mean_acceptance_length": 2.007580665038251 }, { "dataset": "ultrachat", "run": "dspark_k3", "method": "dspark", "num_speculative_tokens": 3, "requests": 5000, "output_tokens": 635846, "request_throughput_rps": 0.6141321767148702, "output_throughput_tps": 78.09869760708868, "speedup_vs_baseline": 1.2431022921283417, "latency_mean_ms": 1628.2056611145947, "latency_p50_ms": 1592.831803496665, "latency_p95_ms": 2287.4158730010095, "latency_p99_ms": 2571.3276440004856, "ttft_mean_ms": 171.98177616532774, "ttft_p50_ms": 172.63183150498662, "ttft_p95_ms": 258.64595805287536, "ttft_p99_ms": 302.3406934656669, "itl_mean_ms": 11.538623067112681, "itl_p50_ms": 11.238307917321478, "itl_p95_ms": 16.24375355278012, "itl_p99_ms": 18.46682838556656, "tpot_mean_ms": 11.5398444087042, "tpot_p50_ms": 11.239705511848223, "tpot_p95_ms": 16.245154050035826, "acceptance_rate": 0.3240773825244086, "acceptance_pos_1": 0.5557454318356237, "acceptance_pos_2": 0.28176742365161084, "acceptance_pos_3": 0.13471929208599118, "mean_acceptance_length": 1.9722321475732256 }, { "dataset": "ultrachat", "run": "dspark_k7", "method": "dspark", "num_speculative_tokens": 7, "requests": 5000, "output_tokens": 636040, "request_throughput_rps": 0.6186701811110291, "output_throughput_tps": 78.69979639877178, "speedup_vs_baseline": 1.2526700225595981, "latency_mean_ms": 1616.259658270469, "latency_p50_ms": 1584.57970800373, "latency_p95_ms": 2290.956548342365, "latency_p99_ms": 2594.4233495666413, "ttft_mean_ms": 176.71817260388633, "ttft_p50_ms": 176.76159800612368, "ttft_p95_ms": 263.7752173024638, "ttft_p99_ms": 303.1083872308955, "itl_mean_ms": 11.399967330431618, "itl_p50_ms": 11.174435653507444, "itl_p95_ms": 16.282335170467917, "itl_p99_ms": 18.624601763757692, "tpot_mean_ms": 11.401195973910607, "tpot_p50_ms": 11.175635023563679, "tpot_p95_ms": 16.283561415453793, "acceptance_rate": 0.15448514765416219, "acceptance_pos_1": 0.5533425372792525, "acceptance_pos_2": 0.28025688169439844, "acceptance_pos_3": 0.13306681148704796, "mean_acceptance_length": 2.0813960335791357 } ]