[ { "dataset": "math500", "run": "baseline", "method": "baseline", "num_speculative_tokens": null, "requests": 500, "output_tokens": 63527, "request_throughput_rps": 2.575660280906157, "output_throughput_tps": 327.24794133025085, "speedup_vs_baseline": 1.0, "latency_mean_ms": 1549.881507652055, "latency_p50_ms": 1541.6923305019736, "latency_p95_ms": 1901.3991399056977, "latency_p99_ms": 2073.3104983172957, "ttft_mean_ms": 147.72230004466837, "ttft_p50_ms": 128.53012699633837, "ttft_p95_ms": 277.2130677534733, "ttft_p99_ms": 387.54749131272547, "itl_mean_ms": 11.11915976237464, "itl_p50_ms": 11.026409515830519, "itl_p95_ms": 13.801987814431913, "itl_p99_ms": 15.665761921923757, "tpot_mean_ms": 11.12106789862526, "tpot_p50_ms": 11.028098413518146, "tpot_p95_ms": 13.803411649602399, "acceptance_rate": null, "acceptance_pos_1": null, "acceptance_pos_2": null, "acceptance_pos_3": null, "mean_acceptance_length": null }, { "dataset": "math500", "run": "dspark_k3", "method": "dspark", "num_speculative_tokens": 3, "requests": 500, "output_tokens": 63539, "request_throughput_rps": 3.34003084405518, "output_throughput_tps": 424.44443960084413, "speedup_vs_baseline": 1.2970117944073019, "latency_mean_ms": 1195.6849749402609, "latency_p50_ms": 1169.5264599984512, "latency_p95_ms": 1773.1451036757787, "latency_p99_ms": 1944.9941020199901, "ttft_mean_ms": 143.30231556599028, "ttft_p50_ms": 116.45177099853754, "ttft_p95_ms": 297.67805395094905, "ttft_p99_ms": 373.04166853777133, "itl_mean_ms": 8.352826905473885, "itl_p50_ms": 8.158615838546549, "itl_p95_ms": 12.354124409472469, "itl_p99_ms": 14.635059152750648, "tpot_mean_ms": 8.35462870098348, "tpot_p50_ms": 8.163252110248358, "tpot_p95_ms": 12.355534362218949, "acceptance_rate": 0.415486407892723, "acceptance_pos_1": 0.6332426800409706, "acceptance_pos_2": 0.38554727510330944, "acceptance_pos_3": 0.22766926853388902, "mean_acceptance_length": 2.246459223678169 }, { "dataset": "math500", "run": "dspark_k7", "method": "dspark", "num_speculative_tokens": 7, "requests": 500, "output_tokens": 63493, "request_throughput_rps": 3.1055917217703812, "output_throughput_tps": 394.36667038073364, "speedup_vs_baseline": 1.2051005386852782, "latency_mean_ms": 1286.8654769104323, "latency_p50_ms": 1246.8089515023166, "latency_p95_ms": 1958.8731202602503, "latency_p99_ms": 2254.721420374407, "ttft_mean_ms": 169.23283278750023, "ttft_p50_ms": 150.04799248708878, "ttft_p95_ms": 310.4628508415771, "ttft_p99_ms": 383.799409790663, "itl_mean_ms": 8.871197280140835, "itl_p50_ms": 8.589971834756103, "itl_p95_ms": 13.716522226358514, "itl_p99_ms": 15.951512530517089, "tpot_mean_ms": 8.87296793212422, "tpot_p50_ms": 8.60067761416536, "tpot_p95_ms": 13.71767588804694, "acceptance_rate": 0.20277687384748888, "acceptance_pos_1": 0.6178056188306758, "acceptance_pos_2": 0.36526195899772207, "acceptance_pos_3": 0.20611237661351556, "mean_acceptance_length": 2.4194381169324224 }, { "dataset": "gsm8k", "run": "baseline", "method": "baseline", "num_speculative_tokens": null, "requests": 1319, "output_tokens": 168533, "request_throughput_rps": 2.699786987185922, "output_throughput_tps": 344.9607280602009, "speedup_vs_baseline": 1.0, "latency_mean_ms": 1479.6602965273742, "latency_p50_ms": 1454.6987209760118, "latency_p95_ms": 1803.740513609955, "latency_p99_ms": 1914.7201473527819, "ttft_mean_ms": 168.7780613106763, "ttft_p50_ms": 145.47203300753608, "ttft_p95_ms": 336.32914681511363, "ttft_p99_ms": 491.5234538837098, "itl_mean_ms": 10.337075278453744, "itl_p50_ms": 10.18635707075055, "itl_p95_ms": 12.4034695235158, "itl_p99_ms": 13.359812111263784, "tpot_mean_ms": 10.338842536074479, "tpot_p50_ms": 10.187574787391716, "tpot_p95_ms": 12.40562485355891, "acceptance_rate": null, "acceptance_pos_1": null, "acceptance_pos_2": null, "acceptance_pos_3": null, "mean_acceptance_length": null }, { "dataset": "gsm8k", "run": "dspark_k3", "method": "dspark", "num_speculative_tokens": 3, "requests": 1319, "output_tokens": 168415, "request_throughput_rps": 2.95131608515978, "output_throughput_tps": 376.8354044595787, "speedup_vs_baseline": 1.0924008845256588, "latency_mean_ms": 1352.7894092793806, "latency_p50_ms": 1323.1997079856228, "latency_p95_ms": 1901.4164583000816, "latency_p99_ms": 2116.5834879834433, "ttft_mean_ms": 146.35591101451402, "ttft_p50_ms": 119.20796197955497, "ttft_p95_ms": 300.50935930921696, "ttft_p99_ms": 387.0458965114083, "itl_mean_ms": 9.517948084655174, "itl_p50_ms": 9.351653133988322, "itl_p95_ms": 13.592088056107666, "itl_p99_ms": 15.135234009802675, "tpot_mean_ms": 9.519542839852642, "tpot_p50_ms": 9.352777228419354, "tpot_p95_ms": 13.593510024599716, "acceptance_rate": 0.27039498963955716, "acceptance_pos_1": 0.48590297454239884, "acceptance_pos_2": 0.220014867325282, "acceptance_pos_3": 0.10526712705099062, "mean_acceptance_length": 1.8111849689186714 }, { "dataset": "gsm8k", "run": "dspark_k7", "method": "dspark", "num_speculative_tokens": 7, "requests": 1319, "output_tokens": 168438, "request_throughput_rps": 2.833061661782182, "output_throughput_tps": 361.7856256158205, "speedup_vs_baseline": 1.0487733709579932, "latency_mean_ms": 1409.892868064269, "latency_p50_ms": 1389.3212949915323, "latency_p95_ms": 2024.2955645022444, "latency_p99_ms": 2287.086103030015, "ttft_mean_ms": 149.37952628049933, "ttft_p50_ms": 121.9771110045258, "ttft_p95_ms": 303.5228224063758, "ttft_p99_ms": 355.9325602342128, "itl_mean_ms": 9.94289976587715, "itl_p50_ms": 9.82163717335926, "itl_p95_ms": 14.449290831630137, "itl_p99_ms": 16.439139549680032, "tpot_mean_ms": 9.944535710893788, "tpot_p50_ms": 9.823021669386277, "tpot_p95_ms": 14.453224437911699, "acceptance_rate": 0.1243881347729851, "acceptance_pos_1": 0.47593713555839395, "acceptance_pos_2": 0.21317265507858055, "acceptance_pos_3": 0.10223801854834232, "mean_acceptance_length": 1.870716943410896 }, { "dataset": "humaneval", "run": "baseline", "method": "baseline", "num_speculative_tokens": null, "requests": 164, "output_tokens": 11241, "request_throughput_rps": 3.867290467605202, "output_throughput_tps": 265.07446430701265, "speedup_vs_baseline": 1.0, "latency_mean_ms": 1027.5289629021356, "latency_p50_ms": 879.5409715094138, "latency_p95_ms": 2018.1860155600586, "latency_p99_ms": 2274.2641233184268, "ttft_mean_ms": 140.52152795707892, "ttft_p50_ms": 120.44445698847994, "ttft_p95_ms": 278.58940106234513, "ttft_p99_ms": 480.18000137846695, "itl_mean_ms": 13.187347418172193, "itl_p50_ms": 12.668324928583226, "itl_p95_ms": 19.83492009233357, "itl_p99_ms": 28.834143813500756, "tpot_mean_ms": 13.194289122212655, "tpot_p50_ms": 12.670714736622909, "tpot_p95_ms": 19.844658022235045, "acceptance_rate": null, "acceptance_pos_1": null, "acceptance_pos_2": null, "acceptance_pos_3": null, "mean_acceptance_length": null }, { "dataset": "humaneval", "run": "dspark_k3", "method": "dspark", "num_speculative_tokens": 3, "requests": 164, "output_tokens": 11202, "request_throughput_rps": 5.440176459827299, "output_throughput_tps": 371.59058965235, "speedup_vs_baseline": 1.4018347283047567, "latency_mean_ms": 731.6780699141573, "latency_p50_ms": 653.8903180044144, "latency_p95_ms": 1438.7055883620633, "latency_p99_ms": 1758.196029517276, "ttft_mean_ms": 153.84280301219493, "ttft_p50_ms": 109.24666900245938, "ttft_p95_ms": 371.5150598072796, "ttft_p99_ms": 427.1234392697809, "itl_mean_ms": 9.217930821861229, "itl_p50_ms": 8.395432637747984, "itl_p95_ms": 16.543497819817173, "itl_p99_ms": 21.385043967787766, "tpot_mean_ms": 9.224179054706173, "tpot_p50_ms": 8.398416746314833, "tpot_p95_ms": 16.54860166646416, "acceptance_rate": 0.5723147241266732, "acceptance_pos_1": 0.7548971596474046, "acceptance_pos_2": 0.5506856023506367, "acceptance_pos_3": 0.41136141038197843, "mean_acceptance_length": 2.7169441723800194 }, { "dataset": "humaneval", "run": "dspark_k7", "method": "dspark", "num_speculative_tokens": 7, "requests": 164, "output_tokens": 11135, "request_throughput_rps": 5.728781174033297, "output_throughput_tps": 388.9632827613461, "speedup_vs_baseline": 1.467373644527463, "latency_mean_ms": 695.9294298721943, "latency_p50_ms": 634.961468007532, "latency_p95_ms": 1258.2039341956258, "latency_p99_ms": 1984.7797918543813, "ttft_mean_ms": 170.66249135340757, "ttft_p50_ms": 130.76092200935818, "ttft_p95_ms": 440.41123243077857, "ttft_p99_ms": 548.8665195452631, "itl_mean_ms": 9.06925487325878, "itl_p50_ms": 7.807209323622503, "itl_p95_ms": 18.29704301704432, "itl_p99_ms": 30.691704785244347, "tpot_mean_ms": 9.07486906638103, "tpot_p50_ms": 7.822465188736613, "tpot_p95_ms": 18.303107731801564, "acceptance_rate": 0.33277512988964747, "acceptance_pos_1": 0.7348963029756538, "acceptance_pos_2": 0.5404268109407875, "acceptance_pos_3": 0.38352870453862337, "mean_acceptance_length": 3.3294259092275325 }, { "dataset": "mbpp", "run": "baseline", "method": "baseline", "num_speculative_tokens": null, "requests": 500, "output_tokens": 61587, "request_throughput_rps": 2.8009211403112766, "output_throughput_tps": 345.00066053670116, "speedup_vs_baseline": 1.0, "latency_mean_ms": 1426.2509520059102, "latency_p50_ms": 1449.8898644960718, "latency_p95_ms": 1792.6750187427385, "latency_p99_ms": 1927.6900081126948, "ttft_mean_ms": 146.00965341442497, "ttft_p50_ms": 129.28127399936784, "ttft_p95_ms": 278.12606179941207, "ttft_p99_ms": 302.88602881017135, "itl_mean_ms": 10.539905415700249, "itl_p50_ms": 10.33841663381217, "itl_p95_ms": 12.799749751224665, "itl_p99_ms": 13.565437609112568, "tpot_mean_ms": 10.546513464177922, "tpot_p50_ms": 10.339677704733752, "tpot_p95_ms": 12.800697692052097, "acceptance_rate": null, "acceptance_pos_1": null, "acceptance_pos_2": null, "acceptance_pos_3": null, "mean_acceptance_length": null }, { "dataset": "mbpp", "run": "dspark_k3", "method": "dspark", "num_speculative_tokens": 3, "requests": 500, "output_tokens": 61810, "request_throughput_rps": 3.8673346513351055, "output_throughput_tps": 478.07990959804573, "speedup_vs_baseline": 1.385736215270775, "latency_mean_ms": 1031.8113529087277, "latency_p50_ms": 1022.938920505112, "latency_p95_ms": 1450.6354179771733, "latency_p99_ms": 1667.4116877623596, "ttft_mean_ms": 149.58043881563935, "ttft_p50_ms": 120.52947049960494, "ttft_p95_ms": 290.24569394096017, "ttft_p99_ms": 465.6294314624392, "itl_mean_ms": 7.4866564872650105, "itl_p50_ms": 7.0952005747625035, "itl_p95_ms": 10.490002014466166, "itl_p99_ms": 15.616462420633345, "tpot_mean_ms": 7.490425515753587, "tpot_p50_ms": 7.09647927951163, "tpot_p95_ms": 10.49589030935033, "acceptance_rate": 0.5115081479053122, "acceptance_pos_1": 0.7031512518952587, "acceptance_pos_2": 0.5011678891939516, "acceptance_pos_3": 0.3302053026267262, "mean_acceptance_length": 2.5345244437159367 }, { "dataset": "mbpp", "run": "dspark_k7", "method": "dspark", "num_speculative_tokens": 7, "requests": 500, "output_tokens": 61424, "request_throughput_rps": 3.9288605862280823, "output_throughput_tps": 482.6526652969474, "speedup_vs_baseline": 1.3989905542386718, "latency_mean_ms": 1016.7640253802527, "latency_p50_ms": 1009.4089240010362, "latency_p95_ms": 1519.8205350112396, "latency_p99_ms": 1755.0660978292578, "ttft_mean_ms": 157.01642078207806, "ttft_p50_ms": 127.09367199568078, "ttft_p95_ms": 310.5101487963111, "ttft_p99_ms": 555.6603015074506, "itl_mean_ms": 7.464304052053599, "itl_p50_ms": 6.92667360632959, "itl_p95_ms": 11.035447346997291, "itl_p99_ms": 15.66697385307634, "tpot_mean_ms": 7.469533758903007, "tpot_p50_ms": 6.928639874063014, "tpot_p95_ms": 11.131861498385357, "acceptance_rate": 0.27595293416337563, "acceptance_pos_1": 0.6748067713025748, "acceptance_pos_2": 0.46773199298212337, "acceptance_pos_3": 0.29465598179145525, "mean_acceptance_length": 2.9316705391436293 }, { "dataset": "mt_bench", "run": "baseline", "method": "baseline", "num_speculative_tokens": null, "requests": 160, "output_tokens": 20102, "request_throughput_rps": 2.865123484091851, "output_throughput_tps": 359.96695173258996, "speedup_vs_baseline": 1.0, "latency_mean_ms": 1379.8839098995813, "latency_p50_ms": 1349.3874825071543, "latency_p95_ms": 1778.2662457597326, "latency_p99_ms": 2064.43852098775, "ttft_mean_ms": 230.34865414410888, "ttft_p50_ms": 188.74082650290802, "ttft_p95_ms": 469.9252384045379, "ttft_p99_ms": 599.9244086013641, "itl_mean_ms": 9.196491927825377, "itl_p50_ms": 8.739159606336653, "itl_p95_ms": 12.184589993809974, "itl_p99_ms": 13.513825134104701, "tpot_mean_ms": 9.201697317973778, "tpot_p50_ms": 8.741404259885389, "tpot_p95_ms": 12.185761682743273, "acceptance_rate": null, "acceptance_pos_1": null, "acceptance_pos_2": null, "acceptance_pos_3": null, "mean_acceptance_length": null }, { "dataset": "mt_bench", "run": "dspark_k3", "method": "dspark", "num_speculative_tokens": 3, "requests": 160, "output_tokens": 20228, "request_throughput_rps": 3.1300295907514477, "output_throughput_tps": 395.7139910107518, "speedup_vs_baseline": 1.0993064477338947, "latency_mean_ms": 1267.3618169241308, "latency_p50_ms": 1240.8099970052717, "latency_p95_ms": 1889.9596753937649, "latency_p99_ms": 2197.152719998557, "ttft_mean_ms": 149.93388268667331, "ttft_p50_ms": 120.75363399344496, "ttft_p95_ms": 296.08547244570207, "ttft_p99_ms": 370.8032631434613, "itl_mean_ms": 8.953320217924055, "itl_p50_ms": 8.487721767728438, "itl_p95_ms": 13.275213856385248, "itl_p99_ms": 16.593942496504845, "tpot_mean_ms": 8.957264486664604, "tpot_p50_ms": 8.49002677173625, "tpot_p95_ms": 13.297254559146252, "acceptance_rate": 0.3200388161086851, "acceptance_pos_1": 0.5434255215914604, "acceptance_pos_2": 0.27666181465308104, "acceptance_pos_3": 0.14002911208151383, "mean_acceptance_length": 1.9601164483260554 }, { "dataset": "mt_bench", "run": "dspark_k7", "method": "dspark", "num_speculative_tokens": 7, "requests": 160, "output_tokens": 20102, "request_throughput_rps": 2.8379656094158685, "output_throughput_tps": 356.5549042529862, "speedup_vs_baseline": 0.9905212201754052, "latency_mean_ms": 1404.4557639432242, "latency_p50_ms": 1343.6405550019117, "latency_p95_ms": 2271.4736478112154, "latency_p99_ms": 2877.03235346009, "ttft_mean_ms": 182.31840012485918, "ttft_p50_ms": 159.79043499100953, "ttft_p95_ms": 363.3274522420834, "ttft_p99_ms": 631.4187010040041, "itl_mean_ms": 9.922806420825252, "itl_p50_ms": 9.43615103145515, "itl_p95_ms": 17.314218461555217, "itl_p99_ms": 20.890300572353894, "tpot_mean_ms": 9.927143236485232, "tpot_p50_ms": 9.4376937125298, "tpot_p95_ms": 17.32922917495226, "acceptance_rate": 0.1459540058496301, "acceptance_pos_1": 0.5190686471296668, "acceptance_pos_2": 0.258129265355279, "acceptance_pos_3": 0.12515054195102368, "mean_acceptance_length": 2.0216780409474104 }, { "dataset": "ultrachat", "run": "baseline", "method": "baseline", "num_speculative_tokens": null, "requests": 5000, "output_tokens": 635560, "request_throughput_rps": 1.3331967798759812, "output_throughput_tps": 169.4653090835957, "speedup_vs_baseline": 1.0, "latency_mean_ms": 2999.209397312632, "latency_p50_ms": 3004.6992159914225, "latency_p95_ms": 3652.681129911798, "latency_p99_ms": 3792.2580027181543, "ttft_mean_ms": 191.445676819887, "ttft_p50_ms": 180.53925450658426, "ttft_p95_ms": 319.0085294045276, "ttft_p99_ms": 436.0589518790949, "itl_mean_ms": 22.255913944838188, "itl_p50_ms": 22.191040874087605, "itl_p95_ms": 26.93092076657395, "itl_p99_ms": 27.96015746255569, "tpot_mean_ms": 22.257870607914793, "tpot_p50_ms": 22.192596555178742, "tpot_p95_ms": 26.93375055238194, "acceptance_rate": null, "acceptance_pos_1": null, "acceptance_pos_2": null, "acceptance_pos_3": null, "mean_acceptance_length": null }, { "dataset": "ultrachat", "run": "dspark_k3", "method": "dspark", "num_speculative_tokens": 3, "requests": 5000, "output_tokens": 636234, "request_throughput_rps": 1.816992286110778, "output_throughput_tps": 231.20645403228093, "speedup_vs_baseline": 1.364329108314604, "latency_mean_ms": 2200.762252031482, "latency_p50_ms": 2194.020344497403, "latency_p95_ms": 2860.881272106781, "latency_p99_ms": 3141.828603268659, "ttft_mean_ms": 203.0626388980716, "ttft_p50_ms": 195.85809300770052, "ttft_p95_ms": 327.66038095142005, "ttft_p99_ms": 418.8016837404577, "itl_mean_ms": 15.817088125396433, "itl_p50_ms": 15.717495220445654, "itl_p95_ms": 20.662014169011663, "itl_p99_ms": 22.842756694141052, "tpot_mean_ms": 15.818637421764908, "tpot_p50_ms": 15.7196392047615, "tpot_p95_ms": 20.663277460188848, "acceptance_rate": 0.3022621517716569, "acceptance_pos_1": 0.5328723151898772, "acceptance_pos_2": 0.2570481426653975, "acceptance_pos_3": 0.11686599745969595, "mean_acceptance_length": 1.9067864553149707 }, { "dataset": "ultrachat", "run": "dspark_k7", "method": "dspark", "num_speculative_tokens": 7, "requests": 5000, "output_tokens": 636210, "request_throughput_rps": 1.8127834563773697, "output_throughput_tps": 230.66219255636926, "speedup_vs_baseline": 1.361117468841872, "latency_mean_ms": 2205.888447939884, "latency_p50_ms": 2192.8646555024898, "latency_p95_ms": 2923.480463263696, "latency_p99_ms": 3272.379539081885, "ttft_mean_ms": 208.6473120003706, "ttft_p50_ms": 199.3813320004847, "ttft_p95_ms": 335.6416549038841, "ttft_p99_ms": 448.6493355646964, "itl_mean_ms": 15.814628500703911, "itl_p50_ms": 15.656986732382371, "itl_p95_ms": 21.199671716881205, "itl_p99_ms": 23.549442959891635, "tpot_mean_ms": 15.81632527800587, "tpot_p50_ms": 15.658670244142634, "tpot_p95_ms": 21.20066768934127, "acceptance_rate": 0.14160791914766432, "acceptance_pos_1": 0.5315988273762544, "acceptance_pos_2": 0.25404655416494404, "acceptance_pos_3": 0.11493529271745531, "mean_acceptance_length": 1.9912554340336501 } ]