[ { "dataset": "math500", "run": "baseline", "method": "baseline", "num_speculative_tokens": null, "requests": 500, "output_tokens": 63642, "request_throughput_rps": 0.9710887696354362, "output_throughput_tps": 123.60406295427687, "speedup_vs_baseline": 1.0, "latency_mean_ms": 1029.6528329752618, "latency_p50_ms": 1011.0086214990588, "latency_p95_ms": 1183.8821351528165, "latency_p99_ms": 1352.1066228914415, "ttft_mean_ms": 137.17931756953476, "ttft_p50_ms": 125.86030399688752, "ttft_p95_ms": 254.1806304405327, "ttft_p99_ms": 259.12194621312665, "itl_mean_ms": 7.065699614096616, "itl_p50_ms": 6.948497625951028, "itl_p95_ms": 8.149714840967464, "itl_p99_ms": 9.294363555888458, "tpot_mean_ms": 7.066986512325109, "tpot_p50_ms": 6.949724716546406, "tpot_p95_ms": 8.150898662537166, "acceptance_rate": null, "acceptance_pos_1": null, "acceptance_pos_2": null, "acceptance_pos_3": null, "mean_acceptance_length": null }, { "dataset": "math500", "run": "dspark_k3", "method": "dspark", "num_speculative_tokens": 3, "requests": 500, "output_tokens": 63731, "request_throughput_rps": 1.1940337177445612, "output_throughput_tps": 152.19392573115726, "speedup_vs_baseline": 1.2313019660806477, "latency_mean_ms": 837.3801779437345, "latency_p50_ms": 816.9088775030104, "latency_p95_ms": 1146.009227049944, "latency_p99_ms": 1370.9137859771727, "ttft_mean_ms": 119.19492300227284, "ttft_p50_ms": 106.72050050197868, "ttft_p95_ms": 213.44604070691275, "ttft_p99_ms": 258.3343633514596, "itl_mean_ms": 5.6791052364180485, "itl_p50_ms": 5.593892366200956, "itl_p95_ms": 8.0383705036205, "itl_p99_ms": 9.9072340297603, "tpot_mean_ms": 5.680338308854861, "tpot_p50_ms": 5.595103622135508, "tpot_p95_ms": 8.040039556758208, "acceptance_rate": 0.4156275683926265, "acceptance_pos_1": 0.6326171187037689, "acceptance_pos_2": 0.3862627685804861, "acceptance_pos_3": 0.2280028178936245, "mean_acceptance_length": 2.2468827051778795 }, { "dataset": "math500", "run": "dspark_k7", "method": "dspark", "num_speculative_tokens": 7, "requests": 500, "output_tokens": 63668, "request_throughput_rps": 1.1775733318385104, "output_throughput_tps": 149.94747778298856, "speedup_vs_baseline": 1.2131274182989966, "latency_mean_ms": 849.0922174340812, "latency_p50_ms": 839.2276294980547, "latency_p95_ms": 1242.3280104500004, "latency_p99_ms": 1512.9399566508073, "ttft_mean_ms": 129.85324780782685, "ttft_p50_ms": 118.26306749571813, "ttft_p95_ms": 256.7381934488367, "ttft_p99_ms": 268.83207743172534, "itl_mean_ms": 5.692451923082981, "itl_p50_ms": 5.603080988136545, "itl_p95_ms": 8.775799772777777, "itl_p99_ms": 10.470896226674313, "tpot_mean_ms": 5.693717860424883, "tpot_p50_ms": 5.604267641709472, "tpot_p95_ms": 8.777032548852267, "acceptance_rate": 0.2032188585715368, "acceptance_pos_1": 0.6206151981210698, "acceptance_pos_2": 0.36616410334116223, "acceptance_pos_3": 0.20482612319115084, "mean_acceptance_length": 2.422532010000758 }, { "dataset": "gsm8k", "run": "baseline", "method": "baseline", "num_speculative_tokens": null, "requests": 1319, "output_tokens": 168500, "request_throughput_rps": 0.99646643680205, "output_throughput_tps": 127.29688749139153, "speedup_vs_baseline": 1.0, "latency_mean_ms": 1003.4296112389243, "latency_p50_ms": 979.7711369901663, "latency_p95_ms": 1147.1265090964152, "latency_p99_ms": 1165.5087842789362, "ttft_mean_ms": 121.82556504392818, "ttft_p50_ms": 98.20042399223894, "ttft_p95_ms": 253.87967379938343, "ttft_p99_ms": 259.2248212755658, "itl_mean_ms": 6.953847864544167, "itl_p50_ms": 6.954055936964903, "itl_p95_ms": 7.2364834158344005, "itl_p99_ms": 7.467789549431415, "tpot_mean_ms": 6.955123454672103, "tpot_p50_ms": 6.955686165340699, "tpot_p95_ms": 7.237693709486319, "acceptance_rate": null, "acceptance_pos_1": null, "acceptance_pos_2": null, "acceptance_pos_3": null, "mean_acceptance_length": null }, { "dataset": "gsm8k", "run": "dspark_k3", "method": "dspark", "num_speculative_tokens": 3, "requests": 1319, "output_tokens": 168418, "request_throughput_rps": 1.0089794437230273, "output_throughput_tps": 128.83267623422654, "speedup_vs_baseline": 1.0120646213202884, "latency_mean_ms": 990.9843431904721, "latency_p50_ms": 969.3124250043184, "latency_p95_ms": 1346.5886881953336, "latency_p99_ms": 1535.857625339122, "ttft_mean_ms": 111.30873365058301, "ttft_p50_ms": 95.77244399406482, "ttft_p95_ms": 207.4599851010134, "ttft_p99_ms": 258.9313290748395, "itl_mean_ms": 6.9409634880545115, "itl_p50_ms": 6.796889598449274, "itl_p95_ms": 9.581487464620439, "itl_p99_ms": 10.733646140910508, "tpot_mean_ms": 6.942225311685386, "tpot_p50_ms": 6.797988936978788, "tpot_p95_ms": 9.582715377937562, "acceptance_rate": 0.27131373077960613, "acceptance_pos_1": 0.48776908551389264, "acceptance_pos_2": 0.22053412462908012, "acceptance_pos_3": 0.10563798219584569, "mean_acceptance_length": 1.8139411923388185 }, { "dataset": "gsm8k", "run": "dspark_k7", "method": "dspark", "num_speculative_tokens": 7, "requests": 1319, "output_tokens": 168457, "request_throughput_rps": 0.9537832973729679, "output_throughput_tps": 121.81309547047616, "speedup_vs_baseline": 0.9569212403462244, "latency_mean_ms": 1048.3418924380537, "latency_p50_ms": 1024.0400490001775, "latency_p95_ms": 1480.1234933955127, "latency_p99_ms": 1657.6216600838231, "ttft_mean_ms": 127.43488244511649, "ttft_p50_ms": 108.85212599532679, "ttft_p95_ms": 265.2473596011987, "ttft_p99_ms": 272.91192704375135, "itl_mean_ms": 7.264521106390136, "itl_p50_ms": 7.101249275637395, "itl_p95_ms": 10.530224299220105, "itl_p99_ms": 11.874427473184914, "tpot_mean_ms": 7.265778957962821, "tpot_p50_ms": 7.102425023638387, "tpot_p95_ms": 10.531432203184004, "acceptance_rate": 0.12425870324814467, "acceptance_pos_1": 0.4784887142079961, "acceptance_pos_2": 0.21339195505667877, "acceptance_pos_3": 0.10025647003963628, "mean_acceptance_length": 1.8698109227370128 }, { "dataset": "humaneval", "run": "baseline", "method": "baseline", "num_speculative_tokens": null, "requests": 164, "output_tokens": 11593, "request_throughput_rps": 1.5860477240562036, "output_throughput_tps": 112.1161662498998, "speedup_vs_baseline": 1.0, "latency_mean_ms": 630.4013797863018, "latency_p50_ms": 523.5549619974336, "latency_p95_ms": 1166.231724152749, "latency_p99_ms": 1309.8181371219107, "ttft_mean_ms": 107.49666936503561, "ttft_p50_ms": 68.79427550302353, "ttft_p95_ms": 235.7000987394713, "ttft_p99_ms": 257.95137990207877, "itl_mean_ms": 7.297341666050263, "itl_p50_ms": 7.1785622135738, "itl_p95_ms": 8.49481829626492, "itl_p99_ms": 9.019322222237124, "tpot_mean_ms": 7.301280505914361, "tpot_p50_ms": 7.182021026551093, "tpot_p95_ms": 8.49614809654986, "acceptance_rate": null, "acceptance_pos_1": null, "acceptance_pos_2": null, "acceptance_pos_3": null, "mean_acceptance_length": null }, { "dataset": "humaneval", "run": "dspark_k3", "method": "dspark", "num_speculative_tokens": 3, "requests": 164, "output_tokens": 11403, "request_throughput_rps": 2.3328174923608316, "output_throughput_tps": 162.20193820360097, "speedup_vs_baseline": 1.4467310436040344, "latency_mean_ms": 428.57000031148254, "latency_p50_ms": 387.70782500796486, "latency_p95_ms": 818.5407340446543, "latency_p99_ms": 898.805770642066, "ttft_mean_ms": 100.60342578719606, "ttft_p50_ms": 86.00447800563416, "ttft_p95_ms": 181.3961499959987, "ttft_p99_ms": 206.1478519160301, "itl_mean_ms": 5.122034968621708, "itl_p50_ms": 4.931764990461113, "itl_p95_ms": 7.476613033529377, "itl_p99_ms": 8.192469397143373, "tpot_mean_ms": 5.12586804041208, "tpot_p50_ms": 4.935711371615048, "tpot_p95_ms": 7.481607739016129, "acceptance_rate": 0.5768270082990895, "acceptance_pos_1": 0.7553782934493595, "acceptance_pos_2": 0.5569253081943437, "acceptance_pos_3": 0.4181774232535654, "mean_acceptance_length": 2.7304810248972684 }, { "dataset": "humaneval", "run": "dspark_k7", "method": "dspark", "num_speculative_tokens": 7, "requests": 164, "output_tokens": 10873, "request_throughput_rps": 2.251028218808984, "output_throughput_tps": 149.24042575067125, "speedup_vs_baseline": 1.3311231621854054, "latency_mean_ms": 444.1504760051896, "latency_p50_ms": 404.9552284996025, "latency_p95_ms": 816.3704766055158, "latency_p99_ms": 1066.9646934645427, "ttft_mean_ms": 160.0552513159417, "ttft_p50_ms": 160.3144644977874, "ttft_p95_ms": 265.6927762531268, "ttft_p99_ms": 272.09467787863105, "itl_mean_ms": 4.944825696685838, "itl_p50_ms": 4.49954001268619, "itl_p95_ms": 8.732022160872603, "itl_p99_ms": 10.318099194431854, "tpot_mean_ms": 4.949079180467763, "tpot_p50_ms": 4.503242393642178, "tpot_p95_ms": 8.749463808814781, "acceptance_rate": 0.33094945594945596, "acceptance_pos_1": 0.7306511056511057, "acceptance_pos_2": 0.538083538083538, "acceptance_pos_3": 0.382985257985258, "mean_acceptance_length": 3.316646191646192 }, { "dataset": "mbpp", "run": "baseline", "method": "baseline", "num_speculative_tokens": null, "requests": 500, "output_tokens": 61467, "request_throughput_rps": 1.060577545227861, "output_throughput_tps": 130.38103994504186, "speedup_vs_baseline": 1.0, "latency_mean_ms": 942.7637862757256, "latency_p50_ms": 959.554503999243, "latency_p95_ms": 1136.071662903123, "latency_p99_ms": 1164.7922664470389, "ttft_mean_ms": 128.43507730966667, "ttft_p50_ms": 113.83070349984337, "ttft_p95_ms": 256.7646375973709, "ttft_p99_ms": 260.46189150976716, "itl_mean_ms": 6.666311532516699, "itl_p50_ms": 6.6432852756234775, "itl_p95_ms": 7.121261330345493, "itl_p99_ms": 7.26582164827209, "tpot_mean_ms": 6.669662882298301, "tpot_p50_ms": 6.644507716595485, "tpot_p95_ms": 7.122378109070412, "acceptance_rate": null, "acceptance_pos_1": null, "acceptance_pos_2": null, "acceptance_pos_3": null, "mean_acceptance_length": null }, { "dataset": "mbpp", "run": "dspark_k3", "method": "dspark", "num_speculative_tokens": 3, "requests": 500, "output_tokens": 61643, "request_throughput_rps": 1.426522036352137, "output_throughput_tps": 175.87019577370955, "speedup_vs_baseline": 1.3488939484440547, "latency_mean_ms": 700.897503109707, "latency_p50_ms": 703.9270094974199, "latency_p95_ms": 919.0087530580053, "latency_p99_ms": 1045.5224799610733, "ttft_mean_ms": 108.79212251797435, "ttft_p50_ms": 91.88079299929086, "ttft_p95_ms": 204.44850095518615, "ttft_p99_ms": 252.88790728605818, "itl_mean_ms": 5.011328370238571, "itl_p50_ms": 4.792381047239568, "itl_p95_ms": 6.631624132701181, "itl_p99_ms": 11.0604020699849, "tpot_mean_ms": 5.013954450995089, "tpot_p50_ms": 4.79360114569697, "tpot_p95_ms": 6.632888974820347, "acceptance_rate": 0.5095989615358338, "acceptance_pos_1": 0.7010043041606887, "acceptance_pos_2": 0.49940561590489857, "acceptance_pos_3": 0.32838696454191435, "mean_acceptance_length": 2.5287968846075017 }, { "dataset": "mbpp", "run": "dspark_k7", "method": "dspark", "num_speculative_tokens": 7, "requests": 500, "output_tokens": 61199, "request_throughput_rps": 1.4351988286444157, "output_throughput_tps": 175.6654662284192, "speedup_vs_baseline": 1.347323708282014, "latency_mean_ms": 696.6608542179456, "latency_p50_ms": 697.1229435002897, "latency_p95_ms": 978.6413718429683, "latency_p99_ms": 1134.9274124257497, "ttft_mean_ms": 144.74703813757515, "ttft_p50_ms": 139.05306548986118, "ttft_p95_ms": 269.57134354670416, "ttft_p99_ms": 274.25451427989174, "itl_mean_ms": 4.7933126349011435, "itl_p50_ms": 4.529900153696456, "itl_p95_ms": 6.829579977194199, "itl_p99_ms": 12.186756946117372, "tpot_mean_ms": 4.796161684500074, "tpot_p50_ms": 4.531059523688732, "tpot_p95_ms": 6.830801591285134, "acceptance_rate": 0.27433826068703066, "acceptance_pos_1": 0.6736976821349007, "acceptance_pos_2": 0.462388017253638, "acceptance_pos_3": 0.2917950419490923, "mean_acceptance_length": 2.9203678248092144 }, { "dataset": "mt_bench", "run": "baseline", "method": "baseline", "num_speculative_tokens": null, "requests": 160, "output_tokens": 20102, "request_throughput_rps": 1.0145474949999134, "output_throughput_tps": 127.46521090305163, "speedup_vs_baseline": 1.0, "latency_mean_ms": 985.5352811689045, "latency_p50_ms": 984.0788365036133, "latency_p95_ms": 1160.0982246920466, "latency_p99_ms": 1228.7251105705216, "ttft_mean_ms": 135.88225862440595, "ttft_p50_ms": 116.72725749667734, "ttft_p95_ms": 258.65039654891007, "ttft_p99_ms": 261.56884412805084, "itl_mean_ms": 6.802873811459109, "itl_p50_ms": 6.694544555138573, "itl_p95_ms": 7.932162838574364, "itl_p99_ms": 8.945381395555675, "tpot_mean_ms": 6.808100097450948, "tpot_p50_ms": 6.695755173252367, "tpot_p95_ms": 7.93327857724984, "acceptance_rate": null, "acceptance_pos_1": null, "acceptance_pos_2": null, "acceptance_pos_3": null, "mean_acceptance_length": null }, { "dataset": "mt_bench", "run": "dspark_k3", "method": "dspark", "num_speculative_tokens": 3, "requests": 160, "output_tokens": 20102, "request_throughput_rps": 1.146245568010398, "output_throughput_tps": 144.01142755090638, "speedup_vs_baseline": 1.1298096675212783, "latency_mean_ms": 872.2913337876889, "latency_p50_ms": 843.7197764942539, "latency_p95_ms": 1313.481805704941, "latency_p99_ms": 1433.86794267004, "ttft_mean_ms": 97.44770207444162, "ttft_p50_ms": 75.82400399405742, "ttft_p95_ms": 185.91803135059308, "ttft_p99_ms": 229.50114874707637, "itl_mean_ms": 6.340698793466556, "itl_p50_ms": 5.966741342590682, "itl_p95_ms": 10.206342749464907, "itl_p99_ms": 12.089351741436634, "tpot_mean_ms": 6.344304410333824, "tpot_p50_ms": 5.967888015807757, "tpot_p95_ms": 10.32622098520203, "acceptance_rate": 0.3227736367202825, "acceptance_pos_1": 0.5441349548842683, "acceptance_pos_2": 0.2823656335817968, "acceptance_pos_3": 0.14182032169478226, "mean_acceptance_length": 1.9683209101608474 }, { "dataset": "mt_bench", "run": "dspark_k7", "method": "dspark", "num_speculative_tokens": 7, "requests": 160, "output_tokens": 20102, "request_throughput_rps": 1.0518269898218957, "output_throughput_tps": 132.14891343374842, "speedup_vs_baseline": 1.0367449478764772, "latency_mean_ms": 950.6035467624315, "latency_p50_ms": 909.5398534991546, "latency_p95_ms": 1560.6508335607941, "latency_p99_ms": 1760.411608337017, "ttft_mean_ms": 132.69152166194544, "ttft_p50_ms": 122.75661650346592, "ttft_p95_ms": 258.31930338608794, "ttft_p99_ms": 269.76503039040836, "itl_mean_ms": 6.65775048881359, "itl_p50_ms": 6.321849094407714, "itl_p95_ms": 11.558155716175955, "itl_p99_ms": 13.248131914494198, "tpot_mean_ms": 6.6624969313339735, "tpot_p50_ms": 6.32326457077476, "tpot_p95_ms": 11.559374125591408, "acceptance_rate": 0.14785346059984694, "acceptance_pos_1": 0.5276458101688062, "acceptance_pos_2": 0.257859092287476, "acceptance_pos_3": 0.1271606186192257, "mean_acceptance_length": 2.034974224198929 }, { "dataset": "ultrachat", "run": "baseline", "method": "baseline", "num_speculative_tokens": null, "requests": 5000, "output_tokens": 636089, "request_throughput_rps": 0.5205862124506341, "output_throughput_tps": 66.22783265830228, "speedup_vs_baseline": 1.0, "latency_mean_ms": 1920.7680450268758, "latency_p50_ms": 1795.524564004154, "latency_p95_ms": 3165.0052286931896, "latency_p99_ms": 3293.7335572592565, "ttft_mean_ms": 134.13350982365372, "ttft_p50_ms": 127.32078000408364, "ttft_p95_ms": 226.59587289090268, "ttft_p99_ms": 258.95474469623883, "itl_mean_ms": 14.14775085821201, "itl_p50_ms": 13.20624435037479, "itl_p95_ms": 23.33603281298938, "itl_p99_ms": 24.09970887749654, "tpot_mean_ms": 14.149112711234228, "tpot_p50_ms": 13.207468862130167, "tpot_p95_ms": 23.337274902295345, "acceptance_rate": null, "acceptance_pos_1": null, "acceptance_pos_2": null, "acceptance_pos_3": null, "mean_acceptance_length": null }, { "dataset": "ultrachat", "run": "dspark_k3", "method": "dspark", "num_speculative_tokens": 3, "requests": 5000, "output_tokens": 635846, "request_throughput_rps": 0.6599485226793063, "output_throughput_tps": 83.92512567030923, "speedup_vs_baseline": 1.2672183627586737, "latency_mean_ms": 1515.1526615684213, "latency_p50_ms": 1470.6976669913274, "latency_p95_ms": 2205.4712326978915, "latency_p99_ms": 2478.7801955970654, "ttft_mean_ms": 154.38469593704212, "ttft_p50_ms": 149.33674249914475, "ttft_p95_ms": 250.97870194367718, "ttft_p99_ms": 286.67091090654156, "itl_mean_ms": 10.78135369824052, "itl_p50_ms": 10.431902874002185, "itl_p95_ms": 15.61618643974625, "itl_p99_ms": 17.696921136958977, "tpot_mean_ms": 10.78267526592669, "tpot_p50_ms": 10.433149968439395, "tpot_p95_ms": 15.617432073222732, "acceptance_rate": 0.30246536910398036, "acceptance_pos_1": 0.5334846321484933, "acceptance_pos_2": 0.2569174118884798, "acceptance_pos_3": 0.11699406327496806, "mean_acceptance_length": 1.907396107311941 }, { "dataset": "ultrachat", "run": "dspark_k7", "method": "dspark", "num_speculative_tokens": 7, "requests": 5000, "output_tokens": 636040, "request_throughput_rps": 0.6585516278400763, "output_throughput_tps": 83.77303547428042, "speedup_vs_baseline": 1.2649218932846762, "latency_mean_ms": 1518.3681885767612, "latency_p50_ms": 1479.9964584963163, "latency_p95_ms": 2203.905933455099, "latency_p99_ms": 2513.144597544161, "ttft_mean_ms": 165.56593623632216, "ttft_p50_ms": 164.75157599779777, "ttft_p95_ms": 262.971773991012, "ttft_p99_ms": 305.4997507331428, "itl_mean_ms": 10.715100522278119, "itl_p50_ms": 10.43376617324282, "itl_p95_ms": 15.631104971251936, "itl_p99_ms": 17.89630725780433, "tpot_mean_ms": 10.71641033546772, "tpot_p50_ms": 10.434974913300184, "tpot_p95_ms": 15.632339404468743, "acceptance_rate": 0.14185244726020002, "acceptance_pos_1": 0.5312465705748265, "acceptance_pos_2": 0.25502381392840456, "acceptance_pos_3": 0.11516911181415417, "mean_acceptance_length": 1.9929671308214 } ]