[ { "dataset": "math500", "run": "baseline", "method": "baseline", "num_speculative_tokens": null, "requests": 500, "output_tokens": 63527, "request_throughput_rps": 2.0994279500650657, "output_throughput_tps": 266.7407187675668, "speedup_vs_baseline": 1.0, "latency_mean_ms": 1901.9527513722423, "latency_p50_ms": 1894.7456854948541, "latency_p95_ms": 2310.2332794354875, "latency_p99_ms": 2623.0439431988634, "ttft_mean_ms": 211.01336648262804, "ttft_p50_ms": 200.41962849791162, "ttft_p95_ms": 379.0829671488606, "ttft_p99_ms": 495.6532387045445, "itl_mean_ms": 13.40169340837536, "itl_p50_ms": 13.301524035513445, "itl_p95_ms": 16.309320616836498, "itl_p99_ms": 19.31773654766166, "tpot_mean_ms": 13.403480814341385, "tpot_p50_ms": 13.303263444846724, "tpot_p95_ms": 16.311841278199893, "acceptance_rate": null, "acceptance_pos_1": null, "acceptance_pos_2": null, "acceptance_pos_3": null, "mean_acceptance_length": null }, { "dataset": "math500", "run": "dflash_k3", "method": "dflash", "num_speculative_tokens": 3, "requests": 500, "output_tokens": 63537, "request_throughput_rps": 1.4572988369276767, "output_throughput_tps": 185.18479240374757, "speedup_vs_baseline": 0.694250181447229, "latency_mean_ms": 2740.524441106001, "latency_p50_ms": 2728.2846120000104, "latency_p95_ms": 3396.0575988000182, "latency_p99_ms": 3908.6018861800358, "ttft_mean_ms": 232.72850664999817, "ttft_p50_ms": 208.99405350002098, "ttft_p95_ms": 422.7843856499816, "ttft_p99_ms": 635.657571979995, "itl_mean_ms": 19.894394063291394, "itl_p50_ms": 19.884664330708496, "itl_p95_ms": 24.709495220472647, "itl_p99_ms": 27.384374742046813, "tpot_mean_ms": 19.89611259958302, "tpot_p50_ms": 19.88607477952736, "tpot_p95_ms": 24.71110046456739, "acceptance_rate": 0.06285151149944762, "acceptance_pos_1": 0.16639047905995782, "acceptance_pos_2": 0.019716028924374813, "acceptance_pos_3": 0.002448026514010244, "mean_acceptance_length": 1.1885545344983428 }, { "dataset": "math500", "run": "dflash_k7", "method": "dflash", "num_speculative_tokens": 7, "requests": 500, "output_tokens": 63468, "request_throughput_rps": 1.484521663636696, "output_throughput_tps": 188.43924189538765, "speedup_vs_baseline": 0.7064509789358043, "latency_mean_ms": 2690.2326844700183, "latency_p50_ms": 2693.1952875002025, "latency_p95_ms": 3411.2693664001654, "latency_p99_ms": 3642.2530803891277, "ttft_mean_ms": 223.1485508320311, "ttft_p50_ms": 201.97454650042346, "ttft_p95_ms": 410.6845789998714, "ttft_p99_ms": 603.7728062999304, "itl_mean_ms": 19.605501851902595, "itl_p50_ms": 19.610514098427146, "itl_p95_ms": 24.830368157877164, "itl_p99_ms": 27.14365097669306, "tpot_mean_ms": 19.607220136180974, "tpot_p50_ms": 19.612673795280877, "tpot_p95_ms": 24.8317818901585, "acceptance_rate": 0.02742245654217219, "acceptance_pos_1": 0.1686266354080012, "acceptance_pos_2": 0.020192089540951374, "acceptance_pos_3": 0.0026090902215836044, "mean_acceptance_length": 1.1919571957952053 }, { "dataset": "math500", "run": "dflare_k3", "method": "dflare", "num_speculative_tokens": 3, "requests": 500, "output_tokens": 63495, "request_throughput_rps": 1.7148645616582274, "output_throughput_tps": 217.7706506849783, "speedup_vs_baseline": 0.8164132258889946, "latency_mean_ms": 2328.9442384679533, "latency_p50_ms": 2273.6253690000012, "latency_p95_ms": 3331.7489405495494, "latency_p99_ms": 3864.866585709678, "ttft_mean_ms": 279.50968451999506, "ttft_p50_ms": 263.16054300059477, "ttft_p95_ms": 506.23001639960404, "ttft_p99_ms": 776.1183005902055, "itl_mean_ms": 16.27184360810761, "itl_p50_ms": 15.999782673234437, "itl_p95_ms": 23.205805455507797, "itl_p99_ms": 27.393199366532944, "tpot_mean_ms": 16.273605324281704, "tpot_p50_ms": 16.000888346459, "tpot_p95_ms": 23.207303030310158, "acceptance_rate": 0.3935291759720552, "acceptance_pos_1": 0.6488919429651263, "acceptance_pos_2": 0.35275725820305787, "acceptance_pos_3": 0.17893832674798144, "mean_acceptance_length": 2.1805875279161655 }, { "dataset": "math500", "run": "dflare_k7", "method": "dflare", "num_speculative_tokens": 7, "requests": 500, "output_tokens": 63622, "request_throughput_rps": 1.7760438758773052, "output_throughput_tps": 225.9909269421318, "speedup_vs_baseline": 0.8472307039820806, "latency_mean_ms": 2249.2024364179815, "latency_p50_ms": 2185.3041670019593, "latency_p95_ms": 3338.340588499702, "latency_p99_ms": 4000.3402635519888, "ttft_mean_ms": 263.0280147520243, "ttft_p50_ms": 252.19799500155204, "ttft_p95_ms": 475.0075314999774, "ttft_p99_ms": 595.4660286509533, "itl_mean_ms": 15.760428864173193, "itl_p50_ms": 15.345385657476402, "itl_p95_ms": 24.247727375981995, "itl_p99_ms": 29.25197200714273, "tpot_mean_ms": 15.762062569323362, "tpot_p50_ms": 15.347322909449966, "tpot_p95_ms": 24.248864364582953, "acceptance_rate": 0.18693377898042707, "acceptance_pos_1": 0.6466625620223824, "acceptance_pos_2": 0.3548223534098729, "acceptance_pos_3": 0.17529245590525516, "mean_acceptance_length": 2.3085364528629895 }, { "dataset": "math500", "run": "dspark_k3", "method": "dspark", "num_speculative_tokens": 3, "requests": 500, "output_tokens": 63636, "request_throughput_rps": 2.2498034619335257, "output_throughput_tps": 286.33698620720367, "speedup_vs_baseline": 1.0734656018405375, "latency_mean_ms": 1774.1428698560267, "latency_p50_ms": 1733.4975750000012, "latency_p95_ms": 2512.932257099783, "latency_p99_ms": 2941.1046552174953, "ttft_mean_ms": 237.00501048202568, "ttft_p50_ms": 212.508828000864, "ttft_p95_ms": 434.44543489968055, "ttft_p99_ms": 581.4404782277412, "itl_mean_ms": 12.18272455456093, "itl_p50_ms": 11.912186838580821, "itl_p95_ms": 17.877204798435372, "itl_p99_ms": 22.473681951484796, "tpot_mean_ms": 12.18441765344298, "tpot_p50_ms": 11.913581936995776, "tpot_p95_ms": 17.87863581495454, "acceptance_rate": 0.4499175952574226, "acceptance_pos_1": 0.6567411999114456, "acceptance_pos_2": 0.42247804590067156, "acceptance_pos_3": 0.27053353996015056, "mean_acceptance_length": 2.3497527857722678 }, { "dataset": "math500", "run": "dspark_k7", "method": "dspark", "num_speculative_tokens": 7, "requests": 500, "output_tokens": 63621, "request_throughput_rps": 2.316449613127783, "output_throughput_tps": 294.7496816736054, "speedup_vs_baseline": 1.1050044516467135, "latency_mean_ms": 1725.2525323180525, "latency_p50_ms": 1675.7616530012456, "latency_p95_ms": 2646.2700289983313, "latency_p99_ms": 3227.1719797111646, "ttft_mean_ms": 247.76108907210437, "ttft_p50_ms": 221.10979600074643, "ttft_p95_ms": 461.0275230510521, "ttft_p99_ms": 670.7492681724033, "itl_mean_ms": 11.723149998862587, "itl_p50_ms": 11.28927224801989, "itl_p95_ms": 18.88152986101506, "itl_p99_ms": 22.771947493768472, "tpot_mean_ms": 11.72477114787681, "tpot_p50_ms": 11.290766421259073, "tpot_p95_ms": 18.882649400401576, "acceptance_rate": 0.22890383017299992, "acceptance_pos_1": 0.6397510474718301, "acceptance_pos_2": 0.3964121547410812, "acceptance_pos_3": 0.24219989423585406, "mean_acceptance_length": 2.6023268112109994 }, { "dataset": "gsm8k", "run": "baseline", "method": "baseline", "num_speculative_tokens": null, "requests": 1319, "output_tokens": 168502, "request_throughput_rps": 2.196360834095345, "output_throughput_tps": 280.5846802628763, "speedup_vs_baseline": 1.0, "latency_mean_ms": 1819.2423071760666, "latency_p50_ms": 1798.3721370110288, "latency_p95_ms": 2167.7596873778384, "latency_p99_ms": 2310.5863884446435, "ttft_mean_ms": 217.04894016683772, "ttft_p50_ms": 191.49014502181672, "ttft_p95_ms": 398.45379750477116, "ttft_p99_ms": 646.2297889695037, "itl_mean_ms": 12.638841870584082, "itl_p50_ms": 12.576321425240883, "itl_p95_ms": 15.217476899920795, "itl_p99_ms": 16.315808897562086, "tpot_mean_ms": 12.640636354922657, "tpot_p50_ms": 12.58000173247566, "tpot_p95_ms": 15.218959284321164, "acceptance_rate": null, "acceptance_pos_1": null, "acceptance_pos_2": null, "acceptance_pos_3": null, "mean_acceptance_length": null }, { "dataset": "gsm8k", "run": "dflash_k3", "method": "dflash", "num_speculative_tokens": 3, "requests": 1319, "output_tokens": 168356, "request_throughput_rps": 1.4790952868779552, "output_throughput_tps": 188.79042162064067, "speedup_vs_baseline": 0.6728465055318247, "latency_mean_ms": 2702.3829456360854, "latency_p50_ms": 2683.6698380000144, "latency_p95_ms": 3317.1016436999594, "latency_p99_ms": 3597.6961572799746, "ttft_mean_ms": 235.6068064935536, "ttft_p50_ms": 221.15605599992705, "ttft_p95_ms": 424.62412700004944, "ttft_p99_ms": 526.131548720104, "itl_mean_ms": 19.477842147847973, "itl_p50_ms": 19.355762133858388, "itl_p95_ms": 24.183653737845873, "itl_p99_ms": 26.07432056251997, "tpot_mean_ms": 19.479491279517458, "tpot_p50_ms": 19.357246826771867, "tpot_p95_ms": 24.18510518785972, "acceptance_rate": 0.0680806946304376, "acceptance_pos_1": 0.18367244085417656, "acceptance_pos_2": 0.018568116693305686, "acceptance_pos_3": 0.002001526343830547, "mean_acceptance_length": 1.2042420838913128 }, { "dataset": "gsm8k", "run": "dflash_k7", "method": "dflash", "num_speculative_tokens": 7, "requests": 1319, "output_tokens": 168367, "request_throughput_rps": 1.4764153266187332, "output_throughput_tps": 188.46066663898122, "speedup_vs_baseline": 0.6716712632436481, "latency_mean_ms": 2705.295749132689, "latency_p50_ms": 2679.983797999739, "latency_p95_ms": 3477.261450399509, "latency_p99_ms": 4079.732661799678, "ttft_mean_ms": 301.1807209363258, "ttft_p50_ms": 224.2310860001453, "ttft_p95_ms": 834.4522548001792, "ttft_p99_ms": 1980.7203211395963, "itl_mean_ms": 18.978770912763007, "itl_p50_ms": 19.034192118114312, "itl_p95_ms": 23.592571637004227, "itl_p99_ms": 25.352459187087952, "tpot_mean_ms": 18.980617877378993, "tpot_p50_ms": 19.03604929134238, "tpot_p95_ms": 23.594471511024295, "acceptance_rate": 0.027380290750817878, "acceptance_pos_1": 0.17135507958901572, "acceptance_pos_2": 0.018069627919570206, "acceptance_pos_3": 0.001987944081056817, "mean_acceptance_length": 1.1916620352557252 }, { "dataset": "gsm8k", "run": "dflare_k3", "method": "dflare", "num_speculative_tokens": 3, "requests": 1319, "output_tokens": 168415, "request_throughput_rps": 1.574435488473932, "output_throughput_tps": 201.0299869532504, "speedup_vs_baseline": 0.7164681505950642, "latency_mean_ms": 2536.437202144023, "latency_p50_ms": 2473.5267859996384, "latency_p95_ms": 3534.031745998982, "latency_p99_ms": 4064.8575092010615, "ttft_mean_ms": 258.1910506959681, "ttft_p50_ms": 238.57910900005663, "ttft_p95_ms": 484.66444300011057, "ttft_p99_ms": 598.7782853190582, "itl_mean_ms": 17.977036765777857, "itl_p50_ms": 17.445515787392107, "itl_p95_ms": 25.81660302204809, "itl_p99_ms": 30.753264190233132, "tpot_mean_ms": 17.978638165785107, "tpot_p50_ms": 17.446966267718064, "tpot_p95_ms": 25.818113115744246, "acceptance_rate": 0.2794753137068277, "acceptance_pos_1": 0.536425000273502, "acceptance_pos_2": 0.21512575623311125, "acceptance_pos_3": 0.08687518461386984, "mean_acceptance_length": 1.838425941120483 }, { "dataset": "gsm8k", "run": "dflare_k7", "method": "dflare", "num_speculative_tokens": 7, "requests": 1319, "output_tokens": 168400, "request_throughput_rps": 1.2094482721851378, "output_throughput_tps": 154.41325931461503, "speedup_vs_baseline": 0.5503267647041427, "latency_mean_ms": 3304.2710242100056, "latency_p50_ms": 3064.5563209982356, "latency_p95_ms": 5885.097530001435, "latency_p99_ms": 7240.322633720641, "ttft_mean_ms": 1205.8129131272765, "ttft_p50_ms": 677.296675999969, "ttft_p95_ms": 4089.5798977991944, "ttft_p99_ms": 5644.379561060411, "itl_mean_ms": 16.556715234828605, "itl_p50_ms": 16.74034803936521, "itl_p95_ms": 27.050037795280637, "itl_p99_ms": 35.51621275700378, "tpot_mean_ms": 16.558478447758667, "tpot_p50_ms": 16.743846314973666, "tpot_p95_ms": 27.058657145679476, "acceptance_rate": 0.12825457741085033, "acceptance_pos_1": 0.5421524916756024, "acceptance_pos_2": 0.21891754613691516, "acceptance_pos_3": 0.09374118178226762, "mean_acceptance_length": 1.8977820418759523 }, { "dataset": "gsm8k", "run": "dspark_k3", "method": "dspark", "num_speculative_tokens": 3, "requests": 1319, "output_tokens": 168413, "request_throughput_rps": 2.0975172075679485, "output_throughput_tps": 267.81589497963677, "speedup_vs_baseline": 0.954492222200882, "latency_mean_ms": 1904.8592331402729, "latency_p50_ms": 1849.926056998811, "latency_p95_ms": 2717.593596396909, "latency_p99_ms": 3128.5920352824137, "ttft_mean_ms": 232.41419941998183, "ttft_p50_ms": 211.30282399826683, "ttft_p95_ms": 436.5587686021171, "ttft_p99_ms": 608.768046818439, "itl_mean_ms": 13.20091798661006, "itl_p50_ms": 12.809261669308327, "itl_p95_ms": 19.22371915434054, "itl_p99_ms": 22.673929599071368, "tpot_mean_ms": 13.202611622505504, "tpot_p50_ms": 12.810728181104036, "tpot_p95_ms": 19.225053650402195, "acceptance_rate": 0.31145141140312554, "acceptance_pos_1": 0.5333057414177645, "acceptance_pos_2": 0.2575245453082247, "acceptance_pos_3": 0.14352394748338737, "mean_acceptance_length": 1.9343542342093767 }, { "dataset": "gsm8k", "run": "dspark_k7", "method": "dspark", "num_speculative_tokens": 7, "requests": 1319, "output_tokens": 168420, "request_throughput_rps": 2.0069879067415446, "output_throughput_tps": 256.2675536417065, "speedup_vs_baseline": 0.9133340900922055, "latency_mean_ms": 1991.4858750174596, "latency_p50_ms": 1940.2353619989299, "latency_p95_ms": 2946.936268898934, "latency_p99_ms": 3409.506298801133, "ttft_mean_ms": 240.2313964965955, "ttft_p50_ms": 217.0051639986923, "ttft_p95_ms": 457.4054478005564, "ttft_p99_ms": 621.9962791216665, "itl_mean_ms": 13.815387803215751, "itl_p50_ms": 13.491749685057384, "itl_p95_ms": 21.069445844088154, "itl_p99_ms": 24.68408968393126, "tpot_mean_ms": 13.817049546266604, "tpot_p50_ms": 13.493896850398981, "tpot_p95_ms": 21.07079229370886, "acceptance_rate": 0.1463064478504878, "acceptance_pos_1": 0.5319253673998655, "acceptance_pos_2": 0.2490394774757468, "acceptance_pos_3": 0.13306838920372682, "mean_acceptance_length": 2.0241451349534145 }, { "dataset": "humaneval", "run": "baseline", "method": "baseline", "num_speculative_tokens": null, "requests": 164, "output_tokens": 11229, "request_throughput_rps": 2.83651766504482, "output_throughput_tps": 194.21498085846514, "speedup_vs_baseline": 1.0, "latency_mean_ms": 1403.542471638946, "latency_p50_ms": 1171.1717744910857, "latency_p95_ms": 2734.2781511018984, "latency_p99_ms": 3048.235554893327, "ttft_mean_ms": 218.20821551103487, "ttft_p50_ms": 209.54365949728526, "ttft_p95_ms": 420.2904565172503, "ttft_p99_ms": 467.85162955609854, "itl_mean_ms": 17.843567054063506, "itl_p50_ms": 17.024532066210227, "itl_p95_ms": 29.900638077389125, "itl_p99_ms": 38.85421750062092, "tpot_mean_ms": 17.849692097307134, "tpot_p50_ms": 17.028038263717008, "tpot_p95_ms": 29.90535920137036, "acceptance_rate": null, "acceptance_pos_1": null, "acceptance_pos_2": null, "acceptance_pos_3": null, "mean_acceptance_length": null }, { "dataset": "humaneval", "run": "dflash_k3", "method": "dflash", "num_speculative_tokens": 3, "requests": 164, "output_tokens": 11292, "request_throughput_rps": 2.1388107405014773, "output_throughput_tps": 147.26494440087, "speedup_vs_baseline": 0.7582573895686753, "latency_mean_ms": 1856.8446467317099, "latency_p50_ms": 1529.8299214999815, "latency_p95_ms": 3609.9989535499276, "latency_p99_ms": 4076.3032318901037, "ttft_mean_ms": 250.89464046951855, "ttft_p50_ms": 233.70097149984304, "ttft_p95_ms": 497.8459723500346, "ttft_p99_ms": 638.7824368799534, "itl_mean_ms": 24.549073372278432, "itl_p50_ms": 22.60172811232828, "itl_p95_ms": 42.012251182637584, "itl_p99_ms": 51.31418752062339, "tpot_mean_ms": 24.555336995750512, "tpot_p50_ms": 22.603792561644838, "tpot_p95_ms": 42.01500447592228, "acceptance_rate": 0.059506172839506176, "acceptance_pos_1": 0.16412698412698412, "acceptance_pos_2": 0.013544973544973546, "acceptance_pos_3": 0.0008465608465608466, "mean_acceptance_length": 1.1785185185185185 }, { "dataset": "humaneval", "run": "dflash_k7", "method": "dflash", "num_speculative_tokens": 7, "requests": 164, "output_tokens": 11052, "request_throughput_rps": 2.1057562192267367, "output_throughput_tps": 141.90742521276766, "speedup_vs_baseline": 0.7306718801274306, "latency_mean_ms": 1877.2794746829145, "latency_p50_ms": 1504.3869174996871, "latency_p95_ms": 3978.088593899155, "latency_p99_ms": 4819.554965540591, "ttft_mean_ms": 343.89938129273037, "ttft_p50_ms": 254.9212224994335, "ttft_p95_ms": 982.0552947493524, "ttft_p99_ms": 1265.2674350103007, "itl_mean_ms": 22.96417710011784, "itl_p50_ms": 22.33989236389506, "itl_p95_ms": 33.208631611878445, "itl_p99_ms": 43.04405845282055, "tpot_mean_ms": 22.970011297649936, "tpot_p50_ms": 22.34099295422598, "tpot_p95_ms": 33.23066878742338, "acceptance_rate": 0.02684898226160627, "acceptance_pos_1": 0.17398888040989863, "acceptance_pos_2": 0.013081870707511174, "acceptance_pos_3": 0.0008721247138340783, "mean_acceptance_length": 1.1879428758312438 }, { "dataset": "humaneval", "run": "dflare_k3", "method": "dflare", "num_speculative_tokens": 3, "requests": 164, "output_tokens": 11277, "request_throughput_rps": 2.8294040460667382, "output_throughput_tps": 194.55603309447932, "speedup_vs_baseline": 1.0017560552461333, "latency_mean_ms": 1401.6537050121503, "latency_p50_ms": 1262.2241380004198, "latency_p95_ms": 2758.0062766009178, "latency_p99_ms": 3165.725670439934, "ttft_mean_ms": 307.73021359141796, "ttft_p50_ms": 282.3090209994916, "ttft_p95_ms": 627.4767150008302, "ttft_p99_ms": 760.1846476996616, "itl_mean_ms": 17.775962665717213, "itl_p50_ms": 16.24701220755945, "itl_p95_ms": 33.04292447846012, "itl_p99_ms": 49.24142361158533, "tpot_mean_ms": 17.781789938797445, "tpot_p50_ms": 16.250892179999738, "tpot_p95_ms": 33.04544840657803, "acceptance_rate": 0.5873682476943346, "acceptance_pos_1": 0.7734683794466403, "acceptance_pos_2": 0.5642292490118577, "acceptance_pos_3": 0.4244071146245059, "mean_acceptance_length": 2.762104743083004 }, { "dataset": "humaneval", "run": "dflare_k7", "method": "dflare", "num_speculative_tokens": 7, "requests": 164, "output_tokens": 11047, "request_throughput_rps": 3.0665775841960174, "output_throughput_tps": 206.5639181256915, "speedup_vs_baseline": 1.0635838554402026, "latency_mean_ms": 1296.2304299939337, "latency_p50_ms": 1225.3228209992812, "latency_p95_ms": 2684.5432901496665, "latency_p99_ms": 3137.9315633195806, "ttft_mean_ms": 395.1852547253976, "ttft_p50_ms": 324.23020049827755, "ttft_p95_ms": 956.4745560996016, "ttft_p99_ms": 1385.3407927481385, "itl_mean_ms": 16.226717747252554, "itl_p50_ms": 13.57385855245207, "itl_p95_ms": 32.95489575833981, "itl_p99_ms": 75.28418661784087, "tpot_mean_ms": 16.23359719581637, "tpot_p50_ms": 13.576510465236616, "tpot_p95_ms": 32.96468897523092, "acceptance_rate": 0.33047302832995556, "acceptance_pos_1": 0.7790522185330516, "acceptance_pos_2": 0.5644431029278599, "acceptance_pos_3": 0.4044672502263809, "mean_acceptance_length": 3.3133111983096892 }, { "dataset": "humaneval", "run": "dspark_k3", "method": "dspark", "num_speculative_tokens": 3, "requests": 164, "output_tokens": 11346, "request_throughput_rps": 3.393797033660498, "output_throughput_tps": 234.79281185312203, "speedup_vs_baseline": 1.2089325489480554, "latency_mean_ms": 1172.9745371464626, "latency_p50_ms": 1033.1892974991206, "latency_p95_ms": 2107.933460299682, "latency_p99_ms": 3156.528969858848, "ttft_mean_ms": 265.29695418294733, "ttft_p50_ms": 228.4132034983486, "ttft_p95_ms": 502.8849432006609, "ttft_p99_ms": 699.91048091968, "itl_mean_ms": 14.77773082701616, "itl_p50_ms": 13.023184389006781, "itl_p95_ms": 29.633302174848463, "itl_p99_ms": 45.321116141877226, "tpot_mean_ms": 14.783925134182368, "tpot_p50_ms": 13.039449575903792, "tpot_p95_ms": 29.64001845995613, "acceptance_rate": 0.5991372158619546, "acceptance_pos_1": 0.7653061224489796, "acceptance_pos_2": 0.587356893977103, "acceptance_pos_3": 0.44474863115978097, "mean_acceptance_length": 2.7974116475858635 }, { "dataset": "humaneval", "run": "dspark_k7", "method": "dspark", "num_speculative_tokens": 7, "requests": 164, "output_tokens": 11058, "request_throughput_rps": 3.1234019762512726, "output_throughput_tps": 210.6010917889425, "speedup_vs_baseline": 1.08437099372071, "latency_mean_ms": 1276.6294960669388, "latency_p50_ms": 1098.64930799813, "latency_p95_ms": 2616.217871799927, "latency_p99_ms": 3188.0119062716053, "ttft_mean_ms": 564.2550010548891, "ttft_p50_ms": 336.2399385005119, "ttft_p95_ms": 1681.4955957521306, "ttft_p99_ms": 2938.6766085621284, "itl_mean_ms": 12.486812538079404, "itl_p50_ms": 11.807729195672675, "itl_p95_ms": 25.278697270681064, "itl_p99_ms": 43.8447163140126, "tpot_mean_ms": 12.494207252966246, "tpot_p50_ms": 11.83652314342243, "tpot_p95_ms": 25.281766504248417, "acceptance_rate": 0.35094729280775794, "acceptance_pos_1": 0.740100565681961, "acceptance_pos_2": 0.5502828409805154, "acceptance_pos_3": 0.39849151477058453, "mean_acceptance_length": 3.4566310496543053 }, { "dataset": "mbpp", "run": "baseline", "method": "baseline", "num_speculative_tokens": null, "requests": 500, "output_tokens": 61587, "request_throughput_rps": 2.23460184339787, "output_throughput_tps": 275.2448474586892, "speedup_vs_baseline": 1.0, "latency_mean_ms": 1788.0222868800047, "latency_p50_ms": 1819.359868997708, "latency_p95_ms": 2216.346881000209, "latency_p99_ms": 2362.3936853421037, "ttft_mean_ms": 221.07102080830373, "ttft_p50_ms": 204.21197451651096, "ttft_p95_ms": 398.6740397042012, "ttft_p99_ms": 655.3067808030755, "itl_mean_ms": 13.201886877023522, "itl_p50_ms": 12.736825452678412, "itl_p95_ms": 15.664009772456069, "itl_p99_ms": 17.321856500478695, "tpot_mean_ms": 13.205948736768402, "tpot_p50_ms": 12.738972996026057, "tpot_p95_ms": 15.67098670536386, "acceptance_rate": null, "acceptance_pos_1": null, "acceptance_pos_2": null, "acceptance_pos_3": null, "mean_acceptance_length": null }, { "dataset": "mbpp", "run": "dflash_k3", "method": "dflash", "num_speculative_tokens": 3, "requests": 500, "output_tokens": 61808, "request_throughput_rps": 1.5365610042290248, "output_throughput_tps": 189.94352509877513, "speedup_vs_baseline": 0.690089303587357, "latency_mean_ms": 2595.6573488480017, "latency_p50_ms": 2617.603001500015, "latency_p95_ms": 3320.9663893999277, "latency_p99_ms": 3516.577585000016, "ttft_mean_ms": 234.1106791739985, "ttft_p50_ms": 218.79009800011318, "ttft_p95_ms": 439.53066730014143, "ttft_p99_ms": 544.9614302001191, "itl_mean_ms": 19.14316073130201, "itl_p50_ms": 18.9773965472434, "itl_p95_ms": 23.96081993622043, "itl_p99_ms": 25.695518167638486, "tpot_mean_ms": 19.146981896371145, "tpot_p50_ms": 18.978798267717032, "tpot_p95_ms": 23.96214876692921, "acceptance_rate": 0.06462320419065401, "acceptance_pos_1": 0.1824529329391629, "acceptance_pos_2": 0.01106659405632488, "acceptance_pos_3": 0.0003500855764742493, "mean_acceptance_length": 1.1938696125719621 }, { "dataset": "mbpp", "run": "dflash_k7", "method": "dflash", "num_speculative_tokens": 7, "requests": 500, "output_tokens": 61377, "request_throughput_rps": 1.387242813763742, "output_throughput_tps": 170.2896043607544, "speedup_vs_baseline": 0.6186840768611036, "latency_mean_ms": 2875.578809605966, "latency_p50_ms": 2855.830152499948, "latency_p95_ms": 4074.322934499559, "latency_p99_ms": 4764.013334749542, "ttft_mean_ms": 634.0442261159515, "ttft_p50_ms": 392.3641729988958, "ttft_p95_ms": 1885.916530999566, "ttft_p99_ms": 2897.4566687989864, "itl_mean_ms": 18.201249949891572, "itl_p50_ms": 18.463196299214943, "itl_p95_ms": 24.69457239645406, "itl_p99_ms": 30.63269140497326, "tpot_mean_ms": 18.205769807911185, "tpot_p50_ms": 18.464951303145412, "tpot_p95_ms": 24.695845573618083, "acceptance_rate": 0.028501713363674443, "acceptance_pos_1": 0.1877840964993408, "acceptance_pos_2": 0.011255632735787795, "acceptance_pos_3": 0.0004525866309844743, "mean_acceptance_length": 1.199511993545721 }, { "dataset": "mbpp", "run": "dflare_k3", "method": "dflare", "num_speculative_tokens": 3, "requests": 500, "output_tokens": 61640, "request_throughput_rps": 1.9801835796759275, "output_throughput_tps": 244.11703170244834, "speedup_vs_baseline": 0.8869086340992713, "latency_mean_ms": 2016.50626224402, "latency_p50_ms": 2021.2945930006754, "latency_p95_ms": 2791.478140551044, "latency_p99_ms": 3086.5239088800263, "ttft_mean_ms": 280.0661946479813, "ttft_p50_ms": 256.2331505005204, "ttft_p95_ms": 557.8820319009537, "ttft_p99_ms": 769.5084362501256, "itl_mean_ms": 14.264965995399256, "itl_p50_ms": 14.109097578741029, "itl_p95_ms": 19.439541319448956, "itl_p99_ms": 23.81797452189929, "tpot_mean_ms": 14.269046870264877, "tpot_p50_ms": 14.11056187795813, "tpot_p95_ms": 19.634308889327635, "acceptance_rate": 0.47048386886358, "acceptance_pos_1": 0.6888967163711792, "acceptance_pos_2": 0.44499236820476695, "acceptance_pos_3": 0.2775625220147939, "mean_acceptance_length": 2.41145160659074 }, { "dataset": "mbpp", "run": "dflare_k7", "method": "dflare", "num_speculative_tokens": 7, "requests": 500, "output_tokens": 61475, "request_throughput_rps": 1.503343533132173, "output_throughput_tps": 184.83608739860065, "speedup_vs_baseline": 0.6715333242572042, "latency_mean_ms": 2657.845550606042, "latency_p50_ms": 2483.5515449995, "latency_p95_ms": 4488.94517174831, "latency_p99_ms": 5523.38156365822, "ttft_mean_ms": 1190.47147037393, "ttft_p50_ms": 932.4905619996571, "ttft_p95_ms": 3259.4635773521077, "ttft_p99_ms": 4279.743841777054, "itl_mean_ms": 12.451201825429687, "itl_p50_ms": 12.228331582672416, "itl_p95_ms": 22.859363721256052, "itl_p99_ms": 36.967242600596585, "tpot_mean_ms": 12.456394792296264, "tpot_p50_ms": 12.230752885830222, "tpot_p95_ms": 22.860772319684717, "acceptance_rate": 0.24657171832973235, "acceptance_pos_1": 0.6712818025486132, "acceptance_pos_2": 0.42739979716918736, "acceptance_pos_3": 0.2544644825609595, "mean_acceptance_length": 2.7260020283081268 }, { "dataset": "mbpp", "run": "dspark_k3", "method": "dspark", "num_speculative_tokens": 3, "requests": 500, "output_tokens": 61981, "request_throughput_rps": 2.507209609073491, "output_throughput_tps": 310.7987175599681, "speedup_vs_baseline": 1.1291717916958104, "latency_mean_ms": 1592.6791832899762, "latency_p50_ms": 1603.4026140005153, "latency_p95_ms": 2178.7681314988727, "latency_p99_ms": 2552.1267430388243, "ttft_mean_ms": 240.7260946480019, "ttft_p50_ms": 224.12168300070334, "ttft_p95_ms": 455.65863744814123, "ttft_p99_ms": 601.2736888401558, "itl_mean_ms": 11.346454396871485, "itl_p50_ms": 10.845360450802424, "itl_p95_ms": 15.682580235835754, "itl_p99_ms": 23.395322236705972, "tpot_mean_ms": 11.350096012527779, "tpot_p50_ms": 10.847383575174785, "tpot_p95_ms": 15.684506253152122, "acceptance_rate": 0.5228963898667771, "acceptance_pos_1": 0.7125284738041002, "acceptance_pos_2": 0.5082211638020294, "acceptance_pos_3": 0.3479395319942017, "mean_acceptance_length": 2.568689169600331 }, { "dataset": "mbpp", "run": "dspark_k7", "method": "dspark", "num_speculative_tokens": 7, "requests": 500, "output_tokens": 61632, "request_throughput_rps": 2.2269171937357544, "output_throughput_tps": 274.498720968644, "speedup_vs_baseline": 0.997289226312739, "latency_mean_ms": 1794.3305799321024, "latency_p50_ms": 1694.5109794996824, "latency_p95_ms": 3171.435100350026, "latency_p99_ms": 4215.3635372184735, "ttft_mean_ms": 643.143013108056, "ttft_p50_ms": 362.4557425009698, "ttft_p95_ms": 2119.0080873015154, "ttft_p99_ms": 3187.935962892621, "itl_mean_ms": 10.101565495090881, "itl_p50_ms": 9.322213830710384, "itl_p95_ms": 14.88066484486098, "itl_p99_ms": 19.67402437656759, "tpot_mean_ms": 10.105878412845934, "tpot_p50_ms": 9.323771708668072, "tpot_p95_ms": 14.882234868113521, "acceptance_rate": 0.28831800034716193, "acceptance_pos_1": 0.6831591737545565, "acceptance_pos_2": 0.47018226002430136, "acceptance_pos_3": 0.3054678007290401, "mean_acceptance_length": 3.0182260024301337 }, { "dataset": "mt_bench", "run": "baseline", "method": "baseline", "num_speculative_tokens": null, "requests": 160, "output_tokens": 20102, "request_throughput_rps": 2.461309935234548, "output_throughput_tps": 309.2328269880305, "speedup_vs_baseline": 1.0, "latency_mean_ms": 1607.6186366983165, "latency_p50_ms": 1548.2021450006869, "latency_p95_ms": 2284.110534451611, "latency_p99_ms": 2534.602298826794, "ttft_mean_ms": 320.49049263077904, "ttft_p50_ms": 293.138627501321, "ttft_p95_ms": 629.1677434302974, "ttft_p99_ms": 759.140422597702, "itl_mean_ms": 11.007379475699068, "itl_p50_ms": 9.985256696983395, "itl_p95_ms": 15.183904158686165, "itl_p99_ms": 26.580996584276846, "tpot_mean_ms": 11.014195983013959, "tpot_p50_ms": 9.988513000022554, "tpot_p95_ms": 15.18576166809785, "acceptance_rate": null, "acceptance_pos_1": null, "acceptance_pos_2": null, "acceptance_pos_3": null, "mean_acceptance_length": null }, { "dataset": "mt_bench", "run": "dflash_k3", "method": "dflash", "num_speculative_tokens": 3, "requests": 160, "output_tokens": 20228, "request_throughput_rps": 1.5473009629136512, "output_throughput_tps": 195.61752423635835, "speedup_vs_baseline": 0.632589774318915, "latency_mean_ms": 2573.229441274998, "latency_p50_ms": 2591.1725164999098, "latency_p95_ms": 3166.3267970500783, "latency_p99_ms": 3608.086452589978, "ttft_mean_ms": 221.35167090626453, "ttft_p50_ms": 203.15486550020978, "ttft_p95_ms": 448.1615023500803, "ttft_p99_ms": 696.169536259867, "itl_mean_ms": 18.676105907776027, "itl_p50_ms": 18.64073133858227, "itl_p95_ms": 23.103904855904904, "itl_p99_ms": 25.284028265118394, "tpot_mean_ms": 18.68020851087627, "tpot_p50_ms": 18.641842641731216, "tpot_p95_ms": 23.105400170866087, "acceptance_rate": 0.060748671490479834, "acceptance_pos_1": 0.16453909053473734, "acceptance_pos_2": 0.01705982704864992, "acceptance_pos_3": 0.0006470968880522384, "mean_acceptance_length": 1.1822460144714395 }, { "dataset": "mt_bench", "run": "dflash_k7", "method": "dflash", "num_speculative_tokens": 7, "requests": 160, "output_tokens": 20102, "request_throughput_rps": 1.4356652700068615, "output_throughput_tps": 180.37339536048705, "speedup_vs_baseline": 0.5832931681844656, "latency_mean_ms": 2753.9001531249596, "latency_p50_ms": 2798.7496705000012, "latency_p95_ms": 3791.8684454991308, "latency_p99_ms": 4289.662929459572, "ttft_mean_ms": 381.8320665374472, "ttft_p50_ms": 295.44119050024165, "ttft_p95_ms": 1026.5876399000251, "ttft_p99_ms": 1367.1522384703346, "itl_mean_ms": 21.709309200685574, "itl_p50_ms": 18.78574032282878, "itl_p95_ms": 25.575840727953157, "itl_p99_ms": 59.525236419324514, "tpot_mean_ms": 21.713930513288513, "tpot_p50_ms": 18.787575688973476, "tpot_p95_ms": 25.57992160196576, "acceptance_rate": 0.024768061794215188, "acceptance_pos_1": 0.15474581251836614, "acceptance_pos_2": 0.016044666470761092, "acceptance_pos_3": 0.0012342051131354686, "mean_acceptance_length": 1.1733764325595064 }, { "dataset": "mt_bench", "run": "dflare_k3", "method": "dflare", "num_speculative_tokens": 3, "requests": 160, "output_tokens": 20228, "request_throughput_rps": 1.7821635622658862, "output_throughput_tps": 225.31002835946467, "speedup_vs_baseline": 0.7286096710818666, "latency_mean_ms": 2231.2470228500047, "latency_p50_ms": 2197.780867999427, "latency_p95_ms": 3333.7097092000463, "latency_p99_ms": 3700.461037129698, "ttft_mean_ms": 248.96705823127832, "ttft_p50_ms": 215.34929249992274, "ttft_p95_ms": 520.0022110006103, "ttft_p99_ms": 662.6540237599145, "itl_mean_ms": 15.77862160127276, "itl_p50_ms": 15.232473944878599, "itl_p95_ms": 23.602753981101593, "itl_p99_ms": 27.57565280062838, "tpot_mean_ms": 15.782222927801888, "tpot_p50_ms": 15.23403437795065, "tpot_p95_ms": 23.60411872952008, "acceptance_rate": 0.3229694606887589, "acceptance_pos_1": 0.5750487329434698, "acceptance_pos_2": 0.27573099415204677, "acceptance_pos_3": 0.11812865497076024, "mean_acceptance_length": 1.9689083820662767 }, { "dataset": "mt_bench", "run": "dflare_k7", "method": "dflare", "num_speculative_tokens": 7, "requests": 160, "output_tokens": 20102, "request_throughput_rps": 1.0658824478410331, "output_throughput_tps": 133.9148060406278, "speedup_vs_baseline": 0.4330549487419434, "latency_mean_ms": 3752.528322143644, "latency_p50_ms": 3464.8722160018224, "latency_p95_ms": 6717.973978601185, "latency_p99_ms": 7876.092720710149, "ttft_mean_ms": 1934.0097326686418, "ttft_p50_ms": 1588.4214640009304, "ttft_p95_ms": 4820.7839886515985, "ttft_p99_ms": 6494.002312830161, "itl_mean_ms": 14.317043586959333, "itl_p50_ms": 13.67209011023508, "itl_p95_ms": 29.82729992639314, "itl_p99_ms": 39.377268431499964, "tpot_mean_ms": 14.323458562974045, "tpot_p50_ms": 13.673784755900577, "tpot_p95_ms": 29.829240646451908, "acceptance_rate": 0.14578666207134242, "acceptance_pos_1": 0.5689585846401287, "acceptance_pos_2": 0.26940088459991957, "acceptance_pos_3": 0.11540008041817451, "mean_acceptance_length": 2.020506634499397 }, { "dataset": "mt_bench", "run": "dspark_k3", "method": "dspark", "num_speculative_tokens": 3, "requests": 160, "output_tokens": 20228, "request_throughput_rps": 2.2379435718508462, "output_throughput_tps": 282.9320160712432, "speedup_vs_baseline": 0.9149481923605564, "latency_mean_ms": 1771.671463362577, "latency_p50_ms": 1775.1809370001865, "latency_p95_ms": 2604.8437047013063, "latency_p99_ms": 3118.5169013906484, "ttft_mean_ms": 252.11589151258522, "ttft_p50_ms": 227.79171349975513, "ttft_p95_ms": 503.8048422973585, "ttft_p99_ms": 627.0037127103565, "itl_mean_ms": 12.360936149822532, "itl_p50_ms": 11.83748445670406, "itl_p95_ms": 19.148658472828007, "itl_p99_ms": 25.009972698654565, "tpot_mean_ms": 12.364664553850584, "tpot_p50_ms": 11.838727370086703, "tpot_p95_ms": 19.150851622824714, "acceptance_rate": 0.36540854417254254, "acceptance_pos_1": 0.5802571547075902, "acceptance_pos_2": 0.3253836582330983, "acceptance_pos_3": 0.19058481957693904, "mean_acceptance_length": 2.0962256325176276 }, { "dataset": "mt_bench", "run": "dspark_k7", "method": "dspark", "num_speculative_tokens": 7, "requests": 160, "output_tokens": 20102, "request_throughput_rps": 1.784863821370322, "output_throughput_tps": 224.24582835741384, "speedup_vs_baseline": 0.7251682511898833, "latency_mean_ms": 2225.1736813125035, "latency_p50_ms": 2117.7182369992806, "latency_p95_ms": 3657.939555099983, "latency_p99_ms": 4345.6770745698295, "ttft_mean_ms": 838.8437911749861, "ttft_p50_ms": 589.9439045006147, "ttft_p95_ms": 2441.8948530525054, "ttft_p99_ms": 2773.190703780456, "itl_mean_ms": 11.426459321042842, "itl_p50_ms": 10.71108703149011, "itl_p95_ms": 22.870212017662954, "itl_p99_ms": 25.594833969671022, "tpot_mean_ms": 11.431539335517765, "tpot_p50_ms": 10.713255098423582, "tpot_p95_ms": 23.0010987298119, "acceptance_rate": 0.17215237263282687, "acceptance_pos_1": 0.5529591613889495, "acceptance_pos_2": 0.2997379340467351, "acceptance_pos_3": 0.16739462764795807, "mean_acceptance_length": 2.205066608429788 }, { "dataset": "ultrachat", "run": "baseline", "method": "baseline", "num_speculative_tokens": null, "requests": 5000, "output_tokens": 635569, "request_throughput_rps": 1.2230023684575178, "output_throughput_tps": 155.4604784636352, "speedup_vs_baseline": 1.0, "latency_mean_ms": 3269.5651201336645, "latency_p50_ms": 3276.8055605120026, "latency_p95_ms": 3934.875892131822, "latency_p99_ms": 4087.044628756121, "ttft_mean_ms": 231.24218063543086, "ttft_p50_ms": 218.61417350010015, "ttft_p95_ms": 386.9901933081565, "ttft_p99_ms": 553.9115706225857, "itl_mean_ms": 24.08620830909222, "itl_p50_ms": 24.040817992012823, "itl_p95_ms": 28.90904858166638, "itl_p99_ms": 29.91989293644905, "tpot_mean_ms": 24.087996070253364, "tpot_p50_ms": 24.042644452736308, "tpot_p95_ms": 28.910471471294663, "acceptance_rate": null, "acceptance_pos_1": null, "acceptance_pos_2": null, "acceptance_pos_3": null, "mean_acceptance_length": null }, { "dataset": "ultrachat", "run": "dflash_k3", "method": "dflash", "num_speculative_tokens": 3, "requests": 5000, "output_tokens": 636175, "request_throughput_rps": 1.0622074332559788, "output_throughput_tps": 135.14996277032446, "speedup_vs_baseline": 0.8693525461002507, "latency_mean_ms": 3763.952480385199, "latency_p50_ms": 3769.2712649998157, "latency_p95_ms": 4515.89870830021, "latency_p99_ms": 4805.0974385998825, "ttft_mean_ms": 257.6036252754003, "ttft_p50_ms": 246.59032999988995, "ttft_p95_ms": 405.558623099887, "ttft_p99_ms": 540.4809878199559, "itl_mean_ms": 27.764084011389272, "itl_p50_ms": 27.68626928346698, "itl_p95_ms": 33.282436218505644, "itl_p99_ms": 35.5445708588995, "tpot_mean_ms": 27.765764208304795, "tpot_p50_ms": 27.687906570868, "tpot_p95_ms": 33.28350510590754, "acceptance_rate": 0.07093823412968715, "acceptance_pos_1": 0.1840484911746139, "acceptance_pos_2": 0.02574222849873457, "acceptance_pos_3": 0.0030239827157130055, "mean_acceptance_length": 1.2128147023890614 }, { "dataset": "ultrachat", "run": "dflash_k7", "method": "dflash", "num_speculative_tokens": 7, "requests": 5000, "output_tokens": 636064, "request_throughput_rps": 1.0536069317356047, "output_throughput_tps": 134.03228788549515, "speedup_vs_baseline": 0.8621630990081349, "latency_mean_ms": 3794.9240800698, "latency_p50_ms": 3797.6578120005797, "latency_p95_ms": 4543.265116648946, "latency_p99_ms": 4838.748624158453, "ttft_mean_ms": 255.93729505000047, "ttft_p50_ms": 246.4688734989977, "ttft_p95_ms": 399.62789619985415, "ttft_p99_ms": 533.8578004602641, "itl_mean_ms": 28.02829548006744, "itl_p50_ms": 27.958994283470407, "itl_p95_ms": 33.55774176338532, "itl_p99_ms": 35.83546665536011, "tpot_mean_ms": 28.030316558857468, "tpot_p50_ms": 27.961111295281434, "tpot_p95_ms": 33.55949413779617, "acceptance_rate": 0.030697806232490035, "acceptance_pos_1": 0.1853871675887231, "acceptance_pos_2": 0.025839366107025186, "acceptance_pos_3": 0.0030756573256390638, "mean_acceptance_length": 1.2148846436274303 }, { "dataset": "ultrachat", "run": "dflare_k3", "method": "dflare", "num_speculative_tokens": 3, "requests": 5000, "output_tokens": 636228, "request_throughput_rps": 1.267241365085764, "output_throughput_tps": 161.25088784515708, "speedup_vs_baseline": 1.0372468259376697, "latency_mean_ms": 3155.8181194270032, "latency_p50_ms": 3158.2490724995296, "latency_p95_ms": 4013.4220384479713, "latency_p99_ms": 4389.7799070399205, "ttft_mean_ms": 282.9019767206311, "ttft_p50_ms": 270.00340999984473, "ttft_p95_ms": 456.05508385024234, "ttft_p99_ms": 595.7626636705755, "itl_mean_ms": 22.751411578135976, "itl_p50_ms": 22.677721858262814, "itl_p95_ms": 29.19536946929717, "itl_p99_ms": 32.0268499520443, "tpot_mean_ms": 22.753117775499057, "tpot_p50_ms": 22.679262984259076, "tpot_p95_ms": 29.196810057103683, "acceptance_rate": 0.31057058500760176, "acceptance_pos_1": 0.5710008489248474, "acceptance_pos_2": 0.2600387645252867, "acceptance_pos_3": 0.10067214157267132, "mean_acceptance_length": 1.9317117550228053 }, { "dataset": "ultrachat", "run": "dflare_k7", "method": "dflare", "num_speculative_tokens": 7, "requests": 5000, "output_tokens": 635796, "request_throughput_rps": 1.2415631421695674, "output_throughput_tps": 157.87617590776844, "speedup_vs_baseline": 1.0155389811481785, "latency_mean_ms": 3221.0815614180046, "latency_p50_ms": 3164.543153499835, "latency_p95_ms": 4123.292088050039, "latency_p99_ms": 4616.265497698114, "ttft_mean_ms": 352.63900740198807, "ttft_p50_ms": 287.9059334991325, "ttft_p95_ms": 612.3089999979129, "ttft_p99_ms": 1196.3210188611995, "itl_mean_ms": 22.7251370702789, "itl_p50_ms": 22.450031862195097, "itl_p95_ms": 29.62520190592225, "itl_p99_ms": 33.215207786221775, "tpot_mean_ms": 22.72683404563422, "tpot_p50_ms": 22.451526275598177, "tpot_p95_ms": 29.626681563398265, "acceptance_rate": 0.1436652803641537, "acceptance_pos_1": 0.5763152990672803, "acceptance_pos_2": 0.26589619110489227, "acceptance_pos_3": 0.10326562455582894, "mean_acceptance_length": 2.005656962549076 }, { "dataset": "ultrachat", "run": "dspark_k3", "method": "dspark", "num_speculative_tokens": 3, "requests": 5000, "output_tokens": 636215, "request_throughput_rps": 1.5004265390600509, "output_throughput_tps": 190.91877410961806, "speedup_vs_baseline": 1.2280855944636573, "latency_mean_ms": 2665.1578789277846, "latency_p50_ms": 2665.9706919999735, "latency_p95_ms": 3441.952741001842, "latency_p99_ms": 3755.308294759525, "ttft_mean_ms": 264.3723180442066, "ttft_p50_ms": 253.26144649989146, "ttft_p95_ms": 429.1125219519018, "ttft_p99_ms": 559.1942446103716, "itl_mean_ms": 19.008812154313944, "itl_p50_ms": 18.913669216526902, "itl_p95_ms": 24.80109081614243, "itl_p99_ms": 27.068464386929747, "tpot_mean_ms": 19.010452079968594, "tpot_p50_ms": 18.915089492132775, "tpot_p95_ms": 24.80313036299977, "acceptance_rate": 0.32401200895393567, "acceptance_pos_1": 0.5557804733415505, "acceptance_pos_2": 0.28182543551086175, "acceptance_pos_3": 0.1344301180093948, "mean_acceptance_length": 1.9720360268618071 }, { "dataset": "ultrachat", "run": "dspark_k7", "method": "dspark", "num_speculative_tokens": 7, "requests": 5000, "output_tokens": 636157, "request_throughput_rps": 1.4902242443540412, "output_throughput_tps": 189.60331692310675, "speedup_vs_baseline": 1.2196239121151176, "latency_mean_ms": 2683.3432791144196, "latency_p50_ms": 2638.018617002672, "latency_p95_ms": 3469.7974141512536, "latency_p99_ms": 3857.7436888694515, "ttft_mean_ms": 287.66377756739894, "ttft_p50_ms": 259.45143649914826, "ttft_p95_ms": 433.6856615469515, "ttft_p99_ms": 568.5856753819111, "itl_mean_ms": 18.969870524426348, "itl_p50_ms": 18.7173821692899, "itl_p95_ms": 25.069323533594044, "itl_p99_ms": 28.087401625863414, "tpot_mean_ms": 18.971464480602076, "tpot_p50_ms": 18.718672405515193, "tpot_p95_ms": 25.071671481768842, "acceptance_rate": 0.15400452166690387, "acceptance_pos_1": 0.5521141942543566, "acceptance_pos_2": 0.27944601414362474, "acceptance_pos_3": 0.13266262718646393, "mean_acceptance_length": 2.078031651668327 } ]