| [ |
| { |
| "dataset": "math500", |
| "run": "baseline", |
| "method": "baseline", |
| "num_speculative_tokens": null, |
| "requests": 500, |
| "output_tokens": 63527, |
| "request_throughput_rps": 2.575660280906157, |
| "output_throughput_tps": 327.24794133025085, |
| "speedup_vs_baseline": 1.0, |
| "latency_mean_ms": 1549.881507652055, |
| "latency_p50_ms": 1541.6923305019736, |
| "latency_p95_ms": 1901.3991399056977, |
| "latency_p99_ms": 2073.3104983172957, |
| "ttft_mean_ms": 147.72230004466837, |
| "ttft_p50_ms": 128.53012699633837, |
| "ttft_p95_ms": 277.2130677534733, |
| "ttft_p99_ms": 387.54749131272547, |
| "itl_mean_ms": 11.11915976237464, |
| "itl_p50_ms": 11.026409515830519, |
| "itl_p95_ms": 13.801987814431913, |
| "itl_p99_ms": 15.665761921923757, |
| "tpot_mean_ms": 11.12106789862526, |
| "tpot_p50_ms": 11.028098413518146, |
| "tpot_p95_ms": 13.803411649602399, |
| "acceptance_rate": null, |
| "acceptance_pos_1": null, |
| "acceptance_pos_2": null, |
| "acceptance_pos_3": null, |
| "mean_acceptance_length": null |
| }, |
| { |
| "dataset": "math500", |
| "run": "dspark_k3", |
| "method": "dspark", |
| "num_speculative_tokens": 3, |
| "requests": 500, |
| "output_tokens": 63539, |
| "request_throughput_rps": 3.34003084405518, |
| "output_throughput_tps": 424.44443960084413, |
| "speedup_vs_baseline": 1.2970117944073019, |
| "latency_mean_ms": 1195.6849749402609, |
| "latency_p50_ms": 1169.5264599984512, |
| "latency_p95_ms": 1773.1451036757787, |
| "latency_p99_ms": 1944.9941020199901, |
| "ttft_mean_ms": 143.30231556599028, |
| "ttft_p50_ms": 116.45177099853754, |
| "ttft_p95_ms": 297.67805395094905, |
| "ttft_p99_ms": 373.04166853777133, |
| "itl_mean_ms": 8.352826905473885, |
| "itl_p50_ms": 8.158615838546549, |
| "itl_p95_ms": 12.354124409472469, |
| "itl_p99_ms": 14.635059152750648, |
| "tpot_mean_ms": 8.35462870098348, |
| "tpot_p50_ms": 8.163252110248358, |
| "tpot_p95_ms": 12.355534362218949, |
| "acceptance_rate": 0.415486407892723, |
| "acceptance_pos_1": 0.6332426800409706, |
| "acceptance_pos_2": 0.38554727510330944, |
| "acceptance_pos_3": 0.22766926853388902, |
| "mean_acceptance_length": 2.246459223678169 |
| }, |
| { |
| "dataset": "math500", |
| "run": "dspark_k7", |
| "method": "dspark", |
| "num_speculative_tokens": 7, |
| "requests": 500, |
| "output_tokens": 63493, |
| "request_throughput_rps": 3.1055917217703812, |
| "output_throughput_tps": 394.36667038073364, |
| "speedup_vs_baseline": 1.2051005386852782, |
| "latency_mean_ms": 1286.8654769104323, |
| "latency_p50_ms": 1246.8089515023166, |
| "latency_p95_ms": 1958.8731202602503, |
| "latency_p99_ms": 2254.721420374407, |
| "ttft_mean_ms": 169.23283278750023, |
| "ttft_p50_ms": 150.04799248708878, |
| "ttft_p95_ms": 310.4628508415771, |
| "ttft_p99_ms": 383.799409790663, |
| "itl_mean_ms": 8.871197280140835, |
| "itl_p50_ms": 8.589971834756103, |
| "itl_p95_ms": 13.716522226358514, |
| "itl_p99_ms": 15.951512530517089, |
| "tpot_mean_ms": 8.87296793212422, |
| "tpot_p50_ms": 8.60067761416536, |
| "tpot_p95_ms": 13.71767588804694, |
| "acceptance_rate": 0.20277687384748888, |
| "acceptance_pos_1": 0.6178056188306758, |
| "acceptance_pos_2": 0.36526195899772207, |
| "acceptance_pos_3": 0.20611237661351556, |
| "mean_acceptance_length": 2.4194381169324224 |
| }, |
| { |
| "dataset": "gsm8k", |
| "run": "baseline", |
| "method": "baseline", |
| "num_speculative_tokens": null, |
| "requests": 1319, |
| "output_tokens": 168533, |
| "request_throughput_rps": 2.699786987185922, |
| "output_throughput_tps": 344.9607280602009, |
| "speedup_vs_baseline": 1.0, |
| "latency_mean_ms": 1479.6602965273742, |
| "latency_p50_ms": 1454.6987209760118, |
| "latency_p95_ms": 1803.740513609955, |
| "latency_p99_ms": 1914.7201473527819, |
| "ttft_mean_ms": 168.7780613106763, |
| "ttft_p50_ms": 145.47203300753608, |
| "ttft_p95_ms": 336.32914681511363, |
| "ttft_p99_ms": 491.5234538837098, |
| "itl_mean_ms": 10.337075278453744, |
| "itl_p50_ms": 10.18635707075055, |
| "itl_p95_ms": 12.4034695235158, |
| "itl_p99_ms": 13.359812111263784, |
| "tpot_mean_ms": 10.338842536074479, |
| "tpot_p50_ms": 10.187574787391716, |
| "tpot_p95_ms": 12.40562485355891, |
| "acceptance_rate": null, |
| "acceptance_pos_1": null, |
| "acceptance_pos_2": null, |
| "acceptance_pos_3": null, |
| "mean_acceptance_length": null |
| }, |
| { |
| "dataset": "gsm8k", |
| "run": "dspark_k3", |
| "method": "dspark", |
| "num_speculative_tokens": 3, |
| "requests": 1319, |
| "output_tokens": 168415, |
| "request_throughput_rps": 2.95131608515978, |
| "output_throughput_tps": 376.8354044595787, |
| "speedup_vs_baseline": 1.0924008845256588, |
| "latency_mean_ms": 1352.7894092793806, |
| "latency_p50_ms": 1323.1997079856228, |
| "latency_p95_ms": 1901.4164583000816, |
| "latency_p99_ms": 2116.5834879834433, |
| "ttft_mean_ms": 146.35591101451402, |
| "ttft_p50_ms": 119.20796197955497, |
| "ttft_p95_ms": 300.50935930921696, |
| "ttft_p99_ms": 387.0458965114083, |
| "itl_mean_ms": 9.517948084655174, |
| "itl_p50_ms": 9.351653133988322, |
| "itl_p95_ms": 13.592088056107666, |
| "itl_p99_ms": 15.135234009802675, |
| "tpot_mean_ms": 9.519542839852642, |
| "tpot_p50_ms": 9.352777228419354, |
| "tpot_p95_ms": 13.593510024599716, |
| "acceptance_rate": 0.27039498963955716, |
| "acceptance_pos_1": 0.48590297454239884, |
| "acceptance_pos_2": 0.220014867325282, |
| "acceptance_pos_3": 0.10526712705099062, |
| "mean_acceptance_length": 1.8111849689186714 |
| }, |
| { |
| "dataset": "gsm8k", |
| "run": "dspark_k7", |
| "method": "dspark", |
| "num_speculative_tokens": 7, |
| "requests": 1319, |
| "output_tokens": 168438, |
| "request_throughput_rps": 2.833061661782182, |
| "output_throughput_tps": 361.7856256158205, |
| "speedup_vs_baseline": 1.0487733709579932, |
| "latency_mean_ms": 1409.892868064269, |
| "latency_p50_ms": 1389.3212949915323, |
| "latency_p95_ms": 2024.2955645022444, |
| "latency_p99_ms": 2287.086103030015, |
| "ttft_mean_ms": 149.37952628049933, |
| "ttft_p50_ms": 121.9771110045258, |
| "ttft_p95_ms": 303.5228224063758, |
| "ttft_p99_ms": 355.9325602342128, |
| "itl_mean_ms": 9.94289976587715, |
| "itl_p50_ms": 9.82163717335926, |
| "itl_p95_ms": 14.449290831630137, |
| "itl_p99_ms": 16.439139549680032, |
| "tpot_mean_ms": 9.944535710893788, |
| "tpot_p50_ms": 9.823021669386277, |
| "tpot_p95_ms": 14.453224437911699, |
| "acceptance_rate": 0.1243881347729851, |
| "acceptance_pos_1": 0.47593713555839395, |
| "acceptance_pos_2": 0.21317265507858055, |
| "acceptance_pos_3": 0.10223801854834232, |
| "mean_acceptance_length": 1.870716943410896 |
| }, |
| { |
| "dataset": "humaneval", |
| "run": "baseline", |
| "method": "baseline", |
| "num_speculative_tokens": null, |
| "requests": 164, |
| "output_tokens": 11241, |
| "request_throughput_rps": 3.867290467605202, |
| "output_throughput_tps": 265.07446430701265, |
| "speedup_vs_baseline": 1.0, |
| "latency_mean_ms": 1027.5289629021356, |
| "latency_p50_ms": 879.5409715094138, |
| "latency_p95_ms": 2018.1860155600586, |
| "latency_p99_ms": 2274.2641233184268, |
| "ttft_mean_ms": 140.52152795707892, |
| "ttft_p50_ms": 120.44445698847994, |
| "ttft_p95_ms": 278.58940106234513, |
| "ttft_p99_ms": 480.18000137846695, |
| "itl_mean_ms": 13.187347418172193, |
| "itl_p50_ms": 12.668324928583226, |
| "itl_p95_ms": 19.83492009233357, |
| "itl_p99_ms": 28.834143813500756, |
| "tpot_mean_ms": 13.194289122212655, |
| "tpot_p50_ms": 12.670714736622909, |
| "tpot_p95_ms": 19.844658022235045, |
| "acceptance_rate": null, |
| "acceptance_pos_1": null, |
| "acceptance_pos_2": null, |
| "acceptance_pos_3": null, |
| "mean_acceptance_length": null |
| }, |
| { |
| "dataset": "humaneval", |
| "run": "dspark_k3", |
| "method": "dspark", |
| "num_speculative_tokens": 3, |
| "requests": 164, |
| "output_tokens": 11202, |
| "request_throughput_rps": 5.440176459827299, |
| "output_throughput_tps": 371.59058965235, |
| "speedup_vs_baseline": 1.4018347283047567, |
| "latency_mean_ms": 731.6780699141573, |
| "latency_p50_ms": 653.8903180044144, |
| "latency_p95_ms": 1438.7055883620633, |
| "latency_p99_ms": 1758.196029517276, |
| "ttft_mean_ms": 153.84280301219493, |
| "ttft_p50_ms": 109.24666900245938, |
| "ttft_p95_ms": 371.5150598072796, |
| "ttft_p99_ms": 427.1234392697809, |
| "itl_mean_ms": 9.217930821861229, |
| "itl_p50_ms": 8.395432637747984, |
| "itl_p95_ms": 16.543497819817173, |
| "itl_p99_ms": 21.385043967787766, |
| "tpot_mean_ms": 9.224179054706173, |
| "tpot_p50_ms": 8.398416746314833, |
| "tpot_p95_ms": 16.54860166646416, |
| "acceptance_rate": 0.5723147241266732, |
| "acceptance_pos_1": 0.7548971596474046, |
| "acceptance_pos_2": 0.5506856023506367, |
| "acceptance_pos_3": 0.41136141038197843, |
| "mean_acceptance_length": 2.7169441723800194 |
| }, |
| { |
| "dataset": "humaneval", |
| "run": "dspark_k7", |
| "method": "dspark", |
| "num_speculative_tokens": 7, |
| "requests": 164, |
| "output_tokens": 11135, |
| "request_throughput_rps": 5.728781174033297, |
| "output_throughput_tps": 388.9632827613461, |
| "speedup_vs_baseline": 1.467373644527463, |
| "latency_mean_ms": 695.9294298721943, |
| "latency_p50_ms": 634.961468007532, |
| "latency_p95_ms": 1258.2039341956258, |
| "latency_p99_ms": 1984.7797918543813, |
| "ttft_mean_ms": 170.66249135340757, |
| "ttft_p50_ms": 130.76092200935818, |
| "ttft_p95_ms": 440.41123243077857, |
| "ttft_p99_ms": 548.8665195452631, |
| "itl_mean_ms": 9.06925487325878, |
| "itl_p50_ms": 7.807209323622503, |
| "itl_p95_ms": 18.29704301704432, |
| "itl_p99_ms": 30.691704785244347, |
| "tpot_mean_ms": 9.07486906638103, |
| "tpot_p50_ms": 7.822465188736613, |
| "tpot_p95_ms": 18.303107731801564, |
| "acceptance_rate": 0.33277512988964747, |
| "acceptance_pos_1": 0.7348963029756538, |
| "acceptance_pos_2": 0.5404268109407875, |
| "acceptance_pos_3": 0.38352870453862337, |
| "mean_acceptance_length": 3.3294259092275325 |
| }, |
| { |
| "dataset": "mbpp", |
| "run": "baseline", |
| "method": "baseline", |
| "num_speculative_tokens": null, |
| "requests": 500, |
| "output_tokens": 61587, |
| "request_throughput_rps": 2.8009211403112766, |
| "output_throughput_tps": 345.00066053670116, |
| "speedup_vs_baseline": 1.0, |
| "latency_mean_ms": 1426.2509520059102, |
| "latency_p50_ms": 1449.8898644960718, |
| "latency_p95_ms": 1792.6750187427385, |
| "latency_p99_ms": 1927.6900081126948, |
| "ttft_mean_ms": 146.00965341442497, |
| "ttft_p50_ms": 129.28127399936784, |
| "ttft_p95_ms": 278.12606179941207, |
| "ttft_p99_ms": 302.88602881017135, |
| "itl_mean_ms": 10.539905415700249, |
| "itl_p50_ms": 10.33841663381217, |
| "itl_p95_ms": 12.799749751224665, |
| "itl_p99_ms": 13.565437609112568, |
| "tpot_mean_ms": 10.546513464177922, |
| "tpot_p50_ms": 10.339677704733752, |
| "tpot_p95_ms": 12.800697692052097, |
| "acceptance_rate": null, |
| "acceptance_pos_1": null, |
| "acceptance_pos_2": null, |
| "acceptance_pos_3": null, |
| "mean_acceptance_length": null |
| }, |
| { |
| "dataset": "mbpp", |
| "run": "dspark_k3", |
| "method": "dspark", |
| "num_speculative_tokens": 3, |
| "requests": 500, |
| "output_tokens": 61810, |
| "request_throughput_rps": 3.8673346513351055, |
| "output_throughput_tps": 478.07990959804573, |
| "speedup_vs_baseline": 1.385736215270775, |
| "latency_mean_ms": 1031.8113529087277, |
| "latency_p50_ms": 1022.938920505112, |
| "latency_p95_ms": 1450.6354179771733, |
| "latency_p99_ms": 1667.4116877623596, |
| "ttft_mean_ms": 149.58043881563935, |
| "ttft_p50_ms": 120.52947049960494, |
| "ttft_p95_ms": 290.24569394096017, |
| "ttft_p99_ms": 465.6294314624392, |
| "itl_mean_ms": 7.4866564872650105, |
| "itl_p50_ms": 7.0952005747625035, |
| "itl_p95_ms": 10.490002014466166, |
| "itl_p99_ms": 15.616462420633345, |
| "tpot_mean_ms": 7.490425515753587, |
| "tpot_p50_ms": 7.09647927951163, |
| "tpot_p95_ms": 10.49589030935033, |
| "acceptance_rate": 0.5115081479053122, |
| "acceptance_pos_1": 0.7031512518952587, |
| "acceptance_pos_2": 0.5011678891939516, |
| "acceptance_pos_3": 0.3302053026267262, |
| "mean_acceptance_length": 2.5345244437159367 |
| }, |
| { |
| "dataset": "mbpp", |
| "run": "dspark_k7", |
| "method": "dspark", |
| "num_speculative_tokens": 7, |
| "requests": 500, |
| "output_tokens": 61424, |
| "request_throughput_rps": 3.9288605862280823, |
| "output_throughput_tps": 482.6526652969474, |
| "speedup_vs_baseline": 1.3989905542386718, |
| "latency_mean_ms": 1016.7640253802527, |
| "latency_p50_ms": 1009.4089240010362, |
| "latency_p95_ms": 1519.8205350112396, |
| "latency_p99_ms": 1755.0660978292578, |
| "ttft_mean_ms": 157.01642078207806, |
| "ttft_p50_ms": 127.09367199568078, |
| "ttft_p95_ms": 310.5101487963111, |
| "ttft_p99_ms": 555.6603015074506, |
| "itl_mean_ms": 7.464304052053599, |
| "itl_p50_ms": 6.92667360632959, |
| "itl_p95_ms": 11.035447346997291, |
| "itl_p99_ms": 15.66697385307634, |
| "tpot_mean_ms": 7.469533758903007, |
| "tpot_p50_ms": 6.928639874063014, |
| "tpot_p95_ms": 11.131861498385357, |
| "acceptance_rate": 0.27595293416337563, |
| "acceptance_pos_1": 0.6748067713025748, |
| "acceptance_pos_2": 0.46773199298212337, |
| "acceptance_pos_3": 0.29465598179145525, |
| "mean_acceptance_length": 2.9316705391436293 |
| }, |
| { |
| "dataset": "mt_bench", |
| "run": "baseline", |
| "method": "baseline", |
| "num_speculative_tokens": null, |
| "requests": 160, |
| "output_tokens": 20102, |
| "request_throughput_rps": 2.865123484091851, |
| "output_throughput_tps": 359.96695173258996, |
| "speedup_vs_baseline": 1.0, |
| "latency_mean_ms": 1379.8839098995813, |
| "latency_p50_ms": 1349.3874825071543, |
| "latency_p95_ms": 1778.2662457597326, |
| "latency_p99_ms": 2064.43852098775, |
| "ttft_mean_ms": 230.34865414410888, |
| "ttft_p50_ms": 188.74082650290802, |
| "ttft_p95_ms": 469.9252384045379, |
| "ttft_p99_ms": 599.9244086013641, |
| "itl_mean_ms": 9.196491927825377, |
| "itl_p50_ms": 8.739159606336653, |
| "itl_p95_ms": 12.184589993809974, |
| "itl_p99_ms": 13.513825134104701, |
| "tpot_mean_ms": 9.201697317973778, |
| "tpot_p50_ms": 8.741404259885389, |
| "tpot_p95_ms": 12.185761682743273, |
| "acceptance_rate": null, |
| "acceptance_pos_1": null, |
| "acceptance_pos_2": null, |
| "acceptance_pos_3": null, |
| "mean_acceptance_length": null |
| }, |
| { |
| "dataset": "mt_bench", |
| "run": "dspark_k3", |
| "method": "dspark", |
| "num_speculative_tokens": 3, |
| "requests": 160, |
| "output_tokens": 20228, |
| "request_throughput_rps": 3.1300295907514477, |
| "output_throughput_tps": 395.7139910107518, |
| "speedup_vs_baseline": 1.0993064477338947, |
| "latency_mean_ms": 1267.3618169241308, |
| "latency_p50_ms": 1240.8099970052717, |
| "latency_p95_ms": 1889.9596753937649, |
| "latency_p99_ms": 2197.152719998557, |
| "ttft_mean_ms": 149.93388268667331, |
| "ttft_p50_ms": 120.75363399344496, |
| "ttft_p95_ms": 296.08547244570207, |
| "ttft_p99_ms": 370.8032631434613, |
| "itl_mean_ms": 8.953320217924055, |
| "itl_p50_ms": 8.487721767728438, |
| "itl_p95_ms": 13.275213856385248, |
| "itl_p99_ms": 16.593942496504845, |
| "tpot_mean_ms": 8.957264486664604, |
| "tpot_p50_ms": 8.49002677173625, |
| "tpot_p95_ms": 13.297254559146252, |
| "acceptance_rate": 0.3200388161086851, |
| "acceptance_pos_1": 0.5434255215914604, |
| "acceptance_pos_2": 0.27666181465308104, |
| "acceptance_pos_3": 0.14002911208151383, |
| "mean_acceptance_length": 1.9601164483260554 |
| }, |
| { |
| "dataset": "mt_bench", |
| "run": "dspark_k7", |
| "method": "dspark", |
| "num_speculative_tokens": 7, |
| "requests": 160, |
| "output_tokens": 20102, |
| "request_throughput_rps": 2.8379656094158685, |
| "output_throughput_tps": 356.5549042529862, |
| "speedup_vs_baseline": 0.9905212201754052, |
| "latency_mean_ms": 1404.4557639432242, |
| "latency_p50_ms": 1343.6405550019117, |
| "latency_p95_ms": 2271.4736478112154, |
| "latency_p99_ms": 2877.03235346009, |
| "ttft_mean_ms": 182.31840012485918, |
| "ttft_p50_ms": 159.79043499100953, |
| "ttft_p95_ms": 363.3274522420834, |
| "ttft_p99_ms": 631.4187010040041, |
| "itl_mean_ms": 9.922806420825252, |
| "itl_p50_ms": 9.43615103145515, |
| "itl_p95_ms": 17.314218461555217, |
| "itl_p99_ms": 20.890300572353894, |
| "tpot_mean_ms": 9.927143236485232, |
| "tpot_p50_ms": 9.4376937125298, |
| "tpot_p95_ms": 17.32922917495226, |
| "acceptance_rate": 0.1459540058496301, |
| "acceptance_pos_1": 0.5190686471296668, |
| "acceptance_pos_2": 0.258129265355279, |
| "acceptance_pos_3": 0.12515054195102368, |
| "mean_acceptance_length": 2.0216780409474104 |
| }, |
| { |
| "dataset": "ultrachat", |
| "run": "baseline", |
| "method": "baseline", |
| "num_speculative_tokens": null, |
| "requests": 5000, |
| "output_tokens": 635560, |
| "request_throughput_rps": 1.3331967798759812, |
| "output_throughput_tps": 169.4653090835957, |
| "speedup_vs_baseline": 1.0, |
| "latency_mean_ms": 2999.209397312632, |
| "latency_p50_ms": 3004.6992159914225, |
| "latency_p95_ms": 3652.681129911798, |
| "latency_p99_ms": 3792.2580027181543, |
| "ttft_mean_ms": 191.445676819887, |
| "ttft_p50_ms": 180.53925450658426, |
| "ttft_p95_ms": 319.0085294045276, |
| "ttft_p99_ms": 436.0589518790949, |
| "itl_mean_ms": 22.255913944838188, |
| "itl_p50_ms": 22.191040874087605, |
| "itl_p95_ms": 26.93092076657395, |
| "itl_p99_ms": 27.96015746255569, |
| "tpot_mean_ms": 22.257870607914793, |
| "tpot_p50_ms": 22.192596555178742, |
| "tpot_p95_ms": 26.93375055238194, |
| "acceptance_rate": null, |
| "acceptance_pos_1": null, |
| "acceptance_pos_2": null, |
| "acceptance_pos_3": null, |
| "mean_acceptance_length": null |
| }, |
| { |
| "dataset": "ultrachat", |
| "run": "dspark_k3", |
| "method": "dspark", |
| "num_speculative_tokens": 3, |
| "requests": 5000, |
| "output_tokens": 636234, |
| "request_throughput_rps": 1.816992286110778, |
| "output_throughput_tps": 231.20645403228093, |
| "speedup_vs_baseline": 1.364329108314604, |
| "latency_mean_ms": 2200.762252031482, |
| "latency_p50_ms": 2194.020344497403, |
| "latency_p95_ms": 2860.881272106781, |
| "latency_p99_ms": 3141.828603268659, |
| "ttft_mean_ms": 203.0626388980716, |
| "ttft_p50_ms": 195.85809300770052, |
| "ttft_p95_ms": 327.66038095142005, |
| "ttft_p99_ms": 418.8016837404577, |
| "itl_mean_ms": 15.817088125396433, |
| "itl_p50_ms": 15.717495220445654, |
| "itl_p95_ms": 20.662014169011663, |
| "itl_p99_ms": 22.842756694141052, |
| "tpot_mean_ms": 15.818637421764908, |
| "tpot_p50_ms": 15.7196392047615, |
| "tpot_p95_ms": 20.663277460188848, |
| "acceptance_rate": 0.3022621517716569, |
| "acceptance_pos_1": 0.5328723151898772, |
| "acceptance_pos_2": 0.2570481426653975, |
| "acceptance_pos_3": 0.11686599745969595, |
| "mean_acceptance_length": 1.9067864553149707 |
| }, |
| { |
| "dataset": "ultrachat", |
| "run": "dspark_k7", |
| "method": "dspark", |
| "num_speculative_tokens": 7, |
| "requests": 5000, |
| "output_tokens": 636210, |
| "request_throughput_rps": 1.8127834563773697, |
| "output_throughput_tps": 230.66219255636926, |
| "speedup_vs_baseline": 1.361117468841872, |
| "latency_mean_ms": 2205.888447939884, |
| "latency_p50_ms": 2192.8646555024898, |
| "latency_p95_ms": 2923.480463263696, |
| "latency_p99_ms": 3272.379539081885, |
| "ttft_mean_ms": 208.6473120003706, |
| "ttft_p50_ms": 199.3813320004847, |
| "ttft_p95_ms": 335.6416549038841, |
| "ttft_p99_ms": 448.6493355646964, |
| "itl_mean_ms": 15.814628500703911, |
| "itl_p50_ms": 15.656986732382371, |
| "itl_p95_ms": 21.199671716881205, |
| "itl_p99_ms": 23.549442959891635, |
| "tpot_mean_ms": 15.81632527800587, |
| "tpot_p50_ms": 15.658670244142634, |
| "tpot_p95_ms": 21.20066768934127, |
| "acceptance_rate": 0.14160791914766432, |
| "acceptance_pos_1": 0.5315988273762544, |
| "acceptance_pos_2": 0.25404655416494404, |
| "acceptance_pos_3": 0.11493529271745531, |
| "mean_acceptance_length": 1.9912554340336501 |
| } |
| ] |
