benchmarks/bs4/comparison.md

Dataset	Run	Req	Output tok/s	Speedup	Latency mean/p95 ms	TTFT mean/p95 ms	ITL mean/p95 ms	TPOT mean/p95 ms	Accept rate	Accept p1/p2/p3	Accept len
math500	baseline	500	266.74	1.000x	1901.95/2310.23	211.01/379.08	13.40/16.31	13.40/16.31	—	-/-/-	—
math500	dflash_k3	500	185.18	0.694x	2740.52/3396.06	232.73/422.78	19.89/24.71	19.90/24.71	6.29%	16.64%/1.97%/0.24%	1.189
math500	dflash_k7	500	188.44	0.706x	2690.23/3411.27	223.15/410.68	19.61/24.83	19.61/24.83	2.74%	16.86%/2.02%/0.26%	1.192
math500	dflare_k3	500	217.77	0.816x	2328.94/3331.75	279.51/506.23	16.27/23.21	16.27/23.21	39.35%	64.89%/35.28%/17.89%	2.181
math500	dflare_k7	500	225.99	0.847x	2249.20/3338.34	263.03/475.01	15.76/24.25	15.76/24.25	18.69%	64.67%/35.48%/17.53%	2.309
math500	dspark_k3	500	286.34	1.073x	1774.14/2512.93	237.01/434.45	12.18/17.88	12.18/17.88	44.99%	65.67%/42.25%/27.05%	2.350
math500	dspark_k7	500	294.75	1.105x	1725.25/2646.27	247.76/461.03	11.72/18.88	11.72/18.88	22.89%	63.98%/39.64%/24.22%	2.602
gsm8k	baseline	1319	280.58	1.000x	1819.24/2167.76	217.05/398.45	12.64/15.22	12.64/15.22	—	-/-/-	—
gsm8k	dflash_k3	1319	188.79	0.673x	2702.38/3317.10	235.61/424.62	19.48/24.18	19.48/24.19	6.81%	18.37%/1.86%/0.20%	1.204
gsm8k	dflash_k7	1319	188.46	0.672x	2705.30/3477.26	301.18/834.45	18.98/23.59	18.98/23.59	2.74%	17.14%/1.81%/0.20%	1.192
gsm8k	dflare_k3	1319	201.03	0.716x	2536.44/3534.03	258.19/484.66	17.98/25.82	17.98/25.82	27.95%	53.64%/21.51%/8.69%	1.838
gsm8k	dflare_k7	1319	154.41	0.550x	3304.27/5885.10	1205.81/4089.58	16.56/27.05	16.56/27.06	12.83%	54.22%/21.89%/9.37%	1.898
gsm8k	dspark_k3	1319	267.82	0.954x	1904.86/2717.59	232.41/436.56	13.20/19.22	13.20/19.23	31.15%	53.33%/25.75%/14.35%	1.934
gsm8k	dspark_k7	1319	256.27	0.913x	1991.49/2946.94	240.23/457.41	13.82/21.07	13.82/21.07	14.63%	53.19%/24.90%/13.31%	2.024
humaneval	baseline	164	194.21	1.000x	1403.54/2734.28	218.21/420.29	17.84/29.90	17.85/29.91	—	-/-/-	—
humaneval	dflash_k3	164	147.26	0.758x	1856.84/3610.00	250.89/497.85	24.55/42.01	24.56/42.02	5.95%	16.41%/1.35%/0.08%	1.179
humaneval	dflash_k7	164	141.91	0.731x	1877.28/3978.09	343.90/982.06	22.96/33.21	22.97/33.23	2.68%	17.40%/1.31%/0.09%	1.188
humaneval	dflare_k3	164	194.56	1.002x	1401.65/2758.01	307.73/627.48	17.78/33.04	17.78/33.05	58.74%	77.35%/56.42%/42.44%	2.762
humaneval	dflare_k7	164	206.56	1.064x	1296.23/2684.54	395.19/956.47	16.23/32.95	16.23/32.96	33.05%	77.91%/56.44%/40.45%	3.313
humaneval	dspark_k3	164	234.79	1.209x	1172.97/2107.93	265.30/502.88	14.78/29.63	14.78/29.64	59.91%	76.53%/58.74%/44.47%	2.797
humaneval	dspark_k7	164	210.60	1.084x	1276.63/2616.22	564.26/1681.50	12.49/25.28	12.49/25.28	35.09%	74.01%/55.03%/39.85%	3.457
mbpp	baseline	500	275.24	1.000x	1788.02/2216.35	221.07/398.67	13.20/15.66	13.21/15.67	—	-/-/-	—
mbpp	dflash_k3	500	189.94	0.690x	2595.66/3320.97	234.11/439.53	19.14/23.96	19.15/23.96	6.46%	18.25%/1.11%/0.04%	1.194
mbpp	dflash_k7	500	170.29	0.619x	2875.58/4074.32	634.04/1885.92	18.20/24.69	18.21/24.70	2.85%	18.78%/1.13%/0.05%	1.200
mbpp	dflare_k3	500	244.12	0.887x	2016.51/2791.48	280.07/557.88	14.26/19.44	14.27/19.63	47.05%	68.89%/44.50%/27.76%	2.411
mbpp	dflare_k7	500	184.84	0.672x	2657.85/4488.95	1190.47/3259.46	12.45/22.86	12.46/22.86	24.66%	67.13%/42.74%/25.45%	2.726
mbpp	dspark_k3	500	310.80	1.129x	1592.68/2178.77	240.73/455.66	11.35/15.68	11.35/15.68	52.29%	71.25%/50.82%/34.79%	2.569
mbpp	dspark_k7	500	274.50	0.997x	1794.33/3171.44	643.14/2119.01	10.10/14.88	10.11/14.88	28.83%	68.32%/47.02%/30.55%	3.018
mt_bench	baseline	160	309.23	1.000x	1607.62/2284.11	320.49/629.17	11.01/15.18	11.01/15.19	—	-/-/-	—
mt_bench	dflash_k3	160	195.62	0.633x	2573.23/3166.33	221.35/448.16	18.68/23.10	18.68/23.11	6.07%	16.45%/1.71%/0.06%	1.182
mt_bench	dflash_k7	160	180.37	0.583x	2753.90/3791.87	381.83/1026.59	21.71/25.58	21.71/25.58	2.48%	15.47%/1.60%/0.12%	1.173
mt_bench	dflare_k3	160	225.31	0.729x	2231.25/3333.71	248.97/520.00	15.78/23.60	15.78/23.60	32.30%	57.50%/27.57%/11.81%	1.969
mt_bench	dflare_k7	160	133.91	0.433x	3752.53/6717.97	1934.01/4820.78	14.32/29.83	14.32/29.83	14.58%	56.90%/26.94%/11.54%	2.021
mt_bench	dspark_k3	160	282.93	0.915x	1771.67/2604.84	252.12/503.80	12.36/19.15	12.36/19.15	36.54%	58.03%/32.54%/19.06%	2.096
mt_bench	dspark_k7	160	224.25	0.725x	2225.17/3657.94	838.84/2441.89	11.43/22.87	11.43/23.00	17.22%	55.30%/29.97%/16.74%	2.205
ultrachat	baseline	5000	155.46	1.000x	3269.57/3934.88	231.24/386.99	24.09/28.91	24.09/28.91	—	-/-/-	—
ultrachat	dflash_k3	5000	135.15	0.869x	3763.95/4515.90	257.60/405.56	27.76/33.28	27.77/33.28	7.09%	18.40%/2.57%/0.30%	1.213
ultrachat	dflash_k7	5000	134.03	0.862x	3794.92/4543.27	255.94/399.63	28.03/33.56	28.03/33.56	3.07%	18.54%/2.58%/0.31%	1.215
ultrachat	dflare_k3	5000	161.25	1.037x	3155.82/4013.42	282.90/456.06	22.75/29.20	22.75/29.20	31.06%	57.10%/26.00%/10.07%	1.932
ultrachat	dflare_k7	5000	157.88	1.016x	3221.08/4123.29	352.64/612.31	22.73/29.63	22.73/29.63	14.37%	57.63%/26.59%/10.33%	2.006
ultrachat	dspark_k3	5000	190.92	1.228x	2665.16/3441.95	264.37/429.11	19.01/24.80	19.01/24.80	32.40%	55.58%/28.18%/13.44%	1.972
ultrachat	dspark_k7	5000	189.60	1.220x	2683.34/3469.80	287.66/433.69	18.97/25.07	18.97/25.07	15.40%	55.21%/27.94%/13.27%	2.078