diff --git "a/README.md" "b/README.md" --- "a/README.md" +++ "b/README.md" @@ -77,355 +77,355 @@ For more details, please refer to our blog post [Qwen3.5](https://qwen.ai/blog?i
- - - - - - - + + + + + + + - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + - - - - - - - + + + + + + + - - - - - - - + + + + + + + - - - - - - - + + + + + + + - - - - - - - + + + + + + + - - - - - - - + + + + + + + - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
GPT-5-mini 2025-08-07GPT-OSS-120BQwen3-235B-A22BQwen3.5-122B-A10BQwen3.5-27BQwen3.5-35B-A3BGPT-5-mini 2025-08-07GPT-OSS-120BQwen3-235B-A22BQwen3.5-122B-A10BQwen3.5-27BQwen3.5-35B-A3B
Knowledge
MMLU-Pro83.780.884.486.786.185.3
MMLU-Redux93.791.093.894.093.293.3
C-Eval82.276.292.191.990.590.2
SuperGPQA58.654.664.967.165.663.4MMLU-Pro83.780.884.486.786.185.3
MMLU-Redux93.791.093.894.093.293.3
C-Eval82.276.292.191.990.590.2
SuperGPQA58.654.664.967.165.663.4
Instruction Following
IFEval93.988.987.893.495.091.9IFEval93.988.987.893.495.091.9
IFBench75.469.051.776.176.570.2IFBench75.469.051.776.176.570.2
MultiChallenge59.045.350.261.560.860.0MultiChallenge59.045.350.261.560.860.0
Long Context
AA-LCR68.050.760.066.966.158.5AA-LCR68.050.760.066.966.158.5
LongBench v256.848.254.860.260.659.0LongBench v256.848.254.860.260.659.0
STEM & Reasoning
HLE w/ COT19.414.918.225.324.322.4
GPQA Diamond82.880.181.186.685.584.2
HMMT Feb 2589.290.085.191.492.089.0
HMMT Nov 2584.290.089.590.389.889.2HLE w/ COT19.414.918.225.324.322.4
GPQA Diamond82.880.181.186.685.584.2
HMMT Feb 2589.290.085.191.492.089.0
HMMT Nov 2584.290.089.590.389.889.2
Coding
SWE-bench Verified72.062.0--72.072.469.2
Terminal Bench 231.918.7--49.441.640.5
LiveCodeBench v680.582.775.178.980.774.6
CodeForces216021572146210018992028
OJBench40.441.532.739.540.136.0
FullStackBench en30.658.961.162.660.158.1
FullStackBench zh35.260.463.158.757.455.0SWE-bench Verified72.062.0--72.072.469.2
Terminal Bench 231.918.7--49.441.640.5
LiveCodeBench v680.582.775.178.980.774.6
CodeForces216021572146210018992028
OJBench40.441.532.739.540.136.0
FullStackBench en30.658.961.162.660.158.1
FullStackBench zh35.260.463.158.757.455.0
General Agent
BFCL-V455.5--54.872.268.567.3
TAU2-Bench69.8--58.579.579.081.2
VITA-Bench13.9--31.633.641.931.9
DeepPlanning17.9--17.124.122.622.8BFCL-V455.5--54.872.268.567.3
TAU2-Bench69.8--58.579.579.081.2
VITA-Bench13.9--31.633.641.931.9
DeepPlanning17.9--17.124.122.622.8
Search Agent
HLE w/ tool35.819.0--47.548.547.4
Browsecomp48.141.1--63.861.061.0
Browsecomp-zh49.542.9--69.962.169.5
WideSearch47.240.4--60.561.157.1
Seal-034.245.1--44.147.241.4HLE w/ tool35.819.0--47.548.547.4
Browsecomp48.141.1--63.861.061.0
Browsecomp-zh49.542.9--69.962.169.5
WideSearch47.240.4--60.561.157.1
Seal-034.245.1--44.147.241.4
Multilingualism
MMMLU86.278.283.486.785.985.2
MMLU-ProX78.574.577.982.282.281.0
NOVA-6351.951.155.458.658.157.1
INCLUDE81.874.081.082.881.679.7
Global PIQA88.584.185.788.487.586.6
PolyMATH67.354.060.168.971.264.4
WMT24++80.774.475.878.377.676.3
MAXIFE85.383.783.287.988.086.6MMMLU86.278.283.486.785.985.2
MMLU-ProX78.574.577.982.282.281.0
NOVA-6351.951.155.458.658.157.1
INCLUDE81.874.081.082.881.679.7
Global PIQA88.584.185.788.487.586.6
PolyMATH67.354.060.168.971.264.4
WMT24++80.774.475.878.377.676.3
MAXIFE85.383.783.287.988.086.6
@@ -448,434 +448,434 @@ For more details, please refer to our blog post [Qwen3.5](https://qwen.ai/blog?i
- - - - - - - + + + + + + + - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + - - - - - - - + + + + + + + - - - - - - - + + + + + + + - - - - - - - + + + + + + + - - - - - - - + + + + + + + - - - - - - - + + + + + + + - - - - - - - + + + + + + + - - - - - - - + + + + + + + - - - - - - - + + + + + + + - - - - - - - + + + + + + + - - - - - - - + + + + + + + - - - - - - - + + + + + + + - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + - - - - - - - + + + + + + + - - - - - - - + + + + + + + - - - - - - - + + + + + + + - - - - - - - + + + + + + + - - - - - - - + + + + + + + - - - - - - - - - - - - - - - - - - - - - - - - - + + + + + + + + + + + + + + + + + + + + + + + + +
GPT-5-mini 2025-08-07Claude-Sonnet-4.5Qwen3-VL-235B-A22BQwen3.5-122B-A10BQwen3.5-27BQwen3.5-35B-A3BGPT-5-mini 2025-08-07Claude-Sonnet-4.5Qwen3-VL-235B-A22BQwen3.5-122B-A10BQwen3.5-27BQwen3.5-35B-A3B
STEM and Puzzle
MMMU79.079.680.683.982.381.4
MMMU-Pro67.368.469.376.975.075.1
MathVision71.971.174.686.286.083.9
Mathvista(mini)79.179.885.887.487.886.2
DynaMath81.478.882.885.987.785.0
ZEROBench3449108
ZEROBench_sub27.326.328.436.236.234.1
VlmsAreBlind75.885.579.596.796.997.0
BabyVision20.9 / 34.518.6 / 34.522.2 / 34.540.2 / 34.544.6 / 34.838.4 / 29.6MMMU79.079.680.683.982.381.4
MMMU-Pro67.368.469.376.975.075.1
MathVision71.971.174.686.286.083.9
Mathvista(mini)79.179.885.887.487.886.2
DynaMath81.478.882.885.987.785.0
ZEROBench3449108
ZEROBench_sub27.326.328.436.236.234.1
VlmsAreBlind75.885.579.596.796.997.0
BabyVision20.9 / 34.518.6 / 34.522.2 / 34.540.2 / 34.544.6 / 34.838.4 / 29.6
General VQA
RealWorldQA79.070.381.385.183.784.1RealWorldQA79.070.381.385.183.784.1
MMStar74.173.878.782.981.081.9MMStar74.173.878.782.981.081.9
MMBenchEN-DEV-v1.186.888.389.792.892.691.5MMBenchEN-DEV-v1.186.888.389.792.892.691.5
SimpleVQA56.857.661.361.756.058.3SimpleVQA56.857.661.361.756.058.3
HallusionBench63.259.966.767.670.067.9HallusionBench63.259.966.767.670.067.9
Text Recognition and Document Understanding
OmniDocBench1.577.085.884.589.888.989.3OmniDocBench1.577.085.884.589.888.989.3
CharXiv(RQ)68.667.266.177.279.577.5CharXiv(RQ)68.667.266.177.279.577.5
MMLongBench-Doc50.3--56.259.060.259.5MMLongBench-Doc50.3--56.259.060.259.5
CC-OCR70.868.181.581.881.080.7CC-OCR70.868.181.581.881.080.7
AI2D_TEST88.287.089.293.392.992.6AI2D_TEST88.287.089.293.392.992.6
OCRBench82176687.592.189.491.0OCRBench82176687.592.189.491.0
Spatial Intelligence
ERQA54.045.052.562.060.564.8
CountBench91.090.093.797.097.897.8
RefCOCO(avg)----91.191.390.989.2
ODInW13----43.244.541.142.6
EmbSpatialBench80.771.884.383.984.583.1
RefSpatialBench9.02.269.969.367.763.5
LingoQA62.412.866.880.882.079.2
Hypersim----11.012.713.013.1
SUNRGBD----34.936.235.433.4
Nuscene----13.915.415.214.6ERQA54.045.052.562.060.564.8
CountBench91.090.093.797.097.897.8
RefCOCO(avg)----91.191.390.989.2
ODInW13----43.244.541.142.6
EmbSpatialBench80.771.884.383.984.583.1
RefSpatialBench9.02.269.969.367.763.5
LingoQA62.412.866.880.882.079.2
Hypersim----11.012.713.013.1
SUNRGBD----34.936.235.433.4
Nuscene----13.915.415.214.6
Video Understanding
VideoMME(w sub.)83.581.183.887.387.086.6
VideoMME(w/o sub.)78.975.379.083.982.882.5
VideoMMMU82.577.680.082.082.380.4
MLVU83.372.883.887.385.985.6
MVBench----75.276.674.674.8
LVBench----63.674.473.671.4
MMVU69.870.671.174.773.372.3VideoMME(w sub.)83.581.183.887.387.086.6
VideoMME(w/o sub.)78.975.379.083.982.882.5
VideoMMMU82.577.680.082.082.380.4
MLVU83.372.883.887.385.985.6
MVBench----75.276.674.674.8
LVBench----63.674.473.671.4
MMVU69.870.671.174.773.372.3
Visual Agent
ScreenSpot Pro--36.262.070.4070.2868.60ScreenSpot Pro--36.262.070.4070.2868.60
OSWorld-Verified--61.438.158.0156.1554.49OSWorld-Verified--61.438.158.0156.1554.49
AndroidWorld----63.766.464.271.1AndroidWorld----63.766.464.271.1
Tool Calling
TIR-Bench24.6 / 42.527.6 / 42.529.8 / 42.553.2 / 42.559.8 / 42.355.5 / 38.0TIR-Bench24.6 / 42.527.6 / 42.529.8 / 42.553.2 / 42.559.8 / 42.355.5 / 38.0
V*71.7 / 90.158.6 / 89.085.9 / 89.593.2 / 90.193.7 / 89.092.7 / 89.5V*71.7 / 90.158.6 / 89.085.9 / 89.593.2 / 90.193.7 / 89.092.7 / 89.5
Medical VQA
SLAKE70.573.654.781.680.078.7
PMC-VQA36.355.941.263.362.462.0
MedXpertQA-MM34.454.047.667.362.461.4SLAKE70.573.654.781.680.078.7
PMC-VQA36.355.941.263.362.462.0
MedXpertQA-MM34.454.047.667.362.461.4