""" MLX Benchmark V2 Leaderboard A leaderboard for evaluating LLM proficiency on Apple's MLX framework. 520 questions across 11 categories, 6 question types, and 4 difficulty levels. """ import gradio as gr import pandas as pd import json import re from pathlib import Path # ────────────────────────────────────────────────────────── # Data Loading — Parses real mlx-bench output format # ────────────────────────────────────────────────────────── RESULTS_DIR = Path("data") def pct(d): """Compute accuracy % from a {total, correct} dict. Returns 0 if total is 0.""" total = d.get("total", 0) correct = d.get("correct", 0) return round(100.0 * correct / total, 1) if total > 0 else 0.0 def parse_model_name(raw_model): """ Parse the model field from bench output. e.g. 'openrouter/openai/gpt-5-nano' -> provider='openai', short_name='gpt-5-nano' e.g. 'ollama/qwen3:32b' -> provider='ollama', short_name='qwen3:32b' """ parts = raw_model.strip().split("/") if len(parts) >= 3: # openrouter/openai/gpt-5-nano or openrouter/google/gemini-... return parts[1], parts[-1], raw_model elif len(parts) == 2: return parts[0], parts[1], raw_model else: return "", raw_model, raw_model def load_results(): """Load all benchmark result JSON files from data/ directory.""" results = [] if not RESULTS_DIR.exists(): return results for f in sorted(RESULTS_DIR.glob("*.json")): try: with open(f, "r") as fp: data = json.load(fp) data["_filename"] = f.name results.append(data) except Exception as e: print(f"Error loading {f}: {e}") return results def build_main_df(results): """Build the main leaderboard DataFrame from real bench output files.""" if not results: return pd.DataFrame() rows = [] for r in results: raw_model = r.get("model", "Unknown") provider, short_name, full_path = parse_model_name(raw_model) judge = r.get("judge", "") timestamp = r.get("timestamp", "") stats = r.get("stats", {}) overall = stats.get("accuracy", 0) total_q = stats.get("total", 520) correct_q = stats.get("correct", 0) # By difficulty — {key: {total, correct}} by_diff = stats.get("by_difficulty", {}) easy = pct(by_diff.get("easy", {})) medium = pct(by_diff.get("medium", {})) hard = pct(by_diff.get("hard", {})) very_hard = pct(by_diff.get("very-hard", {})) # By category — {key: {total, correct}} by_cat = stats.get("by_category", {}) mlx_core = pct(by_cat.get("mlx_core", {})) mlx_nn = pct(by_cat.get("mlx_nn", {})) mlx_lm = pct(by_cat.get("mlx_lm", {})) mlx_lm_lora = pct(by_cat.get("mlx_lm_lora", {})) coding_cat = pct(by_cat.get("coding", {})) debugging = pct(by_cat.get("debugging", {})) mlx_vlm = pct(by_cat.get("mlx_vlm", {})) mlx_optimizers = pct(by_cat.get("mlx_optimizers", {})) mlx_embeddings = pct(by_cat.get("mlx_embeddings", {})) mlx_embeddings_lora = pct(by_cat.get("mlx_embeddings_lora", {})) conceptual = pct(by_cat.get("conceptual", {})) # By type — {key: {total, correct}} by_type = stats.get("by_type", {}) qa_score = pct(by_type.get("qa", {})) coding_type = pct(by_type.get("coding", {})) debug_type = pct(by_type.get("debug", {})) mcq_type = pct(by_type.get("mcq", {})) tf_type = pct(by_type.get("true_false", {})) fb_type = pct(by_type.get("fill_blank", {})) # Display name: make it a nice readable name display_name = short_name rows.append({ "Model": display_name, "Provider": provider.capitalize() if provider else "", "Judge": judge, "Correct": f"{correct_q}/{total_q}", "Overall (%)": round(overall, 1), # Difficulty "Easy (%)": easy, "Medium (%)": medium, "Hard (%)": hard, "Very Hard (%)": very_hard, # Categories "mlx_core": mlx_core, "mlx_nn": mlx_nn, "mlx_lm": mlx_lm, "mlx_lm_lora": mlx_lm_lora, "Coding": coding_cat, "Debugging": debugging, "mlx_vlm": mlx_vlm, "mlx_optimizers": mlx_optimizers, "mlx_embeddings": mlx_embeddings, "mlx_embeddings_lora": mlx_embeddings_lora, "Conceptual": conceptual, # Types "QA": qa_score, "Code Gen": coding_type, "Debug": debug_type, "MCQ": mcq_type, "True/False": tf_type, "Fill Blank": fb_type, }) df = pd.DataFrame(rows) df = df.sort_values("Overall (%)", ascending=False).reset_index(drop=True) df.insert(0, "Rank", range(1, len(df) + 1)) # Medal emojis for top 3 medals = {1: "🥇", 2: "🥈", 3: "🥉"} df["Rank"] = df["Rank"].apply(lambda x: f"{medals.get(x, '')} {x}".strip()) return df # ────────────────────────────────────────────────────────── # Column Definitions # ────────────────────────────────────────────────────────── ALWAYS_SHOWN = ["Rank", "Model", "Overall (%)"] DIFFICULTY_COLS = ["Easy (%)", "Medium (%)", "Hard (%)", "Very Hard (%)"] CATEGORY_COLS = [ "mlx_core", "mlx_nn", "mlx_lm", "mlx_lm_lora", "Coding", "Debugging", "mlx_vlm", "mlx_optimizers", "mlx_embeddings", "mlx_embeddings_lora", "Conceptual" ] TYPE_COLS = ["QA", "Code Gen", "Debug", "MCQ", "True/False", "Fill Blank"] META_COLS = ["Provider", "Judge", "Correct"] ALL_OPTIONAL_COLS = META_COLS + DIFFICULTY_COLS + CATEGORY_COLS + TYPE_COLS DEFAULT_COLS = ALWAYS_SHOWN + ["Provider", "Correct"] + DIFFICULTY_COLS COLUMN_DATATYPES = { "Rank": "str", "Model": "str", "Overall (%)": "number", "Provider": "str", "Judge": "str", "Correct": "str", "Easy (%)": "number", "Medium (%)": "number", "Hard (%)": "number", "Very Hard (%)": "number", "mlx_core": "number", "mlx_nn": "number", "mlx_lm": "number", "mlx_lm_lora": "number", "Coding": "number", "Debugging": "number", "mlx_vlm": "number", "mlx_optimizers": "number", "mlx_embeddings": "number", "mlx_embeddings_lora": "number", "Conceptual": "number", "QA": "number", "Code Gen": "number", "Debug": "number", "MCQ": "number", "True/False": "number", "Fill Blank": "number", } # ────────────────────────────────────────────────────────── # Filter Logic # ────────────────────────────────────────────────────────── def filter_and_select(full_df, search_query, selected_columns): """Filter by search and select columns.""" if full_df is None or full_df.empty: return pd.DataFrame() df = full_df.copy() # Search filter if search_query and search_query.strip(): query = search_query.strip().lower() df = df[df["Model"].str.lower().str.contains(query, na=False)] # Column selection cols_to_show = ALWAYS_SHOWN + [c for c in selected_columns if c in df.columns and c not in ALWAYS_SHOWN] df = df[[c for c in cols_to_show if c in df.columns]] return df # ────────────────────────────────────────────────────────── # Chart Builders # ────────────────────────────────────────────────────────── def build_overall_chart(full_df): """Bar chart of overall accuracy by model.""" import plotly.express as px if full_df is None or full_df.empty: return None chart_df = full_df.copy() chart_df = chart_df.sort_values("Overall (%)", ascending=True) fig = px.bar( chart_df, x="Overall (%)", y="Model", orientation="h", color="Overall (%)", color_continuous_scale=["#FF6B6B", "#FFE66D", "#4ECDC4", "#45B7D1"], title="Overall Accuracy (%)", ) fig.update_layout( height=max(400, len(chart_df) * 50), showlegend=False, xaxis_title="Accuracy (%)", yaxis_title="", coloraxis_showscale=False, template="plotly_white", font=dict(size=13), margin=dict(l=10, r=50, t=50, b=30), ) fig.update_traces(texttemplate="%{x:.1f}%", textposition="outside") return fig def build_difficulty_chart(full_df): """Grouped bar chart: accuracy by difficulty level per model.""" import plotly.graph_objects as go if full_df is None or full_df.empty: return None chart_df = full_df.copy() chart_df = chart_df.sort_values("Overall (%)", ascending=False).head(15) colors = { "Easy (%)": "#4ECDC4", "Medium (%)": "#45B7D1", "Hard (%)": "#FFE66D", "Very Hard (%)": "#FF6B6B", } fig = go.Figure() for col in DIFFICULTY_COLS: if col in chart_df.columns: fig.add_trace(go.Bar( name=col.replace(" (%)", ""), x=chart_df["Model"], y=chart_df[col], marker_color=colors.get(col, "#999"), )) fig.update_layout( barmode="group", title="Accuracy by Difficulty Level", xaxis_title="", yaxis_title="Accuracy (%)", height=500, template="plotly_white", font=dict(size=12), legend=dict(orientation="h", yanchor="bottom", y=1.02, xanchor="right", x=1), margin=dict(l=10, r=10, t=80, b=10), ) fig.update_xaxes(tickangle=30) return fig def build_category_radar(full_df): """Radar chart comparing top models across categories.""" import plotly.graph_objects as go if full_df is None or full_df.empty: return None chart_df = full_df.copy() chart_df = chart_df.sort_values("Overall (%)", ascending=False).head(5) categories_display = [ "mlx_core", "mlx_nn", "mlx_lm", "mlx_lm_lora", "Coding", "Debugging", "mlx_vlm", "mlx_optimizers", "mlx_embeddings", "Conceptual" ] colors = ["#45B7D1", "#FF6B6B", "#4ECDC4", "#FFE66D", "#96CEB4"] fig = go.Figure() for i, (_, row) in enumerate(chart_df.iterrows()): values = [row.get(c, 0) for c in categories_display] values.append(values[0]) # close the polygon cats = categories_display + [categories_display[0]] fig.add_trace(go.Scatterpolar( r=values, theta=cats, fill="toself", name=row["Model"], line=dict(color=colors[i % len(colors)]), opacity=0.6, )) fig.update_layout( polar=dict(radialaxis=dict(visible=True, range=[0, 100])), title="Category Breakdown — Top 5 Models", height=600, template="plotly_white", font=dict(size=12), margin=dict(l=80, r=80, t=60, b=40), ) return fig def build_type_chart(full_df): """Grouped bar chart: accuracy by question type per model.""" import plotly.graph_objects as go if full_df is None or full_df.empty: return None chart_df = full_df.copy() chart_df = chart_df.sort_values("Overall (%)", ascending=False).head(15) colors = { "QA": "#45B7D1", "Code Gen": "#FF6B6B", "Debug": "#FFE66D", "MCQ": "#4ECDC4", "True/False": "#96CEB4", "Fill Blank": "#DDA0DD", } fig = go.Figure() for col in TYPE_COLS: if col in chart_df.columns: fig.add_trace(go.Bar( name=col, x=chart_df["Model"], y=chart_df[col], marker_color=colors.get(col, "#999"), )) fig.update_layout( barmode="group", title="Accuracy by Question Type", xaxis_title="", yaxis_title="Accuracy (%)", height=500, template="plotly_white", font=dict(size=12), legend=dict(orientation="h", yanchor="bottom", y=1.02, xanchor="right", x=1), margin=dict(l=10, r=10, t=80, b=10), ) fig.update_xaxes(tickangle=30) return fig # ────────────────────────────────────────────────────────── # Load Data # ────────────────────────────────────────────────────────── results = load_results() full_df = build_main_df(results) # ────────────────────────────────────────────────────────── # CSS # ────────────────────────────────────────────────────────── CUSTOM_CSS = """ #leaderboard-table { margin-top: 10px; } #leaderboard-table td:first-child, #leaderboard-table th:first-child { max-width: 80px; text-align: center; } #search-bar { padding: 0px; max-width: 500px; } .tabs button { font-size: 18px; } .header-container { text-align: center; padding: 20px 10px 0px 10px; } .header-container h1 { font-size: 2.2em; margin-bottom: 5px; } .header-container p { font-size: 1.1em; color: #666; margin-top: 0; } .stat-box { background: var(--block-background-fill); border: 1px solid var(--border-color-primary); border-radius: 12px; padding: 15px 20px; text-align: center; min-width: 120px; } .stat-box .stat-number { font-size: 2em; font-weight: 700; color: var(--body-text-color); } .stat-box .stat-label { font-size: 0.85em; color: #888; margin-top: 2px; } .about-section { max-width: 900px; margin: 0 auto; padding: 20px; } """ # ────────────────────────────────────────────────────────── # About Page Content # ────────────────────────────────────────────────────────── ABOUT_MD = """ # 🍎 About the MLX Benchmark V2 The **MLX Benchmark V2** is a curated evaluation benchmark of **520 questions** designed to measure LLM proficiency in Apple's **[MLX](https://ml-explore.github.io/mlx/)** machine learning framework. MLX is an array framework for ML on Apple Silicon that leverages unified memory, lazy evaluation, and function transforms — paradigms that differ significantly from PyTorch and JAX. This benchmark directly measures whether models can understand, write, and debug MLX code. --- ## 📊 Benchmark Structure | Dimension | Options | |-----------|---------| | **Categories (11)** | `mlx_core` (188), `mlx_nn` (73), `mlx_lm` (61), `mlx_lm_lora` (55), `coding` (35), `mlx_embeddings` (21), `debugging` (21), `mlx_optimizers` (19), `mlx_vlm` (19), `mlx_embeddings_lora` (15), `conceptual` (13) | | **Question Types (6)** | QA (432), Coding (33), Debug (21), MCQ (12), True/False (12), Fill-in-the-Blank (10) | | **Difficulty Levels (4)** | Easy (180), Medium (181), Hard (109), Very Hard (50) | --- ## 🔬 Why MLX Needs Its Own Benchmark Existing LLM benchmarks (HumanEval, MBPP, MMLU) don't cover MLX-specific patterns: 1. **Lazy Evaluation** — Forgetting `mx.eval()` is the #1 MLX bug, unique to this framework 2. **Unified Memory** — No `.to(device)` calls, which trips up PyTorch-trained models 3. **Function Transforms** — `mx.grad(mx.vmap(f))` differs from JAX's equivalents 4. **MLX Ecosystem** — `mlx-lm`, `mlx-vlm`, `mlx-embeddings` each have their own APIs --- ## 📏 Evaluation Methodology | Question Type | Evaluation Method | |---------------|------------------| | **MCQ, True/False** | Exact matching (letter/keyword extraction) | | **QA, Fill-in-Blank, Coding, Debug** | LLM judge comparing against reference answers | Scoring: Each question → correct/incorrect. Aggregate accuracy computed overall and per breakdown. --- ## 🛠️ Running the Benchmark ```bash pip install mlx-benchmark # Benchmark a local Ollama model mlx-bench --model llama3.2 # Benchmark with a cloud provider mlx-bench --provider anthropic --model claude-sonnet-4-20250514 # Filter by difficulty or type mlx-bench --model llama3.2 --difficulties hard very-hard --types coding debug ``` --- ## 📚 Links - **Dataset:** [Goekdeniz-Guelmez/MLX-Benchmark-V2](https://huggingface.co/datasets/Goekdeniz-Guelmez/MLX-Benchmark-V2) - **CLI Tool:** `pip install mlx-benchmark` - **GitHub:** [Goekdeniz-Guelmez/MLX-Benchmark](https://github.com/Goekdeniz-Guelmez/MLX-Benchmark) - **Created by:** [Gökdeniz Gülmez](https://huggingface.co/Goekdeniz-Guelmez) """ SUBMIT_MD = """ # 📨 Submit Your Results Submit benchmark results for any model to be included on this leaderboard. ## How to Run ```bash pip install mlx-benchmark # Run benchmark and save results mlx-bench --model --provider ``` This produces a JSON file like `bench___.json`. ## Output Format The benchmark CLI automatically generates results in this format: ```json { "model": "openrouter/openai/gpt-5-nano", "judge": "openrouter/google/gemini-3-flash-preview", "timestamp": "20260418_170350", "stats": { "total": 520, "correct": 218, "accuracy": 41.92, "by_type": { "qa": { "total": 432, "correct": 192 }, "mcq": { "total": 12, "correct": 6 }, "fill_blank": { "total": 10, "correct": 8 }, "true_false": { "total": 12, "correct": 9 }, "coding": { "total": 33, "correct": 0 }, "debug": { "total": 21, "correct": 3 } }, "by_difficulty": { "easy": { "total": 180, "correct": 67 }, "medium": { "total": 181, "correct": 89 }, "hard": { "total": 109, "correct": 45 }, "very-hard": { "total": 50, "correct": 17 } }, "by_category": { "mlx_core": { "total": 188, "correct": 88 }, "mlx_nn": { "total": 73, "correct": 32 }, "mlx_optimizers": { "total": 19, "correct": 5 }, "mlx_lm_lora": { "total": 55, "correct": 22 }, "mlx_lm": { "total": 61, "correct": 30 }, "mlx_embeddings_lora": { "total": 15, "correct": 7 }, "mlx_vlm": { "total": 19, "correct": 9 }, "mlx_embeddings": { "total": 21, "correct": 12 }, "coding": { "total": 35, "correct": 2 }, "debugging": { "total": 21, "correct": 3 }, "conceptual": { "total": 13, "correct": 8 } } }, "results": [] } ``` ## How to Submit 1. Run `mlx-bench` on your model 2. Open a [Discussion](https://huggingface.co/spaces/Goekdeniz-Guelmez/mlx-benchmark-leaderboard/discussions) on this Space 3. Attach or paste your output JSON file 4. Results will be reviewed and added to the leaderboard ### Guidelines - Results must be from **MLX Benchmark V2** (520 questions) - Use the standard `mlx-bench` CLI — do not manually edit results - Specify the exact model name/version --- *For questions, contact [Gökdeniz Gülmez](https://huggingface.co/Goekdeniz-Guelmez).* """ # ────────────────────────────────────────────────────────── # Gradio App # ────────────────────────────────────────────────────────── def create_app(): with gr.Blocks( css=CUSTOM_CSS, theme=gr.themes.Soft( primary_hue=gr.themes.colors.orange, secondary_hue=gr.themes.colors.gray, ), title="MLX Benchmark V2 Leaderboard", ) as demo: # ── Header ── gr.HTML("""

🍎 MLX Benchmark V2 Leaderboard

Evaluating LLM proficiency on Apple's MLX machine learning framework
520 questions · 11 categories · 6 question types · 4 difficulty levels

""") # ── Stats Banner ── n_models = len(full_df) if not full_df.empty else 0 top_score = full_df["Overall (%)"].max() if not full_df.empty else 0 avg_score = full_df["Overall (%)"].mean() if not full_df.empty else 0 gr.HTML(f"""
{n_models}
Models Evaluated
520
Questions
{top_score:.1f}%
Top Score
{avg_score:.1f}%
Average Score
""") # ── Tabs ── with gr.Tabs(elem_classes="tabs") as tabs: # ═══════════════════════════════ # TAB: Leaderboard # ═══════════════════════════════ with gr.TabItem("🏅 Leaderboard", id=0): if full_df.empty: gr.Markdown(""" ### No results yet! Be the first to submit benchmark results. Run: ```bash pip install mlx-benchmark mlx-bench --model --provider ``` Then submit your results in the **📨 Submit** tab. """) else: with gr.Row(): search_bar = gr.Textbox( label="🔍 Search Models", placeholder="Type to filter by model name...", elem_id="search-bar", scale=2, ) with gr.Accordion("🔧 Column Selection", open=False): gr.Markdown("*Select which columns to display. Rank, Model, and Overall are always shown.*") with gr.Row(): with gr.Column(): gr.Markdown("**📋 Metadata**") meta_checks = gr.CheckboxGroup( choices=META_COLS, value=["Provider", "Correct"], label="", show_label=False, ) with gr.Column(): gr.Markdown("**📊 Difficulty**") diff_checks = gr.CheckboxGroup( choices=DIFFICULTY_COLS, value=DIFFICULTY_COLS, label="", show_label=False, ) with gr.Column(): gr.Markdown("**📂 Categories**") cat_checks = gr.CheckboxGroup( choices=CATEGORY_COLS, value=[], label="", show_label=False, ) with gr.Column(): gr.Markdown("**❓ Question Types**") type_checks = gr.CheckboxGroup( choices=TYPE_COLS, value=[], label="", show_label=False, ) # Build default view default_view = full_df[[c for c in DEFAULT_COLS if c in full_df.columns]] default_dtypes = [COLUMN_DATATYPES.get(c, "str") for c in default_view.columns] leaderboard_table = gr.Dataframe( value=default_view, datatype=default_dtypes, elem_id="leaderboard-table", interactive=False, wrap=True, ) # Callback to update table def update_table(search_query, meta_sel, diff_sel, cat_sel, type_sel): selected = meta_sel + diff_sel + cat_sel + type_sel result = filter_and_select(full_df, search_query, selected) return result # Wire all inputs to update filter_inputs = [search_bar, meta_checks, diff_checks, cat_checks, type_checks] for inp in filter_inputs: inp.change( update_table, inputs=filter_inputs, outputs=leaderboard_table, ) search_bar.submit( update_table, inputs=filter_inputs, outputs=leaderboard_table, ) # ═══════════════════════════════ # TAB: Charts # ═══════════════════════════════ with gr.TabItem("📊 Charts", id=1): if full_df.empty: gr.Markdown("### No results to chart yet. Submit results first!") else: gr.Markdown("### Visual Comparisons") overall_chart = build_overall_chart(full_df) if overall_chart: gr.Plot(value=overall_chart) diff_chart = build_difficulty_chart(full_df) if diff_chart: gr.Plot(value=diff_chart) type_chart = build_type_chart(full_df) if type_chart: gr.Plot(value=type_chart) radar_chart = build_category_radar(full_df) if radar_chart: gr.Plot(value=radar_chart) # ═══════════════════════════════ # TAB: About # ═══════════════════════════════ with gr.TabItem("📝 About", id=2): with gr.Column(elem_classes="about-section"): gr.Markdown(ABOUT_MD) # ═══════════════════════════════ # TAB: Submit # ═══════════════════════════════ with gr.TabItem("📨 Submit", id=3): with gr.Column(elem_classes="about-section"): gr.Markdown(SUBMIT_MD) return demo demo = create_app() demo.launch()