import json from pathlib import Path import gradio as gr import pandas as pd ROOT = Path(__file__).parent RESULTS_PATH = ROOT / "results.json" MODELS_PATH = ROOT / "models.json" COLUMNS = [ "Model", "Params", "Avg Fair", "Gold69 v2 Fair CER", "Gold69 v2 Fair WER", "FLEURS WER", "FLEURS CER", "Decode (ms)", "Family", "Notes", ] DATATYPES = [ "markdown", "str", "number", "number", "number", "number", "number", "number", "number", "str", "str", ] STATUS_RANK = {"complete": 0, "running": 1, "queued": 2, "failed": 3} def read_json(path: Path) -> list[dict]: if not path.exists(): return [] return json.loads(path.read_text()) def pct(value): return None if value is None or pd.isna(value) else round(float(value), 2) def num(value, digits=3): return None if value is None or pd.isna(value) else round(float(value), digits) def avg_wer(row: dict): gold = row.get("gold69_v2_fair_cer") fleurs = row.get("fleurs_cer") if gold is None or fleurs is None or pd.isna(gold) or pd.isna(fleurs): return None return (float(gold) + float(fleurs)) / 2 def model_link(row: dict) -> str: model = row.get("model", "") repo = row.get("repo", "") if isinstance(repo, str) and repo.startswith("http"): return f"[{model}]({repo})" return model def combined_rows() -> pd.DataFrame: result_rows = {row.get("model"): row for row in read_json(RESULTS_PATH)} model_rows = read_json(MODELS_PATH) merged = [] seen = set() for model_row in model_rows: model = model_row.get("model") merged.append({**model_row, **result_rows.get(model, {})}) seen.add(model) for model, row in result_rows.items(): if model not in seen: merged.append(row) records = [] for row in merged: records.append( { "Model": model_link(row), "Params": num(row.get("params_b"), 3), "Avg Fair": pct(avg_wer(row)), "Gold69 v2 Fair CER": pct(row.get("gold69_v2_fair_cer")), "Gold69 v2 Fair WER": pct(row.get("gold69_v2_fair_wer")), "FLEURS WER": pct(row.get("fleurs_wer")), "FLEURS CER": pct(row.get("fleurs_cer")), "Decode (ms)": num(row.get("mean_decode_ms"), 1), "Family": row.get("family", ""), "Status": row.get("status", "queued"), "Notes": row.get("notes", ""), "_status_rank": STATUS_RANK.get(row.get("status", "queued"), 9), "_model_text": row.get("model", ""), } ) return pd.DataFrame(records) def display_frame() -> pd.DataFrame: df = combined_rows() df = df.sort_values("Avg Fair", ascending=True, na_position="last") display = df[COLUMNS].reset_index(drop=True).astype(object) return display.where(pd.notna(display), None) def summary_cards() -> str: df = combined_rows() completed = df[df["Status"].eq("complete")] best_gold = completed["Gold69 v2 Fair CER"].min() if not completed.empty else None best_fleurs = completed["FLEURS WER"].min() if not completed.empty else None return f"""
A same-test leaderboard for Persian speech recognition models. Gold69 v2 is the corrected tougher real-world set; FLEURS is the cleaner public reference set. Gold69 v2 fair CER ignores punctuation, spaces, half-spaces, and diacritics so models are judged on the words they heard. Lower is better.