import json from pathlib import Path import gradio as gr import pandas as pd ROOT = Path(__file__).parent RESULTS_PATH = ROOT / "results.json" MODELS_PATH = ROOT / "models.json" COLUMNS = [ "Model", "Params", "Triple Threat 🔥", "S³", "Ess Err 🔑", "VisualEars6669 Fair WER", "VisualEars6669 Fair CER", "FLEURS-fa Full WER", "FLEURS-fa Full CER", "Decode (ms)", "Family", "Status", "Notes", ] DATATYPES = [ "markdown", "str", "number", "number", "number", "number", "number", "number", "number", "str", "str", "str", "str", ] STATUS_RANK = {"complete": 0, "running": 1, "queued": 2, "failed": 3} def read_json(path: Path) -> list[dict]: if not path.exists(): return [] return json.loads(path.read_text()) def pct(value): return None if value is None or pd.isna(value) else round(float(value), 2) def num(value, digits=3): return None if value is None or pd.isna(value) else round(float(value), digits) def _avg(*xs): xs = [float(x) for x in xs if x is not None and not pd.isna(x)] return sum(xs) / len(xs) if xs else None def s3_avg(row: dict): return _avg(row.get("visualears6669_s3"), row.get("fleurs_s3")) def triple_threat(row: dict): """60% S³ + 20% WER + 20% CER, with each pillar split 50/50 by dataset. The score is defined only when all six dataset-level inputs exist, so a model is never flattered by a missing penalty term or an incomplete benchmark split. """ s3 = s3_avg(row) p_wer = _avg(row.get("visualears6669_fair_wer"), row.get("fleurs_wer")) p_cer = _avg(row.get("visualears6669_fair_cer"), row.get("fleurs_cer")) required = ( row.get("visualears6669_s3"), row.get("fleurs_s3"), row.get("visualears6669_fair_wer"), row.get("fleurs_wer"), row.get("visualears6669_fair_cer"), row.get("fleurs_cer"), ) if any(value is None or pd.isna(value) for value in required): return None return 0.60 * s3 + 0.20 * p_wer + 0.20 * p_cer def auto_note(row: dict) -> str: note = row.get("notes", "") or "" flags = [] if s3_avg(row) is None and row.get("visualears6669_fair_wer") is not None: flags.append("S³ pending") elif row.get("fleurs_s3") is None and row.get("visualears6669_s3") is not None: flags.append("FLEURS S³ pending") if flags: tag = " · ".join(flags) note = f"{tag} — {note}" if note else tag return note def medal_for_rank(rank: int | None) -> str: return {1: "🥇", 2: "🥈", 3: "🥉"}.get(rank, "") def model_link(row: dict, medal: str = "") -> str: model = row.get("model", "") repo = row.get("repo", "") prefix = "🔒 " if row.get("closed_model") else "" medal_prefix = f"{medal} " if medal else "" if isinstance(repo, str) and repo.startswith("http"): return f"{medal_prefix}{prefix}[{model}]({repo})" return f"{medal_prefix}{prefix}{model}" def combined_rows() -> pd.DataFrame: result_rows = {row.get("model"): row for row in read_json(RESULTS_PATH)} model_rows = read_json(MODELS_PATH) merged = [] seen = set() for model_row in model_rows: model = model_row.get("model") merged.append({**model_row, **result_rows.get(model, {})}) seen.add(model) for model, row in result_rows.items(): if model not in seen: merged.append(row) tt_scores = {row.get("model"): triple_threat(row) for row in merged} ranked = sorted( ((m, s) for m, s in tt_scores.items() if s is not None and not pd.isna(s)), key=lambda item: (item[1], item[0] or ""), ) medal_ranks = {m: rank for rank, (m, _s) in enumerate(ranked[:3], 1)} records = [] for row in merged: records.append( { "Model": model_link(row, medal_for_rank(medal_ranks.get(row.get("model")))), "Params": num(row.get("params_b"), 3), "Triple Threat 🔥": pct(triple_threat(row)), "S³": pct(s3_avg(row)), "Ess Err 🔑": pct(row.get("ess_err")), "VisualEars6669 Fair WER": pct(row.get("visualears6669_fair_wer")), "VisualEars6669 Fair CER": pct(row.get("visualears6669_fair_cer")), "FLEURS-fa Full WER": pct(row.get("fleurs_wer")), "FLEURS-fa Full CER": pct(row.get("fleurs_cer")), "Decode (ms)": ("1×RT" if "Web Speech" in (row.get("model") or "") else ("" if (_d := num(row.get("mean_decode_ms"), 1)) is None else f"{_d}")), "Family": row.get("family", ""), "Status": row.get("status", "queued"), "Notes": auto_note(row), "_status_rank": STATUS_RANK.get(row.get("status", "queued"), 9), "_model_text": row.get("model", ""), } ) return pd.DataFrame(records) def display_frame() -> pd.DataFrame: df = combined_rows() df = df.sort_values( ["Triple Threat 🔥", "S³", "VisualEars6669 Fair WER"], ascending=[True, True, True], na_position="last", ) display = df[COLUMNS].reset_index(drop=True).astype(object) return display.where(pd.notna(display), None) def summary_cards() -> str: df = combined_rows() scored_tt = df[df["Triple Threat 🔥"].notna()] scored_s3 = df[df["S³"].notna()] def champ(frame, metric, icon): if frame.empty: return "—" r = frame.sort_values([metric, "_model_text"], ascending=[True, True]).iloc[0] return f"{icon} {r['_model_text']} · {float(r[metric]):.2f}" return f"""
A same-test leaderboard for Persian speech recognition. Two real-world sets — VisualEars6669 (6,669-row / 10.49h noisy benchmark with clean, farfield & obstructed conditions) and FLEURS-fa Full (4,341-row public Persian FLEURS) — now scored three ways:
S³ is a Persian-first Deaf/hard-of-hearing-focused semantic metric: it weights every ASR error by how much that word matters for understanding, so dropping a keyword costs far more than a filler. 🔑 Ess Err is the error rate on only the most meaning-critical words (importance ≥ 0.8) — the words a DHH reader can least afford to lose. Triple Threat 🔥 = 60% S³ + 20% WER + 20% CER. Each pillar is averaged 50/50 across VisualEars6669 and FLEURS-fa Full, so the two dataset splits remain equally weighted. Fair metrics ignore punctuation, spaces, half-spaces and diacritics, and normalize numbers (spelled-out ↔ digits) so a correctly-heard «صد» isn't penalized against «۱۰۰». Lower is better. 🔒 marks closed paid APIs. S³ is undergoing validation with Persian DHH annotators.