""" FonBench Leaderboard — Gradio app. Interactive leaderboard for ASR models evaluated on the Fon language. """ import gradio as gr import pandas as pd import plotly.express as px import plotly.graph_objects as go from data import PHASE1_DATA, PHASE2_DATA # ============================================================ # DATAFRAMES # ============================================================ df_phase1 = pd.DataFrame(PHASE1_DATA) df_phase2 = pd.DataFrame(PHASE2_DATA) # ============================================================ # FILTERS # ============================================================ def filter_phase1(families, show_disqualified): df = df_phase1.copy() if families: df = df[df["Family"].isin(families)] if not show_disqualified: df = df[~df["Status"].str.contains("Disqualified", na=False)] return df.sort_values("WER (%)").reset_index(drop=True) # ============================================================ # PLOTS — PHASE 1 # ============================================================ def plot_phase1_wer_cer(): df = df_phase1[~df_phase1["Status"].str.contains("Disqualified")].copy() df = df.sort_values("WER (%)") fig = go.Figure() fig.add_trace(go.Bar(name="WER (%)", x=df["Model"], y=df["WER (%)"], marker_color="#1f4e79")) fig.add_trace(go.Bar(name="CER (%)", x=df["Model"], y=df["CER (%)"], marker_color="#2e7d32")) fig.update_layout( title="Phase 1 — WER and CER per model (excluding disqualified)", xaxis_title="Model", yaxis_title="Error rate (%)", barmode="group", template="plotly_white", height=420 ) return fig def plot_phase1_speed_quality(): df = df_phase1[~df_phase1["Status"].str.contains("Disqualified")].copy() fig = px.scatter( df, x="RTFx", y="WER (%)", text="Model", color="Family", size="Params (M)", title="Phase 1 — Speed (RTFx) vs Quality (WER)", labels={"RTFx": "RTFx (higher = faster)", "WER (%)": "WER (%, lower = better)"}, template="plotly_white", height=420 ) fig.update_traces(textposition="top center") return fig # ============================================================ # PLOTS — PHASE 2 # ============================================================ def plot_phase2_progression(): df = df_phase2.copy() fig = go.Figure() fig.add_trace(go.Bar(name="WER (%)", x=df["Configuration"], y=df["WER (%)"], marker_color="#1f4e79", text=df["WER (%)"].apply(lambda x: f"{x:.2f}%"), textposition="outside")) fig.add_trace(go.Bar(name="CER (%)", x=df["Configuration"], y=df["CER (%)"], marker_color="#2e7d32", text=df["CER (%)"].apply(lambda x: f"{x:.2f}%"), textposition="outside")) fig.update_layout( title="Phase 2 : Adaptation progression (zero-shot → FT-2)", xaxis_title="Configuration", yaxis_title="Error rate (%)", barmode="group", template="plotly_white", height=450 ) return fig def plot_phase2_cost_perf(): df = df_phase2.copy() df["Params M"] = df["Params trained"].str.replace(",", "").astype(int) / 1e6 df = df[df["Params M"] > 0] fig = px.scatter( df, x="Params M", y="WER (%)", text="Configuration", size_max=40, log_x=True, title="Phase 2 — Trade-off: Trained parameters vs WER", labels={"Params M": "Trained parameters (millions, log scale)", "WER (%)": "WER (%, lower = better)"}, template="plotly_white", height=420 ) fig.update_traces(textposition="top center", marker=dict(size=18, color="#1f4e79")) return fig # ============================================================ # GRADIO UI # ============================================================ title_md = """ # 🎤 FonBench Leaderboard ### ASR Benchmark and Adaptation for the **Fon language** A low-resource African tonal language (~8M speakers in Benin) **Test set**: 3,541 examples · 7 speakers · stratified by speaker """ about_md = """ ## About FonBench This benchmark was developed as part of a Master's thesis at **ESGIS Bénin**, under the supervision of **Dr Mikaël MOUSSE & Dr Fréjus LALEYE**. ### Methodology 1. **Phase 1 : Zero-shot evaluation** of six pre-trained multilingual ASR models: MMS-1b, XLSR-53, OmniASR-CTC, AfriHuBERT, SeamlessM4T, Whisper-small 2. **Phase 2 : Adaptation** of the best model with four configurations: - Two LoRA variants (r=8, r=32) - Two Full Fine-Tuning variants (LR=1e-5, LR=5e-5) ### Metrics - **WER** (Word Error Rate): standard ASR metric - **CER** (Character Error Rate): sub-word reliability - **T-WER** (Tonal Word Error Rate): proposed composite metric for tonal languages - Requires explicit tonal marking in reference transcriptions (see thesis Ch. 3.2.4.3) - **RTFx** (Real-Time Factor inverse): inference speed - **Score**: composite ranking with logarithmic RTFx normalization ### Resources - 🤗 **Models** on HuggingFace Hub: [FT-2](https://huggingface.co/inesassia/fonbench-mms-1b-fon-ft) & [LoRA-2](https://huggingface.co/inesassia/fonbench-mms-1b-fon-lora) - 💻 **Source code** on GitHub: [FonBench](https://github.com/inesassia/FonBench/tree/main) - 📊 **Experiment tracking** on W&B: [FonBench project](https://wandb.ai/inesassiahounkponou-esgis/FonBench) - 📄 **Dataset**: [alaleye/fon](https://huggingface.co/datasets/alaleye/fon) (CC-BY-4.0) ### Citation *Inès Assia HOUNKPONOU. "Evaluation and adaptation of ASR models for the Fon language." Master's thesis, ESGIS Benin, 2026.* """ with gr.Blocks(theme=gr.themes.Soft(primary_hue="blue"), title="FonBench Leaderboard") as demo: gr.Markdown(title_md) with gr.Tabs(): # ============================================================ # TAB 1 : PHASE 1 # ============================================================ with gr.Tab("📊 Phase 1 : Zero-shot"): gr.Markdown("### Six multilingual ASR models evaluated zero-shot on the Fon test set") with gr.Row(): with gr.Column(scale=1): family_filter = gr.CheckboxGroup( choices=["Encoder-CTC (SSL)", "Encoder-only (SSL)", "Encoder-Decoder"], value=["Encoder-CTC (SSL)", "Encoder-only (SSL)", "Encoder-Decoder"], label="Filter by model family" ) with gr.Column(scale=1): show_dsq = gr.Checkbox(value=True, label="Show disqualified models (CER > 200%)") table_p1 = gr.Dataframe( value=filter_phase1(["Encoder-CTC (SSL)", "Encoder-only (SSL)", "Encoder-Decoder"], True), interactive=False, wrap=True ) family_filter.change(filter_phase1, [family_filter, show_dsq], table_p1) show_dsq.change(filter_phase1, [family_filter, show_dsq], table_p1) with gr.Row(): gr.Plot(plot_phase1_wer_cer()) gr.Plot(plot_phase1_speed_quality()) gr.Markdown(""" **Note on model selection:** Although XLSR-53 obtains the highest composite score, its WER_segmental exceeds 100% (no exploitable transcription). MMS-1b is retained for Phase 2 based on WER and CER (the only diacritic-agnostic metrics in this corpus, whose reference transcriptions do not include systematic tonal marking). See thesis Section 4.1.2 for the full justification. """) # ============================================================ # TAB 2 : PHASE 2 # ============================================================ with gr.Tab("🏆 Phase 2 : Adaptation"): gr.Markdown("### Four adaptation configurations applied to MMS-1b") gr.Dataframe(value=df_phase2, interactive=False, wrap=True) with gr.Row(): gr.Plot(plot_phase2_progression()) gr.Plot(plot_phase2_cost_perf()) gr.Markdown(""" **Key findings:** - 🏆 **FT-2 (LR=5e-5)** reaches **WER 19.27%** and **CER 6.02%** a **64-point WER reduction** over zero-shot. - **LoRA-2** trains only **0.82%** of parameters (~120× less than full FT) while reaching WER 28.75% strong cost/quality trade-off. - The full fine-tuning beats LoRA by ~10 absolute WER points, consistent with the higher capacity of full parameter updates. """) # ============================================================ # TAB 3 : ABOUT # ============================================================ with gr.Tab("ℹ️ About"): gr.Markdown(about_md) if __name__ == "__main__": demo.launch()