Hugging Face's logo

Spaces:
inesassia
/
fonbench-leaderboard
Sleeping

App Files Community
fonbench-leaderboard
File size: 2,917 Bytes
82d3c52
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
"""
FonBench β€” Data tables for Phase 1 and Phase 2 results.
"""

# Phase 1 β€” Zero-shot evaluation (6 models, test set 3541 examples)
PHASE1_DATA = [
    {"Model": "MMS-1b", "Family": "Encoder-CTC (SSL)", "Params (M)": 967,
     "WER (%)": 83.44, "CER (%)": 31.88, "T-WER (%)": 152.41,
     "WER_seg (%)": 59.55, "WER_ton (%)": 46.43,
     "RTFx": 45.40, "Score": 0.4962, "Status": "βœ… Retained for Phase 2"},
    {"Model": "XLSR-53", "Family": "Encoder-CTC (SSL)", "Params (M)": 317,
     "WER (%)": 101.03, "CER (%)": 64.42, "T-WER (%)": 113.63,
     "WER_seg (%)": 100.95, "WER_ton (%)": 6.34,
     "RTFx": 111.81, "Score": 0.5115, "Status": "Evaluated"},
    {"Model": "OmniASR-CTC", "Family": "Encoder-CTC (SSL)", "Params (M)": 600,
     "WER (%)": 92.97, "CER (%)": 42.46, "T-WER (%)": 125.66,
     "WER_seg (%)": 92.18, "WER_ton (%)": 16.74,
     "RTFx": 31.90, "Score": 0.5066, "Status": "Evaluated"},
    {"Model": "AfriHuBERT", "Family": "Encoder-only (SSL)", "Params (M)": 94,
     "WER (%)": 103.03, "CER (%)": 360.12, "T-WER (%)": float('nan'),
     "WER_seg (%)": 103.01, "WER_ton (%)": 42.35,
     "RTFx": 188.80, "Score": 0.0, "Status": "❌ Disqualified (CER>200%)"},
    {"Model": "SeamlessM4T", "Family": "Encoder-Decoder", "Params (M)": 2300,
     "WER (%)": 112.89, "CER (%)": 100.24, "T-WER (%)": 128.12,
     "WER_seg (%)": 111.28, "WER_ton (%)": 8.42,
     "RTFx": 8.93, "Score": 0.4442, "Status": "Evaluated"},
    {"Model": "Whisper-small", "Family": "Encoder-Decoder", "Params (M)": 244,
     "WER (%)": 165.16, "CER (%)": 170.10, "T-WER (%)": 174.14,
     "WER_seg (%)": 162.18, "WER_ton (%)": 5.98,
     "RTFx": 2.16, "Score": 0.4450, "Status": "Evaluated"},
]

# Phase 2 β€” Adaptation (4 configurations on MMS-1b)
PHASE2_DATA = [
    {"Configuration": "MMS-1b zero-shot", "Strategy": "None",
     "Params trained": "0", "Ratio (%)": "0.00",
     "Learning rate": "β€”", "WER (%)": 83.44, "CER (%)": 31.88,
     "Ξ” WER vs zero-shot": "β€”"},
    {"Configuration": "LoRA-1", "Strategy": "LoRA (r=8, Ξ±=16)",
     "Params trained": "2,031,411", "Ratio (%)": "0.21",
     "Learning rate": "2e-4", "WER (%)": 32.20, "CER (%)": 9.08,
     "Ξ” WER vs zero-shot": "βˆ’51.2 pts"},
    {"Configuration": "LoRA-2", "Strategy": "LoRA (r=32, Ξ±=64)",
     "Params trained": "7,929,651", "Ratio (%)": "0.82",
     "Learning rate": "2e-4", "WER (%)": 28.75, "CER (%)": 8.18,
     "Ξ” WER vs zero-shot": "βˆ’54.7 pts"},
    {"Configuration": "FT-1", "Strategy": "Full Fine-Tuning",
     "Params trained": "966,679,987", "Ratio (%)": "100.00",
     "Learning rate": "1e-5", "WER (%)": 24.32, "CER (%)": 7.17,
     "Ξ” WER vs zero-shot": "βˆ’59.1 pts"},
    {"Configuration": "πŸ† FT-2 (Best)", "Strategy": "Full Fine-Tuning",
     "Params trained": "966,679,987", "Ratio (%)": "100.00",
     "Learning rate": "5e-5", "WER (%)": 19.27, "CER (%)": 6.02,
     "Ξ” WER vs zero-shot": "βˆ’64.2 pts"},
]