import csv from collections import defaultdict import gradio as gr import pandas as pd from pathlib import Path def strip_colname(x): if x.startswith("score_"): return x[6:] return x INTRO = """The current leaderboard displays performance across all filtered directions based on the test subset of BOUQuET. The code for reproducing the evaluations or adding a new model is available at https://github.com/facebookresearch/bouquet! """ LANGS_COMMENT = """## Languages For the meaning of the language codes, please refer to the *Languages* tab. """ METRICS_EXPLANATION = """## Metrics 1. `metricx`: [google/metricx-24-hybrid-xl-v2p6](https://huggingface.co/google/metricx-24-hybrid-xl-v2p6) score based on both source and reference. **Attention: lower is better!** 2. `chrfpp`: ChrF++ score ([sacrebleu](https://github.com/mjpost/sacrebleu) implementation) based on reference. We report the mean of sentence-level scores instead of a system-level score. **Attention: values not directly comparable across different target languages!** 3. `glotlid`: probability, as predicted by the [GlotLID model](https://huggingface.co/cis-lmu/glotlid), that translation is in the target language. Note that about 20\% of BOUQuET languages are not supported by GlotLID. 4. `glotlid_x_metricx`: a composite score computed as `glotlid * sqrt(1 - metricx / 25)` to penalize MetricX for off-target translations; higher is better. We report the mean of the sentence-level scores. """ SYSTEMS_EXPLANATION = """## Systems Descriptions of the implementation of the systems will come out later. """ REVERSE_METRICS = {"metricx"} DEFAULT_METRIC = "glotlid_x_metricx" def leaderboard_tab(): # stats = pd.read_csv("data/benchmark_stats.tsv", sep="\t", quoting=csv.QUOTE_NONE) # stats.columns = [strip_colname(c) for c in stats.columns] stats_parts = [] for model_dir in Path("data/mt_benchmark").iterdir(): filename = model_dir / "scores.csv" if not filename.exists(): print(f"No {fn.name} in {model_dir.name} found; skipping") continue part = pd.read_csv(filename) part["system"] = model_dir.name part["level"] = "sentence_level" stats_parts.append(part) stats = pd.concat(stats_parts, ignore_index=True) metrics = ["chrfpp", "metricx", "glotlid_score", "glotlid_x_metricx"] systems = sorted(set(stats["system"])) levels = ["sentence_level"] # TODO: reintroduce "paragraph_level" ALL = "ALL" MEAN = "Average" BEST = "Best" XX2EN = "Everything-into-English" EN2XX = "English-into-Everything" lang_src2tgt = defaultdict(set) lang_tgt2src = defaultdict(set) langs_src = set() langs_tgt = set() for src_lang, tgt_lang in stats[["src_lang", "tgt_lang"]].drop_duplicates().values: lang_src2tgt[src_lang].add(tgt_lang) lang_tgt2src[tgt_lang].add(src_lang) langs_src.add(src_lang) langs_tgt.add(tgt_lang) langs_df = pd.read_csv("data/language_metadata.tsv", sep="\t") lang2name = {} for i, row in langs_df.iterrows(): code = row["Code"] lang2name[code] = row["Language"] if isinstance(row["Secondary ISO 639-3"], str) and len( row["Secondary ISO 639-3"] ): code = row["Secondary ISO 639-3"] + code[3:] lang2name[code] = row["Language"] for lang in langs_src.union(langs_tgt): if lang not in lang2name: print(f"Name not found for {lang}") def named_langs(langs_list): return [ (f"{lang} — {lang2name[lang]}", lang) if lang in lang2name else lang for lang in langs_list ] with gr.Tab("MT leaderboard"): gr.Markdown("# BOUQuET machine translation leaderboard") gr.Markdown(INTRO) gr.Markdown("## Systems ranking") # Inputs gr_level = gr.Dropdown(levels, value="sentence_level", label="Level") gr_src_lang = gr.Dropdown( [ALL] + named_langs(sorted(langs_src)), value=ALL, label="Source lang" ) gr_tgt_lang = gr.Dropdown( [ALL] + named_langs(sorted(langs_tgt)), value=ALL, label="Target lang" ) # Interactivity inputs = [gr_level, gr_src_lang, gr_tgt_lang] def get_lb(level, src_lang, tgt_lang): filtered = stats[stats["level"].eq(level)] if src_lang != ALL: filtered = filtered[filtered["src_lang"].eq(src_lang)] if tgt_lang != ALL: filtered = filtered[filtered["tgt_lang"].eq(tgt_lang)] means = ( filtered.groupby(["system"])[metrics] .mean() .reset_index() .sort_values(DEFAULT_METRIC, ascending=DEFAULT_METRIC in REVERSE_METRICS) ) means.columns = [strip_colname(c) for c in means.columns] styler = means.style.background_gradient().format(precision=4) return styler df_all = get_lb(*[inp.value for inp in inputs]) gr_df = gr.Dataframe(df_all) for inp in inputs: inp.change(fn=get_lb, inputs=inputs, outputs=gr_df) # Interdependecy of the controls def src2tgt(src_lang, tgt_lang): if src_lang == ALL: choices = [ALL] + named_langs(sorted(langs_tgt)) else: choices = [ALL] + named_langs(sorted(lang_src2tgt[src_lang])) return gr.update(choices=choices, value=tgt_lang) def tgt2src(src_lang, tgt_lang): if tgt_lang == ALL: choices = [ALL] + named_langs(sorted(langs_src)) else: choices = [ALL] + named_langs(sorted(lang_tgt2src[tgt_lang])) return gr.update(choices=choices, value=src_lang) gr_src_lang.input( fn=src2tgt, inputs=[gr_src_lang, gr_tgt_lang], outputs=gr_tgt_lang ) gr_tgt_lang.input( fn=tgt2src, inputs=[gr_src_lang, gr_tgt_lang], outputs=gr_src_lang ) gr.Markdown("## Languages difficulty") gr_system = gr.Dropdown( [MEAN, BEST] + systems, value=MEAN, label="Translation system" ) gr_direction = gr.Dropdown( [XX2EN, EN2XX], value=XX2EN, label="Translation direction" ) gr_metric = gr.Dropdown(metrics, label="Quality metric", value=DEFAULT_METRIC) gr_level2 = gr.Dropdown(levels, value="sentence_level", label="Level") bar_controls = [gr_system, gr_direction, gr_metric, gr_level2] def get_hist(system, direction, metric, level): # decide on the data to process if direction == EN2XX: direction_filter = stats["src_lang"].eq("eng_Latn") lang_col = "tgt_lang" else: direction_filter = stats["tgt_lang"].eq("eng_Latn") lang_col = "src_lang" if system in (MEAN, BEST): system_filter = stats["system"].astype(bool) else: system_filter = stats["system"].eq(system) subset = stats[system_filter & direction_filter & stats["level"].eq(level)] # Compute the means and update the plot grouped = subset.groupby(lang_col)[metric] if metric in REVERSE_METRICS: bests = grouped.min() best_sys = grouped.idxmin() else: bests = grouped.max() best_sys = grouped.idxmax() if system == BEST: means = bests else: means = grouped.mean() report = ( pd.DataFrame( { metric: means, "best_system": subset.loc[best_sys]["system"].values, } ) .sort_values(metric, ascending=(metric in REVERSE_METRICS)) .reset_index() ) report["lang_name"] = [lang2name.get(lang, "") for lang in report[lang_col]] tooltip_columns = ["lang_name", "best_system"] # do not return gr.BarPlot, because it gets wrongly cached. return dict( value=report, x=lang_col, y=metric, x_title=lang_col, x_label_angle=-90, height=500, sort="y" if metric in REVERSE_METRICS else "-y", tooltip=tooltip_columns, ) # Use gr.render to fully re-mount the BarPlot on every input change. # This avoids Gradio's BarPlot diff-update bug where changing x/y # leaves the frontend with a stale cached dataset @gr.render(inputs=bar_controls) def render_bar(system, direction, metric, level): if not metric: return gr.BarPlot(**get_hist(system, direction, metric, level)) gr.Markdown(METRICS_EXPLANATION) gr.Markdown(SYSTEMS_EXPLANATION) gr.Markdown(LANGS_COMMENT)