#!/usr/bin/env python3 """Gradio wrapper that serves the HTML eval leaderboard with auth + public URL.""" import argparse import csv import html import json import re from pathlib import Path import gradio as gr STAGES = [ ("all", "All"), ("image", "Stage 1 - Image"), ("video", "Stage 2 - Video"), ("vlm", "Stage 3 - VLM"), ("mllm", "Stage 4 - MLLM"), ] GLOBAL_CATEGORIES = [ ("all", "All"), ("prod", "Prod Models"), ("baseline", "Baselines"), ("research", "Research"), ] GCAT_LABELS = {"prod": "Prod Models", "baseline": "Baselines", "research": "Research"} GCAT_ORDER = {"prod": 0, "baseline": 1, "research": 2} PROD_MODELS = { "prod_v1_100m": { "display_name": "Prod v1, 100M", "wandb_url": "https://czi.wandb.io/ai_imaging/dca-pretraining/runs/7roeq1ro", "base_model": "", "owner": "Ritvik Vasan", }, "fg12bbfr_v2_100m": { "display_name": "Prod v2, 100M", "wandb_url": "https://czi.wandb.io/ai_imaging/dca-pretraining/runs/fg12bbfr", "base_model": "", "owner": "Ritvik Vasan", }, "baseline_fg12bbfr": { "display_name": "Prod v2, 100M (frozen eval)", "wandb_url": "https://czi.wandb.io/ai_imaging/dca-pretraining/runs/fg12bbfr", "base_model": "", "owner": "Markus Marks", }, "prod_v2_300m": { "display_name": "Prod v2, 300M", "wandb_url": "https://czi.wandb.io/ai_imaging/dca-pretraining/runs/fr0a2kxv", "base_model": "Prod v2, 100M", "owner": "Ritvik Vasan", }, "prod_v1_vit_s_250000": { "display_name": "Prod v1, ViT-S (250k)", "wandb_url": "", "base_model": "", "owner": "Ritvik Vasan", }, "prod_v1_vit_b_100000": { "display_name": "Prod v1, ViT-B (100k)", "wandb_url": "", "base_model": "", "owner": "Ritvik Vasan", }, } BASELINE_NAMES = { "dino_vitb16", "dino_vits16", "dinov1_zeroshot", "dinov2_vitb14", "dinov2_vits14", "dinov3_vitb16", "openphenom_alfi_fix", "openphenom_cellphie_h100", "openphenom_fast", "vjepa2_L", "vjepa2_vitb_384", "aot_dinov2s", "aot_dinov3b", "aot_dinov3s", "aot_vjepa2", "dc_dinov2s", "dc_dinov3b", "dc_dinov3s", "dc_vjepa2", "lc_vjepa2", "lc64_vjepa2", } EVAL_SETS = { "image": {"chammi", "cellphie", "hpav23", "idr0017", "rbc-mc", "jumpcp", "allencell_nuclear", "allencell_static", "opencell", "rxrx1", "livecell", "biad2515", "cellstate", "rxrx3_core"}, "video": {"erk", "alfi", "dynacell_annotated", "cbvcc_probe"}, "vlm": {"ret1k", "ret85k", "decoding", "ret500", "vlm"}, } _SCALING_MILESTONES = { "prod_v1_vit_s_3500", "prod_v1_vit_s_10500", "prod_v1_vit_s_50000", "prod_v1_vit_s_100000", "prod_v1_vit_s_150000", "prod_v1_vit_s_200000", "prod_v1_vit_b_1000", "prod_v1_vit_b_3000", "prod_v1_vit_b_9500", "prod_v1_vit_b_50000", } def _global_cat(name: str, ws: str) -> str: if name in _SCALING_MILESTONES: return "research" if name in PROD_MODELS: return "prod" if ws == "ws_prod": return "prod" if name in BASELINE_NAMES or ws == "baseline": return "baseline" return "research" def _detect_checkpoint_series(name: str) -> str | None: m = re.match(r"^(prod_v\d+_vit_[a-z])_\d+$", name) if m: return m.group(1) m = re.match(r"^epoch-\d+-step-\d+(_frozen)?$", name) if m: return "prod_milestones" m = re.match(r"^(.+?)_epoch-\d+-step-\d+(_frozen)?$", name) if m: return m.group(1) return None def _extract_step(name: str) -> int: m = re.search(r"step-(\d+)", name) if m: return int(m.group(1)) m = re.search(r"_(\d+)$", name) if m: return int(m.group(1)) return 0 def _detect_baseline_family(name: str) -> str | None: lower = name.lower() if "morphem" in lower: return "MorphEm" if "openphenom" in lower: return "OpenPhenom" if "vjepa" in lower: return "V-JEPA" if "dinov3" in lower: return "DINOv3" if "dinov2" in lower: return "DINOv2" if "dino" in lower: return "DINO v1" if "clip" in lower: return "CLIP" return None TEAM_MEMBERS = [ "Trang Le", "Ritvik Vasan", "Benjamin Gallusser", "Kyle Harrington", "Alexander Lin", "Markus Marks", ] # Map group name -> column name that is the summary metric (shown when collapsed) WORKSTREAM_MAP = { "baseline_dinov2_vitb14_frozen": "baseline", "baseline_dinov3_vitb16_frozen": "baseline", "baseline_dino_vitb16_frozen": "baseline", "baseline_dino_vits16_frozen": "baseline", "dinov2_vitb14_zeroshot": "baseline", "dinov3_vitb16_zeroshot": "baseline", "dino_vitb16_zeroshot": "baseline", "dino_vits16_zeroshot": "baseline", "prod_v1_100m": "ws_prod", "prod_v2_300m": "ws_prod", "v2_100m_base": "ws_prod", "v2_vits_100k_baseline": "ws1b", "ws4p6_repro3_frozen": "ws4", "ws4p6_twostage_long": "ws4", "ws4p6_weight_schedule": "ws4", "ws4p7_anchor_decay_frozen": "ws4", "ws4p7_strong_anchor_frozen": "ws4", "ws4p8a_decay_high_end_frozen": "ws4", "ws4p8b_constant_frozen": "ws4", "ws4p8c_long_decay_frozen": "ws4", "ws4p9a_very_long_frozen": "ws4", "ws4p9b_dinov3_anchor_frozen": "ws4", "ws4p10a_unmasked_frozen": "ws4", "ws4p10b_twostage_repro_frozen": "ws4", "ws4p11a_multigpu_fixed": "ws4", "ws4p11a_multigpu_frozen": "ws4", "ws4p11b_30k_frozen": "ws4", "ws4p12a_no_anchor_frozen": "ws4", "ws4p12b_8crops_frozen": "ws4", "ws4p13a_two_backward_frozen": "ws4", "ws4p13b_single_bw_frozen": "ws4", "ws4p13d_two_bw_v2_frozen": "ws4", "ws4p13e_2bw_w1_frozen": "ws4", "ws4p14a_fp32_frozen": "ws4", "ws4p14b_2bw_fp32_frozen": "ws4", "v2_baseline_full": "ws1a", "v2_baseline_200k_full": "ws1a", "v2_baseline_s100000": "ws1a", "v2_ccp_full": "ws1a", "v2_ccp_s100000": "ws1a", "v2_cmr_full": "ws1a", "v2_cmr_s100000": "ws1a", "v2_icvit_50k_full": "ws1a", "v2_icvit_aug_full": "ws1a", "v2_icvit_phase2_full": "ws1a", "v2_icvit_phase2_s100000": "ws1a", "v2_icvit_cmr_bf_full": "ws1a", "v2_icvit_cmr_bf_aug_full": "ws1a", "v2_vcsym_full": "ws1a", "v2_vcsym_s100000": "ws1a", "v2_vits_100k_ccp": "ws1b", "v2_vits_100k_cmr": "ws1b", "v2_vits_100k_dino_v17_cls": "ws1b", "v2_vits_100k_dino_v17_spatial": "ws1b", "v2_vits_100k_icvit_50k": "ws1b", "v2_vits_100k_icvit_aug": "ws1b", "v2_vits_100k_icvit_phase2": "ws1b", "v2_vits_100k_icvit_cmr_bf": "ws1b", "v2_vits_100k_virtual_channels_sym": "ws1b", "v2_vits_100k_dino_v18_dinov2_epoch-0000-step-100000_frozen": "ws41", "v2_vits_100k_dino_v18_epoch-0000-step-100000_frozen": "ws41", "v2_vits_100k_dino_v18_koleo_epoch-0000-step-100000_frozen": "ws41", "v2_vits_100k_dino_v19_224_epoch-0000-step-10000_frozen": "ws41", "v2_vits_100k_dino_v19_224_epoch-0000-step-20000_frozen": "ws41", "v2_vits_100k_dino_v19_224_epoch-0000-step-30000_frozen": "ws41", "v2_vits_100k_dino_v19_224_epoch-0000-step-40000_frozen": "ws41", "v2_vits_100k_dino_v19_224_epoch-0000-step-50000_frozen": "ws41", "v2_vits_100k_dino_v19_perchannel_epoch-0000-step-10000_frozen": "ws41", "v2_vits_100k_dino_v19_perchannel_epoch-0000-step-20000_frozen": "ws41", "v2_vits_100k_dino_v19_perchannel_epoch-0000-step-30000_frozen": "ws41", "v2_vits_100k_dino_v19_perchannel_epoch-0000-step-40000_frozen": "ws41", "v2_vits_100k_dino_v19_perchannel_epoch-0000-step-50000_frozen": "ws41", "v2_vits_100k_dino_v19_perchannel_epoch-0000-step-60000_frozen": "ws41", "v2_vits_100k_dino_v19_perchannel_epoch-0000-step-70000_frozen": "ws41", "v2_vits_100k_swav_epoch-0000-step-10000_frozen": "ws1b", "v2_vits_100k_swav_epoch-0000-step-20000_frozen": "ws1b", "v2_vits_100k_swav_epoch-0000-step-30000_frozen": "ws1b", "v2_vits_100k_swav_epoch-0000-step-60000_frozen": "ws1b", "v2_vits_100k_swav_epoch-0000-step-80000_frozen": "ws1b", "v2_vits_100k_swav_epoch-0000-step-90000_frozen": "ws1b", "v2_vits_200k_dino_v18_long_epoch-0000-step-100000_frozen": "ws41", "v2_vits_200k_dino_v18_long_epoch-0000-step-110000_frozen": "ws41", "v2_vits_200k_dino_v18_long_epoch-0000-step-140000_frozen": "ws41", "v2_vits_200k_dino_v18_long_epoch-0000-step-150000_frozen": "ws41", "v2_vits_100k_icvit_phase2_cmr_bf_asym025_epoch-0000-step-100000_frozen": "ws10", "v2_vits_100k_icvit_phase2_cmr_bf_asym025_epoch-0000-step-80000_frozen": "ws10", "v2_vits_100k_icvit_phase2_cmr_bf_asym025_epoch-0000-step-90000_frozen": "ws10", "v2_vits_100k_icvit_phase2_cmr_bf_cvi_epoch-0000-step-60000_frozen": "ws10", "v2_vits_100k_icvit_phase2_cmr_bf_cvi_epoch-0000-step-70000_frozen": "ws10", "v2_vits_100k_icvit_phase2_cmr_bf_cvi_epoch-0000-step-80000_frozen": "ws10", "v2_vits_100k_icvit_phase2_cmr_bf_cvi_epoch-0000-step-90000_frozen": "ws10", "v2_vits_100k_icvit_phase2_cmr_bf_cvi_epoch-0000-step-100000_frozen": "ws10", "v2_vits_100k_icvit_phase2_cmr_bf_gcca_epoch-0000-step-70000_frozen": "ws10", "v2_vits_100k_icvit_phase2_cmr_bf_gcca_epoch-0000-step-80000_frozen": "ws10", "v2_vits_100k_icvit_phase2_cmr_bf_gcca_epoch-0000-step-90000_frozen": "ws10", "v2_vits_100k_icvit_phase2_cmr_bf_gcca_epoch-0000-step-100000_frozen": "ws10", "v2_vits_l1000_pilot_epoch-0000-step-10000_frozen": "ws11", "v2_vits_l1000_control_epoch-0000-step-10000_frozen": "ws11", "v2_vits_esm3_pilot_epoch-0000-step-10000_frozen": "ws7", "pathway_w001_step10000_frozen": "ws7", "pathway_w001_step10000_full": "ws7", "video_mae_10m_no_affine_hetero": "ws10", "video_mae_10m_ccp": "ws1b", "video_mae_10m_cmr": "ws1b", "video_mae_10m_contrastive": "ws1b", "video_mae_10m_ibot_dino1": "ws42", "video_mae_10m_ibot": "ws42", "video_mae_10m_icvit_phase1": "ws1a", "video_mae_10m_icvit_phase2": "ws1a", "video_mae_10m_no_affine_bf16_lr1e3_mae_init": "ws1b", "video_mae_10m_no_affine_bf16": "ws1b", "video_mae_10m_celldino_aug": "ws6", "vlm_coca_100m_phase2": "ws8", "vlm_coca_phase2": "ws8", "vlm_bert_init_100m_500k": "ws8", "vlm_coca_100m_500k": "ws8", "vlm_coca_100m_800k": "ws8", "vlm_coca_300m_200k": "ws8", "vlm_siglip_coca_100m_100k": "ws8", "vlm_siglip_only_100m_100k": "ws8", "vlm_distill_100m_200k": "ws8", "vlm_pe_distill_100m_200k": "ws8", "vlm_llm500k_100m_200k": "ws8", "vlm_llm500k_1000k_100m_300k": "ws8", "vlm_newbio_100m_200k": "ws8", "vlm_rnd500k_100m_200k": "ws8", "vlm_mc_100m_200k_opencell": "ws8", "vlm_dca_distill_pe_100m_200k": "ws8", "vlm_coca_300m_200k_cellphie": "ws8", "vlm_coca_300m_200k_cellphie_v2": "ws8", "vlm_v3_baseline": "ws8", "vlm_v3_bert_init": "ws8", "vlm_v3_distill_kl": "ws8", "vlm_v3_distill_pe": "ws8", "vlm_v3_hardneg": "ws8", "vlm_v3_hardneg_skip5": "ws8", "vlm_v3_mask_reg": "ws8", "vlm_v3_paired_clean": "ws8", "vlm_v3_paired_raw": "ws8", "vlm_v3_siglip": "ws8", "vlm_v4_eval_v4_500k_fast": "ws8", "vlm_v4_eval_v4_baseline_dca_fast": "ws8", "vlm_ret_biomedclip_pretrained": "ws8", "vlm_ret_coca_200k_100m": "ws8", "vlm_ret_coca_300m_phase2": "ws8", "vlm_ret_coca_phase2_100m": "ws8", "vlm_ret_phase1_frozen_100m": "ws8", "vlm_ret_siglip2_100m": "ws8", "vlm_ret_siglip_loca_100m": "ws8", "vlm_ret_siglip_only_100m": "ws8", "vlm_ret500_coca_500k": "ws8", "vlm_ret500_distill_100m": "ws8", "vlm_ret500_distill_pe_100m": "ws8", "vlm_ret500_entity500k_100m": "ws8", "vlm_ret500_llm500k_100m": "ws8", "vlm_ret500_mc_100m": "ws8", "vlm_ret500_mc_native_100m": "ws8", "vlm_ret500_mc_vanilla_100m": "ws8", "vlm_ret500_newbio_100m": "ws8", "vlm_ret500_ontology500k_100m": "ws8", "vlm_ret500_rnd500k_100m": "ws8", "vlm_ret500_siglip2_100m": "ws8", "vlm_ret500_siglip_coca_100m": "ws8", "vlm_ret500_siglip_only_100m": "ws8", "epoch-0000-step-10000_frozen": "ws_prod", "epoch-0000-step-20000_frozen": "ws_prod", "epoch-0000-step-30000_frozen": "ws_prod", "epoch-0000-step-40000_frozen": "ws_prod", "epoch-0000-step-50000_frozen": "ws_prod", "epoch-0000-step-60000_frozen": "ws_prod", "epoch-0000-step-70000_frozen": "ws_prod", "epoch-0000-step-80000_frozen": "ws_prod", "epoch-0000-step-90000_frozen": "ws_prod", "epoch-0000-step-100000_frozen": "ws_prod", "epoch-0000-step-160000_frozen": "ws_prod", "epoch-0000-step-170000_frozen": "ws_prod", "epoch-0000-step-180000_frozen": "ws_prod", "epoch-0000-step-190000_frozen": "ws_prod", "epoch-0000-step-200000_frozen": "ws_prod", "epoch-0439-step-13640_frozen": "ws_prod", "epoch-0617-step-19189_frozen": "ws_prod", "v2_10m_celldino_complete_epoch-0000-step-100000_frozen": "ws6", "v2_10m_100k_shallow_dec": "ws1b", "v2_10m_100k_shallow_dec_epoch-0000-step-80000_frozen": "ws1b", "v2_10m_100k_shallow_dec_epoch-0000-step-90000_frozen": "ws1b", "v2_10m_100k_shallow_dec_epoch-0000-step-100000_frozen": "ws1b", "v17_dino_cls_s100000": "ws1b", "v17_dino_spatial_s100000": "ws1b", } WORKSTREAM_LABELS = { "all": "All", "baseline": "Baselines", "ws_prod": "Prod Runs", "ws1a": "WS1a Virtual Ch.", "ws1b": "WS1b Image Rep.", "ws4": "WS4 DINO-FT", "ws6": "WS6 Augmentations", "ws7": "WS7 BioAlign", "ws8": "WS8 VLM", "ws10": "WS10 Hetero", "ws11": "WS11 L1000", "ws41": "WS41 DINOv2", "ws42": "WS42 iBOT", } SUMMARY_COLS = { "chammi": "chammi/chammi_macro_f1", "cellphie": "cellphie/cellphie_macro_f1", "hpav23": "hpav23/hpav23_Average Precision", "idr0017": "idr0017/idr0017_katamari_early_fusion", "rbc-mc": "rbc-mc/rbcmc_overall_accuracy", "jumpcp": "jumpcp/jumpcp_mean_map", "allencell_nuclear": "allencell_nuclear/allencell_nuclear_psnr", "allencell_static": "allencell_static/allencell_static_ssim", "opencell": "opencell/opencell_micro_f1", "erk": "erk/erk_ssim", "cellstate": "cellstate/cellstate/gene_lr_accuracy", "rxrx1": "rxrx1/rxrx1_percent_matching", "livecell": "livecell/livecell_accuracy", "biad2515": "biad2515/biad2515_accuracy", "rxrx3_core": "rxrx3_core/rxrx3_core_pert_signal_median_magnitude", "alfi": None, "ret1k": "ret1k/ret1k_i2t_r1", "ret85k": "ret85k/ret85k_i2t_r1", "decoding": "decoding/dec_clipscore", "ret500": "ret500/i2t_r@1", } VLM_COL_MAP = { "chammi_macro_f1": "chammi/chammi_macro_f1", "chammi_accuracy": "chammi/chammi_accuracy", "cellphie_macro_f1": "cellphie/cellphie_macro_f1", "cellphie_auc": "cellphie/cellphie_macro_auc", "rbcmc_accuracy": "rbc-mc/rbcmc_overall_accuracy", "rbcmc_swiss": "rbc-mc/rbcmc_swiss_accuracy", "rbcmc_canadian": "rbc-mc/rbcmc_canadian_accuracy", "opencell_micro_f1": "opencell/opencell_micro_f1", "hpav23_ap": "hpav23/hpav23_Average Precision", "idr0017": "idr0017/idr0017_katamari_early_fusion", "jumpcp_map": "jumpcp/jumpcp_mean_map", "allen_structure": "allencell_static/allencell_static_structure_label_accuracy", "allen_mitotic": "allencell_nuclear/allencell_nuclear_mitotic_label_accuracy", } def _parse_float(v: str) -> float | None: v = v.strip() if v == "": return None try: return float(v) except ValueError: return None def load_results(results_dir: Path) -> tuple[list[str], dict[str, list[float | None]]]: header_path = results_dir / "header.txt" raw_columns = header_path.read_text().strip().split(",") keep = [ i for i, c in enumerate(raw_columns) if c.strip() and c.strip() != "placeholder" ] columns = [raw_columns[i] for i in keep] rows: dict[str, list[float | None]] = {} for txt in sorted(results_dir.glob("*.txt")): if txt.name in ("header.txt", "header2.txt", "chammi.txt", "row.txt"): continue model = txt.stem vals_raw = txt.read_text().strip().split(",") all_vals = [_parse_float(v) for v in vals_raw] while len(all_vals) < len(raw_columns): all_vals.append(None) rows[model] = [all_vals[i] for i in keep] # --- Load VLM results from multiple sources --- vlm_dir = results_dir / "vlm" if not vlm_dir.exists(): return columns, rows # 1) Determine VLM-only columns from vlm_eval_all.txt header vlm_all_path = vlm_dir / "vlm_eval_all.txt" vlm_only_col_names: list[str] = [] if vlm_all_path.exists(): vlm_header = vlm_all_path.read_text().strip().splitlines()[0].split(",") for vc in vlm_header: if vc == "checkpoint" or vc in VLM_COL_MAP: continue if vc.startswith("ret1k_"): vlm_only_col_names.append("ret1k/" + vc) elif vc.startswith("ret85k_"): vlm_only_col_names.append("ret85k/" + vc) elif vc.startswith("dec_"): vlm_only_col_names.append("decoding/" + vc) elif vc.startswith("alfi_"): vlm_only_col_names.append("alfi/" + vc) else: vlm_only_col_names.append("vlm/" + vc) # Also add retrieval_evals columns ret_header_path = vlm_dir / "retrieval_evals" / "header.txt" ret_eval_cols: list[str] = [] if ret_header_path.exists(): for rc in ret_header_path.read_text().strip().split(","): full = "ret500/" + rc.strip() if full not in vlm_only_col_names: ret_eval_cols.append(full) all_new_cols = vlm_only_col_names + ret_eval_cols columns.extend(all_new_cols) n_new = len(all_new_cols) for model in rows: rows[model].extend([None] * n_new) def _col_idx(name: str) -> int | None: try: return columns.index(name) except ValueError: return None def _set_val(vals: list, col_name: str, fval: float | None): idx = _col_idx(col_name) if idx is not None and fval is not None: if vals[idx] is None: vals[idx] = fval def _ensure_row(name: str) -> list: if name not in rows: rows[name] = [None] * len(columns) return rows[name] # 2) Load vlm_eval_all.txt if vlm_all_path.exists(): lines = vlm_all_path.read_text().strip().splitlines() vlm_header = lines[0].split(",") for line in lines[1:]: parts = line.split(",") model_name = parts[0].strip() if not model_name: continue prefixed = "vlm_" + model_name vals = _ensure_row(prefixed) for vi, vc in enumerate(vlm_header): if vc == "checkpoint" or vi >= len(parts): continue fval = _parse_float(parts[vi]) main_col = VLM_COL_MAP.get(vc) if main_col: _set_val(vals, main_col, fval) elif vc.startswith("ret1k_"): _set_val(vals, "ret1k/" + vc, fval) elif vc.startswith("ret85k_"): _set_val(vals, "ret85k/" + vc, fval) elif vc.startswith("dec_"): _set_val(vals, "decoding/" + vc, fval) elif vc.startswith("alfi_"): _set_val(vals, "alfi/" + vc, fval) else: _set_val(vals, "vlm/" + vc, fval) # 3) Load retrieval_evals/ (CSV rows with header) ret_evals_dir = vlm_dir / "retrieval_evals" if ret_evals_dir.exists() and ret_header_path.exists(): ret_hdr = ret_header_path.read_text().strip().split(",") for txt in sorted(ret_evals_dir.glob("*.txt")): if txt.name == "header.txt": continue model_name = "vlm_ret_" + txt.stem parts = txt.read_text().strip().split(",") vals = _ensure_row(model_name) for ri, rc in enumerate(ret_hdr): if ri < len(parts): _set_val(vals, "ret500/" + rc.strip(), _parse_float(parts[ri])) # 4) Load vision_evals/*.csv, v3_evals/vision_*.csv, v4_baseline/*.csv (long format) LONG_FORMAT_METRIC_MAP = { "cellphie_macro_f1": "cellphie/cellphie_macro_f1", "cellphie_macro_auc": "cellphie/cellphie_macro_auc", "chammi_macro_f1": "chammi/chammi_macro_f1", "chammi_accuracy": "chammi/chammi_accuracy", "chammi_Allen_Task_one_f1": "chammi/chammi_Allen_Task_one_f1", "chammi_Allen_Task_one_accuracy": "chammi/chammi_Allen_Task_one_accuracy", "chammi_Allen_Task_two_f1": "chammi/chammi_Allen_Task_two_f1", "chammi_Allen_Task_two_accuracy": "chammi/chammi_Allen_Task_two_accuracy", "chammi_HPA_Task_one_f1": "chammi/chammi_HPA_Task_one_f1", "chammi_HPA_Task_one_accuracy": "chammi/chammi_HPA_Task_one_accuracy", "chammi_HPA_Task_two_f1": "chammi/chammi_HPA_Task_two_f1", "chammi_HPA_Task_two_accuracy": "chammi/chammi_HPA_Task_two_accuracy", "chammi_HPA_Task_three_f1": "chammi/chammi_HPA_Task_three_f1", "chammi_HPA_Task_three_accuracy": "chammi/chammi_HPA_Task_three_accuracy", "chammi_CP_Task_one_f1": "chammi/chammi_CP_Task_one_f1", "chammi_CP_Task_one_accuracy": "chammi/chammi_CP_Task_one_accuracy", "chammi_CP_Task_two_f1": "chammi/chammi_CP_Task_two_f1", "chammi_CP_Task_two_accuracy": "chammi/chammi_CP_Task_two_accuracy", "chammi_CP_Task_three_f1": "chammi/chammi_CP_Task_three_f1", "chammi_CP_Task_three_accuracy": "chammi/chammi_CP_Task_three_accuracy", "chammi_CP_Task_four_f1": "chammi/chammi_CP_Task_four_f1", "chammi_CP_Task_four_accuracy": "chammi/chammi_CP_Task_four_accuracy", "hpav23_Average Precision": "hpav23/hpav23_Average Precision", "hpav23_AUC": "hpav23/hpav23_AUC", "hpav23_Label Ranking Average Precision": "hpav23/hpav23_Label Ranking Average Precision", "hpav23_Micro Average Precision": "hpav23/hpav23_Micro Average Precision", "rbcmc_swiss_accuracy": "rbc-mc/rbcmc_swiss_accuracy", "rbcmc_canadian_accuracy": "rbc-mc/rbcmc_canadian_accuracy", "rbcmc_overall_accuracy": "rbc-mc/rbcmc_overall_accuracy", "idr0017_katamari_early_fusion": "idr0017/idr0017_katamari_early_fusion", "jumpcp_mean_map": "jumpcp/jumpcp_mean_map", "jumpcp_active_ratio": "jumpcp/jumpcp_active_ratio", "jumpcp_consistent_ratio": "jumpcp/jumpcp_consistent_ratio", "jumpcp_overall_map": "jumpcp/jumpcp_overall_map", "allencell_static_structure_label_accuracy": "allencell_static/allencell_static_structure_label_accuracy", "allencell_static_cell_stage_label_accuracy": "allencell_static/allencell_static_cell_stage_label_accuracy", "allencell_static_ssim": "allencell_static/allencell_static_ssim", "allencell_nuclear_psnr": "allencell_nuclear/allencell_nuclear_psnr", "allencell_nuclear_mitotic_label_accuracy": "allencell_nuclear/allencell_nuclear_mitotic_label_accuracy", "opencell_micro_f1": "opencell/opencell_micro_f1", "opencell_supervised_roc_auc": "opencell/opencell_supervised_roc_auc", "erk_ssim": "erk/erk_ssim", "erk_psnr": "erk/erk_psnr", "cellstate/gene_lr_accuracy": "cellstate/cellstate/gene_lr_accuracy", "cellstate/minibinder_knn_accuracy": "cellstate/cellstate/minibinder_knn_accuracy", } def _load_long_csv(csv_path: Path, model_prefix: str, model_name: str): vals = _ensure_row(model_prefix + model_name) with open(csv_path) as f: reader = csv.reader(f) next(reader, None) # skip header for row in reader: if len(row) < 3: continue metric, value = row[1].strip(), row[2].strip() mapped = LONG_FORMAT_METRIC_MAP.get(metric) if mapped: _set_val(vals, mapped, _parse_float(value)) for csv_path in sorted((vlm_dir / "vision_evals").glob("*.csv")): name = csv_path.stem if name.endswith("_cellphie") or name.endswith("_cellphie_v2"): continue # duplicates _load_long_csv(csv_path, "vlm_", name) for csv_path in sorted((vlm_dir / "v3_evals").glob("vision_*.csv")): name = csv_path.stem.replace("vision_", "") _load_long_csv(csv_path, "vlm_v3_", name) for csv_path in sorted((vlm_dir / "v4_baseline").glob("*.csv")): _load_long_csv(csv_path, "vlm_v4_", csv_path.stem) # 5) Parse retrieval log files (retrieval_*.txt in vlm/) for log_path in sorted(vlm_dir.glob("retrieval_*.txt")): name = log_path.stem.replace("retrieval_", "") text = log_path.read_text() i2t_match = re.search(r"Image → Text: \{([^}]+)\}", text) t2i_match = re.search(r"Text → Image: \{([^}]+)\}", text) if not i2t_match and not t2i_match: continue prefixed = "vlm_ret500_" + name vals = _ensure_row(prefixed) for label, match, prefix in [ ("i2t", i2t_match, "ret500/i2t_"), ("t2i", t2i_match, "ret500/t2i_"), ]: if not match: continue for kv in match.group(1).split(","): k, v = kv.split(":") k = k.strip().strip("'\"") metric_name = prefix + k.lower() _set_val(vals, metric_name, _parse_float(v.strip())) return columns, rows def load_results_from_manifest( manifest_path: Path, header_path: Path ) -> tuple[list[str], dict[str, list[float | None]], dict[str, dict]]: """Load results from a manifest.jsonl file instead of local results/ glob.""" raw_columns = header_path.read_text().strip().split(",") keep = [ i for i, c in enumerate(raw_columns) if c.strip() and c.strip() != "placeholder" ] columns = [raw_columns[i] for i in keep] rows: dict[str, list[float | None]] = {} metadata: dict[str, dict] = {} for line in manifest_path.read_text().strip().splitlines(): if not line.strip(): continue entry = json.loads(line) name = entry["name"] txt_path = entry.get("txt_path", "") if txt_path and Path(txt_path).exists(): vals_raw = Path(txt_path).read_text().strip().split(",") all_vals = [_parse_float(v) for v in vals_raw] while len(all_vals) < len(raw_columns): all_vals.append(None) rows[name] = [all_vals[i] for i in keep] else: continue ws = entry.get("workstream", "") or WORKSTREAM_MAP.get(name, "") user = entry.get("user", "") ts = entry.get("timestamp", "") date_short = ts[:10] if ts else "" prod_meta = PROD_MODELS.get(name, {}) metadata[name] = { "display_name": prod_meta.get("display_name") or entry.get("display_name", "") or name, "category": WORKSTREAM_LABELS.get(ws, ""), "stage": entry.get("stage", ""), "base_model": prod_meta.get("base_model", ""), "owner": prod_meta.get("owner") or _format_owner(user), "wandb_url": prod_meta.get("wandb_url", ""), "status": "", "eval_source": txt_path, "model_path": entry.get("checkpoint", ""), "date": date_short, "ws": ws, "global_cat": _global_cat(name, ws), } return columns, rows, metadata def _format_owner(username: str) -> str: names = { "mmarks": "Markus Marks", "bgallusser": "Benjamin Gallusser", "rvasan": "Ritvik Vasan", "trang.le": "Trang Le", "kharrington": "Kyle Harrington", "alin": "Alexander Lin", } return names.get(username, username) def load_metadata(results_dir: Path) -> dict[str, dict]: meta_path = results_dir / "metadata.csv" if not meta_path.exists(): return {} meta: dict[str, dict] = {} with open(meta_path) as f: reader = csv.DictReader(f) for row in reader: key = row.get("model_key", "").strip() if not key and not (row.get("display_name") or "").strip(): continue entry = { "display_name": (row.get("display_name") or "").strip(), "category": (row.get("category") or "").strip(), "stage": (row.get("stage") or "").strip(), "base_model": (row.get("base_model") or "").strip(), "owner": (row.get("owner") or "").strip(), "wandb_url": (row.get("wandb_url") or "").strip(), "status": (row.get("status") or "").strip(), "eval_source": (row.get("eval_source") or "").strip(), "model_path": (row.get("model_path") or "").strip(), } if key: meta[key] = entry else: placeholder_key = f"__no_data__{entry['display_name']}" meta[placeholder_key] = entry return meta def compute_column_groups(columns: list[str]) -> list[tuple[str, list[int]]]: groups: list[tuple[str, list[int]]] = [] for i, col in enumerate(columns): group = col.split("/")[0] if "/" in col else col if groups and groups[-1][0] == group: groups[-1][1].append(i) else: groups.append((group, [i])) return groups def short_name(col: str) -> str: if "/" in col: parts = col.split("/", 1) metric = parts[1] prefix = parts[0] + "_" if metric.startswith(prefix): metric = metric[len(prefix) :] return metric return col def column_stats( rows: dict[str, list[float | None]], num_cols: int ) -> list[tuple[float | None, float | None]]: stats: list[tuple[float | None, float | None]] = [] for ci in range(num_cols): vals = [r[ci] for r in rows.values() if r[ci] is not None] if vals: stats.append((min(vals), max(vals))) else: stats.append((None, None)) return stats def build_table_parts( columns: list[str], rows: dict[str, list[float | None]], metadata: dict[str, dict], sort_by: str | None, ) -> tuple[str, str, str]: groups = compute_column_groups(columns) stats = column_stats(rows, len(columns)) col_short = [short_name(c) for c in columns] # Build group info for JS: {name, cols: [indices], summary_idx} js_groups = [] for grp_name, grp_indices in groups: summary_col = SUMMARY_COLS.get(grp_name) summary_idx = None if summary_col and summary_col in columns: summary_idx = columns.index(summary_col) js_groups.append( { "name": grp_name, "cols": grp_indices, "summary": summary_idx, } ) sort_idx = None if sort_by: for i, c in enumerate(columns): if c == sort_by: sort_idx = i break if sort_idx is not None: sorted_models = sorted( rows.keys(), key=lambda m: rows[m][sort_idx] or -999, reverse=True ) else: sorted_models = sorted(rows.keys()) js_data = [] seen_keys = set() for model in sorted_models: meta = metadata.get(model, {}) seen_keys.add(model) eval_source = meta.get("eval_source", "") or f"results/{model}.txt" ws = meta.get("ws", "") or WORKSTREAM_MAP.get(model, "") default_owner = ( "Markus Marks" if ws and ws not in ("baseline", "ws_prod") else "" ) gcat = meta.get("global_cat") or _global_cat(model, ws) js_data.append( { "key": model, "name": meta.get("display_name") or model, "category": meta.get("category", "") or WORKSTREAM_LABELS.get(ws, ""), "stage": meta.get("stage", ""), "base_model": meta.get("base_model", ""), "owner": meta.get("owner", "") or default_owner, "status": meta.get("status", ""), "wandb_url": meta.get("wandb_url", ""), "eval_source": eval_source, "checkpoint": meta.get("model_path", "") or meta.get("checkpoint", ""), "date": meta.get("date", ""), "vals": rows[model], "catalogued": bool(meta), "ws": ws, "gcat": gcat, "ckpt_series": _detect_checkpoint_series(model), "step_num": _extract_step(model), "baseline_family": _detect_baseline_family(model) if gcat == "baseline" else None, } ) for key, meta in metadata.items(): if key not in seen_keys and meta.get("display_name"): ws = meta.get("ws", "") gcat = meta.get("global_cat") or _global_cat(key, ws) js_data.append( { "key": key, "name": meta.get("display_name", key), "category": meta.get("category", ""), "stage": meta.get("stage", ""), "base_model": meta.get("base_model", ""), "owner": meta.get("owner", ""), "status": meta.get("status", ""), "wandb_url": meta.get("wandb_url", ""), "eval_source": meta.get("eval_source", ""), "checkpoint": meta.get("model_path", "") or meta.get("checkpoint", ""), "date": meta.get("date", ""), "vals": [None] * len(columns), "catalogued": True, "ws": ws, "gcat": gcat, "ckpt_series": _detect_checkpoint_series(key), "step_num": _extract_step(key), "baseline_family": _detect_baseline_family(key) if gcat == "baseline" else None, } ) # --- HTML --- # Workstream tabs — only include workstreams that have at least one result ws_in_data = sorted({d["ws"] for d in js_data if d["ws"]}) ws_tabs = '' for ws_key in ws_in_data: label = WORKSTREAM_LABELS.get(ws_key, ws_key) ws_tabs += ( f'' ) stage_tabs = "".join( f'' for key, label in STAGES ) cat_tabs = "".join( f'' for key, label in GLOBAL_CATEGORIES ) owner_tabs = '' for name in TEAM_MEMBERS: first = name.split()[0] owner_tabs += ( f'' ) # Group headers with expand/collapse toggle group_headers = "" for grp in js_groups: n = grp["name"] ncols = len(grp["cols"]) esc_name = html.escape(n) if ncols > 1 and grp["summary"] is not None: group_headers += ( f'' f'{esc_name} +' ) else: group_headers += f'{esc_name}' # Column headers — all rendered, JS controls visibility col_headers = "" for i in range(len(columns)): grp_name = columns[i].split("/")[0] if "/" in columns[i] else columns[i] col_headers += ( f'' f"{html.escape(col_short[i])}" f'' ) table_html = f"""
Evals
Stage
{stage_tabs}
Owner
{owner_tabs}
Category
{cat_tabs}
{group_headers} {col_headers}
# Model info
category stage base model owner date eval source checkpoint
""" # --- CSS (theme-adaptive via CSS custom properties) --- css = """ --bg: #09090b; --bg-alt: #18181b; --bg-header: #27272a; --border: #3f3f46; --text: #fafafa; --text-muted: #a1a1aa; --text-dim: #52525b; --text-meta: #a1a1aa; --accent: #22c55e; --hover: #14291a; --link: #4ade80; --status-ok: #22c55e; --status-run: #f97316; --tab-border: #3f3f46; --tab-hover-bg: #18181b; --input-bg: #18181b; --input-border: #52525b; --best-weight: 800; @media (prefers-color-scheme: light) { --bg: #ffffff; --bg-alt: #fafafa; --bg-header: #f0fdf4; --border: #d4d4d8; --text: #09090b; --text-muted: #71717a; --text-dim: #d4d4d8; --text-meta: #52525b; --accent: #16a34a; --hover: #f0fdf4; --link: #15803d; --status-ok: #16a34a; --status-run: #ea580c; --tab-border: #d4d4d8; --tab-hover-bg: #f0fdf4; --input-bg: #ffffff; --input-border: #d4d4d8; } font-family: "JetBrains Mono", "SF Mono", "Fira Code", monospace; font-size: 12px; color: var(--text); background: var(--bg); padding: 0; width: 100%; .lb-filters { background: var(--bg); padding: 10px 8px 6px 8px; display: flex; gap: 24px; flex-wrap: wrap; border-bottom: 1px solid var(--border); } .filter-group { display: flex; align-items: center; gap: 8px; } .filter-label { color: var(--text-muted); font-size: 11px; text-transform: uppercase; letter-spacing: 0.5px; font-weight: 600; } .tab-bar { display: flex; gap: 4px; } .tab-btn { padding: 4px 12px; border: 1px solid var(--tab-border); border-radius: 4px; background: transparent; color: var(--text-muted); font-size: 12px; cursor: pointer; font-family: inherit; transition: all 0.15s; } .tab-btn:hover { background: var(--tab-hover-bg); color: var(--text); border-color: var(--input-border); } .tab-btn.active { background: var(--bg-header); color: var(--accent); border-color: var(--accent); font-weight: 600; } .lb-controls { background: var(--bg); padding: 8px; display: flex; gap: 12px; align-items: center; } .lb-search { padding: 6px 10px; border: 1px solid var(--input-border); border-radius: 4px; background: var(--input-bg); color: var(--text); font-size: 13px; width: 300px; } .lb-search::placeholder { color: var(--text-muted); } .lb-count { color: var(--text-muted); font-size: 12px; } .expand-all-btn { padding: 4px 12px; border: 1px solid var(--tab-border); border-radius: 4px; background: transparent; color: var(--text-muted); font-size: 12px; cursor: pointer; font-family: inherit; } .expand-all-btn:hover { background: var(--tab-hover-bg); color: var(--text); } .toggle-label { color: var(--text-muted); font-size: 12px; cursor: pointer; display: flex; align-items: center; gap: 4px; } .toggle-label input { cursor: pointer; } .lb-scroll { overflow-x: auto; max-height: 82vh; overflow-y: auto; } .lb-table { border-collapse: collapse; white-space: nowrap; width: auto; } .lb-table th, .lb-table td { padding: 1px 3px; border: 1px solid var(--border); text-align: right; font-size: 11px; } .lb-table th { position: sticky; top: 0; background: var(--bg-alt); cursor: pointer; user-select: none; z-index: 2; } .lb-table th:hover { background: var(--hover); } .lb-table th .arrow { font-size: 10px; margin-left: 3px; color: var(--text-dim); } .lb-table th .arrow.active { color: var(--accent); } .lb-table tr.group-row th.grp-hdr { text-align: center; background: var(--bg-header); font-weight: 700; letter-spacing: 0.5px; text-transform: uppercase; font-size: 11px; color: var(--accent); top: 0; z-index: 3; } .lb-table tr.group-row th.grp-hdr[data-grp] { cursor: pointer; } .lb-table .grp-toggle { font-size: 10px; margin-left: 4px; opacity: 0.7; } .lb-table tr.col-row th { top: 18px; font-size: 11px; font-weight: 600; z-index: 2; } .lb-table th.meta-hdr { top: 18px; font-size: 11px; font-weight: 600; z-index: 2; text-align: left; } .lb-table th.sortable-meta { cursor: pointer; } .lb-table th.sortable-meta:hover { color: var(--accent); } .lb-table th.sort-active { color: var(--accent); } .lb-table td.model, .lb-table th.model { text-align: left; position: sticky; left: 0; z-index: 4; background: var(--bg-alt); min-width: 120px; max-width: 200px; overflow: hidden; text-overflow: ellipsis; font-weight: 500; } .lb-table td.model { background: var(--bg); z-index: 1; } .lb-table td.model a { color: var(--link); text-decoration: none; } .lb-table td.model a:hover { text-decoration: underline; } .lb-table tr:hover td { background: var(--hover) !important; } .lb-table tr:hover td.model { background: var(--hover) !important; } .lb-table td.val { font-variant-numeric: tabular-nums; } .lb-table td.na { color: var(--text-dim); } .lb-table .best { font-weight: var(--best-weight); } .lb-table td.meta { text-align: left; color: var(--text-meta); font-size: 10px; max-width: 90px; overflow: hidden; text-overflow: ellipsis; } .lb-table td.meta.status-completed { color: var(--status-ok); } .lb-table td.meta.status-running { color: var(--status-run); } .lb-table td.meta a { color: var(--link); text-decoration: none; } .lb-table td.meta a:hover { text-decoration: underline; } .eval-src-cell { cursor: pointer; } .eval-src-cell.expanded { max-width: 400px !important; overflow: visible !important; white-space: normal !important; } .eval-src-short { color: var(--link); } .eval-src-full code { font-size: 9px; word-break: break-all; display: block; margin: 2px 0; } .copy-btn { font-size: 9px; padding: 1px 4px; margin-left: 4px; cursor: pointer; border: 1px solid #888; border-radius: 3px; background: #f0f0f0; } .sel-hdr, .sel-cell { width: 20px; min-width: 20px; max-width: 20px; text-align: center; padding: 0 2px !important; } .sel-cb { cursor: pointer; } .compare-btn { padding: 4px 12px; border: 1px solid var(--tab-border); border-radius: 4px; background: var(--tab-bg); color: var(--text-muted); cursor: pointer; font-size: 12px; } .compare-btn:not(:disabled) { color: var(--accent); border-color: var(--accent); } .compare-btn:disabled { opacity: 0.5; cursor: default; } .compare-clear-btn { padding: 4px 8px; border: 1px solid var(--tab-border); border-radius: 4px; background: var(--tab-bg); color: var(--text-muted); cursor: pointer; font-size: 11px; } .lb-table tbody tr:nth-child(even) td { background: var(--bg-alt); } .lb-table tbody tr:nth-child(even) td.model { background: var(--bg-alt); } .lb-table tbody tr.uncatalogued td.model { opacity: 0.6; font-style: italic; } .col-hidden { display: none !important; } .lb-table tr.row-group-header { cursor: pointer; } .lb-table tr.row-group-header:hover td { background: var(--hover) !important; } .lb-table tr.row-group-header td.model { font-weight: 600; } .group-arrow { font-size: 10px; margin-right: 4px; display: inline-block; width: 12px; } .group-count { font-size: 10px; color: var(--text-muted); font-weight: 400; } .group-rep { font-size: 10px; color: var(--text-muted); font-weight: 400; margin-left: 6px; } .lb-table tr.group-member td.model { padding-left: 24px !important; } .lb-table th { position: relative; } .col-resize-handle { position: absolute; right: 0; top: 0; bottom: 0; width: 4px; cursor: col-resize; z-index: 10; background: transparent; } .col-resize-handle:hover, .col-resize-handle.active { background: var(--accent); opacity: 0.4; } .lb-table tr.gcat-header td { background: var(--bg-header) !important; font-weight: 700; font-size: 12px; text-transform: uppercase; letter-spacing: 0.5px; color: var(--accent); padding: 6px 8px; border-top: 2px solid var(--accent); text-align: left; position: sticky; left: 0; } .lb-table tr.gcat-header .gcat-count { font-weight: 400; font-size: 11px; color: var(--text-muted); text-transform: none; } /* #1 Export CSV button */ .export-csv-btn { padding: 4px 12px; border: 1px solid var(--tab-border); border-radius: 4px; background: transparent; color: var(--text-muted); font-size: 12px; cursor: pointer; font-family: inherit; } .export-csv-btn:hover { background: var(--tab-hover-bg); color: var(--text); } /* #2 Rank column */ .lb-table th.rank-hdr, .lb-table td.rank-cell { text-align: center; color: var(--text-muted); font-size: 10px; min-width: 28px; width: 28px; max-width: 36px; padding: 1px 2px !important; } .lb-table td.rank-cell { font-variant-numeric: tabular-nums; } /* #3 Pinned rows */ .lb-table tr.pinned td { border-left: 3px solid #eab308; } .lb-table tr.pinned td:first-child { border-left: 3px solid #eab308; } /* #4 Search highlighting */ .lb-table mark { background: #facc15; color: #000; padding: 0 1px; border-radius: 2px; } /* #5 Metric rank tooltip — no extra CSS needed */ /* #8 Column highlight on hover */ .col-highlight { background: rgba(34, 197, 94, 0.08) !important; } /* #9 URL hash state — no extra CSS needed */ /* #10 Hide meta columns toggle */ .hide-meta-label { color: var(--text-muted); font-size: 12px; cursor: pointer; display: flex; align-items: center; gap: 4px; } .hide-meta-label input { cursor: pointer; } .lb-table .meta-hidden { display: none !important; } """ # --- JS --- eval_set_map = {} for es, grps in EVAL_SETS.items(): for g in grps: eval_set_map[g] = es js_on_load = f""" const COLS = {json.dumps(col_short)}; const FULL_COLS = {json.dumps(columns)}; const DATA = {json.dumps(js_data)}; const STATS = {json.dumps([[s[0], s[1]] for s in stats])}; const GROUPS = {json.dumps(js_groups)}; const GCAT_LABELS = {json.dumps(GCAT_LABELS)}; const GCAT_ORDER = {json.dumps(GCAT_ORDER)}; const EVAL_SET_MAP = {json.dumps(eval_set_map)}; const WS_LABELS = {json.dumps(WORKSTREAM_LABELS)}; function fmt(v) {{ return v === null ? "" : v.toFixed(3); }} const WANDB_BASE = "https://czi.wandb.io"; function wandbUrl(ckpt) {{ if (!ckpt) return null; // W&B artifact: entity/project/artifact-name:version const m = ckpt.match(/^([^\\/]+)\\/([^\\/]+)\\/([^:]+):(.+)$/); if (m) return WANDB_BASE + "/" + m[1] + "/" + m[2] + "/artifacts/model/" + m[3] + "/" + m[4]; return null; }} // Theme-aware cell coloring: detect if dark or light function isDark() {{ return window.matchMedia && window.matchMedia('(prefers-color-scheme: dark)').matches; }} function cellStyle(v, ci) {{ if (v === null) return ""; const [lo, hi] = STATS[ci]; if (lo === null || hi === null || hi === lo) return ""; const t = (v - lo) / (hi - lo); const dark = isDark(); let r, g, b, a; if (dark) {{ a = 0.45; r = Math.round(239 - 205 * t); g = Math.round(68 + 129 * t); b = Math.round(68 + 26 * t); }} else {{ a = 0.35; r = Math.round(254 - 232 * t); g = Math.round(202 + (243 - 202) * t); b = Math.round(202 + (232 - 202) * t); }} return "background:rgba("+r+","+g+","+b+","+a+")"; }} const best = new Array(COLS.length).fill(null); DATA.forEach(d => {{ d.vals.forEach((v, i) => {{ if (v !== null && (best[i] === null || v > best[i])) best[i] = v; }}); }}); let sortCol = -1, sortAsc = false; let sortMeta = null; let filtered = DATA.map((_, i) => i); let activeGcats = new Set(); let activeStage = "all"; let activeOwner = "all"; let activeWs = "all"; let showUncatalogued = false; const selected = new Set(); let compareMode = false; let activeEvalSet = "all"; const rowGroupsExpanded = new Set(); // Track which groups are expanded (default: all collapsed) const expanded = {{}}; GROUPS.forEach(g => {{ expanded[g.name] = false; }}); const tbody = element.querySelector(".lb-tbody"); const countEl = element.querySelector(".lb-count"); function esc(s) {{ const d = document.createElement('div'); d.textContent = s; return d.innerHTML; }} // Column visibility management function updateColumnVisibility() {{ GROUPS.forEach(g => {{ const isExpanded = expanded[g.name]; const hasSummary = g.summary !== null; const evalSet = EVAL_SET_MAP[g.name]; const evalHidden = activeEvalSet !== "all" && evalSet && evalSet !== activeEvalSet; g.cols.forEach(ci => {{ const isSummary = (ci === g.summary); const hide = evalHidden || (hasSummary && !isExpanded && !isSummary); const hdr = element.querySelector('th.metric-hdr[data-col="' + ci + '"]'); if (hdr) hdr.classList.toggle("col-hidden", hide); element.querySelectorAll('td[data-ci="' + ci + '"]').forEach(td => {{ td.classList.toggle("col-hidden", hide); }}); }}); const grpTh = element.querySelector('th.grp-hdr[data-grp="' + g.name + '"]'); if (grpTh) {{ if (evalHidden) {{ grpTh.classList.add("col-hidden"); }} else {{ grpTh.classList.remove("col-hidden"); if (hasSummary) {{ const visibleCount = isExpanded ? g.cols.length : 1; grpTh.setAttribute("colspan", visibleCount); const toggle = grpTh.querySelector(".grp-toggle"); if (toggle) toggle.textContent = isExpanded ? "\\u2212" : "+"; grpTh.classList.toggle("grp-collapsed", !isExpanded); }} }} }} }}); }} // #3 Track pinned rows const pinnedRows = new Set(); function renderRow(d, idx, isGroupMember) {{ const checked = selected.has(idx) ? " checked" : ""; let cells = ''; // #2 Rank column const rank = flatRanks[idx]; cells += '' + (rank != null ? rank : '') + ''; if (d.wandb_url) {{ cells += '' + esc(d.name) + ''; }} else {{ cells += '' + esc(d.name) + ''; }} cells += '' + esc(d.category) + ''; cells += '' + esc(d.stage) + ''; cells += '' + esc(d.base_model) + ''; cells += '' + esc(d.owner) + ''; cells += '' + esc(d.date || '') + ''; if (d.eval_source) {{ const shortSrc = d.eval_source.split('/').slice(-2).join('/'); cells += '' + '' + esc(shortSrc) + '' + ''; }} else {{ cells += ''; }} const ckptUrl = wandbUrl(d.checkpoint); if (d.checkpoint) {{ const shortCkpt = d.checkpoint.split('/').slice(-2).join('/'); let ckptInner; if (ckptUrl) {{ ckptInner = '' + esc(shortCkpt) + ''; }} else {{ ckptInner = esc(shortCkpt); }} cells += '' + '' + ckptInner + '' + ''; }} else {{ cells += ''; }} for (let i = 0; i < COLS.length; i++) {{ const v = d.vals[i]; if (v === null) {{ cells += '\\u2014'; }} else {{ const isBest = (v === best[i]) ? " best" : ""; // #5 Enhanced tooltip with rank const ri = colRanks[i] && colRanks[i][idx]; const rankStr = ri ? " (Rank " + ri.rank + "/" + ri.total + ")" : ""; cells += '' + fmt(v) + ''; }} }} // #3 pinned class const rowCls = (d.catalogued ? "" : " uncatalogued") + (isGroupMember ? " group-member" : "") + (pinnedRows.has(idx) ? " pinned" : ""); return '' + cells + ""; }} function hasMultipleGcats() {{ const cats = new Set(); for (const idx of filtered) {{ cats.add(DATA[idx].gcat); if (cats.size > 1) return true; }} return false; }} function computeRowGroups(indices) {{ if (compareMode) return indices.map(idx => ({{type: "single", idx: idx}})); const groups = {{}}; const idx2g = {{}}; for (const idx of indices) {{ const d = DATA[idx]; let gk = null; if (activeWs === "all" && d.ws && d.ws !== "baseline" && d.ws !== "ws_prod" && d.owner.toLowerCase().includes("markus") && d.gcat === "research") {{ gk = "ws:" + d.ws; }} else if (d.gcat === "baseline" && d.baseline_family) {{ gk = "bl:" + d.baseline_family; }} else if (d.ckpt_series) {{ gk = "ckpt:" + d.ckpt_series; }} if (gk) {{ if (!groups[gk]) groups[gk] = []; groups[gk].push(idx); idx2g[idx] = gk; }} }} for (const [k, m] of Object.entries(groups)) {{ if (m.length <= 1) {{ m.forEach(i => delete idx2g[i]); delete groups[k]; }} }} const grouped = []; const singles = []; const seen = new Set(); for (const idx of indices) {{ const gk = idx2g[idx]; if (gk && !seen.has(gk)) {{ seen.add(gk); grouped.push({{type: "group", key: gk, members: groups[gk]}}); }} else if (!gk) {{ singles.push({{type: "single", idx: idx}}); }} }} return grouped.concat(singles); }} function pickRep(members, byMetric) {{ if (!byMetric) {{ let b = members[0], bs = DATA[b].step_num; for (const idx of members) {{ if (DATA[idx].step_num > bs) {{ b = idx; bs = DATA[idx].step_num; }} }} return b; }} const ci = sortCol >= 0 ? sortCol : FULL_COLS.indexOf("chammi/chammi_macro_f1"); let bi = members[0], bv = -999; for (const idx of members) {{ const v = ci >= 0 ? (DATA[idx].vals[ci] ?? -999) : 0; if (v > bv) {{ bi = idx; bv = v; }} }} return bi; }} function renderGroupRow(groupKey, members) {{ const isCkpt = groupKey.startsWith("ckpt:"); const isWs = groupKey.startsWith("ws:"); const isExp = rowGroupsExpanded.has(groupKey); const rawKey = groupKey.replace(/^(ws:|ckpt:|bl:)/, ""); const label = isWs ? (WS_LABELS[rawKey] || rawKey) : rawKey; const repIdx = pickRep(members, !isCkpt); const d = DATA[repIdx]; const arrow = isExp ? "\\u25BC" : "\\u25B6"; const suffix = isCkpt ? " \\u2014 " + esc(d.name) : " \\u2014 best: " + esc(d.name); let mc = ""; for (let i = 0; i < COLS.length; i++) {{ const v = d.vals[i]; if (v === null) {{ mc += '\\u2014'; }} else {{ const ib = (v === best[i]) ? " best" : ""; mc += '' + fmt(v) + ''; }} }} let r = ''; r += ''; r += ''; r += '' + arrow + ' ' + '' + esc(label) + '' + ' (' + members.length + ')' + '' + suffix + ''; r += ''; r += mc + ''; return r; }} // #5 Precompute per-column ranks among all models for tooltip const colRanks = {{}}; function precomputeRanks() {{ for (let ci = 0; ci < COLS.length; ci++) {{ const vals = []; DATA.forEach((d, idx) => {{ if (d.vals[ci] !== null) vals.push({{idx, v: d.vals[ci]}}); }}); vals.sort((a, b) => b.v - a.v); const ranks = {{}}; vals.forEach((item, rank) => {{ ranks[item.idx] = {{rank: rank + 1, total: vals.length}}; }}); colRanks[ci] = ranks; }} }} precomputeRanks(); // #2 Compute flat rank assignment for visible rows let flatRanks = {{}}; function render() {{ const totalCols = 1 + 1 + 1 + 7 + COLS.length; const showGcatHdrs = hasMultipleGcats(); const items = computeRowGroups(filtered); const parts = []; // #2 Assign flat rank to every visible model index flatRanks = {{}}; let rankCounter = 1; function assignRanks(list) {{ for (const item of list) {{ if (item.type === "single") {{ flatRanks[item.idx] = rankCounter++; }} else {{ for (const idx of item.members) flatRanks[idx] = rankCounter++; }} }} }} function renderItems(list) {{ for (const item of list) {{ if (item.type === "single") {{ parts.push(renderRow(DATA[item.idx], item.idx)); }} else {{ parts.push(renderGroupRow(item.key, item.members)); if (rowGroupsExpanded.has(item.key)) {{ for (const idx of item.members) parts.push(renderRow(DATA[idx], idx, true)); }} }} }} }} if (showGcatHdrs) {{ const buckets = {{}}; for (const item of items) {{ const gc = item.type === "single" ? DATA[item.idx].gcat : DATA[pickRep(item.members, !item.key.startsWith("ckpt:"))].gcat; if (!buckets[gc]) buckets[gc] = []; buckets[gc].push(item); }} const cats = Object.keys(buckets).sort((a, b) => (GCAT_ORDER[a] ?? 99) - (GCAT_ORDER[b] ?? 99)); for (const gc of cats) {{ assignRanks(buckets[gc]); }} for (const gc of cats) {{ const label = GCAT_LABELS[gc] || gc; const count = buckets[gc].reduce((a, it) => a + (it.type === "single" ? 1 : it.members.length), 0); parts.push('' + esc(label) + ' (' + count + ')'); renderItems(buckets[gc]); }} }} else {{ assignRanks(items); renderItems(items); }} const totalModels = items.reduce((a, it) => a + (it.type === "single" ? 1 : it.members.length), 0); tbody.innerHTML = parts.join(""); countEl.textContent = totalModels + " / " + DATA.length + " models"; updateColumnVisibility(); // #4 Apply search highlighting applySearchHighlight(); // #9 Persist state to URL hash saveHashState(); }} function clearSortIndicators() {{ element.querySelectorAll(".arrow").forEach(a => {{ a.textContent = ""; a.classList.remove("active"); }}); element.querySelectorAll(".sortable-meta").forEach(th => th.classList.remove("sort-active")); }} function doSort(ci) {{ sortMeta = null; if (sortCol === ci) {{ sortAsc = !sortAsc; }} else {{ sortCol = ci; sortAsc = false; }} clearSortIndicators(); const arrow = element.querySelector('[data-arrowcol="' + ci + '"]'); if (arrow) {{ arrow.textContent = sortAsc ? " \\u25B2" : " \\u25BC"; arrow.classList.add("active"); }} filtered.sort((a, b) => {{ const va = DATA[a].vals[ci], vb = DATA[b].vals[ci]; if (va === null && vb === null) return 0; if (va === null) return 1; if (vb === null) return -1; return sortAsc ? va - vb : vb - va; }}); render(); }} function doMetaSort(field) {{ sortCol = -1; if (sortMeta === field) {{ sortAsc = !sortAsc; }} else {{ sortMeta = field; sortAsc = false; }} clearSortIndicators(); const th = element.querySelector('[data-meta-sort="' + field + '"]'); if (th) th.classList.add("sort-active"); filtered.sort((a, b) => {{ const va = DATA[a][field] || "", vb = DATA[b][field] || ""; if (!va && !vb) return 0; if (!va) return 1; if (!vb) return -1; return sortAsc ? va.localeCompare(vb) : vb.localeCompare(va); }}); render(); }} function applyAllFilters() {{ const q = (element.querySelector(".lb-search").value || "").toLowerCase(); filtered = []; if (compareMode) {{ for (const idx of selected) filtered.push(idx); }} else {{ const gcatActive = activeGcats.size > 0; for (let i = 0; i < DATA.length; i++) {{ const d = DATA[i]; if (gcatActive && !activeGcats.has(d.gcat)) continue; if (activeWs === "all" && !gcatActive && !showUncatalogued && !d.catalogued) continue; if (activeWs !== "all" && d.ws !== activeWs) continue; if (activeStage !== "all" && d.stage !== activeStage) continue; if (activeOwner !== "all" && !d.owner.toLowerCase().includes(activeOwner)) continue; if (q && !(d.name.toLowerCase().includes(q) || d.key.toLowerCase().includes(q) || d.category.toLowerCase().includes(q) || d.owner.toLowerCase().includes(q))) continue; filtered.push(i); }} }} if (sortCol >= 0) {{ filtered.sort((a, b) => {{ const va = DATA[a].vals[sortCol], vb = DATA[b].vals[sortCol]; if (va === null && vb === null) return 0; if (va === null) return 1; if (vb === null) return -1; return sortAsc ? va - vb : vb - va; }}); }} else if (sortMeta) {{ filtered.sort((a, b) => {{ const va = DATA[a][sortMeta] || "", vb = DATA[b][sortMeta] || ""; if (!va && !vb) return 0; if (!va) return 1; if (!vb) return -1; return sortAsc ? va.localeCompare(vb) : vb.localeCompare(va); }}); }} render(); }} // Tab click handlers element.querySelectorAll(".tab-btn").forEach(btn => {{ btn.addEventListener("click", () => {{ const group = btn.dataset.filterType; const val = btn.dataset.filterVal; const bar = btn.closest(".tab-bar"); if (group === "gcat") {{ // Multi-select: toggle individual buttons, "all" clears const allBtn = bar.querySelector('[data-filter-val="all"]'); if (val === "all") {{ activeGcats.clear(); bar.querySelectorAll(".tab-btn").forEach(b => b.classList.remove("active")); allBtn.classList.add("active"); }} else {{ allBtn.classList.remove("active"); if (activeGcats.has(val)) {{ activeGcats.delete(val); btn.classList.remove("active"); }} else {{ activeGcats.add(val); btn.classList.add("active"); }} if (activeGcats.size === 0) allBtn.classList.add("active"); }} }} else {{ bar.querySelectorAll(".tab-btn").forEach(b => b.classList.remove("active")); btn.classList.add("active"); if (group === "stage") activeStage = val; else if (group === "owner") {{ activeOwner = val; updateWsBar(); }} else if (group === "ws") activeWs = val; else if (group === "evalset") activeEvalSet = val; }} applyAllFilters(); }}); }}); function updateWsBar() {{ const wsGroup = element.querySelector(".ws-filter-group"); if (!wsGroup) return; if (activeOwner === "all") {{ wsGroup.style.display = "none"; activeWs = "all"; wsGroup.querySelectorAll(".tab-btn").forEach(b => {{ b.classList.toggle("active", b.dataset.filterVal === "all"); }}); return; }} const ownerWs = new Set(); DATA.forEach(d => {{ if (d.owner.toLowerCase().includes(activeOwner) && d.ws) ownerWs.add(d.ws); }}); wsGroup.querySelectorAll('.tab-btn').forEach(b => {{ const v = b.dataset.filterVal; b.style.display = (v === "all" || ownerWs.has(v)) ? "" : "none"; }}); if (!ownerWs.has(activeWs) && activeWs !== "all") {{ activeWs = "all"; wsGroup.querySelectorAll(".tab-btn").forEach(b => {{ b.classList.toggle("active", b.dataset.filterVal === "all"); }}); }} wsGroup.style.display = ownerWs.size > 0 ? "" : "none"; }} // Group expand/collapse click handlers element.querySelectorAll("th.grp-hdr[data-grp]").forEach(th => {{ th.addEventListener("click", () => {{ const grpName = th.dataset.grp; expanded[grpName] = !expanded[grpName]; updateColumnVisibility(); }}); }}); // Expand/collapse all button const expandBtn = element.querySelector(".expand-all-btn"); let allExpanded = false; expandBtn.addEventListener("click", () => {{ allExpanded = !allExpanded; GROUPS.forEach(g => {{ expanded[g.name] = allExpanded; }}); if (allExpanded) {{ const items = computeRowGroups(filtered); items.forEach(it => {{ if (it.type === "group") rowGroupsExpanded.add(it.key); }}); }} else {{ rowGroupsExpanded.clear(); }} expandBtn.textContent = allExpanded ? "Collapse all" : "Expand all"; render(); }}); // Copy button handler element.addEventListener("click", (e) => {{ if (e.target.classList.contains("copy-btn")) {{ const path = e.target.dataset.path; navigator.clipboard.writeText(path); e.target.textContent = "copied"; setTimeout(() => e.target.textContent = "copy", 1000); return; }} const cell = e.target.closest(".eval-src-cell"); if (!cell) return; const short = cell.querySelector(".eval-src-short"); const full = cell.querySelector(".eval-src-full"); if (!short || !full) return; const isExpanded = cell.classList.toggle("expanded"); short.style.display = isExpanded ? "none" : ""; full.style.display = isExpanded ? "" : "none"; }}); // Metric column sort element.querySelectorAll("th.metric-hdr").forEach(th => {{ const ci = parseInt(th.dataset.col); if (!isNaN(ci)) th.addEventListener("click", () => doSort(ci)); }}); // Meta column sort (date, etc.) element.querySelectorAll("th.sortable-meta").forEach(th => {{ const field = th.dataset.metaSort; if (field) th.addEventListener("click", () => doMetaSort(field)); }}); element.querySelector(".lb-search").addEventListener("input", () => applyAllFilters()); // Uncatalogued toggle const uncatCheck = element.querySelector(".show-uncatalogued"); if (uncatCheck) {{ uncatCheck.addEventListener("change", () => {{ showUncatalogued = uncatCheck.checked; applyAllFilters(); }}); }} // Compare selection const compareBtn = element.querySelector(".compare-btn"); const compareClearBtn = element.querySelector(".compare-clear-btn"); function updateCompareUI() {{ compareBtn.textContent = "Compare (" + selected.size + ")"; compareBtn.disabled = selected.size < 2; compareClearBtn.style.display = (selected.size > 0 || compareMode) ? "" : "none"; if (compareMode) compareBtn.textContent = "Comparing " + selected.size; }} tbody.addEventListener("change", (e) => {{ if (!e.target.classList.contains("sel-cb")) return; const idx = parseInt(e.target.dataset.idx); if (e.target.checked) selected.add(idx); else selected.delete(idx); updateCompareUI(); }}); tbody.addEventListener("click", (e) => {{ const grpRow = e.target.closest(".row-group-header"); if (!grpRow) return; const gk = grpRow.dataset.group; if (rowGroupsExpanded.has(gk)) rowGroupsExpanded.delete(gk); else rowGroupsExpanded.add(gk); render(); }}); compareBtn.addEventListener("click", () => {{ if (selected.size < 2) return; compareMode = true; applyAllFilters(); updateCompareUI(); }}); compareClearBtn.addEventListener("click", () => {{ selected.clear(); compareMode = false; applyAllFilters(); updateCompareUI(); }}); // Column resize via drag element.querySelectorAll('.lb-table thead th').forEach(th => {{ const handle = document.createElement('div'); handle.className = 'col-resize-handle'; th.appendChild(handle); }}); let _resize = null; element.addEventListener('mousedown', (e) => {{ if (!e.target.classList.contains('col-resize-handle')) return; const th = e.target.parentElement; _resize = {{ th, startX: e.pageX, startW: th.offsetWidth }}; e.target.classList.add('active'); e.preventDefault(); }}); document.addEventListener('mousemove', (e) => {{ if (!_resize) return; const w = Math.max(30, _resize.startW + (e.pageX - _resize.startX)); _resize.th.style.width = w + 'px'; _resize.th.style.minWidth = w + 'px'; }}); document.addEventListener('mouseup', () => {{ if (_resize) {{ _resize.th.querySelector('.col-resize-handle')?.classList.remove('active'); _resize = null; }} }}); // #1 Export CSV element.querySelector(".export-csv-btn").addEventListener("click", () => {{ const header = ["model"].concat(FULL_COLS); const csvRows = [header.join(",")]; for (const idx of filtered) {{ const d = DATA[idx]; const row = ['"' + d.name.replace(/"/g, '""') + '"']; for (let i = 0; i < COLS.length; i++) {{ row.push(d.vals[i] !== null ? d.vals[i] : ""); }} csvRows.push(row.join(",")); }} const blob = new Blob([csvRows.join("\\n")], {{type: "text/csv"}}); const url = URL.createObjectURL(blob); const a = document.createElement("a"); a.href = url; a.download = "leaderboard_export.csv"; a.click(); URL.revokeObjectURL(url); }}); // #3 Double-click to pin/highlight rows tbody.addEventListener("dblclick", (e) => {{ const row = e.target.closest("tr[data-ridx]"); if (!row) return; const idx = parseInt(row.dataset.ridx); if (pinnedRows.has(idx)) {{ pinnedRows.delete(idx); row.classList.remove("pinned"); }} else {{ pinnedRows.add(idx); row.classList.add("pinned"); }} }}); // #4 Search highlighting — applies to model name cells function applySearchHighlight() {{ const q = (element.querySelector(".lb-search").value || "").trim(); if (!q) return; const lower = q.toLowerCase(); element.querySelectorAll(".lb-tbody .model-name-text").forEach(span => {{ const text = span.textContent; const idx = text.toLowerCase().indexOf(lower); if (idx >= 0) {{ const before = text.slice(0, idx); const match = text.slice(idx, idx + q.length); const after = text.slice(idx + q.length); span.innerHTML = esc(before) + '' + esc(match) + '' + esc(after); }} }}); }} // #8 Column highlight on hover element.addEventListener("mouseenter", (e) => {{ const td = e.target.closest("td[data-ci], th[data-col]"); if (!td) return; const ci = td.dataset.ci || td.dataset.col; if (ci == null) return; element.querySelectorAll('td[data-ci="' + ci + '"], th[data-col="' + ci + '"]').forEach(el => el.classList.add("col-highlight")); }}, true); element.addEventListener("mouseleave", (e) => {{ const td = e.target.closest("td[data-ci], th[data-col]"); if (!td) return; const ci = td.dataset.ci || td.dataset.col; if (ci == null) return; element.querySelectorAll('td[data-ci="' + ci + '"], th[data-col="' + ci + '"]').forEach(el => el.classList.remove("col-highlight")); }}, true); // #9 URL hash state persistence function saveHashState() {{ const parts = []; if (sortCol >= 0) parts.push("sc=" + sortCol); if (sortMeta) parts.push("sm=" + sortMeta); if (sortAsc) parts.push("sa=1"); if (activeEvalSet !== "all") parts.push("es=" + encodeURIComponent(activeEvalSet)); if (activeStage !== "all") parts.push("st=" + encodeURIComponent(activeStage)); if (activeOwner !== "all") parts.push("ow=" + encodeURIComponent(activeOwner)); if (activeWs !== "all") parts.push("ws=" + encodeURIComponent(activeWs)); if (activeGcats.size > 0) parts.push("gc=" + encodeURIComponent([...activeGcats].join(","))); const q = (element.querySelector(".lb-search").value || "").trim(); if (q) parts.push("q=" + encodeURIComponent(q)); if (metaHidden) parts.push("hm=1"); location.hash = parts.length > 0 ? parts.join("&") : ""; }} function loadHashState() {{ const hash = location.hash.replace(/^#/, ""); if (!hash) return false; const params = {{}}; hash.split("&").forEach(p => {{ const [k, v] = p.split("="); if (k && v !== undefined) params[k] = decodeURIComponent(v); }}); if (params.sc != null) {{ sortCol = parseInt(params.sc); }} if (params.sm) {{ sortMeta = params.sm; }} if (params.sa === "1") {{ sortAsc = true; }} if (params.es) {{ activeEvalSet = params.es; }} if (params.st) {{ activeStage = params.st; }} if (params.ow) {{ activeOwner = params.ow; }} if (params.ws) {{ activeWs = params.ws; }} if (params.gc) {{ activeGcats = new Set(params.gc.split(",")); }} if (params.q) {{ element.querySelector(".lb-search").value = params.q; }} if (params.hm === "1") {{ metaHidden = true; const cb = element.querySelector(".hide-meta-cb"); if (cb) cb.checked = true; applyMetaHidden(); }} // Sync tab button UI with restored state element.querySelectorAll('.tab-btn[data-filter-type="evalset"]').forEach(b => {{ b.classList.toggle("active", b.dataset.filterVal === activeEvalSet); }}); element.querySelectorAll('.tab-btn[data-filter-type="stage"]').forEach(b => {{ b.classList.toggle("active", b.dataset.filterVal === activeStage); }}); element.querySelectorAll('.tab-btn[data-filter-type="owner"]').forEach(b => {{ b.classList.toggle("active", b.dataset.filterVal === activeOwner); }}); element.querySelectorAll('.tab-btn[data-filter-type="gcat"]').forEach(b => {{ if (activeGcats.size > 0) {{ b.classList.toggle("active", activeGcats.has(b.dataset.filterVal)); }} else {{ b.classList.toggle("active", b.dataset.filterVal === "all"); }} }}); updateWsBar(); if (sortCol >= 0) {{ clearSortIndicators(); const arrow = element.querySelector('[data-arrowcol="' + sortCol + '"]'); if (arrow) {{ arrow.textContent = sortAsc ? " \\u25B2" : " \\u25BC"; arrow.classList.add("active"); }} }} return true; }} // #10 Hide meta columns toggle let metaHidden = false; function applyMetaHidden() {{ element.querySelectorAll(".meta-col, .meta-col-hdr").forEach(el => {{ el.classList.toggle("meta-hidden", metaHidden); }}); // Also update the info group header colspan/visibility const infoHdr = element.querySelector('.grp-hdr.meta-col-hdr'); if (infoHdr) infoHdr.classList.toggle("meta-hidden", metaHidden); }} const hideMetaCb = element.querySelector(".hide-meta-cb"); if (hideMetaCb) {{ hideMetaCb.addEventListener("change", () => {{ metaHidden = hideMetaCb.checked; applyMetaHidden(); saveHashState(); }}); }} // Re-render on theme change to update cell colors if (window.matchMedia) {{ window.matchMedia('(prefers-color-scheme: dark)').addEventListener('change', () => render()); }} // Force full-width: bust out of Gradio container padding let el = element; while (el) {{ if (el.classList) {{ el.style.maxWidth = "100%"; el.style.padding = "0"; el.style.margin = "0"; el.style.width = "100%"; }} el = el.parentElement; }} // #9 Initial render: restore from hash or use default sort const hashRestored = loadHashState(); if (hashRestored) {{ applyAllFilters(); }} else {{ const defaultIdx = FULL_COLS.indexOf({json.dumps(sort_by or "")}); if (defaultIdx >= 0) doSort(defaultIdx); else render(); }} """ return table_html, css, js_on_load def main(): parser = argparse.ArgumentParser(description="Serve eval leaderboard via Gradio") parser.add_argument("--results-dir", type=Path, default=Path("results")) parser.add_argument( "--manifest", type=Path, default=None, help="Path to manifest.jsonl. If provided, loads results from manifest instead of local results/ dir.", ) parser.add_argument("--sort-by", default="chammi/chammi_macro_f1") parser.add_argument("--port", type=int, default=7860) parser.add_argument( "--no-share", action="store_true", help="Disable Gradio public share URL" ) parser.add_argument("--user", default="katamari", help="Auth username") parser.add_argument( "--password", default="katamariRolls4ever!", help="Auth password" ) args = parser.parse_args() if args.manifest and args.manifest.exists(): header_path = args.results_dir / "header.txt" columns, rows, metadata = load_results_from_manifest(args.manifest, header_path) print(f"Loaded {len(rows)} models from manifest, {len(columns)} metrics") else: columns, rows = load_results(args.results_dir) metadata = load_metadata(args.results_dir) print( f"Loaded {len(rows)} models, {len(columns)} metrics, {len(metadata)} metadata entries" ) table_html, css, js_on_load = build_table_parts( columns, rows, metadata, args.sort_by ) fullwidth_css = """ .gradio-container { max-width: 100% !important; padding: 0 !important; margin: 0 !important; } .main { max-width: 100% !important; padding: 0 !important; } .contain { max-width: 100% !important; } footer { display: none !important; } """ with gr.Blocks(title="Katamari Eval Leaderboard") as app: gr.HTML( value=table_html, css_template=css, js_on_load=js_on_load, padding=False, container=False, ) _, local_url, share_url = app.launch( server_name="0.0.0.0", server_port=args.port, share=not args.no_share, auth=(args.user, args.password), css=fullwidth_css, prevent_thread_lock=True, ) url = share_url or local_url url_file = Path("/bio/projects/katamari/results/leaderboard_url.txt") url_file.parent.mkdir(parents=True, exist_ok=True) url_file.write_text(url + "\n") print(f"Leaderboard URL: {url}") print(f"URL written to {url_file}") import threading threading.Event().wait() from pathlib import Path results_dir = Path("results") manifest_path = results_dir / "manifest.jsonl" header_path = results_dir / "header.txt" columns, rows, metadata = load_results_from_manifest(manifest_path, header_path) print(f"Loaded {len(rows)} models from manifest, {len(columns)} metrics") table_html, css, js_on_load = build_table_parts( columns, rows, metadata, "chammi/chammi_macro_f1" ) fullwidth_css = """ .gradio-container { max-width: 100% !important; padding: 0 !important; margin: 0 !important; } .main { max-width: 100% !important; padding: 0 !important; } .contain { max-width: 100% !important; } footer { display: none !important; } """ demo = gr.Blocks(title="Katamari Eval Leaderboard") with demo: gr.HTML( value=table_html, css_template=css, js_on_load=js_on_load, padding=False, container=False, ) demo.launch( auth=("katamari", "katamariRolls4ever!"), css=fullwidth_css, ssr_mode=False, )