merge_gold69_v2_results.py

#!/usr/bin/env python3
from __future__ import annotations

import json
from pathlib import Path


ROOT = Path(__file__).parent
RESULTS = ROOT / "results.json"
REV = "09f80c368937e8899c952774776d3e54cf49768c"


def load_json(path: Path):
    return json.loads(path.read_text()) if path.exists() else []


def main() -> None:
    rows = load_json(RESULTS)
    by_model = {r["model"]: r for r in rows}
    sources = [
        Path(".context/golha_v2_benchmark/out/ctc/ctc_gold69_v2_results.json"),
        Path(".context/golha_v2_benchmark/out/whisper/whisper_gold69_v2_results.json"),
        Path(".context/golha_v2_benchmark/out/public/public_gold69_v2_results.json"),
    ]
    for src in sources:
        if not src.exists():
            continue
        for rec in load_json(src):
            model = rec["model"]
            row = by_model.get(model)
            if row is None:
                row = {"model": model, "repo": rec.get("repo") or rec.get("model")}
                rows.append(row)
                by_model[model] = row
            row.update(
                {
                    "params_b": rec.get("params_b", row.get("params_b")),
                    "gold69_v2_fair_wer": float(rec["gold69_v2_fair_wer"]) * 100.0,
                    "gold69_v2_fair_cer": float(rec["gold69_v2_fair_cer"]) * 100.0,
                    "gold69_v2_revision": REV,
                    "gold69_v2_n": int(rec.get("n", 69)),
                    "mean_decode_ms": rec.get("mean_decode_ms"),
                    "status": "complete",
                    "notes": "Corrected Gold69 v2 rerun completed on Stallion with the base unquantized model. FLEURS is from the earlier full double eval.",
                }
            )
    RESULTS.write_text(json.dumps(rows, ensure_ascii=False, indent=2) + "\n")


if __name__ == "__main__":
    main()