feat: add generation script for language mapping from MADLAD-400 paper

scripts/generate_langmap.py

@@ -0,0 +1,245 @@
+#!/usr/bin/env python3
+"""
+Generate langid_mapping.py from MADLAD-400 paper Table 9.
+
+Parses Table 9 (Section A.1, pages 16-22) from the MADLAD-400 paper PDF,
+assigns regions using script-based defaults and language-specific overrides,
+and writes the result to langmap/langid_mapping.py.
+
+Usage:
+    python scripts/generate_langmap.py <path-to-paper.pdf>
+
+Dev dependencies (requirements-dev.txt):
+    pdfplumber, pycountry
+"""
+
+import sys
+from pathlib import Path
+
+import pdfplumber
+
+# Pages 16-22 of the paper contain Table 9 (0-indexed: 15-21)
+TABLE_PAGES = range(15, 22)
+
+# Default region based on writing script
+SCRIPT_TO_REGION: dict[str, str] = {
+    "Arab": "Middle East & North Africa",
+    "Armn": "Europe",
+    "Beng": "South Asia",
+    "Cans": "Americas",
+    "Cher": "Americas",
+    "Cyrl": "Europe",
+    "Deva": "South Asia",
+    "Ethi": "Africa",
+    "Geor": "Europe",
+    "Grek": "Europe",
+    "Gujr": "South Asia",
+    "Guru": "South Asia",
+    "Hang": "East Asia",
+    "Hans": "East Asia",
+    "Hant": "East Asia",
+    "Hebr": "Middle East & North Africa",
+    "Jpan": "East Asia",
+    "Khmr": "Southeast Asia",
+    "Knda": "South Asia",
+    "Kore": "East Asia",
+    "Laoo": "Southeast Asia",
+    "Mlym": "South Asia",
+    "Mong": "East Asia",
+    "Mymr": "Southeast Asia",
+    "Olck": "South Asia",
+    "Orya": "South Asia",
+    "Sinh": "South Asia",
+    "Taml": "South Asia",
+    "Telu": "South Asia",
+    "Tfng": "Africa",
+    "Thaa": "South Asia",
+    "Thai": "Southeast Asia",
+    "Tibt": "East Asia",
+}
+
+# Language-specific overrides (primarily for Latin-script languages and corrections)
+LANGUAGE_OVERRIDES: dict[str, str] = {
+    # Europe
+    "af": "Africa",
+    "az": "Central Asia",
+    "bs": "Europe",
+    "ca": "Europe",
+    "ceb": "Southeast Asia",
+    "co": "Europe",
+    "cs": "Europe",
+    "cy": "Europe",
+    "da": "Europe",
+    "de": "Europe",
+    "en": "Europe",
+    "es": "Europe",
+    "et": "Europe",
+    "eu": "Europe",
+    "fi": "Europe",
+    "fil": "Southeast Asia",
+    "fr": "Europe",
+    "fy": "Europe",
+    "ga": "Europe",
+    "gd": "Europe",
+    "gl": "Europe",
+    "hr": "Europe",
+    "ht": "Americas",
+    "hu": "Europe",
+    "id": "Southeast Asia",
+    "ig": "Africa",
+    "is": "Europe",
+    "it": "Europe",
+    "jv": "Southeast Asia",
+    "kk": "Central Asia",
+    "ku": "Middle East & North Africa",
+    "ky": "Central Asia",
+    "la": "Europe",
+    "lb": "Europe",
+    "lt": "Europe",
+    "lv": "Europe",
+    "mg": "Africa",
+    "mi": "Oceania",
+    "mk": "Europe",
+    "mn": "East Asia",
+    "ms": "Southeast Asia",
+    "mt": "Europe",
+    "nl": "Europe",
+    "nn": "Europe",
+    "no": "Europe",
+    "ny": "Africa",
+    "pl": "Europe",
+    "pt": "Europe",
+    "qu": "Americas",
+    "ro": "Europe",
+    "rw": "Africa",
+    "sk": "Europe",
+    "sl": "Europe",
+    "sm": "Oceania",
+    "sn": "Africa",
+    "so": "Africa",
+    "sq": "Europe",
+    "sr": "Europe",
+    "st": "Africa",
+    "su": "Southeast Asia",
+    "sv": "Europe",
+    "sw": "Africa",
+    "tg": "Central Asia",
+    "tk": "Central Asia",
+    "tl": "Southeast Asia",
+    "to": "Oceania",
+    "tr": "Middle East & North Africa",
+    "uz": "Central Asia",
+    "vi": "Southeast Asia",
+    "xh": "Africa",
+    "yo": "Africa",
+    "zu": "Africa",
+    # South Asia (non-Devanagari Latin-script entries)
+    "cjm": "Southeast Asia",
+    # Corrections for Cyrillic languages not in Europe
+    "ba": "Central Asia",
+    "ce": "Europe",
+    "cv": "Europe",
+    "kv": "Europe",
+    "os": "Europe",
+    "sah": "East Asia",
+    "tt": "Europe",
+}
+
+
+def parse_table9(pdf_path: str) -> list[dict[str, str]]:
+    """Parse Table 9 from the MADLAD-400 paper."""
+    entries: list[dict[str, str]] = []
+    with pdfplumber.open(pdf_path) as pdf:
+        for page_num in TABLE_PAGES:
+            page = pdf.pages[page_num]
+            table = page.extract_table()
+            if not table:
+                continue
+            for row in table:
+                if not row or not row[0]:
+                    continue
+                bcp47 = row[0].strip()
+                # Skip header rows
+                if bcp47 == "BCP-47" or bcp47 == "":
+                    continue
+                name = row[1].strip() if row[1] else ""
+                script = row[2].strip() if row[2] else ""
+                # Skip rows with "-" in all data columns (last 79 self-audit omissions)
+                data_cols = [c for c in row[3:] if c is not None]
+                if all(c.strip() == "-" for c in data_cols) or not data_cols:
+                    continue
+                entries.append({"bcp47": bcp47, "name": name, "script": script})
+    return entries
+
+
+def assign_region(bcp47: str, script: str) -> str:
+    """Assign a geographic region based on language code and script."""
+    if bcp47 in LANGUAGE_OVERRIDES:
+        return LANGUAGE_OVERRIDES[bcp47]
+    if script in SCRIPT_TO_REGION:
+        return SCRIPT_TO_REGION[script]
+    return "Other"
+
+
+def write_mapping(entries: list[dict[str, str]], output_path: Path) -> None:
+    """Write the language mapping to a Python file."""
+    mapping: list[tuple[str, str, str]] = []
+    for entry in entries:
+        token = f"<2{entry['bcp47']}>"
+        region = assign_region(entry["bcp47"], entry["script"])
+        mapping.append((token, entry["name"], region))
+
+    # Sort by region, then by name within each region
+    mapping.sort(key=lambda x: (x[2], x[1]))
+
+    lines = [
+        "# Auto-generated by scripts/generate_langmap.py from MADLAD-400 paper Table 9 (Section A.1)",
+        "# Source: https://arxiv.org/pdf/2309.04662",
+        f"# {len(mapping)} languages with training data (excludes 79 self-audit omissions)",
+        "#",
+        "# To regenerate:",
+        "#   python scripts/generate_langmap.py <path-to-paper.pdf>",
+        "langid_to_language = {",
+    ]
+    for token, name, region in mapping:
+        # Escape any quotes in names
+        escaped_name = name.replace('"', '\\"')
+        lines.append(f'    "{token}": {{"name": "{escaped_name}", "region": "{region}"}},')
+    lines.append("}")
+    lines.append("")
+
+    output_path.write_text("\n".join(lines))
+
+
+def main() -> None:
+    if len(sys.argv) != 2:
+        print("Usage: python scripts/generate_langmap.py <path-to-paper.pdf>")
+        sys.exit(1)
+
+    pdf_path = sys.argv[1]
+    entries = parse_table9(pdf_path)
+    print(f"Parsed {len(entries)} languages from Table 9")
+
+    # Report region assignments
+    other: list[tuple[str, str]] = []
+    for entry in entries:
+        region = assign_region(entry["bcp47"], entry["script"])
+        if region == "Other":
+            other.append((entry["bcp47"], entry["name"]))
+
+    if other:
+        print(f"\n{len(other)} languages assigned to 'Other' region (need manual override):")
+        for bcp47, name in other:
+            print(f"  {bcp47}: {name}")
+        print("\nAdd overrides to LANGUAGE_OVERRIDES in the script and re-run.")
+
+    output_path = Path(__file__).parent.parent / "langmap" / "langid_mapping.py"
+    write_mapping(entries, output_path)
+    print(f"\nWrote {len(entries)} entries to {output_path}")
+
+    if other:
+        sys.exit(1)
+
+
+if __name__ == "__main__":
+    main()