feat: expand language mapping to 418 languages with regions

Run generate_langmap.py against the MADLAD-400 paper PDF to produce
langid_to_language with 418 entries (up from 22), each having "name"
and "region" keys across 9 geographic regions. Updated parse_table9()
to use word-based PDF extraction (pdfplumber extract_table() returns
None for this PDF) with column x-boundaries and 5px y-tolerance for
multi-line name cells. Added comprehensive LANGUAGE_OVERRIDES covering
all Latin-script languages plus Syrc to SCRIPT_TO_REGION.

Co-Authored-By: Claude Sonnet 4.6 <noreply@anthropic.com>

langmap/langid_mapping.py

@@ -1,27 +1,426 @@
-# Tier 1 production-ready languages: BLEU 35+ in both translation directions
-# (en→xx and xx→en) from Section A.10 of the MADLAD-400 paper.
-# See: https://arxiv.org/pdf/2309.04662
+# Auto-generated by scripts/generate_langmap.py from MADLAD-400 paper Table 9 (Section A.1)
+# Source: https://arxiv.org/pdf/2309.04662
+# 418 languages with training data (excludes 79 self-audit omissions)
+#
+# To regenerate:
+#   python scripts/generate_langmap.py <path-to-paper.pdf>
 langid_to_language = {
-    "<2af>": "Afrikaans",
-    "<2bg>": "Bulgarian",
-    "<2bs>": "Bosnian",
-    "<2ca>": "Catalan",
-    "<2cs>": "Czech",
-    "<2da>": "Danish",
-    "<2de>": "German",
-    "<2es>": "Spanish",
-    "<2fil>": "Filipino",
-    "<2fr>": "French",
-    "<2hr>": "Croatian",
-    "<2id>": "Indonesian",
-    "<2it>": "Italian",
-    "<2mk>": "Macedonian",
-    "<2ms>": "Malay",
-    "<2mt>": "Maltese",
-    "<2nn>": "Norwegian Nynorsk",
-    "<2no>": "Norwegian",
-    "<2pt>": "Portuguese",
-    "<2ro>": "Romanian",
-    "<2sk>": "Slovak",
-    "<2sv>": "Swedish",
+    "<2ach>": {"name": "Acoli", "region": "Africa"},
+    "<2ada>": {"name": "Adangme", "region": "Africa"},
+    "<2adh>": {"name": "Adhola", "region": "Africa"},
+    "<2aa>": {"name": "Afar", "region": "Africa"},
+    "<2af>": {"name": "Afrikaans", "region": "Africa"},
+    "<2alz>": {"name": "Alur", "region": "Africa"},
+    "<2am>": {"name": "Amharic", "region": "Africa"},
+    "<2bm>": {"name": "Bambara", "region": "Africa"},
+    "<2bci>": {"name": "Baoulé", "region": "Africa"},
+    "<2bas>": {"name": "Basa(Cameroon)", "region": "Africa"},
+    "<2bim>": {"name": "Bimoba", "region": "Africa"},
+    "<2bqc>": {"name": "Boko(Benin)", "region": "Africa"},
+    "<2bus>": {"name": "Bokobaru", "region": "Africa"},
+    "<2bum>": {"name": "Bulu", "region": "Africa"},
+    "<2ny>": {"name": "Chichewa", "region": "Africa"},
+    "<2cjk>": {"name": "Chokwe", "region": "Africa"},
+    "<2cce>": {"name": "Chopi", "region": "Africa"},
+    "<2dwr>": {"name": "Dawro", "region": "Africa"},
+    "<2din>": {"name": "Dinka", "region": "Africa"},
+    "<2tbz>": {"name": "Ditammari", "region": "Africa"},
+    "<2dov>": {"name": "Dombe", "region": "Africa"},
+    "<2dyu>": {"name": "Dyula", "region": "Africa"},
+    "<2ee>": {"name": "Ewe", "region": "Africa"},
+    "<2fip>": {"name": "Fipa", "region": "Africa"},
+    "<2fon>": {"name": "Fon", "region": "Africa"},
+    "<2ff>": {"name": "Fulfulde", "region": "Africa"},
+    "<2gof>": {"name": "Gofa", "region": "Africa"},
+    "<2gub>": {"name": "Guajajára", "region": "Africa"},
+    "<2gvl>": {"name": "Gulay", "region": "Africa"},
+    "<2ha>": {"name": "Hausa", "region": "Africa"},
+    "<2ibb>": {"name": "Ibibio", "region": "Africa"},
+    "<2ig>": {"name": "Igbo", "region": "Africa"},
+    "<2iso>": {"name": "Isoko", "region": "Africa"},
+    "<2izz>": {"name": "Izii", "region": "Africa"},
+    "<2kbp>": {"name": "Kabiyè", "region": "Africa"},
+    "<2kmb>": {"name": "Kimbundu", "region": "Africa"},
+    "<2rw>": {"name": "Kinyarwanda", "region": "Africa"},
+    "<2ktu>": {"name": "Kituba(DRC)", "region": "Africa"},
+    "<2kg>": {"name": "Kongo", "region": "Africa"},
+    "<2kri>": {"name": "Krio", "region": "Africa"},
+    "<2kj>": {"name": "Kuanyama", "region": "Africa"},
+    "<2laj>": {"name": "Lango(Uganda)", "region": "Africa"},
+    "<2ln>": {"name": "Lingala", "region": "Africa"},
+    "<2lu>": {"name": "Luba-Katanga", "region": "Africa"},
+    "<2lg>": {"name": "Luganda", "region": "Africa"},
+    "<2ffm>": {"name": "MaasinaFulfulde", "region": "Africa"},
+    "<2mgh>": {"name": "Makhuwa-Meetto", "region": "Africa"},
+    "<2mg>": {"name": "Malagasy", "region": "Africa"},
+    "<2mas>": {"name": "Masai", "region": "Africa"},
+    "<2mfe>": {"name": "Morisien", "region": "Africa"},
+    "<2nnb>": {"name": "Nande", "region": "Africa"},
+    "<2niq>": {"name": "Nandi", "region": "Africa"},
+    "<2nyu>": {"name": "Nyungwe", "region": "Africa"},
+    "<2nzi>": {"name": "Nzima", "region": "Africa"},
+    "<2ann>": {"name": "Obolo", "region": "Africa"},
+    "<2om>": {"name": "Oromo", "region": "Africa"},
+    "<2rn>": {"name": "Rundi", "region": "Africa"},
+    "<2sg>": {"name": "Sango", "region": "Africa"},
+    "<2seh>": {"name": "Sena", "region": "Africa"},
+    "<2nso>": {"name": "Sepedi", "region": "Africa"},
+    "<2srr>": {"name": "Serer", "region": "Africa"},
+    "<2st>": {"name": "Sesotho", "region": "Africa"},
+    "<2sn>": {"name": "Shona", "region": "Africa"},
+    "<2so>": {"name": "Somali", "region": "Africa"},
+    "<2nr>": {"name": "SouthNdebele", "region": "Africa"},
+    "<2spp>": {"name": "SupyireSenoufo", "region": "Africa"},
+    "<2sus>": {"name": "Susu", "region": "Africa"},
+    "<2sw>": {"name": "Swahili", "region": "Africa"},
+    "<2ss>": {"name": "Swati", "region": "Africa"},
+    "<2ber>": {"name": "Tamazight(Tfng)", "region": "Africa"},
+    "<2tdx>": {"name": "Tandroy-MahafalyMalagasy", "region": "Africa"},
+    "<2teo>": {"name": "Teso", "region": "Africa"},
+    "<2tll>": {"name": "Tetela", "region": "Africa"},
+    "<2ti>": {"name": "Tigrinya", "region": "Africa"},
+    "<2tiv>": {"name": "Tiv", "region": "Africa"},
+    "<2ts>": {"name": "Tsonga", "region": "Africa"},
+    "<2tsc>": {"name": "Tswa", "region": "Africa"},
+    "<2tn>": {"name": "Tswana", "region": "Africa"},
+    "<2ak>": {"name": "Twi", "region": "Africa"},
+    "<2ve>": {"name": "Venda", "region": "Africa"},
+    "<2wal>": {"name": "Wolaytta", "region": "Africa"},
+    "<2wo>": {"name": "Wolof", "region": "Africa"},
+    "<2xh>": {"name": "Xhosa", "region": "Africa"},
+    "<2yo>": {"name": "Yoruba", "region": "Africa"},
+    "<2zne>": {"name": "Zande", "region": "Africa"},
+    "<2dje>": {"name": "Zarma", "region": "Africa"},
+    "<2zu>": {"name": "Zulu", "region": "Africa"},
+    "<2agr>": {"name": "Aguaruna", "region": "Americas"},
+    "<2cni>": {"name": "Asháninka", "region": "Americas"},
+    "<2kwi>": {"name": "Awa-Cuaiquer", "region": "Americas"},
+    "<2quy>": {"name": "AyacuchoQuechua", "region": "Americas"},
+    "<2ay>": {"name": "Aymara", "region": "Americas"},
+    "<2bzj>": {"name": "BelizeKriolEnglish", "region": "Americas"},
+    "<2qvc>": {"name": "CajamarcaQuechua", "region": "Americas"},
+    "<2qxr>": {"name": "CañarH.Quichua", "region": "Americas"},
+    "<2maz>": {"name": "CentralMazahua", "region": "Americas"},
+    "<2chr>": {"name": "Cherokee", "region": "Americas"},
+    "<2ctu>": {"name": "Chol", "region": "Americas"},
+    "<2cac>": {"name": "Chuj", "region": "Americas"},
+    "<2gui>": {"name": "E.BolivianGuaraní", "region": "Americas"},
+    "<2nhe>": {"name": "E.HuastecaNahuatl", "region": "Americas"},
+    "<2djk>": {"name": "E.MaroonCreole", "region": "Americas"},
+    "<2sja>": {"name": "Epena", "region": "Americas"},
+    "<2cab>": {"name": "Garifuna", "region": "Americas"},
+    "<2guh>": {"name": "Guahibo", "region": "Americas"},
+    "<2gn>": {"name": "Guarani", "region": "Americas"},
+    "<2amu>": {"name": "GuerreroAmuzgo", "region": "Americas"},
+    "<2ngu>": {"name": "GuerreroNahuatl", "region": "Americas"},
+    "<2gyn>": {"name": "GuyaneseCreoleEnglish", "region": "Americas"},
+    "<2ht>": {"name": "HaitianCreole", "region": "Americas"},
+    "<2haw>": {"name": "Hawaiian", "region": "Americas"},
+    "<2qub>": {"name": "HuallagaHuánucoQuechua", "region": "Americas"},
+    "<2hus>": {"name": "Huastec", "region": "Americas"},
+    "<2qvi>": {"name": "ImbaburaH.Quichua", "region": "Americas"},
+    "<2inb>": {"name": "Inga", "region": "Americas"},
+    "<2iu>": {"name": "Inuktitut", "region": "Americas"},
+    "<2jam>": {"name": "JamaicanCreoleEnglish", "region": "Americas"},
+    "<2kl>": {"name": "Kalaallisut", "region": "Americas"},
+    "<2cak>": {"name": "Kaqchikel", "region": "Americas"},
+    "<2kek>": {"name": "Kekchí", "region": "Americas"},
+    "<2tlh>": {"name": "Klingon", "region": "Americas"},
+    "<2quc>": {"name": "K’iche’", "region": "Americas"},
+    "<2quf>": {"name": "LambayequeQuechua", "region": "Americas"},
+    "<2mam>": {"name": "Mam", "region": "Americas"},
+    "<2arn>": {"name": "Mapudungun", "region": "Americas"},
+    "<2miq>": {"name": "Mískito", "region": "Americas"},
+    "<2emp>": {"name": "N.Emberá", "region": "Americas"},
+    "<2qvz>": {"name": "N.PastazaQuichua", "region": "Americas"},
+    "<2nv>": {"name": "Navajo", "region": "Americas"},
+    "<2gym>": {"name": "Ngäbere", "region": "Americas"},
+    "<2zxx>": {"name": "Noise", "region": "Americas"},
+    "<2oj>": {"name": "Ojibwa", "region": "Americas"},
+    "<2pap>": {"name": "Papiamento", "region": "Americas"},
+    "<2jac>": {"name": "Popti’", "region": "Americas"},
+    "<2qu>": {"name": "Quechua", "region": "Americas"},
+    "<2otq>": {"name": "QuerétaroOtomi", "region": "Americas"},
+    "<2rcf>": {"name": "RéunionCreoleFrench", "region": "Americas"},
+    "<2quh>": {"name": "S.BolivianQuechua", "region": "Americas"},
+    "<2qup>": {"name": "S.PastazaQuechua", "region": "Americas"},
+    "<2cuk>": {"name": "SanBlasKuna", "region": "Americas"},
+    "<2srm>": {"name": "Saramaccan", "region": "Americas"},
+    "<2crs>": {"name": "SeselwaCreoleFrench", "region": "Americas"},
+    "<2shp>": {"name": "Shipibo-Conibo", "region": "Americas"},
+    "<2jiv>": {"name": "Shuar", "region": "Americas"},
+    "<2srn>": {"name": "SrananTongo", "region": "Americas"},
+    "<2acf>": {"name": "StLucianCreoleFrench", "region": "Americas"},
+    "<2tca>": {"name": "Ticuna", "region": "Americas"},
+    "<2toj>": {"name": "Tojolabal", "region": "Americas"},
+    "<2tzh>": {"name": "Tzeltal", "region": "Americas"},
+    "<2tzo>": {"name": "Tzotzil", "region": "Americas"},
+    "<2tzj>": {"name": "Tz’utujil", "region": "Americas"},
+    "<2knj>": {"name": "W.Kanjobal", "region": "Americas"},
+    "<2guc>": {"name": "Wayuu", "region": "Americas"},
+    "<2noa>": {"name": "WounMeu", "region": "Americas"},
+    "<2yua>": {"name": "Yucateco", "region": "Americas"},
+    "<2zap>": {"name": "Zapotec", "region": "Americas"},
+    "<2az>": {"name": "Azerbaijani", "region": "Central Asia"},
+    "<2ba>": {"name": "Bashkir", "region": "Central Asia"},
+    "<2kaa-Latn>": {"name": "Kara-Kalpak(Latn)", "region": "Central Asia"},
+    "<2kk>": {"name": "Kazakh", "region": "Central Asia"},
+    "<2ky>": {"name": "Kyrghyz", "region": "Central Asia"},
+    "<2tg>": {"name": "Tajik", "region": "Central Asia"},
+    "<2tk>": {"name": "Turkmen", "region": "Central Asia"},
+    "<2uz>": {"name": "Uzbek", "region": "Central Asia"},
+    "<2yue>": {"name": "Cantonese", "region": "East Asia"},
+    "<2dz>": {"name": "Dzongkha", "region": "East Asia"},
+    "<2ja>": {"name": "Japanese", "region": "East Asia"},
+    "<2ko>": {"name": "Korean", "region": "East Asia"},
+    "<2zh>": {"name": "MandarinChinese", "region": "East Asia"},
+    "<2mn>": {"name": "Mongolian", "region": "East Asia"},
+    "<2bo>": {"name": "Tibetan", "region": "East Asia"},
+    "<2sah>": {"name": "Yakut", "region": "East Asia"},
+    "<2za>": {"name": "Zhuang", "region": "East Asia"},
+    "<2ady>": {"name": "Adyghe", "region": "Europe"},
+    "<2sq>": {"name": "Albanian", "region": "Europe"},
+    "<2grc>": {"name": "AncientGreek", "region": "Europe"},
+    "<2hy>": {"name": "Armenian", "region": "Europe"},
+    "<2frp>": {"name": "Arpitan", "region": "Europe"},
+    "<2av>": {"name": "Avar", "region": "Europe"},
+    "<2eu>": {"name": "Basque", "region": "Europe"},
+    "<2be>": {"name": "Belarusian", "region": "Europe"},
+    "<2bs>": {"name": "Bosnian", "region": "Europe"},
+    "<2br>": {"name": "Breton", "region": "Europe"},
+    "<2bg>": {"name": "Bulgarian", "region": "Europe"},
+    "<2bua>": {"name": "Buryat", "region": "Europe"},
+    "<2rmc>": {"name": "CarpathianRomani", "region": "Europe"},
+    "<2ca>": {"name": "Catalan", "region": "Europe"},
+    "<2ce>": {"name": "Chechen", "region": "Europe"},
+    "<2cv>": {"name": "Chuvash", "region": "Europe"},
+    "<2kw>": {"name": "Cornish", "region": "Europe"},
+    "<2co>": {"name": "Corsican", "region": "Europe"},
+    "<2crh>": {"name": "CrimeanTatar", "region": "Europe"},
+    "<2hr>": {"name": "Croatian", "region": "Europe"},
+    "<2cs>": {"name": "Czech", "region": "Europe"},
+    "<2da>": {"name": "Danish", "region": "Europe"},
+    "<2nl>": {"name": "Dutch", "region": "Europe"},
+    "<2en>": {"name": "English", "region": "Europe"},
+    "<2myv>": {"name": "Erzya", "region": "Europe"},
+    "<2eo>": {"name": "Esperanto", "region": "Europe"},
+    "<2et>": {"name": "Estonian", "region": "Europe"},
+    "<2fo>": {"name": "Faroese", "region": "Europe"},
+    "<2fi>": {"name": "Finnish", "region": "Europe"},
+    "<2fr>": {"name": "French", "region": "Europe"},
+    "<2gag>": {"name": "Gagauz", "region": "Europe"},
+    "<2gl>": {"name": "Galician", "region": "Europe"},
+    "<2ka>": {"name": "Georgian", "region": "Europe"},
+    "<2de>": {"name": "German", "region": "Europe"},
+    "<2el>": {"name": "Greek", "region": "Europe"},
+    "<2mrj>": {"name": "HillMari", "region": "Europe"},
+    "<2hu>": {"name": "Hungarian", "region": "Europe"},
+    "<2is>": {"name": "Icelandic", "region": "Europe"},
+    "<2ga>": {"name": "Irish", "region": "Europe"},
+    "<2it>": {"name": "Italian", "region": "Europe"},
+    "<2kbd>": {"name": "Kabardian", "region": "Europe"},
+    "<2xal>": {"name": "Kalmyk", "region": "Europe"},
+    "<2kaa>": {"name": "Kara-Kalpak", "region": "Europe"},
+    "<2krc>": {"name": "Karachay-Balkar", "region": "Europe"},
+    "<2kjh>": {"name": "Khakas", "region": "Europe"},
+    "<2kv>": {"name": "Komi", "region": "Europe"},
+    "<2koi>": {"name": "Komi-Permyak", "region": "Europe"},
+    "<2kum>": {"name": "Kumyk", "region": "Europe"},
+    "<2ltg>": {"name": "Latgalian", "region": "Europe"},
+    "<2la>": {"name": "Latin", "region": "Europe"},
+    "<2lv>": {"name": "Latvian", "region": "Europe"},
+    "<2lt>": {"name": "Lithuanian", "region": "Europe"},
+    "<2lb>": {"name": "Luxembourgish", "region": "Europe"},
+    "<2mk>": {"name": "Macedonian", "region": "Europe"},
+    "<2mt>": {"name": "Maltese", "region": "Europe"},
+    "<2gv>": {"name": "Manx", "region": "Europe"},
+    "<2chm>": {"name": "MeadowMari", "region": "Europe"},
+    "<2mdf>": {"name": "Moksha", "region": "Europe"},
+    "<2se>": {"name": "N.Sami", "region": "Europe"},
+    "<2nog>": {"name": "Nogai", "region": "Europe"},
+    "<2no>": {"name": "Norwegian", "region": "Europe"},
+    "<2oc>": {"name": "Occitan", "region": "Europe"},
+    "<2ang>": {"name": "OldEnglish", "region": "Europe"},
+    "<2os>": {"name": "Ossetian", "region": "Europe"},
+    "<2pl>": {"name": "Polish", "region": "Europe"},
+    "<2pt>": {"name": "Portuguese", "region": "Europe"},
+    "<2rom>": {"name": "Romani", "region": "Europe"},
+    "<2ro>": {"name": "Romanian", "region": "Europe"},
+    "<2rm>": {"name": "Romansh", "region": "Europe"},
+    "<2ru>": {"name": "Russian", "region": "Europe"},
+    "<2alt>": {"name": "S.Altai", "region": "Europe"},
+    "<2stq>": {"name": "Saterfriesisch", "region": "Europe"},
+    "<2gd>": {"name": "ScottishGaelic", "region": "Europe"},
+    "<2sr>": {"name": "Serbian", "region": "Europe"},
+    "<2sk>": {"name": "Slovak", "region": "Europe"},
+    "<2sl>": {"name": "Slovenian", "region": "Europe"},
+    "<2es>": {"name": "Spanish", "region": "Europe"},
+    "<2sv>": {"name": "Swedish", "region": "Europe"},
+    "<2gsw>": {"name": "SwissGerman", "region": "Europe"},
+    "<2tab>": {"name": "Tabassaran", "region": "Europe"},
+    "<2tt>": {"name": "Tatar", "region": "Europe"},
+    "<2tyv>": {"name": "Tuvinian", "region": "Europe"},
+    "<2udm>": {"name": "Udmurt", "region": "Europe"},
+    "<2uk>": {"name": "Ukrainian", "region": "Europe"},
+    "<2vec>": {"name": "Venetian", "region": "Europe"},
+    "<2fy>": {"name": "W.Frisian", "region": "Europe"},
+    "<2wa>": {"name": "Walloon", "region": "Europe"},
+    "<2cy>": {"name": "Welsh", "region": "Europe"},
+    "<2ar>": {"name": "Arabic", "region": "Middle East & North Africa"},
+    "<2bgp>": {"name": "E.Baluchi", "region": "Middle East & North Africa"},
+    "<2mey>": {"name": "Hassaniyya", "region": "Middle East & North Africa"},
+    "<2he>": {"name": "Hebrew", "region": "Middle East & North Africa"},
+    "<2ks>": {"name": "Kashmiri", "region": "Middle East & North Africa"},
+    "<2ku>": {"name": "Kurdish(Kurmanji)", "region": "Middle East & North Africa"},
+    "<2ckb>": {"name": "Kurdish(Sorani)", "region": "Middle East & North Africa"},
+    "<2lrc>": {"name": "N.Luri", "region": "Middle East & North Africa"},
+    "<2ps>": {"name": "Pashto", "region": "Middle East & North Africa"},
+    "<2fa>": {"name": "Persian", "region": "Middle East & North Africa"},
+    "<2luz>": {"name": "S.Luri", "region": "Middle East & North Africa"},
+    "<2skr>": {"name": "Saraiki", "region": "Middle East & North Africa"},
+    "<2sd>": {"name": "Sindhi", "region": "Middle East & North Africa"},
+    "<2syr>": {"name": "Syriac", "region": "Middle East & North Africa"},
+    "<2tks>": {"name": "Takestani", "region": "Middle East & North Africa"},
+    "<2tr>": {"name": "Turkish", "region": "Middle East & North Africa"},
+    "<2ur>": {"name": "Urdu", "region": "Middle East & North Africa"},
+    "<2ug>": {"name": "Uyghur", "region": "Middle East & North Africa"},
+    "<2yi>": {"name": "Yiddish", "region": "Middle East & North Africa"},
+    "<2zza>": {"name": "Zaza", "region": "Middle East & North Africa"},
+    "<2abt>": {"name": "Ambulas", "region": "Oceania"},
+    "<2bi>": {"name": "Bislama", "region": "Oceania"},
+    "<2ape>": {"name": "Bukiyip", "region": "Oceania"},
+    "<2ch>": {"name": "Chamorro", "region": "Oceania"},
+    "<2chk>": {"name": "Chuukese", "region": "Oceania"},
+    "<2mps>": {"name": "Dadibi", "region": "Oceania"},
+    "<2enq>": {"name": "Enga", "region": "Oceania"},
+    "<2hif>": {"name": "FijiHindi", "region": "Oceania"},
+    "<2fj>": {"name": "Fijian", "region": "Oceania"},
+    "<2gil>": {"name": "Gilbertese", "region": "Oceania"},
+    "<2ho>": {"name": "HiriMotu", "region": "Oceania"},
+    "<2hui>": {"name": "Huli", "region": "Oceania"},
+    "<2kos>": {"name": "Kosraean", "region": "Oceania"},
+    "<2ksd>": {"name": "Kuanua", "region": "Oceania"},
+    "<2mi>": {"name": "Maori", "region": "Oceania"},
+    "<2mh>": {"name": "Marshallese", "region": "Oceania"},
+    "<2meu>": {"name": "Motu", "region": "Oceania"},
+    "<2tuc>": {"name": "Mutu", "region": "Oceania"},
+    "<2pau>": {"name": "Palauan", "region": "Oceania"},
+    "<2pis>": {"name": "Pijin", "region": "Oceania"},
+    "<2pon>": {"name": "Pohnpeian", "region": "Oceania"},
+    "<2rwo>": {"name": "Rawa", "region": "Oceania"},
+    "<2hvn>": {"name": "Sabu", "region": "Oceania"},
+    "<2sm>": {"name": "Samoan", "region": "Oceania"},
+    "<2twu>": {"name": "Termanu", "region": "Oceania"},
+    "<2to>": {"name": "Tonga(TongaIslands)", "region": "Oceania"},
+    "<2tvl>": {"name": "Tuvalu", "region": "Oceania"},
+    "<2ubu>": {"name": "Umbu-Ungu", "region": "Oceania"},
+    "<2yap>": {"name": "Yapese", "region": "Oceania"},
+    "<2as>": {"name": "Assamese", "region": "South Asia"},
+    "<2awa>": {"name": "Awadhi", "region": "South Asia"},
+    "<2bn>": {"name": "Bengali", "region": "South Asia"},
+    "<2bho>": {"name": "Bhojpuri", "region": "South Asia"},
+    "<2brx>": {"name": "Bodo(India)", "region": "South Asia"},
+    "<2hne>": {"name": "Chhattisgarhi", "region": "South Asia"},
+    "<2dv>": {"name": "Dhivehi", "region": "South Asia"},
+    "<2taj>": {"name": "E.Tamang", "region": "South Asia"},
+    "<2gbm>": {"name": "Garhwali", "region": "South Asia"},
+    "<2gom>": {"name": "GoanKonkani", "region": "South Asia"},
+    "<2gu>": {"name": "Gujarati", "region": "South Asia"},
+    "<2hi>": {"name": "Hindi", "region": "South Asia"},
+    "<2bjj>": {"name": "Kanauji", "region": "South Asia"},
+    "<2kn>": {"name": "Kannada", "region": "South Asia"},
+    "<2kha>": {"name": "Khasi", "region": "South Asia"},
+    "<2trp>": {"name": "KokBorok", "region": "South Asia"},
+    "<2mag>": {"name": "Magahi", "region": "South Asia"},
+    "<2mai>": {"name": "Maithili", "region": "South Asia"},
+    "<2ml>": {"name": "Malayalam", "region": "South Asia"},
+    "<2mr>": {"name": "Marathi", "region": "South Asia"},
+    "<2mni>": {"name": "Meiteilon(Manipuri)", "region": "South Asia"},
+    "<2mtr>": {"name": "Mewari", "region": "South Asia"},
+    "<2lus>": {"name": "Mizo", "region": "South Asia"},
+    "<2ne>": {"name": "Nepali", "region": "South Asia"},
+    "<2new>": {"name": "Newari", "region": "South Asia"},
+    "<2noe>": {"name": "Nimadi", "region": "South Asia"},
+    "<2or>": {"name": "Odia(Oriya)", "region": "South Asia"},
+    "<2pa>": {"name": "Punjabi", "region": "South Asia"},
+    "<2raj>": {"name": "Rajasthani", "region": "South Asia"},
+    "<2sa>": {"name": "Sanskrit", "region": "South Asia"},
+    "<2smt>": {"name": "Simte", "region": "South Asia"},
+    "<2si>": {"name": "Sinhala", "region": "South Asia"},
+    "<2suz>": {"name": "Sunwar", "region": "South Asia"},
+    "<2sjp>": {"name": "Surjapuri", "region": "South Asia"},
+    "<2syl>": {"name": "Sylheti", "region": "South Asia"},
+    "<2ta>": {"name": "Tamil", "region": "South Asia"},
+    "<2te>": {"name": "Telugu", "region": "South Asia"},
+    "<2tcy>": {"name": "Tulu", "region": "South Asia"},
+    "<2ace>": {"name": "Achinese", "region": "Southeast Asia"},
+    "<2msm>": {"name": "AgusanManobo", "region": "Southeast Asia"},
+    "<2ahk>": {"name": "Akha", "region": "Southeast Asia"},
+    "<2ban>": {"name": "Balinese", "region": "Southeast Asia"},
+    "<2bgz>": {"name": "Banggai", "region": "Southeast Asia"},
+    "<2akb>": {"name": "BatakAngkola", "region": "Southeast Asia"},
+    "<2btx>": {"name": "BatakKaro", "region": "Southeast Asia"},
+    "<2bts>": {"name": "BatakSimalungun", "region": "Southeast Asia"},
+    "<2bbc>": {"name": "BatakToba", "region": "Southeast Asia"},
+    "<2bew>": {"name": "Betawi", "region": "Southeast Asia"},
+    "<2jvn>": {"name": "CaribbeanJavanese", "region": "Southeast Asia"},
+    "<2ceb>": {"name": "Cebuano", "region": "Southeast Asia"},
+    "<2bik>": {"name": "CentralBikol", "region": "Southeast Asia"},
+    "<2mel>": {"name": "CentralMelanau", "region": "Southeast Asia"},
+    "<2cbk>": {"name": "Chavacano", "region": "Southeast Asia"},
+    "<2bru>": {"name": "E.Bru", "region": "Southeast Asia"},
+    "<2cfm>": {"name": "FalamChin", "region": "Southeast Asia"},
+    "<2fil>": {"name": "Filipino", "region": "Southeast Asia"},
+    "<2gor>": {"name": "Gorontalo", "region": "Southeast Asia"},
+    "<2cnh>": {"name": "HakhaChin", "region": "Southeast Asia"},
+    "<2hil>": {"name": "Hiligaynon", "region": "Southeast Asia"},
+    "<2hmn>": {"name": "Hmong", "region": "Southeast Asia"},
+    "<2iba>": {"name": "Iban", "region": "Southeast Asia"},
+    "<2ilo>": {"name": "Ilocano", "region": "Southeast Asia"},
+    "<2id>": {"name": "Indonesian", "region": "Southeast Asia"},
+    "<2ium>": {"name": "IuMien", "region": "Southeast Asia"},
+    "<2jax>": {"name": "JambiMalay", "region": "Southeast Asia"},
+    "<2jv>": {"name": "Javanese", "region": "Southeast Asia"},
+    "<2kac>": {"name": "Kachin", "region": "Southeast Asia"},
+    "<2dtp>": {"name": "KadazanDusun", "region": "Southeast Asia"},
+    "<2meo>": {"name": "KedahMalay", "region": "Southeast Asia"},
+    "<2ify>": {"name": "Keley-IKallahan", "region": "Southeast Asia"},
+    "<2km>": {"name": "Khmer", "region": "Southeast Asia"},
+    "<2kjg>": {"name": "Khmu", "region": "Southeast Asia"},
+    "<2krj>": {"name": "Kinaray-A", "region": "Southeast Asia"},
+    "<2mkn>": {"name": "KupangMalay", "region": "Southeast Asia"},
+    "<2lhu>": {"name": "Lahu", "region": "Southeast Asia"},
+    "<2lo>": {"name": "Lao", "region": "Southeast Asia"},
+    "<2mad>": {"name": "Madurese", "region": "Southeast Asia"},
+    "<2mak>": {"name": "Makasar", "region": "Southeast Asia"},
+    "<2ms>": {"name": "Malay", "region": "Southeast Asia"},
+    "<2xmm>": {"name": "ManadoMalay", "region": "Southeast Asia"},
+    "<2mqy>": {"name": "Manggarai", "region": "Southeast Asia"},
+    "<2mrw>": {"name": "Maranao", "region": "Southeast Asia"},
+    "<2msb>": {"name": "Masbatenyo", "region": "Southeast Asia"},
+    "<2mbt>": {"name": "MatigsalugManobo", "region": "Southeast Asia"},
+    "<2min>": {"name": "Minangkabau", "region": "Southeast Asia"},
+    "<2mnw>": {"name": "Mon", "region": "Southeast Asia"},
+    "<2my>": {"name": "Myanmar(Burmese)", "region": "Southeast Asia"},
+    "<2nij>": {"name": "Ngaju", "region": "Southeast Asia"},
+    "<2nia>": {"name": "Nias", "region": "Southeast Asia"},
+    "<2nut>": {"name": "Nung(VietNam)", "region": "Southeast Asia"},
+    "<2pck>": {"name": "PaiteChin", "region": "Southeast Asia"},
+    "<2pam>": {"name": "Pampanga", "region": "Southeast Asia"},
+    "<2pag>": {"name": "Pangasinan", "region": "Southeast Asia"},
+    "<2prk>": {"name": "Parauk", "region": "Southeast Asia"},
+    "<2rki>": {"name": "Rakhine", "region": "Southeast Asia"},
+    "<2msi>": {"name": "SabahMalay", "region": "Southeast Asia"},
+    "<2sxn>": {"name": "Sangir", "region": "Southeast Asia"},
+    "<2shn>": {"name": "Shan", "region": "Southeast Asia"},
+    "<2su>": {"name": "Sundanese", "region": "Southeast Asia"},
+    "<2ksw>": {"name": "S’gawKaren", "region": "Southeast Asia"},
+    "<2tsg>": {"name": "Tausug", "region": "Southeast Asia"},
+    "<2tet>": {"name": "Tetum", "region": "Southeast Asia"},
+    "<2th>": {"name": "Thai", "region": "Southeast Asia"},
+    "<2sda>": {"name": "Toraja-Sa’dan", "region": "Southeast Asia"},
+    "<2tyz>": {"name": "Tày", "region": "Southeast Asia"},
+    "<2ppk>": {"name": "Uma", "region": "Southeast Asia"},
+    "<2vi>": {"name": "Vietnamese", "region": "Southeast Asia"},
+    "<2war>": {"name": "Waray(Philippines)", "region": "Southeast Asia"},
 }

scripts/generate_langmap.py

@@ -13,6 +13,7 @@ Dev dependencies (requirements-dev.txt):
     pdfplumber, pycountry
 """
 
+import re
 import sys
 from pathlib import Path
 
@@ -21,6 +22,26 @@ import pdfplumber
 # Pages 16-22 of the paper contain Table 9 (0-indexed: 15-21)
 TABLE_PAGES = range(15, 22)
 
+# On the last page of the table (page 22, 0-indexed 21), the table ends around
+# y=430 and is followed by Section A.2. Stop processing words below this threshold.
+PAGE_22_MAX_TOP = 440
+
+# Column x-coordinate boundaries for Table 9
+COL_BCP47_MIN = 100
+COL_BCP47_MAX = 140
+COL_NAME_MIN = 140
+COL_NAME_MAX = 220
+COL_SCRIPT_MIN = 220
+COL_SCRIPT_MAX = 260
+COL_DATA_MIN = 260
+
+# Tolerance in pixels for matching name/script cells to BCP-47 cells in the same row.
+# Some multi-line name cells have a slightly different y-position than the BCP-47 cell.
+ROW_TOLERANCE = 5
+
+# BCP-47 code pattern: lowercase base subtag, optional uppercase subtags (e.g. tly-IR, nan-Latn-TW)
+BCP47_RE = re.compile(r"^[a-z]{2,}(-[A-Za-z0-9]+)*$")
+
 # Default region based on writing script
 SCRIPT_TO_REGION: dict[str, str] = {
     "Arab": "Middle East & North Africa",
@@ -50,6 +71,7 @@ SCRIPT_TO_REGION: dict[str, str] = {
     "Olck": "South Asia",
     "Orya": "South Asia",
     "Sinh": "South Asia",
+    "Syrc": "Middle East & North Africa",
     "Taml": "South Asia",
     "Telu": "South Asia",
     "Tfng": "Africa",
@@ -60,81 +82,326 @@ SCRIPT_TO_REGION: dict[str, str] = {
 
 # Language-specific overrides (primarily for Latin-script languages and corrections)
 LANGUAGE_OVERRIDES: dict[str, str] = {
-    # Europe
-    "af": "Africa",
-    "az": "Central Asia",
+    # Europe (Latin-script)
+    "ang": "Europe",
+    "br": "Europe",
     "bs": "Europe",
     "ca": "Europe",
-    "ceb": "Southeast Asia",
     "co": "Europe",
     "cs": "Europe",
     "cy": "Europe",
     "da": "Europe",
     "de": "Europe",
     "en": "Europe",
+    "eo": "Europe",
     "es": "Europe",
     "et": "Europe",
     "eu": "Europe",
     "fi": "Europe",
-    "fil": "Southeast Asia",
+    "fo": "Europe",
     "fr": "Europe",
+    "frp": "Europe",
     "fy": "Europe",
     "ga": "Europe",
+    "gag": "Europe",
     "gd": "Europe",
     "gl": "Europe",
+    "gsw": "Europe",
+    "gv": "Europe",
     "hr": "Europe",
-    "ht": "Americas",
     "hu": "Europe",
-    "id": "Southeast Asia",
-    "ig": "Africa",
     "is": "Europe",
     "it": "Europe",
-    "jv": "Southeast Asia",
-    "kk": "Central Asia",
-    "ku": "Middle East & North Africa",
-    "ky": "Central Asia",
+    "kw": "Europe",
     "la": "Europe",
     "lb": "Europe",
     "lt": "Europe",
+    "ltg": "Europe",
     "lv": "Europe",
-    "mg": "Africa",
-    "mi": "Oceania",
     "mk": "Europe",
-    "mn": "East Asia",
-    "ms": "Southeast Asia",
     "mt": "Europe",
     "nl": "Europe",
     "nn": "Europe",
     "no": "Europe",
-    "ny": "Africa",
+    "oc": "Europe",
     "pl": "Europe",
     "pt": "Europe",
-    "qu": "Americas",
+    "rm": "Europe",
+    "rmc": "Europe",
     "ro": "Europe",
-    "rw": "Africa",
+    "rom": "Europe",
+    "se": "Europe",
     "sk": "Europe",
     "sl": "Europe",
-    "sm": "Oceania",
-    "sn": "Africa",
-    "so": "Africa",
     "sq": "Europe",
     "sr": "Europe",
-    "st": "Africa",
-    "su": "Southeast Asia",
+    "stq": "Europe",
     "sv": "Europe",
+    "vec": "Europe",
+    "wa": "Europe",
+    # Africa (Latin-script)
+    "aa": "Africa",
+    "ach": "Africa",
+    "ada": "Africa",
+    "adh": "Africa",
+    "af": "Africa",
+    "ak": "Africa",
+    "alz": "Africa",
+    "ann": "Africa",
+    "bas": "Africa",
+    "bci": "Africa",
+    "bim": "Africa",
+    "bm": "Africa",
+    "bqc": "Africa",
+    "bum": "Africa",
+    "bus": "Africa",
+    "cce": "Africa",
+    "cjk": "Africa",
+    "din": "Africa",
+    "dje": "Africa",
+    "dov": "Africa",
+    "dwr": "Africa",
+    "dyu": "Africa",
+    "ee": "Africa",
+    "ff": "Africa",
+    "ffm": "Africa",
+    "fip": "Africa",
+    "fon": "Africa",
+    "gub": "Africa",
+    "gvl": "Africa",
+    "ha": "Africa",
+    "ig": "Africa",
+    "ibb": "Africa",
+    "iso": "Africa",
+    "izz": "Africa",
+    "kbp": "Africa",
+    "kg": "Africa",
+    "kj": "Africa",
+    "kmb": "Africa",
+    "kri": "Africa",
+    "ktu": "Africa",
+    "laj": "Africa",
+    "lg": "Africa",
+    "ln": "Africa",
+    "lu": "Africa",
+    "mas": "Africa",
+    "mfe": "Africa",
+    "mg": "Africa",
+    "mgh": "Africa",
+    "niq": "Africa",
+    "nnb": "Africa",
+    "nso": "Africa",
+    "nr": "Africa",
+    "ny": "Africa",
+    "nyu": "Africa",
+    "nzi": "Africa",
+    "om": "Africa",
+    "rn": "Africa",
+    "rw": "Africa",
+    "seh": "Africa",
+    "sg": "Africa",
+    "sn": "Africa",
+    "so": "Africa",
+    "spp": "Africa",
+    "srr": "Africa",
+    "ss": "Africa",
+    "st": "Africa",
+    "sus": "Africa",
     "sw": "Africa",
-    "tg": "Central Asia",
-    "tk": "Central Asia",
-    "tl": "Southeast Asia",
-    "to": "Oceania",
-    "tr": "Middle East & North Africa",
-    "uz": "Central Asia",
-    "vi": "Southeast Asia",
+    "tbz": "Africa",
+    "tdx": "Africa",
+    "teo": "Africa",
+    "tiv": "Africa",
+    "tll": "Africa",
+    "tn": "Africa",
+    "ts": "Africa",
+    "tsc": "Africa",
+    "ve": "Africa",
+    "wal": "Africa",
+    "wo": "Africa",
     "xh": "Africa",
     "yo": "Africa",
+    "zne": "Africa",
     "zu": "Africa",
-    # South Asia (non-Devanagari Latin-script entries)
+    # Americas (Latin-script)
+    "acf": "Americas",
+    "agr": "Americas",
+    "cab": "Americas",
+    "amu": "Americas",
+    "arn": "Americas",
+    "ay": "Americas",
+    "bzj": "Americas",
+    "cac": "Americas",
+    "cak": "Americas",
+    "cni": "Americas",
+    "ctu": "Americas",
+    "cuk": "Americas",
+    "djk": "Americas",
+    "emp": "Americas",
+    "gn": "Americas",
+    "guc": "Americas",
+    "guh": "Americas",
+    "gui": "Americas",
+    "gym": "Americas",
+    "gyn": "Americas",
+    "haw": "Americas",
+    "ht": "Americas",
+    "hus": "Americas",
+    "inb": "Americas",
+    "jac": "Americas",
+    "jam": "Americas",
+    "jiv": "Americas",
+    "kek": "Americas",
+    "kl": "Americas",
+    "knj": "Americas",
+    "kwi": "Americas",
+    "mam": "Americas",
+    "maz": "Americas",
+    "miq": "Americas",
+    "ngu": "Americas",
+    "nhe": "Americas",
+    "noa": "Americas",
+    "nv": "Americas",
+    "otq": "Americas",
+    "pap": "Americas",
+    "qu": "Americas",
+    "qub": "Americas",
+    "quc": "Americas",
+    "quf": "Americas",
+    "quh": "Americas",
+    "qup": "Americas",
+    "quy": "Americas",
+    "qvc": "Americas",
+    "qvi": "Americas",
+    "qvz": "Americas",
+    "qxr": "Americas",
+    "rcf": "Americas",
+    "shp": "Americas",
+    "sja": "Americas",
+    "srm": "Americas",
+    "srn": "Americas",
+    "tca": "Americas",
+    "toj": "Americas",
+    "tzh": "Americas",
+    "tzj": "Americas",
+    "tzo": "Americas",
+    "yua": "Americas",
+    "zap": "Americas",
+    # Southeast Asia (Latin-script)
+    "ace": "Southeast Asia",
+    "ahk": "Southeast Asia",
+    "akb": "Southeast Asia",
+    "ban": "Southeast Asia",
+    "bbc": "Southeast Asia",
+    "bew": "Southeast Asia",
+    "bgz": "Southeast Asia",
+    "bik": "Southeast Asia",
+    "bru": "Southeast Asia",
+    "btx": "Southeast Asia",
+    "bts": "Southeast Asia",
+    "cbk": "Southeast Asia",
+    "ceb": "Southeast Asia",
+    "cfm": "Southeast Asia",
     "cjm": "Southeast Asia",
+    "cnh": "Southeast Asia",
+    "dtp": "Southeast Asia",
+    "fil": "Southeast Asia",
+    "gor": "Southeast Asia",
+    "hil": "Southeast Asia",
+    "hmn": "Southeast Asia",
+    "iba": "Southeast Asia",
+    "id": "Southeast Asia",
+    "ify": "Southeast Asia",
+    "ilo": "Southeast Asia",
+    "ium": "Southeast Asia",
+    "jax": "Southeast Asia",
+    "jv": "Southeast Asia",
+    "jvn": "Southeast Asia",
+    "kac": "Southeast Asia",
+    "krj": "Southeast Asia",
+    "lhu": "Southeast Asia",
+    "mad": "Southeast Asia",
+    "mak": "Southeast Asia",
+    "mbt": "Southeast Asia",
+    "mel": "Southeast Asia",
+    "meo": "Southeast Asia",
+    "min": "Southeast Asia",
+    "mkn": "Southeast Asia",
+    "mqy": "Southeast Asia",
+    "mrw": "Southeast Asia",
+    "ms": "Southeast Asia",
+    "msi": "Southeast Asia",
+    "msb": "Southeast Asia",
+    "msm": "Southeast Asia",
+    "nia": "Southeast Asia",
+    "nij": "Southeast Asia",
+    "nut": "Southeast Asia",
+    "pag": "Southeast Asia",
+    "pam": "Southeast Asia",
+    "pck": "Southeast Asia",
+    "ppk": "Southeast Asia",
+    "prk": "Southeast Asia",
+    "sda": "Southeast Asia",
+    "su": "Southeast Asia",
+    "sxn": "Southeast Asia",
+    "tet": "Southeast Asia",
+    "tl": "Southeast Asia",
+    "tsg": "Southeast Asia",
+    "tyz": "Southeast Asia",
+    "vi": "Southeast Asia",
+    "war": "Southeast Asia",
+    "xmm": "Southeast Asia",
+    # Oceania (Latin-script)
+    "abt": "Oceania",
+    "ape": "Oceania",
+    "bi": "Oceania",
+    "ch": "Oceania",
+    "chk": "Oceania",
+    "enq": "Oceania",
+    "fj": "Oceania",
+    "gil": "Oceania",
+    "hif": "Oceania",
+    "ho": "Oceania",
+    "hui": "Oceania",
+    "hvn": "Oceania",
+    "kos": "Oceania",
+    "ksd": "Oceania",
+    "meu": "Oceania",
+    "mh": "Oceania",
+    "mi": "Oceania",
+    "mps": "Oceania",
+    "pau": "Oceania",
+    "pis": "Oceania",
+    "pon": "Oceania",
+    "rwo": "Oceania",
+    "sm": "Oceania",
+    "to": "Oceania",
+    "tuc": "Oceania",
+    "tvl": "Oceania",
+    "twu": "Oceania",
+    "ubu": "Oceania",
+    "yap": "Oceania",
+    # South Asia (Latin-script)
+    "kha": "South Asia",
+    "lus": "South Asia",
+    "smt": "South Asia",
+    "trp": "South Asia",
+    # East Asia (Latin-script)
+    "mn": "East Asia",
+    "za": "East Asia",
+    # Middle East & North Africa (Latin-script)
+    "bgp": "Middle East & North Africa",
+    "ku": "Middle East & North Africa",
+    "tr": "Middle East & North Africa",
+    "zza": "Middle East & North Africa",
+    # Central Asia (Latin-script)
+    "az": "Central Asia",
+    "kaa-Latn": "Central Asia",
+    "kk": "Central Asia",
+    "ky": "Central Asia",
+    "tg": "Central Asia",
+    "tk": "Central Asia",
+    "uz": "Central Asia",
     # Corrections for Cyrillic languages not in Europe
     "ba": "Central Asia",
     "ce": "Europe",
@@ -143,32 +410,64 @@ LANGUAGE_OVERRIDES: dict[str, str] = {
     "os": "Europe",
     "sah": "East Asia",
     "tt": "Europe",
+    # Non-geographic / constructed languages: assign to Americas (creator's continent)
+    "crs": "Americas",  # Seselwa Creole French (Indian Ocean) — grouped with Atlantic/Indian Ocean creoles
+    "tlh": "Americas",  # Klingon — constructed language
+    "zxx": "Americas",  # Noise/non-linguistic content — placeholder region
 }
 
 
 def parse_table9(pdf_path: str) -> list[dict[str, str]]:
-    """Parse Table 9 from the MADLAD-400 paper."""
+    """Parse Table 9 from the MADLAD-400 paper using word-based extraction.
+
+    pdfplumber's extract_table() does not work on this PDF because the table
+    lines are not detected as table borders. Instead we extract words and
+    assign them to columns by x-coordinate, then group by row using a
+    y-position tolerance to handle multi-line name cells.
+    """
     entries: list[dict[str, str]] = []
     with pdfplumber.open(pdf_path) as pdf:
         for page_num in TABLE_PAGES:
             page = pdf.pages[page_num]
-            table = page.extract_table()
-            if not table:
-                continue
-            for row in table:
-                if not row or not row[0]:
+            words = page.extract_words()
+
+            # On the last table page, cut off below the table (avoids Section A.2 code block)
+            max_top = PAGE_22_MAX_TOP if page_num == 21 else float("inf")
+
+            # Partition words by column
+            bcp47_col = [w for w in words if COL_BCP47_MIN <= w["x0"] < COL_BCP47_MAX and w["top"] <= max_top]
+            name_col = [w for w in words if COL_NAME_MIN <= w["x0"] < COL_NAME_MAX and w["top"] <= max_top]
+            script_col = [w for w in words if COL_SCRIPT_MIN <= w["x0"] < COL_SCRIPT_MAX and w["top"] <= max_top]
+            data_col = [w for w in words if w["x0"] >= COL_DATA_MIN and w["top"] <= max_top]
+
+            for bcp_w in bcp47_col:
+                bcp47 = bcp_w["text"]
+
+                # Skip non-BCP-47 text (headers, prose, code)
+                if not BCP47_RE.match(bcp47) or bcp47 in ("total", "median"):
                     continue
-                bcp47 = row[0].strip()
-                # Skip header rows
-                if bcp47 == "BCP-47" or bcp47 == "":
+
+                bcp_top = bcp_w["top"]
+
+                # Find cells in the same row (within ROW_TOLERANCE pixels)
+                name_ws = [w for w in name_col if abs(w["top"] - bcp_top) <= ROW_TOLERANCE]
+                script_ws = [w for w in script_col if abs(w["top"] - bcp_top) <= ROW_TOLERANCE]
+                data_ws = [w for w in data_col if abs(w["top"] - bcp_top) <= ROW_TOLERANCE]
+
+                name = " ".join(w["text"] for w in sorted(name_ws, key=lambda w: w["x0"]))
+                script = " ".join(w["text"] for w in sorted(script_ws, key=lambda w: w["x0"]))
+                data_texts = [w["text"] for w in sorted(data_ws, key=lambda w: w["x0"])]
+
+                # Require both name and script to be present
+                if not name or not script:
                     continue
-                name = row[1].strip() if row[1] else ""
-                script = row[2].strip() if row[2] else ""
-                # Skip rows with "-" in all data columns (last 79 self-audit omissions)
-                data_cols = [c for c in row[3:] if c is not None]
-                if all(c.strip() == "-" for c in data_cols) or not data_cols:
+
+                # Skip rows where all data columns are "-" (self-audit omissions)
+                if data_texts and all(d.strip() == "-" for d in data_texts):
                     continue
+
                 entries.append({"bcp47": bcp47, "name": name, "script": script})
+
     return entries