Restrict language mapping to Table 14 evaluated languages

Filter langid_to_language from ~400 entries to 183 languages
evaluated in Section A.9 of the MADLAD-400 paper. Only languages
from the WMT, Flores-200, NTREX, and Gatones evaluation sets
are retained.

Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com>

langmap/langid_mapping.py

@@ -1,5 +1,7 @@
-# Mapping the BCP-47 codes used in the MADLAD-400 models to the language names
+# Mapping the BCP-47 codes used in the MADLAD-400 models to the language names.
+# Restricted to languages evaluated in Table 14, Section A.9 of:
 # [MADLAD-400: A Multilingual And Document-Level Large Audited Dataset](https://arxiv.org/pdf/2309.04662)
+# Evaluation sets: WMT, Flores-200, NTREX, Gatones
 langid_to_language = {
     # Page 16
     "<2en>": "English",
@@ -66,12 +68,10 @@ langid_to_language = {
     "<2yue>": "Cantonese",
     "<2ne>": "Nepali",
     "<2kn>": "Kannada",
-    "<2kaa>": "Kara-Kalpak",
     "<2gu>": "Gujarati",
     "<2si>": "Sinhala",
     "<2cy>": "Welsh",
     "<2eo>": "Esperanto",
-    "<2la>": "Latin",
     "<2hy>": "Armenian",
     "<2ky>": "Kyrghyz",
     "<2tg>": "Tajik",
@@ -88,7 +88,6 @@ langid_to_language = {
     "<2lo>": "Lao",
     "<2ha>": "Hausa",
     "<2dv>": "Dhivehi",
-    "<2fy>": "W. Frisian",
     "<2lb>": "Luxembourgish",
     "<2ckb>": "Kurdish (Sorani)",
     "<2mg>": "Malagasy",
@@ -96,7 +95,6 @@ langid_to_language = {
     "<2am>": "Amharic",
     "<2ug>": "Uyghur",
     "<2ht>": "Haitian Creole",
-    "<2grc>": "Ancient Greek",
     "<2hmn>": "Hmong",
     "<2sd>": "Sindhi",
     "<2jv>": "Javanese",
@@ -113,316 +111,85 @@ langid_to_language = {
     "<2ny>": "Chichewa",
     "<2sm>": "Samoan",
     "<2sn>": "Shona",
-    "<2co>": "Corsican",
     "<2zu>": "Zulu",
     "<2ig>": "Igbo",
     "<2yo>": "Yoruba",
     "<2pap>": "Papiamento",
     "<2st>": "Sesotho",
-    "<2haw>": "Hawaiian",
     "<2as>": "Assamese",
     "<2oc>": "Occitan",
     "<2cv>": "Chuvash",
     "<2lus>": "Mizo",
-    "<2tet>": "Tetum",
-    "<2gsw>": "Swiss German",
-    "<2sah>": "Yakut",
-    "<2br>": "Breton",
-    "<2rm>": "Romansh",
     "<2sa>": "Sanskrit",
     "<2bo>": "Tibetan",
     "<2om>": "Oromo",
-    "<2se>": "N. Sami",
     "<2ce>": "Chechen",
-    "<2cnh>": "Hakha Chin",
     # Page 18
     "<2ilo>": "Ilocano",
-    "<2hil>": "Hiligaynon",
-    "<2udm>": "Udmurt",
-    "<2os>": "Ossetian",
     "<2lg>": "Luganda",
     "<2ti>": "Tigrinya",
     "<2vec>": "Venetian",
     "<2ts>": "Tsonga",
-    "<2tyv>": "Tuvinian",
-    "<2kbd>": "Kabardian",
     "<2ee>": "Ewe",
-    "<2iba>": "Iban",
     "<2av>": "Avar",
-    "<2kha>": "Khasi",
     "<2to>": "Tonga (Tonga Islands)",
     "<2tn>": "Tswana",
     "<2nso>": "Sepedi",
     "<2fj>": "Fijian",
     "<2zza>": "Zaza",
     "<2ak>": "Twi",
-    "<2ada>": "Adangme",
-    "<2otq>": "Querétaro Otomi",
     "<2dz>": "Dzongkha",
-    "<2bua>": "Buryat",
-    "<2cfm>": "Falam Chin",
     "<2ln>": "Lingala",
-    "<2chm>": "Meadow Mari",
     "<2gn>": "Guarani",
-    "<2krc>": "Karachay-Balkar",
-    "<2wa>": "Walloon",
-    "<2hif>": "Fiji Hindi",
     "<2yua>": "Yucateco",
-    "<2srn>": "Sranan Tongo",
     "<2war>": "Waray (Philippines)",
-    "<2rom>": "Romani",
-    "<2bik>": "Central Bikol",
-    "<2pam>": "Pampanga",
     "<2sg>": "Sango",
-    "<2lu>": "Luba-Katanga",
     "<2ady>": "Adyghe",
     "<2kbp>": "Kabiyè",
-    "<2syr>": "Syriac",
     "<2ltg>": "Latgalian",
-    "<2myv>": "Erzya",
     "<2iso>": "Isoko",
     "<2kac>": "Kachin",
     "<2bho>": "Bhojpuri",
     "<2ay>": "Aymara",
-    "<2kum>": "Kumyk",
     "<2qu>": "Quechua",
-    "<2za>": "Zhuang",
     "<2pag>": "Pangasinan",
-    "<2ngu>": "Guerrero Nahuatl",
     "<2ve>": "Venda",
-    "<2pck>": "Paite Chin",
-    "<2zap>": "Zapotec",
-    "<2tyz>": "Tày",
-    "<2hui>": "Huli",
     "<2bbc>": "Batak Toba",
-    "<2tzo>": "Tzotzil",
     "<2tiv>": "Tiv",
-    "<2ksd>": "Kuanua",
     "<2gom>": "Goan Konkani",
     "<2min>": "Minangkabau",
-    "<2ang>": "Old English",
-    "<2nhe>": "E. Huasteca Nahuatl",
-    "<2bgp>": "E. Baluchi",
-    "<2nzi>": "Nzima",
-    "<2nnb>": "Nande",
-    "<2nv>": "Navajo",
-    # '<2zxx>': 'Noise',
     "<2bci>": "Baoulé",
-    "<2kv>": "Komi",
-    "<2new>": "Newari",
-    "<2mps>": "Dadibi",
-    "<2alt>": "S. Altai",
-    "<2meu>": "Motu",
     "<2bew>": "Betawi",
     "<2fon>": "Fon",
-    "<2iu>": "Inuktitut",
-    "<2abt>": "Ambulas",
     # Page 19
-    "<2mgh>": "Makhuwa-Meetto",
-    "<2mnw>": "Mon",
-    "<2tvl>": "Tuvalu",
-    "<2dov>": "Dombe",
-    "<2tlh>": "Klingon",
-    "<2ho>": "Hiri Motu",
-    "<2kw>": "Cornish",
-    "<2mrj>": "Hill Mari",
     "<2meo>": "Kedah Malay",
     "<2crh>": "Crimean Tatar",
-    "<2mbt>": "Matigsalug Manobo",
-    "<2emp>": "N. Emberá",
     "<2ace>": "Achinese",
-    "<2ium>": "Iu Mien",
-    "<2mam>": "Mam",
-    "<2gym>": "Ngäbere",
     "<2mai>": "Maithili",
-    "<2crs>": "Seselwa Creole French",
-    "<2pon>": "Pohnpeian",
-    "<2ubu>": "Umbu-Ungu",
-    "<2fip>": "Fipa",
-    "<2quc>": "K’iche’",
-    "<2gv>": "Manx",
-    "<2kj>": "Kuanyama",
-    "<2btx>": "Batak Karo",
-    "<2ape>": "Bukiyip",
-    "<2chk>": "Chuukese",
-    "<2rcf>": "Réunion Creole French",
+    "<2quc>": "K\u2019iche\u2019",
     "<2shn>": "Shan",
-    "<2tzh>": "Tzeltal",
-    "<2mdf>": "Moksha",
-    "<2ppk>": "Uma",
     "<2ss>": "Swati",
-    "<2gag>": "Gagauz",
-    "<2cab>": "Garifuna",
     "<2kri>": "Krio",
-    "<2seh>": "Sena",
-    "<2ibb>": "Ibibio",
-    "<2tbz>": "Ditammari",
-    "<2bru>": "E. Bru",
-    "<2enq>": "Enga",
-    "<2ach>": "Acoli",
-    "<2cuk>": "San Blas Kuna",
     "<2kmb>": "Kimbundu",
     "<2wo>": "Wolof",
-    "<2kek>": "Kekchí",
-    "<2qub>": "Huallaga Huánuco Quechua",
-    "<2tab>": "Tabassaran",
-    "<2bts>": "Batak Simalungun",
-    "<2kos>": "Kosraean",
-    "<2rwo>": "Rawa",
-    "<2cak>": "Kaqchikel",
-    "<2tuc>": "Mutu",
-    "<2bum>": "Bulu",
-    "<2cjk>": "Chokwe",
-    "<2gil>": "Gilbertese",
-    "<2stq>": "Saterfriesisch",
-    "<2tsg>": "Tausug",
-    "<2quh>": "S. Bolivian Quechua",
-    "<2mak>": "Makasar",
-    "<2arn>": "Mapudungun",
     "<2ban>": "Balinese",
-    "<2jiv>": "Shuar",
-    "<2sja>": "Epena",
-    "<2yap>": "Yapese",
-    "<2tcy>": "Tulu",
-    "<2toj>": "Tojolabal",
-    "<2twu>": "Termanu",
-    "<2xal>": "Kalmyk",
-    "<2amu>": "Guerrero Amuzgo",
-    "<2rmc>": "Carpathian Romani",
-    "<2hus>": "Huastec",
-    "<2nia>": "Nias",
-    "<2kjh>": "Khakas",
     "<2bm>": "Bambara",
-    "<2guh>": "Guahibo",
-    "<2mas>": "Masai",
-    "<2acf>": "St Lucian Creole French",
-    "<2dtp>": "Kadazan Dusun",
-    "<2ksw>": "S’gaw Karen",
-    "<2bzj>": "Belize Kriol English",
     # Page 20
     "<2din>": "Dinka",
-    "<2zne>": "Zande",
     "<2mad>": "Madurese",
-    "<2msi>": "Sabah Malay",
     "<2mag>": "Magahi",
-    "<2mkn>": "Kupang Malay",
     "<2kg>": "Kongo",
-    "<2lhu>": "Lahu",
-    "<2ch>": "Chamorro",
-    "<2qvi>": "Imbabura H. Quichua",
-    "<2mh>": "Marshallese",
-    "<2djk>": "E. Maroon Creole",
-    "<2sus>": "Susu",
-    "<2mfe>": "Morisien",
-    "<2srm>": "Saramaccan",
     "<2dyu>": "Dyula",
-    "<2ctu>": "Chol",
-    "<2gui>": "E. Bolivian Guaraní",
-    "<2pau>": "Palauan",
-    "<2inb>": "Inga",
-    "<2bi>": "Bislama",
     "<2mni>": "Meiteilon (Manipuri)",
-    "<2guc>": "Wayuu",
-    "<2jam>": "Jamaican Creole English",
-    "<2wal>": "Wolaytta",
-    "<2jac>": "Popti’",
-    "<2bas>": "Basa (Cameroon)",
-    "<2gor>": "Gorontalo",
     "<2skr>": "Saraiki",
-    "<2nyu>": "Nyungwe",
-    "<2noa>": "Woun Meu",
-    "<2sda>": "Toraja-Sa’dan",
-    "<2gub>": "Guajajára",
-    "<2nog>": "Nogai",
-    "<2cni>": "Asháninka",
-    "<2teo>": "Teso",
-    "<2tdx>": "Tandroy-Mahafaly Malagasy",
-    "<2sxn>": "Sangir",
-    "<2rki>": "Rakhine",
-    "<2nr>": "South Ndebele",
-    "<2frp>": "Arpitan",
-    "<2alz>": "Alur",
-    "<2taj>": "E. Tamang",
-    "<2lrc>": "N. Luri",
-    "<2cce>": "Chopi",
     "<2rn>": "Rundi",
-    "<2jvn>": "Caribbean Javanese",
-    "<2hvn>": "Sabu",
-    "<2nij>": "Ngaju",
-    "<2dwr>": "Dawro",
-    "<2izz>": "Izii",
-    "<2msm>": "Agusan Manobo",
-    "<2bus>": "Bokobaru",
-    "<2ktu>": "Kituba (DRC)",
     "<2chr>": "Cherokee",
-    "<2maz>": "Central Mazahua",
-    "<2tzj>": "Tz’utujil",
-    "<2suz>": "Sunwar",
-    "<2knj>": "W. Kanjobal",
-    "<2bim>": "Bimoba",
-    "<2gvl>": "Gulay",
-    "<2bqc>": "Boko (Benin)",
-    "<2tca>": "Ticuna",
-    "<2pis>": "Pijin",
-    "<2prk>": "Parauk",
-    "<2laj>": "Lango (Uganda)",
-    "<2mel>": "Central Melanau",
-    "<2qxr>": "Cañar H. Quichua",
-    "<2niq>": "Nandi",
-    "<2ahk>": "Akha",
-    "<2shp>": "Shipibo-Conibo",
     "<2hne>": "Chhattisgarhi",
-    "<2spp>": "Supyyire Senoufo",
-    "<2koi>": "Komi-Permyak",
-    "<2krj>": "Kinaray-A",
-    "<2quf>": "Lambayeque Quechua",
-    "<2luz>": "S. Luri",
-    "<2agr>": "Aguaruna",
-    "<2tsc>": "Tswa",
-    "<2mqy>": "Manggarai",
-    "<2gof>": "Gofa",
     # Page 21
-    "<2gbm>": "Garhwali",
-    "<2miq>": "Mískito",
-    "<2dje>": "Zarma",
     "<2awa>": "Awadhi",
-    "<2bjj>": "Kanauji",
-    "<2qvz>": "N. Pastaza Quichua",
-    "<2sjp>": "Surjapuri",
-    "<2tll>": "Tetela",
-    "<2raj>": "Rajasthani",
-    "<2kjg>": "Khmu",
-    "<2bgz>": "Banggai",
     "<2quy>": "Ayacucho Quechua",
-    "<2cbk>": "Chavacano",
-    "<2akb>": "Batak Angkola",
-    "<2oj>": "Ojibwa",
-    "<2ify>": "Keley-I Kallahan",
     "<2mey>": "Hassaniyya",
     "<2ks>": "Kashimiri",
-    "<2cac>": "Chuj",
-    "<2brx>": "Bodo (India)",
-    "<2qup>": "S. Pastaza Quechua",
-    "<2syl>": "Sylheti",
-    "<2jax>": "Jambi Malay",
     "<2ff>": "Fulfulde",
     "<2ber>": "Tamazight (Tfng)",
-    "<2tks>": "Takestani",
-    "<2trp>": "Kok Borok",
-    "<2mrw>": "Maranao",
-    "<2adh>": "Adhola",
-    "<2smt>": "Simte",
-    "<2srr>": "Serer",
-    "<2ffm>": "Maasina Fulfulde",
-    "<2qvc>": "Cajamarca Quechua",
-    "<2mtr>": "Mewari",
-    "<2ann>": "Obolo",
-    "<2kaa-Latn>": "Kara-Kalpak (Latn)",
-    "<2aa>": "Afar",
-    "<2noe>": "Nimadi",
-    "<2nut>": "Nung (Viet Nam)",
-    "<2gyn>": "Guyanese Creole English",
-    "<2kwi>": "Awa-Cuaiquer",
-    "<2xmm>": "Manado Malay",
-    "<2msb>": "Masbatenyo",
 }