langmap/langid_mapping.py

# Tier 1 production-ready languages: BLEU 35+ in both translation directions
# (en→xx and xx→en) from Section A.10 of the MADLAD-400 paper.
# See: https://arxiv.org/pdf/2309.04662
langid_to_language = {
    "<2af>": "Afrikaans",
    "<2bg>": "Bulgarian",
    "<2bs>": "Bosnian",
    "<2ca>": "Catalan",
    "<2cs>": "Czech",
    "<2da>": "Danish",
    "<2de>": "German",
    "<2es>": "Spanish",
    "<2fil>": "Filipino",
    "<2fr>": "French",
    "<2hr>": "Croatian",
    "<2id>": "Indonesian",
    "<2it>": "Italian",
    "<2mk>": "Macedonian",
    "<2ms>": "Malay",
    "<2mt>": "Maltese",
    "<2nn>": "Norwegian Nynorsk",
    "<2no>": "Norwegian",
    "<2pt>": "Portuguese",
    "<2ro>": "Romanian",
    "<2sk>": "Slovak",
    "<2sv>": "Swedish",
}