Daryl Lim Claude Opus 4.6 commited on
Commit
a760c43
·
1 Parent(s): 87a2676

Rename LangMap/ to langmap/ and fix package issues

Browse files

- Rename to lowercase for PEP 8 package naming convention
- Fix leading space in Tz'utujil language name
- Update __init__.py to export langid_to_language directly
- Add py.typed marker for PEP 561 type stub support

Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com>

LangMap/__init__.py DELETED
@@ -1 +0,0 @@
1
- from . import langid_mapping
 
 
LangMap/langid_mapping.py DELETED
@@ -1,428 +0,0 @@
1
- # Mapping the BCP-47 codes used in the MADLAD-400 models to the language names
2
- # [MADLAD-400: A Multilingual And Document-Level Large Audited Dataset](https://arxiv.org/pdf/2309.04662)
3
- langid_to_language = {
4
- # Page 16
5
- '<2en>': 'English',
6
- '<2ru>': 'Russian',
7
- '<2es>': 'Spanish',
8
- '<2fr>': 'French',
9
- '<2de>': 'German',
10
- '<2it>': 'Italian',
11
- '<2pt>': 'Portuguese',
12
- '<2pl>': 'Polish',
13
- '<2nl>': 'Dutch',
14
- '<2vi>': 'Vietnamese',
15
- '<2tr>': 'Turkish',
16
- '<2sv>': 'Swedish',
17
- '<2id>': 'Indonesian',
18
- '<2ro>': 'Romanian',
19
- '<2cs>': 'Czech',
20
- '<2zh>': 'Mandarin Chinese',
21
- '<2hu>': 'Hungarian',
22
- '<2ja>': 'Japanese',
23
- '<2th>': 'Thai',
24
- '<2fi>': 'Finnish',
25
- '<2fa>': 'Persian',
26
- '<2uk>': 'Ukrainian',
27
- '<2da>': 'Danish',
28
- '<2el>': 'Greek',
29
- '<2no>': 'Norwegian',
30
- '<2bg>': 'Bulgarian',
31
- '<2sk>': 'Slovak',
32
- '<2ko>': 'Korean',
33
- '<2ar>': 'Arabic',
34
- '<2lt>': 'Lithuanian',
35
- '<2ca>': 'Catalan',
36
- '<2sl>': 'Slovenian',
37
- '<2he>': 'Hebrew',
38
- '<2et>': 'Estonian',
39
- '<2lv>': 'Latvian',
40
- '<2hi>': 'Hindi',
41
- '<2sq>': 'Albanian',
42
- '<2ms>': 'Malay',
43
- '<2az>': 'Azerbaijani',
44
- '<2sr>': 'Serbian',
45
- '<2ta>': 'Tamil',
46
- '<2hr>': 'Croatian',
47
- '<2kk>': 'Kazakh',
48
- '<2is>': 'Icelandic',
49
- '<2ml>': 'Malayalam',
50
- '<2mr>': 'Marathi',
51
- '<2te>': 'Telugu',
52
- '<2af>': 'Afrikaans',
53
- '<2gl>': 'Galician',
54
- '<2fil>': 'Filipino',
55
- '<2be>': 'Belarusian',
56
- # Page 17
57
- '<2mk>': 'Macedonian',
58
- '<2eu>': 'Basque',
59
- '<2bn>': 'Bengali',
60
- '<2ka>': 'Georgian',
61
- '<2mn>': 'Mongolian',
62
- '<2bs>': 'Bosnian',
63
- '<2uz>': 'Uzbek',
64
- '<2ur>': 'Urdu',
65
- '<2sw>': 'Swahili',
66
- '<2yue>': 'Cantonese',
67
- '<2ne>': 'Nepali',
68
- '<2kn>': 'Kannada',
69
- '<2kaa>': 'Kara-Kalpak',
70
- '<2gu>': 'Gujarati',
71
- '<2si>': 'Sinhala',
72
- '<2cy>': 'Welsh',
73
- '<2eo>': 'Esperanto',
74
- '<2la>': 'Latin',
75
- '<2hy>': 'Armenian',
76
- '<2ky>': 'Kyrghyz',
77
- '<2tg>': 'Tajik',
78
- '<2ga>': 'Irish',
79
- '<2mt>': 'Maltese',
80
- '<2my>': 'Myanmar (Burmese)',
81
- '<2km>': 'Khmer',
82
- '<2tt>': 'Tatar',
83
- '<2so>': 'Somali',
84
- '<2ku>': 'Kurdish (Kurmanji)',
85
- '<2ps>': 'Pashto',
86
- '<2pa>': 'Punjabi',
87
- '<2rw>': 'Kinyarwanda',
88
- '<2lo>': 'Lao',
89
- '<2ha>': 'Hausa',
90
- '<2dv>': 'Dhivehi',
91
- '<2fy>': 'W. Frisian',
92
- '<2lb>': 'Luxembourgish',
93
- '<2ckb>': 'Kurdish (Sorani)',
94
- '<2mg>': 'Malagasy',
95
- '<2gd>': 'Scottish Gaelic',
96
- '<2am>': 'Amharic',
97
- '<2ug>': 'Uyghur',
98
- '<2ht>': 'Haitian Creole',
99
- '<2grc>': 'Ancient Greek',
100
- '<2hmn>': 'Hmong',
101
- '<2sd>': 'Sindhi',
102
- '<2jv>': 'Javanese',
103
- '<2mi>': 'Maori',
104
- '<2tk>': 'Turkmen',
105
- '<2ceb>': 'Cebuano',
106
- '<2yi>': 'Yiddish',
107
- '<2ba>': 'Bashkir',
108
- '<2fo>': 'Faroese',
109
- '<2or>': 'Odia (Oriya)',
110
- '<2xh>': 'Xhosa',
111
- '<2su>': 'Sundanese',
112
- '<2kl>': 'Kalaallisut',
113
- '<2ny>': 'Chichewa',
114
- '<2sm>': 'Samoan',
115
- '<2sn>': 'Shona',
116
- '<2co>': 'Corsican',
117
- '<2zu>': 'Zulu',
118
- '<2ig>': 'Igbo',
119
- '<2yo>': 'Yoruba',
120
- '<2pap>': 'Papiamento',
121
- '<2st>': 'Sesotho',
122
- '<2haw>': 'Hawaiian',
123
- '<2as>': 'Assamese',
124
- '<2oc>': 'Occitan',
125
- '<2cv>': 'Chuvash',
126
- '<2lus>': 'Mizo',
127
- '<2tet>': 'Tetum',
128
- '<2gsw>': 'Swiss German',
129
- '<2sah>': 'Yakut',
130
- '<2br>': 'Breton',
131
- '<2rm>': 'Romansh',
132
- '<2sa>': 'Sanskrit',
133
- '<2bo>': 'Tibetan',
134
- '<2om>': 'Oromo',
135
- '<2se>': 'N. Sami',
136
- '<2ce>': 'Chechen',
137
- '<2cnh>': 'Hakha Chin',
138
- # Page 18
139
- '<2ilo>': 'Ilocano',
140
- '<2hil>': 'Hiligaynon',
141
- '<2udm>': 'Udmurt',
142
- '<2os>': 'Ossetian',
143
- '<2lg>': 'Luganda',
144
- '<2ti>': 'Tigrinya',
145
- '<2vec>': 'Venetian',
146
- '<2ts>': 'Tsonga',
147
- '<2tyv>': 'Tuvinian',
148
- '<2kbd>': 'Kabardian',
149
- '<2ee>': 'Ewe',
150
- '<2iba>': 'Iban',
151
- '<2av>': 'Avar',
152
- '<2kha>': 'Khasi',
153
- '<2to>': 'Tonga (Tonga Islands)',
154
- '<2tn>': 'Tswana',
155
- '<2nso>': 'Sepedi',
156
- '<2fj>': 'Fijian',
157
- '<2zza>': 'Zaza',
158
- '<2ak>': 'Twi',
159
- '<2ada>': 'Adangme',
160
- '<2otq>': 'Querétaro Otomi',
161
- '<2dz>': 'Dzongkha',
162
- '<2bua>': 'Buryat',
163
- '<2cfm>': 'Falam Chin',
164
- '<2ln>': 'Lingala',
165
- '<2chm>': 'Meadow Mari',
166
- '<2gn>': 'Guarani',
167
- '<2krc>': 'Karachay-Balkar',
168
- '<2wa>': 'Walloon',
169
- '<2hif>': 'Fiji Hindi',
170
- '<2yua>': 'Yucateco',
171
- '<2srn>': 'Sranan Tongo',
172
- '<2war>': 'Waray (Philippines)',
173
- '<2rom>': 'Romani',
174
- '<2bik>': 'Central Bikol',
175
- '<2pam>': 'Pampanga',
176
- '<2sg>': 'Sango',
177
- '<2lu>': 'Luba-Katanga',
178
- '<2ady>': 'Adyghe',
179
- '<2kbp>': 'Kabiyè',
180
- '<2syr>': 'Syriac',
181
- '<2ltg>': 'Latgalian',
182
- '<2myv>': 'Erzya',
183
- '<2iso>': 'Isoko',
184
- '<2kac>': 'Kachin',
185
- '<2bho>': 'Bhojpuri',
186
- '<2ay>': 'Aymara',
187
- '<2kum>': 'Kumyk',
188
- '<2qu>': 'Quechua',
189
- '<2za>': 'Zhuang',
190
- '<2pag>': 'Pangasinan',
191
- '<2ngu>': 'Guerrero Nahuatl',
192
- '<2ve>': 'Venda',
193
- '<2pck>': 'Paite Chin',
194
- '<2zap>': 'Zapotec',
195
- '<2tyz>': 'Tày',
196
- '<2hui>': 'Huli',
197
- '<2bbc>': 'Batak Toba',
198
- '<2tzo>': 'Tzotzil',
199
- '<2tiv>': 'Tiv',
200
- '<2ksd>': 'Kuanua',
201
- '<2gom>': 'Goan Konkani',
202
- '<2min>': 'Minangkabau',
203
- '<2ang>': 'Old English',
204
- '<2nhe>': 'E. Huasteca Nahuatl',
205
- '<2bgp>': 'E. Baluchi',
206
- '<2nzi>': 'Nzima',
207
- '<2nnb>': 'Nande',
208
- '<2nv>': 'Navajo',
209
- # '<2zxx>': 'Noise',
210
- '<2bci>': 'Baoulé',
211
- '<2kv>': 'Komi',
212
- '<2new>': 'Newari',
213
- '<2mps>': 'Dadibi',
214
- '<2alt>': 'S. Altai',
215
- '<2meu>': 'Motu',
216
- '<2bew>': 'Betawi',
217
- '<2fon>': 'Fon',
218
- '<2iu>': 'Inuktitut',
219
- '<2abt>': 'Ambulas',
220
- # Page 19
221
- '<2mgh>': 'Makhuwa-Meetto',
222
- '<2mnw>': 'Mon',
223
- '<2tvl>': 'Tuvalu',
224
- '<2dov>': 'Dombe',
225
- '<2tlh>': 'Klingon',
226
- '<2ho>': 'Hiri Motu',
227
- '<2kw>': 'Cornish',
228
- '<2mrj>': 'Hill Mari',
229
- '<2meo>': 'Kedah Malay',
230
- '<2crh>': 'Crimean Tatar',
231
- '<2mbt>': 'Matigsalug Manobo',
232
- '<2emp>': 'N. Emberá',
233
- '<2ace>': 'Achinese',
234
- '<2ium>': 'Iu Mien',
235
- '<2mam>': 'Mam',
236
- '<2gym>': 'Ngäbere',
237
- '<2mai>': 'Maithili',
238
- '<2crs>': 'Seselwa Creole French',
239
- '<2pon>': 'Pohnpeian',
240
- '<2ubu>': 'Umbu-Ungu',
241
- '<2fip>': 'Fipa',
242
- '<2quc>': 'K’iche’',
243
- '<2gv>': 'Manx',
244
- '<2kj>': 'Kuanyama',
245
- '<2btx>': 'Batak Karo',
246
- '<2ape>': 'Bukiyip',
247
- '<2chk>': 'Chuukese',
248
- '<2rcf>': 'Réunion Creole French',
249
- '<2shn>': 'Shan',
250
- '<2tzh>': 'Tzeltal',
251
- '<2mdf>': 'Moksha',
252
- '<2ppk>': 'Uma',
253
- '<2ss>': 'Swati',
254
- '<2gag>': 'Gagauz',
255
- '<2cab>': 'Garifuna',
256
- '<2kri>': 'Krio',
257
- '<2seh>': 'Sena',
258
- '<2ibb>': 'Ibibio',
259
- '<2tbz>': 'Ditammari',
260
- '<2bru>': 'E. Bru',
261
- '<2enq>': 'Enga',
262
- '<2ach>': 'Acoli',
263
- '<2cuk>': 'San Blas Kuna',
264
- '<2kmb>': 'Kimbundu',
265
- '<2wo>': 'Wolof',
266
- '<2kek>': 'Kekchí',
267
- '<2qub>': 'Huallaga Huánuco Quechua',
268
- '<2tab>': 'Tabassaran',
269
- '<2bts>': 'Batak Simalungun',
270
- '<2kos>': 'Kosraean',
271
- '<2rwo>': 'Rawa',
272
- '<2cak>': 'Kaqchikel',
273
- '<2tuc>': 'Mutu',
274
- '<2bum>': 'Bulu',
275
- '<2cjk>': 'Chokwe',
276
- '<2gil>': 'Gilbertese',
277
- '<2stq>': 'Saterfriesisch',
278
- '<2tsg>': 'Tausug',
279
- '<2quh>': 'S. Bolivian Quechua',
280
- '<2mak>': 'Makasar',
281
- '<2arn>': 'Mapudungun',
282
- '<2ban>': 'Balinese',
283
- '<2jiv>': 'Shuar',
284
- '<2sja>': 'Epena',
285
- '<2yap>': 'Yapese',
286
- '<2tcy>': 'Tulu',
287
- '<2toj>': 'Tojolabal',
288
- '<2twu>': 'Termanu',
289
- '<2xal>': 'Kalmyk',
290
- '<2amu>': 'Guerrero Amuzgo',
291
- '<2rmc>': 'Carpathian Romani',
292
- '<2hus>': 'Huastec',
293
- '<2nia>': 'Nias',
294
- '<2kjh>': 'Khakas',
295
- '<2bm>': 'Bambara',
296
- '<2guh>': 'Guahibo',
297
- '<2mas>': 'Masai',
298
- '<2acf>': 'St Lucian Creole French',
299
- '<2dtp>': 'Kadazan Dusun',
300
- '<2ksw>': 'S’gaw Karen',
301
- '<2bzj>': 'Belize Kriol English',
302
- # Page 20
303
- '<2din>': 'Dinka',
304
- '<2zne>': 'Zande',
305
- '<2mad>': 'Madurese',
306
- '<2msi>': 'Sabah Malay',
307
- '<2mag>': 'Magahi',
308
- '<2mkn>': 'Kupang Malay',
309
- '<2kg>': 'Kongo',
310
- '<2lhu>': 'Lahu',
311
- '<2ch>': 'Chamorro',
312
- '<2qvi>': 'Imbabura H. Quichua',
313
- '<2mh>': 'Marshallese',
314
- '<2djk>': 'E. Maroon Creole',
315
- '<2sus>': 'Susu',
316
- '<2mfe>': 'Morisien',
317
- '<2srm>': 'Saramaccan',
318
- '<2dyu>': 'Dyula',
319
- '<2ctu>': 'Chol',
320
- '<2gui>': 'E. Bolivian Guaraní',
321
- '<2pau>': 'Palauan',
322
- '<2inb>': 'Inga',
323
- '<2bi>': 'Bislama',
324
- '<2mni>': 'Meiteilon (Manipuri)',
325
- '<2guc>': 'Wayuu',
326
- '<2jam>': 'Jamaican Creole English',
327
- '<2wal>': 'Wolaytta',
328
- '<2jac>': 'Popti’',
329
- '<2bas>': 'Basa (Cameroon)',
330
- '<2gor>': 'Gorontalo',
331
- '<2skr>': 'Saraiki',
332
- '<2nyu>': 'Nyungwe',
333
- '<2noa>': 'Woun Meu',
334
- '<2sda>': 'Toraja-Sa’dan',
335
- '<2gub>': 'Guajajára',
336
- '<2nog>': 'Nogai',
337
- '<2cni>': 'Asháninka',
338
- '<2teo>': 'Teso',
339
- '<2tdx>': 'Tandroy-Mahafaly Malagasy',
340
- '<2sxn>': 'Sangir',
341
- '<2rki>': 'Rakhine',
342
- '<2nr>': 'South Ndebele',
343
- '<2frp>': 'Arpitan',
344
- '<2alz>': 'Alur',
345
- '<2taj>': 'E. Tamang',
346
- '<2lrc>': 'N. Luri',
347
- '<2cce>': 'Chopi',
348
- '<2rn>': 'Rundi',
349
- '<2jvn>': 'Caribbean Javanese',
350
- '<2hvn>': 'Sabu',
351
- '<2nij>': 'Ngaju',
352
- '<2dwr>': 'Dawro',
353
- '<2izz>': 'Izii',
354
- '<2msm>': 'Agusan Manobo',
355
- '<2bus>': 'Bokobaru',
356
- '<2ktu>': 'Kituba (DRC)',
357
- '<2chr>': 'Cherokee',
358
- '<2maz>': 'Central Mazahua',
359
- '<2tzj>': ' Tz’utujil',
360
- '<2suz>': 'Sunwar',
361
- '<2knj>': 'W. Kanjobal',
362
- '<2bim>': 'Bimoba',
363
- '<2gvl>': 'Gulay',
364
- '<2bqc>': 'Boko (Benin)',
365
- '<2tca>': 'Ticuna',
366
- '<2pis>': 'Pijin',
367
- '<2prk>': 'Parauk',
368
- '<2laj>': 'Lango (Uganda)',
369
- '<2mel>': 'Central Melanau',
370
- '<2qxr>': 'Cañar H. Quichua',
371
- '<2niq>': 'Nandi',
372
- '<2ahk>': 'Akha',
373
- '<2shp>': 'Shipibo-Conibo',
374
- '<2hne>': 'Chhattisgarhi',
375
- '<2spp>': 'Supyyire Senoufo',
376
- '<2koi>': 'Komi-Permyak',
377
- '<2krj>': 'Kinaray-A',
378
- '<2quf>': 'Lambayeque Quechua',
379
- '<2luz>': 'S. Luri',
380
- '<2agr>': 'Aguaruna',
381
- '<2tsc>': 'Tswa',
382
- '<2mqy>': 'Manggarai',
383
- '<2gof>': 'Gofa',
384
- # Page 21
385
- '<2gbm>': 'Garhwali',
386
- '<2miq>': 'Mískito',
387
- '<2dje>': 'Zarma',
388
- '<2awa>': 'Awadhi',
389
- '<2bjj>': 'Kanauji',
390
- '<2qvz>': 'N. Pastaza Quichua',
391
- '<2sjp>': 'Surjapuri',
392
- '<2tll>': 'Tetela',
393
- '<2raj>': 'Rajasthani',
394
- '<2kjg>': 'Khmu',
395
- '<2bgz>': 'Banggai',
396
- '<2quy>': 'Ayacucho Quechua',
397
- '<2cbk>': 'Chavacano',
398
- '<2akb>': 'Batak Angkola',
399
- '<2oj>': 'Ojibwa',
400
- '<2ify>': 'Keley-I Kallahan',
401
- '<2mey>': 'Hassaniyya',
402
- '<2ks>': 'Kashimiri',
403
- '<2cac>': 'Chuj',
404
- '<2brx>': 'Bodo (India)',
405
- '<2qup>': 'S. Pastaza Quechua',
406
- '<2syl>': 'Sylheti',
407
- '<2jax>': 'Jambi Malay',
408
- '<2ff>': 'Fulfulde',
409
- '<2ber>': 'Tamazight (Tfng)',
410
- '<2tks>': 'Takestani',
411
- '<2trp>': 'Kok Borok',
412
- '<2mrw>': 'Maranao',
413
- '<2adh>': 'Adhola',
414
- '<2smt>': 'Simte',
415
- '<2srr>': 'Serer',
416
- '<2ffm>': 'Maasina Fulfulde',
417
- '<2qvc>': 'Cajamarca Quechua',
418
- '<2mtr>': 'Mewari',
419
- '<2ann>': 'Obolo',
420
- '<2kaa-Latn>': 'Kara-Kalpak (Latn)',
421
- '<2aa>': 'Afar',
422
- '<2noe>': 'Nimadi',
423
- '<2nut>': 'Nung (Viet Nam)',
424
- '<2gyn>': 'Guyanese Creole English',
425
- '<2kwi>': 'Awa-Cuaiquer',
426
- '<2xmm>': 'Manado Malay',
427
- '<2msb>': 'Masbatenyo'
428
- }
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
langmap/__init__.py ADDED
@@ -0,0 +1 @@
 
 
1
+ from .langid_mapping import langid_to_language as langid_to_language
langmap/langid_mapping.py ADDED
@@ -0,0 +1,428 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ # Mapping the BCP-47 codes used in the MADLAD-400 models to the language names
2
+ # [MADLAD-400: A Multilingual And Document-Level Large Audited Dataset](https://arxiv.org/pdf/2309.04662)
3
+ langid_to_language = {
4
+ # Page 16
5
+ "<2en>": "English",
6
+ "<2ru>": "Russian",
7
+ "<2es>": "Spanish",
8
+ "<2fr>": "French",
9
+ "<2de>": "German",
10
+ "<2it>": "Italian",
11
+ "<2pt>": "Portuguese",
12
+ "<2pl>": "Polish",
13
+ "<2nl>": "Dutch",
14
+ "<2vi>": "Vietnamese",
15
+ "<2tr>": "Turkish",
16
+ "<2sv>": "Swedish",
17
+ "<2id>": "Indonesian",
18
+ "<2ro>": "Romanian",
19
+ "<2cs>": "Czech",
20
+ "<2zh>": "Mandarin Chinese",
21
+ "<2hu>": "Hungarian",
22
+ "<2ja>": "Japanese",
23
+ "<2th>": "Thai",
24
+ "<2fi>": "Finnish",
25
+ "<2fa>": "Persian",
26
+ "<2uk>": "Ukrainian",
27
+ "<2da>": "Danish",
28
+ "<2el>": "Greek",
29
+ "<2no>": "Norwegian",
30
+ "<2bg>": "Bulgarian",
31
+ "<2sk>": "Slovak",
32
+ "<2ko>": "Korean",
33
+ "<2ar>": "Arabic",
34
+ "<2lt>": "Lithuanian",
35
+ "<2ca>": "Catalan",
36
+ "<2sl>": "Slovenian",
37
+ "<2he>": "Hebrew",
38
+ "<2et>": "Estonian",
39
+ "<2lv>": "Latvian",
40
+ "<2hi>": "Hindi",
41
+ "<2sq>": "Albanian",
42
+ "<2ms>": "Malay",
43
+ "<2az>": "Azerbaijani",
44
+ "<2sr>": "Serbian",
45
+ "<2ta>": "Tamil",
46
+ "<2hr>": "Croatian",
47
+ "<2kk>": "Kazakh",
48
+ "<2is>": "Icelandic",
49
+ "<2ml>": "Malayalam",
50
+ "<2mr>": "Marathi",
51
+ "<2te>": "Telugu",
52
+ "<2af>": "Afrikaans",
53
+ "<2gl>": "Galician",
54
+ "<2fil>": "Filipino",
55
+ "<2be>": "Belarusian",
56
+ # Page 17
57
+ "<2mk>": "Macedonian",
58
+ "<2eu>": "Basque",
59
+ "<2bn>": "Bengali",
60
+ "<2ka>": "Georgian",
61
+ "<2mn>": "Mongolian",
62
+ "<2bs>": "Bosnian",
63
+ "<2uz>": "Uzbek",
64
+ "<2ur>": "Urdu",
65
+ "<2sw>": "Swahili",
66
+ "<2yue>": "Cantonese",
67
+ "<2ne>": "Nepali",
68
+ "<2kn>": "Kannada",
69
+ "<2kaa>": "Kara-Kalpak",
70
+ "<2gu>": "Gujarati",
71
+ "<2si>": "Sinhala",
72
+ "<2cy>": "Welsh",
73
+ "<2eo>": "Esperanto",
74
+ "<2la>": "Latin",
75
+ "<2hy>": "Armenian",
76
+ "<2ky>": "Kyrghyz",
77
+ "<2tg>": "Tajik",
78
+ "<2ga>": "Irish",
79
+ "<2mt>": "Maltese",
80
+ "<2my>": "Myanmar (Burmese)",
81
+ "<2km>": "Khmer",
82
+ "<2tt>": "Tatar",
83
+ "<2so>": "Somali",
84
+ "<2ku>": "Kurdish (Kurmanji)",
85
+ "<2ps>": "Pashto",
86
+ "<2pa>": "Punjabi",
87
+ "<2rw>": "Kinyarwanda",
88
+ "<2lo>": "Lao",
89
+ "<2ha>": "Hausa",
90
+ "<2dv>": "Dhivehi",
91
+ "<2fy>": "W. Frisian",
92
+ "<2lb>": "Luxembourgish",
93
+ "<2ckb>": "Kurdish (Sorani)",
94
+ "<2mg>": "Malagasy",
95
+ "<2gd>": "Scottish Gaelic",
96
+ "<2am>": "Amharic",
97
+ "<2ug>": "Uyghur",
98
+ "<2ht>": "Haitian Creole",
99
+ "<2grc>": "Ancient Greek",
100
+ "<2hmn>": "Hmong",
101
+ "<2sd>": "Sindhi",
102
+ "<2jv>": "Javanese",
103
+ "<2mi>": "Maori",
104
+ "<2tk>": "Turkmen",
105
+ "<2ceb>": "Cebuano",
106
+ "<2yi>": "Yiddish",
107
+ "<2ba>": "Bashkir",
108
+ "<2fo>": "Faroese",
109
+ "<2or>": "Odia (Oriya)",
110
+ "<2xh>": "Xhosa",
111
+ "<2su>": "Sundanese",
112
+ "<2kl>": "Kalaallisut",
113
+ "<2ny>": "Chichewa",
114
+ "<2sm>": "Samoan",
115
+ "<2sn>": "Shona",
116
+ "<2co>": "Corsican",
117
+ "<2zu>": "Zulu",
118
+ "<2ig>": "Igbo",
119
+ "<2yo>": "Yoruba",
120
+ "<2pap>": "Papiamento",
121
+ "<2st>": "Sesotho",
122
+ "<2haw>": "Hawaiian",
123
+ "<2as>": "Assamese",
124
+ "<2oc>": "Occitan",
125
+ "<2cv>": "Chuvash",
126
+ "<2lus>": "Mizo",
127
+ "<2tet>": "Tetum",
128
+ "<2gsw>": "Swiss German",
129
+ "<2sah>": "Yakut",
130
+ "<2br>": "Breton",
131
+ "<2rm>": "Romansh",
132
+ "<2sa>": "Sanskrit",
133
+ "<2bo>": "Tibetan",
134
+ "<2om>": "Oromo",
135
+ "<2se>": "N. Sami",
136
+ "<2ce>": "Chechen",
137
+ "<2cnh>": "Hakha Chin",
138
+ # Page 18
139
+ "<2ilo>": "Ilocano",
140
+ "<2hil>": "Hiligaynon",
141
+ "<2udm>": "Udmurt",
142
+ "<2os>": "Ossetian",
143
+ "<2lg>": "Luganda",
144
+ "<2ti>": "Tigrinya",
145
+ "<2vec>": "Venetian",
146
+ "<2ts>": "Tsonga",
147
+ "<2tyv>": "Tuvinian",
148
+ "<2kbd>": "Kabardian",
149
+ "<2ee>": "Ewe",
150
+ "<2iba>": "Iban",
151
+ "<2av>": "Avar",
152
+ "<2kha>": "Khasi",
153
+ "<2to>": "Tonga (Tonga Islands)",
154
+ "<2tn>": "Tswana",
155
+ "<2nso>": "Sepedi",
156
+ "<2fj>": "Fijian",
157
+ "<2zza>": "Zaza",
158
+ "<2ak>": "Twi",
159
+ "<2ada>": "Adangme",
160
+ "<2otq>": "Querétaro Otomi",
161
+ "<2dz>": "Dzongkha",
162
+ "<2bua>": "Buryat",
163
+ "<2cfm>": "Falam Chin",
164
+ "<2ln>": "Lingala",
165
+ "<2chm>": "Meadow Mari",
166
+ "<2gn>": "Guarani",
167
+ "<2krc>": "Karachay-Balkar",
168
+ "<2wa>": "Walloon",
169
+ "<2hif>": "Fiji Hindi",
170
+ "<2yua>": "Yucateco",
171
+ "<2srn>": "Sranan Tongo",
172
+ "<2war>": "Waray (Philippines)",
173
+ "<2rom>": "Romani",
174
+ "<2bik>": "Central Bikol",
175
+ "<2pam>": "Pampanga",
176
+ "<2sg>": "Sango",
177
+ "<2lu>": "Luba-Katanga",
178
+ "<2ady>": "Adyghe",
179
+ "<2kbp>": "Kabiyè",
180
+ "<2syr>": "Syriac",
181
+ "<2ltg>": "Latgalian",
182
+ "<2myv>": "Erzya",
183
+ "<2iso>": "Isoko",
184
+ "<2kac>": "Kachin",
185
+ "<2bho>": "Bhojpuri",
186
+ "<2ay>": "Aymara",
187
+ "<2kum>": "Kumyk",
188
+ "<2qu>": "Quechua",
189
+ "<2za>": "Zhuang",
190
+ "<2pag>": "Pangasinan",
191
+ "<2ngu>": "Guerrero Nahuatl",
192
+ "<2ve>": "Venda",
193
+ "<2pck>": "Paite Chin",
194
+ "<2zap>": "Zapotec",
195
+ "<2tyz>": "Tày",
196
+ "<2hui>": "Huli",
197
+ "<2bbc>": "Batak Toba",
198
+ "<2tzo>": "Tzotzil",
199
+ "<2tiv>": "Tiv",
200
+ "<2ksd>": "Kuanua",
201
+ "<2gom>": "Goan Konkani",
202
+ "<2min>": "Minangkabau",
203
+ "<2ang>": "Old English",
204
+ "<2nhe>": "E. Huasteca Nahuatl",
205
+ "<2bgp>": "E. Baluchi",
206
+ "<2nzi>": "Nzima",
207
+ "<2nnb>": "Nande",
208
+ "<2nv>": "Navajo",
209
+ # '<2zxx>': 'Noise',
210
+ "<2bci>": "Baoulé",
211
+ "<2kv>": "Komi",
212
+ "<2new>": "Newari",
213
+ "<2mps>": "Dadibi",
214
+ "<2alt>": "S. Altai",
215
+ "<2meu>": "Motu",
216
+ "<2bew>": "Betawi",
217
+ "<2fon>": "Fon",
218
+ "<2iu>": "Inuktitut",
219
+ "<2abt>": "Ambulas",
220
+ # Page 19
221
+ "<2mgh>": "Makhuwa-Meetto",
222
+ "<2mnw>": "Mon",
223
+ "<2tvl>": "Tuvalu",
224
+ "<2dov>": "Dombe",
225
+ "<2tlh>": "Klingon",
226
+ "<2ho>": "Hiri Motu",
227
+ "<2kw>": "Cornish",
228
+ "<2mrj>": "Hill Mari",
229
+ "<2meo>": "Kedah Malay",
230
+ "<2crh>": "Crimean Tatar",
231
+ "<2mbt>": "Matigsalug Manobo",
232
+ "<2emp>": "N. Emberá",
233
+ "<2ace>": "Achinese",
234
+ "<2ium>": "Iu Mien",
235
+ "<2mam>": "Mam",
236
+ "<2gym>": "Ngäbere",
237
+ "<2mai>": "Maithili",
238
+ "<2crs>": "Seselwa Creole French",
239
+ "<2pon>": "Pohnpeian",
240
+ "<2ubu>": "Umbu-Ungu",
241
+ "<2fip>": "Fipa",
242
+ "<2quc>": "K’iche’",
243
+ "<2gv>": "Manx",
244
+ "<2kj>": "Kuanyama",
245
+ "<2btx>": "Batak Karo",
246
+ "<2ape>": "Bukiyip",
247
+ "<2chk>": "Chuukese",
248
+ "<2rcf>": "Réunion Creole French",
249
+ "<2shn>": "Shan",
250
+ "<2tzh>": "Tzeltal",
251
+ "<2mdf>": "Moksha",
252
+ "<2ppk>": "Uma",
253
+ "<2ss>": "Swati",
254
+ "<2gag>": "Gagauz",
255
+ "<2cab>": "Garifuna",
256
+ "<2kri>": "Krio",
257
+ "<2seh>": "Sena",
258
+ "<2ibb>": "Ibibio",
259
+ "<2tbz>": "Ditammari",
260
+ "<2bru>": "E. Bru",
261
+ "<2enq>": "Enga",
262
+ "<2ach>": "Acoli",
263
+ "<2cuk>": "San Blas Kuna",
264
+ "<2kmb>": "Kimbundu",
265
+ "<2wo>": "Wolof",
266
+ "<2kek>": "Kekchí",
267
+ "<2qub>": "Huallaga Huánuco Quechua",
268
+ "<2tab>": "Tabassaran",
269
+ "<2bts>": "Batak Simalungun",
270
+ "<2kos>": "Kosraean",
271
+ "<2rwo>": "Rawa",
272
+ "<2cak>": "Kaqchikel",
273
+ "<2tuc>": "Mutu",
274
+ "<2bum>": "Bulu",
275
+ "<2cjk>": "Chokwe",
276
+ "<2gil>": "Gilbertese",
277
+ "<2stq>": "Saterfriesisch",
278
+ "<2tsg>": "Tausug",
279
+ "<2quh>": "S. Bolivian Quechua",
280
+ "<2mak>": "Makasar",
281
+ "<2arn>": "Mapudungun",
282
+ "<2ban>": "Balinese",
283
+ "<2jiv>": "Shuar",
284
+ "<2sja>": "Epena",
285
+ "<2yap>": "Yapese",
286
+ "<2tcy>": "Tulu",
287
+ "<2toj>": "Tojolabal",
288
+ "<2twu>": "Termanu",
289
+ "<2xal>": "Kalmyk",
290
+ "<2amu>": "Guerrero Amuzgo",
291
+ "<2rmc>": "Carpathian Romani",
292
+ "<2hus>": "Huastec",
293
+ "<2nia>": "Nias",
294
+ "<2kjh>": "Khakas",
295
+ "<2bm>": "Bambara",
296
+ "<2guh>": "Guahibo",
297
+ "<2mas>": "Masai",
298
+ "<2acf>": "St Lucian Creole French",
299
+ "<2dtp>": "Kadazan Dusun",
300
+ "<2ksw>": "S’gaw Karen",
301
+ "<2bzj>": "Belize Kriol English",
302
+ # Page 20
303
+ "<2din>": "Dinka",
304
+ "<2zne>": "Zande",
305
+ "<2mad>": "Madurese",
306
+ "<2msi>": "Sabah Malay",
307
+ "<2mag>": "Magahi",
308
+ "<2mkn>": "Kupang Malay",
309
+ "<2kg>": "Kongo",
310
+ "<2lhu>": "Lahu",
311
+ "<2ch>": "Chamorro",
312
+ "<2qvi>": "Imbabura H. Quichua",
313
+ "<2mh>": "Marshallese",
314
+ "<2djk>": "E. Maroon Creole",
315
+ "<2sus>": "Susu",
316
+ "<2mfe>": "Morisien",
317
+ "<2srm>": "Saramaccan",
318
+ "<2dyu>": "Dyula",
319
+ "<2ctu>": "Chol",
320
+ "<2gui>": "E. Bolivian Guaraní",
321
+ "<2pau>": "Palauan",
322
+ "<2inb>": "Inga",
323
+ "<2bi>": "Bislama",
324
+ "<2mni>": "Meiteilon (Manipuri)",
325
+ "<2guc>": "Wayuu",
326
+ "<2jam>": "Jamaican Creole English",
327
+ "<2wal>": "Wolaytta",
328
+ "<2jac>": "Popti’",
329
+ "<2bas>": "Basa (Cameroon)",
330
+ "<2gor>": "Gorontalo",
331
+ "<2skr>": "Saraiki",
332
+ "<2nyu>": "Nyungwe",
333
+ "<2noa>": "Woun Meu",
334
+ "<2sda>": "Toraja-Sa’dan",
335
+ "<2gub>": "Guajajára",
336
+ "<2nog>": "Nogai",
337
+ "<2cni>": "Asháninka",
338
+ "<2teo>": "Teso",
339
+ "<2tdx>": "Tandroy-Mahafaly Malagasy",
340
+ "<2sxn>": "Sangir",
341
+ "<2rki>": "Rakhine",
342
+ "<2nr>": "South Ndebele",
343
+ "<2frp>": "Arpitan",
344
+ "<2alz>": "Alur",
345
+ "<2taj>": "E. Tamang",
346
+ "<2lrc>": "N. Luri",
347
+ "<2cce>": "Chopi",
348
+ "<2rn>": "Rundi",
349
+ "<2jvn>": "Caribbean Javanese",
350
+ "<2hvn>": "Sabu",
351
+ "<2nij>": "Ngaju",
352
+ "<2dwr>": "Dawro",
353
+ "<2izz>": "Izii",
354
+ "<2msm>": "Agusan Manobo",
355
+ "<2bus>": "Bokobaru",
356
+ "<2ktu>": "Kituba (DRC)",
357
+ "<2chr>": "Cherokee",
358
+ "<2maz>": "Central Mazahua",
359
+ "<2tzj>": "Tz’utujil",
360
+ "<2suz>": "Sunwar",
361
+ "<2knj>": "W. Kanjobal",
362
+ "<2bim>": "Bimoba",
363
+ "<2gvl>": "Gulay",
364
+ "<2bqc>": "Boko (Benin)",
365
+ "<2tca>": "Ticuna",
366
+ "<2pis>": "Pijin",
367
+ "<2prk>": "Parauk",
368
+ "<2laj>": "Lango (Uganda)",
369
+ "<2mel>": "Central Melanau",
370
+ "<2qxr>": "Cañar H. Quichua",
371
+ "<2niq>": "Nandi",
372
+ "<2ahk>": "Akha",
373
+ "<2shp>": "Shipibo-Conibo",
374
+ "<2hne>": "Chhattisgarhi",
375
+ "<2spp>": "Supyyire Senoufo",
376
+ "<2koi>": "Komi-Permyak",
377
+ "<2krj>": "Kinaray-A",
378
+ "<2quf>": "Lambayeque Quechua",
379
+ "<2luz>": "S. Luri",
380
+ "<2agr>": "Aguaruna",
381
+ "<2tsc>": "Tswa",
382
+ "<2mqy>": "Manggarai",
383
+ "<2gof>": "Gofa",
384
+ # Page 21
385
+ "<2gbm>": "Garhwali",
386
+ "<2miq>": "Mískito",
387
+ "<2dje>": "Zarma",
388
+ "<2awa>": "Awadhi",
389
+ "<2bjj>": "Kanauji",
390
+ "<2qvz>": "N. Pastaza Quichua",
391
+ "<2sjp>": "Surjapuri",
392
+ "<2tll>": "Tetela",
393
+ "<2raj>": "Rajasthani",
394
+ "<2kjg>": "Khmu",
395
+ "<2bgz>": "Banggai",
396
+ "<2quy>": "Ayacucho Quechua",
397
+ "<2cbk>": "Chavacano",
398
+ "<2akb>": "Batak Angkola",
399
+ "<2oj>": "Ojibwa",
400
+ "<2ify>": "Keley-I Kallahan",
401
+ "<2mey>": "Hassaniyya",
402
+ "<2ks>": "Kashimiri",
403
+ "<2cac>": "Chuj",
404
+ "<2brx>": "Bodo (India)",
405
+ "<2qup>": "S. Pastaza Quechua",
406
+ "<2syl>": "Sylheti",
407
+ "<2jax>": "Jambi Malay",
408
+ "<2ff>": "Fulfulde",
409
+ "<2ber>": "Tamazight (Tfng)",
410
+ "<2tks>": "Takestani",
411
+ "<2trp>": "Kok Borok",
412
+ "<2mrw>": "Maranao",
413
+ "<2adh>": "Adhola",
414
+ "<2smt>": "Simte",
415
+ "<2srr>": "Serer",
416
+ "<2ffm>": "Maasina Fulfulde",
417
+ "<2qvc>": "Cajamarca Quechua",
418
+ "<2mtr>": "Mewari",
419
+ "<2ann>": "Obolo",
420
+ "<2kaa-Latn>": "Kara-Kalpak (Latn)",
421
+ "<2aa>": "Afar",
422
+ "<2noe>": "Nimadi",
423
+ "<2nut>": "Nung (Viet Nam)",
424
+ "<2gyn>": "Guyanese Creole English",
425
+ "<2kwi>": "Awa-Cuaiquer",
426
+ "<2xmm>": "Manado Malay",
427
+ "<2msb>": "Masbatenyo",
428
+ }
langmap/py.typed ADDED
File without changes