feat: Initial release of EthioBBPE - Ethiopian Language Tokenizer

d681ea8 verified 3 months ago

26.8 kB

	{
	"version": "1.0",
	"truncation": null,
	"padding": null,
	"added_tokens": [
	{
	"id": 0,
	"content": "<pad>",
	"single_word": false,
	"lstrip": false,
	"rstrip": false,
	"normalized": false,
	"special": true
	},
	{
	"id": 1,
	"content": "<unk>",
	"single_word": false,
	"lstrip": false,
	"rstrip": false,
	"normalized": false,
	"special": true
	},
	{
	"id": 2,
	"content": "<s>",
	"single_word": false,
	"lstrip": false,
	"rstrip": false,
	"normalized": false,
	"special": true
	},
	{
	"id": 3,
	"content": "</s>",
	"single_word": false,
	"lstrip": false,
	"rstrip": false,
	"normalized": false,
	"special": true
	},
	{
	"id": 4,
	"content": "<mask>",
	"single_word": false,
	"lstrip": false,
	"rstrip": false,
	"normalized": false,
	"special": true
	}
	],
	"normalizer": null,
	"pre_tokenizer": {
	"type": "ByteLevel",
	"add_prefix_space": false,
	"trim_offsets": true,
	"use_regex": true
	},
	"post_processor": {
	"type": "ByteLevel",
	"add_prefix_space": true,
	"trim_offsets": true,
	"use_regex": true
	},
	"decoder": {
	"type": "ByteLevel",
	"add_prefix_space": true,
	"trim_offsets": true,
	"use_regex": true
	},
	"model": {
	"type": "BPE",
	"dropout": null,
	"unk_token": null,
	"continuing_subword_prefix": null,
	"end_of_word_suffix": null,
	"fuse_unk": false,
	"byte_fallback": false,
	"ignore_merges": false,
	"vocab": {
	"<pad>": 0,
	"<unk>": 1,
	"<s>": 2,
	"</s>": 3,
	"<mask>": 4,
	"!": 5,
	"\"": 6,
	"#": 7,
	"$": 8,
	"%": 9,
	"&": 10,
	"'": 11,
	"(": 12,
	")": 13,
	"*": 14,
	"+": 15,
	",": 16,
	"-": 17,
	".": 18,
	"/": 19,
	"0": 20,
	"1": 21,
	"2": 22,
	"3": 23,
	"4": 24,
	"5": 25,
	"6": 26,
	"7": 27,
	"8": 28,
	"9": 29,
	":": 30,
	";": 31,
	"<": 32,
	"=": 33,
	">": 34,
	"?": 35,
	"@": 36,
	"A": 37,
	"B": 38,
	"C": 39,
	"D": 40,
	"E": 41,
	"F": 42,
	"G": 43,
	"H": 44,
	"I": 45,
	"J": 46,
	"K": 47,
	"L": 48,
	"M": 49,
	"N": 50,
	"O": 51,
	"P": 52,
	"Q": 53,
	"R": 54,
	"S": 55,
	"T": 56,
	"U": 57,
	"V": 58,
	"W": 59,
	"X": 60,
	"Y": 61,
	"Z": 62,
	"[": 63,
	"\\": 64,
	"]": 65,
	"^": 66,
	"_": 67,
	"`": 68,
	"a": 69,
	"b": 70,
	"c": 71,
	"d": 72,
	"e": 73,
	"f": 74,
	"g": 75,
	"h": 76,
	"i": 77,
	"j": 78,
	"k": 79,
	"l": 80,
	"m": 81,
	"n": 82,
	"o": 83,
	"p": 84,
	"q": 85,
	"r": 86,
	"s": 87,
	"t": 88,
	"u": 89,
	"v": 90,
	"w": 91,
	"x": 92,
	"y": 93,
	"z": 94,
	"{": 95,
	"\|": 96,
	"}": 97,
	"~": 98,
	"¡": 99,
	"¢": 100,
	"£": 101,
	"¤": 102,
	"¥": 103,
	"¦": 104,
	"§": 105,
	"¨": 106,
	"©": 107,
	"ª": 108,
	"«": 109,
	"¬": 110,
	"®": 111,
	"¯": 112,
	"°": 113,
	"±": 114,
	"²": 115,
	"³": 116,
	"´": 117,
	"µ": 118,
	"¶": 119,
	"·": 120,
	"¸": 121,
	"¹": 122,
	"º": 123,
	"»": 124,
	"¼": 125,
	"½": 126,
	"¾": 127,
	"¿": 128,
	"À": 129,
	"Á": 130,
	"Â": 131,
	"Ã": 132,
	"Ä": 133,
	"Å": 134,
	"Æ": 135,
	"Ç": 136,
	"È": 137,
	"É": 138,
	"Ê": 139,
	"Ë": 140,
	"Ì": 141,
	"Í": 142,
	"Î": 143,
	"Ï": 144,
	"Ð": 145,
	"Ñ": 146,
	"Ò": 147,
	"Ó": 148,
	"Ô": 149,
	"Õ": 150,
	"Ö": 151,
	"×": 152,
	"Ø": 153,
	"Ù": 154,
	"Ú": 155,
	"Û": 156,
	"Ü": 157,
	"Ý": 158,
	"Þ": 159,
	"ß": 160,
	"à": 161,
	"á": 162,
	"â": 163,
	"ã": 164,
	"ä": 165,
	"å": 166,
	"æ": 167,
	"ç": 168,
	"è": 169,
	"é": 170,
	"ê": 171,
	"ë": 172,
	"ì": 173,
	"í": 174,
	"î": 175,
	"ï": 176,
	"ð": 177,
	"ñ": 178,
	"ò": 179,
	"ó": 180,
	"ô": 181,
	"õ": 182,
	"ö": 183,
	"÷": 184,
	"ø": 185,
	"ù": 186,
	"ú": 187,
	"û": 188,
	"ü": 189,
	"ý": 190,
	"þ": 191,
	"ÿ": 192,
	"Ā": 193,
	"ā": 194,
	"Ă": 195,
	"ă": 196,
	"Ą": 197,
	"ą": 198,
	"Ć": 199,
	"ć": 200,
	"Ĉ": 201,
	"ĉ": 202,
	"Ċ": 203,
	"ċ": 204,
	"Č": 205,
	"č": 206,
	"Ď": 207,
	"ď": 208,
	"Đ": 209,
	"đ": 210,
	"Ē": 211,
	"ē": 212,
	"Ĕ": 213,
	"ĕ": 214,
	"Ė": 215,
	"ė": 216,
	"Ę": 217,
	"ę": 218,
	"Ě": 219,
	"ě": 220,
	"Ĝ": 221,
	"ĝ": 222,
	"Ğ": 223,
	"ğ": 224,
	"Ġ": 225,
	"ġ": 226,
	"Ģ": 227,
	"ģ": 228,
	"Ĥ": 229,
	"ĥ": 230,
	"Ħ": 231,
	"ħ": 232,
	"Ĩ": 233,
	"ĩ": 234,
	"Ī": 235,
	"ī": 236,
	"Ĭ": 237,
	"ĭ": 238,
	"Į": 239,
	"į": 240,
	"İ": 241,
	"ı": 242,
	"Ĳ": 243,
	"ĳ": 244,
	"Ĵ": 245,
	"ĵ": 246,
	"Ķ": 247,
	"ķ": 248,
	"ĸ": 249,
	"Ĺ": 250,
	"ĺ": 251,
	"Ļ": 252,
	"ļ": 253,
	"Ľ": 254,
	"ľ": 255,
	"Ŀ": 256,
	"ŀ": 257,
	"Ł": 258,
	"ł": 259,
	"Ń": 260,
	"Ġt": 261,
	"in": 262,
	"en": 263,
	"Ġa": 264,
	"er": 265,
	"es": 266,
	"Ġo": 267,
	"Ġto": 268,
	"an": 269,
	"ing": 270,
	"ar": 271,
	"at": 272,
	"he": 273,
	"is": 274,
	"or": 275,
	"Ġm": 276,
	"Ġs": 277,
	"al": 278,
	"ken": 279,
	"on": 280,
	"Ġtoken": 281,
	"le": 282,
	"Ġof": 283,
	"Ġb": 284,
	"Ġp": 285,
	"it": 286,
	"ou": 287,
	"Ġc": 288,
	"Ġf": 289,
	"Ġan": 290,
	"ic": 291,
	"iz": 292,
	"ion": 293,
	"ro": 294,
	"ed": 295,
	"el": 296,
	"re": 297,
	"Ġin": 298,
	"Ġthe": 299,
	"Ġis": 300,
	"Ġand": 301,
	"ĠT": 302,
	"Ġu": 303,
	"Ġbe": 304,
	"Ġd": 305,
	"Ġth": 306,
	"ac": 307,
	"mp": 308,
	"od": 309,
	"Ġw": 310,
	"Ġle": 311,
	"enc": 312,
	"ent": 313,
	"Ġtokeniz": 314,
	"as": 315,
	"ex": 316,
	"ter": 317,
	"Ġl": 318,
	"arn": 319,
	"Ġtokens": 320,
	"Ġpro": 321,
	"qu": 322,
	"ul": 323,
	"Ġh": 324,
	"Ġv": 325,
	"ation": 326,
	"Ġthat": 327,
	"Ġlearn": 328,
	"ain": 329,
	"ces": 330,
	"ec": 331,
	"fic": 332,
	"ge": 333,
	"il": 334,
	"res": 335,
	"ua": 336,
	"you": 337,
	"Ġn": 338,
	"Ġre": 339,
	"Ġyou": 340,
	"Ġtr": 341,
	"Ġtex": 342,
	"Ġun": 343,
	"cess": 344,
	"Ġtext": 345,
	"00": 346,
	"am": 347,
	"ds": 348,
	"em": 349,
	"et": 350,
	"gua": 351,
	"ific": 352,
	"oc": 353,
	"per": 354,
	"st": 355,
	"ur": 356,
	"ve": 357,
	"Ġon": 358,
	"angua": 359,
	"ords": 360,
	"Ġmod": 361,
	"ence": 362,
	"Ġtokenizer": 363,
	"ular": 364,
	"Ġlearning": 365,
	"aining": 366,
	"ab": 367,
	"ch": 368,
	"for": 369,
	"hin": 370,
	"his": 371,
	"ial": 372,
	"imp": 373,
	"ks": 374,
	"se": 375,
	"su": 376,
	"us": 377,
	"ĠB": 378,
	"Ġ[": 379,
	"Ġimp": 380,
	"Ġar": 381,
	"Ġor": 382,
	"ans": 383,
	"ate": 384,
	"ith": 385,
	"Ġcan": 386,
	"Ġfor": 387,
	"achin": 388,
	"Ġwords": 389,
	"Ġlangua": 390,
	"Ġprocess": 391,
	"Ġyour": 392,
	"ural": 393,
	"Ġmodel": 394,
	"achine": 395,
	"LP": 396,
	"NLP": 397,
	"PE": 398,
	"The": 399,
	"],": 400,
	"ad": 401,
	"ded": 402,
	"ev": 403,
	"gh": 404,
	"id": 405,
	"im": 406,
	"ld": 407,
	"lp": 408,
	"lu": 409,
	"ly": 410,
	"pl": 411,
	"te": 412,
	"tion": 413,
	"tific": 414,
	"um": 415,
	"vis": 416,
	"wn": 417,
	"yte": 418,
	"Ġen": 419,
	"Ġhe": 420,
	"Ġit": 421,
	"Ġqu": 422,
	"art": 423,
	"ata": 424,
	"orp": 425,
	"Ġmu": 426,
	"Ġman": 427,
	"Ġsu": 428,
	"ont": 429,
	"ity": 430,
	"out": 431,
	"Ġcorp": 432,
	"Ġcont": 433,
	"ions": 434,
	"ell": 435,
	"Ġint": 436,
	"ĠThe": 437,
	"Ġdata": 438,
	"Ġwith": 439,
	"Ġval": 440,
	"Ġvoc": 441,
	"Ġtraining": 442,
	"Ġtrans": 443,
	"pervis": 444,
	"abular": 445,
	"form": 446,
	"Ġartific": 447,
	"Ġlanguage": 448,
	"Ġmodels": 449,
	"Ġvocabular": 450,
	"pervised": 451,
	"Ġartificial": 452,
	"BPE": 453,
	"Byte": 454,
	"Machine": 455,
	"This": 456,
	"ak": 457,
	"amp": 458,
	"ave": 459,
	"clu": 460,
	"ding": 461,
	"dded": 462,
	"een": 463,
	"equ": 464,
	"eural": 465,
	"gr": 466,
	"gence": 467,
	"her": 468,
	"hes": 469,
	"ies": 470,
	"ive": 471,
	"igence": 472,
	"kn": 473,
	"lions": 474,
	"ms": 475,
	"mal": 476,
	"om": 477,
	"oo": 478,
	"op": 479,
	"omp": 480,
	"own": 481,
	"pp": 482,
	"pic": 483,
	"pec": 484,
	"pres": 485,
	"ts": 486,
	"tw": 487,
	"ten": 488,
	"tan": 489,
	"ting": 490,
	"ut": 491,
	"uter": 492,
	"ver": 493,
	"vel": 494,
	"vid": 495,
	"wor": 496,
	"ypic": 497,
	"Ġ(": 498,
	"Ġ2": 499,
	"ĠI": 500,
	"ĠL": 501,
	"ĠP": 502,
	"Ġr": 503,
	"Ġex": 504,
	"ĠNLP": 505,
	"Ġtas": 506,
	"Ġal": 507,
	"Ġat": 508,
	"Ġare": 509,
	"Ġadded": 510,
	"and": 511,
	"atural": 512,
	"ortan": 513,
	"Ġmachine": 514,
	"Ġmak": 515,
	"Ġsiz": 516,
	"Ġsent": 517,
	"Ġsamp": 518,
	"Ġsequ": 519,
	"Ġsmal": 520,
	"alle": 521,
	"level": 522,
	"Ġoften": 523,
	"Ġbro": 524,
	"Ġpart": 525,
	"ous": 526,
	"ough": 527,
	"Ġcomp": 528,
	"Ġfro": 529,
	"Ġfam": 530,
	"Ġfoc": 531,
	"Ġinclu": 532,
	"ĠThis": 533,
	"ĠThes": 534,
	"ĠToo": 535,
	"Ġus": 536,
	"Ġuse": 537,
	"Ġbetw": 538,
	"Ġdi": 539,
	"oding": 540,
	"ences": 541,
	"Ġtokenization": 542,
	"ased": 543,
	"Ġlar": 544,
	"Ġprogr": 545,
	"Ġprovid": 546,
	"Ġhum": 547,
	"Ġhave": 548,
	"ect": 549,
	"illions": 550,
	"Ġnet": 551,
	"Ġneural": 552,
	"Ġrepres": 553,
	"Ġunkn": 554,
	"000": 555,
	"supervised": 556,
	"uses": 557,
	"ĠBBPE": 558,
	"Ġimportan": 559,
	"plic": 560,
	"Ġensu": 561,
	"Ġhelp": 562,
	"Ġmuch": 563,
	"Ġmany": 564,
	"Ġsub": 565,
	"Ġcorpus": 566,
	"elligence": 567,
	"Ġintelligence": 568,
	"Ġwithout": 569,
	"Ġtransform": 570,
	"Ġvocabulary": 571,
	"works": 572,
	"ypical": 573,
	"Ġtasks": 574,
	"Ġsample": 575,
	"Ġsmall": 576,
	"Ġcomputer": 577,
	"Ġfrom": 578,
	"Ġfocuses": 579,
	"ĠThese": 580,
	"Ġbetween": 581,
	"Ġprogram": 582,
	"Ġprovides": 583,
	"Ġnetworks": 584,
	"Ġrepresent": 585,
	"Ġunknown": 586,
	"Ġimportant": 587,
	"Ġtransformer": 588
	},
	"merges": [
	[
	"Ġ",
	"t"
	],
	[
	"i",
	"n"
	],
	[
	"e",
	"n"
	],
	[
	"Ġ",
	"a"
	],
	[
	"e",
	"r"
	],
	[
	"e",
	"s"
	],
	[
	"Ġ",
	"o"
	],
	[
	"Ġt",
	"o"
	],
	[
	"a",
	"n"
	],
	[
	"in",
	"g"
	],
	[
	"a",
	"r"
	],
	[
	"a",
	"t"
	],
	[
	"h",
	"e"
	],
	[
	"i",
	"s"
	],
	[
	"o",
	"r"
	],
	[
	"Ġ",
	"m"
	],
	[
	"Ġ",
	"s"
	],
	[
	"a",
	"l"
	],
	[
	"k",
	"en"
	],
	[
	"o",
	"n"
	],
	[
	"Ġto",
	"ken"
	],
	[
	"l",
	"e"
	],
	[
	"Ġo",
	"f"
	],
	[
	"Ġ",
	"b"
	],
	[
	"Ġ",
	"p"
	],
	[
	"i",
	"t"
	],
	[
	"o",
	"u"
	],
	[
	"Ġ",
	"c"
	],
	[
	"Ġ",
	"f"
	],
	[
	"Ġa",
	"n"
	],
	[
	"i",
	"c"
	],
	[
	"i",
	"z"
	],
	[
	"i",
	"on"
	],
	[
	"r",
	"o"
	],
	[
	"e",
	"d"
	],
	[
	"e",
	"l"
	],
	[
	"r",
	"e"
	],
	[
	"Ġ",
	"in"
	],
	[
	"Ġt",
	"he"
	],
	[
	"Ġ",
	"is"
	],
	[
	"Ġan",
	"d"
	],
	[
	"Ġ",
	"T"
	],
	[
	"Ġ",
	"u"
	],
	[
	"Ġb",
	"e"
	],
	[
	"Ġ",
	"d"
	],
	[
	"Ġt",
	"h"
	],
	[
	"a",
	"c"
	],
	[
	"m",
	"p"
	],
	[
	"o",
	"d"
	],
	[
	"Ġ",
	"w"
	],
	[
	"Ġ",
	"le"
	],
	[
	"en",
	"c"
	],
	[
	"en",
	"t"
	],
	[
	"Ġtoken",
	"iz"
	],
	[
	"a",
	"s"
	],
	[
	"e",
	"x"
	],
	[
	"t",
	"er"
	],
	[
	"Ġ",
	"l"
	],
	[
	"ar",
	"n"
	],
	[
	"Ġtoken",
	"s"
	],
	[
	"Ġp",
	"ro"
	],
	[
	"q",
	"u"
	],
	[
	"u",
	"l"
	],
	[
	"Ġ",
	"h"
	],
	[
	"Ġ",
	"v"
	],
	[
	"at",
	"ion"
	],
	[
	"Ġth",
	"at"
	],
	[
	"Ġle",
	"arn"
	],
	[
	"a",
	"in"
	],
	[
	"c",
	"es"
	],
	[
	"e",
	"c"
	],
	[
	"f",
	"ic"
	],
	[
	"g",
	"e"
	],
	[
	"i",
	"l"
	],
	[
	"r",
	"es"
	],
	[
	"u",
	"a"
	],
	[
	"y",
	"ou"
	],
	[
	"Ġ",
	"n"
	],
	[
	"Ġ",
	"re"
	],
	[
	"Ġ",
	"you"
	],
	[
	"Ġt",
	"r"
	],
	[
	"Ġt",
	"ex"
	],
	[
	"Ġu",
	"n"
	],
	[
	"ces",
	"s"
	],
	[
	"Ġtex",
	"t"
	],
	[
	"0",
	"0"
	],
	[
	"a",
	"m"
	],
	[
	"d",
	"s"
	],
	[
	"e",
	"m"
	],
	[
	"e",
	"t"
	],
	[
	"g",
	"ua"
	],
	[
	"i",
	"fic"
	],
	[
	"o",
	"c"
	],
	[
	"p",
	"er"
	],
	[
	"s",
	"t"
	],
	[
	"u",
	"r"
	],
	[
	"v",
	"e"
	],
	[
	"Ġo",
	"n"
	],
	[
	"an",
	"gua"
	],
	[
	"or",
	"ds"
	],
	[
	"Ġm",
	"od"
	],
	[
	"enc",
	"e"
	],
	[
	"Ġtokeniz",
	"er"
	],
	[
	"ul",
	"ar"
	],
	[
	"Ġlearn",
	"ing"
	],
	[
	"ain",
	"ing"
	],
	[
	"a",
	"b"
	],
	[
	"c",
	"h"
	],
	[
	"f",
	"or"
	],
	[
	"h",
	"in"
	],
	[
	"h",
	"is"
	],
	[
	"i",
	"al"
	],
	[
	"i",
	"mp"
	],
	[
	"k",
	"s"
	],
	[
	"s",
	"e"
	],
	[
	"s",
	"u"
	],
	[
	"u",
	"s"
	],
	[
	"Ġ",
	"B"
	],
	[
	"Ġ",
	"["
	],
	[
	"Ġ",
	"imp"
	],
	[
	"Ġa",
	"r"
	],
	[
	"Ġo",
	"r"
	],
	[
	"an",
	"s"
	],
	[
	"at",
	"e"
	],
	[
	"it",
	"h"
	],
	[
	"Ġc",
	"an"
	],
	[
	"Ġf",
	"or"
	],
	[
	"ac",
	"hin"
	],
	[
	"Ġw",
	"ords"
	],
	[
	"Ġl",
	"angua"
	],
	[
	"Ġpro",
	"cess"
	],
	[
	"Ġyou",
	"r"
	],
	[
	"ur",
	"al"
	],
	[
	"Ġmod",
	"el"
	],
	[
	"achin",
	"e"
	],
	[
	"L",
	"P"
	],
	[
	"N",
	"LP"
	],
	[
	"P",
	"E"
	],
	[
	"T",
	"he"
	],
	[
	"]",
	","
	],
	[
	"a",
	"d"
	],
	[
	"d",
	"ed"
	],
	[
	"e",
	"v"
	],
	[
	"g",
	"h"
	],
	[
	"i",
	"d"
	],
	[
	"i",
	"m"
	],
	[
	"l",
	"d"
	],
	[
	"l",
	"p"
	],
	[
	"l",
	"u"
	],
	[
	"l",
	"y"
	],
	[
	"p",
	"l"
	],
	[
	"t",
	"e"
	],
	[
	"t",
	"ion"
	],
	[
	"t",
	"ific"
	],
	[
	"u",
	"m"
	],
	[
	"v",
	"is"
	],
	[
	"w",
	"n"
	],
	[
	"y",
	"te"
	],
	[
	"Ġ",
	"en"
	],
	[
	"Ġ",
	"he"
	],
	[
	"Ġ",
	"it"
	],
	[
	"Ġ",
	"qu"
	],
	[
	"ar",
	"t"
	],
	[
	"at",
	"a"
	],
	[
	"or",
	"p"
	],
	[
	"Ġm",
	"u"
	],
	[
	"Ġm",
	"an"
	],
	[
	"Ġs",
	"u"
	],
	[
	"on",
	"t"
	],
	[
	"it",
	"y"
	],
	[
	"ou",
	"t"
	],
	[
	"Ġc",
	"orp"
	],
	[
	"Ġc",
	"ont"
	],
	[
	"ion",
	"s"
	],
	[
	"el",
	"l"
	],
	[
	"Ġin",
	"t"
	],
	[
	"ĠT",
	"he"
	],
	[
	"Ġd",
	"ata"
	],
	[
	"Ġw",
	"ith"
	],
	[
	"Ġv",
	"al"
	],
	[
	"Ġv",
	"oc"
	],
	[
	"Ġtr",
	"aining"
	],
	[
	"Ġtr",
	"ans"
	],
	[
	"per",
	"vis"
	],
	[
	"ab",
	"ular"
	],
	[
	"for",
	"m"
	],
	[
	"Ġar",
	"tific"
	],
	[
	"Ġlangua",
	"ge"
	],
	[
	"Ġmodel",
	"s"
	],
	[
	"Ġvoc",
	"abular"
	],
	[
	"pervis",
	"ed"
	],
	[
	"Ġartific",
	"ial"
	],
	[
	"B",
	"PE"
	],
	[
	"B",
	"yte"
	],
	[
	"M",
	"achine"
	],
	[
	"T",
	"his"
	],
	[
	"a",
	"k"
	],
	[
	"a",
	"mp"
	],
	[
	"a",
	"ve"
	],
	[
	"c",
	"lu"
	],
	[
	"d",
	"ing"
	],
	[
	"d",
	"ded"
	],
	[
	"e",
	"en"
	],
	[
	"e",
	"qu"
	],
	[
	"e",
	"ural"
	],
	[
	"g",
	"r"
	],
	[
	"g",
	"ence"
	],
	[
	"h",
	"er"
	],
	[
	"h",
	"es"
	],
	[
	"i",
	"es"
	],
	[
	"i",
	"ve"
	],
	[
	"i",
	"gence"
	],
	[
	"k",
	"n"
	],
	[
	"l",
	"ions"
	],
	[
	"m",
	"s"
	],
	[
	"m",
	"al"
	],
	[
	"o",
	"m"
	],
	[
	"o",
	"o"
	],
	[
	"o",
	"p"
	],
	[
	"o",
	"mp"
	],
	[
	"o",
	"wn"
	],
	[
	"p",
	"p"
	],
	[
	"p",
	"ic"
	],
	[
	"p",
	"ec"
	],
	[
	"p",
	"res"
	],
	[
	"t",
	"s"
	],
	[
	"t",
	"w"
	],
	[
	"t",
	"en"
	],
	[
	"t",
	"an"
	],
	[
	"t",
	"ing"
	],
	[
	"u",
	"t"
	],
	[
	"u",
	"ter"
	],
	[
	"v",
	"er"
	],
	[
	"v",
	"el"
	],
	[
	"v",
	"id"
	],
	[
	"w",
	"or"
	],
	[
	"y",
	"pic"
	],
	[
	"Ġ",
	"("
	],
	[
	"Ġ",
	"2"
	],
	[
	"Ġ",
	"I"
	],
	[
	"Ġ",
	"L"
	],
	[
	"Ġ",
	"P"
	],
	[
	"Ġ",
	"r"
	],
	[
	"Ġ",
	"ex"
	],
	[
	"Ġ",
	"NLP"
	],
	[
	"Ġt",
	"as"
	],
	[
	"Ġa",
	"l"
	],
	[
	"Ġa",
	"t"
	],
	[
	"Ġa",
	"re"
	],
	[
	"Ġa",
	"dded"
	],
	[
	"an",
	"d"
	],
	[
	"at",
	"ural"
	],
	[
	"or",
	"tan"
	],
	[
	"Ġm",
	"achine"
	],
	[
	"Ġm",
	"ak"
	],
	[
	"Ġs",
	"iz"
	],
	[
	"Ġs",
	"ent"
	],
	[
	"Ġs",
	"amp"
	],
	[
	"Ġs",
	"equ"
	],
	[
	"Ġs",
	"mal"
	],
	[
	"al",
	"le"
	],
	[
	"le",
	"vel"
	],
	[
	"Ġof",
	"ten"
	],
	[
	"Ġb",
	"ro"
	],
	[
	"Ġp",
	"art"
	],
	[
	"ou",
	"s"
	],
	[
	"ou",
	"gh"
	],
	[
	"Ġc",
	"omp"
	],
	[
	"Ġf",
	"ro"
	],
	[
	"Ġf",
	"am"
	],
	[
	"Ġf",
	"oc"
	],
	[
	"Ġin",
	"clu"
	],
	[
	"ĠT",
	"his"
	],
	[
	"ĠT",
	"hes"
	],
	[
	"ĠT",
	"oo"
	],
	[
	"Ġu",
	"s"
	],
	[
	"Ġu",
	"se"
	],
	[
	"Ġbe",
	"tw"
	],
	[
	"Ġd",
	"i"
	],
	[
	"od",
	"ing"
	],
	[
	"enc",
	"es"
	],
	[
	"Ġtokeniz",
	"ation"
	],
	[
	"as",
	"ed"
	],
	[
	"Ġl",
	"ar"
	],
	[
	"Ġpro",
	"gr"
	],
	[
	"Ġpro",
	"vid"
	],
	[
	"Ġh",
	"um"
	],
	[
	"Ġh",
	"ave"
	],
	[
	"ec",
	"t"
	],
	[
	"il",
	"lions"
	],
	[
	"Ġn",
	"et"
	],
	[
	"Ġn",
	"eural"
	],
	[
	"Ġre",
	"pres"
	],
	[
	"Ġun",
	"kn"
	],
	[
	"00",
	"0"
	],
	[
	"su",
	"pervised"
	],
	[
	"us",
	"es"
	],
	[
	"ĠB",
	"BPE"
	],
	[
	"Ġimp",
	"ortan"
	],
	[
	"pl",
	"ic"
	],
	[
	"Ġen",
	"su"
	],
	[
	"Ġhe",
	"lp"
	],
	[
	"Ġmu",
	"ch"
	],
	[
	"Ġman",
	"y"
	],
	[
	"Ġsu",
	"b"
	],
	[
	"Ġcorp",
	"us"
	],
	[
	"ell",
	"igence"
	],
	[
	"Ġint",
	"elligence"
	],
	[
	"Ġwith",
	"out"
	],
	[
	"Ġtrans",
	"form"
	],
	[
	"Ġvocabular",
	"y"
	],
	[
	"wor",
	"ks"
	],
	[
	"ypic",
	"al"
	],
	[
	"Ġtas",
	"ks"
	],
	[
	"Ġsamp",
	"le"
	],
	[
	"Ġsmal",
	"l"
	],
	[
	"Ġcomp",
	"uter"
	],
	[
	"Ġfro",
	"m"
	],
	[
	"Ġfoc",
	"uses"
	],
	[
	"ĠThes",
	"e"
	],
	[
	"Ġbetw",
	"een"
	],
	[
	"Ġprogr",
	"am"
	],
	[
	"Ġprovid",
	"es"
	],
	[
	"Ġnet",
	"works"
	],
	[
	"Ġrepres",
	"ent"
	],
	[
	"Ġunkn",
	"own"
	],
	[
	"Ġimportan",
	"t"
	],
	[
	"Ġtransform",
	"er"
	]
	]
	}
	}