{ "version": "1.0", "truncation": null, "padding": null, "added_tokens": [ { "id": 0, "content": "", "single_word": false, "lstrip": false, "rstrip": false, "normalized": false, "special": true }, { "id": 1, "content": "", "single_word": false, "lstrip": false, "rstrip": false, "normalized": false, "special": true }, { "id": 2, "content": "", "single_word": false, "lstrip": false, "rstrip": false, "normalized": false, "special": true }, { "id": 3, "content": "", "single_word": false, "lstrip": false, "rstrip": false, "normalized": false, "special": true }, { "id": 4, "content": "", "single_word": false, "lstrip": false, "rstrip": false, "normalized": false, "special": true } ], "normalizer": null, "pre_tokenizer": { "type": "ByteLevel", "add_prefix_space": false, "trim_offsets": true, "use_regex": true }, "post_processor": { "type": "ByteLevel", "add_prefix_space": true, "trim_offsets": true, "use_regex": true }, "decoder": { "type": "ByteLevel", "add_prefix_space": true, "trim_offsets": true, "use_regex": true }, "model": { "type": "BPE", "dropout": null, "unk_token": null, "continuing_subword_prefix": null, "end_of_word_suffix": null, "fuse_unk": false, "byte_fallback": false, "ignore_merges": false, "vocab": { "": 0, "": 1, "": 2, "": 3, "": 4, "!": 5, "\"": 6, "#": 7, "$": 8, "%": 9, "&": 10, "'": 11, "(": 12, ")": 13, "*": 14, "+": 15, ",": 16, "-": 17, ".": 18, "/": 19, "0": 20, "1": 21, "2": 22, "3": 23, "4": 24, "5": 25, "6": 26, "7": 27, "8": 28, "9": 29, ":": 30, ";": 31, "<": 32, "=": 33, ">": 34, "?": 35, "@": 36, "A": 37, "B": 38, "C": 39, "D": 40, "E": 41, "F": 42, "G": 43, "H": 44, "I": 45, "J": 46, "K": 47, "L": 48, "M": 49, "N": 50, "O": 51, "P": 52, "Q": 53, "R": 54, "S": 55, "T": 56, "U": 57, "V": 58, "W": 59, "X": 60, "Y": 61, "Z": 62, "[": 63, "\\": 64, "]": 65, "^": 66, "_": 67, "`": 68, "a": 69, "b": 70, "c": 71, "d": 72, "e": 73, "f": 74, "g": 75, "h": 76, "i": 77, "j": 78, "k": 79, "l": 80, "m": 81, "n": 82, "o": 83, "p": 84, "q": 85, "r": 86, "s": 87, "t": 88, "u": 89, "v": 90, "w": 91, "x": 92, "y": 93, "z": 94, "{": 95, "|": 96, "}": 97, "~": 98, "¡": 99, "¢": 100, "£": 101, "¤": 102, "¥": 103, "¦": 104, "§": 105, "¨": 106, "©": 107, "ª": 108, "«": 109, "¬": 110, "®": 111, "¯": 112, "°": 113, "±": 114, "²": 115, "³": 116, "´": 117, "µ": 118, "¶": 119, "·": 120, "¸": 121, "¹": 122, "º": 123, "»": 124, "¼": 125, "½": 126, "¾": 127, "¿": 128, "À": 129, "Á": 130, "Â": 131, "Ã": 132, "Ä": 133, "Å": 134, "Æ": 135, "Ç": 136, "È": 137, "É": 138, "Ê": 139, "Ë": 140, "Ì": 141, "Í": 142, "Î": 143, "Ï": 144, "Ð": 145, "Ñ": 146, "Ò": 147, "Ó": 148, "Ô": 149, "Õ": 150, "Ö": 151, "×": 152, "Ø": 153, "Ù": 154, "Ú": 155, "Û": 156, "Ü": 157, "Ý": 158, "Þ": 159, "ß": 160, "à": 161, "á": 162, "â": 163, "ã": 164, "ä": 165, "å": 166, "æ": 167, "ç": 168, "è": 169, "é": 170, "ê": 171, "ë": 172, "ì": 173, "í": 174, "î": 175, "ï": 176, "ð": 177, "ñ": 178, "ò": 179, "ó": 180, "ô": 181, "õ": 182, "ö": 183, "÷": 184, "ø": 185, "ù": 186, "ú": 187, "û": 188, "ü": 189, "ý": 190, "þ": 191, "ÿ": 192, "Ā": 193, "ā": 194, "Ă": 195, "ă": 196, "Ą": 197, "ą": 198, "Ć": 199, "ć": 200, "Ĉ": 201, "ĉ": 202, "Ċ": 203, "ċ": 204, "Č": 205, "č": 206, "Ď": 207, "ď": 208, "Đ": 209, "đ": 210, "Ē": 211, "ē": 212, "Ĕ": 213, "ĕ": 214, "Ė": 215, "ė": 216, "Ę": 217, "ę": 218, "Ě": 219, "ě": 220, "Ĝ": 221, "ĝ": 222, "Ğ": 223, "ğ": 224, "Ġ": 225, "ġ": 226, "Ģ": 227, "ģ": 228, "Ĥ": 229, "ĥ": 230, "Ħ": 231, "ħ": 232, "Ĩ": 233, "ĩ": 234, "Ī": 235, "ī": 236, "Ĭ": 237, "ĭ": 238, "Į": 239, "į": 240, "İ": 241, "ı": 242, "IJ": 243, "ij": 244, "Ĵ": 245, "ĵ": 246, "Ķ": 247, "ķ": 248, "ĸ": 249, "Ĺ": 250, "ĺ": 251, "Ļ": 252, "ļ": 253, "Ľ": 254, "ľ": 255, "Ŀ": 256, "ŀ": 257, "Ł": 258, "ł": 259, "Ń": 260, "Ġt": 261, "in": 262, "en": 263, "Ġa": 264, "er": 265, "es": 266, "Ġo": 267, "Ġto": 268, "an": 269, "ing": 270, "ar": 271, "at": 272, "he": 273, "is": 274, "or": 275, "Ġm": 276, "Ġs": 277, "al": 278, "ken": 279, "on": 280, "Ġtoken": 281, "le": 282, "Ġof": 283, "Ġb": 284, "Ġp": 285, "it": 286, "ou": 287, "Ġc": 288, "Ġf": 289, "Ġan": 290, "ic": 291, "iz": 292, "ion": 293, "ro": 294, "ed": 295, "el": 296, "re": 297, "Ġin": 298, "Ġthe": 299, "Ġis": 300, "Ġand": 301, "ĠT": 302, "Ġu": 303, "Ġbe": 304, "Ġd": 305, "Ġth": 306, "ac": 307, "mp": 308, "od": 309, "Ġw": 310, "Ġle": 311, "enc": 312, "ent": 313, "Ġtokeniz": 314, "as": 315, "ex": 316, "ter": 317, "Ġl": 318, "arn": 319, "Ġtokens": 320, "Ġpro": 321, "qu": 322, "ul": 323, "Ġh": 324, "Ġv": 325, "ation": 326, "Ġthat": 327, "Ġlearn": 328, "ain": 329, "ces": 330, "ec": 331, "fic": 332, "ge": 333, "il": 334, "res": 335, "ua": 336, "you": 337, "Ġn": 338, "Ġre": 339, "Ġyou": 340, "Ġtr": 341, "Ġtex": 342, "Ġun": 343, "cess": 344, "Ġtext": 345, "00": 346, "am": 347, "ds": 348, "em": 349, "et": 350, "gua": 351, "ific": 352, "oc": 353, "per": 354, "st": 355, "ur": 356, "ve": 357, "Ġon": 358, "angua": 359, "ords": 360, "Ġmod": 361, "ence": 362, "Ġtokenizer": 363, "ular": 364, "Ġlearning": 365, "aining": 366, "ab": 367, "ch": 368, "for": 369, "hin": 370, "his": 371, "ial": 372, "imp": 373, "ks": 374, "se": 375, "su": 376, "us": 377, "ĠB": 378, "Ġ[": 379, "Ġimp": 380, "Ġar": 381, "Ġor": 382, "ans": 383, "ate": 384, "ith": 385, "Ġcan": 386, "Ġfor": 387, "achin": 388, "Ġwords": 389, "Ġlangua": 390, "Ġprocess": 391, "Ġyour": 392, "ural": 393, "Ġmodel": 394, "achine": 395, "LP": 396, "NLP": 397, "PE": 398, "The": 399, "],": 400, "ad": 401, "ded": 402, "ev": 403, "gh": 404, "id": 405, "im": 406, "ld": 407, "lp": 408, "lu": 409, "ly": 410, "pl": 411, "te": 412, "tion": 413, "tific": 414, "um": 415, "vis": 416, "wn": 417, "yte": 418, "Ġen": 419, "Ġhe": 420, "Ġit": 421, "Ġqu": 422, "art": 423, "ata": 424, "orp": 425, "Ġmu": 426, "Ġman": 427, "Ġsu": 428, "ont": 429, "ity": 430, "out": 431, "Ġcorp": 432, "Ġcont": 433, "ions": 434, "ell": 435, "Ġint": 436, "ĠThe": 437, "Ġdata": 438, "Ġwith": 439, "Ġval": 440, "Ġvoc": 441, "Ġtraining": 442, "Ġtrans": 443, "pervis": 444, "abular": 445, "form": 446, "Ġartific": 447, "Ġlanguage": 448, "Ġmodels": 449, "Ġvocabular": 450, "pervised": 451, "Ġartificial": 452, "BPE": 453, "Byte": 454, "Machine": 455, "This": 456, "ak": 457, "amp": 458, "ave": 459, "clu": 460, "ding": 461, "dded": 462, "een": 463, "equ": 464, "eural": 465, "gr": 466, "gence": 467, "her": 468, "hes": 469, "ies": 470, "ive": 471, "igence": 472, "kn": 473, "lions": 474, "ms": 475, "mal": 476, "om": 477, "oo": 478, "op": 479, "omp": 480, "own": 481, "pp": 482, "pic": 483, "pec": 484, "pres": 485, "ts": 486, "tw": 487, "ten": 488, "tan": 489, "ting": 490, "ut": 491, "uter": 492, "ver": 493, "vel": 494, "vid": 495, "wor": 496, "ypic": 497, "Ġ(": 498, "Ġ2": 499, "ĠI": 500, "ĠL": 501, "ĠP": 502, "Ġr": 503, "Ġex": 504, "ĠNLP": 505, "Ġtas": 506, "Ġal": 507, "Ġat": 508, "Ġare": 509, "Ġadded": 510, "and": 511, "atural": 512, "ortan": 513, "Ġmachine": 514, "Ġmak": 515, "Ġsiz": 516, "Ġsent": 517, "Ġsamp": 518, "Ġsequ": 519, "Ġsmal": 520, "alle": 521, "level": 522, "Ġoften": 523, "Ġbro": 524, "Ġpart": 525, "ous": 526, "ough": 527, "Ġcomp": 528, "Ġfro": 529, "Ġfam": 530, "Ġfoc": 531, "Ġinclu": 532, "ĠThis": 533, "ĠThes": 534, "ĠToo": 535, "Ġus": 536, "Ġuse": 537, "Ġbetw": 538, "Ġdi": 539, "oding": 540, "ences": 541, "Ġtokenization": 542, "ased": 543, "Ġlar": 544, "Ġprogr": 545, "Ġprovid": 546, "Ġhum": 547, "Ġhave": 548, "ect": 549, "illions": 550, "Ġnet": 551, "Ġneural": 552, "Ġrepres": 553, "Ġunkn": 554, "000": 555, "supervised": 556, "uses": 557, "ĠBBPE": 558, "Ġimportan": 559, "plic": 560, "Ġensu": 561, "Ġhelp": 562, "Ġmuch": 563, "Ġmany": 564, "Ġsub": 565, "Ġcorpus": 566, "elligence": 567, "Ġintelligence": 568, "Ġwithout": 569, "Ġtransform": 570, "Ġvocabulary": 571, "works": 572, "ypical": 573, "Ġtasks": 574, "Ġsample": 575, "Ġsmall": 576, "Ġcomputer": 577, "Ġfrom": 578, "Ġfocuses": 579, "ĠThese": 580, "Ġbetween": 581, "Ġprogram": 582, "Ġprovides": 583, "Ġnetworks": 584, "Ġrepresent": 585, "Ġunknown": 586, "Ġimportant": 587, "Ġtransformer": 588 }, "merges": [ [ "Ġ", "t" ], [ "i", "n" ], [ "e", "n" ], [ "Ġ", "a" ], [ "e", "r" ], [ "e", "s" ], [ "Ġ", "o" ], [ "Ġt", "o" ], [ "a", "n" ], [ "in", "g" ], [ "a", "r" ], [ "a", "t" ], [ "h", "e" ], [ "i", "s" ], [ "o", "r" ], [ "Ġ", "m" ], [ "Ġ", "s" ], [ "a", "l" ], [ "k", "en" ], [ "o", "n" ], [ "Ġto", "ken" ], [ "l", "e" ], [ "Ġo", "f" ], [ "Ġ", "b" ], [ "Ġ", "p" ], [ "i", "t" ], [ "o", "u" ], [ "Ġ", "c" ], [ "Ġ", "f" ], [ "Ġa", "n" ], [ "i", "c" ], [ "i", "z" ], [ "i", "on" ], [ "r", "o" ], [ "e", "d" ], [ "e", "l" ], [ "r", "e" ], [ "Ġ", "in" ], [ "Ġt", "he" ], [ "Ġ", "is" ], [ "Ġan", "d" ], [ "Ġ", "T" ], [ "Ġ", "u" ], [ "Ġb", "e" ], [ "Ġ", "d" ], [ "Ġt", "h" ], [ "a", "c" ], [ "m", "p" ], [ "o", "d" ], [ "Ġ", "w" ], [ "Ġ", "le" ], [ "en", "c" ], [ "en", "t" ], [ "Ġtoken", "iz" ], [ "a", "s" ], [ "e", "x" ], [ "t", "er" ], [ "Ġ", "l" ], [ "ar", "n" ], [ "Ġtoken", "s" ], [ "Ġp", "ro" ], [ "q", "u" ], [ "u", "l" ], [ "Ġ", "h" ], [ "Ġ", "v" ], [ "at", "ion" ], [ "Ġth", "at" ], [ "Ġle", "arn" ], [ "a", "in" ], [ "c", "es" ], [ "e", "c" ], [ "f", "ic" ], [ "g", "e" ], [ "i", "l" ], [ "r", "es" ], [ "u", "a" ], [ "y", "ou" ], [ "Ġ", "n" ], [ "Ġ", "re" ], [ "Ġ", "you" ], [ "Ġt", "r" ], [ "Ġt", "ex" ], [ "Ġu", "n" ], [ "ces", "s" ], [ "Ġtex", "t" ], [ "0", "0" ], [ "a", "m" ], [ "d", "s" ], [ "e", "m" ], [ "e", "t" ], [ "g", "ua" ], [ "i", "fic" ], [ "o", "c" ], [ "p", "er" ], [ "s", "t" ], [ "u", "r" ], [ "v", "e" ], [ "Ġo", "n" ], [ "an", "gua" ], [ "or", "ds" ], [ "Ġm", "od" ], [ "enc", "e" ], [ "Ġtokeniz", "er" ], [ "ul", "ar" ], [ "Ġlearn", "ing" ], [ "ain", "ing" ], [ "a", "b" ], [ "c", "h" ], [ "f", "or" ], [ "h", "in" ], [ "h", "is" ], [ "i", "al" ], [ "i", "mp" ], [ "k", "s" ], [ "s", "e" ], [ "s", "u" ], [ "u", "s" ], [ "Ġ", "B" ], [ "Ġ", "[" ], [ "Ġ", "imp" ], [ "Ġa", "r" ], [ "Ġo", "r" ], [ "an", "s" ], [ "at", "e" ], [ "it", "h" ], [ "Ġc", "an" ], [ "Ġf", "or" ], [ "ac", "hin" ], [ "Ġw", "ords" ], [ "Ġl", "angua" ], [ "Ġpro", "cess" ], [ "Ġyou", "r" ], [ "ur", "al" ], [ "Ġmod", "el" ], [ "achin", "e" ], [ "L", "P" ], [ "N", "LP" ], [ "P", "E" ], [ "T", "he" ], [ "]", "," ], [ "a", "d" ], [ "d", "ed" ], [ "e", "v" ], [ "g", "h" ], [ "i", "d" ], [ "i", "m" ], [ "l", "d" ], [ "l", "p" ], [ "l", "u" ], [ "l", "y" ], [ "p", "l" ], [ "t", "e" ], [ "t", "ion" ], [ "t", "ific" ], [ "u", "m" ], [ "v", "is" ], [ "w", "n" ], [ "y", "te" ], [ "Ġ", "en" ], [ "Ġ", "he" ], [ "Ġ", "it" ], [ "Ġ", "qu" ], [ "ar", "t" ], [ "at", "a" ], [ "or", "p" ], [ "Ġm", "u" ], [ "Ġm", "an" ], [ "Ġs", "u" ], [ "on", "t" ], [ "it", "y" ], [ "ou", "t" ], [ "Ġc", "orp" ], [ "Ġc", "ont" ], [ "ion", "s" ], [ "el", "l" ], [ "Ġin", "t" ], [ "ĠT", "he" ], [ "Ġd", "ata" ], [ "Ġw", "ith" ], [ "Ġv", "al" ], [ "Ġv", "oc" ], [ "Ġtr", "aining" ], [ "Ġtr", "ans" ], [ "per", "vis" ], [ "ab", "ular" ], [ "for", "m" ], [ "Ġar", "tific" ], [ "Ġlangua", "ge" ], [ "Ġmodel", "s" ], [ "Ġvoc", "abular" ], [ "pervis", "ed" ], [ "Ġartific", "ial" ], [ "B", "PE" ], [ "B", "yte" ], [ "M", "achine" ], [ "T", "his" ], [ "a", "k" ], [ "a", "mp" ], [ "a", "ve" ], [ "c", "lu" ], [ "d", "ing" ], [ "d", "ded" ], [ "e", "en" ], [ "e", "qu" ], [ "e", "ural" ], [ "g", "r" ], [ "g", "ence" ], [ "h", "er" ], [ "h", "es" ], [ "i", "es" ], [ "i", "ve" ], [ "i", "gence" ], [ "k", "n" ], [ "l", "ions" ], [ "m", "s" ], [ "m", "al" ], [ "o", "m" ], [ "o", "o" ], [ "o", "p" ], [ "o", "mp" ], [ "o", "wn" ], [ "p", "p" ], [ "p", "ic" ], [ "p", "ec" ], [ "p", "res" ], [ "t", "s" ], [ "t", "w" ], [ "t", "en" ], [ "t", "an" ], [ "t", "ing" ], [ "u", "t" ], [ "u", "ter" ], [ "v", "er" ], [ "v", "el" ], [ "v", "id" ], [ "w", "or" ], [ "y", "pic" ], [ "Ġ", "(" ], [ "Ġ", "2" ], [ "Ġ", "I" ], [ "Ġ", "L" ], [ "Ġ", "P" ], [ "Ġ", "r" ], [ "Ġ", "ex" ], [ "Ġ", "NLP" ], [ "Ġt", "as" ], [ "Ġa", "l" ], [ "Ġa", "t" ], [ "Ġa", "re" ], [ "Ġa", "dded" ], [ "an", "d" ], [ "at", "ural" ], [ "or", "tan" ], [ "Ġm", "achine" ], [ "Ġm", "ak" ], [ "Ġs", "iz" ], [ "Ġs", "ent" ], [ "Ġs", "amp" ], [ "Ġs", "equ" ], [ "Ġs", "mal" ], [ "al", "le" ], [ "le", "vel" ], [ "Ġof", "ten" ], [ "Ġb", "ro" ], [ "Ġp", "art" ], [ "ou", "s" ], [ "ou", "gh" ], [ "Ġc", "omp" ], [ "Ġf", "ro" ], [ "Ġf", "am" ], [ "Ġf", "oc" ], [ "Ġin", "clu" ], [ "ĠT", "his" ], [ "ĠT", "hes" ], [ "ĠT", "oo" ], [ "Ġu", "s" ], [ "Ġu", "se" ], [ "Ġbe", "tw" ], [ "Ġd", "i" ], [ "od", "ing" ], [ "enc", "es" ], [ "Ġtokeniz", "ation" ], [ "as", "ed" ], [ "Ġl", "ar" ], [ "Ġpro", "gr" ], [ "Ġpro", "vid" ], [ "Ġh", "um" ], [ "Ġh", "ave" ], [ "ec", "t" ], [ "il", "lions" ], [ "Ġn", "et" ], [ "Ġn", "eural" ], [ "Ġre", "pres" ], [ "Ġun", "kn" ], [ "00", "0" ], [ "su", "pervised" ], [ "us", "es" ], [ "ĠB", "BPE" ], [ "Ġimp", "ortan" ], [ "pl", "ic" ], [ "Ġen", "su" ], [ "Ġhe", "lp" ], [ "Ġmu", "ch" ], [ "Ġman", "y" ], [ "Ġsu", "b" ], [ "Ġcorp", "us" ], [ "ell", "igence" ], [ "Ġint", "elligence" ], [ "Ġwith", "out" ], [ "Ġtrans", "form" ], [ "Ġvocabular", "y" ], [ "wor", "ks" ], [ "ypic", "al" ], [ "Ġtas", "ks" ], [ "Ġsamp", "le" ], [ "Ġsmal", "l" ], [ "Ġcomp", "uter" ], [ "Ġfro", "m" ], [ "Ġfoc", "uses" ], [ "ĠThes", "e" ], [ "Ġbetw", "een" ], [ "Ġprogr", "am" ], [ "Ġprovid", "es" ], [ "Ġnet", "works" ], [ "Ġrepres", "ent" ], [ "Ġunkn", "own" ], [ "Ġimportan", "t" ], [ "Ġtransform", "er" ] ] } }