{ "add_bos_token": false, "add_eos_token": false, "added_tokens_decoder": { "0": { "content": "<|endoftext|>", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": true }, "1": { "content": "<|pad|>", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": true }, "2": { "content": "<|unk|>", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": true }, "3": { "content": "", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": true }, "4": { "content": "", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": true }, "5": { "content": "", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": true }, "6": { "content": "", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": true }, "32768": { "content": "<|im_start|>", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": true }, "32769": { "content": "<|im_end|>", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": true }, "32770": { "content": "", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": true }, "32771": { "content": "", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": true }, "32772": { "content": "", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": true }, "32773": { "content": "", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": true }, "32774": { "content": "<|fim_prefix|>", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": true }, "32775": { "content": "<|fim_suffix|>", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": true }, "32776": { "content": "<|fim_middle|>", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": true } }, "additional_special_tokens": [ "", "", "", "", "<|im_start|>", "<|im_end|>", "", "", "", "", "<|fim_prefix|>", "<|fim_suffix|>", "<|fim_middle|>" ], "bos_token": "<|endoftext|>", "clean_up_tokenization_spaces": false, "eos_token": "<|im_end|>", "extra_special_tokens": {}, "model_max_length": 131072, "pad_token": "<|pad|>", "tokenizer_class": "PreTrainedTokenizerFast", "unk_token": "<|unk|>", "vocab_size": 32768 }