{ "added_tokens_decoder": { "0": { "content": "", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": true }, "1": { "content": "", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": true }, "2": { "content": "", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": true }, "3": { "content": "", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": true }, "16384": { "content": "<|im_start|>", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": true }, "16385": { "content": "<|im_end|>", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": true }, "16386": { "content": "", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": true }, "16387": { "content": "", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": true }, "16388": { "content": "", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": true }, "16389": { "content": "", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": true }, "16390": { "content": "", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": true }, "16391": { "content": "", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": true }, "16392": { "content": "", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": true }, "16393": { "content": "", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": true }, "16394": { "content": "<|system|>", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": true }, "16395": { "content": "<|user|>", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": true }, "16396": { "content": "<|assistant|>", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": true }, "16397": { "content": "<|fim_prefix|>", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": true }, "16398": { "content": "<|fim_middle|>", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": true }, "16399": { "content": "<|fim_suffix|>", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": true }, "16400": { "content": "<|endoftext|>", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": true } }, "backend": "custom", "bos_token": "", "eos_token": "", "extra_special_tokens": [ "", "", "", "", "", "", "", "", "<|assistant|>", "<|endoftext|>", "<|fim_middle|>", "<|fim_prefix|>", "<|fim_suffix|>", "<|im_end|>", "<|im_start|>", "<|system|>", "<|user|>" ], "model_max_length": 1000000000000000019884624838656, "pad_token": "", "tokenizer_class": "SpikeTokenizer", "unk_token": "", "auto_map": { "AutoTokenizer": [ "spike_tokenizer.SpikeTokenizer", null ] } }