| { | |
| "name": "ZombitX64-Thaitokenizer", | |
| "version": "1.0.0", | |
| "description": "Thai language tokenizer with newline preservation", | |
| "vocab_size": 112, | |
| "max_length": 512, | |
| "pad_token": "[PAD]", | |
| "unk_token": "[UNK]", | |
| "bos_token": "[BOS]", | |
| "eos_token": "[EOS]", | |
| "model_type": "thai_tokenizer" | |
| } |