tokenizer_config.json

{
  "name": "ZombitX64-Thaitokenizer",
  "version": "1.0.0",
  "description": "Thai language tokenizer with newline preservation",
  "vocab_size": 112,
  "max_length": 512,
  "pad_token": "[PAD]",
  "unk_token": "[UNK]",
  "bos_token": "[BOS]",
  "eos_token": "[EOS]",
  "model_type": "thai_tokenizer"
}