# Example configuration for training a SentencePiece tokenizer from JSONL data

# Dataset source - JSONL file
jsonl_path: /home/student/Data/TaoData/pretrain.jsonl
text_field: text  # Field name in JSON for text data

# Tokenizer training parameters
vocab_size: 8192  # Keep aligned with pretrain/sft model vocab_size
model_type: unigram  # SentencePiece model type: unigram, bpe, char, word
character_coverage: 0.9995

# Output configuration
output_dir: tokenizer
tokenizer_prefix: tokenizer

# Custom special tokens
# Built-in tokens are managed by SentencePiece and resolved at runtime.
# Entries here are registered as user-defined symbols and should encode as
# single tokens, but SentencePiece does not guarantee their exact IDs.
# Note: Use \n for newline token, \t for tab, etc.
special_tokens:
  - "\n"        # Newline token - quoted to preserve literal \n in YAML
  - <think>     # Special token for chain-of-thought reasoning
  - <user>      # User message token
  - <assistant> # Assistant message token
  - <image>     # Image token for multimodal models

# Data sampling (optional)
# Set to a number to train on only the first N samples from the JSONL file
# Useful for quick testing or sub-sampling large datasets
# Omit or set to null to use entire file
max_samples: 1000000

# Optional metadata
tokenizer_name: tokenizer