jupimind_tokenizer_report.json

{
  "input_paths": [
    "/home/trainer/jupimind/outputs/pretrain/j1r_t4x4_mix.jsonl"
  ],
  "text_seed_paths": [
    "/home/trainer/jupimind/data/tokenizer/j1r_server_seed.txt",
    "/home/trainer/jupimind/data/tokenizer/j1r_server_seed.txt"
  ],
  "vocab_size": 24576,
  "min_frequency": 2,
  "special_tokens": [
    "<pad>",
    "<unk>",
    "<bos>",
    "<eos>",
    "<|im_start|>",
    "<|im_end|>",
    "<think>",
    "</think>",
    "<tool_call>",
    "</tool_call>",
    "<tool_response>",
    "</tool_response>",
    "<tools>",
    "</tools>"
  ],
  "evaluation": {
    "candidate": {
      "name": "j1r_vi_bytebpe",
      "documents": 50000,
      "avg_tokens_per_doc": 702.28854,
      "avg_chars_per_token": 3.8265045874164487,
      "avg_bytes_per_token": 5.042613795178831,
      "total_tokens": 35114427
    },
    "baseline": {
      "name": "jingyaogong/minimind-3",
      "documents": 50000,
      "avg_tokens_per_doc": 2331.64386,
      "avg_chars_per_token": 1.1525389301949398,
      "avg_bytes_per_token": 1.518829672384015,
      "total_tokens": 116582193
    }
  },
  "recommended_model_vocab_size": 24576,
  "recommended_tokenizer_path": "aevynt/JupiMind-Dense-J1R-Tokenizer-24k",
  "notes": [
    "Tokenizer nay dung byte-level BPE voi byte fallback de xu ly tieng Viet co dau, log, code, JSON, YAML va output he thong.",
    "Tokenizer moi khong tuong thich embedding voi checkpoint vocab_size=6400 cu. Muon dung that phai pretrain lai voi vocab_size moi.",
    "Thiet ke uu tien server-first assistant, khong toi uu cho tieng Trung."
  ]
}