{
  "generated_at": "2026-05-07 15:08:53 UTC",
  "raw_dir": "/data/v2/raw/base_500m_shuffled",
  "output_dir": "/data/v2/tokenizer",
  "raw_meta_path": "/data/v2/raw/base_500m_shuffled/meta.json",
  "raw_meta_summary": {
    "source_ratio_by_chars": {
      "ultra_fineweb_zh": 0.7575749420132658,
      "fineweb_edu_4_5": 0.24242505798673422
    },
    "shuffle_files": true,
    "seed": 42,
    "max_chars_per_file": 50000000
  },
  "vocab_size": 48000,
  "actual_vocab_size": 48000,
  "normalizer": "NFC",
  "pre_tokenizer": "Sequence(Split(Qwen regex, isolated), ByteLevel(add_prefix_space=False, use_regex=False))",
  "decoder": "ByteLevel",
  "min_frequency": 2,
  "initial_alphabet": "ByteLevel.alphabet()",
  "special_tokens": [
    "<|endoftext|>",
    "<|pad|>",
    "<|bos|>",
    "<|unk|>",
    "<|im_start|>",
    "<|im_end|>",
    "<think>",
    "</think>"
  ],
  "raw_shard_count": 10,
  "documents": 445359,
  "chars": 660001818,
  "source_counts": {
    "fineweb_edu_4_5": 71351,
    "ultra_fineweb_zh": 374008
  },
  "source_chars": {
    "fineweb_edu_4_5": 160000979,
    "ultra_fineweb_zh": 500000839
  },
  "elapsed_sec": 2387.6322631835938,
  "validation": {
    "vocab_size": 48000,
    "fast_vocab_size": 48000,
    "special_token_ids": {
      "<|endoftext|>": 0,
      "<|pad|>": 1,
      "<|bos|>": 2,
      "<|unk|>": 3,
      "<|im_start|>": 4,
      "<|im_end|>": 5,
      "<think>": 6,
      "</think>": 7
    },
    "special_single_token": {
      "<|endoftext|>": {
        "ids": [
          0
        ],
        "is_single_token": true
      },
      "<|pad|>": {
        "ids": [
          1
        ],
        "is_single_token": true
      },
      "<|bos|>": {
        "ids": [
          2
        ],
        "is_single_token": true
      },
      "<|unk|>": {
        "ids": [
          3
        ],
        "is_single_token": true
      },
      "<|im_start|>": {
        "ids": [
          4
        ],
        "is_single_token": true
      },
      "<|im_end|>": {
        "ids": [
          5
        ],
        "is_single_token": true
      },
      "<think>": {
        "ids": [
          6
        ],
        "is_single_token": true
      },
      "</think>": {
        "ids": [
          7
        ],
        "is_single_token": true
      }
    },
    "tests": [
      {
        "text": "人工智能是机器学习和深度学习的重要方向。",
        "ids": [
          3648,
          295,
          2707,
          7890,
          10913,
          1972,
          1854,
          275
        ],
        "ids_len": 8,
        "tokens": [
          "人工æĻºèĥ½",
          "æĺ¯",
          "æľºåĻ¨",
          "åŃ¦ä¹łåĴĮ",
          "深度åŃ¦ä¹ł",
          "çļĦéĩįè¦ģ",
          "æĸ¹åIJij",
          "ãĢĤ"
        ],
        "decoded": "人工智能是机器学习和深度学习的重要方向。"
      },
      {
        "text": "北京大学位于北京市海淀区。",
        "ids": [
          10213,
          3116,
          7894,
          47286,
          275
        ],
        "ids_len": 5,
        "tokens": [
          "åĮĹ京大åѦ",
          "ä½įäºİ",
          "åĮĹ京å¸Ĥ",
          "æµ·æ·ĢåĮº",
          "ãĢĤ"
        ],
        "decoded": "北京大学位于北京市海淀区。"
      },
      {
        "text": "Python 是一种常用的编程语言。",
        "ids": [
          15041,
          40479,
          5899,
          23647,
          275
        ],
        "ids_len": 5,
        "tokens": [
          "Python",
          "Ġæĺ¯ä¸Ģç§į",
          "常çĶ¨çļĦ",
          "ç¼ĸç¨ĭè¯Ńè¨Ģ",
          "ãĢĤ"
        ],
        "decoded": "Python 是一种常用的编程语言。"
      },
      {
        "text": "今天天气不错，我准备去图书馆学习。",
        "ids": [
          2117,
          3928,
          4949,
          2375,
          2394,
          720,
          6796,
          874,
          275
        ],
        "ids_len": 9,
        "tokens": [
          "ä»Ĭ天",
          "天æ°Ķ",
          "ä¸įéĶĻ",
          "ï¼ĮæĪij",
          "åĩĨå¤ĩ",
          "åİ»",
          "åĽ¾ä¹¦é¦Ĩ",
          "åŃ¦ä¹ł",
          "ãĢĤ"
        ],
        "decoded": "今天天气不错，我准备去图书馆学习。"
      },
      {
        "text": "<|im_start|>user\n你好<|im_end|>",
        "ids": [
          4,
          42024,
          206,
          17325,
          5
        ],
        "ids_len": 5,
        "tokens": [
          "<|im_start|>",
          "user",
          "Ċ",
          "ä½łå¥½",
          "<|im_end|>"
        ],
        "decoded": "<|im_start|>user\n你好<|im_end|>"
      },
      {
        "text": "<think>先分析问题</think>",
        "ids": [
          6,
          821,
          40075,
          7
        ],
        "ids_len": 4,
        "tokens": [
          "<think>",
          "åħĪ",
          "åĪĨæŀIJéĹ®é¢ĺ",
          "</think>"
        ],
        "decoded": "<think>先分析问题</think>"
      }
    ],
    "passed": true,
    "fail_reasons": []
  },
  "passed": true,
  "fail_reasons": []
}