Upload 5 files

crumb_metadata.json

@@ -0,0 +1,38 @@
+{
+  "name": "CRUMB Python BPE \u2014 TOK-1 v7.4.0",
+  "script_version": "v7.4.0",
+  "vocab_size": 32768,
+  "domain": "python_source_code",
+  "corpus": "stack-dedup-python-parquet-27GB",
+  "min_frequency": 2,
+  "trainer": "ByteLevelBPETokenizer via train_from_iterator (Python str)",
+  "protected_keywords": 38,
+  "protected_dunders": 21,
+  "protected_builtins": 73,
+  "protected_exceptions": 69,
+  "protected_stdlib": 125,
+  "protected_ds_tokens": 119,
+  "total_never_split": 474,
+  "special_tokens": 20,
+  "reserved_future_tokens": 4,
+  "target_model_sizes": [
+    "124M",
+    "250M",
+    "370M",
+    "500M"
+  ],
+  "fim_tokens": [
+    "<fim_prefix>",
+    "<fim_suffix>",
+    "<fim_middle>"
+  ],
+  "notebook_tokens": [
+    "<cell>",
+    "</cell>",
+    "<cell_output>",
+    "</cell_output>"
+  ],
+  "comments_in_training": true,
+  "docstrings_in_training": true,
+  "training_date": "2026-05-11"
+}

tokenizer_config.json

@@ -0,0 +1,69 @@
+{
+  "backend": "tokenizers",
+  "bos_token": "<s>",
+  "eos_token": "</s>",
+  "extra_special_tokens": [
+    "<func>",
+    "</func>",
+    "<class>",
+    "</class>",
+    "<module>",
+    "</module>",
+    "<cell>",
+    "</cell>",
+    "<cell_output>",
+    "</cell_output>",
+    "<traceback>",
+    "</traceback>",
+    "<fim_prefix>",
+    "<fim_suffix>",
+    "<fim_middle>",
+    "<comment>",
+    "<docstring>",
+    "</docstring>",
+    "<type_hint>"
+  ],
+  "mask_token": "<mask>",
+  "model_max_length": 1000000000000000019884624838656,
+  "pad_token": "<pad>",
+  "tokenizer_class": "TokenizersBackend",
+  "unk_token": "<unk>",
+  "crumb_metadata": {
+    "name": "CRUMB Python BPE \u2014 TOK-1 v7.4.0",
+    "script_version": "v7.4.0",
+    "vocab_size": 32768,
+    "domain": "python_source_code",
+    "corpus": "stack-dedup-python-parquet-27GB",
+    "min_frequency": 2,
+    "trainer": "ByteLevelBPETokenizer via train_from_iterator (Python str)",
+    "protected_keywords": 38,
+    "protected_dunders": 21,
+    "protected_builtins": 73,
+    "protected_exceptions": 69,
+    "protected_stdlib": 125,
+    "protected_ds_tokens": 119,
+    "total_never_split": 474,
+    "special_tokens": 20,
+    "reserved_future_tokens": 4,
+    "target_model_sizes": [
+      "124M",
+      "250M",
+      "370M",
+      "500M"
+    ],
+    "fim_tokens": [
+      "<fim_prefix>",
+      "<fim_suffix>",
+      "<fim_middle>"
+    ],
+    "notebook_tokens": [
+      "<cell>",
+      "</cell>",
+      "<cell_output>",
+      "</cell_output>"
+    ],
+    "comments_in_training": true,
+    "docstrings_in_training": true,
+    "training_date": "2026-05-11"
+  }
+}