Upload tokenizer

added_tokens.json

@@ -0,0 +1,3 @@
+{
+  "<|endoftext|>": 59
+}

merges.txt

@@ -0,0 +1 @@
+#version: 0.2

special_tokens_map.json

@@ -0,0 +1,5 @@
+{
+  "bos_token": "<|endoftext|>",
+  "eos_token": "<|endoftext|>",
+  "unk_token": "<|endoftext|>"
+}

tokenizer.json

@@ -0,0 +1,138 @@
+{
+  "version": "1.0",
+  "truncation": null,
+  "padding": null,
+  "added_tokens": [
+    {
+      "id": 0,
+      "content": "[PAD]",
+      "single_word": false,
+      "lstrip": false,
+      "rstrip": false,
+      "normalized": false,
+      "special": true
+    },
+    {
+      "id": 1,
+      "content": "[UNK]",
+      "single_word": false,
+      "lstrip": false,
+      "rstrip": false,
+      "normalized": false,
+      "special": true
+    },
+    {
+      "id": 2,
+      "content": "[MASK]",
+      "single_word": false,
+      "lstrip": false,
+      "rstrip": false,
+      "normalized": false,
+      "special": true
+    },
+    {
+      "id": 3,
+      "content": "[EOS]",
+      "single_word": false,
+      "lstrip": false,
+      "rstrip": false,
+      "normalized": false,
+      "special": true
+    },
+    {
+      "id": 59,
+      "content": "<|endoftext|>",
+      "single_word": false,
+      "lstrip": false,
+      "rstrip": false,
+      "normalized": false,
+      "special": true
+    }
+  ],
+  "normalizer": null,
+  "pre_tokenizer": {
+    "type": "Metaspace",
+    "replacement": "▁",
+    "prepend_scheme": "always",
+    "split": true
+  },
+  "post_processor": null,
+  "decoder": {
+    "type": "Metaspace",
+    "replacement": "▁",
+    "prepend_scheme": "always",
+    "split": true
+  },
+  "model": {
+    "type": "BPE",
+    "dropout": null,
+    "unk_token": null,
+    "continuing_subword_prefix": null,
+    "end_of_word_suffix": null,
+    "fuse_unk": false,
+    "byte_fallback": false,
+    "ignore_merges": false,
+    "vocab": {
+      "[PAD]": 0,
+      "[UNK]": 1,
+      "[MASK]": 2,
+      "[EOS]": 3,
+      "\n": 4,
+      "!": 5,
+      "\"": 6,
+      ",": 7,
+      "-": 8,
+      ".": 9,
+      "0": 10,
+      "1": 11,
+      "2": 12,
+      "3": 13,
+      "4": 14,
+      "5": 15,
+      "6": 16,
+      "7": 17,
+      "8": 18,
+      "9": 19,
+      "?": 20,
+      "а": 21,
+      "б": 22,
+      "в": 23,
+      "г": 24,
+      "д": 25,
+      "е": 26,
+      "ж": 27,
+      "з": 28,
+      "и": 29,
+      "й": 30,
+      "к": 31,
+      "л": 32,
+      "м": 33,
+      "н": 34,
+      "о": 35,
+      "п": 36,
+      "р": 37,
+      "с": 38,
+      "т": 39,
+      "у": 40,
+      "ф": 41,
+      "х": 42,
+      "ц": 43,
+      "ч": 44,
+      "ш": 45,
+      "щ": 46,
+      "ъ": 47,
+      "ы": 48,
+      "ь": 49,
+      "э": 50,
+      "ю": 51,
+      "я": 52,
+      "ё": 53,
+      "ҫ": 54,
+      "ӑ": 55,
+      "ӗ": 56,
+      "ӳ": 57,
+      "▁": 58
+    },
+    "merges": []
+  }
+}

tokenizer_config.json

@@ -0,0 +1,52 @@
+{
+  "add_prefix_space": false,
+  "added_tokens_decoder": {
+    "0": {
+      "content": "[PAD]",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "1": {
+      "content": "[UNK]",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "2": {
+      "content": "[MASK]",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "3": {
+      "content": "[EOS]",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "59": {
+      "content": "<|endoftext|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    }
+  },
+  "bos_token": "<|endoftext|>",
+  "clean_up_tokenization_spaces": false,
+  "eos_token": "<|endoftext|>",
+  "extra_special_tokens": {},
+  "model_max_length": 1000000000000000019884624838656,
+  "tokenizer_class": "GPT2Tokenizer",
+  "unk_token": "<|endoftext|>"
+}

vocab.json

@@ -0,0 +1 @@
+{"[PAD]":0,"[UNK]":1,"[MASK]":2,"[EOS]":3,"\n":4,"!":5,"\"":6,",":7,"-":8,".":9,"0":10,"1":11,"2":12,"3":13,"4":14,"5":15,"6":16,"7":17,"8":18,"9":19,"?":20,"а":21,"б":22,"в":23,"г":24,"д":25,"е":26,"ж":27,"з":28,"и":29,"й":30,"к":31,"л":32,"м":33,"н":34,"о":35,"п":36,"р":37,"с":38,"т":39,"у":40,"ф":41,"х":42,"ц":43,"ч":44,"ш":45,"щ":46,"ъ":47,"ы":48,"ь":49,"э":50,"ю":51,"я":52,"ё":53,"ҫ":54,"ӑ":55,"ӗ":56,"ӳ":57,"▁":58}