Upload tokenizer

tokenization_vnsabsa.py

@@ -8,12 +8,17 @@ import regex as re
 from typing import Tuple, Optional
 import shutil
 import os
+import requests
 
 class VnSmartphoneAbsaTokenizer(PreTrainedTokenizer):
+    pretrained_vocab_files_map = {
+        "vocab_file": "https://huggingface.co/ptdat/vn-smartphone-absa/resolve/main/vocab.txt",
+        "merge_file": "https://huggingface.co/ptdat/vn-smartphone-absa/resolve/main/merge.txt"
+    }
     def __init__(
         self, 
-        vocab_file="vocab.txt", 
-        merge_file="merge.txt", 
+        vocab_file, 
+        merge_file, 
         bos_token="<s>",
         eos_token="</s>",
         sep_token="</s>",

tokenizer_config.json

@@ -41,6 +41,12 @@
       "special": true
     }
   },
+  "auto_map": {
+    "AutoTokenizer": [
+      "tokenization_vnsabsa.VnSmartphoneAbsaTokenizer",
+      null
+    ]
+  },
   "bos_token": "<s>",
   "clean_up_tokenization_spaces": true,
   "cls_token": "<s>",