HuaminChen commited on
Commit
7b7368e
·
verified ·
1 Parent(s): 188fd76

Add tokenizer_config.json to onnx/

Browse files
Files changed (1) hide show
  1. onnx/tokenizer_config.json +31 -0
onnx/tokenizer_config.json ADDED
@@ -0,0 +1,31 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "backend": "tokenizers",
3
+ "bos_token": "<bos>",
4
+ "clean_up_tokenization_spaces": false,
5
+ "cls_token": "<bos>",
6
+ "eos_token": "<eos>",
7
+ "extra_special_tokens": [
8
+ "<start_of_turn>",
9
+ "<end_of_turn>"
10
+ ],
11
+ "is_local": true,
12
+ "mask_token": "<mask>",
13
+ "max_length": 32768,
14
+ "model_input_names": [
15
+ "input_ids",
16
+ "attention_mask"
17
+ ],
18
+ "model_max_length": 32768,
19
+ "model_specific_special_tokens": {},
20
+ "pad_to_multiple_of": null,
21
+ "pad_token": "<pad>",
22
+ "pad_token_type_id": 0,
23
+ "padding_side": "right",
24
+ "sep_token": "<eos>",
25
+ "spaces_between_special_tokens": false,
26
+ "stride": 0,
27
+ "tokenizer_class": "TokenizersBackend",
28
+ "truncation_side": "right",
29
+ "truncation_strategy": "longest_first",
30
+ "unk_token": "<unk>"
31
+ }