File size: 2,237 Bytes
1c9c7eb 594f96a 1c9c7eb 594f96a 1c9c7eb 594f96a 1c9c7eb 594f96a 1c9c7eb 594f96a 1c9c7eb 594f96a 1c9c7eb 594f96a 1c9c7eb 594f96a 1c9c7eb 594f96a 1c9c7eb 594f96a 1c9c7eb 594f96a 1c9c7eb 594f96a 1c9c7eb 594f96a 1c9c7eb 594f96a 1c9c7eb 594f96a 1c9c7eb 594f96a 1c9c7eb 594f96a 1c9c7eb 594f96a 1c9c7eb 594f96a 1c9c7eb 594f96a 1c9c7eb 594f96a 1c9c7eb 594f96a 1c9c7eb | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 | {
"added_tokens_decoder": {
"0": {
"content": "<s>",
"lstrip": false,
"normalized": false,
"rstrip": false,
"single_word": false,
"special": true
},
"1": {
"content": "<pad>",
"lstrip": false,
"normalized": false,
"rstrip": false,
"single_word": false,
"special": true
},
"2": {
"content": "</s>",
"lstrip": false,
"normalized": false,
"rstrip": false,
"single_word": false,
"special": true
},
"3": {
"content": "<unk>",
"lstrip": false,
"normalized": false,
"rstrip": false,
"single_word": false,
"special": true
},
"32769": {
"content": "<mask>",
"lstrip": true,
"normalized": true,
"rstrip": false,
"single_word": false,
"special": true
},
"32770": {
"content": "<ent>",
"lstrip": false,
"normalized": true,
"rstrip": false,
"single_word": false,
"special": true
},
"32771": {
"content": "<ent2>",
"lstrip": false,
"normalized": true,
"rstrip": false,
"single_word": false,
"special": true
}
},
"additional_special_tokens": [
"<ent>",
"<ent2>",
"<ent>",
"<ent2>",
"<ent>",
"<ent2>",
"<ent>",
"<ent2>"
],
"bos_token": "<s>",
"clean_up_tokenization_spaces": true,
"cls_token": "<s>",
"entity_mask2_token": "[MASK2]",
"entity_mask_token": "[MASK]",
"entity_pad_token": "[PAD]",
"entity_token_1": {
"__type": "AddedToken",
"content": "<ent>",
"lstrip": false,
"normalized": true,
"rstrip": false,
"single_word": false,
"special": false
},
"entity_token_2": {
"__type": "AddedToken",
"content": "<ent2>",
"lstrip": false,
"normalized": true,
"rstrip": false,
"single_word": false,
"special": false
},
"entity_unk_token": "[UNK]",
"eos_token": "</s>",
"mask_token": "<mask>",
"max_entity_length": 32,
"max_mention_length": 30,
"model_max_length": 512,
"pad_token": "<pad>",
"sep_token": "</s>",
"sp_model_kwargs": {},
"task": null,
"tokenizer_class": "MLukeTokenizer",
"unk_token": "<unk>"
}
|