Hugging Face's logo

flokymind
/
mishkala

Arabic
arabic
diacritization
tashkeel
nlp
transformer
Model card Files
xet
 Community
flokymind commited on
Commit
8a9a2f7
·
1 Parent(s): a1e3dd8

Upload tokenizer.json

Browse files
Files changed (1) hide show
  1. tokenizer.json +81 -0
tokenizer.json ADDED
@@ -0,0 +1,81 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "char_to_id": {
3
+ "[PAD]": 0,
4
+ "[UNK]": 1,
5
+ "[BOS]": 2,
6
+ "[EOS]": 3,
7
+ "[MASK]": 4,
8
+ " ": 5,
9
+ "ا": 6,
10
+ "ل": 7,
11
+ "ي": 8,
12
+ "م": 9,
13
+ "و": 10,
14
+ "ن": 11,
15
+ "ه": 12,
16
+ "ب": 13,
17
+ "ر": 14,
18
+ "ع": 15,
19
+ "ف": 16,
20
+ "أ": 17,
21
+ "ق": 18,
22
+ "ت": 19,
23
+ "د": 20,
24
+ "ك": 21,
25
+ "ح": 22,
26
+ "ة": 23,
27
+ "س": 24,
28
+ "ج": 25,
29
+ "إ": 26,
30
+ "ص": 27,
31
+ "ذ": 28,
32
+ "ى": 29,
33
+ "خ": 30,
34
+ "ش": 31,
35
+ "ث": 32,
36
+ "ض": 33,
37
+ "ط": 34,
38
+ "ز": 35,
39
+ ":": 36,
40
+ "غ": 37,
41
+ "ء": 38,
42
+ "ئ": 39,
43
+ "ظ": 40,
44
+ "؛": 41,
45
+ "آ": 42,
46
+ "-": 43,
47
+ "ؤ": 44,
48
+ ",": 45,
49
+ "ٰ": 46,
50
+ "ۚ": 47,
51
+ "ۖ": 48,
52
+ "ۗ": 49
53
+ },
54
+ "vocab_size": 50,
55
+ "special_tokens": {
56
+ "PAD": 0,
57
+ "UNK": 1,
58
+ "BOS": 2,
59
+ "EOS": 3,
60
+ "MASK": 4,
61
+ "SPACE": 5
62
+ },
63
+ "diacritic_classes": [
64
+ "NO_DIACRITIC",
65
+ "FATHA",
66
+ "FATHATAN",
67
+ "DAMMA",
68
+ "DAMMATAN",
69
+ "KASRA",
70
+ "KASRATAN",
71
+ "SUKUN",
72
+ "SHADDA",
73
+ "SHADDA_FATHA",
74
+ "SHADDA_FATHATAN",
75
+ "SHADDA_DAMMA",
76
+ "SHADDA_DAMMATAN",
77
+ "SHADDA_KASRA",
78
+ "SHADDA_KASRATAN"
79
+ ],
80
+ "num_labels": 15
81
+ }