JonusNattapong's picture
Upload Thai tokenizer v1.0.0
6a65062 verified
{
"[PAD]": 0,
"[UNK]": 1,
"[BOS]": 2,
"[EOS]": 3,
"\n": 4,
"\r\n": 5,
"\r": 6,
" ": 7,
"\t": 8,
" ": 9,
" ": 10,
"ก": 11,
"ข": 12,
"ฃ": 13,
"ค": 14,
"ฅ": 15,
"ฆ": 16,
"ง": 17,
"จ": 18,
"ฉ": 19,
"ช": 20,
"ซ": 21,
"ฌ": 22,
"ญ": 23,
"ฎ": 24,
"ฏ": 25,
"ฐ": 26,
"ฑ": 27,
"ฒ": 28,
"ณ": 29,
"ด": 30,
"ต": 31,
"ถ": 32,
"ท": 33,
"ธ": 34,
"น": 35,
"บ": 36,
"ป": 37,
"ผ": 38,
"ฝ": 39,
"พ": 40,
"ฟ": 41,
"ภ": 42,
"ม": 43,
"ย": 44,
"ร": 45,
"ฤ": 46,
"ล": 47,
"ฦ": 48,
"ว": 49,
"ศ": 50,
"ษ": 51,
"ส": 52,
"ห": 53,
"ฬ": 54,
"อ": 55,
"ฮ": 56,
"ะ": 57,
"ั": 58,
"า": 59,
"ำ": 60,
"ิ": 61,
"ี": 62,
"ึ": 63,
"ื": 64,
"ุ": 65,
"ู": 66,
"ฺ": 67,
"฻": 68,
"฼": 69,
"฽": 70,
"฾": 71,
"฿": 72,
"เ": 73,
"แ": 74,
"โ": 75,
"ใ": 76,
"ไ": 77,
"ๅ": 78,
"ๆ": 79,
"็": 80,
"่": 81,
"้": 82,
"๊": 83,
"๋": 84,
"์": 85,
"ํ": 86,
"๎": 87,
"๐": 88,
"๑": 89,
"๒": 90,
"๓": 91,
"๔": 92,
"๕": 93,
"๖": 94,
"๗": 95,
"๘": 96,
"๙": 97,
".": 98,
",": 99,
"!": 100,
"?": 101,
";": 102,
":": 103,
"\"": 104,
"'": 105,
"(": 106,
")": 107,
"[": 108,
"]": 109,
"{": 110,
"}": 111
}