fusion-llm-demo / tokenizers /tokenizer.vocab
zhan1206
feat: train SentencePiece tokenizer + update README
ff4d952
Raw
History Blame
5.49 kB
<unk> 0
<s> 0
</s> 0
<pad> 0
<|pad|> 0
<|start|> 0
<|end|> 0
<|think_depth_0|> 0
<|think_depth_1|> 0
<|think_depth_2|> 0
<|think_depth_3|> 0
<0x00> 0
<0x01> 0
<0x02> 0
<0x03> 0
<0x04> 0
<0x05> 0
<0x06> 0
<0x07> 0
<0x08> 0
<0x09> 0
<0x0A> 0
<0x0B> 0
<0x0C> 0
<0x0D> 0
<0x0E> 0
<0x0F> 0
<0x10> 0
<0x11> 0
<0x12> 0
<0x13> 0
<0x14> 0
<0x15> 0
<0x16> 0
<0x17> 0
<0x18> 0
<0x19> 0
<0x1A> 0
<0x1B> 0
<0x1C> 0
<0x1D> 0
<0x1E> 0
<0x1F> 0
<0x20> 0
<0x21> 0
<0x22> 0
<0x23> 0
<0x24> 0
<0x25> 0
<0x26> 0
<0x27> 0
<0x28> 0
<0x29> 0
<0x2A> 0
<0x2B> 0
<0x2C> 0
<0x2D> 0
<0x2E> 0
<0x2F> 0
<0x30> 0
<0x31> 0
<0x32> 0
<0x33> 0
<0x34> 0
<0x35> 0
<0x36> 0
<0x37> 0
<0x38> 0
<0x39> 0
<0x3A> 0
<0x3B> 0
<0x3C> 0
<0x3D> 0
<0x3E> 0
<0x3F> 0
<0x40> 0
<0x41> 0
<0x42> 0
<0x43> 0
<0x44> 0
<0x45> 0
<0x46> 0
<0x47> 0
<0x48> 0
<0x49> 0
<0x4A> 0
<0x4B> 0
<0x4C> 0
<0x4D> 0
<0x4E> 0
<0x4F> 0
<0x50> 0
<0x51> 0
<0x52> 0
<0x53> 0
<0x54> 0
<0x55> 0
<0x56> 0
<0x57> 0
<0x58> 0
<0x59> 0
<0x5A> 0
<0x5B> 0
<0x5C> 0
<0x5D> 0
<0x5E> 0
<0x5F> 0
<0x60> 0
<0x61> 0
<0x62> 0
<0x63> 0
<0x64> 0
<0x65> 0
<0x66> 0
<0x67> 0
<0x68> 0
<0x69> 0
<0x6A> 0
<0x6B> 0
<0x6C> 0
<0x6D> 0
<0x6E> 0
<0x6F> 0
<0x70> 0
<0x71> 0
<0x72> 0
<0x73> 0
<0x74> 0
<0x75> 0
<0x76> 0
<0x77> 0
<0x78> 0
<0x79> 0
<0x7A> 0
<0x7B> 0
<0x7C> 0
<0x7D> 0
<0x7E> 0
<0x7F> 0
<0x80> 0
<0x81> 0
<0x82> 0
<0x83> 0
<0x84> 0
<0x85> 0
<0x86> 0
<0x87> 0
<0x88> 0
<0x89> 0
<0x8A> 0
<0x8B> 0
<0x8C> 0
<0x8D> 0
<0x8E> 0
<0x8F> 0
<0x90> 0
<0x91> 0
<0x92> 0
<0x93> 0
<0x94> 0
<0x95> 0
<0x96> 0
<0x97> 0
<0x98> 0
<0x99> 0
<0x9A> 0
<0x9B> 0
<0x9C> 0
<0x9D> 0
<0x9E> 0
<0x9F> 0
<0xA0> 0
<0xA1> 0
<0xA2> 0
<0xA3> 0
<0xA4> 0
<0xA5> 0
<0xA6> 0
<0xA7> 0
<0xA8> 0
<0xA9> 0
<0xAA> 0
<0xAB> 0
<0xAC> 0
<0xAD> 0
<0xAE> 0
<0xAF> 0
<0xB0> 0
<0xB1> 0
<0xB2> 0
<0xB3> 0
<0xB4> 0
<0xB5> 0
<0xB6> 0
<0xB7> 0
<0xB8> 0
<0xB9> 0
<0xBA> 0
<0xBB> 0
<0xBC> 0
<0xBD> 0
<0xBE> 0
<0xBF> 0
<0xC0> 0
<0xC1> 0
<0xC2> 0
<0xC3> 0
<0xC4> 0
<0xC5> 0
<0xC6> 0
<0xC7> 0
<0xC8> 0
<0xC9> 0
<0xCA> 0
<0xCB> 0
<0xCC> 0
<0xCD> 0
<0xCE> 0
<0xCF> 0
<0xD0> 0
<0xD1> 0
<0xD2> 0
<0xD3> 0
<0xD4> 0
<0xD5> 0
<0xD6> 0
<0xD7> 0
<0xD8> 0
<0xD9> 0
<0xDA> 0
<0xDB> 0
<0xDC> 0
<0xDD> 0
<0xDE> 0
<0xDF> 0
<0xE0> 0
<0xE1> 0
<0xE2> 0
<0xE3> 0
<0xE4> 0
<0xE5> 0
<0xE6> 0
<0xE7> 0
<0xE8> 0
<0xE9> 0
<0xEA> 0
<0xEB> 0
<0xEC> 0
<0xED> 0
<0xEE> 0
<0xEF> 0
<0xF0> 0
<0xF1> 0
<0xF2> 0
<0xF3> 0
<0xF4> 0
<0xF5> 0
<0xF6> 0
<0xF7> 0
<0xF8> 0
<0xF9> 0
<0xFA> 0
<0xFB> 0
<0xFC> 0
<0xFD> 0
<0xFE> 0
<0xFF> 0
▁ -2.44612
s -2.94286
l -3.33855
a -3.77109
t -3.82259
e -4.00136
o -4.03383
p -4.30998
f -4.40349
mode -4.40746
t -4.60183
language -4.62083
k -4.62094
- -4.62445
tion -4.7007
ge -4.72992
ce -4.85343
ra -4.86504
re -4.8947
计算机 -4.89926
的 -4.90304
learning -4.91083
computer -4.91117
A -4.92103
he -4.92372
ta -4.92573
u -4.99341
ch -5.00304
ine -5.15633
en -5.18019
te -5.18026
m -5.18134
ed -5.18892
i -5.19778
n -5.23153
c -5.23766
学习 -5.29817
语言 -5.30085
是 -5.30174
P -5.3058
N -5.30606
大 -5.30708
机 -5.30758
a -5.30847
人工智能 -5.30859
模型 -5.30988
rom -5.31076
中 -5.31088
任务 -5.31127
现 -5.31167
语言模型 -5.31206
learn -5.31216
L -5.31437
技术 -5.31487
使 -5.31626
Transformer -5.31634
模 -5.31696
b -5.31828
自然语言处理 -5.31847
深度学习 -5.32159
ow -5.32251
了 -5.32341
人 -5.32361
pro -5.32703
ex -5.32706
ne -5.34816
cor -5.35055
ific -5.35407
de -5.35822
nd -5.39694
mo -5.4074
r -5.41932
ur -5.43363
u -5.48994
is -5.49832
ar -5.53788
da -5.55829
ar -5.59715
m -5.65626
tu -5.66034
d -5.72945
tur -5.754
er -5.76365
chi -5.78109
es -5.78847
gen -5.83188
in -5.84722
ence -5.84828
at -5.87637
iz -5.92451
on -5.95904
ec -5.96185
ent -5.97889
w -5.97954
ing -5.98389
ti -5.98673
学 -5.98833
一 -5.98834
个 -5.98834
分 -5.98834
支 -5.98834
科 -5.98834
重 -5.98834
要 -5.98835
语 -5.99244
料 -5.99246
练 -5.99246
规 -5.99246
过 -5.99246
通 -5.99246
训 -5.99247
预 -5.99247
知 -5.99247
识 -5.99247
ci -5.99501
v -5.99642
D -5.99741
y -5.99741
T -5.99776
从 -5.99998
器 -5.99998
够 -5.99998
据 -5.99998
数 -5.99998
能 -5.99998
下 -6.00078
定 -6.00078
应 -6.00078
游 -6.00078
特 -6.00078
适 -6.00078
微 -6.00079
调 -6.00079
代 -6.00158
件 -6.00158
制 -6.00158
力 -6.00158
心 -6.00158
意 -6.00158
核 -6.00158
注 -6.00158
组 -6.00158
助 -6.00556
帮 -6.00556
理 -6.00556
类 -6.00557
解 -6.00557
or -6.00794
在 -6.00816
文 -6.00816
出 -6.00816
成 -6.00816
本 -6.00816
生 -6.00816
色 -6.00816
表 -6.00816
M -6.00977
学习使 -6.00979
comp -6.01112
F -6.01157
动 -6.0154
及 -6.0154
开 -6.0154
推 -6.0154
普 -6.0154
源 -6.0154
底 -6.01781
彻 -6.01781
改 -6.01781
变 -6.01782
域 -6.01782
构 -6.01782
架 -6.01782
领 -6.01782
ning -6.02384
多 -6.02818
层 -6.02818
拟 -6.02818
来 -6.02818
用 -6.02818
神 -6.02818
经 -6.02818
络 -6.02818
网 -6.02818
脑 -6.02818
rn -6.02895
I -6.03309
O -6.0331
rati -6.03329
s -6.0389
ran -6.04547
an -6.05951
d -6.06651
z -6.1136
po -6.12669
la -6.16874
tr -6.19337
ns -6.23596
ers -6.24996
pe -6.44462
le -6.48899
ne -6.54229
el -6.61066
ea -6.67065
me -6.87077
la -6.91637
i -7.05299
nce -7.16604
g -7.59023
nt -7.68809
ion -7.75095
l -7.82733
h -8.89587
A -11.5543
x -11.9128
N -11.9523
T -12.6852
深 -16.3599
度 -16.36
自 -16.3601
然 -16.3602
处 -16.3603
术 -16.3604
技 -16.3605
务 -16.3606
任 -16.3607
智 -16.3608
工 -16.3609
计 -16.361
算 -16.3611
型 -16.3612
习 -16.3613
言 -16.3614