Hugging Face's logo

Nexuss0781
/
Ethio-BBPE

Amharic
tokenizers
amharic
geez
ethiopic
biblical-texts
synaxarium
byte-level-bpe
Model card Files
xet
 Community
Nexuss0781 commited on
Commit
3c38dcf
·
verified ·
1 Parent(s): bfaa93b

Delete models/demo_tokenizer/tokenizer.json with huggingface_hub

Browse files
Files changed (1) hide show
  1. models/demo_tokenizer/tokenizer.json +0 -1986
models/demo_tokenizer/tokenizer.json DELETED
@@ -1,1986 +0,0 @@
1
- {
2
- "version": "1.0",
3
- "truncation": null,
4
- "padding": null,
5
- "added_tokens": [
6
- {
7
- "id": 0,
8
- "content": "<pad>",
9
- "single_word": false,
10
- "lstrip": false,
11
- "rstrip": false,
12
- "normalized": false,
13
- "special": true
14
- },
15
- {
16
- "id": 1,
17
- "content": "<unk>",
18
- "single_word": false,
19
- "lstrip": false,
20
- "rstrip": false,
21
- "normalized": false,
22
- "special": true
23
- },
24
- {
25
- "id": 2,
26
- "content": "<s>",
27
- "single_word": false,
28
- "lstrip": false,
29
- "rstrip": false,
30
- "normalized": false,
31
- "special": true
32
- },
33
- {
34
- "id": 3,
35
- "content": "</s>",
36
- "single_word": false,
37
- "lstrip": false,
38
- "rstrip": false,
39
- "normalized": false,
40
- "special": true
41
- },
42
- {
43
- "id": 4,
44
- "content": "<mask>",
45
- "single_word": false,
46
- "lstrip": false,
47
- "rstrip": false,
48
- "normalized": false,
49
- "special": true
50
- }
51
- ],
52
- "normalizer": null,
53
- "pre_tokenizer": {
54
- "type": "ByteLevel",
55
- "add_prefix_space": false,
56
- "trim_offsets": true,
57
- "use_regex": true
58
- },
59
- "post_processor": {
60
- "type": "ByteLevel",
61
- "add_prefix_space": true,
62
- "trim_offsets": true,
63
- "use_regex": true
64
- },
65
- "decoder": {
66
- "type": "ByteLevel",
67
- "add_prefix_space": true,
68
- "trim_offsets": true,
69
- "use_regex": true
70
- },
71
- "model": {
72
- "type": "BPE",
73
- "dropout": null,
74
- "unk_token": null,
75
- "continuing_subword_prefix": null,
76
- "end_of_word_suffix": null,
77
- "fuse_unk": false,
78
- "byte_fallback": false,
79
- "ignore_merges": false,
80
- "vocab": {
81
- "<pad>": 0,
82
- "<unk>": 1,
83
- "<s>": 2,
84
- "</s>": 3,
85
- "<mask>": 4,
86
- "!": 5,
87
- "\"": 6,
88
- "#": 7,
89
- "$": 8,
90
- "%": 9,
91
- "&": 10,
92
- "'": 11,
93
- "(": 12,
94
- ")": 13,
95
- "*": 14,
96
- "+": 15,
97
- ",": 16,
98
- "-": 17,
99
- ".": 18,
100
- "/": 19,
101
- "0": 20,
102
- "1": 21,
103
- "2": 22,
104
- "3": 23,
105
- "4": 24,
106
- "5": 25,
107
- "6": 26,
108
- "7": 27,
109
- "8": 28,
110
- "9": 29,
111
- ":": 30,
112
- ";": 31,
113
- "<": 32,
114
- "=": 33,
115
- ">": 34,
116
- "?": 35,
117
- "@": 36,
118
- "A": 37,
119
- "B": 38,
120
- "C": 39,
121
- "D": 40,
122
- "E": 41,
123
- "F": 42,
124
- "G": 43,
125
- "H": 44,
126
- "I": 45,
127
- "J": 46,
128
- "K": 47,
129
- "L": 48,
130
- "M": 49,
131
- "N": 50,
132
- "O": 51,
133
- "P": 52,
134
- "Q": 53,
135
- "R": 54,
136
- "S": 55,
137
- "T": 56,
138
- "U": 57,
139
- "V": 58,
140
- "W": 59,
141
- "X": 60,
142
- "Y": 61,
143
- "Z": 62,
144
- "[": 63,
145
- "\\": 64,
146
- "]": 65,
147
- "^": 66,
148
- "_": 67,
149
- "`": 68,
150
- "a": 69,
151
- "b": 70,
152
- "c": 71,
153
- "d": 72,
154
- "e": 73,
155
- "f": 74,
156
- "g": 75,
157
- "h": 76,
158
- "i": 77,
159
- "j": 78,
160
- "k": 79,
161
- "l": 80,
162
- "m": 81,
163
- "n": 82,
164
- "o": 83,
165
- "p": 84,
166
- "q": 85,
167
- "r": 86,
168
- "s": 87,
169
- "t": 88,
170
- "u": 89,
171
- "v": 90,
172
- "w": 91,
173
- "x": 92,
174
- "y": 93,
175
- "z": 94,
176
- "{": 95,
177
- "|": 96,
178
- "}": 97,
179
- "~": 98,
180
- "¡": 99,
181
- "¢": 100,
182
- "£": 101,
183
- "¤": 102,
184
- "¥": 103,
185
- "¦": 104,
186
- "§": 105,
187
- "¨": 106,
188
- "©": 107,
189
- "ª": 108,
190
- "«": 109,
191
- "¬": 110,
192
- "®": 111,
193
- "¯": 112,
194
- "°": 113,
195
- "±": 114,
196
- "²": 115,
197
- "³": 116,
198
- "´": 117,
199
- "µ": 118,
200
- "¶": 119,
201
- "·": 120,
202
- "¸": 121,
203
- "¹": 122,
204
- "º": 123,
205
- "»": 124,
206
- "¼": 125,
207
- "½": 126,
208
- "¾": 127,
209
- "¿": 128,
210
- "À": 129,
211
- "Á": 130,
212
- "Â": 131,
213
- "Ã": 132,
214
- "Ä": 133,
215
- "Å": 134,
216
- "Æ": 135,
217
- "Ç": 136,
218
- "È": 137,
219
- "É": 138,
220
- "Ê": 139,
221
- "Ë": 140,
222
- "Ì": 141,
223
- "Í": 142,
224
- "Î": 143,
225
- "Ï": 144,
226
- "Ð": 145,
227
- "Ñ": 146,
228
- "Ò": 147,
229
- "Ó": 148,
230
- "Ô": 149,
231
- "Õ": 150,
232
- "Ö": 151,
233
- "×": 152,
234
- "Ø": 153,
235
- "Ù": 154,
236
- "Ú": 155,
237
- "Û": 156,
238
- "Ü": 157,
239
- "Ý": 158,
240
- "Þ": 159,
241
- "ß": 160,
242
- "à": 161,
243
- "á": 162,
244
- "â": 163,
245
- "ã": 164,
246
- "ä": 165,
247
- "å": 166,
248
- "æ": 167,
249
- "ç": 168,
250
- "è": 169,
251
- "é": 170,
252
- "ê": 171,
253
- "ë": 172,
254
- "ì": 173,
255
- "í": 174,
256
- "î": 175,
257
- "ï": 176,
258
- "ð": 177,
259
- "ñ": 178,
260
- "ò": 179,
261
- "ó": 180,
262
- "ô": 181,
263
- "õ": 182,
264
- "ö": 183,
265
- "÷": 184,
266
- "ø": 185,
267
- "ù": 186,
268
- "ú": 187,
269
- "û": 188,
270
- "ü": 189,
271
- "ý": 190,
272
- "þ": 191,
273
- "ÿ": 192,
274
- "Ā": 193,
275
- "ā": 194,
276
- "Ă": 195,
277
- "ă": 196,
278
- "Ą": 197,
279
- "ą": 198,
280
- "Ć": 199,
281
- "ć": 200,
282
- "Ĉ": 201,
283
- "ĉ": 202,
284
- "Ċ": 203,
285
- "ċ": 204,
286
- "Č": 205,
287
- "č": 206,
288
- "Ď": 207,
289
- "ď": 208,
290
- "Đ": 209,
291
- "đ": 210,
292
- "Ē": 211,
293
- "ē": 212,
294
- "Ĕ": 213,
295
- "ĕ": 214,
296
- "Ė": 215,
297
- "ė": 216,
298
- "Ę": 217,
299
- "ę": 218,
300
- "Ě": 219,
301
- "ě": 220,
302
- "Ĝ": 221,
303
- "ĝ": 222,
304
- "Ğ": 223,
305
- "ğ": 224,
306
- "Ġ": 225,
307
- "ġ": 226,
308
- "Ģ": 227,
309
- "ģ": 228,
310
- "Ĥ": 229,
311
- "ĥ": 230,
312
- "Ħ": 231,
313
- "ħ": 232,
314
- "Ĩ": 233,
315
- "ĩ": 234,
316
- "Ī": 235,
317
- "ī": 236,
318
- "Ĭ": 237,
319
- "ĭ": 238,
320
- "Į": 239,
321
- "į": 240,
322
- "İ": 241,
323
- "ı": 242,
324
- "IJ": 243,
325
- "ij": 244,
326
- "Ĵ": 245,
327
- "ĵ": 246,
328
- "Ķ": 247,
329
- "ķ": 248,
330
- "ĸ": 249,
331
- "Ĺ": 250,
332
- "ĺ": 251,
333
- "Ļ": 252,
334
- "ļ": 253,
335
- "Ľ": 254,
336
- "ľ": 255,
337
- "Ŀ": 256,
338
- "ŀ": 257,
339
- "Ł": 258,
340
- "ł": 259,
341
- "Ń": 260,
342
- "Ġt": 261,
343
- "in": 262,
344
- "en": 263,
345
- "Ġa": 264,
346
- "er": 265,
347
- "es": 266,
348
- "Ġo": 267,
349
- "Ġto": 268,
350
- "an": 269,
351
- "ing": 270,
352
- "ar": 271,
353
- "at": 272,
354
- "he": 273,
355
- "is": 274,
356
- "or": 275,
357
- "Ġm": 276,
358
- "Ġs": 277,
359
- "al": 278,
360
- "ken": 279,
361
- "on": 280,
362
- "Ġtoken": 281,
363
- "le": 282,
364
- "Ġof": 283,
365
- "Ġb": 284,
366
- "Ġp": 285,
367
- "it": 286,
368
- "ou": 287,
369
- "Ġc": 288,
370
- "Ġf": 289,
371
- "Ġan": 290,
372
- "ic": 291,
373
- "iz": 292,
374
- "ion": 293,
375
- "ro": 294,
376
- "ed": 295,
377
- "el": 296,
378
- "re": 297,
379
- "Ġin": 298,
380
- "Ġthe": 299,
381
- "Ġis": 300,
382
- "Ġand": 301,
383
- "ĠT": 302,
384
- "Ġu": 303,
385
- "Ġbe": 304,
386
- "Ġd": 305,
387
- "Ġth": 306,
388
- "ac": 307,
389
- "mp": 308,
390
- "od": 309,
391
- "Ġw": 310,
392
- "Ġle": 311,
393
- "enc": 312,
394
- "ent": 313,
395
- "Ġtokeniz": 314,
396
- "as": 315,
397
- "ex": 316,
398
- "ter": 317,
399
- "Ġl": 318,
400
- "arn": 319,
401
- "Ġtokens": 320,
402
- "Ġpro": 321,
403
- "qu": 322,
404
- "ul": 323,
405
- "Ġh": 324,
406
- "Ġv": 325,
407
- "ation": 326,
408
- "Ġthat": 327,
409
- "Ġlearn": 328,
410
- "ain": 329,
411
- "ces": 330,
412
- "ec": 331,
413
- "fic": 332,
414
- "ge": 333,
415
- "il": 334,
416
- "res": 335,
417
- "ua": 336,
418
- "you": 337,
419
- "Ġn": 338,
420
- "Ġre": 339,
421
- "Ġyou": 340,
422
- "Ġtr": 341,
423
- "Ġtex": 342,
424
- "Ġun": 343,
425
- "cess": 344,
426
- "Ġtext": 345,
427
- "00": 346,
428
- "am": 347,
429
- "ds": 348,
430
- "em": 349,
431
- "et": 350,
432
- "gua": 351,
433
- "ific": 352,
434
- "oc": 353,
435
- "per": 354,
436
- "st": 355,
437
- "ur": 356,
438
- "ve": 357,
439
- "Ġon": 358,
440
- "angua": 359,
441
- "ords": 360,
442
- "Ġmod": 361,
443
- "ence": 362,
444
- "Ġtokenizer": 363,
445
- "ular": 364,
446
- "Ġlearning": 365,
447
- "aining": 366,
448
- "ab": 367,
449
- "ch": 368,
450
- "for": 369,
451
- "hin": 370,
452
- "his": 371,
453
- "ial": 372,
454
- "imp": 373,
455
- "ks": 374,
456
- "se": 375,
457
- "su": 376,
458
- "us": 377,
459
- "ĠB": 378,
460
- "Ġ[": 379,
461
- "Ġimp": 380,
462
- "Ġar": 381,
463
- "Ġor": 382,
464
- "ans": 383,
465
- "ate": 384,
466
- "ith": 385,
467
- "Ġcan": 386,
468
- "Ġfor": 387,
469
- "achin": 388,
470
- "Ġwords": 389,
471
- "Ġlangua": 390,
472
- "Ġprocess": 391,
473
- "Ġyour": 392,
474
- "ural": 393,
475
- "Ġmodel": 394,
476
- "achine": 395,
477
- "LP": 396,
478
- "NLP": 397,
479
- "PE": 398,
480
- "The": 399,
481
- "],": 400,
482
- "ad": 401,
483
- "ded": 402,
484
- "ev": 403,
485
- "gh": 404,
486
- "id": 405,
487
- "im": 406,
488
- "ld": 407,
489
- "lp": 408,
490
- "lu": 409,
491
- "ly": 410,
492
- "pl": 411,
493
- "te": 412,
494
- "tion": 413,
495
- "tific": 414,
496
- "um": 415,
497
- "vis": 416,
498
- "wn": 417,
499
- "yte": 418,
500
- "Ġen": 419,
501
- "Ġhe": 420,
502
- "Ġit": 421,
503
- "Ġqu": 422,
504
- "art": 423,
505
- "ata": 424,
506
- "orp": 425,
507
- "Ġmu": 426,
508
- "Ġman": 427,
509
- "Ġsu": 428,
510
- "ont": 429,
511
- "ity": 430,
512
- "out": 431,
513
- "Ġcorp": 432,
514
- "Ġcont": 433,
515
- "ions": 434,
516
- "ell": 435,
517
- "Ġint": 436,
518
- "ĠThe": 437,
519
- "Ġdata": 438,
520
- "Ġwith": 439,
521
- "Ġval": 440,
522
- "Ġvoc": 441,
523
- "Ġtraining": 442,
524
- "Ġtrans": 443,
525
- "pervis": 444,
526
- "abular": 445,
527
- "form": 446,
528
- "Ġartific": 447,
529
- "Ġlanguage": 448,
530
- "Ġmodels": 449,
531
- "Ġvocabular": 450,
532
- "pervised": 451,
533
- "Ġartificial": 452,
534
- "BPE": 453,
535
- "Byte": 454,
536
- "Machine": 455,
537
- "This": 456,
538
- "ak": 457,
539
- "amp": 458,
540
- "ave": 459,
541
- "clu": 460,
542
- "ding": 461,
543
- "dded": 462,
544
- "een": 463,
545
- "equ": 464,
546
- "eural": 465,
547
- "gr": 466,
548
- "gence": 467,
549
- "her": 468,
550
- "hes": 469,
551
- "ies": 470,
552
- "ive": 471,
553
- "igence": 472,
554
- "kn": 473,
555
- "lions": 474,
556
- "ms": 475,
557
- "mal": 476,
558
- "om": 477,
559
- "oo": 478,
560
- "op": 479,
561
- "omp": 480,
562
- "own": 481,
563
- "pp": 482,
564
- "pic": 483,
565
- "pec": 484,
566
- "pres": 485,
567
- "ts": 486,
568
- "tw": 487,
569
- "ten": 488,
570
- "tan": 489,
571
- "ting": 490,
572
- "ut": 491,
573
- "uter": 492,
574
- "ver": 493,
575
- "vel": 494,
576
- "vid": 495,
577
- "wor": 496,
578
- "ypic": 497,
579
- "Ġ(": 498,
580
- "Ġ2": 499,
581
- "ĠI": 500,
582
- "ĠL": 501,
583
- "ĠP": 502,
584
- "Ġr": 503,
585
- "Ġex": 504,
586
- "ĠNLP": 505,
587
- "Ġtas": 506,
588
- "Ġal": 507,
589
- "Ġat": 508,
590
- "Ġare": 509,
591
- "Ġadded": 510,
592
- "and": 511,
593
- "atural": 512,
594
- "ortan": 513,
595
- "Ġmachine": 514,
596
- "Ġmak": 515,
597
- "Ġsiz": 516,
598
- "Ġsent": 517,
599
- "Ġsamp": 518,
600
- "Ġsequ": 519,
601
- "Ġsmal": 520,
602
- "alle": 521,
603
- "level": 522,
604
- "Ġoften": 523,
605
- "Ġbro": 524,
606
- "Ġpart": 525,
607
- "ous": 526,
608
- "ough": 527,
609
- "Ġcomp": 528,
610
- "Ġfro": 529,
611
- "Ġfam": 530,
612
- "Ġfoc": 531,
613
- "Ġinclu": 532,
614
- "ĠThis": 533,
615
- "ĠThes": 534,
616
- "ĠToo": 535,
617
- "Ġus": 536,
618
- "Ġuse": 537,
619
- "Ġbetw": 538,
620
- "Ġdi": 539,
621
- "oding": 540,
622
- "ences": 541,
623
- "Ġtokenization": 542,
624
- "ased": 543,
625
- "Ġlar": 544,
626
- "Ġprogr": 545,
627
- "Ġprovid": 546,
628
- "Ġhum": 547,
629
- "Ġhave": 548,
630
- "ect": 549,
631
- "illions": 550,
632
- "Ġnet": 551,
633
- "Ġneural": 552,
634
- "Ġrepres": 553,
635
- "Ġunkn": 554,
636
- "000": 555,
637
- "supervised": 556,
638
- "uses": 557,
639
- "ĠBBPE": 558,
640
- "Ġimportan": 559,
641
- "plic": 560,
642
- "Ġensu": 561,
643
- "Ġhelp": 562,
644
- "Ġmuch": 563,
645
- "Ġmany": 564,
646
- "Ġsub": 565,
647
- "Ġcorpus": 566,
648
- "elligence": 567,
649
- "Ġintelligence": 568,
650
- "Ġwithout": 569,
651
- "Ġtransform": 570,
652
- "Ġvocabulary": 571,
653
- "works": 572,
654
- "ypical": 573,
655
- "Ġtasks": 574,
656
- "Ġsample": 575,
657
- "Ġsmall": 576,
658
- "Ġcomputer": 577,
659
- "Ġfrom": 578,
660
- "Ġfocuses": 579,
661
- "ĠThese": 580,
662
- "Ġbetween": 581,
663
- "Ġprogram": 582,
664
- "Ġprovides": 583,
665
- "Ġnetworks": 584,
666
- "Ġrepresent": 585,
667
- "Ġunknown": 586,
668
- "Ġimportant": 587,
669
- "Ġtransformer": 588
670
- },
671
- "merges": [
672
- [
673
- "Ġ",
674
- "t"
675
- ],
676
- [
677
- "i",
678
- "n"
679
- ],
680
- [
681
- "e",
682
- "n"
683
- ],
684
- [
685
- "Ġ",
686
- "a"
687
- ],
688
- [
689
- "e",
690
- "r"
691
- ],
692
- [
693
- "e",
694
- "s"
695
- ],
696
- [
697
- "Ġ",
698
- "o"
699
- ],
700
- [
701
- "Ġt",
702
- "o"
703
- ],
704
- [
705
- "a",
706
- "n"
707
- ],
708
- [
709
- "in",
710
- "g"
711
- ],
712
- [
713
- "a",
714
- "r"
715
- ],
716
- [
717
- "a",
718
- "t"
719
- ],
720
- [
721
- "h",
722
- "e"
723
- ],
724
- [
725
- "i",
726
- "s"
727
- ],
728
- [
729
- "o",
730
- "r"
731
- ],
732
- [
733
- "Ġ",
734
- "m"
735
- ],
736
- [
737
- "Ġ",
738
- "s"
739
- ],
740
- [
741
- "a",
742
- "l"
743
- ],
744
- [
745
- "k",
746
- "en"
747
- ],
748
- [
749
- "o",
750
- "n"
751
- ],
752
- [
753
- "Ġto",
754
- "ken"
755
- ],
756
- [
757
- "l",
758
- "e"
759
- ],
760
- [
761
- "Ġo",
762
- "f"
763
- ],
764
- [
765
- "Ġ",
766
- "b"
767
- ],
768
- [
769
- "Ġ",
770
- "p"
771
- ],
772
- [
773
- "i",
774
- "t"
775
- ],
776
- [
777
- "o",
778
- "u"
779
- ],
780
- [
781
- "Ġ",
782
- "c"
783
- ],
784
- [
785
- "Ġ",
786
- "f"
787
- ],
788
- [
789
- "Ġa",
790
- "n"
791
- ],
792
- [
793
- "i",
794
- "c"
795
- ],
796
- [
797
- "i",
798
- "z"
799
- ],
800
- [
801
- "i",
802
- "on"
803
- ],
804
- [
805
- "r",
806
- "o"
807
- ],
808
- [
809
- "e",
810
- "d"
811
- ],
812
- [
813
- "e",
814
- "l"
815
- ],
816
- [
817
- "r",
818
- "e"
819
- ],
820
- [
821
- "Ġ",
822
- "in"
823
- ],
824
- [
825
- "Ġt",
826
- "he"
827
- ],
828
- [
829
- "Ġ",
830
- "is"
831
- ],
832
- [
833
- "Ġan",
834
- "d"
835
- ],
836
- [
837
- "Ġ",
838
- "T"
839
- ],
840
- [
841
- "Ġ",
842
- "u"
843
- ],
844
- [
845
- "Ġb",
846
- "e"
847
- ],
848
- [
849
- "Ġ",
850
- "d"
851
- ],
852
- [
853
- "Ġt",
854
- "h"
855
- ],
856
- [
857
- "a",
858
- "c"
859
- ],
860
- [
861
- "m",
862
- "p"
863
- ],
864
- [
865
- "o",
866
- "d"
867
- ],
868
- [
869
- "Ġ",
870
- "w"
871
- ],
872
- [
873
- "Ġ",
874
- "le"
875
- ],
876
- [
877
- "en",
878
- "c"
879
- ],
880
- [
881
- "en",
882
- "t"
883
- ],
884
- [
885
- "Ġtoken",
886
- "iz"
887
- ],
888
- [
889
- "a",
890
- "s"
891
- ],
892
- [
893
- "e",
894
- "x"
895
- ],
896
- [
897
- "t",
898
- "er"
899
- ],
900
- [
901
- "Ġ",
902
- "l"
903
- ],
904
- [
905
- "ar",
906
- "n"
907
- ],
908
- [
909
- "Ġtoken",
910
- "s"
911
- ],
912
- [
913
- "Ġp",
914
- "ro"
915
- ],
916
- [
917
- "q",
918
- "u"
919
- ],
920
- [
921
- "u",
922
- "l"
923
- ],
924
- [
925
- "Ġ",
926
- "h"
927
- ],
928
- [
929
- "Ġ",
930
- "v"
931
- ],
932
- [
933
- "at",
934
- "ion"
935
- ],
936
- [
937
- "Ġth",
938
- "at"
939
- ],
940
- [
941
- "Ġle",
942
- "arn"
943
- ],
944
- [
945
- "a",
946
- "in"
947
- ],
948
- [
949
- "c",
950
- "es"
951
- ],
952
- [
953
- "e",
954
- "c"
955
- ],
956
- [
957
- "f",
958
- "ic"
959
- ],
960
- [
961
- "g",
962
- "e"
963
- ],
964
- [
965
- "i",
966
- "l"
967
- ],
968
- [
969
- "r",
970
- "es"
971
- ],
972
- [
973
- "u",
974
- "a"
975
- ],
976
- [
977
- "y",
978
- "ou"
979
- ],
980
- [
981
- "Ġ",
982
- "n"
983
- ],
984
- [
985
- "Ġ",
986
- "re"
987
- ],
988
- [
989
- "Ġ",
990
- "you"
991
- ],
992
- [
993
- "Ġt",
994
- "r"
995
- ],
996
- [
997
- "Ġt",
998
- "ex"
999
- ],
1000
- [
1001
- "Ġu",
1002
- "n"
1003
- ],
1004
- [
1005
- "ces",
1006
- "s"
1007
- ],
1008
- [
1009
- "Ġtex",
1010
- "t"
1011
- ],
1012
- [
1013
- "0",
1014
- "0"
1015
- ],
1016
- [
1017
- "a",
1018
- "m"
1019
- ],
1020
- [
1021
- "d",
1022
- "s"
1023
- ],
1024
- [
1025
- "e",
1026
- "m"
1027
- ],
1028
- [
1029
- "e",
1030
- "t"
1031
- ],
1032
- [
1033
- "g",
1034
- "ua"
1035
- ],
1036
- [
1037
- "i",
1038
- "fic"
1039
- ],
1040
- [
1041
- "o",
1042
- "c"
1043
- ],
1044
- [
1045
- "p",
1046
- "er"
1047
- ],
1048
- [
1049
- "s",
1050
- "t"
1051
- ],
1052
- [
1053
- "u",
1054
- "r"
1055
- ],
1056
- [
1057
- "v",
1058
- "e"
1059
- ],
1060
- [
1061
- "Ġo",
1062
- "n"
1063
- ],
1064
- [
1065
- "an",
1066
- "gua"
1067
- ],
1068
- [
1069
- "or",
1070
- "ds"
1071
- ],
1072
- [
1073
- "Ġm",
1074
- "od"
1075
- ],
1076
- [
1077
- "enc",
1078
- "e"
1079
- ],
1080
- [
1081
- "Ġtokeniz",
1082
- "er"
1083
- ],
1084
- [
1085
- "ul",
1086
- "ar"
1087
- ],
1088
- [
1089
- "Ġlearn",
1090
- "ing"
1091
- ],
1092
- [
1093
- "ain",
1094
- "ing"
1095
- ],
1096
- [
1097
- "a",
1098
- "b"
1099
- ],
1100
- [
1101
- "c",
1102
- "h"
1103
- ],
1104
- [
1105
- "f",
1106
- "or"
1107
- ],
1108
- [
1109
- "h",
1110
- "in"
1111
- ],
1112
- [
1113
- "h",
1114
- "is"
1115
- ],
1116
- [
1117
- "i",
1118
- "al"
1119
- ],
1120
- [
1121
- "i",
1122
- "mp"
1123
- ],
1124
- [
1125
- "k",
1126
- "s"
1127
- ],
1128
- [
1129
- "s",
1130
- "e"
1131
- ],
1132
- [
1133
- "s",
1134
- "u"
1135
- ],
1136
- [
1137
- "u",
1138
- "s"
1139
- ],
1140
- [
1141
- "Ġ",
1142
- "B"
1143
- ],
1144
- [
1145
- "Ġ",
1146
- "["
1147
- ],
1148
- [
1149
- "Ġ",
1150
- "imp"
1151
- ],
1152
- [
1153
- "Ġa",
1154
- "r"
1155
- ],
1156
- [
1157
- "Ġo",
1158
- "r"
1159
- ],
1160
- [
1161
- "an",
1162
- "s"
1163
- ],
1164
- [
1165
- "at",
1166
- "e"
1167
- ],
1168
- [
1169
- "it",
1170
- "h"
1171
- ],
1172
- [
1173
- "Ġc",
1174
- "an"
1175
- ],
1176
- [
1177
- "Ġf",
1178
- "or"
1179
- ],
1180
- [
1181
- "ac",
1182
- "hin"
1183
- ],
1184
- [
1185
- "Ġw",
1186
- "ords"
1187
- ],
1188
- [
1189
- "Ġl",
1190
- "angua"
1191
- ],
1192
- [
1193
- "Ġpro",
1194
- "cess"
1195
- ],
1196
- [
1197
- "Ġyou",
1198
- "r"
1199
- ],
1200
- [
1201
- "ur",
1202
- "al"
1203
- ],
1204
- [
1205
- "Ġmod",
1206
- "el"
1207
- ],
1208
- [
1209
- "achin",
1210
- "e"
1211
- ],
1212
- [
1213
- "L",
1214
- "P"
1215
- ],
1216
- [
1217
- "N",
1218
- "LP"
1219
- ],
1220
- [
1221
- "P",
1222
- "E"
1223
- ],
1224
- [
1225
- "T",
1226
- "he"
1227
- ],
1228
- [
1229
- "]",
1230
- ","
1231
- ],
1232
- [
1233
- "a",
1234
- "d"
1235
- ],
1236
- [
1237
- "d",
1238
- "ed"
1239
- ],
1240
- [
1241
- "e",
1242
- "v"
1243
- ],
1244
- [
1245
- "g",
1246
- "h"
1247
- ],
1248
- [
1249
- "i",
1250
- "d"
1251
- ],
1252
- [
1253
- "i",
1254
- "m"
1255
- ],
1256
- [
1257
- "l",
1258
- "d"
1259
- ],
1260
- [
1261
- "l",
1262
- "p"
1263
- ],
1264
- [
1265
- "l",
1266
- "u"
1267
- ],
1268
- [
1269
- "l",
1270
- "y"
1271
- ],
1272
- [
1273
- "p",
1274
- "l"
1275
- ],
1276
- [
1277
- "t",
1278
- "e"
1279
- ],
1280
- [
1281
- "t",
1282
- "ion"
1283
- ],
1284
- [
1285
- "t",
1286
- "ific"
1287
- ],
1288
- [
1289
- "u",
1290
- "m"
1291
- ],
1292
- [
1293
- "v",
1294
- "is"
1295
- ],
1296
- [
1297
- "w",
1298
- "n"
1299
- ],
1300
- [
1301
- "y",
1302
- "te"
1303
- ],
1304
- [
1305
- "Ġ",
1306
- "en"
1307
- ],
1308
- [
1309
- "Ġ",
1310
- "he"
1311
- ],
1312
- [
1313
- "Ġ",
1314
- "it"
1315
- ],
1316
- [
1317
- "Ġ",
1318
- "qu"
1319
- ],
1320
- [
1321
- "ar",
1322
- "t"
1323
- ],
1324
- [
1325
- "at",
1326
- "a"
1327
- ],
1328
- [
1329
- "or",
1330
- "p"
1331
- ],
1332
- [
1333
- "Ġm",
1334
- "u"
1335
- ],
1336
- [
1337
- "Ġm",
1338
- "an"
1339
- ],
1340
- [
1341
- "Ġs",
1342
- "u"
1343
- ],
1344
- [
1345
- "on",
1346
- "t"
1347
- ],
1348
- [
1349
- "it",
1350
- "y"
1351
- ],
1352
- [
1353
- "ou",
1354
- "t"
1355
- ],
1356
- [
1357
- "Ġc",
1358
- "orp"
1359
- ],
1360
- [
1361
- "Ġc",
1362
- "ont"
1363
- ],
1364
- [
1365
- "ion",
1366
- "s"
1367
- ],
1368
- [
1369
- "el",
1370
- "l"
1371
- ],
1372
- [
1373
- "Ġin",
1374
- "t"
1375
- ],
1376
- [
1377
- "ĠT",
1378
- "he"
1379
- ],
1380
- [
1381
- "Ġd",
1382
- "ata"
1383
- ],
1384
- [
1385
- "Ġw",
1386
- "ith"
1387
- ],
1388
- [
1389
- "Ġv",
1390
- "al"
1391
- ],
1392
- [
1393
- "Ġv",
1394
- "oc"
1395
- ],
1396
- [
1397
- "Ġtr",
1398
- "aining"
1399
- ],
1400
- [
1401
- "Ġtr",
1402
- "ans"
1403
- ],
1404
- [
1405
- "per",
1406
- "vis"
1407
- ],
1408
- [
1409
- "ab",
1410
- "ular"
1411
- ],
1412
- [
1413
- "for",
1414
- "m"
1415
- ],
1416
- [
1417
- "Ġar",
1418
- "tific"
1419
- ],
1420
- [
1421
- "Ġlangua",
1422
- "ge"
1423
- ],
1424
- [
1425
- "Ġmodel",
1426
- "s"
1427
- ],
1428
- [
1429
- "Ġvoc",
1430
- "abular"
1431
- ],
1432
- [
1433
- "pervis",
1434
- "ed"
1435
- ],
1436
- [
1437
- "Ġartific",
1438
- "ial"
1439
- ],
1440
- [
1441
- "B",
1442
- "PE"
1443
- ],
1444
- [
1445
- "B",
1446
- "yte"
1447
- ],
1448
- [
1449
- "M",
1450
- "achine"
1451
- ],
1452
- [
1453
- "T",
1454
- "his"
1455
- ],
1456
- [
1457
- "a",
1458
- "k"
1459
- ],
1460
- [
1461
- "a",
1462
- "mp"
1463
- ],
1464
- [
1465
- "a",
1466
- "ve"
1467
- ],
1468
- [
1469
- "c",
1470
- "lu"
1471
- ],
1472
- [
1473
- "d",
1474
- "ing"
1475
- ],
1476
- [
1477
- "d",
1478
- "ded"
1479
- ],
1480
- [
1481
- "e",
1482
- "en"
1483
- ],
1484
- [
1485
- "e",
1486
- "qu"
1487
- ],
1488
- [
1489
- "e",
1490
- "ural"
1491
- ],
1492
- [
1493
- "g",
1494
- "r"
1495
- ],
1496
- [
1497
- "g",
1498
- "ence"
1499
- ],
1500
- [
1501
- "h",
1502
- "er"
1503
- ],
1504
- [
1505
- "h",
1506
- "es"
1507
- ],
1508
- [
1509
- "i",
1510
- "es"
1511
- ],
1512
- [
1513
- "i",
1514
- "ve"
1515
- ],
1516
- [
1517
- "i",
1518
- "gence"
1519
- ],
1520
- [
1521
- "k",
1522
- "n"
1523
- ],
1524
- [
1525
- "l",
1526
- "ions"
1527
- ],
1528
- [
1529
- "m",
1530
- "s"
1531
- ],
1532
- [
1533
- "m",
1534
- "al"
1535
- ],
1536
- [
1537
- "o",
1538
- "m"
1539
- ],
1540
- [
1541
- "o",
1542
- "o"
1543
- ],
1544
- [
1545
- "o",
1546
- "p"
1547
- ],
1548
- [
1549
- "o",
1550
- "mp"
1551
- ],
1552
- [
1553
- "o",
1554
- "wn"
1555
- ],
1556
- [
1557
- "p",
1558
- "p"
1559
- ],
1560
- [
1561
- "p",
1562
- "ic"
1563
- ],
1564
- [
1565
- "p",
1566
- "ec"
1567
- ],
1568
- [
1569
- "p",
1570
- "res"
1571
- ],
1572
- [
1573
- "t",
1574
- "s"
1575
- ],
1576
- [
1577
- "t",
1578
- "w"
1579
- ],
1580
- [
1581
- "t",
1582
- "en"
1583
- ],
1584
- [
1585
- "t",
1586
- "an"
1587
- ],
1588
- [
1589
- "t",
1590
- "ing"
1591
- ],
1592
- [
1593
- "u",
1594
- "t"
1595
- ],
1596
- [
1597
- "u",
1598
- "ter"
1599
- ],
1600
- [
1601
- "v",
1602
- "er"
1603
- ],
1604
- [
1605
- "v",
1606
- "el"
1607
- ],
1608
- [
1609
- "v",
1610
- "id"
1611
- ],
1612
- [
1613
- "w",
1614
- "or"
1615
- ],
1616
- [
1617
- "y",
1618
- "pic"
1619
- ],
1620
- [
1621
- "Ġ",
1622
- "("
1623
- ],
1624
- [
1625
- "Ġ",
1626
- "2"
1627
- ],
1628
- [
1629
- "Ġ",
1630
- "I"
1631
- ],
1632
- [
1633
- "Ġ",
1634
- "L"
1635
- ],
1636
- [
1637
- "Ġ",
1638
- "P"
1639
- ],
1640
- [
1641
- "Ġ",
1642
- "r"
1643
- ],
1644
- [
1645
- "Ġ",
1646
- "ex"
1647
- ],
1648
- [
1649
- "Ġ",
1650
- "NLP"
1651
- ],
1652
- [
1653
- "Ġt",
1654
- "as"
1655
- ],
1656
- [
1657
- "Ġa",
1658
- "l"
1659
- ],
1660
- [
1661
- "Ġa",
1662
- "t"
1663
- ],
1664
- [
1665
- "Ġa",
1666
- "re"
1667
- ],
1668
- [
1669
- "Ġa",
1670
- "dded"
1671
- ],
1672
- [
1673
- "an",
1674
- "d"
1675
- ],
1676
- [
1677
- "at",
1678
- "ural"
1679
- ],
1680
- [
1681
- "or",
1682
- "tan"
1683
- ],
1684
- [
1685
- "Ġm",
1686
- "achine"
1687
- ],
1688
- [
1689
- "Ġm",
1690
- "ak"
1691
- ],
1692
- [
1693
- "Ġs",
1694
- "iz"
1695
- ],
1696
- [
1697
- "Ġs",
1698
- "ent"
1699
- ],
1700
- [
1701
- "Ġs",
1702
- "amp"
1703
- ],
1704
- [
1705
- "Ġs",
1706
- "equ"
1707
- ],
1708
- [
1709
- "Ġs",
1710
- "mal"
1711
- ],
1712
- [
1713
- "al",
1714
- "le"
1715
- ],
1716
- [
1717
- "le",
1718
- "vel"
1719
- ],
1720
- [
1721
- "Ġof",
1722
- "ten"
1723
- ],
1724
- [
1725
- "Ġb",
1726
- "ro"
1727
- ],
1728
- [
1729
- "Ġp",
1730
- "art"
1731
- ],
1732
- [
1733
- "ou",
1734
- "s"
1735
- ],
1736
- [
1737
- "ou",
1738
- "gh"
1739
- ],
1740
- [
1741
- "Ġc",
1742
- "omp"
1743
- ],
1744
- [
1745
- "Ġf",
1746
- "ro"
1747
- ],
1748
- [
1749
- "Ġf",
1750
- "am"
1751
- ],
1752
- [
1753
- "Ġf",
1754
- "oc"
1755
- ],
1756
- [
1757
- "Ġin",
1758
- "clu"
1759
- ],
1760
- [
1761
- "ĠT",
1762
- "his"
1763
- ],
1764
- [
1765
- "ĠT",
1766
- "hes"
1767
- ],
1768
- [
1769
- "ĠT",
1770
- "oo"
1771
- ],
1772
- [
1773
- "Ġu",
1774
- "s"
1775
- ],
1776
- [
1777
- "Ġu",
1778
- "se"
1779
- ],
1780
- [
1781
- "Ġbe",
1782
- "tw"
1783
- ],
1784
- [
1785
- "Ġd",
1786
- "i"
1787
- ],
1788
- [
1789
- "od",
1790
- "ing"
1791
- ],
1792
- [
1793
- "enc",
1794
- "es"
1795
- ],
1796
- [
1797
- "Ġtokeniz",
1798
- "ation"
1799
- ],
1800
- [
1801
- "as",
1802
- "ed"
1803
- ],
1804
- [
1805
- "Ġl",
1806
- "ar"
1807
- ],
1808
- [
1809
- "Ġpro",
1810
- "gr"
1811
- ],
1812
- [
1813
- "Ġpro",
1814
- "vid"
1815
- ],
1816
- [
1817
- "Ġh",
1818
- "um"
1819
- ],
1820
- [
1821
- "Ġh",
1822
- "ave"
1823
- ],
1824
- [
1825
- "ec",
1826
- "t"
1827
- ],
1828
- [
1829
- "il",
1830
- "lions"
1831
- ],
1832
- [
1833
- "Ġn",
1834
- "et"
1835
- ],
1836
- [
1837
- "Ġn",
1838
- "eural"
1839
- ],
1840
- [
1841
- "Ġre",
1842
- "pres"
1843
- ],
1844
- [
1845
- "Ġun",
1846
- "kn"
1847
- ],
1848
- [
1849
- "00",
1850
- "0"
1851
- ],
1852
- [
1853
- "su",
1854
- "pervised"
1855
- ],
1856
- [
1857
- "us",
1858
- "es"
1859
- ],
1860
- [
1861
- "ĠB",
1862
- "BPE"
1863
- ],
1864
- [
1865
- "Ġimp",
1866
- "ortan"
1867
- ],
1868
- [
1869
- "pl",
1870
- "ic"
1871
- ],
1872
- [
1873
- "Ġen",
1874
- "su"
1875
- ],
1876
- [
1877
- "Ġhe",
1878
- "lp"
1879
- ],
1880
- [
1881
- "Ġmu",
1882
- "ch"
1883
- ],
1884
- [
1885
- "Ġman",
1886
- "y"
1887
- ],
1888
- [
1889
- "Ġsu",
1890
- "b"
1891
- ],
1892
- [
1893
- "Ġcorp",
1894
- "us"
1895
- ],
1896
- [
1897
- "ell",
1898
- "igence"
1899
- ],
1900
- [
1901
- "Ġint",
1902
- "elligence"
1903
- ],
1904
- [
1905
- "Ġwith",
1906
- "out"
1907
- ],
1908
- [
1909
- "Ġtrans",
1910
- "form"
1911
- ],
1912
- [
1913
- "Ġvocabular",
1914
- "y"
1915
- ],
1916
- [
1917
- "wor",
1918
- "ks"
1919
- ],
1920
- [
1921
- "ypic",
1922
- "al"
1923
- ],
1924
- [
1925
- "Ġtas",
1926
- "ks"
1927
- ],
1928
- [
1929
- "Ġsamp",
1930
- "le"
1931
- ],
1932
- [
1933
- "Ġsmal",
1934
- "l"
1935
- ],
1936
- [
1937
- "Ġcomp",
1938
- "uter"
1939
- ],
1940
- [
1941
- "Ġfro",
1942
- "m"
1943
- ],
1944
- [
1945
- "Ġfoc",
1946
- "uses"
1947
- ],
1948
- [
1949
- "ĠThes",
1950
- "e"
1951
- ],
1952
- [
1953
- "Ġbetw",
1954
- "een"
1955
- ],
1956
- [
1957
- "Ġprogr",
1958
- "am"
1959
- ],
1960
- [
1961
- "Ġprovid",
1962
- "es"
1963
- ],
1964
- [
1965
- "Ġnet",
1966
- "works"
1967
- ],
1968
- [
1969
- "Ġrepres",
1970
- "ent"
1971
- ],
1972
- [
1973
- "Ġunkn",
1974
- "own"
1975
- ],
1976
- [
1977
- "Ġimportan",
1978
- "t"
1979
- ],
1980
- [
1981
- "Ġtransform",
1982
- "er"
1983
- ]
1984
- ]
1985
- }
1986
- }