Upload tokenizer
Browse files- tokenizer.json +0 -162
- vocab.json +0 -0
tokenizer.json
CHANGED
|
@@ -14492,168 +14492,6 @@
|
|
| 14492 |
"rstrip": false,
|
| 14493 |
"normalized": true,
|
| 14494 |
"special": false
|
| 14495 |
-
},
|
| 14496 |
-
{
|
| 14497 |
-
"id": 51867,
|
| 14498 |
-
"content": "<|transcribenonverbal|>",
|
| 14499 |
-
"single_word": false,
|
| 14500 |
-
"lstrip": false,
|
| 14501 |
-
"rstrip": false,
|
| 14502 |
-
"normalized": true,
|
| 14503 |
-
"special": false
|
| 14504 |
-
},
|
| 14505 |
-
{
|
| 14506 |
-
"id": 51868,
|
| 14507 |
-
"content": "<|speech_effect|>",
|
| 14508 |
-
"single_word": false,
|
| 14509 |
-
"lstrip": false,
|
| 14510 |
-
"rstrip": false,
|
| 14511 |
-
"normalized": true,
|
| 14512 |
-
"special": false
|
| 14513 |
-
},
|
| 14514 |
-
{
|
| 14515 |
-
"id": 51869,
|
| 14516 |
-
"content": "<|blowing|>",
|
| 14517 |
-
"single_word": false,
|
| 14518 |
-
"lstrip": false,
|
| 14519 |
-
"rstrip": false,
|
| 14520 |
-
"normalized": true,
|
| 14521 |
-
"special": false
|
| 14522 |
-
},
|
| 14523 |
-
{
|
| 14524 |
-
"id": 51870,
|
| 14525 |
-
"content": "<|snorting|>",
|
| 14526 |
-
"single_word": false,
|
| 14527 |
-
"lstrip": false,
|
| 14528 |
-
"rstrip": false,
|
| 14529 |
-
"normalized": true,
|
| 14530 |
-
"special": false
|
| 14531 |
-
},
|
| 14532 |
-
{
|
| 14533 |
-
"id": 51871,
|
| 14534 |
-
"content": "<|imitative_sound|>",
|
| 14535 |
-
"single_word": false,
|
| 14536 |
-
"lstrip": false,
|
| 14537 |
-
"rstrip": false,
|
| 14538 |
-
"normalized": true,
|
| 14539 |
-
"special": false
|
| 14540 |
-
},
|
| 14541 |
-
{
|
| 14542 |
-
"id": 51872,
|
| 14543 |
-
"content": "<|whistling|>",
|
| 14544 |
-
"single_word": false,
|
| 14545 |
-
"lstrip": false,
|
| 14546 |
-
"rstrip": false,
|
| 14547 |
-
"normalized": true,
|
| 14548 |
-
"special": false
|
| 14549 |
-
},
|
| 14550 |
-
{
|
| 14551 |
-
"id": 51873,
|
| 14552 |
-
"content": "<|vocal_noise|>",
|
| 14553 |
-
"single_word": false,
|
| 14554 |
-
"lstrip": false,
|
| 14555 |
-
"rstrip": false,
|
| 14556 |
-
"normalized": true,
|
| 14557 |
-
"special": false
|
| 14558 |
-
},
|
| 14559 |
-
{
|
| 14560 |
-
"id": 51874,
|
| 14561 |
-
"content": "<|sound_effect|>",
|
| 14562 |
-
"single_word": false,
|
| 14563 |
-
"lstrip": false,
|
| 14564 |
-
"rstrip": false,
|
| 14565 |
-
"normalized": true,
|
| 14566 |
-
"special": false
|
| 14567 |
-
},
|
| 14568 |
-
{
|
| 14569 |
-
"id": 51875,
|
| 14570 |
-
"content": "<|sneeze|>",
|
| 14571 |
-
"single_word": false,
|
| 14572 |
-
"lstrip": false,
|
| 14573 |
-
"rstrip": false,
|
| 14574 |
-
"normalized": true,
|
| 14575 |
-
"special": false
|
| 14576 |
-
},
|
| 14577 |
-
{
|
| 14578 |
-
"id": 51876,
|
| 14579 |
-
"content": "<|cough|>",
|
| 14580 |
-
"single_word": false,
|
| 14581 |
-
"lstrip": false,
|
| 14582 |
-
"rstrip": false,
|
| 14583 |
-
"normalized": true,
|
| 14584 |
-
"special": false
|
| 14585 |
-
},
|
| 14586 |
-
{
|
| 14587 |
-
"id": 51877,
|
| 14588 |
-
"content": "<|laugh|>",
|
| 14589 |
-
"single_word": false,
|
| 14590 |
-
"lstrip": false,
|
| 14591 |
-
"rstrip": false,
|
| 14592 |
-
"normalized": true,
|
| 14593 |
-
"special": false
|
| 14594 |
-
},
|
| 14595 |
-
{
|
| 14596 |
-
"id": 51878,
|
| 14597 |
-
"content": "<|yawning|>",
|
| 14598 |
-
"single_word": false,
|
| 14599 |
-
"lstrip": false,
|
| 14600 |
-
"rstrip": false,
|
| 14601 |
-
"normalized": true,
|
| 14602 |
-
"special": false
|
| 14603 |
-
},
|
| 14604 |
-
{
|
| 14605 |
-
"id": 51879,
|
| 14606 |
-
"content": "<|mouth_sound|>",
|
| 14607 |
-
"single_word": false,
|
| 14608 |
-
"lstrip": false,
|
| 14609 |
-
"rstrip": false,
|
| 14610 |
-
"normalized": true,
|
| 14611 |
-
"special": false
|
| 14612 |
-
},
|
| 14613 |
-
{
|
| 14614 |
-
"id": 51880,
|
| 14615 |
-
"content": "<|sniff|>",
|
| 14616 |
-
"single_word": false,
|
| 14617 |
-
"lstrip": false,
|
| 14618 |
-
"rstrip": false,
|
| 14619 |
-
"normalized": true,
|
| 14620 |
-
"special": false
|
| 14621 |
-
},
|
| 14622 |
-
{
|
| 14623 |
-
"id": 51881,
|
| 14624 |
-
"content": "<|clicking|>",
|
| 14625 |
-
"single_word": false,
|
| 14626 |
-
"lstrip": false,
|
| 14627 |
-
"rstrip": false,
|
| 14628 |
-
"normalized": true,
|
| 14629 |
-
"special": false
|
| 14630 |
-
},
|
| 14631 |
-
{
|
| 14632 |
-
"id": 51882,
|
| 14633 |
-
"content": "<|nonverbal_expression|>",
|
| 14634 |
-
"single_word": false,
|
| 14635 |
-
"lstrip": false,
|
| 14636 |
-
"rstrip": false,
|
| 14637 |
-
"normalized": true,
|
| 14638 |
-
"special": false
|
| 14639 |
-
},
|
| 14640 |
-
{
|
| 14641 |
-
"id": 51883,
|
| 14642 |
-
"content": "<|sigh|>",
|
| 14643 |
-
"single_word": false,
|
| 14644 |
-
"lstrip": false,
|
| 14645 |
-
"rstrip": false,
|
| 14646 |
-
"normalized": true,
|
| 14647 |
-
"special": false
|
| 14648 |
-
},
|
| 14649 |
-
{
|
| 14650 |
-
"id": 51884,
|
| 14651 |
-
"content": "<|humming|>",
|
| 14652 |
-
"single_word": false,
|
| 14653 |
-
"lstrip": false,
|
| 14654 |
-
"rstrip": false,
|
| 14655 |
-
"normalized": true,
|
| 14656 |
-
"special": false
|
| 14657 |
}
|
| 14658 |
],
|
| 14659 |
"normalizer": null,
|
|
|
|
| 14492 |
"rstrip": false,
|
| 14493 |
"normalized": true,
|
| 14494 |
"special": false
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 14495 |
}
|
| 14496 |
],
|
| 14497 |
"normalizer": null,
|
vocab.json
CHANGED
|
The diff for this file is too large to render.
See raw diff
|
|
|