Update tokenization_small100.py
Browse files- tokenization_small100.py +1 -1
tokenization_small100.py
CHANGED
|
@@ -24,7 +24,7 @@ from typing import Any, Dict, List, Optional, Tuple, Union
|
|
| 24 |
|
| 25 |
import sentencepiece
|
| 26 |
|
| 27 |
-
from transformers
|
| 28 |
from transformers.utils import logging
|
| 29 |
|
| 30 |
|
|
|
|
| 24 |
|
| 25 |
import sentencepiece
|
| 26 |
|
| 27 |
+
from transformers import BatchEncoding, PreTrainedTokenizer
|
| 28 |
from transformers.utils import logging
|
| 29 |
|
| 30 |
|