Update pipeline.py
Browse files- pipeline.py +2 -0
pipeline.py
CHANGED
|
@@ -2,6 +2,7 @@ from transformers import AutoModelForSeq2SeqLM, AutoTokenizer
|
|
| 2 |
from typing import Dict, List, Any
|
| 3 |
import itertools
|
| 4 |
from nltk import sent_tokenize
|
|
|
|
| 5 |
|
| 6 |
class PreTrainedPipeline():
|
| 7 |
def __init__(self, path=""):
|
|
@@ -9,6 +10,7 @@ class PreTrainedPipeline():
|
|
| 9 |
# Preload all the elements you are going to need at inference.
|
| 10 |
# For instance your model, processors, tokenizer that might be needed.
|
| 11 |
# This function is only called once, so do all the heavy processing I/O here"""
|
|
|
|
| 12 |
self.model = AutoModelForSeq2SeqLM.from_pretrained(path)
|
| 13 |
self.tokenizer = AutoTokenizer.from_pretrained(path)
|
| 14 |
|
|
|
|
| 2 |
from typing import Dict, List, Any
|
| 3 |
import itertools
|
| 4 |
from nltk import sent_tokenize
|
| 5 |
+
import nltk
|
| 6 |
|
| 7 |
class PreTrainedPipeline():
|
| 8 |
def __init__(self, path=""):
|
|
|
|
| 10 |
# Preload all the elements you are going to need at inference.
|
| 11 |
# For instance your model, processors, tokenizer that might be needed.
|
| 12 |
# This function is only called once, so do all the heavy processing I/O here"""
|
| 13 |
+
nltk.download('punkt')
|
| 14 |
self.model = AutoModelForSeq2SeqLM.from_pretrained(path)
|
| 15 |
self.tokenizer = AutoTokenizer.from_pretrained(path)
|
| 16 |
|