app.py

from vllm import LLM, SamplingParams
import gradio as gr
import os
from huggingface_hub import login


class TextCompletion:
    def __init__(self, model, sampling_params):
        self.model = model
        self.sampling_params = sampling_params

    def generate(self, prompt: str):
        output = self.model.generate(prompt, self.sampling_params)
        response = output[0].outputs[0].text
        return response


if __name__ == "__main__":
    HF_TOKEN = os.getenv('HF_TOKEN')
    login(token=HF_TOKEN)

    model = LLM(
        model="mep296/llama-3-8b-rephrase-summarize-quality",
        tokenizer="meta-llama/Meta-Llama-3-8B",
        device="cuda"
    )
    tokenizer = model.get_tokenizer()
    sampling_params = SamplingParams(
        temperature=0.1,
        max_tokens=500,
        stop=[tokenizer.eos_token, "## Example 7", "##"]
    )

    def text_completion_fn(prompt):
        text_completer = TextCompletion(model, sampling_params)
        return text_completer.generate(prompt)
    demo = gr.Interface(fn=text_completion_fn, inputs="textbox", outputs="textbox")
    demo.launch()