Update app.py

app.py

@@ -1,13 +1,36 @@
+import os
 import gradio as gr
-from transformers import pipeline
+from llama_cpp import Llama
 
-qa = pipeline("text2text-generation", model="google/flan-t5-base")
+# Load GGUF model
+MODEL_PATH = "./models/mistral.gguf"
 
-def answer_question(prompt):
-    output = qa(prompt, max_length=256, do_sample=False)
-    return output[0]['generated_text']
+llm = Llama(
+    model_path=MODEL_PATH,
+    n_ctx=2048,
+    n_threads=9,       # Increase for more speed if CPU allows
+    n_batch=128,
+    use_mlock=True,
+    use_mmap=True,
+    verbose=False
+)
 
-gr.Interface(fn=answer_question, 
-             inputs=gr.Textbox(label="Ask EduBot"),
-             outputs=gr.Textbox(label="Answer"),
-             title="Leo9 EduBot (CPU)").launch()
+# Streaming generator
+def generate_response(prompt):
+    stream = llm(
+        prompt=f"[INST] {prompt.strip()} [/INST]",
+        max_tokens=512,
+        stop=["</s>"],
+        stream=True
+    )
+    partial = ""
+    for chunk in stream:
+        partial += chunk["choices"][0]["text"]
+        yield partial
+
+# Gradio UI
+gr.ChatInterface(
+    fn=generate_response,
+    title="Leo9 AI Tutor",
+    description="An ai chatbots who answer any question.",
+).launch()