app.py

import gradio as gr
from llama_cpp import Llama

# 1. Load the quantized model
# Make sure to pick a file like Q4_K_M.gguf from the repo's 'Files' tab
llm = Llama.from_pretrained(
    repo_id="empero-ai/Qwythos-9B-Claude-Mythos-5-1M-GGUF",
    filename="Qwythos-9B-Claude-Mythos-5-1M-Q4_K_M.gguf", 
    n_ctx=2048, # Small context to save RAM
    n_threads=2 # Keep threads low to avoid CPU spikes
)

def respond(message, history):
    # Construct the prompt
    prompt = f"User: {message}\nAssistant:"
    
    # Generate
    output = llm(prompt, max_tokens=256)
    return output['choices'][0]['text']

# 2. Launch the Web UI
demo = gr.ChatInterface(fn=respond, title="Qwythos 9B Chat")
demo.launch()