| import gradio as gr | |
| from llama_cpp import Llama | |
| # 1. Load the quantized model | |
| # Make sure to pick a file like Q4_K_M.gguf from the repo's 'Files' tab | |
| llm = Llama.from_pretrained( | |
| repo_id="empero-ai/Qwythos-9B-Claude-Mythos-5-1M-GGUF", | |
| filename="Qwythos-9B-Claude-Mythos-5-1M-Q4_K_M.gguf", | |
| n_ctx=2048, # Small context to save RAM | |
| n_threads=2 # Keep threads low to avoid CPU spikes | |
| ) | |
| def respond(message, history): | |
| # Construct the prompt | |
| prompt = f"User: {message}\nAssistant:" | |
| # Generate | |
| output = llm(prompt, max_tokens=256) | |
| return output['choices'][0]['text'] | |
| # 2. Launch the Web UI | |
| demo = gr.ChatInterface(fn=respond, title="Qwythos 9B Chat") | |
| demo.launch() | |