app.py

import gradio as gr
from llama_cpp import Llama
from huggingface_hub import hf_hub_download

# --- SETTINGS ---
# This is the Qwen3-VL 8B model (Quantized to fit in Free Space)
REPO_ID = "Qwen/Qwen3-VL-8B-Instruct-GGUF" 
FILENAME = "*q4_k_m.gguf" 

print("⬇️ Downloading model... (This mimics 'ollama pull')")
try:
    model_path = hf_hub_download(repo_id=REPO_ID, filename=FILENAME)
except:
    # Fallback to Qwen2.5-VL if Qwen3 isn't fully indexed yet
    print("⚠️ Qwen3 GGUF not found, falling back to Qwen2.5-VL")
    model_path = hf_hub_download(repo_id="Qwen/Qwen2.5-VL-7B-Instruct-GGUF", filename="*q4_k_m.gguf")

print("✅ Model downloaded!")

# --- LOAD ENGINE ---
# n_gpu_layers=0 forces CPU mode (Free Tier)
llm = Llama(
    model_path=model_path,
    n_ctx=2048,       
    n_gpu_layers=0,  
    verbose=True
)

# --- CHAT UI ---
def chat(message, history):
    prompt = f"<|im_start|>user\n{message}<|im_end|>\n<|im_start|>assistant\n"
    output = llm(prompt, max_tokens=512, stop=["<|im_end|>"], echo=False)
    return output['choices'][0]['text']

gr.ChatInterface(chat).launch(server_name="0.0.0.0", server_port=7860)