Add Gradio chatbot for fine-tuned Zephyr model new space

app.py

@@ -1,29 +1,21 @@
 import gradio as gr
 import torch
-from transformers import AutoTokenizer, AutoModelForCausalLM, BitsAndBytesConfig
+from transformers import AutoTokenizer, AutoModelForCausalLM
 from peft import PeftModel
 
 # Model setup
 base_model = "HuggingFaceH4/zephyr-7b-beta"
-model_path = r"hmorad/zephyr-geo-finetuned/tree/main/checkpoint-161"  # Update to your repo ID
-
-# Quantization config
-bnb_config = BitsAndBytesConfig(
-    load_in_4bit=True,
-    bnb_4bit_compute_dtype=torch.float16,
-    bnb_4bit_use_double_quant=True,
-    bnb_4bit_quant_type="nf4"
-)
+model_path = "your-username/zephyr-geo-finetuned"  # Update with your Hugging Face repo ID
 
 # Load tokenizer
 tokenizer = AutoTokenizer.from_pretrained(base_model)
 tokenizer.pad_token = tokenizer.eos_token
 
-# Load base model
+# Load base model on CPU
 base = AutoModelForCausalLM.from_pretrained(
     base_model,
-    quantization_config=bnb_config,
-    device_map="auto",
+    torch_dtype=torch.float16,  # Use FP16 to reduce memory
+    device_map="cpu",  # Force CPU
     trust_remote_code=True
 )
 
@@ -33,7 +25,7 @@ model.eval()
 
 def generate_response(prompt, max_new_tokens=200):
     input_text = f"user: {prompt}\nassistant:"
-    inputs = tokenizer(input_text, return_tensors="pt").to("cuda" if torch.cuda.is_available() else "cpu")
+    inputs = tokenizer(input_text, return_tensors="pt").to("cpu")
     with torch.no_grad():
         output = model.generate(
             **inputs,

requirements.txt

@@ -1,6 +1,5 @@
 torch>=2.0.0
 transformers==4.52.4
 peft==0.15.2
-bitsandbytes==0.44.0
 gradio==4.44.0
 huggingface_hub>=0.30.0