feat: switch demo to lighter GGUF (llama.cpp) — 1.06GB vs 3.3GB fp16; add PyPI link

Dockerfile

@@ -1,6 +1,6 @@
 FROM python:3.12-slim
 
-# Fix fused model (v2) — direct safetensors save, no corruption
+# Bharat-Tiny-LLM demo — runs the GGUF build via llama.cpp (lighter than fp16)
 WORKDIR /app
 
 # System dependencies
@@ -8,11 +8,11 @@ RUN apt-get update && apt-get install -y --no-install-recommends \
     git git-lfs ffmpeg libsm6 libxext6 cmake rsync libgl1 \
     && rm -rf /var/lib/apt/lists/* && git lfs install
 
-# Pre-download model at build time
-RUN pip install --no-cache-dir "huggingface-hub<1.22.0"
+# Pre-download the GGUF model at build time so the Space starts fast
+RUN pip install --no-cache-dir "huggingface-hub<1.22.0" "llama-cpp-python>=0.3.0"
 RUN python3 -c "\
-from huggingface_hub import snapshot_download; \
-snapshot_download('eulogik/Bharat-Tiny-LLM-fused'); \
+from huggingface_hub import hf_hub_download; \
+hf_hub_download(repo_id='eulogik/Bharat-Tiny-LLM-GGUF', filename='bharat-tiny-llm-q4_k_m.gguf'); \
 "
 
 # Copy app files

app.py

@@ -16,10 +16,11 @@ def _json_schema_to_python_type_patched(schema, defs):
 _gcu._json_schema_to_python_type = _json_schema_to_python_type_patched
 
 import gradio as gr
-import torch
-from transformers import AutoModelForCausalLM, AutoTokenizer
+from llama_cpp import Llama
+from huggingface_hub import hf_hub_download
 
-MODEL = "eulogik/Bharat-Tiny-LLM-fused"
+MODEL_REPO = "eulogik/Bharat-Tiny-LLM-GGUF"
+MODEL_FILE = "bharat-tiny-llm-q4_k_m.gguf"
 IM_END = 151645
 
 DESCRIPTION = """
@@ -32,6 +33,7 @@ DESCRIPTION = """
     Built by <a href="https://eulogik.com" target="_blank"><strong>eulogik</strong></a>
     · <a href="https://huggingface.co/eulogik/Bharat-Tiny-LLM" target="_blank">Download Model</a>
     · <a href="https://github.com/eulogik/Bharat-Tiny-LLM" target="_blank">GitHub</a>
+    · <a href="https://pypi.org/project/bharat-tiny-llm/" target="_blank">PyPI</a>
   </p>
 </div>
 """
@@ -57,45 +59,25 @@ CUSTOM_CSS = """
 .examples-section { margin: 1rem 0; }
 """
 
-def load_pipeline():
-    model = AutoModelForCausalLM.from_pretrained(MODEL, torch_dtype=torch.float32)
-    tokenizer = AutoTokenizer.from_pretrained(MODEL)
-    if tokenizer.pad_token_id is None:
-        tokenizer.pad_token_id = tokenizer.eos_token_id
-    model.eval()
-    return model, tokenizer
-
-print("Loading model...")
-model, tokenizer = load_pipeline()
+print("Loading GGUF model...")
+gguf_path = hf_hub_download(repo_id=MODEL_REPO, filename=MODEL_FILE)
+llm = Llama(model_path=gguf_path, n_ctx=1024, n_gpu_layers=0, verbose=False)
 print("Model loaded!")
 
+
 def generate(message, history, temperature, max_tokens):
     messages = [{"role": "user", "content": message}]
     if history:
         messages = [m for m in history] + messages
-    prompt = ""
-    for m in messages:
-        prompt += f"<|im_start|>{m['role']}\n{m['content']}<|im_end|>\n"
-    prompt += "<|im_start|>assistant\n"
-    inputs = tokenizer(prompt, return_tensors="pt")
-
-    with torch.no_grad():
-        outputs = model.generate(
-            **inputs,
-            max_new_tokens=max_tokens,
-            temperature=temperature,
-            do_sample=True,
-            top_p=0.85,
-            repetition_penalty=1.25,
-            no_repeat_ngram_size=3,
-            pad_token_id=tokenizer.pad_token_id,
-            eos_token_id=IM_END,
-        )
-
-    response = tokenizer.decode(
-        outputs[0][inputs.input_ids.shape[1]:], skip_special_tokens=True
+    out = llm.create_chat_completion(
+        messages=messages,
+        temperature=temperature,
+        top_p=0.85,
+        repeat_penalty=1.25,
+        max_tokens=max_tokens,
     )
-    return response
+    return out["choices"][0]["message"]["content"]
+
 
 def format_chat(message, history):
     history = history or []
@@ -105,6 +87,7 @@ def format_chat(message, history):
     history.append({"role": "assistant", "content": response})
     yield history, ""
 
+
 with gr.Blocks(css=CUSTOM_CSS, theme=gr.themes.Soft(primary_hue="blue", neutral_hue="gray")) as demo:
     gr.HTML(DESCRIPTION)
 
@@ -128,8 +111,8 @@ with gr.Blocks(css=CUSTOM_CSS, theme=gr.themes.Soft(primary_hue="blue", neutral_
 
     with gr.Accordion("Settings", open=False):
         with gr.Row():
-        temperature_slider = gr.Slider(0.1, 1.0, value=0.3, step=0.05, label="Temperature")
-        max_tokens_slider = gr.Slider(8, 256, value=128, step=8, label="Max Tokens")
+            temperature_slider = gr.Slider(0.1, 1.0, value=0.3, step=0.05, label="Temperature")
+            max_tokens_slider = gr.Slider(8, 256, value=128, step=8, label="Max Tokens")
 
     gr.Examples(
         examples=EXAMPLES,
@@ -149,4 +132,4 @@ with gr.Blocks(css=CUSTOM_CSS, theme=gr.themes.Soft(primary_hue="blue", neutral_
     send.click(format_chat, [msg, chatbot], [chatbot, msg])
 
 if __name__ == "__main__":
-    demo.launch(server_name="0.0.0.0", server_port=7860)
+    demo.launch()

requirements.txt

@@ -2,6 +2,4 @@ gradio==4.44.1
 starlette==0.51.0
 requests>=2.31
 huggingface_hub<1.22.0
-transformers>=4.40
-torch>=2.1
-sentencepiece>=0.2
+llama-cpp-python>=0.3.0