import huggingface_hub if not hasattr(huggingface_hub, 'HfFolder'): class HfFolder: @staticmethod def path(): return None @staticmethod def get_token(): return None huggingface_hub.HfFolder = HfFolder import gradio_client.utils as _gcu _json_schema_to_python_type_orig = _gcu._json_schema_to_python_type def _json_schema_to_python_type_patched(schema, defs): if not isinstance(schema, dict): return str(schema) return _json_schema_to_python_type_orig(schema, defs) _gcu._json_schema_to_python_type = _json_schema_to_python_type_patched import gradio as gr from llama_cpp import Llama from huggingface_hub import hf_hub_download MODEL_REPO = "eulogik/Bharat-Tiny-LLM-GGUF" MODEL_FILE = "bharat-tiny-llm-q4_k_m.gguf" IM_END = 151645 DESCRIPTION = """

🇮🇳 Bharat-Tiny-LLM

Hinglish AI — trained on a Mac Mini M4, runs on ₹8,000 phones

Built by eulogik · Download Model · GitHub · PyPI

""" EXAMPLES = [ ["Chai peete hain?", 0.3, 128], ["Kal interview hai, nervous ho raha hoon", 0.3, 128], ["Biryani kaise banate hain?", 0.3, 128], ["Delhi me rehne ke liye kya karna padega?", 0.3, 128], ["Mujhe Hindi seekhni hai", 0.3, 128], ["Weekend pe kya karein?", 0.3, 128], ] CUSTOM_CSS = """ #title { text-align: center; margin-top: 2rem; } #title h1 { font-weight: 700; letter-spacing: -0.02em; } #chatbot { min-height: 400px; } #chatbot .message { border-radius: 18px !important; } #chatbot .user { background: var(--primary-400) !important; } .footer { text-align: center; font-size: 0.8rem; color: var(--body-text-color-subdued); padding: 1rem 0; } .footer a { color: var(--primary-500); text-decoration: none; } .footer a:hover { text-decoration: underline; } .examples-section { margin: 1rem 0; } """ print("Loading GGUF model...") gguf_path = hf_hub_download(repo_id=MODEL_REPO, filename=MODEL_FILE) llm = Llama(model_path=gguf_path, n_ctx=1024, n_gpu_layers=0, verbose=False) print("Model loaded!") def generate(message, history, temperature, max_tokens): messages = [{"role": "user", "content": message}] if history: messages = [m for m in history] + messages out = llm.create_chat_completion( messages=messages, temperature=temperature, top_p=0.85, repeat_penalty=1.25, max_tokens=max_tokens, ) return out["choices"][0]["message"]["content"] def format_chat(message, history): history = history or [] history.append({"role": "user", "content": message}) yield history, "" response = generate(message, history, temperature_slider.value, max_tokens_slider.value) history.append({"role": "assistant", "content": response}) yield history, "" with gr.Blocks(css=CUSTOM_CSS, theme=gr.themes.Soft(primary_hue="blue", neutral_hue="gray")) as demo: gr.HTML(DESCRIPTION) chatbot = gr.Chatbot( type="messages", label="Chat", elem_id="chatbot", height=450, bubble_full_width=False, ) with gr.Row(): msg = gr.Textbox( placeholder="Hinglish mein kuch bhi poochhein...", show_label=False, container=False, scale=8, ) send = gr.Button("Send", scale=1, variant="primary") clear = gr.ClearButton([msg, chatbot], scale=1) with gr.Accordion("Settings", open=False): with gr.Row(): temperature_slider = gr.Slider(0.1, 1.0, value=0.3, step=0.05, label="Temperature") max_tokens_slider = gr.Slider(8, 256, value=128, step=8, label="Max Tokens") gr.Examples( examples=EXAMPLES, inputs=[msg, temperature_slider, max_tokens_slider], label="Try these examples", ) gr.HTML( '' ) msg.submit(format_chat, [msg, chatbot], [chatbot, msg]) send.click(format_chat, [msg, chatbot], [chatbot, msg]) if __name__ == "__main__": demo.launch()