chat.py

import os
from huggingface_hub import InferenceClient

client = InferenceClient(
    provider="featherless-ai",
    api_key=os.environ["HF_TOKEN_inf"]
)

def chat_with_model(message, history_messages, perspective):
    """
    Streaming generator for Gradio chatbot.
    Inputs:
      - message: str
      - history_messages: list[{"role": ..., "content": ...}]
      - perspective: str
    Yields:
      - (updated_messages_for_chatbot, updated_messages_for_state)
    """
    # Build messages for the API
    messages = []
    if perspective and perspective.strip():
        messages.append({"role": "system", "content": f"Adopt this perspective: {perspective.strip()}"})
    for m in history_messages:
        if "role" in m and "content" in m:
            messages.append({"role": m["role"], "content": m["content"]})
    messages.append({"role": "user", "content": message})

    # Prepare base history
    reply = ""
    base = history_messages + [{"role": "user", "content": message}]

    # Start streaming from HF Inference
    stream = client.chat.completions.create(
        model="mistralai/Mistral-7B-Instruct-v0.2",
        messages=messages,
        max_tokens=512,
        stream=True,
    )

    for event in stream:
        if event.choices and event.choices[0].delta:
            token = event.choices[0].delta.content or ""
            if token:
                reply += token
                updated = base + [{"role": "assistant", "content": reply}]
                yield updated, updated