Spaces:
Sleeping
Sleeping
File size: 1,511 Bytes
c288578 15b0e40 c288578 15b0e40 c288578 bde6c09 15b0e40 bde6c09 15b0e40 c288578 15b0e40 bde6c09 | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 | import os
from huggingface_hub import InferenceClient
client = InferenceClient(
provider="featherless-ai",
api_key=os.environ["HF_TOKEN_inf"]
)
def chat_with_model(message, history_messages, perspective):
"""
Streaming generator for Gradio chatbot.
Inputs:
- message: str
- history_messages: list[{"role": ..., "content": ...}]
- perspective: str
Yields:
- (updated_messages_for_chatbot, updated_messages_for_state)
"""
# Build messages for the API
messages = []
if perspective and perspective.strip():
messages.append({"role": "system", "content": f"Adopt this perspective: {perspective.strip()}"})
for m in history_messages:
if "role" in m and "content" in m:
messages.append({"role": m["role"], "content": m["content"]})
messages.append({"role": "user", "content": message})
# Prepare base history
reply = ""
base = history_messages + [{"role": "user", "content": message}]
# Start streaming from HF Inference
stream = client.chat.completions.create(
model="mistralai/Mistral-7B-Instruct-v0.2",
messages=messages,
max_tokens=512,
stream=True,
)
for event in stream:
if event.choices and event.choices[0].delta:
token = event.choices[0].delta.content or ""
if token:
reply += token
updated = base + [{"role": "assistant", "content": reply}]
yield updated, updated
|