# ============================================================
# core/api_clients.py  --  Titan Gateway V23.0  "Pexels First"
# ============================================================
# CHANGELOG V23.0:
#   [IMAGE]   Replaced AI image generation (HF SDXL) and Pollinations
#             with a real-photo hybrid system:
#               1. Pexels API  — high-quality real photos (PEXELS_API_KEY env var)
#               2. Wikimedia Commons — public-domain fallback
#             The Director now produces `pexels_search_query` (1-3 words)
#             instead of long cinematic prompts. The `cinematic_image_prompt`
#             field is retained for schema compatibility but is no longer
#             used for image generation.
#   [CLEANUP] Pollinations, HF SDXL endpoints, token pool, _bouncer(),
#             _hf_infer_image_with_token(), _generate_via_pollinations()
#             all removed. No breaking change to other pipeline functions.
#   [KEPT]    call_groq_json, call_gemini_json, call_groq_chat_completion,
#             call_compound_researcher, dispatch_transcription,
#             generate_audio_ldm2, call_gemini_translation_batch,
#             call_gemini_chunk_translation — all unchanged.
#
# CHANGELOG V16.6:
#   [IMAGE]   Replaced Pollinations + Hercai with a Hugging Face
#             Inference API system backed by a token rotation pool.
#
#             TOKEN POOL:
#               Scans env vars HF_TOKEN1 ... HF_TOKEN8 at module load.
#               NOTE: HF_TOKEN (no number) is intentionally excluded.
#               It is reserved for the HuggingFace dataset/vault client
#               used elsewhere in the pipeline and must never be consumed
#               by image generation.
#               All found numbered tokens are stored in _HF_IMAGE_TOKENS.
#
#             ROUND-ROBIN ROTATION:
#               _hf_image_token_index (module-level int) advances by 1
#               on every call to _acquire_hf_image_token(), wrapping
#               around the list with modulo.
#               On HTTP 429 the current token is considered exhausted
#               for this request; the outer loop immediately picks the
#               next token without sleeping.
#
#             MODEL:
#               stabilityai/stable-diffusion-xl-base-1.0
#               Endpoint:
#                 https://api-inference.huggingface.co/models/
#                 stabilityai/stable-diffusion-xl-base-1.0
#
#             503 HANDLING (model cold-start):
#               HF returns HTTP 503 + {"estimated_time": N} while the
#               model loads. Retried up to HF_IMAGE_503_RETRIES (3)
#               times on the SAME token with a fixed HF_IMAGE_503_WAIT
#               (5 s) delay. 503 does NOT advance the token index.
#
#             THE BOUNCER:
#               After saving any file, os.path.getsize() is checked.
#               Files < MIN_REAL_IMAGE_BYTES (30 KB) are deleted and
#               False is returned. Real SDXL outputs are 200 KB-1 MB;
#               error/placeholder bodies are always smaller than 30 KB.
#
#             PROMPT:
#               Truncated to <= HF_IMAGE_MAX_KEYWORDS (15) keywords
#               using the existing _truncate_to_keywords() helper.
#               Sent as JSON payload {"inputs": "<prompt>"} -- NOT
#               URL-encoded in a query string, because the HF
#               Inference API accepts a POST body, not a GET param.
#
#             Entry point: generate_truly_free_image(prompt, out_path)
#             Returns False (never raises) -- pipeline never crashes.
#
#   [CLEANUP] _image_via_pollinations_turbo(), _image_via_hercai(),
#             POLLINATIONS_IMAGE_TIMEOUT, HERCAI_IMAGE_TIMEOUT,
#             FREE_IMAGE_MAX_KEYWORDS, MIN_REAL_IMAGE_BYTES (50 KB)
#             all replaced or renamed. No Pollinations or Hercai
#             URLs remain anywhere in the file.
#   [GROQ]    frequency_penalty=0.9, presence_penalty=0.7 unchanged.
# ============================================================

import asyncio
import httpx
import json
import logging
import os
import random
import re
import urllib.parse
from typing import Any, Dict, List, Optional

from google import genai
from groq import AsyncGroq, RateLimitError as GroqRateLimitError  # noqa: F401

logger = logging.getLogger("titan.api_clients")

# ------------------------------------------------------------------
# Constants -- audio (unchanged)
# ------------------------------------------------------------------
HF_AUDIOLDM2_URL:        str   = "https://api-inference.huggingface.co/models/cvssp/audioldm2"
HF_AUDIO_DURATION:       float = 10.0
HF_AUDIO_GUIDANCE:       float = 3.5
HF_AUDIO_RETRIES:        int   = 3
HF_AUDIO_TIMEOUT:        float = 90.0

# ------------------------------------------------------------------
# Constants -- V16.6 HF image generation
# ------------------------------------------------------------------

# Model endpoint (lightweight, stable SDXL variant) – UPDATED to router URL
# Primary: free serverless endpoint (no credits consumed)
HF_IMAGE_MODEL_URL:      str   = (
    "https://api-inference.huggingface.co/models/stabilityai/stable-diffusion-xl-base-1.0"
)
# Fallback: paid inference provider (used only if primary returns 404/unavailable)
HF_IMAGE_MODEL_URL_PAID: str   = (
    "https://router.huggingface.co/hf-inference/models/stabilityai/stable-diffusion-xl-base-1.0"
)
HF_IMAGE_TIMEOUT:       float = 120.0  # SDXL inference can take 60-90 s
HF_IMAGE_503_RETRIES:   int   = 3      # retries on model cold-start (503)
HF_IMAGE_503_WAIT:      float = 5.0    # seconds between 503 retries

# Max keywords sent to HF -- keeps the prompt concise
HF_IMAGE_MAX_KEYWORDS:  int   = 15

# The Bouncer threshold -- real SDXL images are 200 KB-1 MB;
# HF error / placeholder bodies are always < 30 KB.
MIN_REAL_IMAGE_BYTES:   int   = 30 * 1024  # 30 KB

# ------------------------------------------------------------------
# Constants -- LLM / translation (unchanged)
# ------------------------------------------------------------------
IMMERSION_AUTHOR_MODEL:  str   = "llama-3.3-70b-versatile"  # Director + Dice — json_object calls
GEMMA_MODEL_ID:          str   = os.environ.get("GEMINI_MODEL", "gemma-3-27b-it")
TRANSLATION_TEMPERATURE: float = 0.10

# MIN_IMAGE_BYTES retained for AudioLDM2 audio-payload validation only
MIN_IMAGE_BYTES:         int   = 1024


# ══════════════════════════════════════════════════════════════════
#
#   V16.6 -- HF IMAGE TOKEN POOL  (initialised once at module load)
#
#   Scans HF_TOKEN1 ... HF_TOKEN8.
#   HF_TOKEN (no suffix) is deliberately excluded: it belongs to
#   the HuggingFace dataset / vault client, not image generation.
#
# ══════════════════════════════════════════════════════════════════

_HF_IMAGE_TOKENS: List[str] = [
    tok
    for tok in (
        os.environ.get(f"HF_TOKEN{i}", "").strip()
        for i in range(1, 9)   # HF_TOKEN1 .. HF_TOKEN8
    )
    if tok                     # skip empty / unset variables
]

# Round-robin cursor -- shared across all coroutines in this process.
# Protected by _hf_token_lock to be safe under async concurrency.
_hf_image_token_index: int = 0
_hf_token_lock = asyncio.Lock()

if _HF_IMAGE_TOKENS:
    logger.info(
        "[HF-Image] Token pool ready: %d token(s) loaded "
        "(HF_TOKEN1-HF_TOKEN8). HF_TOKEN (no suffix) excluded.",
        len(_HF_IMAGE_TOKENS),
    )
else:
    logger.warning(
        "[HF-Image] No tokens found (HF_TOKEN1-HF_TOKEN8 not set). "
        "Image generation will return False for every chapter."
    )


async def _acquire_hf_image_token() -> Optional[str]:
    """
    Return the next token from the round-robin pool and advance the
    cursor.  Thread-safe via _hf_token_lock.
    Returns None if the pool is empty.
    """
    if not _HF_IMAGE_TOKENS:
        return None
    global _hf_image_token_index
    async with _hf_token_lock:
        token = _HF_IMAGE_TOKENS[_hf_image_token_index % len(_HF_IMAGE_TOKENS)]
        _hf_image_token_index += 1
    return token


# ------------------------------------------------------------------
# Utility helpers (unchanged)
# ------------------------------------------------------------------

def clean_json(raw: str) -> str:
    """Strip markdown code fences and trim to first {...} or [...]."""
    raw = re.sub(r"^```(?:json)?\s*\n?", "", raw, flags=re.MULTILINE)
    raw = re.sub(r"\n?\s*```\s*$",        "", raw, flags=re.MULTILINE)
    raw = raw.strip()
    starts = [i for i in [raw.find("{"), raw.find("[")] if i >= 0]
    if starts:
        raw = raw[min(starts):]
    end = max(raw.rfind("}"), raw.rfind("]"))
    if end >= 0:
        raw = raw[: end + 1]
    return raw.strip()


_CONTAMINATION_PATTERNS: re.Pattern = re.compile(
    r"https?://\S+"
    r"|www\.\S+"
    r"|\{[^}]{0,500}\}"
    r"|\b[45]\d{2}\s+(?:Internal|Not\s+Found|Bad\s+Request|"
    r"Forbidden|Unauthorized|Service\s+Unavailable|Error)\b"
    r"|(?:^|\n)\s*(?:Error|Exception|Traceback)\s*[:(]"
    r'|"(?:error|detail|message|status)"\s*:\s*"',
    re.IGNORECASE | re.MULTILINE,
)


def text_is_contaminated(text: str) -> bool:
    """Return True if the text contains technical artifacts."""
    return bool(_CONTAMINATION_PATTERNS.search(text))


def _truncate_to_keywords(prompt: str, max_kw: int = HF_IMAGE_MAX_KEYWORDS) -> str:
    """
    Reduce an image prompt to at most `max_kw` comma-separated keywords.

    Strategy:
      1. Commas present -> split on commas, take first max_kw parts.
      2. No commas     -> split on whitespace, take first max_kw words.

    Special characters that could break parsers are stripped from each
    token before reassembly.  Returns a clean ASCII-safe string.
    """
    if "," in prompt:
        parts = [p.strip() for p in prompt.split(",") if p.strip()]
    else:
        parts = prompt.split()

    selected = parts[:max_kw]
    cleaned  = [re.sub(r"[^\w\s-]", "", p).strip() for p in selected]
    return ", ".join(w for w in cleaned if w)


# ------------------------------------------------------------------
# Groq -- JSON mode (unchanged)
# ------------------------------------------------------------------

async def call_groq_json(
    api_key:     str,
    prompt:      str,
    caller:      str,
    max_tokens:  int   = 4000,
    temperature: float = 0.4,
    model:       str   = IMMERSION_AUTHOR_MODEL,   # override per-caller if needed
) -> Dict[str, Any]:
    """Call Groq in JSON-mode. Raises on failure; caller handles retries."""
    client   = AsyncGroq(api_key=api_key)
    response = await client.chat.completions.create(
        model=model,
        response_format={"type": "json_object"},
        temperature=temperature,
        max_tokens=max_tokens,
        messages=[
            {"role": "system", "content": "You are a helpful assistant. Output only valid JSON."},
            {"role": "user",   "content": prompt},
        ],
    )
    raw = clean_json(response.choices[0].message.content.strip())
    result = json.loads(raw)
    # GPT OSS 120B sometimes wraps the JSON in a list — unwrap it
    if isinstance(result, list):
        for item in result:
            if isinstance(item, dict):
                result = item
                break
    if not isinstance(result, dict):
        raise ValueError(f"[{caller}] Expected JSON object, got {type(result).__name__}. Raw: {raw[:120]}")
    return result


# ------------------------------------------------------------------
# Gemini -- JSON mode  (unchanged)
# ------------------------------------------------------------------

async def call_gemini_json(
    api_key:     str,
    prompt:      str,
    caller:      str   = "",
    max_tokens:  int   = 4000,
    temperature: float = 0.25,
) -> Dict[str, Any]:
    """Call Gemini expecting JSON output. Uses the google.genai client."""
    client = genai.Client(api_key=api_key)
    try:
        from google.genai import types as _gt
        cfg = _gt.GenerateContentConfig(
            temperature=temperature,
            top_p=0.95,
            max_output_tokens=max_tokens,
        )
        response = await asyncio.to_thread(
            client.models.generate_content,
            model=GEMMA_MODEL_ID,
            contents=prompt,
            config=cfg,
        )
    except Exception:
        response = await asyncio.to_thread(
            client.models.generate_content,
            model=GEMMA_MODEL_ID,
            contents=prompt,
        )
    raw = clean_json(response.text.strip())
    return json.loads(raw)


# ------------------------------------------------------------------
# Groq -- Whisper transcription (unchanged)
# ------------------------------------------------------------------

async def dispatch_transcription(
    api_key:     str,
    audio_bytes: bytes,
    filename:    str,
    language:    Optional[str] = None,
    prompt:      Optional[str] = None,
) -> Dict[str, Any]:
    """Transcribe audio using Groq Whisper-large-v3-turbo."""
    import io
    client        = AsyncGroq(api_key=api_key)
    audio_stream  = io.BytesIO(audio_bytes)
    audio_stream.name = filename
    kwargs: Dict[str, Any] = {
        "file":            (filename, audio_stream),
        "model":           "whisper-large-v3-turbo",
        "language":        language,
        "response_format": "verbose_json",
    }
    if prompt:
        kwargs["prompt"] = prompt[-800:]
    tx = await client.audio.transcriptions.create(**kwargs)

    if hasattr(tx, "model_dump"):
        result = tx.model_dump()
    elif hasattr(tx, "to_dict"):
        result = tx.to_dict()
    else:
        result = {
            "text":     getattr(tx, "text",     ""),
            "segments": getattr(tx, "segments", []),
            "language": getattr(tx, "language", "unknown"),
        }
    for idx, seg in enumerate(result.get("segments", [])):
        if "id" not in seg:
            seg["id"] = str(idx)
    return result


# ------------------------------------------------------------------
# HF AudioLDM2 ambient generation (unchanged)
# ------------------------------------------------------------------

async def generate_audio_ldm2(
    hf_token:     str,
    audio_prompt: str,
    out_path:     str,
) -> bool:
    """Generate a 10-second ambient clip via HF AudioLDM 2."""
    headers = {
        "Authorization": f"Bearer {hf_token}",
        "Content-Type":  "application/json",
    }
    payload = {
        "inputs": audio_prompt,
        "parameters": {
            "audio_length_in_s":        HF_AUDIO_DURATION,
            "guidance_scale":           HF_AUDIO_GUIDANCE,
            "num_waveforms_per_prompt": 1,
        },
    }

    for attempt in range(1, HF_AUDIO_RETRIES + 1):
        try:
            logger.info("[AudioLDM2] attempt %d/%d", attempt, HF_AUDIO_RETRIES)
            async with httpx.AsyncClient(timeout=HF_AUDIO_TIMEOUT) as client:
                resp = await client.post(HF_AUDIOLDM2_URL, headers=headers, json=payload)

            if resp.status_code == 503:
                try:
                    wait = min(float(resp.json().get("estimated_time", 30.0)), 60.0)
                except Exception:
                    wait = 30.0
                await asyncio.sleep(wait)
                continue

            if resp.status_code == 429:
                await asyncio.sleep(65.0 * attempt)
                continue

            if resp.status_code != 200:
                logger.error("[AudioLDM2] HTTP %d", resp.status_code)
                if attempt < HF_AUDIO_RETRIES:
                    await asyncio.sleep(10.0 * attempt)
                continue

            audio_bytes = resp.content
            if len(audio_bytes) < 512:
                await asyncio.sleep(10.0 * attempt)
                continue

            with open(out_path, "wb") as fh:
                fh.write(audio_bytes)
            logger.info(
                "[AudioLDM2] Saved -> %s (%.0f KB)", out_path, len(audio_bytes) / 1024
            )
            return True

        except httpx.TimeoutException:
            await asyncio.sleep(10.0 * attempt)
        except Exception as exc:
            logger.error("[AudioLDM2] Error attempt=%d: %s", attempt, exc)
            await asyncio.sleep(10.0 * attempt)

    logger.error("[AudioLDM2] Failed after %d attempts", HF_AUDIO_RETRIES)
    return False


# ------------------------------------------------------------------
# Gemini -- subtitle translation batch (unchanged)
# ------------------------------------------------------------------

async def call_gemini_translation_batch(
    api_key: str,
    prompt:  str,
) -> Dict[str, str]:
    """Translate subtitle segments via Gemini. Returns id -> text dict."""
    client = genai.Client(api_key=api_key)
    try:
        from google.genai import types as _gt
        cfg = _gt.GenerateContentConfig(
            temperature=TRANSLATION_TEMPERATURE,
            top_p=0.95,
            max_output_tokens=8192,
        )
        response = await asyncio.to_thread(
            client.models.generate_content,
            model=GEMMA_MODEL_ID,
            contents=prompt,
            config=cfg,
        )
    except Exception:
        response = await asyncio.to_thread(
            client.models.generate_content,
            model=GEMMA_MODEL_ID,
            contents=prompt,
        )
    parsed: Dict[str, str] = {}
    for line in response.text.strip().splitlines():
        m = re.match(r"^\[([^\]]+)\]\s*(.*)", line.strip())
        if m:
            parsed[m.group(1)] = m.group(2).strip()
    return parsed


async def call_gemini_chunk_translation(
    api_key: str,
    prefix:  str,
    chunk:   str,
) -> str:
    """Translate a single text chunk via Gemini."""
    client = genai.Client(api_key=api_key)
    try:
        from google.genai import types as _gt
        cfg = _gt.GenerateContentConfig(
            temperature=0.15,
            top_p=0.95,
            max_output_tokens=3000,
        )
        response = await asyncio.to_thread(
            client.models.generate_content,
            model=GEMMA_MODEL_ID,
            contents=prefix + chunk,
            config=cfg,
        )
    except Exception:
        response = await asyncio.to_thread(
            client.models.generate_content,
            model=GEMMA_MODEL_ID,
            contents=prefix + chunk,
        )
    return (response.text or "").strip()


# ------------------------------------------------------------------
# Groq -- Chat completion  (Narrator / Novelist)  (unchanged)
# ------------------------------------------------------------------

async def call_groq_chat_completion(
    api_key:       str,
    system_prompt: str,
    user_prompt:   str,
    model:         str,
    max_tokens:    int,
    temperature:   float,
) -> str:
    """
    Groq chat completion for The Narrator.
    V18.2: Robust content extraction —
    - No frequency_penalty / presence_penalty (not supported by GPT OSS 120B)
    - Handles None content via reasoning_content fallback
    - Full response logged on empty to diagnose root cause
    """
    client   = AsyncGroq(api_key=api_key)
    response = await client.chat.completions.create(
        messages=[
            {"role": "system", "content": system_prompt},
            {"role": "user",   "content": user_prompt},
        ],
        model=model,
        temperature=temperature,
        max_tokens=max_tokens,
    )

    if not response.choices:
        logger.error(
            "[Narrator] No choices in response — model=%s | response=%s",
            model, str(response)[:500]
        )
        return ""

    choice = response.choices[0]
    msg    = choice.message

    # ONLY use msg.content — never reasoning/reasoning_content.
    # reasoning field contains the model's thinking process, not the narration.
    # If content is empty → max_tokens too low → retry with more tokens.
    raw = msg.content if msg.content else None

    if not raw:
        logger.error(
            "[Narrator] EMPTY CONTENT — model=%s | finish_reason=%s | "
            "msg_fields=%s | full_response=%s",
            model,
            getattr(choice, "finish_reason", "?"),
            list((msg.model_dump() or {}).keys()) if hasattr(msg, "model_dump") else "?",
            str(response)[:1000],
        )
        return ""

    return raw.strip()


# ------------------------------------------------------------------
# Groq Compound — Autonomous Web Researcher  (V22.0)
# ------------------------------------------------------------------
# groq/compound-beta has built-in web search (tool_use).
# We call it, collect the final text response (which includes
# web-searched facts + real Wikimedia image URLs).
# ------------------------------------------------------------------

COMPOUND_MODEL:      str   = "compound-beta"
COMPOUND_MAX_TOKENS: int   = 4000
COMPOUND_TIMEOUT:    float = 90.0   # compound is slower — allow 90 s


async def call_compound_researcher(
    api_key:       str,
    system_prompt: str,
    user_prompt:   str,
    caller:        str = "CompoundResearcher",
) -> str:
    """
    V22.0 — Call groq/compound-beta as an autonomous web researcher.

    compound-beta uses built-in web search automatically.
    We do NOT pass explicit tool definitions — the model decides when
    to search. We collect the final text response (Markdown dossier).

    Returns the full Markdown dossier string.
    Raises on failure — caller (_run_historian) handles retries.
    """
    client = AsyncGroq(api_key=api_key)

    response = await client.chat.completions.create(
        model=COMPOUND_MODEL,
        messages=[
            {"role": "system", "content": system_prompt},
            {"role": "user",   "content": user_prompt},
        ],
        max_tokens=COMPOUND_MAX_TOKENS,
        temperature=0.3,
    )

    if not response.choices:
        raise ValueError(f"[{caller}] compound-beta returned no choices")

    choice = response.choices[0]
    msg    = choice.message

    # compound returns the final answer in msg.content
    # (tool calls are handled internally by the model, not returned to us)
    raw = msg.content if msg.content else None

    if not raw or len(raw.strip()) < 100:
        logger.error(
            "[%s] compound-beta empty/short content — finish_reason=%s | len=%d",
            caller,
            getattr(choice, "finish_reason", "?"),
            len(raw or ""),
        )
        raise ValueError(f"[{caller}] compound-beta returned empty dossier")

    logger.info(
        "[%s] compound-beta dossier ready — %d chars | finish=%s",
        caller, len(raw), getattr(choice, "finish_reason", "?"),
    )
    return raw.strip()


# ══════════════════════════════════════════════════════════════════
#
#   V16.6 -- HF INFERENCE IMAGE GENERATION  (new)
#
#   Token pool:  HF_TOKEN1 ... HF_TOKEN8  (HF_TOKEN excluded)
#   Model:       stabilityai/stable-diffusion-xl-base-1.0
#   Rotation:    round-robin; 429 -> skip token, try next
#   503 retry:   up to 3x with 5 s delay per token (model warm-up)
#   Bouncer:     reject + delete files < 30 KB
#
# ══════════════════════════════════════════════════════════════════

def _bouncer(out_path: str, token_label: str) -> bool:
    """
    Post-write file-size gate (The Bouncer).

    Reads os.path.getsize() on the already-written file.
    Rejects and deletes anything < MIN_REAL_IMAGE_BYTES (30 KB).

    Real SDXL outputs are 200 KB - 1 MB.
    HF error JSON bodies and placeholder images are always < 30 KB.

    Returns True if the file is large enough to be a genuine image.
    Returns False (and deletes the file) otherwise.
    `token_label` is used only in log messages.
    """
    try:
        size = os.path.getsize(out_path)
    except OSError as exc:
        logger.error("[Bouncer/%s] Cannot stat '%s': %s", token_label, out_path, exc)
        return False

    if size < MIN_REAL_IMAGE_BYTES:
        logger.error(
            "[Bouncer/%s] File too small (%d B < %d B minimum) -- "
            "placeholder or error body. Deleting.",
            token_label, size, MIN_REAL_IMAGE_BYTES,
        )
        try:
            os.remove(out_path)
        except OSError:
            pass
        return False

    logger.info("[Bouncer/%s] Size OK -- %.1f KB", token_label, size / 1024)
    return True


async def _hf_infer_image_with_token(
    token:       str,
    token_label: str,
    prompt:      str,
    out_path:    str,
) -> bool:
    """
    Attempt one image generation call using a single HF Bearer token.

    Tries TWO endpoints in order:
      1. HF_IMAGE_MODEL_URL      — free serverless (api-inference.huggingface.co)
      2. HF_IMAGE_MODEL_URL_PAID — paid inference provider (router.huggingface.co)
         Used only if primary returns 402/404/503-exhausted.

    402 on primary  → silently try paid URL (credits on paid, not free)
    402 on paid     → token genuinely exhausted, return False
    503 anywhere    → cold-start, retry same URL up to HF_IMAGE_503_RETRIES
    429 anywhere    → rate-limited, return False (caller rotates token)
    200 + bouncer   → True
    """
    headers = {
        "Authorization": f"Bearer {token}",
        "Content-Type":  "application/json",
        "Accept":        "image/jpeg",
    }
    payload = {"inputs": prompt}

    # V16.6: Free endpoint (api-inference.huggingface.co) returned HTTP 410 Gone.
    endpoints = [
        (HF_IMAGE_MODEL_URL_PAID, "paid"),
    ]

    for endpoint_url, endpoint_label in endpoints:
        logger.info(
            "[HF-Image/%s] Trying %s endpoint | prompt_len=%d",
            token_label, endpoint_label, len(prompt),
        )

        for cold_attempt in range(1, HF_IMAGE_503_RETRIES + 1):
            try:
                logger.info(
                    "[HF-Image/%s] POST | endpoint=%s | cold_attempt=%d/%d | prompt_len=%d",
                    token_label, endpoint_label, cold_attempt, HF_IMAGE_503_RETRIES, len(prompt),
                )
                async with httpx.AsyncClient(timeout=HF_IMAGE_TIMEOUT) as client:
                    resp = await client.post(
                        endpoint_url, headers=headers, json=payload
                    )

            # -- 503: model loading; retry same URL after a short wait --
                if resp.status_code == 503:
                    try:
                        wait = min(
                            float(resp.json().get("estimated_time", HF_IMAGE_503_WAIT)),
                            60.0,
                        )
                    except Exception:
                        wait = HF_IMAGE_503_WAIT
                    if cold_attempt < HF_IMAGE_503_RETRIES:
                        logger.info(
                            "[HF-Image/%s] 503 model loading (%s) -- waiting %.0f s "
                            "(retry %d/%d)",
                            token_label, endpoint_label, wait, cold_attempt, HF_IMAGE_503_RETRIES,
                        )
                        await asyncio.sleep(wait)
                        continue   # retry same endpoint
                    else:
                        logger.warning(
                            "[HF-Image/%s] 503 persisted on %s -- trying next endpoint",
                            token_label, endpoint_label,
                        )
                        break  # try next endpoint

                # -- 402: credits exhausted on this endpoint -- try next ----
                if resp.status_code == 402:
                    logger.warning(
                        "[HF-Image/%s] 402 credits exhausted on %s -- trying next endpoint",
                        token_label, endpoint_label,
                    )
                    break  # try next endpoint

                # -- 429: rate-limited; signal caller to rotate token -------
                if resp.status_code == 429:
                    logger.warning(
                        "[HF-Image/%s] 429 rate-limited on %s -- rotating to next token",
                        token_label, endpoint_label,
                    )
                    return False

                # -- Any other non-200 error --------------------------------
                if resp.status_code != 200:
                    logger.error(
                        "[HF-Image/%s] HTTP %d on %s -- %.200s",
                        token_label, resp.status_code, endpoint_label, resp.text,
                    )
                    break  # try next endpoint

                # -- 200 success: write file and validate ---------------
                image_bytes = resp.content
                if not image_bytes:
                    logger.error(
                        "[HF-Image/%s] HTTP 200 but empty body on %s",
                        token_label, endpoint_label,
                    )
                    break  # try next endpoint

                with open(out_path, "wb") as fh:
                    fh.write(image_bytes)

                if not _bouncer(out_path, token_label):
                    break  # try next endpoint

                logger.info(
                    "[HF-Image/%s] Image saved via %s -> %s (%.1f KB)",
                    token_label, endpoint_label, out_path, len(image_bytes) / 1024,
                )
                return True

            except httpx.TimeoutException:
                logger.error(
                    "[HF-Image/%s] Timed out on %s after %.0f s",
                    token_label, endpoint_label, HF_IMAGE_TIMEOUT,
                )
                break  # try next endpoint
            except Exception as exc:
                logger.error(
                    "[HF-Image/%s] Unexpected exception on %s: %s",
                    token_label, endpoint_label, exc,
                )
                break  # try next endpoint

    # All endpoints exhausted for this token
    logger.error("[HF-Image/%s] All endpoints exhausted", token_label)
    return False



# ══════════════════════════════════════════════════════════════════
#
#   V18.5 -- HYBRID IMAGE SYSTEM
#
#   Priority order:
#     1. Wikimedia Commons — real historical images, public domain, no key
#     2. Pollinations.ai   — free AI generation, no key, no limit
#     3. HF Inference      — paid fallback (existing system)
#
# ══════════════════════════════════════════════════════════════════

WIKIMEDIA_API:      str   = "https://en.wikipedia.org/w/api.php"
WIKIMEDIA_TIMEOUT:  float = 15.0
MIN_WIKIMEDIA_BYTES: int  = 50 * 1024   # 50 KB minimum — rejects thumbnails/icons

# ------------------------------------------------------------------
# V23.0 — Pexels API constants
# ------------------------------------------------------------------
PEXELS_API_URL:     str   = "https://api.pexels.com/v1/search"
PEXELS_TIMEOUT:     float = 15.0
PEXELS_PER_PAGE:    int   = 5           # fetch top-5 results, pick first valid
MIN_PEXELS_BYTES:   int   = 20 * 1024   # 20 KB — rejects broken/empty responses

POLLINATIONS_URL:   str   = "https://image.pollinations.ai/prompt/{prompt}"
POLLINATIONS_TIMEOUT: float = 60.0


async def _search_pexels_image(search_query: str, out_path: str) -> bool:
    """
    V23.0 — Fetch a real photo from Pexels API.

    Uses PEXELS_API_KEY environment variable.
    Searches for `search_query` (expected: 1-3 keywords from Director).
    Downloads the first result with src.large or src.original URL.
    Rejects files < MIN_PEXELS_BYTES (20 KB).

    Returns True if a valid image was saved to out_path.
    Returns False on any error (non-fatal — pipeline falls back to Wikimedia).
    """
    api_key = os.environ.get("PEXELS_API_KEY", "").strip()
    if not api_key:
        logger.warning("[Pexels] PEXELS_API_KEY not set — skipping Pexels")
        return False

    if not search_query.strip():
        logger.warning("[Pexels] Empty search query — skipping")
        return False

    try:
        logger.info("[Pexels] Searching: '%s'", search_query[:80])
        async with httpx.AsyncClient(timeout=PEXELS_TIMEOUT) as client:
            resp = await client.get(
                PEXELS_API_URL,
                params={
                    "query":    search_query.strip(),
                    "per_page": PEXELS_PER_PAGE,
                    "page":     1,
                },
                headers={
                    "Authorization": api_key,
                },
            )

        if resp.status_code == 401:
            logger.error("[Pexels] 401 Unauthorized — check PEXELS_API_KEY")
            return False
        if resp.status_code == 429:
            logger.warning("[Pexels] 429 Rate limited")
            return False
        if resp.status_code != 200:
            logger.warning("[Pexels] HTTP %d for query='%s'", resp.status_code, search_query[:60])
            return False

        data   = resp.json()
        photos = data.get("photos", [])
        if not photos:
            logger.warning("[Pexels] No photos found for: %s", search_query[:60])
            return False

        # Try each photo in order until one downloads cleanly
        for photo in photos:
            src     = photo.get("src", {})
            img_url = src.get("large2x") or src.get("large") or src.get("original") or ""
            if not img_url:
                continue

            try:
                async with httpx.AsyncClient(timeout=PEXELS_TIMEOUT) as client:
                    img_resp = await client.get(img_url, follow_redirects=True)

                if img_resp.status_code != 200:
                    continue

                img_bytes = img_resp.content
                if len(img_bytes) < MIN_PEXELS_BYTES:
                    logger.warning("[Pexels] Image too small (%d B) — trying next", len(img_bytes))
                    continue

                with open(out_path, "wb") as fh:
                    fh.write(img_bytes)

                photographer = photo.get("photographer", "Unknown")
                logger.info(
                    "[Pexels] ✅ Saved %s (%.1f KB) — photo by %s | query='%s'",
                    out_path, len(img_bytes) / 1024, photographer, search_query[:60],
                )
                return True

            except Exception as dl_exc:
                logger.warning("[Pexels] Download error for photo_id=%s: %s", photo.get("id"), dl_exc)
                continue

        logger.warning("[Pexels] All photos failed for query: %s", search_query[:60])
        return False

    except Exception as exc:
        logger.warning("[Pexels] Exception: %s", exc)
        return False


async def _search_wikimedia_image(search_query: str, out_path: str, image_index: int = 0) -> bool:
    """
    V22.0 — Search Wikimedia Commons for a relevant image.

    Improvements over V18.9:
    - Extended _SKIP_WORDS to reject anatomy/body diagrams, portraits of unrelated people,
      and other clearly irrelevant images (the human body diagram problem).
    - Image title must contain at least one keyword from the search query (relevance gate).
    - srlimit=5: tries up to 5 articles for better coverage.
    - imlimit=30: fetches more images per article.
    Returns True if a valid relevant image >= 50 KB was saved.
    """
    _SKIP_WORDS = [
        "flag", "icon", "logo", "map", "seal", "coat", "stub",
        "commons", "wikimedia", ".svg", "portrait", "anatomy",
        "body", "human_body", "diagram_of", "chart", "template",
        "blank", "placeholder", "symbol", "emblem", "crest",
    ]

    # Build relevance keywords from search query (lowercase, 3+ chars)
    _query_keywords = [w.lower() for w in re.split(r"\W+", search_query) if len(w) >= 3]

    def _is_relevant(img_title: str) -> bool:
        """Return True if the image title has at least one query keyword match."""
        title_lower = img_title.lower()
        # Always reject skip words
        if any(skip in title_lower for skip in _SKIP_WORDS):
            return False
        # Must end in .jpg/.jpeg/.png
        if not title_lower.endswith((".jpg", ".jpeg", ".png")):
            return False
        # Relevance check: at least one query keyword in the title
        if _query_keywords:
            return any(kw in title_lower for kw in _query_keywords)
        return True

    async def _try_download(img_title: str) -> bytes | None:
        """Download one image, return bytes if >= MIN_WIKIMEDIA_BYTES else None."""
        try:
            async with httpx.AsyncClient(timeout=WIKIMEDIA_TIMEOUT) as client:
                r = await client.get(
                    WIKIMEDIA_API,
                    params={
                        "action": "query", "titles": img_title,
                        "prop": "imageinfo", "iiprop": "url|size",
                        "iiurlwidth": 800, "format": "json",
                    },
                    headers={"User-Agent": "TitanImmersion/1.0 (educational)"},
                )
            pages = r.json().get("query", {}).get("pages", {})
            img_url = None
            for page in pages.values():
                info = page.get("imageinfo", [])
                if info:
                    img_url = info[0].get("thumburl") or info[0].get("url")
                    break
            if not img_url:
                return None
            async with httpx.AsyncClient(timeout=WIKIMEDIA_TIMEOUT) as client:
                r2 = await client.get(
                    img_url,
                    headers={"User-Agent": "TitanImmersion/1.0 (educational)"},
                    follow_redirects=True,
                )
            if r2.status_code != 200:
                return None
            data = r2.content
            return data if len(data) >= MIN_WIKIMEDIA_BYTES else None
        except Exception:
            return None

    try:
        # Step 1: Search up to 5 articles
        async with httpx.AsyncClient(timeout=WIKIMEDIA_TIMEOUT) as client:
            resp = await client.get(
                WIKIMEDIA_API,
                params={
                    "action": "query", "list": "search",
                    "srsearch": search_query, "srlimit": 5, "format": "json",
                },
                headers={"User-Agent": "TitanImmersion/1.0 (educational)"},
            )
        if resp.status_code != 200:
            logger.warning("[Wikimedia] Search HTTP %d", resp.status_code)
            return False

        results = resp.json().get("query", {}).get("search", [])
        if not results:
            logger.warning("[Wikimedia] No results for: %s", search_query[:80])
            return False

        # Step 2: Try each article until we find a relevant image
        for article in results:
            page_title = article["title"]
            logger.info("[Wikimedia] Trying article: %s", page_title)

            async with httpx.AsyncClient(timeout=WIKIMEDIA_TIMEOUT) as client:
                resp = await client.get(
                    WIKIMEDIA_API,
                    params={
                        "action": "query", "titles": page_title,
                        "prop": "images", "imlimit": 30, "format": "json",
                    },
                    headers={"User-Agent": "TitanImmersion/1.0 (educational)"},
                )

            pages = resp.json().get("query", {}).get("pages", {})
            images = []
            for page in pages.values():
                for img in page.get("images", []):
                    title = img.get("title", "")
                    if _is_relevant(title):
                        images.append(title)

            if not images:
                logger.warning("[Wikimedia] No relevant images in: %s — trying next article", page_title)
                continue

            # Step 3: Try image at image_index, then adjacent slots
            _slots = []
            _base = min(image_index, len(images) - 1)
            for _offset in range(len(images)):
                _slots.append((_base + _offset) % len(images))

            for _slot in _slots:
                img_bytes = await _try_download(images[_slot])
                if img_bytes:
                    with open(out_path, "wb") as fh:
                        fh.write(img_bytes)
                    logger.info(
                        "[Wikimedia] ✅ Saved %s (%.1f KB) from: %s | slot=%d/%d",
                        out_path, len(img_bytes) / 1024, page_title, _slot, len(images),
                    )
                    return True
                logger.warning("[Wikimedia] Slot %d too small or failed — trying next", _slot)

            logger.warning("[Wikimedia] All images in '%s' failed — trying next article", page_title)

        logger.warning("[Wikimedia] All articles exhausted for query: %s", search_query[:80])
        return False

    except Exception as exc:
        logger.warning("[Wikimedia] Exception: %s", exc)
        return False


async def _generate_via_pollinations(prompt: str, out_path: str) -> bool:
    """
    V22.0 — Generate image via Pollinations.ai.
    Updated URL: gen.pollinations.ai (image.pollinations.ai redirects → 404).
    3 retry attempts with exponential back-off.
    """
    safe = urllib.parse.quote(prompt[:500])
    url  = f"https://image.pollinations.ai/prompt/{safe}?width=800&height=600&nologo=true&model=flux"
    WAIT_TIMES = [5.0, 15.0, 30.0]

    for attempt in range(1, 4):
        try:
            logger.info(
                "[Pollinations] attempt %d/3 | prompt_len=%d",
                attempt, len(prompt)
            )
            async with httpx.AsyncClient(timeout=POLLINATIONS_TIMEOUT) as client:
                resp = await client.get(url, follow_redirects=True)

            if resp.status_code == 429:
                wait = WAIT_TIMES[attempt - 1]
                logger.warning("[Pollinations] 429 — waiting %.0f s (attempt %d/3)", wait, attempt)
                await asyncio.sleep(wait)
                continue

            if resp.status_code == 500:
                wait = WAIT_TIMES[attempt - 1]
                logger.warning("[Pollinations] 500 server error — waiting %.0f s (attempt %d/3)", wait, attempt)
                await asyncio.sleep(wait)
                continue

            if resp.status_code != 200:
                logger.warning("[Pollinations] HTTP %d — attempt %d/3", resp.status_code, attempt)
                if attempt < 3:
                    await asyncio.sleep(WAIT_TIMES[attempt - 1])
                continue

            img_bytes = resp.content
            if len(img_bytes) < MIN_WIKIMEDIA_BYTES:
                logger.warning("[Pollinations] Image too small (%d B) — attempt %d/3", len(img_bytes), attempt)
                if attempt < 3:
                    await asyncio.sleep(WAIT_TIMES[attempt - 1])
                continue

            with open(out_path, "wb") as fh:
                fh.write(img_bytes)
            logger.info("[Pollinations] ✅ Saved %s (%.1f KB)", out_path, len(img_bytes) / 1024)
            return True

        except httpx.TimeoutException:
            wait = WAIT_TIMES[attempt - 1]
            logger.warning("[Pollinations] Timeout attempt %d/3 — waiting %.0f s", attempt, wait)
            await asyncio.sleep(wait)
        except Exception as exc:
            logger.warning("[Pollinations] Exception attempt %d/3: %s", attempt, exc)
            if attempt < 3:
                await asyncio.sleep(WAIT_TIMES[attempt - 1])

    logger.warning("[Pollinations] All 3 attempts failed")
    return False


# ------------------------------------------------------------------
# Gemini — Plain text mode (Titan Mentor: Lecturer + Translator)
# ------------------------------------------------------------------

async def call_gemini_text(
    api_key:     str,
    prompt:      str,
    caller:      str   = "",
    max_tokens:  int   = 2000,
    temperature: float = 0.4,
) -> str:
    """
    Call Gemini and return plain text (not JSON).
    Used by Titan Mentor Lecturer and Translator agents.
    Raises RuntimeError on failure — caller handles retries.
    """
    import asyncio as _asyncio

    client = genai.Client(api_key=api_key)
    try:
        from google.genai import types as _gt
        cfg = _gt.GenerateContentConfig(
            temperature=temperature,
            top_p=0.95,
            max_output_tokens=max_tokens,
        )
        response = await _asyncio.to_thread(
            client.models.generate_content,
            model=GEMMA_MODEL_ID,
            contents=prompt,
            config=cfg,
        )
        text = (response.text or "").strip()
        if not text:
            raise ValueError(f"[{caller}] Gemini returned empty text response")
        return text
    except Exception as exc:
        raise RuntimeError(
            f"[{caller}] call_gemini_text failed: {exc}"
        ) from exc


# ------------------------------------------------------------------
# update_daily_decor — Pexels landscape photos → config/decor.json on HF
# ------------------------------------------------------------------

_DECOR_QUERIES: list = [
    "misty morning nature",
    "golden hour aesthetic",
    "starry night sky",
]


async def update_daily_decor() -> bool:
    """
    Titan Mentor daily job — called once per day by the scheduler in immersion.py.

    Fetches 3 high-quality landscape photos from Pexels (one per query in
    _DECOR_QUERIES) and uploads their metadata + URLs directly to
    config/decor.json on the HuggingFace dataset.

    Uses the existing PEXELS_API_KEY, HF_TOKEN, and HF_DATASET_ID env vars
    already present in the server — no new dependencies needed.

    Returns True on full success, False on any failure (non-fatal).
    """
    import datetime as _dt
    import io         as _io

    pexels_key = os.environ.get("PEXELS_API_KEY", "").strip()
    hf_token   = os.environ.get("HF_TOKEN",       "").strip()
    dataset_id = os.environ.get("HF_DATASET_ID",  "").strip()

    if not pexels_key:
        logger.warning("[Decor] PEXELS_API_KEY not set — skipping daily decor update")
        return False
    if not hf_token or not dataset_id:
        logger.warning(
            "[Decor] HF_TOKEN or HF_DATASET_ID not set — cannot upload decor.json"
        )
        return False

    decor_photos: list = []

    for query in _DECOR_QUERIES:
        try:
            async with httpx.AsyncClient(timeout=PEXELS_TIMEOUT) as client:
                resp = await client.get(
                    PEXELS_API_URL,
                    headers={"Authorization": pexels_key},
                    params={
                        "query":       query,
                        "per_page":    3,
                        "orientation": "landscape",
                    },
                )

            if resp.status_code == 401:
                logger.error("[Decor] Pexels 401 Unauthorized — check PEXELS_API_KEY")
                return False
            if resp.status_code != 200:
                logger.warning(
                    "[Decor] Pexels HTTP %d for query '%s'", resp.status_code, query
                )
                continue

            photos = resp.json().get("photos", [])
            if not photos:
                logger.warning("[Decor] No Pexels photos returned for query: %s", query)
                continue

            photo = photos[0]
            src   = photo.get("src", {})
            url   = (
                src.get("large2x")
                or src.get("large")
                or src.get("original")
                or ""
            )
            if not url:
                logger.warning("[Decor] No usable URL in photo for query: %s", query)
                continue

            decor_photos.append({
                "query":        query,
                "url":          url,
                "photographer": photo.get("photographer", "Unknown"),
                "photo_id":     photo.get("id", ""),
                "width":        photo.get("width", 0),
                "height":       photo.get("height", 0),
            })
            logger.info("[Decor] ✅ '%s' → %s", query, url[:80])

        except Exception as exc:
            logger.warning("[Decor] Exception for query '%s': %s", query, exc)

    if not decor_photos:
        logger.warning("[Decor] No photos fetched — decor.json NOT updated")
        return False

    # ── Build decor.json payload ───────────────────────────────────
    decor_payload = {
        "updated_at": _dt.datetime.utcnow().isoformat() + "Z",
        "count":      len(decor_photos),
        "photos":     decor_photos,
    }

    # ── Upload to HuggingFace dataset: config/decor.json ──────────
    try:
        from huggingface_hub import HfApi as _HfApi
        import asyncio as _asyncio2

        api  = _HfApi(token=hf_token)
        data = json.dumps(decor_payload, ensure_ascii=False, indent=2).encode("utf-8")

        await _asyncio2.to_thread(
            api.upload_file,
            path_or_fileobj=_io.BytesIO(data),
            path_in_repo="config/decor.json",
            repo_id=dataset_id,
            repo_type="dataset",
            commit_message=(
                f"Daily decor update — "
                f"{_dt.datetime.utcnow().strftime('%Y-%m-%d')} "
                f"({len(decor_photos)} photos)"
            ),
        )
        logger.info(
            "[Decor] ✅ decor.json uploaded to HF '%s' with %d photos",
            dataset_id, len(decor_photos),
        )
        return True

    except Exception as exc:
        logger.error("[Decor] Failed to upload decor.json to HF dataset: %s", exc)
        return False


async def generate_truly_free_image(
    prompt: str,
    out_path: str,
    search_query: str = "",
    chapter_num: int = 1,
    story_type: str = "historical",
) -> bool:
    """
    V23.1 -- Real-Photo Hybrid Image System.

    Priority order for ALL story types:
      1. Wikimedia Commons — topically accurate real/historical photos,
         public domain, directly tied to the story subject.
      2. Pexels API        — high-quality real photos (PEXELS_API_KEY),
         fallback when Wikimedia finds nothing relevant.

    `prompt` (cinematic_image_prompt) is retained for schema compatibility
    but is NOT used for image fetching in this version.

    Returns True when any provider saves a valid image.
    Returns False (non-fatal) — pipeline sets image_url="" and continues.
    """
    if not search_query.strip():
        logger.info("[Image/Hybrid] No search_query — image skipped (non-fatal)")
        return False

    # ── Stage 1: Wikimedia Commons ───────────────────────────────────
    # Best for historical/scientific topics — real photos tied to subject.
    logger.info(
        "[Image/Hybrid] Stage 1: Wikimedia | query='%.80s' | ch=%d | type=%s",
        search_query, chapter_num, story_type
    )
    ok = await _search_wikimedia_image(search_query, out_path, image_index=chapter_num - 1)
    if ok:
        logger.info("[Image/Hybrid] ✅ Wikimedia success (ch%d)", chapter_num)
        return True
    logger.info("[Image/Hybrid] Wikimedia failed — trying Pexels fallback")

    # ── Stage 2: Pexels API (fallback) ───────────────────────────────
    # Generic but high-quality real photos when Wikimedia has no match.
    logger.info(
        "[Image/Hybrid] Stage 2: Pexels | query='%.80s' | ch=%d | type=%s",
        search_query, chapter_num, story_type
    )
    ok = await _search_pexels_image(search_query, out_path)
    if ok:
        logger.info("[Image/Hybrid] ✅ Pexels success (ch%d)", chapter_num)
        return True

    logger.warning(
        "[Image/Hybrid] All sources failed for ch%d type=%s — image_url='' (non-fatal)",
        chapter_num, story_type
    )
    return False