core/api_clients.py

# ============================================================
# core/api_clients.py  --  Titan Gateway V16.6  "HF Rotation"
# ============================================================
# CHANGELOG V16.6:
#   [IMAGE]   Replaced Pollinations + Hercai with a Hugging Face
#             Inference API system backed by a token rotation pool.
#
#             TOKEN POOL:
#               Scans env vars HF_TOKEN1 ... HF_TOKEN8 at module load.
#               NOTE: HF_TOKEN (no number) is intentionally excluded.
#               It is reserved for the HuggingFace dataset/vault client
#               used elsewhere in the pipeline and must never be consumed
#               by image generation.
#               All found numbered tokens are stored in _HF_IMAGE_TOKENS.
#
#             ROUND-ROBIN ROTATION:
#               _hf_image_token_index (module-level int) advances by 1
#               on every call to _acquire_hf_image_token(), wrapping
#               around the list with modulo.
#               On HTTP 429 the current token is considered exhausted
#               for this request; the outer loop immediately picks the
#               next token without sleeping.
#
#             MODEL:
#               stabilityai/stable-diffusion-xl-base-1.0
#               Endpoint:
#                 https://api-inference.huggingface.co/models/
#                 stabilityai/stable-diffusion-xl-base-1.0
#
#             503 HANDLING (model cold-start):
#               HF returns HTTP 503 + {"estimated_time": N} while the
#               model loads. Retried up to HF_IMAGE_503_RETRIES (3)
#               times on the SAME token with a fixed HF_IMAGE_503_WAIT
#               (5 s) delay. 503 does NOT advance the token index.
#
#             THE BOUNCER:
#               After saving any file, os.path.getsize() is checked.
#               Files < MIN_REAL_IMAGE_BYTES (30 KB) are deleted and
#               False is returned. Real SDXL outputs are 200 KB-1 MB;
#               error/placeholder bodies are always smaller than 30 KB.
#
#             PROMPT:
#               Truncated to <= HF_IMAGE_MAX_KEYWORDS (15) keywords
#               using the existing _truncate_to_keywords() helper.
#               Sent as JSON payload {"inputs": "<prompt>"} -- NOT
#               URL-encoded in a query string, because the HF
#               Inference API accepts a POST body, not a GET param.
#
#             Entry point: generate_truly_free_image(prompt, out_path)
#             Returns False (never raises) -- pipeline never crashes.
#
#   [CLEANUP] _image_via_pollinations_turbo(), _image_via_hercai(),
#             POLLINATIONS_IMAGE_TIMEOUT, HERCAI_IMAGE_TIMEOUT,
#             FREE_IMAGE_MAX_KEYWORDS, MIN_REAL_IMAGE_BYTES (50 KB)
#             all replaced or renamed. No Pollinations or Hercai
#             URLs remain anywhere in the file.
#   [GROQ]    frequency_penalty=0.9, presence_penalty=0.7 unchanged.
# ============================================================

import asyncio
import httpx
import json
import logging
import os
import random
import re
import urllib.parse
from typing import Any, Dict, List, Optional

from google import genai
from groq import AsyncGroq, RateLimitError as GroqRateLimitError  # noqa: F401

logger = logging.getLogger("titan.api_clients")

# ------------------------------------------------------------------
# Constants -- audio (unchanged)
# ------------------------------------------------------------------
HF_AUDIOLDM2_URL:        str   = "https://api-inference.huggingface.co/models/cvssp/audioldm2"
HF_AUDIO_DURATION:       float = 10.0
HF_AUDIO_GUIDANCE:       float = 3.5
HF_AUDIO_RETRIES:        int   = 3
HF_AUDIO_TIMEOUT:        float = 90.0

# ------------------------------------------------------------------
# Constants -- V16.6 HF image generation
# ------------------------------------------------------------------

# Model endpoint (lightweight, stable SDXL variant) – UPDATED to router URL
# Primary: free serverless endpoint (no credits consumed)
HF_IMAGE_MODEL_URL:      str   = (
    "https://api-inference.huggingface.co/models/stabilityai/stable-diffusion-xl-base-1.0"
)
# Fallback: paid inference provider (used only if primary returns 404/unavailable)
HF_IMAGE_MODEL_URL_PAID: str   = (
    "https://router.huggingface.co/hf-inference/models/stabilityai/stable-diffusion-xl-base-1.0"
)
HF_IMAGE_TIMEOUT:       float = 120.0  # SDXL inference can take 60-90 s
HF_IMAGE_503_RETRIES:   int   = 3      # retries on model cold-start (503)
HF_IMAGE_503_WAIT:      float = 5.0    # seconds between 503 retries

# Max keywords sent to HF -- keeps the prompt concise
HF_IMAGE_MAX_KEYWORDS:  int   = 15

# The Bouncer threshold -- real SDXL images are 200 KB-1 MB;
# HF error / placeholder bodies are always < 30 KB.
MIN_REAL_IMAGE_BYTES:   int   = 30 * 1024  # 30 KB

# ------------------------------------------------------------------
# Constants -- LLM / translation (unchanged)
# ------------------------------------------------------------------
IMMERSION_AUTHOR_MODEL:  str   = "llama-3.3-70b-versatile"  # Director + Dice — json_object calls
GEMMA_MODEL_ID:          str   = os.environ.get("GEMINI_MODEL", "gemma-3-27b-it")
TRANSLATION_TEMPERATURE: float = 0.10

# MIN_IMAGE_BYTES retained for AudioLDM2 audio-payload validation only
MIN_IMAGE_BYTES:         int   = 1024


# ══════════════════════════════════════════════════════════════════
#
#   V16.6 -- HF IMAGE TOKEN POOL  (initialised once at module load)
#
#   Scans HF_TOKEN1 ... HF_TOKEN8.
#   HF_TOKEN (no suffix) is deliberately excluded: it belongs to
#   the HuggingFace dataset / vault client, not image generation.
#
# ══════════════════════════════════════════════════════════════════

_HF_IMAGE_TOKENS: List[str] = [
    tok
    for tok in (
        os.environ.get(f"HF_TOKEN{i}", "").strip()
        for i in range(1, 9)   # HF_TOKEN1 .. HF_TOKEN8
    )
    if tok                     # skip empty / unset variables
]

# Round-robin cursor -- shared across all coroutines in this process.
# Protected by _hf_token_lock to be safe under async concurrency.
_hf_image_token_index: int = 0
_hf_token_lock = asyncio.Lock()

if _HF_IMAGE_TOKENS:
    logger.info(
        "[HF-Image] Token pool ready: %d token(s) loaded "
        "(HF_TOKEN1-HF_TOKEN8). HF_TOKEN (no suffix) excluded.",
        len(_HF_IMAGE_TOKENS),
    )
else:
    logger.warning(
        "[HF-Image] No tokens found (HF_TOKEN1-HF_TOKEN8 not set). "
        "Image generation will return False for every chapter."
    )


async def _acquire_hf_image_token() -> Optional[str]:
    """
    Return the next token from the round-robin pool and advance the
    cursor.  Thread-safe via _hf_token_lock.
    Returns None if the pool is empty.
    """
    if not _HF_IMAGE_TOKENS:
        return None
    global _hf_image_token_index
    async with _hf_token_lock:
        token = _HF_IMAGE_TOKENS[_hf_image_token_index % len(_HF_IMAGE_TOKENS)]
        _hf_image_token_index += 1
    return token


# ------------------------------------------------------------------
# Utility helpers (unchanged)
# ------------------------------------------------------------------

def clean_json(raw: str) -> str:
    """Strip markdown code fences and trim to first {...} or [...]."""
    raw = re.sub(r"^```(?:json)?\s*\n?", "", raw, flags=re.MULTILINE)
    raw = re.sub(r"\n?\s*```\s*$",        "", raw, flags=re.MULTILINE)
    raw = raw.strip()
    starts = [i for i in [raw.find("{"), raw.find("[")] if i >= 0]
    if starts:
        raw = raw[min(starts):]
    end = max(raw.rfind("}"), raw.rfind("]"))
    if end >= 0:
        raw = raw[: end + 1]
    return raw.strip()


_CONTAMINATION_PATTERNS: re.Pattern = re.compile(
    r"https?://\S+"
    r"|www\.\S+"
    r"|\{[^}]{0,500}\}"
    r"|\b[45]\d{2}\s+(?:Internal|Not\s+Found|Bad\s+Request|"
    r"Forbidden|Unauthorized|Service\s+Unavailable|Error)\b"
    r"|(?:^|\n)\s*(?:Error|Exception|Traceback)\s*[:(]"
    r'|"(?:error|detail|message|status)"\s*:\s*"',
    re.IGNORECASE | re.MULTILINE,
)


def text_is_contaminated(text: str) -> bool:
    """Return True if the text contains technical artifacts."""
    return bool(_CONTAMINATION_PATTERNS.search(text))


def _truncate_to_keywords(prompt: str, max_kw: int = HF_IMAGE_MAX_KEYWORDS) -> str:
    """
    Reduce an image prompt to at most `max_kw` comma-separated keywords.

    Strategy:
      1. Commas present -> split on commas, take first max_kw parts.
      2. No commas     -> split on whitespace, take first max_kw words.

    Special characters that could break parsers are stripped from each
    token before reassembly.  Returns a clean ASCII-safe string.
    """
    if "," in prompt:
        parts = [p.strip() for p in prompt.split(",") if p.strip()]
    else:
        parts = prompt.split()

    selected = parts[:max_kw]
    cleaned  = [re.sub(r"[^\w\s-]", "", p).strip() for p in selected]
    return ", ".join(w for w in cleaned if w)


# ------------------------------------------------------------------
# Groq -- JSON mode (unchanged)
# ------------------------------------------------------------------

async def call_groq_json(
    api_key:     str,
    prompt:      str,
    caller:      str,
    max_tokens:  int   = 4000,
    temperature: float = 0.4,
    model:       str   = IMMERSION_AUTHOR_MODEL,   # override per-caller if needed
) -> Dict[str, Any]:
    """Call Groq in JSON-mode. Raises on failure; caller handles retries."""
    client   = AsyncGroq(api_key=api_key)
    response = await client.chat.completions.create(
        model=model,
        response_format={"type": "json_object"},
        temperature=temperature,
        max_tokens=max_tokens,
        messages=[
            {"role": "system", "content": "You are a helpful assistant. Output only valid JSON."},
            {"role": "user",   "content": prompt},
        ],
    )
    raw = clean_json(response.choices[0].message.content.strip())
    result = json.loads(raw)
    # GPT OSS 120B sometimes wraps the JSON in a list — unwrap it
    if isinstance(result, list) and len(result) == 1 and isinstance(result[0], dict):
        result = result[0]
    return result


# ------------------------------------------------------------------
# Gemini -- JSON mode  (unchanged)
# ------------------------------------------------------------------

async def call_gemini_json(
    api_key:     str,
    prompt:      str,
    caller:      str   = "",
    max_tokens:  int   = 4000,
    temperature: float = 0.25,
) -> Dict[str, Any]:
    """Call Gemini expecting JSON output. Uses the google.genai client."""
    client = genai.Client(api_key=api_key)
    try:
        from google.genai import types as _gt
        cfg = _gt.GenerateContentConfig(
            temperature=temperature,
            top_p=0.95,
            max_output_tokens=max_tokens,
        )
        response = await asyncio.to_thread(
            client.models.generate_content,
            model=GEMMA_MODEL_ID,
            contents=prompt,
            config=cfg,
        )
    except Exception:
        response = await asyncio.to_thread(
            client.models.generate_content,
            model=GEMMA_MODEL_ID,
            contents=prompt,
        )
    raw = clean_json(response.text.strip())
    return json.loads(raw)


# ------------------------------------------------------------------
# Groq -- Whisper transcription (unchanged)
# ------------------------------------------------------------------

async def dispatch_transcription(
    api_key:     str,
    audio_bytes: bytes,
    filename:    str,
    language:    Optional[str] = None,
    prompt:      Optional[str] = None,
) -> Dict[str, Any]:
    """Transcribe audio using Groq Whisper-large-v3-turbo."""
    import io
    client        = AsyncGroq(api_key=api_key)
    audio_stream  = io.BytesIO(audio_bytes)
    audio_stream.name = filename
    kwargs: Dict[str, Any] = {
        "file":            (filename, audio_stream),
        "model":           "whisper-large-v3-turbo",
        "language":        language,
        "response_format": "verbose_json",
    }
    if prompt:
        kwargs["prompt"] = prompt[-800:]
    tx = await client.audio.transcriptions.create(**kwargs)

    if hasattr(tx, "model_dump"):
        result = tx.model_dump()
    elif hasattr(tx, "to_dict"):
        result = tx.to_dict()
    else:
        result = {
            "text":     getattr(tx, "text",     ""),
            "segments": getattr(tx, "segments", []),
            "language": getattr(tx, "language", "unknown"),
        }
    for idx, seg in enumerate(result.get("segments", [])):
        if "id" not in seg:
            seg["id"] = str(idx)
    return result


# ------------------------------------------------------------------
# HF AudioLDM2 ambient generation (unchanged)
# ------------------------------------------------------------------

async def generate_audio_ldm2(
    hf_token:     str,
    audio_prompt: str,
    out_path:     str,
) -> bool:
    """Generate a 10-second ambient clip via HF AudioLDM 2."""
    headers = {
        "Authorization": f"Bearer {hf_token}",
        "Content-Type":  "application/json",
    }
    payload = {
        "inputs": audio_prompt,
        "parameters": {
            "audio_length_in_s":        HF_AUDIO_DURATION,
            "guidance_scale":           HF_AUDIO_GUIDANCE,
            "num_waveforms_per_prompt": 1,
        },
    }

    for attempt in range(1, HF_AUDIO_RETRIES + 1):
        try:
            logger.info("[AudioLDM2] attempt %d/%d", attempt, HF_AUDIO_RETRIES)
            async with httpx.AsyncClient(timeout=HF_AUDIO_TIMEOUT) as client:
                resp = await client.post(HF_AUDIOLDM2_URL, headers=headers, json=payload)

            if resp.status_code == 503:
                try:
                    wait = min(float(resp.json().get("estimated_time", 30.0)), 60.0)
                except Exception:
                    wait = 30.0
                await asyncio.sleep(wait)
                continue

            if resp.status_code == 429:
                await asyncio.sleep(65.0 * attempt)
                continue

            if resp.status_code != 200:
                logger.error("[AudioLDM2] HTTP %d", resp.status_code)
                if attempt < HF_AUDIO_RETRIES:
                    await asyncio.sleep(10.0 * attempt)
                continue

            audio_bytes = resp.content
            if len(audio_bytes) < 512:
                await asyncio.sleep(10.0 * attempt)
                continue

            with open(out_path, "wb") as fh:
                fh.write(audio_bytes)
            logger.info(
                "[AudioLDM2] Saved -> %s (%.0f KB)", out_path, len(audio_bytes) / 1024
            )
            return True

        except httpx.TimeoutException:
            await asyncio.sleep(10.0 * attempt)
        except Exception as exc:
            logger.error("[AudioLDM2] Error attempt=%d: %s", attempt, exc)
            await asyncio.sleep(10.0 * attempt)

    logger.error("[AudioLDM2] Failed after %d attempts", HF_AUDIO_RETRIES)
    return False


# ------------------------------------------------------------------
# Gemini -- subtitle translation batch (unchanged)
# ------------------------------------------------------------------

async def call_gemini_translation_batch(
    api_key: str,
    prompt:  str,
) -> Dict[str, str]:
    """Translate subtitle segments via Gemini. Returns id -> text dict."""
    client = genai.Client(api_key=api_key)
    try:
        from google.genai import types as _gt
        cfg = _gt.GenerateContentConfig(
            temperature=TRANSLATION_TEMPERATURE,
            top_p=0.95,
            max_output_tokens=8192,
        )
        response = await asyncio.to_thread(
            client.models.generate_content,
            model=GEMMA_MODEL_ID,
            contents=prompt,
            config=cfg,
        )
    except Exception:
        response = await asyncio.to_thread(
            client.models.generate_content,
            model=GEMMA_MODEL_ID,
            contents=prompt,
        )
    parsed: Dict[str, str] = {}
    for line in response.text.strip().splitlines():
        m = re.match(r"^\[([^\]]+)\]\s*(.*)", line.strip())
        if m:
            parsed[m.group(1)] = m.group(2).strip()
    return parsed


async def call_gemini_chunk_translation(
    api_key: str,
    prefix:  str,
    chunk:   str,
) -> str:
    """Translate a single text chunk via Gemini."""
    client = genai.Client(api_key=api_key)
    try:
        from google.genai import types as _gt
        cfg = _gt.GenerateContentConfig(
            temperature=0.15,
            top_p=0.95,
            max_output_tokens=3000,
        )
        response = await asyncio.to_thread(
            client.models.generate_content,
            model=GEMMA_MODEL_ID,
            contents=prefix + chunk,
            config=cfg,
        )
    except Exception:
        response = await asyncio.to_thread(
            client.models.generate_content,
            model=GEMMA_MODEL_ID,
            contents=prefix + chunk,
        )
    return response.text.strip()


# ------------------------------------------------------------------
# Groq -- Chat completion  (Narrator / Novelist)  (unchanged)
# ------------------------------------------------------------------

async def call_groq_chat_completion(
    api_key:       str,
    system_prompt: str,
    user_prompt:   str,
    model:         str,
    max_tokens:    int,
    temperature:   float,
) -> str:
    """
    Groq chat completion for The Narrator.
    V18.2: Robust content extraction —
    - No frequency_penalty / presence_penalty (not supported by GPT OSS 120B)
    - Handles None content via reasoning_content fallback
    - Full response logged on empty to diagnose root cause
    """
    client   = AsyncGroq(api_key=api_key)
    response = await client.chat.completions.create(
        messages=[
            {"role": "system", "content": system_prompt},
            {"role": "user",   "content": user_prompt},
        ],
        model=model,
        temperature=temperature,
        max_tokens=max_tokens,
    )

    if not response.choices:
        logger.error(
            "[Narrator] No choices in response — model=%s | response=%s",
            model, str(response)[:500]
        )
        return ""

    choice = response.choices[0]
    msg    = choice.message

    # ONLY use msg.content — never reasoning/reasoning_content.
    # reasoning field contains the model's thinking process, not the narration.
    # If content is empty → max_tokens too low → retry with more tokens.
    raw = msg.content if msg.content else None

    if not raw:
        logger.error(
            "[Narrator] EMPTY CONTENT — model=%s | finish_reason=%s | "
            "msg_fields=%s | full_response=%s",
            model,
            getattr(choice, "finish_reason", "?"),
            list((msg.model_dump() or {}).keys()) if hasattr(msg, "model_dump") else "?",
            str(response)[:1000],
        )
        return ""

    return raw.strip()


# ══════════════════════════════════════════════════════════════════
#
#   V16.6 -- HF INFERENCE IMAGE GENERATION  (new)
#
#   Token pool:  HF_TOKEN1 ... HF_TOKEN8  (HF_TOKEN excluded)
#   Model:       stabilityai/stable-diffusion-xl-base-1.0
#   Rotation:    round-robin; 429 -> skip token, try next
#   503 retry:   up to 3x with 5 s delay per token (model warm-up)
#   Bouncer:     reject + delete files < 30 KB
#
# ══════════════════════════════════════════════════════════════════

def _bouncer(out_path: str, token_label: str) -> bool:
    """
    Post-write file-size gate (The Bouncer).

    Reads os.path.getsize() on the already-written file.
    Rejects and deletes anything < MIN_REAL_IMAGE_BYTES (30 KB).

    Real SDXL outputs are 200 KB - 1 MB.
    HF error JSON bodies and placeholder images are always < 30 KB.

    Returns True if the file is large enough to be a genuine image.
    Returns False (and deletes the file) otherwise.
    `token_label` is used only in log messages.
    """
    try:
        size = os.path.getsize(out_path)
    except OSError as exc:
        logger.error("[Bouncer/%s] Cannot stat '%s': %s", token_label, out_path, exc)
        return False

    if size < MIN_REAL_IMAGE_BYTES:
        logger.error(
            "[Bouncer/%s] File too small (%d B < %d B minimum) -- "
            "placeholder or error body. Deleting.",
            token_label, size, MIN_REAL_IMAGE_BYTES,
        )
        try:
            os.remove(out_path)
        except OSError:
            pass
        return False

    logger.info("[Bouncer/%s] Size OK -- %.1f KB", token_label, size / 1024)
    return True


async def _hf_infer_image_with_token(
    token:       str,
    token_label: str,
    prompt:      str,
    out_path:    str,
) -> bool:
    """
    Attempt one image generation call using a single HF Bearer token.

    Tries TWO endpoints in order:
      1. HF_IMAGE_MODEL_URL      — free serverless (api-inference.huggingface.co)
      2. HF_IMAGE_MODEL_URL_PAID — paid inference provider (router.huggingface.co)
         Used only if primary returns 402/404/503-exhausted.

    402 on primary  → silently try paid URL (credits on paid, not free)
    402 on paid     → token genuinely exhausted, return False
    503 anywhere    → cold-start, retry same URL up to HF_IMAGE_503_RETRIES
    429 anywhere    → rate-limited, return False (caller rotates token)
    200 + bouncer   → True
    """
    headers = {
        "Authorization": f"Bearer {token}",
        "Content-Type":  "application/json",
        "Accept":        "image/jpeg",
    }
    payload = {"inputs": prompt}

    # Try free endpoint first, then paid fallback
    endpoints = [
        (HF_IMAGE_MODEL_URL,      "free"),
        (HF_IMAGE_MODEL_URL_PAID, "paid"),
    ]

    for endpoint_url, endpoint_label in endpoints:
        logger.info(
            "[HF-Image/%s] Trying %s endpoint | prompt_len=%d",
            token_label, endpoint_label, len(prompt),
        )

        for cold_attempt in range(1, HF_IMAGE_503_RETRIES + 1):
            try:
                logger.info(
                    "[HF-Image/%s] POST | endpoint=%s | cold_attempt=%d/%d | prompt_len=%d",
                    token_label, endpoint_label, cold_attempt, HF_IMAGE_503_RETRIES, len(prompt),
                )
                async with httpx.AsyncClient(timeout=HF_IMAGE_TIMEOUT) as client:
                    resp = await client.post(
                        endpoint_url, headers=headers, json=payload
                    )

            # -- 503: model loading; retry same URL after a short wait --
                if resp.status_code == 503:
                    try:
                        wait = min(
                            float(resp.json().get("estimated_time", HF_IMAGE_503_WAIT)),
                            60.0,
                        )
                    except Exception:
                        wait = HF_IMAGE_503_WAIT
                    if cold_attempt < HF_IMAGE_503_RETRIES:
                        logger.info(
                            "[HF-Image/%s] 503 model loading (%s) -- waiting %.0f s "
                            "(retry %d/%d)",
                            token_label, endpoint_label, wait, cold_attempt, HF_IMAGE_503_RETRIES,
                        )
                        await asyncio.sleep(wait)
                        continue   # retry same endpoint
                    else:
                        logger.warning(
                            "[HF-Image/%s] 503 persisted on %s -- trying next endpoint",
                            token_label, endpoint_label,
                        )
                        break  # try next endpoint

                # -- 402: credits exhausted on this endpoint -- try next ----
                if resp.status_code == 402:
                    logger.warning(
                        "[HF-Image/%s] 402 credits exhausted on %s -- trying next endpoint",
                        token_label, endpoint_label,
                    )
                    break  # try next endpoint

                # -- 429: rate-limited; signal caller to rotate token -------
                if resp.status_code == 429:
                    logger.warning(
                        "[HF-Image/%s] 429 rate-limited on %s -- rotating to next token",
                        token_label, endpoint_label,
                    )
                    return False

                # -- Any other non-200 error --------------------------------
                if resp.status_code != 200:
                    logger.error(
                        "[HF-Image/%s] HTTP %d on %s -- %.200s",
                        token_label, resp.status_code, endpoint_label, resp.text,
                    )
                    break  # try next endpoint

                # -- 200 success: write file and validate ---------------
                image_bytes = resp.content
                if not image_bytes:
                    logger.error(
                        "[HF-Image/%s] HTTP 200 but empty body on %s",
                        token_label, endpoint_label,
                    )
                    break  # try next endpoint

                with open(out_path, "wb") as fh:
                    fh.write(image_bytes)

                if not _bouncer(out_path, token_label):
                    break  # try next endpoint

                logger.info(
                    "[HF-Image/%s] Image saved via %s -> %s (%.1f KB)",
                    token_label, endpoint_label, out_path, len(image_bytes) / 1024,
                )
                return True

            except httpx.TimeoutException:
                logger.error(
                    "[HF-Image/%s] Timed out on %s after %.0f s",
                    token_label, endpoint_label, HF_IMAGE_TIMEOUT,
                )
                break  # try next endpoint
            except Exception as exc:
                logger.error(
                    "[HF-Image/%s] Unexpected exception on %s: %s",
                    token_label, endpoint_label, exc,
                )
                break  # try next endpoint

    # All endpoints exhausted for this token
    logger.error("[HF-Image/%s] All endpoints exhausted", token_label)
    return False



# ══════════════════════════════════════════════════════════════════
#
#   V18.5 -- HYBRID IMAGE SYSTEM
#
#   Priority order:
#     1. Wikimedia Commons — real historical images, public domain, no key
#     2. Pollinations.ai   — free AI generation, no key, no limit
#     3. HF Inference      — paid fallback (existing system)
#
# ══════════════════════════════════════════════════════════════════

WIKIMEDIA_API:      str   = "https://en.wikipedia.org/w/api.php"
WIKIMEDIA_TIMEOUT:  float = 15.0
MIN_WIKIMEDIA_BYTES: int  = 20 * 1024   # 20 KB minimum

POLLINATIONS_URL:   str   = "https://image.pollinations.ai/prompt/{prompt}"
POLLINATIONS_TIMEOUT: float = 60.0


async def _search_wikimedia_image(search_query: str, out_path: str) -> bool:
    """
    Search Wikimedia Commons for a relevant historical image.
    Uses the Wikipedia opensearch + imageinfo APIs.
    Returns True if a valid image >= 20 KB was saved.
    """
    try:
        # Step 1: Find the most relevant Wikipedia article
        async with httpx.AsyncClient(timeout=WIKIMEDIA_TIMEOUT) as client:
            resp = await client.get(
                WIKIMEDIA_API,
                params={
                    "action":   "query",
                    "list":     "search",
                    "srsearch": search_query,
                    "srlimit":  1,
                    "format":   "json",
                },
                headers={"User-Agent": "TitanImmersion/1.0 (educational)"},
            )
        if resp.status_code != 200:
            logger.warning("[Wikimedia] Search HTTP %d", resp.status_code)
            return False

        results = resp.json().get("query", {}).get("search", [])
        if not results:
            logger.warning("[Wikimedia] No results for: %s", search_query[:80])
            return False

        page_title = results[0]["title"]
        logger.info("[Wikimedia] Found article: %s", page_title)

        # Step 2: Get images from that article
        async with httpx.AsyncClient(timeout=WIKIMEDIA_TIMEOUT) as client:
            resp = await client.get(
                WIKIMEDIA_API,
                params={
                    "action":   "query",
                    "titles":   page_title,
                    "prop":     "images",
                    "imlimit":  10,
                    "format":   "json",
                },
                headers={"User-Agent": "TitanImmersion/1.0 (educational)"},
            )

        pages = resp.json().get("query", {}).get("pages", {})
        images = []
        for page in pages.values():
            for img in page.get("images", []):
                title = img.get("title", "")
                # Skip icons, logos, flags, small decorative images
                if any(skip in title.lower() for skip in [
                    "flag", "icon", "logo", "map", "seal", "coat",
                    "stub", "commons", "wikimedia", ".svg"
                ]):
                    continue
                if title.lower().endswith((".jpg", ".jpeg", ".png")):
                    images.append(title)

        if not images:
            logger.warning("[Wikimedia] No usable images for: %s", page_title)
            return False

        # Step 3: Get the actual URL for the first good image
        async with httpx.AsyncClient(timeout=WIKIMEDIA_TIMEOUT) as client:
            resp = await client.get(
                WIKIMEDIA_API,
                params={
                    "action":    "query",
                    "titles":    images[0],
                    "prop":      "imageinfo",
                    "iiprop":    "url|size",
                    "iiurlwidth": 800,
                    "format":    "json",
                },
                headers={"User-Agent": "TitanImmersion/1.0 (educational)"},
            )

        pages = resp.json().get("query", {}).get("pages", {})
        img_url = None
        for page in pages.values():
            info = page.get("imageinfo", [])
            if info:
                img_url = info[0].get("thumburl") or info[0].get("url")
                break

        if not img_url:
            logger.warning("[Wikimedia] No URL for image: %s", images[0])
            return False

        # Step 4: Download the image
        async with httpx.AsyncClient(timeout=WIKIMEDIA_TIMEOUT) as client:
            resp = await client.get(
                img_url,
                headers={"User-Agent": "TitanImmersion/1.0 (educational)"},
                follow_redirects=True,
            )

        if resp.status_code != 200:
            logger.warning("[Wikimedia] Download HTTP %d", resp.status_code)
            return False

        img_bytes = resp.content
        if len(img_bytes) < MIN_WIKIMEDIA_BYTES:
            logger.warning("[Wikimedia] Image too small: %d B", len(img_bytes))
            return False

        with open(out_path, "wb") as fh:
            fh.write(img_bytes)

        logger.info(
            "[Wikimedia] ✅ Saved %s (%.1f KB) from: %s",
            out_path, len(img_bytes) / 1024, page_title,
        )
        return True

    except Exception as exc:
        logger.warning("[Wikimedia] Exception: %s", exc)
        return False


async def _generate_via_pollinations(prompt: str, out_path: str) -> bool:
    """
    Generate image via Pollinations.ai — free, no API key, no rate limit.
    Returns True if a valid image >= 20 KB was saved.
    """
    try:
        safe = urllib.parse.quote(prompt[:500])
        url  = f"https://image.pollinations.ai/prompt/{safe}?width=800&height=600&nologo=true"

        logger.info("[Pollinations] Requesting image | prompt_len=%d", len(prompt))
        async with httpx.AsyncClient(timeout=POLLINATIONS_TIMEOUT) as client:
            resp = await client.get(url, follow_redirects=True)

        if resp.status_code != 200:
            logger.warning("[Pollinations] HTTP %d", resp.status_code)
            return False

        img_bytes = resp.content
        if len(img_bytes) < MIN_WIKIMEDIA_BYTES:
            logger.warning("[Pollinations] Image too small: %d B", len(img_bytes))
            return False

        with open(out_path, "wb") as fh:
            fh.write(img_bytes)

        logger.info(
            "[Pollinations] ✅ Saved %s (%.1f KB)",
            out_path, len(img_bytes) / 1024,
        )
        return True

    except Exception as exc:
        logger.warning("[Pollinations] Exception: %s", exc)
        return False


async def generate_truly_free_image(
    prompt: str,
    out_path: str,
    search_query: str = "",
) -> bool:
    """
    V18.5 -- Hybrid Image System.

    Priority order:
      1. Wikimedia Commons  — real historical images, public domain, no key
         Uses search_query (historical_anchor event + year) if provided.
      2. Pollinations.ai    — free AI generation, no key, no rate limit
         Uses the cinematic_image_prompt.
      3. HF Inference       — paid fallback (existing token pool system)

    Returns True when any provider saves a valid image.
    Returns False (non-fatal) — pipeline sets image_url="" and continues.
    """
    # ── Stage 1: Wikimedia Commons ──────────────────────────────────
    if search_query.strip():
        logger.info("[Image/Hybrid] Stage 1: Wikimedia | query='%.80s'", search_query)
        ok = await _search_wikimedia_image(search_query, out_path)
        if ok:
            logger.info("[Image/Hybrid] ✅ Wikimedia success")
            return True
        logger.info("[Image/Hybrid] Wikimedia failed — trying Pollinations")
    else:
        logger.info("[Image/Hybrid] No search_query — skipping Wikimedia")

    # ── Stage 2: Pollinations.ai ─────────────────────────────────────
    logger.info("[Image/Hybrid] Stage 2: Pollinations | prompt_len=%d", len(prompt))
    ok = await _generate_via_pollinations(prompt, out_path)
    if ok:
        logger.info("[Image/Hybrid] ✅ Pollinations success")
        return True
    logger.info("[Image/Hybrid] Pollinations failed — trying HF tokens")

    # ── Stage 3: HF Inference (existing paid system) ─────────────────
    if not _HF_IMAGE_TOKENS:
        logger.error(
            "[Image/Hybrid] All stages failed — no HF tokens available. "
            "Setting image_url='' for this chapter."
        )
        return False

    # Truncate once; every token attempt uses this same string
    safe_prompt = _truncate_to_keywords(prompt, HF_IMAGE_MAX_KEYWORDS)
    n_tokens    = len(_HF_IMAGE_TOKENS)

    logger.info(
        "[HF-Image] Starting | pool=%d token(s) | prompt='%.80s'",
        n_tokens, safe_prompt,
    )

    for attempt in range(n_tokens):
        token = await _acquire_hf_image_token()
        if token is None:
            break  # pool emptied concurrently -- should not happen

        token_label = f"TOKEN...{token[-4:]}"  # last 4 chars for safe logging

        ok = await _hf_infer_image_with_token(
            token=token,
            token_label=token_label,
            prompt=safe_prompt,
            out_path=out_path,
        )

        if ok:
            logger.info(
                "[HF-Image] Success with %s (attempt %d/%d)",
                token_label, attempt + 1, n_tokens,
            )
            return True

        remaining = n_tokens - attempt - 1
        logger.warning(
            "[HF-Image] %s failed (attempt %d/%d) -- %s",
            token_label,
            attempt + 1,
            n_tokens,
            f"{remaining} token(s) remaining" if remaining else "all tokens exhausted",
        )

    logger.error(
        "[HF-Image] All %d token(s) exhausted. "
        "Setting image_url='' for this chapter.",
        n_tokens,
    )
    return False