# backend/tasks/tools/web_search_tools.py
import asyncio
import json
import urllib.request
import urllib.parse
import os
import logging

import html as _html
import re as _re

# DuckDuckGo's HTML endpoint serves the classic markup; a real browser UA is
# required or it returns a stripped page with no results.
_WIKI_UA = "JARVIS_OMEGA/1.0 (+https://huggingface.co/spaces/Jarvis2345/jarvis-cloud)"

_BROWSER_UA = ("Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 "
               "(KHTML, like Gecko) Chrome/120.0 Safari/537.36")

_DDG_RESULT_RE = _re.compile(
    r'<a[^>]+class="[^"]*result__a[^"]*"[^>]*href="([^"]+)"[^>]*>(.*?)</a>', _re.S)
_DDG_SNIPPET_RE = _re.compile(
    r'<a[^>]+class="[^"]*result__snippet[^"]*"[^>]*>(.*?)</a>', _re.S)


def _strip_html(fragment: str) -> str:
    return _html.unescape(_re.sub(r"<[^>]+>", "", fragment or "")).strip()


def _ddg_unwrap(href: str) -> str:
    """DuckDuckGo wraps outbound links as //duckduckgo.com/l/?uddg=<encoded>."""
    if "duckduckgo.com/l/" in href:
        target = "https:" + href if href.startswith("//") else href
        query = urllib.parse.urlparse(target).query
        decoded = urllib.parse.parse_qs(query).get("uddg", [""])[0]
        return decoded or href
    return href


def _resolve_key(*names: str) -> str | None:
    """Resolve a credential the way the rest of the backend does.

    `get_secret` checks real environment variables first (how HF Spaces inject
    secrets) and then the encrypted SQLite vault. Reading `os.environ` directly —
    as this module used to — bypasses the vault entirely, so a key the user had
    actually configured through the UI was invisible here.
    """
    for name in names:
        if not name:
            continue
        try:
            from backend.services.usb_vault import get_secret
            value = get_secret(name)
        except Exception:
            value = os.environ.get(name)
        if value:
            return value
    return None


async def search_web(query: str, num_results: int = 5) -> dict:
    """Web search: Brave (primary) -> SerpAPI (fallback) -> DuckDuckGo (last resort).

    Two real bugs fixed here.

    1. ``num_results`` did not exist, yet every caller passed it —
       internet_routes.py (search + osint) and sentinel_routes.py. All three
       routes were a hard 500:
       ``search_web() got an unexpected keyword argument 'num_results'``.
       The result cap was also hardcoded at 5; it is now honoured.

    2. The Brave credential was looked up as ``BRAVE_API_KEY`` via a raw
       ``os.environ`` read, but the system provisions it as
       ``BRAVE_SEARCH_API_KEY`` (src-tauri/src/key_registry.rs) and stores it in
       the encrypted vault. The name never matched and the vault was never
       consulted, so Brave was *always* skipped and every search silently fell
       through to DuckDuckGo's instant-answer endpoint — which returns a single,
       usually empty "Abstract" and ignores the result count. For an assistant
       whose whole point is live research, that is a broken capability that
       reports success.

    SerpAPI is now wired as the real fallback; key_registry.rs already describes
    ``SERPAPI_KEY`` as the "Google/Bing search fallback", but nothing used it, so
    the documented main+fallback pair did not actually exist.
    """
    try:
        num_results = max(1, min(int(num_results), 20))
    except (TypeError, ValueError):
        num_results = 5

    brave_api_key = _resolve_key("BRAVE_SEARCH_API_KEY", "BRAVE_API_KEY")
    serpapi_key = _resolve_key("SERPAPI_KEY")

    def _duck_html_search():
        """Real organic results from DuckDuckGo's HTML endpoint.

        The keyless fallback used to hit api.duckduckgo.com (the *instant answer*
        API), which only ever returns a single "Abstract" and is empty for almost
        every real query — measured: 0 usable results for OSINT-shaped searches.
        That is why /internet/osint and /sentinel/threat_model came back with
        empty findings even though the routes were working.

        The HTML endpoint returns actual ranked web results and needs no API key,
        no account and no payment card — which keeps OSINT working when neither
        BRAVE_SEARCH_API_KEY nor SERPAPI_KEY is configured.
        """
        # Short timeout on purpose. Callers such as the research engine issue
        # several searches per request and then fetch pages on top; a 25s budget
        # here stacked up and pushed /omega/research past its own limit, turning a
        # fast "no results" into a hung request. Fail fast and let the caller
        # continue with whatever it has.
        url = "https://html.duckduckgo.com/html/?q=" + urllib.parse.quote(query)
        req = urllib.request.Request(url, headers={"User-Agent": _BROWSER_UA})
        try:
            with urllib.request.urlopen(req, timeout=8) as response:
                body = response.read().decode("utf-8", "replace")
        except Exception as e:
            return {"error": str(e)}

        # DuckDuckGo throttles automated use: it answers 200 with a much smaller
        # challenge page containing an "anomaly" marker and no results. Report
        # that distinctly instead of returning an empty list, so a throttled
        # keyless fallback is never mistaken for "the web has nothing".
        if "result__a" not in body and "anomaly" in body.lower():
            return {"error": "duckduckgo rate-limited this client (anti-automation challenge); "
                             "configure BRAVE_SEARCH_API_KEY or SERPAPI_KEY for reliable search"}

        titles = _DDG_RESULT_RE.findall(body)
        snippets = _DDG_SNIPPET_RE.findall(body)
        results = []
        for i, (href, raw_title) in enumerate(titles):
            link = _ddg_unwrap(href)
            # Skip sponsored placements — they are ads, not search results.
            if "duckduckgo.com/y.js" in link or "ad_domain=" in link:
                continue
            results.append({
                "title": _strip_html(raw_title)[:160],
                "snippet": _strip_html(snippets[i])[:320] if i < len(snippets) else "",
                "url": link,
            })
            if len(results) >= num_results:
                break
        return {"results": results} if results else {"error": "duckduckgo returned no results"}

    def _wikipedia_search():
        """Free, keyless, genuinely unlimited — and rate-limit free.

        Every commercial option now gates access (Brave/SerpAPI need a card or
        signup, Gemini search-grounding 429s without billing) and every keyless
        one throttles (DuckDuckGo challenges automated use, public SearXNG
        instances 403 the JSON API). Wikipedia's REST/Action API does not, so it
        is the one source that keeps research working with zero credentials.

        Deliberately placed above the DuckDuckGo scrape: authoritative, fast
        JSON, and it cannot rate-limit the assistant out of working.
        """
        try:
            api = ("https://en.wikipedia.org/w/api.php?action=query&list=search"
                   "&srsearch=" + urllib.parse.quote(query) +
                   f"&srlimit={num_results}&format=json&origin=*")
            req = urllib.request.Request(api, headers={"User-Agent": _WIKI_UA})
            with urllib.request.urlopen(req, timeout=12) as resp:
                data = json.loads(resp.read().decode("utf-8", "replace"))
            hits = data.get("query", {}).get("search", [])
            results = []
            for h in hits[:num_results]:
                title = h.get("title", "")
                results.append({
                    "title": title,
                    # Wikipedia returns the snippet with HTML search highlighting.
                    "snippet": _strip_html(h.get("snippet", "")),
                    "url": "https://en.wikipedia.org/wiki/" + urllib.parse.quote(title.replace(" ", "_")),
                })
            return {"results": results} if results else {"error": "wikipedia returned no results"}
        except Exception as e:
            return {"error": str(e)}

    def _serpapi_search():
        url = ("https://serpapi.com/search.json?q=" + urllib.parse.quote(query)
               + f"&num={num_results}&api_key={urllib.parse.quote(serpapi_key)}")
        try:
            with urllib.request.urlopen(url, timeout=12) as response:
                parsed = json.loads(response.read().decode('utf-8'))
            results = [{"title": i.get("title"), "snippet": i.get("snippet"), "url": i.get("link")}
                       for i in parsed.get("organic_results", [])[:num_results]]
            return {"results": results} if results else {"error": "serpapi returned no results"}
        except Exception as e:
            return {"error": str(e)}

    def _brave_search():
        url = f"https://api.search.brave.com/res/v1/web/search?q={urllib.parse.quote(query)}"
        req = urllib.request.Request(url, headers={
            'Accept': 'application/json',
            'Accept-Encoding': 'gzip',
            'X-Subscription-Token': brave_api_key
        })
        try:
            with urllib.request.urlopen(req, timeout=10) as response:
                if response.info().get('Content-Encoding') == 'gzip':
                    import gzip
                    data = gzip.decompress(response.read()).decode('utf-8')
                else:
                    data = response.read().decode('utf-8')
                
                parsed = json.loads(data)
                results = []
                for item in parsed.get('web', {}).get('results', [])[:num_results]:
                    results.append({
                        "title": item.get("title"),
                        "snippet": item.get("description"),
                        "url": item.get("url")
                    })
                return {"results": results} if results else {"error": "brave returned no results"}
        except Exception as e:
            return {"error": str(e)}

    # Primary -> fallback -> last resort. A provider is only considered to have
    # worked if it actually returned results; an empty 200 falls through, so a
    # rate-limited or misconfigured primary degrades instead of silently
    # returning nothing to the assistant.
    for provider, available in ((_brave_search, brave_api_key),
                                (_serpapi_search, serpapi_key),
                                # Keyless and unthrottled — the reason research
                                # keeps working with no credentials configured.
                                (_wikipedia_search, True)):
        if not available:
            continue
        out = await asyncio.to_thread(provider)
        if out.get("results"):
            return out
        logging.warning("web search provider %s failed (%s); trying next",
                        provider.__name__, out.get("error"))
    # Keyless last resort. Unlike the old instant-answer endpoint this returns
    # real ranked results, so search/OSINT/threat-modelling still function with
    # no credential configured at all.
    return await asyncio.to_thread(_duck_html_search)

async def fetch_page(url: str) -> dict:
    """Fetch page via Playwright, bypassing CAPTCHAs if encountered"""
    from backend.tools.browser_tools import get_ephemeral_context, stealth_async
    from backend.tools.captcha_solver import solve_captcha_if_present
    
    try:
        async with get_ephemeral_context() as context:
            page = await context.new_page()
            if stealth_async:
                await stealth_async(page)
                
            await page.goto(url, wait_until="domcontentloaded", timeout=15000)
            await page.wait_for_timeout(2000)
            
            # Auto-solve CAPTCHA if Cloudflare/Turnstile/hCaptcha hard-blocks the article
            await solve_captcha_if_present(page)
            
            # Extract main text
            content = await page.evaluate("document.body.innerText")
            await page.close()
            
            return {"status": "success", "content": content[:5000]} # Limit to 5000 chars for context limits
    except Exception as e:
        return {"error": str(e)}