Spaces:
Running
Running
deploy(S4): Blender headless pipeline + WebAR client + backend fixes
Browse files
backend/tools/web_search_tools.py
CHANGED
|
@@ -11,6 +11,8 @@ import re as _re
|
|
| 11 |
|
| 12 |
# DuckDuckGo's HTML endpoint serves the classic markup; a real browser UA is
|
| 13 |
# required or it returns a stripped page with no results.
|
|
|
|
|
|
|
| 14 |
_BROWSER_UA = ("Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 "
|
| 15 |
"(KHTML, like Gecko) Chrome/120.0 Safari/537.36")
|
| 16 |
|
|
@@ -139,6 +141,39 @@ async def search_web(query: str, num_results: int = 5) -> dict:
|
|
| 139 |
break
|
| 140 |
return {"results": results} if results else {"error": "duckduckgo returned no results"}
|
| 141 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 142 |
def _serpapi_search():
|
| 143 |
url = ("https://serpapi.com/search.json?q=" + urllib.parse.quote(query)
|
| 144 |
+ f"&num={num_results}&api_key={urllib.parse.quote(serpapi_key)}")
|
|
@@ -183,7 +218,10 @@ async def search_web(query: str, num_results: int = 5) -> dict:
|
|
| 183 |
# rate-limited or misconfigured primary degrades instead of silently
|
| 184 |
# returning nothing to the assistant.
|
| 185 |
for provider, available in ((_brave_search, brave_api_key),
|
| 186 |
-
(_serpapi_search, serpapi_key)
|
|
|
|
|
|
|
|
|
|
| 187 |
if not available:
|
| 188 |
continue
|
| 189 |
out = await asyncio.to_thread(provider)
|
|
|
|
| 11 |
|
| 12 |
# DuckDuckGo's HTML endpoint serves the classic markup; a real browser UA is
|
| 13 |
# required or it returns a stripped page with no results.
|
| 14 |
+
_WIKI_UA = "JARVIS_OMEGA/1.0 (+https://huggingface.co/spaces/Jarvis2345/jarvis-cloud)"
|
| 15 |
+
|
| 16 |
_BROWSER_UA = ("Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 "
|
| 17 |
"(KHTML, like Gecko) Chrome/120.0 Safari/537.36")
|
| 18 |
|
|
|
|
| 141 |
break
|
| 142 |
return {"results": results} if results else {"error": "duckduckgo returned no results"}
|
| 143 |
|
| 144 |
+
def _wikipedia_search():
|
| 145 |
+
"""Free, keyless, genuinely unlimited — and rate-limit free.
|
| 146 |
+
|
| 147 |
+
Every commercial option now gates access (Brave/SerpAPI need a card or
|
| 148 |
+
signup, Gemini search-grounding 429s without billing) and every keyless
|
| 149 |
+
one throttles (DuckDuckGo challenges automated use, public SearXNG
|
| 150 |
+
instances 403 the JSON API). Wikipedia's REST/Action API does not, so it
|
| 151 |
+
is the one source that keeps research working with zero credentials.
|
| 152 |
+
|
| 153 |
+
Deliberately placed above the DuckDuckGo scrape: authoritative, fast
|
| 154 |
+
JSON, and it cannot rate-limit the assistant out of working.
|
| 155 |
+
"""
|
| 156 |
+
try:
|
| 157 |
+
api = ("https://en.wikipedia.org/w/api.php?action=query&list=search"
|
| 158 |
+
"&srsearch=" + urllib.parse.quote(query) +
|
| 159 |
+
f"&srlimit={num_results}&format=json&origin=*")
|
| 160 |
+
req = urllib.request.Request(api, headers={"User-Agent": _WIKI_UA})
|
| 161 |
+
with urllib.request.urlopen(req, timeout=12) as resp:
|
| 162 |
+
data = json.loads(resp.read().decode("utf-8", "replace"))
|
| 163 |
+
hits = data.get("query", {}).get("search", [])
|
| 164 |
+
results = []
|
| 165 |
+
for h in hits[:num_results]:
|
| 166 |
+
title = h.get("title", "")
|
| 167 |
+
results.append({
|
| 168 |
+
"title": title,
|
| 169 |
+
# Wikipedia returns the snippet with HTML search highlighting.
|
| 170 |
+
"snippet": _strip_html(h.get("snippet", "")),
|
| 171 |
+
"url": "https://en.wikipedia.org/wiki/" + urllib.parse.quote(title.replace(" ", "_")),
|
| 172 |
+
})
|
| 173 |
+
return {"results": results} if results else {"error": "wikipedia returned no results"}
|
| 174 |
+
except Exception as e:
|
| 175 |
+
return {"error": str(e)}
|
| 176 |
+
|
| 177 |
def _serpapi_search():
|
| 178 |
url = ("https://serpapi.com/search.json?q=" + urllib.parse.quote(query)
|
| 179 |
+ f"&num={num_results}&api_key={urllib.parse.quote(serpapi_key)}")
|
|
|
|
| 218 |
# rate-limited or misconfigured primary degrades instead of silently
|
| 219 |
# returning nothing to the assistant.
|
| 220 |
for provider, available in ((_brave_search, brave_api_key),
|
| 221 |
+
(_serpapi_search, serpapi_key),
|
| 222 |
+
# Keyless and unthrottled — the reason research
|
| 223 |
+
# keeps working with no credentials configured.
|
| 224 |
+
(_wikipedia_search, True)):
|
| 225 |
if not available:
|
| 226 |
continue
|
| 227 |
out = await asyncio.to_thread(provider)
|