omkarkudalkar222 commited on
Commit
057a2a1
·
1 Parent(s): 4ab9a11

feat: remove Semantic Scholar; add keyless providers Europe PMC, PubMed, Unpaywall, Wikipedia, Wikidata for verification, gaps, and novelty

Browse files
agents/citation_gap_agent.py CHANGED
@@ -5,7 +5,6 @@ Output: CitationGap nodes + SuggestedPaper nodes (with direct links)
5
  """
6
  from typing import Any, Dict, List
7
  from agents.base_agent import BaseAgent, AgentContext
8
- from tools.search_tool import SemanticScholarTool
9
  from services.paper_recommender import PaperRecommender
10
  from utils.helpers import generate_id
11
 
@@ -20,17 +19,11 @@ class CitationGapAgent(BaseAgent):
20
 
21
  def __init__(self):
22
  super().__init__()
23
- self._search_tool = None
24
  self._recommender = None
25
 
26
- def _get_search_tool(self) -> SemanticScholarTool:
27
- if self._search_tool is None:
28
- self._search_tool = SemanticScholarTool()
29
- return self._search_tool
30
-
31
  def _get_recommender(self) -> PaperRecommender:
32
  if self._recommender is None:
33
- self._recommender = PaperRecommender(self._get_search_tool())
34
  return self._recommender
35
 
36
  async def execute(self, ctx: AgentContext) -> Dict[str, Any]:
@@ -63,22 +56,22 @@ class CitationGapAgent(BaseAgent):
63
  self.logger.warning("No keywords found; skipping citation gap analysis")
64
  return {"gap_count": 0}
65
 
66
- # 3. Search Semantic Scholar for each top keyword
67
- search_tool = self._get_search_tool()
68
  all_found_papers = []
69
  query_terms = keywords[:5] # limit API calls
70
 
71
  for term in query_terms:
72
- result = await search_tool.run(query=term, limit=5)
73
- if result.success and result.data:
74
- all_found_papers.extend(result.data)
 
75
 
76
  # 4. Use LLM to identify gaps (Graph-RAG context: keywords + refs only)
77
  gaps = await self._identify_gaps(ctx, keywords, existing_refs, all_found_papers)
78
 
79
  # 4b. For every gap, search the web for similar papers and recommend
80
  # direct links the researcher can open and cite.
81
- recommender = self._get_recommender()
82
  for gap in gaps:
83
  recs = await recommender.recommend(
84
  gap.get("description", ""), doc_keywords=keywords, limit=3
 
5
  """
6
  from typing import Any, Dict, List
7
  from agents.base_agent import BaseAgent, AgentContext
 
8
  from services.paper_recommender import PaperRecommender
9
  from utils.helpers import generate_id
10
 
 
19
 
20
  def __init__(self):
21
  super().__init__()
 
22
  self._recommender = None
23
 
 
 
 
 
 
24
  def _get_recommender(self) -> PaperRecommender:
25
  if self._recommender is None:
26
+ self._recommender = PaperRecommender()
27
  return self._recommender
28
 
29
  async def execute(self, ctx: AgentContext) -> Dict[str, Any]:
 
56
  self.logger.warning("No keywords found; skipping citation gap analysis")
57
  return {"gap_count": 0}
58
 
59
+ # 3. Search keyless academic providers for each top keyword
60
+ recommender = self._get_recommender()
61
  all_found_papers = []
62
  query_terms = keywords[:5] # limit API calls
63
 
64
  for term in query_terms:
65
+ found = await recommender.search_web(
66
+ term, limit=5, keywords=keywords
67
+ )
68
+ all_found_papers.extend(found)
69
 
70
  # 4. Use LLM to identify gaps (Graph-RAG context: keywords + refs only)
71
  gaps = await self._identify_gaps(ctx, keywords, existing_refs, all_found_papers)
72
 
73
  # 4b. For every gap, search the web for similar papers and recommend
74
  # direct links the researcher can open and cite.
 
75
  for gap in gaps:
76
  recs = await recommender.recommend(
77
  gap.get("description", ""), doc_keywords=keywords, limit=3
agents/claim_verifier_agent.py CHANGED
@@ -1,7 +1,9 @@
1
  """
2
  Claim Verifier Agent (Wave 3 — runs after ClaimAgent)
3
- For each extracted claim, searches Semantic Scholar and Tavily for supporting
4
- or contradicting evidence, then asks the LLM to classify the claim as:
 
 
5
  SUPPORTED | CONTRADICTED | UNVERIFIED
6
 
7
  Results are stored on the Claim node in Neo4j and returned for the
@@ -13,8 +15,9 @@ from typing import Any, Dict, List, Optional
13
  from agents.base_agent import BaseAgent, AgentContext
14
  from services.paper_recommender import PaperRecommender, clean_title, extract_query
15
  from tools.ddg_tool import DuckDuckGoTool
16
- from tools.search_tool import SemanticScholarTool
17
  from tools.tavily_tool import TavilyTool
 
18
  from utils.logger import get_logger
19
 
20
  VERIFY_SYSTEM = """You are a scientific fact-checker.
@@ -47,15 +50,23 @@ class ClaimVerifierAgent(BaseAgent):
47
 
48
  def __init__(self):
49
  super().__init__()
50
- self._ss_tool = None
 
51
  self._tavily_tool = None
52
  self._ddg_tool = None
 
 
53
  self._rec_tool = None
54
 
55
- def _ss(self) -> SemanticScholarTool:
56
- if self._ss_tool is None:
57
- self._ss_tool = SemanticScholarTool()
58
- return self._ss_tool
 
 
 
 
 
59
 
60
  def _tavily(self) -> TavilyTool:
61
  if self._tavily_tool is None:
@@ -67,6 +78,16 @@ class ClaimVerifierAgent(BaseAgent):
67
  self._ddg_tool = DuckDuckGoTool()
68
  return self._ddg_tool
69
 
 
 
 
 
 
 
 
 
 
 
70
  def _get_recommender(self) -> PaperRecommender:
71
  if self._rec_tool is None:
72
  self._rec_tool = PaperRecommender()
@@ -196,7 +217,8 @@ class ClaimVerifierAgent(BaseAgent):
196
  self, claim_text: str, doc_keywords: Optional[List[str]] = None
197
  ) -> List[Dict]:
198
  """
199
- Evidence chain: Semantic Scholar → OpenAlex/Crossref/arXiv → Tavily → DDG.
 
200
  Queries are topic-based (document keywords + claim key terms), since
201
  full claim sentences make poor search queries.
202
  """
@@ -221,13 +243,21 @@ class ClaimVerifierAgent(BaseAgent):
221
  queries = list(dict.fromkeys(q for q in (topic_query, claim_query) if q))
222
  if not queries:
223
  queries = [claim_text[:100]]
 
224
 
225
- # 1. Semantic Scholar on the claim topic
226
- ss_result = await self._ss().run(query=queries[0], limit=5)
227
- if ss_result.success and ss_result.data:
228
- for p in ss_result.data:
229
  if p.get("title"):
230
- add("Semantic Scholar", p["title"], (p.get("abstract") or "")[:300])
 
 
 
 
 
 
 
231
 
232
  # 2. Academic providers (OpenAlex/Crossref/arXiv) on topic + claim queries
233
  if len(evidence) < 2:
@@ -245,6 +275,23 @@ class ClaimVerifierAgent(BaseAgent):
245
  except Exception as e:
246
  self.logger.debug(f"Academic evidence search failed: {e}")
247
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
248
  if len(evidence) < 2:
249
  tv_result = await self._tavily().run(query=queries[0], max_results=4)
250
  if tv_result.success and tv_result.data:
 
1
  """
2
  Claim Verifier Agent (Wave 3 — runs after ClaimAgent)
3
+ For each extracted claim, searches keyless academic providers (Europe PMC,
4
+ PubMed), web recommender sources (OpenAlex/Crossref/arXiv), Wikipedia and
5
+ Wikidata for supporting or contradicting evidence, then asks the LLM to
6
+ classify the claim as:
7
  SUPPORTED | CONTRADICTED | UNVERIFIED
8
 
9
  Results are stored on the Claim node in Neo4j and returned for the
 
15
  from agents.base_agent import BaseAgent, AgentContext
16
  from services.paper_recommender import PaperRecommender, clean_title, extract_query
17
  from tools.ddg_tool import DuckDuckGoTool
18
+ from tools.search_tool import EuropePMCTool, PubMedTool
19
  from tools.tavily_tool import TavilyTool
20
+ from tools.wikipedia_tool import WikipediaTool, WikidataTool
21
  from utils.logger import get_logger
22
 
23
  VERIFY_SYSTEM = """You are a scientific fact-checker.
 
50
 
51
  def __init__(self):
52
  super().__init__()
53
+ self._epmc_tool = None
54
+ self._pubmed_tool = None
55
  self._tavily_tool = None
56
  self._ddg_tool = None
57
+ self._wiki_tool = None
58
+ self._wikidata_tool = None
59
  self._rec_tool = None
60
 
61
+ def _epmc(self) -> EuropePMCTool:
62
+ if self._epmc_tool is None:
63
+ self._epmc_tool = EuropePMCTool()
64
+ return self._epmc_tool
65
+
66
+ def _pubmed(self) -> PubMedTool:
67
+ if self._pubmed_tool is None:
68
+ self._pubmed_tool = PubMedTool()
69
+ return self._pubmed_tool
70
 
71
  def _tavily(self) -> TavilyTool:
72
  if self._tavily_tool is None:
 
78
  self._ddg_tool = DuckDuckGoTool()
79
  return self._ddg_tool
80
 
81
+ def _wiki(self) -> WikipediaTool:
82
+ if self._wiki_tool is None:
83
+ self._wiki_tool = WikipediaTool()
84
+ return self._wiki_tool
85
+
86
+ def _wikidata(self) -> WikidataTool:
87
+ if self._wikidata_tool is None:
88
+ self._wikidata_tool = WikidataTool()
89
+ return self._wikidata_tool
90
+
91
  def _get_recommender(self) -> PaperRecommender:
92
  if self._rec_tool is None:
93
  self._rec_tool = PaperRecommender()
 
217
  self, claim_text: str, doc_keywords: Optional[List[str]] = None
218
  ) -> List[Dict]:
219
  """
220
+ Evidence chain: Europe PMCPubMed → OpenAlex/Crossref/arXiv →
221
+ Wikipedia → Wikidata → Tavily → DDG.
222
  Queries are topic-based (document keywords + claim key terms), since
223
  full claim sentences make poor search queries.
224
  """
 
243
  queries = list(dict.fromkeys(q for q in (topic_query, claim_query) if q))
244
  if not queries:
245
  queries = [claim_text[:100]]
246
+ web_query = claim_query or topic_query or queries[0]
247
 
248
+ # 1. Keyless biomedical providers on the claim topic
249
+ epmc_result = await self._epmc().run(query=queries[0], limit=4)
250
+ if epmc_result.success and epmc_result.data:
251
+ for p in epmc_result.data:
252
  if p.get("title"):
253
+ add("Europe PMC", p["title"], (p.get("abstract") or "")[:300])
254
+
255
+ if len(evidence) < 2:
256
+ pm_result = await self._pubmed().run(query=queries[0], limit=4)
257
+ if pm_result.success and pm_result.data:
258
+ for p in pm_result.data:
259
+ if p.get("title"):
260
+ add("PubMed", p["title"], (p.get("abstract") or "")[:300])
261
 
262
  # 2. Academic providers (OpenAlex/Crossref/arXiv) on topic + claim queries
263
  if len(evidence) < 2:
 
275
  except Exception as e:
276
  self.logger.debug(f"Academic evidence search failed: {e}")
277
 
278
+ # 3. Wikipedia encyclopedia evidence on the claim's key terms
279
+ if len(evidence) < 2:
280
+ wiki_result = await self._wiki().run(query=web_query, limit=3)
281
+ if wiki_result.success and wiki_result.data:
282
+ for r in wiki_result.data:
283
+ add("Wikipedia", r.get("title", ""), r.get("snippet", ""))
284
+
285
+ # 4. Wikidata entity facts
286
+ if len(evidence) < 2:
287
+ wd_result = await self._wikidata().run(query=web_query, limit=3)
288
+ if wd_result.success and wd_result.data:
289
+ for r in wd_result.data:
290
+ desc = r.get("description") or r.get("label", "")
291
+ if desc:
292
+ add("Wikidata", r.get("label", ""), desc)
293
+
294
+ # 5. Web search (Tavily, then DuckDuckGo as keyless fallback)
295
  if len(evidence) < 2:
296
  tv_result = await self._tavily().run(query=queries[0], max_results=4)
297
  if tv_result.success and tv_result.data:
agents/custom/research_agent.py CHANGED
@@ -5,7 +5,7 @@ This agent performs deep literature review for a given topic.
5
  """
6
  from typing import Any, Dict
7
  from agents.base_agent import BaseAgent, AgentContext
8
- from tools.search_tool import SemanticScholarTool
9
 
10
  SYSTEM = (
11
  "You are a research synthesis expert. "
@@ -22,7 +22,7 @@ class ResearchAgent(BaseAgent):
22
 
23
  def __init__(self):
24
  super().__init__()
25
- self._search_tool = SemanticScholarTool()
26
 
27
  async def execute(self, ctx: AgentContext) -> Dict[str, Any]:
28
  # Get topic from keywords in graph
@@ -35,16 +35,19 @@ class ResearchAgent(BaseAgent):
35
  return {"status": "skipped", "reason": "no keywords"}
36
 
37
  topic = " ".join(r["term"] for r in kw_rows[:3])
38
- search_result = await self._search_tool.run(query=topic, limit=10)
 
 
 
39
 
40
- if not search_result.success:
41
- return {"status": "failed", "error": search_result.error}
42
 
43
  papers_text = "\n".join(
44
- f"- {p['title']} ({p.get('year', 'N/A')})"
45
- for p in search_result.data[:10]
46
  )
47
  prompt = f"Topic: {topic}\n\nRelated papers:\n{papers_text}"
48
  analysis = await ctx.llm.complete_json(SYSTEM, prompt)
49
 
50
- return {"topic": topic, "papers_found": len(search_result.data), "analysis": analysis}
 
5
  """
6
  from typing import Any, Dict
7
  from agents.base_agent import BaseAgent, AgentContext
8
+ from services.paper_recommender import PaperRecommender
9
 
10
  SYSTEM = (
11
  "You are a research synthesis expert. "
 
22
 
23
  def __init__(self):
24
  super().__init__()
25
+ self._recommender = PaperRecommender()
26
 
27
  async def execute(self, ctx: AgentContext) -> Dict[str, Any]:
28
  # Get topic from keywords in graph
 
35
  return {"status": "skipped", "reason": "no keywords"}
36
 
37
  topic = " ".join(r["term"] for r in kw_rows[:3])
38
+ try:
39
+ papers = await self._recommender.search_web(topic, limit=10)
40
+ except Exception as e:
41
+ return {"status": "failed", "error": str(e)}
42
 
43
+ if not papers:
44
+ return {"status": "failed", "error": "no results from academic providers"}
45
 
46
  papers_text = "\n".join(
47
+ f"- {p.get('title', 'Unknown')} ({p.get('year', 'N/A')})"
48
+ for p in papers[:10]
49
  )
50
  prompt = f"Topic: {topic}\n\nRelated papers:\n{papers_text}"
51
  analysis = await ctx.llm.complete_json(SYSTEM, prompt)
52
 
53
+ return {"topic": topic, "papers_found": len(papers), "analysis": analysis}
agents/novelty_agent.py CHANGED
@@ -11,7 +11,7 @@ from typing import Any, Dict, List
11
  import json
12
 
13
  from agents.base_agent import BaseAgent, AgentContext
14
- from tools.search_tool import SemanticScholarTool
15
 
16
  EXTRACT_SYSTEM = (
17
  "You are an expert scientific reviewer. Extract the precise problem "
@@ -36,12 +36,12 @@ class NoveltyAgent(BaseAgent):
36
 
37
  def __init__(self):
38
  super().__init__()
39
- self._search_tool = None
40
 
41
- def _get_search_tool(self) -> SemanticScholarTool:
42
- if self._search_tool is None:
43
- self._search_tool = SemanticScholarTool()
44
- return self._search_tool
45
 
46
  async def execute(self, ctx: AgentContext) -> Dict[str, Any]:
47
  # 1. Paper metadata
@@ -139,17 +139,20 @@ class NoveltyAgent(BaseAgent):
139
  queries.append(" ".join(kw_terms[:4])[:120])
140
 
141
  found: List[Dict] = []
142
- tool = self._get_search_tool()
143
  for q in queries[:2]:
144
- result = await tool.run(query=q, limit=4)
145
- if result.success and result.data:
146
- for p in result.data:
147
- found.append({
148
- "title": p.get("title", ""),
149
- "year": p.get("year"),
150
- "url": p.get("url", ""),
151
- "why_similar": "search hit for the paper's problem area",
152
- })
 
 
 
153
  return found
154
 
155
  async def _assess_novelty(
 
11
  import json
12
 
13
  from agents.base_agent import BaseAgent, AgentContext
14
+ from services.paper_recommender import PaperRecommender
15
 
16
  EXTRACT_SYSTEM = (
17
  "You are an expert scientific reviewer. Extract the precise problem "
 
36
 
37
  def __init__(self):
38
  super().__init__()
39
+ self._recommender = None
40
 
41
+ def _get_recommender(self) -> PaperRecommender:
42
+ if self._recommender is None:
43
+ self._recommender = PaperRecommender()
44
+ return self._recommender
45
 
46
  async def execute(self, ctx: AgentContext) -> Dict[str, Any]:
47
  # 1. Paper metadata
 
139
  queries.append(" ".join(kw_terms[:4])[:120])
140
 
141
  found: List[Dict] = []
142
+ recommender = self._get_recommender()
143
  for q in queries[:2]:
144
+ try:
145
+ results = await recommender.search_web(q, limit=4)
146
+ except Exception as e:
147
+ self.logger.debug(f"Similar-work search failed for '{q[:50]}': {e}")
148
+ results = []
149
+ for p in results:
150
+ found.append({
151
+ "title": p.get("title", ""),
152
+ "year": p.get("year"),
153
+ "url": p.get("url", ""),
154
+ "why_similar": "search hit for the paper's problem area",
155
+ })
156
  return found
157
 
158
  async def _assess_novelty(
scripts/generate_eval_pdf.py CHANGED
@@ -303,7 +303,7 @@ CROSS_PAPER_MD = """
303
  Architecture and theory-driven papers (gnn_deeper, yolov10) score highest (8.1+).
304
  Survey papers (kg_embedding) score at the same level when the field is mature.
305
  Empirical/applied papers (llm_reasoning, federated_chest) score slightly lower (7-8 range) —
306
- not because the papers are weaker, but because novel empirical methods are not yet indexed in Semantic Scholar.
307
 
308
  ### Pattern 2: Counterfactuality
309
  All 5 papers score very low on counterfactuality (0.00–0.18), which is the correct result.
@@ -324,14 +324,14 @@ for papers that store references differently (inline footnotes vs numbered lists
324
  | Issue | Papers Affected | Impact |
325
  |-------|----------------|--------|
326
  | KBIR incompatible with transformers 5.8 | ALL 5 | 0 keywords → citation gap disabled |
327
- | Semantic Scholar 429 rate limits | ALL 5 | Reduced verification confidence |
328
  | Reference parser bug (0 refs) | 3/5 papers | Citation completeness score misleading |
329
  | KeyBERT snapshot path error | ALL 5 | No keyword fallback |
330
 
331
  ### Recommended Priority Fixes
332
  1. **KBIR**: Pin transformers==4.44.0 or replace with KeyBERT + cached all-MiniLM-L6-v2
333
  2. **Reference parser**: Add fallback regex patterns for footnote/inline reference formats
334
- 3. **Semantic Scholar API key**: Register for free key to eliminate 429 rate limits
335
  4. **KeyBERT cache**: Pre-download all-MiniLM-L6-v2 to avoid snapshot path errors
336
  """
337
 
 
303
  Architecture and theory-driven papers (gnn_deeper, yolov10) score highest (8.1+).
304
  Survey papers (kg_embedding) score at the same level when the field is mature.
305
  Empirical/applied papers (llm_reasoning, federated_chest) score slightly lower (7-8 range) —
306
+ not because the papers are weaker, but because novel empirical methods are not yet indexed in the academic search providers.
307
 
308
  ### Pattern 2: Counterfactuality
309
  All 5 papers score very low on counterfactuality (0.00–0.18), which is the correct result.
 
324
  | Issue | Papers Affected | Impact |
325
  |-------|----------------|--------|
326
  | KBIR incompatible with transformers 5.8 | ALL 5 | 0 keywords → citation gap disabled |
327
+ | Academic search provider rate limits | ALL 5 | Reduced verification confidence |
328
  | Reference parser bug (0 refs) | 3/5 papers | Citation completeness score misleading |
329
  | KeyBERT snapshot path error | ALL 5 | No keyword fallback |
330
 
331
  ### Recommended Priority Fixes
332
  1. **KBIR**: Pin transformers==4.44.0 or replace with KeyBERT + cached all-MiniLM-L6-v2
333
  2. **Reference parser**: Add fallback regex patterns for footnote/inline reference formats
334
+ 3. **Keyless providers (Europe PMC, PubMed, OpenAlex)**: throttled keyless search replaced Semantic Scholar 429s
335
  4. **KeyBERT cache**: Pre-download all-MiniLM-L6-v2 to avoid snapshot path errors
336
  """
337
 
services/paper_recommender.py CHANGED
@@ -11,7 +11,8 @@ Providers:
11
  1. OpenAlex — broad aggregator (publishers, repositories)
12
  2. Crossref — publisher metadata (Elsevier, Springer, IEEE, Wiley, ...)
13
  3. arXiv — open-access preprint repository (arxiv.org direct links)
14
- 4. Semantic Scholarfallback (unauthenticated rate limits are tight)
 
15
  """
16
  import asyncio
17
  import math
@@ -21,7 +22,7 @@ from xml.etree import ElementTree
21
 
22
  import httpx
23
 
24
- from tools.search_tool import SemanticScholarTool
25
  from utils.logger import get_logger
26
  from utils.retry import async_retry
27
 
@@ -306,16 +307,24 @@ _PROVIDER_SEM = asyncio.Semaphore(2)
306
  async def _search_all_providers(
307
  client: httpx.AsyncClient, query: str, limit: int = 10
308
  ) -> List[Dict]:
309
- """Query OpenAlex, Crossref, arXiv sequentially with a small stagger
310
- (kind to rate limits), reusing a shared semaphore across callers."""
311
  results: List[Dict] = []
 
 
312
  async with _PROVIDER_SEM:
313
- for i, fn in enumerate((_search_openalex, _search_crossref, _search_arxiv)):
314
  try:
315
  results.extend(await fn(client, query, limit))
316
  except Exception as e:
317
  logger.debug(f"Provider {fn.__name__} failed for '{query[:50]}': {e}")
318
  await asyncio.sleep(0.25)
 
 
 
 
 
 
319
  return results
320
 
321
  def _reconstruct_openalex_abstract(inverted: Optional[Dict]) -> str:
@@ -410,8 +419,10 @@ def source_for(paper: Dict) -> str:
410
  url = (paper.get("url") or "").lower()
411
  if "arxiv.org" in url:
412
  return "arXiv"
413
- if "semanticscholar.org" in url:
414
- return "Semantic Scholar"
 
 
415
  if "openalex.org" in url:
416
  return "OpenAlex"
417
  if "crossref.org" in url or "doi.org" in url:
@@ -464,10 +475,11 @@ async def _search_arxiv(client: httpx.AsyncClient, query: str, limit: int = 10)
464
 
465
 
466
  class PaperRecommender:
467
- """Web-based paper recommender (OpenAlex + Crossref + arXiv + fallbacks)."""
 
468
 
469
- def __init__(self, search_tool: Optional[SemanticScholarTool] = None):
470
- self._search_tool = search_tool or SemanticScholarTool()
471
 
472
  async def recommend(
473
  self,
@@ -501,16 +513,6 @@ class PaperRecommender:
501
  except Exception as e:
502
  logger.warning(f"Paper recommender HTTP error: {e}")
503
 
504
- if not candidates:
505
- # Fall back to Semantic Scholar
506
- try:
507
- for q in queries[:1]:
508
- result = await self._search_tool.run(query=q, limit=10)
509
- if result.success and result.data:
510
- candidates.extend(result.data)
511
- except Exception as e:
512
- logger.warning(f"Semantic Scholar fallback failed: {e}")
513
-
514
  if not candidates:
515
  return []
516
 
@@ -518,8 +520,9 @@ class PaperRecommender:
518
  c["source"] = source_for(c)
519
 
520
  # Same paper from multiple providers (e.g. arXiv + OpenAlex copy):
521
- # keep the entry with the most direct link (arXiv > DOI > landing page).
522
- _SOURCE_PRIORITY = {"arXiv": 0, "Semantic Scholar": 1, "Crossref": 2}
 
523
  by_title: Dict[str, Dict] = {}
524
  for c in candidates:
525
  key = clean_title(c.get("title", "")).lower()
@@ -552,6 +555,17 @@ class PaperRecommender:
552
  chosen = {r["title"].lower() for r in recs}
553
  recs += [r for r in filled if r["title"].lower() not in chosen]
554
 
 
 
 
 
 
 
 
 
 
 
 
555
  return recs[:limit]
556
 
557
  async def run(self, query: str, limit: int = 5) -> List[Dict]:
@@ -565,8 +579,9 @@ class PaperRecommender:
565
  keywords: Optional[List[str]] = None,
566
  ) -> List[Dict]:
567
  """
568
- Raw academic search (OpenAlex + Crossref + arXiv) without ranking:
569
- returns candidates labeled with source, for evidence gathering.
 
570
  """
571
  query_terms = extract_query(query, keywords)
572
  q = " ".join(query_terms[:4]) if query_terms else query.strip()
@@ -586,7 +601,7 @@ class PaperRecommender:
586
  for c in candidates:
587
  c["source"] = source_for(c)
588
 
589
- _SOURCE_PRIORITY = {"arXiv": 0, "Semantic Scholar": 1, "Crossref": 2}
590
  by_title: Dict[str, Dict] = {}
591
  for c in candidates:
592
  key = clean_title(c.get("title", "")).lower()
 
11
  1. OpenAlex — broad aggregator (publishers, repositories)
12
  2. Crossref — publisher metadata (Elsevier, Springer, IEEE, Wiley, ...)
13
  3. arXiv — open-access preprint repository (arxiv.org direct links)
14
+ 4. Europe PMCbiomedical literature + preprints + patents (keyless)
15
+ 5. PubMed — MEDLINE abstracts via NCBI E-utilities (keyless)
16
  """
17
  import asyncio
18
  import math
 
22
 
23
  import httpx
24
 
25
+ from tools.search_tool import EuropePMCTool, PubMedTool, UnpaywallTool
26
  from utils.logger import get_logger
27
  from utils.retry import async_retry
28
 
 
307
  async def _search_all_providers(
308
  client: httpx.AsyncClient, query: str, limit: int = 10
309
  ) -> List[Dict]:
310
+ """Query OpenAlex, Crossref, arXiv, Europe PMC, PubMed sequentially with
311
+ a small stagger (kind to rate limits), reusing a shared semaphore."""
312
  results: List[Dict] = []
313
+ epmc = EuropePMCTool()
314
+ pubmed = PubMedTool()
315
  async with _PROVIDER_SEM:
316
+ for fn in (_search_openalex, _search_crossref, _search_arxiv):
317
  try:
318
  results.extend(await fn(client, query, limit))
319
  except Exception as e:
320
  logger.debug(f"Provider {fn.__name__} failed for '{query[:50]}': {e}")
321
  await asyncio.sleep(0.25)
322
+ for tool in (epmc, pubmed):
323
+ try:
324
+ results.extend(await tool.search(query, limit))
325
+ except Exception as e:
326
+ logger.debug(f"Provider {type(tool).__name__} failed for '{query[:50]}': {e}")
327
+ await asyncio.sleep(0.25)
328
  return results
329
 
330
  def _reconstruct_openalex_abstract(inverted: Optional[Dict]) -> str:
 
419
  url = (paper.get("url") or "").lower()
420
  if "arxiv.org" in url:
421
  return "arXiv"
422
+ if "europepmc.org" in url or "pmc.ncbi.nlm.nih.gov" in url:
423
+ return "Europe PMC"
424
+ if "pubmed.ncbi.nlm.nih.gov" in url:
425
+ return "PubMed"
426
  if "openalex.org" in url:
427
  return "OpenAlex"
428
  if "crossref.org" in url or "doi.org" in url:
 
475
 
476
 
477
  class PaperRecommender:
478
+ """Web-based paper recommender (OpenAlex + Crossref + arXiv + Europe PMC
479
+ + PubMed, with Unpaywall open-access enrichment)."""
480
 
481
+ def __init__(self):
482
+ self._unpaywall = UnpaywallTool()
483
 
484
  async def recommend(
485
  self,
 
513
  except Exception as e:
514
  logger.warning(f"Paper recommender HTTP error: {e}")
515
 
 
 
 
 
 
 
 
 
 
 
516
  if not candidates:
517
  return []
518
 
 
520
  c["source"] = source_for(c)
521
 
522
  # Same paper from multiple providers (e.g. arXiv + OpenAlex copy):
523
+ # keep the entry with the most direct link (arXiv > Europe PMC >
524
+ # PubMed > Crossref landing page).
525
+ _SOURCE_PRIORITY = {"arXiv": 0, "Europe PMC": 1, "PubMed": 2, "Crossref": 3}
526
  by_title: Dict[str, Dict] = {}
527
  for c in candidates:
528
  key = clean_title(c.get("title", "")).lower()
 
555
  chosen = {r["title"].lower() for r in recs}
556
  recs += [r for r in filled if r["title"].lower() not in chosen]
557
 
558
+ # Enrich top recommendations with open-access copies (silent on failure)
559
+ for rec in recs:
560
+ if not rec.get("doi"):
561
+ continue
562
+ try:
563
+ result = await self._unpaywall.run(rec["doi"])
564
+ if result.success and result.data and result.data.get("url"):
565
+ rec["oa_url"] = result.data["url"]
566
+ except Exception as e:
567
+ logger.debug(f"Unpaywall enrichment failed for {rec['doi']}: {e}")
568
+
569
  return recs[:limit]
570
 
571
  async def run(self, query: str, limit: int = 5) -> List[Dict]:
 
579
  keywords: Optional[List[str]] = None,
580
  ) -> List[Dict]:
581
  """
582
+ Raw academic search (OpenAlex + Crossref + arXiv + Europe PMC +
583
+ PubMed) without ranking: returns candidates labeled with source,
584
+ for evidence gathering.
585
  """
586
  query_terms = extract_query(query, keywords)
587
  q = " ".join(query_terms[:4]) if query_terms else query.strip()
 
601
  for c in candidates:
602
  c["source"] = source_for(c)
603
 
604
+ _SOURCE_PRIORITY = {"arXiv": 0, "Europe PMC": 1, "PubMed": 2, "Crossref": 3}
605
  by_title: Dict[str, Dict] = {}
606
  for c in candidates:
607
  key = clean_title(c.get("title", "")).lower()
tests/test_agents.py CHANGED
@@ -168,14 +168,12 @@ async def test_novelty_assesses_and_stores(agent_ctx, mock_neo4j, mock_llm):
168
  },
169
  ]
170
  agent = NoveltyAgent()
171
- agent._search_tool = type(
172
- "FakeTool",
173
  (),
174
  {
175
- "run": AsyncMock(
176
- return_value=type(
177
- "Res", (), {"success": True, "data": [{"title": "Old Work", "year": 2020, "url": "x"}], "error": ""}
178
- )()
179
  )
180
  },
181
  )()
@@ -214,12 +212,10 @@ async def test_novelty_survives_llm_and_search_failure(agent_ctx, mock_neo4j, mo
214
  ]
215
  mock_llm.complete_json.side_effect = RuntimeError("LLM down")
216
  agent = NoveltyAgent()
217
- agent._search_tool = type(
218
- "FakeTool",
219
  (),
220
- {
221
- "run": AsyncMock(return_value=type("Res", (), {"success": False, "data": None, "error": "429 rate limit"})())
222
- },
223
  )()
224
  result = await agent.run(agent_ctx)
225
  assert result.status == AgentStatus.COMPLETED
 
168
  },
169
  ]
170
  agent = NoveltyAgent()
171
+ agent._recommender = type(
172
+ "FakeRecommender",
173
  (),
174
  {
175
+ "search_web": AsyncMock(
176
+ return_value=[{"title": "Old Work", "year": 2020, "url": "x"}]
 
 
177
  )
178
  },
179
  )()
 
212
  ]
213
  mock_llm.complete_json.side_effect = RuntimeError("LLM down")
214
  agent = NoveltyAgent()
215
+ agent._recommender = type(
216
+ "FakeRecommender",
217
  (),
218
+ {"search_web": AsyncMock(return_value=[])},
 
 
219
  )()
220
  result = await agent.run(agent_ctx)
221
  assert result.status == AgentStatus.COMPLETED
tests/test_tools.py CHANGED
@@ -1,65 +1,192 @@
1
  import pytest
2
  from unittest.mock import AsyncMock, MagicMock, patch
3
- from tools.search_tool import SemanticScholarTool
 
4
  from tools.database_tool import Neo4jTool, MongoTool
5
 
6
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
7
  @pytest.mark.asyncio
8
- async def test_search_tool_returns_papers():
9
- tool = SemanticScholarTool(api_key="test")
10
  mock_response = {
11
- "data": [
12
- {
13
- "title": "Test Paper",
14
- "authors": [{"name": "Author A"}],
15
- "year": 2023,
16
- "abstract": "Test abstract",
17
- "citationCount": 10,
18
- "externalIds": {"DOI": "10.1234/test"},
19
- "paperId": "abc123",
20
- }
21
- ]
 
 
 
22
  }
23
- with patch("httpx.AsyncClient") as mock_client:
24
- mock_resp = MagicMock()
25
- mock_resp.json.return_value = mock_response
26
- mock_resp.raise_for_status = MagicMock()
27
- mock_client.return_value.__aenter__ = AsyncMock(return_value=MagicMock(
28
- get=AsyncMock(return_value=mock_resp)
29
- ))
30
- mock_client.return_value.__aexit__ = AsyncMock(return_value=None)
31
- result = await tool.run(query="transformer nlp", limit=1)
32
 
33
  assert result.success
34
  assert len(result.data) == 1
35
- assert result.data[0]["title"] == "Test Paper"
 
 
36
 
37
 
38
  @pytest.mark.asyncio
39
- async def test_neo4j_tool_query(mock_neo4j):
40
- mock_neo4j.run.return_value = [{"n": "value"}]
41
- tool = Neo4jTool(neo4j=mock_neo4j)
42
- result = await tool.run(query="MATCH (n) RETURN n LIMIT 1")
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
43
  assert result.success
44
- assert result.data == [{"n": "value"}]
 
 
 
 
45
 
46
 
47
  @pytest.mark.asyncio
48
- async def test_search_tool_serializes_parallel_calls():
49
- """Concurrent Semantic Scholar searches must be spaced out (429 flood regression)."""
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
50
  import asyncio
51
  import time
52
  import tools.search_tool as st
53
 
54
- original_interval = st._SS_MIN_INTERVAL
55
- original_last = st._ss_last_request_at
56
- st._SS_MIN_INTERVAL = 0.2
57
- st._ss_last_request_at = 0.0
58
  starts = []
59
 
60
- mock_resp = MagicMock()
61
- mock_resp.json.return_value = {"data": []}
62
- mock_resp.raise_for_status = MagicMock()
63
 
64
  def make_ctx():
65
  client = MagicMock()
@@ -67,7 +194,10 @@ async def test_search_tool_serializes_parallel_calls():
67
 
68
  async def get(*args, **kwargs):
69
  starts.append(time.monotonic())
70
- return mock_resp
 
 
 
71
 
72
  ctx.get = AsyncMock(side_effect=get)
73
  client.__aenter__ = AsyncMock(return_value=ctx)
@@ -75,14 +205,23 @@ async def test_search_tool_serializes_parallel_calls():
75
  return client
76
 
77
  try:
78
- with patch("httpx.AsyncClient", side_effect=[make_ctx() for _ in range(3)]):
79
- tool = SemanticScholarTool(api_key="test")
80
  results = await asyncio.gather(*[tool.run(query="q", limit=5) for _ in range(3)])
81
  finally:
82
- st._SS_MIN_INTERVAL = original_interval
83
- st._ss_last_request_at = original_last
84
 
85
  assert all(r.success for r in results)
86
  assert len(starts) == 3
87
  deltas = [starts[i + 1] - starts[i] for i in range(2)]
88
  assert all(d >= 0.18 for d in deltas)
 
 
 
 
 
 
 
 
 
 
1
  import pytest
2
  from unittest.mock import AsyncMock, MagicMock, patch
3
+ from tools.search_tool import EuropePMCTool, PubMedTool, UnpaywallTool
4
+ from tools.wikipedia_tool import WikipediaTool, WikidataTool
5
  from tools.database_tool import Neo4jTool, MongoTool
6
 
7
 
8
+ def _client_ctx(responses):
9
+ """Build a mocked httpx.AsyncClient whose GET returns `responses` in order."""
10
+ client = MagicMock()
11
+ ctx = MagicMock()
12
+ it = iter(responses)
13
+
14
+ async def get(*args, **kwargs):
15
+ return next(it)
16
+
17
+ ctx.get = AsyncMock(side_effect=get)
18
+ client.__aenter__ = AsyncMock(return_value=ctx)
19
+ client.__aexit__ = AsyncMock(return_value=None)
20
+ return client
21
+
22
+
23
+ def _resp(status=200, json=None, text=""):
24
+ r = MagicMock()
25
+ r.status_code = status
26
+ r.json.return_value = json if json is not None else {}
27
+ r.text = text
28
+ r.raise_for_status = MagicMock()
29
+ if status >= 400:
30
+ r.raise_for_status.side_effect = RuntimeError(f"HTTP {status}")
31
+ return r
32
+
33
+
34
  @pytest.mark.asyncio
35
+ async def test_europepmc_tool_returns_papers():
36
+ tool = EuropePMCTool()
37
  mock_response = {
38
+ "resultList": {
39
+ "result": [
40
+ {
41
+ "title": "A Study on Federated Learning",
42
+ "authorString": "Alice A, Bob B",
43
+ "pubYear": "2023",
44
+ "abstractText": "We study federated learning.",
45
+ "citedByCount": 12,
46
+ "doi": "10.1234/test",
47
+ "pmcid": "PMC1234567",
48
+ "id": "PMC1234567",
49
+ }
50
+ ]
51
+ }
52
  }
53
+ with patch("httpx.AsyncClient", return_value=_client_ctx([_resp(json=mock_response)])):
54
+ result = await tool.run(query="federated learning", limit=5)
 
 
 
 
 
 
 
55
 
56
  assert result.success
57
  assert len(result.data) == 1
58
+ assert result.data[0]["title"] == "A Study on Federated Learning"
59
+ assert result.data[0]["url"] == "https://europepmc.org/article/PMC/PMC1234567"
60
+ assert result.data[0]["citation_count"] == 12
61
 
62
 
63
  @pytest.mark.asyncio
64
+ async def test_pubmed_tool_returns_papers():
65
+ tool = PubMedTool()
66
+ esearch_json = {"esearchresult": {"idlist": ["33000001"]}}
67
+ efetch_xml = """<?xml version="1.0"?>
68
+ <PubmedArticleSet>
69
+ <PubmedArticle>
70
+ <MedlineCitation>
71
+ <PMID>33000001</PMID>
72
+ <Article>
73
+ <ArticleTitle>Neural <i>networks</i> for medical imaging</ArticleTitle>
74
+ <Abstract><AbstractText>We train CNNs on scans.</AbstractText></Abstract>
75
+ <ArticleDate><Year>2021</Year></ArticleDate>
76
+ <AuthorList><Author><ForeName>Jane</ForeName><LastName>Doe</LastName></Author></AuthorList>
77
+ <ELocationID EIdType="doi">10.9999/medimg</ELocationID>
78
+ </Article>
79
+ </MedlineCitation>
80
+ </PubmedArticle>
81
+ </PubmedArticleSet>"""
82
+ with patch(
83
+ "httpx.AsyncClient",
84
+ return_value=_client_ctx([_resp(json=esearch_json), _resp(text=efetch_xml)]),
85
+ ):
86
+ result = await tool.run(query="medical imaging", limit=5)
87
+
88
  assert result.success
89
+ assert len(result.data) == 1
90
+ assert result.data[0]["title"] == "Neural networks for medical imaging"
91
+ assert result.data[0]["abstract"] == "We train CNNs on scans."
92
+ assert result.data[0]["doi"] == "10.9999/medimg"
93
+ assert result.data[0]["url"].startswith("https://pubmed.ncbi.nlm.nih.gov/")
94
 
95
 
96
  @pytest.mark.asyncio
97
+ async def test_pubmed_tool_no_ids_returns_empty():
98
+ tool = PubMedTool()
99
+ with patch(
100
+ "httpx.AsyncClient",
101
+ return_value=_client_ctx([_resp(json={"esearchresult": {"idlist": []}})]),
102
+ ):
103
+ result = await tool.run(query="nothing found", limit=5)
104
+ assert result.success
105
+ assert result.data == []
106
+
107
+
108
+ @pytest.mark.asyncio
109
+ async def test_unpaywall_lookup_returns_oa_url():
110
+ tool = UnpaywallTool()
111
+ mock_response = {
112
+ "is_oa": True,
113
+ "best_oa_location": {"url": "https://oa.example.com/paper.pdf"},
114
+ }
115
+ with patch("httpx.AsyncClient", return_value=_client_ctx([_resp(json=mock_response)])):
116
+ result = await tool.run("10.1234/test")
117
+ assert result.success
118
+ assert result.data["is_oa"] is True
119
+ assert result.data["url"] == "https://oa.example.com/paper.pdf"
120
+
121
+
122
+ @pytest.mark.asyncio
123
+ async def test_unpaywall_lookup_not_oa_404():
124
+ tool = UnpaywallTool()
125
+ with patch("httpx.AsyncClient", return_value=_client_ctx([_resp(status=404)])):
126
+ result = await tool.run("10.1234/missing")
127
+ assert result.success
128
+ assert result.data["is_oa"] is False
129
+ assert result.data["url"] == ""
130
+
131
+
132
+ @pytest.mark.asyncio
133
+ async def test_wikipedia_tool_returns_snippets():
134
+ tool = WikipediaTool()
135
+ search_json = {
136
+ "query": {
137
+ "search": [
138
+ {"title": "Momentum Investing", "snippet": "<span>Momentum</span> investing basics"},
139
+ ]
140
+ }
141
+ }
142
+ extracts_json = {
143
+ "query": {
144
+ "pages": {
145
+ "1": {"title": "Momentum Investing", "extract": "Momentum investing is a strategy..."}
146
+ }
147
+ }
148
+ }
149
+ with patch(
150
+ "httpx.AsyncClient",
151
+ return_value=_client_ctx([_resp(json=search_json), _resp(json=extracts_json)]),
152
+ ):
153
+ result = await tool.run(query="momentum investing", limit=3)
154
+ assert result.success
155
+ assert len(result.data) == 1
156
+ assert result.data[0]["title"] == "Momentum Investing"
157
+ assert result.data[0]["url"] == "https://en.wikipedia.org/wiki/Momentum_Investing"
158
+ assert "Momentum investing is a strategy" in result.data[0]["snippet"]
159
+
160
+
161
+ @pytest.mark.asyncio
162
+ async def test_wikidata_tool_returns_entities():
163
+ tool = WikidataTool()
164
+ mock_response = {
165
+ "search": [
166
+ {"id": "Q42", "label": "Douglas Adams", "description": "English writer and humorist"},
167
+ ]
168
+ }
169
+ with patch("httpx.AsyncClient", return_value=_client_ctx([_resp(json=mock_response)])):
170
+ result = await tool.run(query="douglas adams", limit=3)
171
+ assert result.success
172
+ assert result.data[0]["id"] == "Q42"
173
+ assert result.data[0]["url"] == "https://www.wikidata.org/wiki/Q42"
174
+
175
+
176
+ @pytest.mark.asyncio
177
+ async def test_academic_tools_serialize_parallel_calls():
178
+ """Concurrent keyless academic searches must be spaced out (rate-limit regression)."""
179
  import asyncio
180
  import time
181
  import tools.search_tool as st
182
 
183
+ original_interval = st._ACADEMIC_MIN_INTERVAL
184
+ original_last = st._academic_last_request_at
185
+ st._ACADEMIC_MIN_INTERVAL = 0.2
186
+ st._academic_last_request_at = 0.0
187
  starts = []
188
 
189
+ empty_ok = {"resultList": {"result": []}}
 
 
190
 
191
  def make_ctx():
192
  client = MagicMock()
 
194
 
195
  async def get(*args, **kwargs):
196
  starts.append(time.monotonic())
197
+ r = MagicMock()
198
+ r.json.return_value = empty_ok
199
+ r.raise_for_status = MagicMock()
200
+ return r
201
 
202
  ctx.get = AsyncMock(side_effect=get)
203
  client.__aenter__ = AsyncMock(return_value=ctx)
 
205
  return client
206
 
207
  try:
208
+ with patch("httpx.AsyncClient", return_value=make_ctx()):
209
+ tool = EuropePMCTool()
210
  results = await asyncio.gather(*[tool.run(query="q", limit=5) for _ in range(3)])
211
  finally:
212
+ st._ACADEMIC_MIN_INTERVAL = original_interval
213
+ st._academic_last_request_at = original_last
214
 
215
  assert all(r.success for r in results)
216
  assert len(starts) == 3
217
  deltas = [starts[i + 1] - starts[i] for i in range(2)]
218
  assert all(d >= 0.18 for d in deltas)
219
+
220
+
221
+ @pytest.mark.asyncio
222
+ async def test_neo4j_tool_query(mock_neo4j):
223
+ mock_neo4j.run.return_value = [{"n": "value"}]
224
+ tool = Neo4jTool(neo4j=mock_neo4j)
225
+ result = await tool.run(query="MATCH (n) RETURN n LIMIT 1")
226
+ assert result.success
227
+ assert result.data == [{"n": "value"}]
tools/__init__.py CHANGED
@@ -1,3 +1,3 @@
1
  from tools.base_tool import BaseTool, ToolResult
2
- from tools.search_tool import SemanticScholarTool
3
  from tools.database_tool import Neo4jTool, MongoTool
 
1
  from tools.base_tool import BaseTool, ToolResult
2
+ from tools.search_tool import EuropePMCTool, PubMedTool, UnpaywallTool
3
  from tools.database_tool import Neo4jTool, MongoTool
tools/ddg_tool.py CHANGED
@@ -1,6 +1,6 @@
1
  """
2
  DuckDuckGo HTML search tool — no-API-key web fallback for claim verification.
3
- Used when Semantic Scholar (rate-limited) and Tavily (quota) both fail.
4
  """
5
  from typing import Dict, List
6
 
 
1
  """
2
  DuckDuckGo HTML search tool — no-API-key web fallback for claim verification.
3
+ Used when the academic providers and Tavily (quota) both fail.
4
  """
5
  from typing import Dict, List
6
 
tools/search_tool.py CHANGED
@@ -1,79 +1,234 @@
1
  """
2
- Semantic Scholar search tool for citation gap detection.
 
 
 
 
 
 
 
 
3
  """
4
  import asyncio
5
  import os
6
  import time
7
  from typing import Dict, List, Optional
 
 
8
  import httpx
 
9
  from tools.base_tool import BaseTool, ToolResult
10
  from utils.logger import get_logger
11
  from utils.retry import async_retry
12
 
13
  logger = get_logger("search_tool")
14
 
15
- SS_BASE = "https://api.semanticscholar.org/graph/v1"
16
- SS_FIELDS = "title,authors,year,abstract,citationCount,externalIds,url"
 
 
 
 
17
 
18
- # Semantic Scholar's unauthenticated quota (~100 requests / 5 min / IP) is
19
- # shared process-wide AND with other tenants on the host egress IP. Multiple
20
- # pipeline agents run in parallel, so every request goes through a global
21
- # throttle to avoid self-inflicted 429 floods.
22
- _SS_MIN_INTERVAL = 1.5
23
- _ss_rate_lock = asyncio.Lock()
24
- _ss_last_request_at = 0.0
25
 
26
 
27
- async def _throttle_semantic_scholar() -> None:
28
- global _ss_last_request_at
29
- async with _ss_rate_lock:
30
  now = time.monotonic()
31
- wait = _SS_MIN_INTERVAL - (now - _ss_last_request_at)
32
  if wait > 0:
33
  await asyncio.sleep(wait)
34
- _ss_last_request_at = time.monotonic()
35
 
36
 
37
- class SemanticScholarTool(BaseTool):
38
- name = "semantic_scholar_search"
39
- description = "Search Semantic Scholar for research papers by keyword or title."
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
40
 
41
- def __init__(self, api_key: Optional[str] = None):
42
- self.api_key = api_key or os.getenv("SEMANTIC_SCHOLAR_API_KEY", "")
43
- self._headers = {"x-api-key": self.api_key} if self.api_key else {}
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
44
 
45
- async def run(self, query: str, limit: int = 10) -> ToolResult:
46
  try:
47
- results = await self._search(query, limit)
48
- return self._ok(results)
49
  except Exception as e:
50
- logger.error(f"Semantic Scholar search failed: {e}")
51
  return self._err(str(e))
52
 
53
- @async_retry(max_attempts=4, delay=2.0, backoff=2.5)
54
- async def _search(self, query: str, limit: int) -> List[Dict]:
55
- await _throttle_semantic_scholar()
56
- async with httpx.AsyncClient(headers=self._headers, timeout=30) as client:
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
57
  resp = await client.get(
58
- f"{SS_BASE}/paper/search",
59
- params={"query": query, "limit": limit, "fields": SS_FIELDS},
60
  )
 
 
61
  resp.raise_for_status()
62
  data = resp.json()
63
- papers = data.get("data", [])
64
- logger.debug(f"Semantic Scholar: {len(papers)} results for '{query}'")
65
- return [self._normalize(p) for p in papers]
66
-
67
- def _normalize(self, paper: Dict) -> Dict:
68
- authors = [a.get("name", "") for a in paper.get("authors", [])]
69
- doi = paper.get("externalIds", {}).get("DOI", "")
70
  return {
71
- "title": paper.get("title", ""),
72
- "authors": authors,
73
- "year": paper.get("year"),
74
- "abstract": paper.get("abstract", ""),
75
- "citation_count": paper.get("citationCount", 0),
76
  "doi": doi,
77
- "paper_id": paper.get("paperId", ""),
78
- "url": paper.get("url", ""),
79
  }
 
 
 
 
 
 
 
 
 
 
 
1
  """
2
+ Keyless academic search tools (Europe PMC, PubMed, Unpaywall).
3
+
4
+ Semantic Scholar was removed: its unauthenticated quota is shared per-IP
5
+ and routinely 429s when parallel pipeline agents search together. Every
6
+ tool here needs no API key.
7
+
8
+ Europe PMC — life-science literature (papers, preprints, patents)
9
+ PubMed — MEDLINE biomedical abstracts (E-utilities, email-tagged)
10
+ Unpaywall — open-access copy lookup by DOI (email-tagged)
11
  """
12
  import asyncio
13
  import os
14
  import time
15
  from typing import Dict, List, Optional
16
+ from xml.etree import ElementTree
17
+
18
  import httpx
19
+
20
  from tools.base_tool import BaseTool, ToolResult
21
  from utils.logger import get_logger
22
  from utils.retry import async_retry
23
 
24
  logger = get_logger("search_tool")
25
 
26
+ EUROPE_PMC_BASE = "https://www.ebi.ac.uk/europepmc/webservices/rest"
27
+ PUBMED_EUTILS = "https://eutils.ncbi.nlm.nih.gov/entrez/eutils"
28
+ UNPAYWALL_BASE = "https://api.unpaywall.org/v2"
29
+
30
+ # Public APIs want an email + tool name for fair-use tracking (no key).
31
+ CONTACT_EMAIL = os.getenv("PUBMED_EMAIL", "citationedge@app.local")
32
 
33
+ # Europe PMC and PubMed are burst-sensitive (1-3 req/s without keys), and
34
+ # multiple pipeline agents search in parallel serialize all calls through
35
+ # a process-wide throttle.
36
+ _ACADEMIC_MIN_INTERVAL = 1.2
37
+ _academic_rate_lock = asyncio.Lock()
38
+ _academic_last_request_at = 0.0
 
39
 
40
 
41
+ async def _throttle_academic() -> None:
42
+ global _academic_last_request_at
43
+ async with _academic_rate_lock:
44
  now = time.monotonic()
45
+ wait = _ACADEMIC_MIN_INTERVAL - (now - _academic_last_request_at)
46
  if wait > 0:
47
  await asyncio.sleep(wait)
48
+ _academic_last_request_at = time.monotonic()
49
 
50
 
51
+ class EuropePMCTool(BaseTool):
52
+ name = "europepmc_search"
53
+ description = "Search Europe PMC (biomedical literature, preprints, patents) by keyword or title."
54
+
55
+ async def search(self, query: str, limit: int = 5) -> List[Dict]:
56
+ return await self._fetch(query, min(limit, 25))
57
+
58
+ @async_retry(max_attempts=3, delay=1.5)
59
+ async def _fetch(self, query: str, limit: int) -> List[Dict]:
60
+ await _throttle_academic()
61
+ params = {
62
+ "query": query,
63
+ "format": "json",
64
+ "pageSize": limit,
65
+ "resultType": "core",
66
+ }
67
+ async with httpx.AsyncClient(timeout=30) as client:
68
+ resp = await client.get(f"{EUROPE_PMC_BASE}/search", params=params)
69
+ resp.raise_for_status()
70
+ data = resp.json()
71
 
72
+ papers: List[Dict] = []
73
+ results = ((data.get("resultList") or {}).get("result") or []) or []
74
+ for r in results:
75
+ title = (r.get("title") or "").strip()
76
+ if not title:
77
+ continue
78
+ pmcid = r.get("pmcid") or ""
79
+ doi = r.get("doi") or ""
80
+ url = (
81
+ f"https://europepmc.org/article/PMC/{pmcid}"
82
+ if pmcid else f"https://doi.org/{doi}" if doi else ""
83
+ )
84
+ papers.append({
85
+ "title": title,
86
+ "authors": [
87
+ a.strip() for a in (r.get("authorString") or "").split(",") if a.strip()
88
+ ][:5],
89
+ "year": r.get("pubYear"),
90
+ "abstract": r.get("abstractText") or "",
91
+ "citation_count": int(r.get("citedByCount") or 0),
92
+ "doi": doi,
93
+ "paper_id": r.get("pmcid") or r.get("id") or "",
94
+ "url": url,
95
+ "source": "Europe PMC",
96
+ })
97
+ return papers
98
 
99
+ async def run(self, query: str, limit: int = 5) -> ToolResult:
100
  try:
101
+ return self._ok(await self.search(query, limit))
 
102
  except Exception as e:
103
+ logger.error(f"Europe PMC search failed: {e}")
104
  return self._err(str(e))
105
 
106
+
107
+ class PubMedTool(BaseTool):
108
+ name = "pubmed_search"
109
+ description = "Search PubMed (MEDLINE biomedical abstracts) by keyword or title."
110
+
111
+ async def search(self, query: str, limit: int = 5) -> List[Dict]:
112
+ return await self._fetch(query, min(limit, 10))
113
+
114
+ @async_retry(max_attempts=3, delay=1.5)
115
+ async def _fetch(self, query: str, limit: int) -> List[Dict]:
116
+ async with httpx.AsyncClient(timeout=30) as client:
117
+ await _throttle_academic()
118
+ esearch = await client.get(
119
+ f"{PUBMED_EUTILS}/esearch.fcgi",
120
+ params={
121
+ "db": "pubmed", "term": query, "retmode": "json",
122
+ "retmax": limit, "tool": "citationedge", "email": CONTACT_EMAIL,
123
+ },
124
+ )
125
+ esearch.raise_for_status()
126
+ ids = (esearch.json().get("esearchresult") or {}).get("idlist") or []
127
+ if not ids:
128
+ return []
129
+
130
+ await _throttle_academic()
131
+ efetch = await client.get(
132
+ f"{PUBMED_EUTILS}/efetch.fcgi",
133
+ params={
134
+ "db": "pubmed", "id": ",".join(ids),
135
+ "rettype": "abstract", "retmode": "xml",
136
+ "tool": "citationedge", "email": CONTACT_EMAIL,
137
+ },
138
+ )
139
+ efetch.raise_for_status()
140
+ xml_text = efetch.text
141
+
142
+ return self._parse_efetch(xml_text)
143
+
144
+ def _parse_efetch(self, xml_text: str) -> List[Dict]:
145
+ papers: List[Dict] = []
146
+ try:
147
+ root = ElementTree.fromstring(xml_text)
148
+ except ElementTree.ParseError as e:
149
+ logger.warning(f"PubMed efetch XML parse failed: {e}")
150
+ return []
151
+ for art in root.findall(".//PubmedArticle"):
152
+ medline = art.find("MedlineCitation")
153
+ if medline is None:
154
+ continue
155
+ title_node = medline.find("Article/ArticleTitle")
156
+ title = "".join(title_node.itertext()).strip() if title_node is not None else ""
157
+ if not title:
158
+ continue
159
+ pmid = (medline.findtext("PMID") or "").strip()
160
+ doi = ""
161
+ doi_node = medline.find("Article/ELocationID[@EIdType='doi']")
162
+ if doi_node is not None and doi_node.text:
163
+ doi = doi_node.text.strip()
164
+ if not doi:
165
+ doi_node = art.find("PubmedData/ArticleIdList/ArticleId[@IdType='doi']")
166
+ if doi_node is not None and doi_node.text:
167
+ doi = doi_node.text.strip()
168
+
169
+ abstract = " ".join(
170
+ "".join(t.itertext()) for t in medline.findall("Article/Abstract/AbstractText")
171
+ ).strip()
172
+ year = (
173
+ medline.findtext("Article/ArticleDate/Year")
174
+ or medline.findtext("Article/JournalIssue/PubDate/Year")
175
+ or ""
176
+ )
177
+ authors = [
178
+ f"{a.findtext('ForeName', '')} {a.findtext('LastName', '')}".strip()
179
+ for a in medline.findall("Article/AuthorList/Author")
180
+ if a.findtext("LastName", "")
181
+ ]
182
+ authors = [a for a in authors if a][:5]
183
+
184
+ papers.append({
185
+ "title": title,
186
+ "authors": authors,
187
+ "year": int(year) if year and year.isdigit() else None,
188
+ "abstract": abstract,
189
+ "citation_count": 0, # PubMed does not expose cited-by here
190
+ "doi": doi,
191
+ "paper_id": pmid,
192
+ "url": f"https://pubmed.ncbi.nlm.nih.gov/{pmid}/" if pmid else "",
193
+ "source": "PubMed",
194
+ })
195
+ return papers
196
+
197
+ async def run(self, query: str, limit: int = 5) -> ToolResult:
198
+ try:
199
+ return self._ok(await self.search(query, limit))
200
+ except Exception as e:
201
+ logger.error(f"PubMed search failed: {e}")
202
+ return self._err(str(e))
203
+
204
+
205
+ class UnpaywallTool(BaseTool):
206
+ name = "unpaywall_lookup"
207
+ description = "Look up the open-access copy of a paper by DOI (Unpaywall)."
208
+
209
+ @async_retry(max_attempts=2, delay=1.0)
210
+ async def _lookup(self, doi: str) -> Dict:
211
+ async with httpx.AsyncClient(timeout=30) as client:
212
  resp = await client.get(
213
+ f"{UNPAYWALL_BASE}/{doi}", params={"email": CONTACT_EMAIL}
 
214
  )
215
+ if resp.status_code == 404:
216
+ return {"doi": doi, "is_oa": False, "url": ""}
217
  resp.raise_for_status()
218
  data = resp.json()
219
+ loc = data.get("best_oa_location") or {}
 
 
 
 
 
 
220
  return {
 
 
 
 
 
221
  "doi": doi,
222
+ "is_oa": bool(data.get("is_oa")),
223
+ "url": loc.get("url") or loc.get("url_for_pdf") or "",
224
  }
225
+
226
+ async def run(self, doi: str) -> ToolResult:
227
+ doi = (doi or "").strip()
228
+ if not doi:
229
+ return self._err("DOI required")
230
+ try:
231
+ return self._ok(await self._lookup(doi))
232
+ except Exception as e:
233
+ logger.error(f"Unpaywall lookup failed for {doi}: {e}")
234
+ return self._err(str(e))
tools/tavily_tool.py CHANGED
@@ -1,6 +1,6 @@
1
  """
2
  Tavily Search tool — web search fallback for claim verification.
3
- Used when Semantic Scholar returns no results for a specific claim.
4
  """
5
  import os
6
  from typing import Dict, List, Optional
 
1
  """
2
  Tavily Search tool — web search fallback for claim verification.
3
+ Used when the keyless academic providers return no results for a specific claim.
4
  """
5
  import os
6
  from typing import Dict, List, Optional
tools/wikipedia_tool.py ADDED
@@ -0,0 +1,151 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ """
2
+ Keyless Wikipedia + Wikidata evidence tools for claim verification.
3
+
4
+ Wikipedia — encyclopedia search + lead-section extracts
5
+ Wikidata — entity search (people, places, concepts) with descriptions
6
+ """
7
+ import asyncio
8
+ import re
9
+ import time
10
+ from typing import Dict, List
11
+
12
+ import httpx
13
+
14
+ from tools.base_tool import BaseTool, ToolResult
15
+ from utils.logger import get_logger
16
+ from utils.retry import async_retry
17
+
18
+ logger = get_logger("wikipedia_tool")
19
+
20
+ WIKIPEDIA_API = "https://en.wikipedia.org/w/api.php"
21
+ WIKIDATA_API = "https://www.wikidata.org/w/api.php"
22
+
23
+ # Wikimedia policy requires a descriptive User-Agent (contact info included);
24
+ # generic client UAs (e.g. python-httpx) are rejected with 403.
25
+ WIKI_HEADERS = {
26
+ "User-Agent": "CitationEdge/1.0 (citationedge@app.local; academic citation analysis)"
27
+ }
28
+
29
+ _TAG_RE = re.compile(r"<[^>]+>")
30
+
31
+ _WIKI_MIN_INTERVAL = 1.0
32
+ _wiki_rate_lock = asyncio.Lock()
33
+ _wiki_last_request_at = 0.0
34
+
35
+
36
+ async def _throttle_wiki() -> None:
37
+ global _wiki_last_request_at
38
+ async with _wiki_rate_lock:
39
+ now = time.monotonic()
40
+ wait = _WIKI_MIN_INTERVAL - (now - _wiki_last_request_at)
41
+ if wait > 0:
42
+ await asyncio.sleep(wait)
43
+ _wiki_last_request_at = time.monotonic()
44
+
45
+
46
+ def _strip_html(text: str) -> str:
47
+ return _TAG_RE.sub("", text or "").strip()
48
+
49
+
50
+ class WikipediaTool(BaseTool):
51
+ name = "wikipedia_search"
52
+ description = "Search Wikipedia for encyclopedia evidence on a topic."
53
+
54
+ async def search(self, query: str, limit: int = 3) -> List[Dict]:
55
+ return await self._fetch(query, min(limit, 10))
56
+
57
+ @async_retry(max_attempts=3, delay=1.5)
58
+ async def _fetch(self, query: str, limit: int) -> List[Dict]:
59
+ async with httpx.AsyncClient(timeout=30, headers=WIKI_HEADERS) as client:
60
+ await _throttle_wiki()
61
+ search = await client.get(
62
+ WIKIPEDIA_API,
63
+ params={
64
+ "action": "query", "list": "search", "srsearch": query,
65
+ "srlimit": limit, "format": "json",
66
+ },
67
+ )
68
+ search.raise_for_status()
69
+ hits = (search.json().get("query") or {}).get("search") or []
70
+ titles = [h.get("title") for h in hits if h.get("title")]
71
+ if not titles:
72
+ return []
73
+
74
+ await _throttle_wiki()
75
+ extracts = await client.get(
76
+ WIKIPEDIA_API,
77
+ params={
78
+ "action": "query", "prop": "extracts", "exintro": 1,
79
+ "explaintext": 1, "titles": "|".join(titles[:10]),
80
+ "format": "json",
81
+ },
82
+ )
83
+ extracts.raise_for_status()
84
+ pages = (extracts.json().get("query") or {}).get("pages") or {}
85
+
86
+ by_title = {
87
+ (p.get("title") or ""): p
88
+ for p in pages.values() if isinstance(p, dict)
89
+ }
90
+ results: List[Dict] = []
91
+ for hit in hits:
92
+ title = hit.get("title") or ""
93
+ if not title:
94
+ continue
95
+ page = by_title.get(title) or {}
96
+ snippet = (
97
+ (page.get("extract") or "").strip()
98
+ or _strip_html(hit.get("snippet") or "")
99
+ )
100
+ results.append({
101
+ "title": title,
102
+ "url": f"https://en.wikipedia.org/wiki/{title.replace(' ', '_')}",
103
+ "snippet": snippet[:400],
104
+ "source": "Wikipedia",
105
+ })
106
+ return results
107
+
108
+ async def run(self, query: str, limit: int = 3) -> ToolResult:
109
+ try:
110
+ return self._ok(await self.search(query, limit))
111
+ except Exception as e:
112
+ logger.error(f"Wikipedia search failed: {e}")
113
+ return self._err(str(e))
114
+
115
+
116
+ class WikidataTool(BaseTool):
117
+ name = "wikidata_lookup"
118
+ description = "Search Wikidata for entity facts (people, places, concepts)."
119
+
120
+ @async_retry(max_attempts=3, delay=1.5)
121
+ async def _fetch(self, query: str, limit: int) -> List[Dict]:
122
+ await _throttle_wiki()
123
+ params = {
124
+ "action": "wbsearchentities", "search": query, "language": "en",
125
+ "format": "json", "limit": limit,
126
+ }
127
+ async with httpx.AsyncClient(timeout=30, headers=WIKI_HEADERS) as client:
128
+ resp = await client.get(WIKIDATA_API, params=params)
129
+ resp.raise_for_status()
130
+ hits = resp.json().get("search") or []
131
+
132
+ results: List[Dict] = []
133
+ for h in hits:
134
+ qid = h.get("id") or ""
135
+ if not qid:
136
+ continue
137
+ results.append({
138
+ "id": qid,
139
+ "label": h.get("label") or qid,
140
+ "description": h.get("description") or "",
141
+ "url": f"https://www.wikidata.org/wiki/{qid}",
142
+ "source": "Wikidata",
143
+ })
144
+ return results
145
+
146
+ async def run(self, query: str, limit: int = 3) -> ToolResult:
147
+ try:
148
+ return self._ok(await self._fetch(query, min(limit, 10)))
149
+ except Exception as e:
150
+ logger.error(f"Wikidata search failed: {e}")
151
+ return self._err(str(e))