MukulRay commited on
Commit
6a63556
·
1 Parent(s): a90ef68

Phase 2.5: wire reliability scorer into critic — edge reliability replaces mean-age STALE threshold

Browse files
Files changed (2) hide show
  1. src/agents/critic.py +14 -1
  2. src/state.py +2 -1
src/agents/critic.py CHANGED
@@ -7,6 +7,7 @@ from langchain_groq import ChatGroq
7
  from langchain_core.messages import SystemMessage, HumanMessage
8
 
9
  from src.state import ResearchState, Paper, Verdict
 
10
 
11
  load_dotenv()
12
  logger = logging.getLogger(__name__)
@@ -189,9 +190,19 @@ def critic_node(state: ResearchState) -> ResearchState:
189
  "calibration_bin": Verdict.INSUFFICIENT,
190
  }
191
 
 
 
 
 
192
  # --- Run STALE and CONTRADICTED checks in parallel (both always run) ---
193
  mean_age = _mean_age_months(papers)
194
- is_stale = mean_age > 24
 
 
 
 
 
 
195
 
196
  contradictions = _detect_contradictions(papers)
197
  is_contradicted = len(contradictions) > 0
@@ -219,6 +230,7 @@ def critic_node(state: ResearchState) -> ResearchState:
219
  "retry_count": retry_count,
220
  "rewritten_questions": [],
221
  "calibration_bin": Verdict.PASS,
 
222
  }
223
 
224
  # --- Non-PASS path: rewrite questions and return ---
@@ -230,4 +242,5 @@ def critic_node(state: ResearchState) -> ResearchState:
230
  "rewritten_questions": rewritten,
231
  "retry_count": retry_count + 1,
232
  "calibration_bin": verdict,
 
233
  }
 
7
  from langchain_core.messages import SystemMessage, HumanMessage
8
 
9
  from src.state import ResearchState, Paper, Verdict
10
+ from src.reliability import score_papers, ReliabilityScore
11
 
12
  load_dotenv()
13
  logger = logging.getLogger(__name__)
 
190
  "calibration_bin": Verdict.INSUFFICIENT,
191
  }
192
 
193
+ # --- Phase 2.4: Compute edge reliability scores for all papers ---
194
+ original_query = state.get("original_query", "")
195
+ reliability_scores = score_papers(papers, query=original_query, use_llm=True)
196
+
197
  # --- Run STALE and CONTRADICTED checks in parallel (both always run) ---
198
  mean_age = _mean_age_months(papers)
199
+ # v2: STALE = low reliability across retrieved papers (not just mean age)
200
+ if reliability_scores:
201
+ mean_reliability = sum(rs.score for rs in reliability_scores.values()) / len(reliability_scores)
202
+ is_stale = mean_reliability < 0.40
203
+ else:
204
+ # fallback to v1 age threshold if scorer failed
205
+ is_stale = mean_age > 24
206
 
207
  contradictions = _detect_contradictions(papers)
208
  is_contradicted = len(contradictions) > 0
 
230
  "retry_count": retry_count,
231
  "rewritten_questions": [],
232
  "calibration_bin": Verdict.PASS,
233
+ "paper_reliability_scores": {pid: rs.__dict__ for pid, rs in reliability_scores.items()},
234
  }
235
 
236
  # --- Non-PASS path: rewrite questions and return ---
 
242
  "rewritten_questions": rewritten,
243
  "retry_count": retry_count + 1,
244
  "calibration_bin": verdict,
245
+ "paper_reliability_scores": {pid: rs.__dict__ for pid, rs in reliability_scores.items()},
246
  }
src/state.py CHANGED
@@ -99,4 +99,5 @@ class ResearchState(TypedDict):
99
  # --- Eval / config ---
100
  decay_config: str # "none" | "linear" | "log"
101
  calibration_bin: str # filled by critic for eval aggregation
102
- latency_ms: float
 
 
99
  # --- Eval / config ---
100
  decay_config: str # "none" | "linear" | "log"
101
  calibration_bin: str # filled by critic for eval aggregation
102
+ latency_ms: float
103
+ paper_reliability_scores: dict # {paper_id: ReliabilityScore.__dict__}