Phase 2.5: wire reliability scorer into critic — edge reliability replaces mean-age STALE threshold
Browse files- src/agents/critic.py +14 -1
- src/state.py +2 -1
src/agents/critic.py
CHANGED
|
@@ -7,6 +7,7 @@ from langchain_groq import ChatGroq
|
|
| 7 |
from langchain_core.messages import SystemMessage, HumanMessage
|
| 8 |
|
| 9 |
from src.state import ResearchState, Paper, Verdict
|
|
|
|
| 10 |
|
| 11 |
load_dotenv()
|
| 12 |
logger = logging.getLogger(__name__)
|
|
@@ -189,9 +190,19 @@ def critic_node(state: ResearchState) -> ResearchState:
|
|
| 189 |
"calibration_bin": Verdict.INSUFFICIENT,
|
| 190 |
}
|
| 191 |
|
|
|
|
|
|
|
|
|
|
|
|
|
| 192 |
# --- Run STALE and CONTRADICTED checks in parallel (both always run) ---
|
| 193 |
mean_age = _mean_age_months(papers)
|
| 194 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 195 |
|
| 196 |
contradictions = _detect_contradictions(papers)
|
| 197 |
is_contradicted = len(contradictions) > 0
|
|
@@ -219,6 +230,7 @@ def critic_node(state: ResearchState) -> ResearchState:
|
|
| 219 |
"retry_count": retry_count,
|
| 220 |
"rewritten_questions": [],
|
| 221 |
"calibration_bin": Verdict.PASS,
|
|
|
|
| 222 |
}
|
| 223 |
|
| 224 |
# --- Non-PASS path: rewrite questions and return ---
|
|
@@ -230,4 +242,5 @@ def critic_node(state: ResearchState) -> ResearchState:
|
|
| 230 |
"rewritten_questions": rewritten,
|
| 231 |
"retry_count": retry_count + 1,
|
| 232 |
"calibration_bin": verdict,
|
|
|
|
| 233 |
}
|
|
|
|
| 7 |
from langchain_core.messages import SystemMessage, HumanMessage
|
| 8 |
|
| 9 |
from src.state import ResearchState, Paper, Verdict
|
| 10 |
+
from src.reliability import score_papers, ReliabilityScore
|
| 11 |
|
| 12 |
load_dotenv()
|
| 13 |
logger = logging.getLogger(__name__)
|
|
|
|
| 190 |
"calibration_bin": Verdict.INSUFFICIENT,
|
| 191 |
}
|
| 192 |
|
| 193 |
+
# --- Phase 2.4: Compute edge reliability scores for all papers ---
|
| 194 |
+
original_query = state.get("original_query", "")
|
| 195 |
+
reliability_scores = score_papers(papers, query=original_query, use_llm=True)
|
| 196 |
+
|
| 197 |
# --- Run STALE and CONTRADICTED checks in parallel (both always run) ---
|
| 198 |
mean_age = _mean_age_months(papers)
|
| 199 |
+
# v2: STALE = low reliability across retrieved papers (not just mean age)
|
| 200 |
+
if reliability_scores:
|
| 201 |
+
mean_reliability = sum(rs.score for rs in reliability_scores.values()) / len(reliability_scores)
|
| 202 |
+
is_stale = mean_reliability < 0.40
|
| 203 |
+
else:
|
| 204 |
+
# fallback to v1 age threshold if scorer failed
|
| 205 |
+
is_stale = mean_age > 24
|
| 206 |
|
| 207 |
contradictions = _detect_contradictions(papers)
|
| 208 |
is_contradicted = len(contradictions) > 0
|
|
|
|
| 230 |
"retry_count": retry_count,
|
| 231 |
"rewritten_questions": [],
|
| 232 |
"calibration_bin": Verdict.PASS,
|
| 233 |
+
"paper_reliability_scores": {pid: rs.__dict__ for pid, rs in reliability_scores.items()},
|
| 234 |
}
|
| 235 |
|
| 236 |
# --- Non-PASS path: rewrite questions and return ---
|
|
|
|
| 242 |
"rewritten_questions": rewritten,
|
| 243 |
"retry_count": retry_count + 1,
|
| 244 |
"calibration_bin": verdict,
|
| 245 |
+
"paper_reliability_scores": {pid: rs.__dict__ for pid, rs in reliability_scores.items()},
|
| 246 |
}
|
src/state.py
CHANGED
|
@@ -99,4 +99,5 @@ class ResearchState(TypedDict):
|
|
| 99 |
# --- Eval / config ---
|
| 100 |
decay_config: str # "none" | "linear" | "log"
|
| 101 |
calibration_bin: str # filled by critic for eval aggregation
|
| 102 |
-
latency_ms: float
|
|
|
|
|
|
| 99 |
# --- Eval / config ---
|
| 100 |
decay_config: str # "none" | "linear" | "log"
|
| 101 |
calibration_bin: str # filled by critic for eval aggregation
|
| 102 |
+
latency_ms: float
|
| 103 |
+
paper_reliability_scores: dict # {paper_id: ReliabilityScore.__dict__}
|