"""Load curated System Run case studies for the Streamlit demo.
Labels and section structure follow the SciPaths Dev · CodeAgent Case Studies HTML report.
"""
from __future__ import annotations
import json
from functools import lru_cache
from pathlib import Path
from typing import Any, Optional
CASES_DIR = Path(__file__).resolve().parent / "system_run_cases"
ANALYSIS_PATH = CASES_DIR / "case_study_analysis.json"
SYSTEM_RUN_METHODS: list[dict[str, str]] = [
{
"id": "codeagent_parametric",
"label": "Parametric - LLM Only",
"banner_title": "1 · codeagent_parametric",
"banner_desc": "Parametric-only generation (no search/retrieval). Mean F1 ≈ 0.20.",
},
{
"id": "codeagent_websearch_deep",
# Audience label: "Crawl" = deep_web_search_tool (page/PDF fetch; no separate crawl tool).
"label": "Websearch Deep - LLM + Web Search + Crawl",
"banner_title": "2 · codeagent_websearch_deep",
"banner_desc": (
"Tools: web_search_tool + deep_web_search_tool (page fetch / crawl). "
"Mean F1 ≈ 0.32."
),
},
]
DEFAULT_SYSTEM_METHOD = SYSTEM_RUN_METHODS[0]["id"]
DEFAULT_SYSTEM_CASE = "success"
# Case keys exposed in the UI — labels match the HTML report.
_METHOD_CASES: dict[str, list[tuple[str, str]]] = {
"codeagent_parametric": [("success", "Success"), ("failure", "Failure")],
"codeagent_websearch_deep": [("success", "Success"), ("failure", "Failure")],
}
CASE_OUTCOME_LABEL: dict[str, str] = {
"success": "Success",
"failure": "Failure",
}
TOOLING_NOTE_HTML = (
"Tooling by setting."
"
· parametric: parametric knowledge only (no search/retrieval)."
"
· websearch_deep: web_search_tool + "
"deep_web_search_tool (URL page fetch)."
)
@lru_cache(maxsize=1)
def load_analysis() -> dict[str, Any]:
with ANALYSIS_PATH.open("r", encoding="utf-8") as f:
data = json.load(f)
if not isinstance(data, dict):
raise ValueError(f"Invalid case study analysis at {ANALYSIS_PATH}")
return data
def method_ids() -> list[str]:
return [m["id"] for m in SYSTEM_RUN_METHODS]
def method_meta(method_id: str) -> dict[str, str]:
for m in SYSTEM_RUN_METHODS:
if m["id"] == method_id:
return m
return {"id": method_id, "label": method_id, "banner_title": method_id, "banner_desc": ""}
def method_label(method_id: str) -> str:
return method_meta(method_id).get("label") or method_id
def case_chip_label(method_id: str, case_key: str) -> str:
"""Top-of-card label, e.g. 'Parametric · Success'."""
pretty = {
"codeagent_parametric": "Parametric",
"codeagent_websearch_deep": "WebSearch Deep",
}.get(method_id, method_label(method_id))
outcome = CASE_OUTCOME_LABEL.get(case_key, case_key)
return f"{pretty} · {outcome}"
def list_cases(method_id: str) -> list[tuple[str, str]]:
return list(_METHOD_CASES.get(method_id, []))
def default_case_for_method(method_id: str) -> str:
cases = list_cases(method_id)
return cases[0][0] if cases else DEFAULT_SYSTEM_CASE
def outcome_tone(case_key: str, f1: float) -> str:
if case_key == "success":
return "ok"
if case_key == "failure":
return "bad"
if f1 >= 0.75:
return "ok"
if f1 <= 0.05:
return "bad"
return "part"
def _setting(method_id: str) -> dict[str, Any]:
settings = load_analysis().get("settings") or {}
block = settings.get(method_id)
if not isinstance(block, dict):
raise KeyError(f"Unknown system-run method: {method_id}")
return block
def _raw_case_block(method_id: str, case_key: str) -> dict[str, Any]:
setting = _setting(method_id)
if case_key == "featured":
block = setting.get("case")
else:
block = setting.get(case_key)
if not isinstance(block, dict):
raise KeyError(f"Unknown case {case_key!r} for method {method_id}")
return block
def _gold_ingredients(case: dict[str, Any]) -> list[dict[str, Any]]:
"""Gold column with full/partial/miss badges and match notes (HTML report style)."""
predicted = case.get("predicted_ingredients") or []
gold: list[dict[str, Any]] = []
for judgment in case.get("recall_judgments") or []:
if not isinstance(judgment, dict):
continue
desc = str(judgment.get("reference_ingredient") or "").strip()
role = str(judgment.get("reference_role") or "").strip()
if not desc:
continue
covered = bool(judgment.get("covered"))
partials = judgment.get("partial_matches") or []
if not isinstance(partials, list):
partials = []
match_notes: list[str] = []
judge_note = ""
pair_pred_idx: Optional[int] = None
if covered:
badge = "full"
best = judgment.get("best_match") if isinstance(judgment.get("best_match"), dict) else {}
best_desc = str(best.get("description") or "").strip()
if best_desc:
match_notes.append(f"Matched to pred: {best_desc}")
judge_note = str(judgment.get("reasoning") or "").strip()
raw_idx = judgment.get("best_match_idx")
if isinstance(raw_idx, int) and raw_idx > 0:
pair_pred_idx = raw_idx
elif best_desc:
# Fall back to description match against predicted list (1-based).
for i, pred in enumerate(predicted, start=1):
if isinstance(pred, dict) and str(pred.get("description") or "").strip() == best_desc:
pair_pred_idx = i
break
elif partials:
badge = "partial"
for pm in partials:
if not isinstance(pm, dict):
continue
pred_idx = pm.get("predicted_idx")
reason = str(pm.get("reasoning") or "").strip()
if pred_idx is not None and reason:
match_notes.append(f"Partial ↔ pred #{pred_idx}: {reason}")
elif reason:
match_notes.append(reason)
else:
badge = "miss"
gold.append(
{
"description": desc,
"role": role,
"badge": badge,
"match_notes": match_notes,
"judge_note": judge_note,
"pair_pred_idx": pair_pred_idx,
}
)
return gold
def build_pair_rows(
gold: list[dict[str, Any]], predicted: list[dict[str, Any]]
) -> list[dict[str, Any]]:
"""Align gold↔prediction rows for success-case connector lines."""
by_idx = {
int(p["idx"]): p
for p in predicted
if isinstance(p.get("idx"), int)
}
used: set[int] = set()
rows: list[dict[str, Any]] = []
for g in gold:
pred_idx = g.get("pair_pred_idx")
pred = by_idx.get(int(pred_idx)) if isinstance(pred_idx, int) else None
linked = bool(pred is not None and g.get("badge") == "full")
if linked and isinstance(pred_idx, int):
used.add(pred_idx)
rows.append({"gold": g, "pred": pred, "linked": linked})
for p in predicted:
idx = p.get("idx")
if isinstance(idx, int) and idx not in used:
rows.append({"gold": None, "pred": p, "linked": False})
return rows
def _tool_panel(method_id: str, case_key: str, case: dict[str, Any], setting: dict[str, Any]) -> dict[str, Any]:
"""Structured tool evidence / traces for the case card."""
if method_id == "codeagent_retrieval_plus_websearch":
analysis = case.get("gemini_analysis") if isinstance(case.get("gemini_analysis"), dict) else {}
return {
"kind": "retrieval",
"intro": (
"This run used both retrieve_documents and web_search_tool. "
"Web search continued in later steps after documents were returned."
),
"queries": list(analysis.get("web_search_queries_example") or []),
"retrieved_docs": list(case.get("retrieved_doc_titles") or []),
}
if method_id == "codeagent_websearch_deep":
traces = setting.get("tool_traces") if isinstance(setting.get("tool_traces"), dict) else {}
trace = traces.get("success" if case_key == "success" else "failure")
if not isinstance(trace, dict):
return {"kind": "none"}
deep_fetches = []
for item in trace.get("deep_fetches") or []:
if not isinstance(item, dict):
continue
outcome = str(item.get("outcome") or "")
ok = outcome.upper().startswith("SUCCESS")
deep_fetches.append(
{
"url": str(item.get("url") or ""),
"outcome": outcome,
"ok": ok,
}
)
return {
"kind": "deep",
"queries": list(
trace.get("web_search_queries")
or trace.get("web_search_queries_sample")
or []
),
"hits": list(
trace.get("web_search_hits_notable")
or trace.get("web_search_correct_hits_but_underused")
or []
),
"deep_fetches": deep_fetches,
}
return {"kind": "none"}
def get_case(method_id: str, case_key: str) -> dict[str, Any]:
"""Normalized case card payload for the UI."""
setting = _setting(method_id)
case = _raw_case_block(method_id, case_key)
tools = [str(t) for t in (setting.get("tools") or [])]
predicted = []
for idx, item in enumerate(case.get("predicted_ingredients") or [], start=1):
if not isinstance(item, dict):
continue
predicted.append(
{
"idx": idx,
"description": str(item.get("description") or "").strip(),
"role": str(item.get("role") or "").strip(),
"rationale": str(item.get("rationale") or "").strip(),
}
)
f1 = float(case.get("f1") or 0.0)
return {
"method_id": method_id,
"method_label": method_label(method_id),
"case_key": case_key,
"case_chip": case_chip_label(method_id, case_key),
"outcome_tone": outcome_tone(case_key, f1),
"tools": tools,
"paper_id": str(case.get("paper_id") or ""),
"claim_idx": case.get("claim_idx"),
"paper_title": str(case.get("paper_title") or ""),
"claim": str(case.get("claim") or ""),
"recall": float(case.get("recall") or 0.0),
"precision": float(case.get("precision") or 0.0),
"f1": f1,
"gold_ingredients": _gold_ingredients(case),
"predicted_ingredients": predicted,
"tool_panel": _tool_panel(method_id, case_key, case, setting),
"banner": method_meta(method_id),
}
def get_analysis(method_id: str, case_key: Optional[str] = None) -> dict[str, Any]:
setting = _setting(method_id)
if method_id == "codeagent_retrieval_plus_websearch":
case = setting.get("case") or {}
analysis = case.get("gemini_analysis") if isinstance(case, dict) else None
return dict(analysis) if isinstance(analysis, dict) else {}
analysis = setting.get("gemini_analysis")
if not isinstance(analysis, dict):
return {}
out = dict(analysis)
out["_case_key"] = case_key or ""
return out
def analysis_view_for_case(method_id: str, case_key: str) -> dict[str, Any]:
"""Structured analysis view matching HTML report headings."""
analysis = get_analysis(method_id, case_key)
if not analysis:
return {"model_tag": "Gemini 3.1 Pro analysis", "blocks": []}
blocks: list[dict[str, Any]] = []
if method_id == "codeagent_retrieval_plus_websearch":
headline = str(analysis.get("headline") or "").strip()
takeaway = str(analysis.get("takeaway") or "").strip()
diag = headline
if takeaway:
diag = f"{headline}\n{takeaway}" if headline else takeaway
if diag:
blocks.append({"type": "diagnosis", "text": diag, "strong_first": True})
two_col = []
for key, title in (
("what_was_retrieved", "What the tools returned"),
("effect_on_prediction", "Effect on prediction"),
):
items = analysis.get(key)
if isinstance(items, list) and items:
two_col.append({"title": title, "items": [str(x) for x in items]})
if two_col:
blocks.append({"type": "two_col", "cols": two_col})
errs = analysis.get("error_taxonomy_bullets")
if isinstance(errs, list) and errs:
blocks.append(
{"type": "list", "title": "Error analysis", "items": [str(x) for x in errs]}
)
return {"model_tag": "Gemini 3.1 Pro analysis", "blocks": blocks}
if method_id == "codeagent_parametric":
if case_key == "success":
why = analysis.get("success_why")
if isinstance(why, list) and why:
blocks.append(
{
"type": "list",
"title": "Why this succeeded",
"items": [str(x) for x in why],
}
)
else:
if analysis.get("failure_diagnosis"):
blocks.append(
{
"type": "diagnosis",
"text": f"Diagnosis. {analysis['failure_diagnosis']}",
"strong_first": True,
}
)
errs = analysis.get("failure_error_bullets")
if isinstance(errs, list) and errs:
blocks.append(
{
"type": "list",
"title": "Error taxonomy",
"items": [str(x) for x in errs],
}
)
needed = analysis.get("failure_what_needed")
if isinstance(needed, list) and needed:
blocks.append(
{
"type": "list",
"title": "What recovery would have required",
"items": [str(x) for x in needed],
}
)
return {"model_tag": "Gemini 3.1 Pro analysis", "blocks": blocks}
# websearch_deep
if case_key == "success":
two_col = []
for key, title in (
("success_search_contribution", "web_search_tool"),
("success_deep_contribution", "deep_web_search_tool"),
):
items = analysis.get(key)
if isinstance(items, list) and items:
two_col.append({"title": title, "items": [str(x) for x in items]})
if two_col:
blocks.append({"type": "two_col", "cols": two_col})
why = analysis.get("success_why_overall")
if isinstance(why, list) and why:
blocks.append(
{"type": "list", "title": "Why this succeeded", "items": [str(x) for x in why]}
)
else:
if analysis.get("failure_diagnosis"):
blocks.append(
{
"type": "diagnosis",
"text": f"Diagnosis. {analysis['failure_diagnosis']}",
"strong_first": True,
}
)
two_col = []
for key, title in (
("failure_search_what_went_wrong", "web_search_tool"),
("failure_deep_what_went_wrong", "deep_web_search_tool"),
):
items = analysis.get(key)
if isinstance(items, list) and items:
two_col.append({"title": title, "items": [str(x) for x in items]})
if two_col:
blocks.append({"type": "two_col", "cols": two_col})
errs = analysis.get("failure_error_bullets")
if isinstance(errs, list) and errs:
blocks.append(
{"type": "list", "title": "Error taxonomy", "items": [str(x) for x in errs]}
)
needed = analysis.get("failure_what_needed")
if isinstance(needed, list) and needed:
blocks.append(
{
"type": "list",
"title": "What recovery would have required",
"items": [str(x) for x in needed],
}
)
return {"model_tag": "Gemini 3.1 Pro analysis", "blocks": blocks}