"""Streamlit helpers for System Run → Live Run (self-contained scipath_live).""" from __future__ import annotations import json import os import subprocess import sys from pathlib import Path from typing import Any, Callable, Optional from scipath_live.pipeline import ( DEFAULT_RUNS_ROOT, load_example_claim, new_job_dir, ) HF_SPACE = Path(__file__).resolve().parent DEMO_ROOT = HF_SPACE.parent CLI = HF_SPACE / "scipath_live" / "cli.py" # Stable browser cache for this demo (avoids Cursor sandbox temp paths). PLAYWRIGHT_BROWSERS_DIR = DEMO_ROOT / ".playwright" METHOD_UI = [ {"id": "codeagent_parametric", "cli": "parametric", "label": "Parametric - LLM Only"}, { "id": "codeagent_websearch_deep", "cli": "websearch_deep", "label": "Websearch Deep - LLM + Web Search + Crawl", }, ] def example_claim() -> dict[str, Any]: return load_example_claim() def _env_for_child() -> dict[str, str]: env = dict(os.environ) # Prefer demo-local .env if present. for candidate in (HF_SPACE / ".env", HF_SPACE.parent / ".env"): if not candidate.exists(): continue try: from dotenv import dotenv_values for key, value in dotenv_values(candidate).items(): if value and key not in env: env[key] = value except Exception: pass google = ( env.get("GOOGLE_GENAI_API_KEY") or env.get("GEMINI_API_KEY") or env.get("GOOGLE_API_KEY") or "" ).strip() if google: env["GEMINI_API_KEY"] = google env["GOOGLE_API_KEY"] = google env["GOOGLE_GENAI_API_KEY"] = google env["PYTHONPATH"] = ( f"{HF_SPACE}{os.pathsep}{env.get('PYTHONPATH', '')}".rstrip(os.pathsep) ) # Stream agent prints line-by-line into the Live Run terminal. env["PYTHONUNBUFFERED"] = "1" env["PYTHONIOENCODING"] = "utf-8" # Always use demo-local Playwright browsers (override Cursor sandbox cache). PLAYWRIGHT_BROWSERS_DIR.mkdir(parents=True, exist_ok=True) env["PLAYWRIGHT_BROWSERS_PATH"] = str(PLAYWRIGHT_BROWSERS_DIR) return env def _is_result_payload_line(line: str) -> bool: text = (line or "").strip() return text.startswith("{") and '"ok"' in text def _append_trace_from_logs_jsonl( job_dir: Path, log_callback: Optional[Callable[[str], None]] ) -> None: """Fallback: if stdout missed the trace, reconstruct from logs.jsonl.""" if not log_callback: return path = job_dir / "logs.jsonl" if not path.exists(): return try: from scipath_live.agent import format_agent_trace payload = json.loads(path.read_text(encoding="utf-8")) steps = payload.get("steps") or [] for line in format_agent_trace(steps): log_callback(line) except Exception as exc: log_callback(f"[warn] Could not load agent trace: {exc}") def run_live_method( method_id: str, *, log_callback: Optional[Callable[[str], None]] = None, model_name: str = "gemini-3-flash-preview", ) -> dict[str, Any]: """Subprocess-run a method; stream stdout lines via log_callback.""" cli_method = next((m["cli"] for m in METHOD_UI if m["id"] == method_id), None) if not cli_method: raise ValueError(f"Unknown method: {method_id}") job_dir = new_job_dir(DEFAULT_RUNS_ROOT, method=cli_method) cmd = [ sys.executable, str(CLI), "run", "--method", cli_method, "--output-dir", str(job_dir), "--model", model_name, ] if log_callback: log_callback(f"$ {' '.join(cmd)}") proc = subprocess.Popen( cmd, cwd=str(HF_SPACE), env=_env_for_child(), stdout=subprocess.PIPE, stderr=subprocess.STDOUT, text=True, bufsize=1, ) assert proc.stdout is not None last_json = "" saw_trace = False for line in proc.stdout: line = line.rstrip("\n") if not line: continue if _is_result_payload_line(line): # Keep machine payload out of the human terminal. last_json = line continue if "SciFy CodeAgent trace" in line: saw_trace = True if log_callback: log_callback(line) code = proc.wait() if code != 0: raise RuntimeError(f"Live run failed with exit code {code}") if last_json: payload = json.loads(last_json) if not payload.get("ok"): raise RuntimeError(payload.get("error") or "Live run failed") result = payload["result"] else: # Fallback: read result.json written by the agent. result_path = job_dir / "result.json" if not result_path.exists(): raise RuntimeError("Live run produced no result.json") result = json.loads(result_path.read_text(encoding="utf-8")) result["output_dir"] = str(job_dir) result["method_id"] = method_id if not saw_trace: _append_trace_from_logs_jsonl(job_dir, log_callback) result.setdefault("output_dir", str(job_dir)) result.setdefault("method_id", method_id) return result def evaluate_live_run( run_result: dict[str, Any], *, log_callback: Optional[Callable[[str], None]] = None, judge_model: str = "gemini/gemini-3.1-pro-preview", ) -> dict[str, Any]: result_json = Path(run_result.get("output_dir") or ".") / "result.json" if not result_json.exists(): # Write a minimal result file for the CLI. result_json.parent.mkdir(parents=True, exist_ok=True) result_json.write_text( json.dumps(run_result, indent=2, ensure_ascii=False) + "\n", encoding="utf-8" ) cmd = [ sys.executable, str(CLI), "evaluate", "--result-json", str(result_json), "--judge-model", judge_model, ] if log_callback: log_callback(f"$ {' '.join(cmd)}") proc = subprocess.run( cmd, cwd=str(HF_SPACE), env=_env_for_child(), capture_output=True, text=True, ) if log_callback and proc.stdout: for line in proc.stdout.splitlines(): log_callback(line) if proc.returncode != 0: err = (proc.stderr or proc.stdout or "").strip() raise RuntimeError(err or f"Evaluate failed with exit code {proc.returncode}") # Last JSON line is the payload. payload = None for line in reversed((proc.stdout or "").splitlines()): if line.startswith("{"): payload = json.loads(line) break if not payload or not payload.get("ok"): raise RuntimeError((payload or {}).get("error") or "Evaluate produced no result") return payload["judged"] def live_result_to_case_card(run_result: dict[str, Any], judged: Optional[dict] = None) -> dict[str, Any]: """Normalize live artifacts into the System Run case-card shape.""" method_id = run_result.get("method_id") or "codeagent_parametric" method_label = next( (m["label"] for m in METHOD_UI if m["id"] == method_id), method_id ) predicted = [] source_pred = (judged or {}).get("predicted_ingredients") or run_result.get("ingredients") or [] for i, item in enumerate(source_pred, start=1): if not isinstance(item, dict): continue predicted.append( { "idx": i, "description": str(item.get("description") or ""), "role": str(item.get("role") or ""), "rationale": str(item.get("rationale") or ""), } ) if judged: gold = [] for j in judged.get("recall_judgments") or []: gold.append( { "description": str(j.get("reference_ingredient") or ""), "role": str(j.get("reference_role") or ""), "badge": ( "full" if j.get("covered") else ("partial" if j.get("partial_matches") else "miss") ), "match_notes": [], "judge_note": str(j.get("reasoning") or ""), "pair_pred_idx": j.get("best_match_idx"), } ) # Reconstruct match notes for partials for pm in j.get("partial_matches") or []: gold[-1]["match_notes"].append( f"Partial ↔ pred #{pm.get('predicted_idx')}: {pm.get('reasoning', '')}" ) if gold[-1]["badge"] == "miss": gold[-1]["badge"] = "partial" # Live Run is not framed as success/failure — always yellow (part). f1 = float(judged.get("f1") or 0.0) return { "method_id": method_id, "method_label": method_label.split(" - ")[0], "case_key": "live", "case_chip": f"{method_label.split(' - ')[0]} · Live", "outcome_tone": "part", "tools": ( ["check_answer_format"] if "parametric" in method_id else ["web_search_tool", "deep_web_search_tool", "check_answer_format"] ), "paper_id": judged.get("paper_id") or run_result.get("paper_id"), "claim_idx": judged.get("claim_idx") or run_result.get("claim_idx"), "paper_title": judged.get("paper_title") or run_result.get("paper_title"), "claim": judged.get("claim") or run_result.get("claim"), "recall": float(judged.get("recall") or 0.0), "precision": float(judged.get("precision") or 0.0), "f1": f1, "gold_ingredients": gold, "predicted_ingredients": predicted, "tool_panel": {"kind": "none"}, } # Pre-evaluate: predictions only. return { "method_id": method_id, "method_label": method_label.split(" - ")[0], "case_key": "featured", "case_chip": f"{method_label.split(' - ')[0]} · Live (unevaluated)", "outcome_tone": "part", "tools": ( ["check_answer_format"] if "parametric" in method_id else ["web_search_tool", "deep_web_search_tool", "check_answer_format"] ), "paper_id": run_result.get("paper_id"), "claim_idx": run_result.get("claim_idx"), "paper_title": run_result.get("paper_title"), "claim": run_result.get("claim"), "recall": 0.0, "precision": 0.0, "f1": 0.0, "gold_ingredients": [], "predicted_ingredients": predicted, "tool_panel": {"kind": "none"}, }