eval/run_curated_retrieval_audit.py

"""
Audit curated-support retrieval without running the generator.

Run after Karthik's corpus has been validated and indexed:
    python eval/run_curated_retrieval_audit.py
"""

from __future__ import annotations

import json
import sys
from pathlib import Path

sys.path.insert(0, "src")

from pipeline.pipeline import EmpathRAGPipeline
from pipeline.query_router import route_query


PROMPTS = [
    "I'm so anxious about my thesis defense next week, I can't sleep.",
    "My advisor rejected my work again and I don't know what to do.",
    "I feel isolated in my program and I don't have anyone to talk to.",
    "I think I might need counseling but I'm not sure where to start.",
    "I'm burned out and falling behind on everything.",
    "Can you help me find support for disability accommodations?",
    "I don't know who to contact after hours if things get worse.",
]

RESULTS_PATH = Path("eval/curated_retrieval_audit.json")


def main() -> int:
    pipeline = EmpathRAGPipeline(
        retrieval_corpus="curated_support",
        use_real_guardrail=True,
        guardrail_threshold=0.5,
    )

    rows = []
    for prompt in PROMPTS:
        emotion = pipeline._classify_emotion(prompt)
        pipeline.tracker.update(emotion, len(prompt.split()))
        trajectory = pipeline.tracker.trajectory()
        routed = route_query(prompt, emotion, trajectory)
        retrieved = pipeline._retrieve(routed, emotion)
        sources = pipeline._source_summaries(retrieved)
        rows.append(
            {
                "prompt": prompt,
                "emotion": emotion,
                "trajectory": trajectory,
                "routed_query": routed,
                "sources": sources,
            }
        )

        print("\nPROMPT:", prompt)
        if not sources:
            print("  NO SOURCES")
            continue
        for i, source in enumerate(sources[:3], 1):
            print(
                f"  {i}. {source['source_name']} | {source['title']} | "
                f"{source['topic']} | {source['risk_level']}"
            )
            if source["url"]:
                print(f"     {source['url']}")

    RESULTS_PATH.write_text(json.dumps(rows, indent=2), encoding="utf-8")
    print(f"\nSaved audit: {RESULTS_PATH}")
    return 0


if __name__ == "__main__":
    raise SystemExit(main())