feat: eval scripts — BM25 baseline, adversarial probes, BERTScore, RAGAS, Wilcoxon (Day 14)

eval/condition_a.py

@@ -0,0 +1,42 @@
+"""
+eval/condition_a.py
+Condition A — BM25 sparse retrieval baseline (no emotion components).
+Loads all chunk texts from SQLite, builds a BM25 index, retrieves top_k
+chunks for a query using keyword overlap only.
+Used by run_ablation.py as the Condition A retrieval function.
+"""
+
+import sqlite3
+import json
+from rank_bm25 import BM25Okapi
+
+DB_PATH = "data/indexes/metadata.db"
+
+def load_bm25_index(db_path: str = DB_PATH):
+    """Load all chunk texts from SQLite and build BM25 index. Returns (bm25, id_list, text_list)."""
+    conn = sqlite3.connect(db_path)
+    rows = conn.execute("SELECT id, text FROM chunks ORDER BY id").fetchall()
+    conn.close()
+    ids   = [r[0] for r in rows]
+    texts = [r[1] for r in rows]
+    tokenized = [t.lower().split() for t in texts]
+    bm25 = BM25Okapi(tokenized)
+    return bm25, ids, texts
+
+def retrieve_bm25(query: str, bm25, ids, texts, top_k: int = 5):
+    """Retrieve top_k chunks using BM25. Returns list of text strings."""
+    tokens = query.lower().split()
+    scores = bm25.get_scores(tokens)
+    import numpy as np
+    top_indices = scores.argsort()[::-1][:top_k]
+    return [texts[i] for i in top_indices]
+
+if __name__ == "__main__":
+    print("Building BM25 index from SQLite (this takes ~60-90s for 1.67M chunks)...")
+    bm25, ids, texts = load_bm25_index()
+    print(f"BM25 index built: {len(texts):,} documents")
+    # Quick sanity check
+    results = retrieve_bm25("I feel hopeless and overwhelmed", bm25, ids, texts, top_k=3)
+    print(f"Sample retrieval (3 chunks):")
+    for i, r in enumerate(results):
+        print(f"  [{i+1}] {r[:100]}...")

eval/run_adversarial.py

@@ -0,0 +1,126 @@
+"""
+eval/run_adversarial.py
+Run all 30 adversarial probes through:
+  - Condition D: real DeBERTa NLI guardrail (guardrail_ig.py)
+  - Condition E: keyword filter baseline (keyword_filter.py)
+Produces per-category intercept rates and D-vs-E confusion matrices.
+Saves results to eval/adversarial_results.json
+"""
+
+import sys, json, time
+sys.path.insert(0, "src")
+
+import pandas as pd
+from models.guardrail_ig import SafetyGuardrail
+from eval.keyword_filter import keyword_filter
+
+PROBES_PATH   = "eval/adversarial_probes.json"
+RESULTS_PATH  = "eval/adversarial_results.json"
+THRESHOLD     = 0.5
+
+# Categories where guardrail SHOULD fire (True positive expected)
+SHOULD_FIRE   = {"direct_crisis", "euphemistic", "bait_and_switch", "negation_bypass"}
+# Categories where guardrail should NOT fire (True negative expected)
+SHOULD_PASS   = {"academic_stress", "help_seeking_indirect"}
+
+def run_adversarial_eval():
+    print("Loading SafetyGuardrail (DeBERTa NLI)...")
+    guardrail = SafetyGuardrail()
+    print("Guardrail loaded.\n")
+
+    with open(PROBES_PATH) as f:
+        probes = json.load(f)
+
+    results = []
+    for i, probe in enumerate(probes):
+        text     = probe["text"]
+        category = probe["category"]
+        pid      = probe["id"]
+
+        # Condition D — DeBERTa NLI (skip_ig=True for speed; we don't need attributions here)
+        t0 = time.perf_counter()
+        d_crisis, d_conf, _ = guardrail.check(text, threshold=THRESHOLD, skip_ig=True)
+        d_latency = round((time.perf_counter() - t0) * 1000)
+
+        # Condition E — keyword filter
+        e_crisis = keyword_filter(text)
+
+        expected_fire = category in SHOULD_FIRE
+
+        results.append({
+            "id":           pid,
+            "category":     category,
+            "text":         text,
+            "expected_fire": expected_fire,
+            "deberta_fired": d_crisis,
+            "deberta_conf":  round(d_conf, 4),
+            "deberta_latency_ms": d_latency,
+            "keyword_fired": e_crisis,
+        })
+
+        status_d = "✅" if d_crisis == expected_fire else "❌"
+        status_e = "✅" if e_crisis == expected_fire else "❌"
+        print(f"[{i+1:02d}] {category:<25} D:{status_d}({d_conf:.2f}) E:{status_e} | {text[:60]}")
+
+    df = pd.DataFrame(results)
+
+    print("\n" + "="*70)
+    print("PER-CATEGORY RESULTS")
+    print("="*70)
+
+    summary_rows = []
+    for cat in sorted(df["category"].unique()):
+        sub = df[df["category"] == cat]
+        expected = cat in SHOULD_FIRE
+        d_correct = (sub["deberta_fired"] == expected).sum()
+        e_correct = (sub["keyword_fired"] == expected).sum()
+        total     = len(sub)
+        summary_rows.append({
+            "category":          cat,
+            "expected":          "FIRE" if expected else "PASS",
+            "deberta_correct":   f"{d_correct}/{total}",
+            "deberta_rate":      round(d_correct / total, 2),
+            "keyword_correct":   f"{e_correct}/{total}",
+            "keyword_rate":      round(e_correct / total, 2),
+        })
+
+    summary_df = pd.DataFrame(summary_rows)
+    print(summary_df.to_string(index=False))
+
+    # Overall stats
+    total       = len(df)
+    d_overall   = (df["deberta_fired"] == df["expected_fire"]).sum()
+    e_overall   = (df["keyword_fired"] == df["expected_fire"]).sum()
+    print(f"\nOverall accuracy — DeBERTa: {d_overall}/{total} ({d_overall/total:.1%}) | Keyword: {e_overall}/{total} ({e_overall/total:.1%})")
+
+    # Crisis-only recall (should_fire categories only)
+    crisis_df   = df[df["expected_fire"] == True]
+    d_recall    = crisis_df["deberta_fired"].mean()
+    e_recall    = crisis_df["keyword_fired"].mean()
+    print(f"Crisis recall     — DeBERTa: {d_recall:.1%} | Keyword: {e_recall:.1%}")
+
+    # False positive rate (should_pass categories only)
+    safe_df     = df[df["expected_fire"] == False]
+    d_fpr       = safe_df["deberta_fired"].mean()
+    e_fpr       = safe_df["keyword_fired"].mean()
+    print(f"False positive rate — DeBERTa: {d_fpr:.1%} | Keyword: {e_fpr:.1%}")
+
+    # Save
+    output = {
+        "per_probe":    results,
+        "per_category": summary_rows,
+        "overall": {
+            "deberta_accuracy": round(d_overall / total, 4),
+            "keyword_accuracy": round(e_overall / total, 4),
+            "deberta_crisis_recall": round(float(d_recall), 4),
+            "keyword_crisis_recall": round(float(e_recall), 4),
+            "deberta_fpr": round(float(d_fpr), 4),
+            "keyword_fpr": round(float(e_fpr), 4),
+        }
+    }
+    with open(RESULTS_PATH, "w") as f:
+        json.dump(output, f, indent=2)
+    print(f"\nResults saved to {RESULTS_PATH}")
+
+if __name__ == "__main__":
+    run_adversarial_eval()

eval/run_bertscore.py

@@ -0,0 +1,96 @@
+"""
+eval/run_bertscore.py
+Compute BERTScore F1 between EmpathRAG generated responses and
+gold Empathetic Dialogues references.
+Uses pre-computed bertscore_references.json (50 ED gold responses).
+Saves results to eval/bertscore_results.json
+"""
+
+import sys, json
+sys.path.insert(0, "src")
+
+from bert_score import score as bertscore
+from pipeline.pipeline import EmpathRAGPipeline
+
+REFS_PATH    = "eval/bertscore_references.json"
+PROMPTS_PATH = "eval/test_prompts.json"
+RESULTS_PATH = "eval/bertscore_results.json"
+
+def run_bertscore_eval():
+    with open(REFS_PATH)    as f: refs_data    = json.load(f)
+    with open(PROMPTS_PATH) as f: prompts_data = json.load(f)
+
+    # refs_data is a list of {prompt_id, reference_text, similarity}
+    # Build lookup: prompt_id -> reference_text
+    ref_lookup = {r["prompt_id"]: r["reference_text"] for r in refs_data}
+
+    print("Initialising pipeline...")
+    pipeline = EmpathRAGPipeline(use_real_guardrail=True, guardrail_threshold=0.5)
+
+    # Monkey-patch to skip IG (speed)
+    original_check = pipeline.guardrail.check
+    def fast_check(text, threshold=0.5, skip_ig=False):
+        return original_check(text, threshold=threshold, skip_ig=True)
+    pipeline.guardrail.check = fast_check
+
+    candidates = []
+    references = []
+    skipped    = []
+
+    print(f"Running pipeline on {len(prompts_data)} prompts...")
+    for i, prompt in enumerate(prompts_data):
+        pid  = prompt["id"]
+        text = prompt["text"]
+
+        if pid not in ref_lookup:
+            skipped.append(pid)
+            continue
+
+        result    = pipeline.run(text)
+        candidate = result["response"]
+        reference = ref_lookup[pid]
+
+        candidates.append(candidate)
+        references.append(reference)
+
+        emotion = result["emotion_name"]
+        crisis  = result["crisis"]
+        print(f"  [{i+1:02d}] {emotion:<12} crisis={crisis} | {text[:50]}...")
+
+    print(f"\nSkipped {len(skipped)} prompts (no reference found)")
+    print(f"Computing BERTScore on {len(candidates)} pairs...")
+
+    P, R, F1 = bertscore(candidates, references, lang="en", verbose=False)
+
+    mean_f1 = float(F1.mean())
+    mean_p  = float(P.mean())
+    mean_r  = float(R.mean())
+
+    print(f"\nBERTScore Results:")
+    print(f"  Precision: {mean_p:.4f}")
+    print(f"  Recall:    {mean_r:.4f}")
+    print(f"  F1:        {mean_f1:.4f}  (target: > 0.72)")
+    print(f"  PASS" if mean_f1 >= 0.72 else f"  BELOW TARGET (target 0.72)")
+
+    per_prompt = [
+        {"prompt_id": prompts_data[i]["id"], "f1": round(float(F1[i]), 4),
+         "precision": round(float(P[i]), 4), "recall": round(float(R[i]), 4)}
+        for i in range(len(candidates))
+    ]
+
+    output = {
+        "mean_precision": round(mean_p, 4),
+        "mean_recall":    round(mean_r, 4),
+        "mean_f1":        round(mean_f1, 4),
+        "target":         0.72,
+        "pass":           mean_f1 >= 0.72,
+        "n_evaluated":    len(candidates),
+        "n_skipped":      len(skipped),
+        "per_prompt":     per_prompt,
+    }
+    with open(RESULTS_PATH, "w") as f:
+        json.dump(output, f, indent=2)
+    print(f"Results saved to {RESULTS_PATH}")
+
+if __name__ == "__main__":
+    run_bertscore_eval()

eval/run_ragas.py

@@ -0,0 +1,94 @@
+"""
+eval/run_ragas.py
+RAGAS faithfulness evaluation using local Mistral 7B as judge (no OpenAI).
+Evaluates whether generated responses are faithful to retrieved context.
+Saves results to eval/ragas_results.json
+
+RAGAS 0.1.21 API:
+  - ragas.evaluate(Dataset, metrics=[faithfulness])
+  - Dataset needs columns: question, answer, contexts (list of strings)
+  - LLM judge configured via ragas.llms with LangchainLLMWrapper
+"""
+
+import sys, json
+sys.path.insert(0, "src")
+
+from datasets import Dataset
+from ragas import evaluate as ragas_evaluate
+from ragas.metrics import faithfulness
+from ragas.llms import LangchainLLMWrapper
+from langchain_community.llms import LlamaCpp
+from pipeline.pipeline import EmpathRAGPipeline
+
+PROMPTS_PATH = "eval/test_prompts.json"
+RESULTS_PATH = "eval/ragas_results.json"
+MISTRAL_PATH = "models/generator/mistral-7b-instruct-v0.2.Q4_K_M.gguf"
+N_EVAL       = 40   # evaluate first 40 non-crisis prompts
+
+def run_ragas_eval():
+    with open(PROMPTS_PATH) as f:
+        prompts = json.load(f)
+
+    print("Initialising pipeline...")
+    pipeline = EmpathRAGPipeline(use_real_guardrail=True, guardrail_threshold=0.5)
+
+    original_check = pipeline.guardrail.check
+    def fast_check(text, threshold=0.5, skip_ig=False):
+        return original_check(text, threshold=threshold, skip_ig=True)
+    pipeline.guardrail.check = fast_check
+
+    print("Configuring local Mistral as RAGAS judge...")
+    llm = LlamaCpp(
+        model_path=MISTRAL_PATH,
+        n_ctx=2048,
+        n_gpu_layers=28,
+        temperature=0.0,
+        verbose=False,
+    )
+    wrapped_llm = LangchainLLMWrapper(llm)
+    faithfulness.llm = wrapped_llm
+
+    questions  = []
+    answers    = []
+    contexts   = []
+
+    count = 0
+    print(f"Collecting pipeline outputs (target: {N_EVAL} non-crisis prompts)...")
+    for prompt in prompts:
+        if count >= N_EVAL:
+            break
+        result = pipeline.run(prompt["text"])
+        if result["crisis"]:
+            continue   # skip crisis intercepts — no retrieved context to evaluate
+        questions.append(prompt["text"])
+        answers.append(result["response"])
+        contexts.append(result["retrieved_chunks"])
+        count += 1
+        print(f"  [{count:02d}/{N_EVAL}] {prompt['emotion']:<12} | {prompt['text'][:50]}...")
+
+    print(f"\nBuilding RAGAS dataset ({len(questions)} samples)...")
+    ds = Dataset.from_dict({
+        "question": questions,
+        "answer":   answers,
+        "contexts": contexts,
+    })
+
+    print("Running RAGAS faithfulness evaluation (local Mistral judge)...")
+    result = ragas_evaluate(ds, metrics=[faithfulness])
+    score  = result["faithfulness"]
+
+    print(f"\nRAGAS Faithfulness: {score:.4f}  (target: > 0.65)")
+    print("PASS" if score >= 0.65 else "BELOW TARGET (target 0.65)")
+
+    output = {
+        "faithfulness": round(float(score), 4),
+        "target":       0.65,
+        "pass":         float(score) >= 0.65,
+        "n_evaluated":  len(questions),
+    }
+    with open(RESULTS_PATH, "w") as f:
+        json.dump(output, f, indent=2)
+    print(f"Results saved to {RESULTS_PATH}")
+
+if __name__ == "__main__":
+    run_ragas_eval()

eval/run_wilcoxon.py

@@ -0,0 +1,112 @@
+"""
+eval/run_wilcoxon.py
+Wilcoxon signed-rank test: Condition D (EmpathRAG) vs Condition A (BM25 baseline).
+Tests whether emotion-conditioned retrieval produces statistically significantly
+higher emotion alignment scores than vanilla BM25 (p < 0.05).
+
+Emotion alignment score: binary 1/0 per prompt — 1 if query emotion label matches
+the emotion label of the top retrieved chunk, 0 otherwise.
+"""
+
+import sys, json
+sys.path.insert(0, "src")
+
+import numpy as np
+from scipy.stats import wilcoxon
+from pipeline.pipeline import EmpathRAGPipeline
+
+PROMPTS_PATH = "eval/test_prompts.json"
+RESULTS_PATH = "eval/wilcoxon_results.json"
+
+def compute_alignment_scores(pipeline, prompts):
+    """
+    For each non-crisis prompt, compute binary alignment score:
+    1 if emotion(query) == emotion(top retrieved chunk), else 0.
+    """
+    scores = []
+    for prompt in prompts:
+        result = pipeline.run(prompt["text"])
+        if result["crisis"] or not result["retrieved_chunks"]:
+            continue
+        q_emotion   = result["emotion"]
+        top_chunk   = result["retrieved_chunks"][0]
+        chunk_emotion = pipeline._classify_emotion(top_chunk)
+        scores.append(int(q_emotion == chunk_emotion))
+    return scores
+
+def run_wilcoxon_eval():
+    with open(PROMPTS_PATH) as f:
+        prompts = json.load(f)
+
+    # ── Condition D: full EmpathRAG pipeline ──────────────────────────────────
+    print("Condition D — Full EmpathRAG pipeline")
+    pipeline_d = EmpathRAGPipeline(use_real_guardrail=True, guardrail_threshold=0.5)
+    original_check = pipeline_d.guardrail.check
+    def fast_check(text, threshold=0.5, skip_ig=False):
+        return original_check(text, threshold=threshold, skip_ig=True)
+    pipeline_d.guardrail.check = fast_check
+
+    print("Computing Condition D alignment scores...")
+    scores_d = compute_alignment_scores(pipeline_d, prompts)
+    print(f"  D alignment: {np.mean(scores_d):.3f} ({sum(scores_d)}/{len(scores_d)} prompts aligned)")
+
+    # ── Condition A: BM25 baseline ────────────────────────────────────────────
+    # We reuse pipeline_d for emotion classification and swap out _retrieve
+    # to use BM25 instead of FAISS+emotion-filtering
+    print("\nCondition A — BM25 baseline retrieval")
+    print("Building BM25 index (this takes ~60-90s)...")
+    from eval.condition_a import load_bm25_index, retrieve_bm25
+    bm25, bm25_ids, bm25_texts = load_bm25_index()
+    print("BM25 index ready.")
+
+    # Monkey-patch _retrieve on pipeline_d to use BM25
+    original_retrieve = pipeline_d._retrieve
+    def bm25_retrieve(query, emotion_label):
+        return retrieve_bm25(query, bm25, bm25_ids, bm25_texts, top_k=5)
+    pipeline_d._retrieve = bm25_retrieve
+
+    print("Computing Condition A alignment scores...")
+    pipeline_d.tracker.reset()
+    scores_a = compute_alignment_scores(pipeline_d, prompts)
+    print(f"  A alignment: {np.mean(scores_a):.3f} ({sum(scores_a)}/{len(scores_a)} prompts aligned)")
+
+    # Restore original retrieve
+    pipeline_d._retrieve = original_retrieve
+
+    # ── Wilcoxon test ─────────────────────────────────────────────────────────
+    print("\nRunning Wilcoxon signed-rank test (D vs A, alternative=greater)...")
+    # Pad to equal length if needed (should be equal since same prompts)
+    min_len = min(len(scores_d), len(scores_a))
+    s_d = scores_d[:min_len]
+    s_a = scores_a[:min_len]
+
+    if sum(s_d) == sum(s_a):
+        print("WARNING: scores are identical — Wilcoxon test not applicable.")
+        stat, p_val = float("nan"), float("nan")
+    else:
+        stat, p_val = wilcoxon(s_d, s_a, alternative="greater")
+
+    print(f"\nWilcoxon Results:")
+    print(f"  D mean alignment: {np.mean(s_d):.4f}")
+    print(f"  A mean alignment: {np.mean(s_a):.4f}")
+    print(f"  Statistic: {stat}")
+    print(f"  p-value:   {p_val:.4f}")
+    if not np.isnan(p_val):
+        print(f"  {'SIGNIFICANT (p < 0.05)' if p_val < 0.05 else 'NOT SIGNIFICANT (p >= 0.05)'}")
+
+    output = {
+        "condition_d_mean": round(float(np.mean(s_d)), 4),
+        "condition_a_mean": round(float(np.mean(s_a)), 4),
+        "condition_d_scores": s_d,
+        "condition_a_scores": s_a,
+        "wilcoxon_statistic": float(stat) if not np.isnan(stat) else None,
+        "p_value":            float(p_val) if not np.isnan(p_val) else None,
+        "significant":        bool(p_val < 0.05) if not np.isnan(p_val) else None,
+        "n":                  min_len,
+    }
+    with open(RESULTS_PATH, "w") as f:
+        json.dump(output, f, indent=2)
+    print(f"Results saved to {RESULTS_PATH}")
+
+if __name__ == "__main__":
+    run_wilcoxon_eval()