File size: 5,657 Bytes
78fc1e6 7fc5654 78fc1e6 68b0d74 81deeef 78fc1e6 d5f8958 78fc1e6 d5f8958 78fc1e6 7fc5654 78fc1e6 | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 | """
eval/run_wilcoxon.py
Wilcoxon signed-rank test: Condition D (EmpathRAG) vs Condition A (BM25 baseline).
Tests whether emotion-conditioned retrieval produces statistically significantly
higher emotion alignment scores than vanilla BM25 (p < 0.05).
Emotion alignment score: binary 1/0 per prompt — 1 if query emotion label matches
the emotion label of the top retrieved chunk, 0 otherwise.
"""
import sys, json
sys.path.insert(0, "src")
sys.path.insert(0, ".")
sys.path.insert(0, "eval")
import numpy as np
from scipy.stats import wilcoxon
from pipeline.pipeline import EmpathRAGPipeline
PROMPTS_PATH = "eval/test_prompts.json"
RESULTS_PATH = "eval/wilcoxon_results.json"
def compute_alignment_scores(pipeline, prompts):
"""
For each non-crisis prompt, compute binary alignment score:
1 if emotion(query) == emotion(top retrieved chunk), else 0.
"""
scores = []
for prompt in prompts:
result = pipeline.run(prompt["text"])
if result["crisis"] or not result["retrieved_chunks"]:
continue
q_emotion = result["emotion"]
top_chunk = result["retrieved_chunks"][0]
chunk_emotion = pipeline._classify_emotion(top_chunk)
scores.append(int(q_emotion == chunk_emotion))
return scores
def run_wilcoxon_eval():
with open(PROMPTS_PATH) as f:
prompts = json.load(f)
# ── Condition D: full EmpathRAG pipeline ──────────────────────────────────
print("Condition D — Full EmpathRAG pipeline")
# use_real_guardrail=False: Wilcoxon tests RETRIEVAL quality (Stage 4),
# not guardrail behavior (Stage 2). With real guardrail at t=0.50, 37/50
# prompts are intercepted before retrieval — leaving only 13 samples.
# Guardrail and retrieval are independent components; disabling guardrail
# here lets all 50 prompts reach the retrieval stage as intended.
pipeline_d = EmpathRAGPipeline(
use_real_guardrail=False,
allow_stub_guardrail=True,
guardrail_threshold=0.5,
)
original_check = pipeline_d.guardrail.check
def fast_check(text, threshold=0.5, skip_ig=False):
return original_check(text, threshold=threshold, skip_ig=True)
pipeline_d.guardrail.check = fast_check
print("Computing Condition D alignment scores...")
scores_d = compute_alignment_scores(pipeline_d, prompts)
print(f" D alignment: {np.mean(scores_d):.3f} ({sum(scores_d)}/{len(scores_d)} prompts aligned)")
# ── Condition A: BM25 baseline ────────────────────────────────────────────
# We reuse pipeline_d for emotion classification and swap out _retrieve
# to use BM25 instead of FAISS+emotion-filtering
print("\nCondition A — BM25 baseline retrieval")
print("Building BM25 index (this takes ~60-90s)...")
import condition_a
bm25, bm25_ids, bm25_texts = condition_a.load_bm25_index()
print("BM25 index ready.")
# Monkey-patch _retrieve on pipeline_d to use BM25
original_retrieve = pipeline_d._retrieve
def bm25_retrieve(query, emotion_label):
return condition_a.retrieve_bm25(query, bm25, bm25_ids, bm25_texts, top_k=5)
pipeline_d._retrieve = bm25_retrieve
print("Computing Condition A alignment scores...")
pipeline_d.tracker.reset()
scores_a = compute_alignment_scores(pipeline_d, prompts)
print(f" A alignment: {np.mean(scores_a):.3f} ({sum(scores_a)}/{len(scores_a)} prompts aligned)")
# Restore original retrieve
pipeline_d._retrieve = original_retrieve
# ── Wilcoxon test ─────────────────────────────────────────────────────────
print("\nRunning Wilcoxon signed-rank test (D vs A, alternative=greater)...")
# Pad to equal length if needed (should be equal since same prompts)
min_len = min(len(scores_d), len(scores_a))
s_d = scores_d[:min_len]
s_a = scores_a[:min_len]
if sum(s_d) == sum(s_a):
print("WARNING: scores are identical — Wilcoxon test not applicable.")
stat, p_val = float("nan"), float("nan")
else:
try:
# zero_method=pratt handles tied differences correctly for binary 0/1 scores
stat, p_val = wilcoxon(s_d, s_a, alternative="greater", zero_method="pratt")
except ValueError as e:
print(f"WARNING: Wilcoxon failed ({e}) — scores may be too similar.")
stat, p_val = float("nan"), float("nan")
print(f"\nWilcoxon Results:")
print(f" D mean alignment: {np.mean(s_d):.4f}")
print(f" A mean alignment: {np.mean(s_a):.4f}")
print(f" Statistic: {stat}")
print(f" p-value: {p_val:.4f}")
if not np.isnan(p_val):
print(f" {'SIGNIFICANT (p < 0.05)' if p_val < 0.05 else 'NOT SIGNIFICANT (p >= 0.05)'}")
output = {
"condition_d_mean": round(float(np.mean(s_d)), 4),
"condition_a_mean": round(float(np.mean(s_a)), 4),
"condition_d_scores": s_d,
"condition_a_scores": s_a,
"wilcoxon_statistic": float(stat) if not np.isnan(stat) else None,
"p_value": float(p_val) if not np.isnan(p_val) else None,
"significant": bool(p_val < 0.05) if not np.isnan(p_val) else None,
"n": min_len,
}
with open(RESULTS_PATH, "w") as f:
json.dump(output, f, indent=2)
print(f"Results saved to {RESULTS_PATH}")
if __name__ == "__main__":
run_wilcoxon_eval()
|