Mukul Rayana commited on
Commit
68b0d74
·
1 Parent(s): d02b074

fix: Wilcoxon uses stub guardrail — tests retrieval quality on all 50 prompts not 13 (Day 15)

Browse files
Files changed (1) hide show
  1. eval/run_wilcoxon.py +6 -1
eval/run_wilcoxon.py CHANGED
@@ -40,7 +40,12 @@ def run_wilcoxon_eval():
40
 
41
  # ── Condition D: full EmpathRAG pipeline ──────────────────────────────────
42
  print("Condition D — Full EmpathRAG pipeline")
43
- pipeline_d = EmpathRAGPipeline(use_real_guardrail=True, guardrail_threshold=0.5)
 
 
 
 
 
44
  original_check = pipeline_d.guardrail.check
45
  def fast_check(text, threshold=0.5, skip_ig=False):
46
  return original_check(text, threshold=threshold, skip_ig=True)
 
40
 
41
  # ── Condition D: full EmpathRAG pipeline ──────────────────────────────────
42
  print("Condition D — Full EmpathRAG pipeline")
43
+ # use_real_guardrail=False: Wilcoxon tests RETRIEVAL quality (Stage 4),
44
+ # not guardrail behavior (Stage 2). With real guardrail at t=0.50, 37/50
45
+ # prompts are intercepted before retrieval — leaving only 13 samples.
46
+ # Guardrail and retrieval are independent components; disabling guardrail
47
+ # here lets all 50 prompts reach the retrieval stage as intended.
48
+ pipeline_d = EmpathRAGPipeline(use_real_guardrail=False, guardrail_threshold=0.5)
49
  original_check = pipeline_d.guardrail.check
50
  def fast_check(text, threshold=0.5, skip_ig=False):
51
  return original_check(text, threshold=threshold, skip_ig=True)