Add curated corpus integration scaffold

.gitattributes

@@ -0,0 +1 @@
+*.pdf binary

.gitignore

@@ -61,7 +61,13 @@ service-account*.json
 
 # Data artifacts
 data/processed/
+data/curated/resources_seed.jsonl
+data/curated/source_inventory.csv
+data/curated/excluded_sources.csv
+data/curated/raw_pages/
+data/curated/indexes/
 eval/ragas_results.json
+eval/curated_retrieval_audit.json
 
 # Session artifacts
 CHANGES_APPLIED.md

data/curated/resources_seed.example.jsonl

@@ -0,0 +1 @@
+{"id":"umd_counseling_services_example_001","source_id":"src_example_001","source_name":"UMD Counseling Center","source_type":"university_resource","title":"Counseling Services","url":"https://counseling.umd.edu/","topic":"counseling_services","audience":"umd_student","risk_level":"safe","usage_mode":"retrieval","text":"The University of Maryland Counseling Center provides support services for students dealing with emotional, academic, social, or personal concerns. Students can use counseling resources to talk through stress, relationship concerns, adjustment difficulties, anxiety, depression, and other challenges that may affect wellbeing or academic life. The Counseling Center can also help students understand what type of support may fit their situation and connect them with appropriate campus or community resources.","summary":"Overview of UMD Counseling Center support services for student wellbeing.","last_checked":"2026-04-27","notes":"Example row only. Replace with Karthik's reviewed corpus before building the curated index."}

demo/app.py

@@ -12,6 +12,7 @@ import uuid
 import datetime
 import os
 import threading
+from html import escape
 from pipeline.pipeline import EmpathRAGPipeline
 
 # Constants
@@ -26,10 +27,15 @@ LABEL_COLORS = {
 LOG_PATH = "eval/human_eval_log.jsonl"
 LOG_TURNS = os.getenv("EMPATHRAG_LOG_TURNS") == "1"
 SHARE_DEMO = os.getenv("EMPATHRAG_SHARE") == "1"
+RETRIEVAL_CORPUS = os.getenv("EMPATHRAG_RETRIEVAL_CORPUS", "auto")
 
 # Initialize pipeline (runs once at module load)
 print("[Demo] Initialising EmpathRAG pipeline...")
-pipeline = EmpathRAGPipeline(use_real_guardrail=True, guardrail_threshold=0.5)
+pipeline = EmpathRAGPipeline(
+    use_real_guardrail=True,
+    guardrail_threshold=0.5,
+    retrieval_corpus=RETRIEVAL_CORPUS,
+)
 pipeline_lock = threading.Lock()
 print("[Demo] Pipeline ready.")
 
@@ -127,10 +133,51 @@ def format_ig_panel(is_crisis, confidence, ig_tokens, loading) -> str:
     return html
 
 
+def format_retrieval_panel(result=None) -> str:
+    """Format retrieval corpus and source metadata for the demo side panel."""
+    if not result:
+        return "<div style='color:#888;font-size:13px;padding:8px;'>No retrieval yet.</div>"
+
+    safety_level = escape(str(result.get("safety_level", "unknown")))
+    safety_reason = escape(str(result.get("safety_reason", "")))
+    corpus = escape(str(result.get("retrieval_corpus", "unknown")))
+    html = (
+        "<div style='font-size:12px;line-height:1.35;'>"
+        f"<div><strong>Corpus:</strong> {corpus}</div>"
+        f"<div><strong>Safety:</strong> {safety_level}</div>"
+        f"<div><strong>Reason:</strong> {safety_reason}</div>"
+    )
+
+    sources = result.get("retrieved_sources", [])
+    if not sources:
+        html += "<div style='color:#888;margin-top:8px;'>No sources retrieved.</div></div>"
+        return html
+
+    html += "<div style='margin-top:10px;font-weight:600;'>Top Sources</div>"
+    for source in sources[:3]:
+        title = escape(str(source.get("title", "") or "Untitled source"))
+        source_name = escape(str(source.get("source_name", "") or "Unknown source"))
+        topic = escape(str(source.get("topic", "") or ""))
+        risk = escape(str(source.get("risk_level", "") or ""))
+        url = escape(str(source.get("url", "") or ""))
+        html += (
+            "<div style='border-top:1px solid #ddd;padding-top:6px;margin-top:6px;'>"
+            f"<div><strong>{title}</strong></div>"
+            f"<div>{source_name}</div>"
+            f"<div style='color:#666;'>topic={topic} · risk={risk}</div>"
+        )
+        if url:
+            html += f"<div><a href='{url}' target='_blank'>source link</a></div>"
+        html += "</div>"
+    html += "</div>"
+    return html
+
+
 def respond(message, chat_history, session_state):
     """
     Generator function - yields UI state after each update.
-    Yields tuple of 5 values: (chatbot, timeline_html, trajectory, crisis_html, session_id)
+    Yields chatbot, emotion timeline, trajectory, safety panel, retrieval panel,
+    session ID, and per-user session state.
     """
     if not session_state:
         session_state = new_session_state()
@@ -144,6 +191,7 @@ def respond(message, chat_history, session_state):
                format_emotion_timeline(emotion_history, pipeline.tracker.trajectory()),
                pipeline.tracker.trajectory(),
                format_ig_panel(False, 0.0, [], False),
+               format_retrieval_panel(),
                session_id,
                session_state)
         return
@@ -189,6 +237,7 @@ def respond(message, chat_history, session_state):
                timeline_html,
                result["trajectory"],
                format_ig_panel(True, result["crisis_confidence"], [], loading=True),
+               format_retrieval_panel(result),
                session_id,
                session_state)
 
@@ -201,6 +250,7 @@ def respond(message, chat_history, session_state):
                timeline_html,
                result["trajectory"],
                format_ig_panel(True, confidence, ig_tokens, loading=False),
+               format_retrieval_panel(result),
                session_id,
                session_state)
     else:
@@ -209,6 +259,7 @@ def respond(message, chat_history, session_state):
                timeline_html,
                result["trajectory"],
                format_ig_panel(False, 0.0, [], False),
+               format_retrieval_panel(result),
                session_id,
                session_state)
 
@@ -219,8 +270,9 @@ def reset_session_handler():
 
     placeholder_timeline = "<div style='color:#888;font-size:13px;padding:8px;'>No emotions detected yet.</div>"
     placeholder_crisis = "<div style='color:#888;font-size:13px;padding:8px;'>No crisis detected this session.</div>"
+    placeholder_retrieval = "<div style='color:#888;font-size:13px;padding:8px;'>No retrieval yet.</div>"
 
-    return ([], placeholder_timeline, "stable", placeholder_crisis, session_state["session_id"], session_state)
+    return ([], placeholder_timeline, "stable", placeholder_crisis, placeholder_retrieval, session_state["session_id"], session_state)
 
 
 # Gradio UI
@@ -259,12 +311,14 @@ with gr.Blocks(theme=gr.themes.Soft(), title="EmpathRAG Demo") as demo:
 
             gr.Markdown("### Safety Guardrail")
             crisis_out = gr.HTML(value="<div style='color:#888;font-size:13px;padding:8px;'>No crisis detected this session.</div>")
+            gr.Markdown("### Retrieval Sources")
+            retrieval_out = gr.HTML(value="<div style='color:#888;font-size:13px;padding:8px;'>No retrieval yet.</div>")
 
     # Wire up interactions
     msg_box.submit(
         respond,
         inputs=[msg_box, chatbot, session_state],
-        outputs=[chatbot, timeline_out, trajectory_out, crisis_out, session_id_box, session_state]
+        outputs=[chatbot, timeline_out, trajectory_out, crisis_out, retrieval_out, session_id_box, session_state]
     ).then(
         lambda: "",
         outputs=msg_box
@@ -273,7 +327,7 @@ with gr.Blocks(theme=gr.themes.Soft(), title="EmpathRAG Demo") as demo:
     send_btn.click(
         respond,
         inputs=[msg_box, chatbot, session_state],
-        outputs=[chatbot, timeline_out, trajectory_out, crisis_out, session_id_box, session_state]
+        outputs=[chatbot, timeline_out, trajectory_out, crisis_out, retrieval_out, session_id_box, session_state]
     ).then(
         lambda: "",
         outputs=msg_box
@@ -281,7 +335,7 @@ with gr.Blocks(theme=gr.themes.Soft(), title="EmpathRAG Demo") as demo:
 
     reset_btn.click(
         reset_session_handler,
-        outputs=[chatbot, timeline_out, trajectory_out, crisis_out, session_id_box, session_state]
+        outputs=[chatbot, timeline_out, trajectory_out, crisis_out, retrieval_out, session_id_box, session_state]
     )
 
 

docs/KARTHIK_CORPUS_INTEGRATION_STEPS.md

@@ -0,0 +1,82 @@
+# Karthik Corpus Integration Steps
+
+Use this after Karthik sends the curated corpus delivery.
+
+## Expected Files
+
+Place the files here:
+
+```text
+data/curated/resources_seed.jsonl
+data/curated/source_inventory.csv
+data/curated/excluded_sources.csv
+data/curated/raw_pages/
+```
+
+The only required file for indexing is `resources_seed.jsonl`. The others are
+for review, reproducibility, and the class/research writeup.
+
+## Validate
+
+```bash
+python -m src.data.curated_resources data/curated/resources_seed.jsonl
+```
+
+If validation fails, fix the reported line numbers before indexing.
+
+## Build Curated Index
+
+```bash
+python -m src.data.build_curated_index
+```
+
+This creates:
+
+```text
+data/curated/indexes/faiss_curated.index
+data/curated/indexes/metadata_curated.db
+```
+
+These artifacts are intentionally separate from the existing Reddit research
+index. Do not delete or overwrite `data/indexes/faiss_flat.index` or
+`data/indexes/metadata.db`.
+
+## Run Retrieval Audit
+
+```bash
+python eval/run_curated_retrieval_audit.py
+```
+
+Review `eval/curated_retrieval_audit.json` and confirm top sources are safe,
+official, and relevant.
+
+## Run Demo
+
+Default demo behavior uses `auto` retrieval mode:
+
+- If curated index exists, use `curated_support`.
+- If curated index is missing, fall back to `reddit_research`.
+
+```bash
+python demo/app.py
+```
+
+To force a mode:
+
+```bash
+set EMPATHRAG_RETRIEVAL_CORPUS=curated_support
+python demo/app.py
+```
+
+or:
+
+```bash
+set EMPATHRAG_RETRIEVAL_CORPUS=reddit_research
+python demo/app.py
+```
+
+## Presentation Safety Note
+
+For the MSML demo, describe Reddit as the original research corpus and curated
+resources as the safer V2 student-support corpus. The curated corpus is the
+preferred path for any UMD counseling conversation or future publication work.

docs/TEAMMATE_CURATED_CORPUS_HANDOFF.md

@@ -0,0 +1,560 @@
+# EmpathRAG V2 Curated Corpus Handoff
+
+This document is the teammate handoff for building the first curated support
+corpus for EmpathRAG V2. The goal is to produce clean, source-cited,
+student-support content that can be directly ingested into a separate curated
+FAISS + SQLite retrieval index.
+
+## 1. Objective
+
+EmpathRAG currently has a large Reddit-based retrieval corpus. That corpus is
+useful for research comparison and ablation, but it should not be the primary
+student-facing support source for a mental-health-adjacent demo.
+
+Your task is to create a safer curated corpus from official and reputable public
+resources. This corpus should help EmpathRAG retrieve grounding context for:
+
+- anxiety before exams, thesis defense, or presentations
+- advisor conflict and academic frustration
+- burnout, isolation, loneliness, and imposter feelings
+- depression/help-seeking language
+- campus counseling navigation
+- after-hours and crisis support
+- disability/accessibility support
+- graduate student stress, funding stress, and academic pressure
+- grounding exercises and help-seeking scripts
+
+Prioritize correctness, source quality, and safety over volume.
+
+## 2. Deliverables
+
+Please deliver a folder like this:
+
+```text
+curated_corpus_delivery/
+  README_corpus_notes.md
+  source_inventory.csv
+  excluded_sources.csv
+  resources_seed.jsonl
+  raw_pages/
+    src_001.txt
+    src_002.txt
+```
+
+Required files:
+
+- `source_inventory.csv`: every source/page reviewed.
+- `resources_seed.jsonl`: final clean chunks for ingestion.
+- `README_corpus_notes.md`: summary of what was collected and any concerns.
+
+Optional but helpful:
+
+- `raw_pages/`: saved raw text snapshots from pages.
+- `excluded_sources.csv`: pages reviewed but rejected.
+
+## 3. Source Priority
+
+Use official/public sources first.
+
+Priority order:
+
+1. UMD Counseling Center
+2. UMD after-hours or crisis support pages
+3. UMD Graduate School resources
+4. UMD Accessibility and Disability Service
+5. UMD Ombuds, conflict-resolution, or student support resources
+6. 988 Lifeline
+7. NIMH
+8. SAMHSA
+9. CDC mental health or suicide prevention resources
+10. Other reputable nonprofit/clinical education resources only if they fill a
+    real coverage gap
+
+Do not use:
+
+- Reddit, Quora, forums, or social media
+- random blogs
+- commercial therapy marketing pages
+- personal stories with graphic crisis details
+- pages that describe self-harm methods
+- content that gives diagnosis, treatment, or medication instructions
+- login-gated or private pages
+
+## 4. Source Inventory Format
+
+Create `source_inventory.csv`.
+
+Required columns:
+
+```csv
+source_id,source_name,source_type,title,url,domain,date_accessed,include_status,reason,license_or_terms_notes
+```
+
+Allowed `include_status` values:
+
+```text
+include
+partial
+exclude
+needs_review
+```
+
+Example:
+
+```csv
+src_001,UMD Counseling Center,university_resource,Counseling Services,https://counseling.umd.edu/,counseling.umd.edu,2026-04-27,include,Official student counseling resource,Public webpage
+```
+
+Use `needs_review` when the page seems useful but contains sensitive, ambiguous,
+or policy-heavy content that should be checked before inclusion.
+
+## 5. Main Corpus Format
+
+Create `resources_seed.jsonl`.
+
+Use JSONL format: one valid JSON object per line. Do not wrap the file in a JSON
+array.
+
+Required schema:
+
+```json
+{
+  "id": "umd_counseling_001",
+  "source_id": "src_001",
+  "source_name": "UMD Counseling Center",
+  "source_type": "university_resource",
+  "title": "Counseling Services",
+  "url": "https://example.edu/page",
+  "topic": "counseling_services",
+  "audience": "umd_student",
+  "risk_level": "safe",
+  "usage_mode": "retrieval",
+  "text": "Clean paragraph-sized text chunk suitable for retrieval.",
+  "summary": "One sentence summary of the chunk.",
+  "last_checked": "2026-04-27",
+  "notes": "Why this chunk is useful."
+}
+```
+
+Every field is required. Every `id` must be unique.
+
+## 6. Allowed Field Values
+
+### `source_type`
+
+Use one of:
+
+```text
+university_resource
+crisis_resource
+government_public_health
+student_support
+clinician_review_candidate
+```
+
+### `topic`
+
+Use one primary topic per chunk:
+
+```text
+crisis_immediate_help
+counseling_services
+after_hours_support
+academic_burnout
+advisor_conflict
+isolation_loneliness
+anxiety_stress
+depression_support
+accessibility_disability
+graduate_student_support
+help_seeking_script
+grounding_exercise
+campus_navigation
+therapy_expectations
+peer_support
+emergency_services
+```
+
+If a chunk fits multiple topics, choose the most specific one.
+
+### `audience`
+
+Use one of:
+
+```text
+umd_student
+graduate_student
+student_general
+crisis_support
+supporter_or_friend
+```
+
+### `risk_level`
+
+Use one of:
+
+```text
+safe
+wellbeing
+crisis_resource
+exclude
+```
+
+Meanings:
+
+- `safe`: normal support/resource text.
+- `wellbeing`: distress/help-seeking content, but not crisis.
+- `crisis_resource`: crisis resource or urgent-support content.
+- `exclude`: do not put this in the final retrieval corpus.
+
+Prefer putting excluded content in `excluded_sources.csv` instead of
+`resources_seed.jsonl`.
+
+### `usage_mode`
+
+Use one of:
+
+```text
+retrieval
+wellbeing_only
+crisis_only
+metadata_only
+```
+
+Meanings:
+
+- `retrieval`: safe for normal RAG retrieval.
+- `wellbeing_only`: use only when the user is distressed/help-seeking.
+- `crisis_only`: use only when triage detects crisis or emergency.
+- `metadata_only`: useful as source/contact metadata, not generation context.
+
+Examples:
+
+- UMD counseling overview: `retrieval`
+- 988 immediate crisis guidance: `crisis_only`
+- grounding exercise: `wellbeing_only`
+- phone-number-only contact page: `metadata_only`
+
+## 7. Chunking Rules
+
+Do not dump full webpages.
+
+Each chunk should be:
+
+- 80-250 words
+- focused on one useful idea
+- understandable without the full page
+- source-cited with URL
+- safe, factual, and student-appropriate
+
+Remove:
+
+- nav menus
+- footers
+- cookie banners
+- sidebars
+- repeated boilerplate
+- irrelevant event calendars
+- legal disclaimers with no support value
+- raw HTML artifacts
+
+If a useful page has 1,000 words, split it into 4-8 focused chunks.
+
+## 8. Safety Filtering
+
+Do not include content that:
+
+- describes self-harm methods in detail
+- gives instructions for suicide or self-harm
+- includes graphic personal crisis stories
+- makes diagnosis claims
+- claims to replace therapy, counseling, or emergency care
+- gives medication instructions
+- sounds judgmental, stigmatizing, or moralizing
+- is outdated, unofficial, or unsupported
+
+For crisis resources, include only safe action-oriented guidance:
+
+- call or text 988
+- contact emergency services
+- contact campus crisis or after-hours support
+- seek immediate support
+- stay with another person
+- ask someone nearby for help
+
+## 9. Recommended Corpus Size
+
+First usable version:
+
+```text
+Minimum: 80 chunks
+Good target: 150-250 chunks
+Excellent first pass: 300-500 chunks
+```
+
+Suggested distribution:
+
+```text
+UMD-specific resources: 40-80 chunks
+Crisis resources: 20-40 chunks
+Government/public health: 40-80 chunks
+Graduate/student academic support: 40-80 chunks
+Grounding/help-seeking snippets: 20-60 chunks
+```
+
+A clean 150-chunk corpus is better than a noisy 1,000-chunk scrape.
+
+## 10. Collection and Scraping Method
+
+Preferred workflow:
+
+1. Manually collect official source URLs.
+2. Record every reviewed URL in `source_inventory.csv`.
+3. Use simple scraping only for public pages.
+4. Save raw extracted text in `raw_pages/` when possible.
+5. Manually clean and chunk the useful text.
+6. Annotate every chunk with topic, audience, risk level, and usage mode.
+7. Validate JSONL before handoff.
+
+Recommended Python tools:
+
+```text
+requests
+beautifulsoup4
+trafilatura
+pandas
+```
+
+Basic scraping example:
+
+```python
+import requests
+from bs4 import BeautifulSoup
+
+url = "https://example.edu/page"
+html = requests.get(url, timeout=20).text
+soup = BeautifulSoup(html, "html.parser")
+
+for tag in soup(["script", "style", "nav", "footer", "header"]):
+    tag.decompose()
+
+text = soup.get_text("\n")
+lines = [line.strip() for line in text.splitlines() if line.strip()]
+clean_text = "\n".join(lines)
+```
+
+Do not trust scraper output directly. Every final chunk should be manually
+reviewed.
+
+Respect:
+
+- public pages only
+- no login-gated content
+- no private student information
+- no heavy request volume
+- robots/terms restrictions when applicable
+
+## 11. Manual Annotation Process
+
+For each chunk, answer:
+
+1. What student problem does this help with?
+2. Is this UMD-specific, general student support, or crisis support?
+3. Is it safe for normal retrieval?
+4. Should it appear only in wellbeing/crisis contexts?
+5. Does it contain actionable resource/navigation information?
+6. Is the URL official and useful?
+
+Then fill:
+
+```json
+"topic": "...",
+"audience": "...",
+"risk_level": "...",
+"usage_mode": "..."
+```
+
+LLMs may help draft labels, but a human should approve every final row.
+
+## 12. Compatibility With EmpathRAG
+
+EmpathRAG currently uses a FAISS index plus SQLite metadata.
+
+The current v1 SQLite metadata roughly contains:
+
+```text
+id
+text
+emotion_label
+safety_score
+source
+```
+
+For v2, we will build a separate curated index from `resources_seed.jsonl`.
+
+Field mapping:
+
+```text
+id -> metadata ID
+text -> embedded retrieval chunk
+source_name -> source display name
+url -> citation URL
+title -> citation title
+topic -> retrieval/evaluation filter
+risk_level -> safety filter
+usage_mode -> routing filter
+summary -> compact display/eval text
+```
+
+Do not build FAISS unless asked. Deliver clean structured data; integration will
+happen in EmpathRAG.
+
+## 13. Current Reddit Corpus Policy
+
+Do not mix Reddit into the curated corpus.
+
+Current Reddit corpus role:
+
+```text
+research baseline
+ablation comparison
+emotion/retrieval experiment
+not primary student-facing support source
+```
+
+New curated corpus role:
+
+```text
+safer student-support retrieval
+MSML class demo
+future UMD counseling stakeholder discussion
+publication-oriented system improvement
+```
+
+If you find useful Reddit-like examples, save them separately only if needed for
+future research evaluation. Do not include them in `resources_seed.jsonl`.
+
+## 14. Quality Checklist
+
+Before handoff, verify:
+
+- [ ] Every JSONL line is valid JSON.
+- [ ] Every row has a unique `id`.
+- [ ] Every row has a matching `source_id` in `source_inventory.csv`.
+- [ ] Every row has a working `url`.
+- [ ] Every row has `source_name`.
+- [ ] Every row has one allowed `source_type`.
+- [ ] Every row has one allowed `topic`.
+- [ ] Every row has one allowed `audience`.
+- [ ] Every row has one allowed `risk_level`.
+- [ ] Every row has one allowed `usage_mode`.
+- [ ] Text chunks are usually 80-250 words.
+- [ ] No nav/footer/cookie/sidebar junk remains.
+- [ ] No Reddit/social media/random blog content appears.
+- [ ] No graphic self-harm details appear.
+- [ ] No diagnosis/treatment/medication instructions appear.
+- [ ] Crisis resources are marked `crisis_resource` and `crisis_only`.
+- [ ] UMD-specific resources are prioritized.
+- [ ] `README_corpus_notes.md` is included.
+
+## 15. README Notes Template
+
+In `README_corpus_notes.md`, include:
+
+```text
+Corpus creator:
+Date:
+Total sources reviewed:
+Total chunks included:
+Total chunks excluded:
+Main source domains:
+Known limitations:
+Sources needing review:
+Pages that were hard to scrape:
+Content you were unsure about:
+Suggested next sources:
+```
+
+## 16. Good Example Row
+
+```json
+{
+  "id": "umd_counseling_services_001",
+  "source_id": "src_001",
+  "source_name": "UMD Counseling Center",
+  "source_type": "university_resource",
+  "title": "Counseling Services",
+  "url": "https://counseling.umd.edu/",
+  "topic": "counseling_services",
+  "audience": "umd_student",
+  "risk_level": "safe",
+  "usage_mode": "retrieval",
+  "text": "The University of Maryland Counseling Center provides support services for students dealing with emotional, academic, social, or personal concerns. Students can use counseling resources to talk through stress, relationship concerns, adjustment difficulties, anxiety, depression, and other challenges that may affect wellbeing or academic life. The Counseling Center can also help students understand what type of support may fit their situation and connect them with appropriate campus or community resources.",
+  "summary": "Overview of UMD Counseling Center support services for student wellbeing.",
+  "last_checked": "2026-04-27",
+  "notes": "Useful for routing students toward official campus counseling support."
+}
+```
+
+## 17. Bad Example Row
+
+Do not include:
+
+```json
+{
+  "text": "I saw someone on Reddit say they fixed depression by ignoring everyone and taking random supplements..."
+}
+```
+
+Why this is bad:
+
+- unofficial
+- anecdotal
+- potentially unsafe
+- not student-resource grounded
+- no source metadata
+- not appropriate for mental-health support retrieval
+
+## 18. Validation Commands
+
+If using Python, validate JSONL with:
+
+```python
+import json
+from pathlib import Path
+
+path = Path("resources_seed.jsonl")
+seen = set()
+
+for line_no, line in enumerate(path.read_text(encoding="utf-8").splitlines(), 1):
+    row = json.loads(line)
+    assert row["id"] not in seen, f"duplicate id on line {line_no}"
+    seen.add(row["id"])
+    for field in [
+        "id", "source_id", "source_name", "source_type", "title", "url",
+        "topic", "audience", "risk_level", "usage_mode", "text",
+        "summary", "last_checked", "notes"
+    ]:
+        assert row.get(field), f"missing {field} on line {line_no}"
+
+print(f"Valid rows: {len(seen)}")
+```
+
+## 19. Final Handoff Standard
+
+The handoff is ready when:
+
+- `resources_seed.jsonl` is valid JSONL.
+- `source_inventory.csv` explains every source.
+- UMD resources are clearly prioritized.
+- Crisis resources are separated by `risk_level` and `usage_mode`.
+- No unsafe or noisy web content is included.
+- The data can be ingested without further scraping.
+
+Once delivered, EmpathRAG can build:
+
+- curated FAISS index
+- curated SQLite metadata
+- source-cited retrieval
+- safer MSML class demo
+- stronger research direction for publication

eval/run_curated_retrieval_audit.py

@@ -0,0 +1,76 @@
+"""
+Audit curated-support retrieval without running the generator.
+
+Run after Karthik's corpus has been validated and indexed:
+    python eval/run_curated_retrieval_audit.py
+"""
+
+from __future__ import annotations
+
+import json
+import sys
+from pathlib import Path
+
+sys.path.insert(0, "src")
+
+from pipeline.pipeline import EmpathRAGPipeline
+from pipeline.query_router import route_query
+
+
+PROMPTS = [
+    "I'm so anxious about my thesis defense next week, I can't sleep.",
+    "My advisor rejected my work again and I don't know what to do.",
+    "I feel isolated in my program and I don't have anyone to talk to.",
+    "I think I might need counseling but I'm not sure where to start.",
+    "I'm burned out and falling behind on everything.",
+    "Can you help me find support for disability accommodations?",
+    "I don't know who to contact after hours if things get worse.",
+]
+
+RESULTS_PATH = Path("eval/curated_retrieval_audit.json")
+
+
+def main() -> int:
+    pipeline = EmpathRAGPipeline(
+        retrieval_corpus="curated_support",
+        use_real_guardrail=True,
+        guardrail_threshold=0.5,
+    )
+
+    rows = []
+    for prompt in PROMPTS:
+        emotion = pipeline._classify_emotion(prompt)
+        pipeline.tracker.update(emotion, len(prompt.split()))
+        trajectory = pipeline.tracker.trajectory()
+        routed = route_query(prompt, emotion, trajectory)
+        retrieved = pipeline._retrieve(routed, emotion)
+        sources = pipeline._source_summaries(retrieved)
+        rows.append(
+            {
+                "prompt": prompt,
+                "emotion": emotion,
+                "trajectory": trajectory,
+                "routed_query": routed,
+                "sources": sources,
+            }
+        )
+
+        print("\nPROMPT:", prompt)
+        if not sources:
+            print("  NO SOURCES")
+            continue
+        for i, source in enumerate(sources[:3], 1):
+            print(
+                f"  {i}. {source['source_name']} | {source['title']} | "
+                f"{source['topic']} | {source['risk_level']}"
+            )
+            if source["url"]:
+                print(f"     {source['url']}")
+
+    RESULTS_PATH.write_text(json.dumps(rows, indent=2), encoding="utf-8")
+    print(f"\nSaved audit: {RESULTS_PATH}")
+    return 0
+
+
+if __name__ == "__main__":
+    raise SystemExit(main())

src/data/build_curated_index.py

@@ -0,0 +1,148 @@
+"""
+Build a separate FAISS + SQLite index for the curated EmpathRAG corpus.
+
+Run from repo root:
+    python -m src.data.build_curated_index
+"""
+
+from __future__ import annotations
+
+import argparse
+import os
+import sqlite3
+from pathlib import Path
+
+import faiss
+import numpy as np
+from sentence_transformers import SentenceTransformer
+
+from .curated_resources import ingestion_rows, validate_file
+
+
+MODEL_NAME = "sentence-transformers/all-mpnet-base-v2"
+DEFAULT_INPUT = "data/curated/resources_seed.jsonl"
+DEFAULT_INDEX = "data/curated/indexes/faiss_curated.index"
+DEFAULT_DB = "data/curated/indexes/metadata_curated.db"
+
+
+def build_curated_index(
+    input_path: str = DEFAULT_INPUT,
+    index_path: str = DEFAULT_INDEX,
+    db_path: str = DEFAULT_DB,
+    model_name: str = MODEL_NAME,
+) -> None:
+    rows, _ = validate_file(input_path, strict=True)
+    usable = ingestion_rows(rows)
+    if not usable:
+        raise ValueError("No usable curated rows found after validation/filtering.")
+
+    texts = [row["text"] for row in usable]
+    print(f"Curated rows loaded: {len(rows)}")
+    print(f"Rows entering retrieval index: {len(usable)}")
+
+    encoder = SentenceTransformer(model_name)
+    embeddings = encoder.encode(
+        texts,
+        batch_size=64,
+        show_progress_bar=True,
+        normalize_embeddings=True,
+    )
+    embeddings = np.array(embeddings, dtype=np.float32)
+
+    dim = embeddings.shape[1]
+    index = faiss.IndexFlatL2(dim)
+    index.add(embeddings)
+
+    Path(index_path).parent.mkdir(parents=True, exist_ok=True)
+    faiss.write_index(index, index_path)
+
+    _write_metadata(db_path, usable)
+
+    print(f"Curated FAISS index saved: {index_path}")
+    print(f"Curated metadata DB saved: {db_path}")
+    print(f"Vectors indexed: {index.ntotal}")
+
+
+def _write_metadata(db_path: str, rows: list[dict]) -> None:
+    Path(db_path).parent.mkdir(parents=True, exist_ok=True)
+    if os.path.exists(db_path):
+        os.remove(db_path)
+
+    conn = sqlite3.connect(db_path)
+    c = conn.cursor()
+    c.execute(
+        """
+        CREATE TABLE chunks (
+            id INTEGER PRIMARY KEY,
+            resource_id TEXT UNIQUE NOT NULL,
+            text TEXT NOT NULL,
+            source_id TEXT NOT NULL,
+            source_name TEXT NOT NULL,
+            source_type TEXT NOT NULL,
+            title TEXT NOT NULL,
+            url TEXT NOT NULL,
+            topic TEXT NOT NULL,
+            audience TEXT NOT NULL,
+            risk_level TEXT NOT NULL,
+            usage_mode TEXT NOT NULL,
+            summary TEXT NOT NULL,
+            last_checked TEXT NOT NULL,
+            notes TEXT NOT NULL
+        )
+        """
+    )
+    c.execute("CREATE INDEX idx_chunks_topic ON chunks(topic)")
+    c.execute("CREATE INDEX idx_chunks_risk ON chunks(risk_level)")
+    c.execute("CREATE INDEX idx_chunks_usage ON chunks(usage_mode)")
+
+    for idx, row in enumerate(rows):
+        c.execute(
+            """
+            INSERT INTO chunks (
+                id, resource_id, text, source_id, source_name, source_type,
+                title, url, topic, audience, risk_level, usage_mode, summary,
+                last_checked, notes
+            ) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
+            """,
+            (
+                idx,
+                row["id"],
+                row["text"],
+                row["source_id"],
+                row["source_name"],
+                row["source_type"],
+                row["title"],
+                row["url"],
+                row["topic"],
+                row["audience"],
+                row["risk_level"],
+                row["usage_mode"],
+                row["summary"],
+                row["last_checked"],
+                row["notes"],
+            ),
+        )
+
+    conn.commit()
+    conn.close()
+
+
+def main() -> int:
+    parser = argparse.ArgumentParser(description="Build curated EmpathRAG FAISS index.")
+    parser.add_argument("--input", default=DEFAULT_INPUT)
+    parser.add_argument("--index", default=DEFAULT_INDEX)
+    parser.add_argument("--db", default=DEFAULT_DB)
+    parser.add_argument("--model", default=MODEL_NAME)
+    args = parser.parse_args()
+
+    build_curated_index(
+        input_path=args.input,
+        index_path=args.index,
+        db_path=args.db,
+        model_name=args.model,
+    )
+    return 0
+
+
+if __name__ == "__main__":
+    raise SystemExit(main())

src/data/curated_resources.py

@@ -0,0 +1,206 @@
+"""
+Utilities for EmpathRAG curated resource corpora.
+
+The curated corpus is a JSONL file prepared from official/student-support
+resources. It intentionally stays separate from the Reddit research corpus.
+"""
+
+from __future__ import annotations
+
+import argparse
+import json
+from dataclasses import dataclass
+from pathlib import Path
+from typing import Iterable
+
+
+REQUIRED_FIELDS = (
+    "id",
+    "source_id",
+    "source_name",
+    "source_type",
+    "title",
+    "url",
+    "topic",
+    "audience",
+    "risk_level",
+    "usage_mode",
+    "text",
+    "summary",
+    "last_checked",
+    "notes",
+)
+
+SOURCE_TYPES = {
+    "university_resource",
+    "crisis_resource",
+    "government_public_health",
+    "student_support",
+    "clinician_review_candidate",
+}
+
+TOPICS = {
+    "crisis_immediate_help",
+    "counseling_services",
+    "after_hours_support",
+    "academic_burnout",
+    "advisor_conflict",
+    "isolation_loneliness",
+    "anxiety_stress",
+    "depression_support",
+    "accessibility_disability",
+    "graduate_student_support",
+    "help_seeking_script",
+    "grounding_exercise",
+    "campus_navigation",
+    "therapy_expectations",
+    "peer_support",
+    "emergency_services",
+}
+
+AUDIENCES = {
+    "umd_student",
+    "graduate_student",
+    "student_general",
+    "crisis_support",
+    "supporter_or_friend",
+}
+
+RISK_LEVELS = {"safe", "wellbeing", "crisis_resource", "exclude"}
+USAGE_MODES = {"retrieval", "wellbeing_only", "crisis_only", "metadata_only"}
+
+
+@dataclass(frozen=True)
+class ValidationIssue:
+    line_no: int
+    row_id: str
+    message: str
+
+
+def load_jsonl(path: str | Path) -> list[dict]:
+    rows = []
+    path = Path(path)
+    for line_no, line in enumerate(path.read_text(encoding="utf-8").splitlines(), 1):
+        if not line.strip():
+            continue
+        try:
+            row = json.loads(line)
+        except json.JSONDecodeError as exc:
+            raise ValueError(f"Invalid JSON on line {line_no}: {exc}") from exc
+        if not isinstance(row, dict):
+            raise ValueError(f"Line {line_no} must be a JSON object.")
+        row["_line_no"] = line_no
+        rows.append(row)
+    return rows
+
+
+def validate_rows(rows: Iterable[dict]) -> list[ValidationIssue]:
+    issues: list[ValidationIssue] = []
+    seen_ids: set[str] = set()
+
+    for row in rows:
+        line_no = int(row.get("_line_no", 0))
+        row_id = str(row.get("id", "")).strip()
+
+        for field in REQUIRED_FIELDS:
+            if not str(row.get(field, "")).strip():
+                issues.append(ValidationIssue(line_no, row_id, f"missing field: {field}"))
+
+        if row_id in seen_ids:
+            issues.append(ValidationIssue(line_no, row_id, "duplicate id"))
+        if row_id:
+            seen_ids.add(row_id)
+
+        _check_allowed(issues, row, line_no, row_id, "source_type", SOURCE_TYPES)
+        _check_allowed(issues, row, line_no, row_id, "topic", TOPICS)
+        _check_allowed(issues, row, line_no, row_id, "audience", AUDIENCES)
+        _check_allowed(issues, row, line_no, row_id, "risk_level", RISK_LEVELS)
+        _check_allowed(issues, row, line_no, row_id, "usage_mode", USAGE_MODES)
+
+        text = str(row.get("text", "")).strip()
+        word_count = len(text.split())
+        if text and not (40 <= word_count <= 300):
+            issues.append(
+                ValidationIssue(
+                    line_no,
+                    row_id,
+                    f"text length {word_count} words outside review band 40-300",
+                )
+            )
+        if row.get("risk_level") == "exclude" and row.get("usage_mode") != "metadata_only":
+            issues.append(
+                ValidationIssue(
+                    line_no,
+                    row_id,
+                    "exclude rows must use usage_mode=metadata_only or be removed",
+                )
+            )
+
+    return issues
+
+
+def ingestion_rows(rows: Iterable[dict]) -> list[dict]:
+    """Rows safe to embed into the curated retrieval index."""
+    usable = []
+    for row in rows:
+        if row.get("risk_level") == "exclude":
+            continue
+        if row.get("usage_mode") == "metadata_only":
+            continue
+        usable.append({k: v for k, v in row.items() if not k.startswith("_")})
+    return usable
+
+
+def validate_file(path: str | Path, strict: bool = True) -> tuple[list[dict], list[ValidationIssue]]:
+    rows = load_jsonl(path)
+    issues = validate_rows(rows)
+    if strict and issues:
+        messages = "\n".join(
+            f"line {i.line_no} ({i.row_id or 'no id'}): {i.message}" for i in issues
+        )
+        raise ValueError(f"Curated corpus validation failed:\n{messages}")
+    return rows, issues
+
+
+def _check_allowed(
+    issues: list[ValidationIssue],
+    row: dict,
+    line_no: int,
+    row_id: str,
+    field: str,
+    allowed: set[str],
+) -> None:
+    value = row.get(field)
+    if value and value not in allowed:
+        issues.append(
+            ValidationIssue(
+                line_no,
+                row_id,
+                f"{field}={value!r} is not one of {sorted(allowed)}",
+            )
+        )
+
+
+def main() -> int:
+    parser = argparse.ArgumentParser(description="Validate EmpathRAG curated JSONL corpus.")
+    parser.add_argument("path", help="Path to resources_seed.jsonl")
+    parser.add_argument("--non-strict", action="store_true", help="Print issues but exit 0.")
+    args = parser.parse_args()
+
+    rows, issues = validate_file(args.path, strict=False)
+    usable = ingestion_rows(rows)
+    print(f"Rows: {len(rows)}")
+    print(f"Usable retrieval rows: {len(usable)}")
+
+    if issues:
+        print(f"Issues: {len(issues)}")
+        for issue in issues:
+            print(f"- line {issue.line_no} ({issue.row_id or 'no id'}): {issue.message}")
+        return 0 if args.non_strict else 1
+
+    print("Validation passed.")
+    return 0
+
+
+if __name__ == "__main__":
+    raise SystemExit(main())

src/pipeline/pipeline.py

@@ -20,6 +20,7 @@ VRAM sequencing on RTX 3060 6GB:
 import asyncio
 import sqlite3
 import time
+from pathlib import Path
 import torch
 import numpy as np
 import faiss
@@ -106,6 +107,9 @@ class EmpathRAGPipeline:
         guardrail_ckpt:  str = "models/safety_guardrail",
         faiss_index_path:str = "data/indexes/faiss_flat.index",
         db_path:         str = "data/indexes/metadata.db",
+        retrieval_corpus: str = "reddit_research",
+        curated_index_path: str = "data/curated/indexes/faiss_curated.index",
+        curated_db_path: str = "data/curated/indexes/metadata_curated.db",
         mistral_path:    str = "models/generator/mistral-7b-instruct-v0.2.Q4_K_M.gguf",
         st_model:        str = "sentence-transformers/all-mpnet-base-v2",
         n_gpu_layers:    int = 28,
@@ -118,7 +122,11 @@ class EmpathRAGPipeline:
     ):
         self.top_k               = top_k
         self.guardrail_threshold = guardrail_threshold
-        self.db_path             = db_path
+        self.retrieval_corpus    = self._resolve_retrieval_corpus(
+            retrieval_corpus, curated_index_path, curated_db_path
+        )
+        self.faiss_index_path    = curated_index_path if self.retrieval_corpus == "curated_support" else faiss_index_path
+        self.db_path             = curated_db_path if self.retrieval_corpus == "curated_support" else db_path
         self.safety_policy       = SafetyTriagePolicy(
             support_threshold=guardrail_threshold
         )
@@ -170,7 +178,8 @@ class EmpathRAGPipeline:
         # Start on CPU — we move to GPU only during encode(), then back
 
         print("[EmpathRAG] Loading FAISS index...")
-        self.faiss_index = faiss.read_index(faiss_index_path)
+        self.faiss_index = faiss.read_index(self.faiss_index_path)
+        print(f"[EmpathRAG] Retrieval corpus: {self.retrieval_corpus}")
         print(f"[EmpathRAG] FAISS: {self.faiss_index.ntotal:,} vectors")
 
         print("[EmpathRAG] Loading Mistral 7B (GPU)...")
@@ -187,6 +196,20 @@ class EmpathRAGPipeline:
         self.conv_history = []  # list of {"role": "user"|"assistant", "content": str}
         print("[EmpathRAG] Pipeline initialised. Ready for inference.")
 
+    def _resolve_retrieval_corpus(
+        self,
+        retrieval_corpus: str,
+        curated_index_path: str,
+        curated_db_path: str,
+    ) -> str:
+        allowed = {"reddit_research", "curated_support", "auto"}
+        if retrieval_corpus not in allowed:
+            raise ValueError(f"retrieval_corpus must be one of {sorted(allowed)}")
+        if retrieval_corpus == "auto":
+            curated_ready = Path(curated_index_path).exists() and Path(curated_db_path).exists()
+            return "curated_support" if curated_ready else "reddit_research"
+        return retrieval_corpus
+
     # ── Stage 1: Emotion classification ───────────────────────────────────────
 
     def _classify_emotion(self, text: str) -> int:
@@ -203,10 +226,10 @@ class EmpathRAGPipeline:
 
     # ── Stage 4: FAISS retrieval ───────────────────────────────────────────────
 
-    def _retrieve(self, query: str, emotion_label: int) -> list[str]:
+    def _retrieve(self, query: str, emotion_label: int) -> list[dict]:
         """
         Encodes query on GPU, searches FAISS, filters via SQLite.
-        Returns top_k chunk texts ranked by emotion match + safety score.
+        Returns top_k chunk metadata dicts.
         GPU usage: ~440 MB during encode, freed before returning.
         """
         # Move encoder to GPU for this call only
@@ -229,6 +252,9 @@ class EmpathRAGPipeline:
         if not candidate_ids:
             return []
 
+        if self.retrieval_corpus == "curated_support":
+            return self._fetch_curated_rows(candidate_ids)
+
         # Fetch metadata from SQLite
         placeholders = ",".join("?" * len(candidate_ids))
         conn = sqlite3.connect(self.db_path)
@@ -246,7 +272,64 @@ class EmpathRAGPipeline:
             return match_bonus + safety
 
         rows_sorted = sorted(rows, key=_score, reverse=True)[: self.top_k]
-        return [r[1] for r in rows_sorted]
+        return [
+            {
+                "id": r[0],
+                "text": r[1],
+                "emotion_label": r[2],
+                "safety_score": r[3],
+                "source_name": "Reddit Mental Health",
+                "source_type": "research_corpus",
+                "title": "Reddit Mental Health chunk",
+                "url": "",
+                "topic": "",
+                "risk_level": "research_only",
+                "usage_mode": "retrieval",
+            }
+            for r in rows_sorted
+        ]
+
+    def _fetch_curated_rows(self, candidate_ids: list[int]) -> list[dict]:
+        placeholders = ",".join("?" * len(candidate_ids))
+        conn = sqlite3.connect(self.db_path)
+        rows = conn.execute(
+            f"""
+            SELECT id, resource_id, text, source_id, source_name, source_type,
+                   title, url, topic, audience, risk_level, usage_mode, summary,
+                   last_checked, notes
+            FROM chunks
+            WHERE id IN ({placeholders})
+            """,
+            candidate_ids,
+        ).fetchall()
+        conn.close()
+
+        by_id = {row[0]: row for row in rows}
+        ordered = [by_id[i] for i in candidate_ids if i in by_id]
+        filtered = [
+            row for row in ordered
+            if row[10] != "exclude" and row[11] != "metadata_only"
+        ][: self.top_k]
+        return [
+            {
+                "id": row[0],
+                "resource_id": row[1],
+                "text": row[2],
+                "source_id": row[3],
+                "source_name": row[4],
+                "source_type": row[5],
+                "title": row[6],
+                "url": row[7],
+                "topic": row[8],
+                "audience": row[9],
+                "risk_level": row[10],
+                "usage_mode": row[11],
+                "summary": row[12],
+                "last_checked": row[13],
+                "notes": row[14],
+            }
+            for row in filtered
+        ]
 
     # ── Stage 5: Generation ────────────────────────────────────────────────────
 
@@ -362,6 +445,8 @@ class EmpathRAGPipeline:
                 "safety_reason":     safety_decision.reason,
                 "ig_highlights":     ig_highlights,
                 "retrieved_chunks":  [],
+                "retrieved_sources": [],
+                "retrieval_corpus":   self.retrieval_corpus,
                 "latency_ms":        latency,
             }
 
@@ -372,7 +457,8 @@ class EmpathRAGPipeline:
 
         # ── Stage 4: Retrieval ─────────────────────────────────────────────────
         t0 = time.perf_counter()
-        chunks = self._retrieve(routed_query, emotion_label)
+        retrieved = self._retrieve(routed_query, emotion_label)
+        chunks = [row["text"] for row in retrieved]
         latency["retrieval_ms"] = round((time.perf_counter() - t0) * 1000)
 
         # ── Stage 5: Generation ────────────────────────────────────────────────
@@ -398,9 +484,24 @@ class EmpathRAGPipeline:
             "safety_reason":     safety_decision.reason,
             "ig_highlights":     [],
             "retrieved_chunks":  chunks,
+            "retrieved_sources": self._source_summaries(retrieved),
+            "retrieval_corpus":  self.retrieval_corpus,
             "latency_ms":        latency,
         }
 
+    def _source_summaries(self, retrieved: list[dict]) -> list[dict]:
+        return [
+            {
+                "title": row.get("title", ""),
+                "source_name": row.get("source_name", ""),
+                "url": row.get("url", ""),
+                "topic": row.get("topic", ""),
+                "risk_level": row.get("risk_level", ""),
+                "usage_mode": row.get("usage_mode", ""),
+            }
+            for row in retrieved
+        ]
+
     def reset_session(self):
         """Clear session emotion history and conversation history."""
         self.tracker.reset()