/** * Day-3 live gate (half 2): fabricated-citation rejection. * * Drives the LegalRetrievalPort verifier with two payloads against the same * real chunk: * - GENUINE quote (exact verbatim substring of chunk.text) → must be supported=true * - FABRICATED quote (plausible-sounding but NOT in chunk.text) → must be supported=false * * Asserts both, prints the legal_claim_verifications row, exits non-zero on * regression. Runs against the local Postgres (DATABASE_URL must be exported). */ import { getDb, schema } from "@courtmitra/db"; import { createOpenAiCompatibleEmbedder, createPgvectorFtsLegalRetrieval, } from "@courtmitra/adapters"; import { eq } from "drizzle-orm"; async function main() { const db = getDb(); const embedder = createOpenAiCompatibleEmbedder(); const retrieval = createPgvectorFtsLegalRetrieval(db, embedder); // Pick a real seeded chunk to anchor both tests. const [chunk] = await db .select() .from(schema.legalChunks) .where(eq(schema.legalChunks.sectionNo, "award")) .limit(1); if (!chunk) throw new Error("No 'award' chunk seeded — run pnpm db:seed first."); // Pick any mission to satisfy FK. const [mission] = await db.select().from(schema.missions).limit(1); if (!mission) throw new Error("No mission in DB."); // 1. GENUINE quote — verbatim substring of chunk.text. const genuineQuote = chunk.text.slice(60, 180).trim(); const genuine = await retrieval.verifyClaimSupport({ missionId: mission.id, claim: "Ombudsman can direct the regulated entity to pay an award (test-genuine).", quoteSpans: [genuineQuote], chunkIds: [chunk.id], }); // 2. FABRICATED quote — plausible legalese but NOT in chunk.text. const fabricatedQuote = "The Ombudsman shall award treble damages plus punitive interest at 24% per annum on every claim, payable within seventy-two hours."; const fabricated = await retrieval.verifyClaimSupport({ missionId: mission.id, claim: "Ombudsman awards treble damages plus 24% interest (test-fabricated).", quoteSpans: [fabricatedQuote], chunkIds: [chunk.id], }); console.log("\n=== DAY-3 FABRICATION GATE ==="); console.log("Genuine →", { supported: genuine.supported, supportLevel: genuine.supportLevel }); console.log("Fabricated →", { supported: fabricated.supported, supportLevel: fabricated.supportLevel, failureReason: fabricated.failureReason, }); const ok = genuine.supported === true && fabricated.supported === false; console.log(ok ? "\nPASS ✓ verifier rejects fabricated citation" : "\nFAIL ✗"); process.exit(ok ? 0 : 1); } main().catch((e) => { console.error(e); process.exit(1); });