/** * Qdrant seed script: populates the Qdrant collection with existing legal chunks. * Reads chunks from Postgres, embeds them, and upserts to Qdrant. * Usage: tsx scripts/seed-qdrant.ts */ import { readFileSync } from "node:fs"; import { resolve, dirname } from "node:path"; import { fileURLToPath } from "node:url"; import { sql } from "drizzle-orm"; import { getDb, closeDb, schema } from "@courtmitra/db"; import { createOpenAiCompatibleEmbedder } from "@courtmitra/adapters"; import { QdrantClient } from "@qdrant/qdrant-js"; const __filename = fileURLToPath(import.meta.url); const __dirname = dirname(__filename); const envPath = resolve(__dirname, "../infra/.env"); function loadEnv(path: string): void { try { const raw = readFileSync(path, "utf8"); for (const line of raw.split("\n")) { const trimmed = line.trim(); if (!trimmed || trimmed.startsWith("#")) continue; const eqIdx = trimmed.indexOf("="); if (eqIdx === -1) continue; const key = trimmed.slice(0, eqIdx).trim(); const val = trimmed.slice(eqIdx + 1).trim(); if (!process.env[key]) process.env[key] = val; } } catch { // env file not found } } loadEnv(envPath); async function main(): Promise { console.log("[seed-qdrant] Starting Qdrant seed..."); const db = getDb(); const embedder = createOpenAiCompatibleEmbedder(); const qdrant = new QdrantClient({ url: process.env["QDRANT_URL"] ?? "http://localhost:6333", apiKey: process.env["QDRANT_API_KEY"], }); const collectionName = process.env["QDRANT_COLLECTION"] ?? "courtmitra_legal"; // Ensure collection exists const collections = await qdrant.getCollections(); const exists = collections.collections?.some((c) => c.name === collectionName); if (!exists) { await qdrant.createCollection(collectionName, { vectors: { dense: { size: embedder.dimensions, distance: "Cosine", }, }, }); console.log(`[seed-qdrant] Created collection "${collectionName}".`); } else { console.log(`[seed-qdrant] Collection "${collectionName}" already exists.`); } // Load all chunks from Postgres with source info const rows = await db .select({ chunkId: schema.legalChunks.id, text: schema.legalChunks.text, actName: schema.legalChunks.actName, sectionNo: schema.legalChunks.sectionNo, heading: schema.legalChunks.heading, chunkNo: schema.legalChunks.chunkNo, sourceId: schema.legalChunks.sourceId, sourceTitle: schema.legalSources.title, tier: schema.legalSources.tier, sourceType: schema.legalSources.sourceType, jurisdiction: schema.legalSources.jurisdiction, metadata: schema.legalChunks.metadata, }) .from(schema.legalChunks) .innerJoin(schema.legalSources, sql`${schema.legalSources.id} = ${schema.legalChunks.sourceId}`); console.log(`[seed-qdrant] Loaded ${rows.length} chunks from Postgres.`); // Embed all chunk texts const texts = rows.map((r) => r.text); let embeddings: (number[] | null)[]; try { embeddings = await embedder.embed(texts); console.log("[seed-qdrant] Embeddings computed."); } catch (err) { console.error("[seed-qdrant] Embedding failed:", (err as Error).message); await closeDb(); process.exit(1); } // Upsert to Qdrant let upserted = 0; for (let i = 0; i < rows.length; i++) { const row = rows[i]!; const embedding = embeddings[i]; if (!embedding) continue; await qdrant.upsert(collectionName, { wait: true, points: [ { id: row.chunkId, vector: { dense: embedding, }, payload: { source_id: row.sourceId, source_title: row.sourceTitle, act_name: row.actName, section_no: row.sectionNo, heading: row.heading, chunk_no: row.chunkNo, text: row.text, tier: row.tier, source_type: row.sourceType, jurisdiction: row.jurisdiction, issue_domain: (row.metadata as Record)?.["issue_domain"] ?? null, }, }, ], }); upserted++; if (upserted % 10 === 0) { console.log(`[seed-qdrant] ${upserted}/${rows.length}`); } } console.log(`[seed-qdrant] Done. ${upserted} chunks upserted.`); await closeDb(); } main().catch((err) => { console.error("[seed-qdrant] FATAL:", err); process.exit(1); });