Jakaria commited on
Commit
8d60d43
·
1 Parent(s): 6a6f5ec

Add Bangla model API

Browse files
Files changed (1) hide show
  1. ingestion.py +29 -0
ingestion.py CHANGED
@@ -0,0 +1,29 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ import os
2
+ from langchain_community.document_loaders import PyPDFLoader
3
+ from langchain.text_splitter import RecursiveCharacterTextSplitter
4
+ from langchain_community.vectorstores import FAISS
5
+ from langchain_huggingface import HuggingFaceEmbeddings
6
+ from config import VECTOR_DIR, EMBED_MODEL
7
+
8
+ def ingest_pdf(pdf_path: str) -> str:
9
+ """
10
+ Load PDF, split into chunks, embed, and store in FAISS.
11
+ Overwrites previous FAISS index.
12
+ """
13
+ if not os.path.exists(pdf_path):
14
+ return "PDF file not found."
15
+
16
+ # Load PDF
17
+ loader = PyPDFLoader(pdf_path)
18
+ docs = loader.load()
19
+
20
+ # Split text
21
+ splitter = RecursiveCharacterTextSplitter(chunk_size=1000, chunk_overlap=200)
22
+ chunks = splitter.split_documents(docs)
23
+
24
+ # Embeddings + FAISS
25
+ embeddings = HuggingFaceEmbeddings(model_name=EMBED_MODEL)
26
+ vectordb = FAISS.from_documents(chunks, embeddings)
27
+ vectordb.save_local(VECTOR_DIR)
28
+
29
+ return f"Indexed {len(chunks)} chunks successfully."