bm25

langchain-streamlit-demo/app.py

@@ -18,6 +18,7 @@ from langchain.document_loaders import PyPDFLoader
 from langchain.embeddings import OpenAIEmbeddings
 from langchain.memory import ConversationBufferMemory, StreamlitChatMessageHistory
 from langchain.prompts import ChatPromptTemplate, MessagesPlaceholder
+from langchain.retrievers import BM25Retriever, EnsembleRetriever
 from langchain.schema.document import Document
 from langchain.schema.retriever import BaseRetriever
 from langchain.text_splitter import RecursiveCharacterTextSplitter
@@ -142,8 +143,19 @@ def get_texts_and_retriever(
         )
         texts = text_splitter.split_documents(documents)
         embeddings = OpenAIEmbeddings(openai_api_key=openai_api_key)
-        db = FAISS.from_documents(texts, embeddings)
-        return texts, db.as_retriever()
+
+        bm25_retriever = BM25Retriever.from_documents(texts)
+        bm25_retriever.k = 4
+
+        faiss_vectorstore = FAISS.from_documents(texts, embeddings)
+        faiss_retriever = faiss_vectorstore.as_retriever(search_kwargs={"k": 4})
+
+        ensemble_retriever = EnsembleRetriever(
+            retrievers=[bm25_retriever, faiss_retriever],
+            weights=[0.5, 0.5],
+        )
+
+        return texts, ensemble_retriever
 
 
 # --- Sidebar ---

langchain-streamlit-demo/qagen.py

@@ -7,19 +7,13 @@ from langchain.prompts.chat import (
 )
 from langchain.schema.language_model import BaseLanguageModel
 from langchain.schema.runnable import RunnableSequence
-from pydantic import BaseModel, field_validator, Field
+from pydantic import BaseModel, Field
 
 
 class QuestionAnswerPair(BaseModel):
     question: str = Field(..., description="The question that will be answered.")
     answer: str = Field(..., description="The answer to the question that was asked.")
 
-    @field_validator("question")
-    def validate_question(cls, v: str) -> str:
-        if not v.endswith("?"):
-            raise ValueError("Question must end with a question mark.")
-        return v
-
 
 class QuestionAnswerPairList(BaseModel):
     QuestionAnswerPairs: List[QuestionAnswerPair]

requirements.txt

@@ -5,7 +5,8 @@ langsmith==0.0.40
 numpy>=1.22.2 # not directly required, pinned by Snyk to avoid a vulnerability
 openai==0.28.0
 pypdf==3.16.2
-streamlit==1.27.0
+rank_bm25==0.2.2
+streamlit==1.27.1
 streamlit-feedback==0.1.2
 tiktoken==0.5.1
 tornado>=6.3.3 # not directly required, pinned by Snyk to avoid a vulnerability