Upload 4 files

.gitattributes

@@ -33,3 +33,5 @@ saved_model/**/* filter=lfs diff=lfs merge=lfs -text
 *.zip filter=lfs diff=lfs merge=lfs -text
 *.zst filter=lfs diff=lfs merge=lfs -text
 *tfevents* filter=lfs diff=lfs merge=lfs -text
+data/Medical_book.pdf filter=lfs diff=lfs merge=lfs -text
+data/vector/index.faiss filter=lfs diff=lfs merge=lfs -text

data/Medical_book.pdf

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:753cd53b7a3020bbd91f05629b0e3ddcfb6a114d7bbedb22c2298b66f5dd00cc
+size 16127037

data/vector/index.faiss

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:eefa9ec7a37afdb420c2071d7ad6152d242a556e8a8dcfa4a88716714afc4ca3
+size 9001005

data/vector/index.pkl

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:746288fa8a336fe54359fd5ed7d43cc71fe78e463a389fc28f20e298dfc85082
+size 3283602

store_index.py

@@ -0,0 +1,48 @@
+from langchain_community.embeddings import HuggingFaceEmbeddings
+from langchain_community.vectorstores import FAISS
+from langchain_community.document_loaders import PyPDFLoader, DirectoryLoader
+from langchain.text_splitter import RecursiveCharacterTextSplitter
+import os
+from src.helper import load_pdf, text_split, download_hugging_face_embeddings
+DATA_PATH = r'G:\Chatbot\data'
+DB_FAISS_PATH = r'G:\Chatbot\data\vector'
+
+
+'''extracted_data = load_pdf(r"G:\Chatbot\data")
+text_chunks = text_split(extracted_data)
+embeddings = download_hugging_face_embeddings()
+# Initializing the Faiss
+db = FAISS.from_documents(text_chunks, embeddings)
+db.save_local(DB_FAISS_PATH)
+# I change the above DB_FAISS_PATH
+# db.save_local(r"G:\Chatbot\DB_FAISS_PATH")'''
+
+
+# Load the data from the PDF file
+def create_vector_db():
+    extracted_data = load_pdf(DATA_PATH)
+    text_chunks = text_split(extracted_data)
+    embeddings = download_hugging_face_embeddings()
+    db = FAISS.from_documents(text_chunks, embeddings)
+    db.save_local(DB_FAISS_PATH)
+    print("### db is created")
+
+
+'''# Create vector database
+def create_vector_db():
+    loader = DirectoryLoader(DATA_PATH, 
+                            glob='*.pdf', 
+                            loader_cls=PyPDFLoader)
+
+    documents = loader.load()
+    text_splitter = RecursiveCharacterTextSplitter(chunk_size=500, 
+                                                    chunk_overlap=50)
+    texts = text_splitter.split_documents(documents)
+
+    embeddings = HuggingFaceEmbeddings(model_name='sentence-transformers/all-MiniLM-L6-v2',
+                                        model_kwargs={'device': 'cuda'})
+
+    db = FAISS.from_documents(texts, embeddings)
+    db.save_local(DB_FAISS_PATH)
+
+create_vector_db()  # Call the function directly in the cell'''