Update

Author: mrshakil015

Democehk/__pycache__/langchain.cpython-39.pyc

@@ -0,0 +1,54 @@
+from dotenv import load_dotenv
+import streamlit as st
+from PyPDF2 import PdfReader
+from langchain.text_splitter import CharacterTextSplitter
+from langchain.embeddings.openai import OpenAIEmbeddings
+from langchain.vectorstores import FAISS
+from langchain.chains.question_answering import load_qa_chain
+from langchain.llms import OpenAI
+from langchain.callbacks import get_openai_callback
+
+
+def main():
+    load_dotenv()
+    st.set_page_config(page_title="Ask your PDF")
+    st.header("Ask your PDF 💬")
+    
+    # upload file
+    pdf = st.file_uploader("Upload your PDF", type="pdf")
+    
+    # extract the text
+    if pdf is not None:
+      pdf_reader = PdfReader(pdf)
+      text = ""
+      for page in pdf_reader.pages:
+        text += page.extract_text()
+        
+      # split into chunks
+      text_splitter = CharacterTextSplitter(
+        separator="\n",
+        chunk_size=1000,
+        chunk_overlap=200,
+        length_function=len
+      )
+      chunks = text_splitter.split_text(text)
+      
+      # create embeddings
+      embeddings = OpenAIEmbeddings()
+      knowledge_base = FAISS.from_texts(chunks, embeddings)
+      
+      # show user input
+      user_question = st.text_input("Ask a question about your PDF:")
+      if user_question:
+        docs = knowledge_base.similarity_search(user_question)
+        
+        llm = OpenAI()
+        chain = load_qa_chain(llm, chain_type="stuff")
+        with get_openai_callback() as cb:
+          response = chain.run(input_documents=docs, question=user_question)
+          print(cb)
+           
+        st.write(response)
+    
+if __name__ == '__main__':
+    main()

Democehk/langchain.py

@@ -0,0 +1,5 @@
+pip install google-cloud-aiplatform --upgrade
+pip install langchain --upgrade
+pip install bs4 docarray tiktoken streamlit
+pip install faiss-cpu
+pip install pypdf

Democehk/requirements.txt

@@ -0,0 +1,69 @@
+# from langchain.document_loaders import PyPDFLoader
+# from langchain.vectorstores import FAISS
+# from langchain.embeddings.openai import OpenAIEmbeddings
+
+# loader = PyPDFLoader("samplepdf.pdf")
+# pages = loader.load_and_split()
+
+# # Set your OpenAI API key here
+# openai_api_key = "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"
+
+# faiss_index = FAISS.from_documents(pages, OpenAIEmbeddings(openai_api_key=openai_api_key))
+# docs = faiss_index.similarity_search("How will the community be engaged?", k=2)
+# for doc in docs:
+#     print(str(doc.metadata["page"]) + ":", doc.page_content[:300])
+
+from langchain.text_splitter import CharacterTextSplitter
+from langchain.embeddings.openai import OpenAIEmbeddings
+from langchain.vectorstores import FAISS
+from langchain.chains.question_answering import load_qa_chain
+from langchain.llms import OpenAI
+from langchain.callbacks import get_openai_callback
+
+
+
+def main():
+    load_dotenv()
+    st.set_page_config(page_title="Ask your PDF")
+    st.header("Ask your PDF 💬")
+    
+    # upload file
+    pdf = st.file_uploader("Upload your PDF", type="pdf")
+    
+    # extract the text
+    if pdf is not None:
+      pdf_reader = PdfReader(pdf)
+      text = ""
+      for page in pdf_reader.pages:
+        text += page.extract_text()
+        
+      # split into chunks
+      text_splitter = CharacterTextSplitter(
+        separator="\n",
+        chunk_size=1000,
+        chunk_overlap=200,
+        length_function=len
+      )
+      chunks = text_splitter.split_text(text)
+      
+      # create embeddings
+      openai_api_key = "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"
+      embeddings = OpenAIEmbeddings(openai_api_key=openai_api_key)
+      knowledge_base = FAISS.from_texts(chunks, embeddings)
+      
+      # show user input
+      user_question = st.text_input("Ask a question about your PDF:")
+      if user_question:
+        docs = knowledge_base.similarity_search(user_question)
+        
+        llm = OpenAI()
+        chain = load_qa_chain(llm, chain_type="stuff")
+        with get_openai_callback() as cb:
+          response = chain.run(input_documents=docs, question=user_question)
+          print(cb)
+           
+        st.write(response)
+    
+
+if __name__ == '__main__':
+    main()

Langchain/pdf test.ipynb

@@ -62,7 +62,6 @@
 import os
 from pprint import pprint
 
-
 # this is a key file for a service account, which only has the role "Vertex AI User"
 os.environ['GOOGLE_APPLICATION_CREDENTIALS'] = 'my_credentials.json'