Implement a user assistant chatbot
Using LangChain and Q&A with RAG, develop a user assistant chatbot that leverages all the software documentation accessible within our Framagit project.
A first test version:
import os
from langchain.document_loaders import UnstructuredMarkdownLoader
from langchain.document_loaders import TextLoader
from langchain.text_splitter import RecursiveCharacterTextSplitter
from langchain.embeddings.sentence_transformer import SentenceTransformerEmbeddings
from langchain.vectorstores import Chroma
from langchain.chat_models import ChatOpenAI
from langchain import PromptTemplate, LLMChain
from langchain.chains import RetrievalQA, RetrievalQAWithSourcesChain
app_dir = ".../linkr"
# Readme.Md
loader = UnstructuredMarkdownLoader(app_dir + "/Readme.Md", encoding = "utf-8")
docs = loader.load()
# DESCRIPTION
loader = TextLoader(app_dir + "/DESCRIPTION", encoding = "utf-8")
docs.extend(loader.load())
# R help files
for dirpath, dirnames, filenames in os.walk(app_dir + "/R"):
for file in filenames:
if file.startswith("help_") and file.endswith(".R"):
loader = TextLoader(os.path.join(dirpath, file), encoding = "utf-8")
docs.extend(loader.load())
# Rd doc files
for dirpath, dirnames, filenames in os.walk(app_dir + "/man"):
for file in filenames:
if file.endswith(".Rd"):
loader = TextLoader(os.path.join(dirpath, file), encoding = "utf-8")
docs.extend(loader.load())
# R files
for dirpath, dirnames, filenames in os.walk(app_dir + "/R"):
for file in filenames:
if not file.startswith("help_") and file.endswith(".R"):
loader = TextLoader(os.path.join(dirpath, file), encoding = "utf-8")
docs.extend(loader.load())
text_splitter = RecursiveCharacterTextSplitter(chunk_size = 500, chunk_overlap = 0, length_function = len)
documents = text_splitter.split_documents(docs)
embedding_function = SentenceTransformerEmbeddings(model_name = "all-MiniLM-L6-v2")
vectorstore = Chroma.from_documents(documents, embedding_function)
llm = ChatOpenAI(model_name = "gpt-3.5-turbo", temperature = 0)
qa = RetrievalQA.from_chain_type(llm, retriever = vectorstore.as_retriever())
prompt = "What is LinkR ?"
qa({"query": prompt})
Edited by Boris Delange