pip install beautifulsoup4

# scrape.py

import os
import requests
import json

def get_response_and_save(url: str):
    response = requests.get(url)

    # create the scrape dir (if not found)
    if not os.path.exists("./scrape"):
        os.mkdir("./scrape")

    # save scraped content to a cleaned filename
    parsedUrl = cleanUrl(url)
    with open("./scrape/" + parsedUrl + ".html", "wb") as f:
        f.write(response.content)

    return response

def cleanUrl(url: str):
    return url.replace("https://", "").replace("/", "-").replace(".", "_")

# scrape.py

from urllib.parse import urlparse
from collections import defaultdict
from bs4 import BeautifulSoup

...

def scrape_links(
    scheme: str,
    origin: str,
    path: str,
    depth=3,
    sitemap: dict = defaultdict(lambda: ""),
):
    siteUrl = scheme + "://" + origin + path
    cleanedUrl = cleanUrl(siteUrl)

    if depth < 0:
        return
    if sitemap[cleanedUrl] != "":
        return

    sitemap[cleanedUrl] = siteUrl
    response = get_response_and_save(siteUrl)
    soup = BeautifulSoup(response.content, "html.parser")
    links = soup.find_all("a")

    for link in links:
        href = urlparse(link.get("href"))
        if (href.netloc != origin and href.netloc != "") or (
            href.scheme != "" and href.scheme != "https"
        ):
            # don't scrape external links
            continue

        scrape_links(
            href.scheme or "https",
            href.netloc or origin,
            href.path,
            depth=depth - 1,
            sitemap=sitemap,
        )

    return sitemap

# scrape.py

import argparse

parser = argparse.ArgumentParser()
parser.add_argument("--site", type=str, required=True)
parser.add_argument("--depth", type=int, default=3)

if __name__ == "__main__":
    args = parser.parse_args()
    url = urlparse(args.site)
    sitemap = scrape_links(url.scheme, url.netloc, url.path, depth=args.depth)
    with open("./scrape/sitemap.json", "w") as f:
        f.write(json.dumps(sitemap))

python scrape.py --site <your_site_url> --depth 3

python scrape.py \
  --site https://python.langchain.com/docs/get_started/introduction.html \
  --depth 10

pip install chroma langchain

pip install openai

# .env

OPENAI_API_KEY=<your_openai_api_key_here>

# embed.py

import os
import json

from langchain.document_loaders import (
    BSHTMLLoader,
    DirectoryLoader,
)
from langchain.embeddings.openai import OpenAIEmbeddings
from langchain.text_splitter import RecursiveCharacterTextSplitter
from langchain.vectorstores import Chroma

from dotenv import load_dotenv
load_dotenv()

loader = DirectoryLoader(
        "./scrape",
        glob="*.html",
        loader_cls=BSHTMLLoader,
        show_progress=True,
        loader_kwargs={"get_text_separator": " "},
)
data = loader.load()

text_splitter = RecursiveCharacterTextSplitter(
        chunk_size=1000,
        chunk_overlap=200,
)
documents = text_splitter.split_documents(data)

# map sources from file directory to web source
with open("./scrape/sitemap.json", "r") as f:
        sitemap = json.loads(f.read())
for document in documents:
        document.metadata["source"] = sitemap[
                document.metadata["source"].replace(".html", "").replace("scrape/", "")
        ]

embedding_model = OpenAIEmbeddings(model="text-embedding-ada-002")
db = Chroma.from_documents(
    documents, 
    embedding_model,
    persist_directory="./chroma"
)
db.persist()

python embed.py

# main.py
from dotenv import load_dotenv
load_dotenv()

from langchain.chat_models import ChatOpenAI

llm = ChatOpenAI(temperature=0, model="gpt-3.5-turbo")

# main.py

from langchain.chains import LLMChain

condense_question_prompt = """Given the following conversation and a follow up question, rephrase the follow up question to be a standalone question, in its original language.\
Make sure to avoid using any unclear pronouns.

Chat History:
{chat_history}
Follow Up Input: {question}
Standalone question:"""

condense_question_prompt = PromptTemplate.from_template(condense_question_prompt)

condense_question_chain = LLMChain(
    llm=llm,
    prompt=condense_question_prompt,
)

# main.py

from langchain.memory import ConversationBufferMemory

memory = ConversationBufferMemory(memory_key="chat_history", return_messages=True)

# example chat_history injected by theConversationBufferMemory object
chat_history = """
Human: Hi bot!
AI: Hi human! What can I assist you with today?
Human: What is the answer to life, the universe, and everything else?
AI: 42
...
"""

# main.py

from langchain.chains import create_qa_with_sources_chain

qa_chain = create_qa_with_sources_chain(llm)

# main.py

from langchain.prompts import PromptTemplate
from langchain.chains import create_qa_with_sources_chain
from langchain.chains.combine_documents.stuff import StuffDocumentsChain

qa_chain = create_qa_with_sources_chain(llm)

doc_prompt = PromptTemplate(
    template="Content: {page_content}\nSource: {source}",
    input_variables=["page_content", "source"],
)

final_qa_chain = StuffDocumentsChain(
    llm_chain=qa_chain,
    document_variable_name="context",
    document_prompt=doc_prompt,
)

# main.py

from langchain.chains import ConversationalRetrievalChain
from langchain.vectorstores import Chroma

db = Chroma(
    persist_directory="./chroma",
    embedding_function=OpenAIEmbeddings(model="text-embedding-ada-002"),
)

retrieval_qa = ConversationalRetrievalChain(
    question_generator=condense_question_chain,
    retriever=db.as_retriever(),
    memory=memory,
    combine_docs_chain=final_qa_chain,
)

response = retrieval_qa.run({question: 'How can I use LangChain with LLMs?'})
print(response)

# output:
"""
{
  "answer": "LangChain provides a standard interface for LLMs, which are language models that take a string as input and return a string as output. To use LangChain with LLMs, you need to understand the different types of language models and how to work with them. You can configure the LLM and/or the prompt used in LangChain applications to customize the output. Additionally, LangChain provides prompt management, prompt optimization, and common utilities for working with LLMs. By combining LLMs with other modules in LangChain, such as chains and agents, you can create more complex applications.",
  "sources": [
    "https://python.langchain.com/docs/get_started/quickstart", "https://python.langchain.com/docs/modules/data_connection/document_loaders/markdown"
  ]
}
"""

import json

response = retrieval_qa.run({question: 'How can I use LangChain with LLMs?'})
responseDict = json.loads(response)
answer = responseDict["answer"]
sources = responseDict["sources"]

print(answer)
> """LangChain provides a standard interface for LLMs, which are language models that take a string as input and return a string as output. To use LangChain with LLMs, you need to understand the different types of language models and how to work with them. You can configure the LLM and/or the prompt used in LangChain applications to customize the output. Additionally, LangChain provides prompt management, prompt optimization, and common utilities for working with LLMs. By combining LLMs with other modules in LangChain, such as chains and agents, you can create more complex applications."""

print(sources)
> [
      "https://python.langchain.com/docs/get_started/quickstart",
      ...
  ]

pip install gradio

# main.py

import gradio

def predict(message, history):
    response = retrieval_qa.run({"question": message})
    responseDict = json.loads(response)
    answer = responseDict["answer"]
    sources = responseDict["sources"]

    if type(sources) == list:
        sources = "\n".join(sources)

    if sources:
        return answer + "\n\nSee more:\n" + sources
    return answer


gradio.ChatInterface(predict).launch()

python main.py

python scrape.py \
  --site https://python.langchain.com/docs/get_started/quickstart \
  --depth 10

python embed.py

python main.py