02_document_splitting.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "id": "bafb515b",
   "metadata": {},
   "source": [
    "# Document Splitting"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "id": "1f9ad2a1-2c8e-471e-8174-338cf2e3e92d",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "import os\n",
    "import sys\n",
    "sys.path.append('../..')\n",
    "import openai\n",
    "\n",
    "from dotenv import load_dotenv, find_dotenv\n",
    "_ = load_dotenv(find_dotenv()) # read local .env file\n",
    "openai.api_key  = os.environ['OPENAI_API_KEY']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "06ea702c",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "from langchain.text_splitter import RecursiveCharacterTextSplitter, CharacterTextSplitter"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 49,
   "id": "2d587fb0",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "chunk_size =26\n",
    "chunk_overlap = 5"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 53,
   "id": "2723bf02",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "r_splitter = RecursiveCharacterTextSplitter(\n",
    "    chunk_size=chunk_size,\n",
    "    chunk_overlap=chunk_overlap\n",
    ")\n",
    "c_splitter = CharacterTextSplitter(\n",
    "    chunk_size=chunk_size,\n",
    "    chunk_overlap=chunk_overlap\n",
    ")"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "a61167fc",
   "metadata": {},
   "source": [
    "Why doesn't this split the string below?"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "id": "dd21401b",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "text1 = 'abcdefghijklmnopqrstuvwxyz'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "id": "fa92a860",
   "metadata": {
    "tags": []
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['abcdefghijklmnopqrstuvwxyz']"
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "r_splitter.split_text(text1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "id": "8b5da0f9",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "text2 = 'abcdefghijklmnopqrstuvwxyzabcdefg'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "id": "4ea0d456",
   "metadata": {
    "tags": []
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['abcdefghijklmnopqrstuvwxyz', 'wxyzabcdefg']"
      ]
     },
     "execution_count": 8,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "r_splitter.split_text(text2)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "87200022",
   "metadata": {},
   "source": [
    "Ok, this splits the string but we have an overlap specified as 5, but it looks like 3? (try an even number)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 54,
   "id": "dce39d64",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "text3 = \"a b c d e f g h i j k l m n o p q r s t u v w x y z\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 55,
   "id": "d96c0ab9",
   "metadata": {
    "tags": []
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['a b c d e f g h i j k l m', 'l m n o p q r s t u v w x', 'w x y z']"
      ]
     },
     "execution_count": 55,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "r_splitter.split_text(text3)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 56,
   "id": "0dbea42c",
   "metadata": {
    "tags": []
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['a b c d e f g h i j k l m n o p q r s t u v w x y z']"
      ]
     },
     "execution_count": 56,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "c_splitter.split_text(text3)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "id": "63e71644",
   "metadata": {
    "tags": []
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['a b c d e f g h i j k l m', 'l m n o p q r s t u v w x', 'w x y z']"
      ]
     },
     "execution_count": 12,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "c_splitter = CharacterTextSplitter(\n",
    "    chunk_size=chunk_size,\n",
    "    chunk_overlap=chunk_overlap,\n",
    "    separator = ' '\n",
    ")\n",
    "c_splitter.split_text(text3)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "8b613ee6-6d34-4504-b00d-bb0dbf414cfe",
   "metadata": {},
   "source": [
    "Try your own examples!"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "c1a4d0ed",
   "metadata": {},
   "source": [
    "## Recursive splitting details\n",
    "\n",
    "`RecursiveCharacterTextSplitter` is recommended for generic text. "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "id": "96926a42-92e6-4ad4-9946-908a075fb32d",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "some_text = \"\"\"When writing documents, writers will use document structure to group content. \\\n",
    "This can convey to the reader, which idea's are related. For example, closely related ideas \\\n",
    "are in sentances. Similar ideas are in paragraphs. Paragraphs form a document. \\n\\n  \\\n",
    "Paragraphs are often delimited with a carriage return or two carriage returns. \\\n",
    "Carriage returns are the \"backslash n\" you see embedded in this string. \\\n",
    "Sentences have a period at the end, but also, have a space.\\\n",
    "and words are separated by space.\"\"\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "id": "18706d73",
   "metadata": {
    "tags": []
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "496"
      ]
     },
     "execution_count": 14,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(some_text)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "id": "5083a172-ba0a-4d9f-b561-18f0d2e6d9ce",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "c_splitter = CharacterTextSplitter(\n",
    "    chunk_size=450,\n",
    "    chunk_overlap=0,\n",
    "    separator = ' '\n",
    ")\n",
    "r_splitter = RecursiveCharacterTextSplitter(\n",
    "    chunk_size=450,\n",
    "    chunk_overlap=0, \n",
    "    separators=[\"\\n\\n\", \"\\n\", \" \", \"\"]\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "id": "7f8aa4ed-5347-4819-b3c1-d15696832a65",
   "metadata": {
    "tags": []
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['When writing documents, writers will use document structure to group content. This can convey to the reader, which idea\\'s are related. For example, closely related ideas are in sentances. Similar ideas are in paragraphs. Paragraphs form a document. \\n\\n Paragraphs are often delimited with a carriage return or two carriage returns. Carriage returns are the \"backslash n\" you see embedded in this string. Sentences have a period at the end, but also,',\n",
       " 'have a space.and words are separated by space.']"
      ]
     },
     "execution_count": 16,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "c_splitter.split_text(some_text)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "id": "6af907f2-4110-4e40-a831-593f8592bd83",
   "metadata": {
    "tags": []
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[\"When writing documents, writers will use document structure to group content. This can convey to the reader, which idea's are related. For example, closely related ideas are in sentances. Similar ideas are in paragraphs. Paragraphs form a document.\",\n",
       " 'Paragraphs are often delimited with a carriage return or two carriage returns. Carriage returns are the \"backslash n\" you see embedded in this string. Sentences have a period at the end, but also, have a space.and words are separated by space.']"
      ]
     },
     "execution_count": 17,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "r_splitter.split_text(some_text)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "3b3720ef-2a97-4e0f-a2da-660b274b2f00",
   "metadata": {
    "tags": []
   },
   "source": [
    "Let's reduce the chunk size a bit and add a period to our separators:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "id": "1e5496ea-d34e-4ed9-8f68-9dadd86e8da7",
   "metadata": {
    "tags": []
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[\"When writing documents, writers will use document structure to group content. This can convey to the reader, which idea's are related. For example,\",\n",
       " 'closely related ideas are in sentances. Similar ideas are in paragraphs. Paragraphs form a document.',\n",
       " 'Paragraphs are often delimited with a carriage return or two carriage returns. Carriage returns are the \"backslash n\" you see embedded in this',\n",
       " 'string. Sentences have a period at the end, but also, have a space.and words are separated by space.']"
      ]
     },
     "execution_count": 18,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "r_splitter = RecursiveCharacterTextSplitter(\n",
    "    chunk_size=150,\n",
    "    chunk_overlap=0,\n",
    "    separators=[\"\\n\\n\", \"\\n\", \"\\. \", \" \", \"\"]\n",
    ")\n",
    "r_splitter.split_text(some_text)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "id": "31da7561",
   "metadata": {
    "tags": []
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[\"When writing documents, writers will use document structure to group content. This can convey to the reader, which idea's are related. For example,\",\n",
       " 'closely related ideas are in sentances. Similar ideas are in paragraphs. Paragraphs form a document.',\n",
       " 'Paragraphs are often delimited with a carriage return or two carriage returns. Carriage returns are the \"backslash n\" you see embedded in this',\n",
       " 'string. Sentences have a period at the end, but also, have a space.and words are separated by space.']"
      ]
     },
     "execution_count": 19,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "r_splitter = RecursiveCharacterTextSplitter(\n",
    "    chunk_size=150,\n",
    "    chunk_overlap=0,\n",
    "    separators=[\"\\n\\n\", \"\\n\", \"(?<=\\. )\", \" \", \"\"]\n",
    ")\n",
    "r_splitter.split_text(some_text)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "id": "a469e07d",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "from langchain.document_loaders import PyPDFLoader\n",
    "loader = PyPDFLoader(\"docs/MachineLearning-Lecture01.pdf\")\n",
    "pages = loader.load()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "id": "bba6f05d",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "from langchain.text_splitter import CharacterTextSplitter\n",
    "text_splitter = CharacterTextSplitter(\n",
    "    separator=\"\\n\",\n",
    "    chunk_size=1000,\n",
    "    chunk_overlap=150,\n",
    "    length_function=len\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "id": "c752a663-896b-4a24-9ffb-7640642b7a3f",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "docs = text_splitter.split_documents(pages)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "id": "7474a52c",
   "metadata": {
    "tags": []
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "77"
      ]
     },
     "execution_count": 23,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(docs)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "id": "d16046b2",
   "metadata": {
    "tags": []
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "22"
      ]
     },
     "execution_count": 24,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(pages)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "68f720db",
   "metadata": {},
   "source": [
    "## Token splitting\n",
    "\n",
    "We can also split on token count explicity, if we want.\n",
    "\n",
    "This can be useful because LLMs often have context windows designated in tokens.\n",
    "\n",
    "Tokens are often ~4 characters."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "id": "da0bcc05",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "from langchain.text_splitter import TokenTextSplitter\n",
    "# tiktoken is used under hode \n",
    "# tiktoken is a fast BPE tokeniser for use with OpenAI's models by default we use gpt2 (open source)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "id": "0116b01b-2b0b-47a7-a107-bbc497029713",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "text_splitter = TokenTextSplitter(chunk_size=1, chunk_overlap=0)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "id": "07a95e78",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "text1 = \"foo bar bazzyfoo\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "id": "8eec0912",
   "metadata": {
    "tags": []
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['foo', ' bar', ' b', 'az', 'zy', 'foo']"
      ]
     },
     "execution_count": 33,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "text_splitter.split_text(text1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 34,
   "id": "ffa29d43",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "text_splitter = TokenTextSplitter(chunk_size=10, chunk_overlap=0)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 35,
   "id": "e53e203a",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "docs = text_splitter.split_documents(pages)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 36,
   "id": "3b186c5f",
   "metadata": {
    "tags": []
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "Document(metadata={'source': 'docs/MachineLearning-Lecture01.pdf', 'page': 0}, page_content='MachineLearning-Lecture01  \\n')"
      ]
     },
     "execution_count": 36,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "docs[0]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 37,
   "id": "917f7abc",
   "metadata": {
    "tags": []
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'source': 'docs/MachineLearning-Lecture01.pdf', 'page': 0}"
      ]
     },
     "execution_count": 37,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "pages[0].metadata"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "282d9bfa",
   "metadata": {},
   "source": [
    "## Context aware splitting\n",
    "\n",
    "Chunking aims to keep text with common context together.\n",
    "\n",
    "A text splitting often uses sentences or other delimiters to keep related text together but many documents (such as Markdown) have structure (headers) that can be explicitly used in splitting.\n",
    "\n",
    "We can use `MarkdownHeaderTextSplitter` to preserve header metadata in our chunks, as show below."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 38,
   "id": "aa3b93d9",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "from langchain.document_loaders import NotionDirectoryLoader\n",
    "from langchain.text_splitter import MarkdownHeaderTextSplitter"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 39,
   "id": "d2c73a9c",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "markdown_document = \"\"\"# Title\\n\\n \\\n",
    "## Chapter 1\\n\\n \\\n",
    "Hi this is Jim\\n\\n Hi this is Joe\\n\\n \\\n",
    "### Section \\n\\n \\\n",
    "Hi this is Lance \\n\\n \n",
    "## Chapter 2\\n\\n \\\n",
    "Hi this is Molly\"\"\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 40,
   "id": "eadb7740",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "headers_to_split_on = [\n",
    "    (\"#\", \"Header 1\"),\n",
    "    (\"##\", \"Header 2\"),\n",
    "    (\"###\", \"Header 3\"),\n",
    "]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 41,
   "id": "a789eede",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "markdown_splitter = MarkdownHeaderTextSplitter(\n",
    "    headers_to_split_on=headers_to_split_on\n",
    ")\n",
    "md_header_splits = markdown_splitter.split_text(markdown_document)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 42,
   "id": "27053c17",
   "metadata": {
    "tags": []
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "Document(metadata={'Header 1': 'Title', 'Header 2': 'Chapter 1'}, page_content='Hi this is Jim  \\nHi this is Joe')"
      ]
     },
     "execution_count": 42,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "md_header_splits[0]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 43,
   "id": "a5b6b903",
   "metadata": {
    "tags": []
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "Document(metadata={'Header 1': 'Title', 'Header 2': 'Chapter 1', 'Header 3': 'Section'}, page_content='Hi this is Lance')"
      ]
     },
     "execution_count": 43,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "md_header_splits[1]"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "8d0c6111",
   "metadata": {},
   "source": [
    "Try on a real Markdown file, like a Notion database."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 44,
   "id": "b238b16f",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "loader = NotionDirectoryLoader(\"docs/Notion_DB\")\n",
    "docs = loader.load()\n",
    "txt = ' '.join([d.page_content for d in docs])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 45,
   "id": "9fdab1d4",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "headers_to_split_on = [\n",
    "    (\"#\", \"Header 1\"),\n",
    "    (\"##\", \"Header 2\"),\n",
    "]\n",
    "markdown_splitter = MarkdownHeaderTextSplitter(\n",
    "    headers_to_split_on=headers_to_split_on\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 46,
   "id": "e722c39c",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "md_header_splits = markdown_splitter.split_text(txt)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "3f2b6df5",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "md_header_splits[0]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "9f97c85a-78e8-4c63-bd45-ce585e26b63a",
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.11.5"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}