03-vector-search/eval/ground-truth-data.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "7ba3282a-6099-44b3-81c6-ba4b73d80e7b",
   "metadata": {},
   "outputs": [],
   "source": [
    "import requests \n",
    "\n",
    "docs_url = 'https://github.com/DataTalksClub/llm-zoomcamp/blob/main/01-intro/documents.json?raw=1'\n",
    "docs_response = requests.get(docs_url)\n",
    "documents_raw = docs_response.json()\n",
    "\n",
    "documents = []\n",
    "\n",
    "for course in documents_raw:\n",
    "    course_name = course['course']\n",
    "\n",
    "    for doc in course['documents']:\n",
    "        doc['course'] = course_name\n",
    "        documents.append(doc)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "id": "9880aca9-ae97-42f5-9aa5-37bb46448841",
   "metadata": {},
   "outputs": [],
   "source": [
    "import hashlib\n",
    "\n",
    "def generate_document_id(doc):\n",
    "    # combined = f\"{doc['course']}-{doc['question']}\"\n",
    "    combined = f\"{doc['course']}-{doc['question']}-{doc['text'][:10]}\"\n",
    "    hash_object = hashlib.md5(combined.encode())\n",
    "    hash_hex = hash_object.hexdigest()\n",
    "    document_id = hash_hex[:8]\n",
    "    return document_id"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "id": "66719b38-04f3-41a8-bdd8-f33f04fe9367",
   "metadata": {},
   "outputs": [],
   "source": [
    "for doc in documents:\n",
    "    doc['id'] = generate_document_id(doc)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "id": "d4672632-acc2-4c1c-96b4-d30f24598aa1",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'text': \"You don't need it. You're accepted. You can also just start learning and submitting homework without registering. It is not checked against any registered list. Registration is just to gauge interest before the start date.\",\n",
       " 'section': 'General course-related questions',\n",
       " 'question': 'Course - I have registered for the Data Engineering Bootcamp. When can I expect to receive the confirmation email?',\n",
       " 'course': 'data-engineering-zoomcamp',\n",
       " 'id': '0bbf41ec'}"
      ]
     },
     "execution_count": 23,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "documents[3]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "id": "470b703c-29d6-4ca8-a68e-4c461b3e7a90",
   "metadata": {},
   "outputs": [],
   "source": [
    "from collections import defaultdict"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "id": "410f08db-2302-4c50-926c-511037b46c90",
   "metadata": {},
   "outputs": [],
   "source": [
    "hashes = defaultdict(list)\n",
    "\n",
    "for doc in documents:\n",
    "    doc_id = doc['id']\n",
    "    hashes[doc_id].append(doc)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "id": "47c69c01-e952-4818-a307-94ea224ca423",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(947, 948)"
      ]
     },
     "execution_count": 25,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(hashes), len(documents)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "id": "c3e495f9-fdac-436f-88e9-44ae68844ee3",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "593f7569 2\n"
     ]
    }
   ],
   "source": [
    "for k, values in hashes.items():\n",
    "    if len(values) > 1:\n",
    "        print(k, len(values))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "id": "56bed2ec-9b14-4e7c-9db4-87900b30a674",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[{'text': \"They both do the same, it's just less typing from the script.\\nAsked by Andrew Katoch, Added by Edidiong Esu\",\n",
       "  'section': '6. Decision Trees and Ensemble Learning',\n",
       "  'question': 'Does it matter if we let the Python file create the server or if we run gunicorn directly?',\n",
       "  'course': 'machine-learning-zoomcamp',\n",
       "  'id': '593f7569'},\n",
       " {'text': \"They both do the same, it's just less typing from the script.\",\n",
       "  'section': '6. Decision Trees and Ensemble Learning',\n",
       "  'question': 'Does it matter if we let the Python file create the server or if we run gunicorn directly?',\n",
       "  'course': 'machine-learning-zoomcamp',\n",
       "  'id': '593f7569'}]"
      ]
     },
     "execution_count": 28,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "hashes['593f7569']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "id": "4d25e980-8a04-4186-94ef-c357bdc1b255",
   "metadata": {},
   "outputs": [],
   "source": [
    "import json"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "id": "686a488a-67af-4f21-8538-2180dc085fa2",
   "metadata": {},
   "outputs": [],
   "source": [
    "with open('documents-with-ids.json', 'wt') as f_out:\n",
    "    json.dump(documents, f_out, indent=2)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 34,
   "id": "c567515e-a923-487f-9b4d-9f4ce370e2ca",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[\n",
      "  {\n",
      "    \"text\": \"The purpose of this document is to capture frequently asked technical questions\\nThe exact day and hour of the course will be 15th Jan 2024 at 17h00. The course will start with the first  \\u201cOffice Hours'' live.1\\nSubscribe to course public Google Calendar (it works from Desktop only).\\nRegister before the course starts using this link.\\nJoin the course Telegram channel with announcements.\\nDon\\u2019t forget to register in DataTalks.Club's Slack and join the channel.\",\n",
      "    \"section\": \"General course-related questions\",\n",
      "    \"question\": \"Course - When will the course start?\",\n",
      "    \"course\": \"data-engineering-zoomcamp\",\n",
      "    \"id\": \"c02e79ef\"\n",
      "  },\n",
      "  {\n",
      "    \"text\": \"GitHub - DataTalksClub data-engineering-zoomcamp#prerequisites\",\n"
     ]
    }
   ],
   "source": [
    "!head documents-with-ids.json"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 35,
   "id": "1c062325-5608-4da6-80bf-b9ac371bc17f",
   "metadata": {},
   "outputs": [],
   "source": [
    "prompt_template = \"\"\"\n",
    "You emulate a student who's taking our course.\n",
    "Formulate 5 questions this student might ask based on a FAQ record. The record\n",
    "should contain the answer to the questions, and the questions should be complete and not too short.\n",
    "If possible, use as fewer words as possible from the record. \n",
    "\n",
    "The record:\n",
    "\n",
    "section: {section}\n",
    "question: {question}\n",
    "answer: {text}\n",
    "\n",
    "Provide the output in parsable JSON without using code blocks:\n",
    "\n",
    "[\"question1\", \"question2\", ..., \"question5\"]\n",
    "\"\"\".strip()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 38,
   "id": "51b8f259-eed8-4a50-b50c-2186fb154853",
   "metadata": {},
   "outputs": [],
   "source": [
    "from openai import OpenAI\n",
    "client = OpenAI()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 48,
   "id": "534ac1ae-b5e0-43a7-b8ad-103fd56ced54",
   "metadata": {},
   "outputs": [],
   "source": [
    "def generate_questions(doc):\n",
    "    prompt = prompt_template.format(**doc)\n",
    "\n",
    "    response = client.chat.completions.create(\n",
    "        model='gpt-4o',\n",
    "        messages=[{\"role\": \"user\", \"content\": prompt}]\n",
    "    )\n",
    "\n",
    "    json_response = response.choices[0].message.content\n",
    "    return json_response"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 43,
   "id": "eb2353f8-411b-4ab9-a4c2-0d158495491e",
   "metadata": {},
   "outputs": [],
   "source": [
    "from tqdm.auto import tqdm"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 49,
   "id": "25a48077-18c0-4d31-82ab-00a44d426279",
   "metadata": {},
   "outputs": [],
   "source": [
    "results = {}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "742f4b7c-7632-4475-b6df-b3c02d343287",
   "metadata": {},
   "outputs": [],
   "source": [
    "for doc in tqdm(documents): \n",
    "    doc_id = doc['id']\n",
    "    if doc_id in results:\n",
    "        continue\n",
    "\n",
    "    questions = generate_questions(doc)\n",
    "    results[doc_id] = questions"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 52,
   "id": "031ecc50-a9a3-4508-8149-cf716e045d20",
   "metadata": {},
   "outputs": [],
   "source": [
    "import pickle"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 53,
   "id": "de64d355-a4c8-4633-b179-952bb38923c5",
   "metadata": {},
   "outputs": [],
   "source": [
    "with open('results.bin', 'rb') as f_in:\n",
    "    results = pickle.load(f_in)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 55,
   "id": "2d5cd8f8-ddfd-4802-908b-504722511a04",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'[\"Where can I find the prerequisites for this course?\", \"How do I check the prerequisites for this course?\", \"Where are the course prerequisites listed?\", \"What are the requirements for joining this course?\", \"Where is the list of prerequisites for the course?\"]'"
      ]
     },
     "execution_count": 55,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "results['1f6520ca']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 65,
   "id": "d265559b-de13-45a3-ba4d-b5f84a3c070a",
   "metadata": {},
   "outputs": [],
   "source": [
    "parsed_resulst = {}\n",
    "\n",
    "for doc_id, json_questions in results.items():\n",
    "    parsed_resulst[doc_id] = json.loads(json_questions)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 67,
   "id": "6c928923-d3d8-4b8f-b093-887dffc8e6cb",
   "metadata": {},
   "outputs": [],
   "source": [
    "doc_index = {d['id']: d for d in documents}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 69,
   "id": "1ad018b8-33d7-4b80-85df-de3a115aa2b7",
   "metadata": {},
   "outputs": [],
   "source": [
    "final_results = []\n",
    "\n",
    "for doc_id, questions in parsed_resulst.items():\n",
    "    course = doc_index[doc_id]['course']\n",
    "    for q in questions:\n",
    "        final_results.append((q, course, doc_id))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 71,
   "id": "f7e44c9c-c383-4b7a-8c7a-404d9c2ec8d2",
   "metadata": {},
   "outputs": [],
   "source": [
    "import pandas as pd"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 75,
   "id": "c5adba2e-d628-47e5-a107-0ddad14fd667",
   "metadata": {},
   "outputs": [],
   "source": [
    "df = pd.DataFrame(final_results, columns=['question', 'course', 'document'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 77,
   "id": "9e4d5ba5-c59c-4e0f-9e79-8f4cd86bb5f0",
   "metadata": {},
   "outputs": [],
   "source": [
    "df.to_csv('ground-truth-data.csv', index=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 78,
   "id": "f88e2b0d-53ba-4766-9f5c-aa5d85eff47b",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "question,course,document\n",
      "When does the course begin?,data-engineering-zoomcamp,c02e79ef\n",
      "How can I get the course schedule?,data-engineering-zoomcamp,c02e79ef\n",
      "What is the link for course registration?,data-engineering-zoomcamp,c02e79ef\n",
      "How can I receive course announcements?,data-engineering-zoomcamp,c02e79ef\n",
      "Where do I join the Slack channel?,data-engineering-zoomcamp,c02e79ef\n",
      "Where can I find the prerequisites for this course?,data-engineering-zoomcamp,1f6520ca\n",
      "How do I check the prerequisites for this course?,data-engineering-zoomcamp,1f6520ca\n",
      "Where are the course prerequisites listed?,data-engineering-zoomcamp,1f6520ca\n",
      "What are the requirements for joining this course?,data-engineering-zoomcamp,1f6520ca\n"
     ]
    }
   ],
   "source": [
    "!head ground-truth-data.csv"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "46afefd7-2230-4a08-ae31-5600de189d6f",
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.12.3"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}