AgentQA.ipynb

{
 "cells": [
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Sign up for APIs\n",
    "\n",
    "- Add OPENAI_API_KEY to .env file from OpenAI\n",
    "- Add SERPAPI_API_KEY to .env file from https://serpapi.com/"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "True"
      ]
     },
     "execution_count": 1,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "from dotenv import load_dotenv\n",
    "load_dotenv() "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "WARNING:root:Failed to default session, using empty session: HTTPConnectionPool(host='localhost', port=8000): Max retries exceeded with url: /sessions (Caused by NewConnectionError('<urllib3.connection.HTTPConnection object at 0x1229960b0>: Failed to establish a new connection: [Errno 61] Connection refused'))\n"
     ]
    }
   ],
   "source": [
    "from langchain.utilities import ApifyWrapper\n",
    "from langchain.document_loaders.base import Document\n",
    "from langchain.indexes import VectorstoreIndexCreator\n",
    "from langchain.document_loaders import ApifyDatasetLoader\n",
    "from langchain.embeddings.openai import OpenAIEmbeddings\n",
    "from langchain.text_splitter import CharacterTextSplitter, RecursiveCharacterTextSplitter\n",
    "from langchain.vectorstores import Chroma\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "from langchain.agents import load_tools, Tool\n",
    "from langchain.agents import initialize_agent\n",
    "from langchain.agents import AgentType\n",
    "from langchain.llms import OpenAI\n",
    "from langchain.chat_models import ChatOpenAI"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "from langchain_visualizer.jupyter import visualize\n",
    "import asyncio\n",
    "import tqdm as notebook_tqdm"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "llm_openai_davinci_3 = OpenAI(temperature=0)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "\n",
    "llm_agent = llm_openai_davinci_3\n",
    "llm_eval = llm_openai_davinci_3"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Let's use an agent that can search the internet and do Maths in order to answer a simple question about our conference."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "question = \"When is the PyCon DE & PyData Berlin 2023 conference? How many days are between that date and today?\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "\n",
      "\u001b[1m> Entering new AgentExecutor chain...\u001b[0m\n",
      "\u001b[32;1m\u001b[1;3m I need to find the date of the conference and then calculate the number of days between that date and today.\n",
      "Action: Search\n",
      "Action Input: \"PyCon DE & PyData Berlin 2023\"\u001b[0m\u001b[36;1m\u001b[1;3mPyConDE & PyData Berlin 2023, Berlin Germany. Where Pythonistas in Germany can meet to learn about new and upcoming Python libraries, tools, software and ...\u001b[0m\u001b[32;1m\u001b[1;3m I need to find the exact date of the conference\n",
      "Action: Search\n",
      "Action Input: \"PyCon DE & PyData Berlin 2023 date\"\u001b[0m\u001b[36;1m\u001b[1;3mPyCon DE & PyData Berlin 2023 · From 17 April through 19 April , 2023 · Explore events · More events at Berlin, Germany.\u001b[0m\u001b[32;1m\u001b[1;3m I need to calculate the number of days between the conference date and today\n",
      "Action: Calculator\n",
      "Action Input: 17 April 2023 - today's date\u001b[0m"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "WARNING:root:Failed to persist run: HTTPConnectionPool(host='localhost', port=8000): Max retries exceeded with url: /chain-runs (Caused by NewConnectionError('<urllib3.connection.HTTPConnection object at 0x2a1802f80>: Failed to establish a new connection: [Errno 61] Connection refused'))\n"
     ]
    },
    {
     "ename": "ValueError",
     "evalue": "invalid syntax (<expr>, line 1). Please try again with a valid numerical expression",
     "output_type": "error",
     "traceback": [
      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[0;31mSyntaxError\u001b[0m                               Traceback (most recent call last)",
      "File \u001b[0;32m~/Documents/GitHub/langchain-agent-qa/myenv/lib/python3.10/site-packages/langchain/chains/llm_math/base.py:59\u001b[0m, in \u001b[0;36mLLMMathChain._evaluate_expression\u001b[0;34m(self, expression)\u001b[0m\n\u001b[1;32m     57\u001b[0m     local_dict \u001b[39m=\u001b[39m {\u001b[39m\"\u001b[39m\u001b[39mpi\u001b[39m\u001b[39m\"\u001b[39m: math\u001b[39m.\u001b[39mpi, \u001b[39m\"\u001b[39m\u001b[39me\u001b[39m\u001b[39m\"\u001b[39m: math\u001b[39m.\u001b[39me}\n\u001b[1;32m     58\u001b[0m     output \u001b[39m=\u001b[39m \u001b[39mstr\u001b[39m(\n\u001b[0;32m---> 59\u001b[0m         numexpr\u001b[39m.\u001b[39;49mevaluate(\n\u001b[1;32m     60\u001b[0m             expression\u001b[39m.\u001b[39;49mstrip(),\n\u001b[1;32m     61\u001b[0m             global_dict\u001b[39m=\u001b[39;49m{},  \u001b[39m# restrict access to globals\u001b[39;49;00m\n\u001b[1;32m     62\u001b[0m             local_dict\u001b[39m=\u001b[39;49mlocal_dict,  \u001b[39m# add common mathematical functions\u001b[39;49;00m\n\u001b[1;32m     63\u001b[0m         )\n\u001b[1;32m     64\u001b[0m     )\n\u001b[1;32m     65\u001b[0m \u001b[39mexcept\u001b[39;00m \u001b[39mException\u001b[39;00m \u001b[39mas\u001b[39;00m e:\n",
      "File \u001b[0;32m~/Documents/GitHub/langchain-agent-qa/myenv/lib/python3.10/site-packages/numexpr/necompiler.py:817\u001b[0m, in \u001b[0;36mevaluate\u001b[0;34m(ex, local_dict, global_dict, out, order, casting, **kwargs)\u001b[0m\n\u001b[1;32m    816\u001b[0m \u001b[39mif\u001b[39;00m expr_key \u001b[39mnot\u001b[39;00m \u001b[39min\u001b[39;00m _names_cache:\n\u001b[0;32m--> 817\u001b[0m     _names_cache[expr_key] \u001b[39m=\u001b[39m getExprNames(ex, context)\n\u001b[1;32m    818\u001b[0m names, ex_uses_vml \u001b[39m=\u001b[39m _names_cache[expr_key]\n",
      "File \u001b[0;32m~/Documents/GitHub/langchain-agent-qa/myenv/lib/python3.10/site-packages/numexpr/necompiler.py:704\u001b[0m, in \u001b[0;36mgetExprNames\u001b[0;34m(text, context)\u001b[0m\n\u001b[1;32m    703\u001b[0m \u001b[39mdef\u001b[39;00m \u001b[39mgetExprNames\u001b[39m(text, context):\n\u001b[0;32m--> 704\u001b[0m     ex \u001b[39m=\u001b[39m stringToExpression(text, {}, context)\n\u001b[1;32m    705\u001b[0m     ast \u001b[39m=\u001b[39m expressionToAST(ex)\n",
      "File \u001b[0;32m~/Documents/GitHub/langchain-agent-qa/myenv/lib/python3.10/site-packages/numexpr/necompiler.py:274\u001b[0m, in \u001b[0;36mstringToExpression\u001b[0;34m(s, types, context)\u001b[0m\n\u001b[1;32m    273\u001b[0m     flags \u001b[39m=\u001b[39m \u001b[39m0\u001b[39m\n\u001b[0;32m--> 274\u001b[0m c \u001b[39m=\u001b[39m \u001b[39mcompile\u001b[39;49m(s, \u001b[39m'\u001b[39;49m\u001b[39m<expr>\u001b[39;49m\u001b[39m'\u001b[39;49m, \u001b[39m'\u001b[39;49m\u001b[39meval\u001b[39;49m\u001b[39m'\u001b[39;49m, flags)\n\u001b[1;32m    275\u001b[0m \u001b[39m# make VariableNode's for the names\u001b[39;00m\n",
      "\u001b[0;31mSyntaxError\u001b[0m: invalid syntax (<expr>, line 1)",
      "\nDuring handling of the above exception, another exception occurred:\n",
      "\u001b[0;31mValueError\u001b[0m                                Traceback (most recent call last)",
      "Cell \u001b[0;32mIn[8], line 11\u001b[0m\n\u001b[1;32m      1\u001b[0m tools \u001b[39m=\u001b[39m load_tools([\n\u001b[1;32m      2\u001b[0m     \u001b[39m\"\u001b[39m\u001b[39mserpapi\u001b[39m\u001b[39m\"\u001b[39m,\n\u001b[1;32m      3\u001b[0m     \u001b[39m\"\u001b[39m\u001b[39mllm-math\u001b[39m\u001b[39m\"\u001b[39m],\n\u001b[1;32m      4\u001b[0m     llm\u001b[39m=\u001b[39mllm_agent)\n\u001b[1;32m      5\u001b[0m agent \u001b[39m=\u001b[39m initialize_agent(\n\u001b[1;32m      6\u001b[0m     tools,\n\u001b[1;32m      7\u001b[0m     llm_agent,\n\u001b[1;32m      8\u001b[0m     agent\u001b[39m=\u001b[39m\u001b[39m\"\u001b[39m\u001b[39mzero-shot-react-description\u001b[39m\u001b[39m\"\u001b[39m,\n\u001b[1;32m      9\u001b[0m       verbose\u001b[39m=\u001b[39m\u001b[39mTrue\u001b[39;00m)\n\u001b[0;32m---> 11\u001b[0m output \u001b[39m=\u001b[39m agent\u001b[39m.\u001b[39;49mrun(question)\n",
      "File \u001b[0;32m~/Documents/GitHub/langchain-agent-qa/myenv/lib/python3.10/site-packages/langchain/chains/base.py:213\u001b[0m, in \u001b[0;36mChain.run\u001b[0;34m(self, *args, **kwargs)\u001b[0m\n\u001b[1;32m    211\u001b[0m     \u001b[39mif\u001b[39;00m \u001b[39mlen\u001b[39m(args) \u001b[39m!=\u001b[39m \u001b[39m1\u001b[39m:\n\u001b[1;32m    212\u001b[0m         \u001b[39mraise\u001b[39;00m \u001b[39mValueError\u001b[39;00m(\u001b[39m\"\u001b[39m\u001b[39m`run` supports only one positional argument.\u001b[39m\u001b[39m\"\u001b[39m)\n\u001b[0;32m--> 213\u001b[0m     \u001b[39mreturn\u001b[39;00m \u001b[39mself\u001b[39;49m(args[\u001b[39m0\u001b[39;49m])[\u001b[39mself\u001b[39m\u001b[39m.\u001b[39moutput_keys[\u001b[39m0\u001b[39m]]\n\u001b[1;32m    215\u001b[0m \u001b[39mif\u001b[39;00m kwargs \u001b[39mand\u001b[39;00m \u001b[39mnot\u001b[39;00m args:\n\u001b[1;32m    216\u001b[0m     \u001b[39mreturn\u001b[39;00m \u001b[39mself\u001b[39m(kwargs)[\u001b[39mself\u001b[39m\u001b[39m.\u001b[39moutput_keys[\u001b[39m0\u001b[39m]]\n",
      "File \u001b[0;32m~/Documents/GitHub/langchain-agent-qa/myenv/lib/python3.10/site-packages/langchain_visualizer/hijacking.py:44\u001b[0m, in \u001b[0;36mget_overridden_call.<locals>.overridden_call\u001b[0;34m(og_self, *args, **kwargs)\u001b[0m\n\u001b[1;32m     38\u001b[0m \u001b[39mif\u001b[39;00m (\n\u001b[1;32m     39\u001b[0m     \u001b[39mnot\u001b[39;00m \u001b[39mhasattr\u001b[39m(og_self\u001b[39m.\u001b[39m\u001b[39m__class__\u001b[39m, \u001b[39m\"\u001b[39m\u001b[39m_should_trace\u001b[39m\u001b[39m\"\u001b[39m)\n\u001b[1;32m     40\u001b[0m     \u001b[39mor\u001b[39;00m og_self\u001b[39m.\u001b[39m\u001b[39m__class__\u001b[39m\u001b[39m.\u001b[39m_should_trace\n\u001b[1;32m     41\u001b[0m ):\n\u001b[1;32m     42\u001b[0m     \u001b[39m# ICE displays class name in visualization\u001b[39;00m\n\u001b[1;32m     43\u001b[0m     ice_agent\u001b[39m.\u001b[39m\u001b[39m__class__\u001b[39m\u001b[39m.\u001b[39m\u001b[39m__name__\u001b[39m \u001b[39m=\u001b[39m og_self\u001b[39m.\u001b[39m\u001b[39m__class__\u001b[39m\u001b[39m.\u001b[39m\u001b[39m__name__\u001b[39m\n\u001b[0;32m---> 44\u001b[0m     \u001b[39mreturn\u001b[39;00m asyncio\u001b[39m.\u001b[39;49mget_event_loop()\u001b[39m.\u001b[39;49mrun_until_complete(\n\u001b[1;32m     45\u001b[0m         ice_agent\u001b[39m.\u001b[39;49mrun(\u001b[39m*\u001b[39;49margs, \u001b[39m*\u001b[39;49m\u001b[39m*\u001b[39;49mkwargs)\n\u001b[1;32m     46\u001b[0m     )\n\u001b[1;32m     48\u001b[0m \u001b[39mreturn\u001b[39;00m ice_agent\u001b[39m.\u001b[39mog_fn(og_self, \u001b[39m*\u001b[39margs, \u001b[39m*\u001b[39m\u001b[39m*\u001b[39mkwargs)\n",
      "File \u001b[0;32m~/Documents/GitHub/langchain-agent-qa/myenv/lib/python3.10/site-packages/nest_asyncio.py:90\u001b[0m, in \u001b[0;36m_patch_loop.<locals>.run_until_complete\u001b[0;34m(self, future)\u001b[0m\n\u001b[1;32m     87\u001b[0m \u001b[39mif\u001b[39;00m \u001b[39mnot\u001b[39;00m f\u001b[39m.\u001b[39mdone():\n\u001b[1;32m     88\u001b[0m     \u001b[39mraise\u001b[39;00m \u001b[39mRuntimeError\u001b[39;00m(\n\u001b[1;32m     89\u001b[0m         \u001b[39m'\u001b[39m\u001b[39mEvent loop stopped before Future completed.\u001b[39m\u001b[39m'\u001b[39m)\n\u001b[0;32m---> 90\u001b[0m \u001b[39mreturn\u001b[39;00m f\u001b[39m.\u001b[39;49mresult()\n",
      "File \u001b[0;32m~/miniconda3/lib/python3.10/asyncio/futures.py:201\u001b[0m, in \u001b[0;36mFuture.result\u001b[0;34m(self)\u001b[0m\n\u001b[1;32m    199\u001b[0m \u001b[39mself\u001b[39m\u001b[39m.\u001b[39m__log_traceback \u001b[39m=\u001b[39m \u001b[39mFalse\u001b[39;00m\n\u001b[1;32m    200\u001b[0m \u001b[39mif\u001b[39;00m \u001b[39mself\u001b[39m\u001b[39m.\u001b[39m_exception \u001b[39mis\u001b[39;00m \u001b[39mnot\u001b[39;00m \u001b[39mNone\u001b[39;00m:\n\u001b[0;32m--> 201\u001b[0m     \u001b[39mraise\u001b[39;00m \u001b[39mself\u001b[39m\u001b[39m.\u001b[39m_exception\u001b[39m.\u001b[39mwith_traceback(\u001b[39mself\u001b[39m\u001b[39m.\u001b[39m_exception_tb)\n\u001b[1;32m    202\u001b[0m \u001b[39mreturn\u001b[39;00m \u001b[39mself\u001b[39m\u001b[39m.\u001b[39m_result\n",
      "File \u001b[0;32m~/miniconda3/lib/python3.10/asyncio/tasks.py:232\u001b[0m, in \u001b[0;36mTask.__step\u001b[0;34m(***failed resolving arguments***)\u001b[0m\n\u001b[1;32m    228\u001b[0m \u001b[39mtry\u001b[39;00m:\n\u001b[1;32m    229\u001b[0m     \u001b[39mif\u001b[39;00m exc \u001b[39mis\u001b[39;00m \u001b[39mNone\u001b[39;00m:\n\u001b[1;32m    230\u001b[0m         \u001b[39m# We use the `send` method directly, because coroutines\u001b[39;00m\n\u001b[1;32m    231\u001b[0m         \u001b[39m# don't have `__iter__` and `__next__` methods.\u001b[39;00m\n\u001b[0;32m--> 232\u001b[0m         result \u001b[39m=\u001b[39m coro\u001b[39m.\u001b[39;49msend(\u001b[39mNone\u001b[39;49;00m)\n\u001b[1;32m    233\u001b[0m     \u001b[39melse\u001b[39;00m:\n\u001b[1;32m    234\u001b[0m         result \u001b[39m=\u001b[39m coro\u001b[39m.\u001b[39mthrow(exc)\n",
      "File \u001b[0;32m~/Documents/GitHub/langchain-agent-qa/myenv/lib/python3.10/site-packages/ice/trace.py:231\u001b[0m, in \u001b[0;36mtrace.<locals>.wrapper\u001b[0;34m(*args, **kwargs)\u001b[0m\n\u001b[1;32m    228\u001b[0m \u001b[39m@wraps\u001b[39m(fn)\n\u001b[1;32m    229\u001b[0m \u001b[39masync\u001b[39;00m \u001b[39mdef\u001b[39;00m \u001b[39mwrapper\u001b[39m(\u001b[39m*\u001b[39margs, \u001b[39m*\u001b[39m\u001b[39m*\u001b[39mkwargs):\n\u001b[1;32m    230\u001b[0m     \u001b[39mif\u001b[39;00m \u001b[39mnot\u001b[39;00m trace_enabled():\n\u001b[0;32m--> 231\u001b[0m         \u001b[39mreturn\u001b[39;00m \u001b[39mawait\u001b[39;00m fn(\u001b[39m*\u001b[39margs, \u001b[39m*\u001b[39m\u001b[39m*\u001b[39mkwargs)\n\u001b[1;32m    233\u001b[0m     \u001b[39m@wraps\u001b[39m(fn)\n\u001b[1;32m    234\u001b[0m     \u001b[39masync\u001b[39;00m \u001b[39mdef\u001b[39;00m \u001b[39minner_wrapper\u001b[39m(\u001b[39m*\u001b[39margs, \u001b[39m*\u001b[39m\u001b[39m*\u001b[39mkwargs):\n\u001b[1;32m    235\u001b[0m         \u001b[39mid\u001b[39m \u001b[39m=\u001b[39m make_id()\n",
      "File \u001b[0;32m~/Documents/GitHub/langchain-agent-qa/myenv/lib/python3.10/site-packages/langchain_visualizer/hijacking.py:18\u001b[0m, in \u001b[0;36mVisualizationWrapper.run\u001b[0;34m(self, *args, **kwargs)\u001b[0m\n\u001b[1;32m     17\u001b[0m \u001b[39masync\u001b[39;00m \u001b[39mdef\u001b[39;00m \u001b[39mrun\u001b[39m(\u001b[39mself\u001b[39m, \u001b[39m*\u001b[39margs, \u001b[39m*\u001b[39m\u001b[39m*\u001b[39mkwargs):\n\u001b[0;32m---> 18\u001b[0m     \u001b[39mreturn\u001b[39;00m \u001b[39mself\u001b[39;49m\u001b[39m.\u001b[39;49mog_fn(\u001b[39mself\u001b[39;49m\u001b[39m.\u001b[39;49mog_obj, \u001b[39m*\u001b[39;49margs, \u001b[39m*\u001b[39;49m\u001b[39m*\u001b[39;49mkwargs)\n",
      "File \u001b[0;32m~/Documents/GitHub/langchain-agent-qa/myenv/lib/python3.10/site-packages/langchain/chains/base.py:116\u001b[0m, in \u001b[0;36mChain.__call__\u001b[0;34m(self, inputs, return_only_outputs)\u001b[0m\n\u001b[1;32m    114\u001b[0m \u001b[39mexcept\u001b[39;00m (\u001b[39mKeyboardInterrupt\u001b[39;00m, \u001b[39mException\u001b[39;00m) \u001b[39mas\u001b[39;00m e:\n\u001b[1;32m    115\u001b[0m     \u001b[39mself\u001b[39m\u001b[39m.\u001b[39mcallback_manager\u001b[39m.\u001b[39mon_chain_error(e, verbose\u001b[39m=\u001b[39m\u001b[39mself\u001b[39m\u001b[39m.\u001b[39mverbose)\n\u001b[0;32m--> 116\u001b[0m     \u001b[39mraise\u001b[39;00m e\n\u001b[1;32m    117\u001b[0m \u001b[39mself\u001b[39m\u001b[39m.\u001b[39mcallback_manager\u001b[39m.\u001b[39mon_chain_end(outputs, verbose\u001b[39m=\u001b[39m\u001b[39mself\u001b[39m\u001b[39m.\u001b[39mverbose)\n\u001b[1;32m    118\u001b[0m \u001b[39mreturn\u001b[39;00m \u001b[39mself\u001b[39m\u001b[39m.\u001b[39mprep_outputs(inputs, outputs, return_only_outputs)\n",
      "File \u001b[0;32m~/Documents/GitHub/langchain-agent-qa/myenv/lib/python3.10/site-packages/langchain/chains/base.py:113\u001b[0m, in \u001b[0;36mChain.__call__\u001b[0;34m(self, inputs, return_only_outputs)\u001b[0m\n\u001b[1;32m    107\u001b[0m \u001b[39mself\u001b[39m\u001b[39m.\u001b[39mcallback_manager\u001b[39m.\u001b[39mon_chain_start(\n\u001b[1;32m    108\u001b[0m     {\u001b[39m\"\u001b[39m\u001b[39mname\u001b[39m\u001b[39m\"\u001b[39m: \u001b[39mself\u001b[39m\u001b[39m.\u001b[39m\u001b[39m__class__\u001b[39m\u001b[39m.\u001b[39m\u001b[39m__name__\u001b[39m},\n\u001b[1;32m    109\u001b[0m     inputs,\n\u001b[1;32m    110\u001b[0m     verbose\u001b[39m=\u001b[39m\u001b[39mself\u001b[39m\u001b[39m.\u001b[39mverbose,\n\u001b[1;32m    111\u001b[0m )\n\u001b[1;32m    112\u001b[0m \u001b[39mtry\u001b[39;00m:\n\u001b[0;32m--> 113\u001b[0m     outputs \u001b[39m=\u001b[39m \u001b[39mself\u001b[39;49m\u001b[39m.\u001b[39;49m_call(inputs)\n\u001b[1;32m    114\u001b[0m \u001b[39mexcept\u001b[39;00m (\u001b[39mKeyboardInterrupt\u001b[39;00m, \u001b[39mException\u001b[39;00m) \u001b[39mas\u001b[39;00m e:\n\u001b[1;32m    115\u001b[0m     \u001b[39mself\u001b[39m\u001b[39m.\u001b[39mcallback_manager\u001b[39m.\u001b[39mon_chain_error(e, verbose\u001b[39m=\u001b[39m\u001b[39mself\u001b[39m\u001b[39m.\u001b[39mverbose)\n",
      "File \u001b[0;32m~/Documents/GitHub/langchain-agent-qa/myenv/lib/python3.10/site-packages/langchain/agents/agent.py:792\u001b[0m, in \u001b[0;36mAgentExecutor._call\u001b[0;34m(self, inputs)\u001b[0m\n\u001b[1;32m    790\u001b[0m \u001b[39m# We now enter the agent loop (until it returns something).\u001b[39;00m\n\u001b[1;32m    791\u001b[0m \u001b[39mwhile\u001b[39;00m \u001b[39mself\u001b[39m\u001b[39m.\u001b[39m_should_continue(iterations, time_elapsed):\n\u001b[0;32m--> 792\u001b[0m     next_step_output \u001b[39m=\u001b[39m \u001b[39mself\u001b[39;49m\u001b[39m.\u001b[39;49m_take_next_step(\n\u001b[1;32m    793\u001b[0m         name_to_tool_map, color_mapping, inputs, intermediate_steps\n\u001b[1;32m    794\u001b[0m     )\n\u001b[1;32m    795\u001b[0m     \u001b[39mif\u001b[39;00m \u001b[39misinstance\u001b[39m(next_step_output, AgentFinish):\n\u001b[1;32m    796\u001b[0m         \u001b[39mreturn\u001b[39;00m \u001b[39mself\u001b[39m\u001b[39m.\u001b[39m_return(next_step_output, intermediate_steps)\n",
      "File \u001b[0;32m~/Documents/GitHub/langchain-agent-qa/myenv/lib/python3.10/site-packages/langchain/agents/agent.py:695\u001b[0m, in \u001b[0;36mAgentExecutor._take_next_step\u001b[0;34m(self, name_to_tool_map, color_mapping, inputs, intermediate_steps)\u001b[0m\n\u001b[1;32m    693\u001b[0m         tool_run_kwargs[\u001b[39m\"\u001b[39m\u001b[39mllm_prefix\u001b[39m\u001b[39m\"\u001b[39m] \u001b[39m=\u001b[39m \u001b[39m\"\u001b[39m\u001b[39m\"\u001b[39m\n\u001b[1;32m    694\u001b[0m     \u001b[39m# We then call the tool on the tool input to get an observation\u001b[39;00m\n\u001b[0;32m--> 695\u001b[0m     observation \u001b[39m=\u001b[39m tool\u001b[39m.\u001b[39;49mrun(\n\u001b[1;32m    696\u001b[0m         agent_action\u001b[39m.\u001b[39;49mtool_input,\n\u001b[1;32m    697\u001b[0m         verbose\u001b[39m=\u001b[39;49m\u001b[39mself\u001b[39;49m\u001b[39m.\u001b[39;49mverbose,\n\u001b[1;32m    698\u001b[0m         color\u001b[39m=\u001b[39;49mcolor,\n\u001b[1;32m    699\u001b[0m         \u001b[39m*\u001b[39;49m\u001b[39m*\u001b[39;49mtool_run_kwargs,\n\u001b[1;32m    700\u001b[0m     )\n\u001b[1;32m    701\u001b[0m \u001b[39melse\u001b[39;00m:\n\u001b[1;32m    702\u001b[0m     tool_run_kwargs \u001b[39m=\u001b[39m \u001b[39mself\u001b[39m\u001b[39m.\u001b[39magent\u001b[39m.\u001b[39mtool_run_logging_kwargs()\n",
      "File \u001b[0;32m~/Documents/GitHub/langchain-agent-qa/myenv/lib/python3.10/site-packages/langchain/tools/base.py:163\u001b[0m, in \u001b[0;36mBaseTool.run\u001b[0;34m(self, tool_input, verbose, start_color, color, **kwargs)\u001b[0m\n\u001b[1;32m    161\u001b[0m \u001b[39mexcept\u001b[39;00m (\u001b[39mException\u001b[39;00m, \u001b[39mKeyboardInterrupt\u001b[39;00m) \u001b[39mas\u001b[39;00m e:\n\u001b[1;32m    162\u001b[0m     \u001b[39mself\u001b[39m\u001b[39m.\u001b[39mcallback_manager\u001b[39m.\u001b[39mon_tool_error(e, verbose\u001b[39m=\u001b[39mverbose_)\n\u001b[0;32m--> 163\u001b[0m     \u001b[39mraise\u001b[39;00m e\n\u001b[1;32m    164\u001b[0m \u001b[39mself\u001b[39m\u001b[39m.\u001b[39mcallback_manager\u001b[39m.\u001b[39mon_tool_end(\n\u001b[1;32m    165\u001b[0m     observation, verbose\u001b[39m=\u001b[39mverbose_, color\u001b[39m=\u001b[39mcolor, name\u001b[39m=\u001b[39m\u001b[39mself\u001b[39m\u001b[39m.\u001b[39mname, \u001b[39m*\u001b[39m\u001b[39m*\u001b[39mkwargs\n\u001b[1;32m    166\u001b[0m )\n\u001b[1;32m    167\u001b[0m \u001b[39mreturn\u001b[39;00m observation\n",
      "File \u001b[0;32m~/Documents/GitHub/langchain-agent-qa/myenv/lib/python3.10/site-packages/langchain/tools/base.py:160\u001b[0m, in \u001b[0;36mBaseTool.run\u001b[0;34m(self, tool_input, verbose, start_color, color, **kwargs)\u001b[0m\n\u001b[1;32m    158\u001b[0m \u001b[39mtry\u001b[39;00m:\n\u001b[1;32m    159\u001b[0m     args, kwargs \u001b[39m=\u001b[39m _to_args_and_kwargs(run_input)\n\u001b[0;32m--> 160\u001b[0m     observation \u001b[39m=\u001b[39m \u001b[39mself\u001b[39;49m\u001b[39m.\u001b[39;49m_run(\u001b[39m*\u001b[39;49margs, \u001b[39m*\u001b[39;49m\u001b[39m*\u001b[39;49mkwargs)\n\u001b[1;32m    161\u001b[0m \u001b[39mexcept\u001b[39;00m (\u001b[39mException\u001b[39;00m, \u001b[39mKeyboardInterrupt\u001b[39;00m) \u001b[39mas\u001b[39;00m e:\n\u001b[1;32m    162\u001b[0m     \u001b[39mself\u001b[39m\u001b[39m.\u001b[39mcallback_manager\u001b[39m.\u001b[39mon_tool_error(e, verbose\u001b[39m=\u001b[39mverbose_)\n",
      "File \u001b[0;32m~/Documents/GitHub/langchain-agent-qa/myenv/lib/python3.10/site-packages/langchain/agents/tools.py:30\u001b[0m, in \u001b[0;36mTool._run\u001b[0;34m(self, *args, **kwargs)\u001b[0m\n\u001b[1;32m     28\u001b[0m \u001b[39mdef\u001b[39;00m \u001b[39m_run\u001b[39m(\u001b[39mself\u001b[39m, \u001b[39m*\u001b[39margs: Any, \u001b[39m*\u001b[39m\u001b[39m*\u001b[39mkwargs: Any) \u001b[39m-\u001b[39m\u001b[39m>\u001b[39m \u001b[39mstr\u001b[39m:\n\u001b[1;32m     29\u001b[0m \u001b[39m    \u001b[39m\u001b[39m\"\"\"Use the tool.\"\"\"\u001b[39;00m\n\u001b[0;32m---> 30\u001b[0m     \u001b[39mreturn\u001b[39;00m \u001b[39mself\u001b[39;49m\u001b[39m.\u001b[39;49mfunc(\u001b[39m*\u001b[39;49margs, \u001b[39m*\u001b[39;49m\u001b[39m*\u001b[39;49mkwargs)\n",
      "File \u001b[0;32m~/Documents/GitHub/langchain-agent-qa/myenv/lib/python3.10/site-packages/langchain/chains/base.py:213\u001b[0m, in \u001b[0;36mChain.run\u001b[0;34m(self, *args, **kwargs)\u001b[0m\n\u001b[1;32m    211\u001b[0m     \u001b[39mif\u001b[39;00m \u001b[39mlen\u001b[39m(args) \u001b[39m!=\u001b[39m \u001b[39m1\u001b[39m:\n\u001b[1;32m    212\u001b[0m         \u001b[39mraise\u001b[39;00m \u001b[39mValueError\u001b[39;00m(\u001b[39m\"\u001b[39m\u001b[39m`run` supports only one positional argument.\u001b[39m\u001b[39m\"\u001b[39m)\n\u001b[0;32m--> 213\u001b[0m     \u001b[39mreturn\u001b[39;00m \u001b[39mself\u001b[39;49m(args[\u001b[39m0\u001b[39;49m])[\u001b[39mself\u001b[39m\u001b[39m.\u001b[39moutput_keys[\u001b[39m0\u001b[39m]]\n\u001b[1;32m    215\u001b[0m \u001b[39mif\u001b[39;00m kwargs \u001b[39mand\u001b[39;00m \u001b[39mnot\u001b[39;00m args:\n\u001b[1;32m    216\u001b[0m     \u001b[39mreturn\u001b[39;00m \u001b[39mself\u001b[39m(kwargs)[\u001b[39mself\u001b[39m\u001b[39m.\u001b[39moutput_keys[\u001b[39m0\u001b[39m]]\n",
      "File \u001b[0;32m~/Documents/GitHub/langchain-agent-qa/myenv/lib/python3.10/site-packages/langchain_visualizer/hijacking.py:44\u001b[0m, in \u001b[0;36mget_overridden_call.<locals>.overridden_call\u001b[0;34m(og_self, *args, **kwargs)\u001b[0m\n\u001b[1;32m     38\u001b[0m \u001b[39mif\u001b[39;00m (\n\u001b[1;32m     39\u001b[0m     \u001b[39mnot\u001b[39;00m \u001b[39mhasattr\u001b[39m(og_self\u001b[39m.\u001b[39m\u001b[39m__class__\u001b[39m, \u001b[39m\"\u001b[39m\u001b[39m_should_trace\u001b[39m\u001b[39m\"\u001b[39m)\n\u001b[1;32m     40\u001b[0m     \u001b[39mor\u001b[39;00m og_self\u001b[39m.\u001b[39m\u001b[39m__class__\u001b[39m\u001b[39m.\u001b[39m_should_trace\n\u001b[1;32m     41\u001b[0m ):\n\u001b[1;32m     42\u001b[0m     \u001b[39m# ICE displays class name in visualization\u001b[39;00m\n\u001b[1;32m     43\u001b[0m     ice_agent\u001b[39m.\u001b[39m\u001b[39m__class__\u001b[39m\u001b[39m.\u001b[39m\u001b[39m__name__\u001b[39m \u001b[39m=\u001b[39m og_self\u001b[39m.\u001b[39m\u001b[39m__class__\u001b[39m\u001b[39m.\u001b[39m\u001b[39m__name__\u001b[39m\n\u001b[0;32m---> 44\u001b[0m     \u001b[39mreturn\u001b[39;00m asyncio\u001b[39m.\u001b[39;49mget_event_loop()\u001b[39m.\u001b[39;49mrun_until_complete(\n\u001b[1;32m     45\u001b[0m         ice_agent\u001b[39m.\u001b[39;49mrun(\u001b[39m*\u001b[39;49margs, \u001b[39m*\u001b[39;49m\u001b[39m*\u001b[39;49mkwargs)\n\u001b[1;32m     46\u001b[0m     )\n\u001b[1;32m     48\u001b[0m \u001b[39mreturn\u001b[39;00m ice_agent\u001b[39m.\u001b[39mog_fn(og_self, \u001b[39m*\u001b[39margs, \u001b[39m*\u001b[39m\u001b[39m*\u001b[39mkwargs)\n",
      "File \u001b[0;32m~/Documents/GitHub/langchain-agent-qa/myenv/lib/python3.10/site-packages/nest_asyncio.py:90\u001b[0m, in \u001b[0;36m_patch_loop.<locals>.run_until_complete\u001b[0;34m(self, future)\u001b[0m\n\u001b[1;32m     87\u001b[0m \u001b[39mif\u001b[39;00m \u001b[39mnot\u001b[39;00m f\u001b[39m.\u001b[39mdone():\n\u001b[1;32m     88\u001b[0m     \u001b[39mraise\u001b[39;00m \u001b[39mRuntimeError\u001b[39;00m(\n\u001b[1;32m     89\u001b[0m         \u001b[39m'\u001b[39m\u001b[39mEvent loop stopped before Future completed.\u001b[39m\u001b[39m'\u001b[39m)\n\u001b[0;32m---> 90\u001b[0m \u001b[39mreturn\u001b[39;00m f\u001b[39m.\u001b[39;49mresult()\n",
      "File \u001b[0;32m~/miniconda3/lib/python3.10/asyncio/futures.py:201\u001b[0m, in \u001b[0;36mFuture.result\u001b[0;34m(self)\u001b[0m\n\u001b[1;32m    199\u001b[0m \u001b[39mself\u001b[39m\u001b[39m.\u001b[39m__log_traceback \u001b[39m=\u001b[39m \u001b[39mFalse\u001b[39;00m\n\u001b[1;32m    200\u001b[0m \u001b[39mif\u001b[39;00m \u001b[39mself\u001b[39m\u001b[39m.\u001b[39m_exception \u001b[39mis\u001b[39;00m \u001b[39mnot\u001b[39;00m \u001b[39mNone\u001b[39;00m:\n\u001b[0;32m--> 201\u001b[0m     \u001b[39mraise\u001b[39;00m \u001b[39mself\u001b[39m\u001b[39m.\u001b[39m_exception\u001b[39m.\u001b[39mwith_traceback(\u001b[39mself\u001b[39m\u001b[39m.\u001b[39m_exception_tb)\n\u001b[1;32m    202\u001b[0m \u001b[39mreturn\u001b[39;00m \u001b[39mself\u001b[39m\u001b[39m.\u001b[39m_result\n",
      "File \u001b[0;32m~/miniconda3/lib/python3.10/asyncio/tasks.py:232\u001b[0m, in \u001b[0;36mTask.__step\u001b[0;34m(***failed resolving arguments***)\u001b[0m\n\u001b[1;32m    228\u001b[0m \u001b[39mtry\u001b[39;00m:\n\u001b[1;32m    229\u001b[0m     \u001b[39mif\u001b[39;00m exc \u001b[39mis\u001b[39;00m \u001b[39mNone\u001b[39;00m:\n\u001b[1;32m    230\u001b[0m         \u001b[39m# We use the `send` method directly, because coroutines\u001b[39;00m\n\u001b[1;32m    231\u001b[0m         \u001b[39m# don't have `__iter__` and `__next__` methods.\u001b[39;00m\n\u001b[0;32m--> 232\u001b[0m         result \u001b[39m=\u001b[39m coro\u001b[39m.\u001b[39;49msend(\u001b[39mNone\u001b[39;49;00m)\n\u001b[1;32m    233\u001b[0m     \u001b[39melse\u001b[39;00m:\n\u001b[1;32m    234\u001b[0m         result \u001b[39m=\u001b[39m coro\u001b[39m.\u001b[39mthrow(exc)\n",
      "File \u001b[0;32m~/Documents/GitHub/langchain-agent-qa/myenv/lib/python3.10/site-packages/ice/trace.py:231\u001b[0m, in \u001b[0;36mtrace.<locals>.wrapper\u001b[0;34m(*args, **kwargs)\u001b[0m\n\u001b[1;32m    228\u001b[0m \u001b[39m@wraps\u001b[39m(fn)\n\u001b[1;32m    229\u001b[0m \u001b[39masync\u001b[39;00m \u001b[39mdef\u001b[39;00m \u001b[39mwrapper\u001b[39m(\u001b[39m*\u001b[39margs, \u001b[39m*\u001b[39m\u001b[39m*\u001b[39mkwargs):\n\u001b[1;32m    230\u001b[0m     \u001b[39mif\u001b[39;00m \u001b[39mnot\u001b[39;00m trace_enabled():\n\u001b[0;32m--> 231\u001b[0m         \u001b[39mreturn\u001b[39;00m \u001b[39mawait\u001b[39;00m fn(\u001b[39m*\u001b[39margs, \u001b[39m*\u001b[39m\u001b[39m*\u001b[39mkwargs)\n\u001b[1;32m    233\u001b[0m     \u001b[39m@wraps\u001b[39m(fn)\n\u001b[1;32m    234\u001b[0m     \u001b[39masync\u001b[39;00m \u001b[39mdef\u001b[39;00m \u001b[39minner_wrapper\u001b[39m(\u001b[39m*\u001b[39margs, \u001b[39m*\u001b[39m\u001b[39m*\u001b[39mkwargs):\n\u001b[1;32m    235\u001b[0m         \u001b[39mid\u001b[39m \u001b[39m=\u001b[39m make_id()\n",
      "File \u001b[0;32m~/Documents/GitHub/langchain-agent-qa/myenv/lib/python3.10/site-packages/langchain_visualizer/hijacking.py:18\u001b[0m, in \u001b[0;36mVisualizationWrapper.run\u001b[0;34m(self, *args, **kwargs)\u001b[0m\n\u001b[1;32m     17\u001b[0m \u001b[39masync\u001b[39;00m \u001b[39mdef\u001b[39;00m \u001b[39mrun\u001b[39m(\u001b[39mself\u001b[39m, \u001b[39m*\u001b[39margs, \u001b[39m*\u001b[39m\u001b[39m*\u001b[39mkwargs):\n\u001b[0;32m---> 18\u001b[0m     \u001b[39mreturn\u001b[39;00m \u001b[39mself\u001b[39;49m\u001b[39m.\u001b[39;49mog_fn(\u001b[39mself\u001b[39;49m\u001b[39m.\u001b[39;49mog_obj, \u001b[39m*\u001b[39;49margs, \u001b[39m*\u001b[39;49m\u001b[39m*\u001b[39;49mkwargs)\n",
      "File \u001b[0;32m~/Documents/GitHub/langchain-agent-qa/myenv/lib/python3.10/site-packages/langchain/chains/base.py:116\u001b[0m, in \u001b[0;36mChain.__call__\u001b[0;34m(self, inputs, return_only_outputs)\u001b[0m\n\u001b[1;32m    114\u001b[0m \u001b[39mexcept\u001b[39;00m (\u001b[39mKeyboardInterrupt\u001b[39;00m, \u001b[39mException\u001b[39;00m) \u001b[39mas\u001b[39;00m e:\n\u001b[1;32m    115\u001b[0m     \u001b[39mself\u001b[39m\u001b[39m.\u001b[39mcallback_manager\u001b[39m.\u001b[39mon_chain_error(e, verbose\u001b[39m=\u001b[39m\u001b[39mself\u001b[39m\u001b[39m.\u001b[39mverbose)\n\u001b[0;32m--> 116\u001b[0m     \u001b[39mraise\u001b[39;00m e\n\u001b[1;32m    117\u001b[0m \u001b[39mself\u001b[39m\u001b[39m.\u001b[39mcallback_manager\u001b[39m.\u001b[39mon_chain_end(outputs, verbose\u001b[39m=\u001b[39m\u001b[39mself\u001b[39m\u001b[39m.\u001b[39mverbose)\n\u001b[1;32m    118\u001b[0m \u001b[39mreturn\u001b[39;00m \u001b[39mself\u001b[39m\u001b[39m.\u001b[39mprep_outputs(inputs, outputs, return_only_outputs)\n",
      "File \u001b[0;32m~/Documents/GitHub/langchain-agent-qa/myenv/lib/python3.10/site-packages/langchain/chains/base.py:113\u001b[0m, in \u001b[0;36mChain.__call__\u001b[0;34m(self, inputs, return_only_outputs)\u001b[0m\n\u001b[1;32m    107\u001b[0m \u001b[39mself\u001b[39m\u001b[39m.\u001b[39mcallback_manager\u001b[39m.\u001b[39mon_chain_start(\n\u001b[1;32m    108\u001b[0m     {\u001b[39m\"\u001b[39m\u001b[39mname\u001b[39m\u001b[39m\"\u001b[39m: \u001b[39mself\u001b[39m\u001b[39m.\u001b[39m\u001b[39m__class__\u001b[39m\u001b[39m.\u001b[39m\u001b[39m__name__\u001b[39m},\n\u001b[1;32m    109\u001b[0m     inputs,\n\u001b[1;32m    110\u001b[0m     verbose\u001b[39m=\u001b[39m\u001b[39mself\u001b[39m\u001b[39m.\u001b[39mverbose,\n\u001b[1;32m    111\u001b[0m )\n\u001b[1;32m    112\u001b[0m \u001b[39mtry\u001b[39;00m:\n\u001b[0;32m--> 113\u001b[0m     outputs \u001b[39m=\u001b[39m \u001b[39mself\u001b[39;49m\u001b[39m.\u001b[39;49m_call(inputs)\n\u001b[1;32m    114\u001b[0m \u001b[39mexcept\u001b[39;00m (\u001b[39mKeyboardInterrupt\u001b[39;00m, \u001b[39mException\u001b[39;00m) \u001b[39mas\u001b[39;00m e:\n\u001b[1;32m    115\u001b[0m     \u001b[39mself\u001b[39m\u001b[39m.\u001b[39mcallback_manager\u001b[39m.\u001b[39mon_chain_error(e, verbose\u001b[39m=\u001b[39m\u001b[39mself\u001b[39m\u001b[39m.\u001b[39mverbose)\n",
      "File \u001b[0;32m~/Documents/GitHub/langchain-agent-qa/myenv/lib/python3.10/site-packages/langchain/chains/llm_math/base.py:130\u001b[0m, in \u001b[0;36mLLMMathChain._call\u001b[0;34m(self, inputs)\u001b[0m\n\u001b[1;32m    126\u001b[0m \u001b[39mself\u001b[39m\u001b[39m.\u001b[39mcallback_manager\u001b[39m.\u001b[39mon_text(inputs[\u001b[39mself\u001b[39m\u001b[39m.\u001b[39minput_key], verbose\u001b[39m=\u001b[39m\u001b[39mself\u001b[39m\u001b[39m.\u001b[39mverbose)\n\u001b[1;32m    127\u001b[0m llm_output \u001b[39m=\u001b[39m llm_executor\u001b[39m.\u001b[39mpredict(\n\u001b[1;32m    128\u001b[0m     question\u001b[39m=\u001b[39minputs[\u001b[39mself\u001b[39m\u001b[39m.\u001b[39minput_key], stop\u001b[39m=\u001b[39m[\u001b[39m\"\u001b[39m\u001b[39m```output\u001b[39m\u001b[39m\"\u001b[39m]\n\u001b[1;32m    129\u001b[0m )\n\u001b[0;32m--> 130\u001b[0m \u001b[39mreturn\u001b[39;00m \u001b[39mself\u001b[39;49m\u001b[39m.\u001b[39;49m_process_llm_result(llm_output)\n",
      "File \u001b[0;32m~/Documents/GitHub/langchain-agent-qa/myenv/lib/python3.10/site-packages/langchain/chains/llm_math/base.py:77\u001b[0m, in \u001b[0;36mLLMMathChain._process_llm_result\u001b[0;34m(self, llm_output)\u001b[0m\n\u001b[1;32m     75\u001b[0m \u001b[39mif\u001b[39;00m text_match:\n\u001b[1;32m     76\u001b[0m     expression \u001b[39m=\u001b[39m text_match\u001b[39m.\u001b[39mgroup(\u001b[39m1\u001b[39m)\n\u001b[0;32m---> 77\u001b[0m     output \u001b[39m=\u001b[39m \u001b[39mself\u001b[39;49m\u001b[39m.\u001b[39;49m_evaluate_expression(expression)\n\u001b[1;32m     78\u001b[0m     \u001b[39mself\u001b[39m\u001b[39m.\u001b[39mcallback_manager\u001b[39m.\u001b[39mon_text(\u001b[39m\"\u001b[39m\u001b[39m\\n\u001b[39;00m\u001b[39mAnswer: \u001b[39m\u001b[39m\"\u001b[39m, verbose\u001b[39m=\u001b[39m\u001b[39mself\u001b[39m\u001b[39m.\u001b[39mverbose)\n\u001b[1;32m     79\u001b[0m     \u001b[39mself\u001b[39m\u001b[39m.\u001b[39mcallback_manager\u001b[39m.\u001b[39mon_text(output, color\u001b[39m=\u001b[39m\u001b[39m\"\u001b[39m\u001b[39myellow\u001b[39m\u001b[39m\"\u001b[39m, verbose\u001b[39m=\u001b[39m\u001b[39mself\u001b[39m\u001b[39m.\u001b[39mverbose)\n",
      "File \u001b[0;32m~/Documents/GitHub/langchain-agent-qa/myenv/lib/python3.10/site-packages/langchain/chains/llm_math/base.py:66\u001b[0m, in \u001b[0;36mLLMMathChain._evaluate_expression\u001b[0;34m(self, expression)\u001b[0m\n\u001b[1;32m     58\u001b[0m     output \u001b[39m=\u001b[39m \u001b[39mstr\u001b[39m(\n\u001b[1;32m     59\u001b[0m         numexpr\u001b[39m.\u001b[39mevaluate(\n\u001b[1;32m     60\u001b[0m             expression\u001b[39m.\u001b[39mstrip(),\n\u001b[0;32m   (...)\u001b[0m\n\u001b[1;32m     63\u001b[0m         )\n\u001b[1;32m     64\u001b[0m     )\n\u001b[1;32m     65\u001b[0m \u001b[39mexcept\u001b[39;00m \u001b[39mException\u001b[39;00m \u001b[39mas\u001b[39;00m e:\n\u001b[0;32m---> 66\u001b[0m     \u001b[39mraise\u001b[39;00m \u001b[39mValueError\u001b[39;00m(\u001b[39mf\u001b[39m\u001b[39m\"\u001b[39m\u001b[39m{\u001b[39;00me\u001b[39m}\u001b[39;00m\u001b[39m. Please try again with a valid numerical expression\u001b[39m\u001b[39m\"\u001b[39m)\n\u001b[1;32m     68\u001b[0m \u001b[39m# Remove any leading and trailing brackets from the output\u001b[39;00m\n\u001b[1;32m     69\u001b[0m \u001b[39mreturn\u001b[39;00m re\u001b[39m.\u001b[39msub(\u001b[39mr\u001b[39m\u001b[39m\"\u001b[39m\u001b[39m^\u001b[39m\u001b[39m\\\u001b[39m\u001b[39m[|\u001b[39m\u001b[39m\\\u001b[39m\u001b[39m]$\u001b[39m\u001b[39m\"\u001b[39m, \u001b[39m\"\u001b[39m\u001b[39m\"\u001b[39m, output)\n",
      "\u001b[0;31mValueError\u001b[0m: invalid syntax (<expr>, line 1). Please try again with a valid numerical expression"
     ]
    }
   ],
   "source": [
    "tools = load_tools([\n",
    "    \"serpapi\",\n",
    "    \"llm-math\"],\n",
    "    llm=llm_agent)\n",
    "agent = initialize_agent(\n",
    "    tools,\n",
    "    llm_agent,\n",
    "    agent=\"zero-shot-react-description\",\n",
    "      verbose=True)\n",
    "\n",
    "output = agent.run(question)\n"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "What is actually happening under the hood? Let's see it in the trace visualiser"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "2023-04-19 10:22.53.868678 [info     ] Trace: http://0.0.0.0:8935/traces/01GYC9R0NBYMHRQMF6Q41J62HP\n",
      "\n",
      "\n",
      "\u001b[1m> Entering new AgentExecutor chain...\u001b[0m\n",
      "Rendering http://127.0.0.1:8935/traces/01GYC9R0NBYMHRQMF6Q41J62HP in notebook\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "\n",
       "        <iframe\n",
       "            width=\"1000\"\n",
       "            height=\"500\"\n",
       "            src=\"http://127.0.0.1:8935/traces/01GYC9R0NBYMHRQMF6Q41J62HP\"\n",
       "            frameborder=\"0\"\n",
       "            allowfullscreen\n",
       "            \n",
       "        ></iframe>\n",
       "        "
      ],
      "text/plain": [
       "<IPython.lib.display.IFrame at 0x2a164e7d0>"
      ]
     },
     "execution_count": 9,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\u001b[32;1m\u001b[1;3m I need to find the date of the conference and then calculate the number of days between that date and today.\n",
      "Action: Search\n",
      "Action Input: \"PyCon DE & PyData Berlin 2023\"\u001b[0m\u001b[36;1m\u001b[1;3mPyConDE & PyData Berlin 2023, Berlin Germany. Where Pythonistas in Germany can meet to learn about new and upcoming Python libraries, tools, software and ...\u001b[0m\u001b[32;1m\u001b[1;3m I need to find the exact date of the conference\n",
      "Action: Search\n",
      "Action Input: \"PyCon DE & PyData Berlin 2023 date\"\u001b[0m\u001b[36;1m\u001b[1;3mPyCon DE & PyData Berlin 2023 · From 17 April through 19 April , 2023 · Explore events · More events at Berlin, Germany.\u001b[0m\u001b[32;1m\u001b[1;3m I need to calculate the number of days between the conference date and today\n",
      "Action: Calculator\n",
      "Action Input: 17 April 2023 - today's date\u001b[0m"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "WARNING:root:Failed to persist run: HTTPConnectionPool(host='localhost', port=8000): Max retries exceeded with url: /chain-runs (Caused by NewConnectionError('<urllib3.connection.HTTPConnection object at 0x2a2671c60>: Failed to establish a new connection: [Errno 61] Connection refused'))\n",
      "Exception in thread Thread-4 (visualize):\n",
      "Traceback (most recent call last):\n",
      "  File \"/Users/levkonstantinovskiy/Documents/GitHub/langchain-agent-qa/myenv/lib/python3.10/site-packages/langchain/chains/llm_math/base.py\", line 59, in _evaluate_expression\n",
      "    numexpr.evaluate(\n",
      "  File \"/Users/levkonstantinovskiy/Documents/GitHub/langchain-agent-qa/myenv/lib/python3.10/site-packages/numexpr/necompiler.py\", line 817, in evaluate\n",
      "    _names_cache[expr_key] = getExprNames(ex, context)\n",
      "  File \"/Users/levkonstantinovskiy/Documents/GitHub/langchain-agent-qa/myenv/lib/python3.10/site-packages/numexpr/necompiler.py\", line 704, in getExprNames\n",
      "    ex = stringToExpression(text, {}, context)\n",
      "  File \"/Users/levkonstantinovskiy/Documents/GitHub/langchain-agent-qa/myenv/lib/python3.10/site-packages/numexpr/necompiler.py\", line 274, in stringToExpression\n",
      "    c = compile(s, '<expr>', 'eval', flags)\n",
      "  File \"<expr>\", line 1\n",
      "    17 April 2023 - today()\n",
      "       ^^^^^\n",
      "SyntaxError: invalid syntax\n",
      "\n",
      "During handling of the above exception, another exception occurred:\n",
      "\n",
      "Traceback (most recent call last):\n",
      "  File \"/Users/levkonstantinovskiy/miniconda3/lib/python3.10/threading.py\", line 1016, in _bootstrap_inner\n",
      "    self.run()\n",
      "  File \"/Users/levkonstantinovskiy/miniconda3/lib/python3.10/threading.py\", line 953, in run\n",
      "    self._target(*self._args, **self._kwargs)\n",
      "  File \"/Users/levkonstantinovskiy/Documents/GitHub/langchain-agent-qa/myenv/lib/python3.10/site-packages/langchain_visualizer/visualize.py\", line 69, in visualize\n",
      "    return new_main(recipe, fn)\n",
      "  File \"/Users/levkonstantinovskiy/Documents/GitHub/langchain-agent-qa/myenv/lib/python3.10/site-packages/langchain_visualizer/visualize.py\", line 67, in new_main\n",
      "    cli()\n",
      "  File \"/Users/levkonstantinovskiy/Documents/GitHub/langchain-agent-qa/myenv/lib/python3.10/site-packages/langchain_visualizer/visualize.py\", line 65, in cli\n",
      "    asyncio.run(untraced_wrapper(*args, **kwargs))\n",
      "  File \"/Users/levkonstantinovskiy/Documents/GitHub/langchain-agent-qa/myenv/lib/python3.10/site-packages/nest_asyncio.py\", line 35, in run\n",
      "    return loop.run_until_complete(task)\n",
      "  File \"/Users/levkonstantinovskiy/Documents/GitHub/langchain-agent-qa/myenv/lib/python3.10/site-packages/nest_asyncio.py\", line 90, in run_until_complete\n",
      "    return f.result()\n",
      "  File \"/Users/levkonstantinovskiy/miniconda3/lib/python3.10/asyncio/futures.py\", line 201, in result\n",
      "    raise self._exception.with_traceback(self._exception_tb)\n",
      "  File \"/Users/levkonstantinovskiy/miniconda3/lib/python3.10/asyncio/tasks.py\", line 234, in __step\n",
      "    result = coro.throw(exc)\n",
      "  File \"/Users/levkonstantinovskiy/Documents/GitHub/langchain-agent-qa/myenv/lib/python3.10/site-packages/langchain_visualizer/visualize.py\", line 53, in untraced_wrapper\n",
      "    return await hidden_wrapper(*args, **kwargs)\n",
      "  File \"/Users/levkonstantinovskiy/Documents/GitHub/langchain-agent-qa/myenv/lib/python3.10/site-packages/ice/trace.py\", line 287, in wrapper\n",
      "    return await create_task(inner_wrapper(*args, **kwargs))\n",
      "  File \"/Users/levkonstantinovskiy/miniconda3/lib/python3.10/asyncio/futures.py\", line 285, in __await__\n",
      "    yield self  # This tells Task to wait for completion.\n",
      "  File \"/Users/levkonstantinovskiy/miniconda3/lib/python3.10/asyncio/tasks.py\", line 304, in __wakeup\n",
      "    future.result()\n",
      "  File \"/Users/levkonstantinovskiy/miniconda3/lib/python3.10/asyncio/futures.py\", line 201, in result\n",
      "    raise self._exception.with_traceback(self._exception_tb)\n",
      "  File \"/Users/levkonstantinovskiy/miniconda3/lib/python3.10/asyncio/tasks.py\", line 234, in __step\n",
      "    result = coro.throw(exc)\n",
      "  File \"/Users/levkonstantinovskiy/Documents/GitHub/langchain-agent-qa/myenv/lib/python3.10/site-packages/ice/trace.py\", line 276, in inner_wrapper\n",
      "    result = await fn(*args, **kwargs)\n",
      "  File \"/Users/levkonstantinovskiy/Documents/GitHub/langchain-agent-qa/myenv/lib/python3.10/site-packages/langchain_visualizer/visualize.py\", line 38, in hidden_wrapper\n",
      "    result = await traced_main(*args, **kwargs)\n",
      "  File \"/Users/levkonstantinovskiy/Documents/GitHub/langchain-agent-qa/myenv/lib/python3.10/site-packages/ice/trace.py\", line 287, in wrapper\n",
      "    return await create_task(inner_wrapper(*args, **kwargs))\n",
      "  File \"/Users/levkonstantinovskiy/miniconda3/lib/python3.10/asyncio/futures.py\", line 285, in __await__\n",
      "    yield self  # This tells Task to wait for completion.\n",
      "  File \"/Users/levkonstantinovskiy/miniconda3/lib/python3.10/asyncio/tasks.py\", line 304, in __wakeup\n",
      "    future.result()\n",
      "  File \"/Users/levkonstantinovskiy/miniconda3/lib/python3.10/asyncio/futures.py\", line 201, in result\n",
      "    raise self._exception.with_traceback(self._exception_tb)\n",
      "  File \"/Users/levkonstantinovskiy/miniconda3/lib/python3.10/asyncio/tasks.py\", line 232, in __step\n",
      "    result = coro.send(None)\n",
      "  File \"/Users/levkonstantinovskiy/Documents/GitHub/langchain-agent-qa/myenv/lib/python3.10/site-packages/ice/trace.py\", line 276, in inner_wrapper\n",
      "    result = await fn(*args, **kwargs)\n",
      "  File \"/var/folders/5k/k58wnkmx6x1cdgy02s9v8hnr0000gn/T/ipykernel_23030/1657020887.py\", line 2, in search_agent_demo\n",
      "  File \"/Users/levkonstantinovskiy/Documents/GitHub/langchain-agent-qa/myenv/lib/python3.10/site-packages/langchain/chains/base.py\", line 213, in run\n",
      "    return self(args[0])[self.output_keys[0]]\n",
      "  File \"/Users/levkonstantinovskiy/Documents/GitHub/langchain-agent-qa/myenv/lib/python3.10/site-packages/langchain_visualizer/hijacking.py\", line 44, in overridden_call\n",
      "    return asyncio.get_event_loop().run_until_complete(\n",
      "  File \"/Users/levkonstantinovskiy/Documents/GitHub/langchain-agent-qa/myenv/lib/python3.10/site-packages/nest_asyncio.py\", line 90, in run_until_complete\n",
      "    return f.result()\n",
      "  File \"/Users/levkonstantinovskiy/miniconda3/lib/python3.10/asyncio/futures.py\", line 201, in result\n",
      "    raise self._exception.with_traceback(self._exception_tb)\n",
      "  File \"/Users/levkonstantinovskiy/miniconda3/lib/python3.10/asyncio/tasks.py\", line 234, in __step\n",
      "    result = coro.throw(exc)\n",
      "  File \"/Users/levkonstantinovskiy/Documents/GitHub/langchain-agent-qa/myenv/lib/python3.10/site-packages/ice/trace.py\", line 287, in wrapper\n",
      "    return await create_task(inner_wrapper(*args, **kwargs))\n",
      "  File \"/Users/levkonstantinovskiy/miniconda3/lib/python3.10/asyncio/futures.py\", line 285, in __await__\n",
      "    yield self  # This tells Task to wait for completion.\n",
      "  File \"/Users/levkonstantinovskiy/miniconda3/lib/python3.10/asyncio/tasks.py\", line 304, in __wakeup\n",
      "    future.result()\n",
      "  File \"/Users/levkonstantinovskiy/miniconda3/lib/python3.10/asyncio/futures.py\", line 201, in result\n",
      "    raise self._exception.with_traceback(self._exception_tb)\n",
      "  File \"/Users/levkonstantinovskiy/miniconda3/lib/python3.10/asyncio/tasks.py\", line 232, in __step\n",
      "    result = coro.send(None)\n",
      "  File \"/Users/levkonstantinovskiy/Documents/GitHub/langchain-agent-qa/myenv/lib/python3.10/site-packages/ice/trace.py\", line 276, in inner_wrapper\n",
      "    result = await fn(*args, **kwargs)\n",
      "  File \"/Users/levkonstantinovskiy/Documents/GitHub/langchain-agent-qa/myenv/lib/python3.10/site-packages/langchain_visualizer/hijacking.py\", line 18, in run\n",
      "    return self.og_fn(self.og_obj, *args, **kwargs)\n",
      "  File \"/Users/levkonstantinovskiy/Documents/GitHub/langchain-agent-qa/myenv/lib/python3.10/site-packages/langchain/chains/base.py\", line 116, in __call__\n",
      "    raise e\n",
      "  File \"/Users/levkonstantinovskiy/Documents/GitHub/langchain-agent-qa/myenv/lib/python3.10/site-packages/langchain/chains/base.py\", line 113, in __call__\n",
      "    outputs = self._call(inputs)\n",
      "  File \"/Users/levkonstantinovskiy/Documents/GitHub/langchain-agent-qa/myenv/lib/python3.10/site-packages/langchain/agents/agent.py\", line 792, in _call\n",
      "    next_step_output = self._take_next_step(\n",
      "  File \"/Users/levkonstantinovskiy/Documents/GitHub/langchain-agent-qa/myenv/lib/python3.10/site-packages/langchain/agents/agent.py\", line 695, in _take_next_step\n",
      "    observation = tool.run(\n",
      "  File \"/Users/levkonstantinovskiy/Documents/GitHub/langchain-agent-qa/myenv/lib/python3.10/site-packages/langchain/tools/base.py\", line 163, in run\n",
      "    raise e\n",
      "  File \"/Users/levkonstantinovskiy/Documents/GitHub/langchain-agent-qa/myenv/lib/python3.10/site-packages/langchain/tools/base.py\", line 160, in run\n",
      "    observation = self._run(*args, **kwargs)\n",
      "  File \"/Users/levkonstantinovskiy/Documents/GitHub/langchain-agent-qa/myenv/lib/python3.10/site-packages/langchain/agents/tools.py\", line 30, in _run\n",
      "    return self.func(*args, **kwargs)\n",
      "  File \"/Users/levkonstantinovskiy/Documents/GitHub/langchain-agent-qa/myenv/lib/python3.10/site-packages/langchain/chains/base.py\", line 213, in run\n",
      "    return self(args[0])[self.output_keys[0]]\n",
      "  File \"/Users/levkonstantinovskiy/Documents/GitHub/langchain-agent-qa/myenv/lib/python3.10/site-packages/langchain_visualizer/hijacking.py\", line 44, in overridden_call\n",
      "    return asyncio.get_event_loop().run_until_complete(\n",
      "  File \"/Users/levkonstantinovskiy/Documents/GitHub/langchain-agent-qa/myenv/lib/python3.10/site-packages/nest_asyncio.py\", line 90, in run_until_complete\n",
      "    return f.result()\n",
      "  File \"/Users/levkonstantinovskiy/miniconda3/lib/python3.10/asyncio/futures.py\", line 201, in result\n",
      "    raise self._exception.with_traceback(self._exception_tb)\n",
      "  File \"/Users/levkonstantinovskiy/miniconda3/lib/python3.10/asyncio/tasks.py\", line 234, in __step\n",
      "    result = coro.throw(exc)\n",
      "  File \"/Users/levkonstantinovskiy/Documents/GitHub/langchain-agent-qa/myenv/lib/python3.10/site-packages/ice/trace.py\", line 287, in wrapper\n",
      "    return await create_task(inner_wrapper(*args, **kwargs))\n",
      "  File \"/Users/levkonstantinovskiy/miniconda3/lib/python3.10/asyncio/futures.py\", line 285, in __await__\n",
      "    yield self  # This tells Task to wait for completion.\n",
      "  File \"/Users/levkonstantinovskiy/miniconda3/lib/python3.10/asyncio/tasks.py\", line 304, in __wakeup\n",
      "    future.result()\n",
      "  File \"/Users/levkonstantinovskiy/miniconda3/lib/python3.10/asyncio/futures.py\", line 201, in result\n",
      "    raise self._exception.with_traceback(self._exception_tb)\n",
      "  File \"/Users/levkonstantinovskiy/miniconda3/lib/python3.10/asyncio/tasks.py\", line 232, in __step\n",
      "    result = coro.send(None)\n",
      "  File \"/Users/levkonstantinovskiy/Documents/GitHub/langchain-agent-qa/myenv/lib/python3.10/site-packages/ice/trace.py\", line 276, in inner_wrapper\n",
      "    result = await fn(*args, **kwargs)\n",
      "  File \"/Users/levkonstantinovskiy/Documents/GitHub/langchain-agent-qa/myenv/lib/python3.10/site-packages/langchain_visualizer/hijacking.py\", line 18, in run\n",
      "    return self.og_fn(self.og_obj, *args, **kwargs)\n",
      "  File \"/Users/levkonstantinovskiy/Documents/GitHub/langchain-agent-qa/myenv/lib/python3.10/site-packages/langchain/chains/base.py\", line 116, in __call__\n",
      "    raise e\n",
      "  File \"/Users/levkonstantinovskiy/Documents/GitHub/langchain-agent-qa/myenv/lib/python3.10/site-packages/langchain/chains/base.py\", line 113, in __call__\n",
      "    outputs = self._call(inputs)\n",
      "  File \"/Users/levkonstantinovskiy/Documents/GitHub/langchain-agent-qa/myenv/lib/python3.10/site-packages/langchain/chains/llm_math/base.py\", line 130, in _call\n",
      "    return self._process_llm_result(llm_output)\n",
      "  File \"/Users/levkonstantinovskiy/Documents/GitHub/langchain-agent-qa/myenv/lib/python3.10/site-packages/langchain/chains/llm_math/base.py\", line 77, in _process_llm_result\n",
      "    output = self._evaluate_expression(expression)\n",
      "  File \"/Users/levkonstantinovskiy/Documents/GitHub/langchain-agent-qa/myenv/lib/python3.10/site-packages/langchain/chains/llm_math/base.py\", line 66, in _evaluate_expression\n",
      "    raise ValueError(f\"{e}. Please try again with a valid numerical expression\")\n",
      "ValueError: invalid syntax (<expr>, line 1). Please try again with a valid numerical expression\n"
     ]
    }
   ],
   "source": [
    "async def search_agent_demo():\n",
    "    return agent.run(\n",
    "        question\n",
    "    )\n",
    "\n",
    "visualize(search_agent_demo)"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "So it is the Google's fault that it gives us the incorrect date!\n",
    "Let's build our own index of our conference website, maybe that would help"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Create Index Database from PyCon Website. Only run once"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [],
   "source": [
    "# \n",
    "# for a production-ready document index, use HayStack by Deepset\n",
    "\n",
    "# only run once to create index db\n",
    "# sign up for Apify.com and add  APIFY_API_TOKEN to .env file\n",
    "# # costs 50 cents to scrape 704 pages\n",
    "# apify = ApifyWrapper()\n",
    "# loader = apify.call_actor(\n",
    "#     actor_id=\"apify/website-content-crawler\",\n",
    "#     run_input={\"startUrls\": [{\"url\": \"https://2023.pycon.de/\"}]},\n",
    "#     dataset_mapping_function=lambda item: Document(\n",
    "#         page_content=item[\"text\"] or \"\", metadata={\"source\": item[\"url\"]}\n",
    "#     ),  \n",
    "# )\n",
    "\n",
    "# persist_directory = 'db'\n",
    "# embedding = OpenAIEmbeddings()\n",
    "\n",
    "# loader = ApifyDatasetLoader(\n",
    "#     dataset_id=\"jjA7bt8A2W30ekM5t\",\n",
    "#     dataset_mapping_function=lambda dataset_item: Document(\n",
    "#         page_content=dataset_item[\"text\"], metadata={\"source\": dataset_item[\"url\"]}\n",
    "#     ),\n",
    "# )\n",
    "# documents = loader.load()\n",
    "# text_splitter = RecursiveCharacterTextSplitter(chunk_size=1000, chunk_overlap=200)\n",
    "# split_docs = text_splitter.split_documents(documents)\n",
    "\n",
    "\n",
    "\n",
    "# vectordb = Chroma.from_documents(documents=split_docs, embedding=embedding, persist_directory=persist_directory)\n",
    "\n",
    "# vectordb.persist()"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Load already created index"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "WARNING:chromadb:Using embedded DuckDB with persistence: data will be stored in: db\n"
     ]
    }
   ],
   "source": [
    "persist_directory = 'db'\n",
    "embedding = OpenAIEmbeddings()\n",
    "vectordb = Chroma(persist_directory=persist_directory, embedding_function=embedding)\n",
    "retriever = vectordb.as_retriever(search_type=\"mmr\", search_kwargs={\"k\": 3})\n",
    "result = retriever.get_relevant_documents(\"When is the PyCon DE & PyData Berlin 2023 conference?\")\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'🎉 🤗 April 17-19 🤗 🎉 \\nPyCon DE & PyData Berlin \\n2023 \\nThanks to the European Union and Europäischen Fonds für regionale Entwicklung for their support.\\nDieses Vorhaben wurde als Teil der Reaktion der Union auf die Covid-19-Pandemie finanziert.\\nSee you soon!\\nCheck the Blog and FAQ for the most recent infos. \\nAbout \\nWhat can I expect from the conference? \\nOver the span of 3 days, attendees will have the opportunity to participate in workshops, attend live keynote sessions and talks, as well as get to know fellow members of the Python and PyData Communities. We aim to be an accessible, community-driven conference, with novice to advanced level presentations. Tutorials and talks bring attendees the latest project features along with cutting-edge use cases. The conference is organised by experts for other experts providing an excellent quality level of content. Newcomers explictly welcome!\\nCheck out the previous 2022 conference\\nPyConDE videos\\nPyData Berlin videos\\nOur amazing venue \\nbcc'"
      ]
     },
     "execution_count": 11,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "str(result[0].page_content)"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "The retrieved results look very good. They have the answer. Great hope!"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "2023-04-19 00:45.48.740027 [info     ] Trace: http://0.0.0.0:8935/traces/01GYB8QB22VE8KCR1ZHD5NZ1MN\n",
      "\n",
      "\n",
      "\u001b[1m> Entering new AgentExecutor chain...\u001b[0m\n",
      "Rendering http://127.0.0.1:8935/traces/01GYB8QB22VE8KCR1ZHD5NZ1MN in notebook\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "\n",
       "        <iframe\n",
       "            width=\"1000\"\n",
       "            height=\"500\"\n",
       "            src=\"http://127.0.0.1:8935/traces/01GYB8QB22VE8KCR1ZHD5NZ1MN\"\n",
       "            frameborder=\"0\"\n",
       "            allowfullscreen\n",
       "            \n",
       "        ></iframe>\n",
       "        "
      ],
      "text/plain": [
       "<IPython.lib.display.IFrame at 0x2abb520d0>"
      ]
     },
     "execution_count": 17,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\u001b[32;1m\u001b[1;3m I need to find out when the conference is and then calculate the number of days between that date and today.\n",
      "Action: PyCon and PyData DE 2023 Website Index\n",
      "Action Input: PyCon DE & PyData Berlin 2023\u001b[0m\u001b[38;5;200m\u001b[1;3mPyConDE & PyData Berlin 2023 Tickets \n",
      "Read more \n",
      "Jan 13, 2023 by Organisers \n",
      "Community Voting on Submissions \n",
      "Read more \n",
      "Dec 14, 2022 by Organisers \n",
      "Financial Aid Programme \n",
      "Read more \n",
      "Dec 11, 2022 by Organisers \n",
      "Call For Proposals \n",
      "Read more \n",
      "Nov 18, 2022 by Organisers \n",
      "Call for Sponsors \n",
      "Read more \n",
      "Nov 1, 2022 by ORGANISERS \n",
      "Call for Conference Chairs, Committee Members & Volunteers \n",
      "Read more \n",
      "Register now;🎉 🤗 April 17-19 🤗 🎉 \n",
      "PyCon DE & PyData Berlin \n",
      "2023 \n",
      "Thanks to the European Union and Europäischen Fonds für regionale Entwicklung for their support.\n",
      "Dieses Vorhaben wurde als Teil der Reaktion der Union auf die Covid-19-Pandemie finanziert.\n",
      "See you soon!\n",
      "Check the Blog and FAQ for the most recent infos. \n",
      "About \n",
      "What can I expect from the conference? \n",
      "Over the span of 3 days, attendees will have the opportunity to participate in workshops, attend live keynote sessions and talks, as well as get to know fellow members of the Python and PyData Communities. We aim to be an accessible, community-driven conference, with novice to advanced level presentations. Tutorials and talks bring attendees the latest project features along with cutting-edge use cases. The conference is organised by experts for other experts providing an excellent quality level of content. Newcomers explictly welcome!\n",
      "Check out the previous 2022 conference\n",
      "PyConDE videos\n",
      "PyData Berlin videos\n",
      "Our amazing venue \n",
      "bcc;We are happy to announce our team of keynote speakers for the PyCon.DE & PyData Berlin, 2023.\n",
      "Carsten Binnig is a Full Professor in the Computer Science department at TU Darmstadt and a Visiting Researcher at the Google Systems Research Group. Carsten received his Ph.D. at the University of Heidelberg in 2008. Afterwards, he spent time as a postdoctoral researcher in the Systems Group at ETH Zurich and at SAP working on in-memory databases. Currently, his research focus is on the design of scalable data systems on modern hardware as well as machine learning for scalable data systems. His work has been awarded a Google Faculty Award, as well as multiple best paper and best demo awards.\n",
      "Towards Learned Database Systems\u001b[0m\u001b[32;1m\u001b[1;3m The conference is from April 17-19, 2023.\n",
      "Action: Calculator\n",
      "Action Input: Today's date (April 8, 2021) and April 17, 2023\u001b[0m\u001b[33;1m\u001b[1;3mAnswer: 739\u001b[0m"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "WARNING:root:Failed to persist run: HTTPConnectionPool(host='localhost', port=8000): Max retries exceeded with url: /chain-runs (Caused by NewConnectionError('<urllib3.connection.HTTPConnection object at 0x2b196cc70>: Failed to establish a new connection: [Errno 61] Connection refused'))\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\u001b[32;1m\u001b[1;3m I now know the number of days between today and the conference.\n",
      "Final Answer: The PyCon DE & PyData Berlin 2023 conference is 739 days away from today (April 8, 2021).\u001b[0m\n",
      "\n",
      "\u001b[1m> Finished chain.\u001b[0m\n",
      "The PyCon DE & PyData Berlin \u001b[1;36m2023\u001b[0m conference is \u001b[1;36m739\u001b[0m days away from today \u001b[1m(\u001b[0mApril \u001b[1;36m8\u001b[0m, \u001b[1;36m2021\u001b[0m\u001b[1m)\u001b[0m.\n"
     ]
    }
   ],
   "source": [
    "tools = load_tools([\"serpapi\", \"llm-math\"], llm=llm_agent)\n",
    "tools.append(\n",
    "    Tool(\n",
    "        name=\"PyCon and PyData DE 2023 Website Index\",\n",
    "        func=lambda q: \";\".join([x.page_content for x in retriever.get_relevant_documents(q)]),\n",
    "        description=f\"Useful when you want answer questions about PyCon and PyData DE 2023.\",\n",
    "    ),\n",
    ")\n",
    "index_agent = initialize_agent(tools, llm_agent, agent=\"zero-shot-react-description\", verbose=True)\n",
    "async def search_agent_with_index_demo():\n",
    "    return index_agent.run(\n",
    "        question\n",
    "    )\n",
    "\n",
    "visualize(search_agent_with_index_demo)"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Yay! We have the correct date. Just a minor problem with todays date. Maybe we just need to give it a better tool for finding out todays date? And then we are done?\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[Tool(name='Search', description='A search engine. Useful for when you need to answer questions about current events. Input should be a search query.', args_schema=None, return_direct=False, verbose=False, callback_manager=<langchain.callbacks.shared.SharedCallbackManager object at 0x104bc2250>, func=<bound method get_overridden_call.<locals>.overridden_call of SerpAPIWrapper(search_engine=<class 'serpapi.google_search.GoogleSearch'>, params={'engine': 'google', 'google_domain': 'google.com', 'gl': 'us', 'hl': 'en'}, serpapi_api_key='0d123db3ed3687301055b015060fdb0287e675bf25c83131175557e6ed4ff9a6', aiosession=None)>, coroutine=<bound method SerpAPIWrapper.arun of SerpAPIWrapper(search_engine=<class 'serpapi.google_search.GoogleSearch'>, params={'engine': 'google', 'google_domain': 'google.com', 'gl': 'us', 'hl': 'en'}, serpapi_api_key='0d123db3ed3687301055b015060fdb0287e675bf25c83131175557e6ed4ff9a6', aiosession=None)>), Tool(name='Calculator', description='Useful for when you need to answer questions about math.', args_schema=None, return_direct=False, verbose=False, callback_manager=<langchain.callbacks.shared.SharedCallbackManager object at 0x104bc2250>, func=<bound method Chain.run of LLMMathChain(memory=None, callback_manager=<langchain.callbacks.shared.SharedCallbackManager object at 0x104bc2250>, verbose=False, llm=OpenAI(cache=None, verbose=False, callback_manager=<langchain.callbacks.shared.SharedCallbackManager object at 0x104bc2250>, client=<class 'openai.api_resources.completion.Completion'>, model_name='text-davinci-003', temperature=0.0, max_tokens=256, top_p=1, frequency_penalty=0, presence_penalty=0, n=1, best_of=1, model_kwargs={}, openai_api_key=None, openai_api_base=None, openai_organization=None, batch_size=20, request_timeout=None, logit_bias={}, max_retries=6, streaming=False), prompt=PromptTemplate(input_variables=['question'], output_parser=None, partial_variables={}, template='Translate a math problem into a expression that can be executed using Python\\'s numexpr library. Use the output of running this code to answer the question.\\n\\nQuestion: ${{Question with math problem.}}\\n```text\\n${{single line mathematical expression that solves the problem}}\\n```\\n...numexpr.evaluate(text)...\\n```output\\n${{Output of running the code}}\\n```\\nAnswer: ${{Answer}}\\n\\nBegin.\\n\\nQuestion: What is 37593 * 67?\\n\\n```text\\n37593 * 67\\n```\\n...numexpr.evaluate(\"37593 * 67\")...\\n```output\\n2518731\\n```\\nAnswer: 2518731\\n\\nQuestion: {question}\\n', template_format='f-string', validate_template=True), input_key='question', output_key='answer')>, coroutine=<bound method Chain.arun of LLMMathChain(memory=None, callback_manager=<langchain.callbacks.shared.SharedCallbackManager object at 0x104bc2250>, verbose=False, llm=OpenAI(cache=None, verbose=False, callback_manager=<langchain.callbacks.shared.SharedCallbackManager object at 0x104bc2250>, client=<class 'openai.api_resources.completion.Completion'>, model_name='text-davinci-003', temperature=0.0, max_tokens=256, top_p=1, frequency_penalty=0, presence_penalty=0, n=1, best_of=1, model_kwargs={}, openai_api_key=None, openai_api_base=None, openai_organization=None, batch_size=20, request_timeout=None, logit_bias={}, max_retries=6, streaming=False), prompt=PromptTemplate(input_variables=['question'], output_parser=None, partial_variables={}, template='Translate a math problem into a expression that can be executed using Python\\'s numexpr library. Use the output of running this code to answer the question.\\n\\nQuestion: ${{Question with math problem.}}\\n```text\\n${{single line mathematical expression that solves the problem}}\\n```\\n...numexpr.evaluate(text)...\\n```output\\n${{Output of running the code}}\\n```\\nAnswer: ${{Answer}}\\n\\nBegin.\\n\\nQuestion: What is 37593 * 67?\\n\\n```text\\n37593 * 67\\n```\\n...numexpr.evaluate(\"37593 * 67\")...\\n```output\\n2518731\\n```\\nAnswer: 2518731\\n\\nQuestion: {question}\\n', template_format='f-string', validate_template=True), input_key='question', output_key='answer')>), Tool(name='PyCon and PyData DE 2023 Website Index', description='Useful when you want answer questions about PyCon and PyData DE 2023.', args_schema=None, return_direct=False, verbose=False, callback_manager=<langchain.callbacks.shared.SharedCallbackManager object at 0x104bc2250>, func=<function <lambda> at 0x2b1a30040>, coroutine=None), Tool(name='A date finder.', description='Useful for when you need to calculate any date.', args_schema=None, return_direct=False, verbose=False, callback_manager=<langchain.callbacks.shared.SharedCallbackManager object at 0x104bc2250>, func=<bound method Chain.run of LLMMathChain(memory=None, callback_manager=<langchain.callbacks.shared.SharedCallbackManager object at 0x104bc2250>, verbose=False, llm=OpenAI(cache=None, verbose=False, callback_manager=<langchain.callbacks.shared.SharedCallbackManager object at 0x104bc2250>, client=<class 'openai.api_resources.completion.Completion'>, model_name='text-davinci-003', temperature=0.0, max_tokens=256, top_p=1, frequency_penalty=0, presence_penalty=0, n=1, best_of=1, model_kwargs={}, openai_api_key=None, openai_api_base=None, openai_organization=None, batch_size=20, request_timeout=None, logit_bias={}, max_retries=6, streaming=False), prompt=PromptTemplate(input_variables=['question'], output_parser=None, partial_variables={}, template='Translate a math problem into a expression that can be executed using Python\\'s numexpr library. Use the output of running this code to answer the question.\\n\\nQuestion: ${{Question with math problem.}}\\n```text\\n${{single line mathematical expression that solves the problem}}\\n```\\n...numexpr.evaluate(text)...\\n```output\\n${{Output of running the code}}\\n```\\nAnswer: ${{Answer}}\\n\\nBegin.\\n\\nQuestion: What is 37593 * 67?\\n\\n```text\\n37593 * 67\\n```\\n...numexpr.evaluate(\"37593 * 67\")...\\n```output\\n2518731\\n```\\nAnswer: 2518731\\n\\nQuestion: {question}\\n', template_format='f-string', validate_template=True), input_key='question', output_key='answer')>, coroutine=<bound method Chain.arun of LLMMathChain(memory=None, callback_manager=<langchain.callbacks.shared.SharedCallbackManager object at 0x104bc2250>, verbose=False, llm=OpenAI(cache=None, verbose=False, callback_manager=<langchain.callbacks.shared.SharedCallbackManager object at 0x104bc2250>, client=<class 'openai.api_resources.completion.Completion'>, model_name='text-davinci-003', temperature=0.0, max_tokens=256, top_p=1, frequency_penalty=0, presence_penalty=0, n=1, best_of=1, model_kwargs={}, openai_api_key=None, openai_api_base=None, openai_organization=None, batch_size=20, request_timeout=None, logit_bias={}, max_retries=6, streaming=False), prompt=PromptTemplate(input_variables=['question'], output_parser=None, partial_variables={}, template='Translate a math problem into a expression that can be executed using Python\\'s numexpr library. Use the output of running this code to answer the question.\\n\\nQuestion: ${{Question with math problem.}}\\n```text\\n${{single line mathematical expression that solves the problem}}\\n```\\n...numexpr.evaluate(text)...\\n```output\\n${{Output of running the code}}\\n```\\nAnswer: ${{Answer}}\\n\\nBegin.\\n\\nQuestion: What is 37593 * 67?\\n\\n```text\\n37593 * 67\\n```\\n...numexpr.evaluate(\"37593 * 67\")...\\n```output\\n2518731\\n```\\nAnswer: 2518731\\n\\nQuestion: {question}\\n', template_format='f-string', validate_template=True), input_key='question', output_key='answer')>)]\n",
      "Rendering http://127.0.0.1:8935/traces/01GYB8QB22VE8KCR1ZHD5NZ1MN in notebook\n",
      "2023-04-19 00:46.15.470769 [info     ] Trace: http://0.0.0.0:8935/traces/01GYB8R55CW363RYJZ2PDP733J\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "\n",
       "        <iframe\n",
       "            width=\"1000\"\n",
       "            height=\"500\"\n",
       "            src=\"http://127.0.0.1:8935/traces/01GYB8QB22VE8KCR1ZHD5NZ1MN\"\n",
       "            frameborder=\"0\"\n",
       "            allowfullscreen\n",
       "            \n",
       "        ></iframe>\n",
       "        "
      ],
      "text/plain": [
       "<IPython.lib.display.IFrame at 0x2aa638fd0>"
      ]
     },
     "execution_count": 18,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "\n",
      "\u001b[1m> Entering new AgentExecutor chain...\u001b[0m\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\u001b[32;1m\u001b[1;3m I need to find the date of the conference and then calculate the number of days between that date and today.\n",
      "Action: PyCon and PyData DE 2023 Website Index\n",
      "Action Input: Date of PyCon DE & PyData Berlin 2023\u001b[0m\u001b[38;5;200m\u001b[1;3mPyConDE & PyData Berlin 2023 Tickets \n",
      "Read more \n",
      "Jan 13, 2023 by Organisers \n",
      "Community Voting on Submissions \n",
      "Read more \n",
      "Dec 14, 2022 by Organisers \n",
      "Financial Aid Programme \n",
      "Read more \n",
      "Dec 11, 2022 by Organisers \n",
      "Call For Proposals \n",
      "Read more \n",
      "Nov 18, 2022 by Organisers \n",
      "Call for Sponsors \n",
      "Read more \n",
      "Nov 1, 2022 by ORGANISERS \n",
      "Call for Conference Chairs, Committee Members & Volunteers \n",
      "Read more \n",
      "Register now;🎉 🤗 April 17-19 🤗 🎉 \n",
      "PyCon DE & PyData Berlin \n",
      "2023 \n",
      "Thanks to the European Union and Europäischen Fonds für regionale Entwicklung for their support.\n",
      "Dieses Vorhaben wurde als Teil der Reaktion der Union auf die Covid-19-Pandemie finanziert.\n",
      "See you soon!\n",
      "Check the Blog and FAQ for the most recent infos. \n",
      "About \n",
      "What can I expect from the conference? \n",
      "Over the span of 3 days, attendees will have the opportunity to participate in workshops, attend live keynote sessions and talks, as well as get to know fellow members of the Python and PyData Communities. We aim to be an accessible, community-driven conference, with novice to advanced level presentations. Tutorials and talks bring attendees the latest project features along with cutting-edge use cases. The conference is organised by experts for other experts providing an excellent quality level of content. Newcomers explictly welcome!\n",
      "Check out the previous 2022 conference\n",
      "PyConDE videos\n",
      "PyData Berlin videos\n",
      "Our amazing venue \n",
      "bcc;We are happy to announce our team of keynote speakers for the PyCon.DE & PyData Berlin, 2023.\n",
      "Carsten Binnig is a Full Professor in the Computer Science department at TU Darmstadt and a Visiting Researcher at the Google Systems Research Group. Carsten received his Ph.D. at the University of Heidelberg in 2008. Afterwards, he spent time as a postdoctoral researcher in the Systems Group at ETH Zurich and at SAP working on in-memory databases. Currently, his research focus is on the design of scalable data systems on modern hardware as well as machine learning for scalable data systems. His work has been awarded a Google Faculty Award, as well as multiple best paper and best demo awards.\n",
      "Towards Learned Database Systems\u001b[0m\u001b[32;1m\u001b[1;3m The date of the conference is January 13, 2023.\n",
      "Action: A date finder.\n",
      "Action Input: January 13, 2023\u001b[0m\u001b[31;1m\u001b[1;3mAnswer: 20230113\u001b[0m\u001b[32;1m\u001b[1;3m I need to calculate the number of days between January 13, 2023 and today.\n",
      "Action: Calculator\n",
      "Action Input: 20230113 - today's date\u001b[0m"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "WARNING:root:Failed to persist run: HTTPConnectionPool(host='localhost', port=8000): Max retries exceeded with url: /chain-runs (Caused by NewConnectionError('<urllib3.connection.HTTPConnection object at 0x2b2b5d0d0>: Failed to establish a new connection: [Errno 61] Connection refused'))\n",
      "Exception in thread Thread-7:\n",
      "Traceback (most recent call last):\n",
      "  File \"/Users/levkonstantinovskiy/Documents/GitHub/langchain-agent-qa/lib/python3.9/site-packages/langchain/chains/llm_math/base.py\", line 59, in _evaluate_expression\n",
      "    numexpr.evaluate(\n",
      "  File \"/Users/levkonstantinovskiy/Documents/GitHub/langchain-agent-qa/lib/python3.9/site-packages/numexpr/necompiler.py\", line 817, in evaluate\n",
      "    _names_cache[expr_key] = getExprNames(ex, context)\n",
      "  File \"/Users/levkonstantinovskiy/Documents/GitHub/langchain-agent-qa/lib/python3.9/site-packages/numexpr/necompiler.py\", line 704, in getExprNames\n",
      "    ex = stringToExpression(text, {}, context)\n",
      "  File \"/Users/levkonstantinovskiy/Documents/GitHub/langchain-agent-qa/lib/python3.9/site-packages/numexpr/necompiler.py\", line 289, in stringToExpression\n",
      "    ex = eval(c, names)\n",
      "  File \"<expr>\", line 1, in <module>\n",
      "AttributeError: 'VariableNode' object has no attribute 'today'\n",
      "\n",
      "During handling of the above exception, another exception occurred:\n",
      "\n",
      "Traceback (most recent call last):\n",
      "  File \"/opt/homebrew/Cellar/python@3.9/3.9.16/Frameworks/Python.framework/Versions/3.9/lib/python3.9/threading.py\", line 980, in _bootstrap_inner\n",
      "    self.run()\n",
      "  File \"/opt/homebrew/Cellar/python@3.9/3.9.16/Frameworks/Python.framework/Versions/3.9/lib/python3.9/threading.py\", line 917, in run\n",
      "    self._target(*self._args, **self._kwargs)\n",
      "  File \"/Users/levkonstantinovskiy/Documents/GitHub/langchain-agent-qa/lib/python3.9/site-packages/langchain_visualizer/visualize.py\", line 69, in visualize\n",
      "    return new_main(recipe, fn)\n",
      "  File \"/Users/levkonstantinovskiy/Documents/GitHub/langchain-agent-qa/lib/python3.9/site-packages/langchain_visualizer/visualize.py\", line 67, in new_main\n",
      "    cli()\n",
      "  File \"/Users/levkonstantinovskiy/Documents/GitHub/langchain-agent-qa/lib/python3.9/site-packages/langchain_visualizer/visualize.py\", line 65, in cli\n",
      "    asyncio.run(untraced_wrapper(*args, **kwargs))\n",
      "  File \"/Users/levkonstantinovskiy/Documents/GitHub/langchain-agent-qa/lib/python3.9/site-packages/nest_asyncio.py\", line 35, in run\n",
      "    return loop.run_until_complete(task)\n",
      "  File \"/Users/levkonstantinovskiy/Documents/GitHub/langchain-agent-qa/lib/python3.9/site-packages/nest_asyncio.py\", line 90, in run_until_complete\n",
      "    return f.result()\n",
      "  File \"/opt/homebrew/Cellar/python@3.9/3.9.16/Frameworks/Python.framework/Versions/3.9/lib/python3.9/asyncio/futures.py\", line 201, in result\n",
      "    raise self._exception\n",
      "  File \"/opt/homebrew/Cellar/python@3.9/3.9.16/Frameworks/Python.framework/Versions/3.9/lib/python3.9/asyncio/tasks.py\", line 258, in __step\n",
      "    result = coro.throw(exc)\n",
      "  File \"/Users/levkonstantinovskiy/Documents/GitHub/langchain-agent-qa/lib/python3.9/site-packages/langchain_visualizer/visualize.py\", line 53, in untraced_wrapper\n",
      "    return await hidden_wrapper(*args, **kwargs)\n",
      "  File \"/Users/levkonstantinovskiy/Documents/GitHub/langchain-agent-qa/lib/python3.9/site-packages/ice/trace.py\", line 287, in wrapper\n",
      "    return await create_task(inner_wrapper(*args, **kwargs))\n",
      "  File \"/opt/homebrew/Cellar/python@3.9/3.9.16/Frameworks/Python.framework/Versions/3.9/lib/python3.9/asyncio/futures.py\", line 284, in __await__\n",
      "    yield self  # This tells Task to wait for completion.\n",
      "  File \"/opt/homebrew/Cellar/python@3.9/3.9.16/Frameworks/Python.framework/Versions/3.9/lib/python3.9/asyncio/tasks.py\", line 328, in __wakeup\n",
      "    future.result()\n",
      "  File \"/opt/homebrew/Cellar/python@3.9/3.9.16/Frameworks/Python.framework/Versions/3.9/lib/python3.9/asyncio/futures.py\", line 201, in result\n",
      "    raise self._exception\n",
      "  File \"/opt/homebrew/Cellar/python@3.9/3.9.16/Frameworks/Python.framework/Versions/3.9/lib/python3.9/asyncio/tasks.py\", line 258, in __step\n",
      "    result = coro.throw(exc)\n",
      "  File \"/Users/levkonstantinovskiy/Documents/GitHub/langchain-agent-qa/lib/python3.9/site-packages/ice/trace.py\", line 276, in inner_wrapper\n",
      "    result = await fn(*args, **kwargs)\n",
      "  File \"/Users/levkonstantinovskiy/Documents/GitHub/langchain-agent-qa/lib/python3.9/site-packages/langchain_visualizer/visualize.py\", line 38, in hidden_wrapper\n",
      "    result = await traced_main(*args, **kwargs)\n",
      "  File \"/Users/levkonstantinovskiy/Documents/GitHub/langchain-agent-qa/lib/python3.9/site-packages/ice/trace.py\", line 287, in wrapper\n",
      "    return await create_task(inner_wrapper(*args, **kwargs))\n",
      "  File \"/opt/homebrew/Cellar/python@3.9/3.9.16/Frameworks/Python.framework/Versions/3.9/lib/python3.9/asyncio/futures.py\", line 284, in __await__\n",
      "    yield self  # This tells Task to wait for completion.\n",
      "  File \"/opt/homebrew/Cellar/python@3.9/3.9.16/Frameworks/Python.framework/Versions/3.9/lib/python3.9/asyncio/tasks.py\", line 328, in __wakeup\n",
      "    future.result()\n",
      "  File \"/opt/homebrew/Cellar/python@3.9/3.9.16/Frameworks/Python.framework/Versions/3.9/lib/python3.9/asyncio/futures.py\", line 201, in result\n",
      "    raise self._exception\n",
      "  File \"/opt/homebrew/Cellar/python@3.9/3.9.16/Frameworks/Python.framework/Versions/3.9/lib/python3.9/asyncio/tasks.py\", line 256, in __step\n",
      "    result = coro.send(None)\n",
      "  File \"/Users/levkonstantinovskiy/Documents/GitHub/langchain-agent-qa/lib/python3.9/site-packages/ice/trace.py\", line 276, in inner_wrapper\n",
      "    result = await fn(*args, **kwargs)\n",
      "  File \"/var/folders/5k/k58wnkmx6x1cdgy02s9v8hnr0000gn/T/ipykernel_15057/3540474874.py\", line 20, in search_agent_with_index_and_date_finder_demo_\n",
      "  File \"/Users/levkonstantinovskiy/Documents/GitHub/langchain-agent-qa/lib/python3.9/site-packages/langchain/chains/base.py\", line 213, in run\n",
      "    return self(args[0])[self.output_keys[0]]\n",
      "  File \"/Users/levkonstantinovskiy/Documents/GitHub/langchain-agent-qa/lib/python3.9/site-packages/langchain_visualizer/hijacking.py\", line 44, in overridden_call\n",
      "    return asyncio.get_event_loop().run_until_complete(\n",
      "  File \"/Users/levkonstantinovskiy/Documents/GitHub/langchain-agent-qa/lib/python3.9/site-packages/nest_asyncio.py\", line 90, in run_until_complete\n",
      "    return f.result()\n",
      "  File \"/opt/homebrew/Cellar/python@3.9/3.9.16/Frameworks/Python.framework/Versions/3.9/lib/python3.9/asyncio/futures.py\", line 201, in result\n",
      "    raise self._exception\n",
      "  File \"/opt/homebrew/Cellar/python@3.9/3.9.16/Frameworks/Python.framework/Versions/3.9/lib/python3.9/asyncio/tasks.py\", line 258, in __step\n",
      "    result = coro.throw(exc)\n",
      "  File \"/Users/levkonstantinovskiy/Documents/GitHub/langchain-agent-qa/lib/python3.9/site-packages/ice/trace.py\", line 287, in wrapper\n",
      "    return await create_task(inner_wrapper(*args, **kwargs))\n",
      "  File \"/opt/homebrew/Cellar/python@3.9/3.9.16/Frameworks/Python.framework/Versions/3.9/lib/python3.9/asyncio/futures.py\", line 284, in __await__\n",
      "    yield self  # This tells Task to wait for completion.\n",
      "  File \"/opt/homebrew/Cellar/python@3.9/3.9.16/Frameworks/Python.framework/Versions/3.9/lib/python3.9/asyncio/tasks.py\", line 328, in __wakeup\n",
      "    future.result()\n",
      "  File \"/opt/homebrew/Cellar/python@3.9/3.9.16/Frameworks/Python.framework/Versions/3.9/lib/python3.9/asyncio/futures.py\", line 201, in result\n",
      "    raise self._exception\n",
      "  File \"/opt/homebrew/Cellar/python@3.9/3.9.16/Frameworks/Python.framework/Versions/3.9/lib/python3.9/asyncio/tasks.py\", line 256, in __step\n",
      "    result = coro.send(None)\n",
      "  File \"/Users/levkonstantinovskiy/Documents/GitHub/langchain-agent-qa/lib/python3.9/site-packages/ice/trace.py\", line 276, in inner_wrapper\n",
      "    result = await fn(*args, **kwargs)\n",
      "  File \"/Users/levkonstantinovskiy/Documents/GitHub/langchain-agent-qa/lib/python3.9/site-packages/langchain_visualizer/hijacking.py\", line 18, in run\n",
      "    return self.og_fn(self.og_obj, *args, **kwargs)\n",
      "  File \"/Users/levkonstantinovskiy/Documents/GitHub/langchain-agent-qa/lib/python3.9/site-packages/langchain/chains/base.py\", line 116, in __call__\n",
      "    raise e\n",
      "  File \"/Users/levkonstantinovskiy/Documents/GitHub/langchain-agent-qa/lib/python3.9/site-packages/langchain/chains/base.py\", line 113, in __call__\n",
      "    outputs = self._call(inputs)\n",
      "  File \"/Users/levkonstantinovskiy/Documents/GitHub/langchain-agent-qa/lib/python3.9/site-packages/langchain/agents/agent.py\", line 792, in _call\n",
      "    next_step_output = self._take_next_step(\n",
      "  File \"/Users/levkonstantinovskiy/Documents/GitHub/langchain-agent-qa/lib/python3.9/site-packages/langchain/agents/agent.py\", line 695, in _take_next_step\n",
      "    observation = tool.run(\n",
      "  File \"/Users/levkonstantinovskiy/Documents/GitHub/langchain-agent-qa/lib/python3.9/site-packages/langchain/tools/base.py\", line 163, in run\n",
      "    raise e\n",
      "  File \"/Users/levkonstantinovskiy/Documents/GitHub/langchain-agent-qa/lib/python3.9/site-packages/langchain/tools/base.py\", line 160, in run\n",
      "    observation = self._run(*args, **kwargs)\n",
      "  File \"/Users/levkonstantinovskiy/Documents/GitHub/langchain-agent-qa/lib/python3.9/site-packages/langchain/agents/tools.py\", line 30, in _run\n",
      "    return self.func(*args, **kwargs)\n",
      "  File \"/Users/levkonstantinovskiy/Documents/GitHub/langchain-agent-qa/lib/python3.9/site-packages/langchain/chains/base.py\", line 213, in run\n",
      "    return self(args[0])[self.output_keys[0]]\n",
      "  File \"/Users/levkonstantinovskiy/Documents/GitHub/langchain-agent-qa/lib/python3.9/site-packages/langchain_visualizer/hijacking.py\", line 44, in overridden_call\n",
      "    return asyncio.get_event_loop().run_until_complete(\n",
      "  File \"/Users/levkonstantinovskiy/Documents/GitHub/langchain-agent-qa/lib/python3.9/site-packages/nest_asyncio.py\", line 90, in run_until_complete\n",
      "    return f.result()\n",
      "  File \"/opt/homebrew/Cellar/python@3.9/3.9.16/Frameworks/Python.framework/Versions/3.9/lib/python3.9/asyncio/futures.py\", line 201, in result\n",
      "    raise self._exception\n",
      "  File \"/opt/homebrew/Cellar/python@3.9/3.9.16/Frameworks/Python.framework/Versions/3.9/lib/python3.9/asyncio/tasks.py\", line 258, in __step\n",
      "    result = coro.throw(exc)\n",
      "  File \"/Users/levkonstantinovskiy/Documents/GitHub/langchain-agent-qa/lib/python3.9/site-packages/ice/trace.py\", line 287, in wrapper\n",
      "    return await create_task(inner_wrapper(*args, **kwargs))\n",
      "  File \"/opt/homebrew/Cellar/python@3.9/3.9.16/Frameworks/Python.framework/Versions/3.9/lib/python3.9/asyncio/futures.py\", line 284, in __await__\n",
      "    yield self  # This tells Task to wait for completion.\n",
      "  File \"/opt/homebrew/Cellar/python@3.9/3.9.16/Frameworks/Python.framework/Versions/3.9/lib/python3.9/asyncio/tasks.py\", line 328, in __wakeup\n",
      "    future.result()\n",
      "  File \"/opt/homebrew/Cellar/python@3.9/3.9.16/Frameworks/Python.framework/Versions/3.9/lib/python3.9/asyncio/futures.py\", line 201, in result\n",
      "    raise self._exception\n",
      "  File \"/opt/homebrew/Cellar/python@3.9/3.9.16/Frameworks/Python.framework/Versions/3.9/lib/python3.9/asyncio/tasks.py\", line 256, in __step\n",
      "    result = coro.send(None)\n",
      "  File \"/Users/levkonstantinovskiy/Documents/GitHub/langchain-agent-qa/lib/python3.9/site-packages/ice/trace.py\", line 276, in inner_wrapper\n",
      "    result = await fn(*args, **kwargs)\n",
      "  File \"/Users/levkonstantinovskiy/Documents/GitHub/langchain-agent-qa/lib/python3.9/site-packages/langchain_visualizer/hijacking.py\", line 18, in run\n",
      "    return self.og_fn(self.og_obj, *args, **kwargs)\n",
      "  File \"/Users/levkonstantinovskiy/Documents/GitHub/langchain-agent-qa/lib/python3.9/site-packages/langchain/chains/base.py\", line 116, in __call__\n",
      "    raise e\n",
      "  File \"/Users/levkonstantinovskiy/Documents/GitHub/langchain-agent-qa/lib/python3.9/site-packages/langchain/chains/base.py\", line 113, in __call__\n",
      "    outputs = self._call(inputs)\n",
      "  File \"/Users/levkonstantinovskiy/Documents/GitHub/langchain-agent-qa/lib/python3.9/site-packages/langchain/chains/llm_math/base.py\", line 130, in _call\n",
      "    return self._process_llm_result(llm_output)\n",
      "  File \"/Users/levkonstantinovskiy/Documents/GitHub/langchain-agent-qa/lib/python3.9/site-packages/langchain/chains/llm_math/base.py\", line 77, in _process_llm_result\n",
      "    output = self._evaluate_expression(expression)\n",
      "  File \"/Users/levkonstantinovskiy/Documents/GitHub/langchain-agent-qa/lib/python3.9/site-packages/langchain/chains/llm_math/base.py\", line 66, in _evaluate_expression\n",
      "    raise ValueError(f\"{e}. Please try again with a valid numerical expression\")\n",
      "ValueError: 'VariableNode' object has no attribute 'today'. Please try again with a valid numerical expression\n"
     ]
    }
   ],
   "source": [
    "\n",
    "tools = load_tools([\"serpapi\", \"llm-math\"], llm=llm_agent)\n",
    "tools.append(\n",
    "    Tool(\n",
    "        name=\"PyCon and PyData DE 2023 Website Index\",\n",
    "        func=lambda q: \";\".join([x.page_content for x in retriever.get_relevant_documents(q)]),\n",
    "        description=f\"Useful when you want answer questions about PyCon and PyData DE 2023.\",\n",
    "    ),\n",
    "    \n",
    ")\n",
    "\n",
    "date_finder = load_tools([\"llm-math\"], llm=llm_agent)[0]\n",
    "date_finder.name = 'A date finder.'\n",
    "date_finder.description = 'Useful for when you need to calculate any date.'\n",
    "tools.append(date_finder)\n",
    "\n",
    "print(tools)\n",
    "\n",
    "index_agent_with_date_finder = initialize_agent(tools, llm_agent, agent=\"zero-shot-react-description\", verbose=True)\n",
    "async def search_agent_with_index_and_date_finder_demo_():\n",
    "    return index_agent_with_date_finder.run(\n",
    "        question\n",
    "    )\n",
    "\n",
    "visualize(search_agent_with_index_and_date_finder_demo_)"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Now it messed up the conference date again. However todays date is now correct.\n",
    "Weird!\n",
    "\n",
    "Not even clear how to fix this dark magic.... :("
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Self - Evaluation\n",
    "\n",
    "If we know a correct answer, then GPT is really good at grading predictions and doing root cause analysis."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [],
   "source": [
    "outputs = [\"-2\", \"2\", \"blue sky\", output]\n",
    "\n",
    "correct_answer = \"On 17 April. There are -2 days\"\n",
    "predictions = [{\"input\": question, \"answer\": correct_answer, \"output\": output} for output in outputs]\n",
    "dataset = [{\"input\": d['input'], \"answer\": d['answer'] } for d in predictions ] \n",
    "\n",
    "\n",
    "from langchain.evaluation.qa import QAEvalChain\n",
    "from langchain.prompts import PromptTemplate\n",
    "\n",
    "eval_template = \"\"\"You are a teacher grading a quiz.\n",
    "You are given a question, the context the question is about, and the student's answer. You are asked to score the student's answer as either CORRECT or INCORRECT, based on the context.\n",
    "Write out in a step by step manner your reasoning to be sure that your conclusion is correct. Avoid simply stating the correct answer at the outset.\n",
    "\n",
    "Example Format:\n",
    "QUESTION: question here\n",
    "CORRECT ANSWER: correct answer\n",
    "STUDENT ANSWER: student's answer here\n",
    "EXPLANATION: step by step reasoning here\n",
    "GRADE: Grade from 0 to 10, where 0 is the lowest (very low similarity) and 10 is the highest (very high similarity)\n",
    "\n",
    "Grade the student answers based ONLY on their factual accuracy. Ignore differences in punctuation and phrasing between the student answer and true answer. It is OK if the student answer contains more information than the true answer, as long as it does not contain any conflicting statements. Begin! \n",
    "\n",
    "\n",
    "\n",
    "QUESTION: {query}\n",
    "CORRECT ANSWER: {answer}\n",
    "STUDENT ANSWER: {result}\n",
    "EXPLANATION:\"\"\"\n",
    "eval_prompt_template = PromptTemplate(\n",
    "    input_variables=[\"query\", \"answer\",  \"result\"], template=eval_template\n",
    ")\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "WARNING:root:Failed to persist run: HTTPConnectionPool(host='localhost', port=8000): Max retries exceeded with url: /llm-runs (Caused by NewConnectionError('<urllib3.connection.HTTPConnection object at 0x2ad9daf20>: Failed to establish a new connection: [Errno 61] Connection refused'))\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "{'text': \" The student's answer is correct, as the PyCon DE & PyData Berlin 2023 conference is on 17 April, and there are -2 days between that date and today.\\nGRADE: 10\"}"
      ]
     },
     "execution_count": 16,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "\n",
    "eval_chain = QAEvalChain.from_llm(llm_eval, prompt=eval_prompt_template)\n",
    "graded_outputs = eval_chain.evaluate(dataset, predictions, question_key=\"input\", prediction_key=\"output\")\n"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Fair grade for its own answer."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      " \n",
      "1. Check the date of the conference: The correct answer is 17 April, while the student answer is 16 January. \n",
      "2. Check the number of days between the conference and today: The correct answer is -2 days, while the student answer is 45017 days. \n",
      "GRADE: 0\n"
     ]
    }
   ],
   "source": [
    "print(graded_outputs[3]['text'])"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "For more ways to evaluate QA see https://github.com/PineappleExpress808/auto-evaluator"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "venv",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.10.8"
  },
  "orig_nbformat": 4
 },
 "nbformat": 4,
 "nbformat_minor": 2
}