pagerank-retrieval-system-reranked.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 104,
   "metadata": {},
   "outputs": [],
   "source": [
    "import json\n",
    "import os\n",
    "from collections import defaultdict\n",
    "import numpy as np\n",
    "import pandas as pd\n",
    "\n",
    "from tira.third_party_integrations import ensure_pyterrier_is_loaded, persist_and_normalize_run\n",
    "from tira.rest_api_client import Client\n",
    "import pyterrier as pt"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 105,
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.linear_model import LogisticRegression\n",
    "from sklearn.model_selection import train_test_split"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 106,
   "metadata": {},
   "outputs": [],
   "source": [
    "ensure_pyterrier_is_loaded()\n",
    "tira = Client()\n",
    "pt_dataset = pt.get_dataset('irds:ir-lab-sose-2024/ir-acl-anthology-20240504-training')\n",
    "topics = pt_dataset.get_topics('text')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 107,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "2623\n"
     ]
    }
   ],
   "source": [
    "qrels_df = pt_dataset.get_qrels()\n",
    "\n",
    "# qrels_dict: {qid: {docno: label}}\n",
    "qrels_dict = defaultdict(dict)\n",
    "\n",
    "for i, row in qrels_df.iterrows():\n",
    "    qrels_dict[row['qid']][row['docno']] = row['label']\n",
    "\n",
    "print(len(qrels_df))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 108,
   "metadata": {},
   "outputs": [],
   "source": [
    "if os.getcwd().endswith('pagerank'):\n",
    "    os.chdir('..')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 109,
   "metadata": {},
   "outputs": [],
   "source": [
    "with open('data/docs_with_all_info.json', 'r') as file:\n",
    "    documents = json.load(file)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 110,
   "metadata": {},
   "outputs": [],
   "source": [
    "df_docs = pd.DataFrame(documents)\n",
    "df_docs = df_docs.transpose()\n",
    "\n",
    "df_docs['docno'] = df_docs.index\n",
    "\n",
    "df_docs['text'] = df_docs['title'] + ' ' + df_docs['abstract']\n",
    "df_docs.drop(columns=['pagerank', 'title', 'abstract', 'discounted_pagerank', 'pub_date'], inplace=True)\n",
    "\n",
    "df_docs.reset_index(drop=True, inplace=True)\n",
    "\n",
    "df_docs = df_docs.to_dict(orient='records')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 111,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "/workspaces/SE-shared-task-v2/ir-lab-sose-2024-augsburg-information-retrieval\n"
     ]
    }
   ],
   "source": [
    "print(os.getcwd())\n",
    "\n",
    "#Create the index using PyTerrier\n",
    "indexer = pt.IterDictIndexer(\n",
    "    \"../index/index_retrieved_docs\",\n",
    "    overwrite=True,\n",
    "    fields=[\"text\"],\n",
    "    meta=[\"docno\"]\n",
    ")\n",
    "\n",
    "# Index the documents\n",
    "index = indexer.index(df_docs)\n",
    "\n",
    "#Retrieve documents using BM25\n",
    "bm25 = pt.BatchRetrieve(index, wmodel=\"BM25\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 112,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "/workspaces/SE-shared-task-v2/ir-lab-sose-2024-augsburg-information-retrieval\n"
     ]
    }
   ],
   "source": [
    "if os.getcwd().endswith('pagerank'):\n",
    "    os.chdir('..')\n",
    "\n",
    "print(os.getcwd())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 113,
   "metadata": {},
   "outputs": [],
   "source": [
    "bm25_run = bm25(pt_dataset.get_topics('text'))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 114,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "66391\n"
     ]
    }
   ],
   "source": [
    "print(len(bm25_run))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Combine all retrieved information in one Dictionary"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 115,
   "metadata": {},
   "outputs": [],
   "source": [
    "bm25_dict = defaultdict(dict)\n",
    "\n",
    "for i, row in bm25_run.iterrows():\n",
    "    bm25_dict[row['qid']][row['docno']] = {'bm25_score': row['score'], \n",
    "                                           'bm25_rank': row['rank'], \n",
    "                                           'pagerank_score': documents[row['docno']]['pagerank'],\n",
    "                                           'discounted_pagerank_score': documents[row['docno']]['discounted_pagerank']}"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Impute missing values"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 116,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "BM25 Imputation:  497.66620475666883\n",
      "Pagerank Imputation:  2.0335444620543548e-06\n",
      "Discounted Pagerank Imputation:  8.441582033060812e-07\n"
     ]
    }
   ],
   "source": [
    "bm25_imputation = np.mean([scores['bm25_rank'] for doc_dict in bm25_dict.values() for scores in doc_dict.values()])\n",
    "pagerank_imputation = np.mean([documents[docno]['pagerank'] for docno in documents])\n",
    "discounted_pagerank_imputation = np.mean([documents[docno]['discounted_pagerank'] for docno in documents])\n",
    "\n",
    "print(\"BM25 Imputation: \", bm25_imputation)\n",
    "print(\"Pagerank Imputation: \", pagerank_imputation)\n",
    "print(\"Discounted Pagerank Imputation: \", discounted_pagerank_imputation)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Include the Labels"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 117,
   "metadata": {},
   "outputs": [],
   "source": [
    "data_dict = defaultdict(dict)\n",
    "\n",
    "for qid, doc_dict in qrels_dict.items():\n",
    "    for docno, label in doc_dict.items():\n",
    "        data_dict[qid][docno] = {}\n",
    "        try:\n",
    "            data_dict[qid][docno]['bm25_score'] = bm25_dict[qid][docno]['bm25_score']\n",
    "        except:\n",
    "            data_dict[qid][docno]['bm25_score'] =  bm25_imputation\n",
    "\n",
    "        try:\n",
    "            data_dict[qid][docno]['pagerank_score'] = bm25_dict[qid][docno]['pagerank_score']\n",
    "        except:\n",
    "            data_dict[qid][docno]['pagerank_score'] = pagerank_imputation\n",
    "\n",
    "        try:\n",
    "            data_dict[qid][docno]['discounted_pagerank_score'] = bm25_dict[qid][docno]['discounted_pagerank_score']\n",
    "        except:\n",
    "            data_dict[qid][docno]['discounted_pagerank_score'] = discounted_pagerank_imputation\n",
    "        \n",
    "        try:\n",
    "            data_dict[qid][docno]['label'] = qrels_dict[qid][docno]\n",
    "        except:\n",
    "            data_dict[qid][docno]['label'] = 0"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Flatten the data for scikit"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 118,
   "metadata": {},
   "outputs": [],
   "source": [
    "flattened_data = []\n",
    "for qid, docs in data_dict.items():\n",
    "    for docno, values in docs.items():\n",
    "        flattened_data.append({\n",
    "            'qid': qid,\n",
    "            'docno': docno,\n",
    "            'bm25': values['bm25_score'],\n",
    "            'pagerank': values['pagerank_score'],\n",
    "            'discounted_pagerank': values['discounted_pagerank_score'],\n",
    "            'label': values['label']\n",
    "        })\n",
    "\n",
    "df = pd.DataFrame(flattened_data)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Use Logistic Regression to use both BM25 and PageRank to retrieve the most relevant documents"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 119,
   "metadata": {},
   "outputs": [],
   "source": [
    "#Prepare training data\n",
    "X = df[['bm25', 'discounted_pagerank']]\n",
    "y = df['label']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 120,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<style>#sk-container-id-5 {color: black;}#sk-container-id-5 pre{padding: 0;}#sk-container-id-5 div.sk-toggleable {background-color: white;}#sk-container-id-5 label.sk-toggleable__label {cursor: pointer;display: block;width: 100%;margin-bottom: 0;padding: 0.3em;box-sizing: border-box;text-align: center;}#sk-container-id-5 label.sk-toggleable__label-arrow:before {content: \"▸\";float: left;margin-right: 0.25em;color: #696969;}#sk-container-id-5 label.sk-toggleable__label-arrow:hover:before {color: black;}#sk-container-id-5 div.sk-estimator:hover label.sk-toggleable__label-arrow:before {color: black;}#sk-container-id-5 div.sk-toggleable__content {max-height: 0;max-width: 0;overflow: hidden;text-align: left;background-color: #f0f8ff;}#sk-container-id-5 div.sk-toggleable__content pre {margin: 0.2em;color: black;border-radius: 0.25em;background-color: #f0f8ff;}#sk-container-id-5 input.sk-toggleable__control:checked~div.sk-toggleable__content {max-height: 200px;max-width: 100%;overflow: auto;}#sk-container-id-5 input.sk-toggleable__control:checked~label.sk-toggleable__label-arrow:before {content: \"▾\";}#sk-container-id-5 div.sk-estimator input.sk-toggleable__control:checked~label.sk-toggleable__label {background-color: #d4ebff;}#sk-container-id-5 div.sk-label input.sk-toggleable__control:checked~label.sk-toggleable__label {background-color: #d4ebff;}#sk-container-id-5 input.sk-hidden--visually {border: 0;clip: rect(1px 1px 1px 1px);clip: rect(1px, 1px, 1px, 1px);height: 1px;margin: -1px;overflow: hidden;padding: 0;position: absolute;width: 1px;}#sk-container-id-5 div.sk-estimator {font-family: monospace;background-color: #f0f8ff;border: 1px dotted black;border-radius: 0.25em;box-sizing: border-box;margin-bottom: 0.5em;}#sk-container-id-5 div.sk-estimator:hover {background-color: #d4ebff;}#sk-container-id-5 div.sk-parallel-item::after {content: \"\";width: 100%;border-bottom: 1px solid gray;flex-grow: 1;}#sk-container-id-5 div.sk-label:hover label.sk-toggleable__label {background-color: #d4ebff;}#sk-container-id-5 div.sk-serial::before {content: \"\";position: absolute;border-left: 1px solid gray;box-sizing: border-box;top: 0;bottom: 0;left: 50%;z-index: 0;}#sk-container-id-5 div.sk-serial {display: flex;flex-direction: column;align-items: center;background-color: white;padding-right: 0.2em;padding-left: 0.2em;position: relative;}#sk-container-id-5 div.sk-item {position: relative;z-index: 1;}#sk-container-id-5 div.sk-parallel {display: flex;align-items: stretch;justify-content: center;background-color: white;position: relative;}#sk-container-id-5 div.sk-item::before, #sk-container-id-5 div.sk-parallel-item::before {content: \"\";position: absolute;border-left: 1px solid gray;box-sizing: border-box;top: 0;bottom: 0;left: 50%;z-index: -1;}#sk-container-id-5 div.sk-parallel-item {display: flex;flex-direction: column;z-index: 1;position: relative;background-color: white;}#sk-container-id-5 div.sk-parallel-item:first-child::after {align-self: flex-end;width: 50%;}#sk-container-id-5 div.sk-parallel-item:last-child::after {align-self: flex-start;width: 50%;}#sk-container-id-5 div.sk-parallel-item:only-child::after {width: 0;}#sk-container-id-5 div.sk-dashed-wrapped {border: 1px dashed gray;margin: 0 0.4em 0.5em 0.4em;box-sizing: border-box;padding-bottom: 0.4em;background-color: white;}#sk-container-id-5 div.sk-label label {font-family: monospace;font-weight: bold;display: inline-block;line-height: 1.2em;}#sk-container-id-5 div.sk-label-container {text-align: center;}#sk-container-id-5 div.sk-container {/* jupyter's `normalize.less` sets `[hidden] { display: none; }` but bootstrap.min.css set `[hidden] { display: none !important; }` so we also need the `!important` here to be able to override the default hidden behavior on the sphinx rendered scikit-learn.org. See: https://github.com/scikit-learn/scikit-learn/issues/21755 */display: inline-block !important;position: relative;}#sk-container-id-5 div.sk-text-repr-fallback {display: none;}</style><div id=\"sk-container-id-5\" class=\"sk-top-container\"><div class=\"sk-text-repr-fallback\"><pre>LogisticRegression()</pre><b>In a Jupyter environment, please rerun this cell to show the HTML representation or trust the notebook. <br />On GitHub, the HTML representation is unable to render, please try loading this page with nbviewer.org.</b></div><div class=\"sk-container\" hidden><div class=\"sk-item\"><div class=\"sk-estimator sk-toggleable\"><input class=\"sk-toggleable__control sk-hidden--visually\" id=\"sk-estimator-id-5\" type=\"checkbox\" checked><label for=\"sk-estimator-id-5\" class=\"sk-toggleable__label sk-toggleable__label-arrow\">LogisticRegression</label><div class=\"sk-toggleable__content\"><pre>LogisticRegression()</pre></div></div></div></div></div>"
      ],
      "text/plain": [
       "LogisticRegression()"
      ]
     },
     "execution_count": 120,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.2, random_state=420)\n",
    "\n",
    "model = LogisticRegression()\n",
    "model.fit(X, y)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 121,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Predict relevance scores\n",
    "df['predicted_relevance'] = model.predict_proba(X)[:, 1]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 122,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Select top 10 documents for each query\n",
    "top_10_docs = df.sort_values(by=['qid', 'predicted_relevance'], ascending=[True, False]).groupby('qid').head(10)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 123,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Learned weights for BM25 and PageRank: [[-1.68452733e-03  2.78602961e-07]]\n"
     ]
    }
   ],
   "source": [
    "# Save the resulting document orders in a JSON file\n",
    "result = {}\n",
    "for qid, group in top_10_docs.groupby('qid'):\n",
    "    result[qid] = group[['docno', 'predicted_relevance']].to_dict(orient='records')\n",
    "\n",
    "with open('top_10_docs.json', 'w') as f:\n",
    "    json.dump(result, f, indent=4)\n",
    "\n",
    "# Print learned weights\n",
    "print(\"Learned weights for BM25 and PageRank:\", model.coef_)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 124,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "   qid                               docno     score      bm25  rank\n",
      "1    1  2019.tois_journal-ir0volumeA37A1.2  0.575487  8.840352     0\n",
      "3    1   2015.ipm_journal-ir0volumeA51A5.7  0.575381  9.098508     1\n",
      "39   1     1998.sigirconf_conference-98.22  0.575288  9.323570     2\n",
      "17   1        2008.cikm_conference-2008.59  0.575184  9.576583     3\n",
      "40   1       2009.cikm_conference-2009.277  0.575121  9.730872     4\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/tmp/ipykernel_3914/4026046920.py:3: SettingWithCopyWarning: \n",
      "A value is trying to be set on a copy of a slice from a DataFrame\n",
      "\n",
      "See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy\n",
      "  run.rename(columns={'predicted_relevance': 'score'}, inplace=True)\n",
      "/tmp/ipykernel_3914/4026046920.py:5: SettingWithCopyWarning: \n",
      "A value is trying to be set on a copy of a slice from a DataFrame\n",
      "\n",
      "See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy\n",
      "  run.sort_values(by=['qid', 'score'], ascending=[True, False], inplace=True)\n",
      "/tmp/ipykernel_3914/4026046920.py:7: SettingWithCopyWarning: \n",
      "A value is trying to be set on a copy of a slice from a DataFrame.\n",
      "Try using .loc[row_indexer,col_indexer] = value instead\n",
      "\n",
      "See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy\n",
      "  run['rank'] = run.groupby('qid').cumcount()\n"
     ]
    }
   ],
   "source": [
    "run = df[['qid', 'docno', 'predicted_relevance', 'bm25']]\n",
    "\n",
    "run.rename(columns={'predicted_relevance': 'score'}, inplace=True)\n",
    "\n",
    "run.sort_values(by=['qid', 'score'], ascending=[True, False], inplace=True)\n",
    "\n",
    "run['rank'] = run.groupby('qid').cumcount()\n",
    "\n",
    "print(run.head(5))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 125,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Evaluation Metrics:\n",
      "{'map': 0.547369892273752, 'ndcg': 0.7323267767648299, 'ndcg_cut.10': 0.45129355824419753, 'recip_rank': 0.5550540146128383, 'recall_100': 0.9852941176470589}\n"
     ]
    }
   ],
   "source": [
    "eval = pt.Evaluate(run, qrels_df, metrics=[\"map\", \"ndcg\", \"ndcg_cut.10\", \"recip_rank\", \"recall_100\"])\n",
    "print(\"Evaluation Metrics:\")\n",
    "print(eval)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 126,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "The run file is normalized outside the TIRA sandbox, I will store it at \"runs/bm25_and_discounted_pagerank_run.txt\".\n",
      "Done. run file is stored under \"runs/bm25_and_discounted_pagerank_run.txt\".\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/usr/local/lib/python3.10/dist-packages/tira/third_party_integrations.py:129: SettingWithCopyWarning: \n",
      "A value is trying to be set on a copy of a slice from a DataFrame.\n",
      "Try using .loc[row_indexer,col_indexer] = value instead\n",
      "\n",
      "See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy\n",
      "  run['qid'] = run['qid'].astype(int)\n",
      "/usr/local/lib/python3.10/dist-packages/tira/third_party_integrations.py:133: SettingWithCopyWarning: \n",
      "A value is trying to be set on a copy of a slice from a DataFrame.\n",
      "Try using .loc[row_indexer,col_indexer] = value instead\n",
      "\n",
      "See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy\n",
      "  run['system'] = system_name\n"
     ]
    }
   ],
   "source": [
    "persist_and_normalize_run(run, system_name='BM25_and_discounted_pagerank', default_output='runs/bm25_and_discounted_pagerank_run.txt')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Use ReRank to further improve results"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 127,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "ir-lab-sose-2024/ir-acl-anthology-20240504-training documents: 100%|██████████| 126958/126958 [00:01<00:00, 74321.01it/s]\n",
      "/tmp/ipykernel_3914/859766974.py:12: SettingWithCopyWarning: \n",
      "A value is trying to be set on a copy of a slice from a DataFrame.\n",
      "Try using .loc[row_indexer,col_indexer] = value instead\n",
      "\n",
      "See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy\n",
      "  run['qid'] = run['qid'].astype(str)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Done. Here are the first 10 entries of the run\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>qid</th>\n",
       "      <th>docno</th>\n",
       "      <th>score</th>\n",
       "      <th>bm25</th>\n",
       "      <th>rank</th>\n",
       "      <th>system</th>\n",
       "      <th>text</th>\n",
       "      <th>query</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>1</td>\n",
       "      <td>2019.tois_journal-ir0volumeA37A1.2</td>\n",
       "      <td>0.575487</td>\n",
       "      <td>8.840352</td>\n",
       "      <td>0</td>\n",
       "      <td>BM25_and_discounted_pagerank</td>\n",
       "      <td>Learning to Adaptively Rank Document Retrieval...</td>\n",
       "      <td>retrieval system improving effectiveness</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>1</td>\n",
       "      <td>2015.ipm_journal-ir0volumeA51A5.7</td>\n",
       "      <td>0.575381</td>\n",
       "      <td>9.098508</td>\n",
       "      <td>1</td>\n",
       "      <td>BM25_and_discounted_pagerank</td>\n",
       "      <td>Statistical comparisons of non-deterministic I...</td>\n",
       "      <td>retrieval system improving effectiveness</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>1</td>\n",
       "      <td>1998.sigirconf_conference-98.22</td>\n",
       "      <td>0.575288</td>\n",
       "      <td>9.323570</td>\n",
       "      <td>2</td>\n",
       "      <td>BM25_and_discounted_pagerank</td>\n",
       "      <td>Aspect Windows, 3-D Visualizations, and Indire...</td>\n",
       "      <td>retrieval system improving effectiveness</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "  qid                               docno     score      bm25  rank  \\\n",
       "0   1  2019.tois_journal-ir0volumeA37A1.2  0.575487  8.840352     0   \n",
       "1   1   2015.ipm_journal-ir0volumeA51A5.7  0.575381  9.098508     1   \n",
       "2   1     1998.sigirconf_conference-98.22  0.575288  9.323570     2   \n",
       "\n",
       "                         system  \\\n",
       "0  BM25_and_discounted_pagerank   \n",
       "1  BM25_and_discounted_pagerank   \n",
       "2  BM25_and_discounted_pagerank   \n",
       "\n",
       "                                                text  \\\n",
       "0  Learning to Adaptively Rank Document Retrieval...   \n",
       "1  Statistical comparisons of non-deterministic I...   \n",
       "2  Aspect Windows, 3-D Visualizations, and Indire...   \n",
       "\n",
       "                                      query  \n",
       "0  retrieval system improving effectiveness  \n",
       "1  retrieval system improving effectiveness  \n",
       "2  retrieval system improving effectiveness  "
      ]
     },
     "execution_count": 127,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "from ReRank.ReRank import ReRank\n",
    "\n",
    "# Create a df with the text documents\n",
    "documents = pt_dataset.get_corpus_iter() \n",
    "# Extract docno and text into a DataFrame\n",
    "doc_list = []\n",
    "for doc in documents:\n",
    "    doc_list.append(doc)\n",
    "documents = pd.DataFrame(doc_list)\n",
    "\n",
    "queries = pt_dataset.get_topics('text')\n",
    "run['qid'] = run['qid'].astype(str)\n",
    "\n",
    "\n",
    "run = run.merge(documents, on='docno', how='left') # Merge the run and the documents\n",
    "run = run.merge(queries, on='qid', how='left') # Merge the run and the queries\n",
    "print('Done. Here are the first 10 entries of the run')\n",
    "run.head(3)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 128,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "A parameter name that contains `beta` will be renamed internally to `bias`. Please use a different name to suppress this warning.\n",
      "A parameter name that contains `gamma` will be renamed internally to `weight`. Please use a different name to suppress this warning.\n",
      "A parameter name that contains `beta` will be renamed internally to `bias`. Please use a different name to suppress this warning.\n",
      "A parameter name that contains `gamma` will be renamed internally to `weight`. Please use a different name to suppress this warning.\n",
      "A parameter name that contains `beta` will be renamed internally to `bias`. Please use a different name to suppress this warning.\n",
      "A parameter name that contains `gamma` will be renamed internally to `weight`. Please use a different name to suppress this warning.\n",
      "A parameter name that contains `beta` will be renamed internally to `bias`. Please use a different name to suppress this warning.\n",
      "A parameter name that contains `gamma` will be renamed internally to `weight`. Please use a different name to suppress this warning.\n",
      "A parameter name that contains `beta` will be renamed internally to `bias`. Please use a different name to suppress this warning.\n",
      "A parameter name that contains `gamma` will be renamed internally to `weight`. Please use a different name to suppress this warning.\n",
      "A parameter name that contains `beta` will be renamed internally to `bias`. Please use a different name to suppress this warning.\n",
      "A parameter name that contains `gamma` will be renamed internally to `weight`. Please use a different name to suppress this warning.\n",
      "A parameter name that contains `beta` will be renamed internally to `bias`. Please use a different name to suppress this warning.\n",
      "A parameter name that contains `gamma` will be renamed internally to `weight`. Please use a different name to suppress this warning.\n",
      "A parameter name that contains `beta` will be renamed internally to `bias`. Please use a different name to suppress this warning.\n",
      "A parameter name that contains `gamma` will be renamed internally to `weight`. Please use a different name to suppress this warning.\n",
      "A parameter name that contains `beta` will be renamed internally to `bias`. Please use a different name to suppress this warning.\n",
      "A parameter name that contains `gamma` will be renamed internally to `weight`. Please use a different name to suppress this warning.\n",
      "A parameter name that contains `beta` will be renamed internally to `bias`. Please use a different name to suppress this warning.\n",
      "A parameter name that contains `gamma` will be renamed internally to `weight`. Please use a different name to suppress this warning.\n",
      "A parameter name that contains `beta` will be renamed internally to `bias`. Please use a different name to suppress this warning.\n",
      "A parameter name that contains `gamma` will be renamed internally to `weight`. Please use a different name to suppress this warning.\n",
      "A parameter name that contains `beta` will be renamed internally to `bias`. Please use a different name to suppress this warning.\n",
      "A parameter name that contains `gamma` will be renamed internally to `weight`. Please use a different name to suppress this warning.\n",
      "A parameter name that contains `beta` will be renamed internally to `bias`. Please use a different name to suppress this warning.\n",
      "A parameter name that contains `gamma` will be renamed internally to `weight`. Please use a different name to suppress this warning.\n",
      "A parameter name that contains `beta` will be renamed internally to `bias`. Please use a different name to suppress this warning.\n",
      "A parameter name that contains `gamma` will be renamed internally to `weight`. Please use a different name to suppress this warning.\n",
      "A parameter name that contains `beta` will be renamed internally to `bias`. Please use a different name to suppress this warning.\n",
      "A parameter name that contains `gamma` will be renamed internally to `weight`. Please use a different name to suppress this warning.\n",
      "A parameter name that contains `beta` will be renamed internally to `bias`. Please use a different name to suppress this warning.\n",
      "A parameter name that contains `gamma` will be renamed internally to `weight`. Please use a different name to suppress this warning.\n",
      "A parameter name that contains `beta` will be renamed internally to `bias`. Please use a different name to suppress this warning.\n",
      "A parameter name that contains `gamma` will be renamed internally to `weight`. Please use a different name to suppress this warning.\n",
      "A parameter name that contains `beta` will be renamed internally to `bias`. Please use a different name to suppress this warning.\n",
      "A parameter name that contains `gamma` will be renamed internally to `weight`. Please use a different name to suppress this warning.\n",
      "A parameter name that contains `beta` will be renamed internally to `bias`. Please use a different name to suppress this warning.\n",
      "A parameter name that contains `gamma` will be renamed internally to `weight`. Please use a different name to suppress this warning.\n",
      "A parameter name that contains `beta` will be renamed internally to `bias`. Please use a different name to suppress this warning.\n",
      "A parameter name that contains `gamma` will be renamed internally to `weight`. Please use a different name to suppress this warning.\n",
      "A parameter name that contains `beta` will be renamed internally to `bias`. Please use a different name to suppress this warning.\n",
      "A parameter name that contains `gamma` will be renamed internally to `weight`. Please use a different name to suppress this warning.\n",
      "A parameter name that contains `beta` will be renamed internally to `bias`. Please use a different name to suppress this warning.\n",
      "A parameter name that contains `gamma` will be renamed internally to `weight`. Please use a different name to suppress this warning.\n",
      "A parameter name that contains `beta` will be renamed internally to `bias`. Please use a different name to suppress this warning.\n",
      "A parameter name that contains `gamma` will be renamed internally to `weight`. Please use a different name to suppress this warning.\n",
      "A parameter name that contains `beta` will be renamed internally to `bias`. Please use a different name to suppress this warning.\n",
      "A parameter name that contains `gamma` will be renamed internally to `weight`. Please use a different name to suppress this warning.\n",
      "A parameter name that contains `beta` will be renamed internally to `bias`. Please use a different name to suppress this warning.\n",
      "A parameter name that contains `gamma` will be renamed internally to `weight`. Please use a different name to suppress this warning.\n",
      "A parameter name that contains `beta` will be renamed internally to `bias`. Please use a different name to suppress this warning.\n",
      "A parameter name that contains `gamma` will be renamed internally to `weight`. Please use a different name to suppress this warning.\n"
     ]
    }
   ],
   "source": [
    "# Create ReRank object\n",
    "ReRank_object = ReRank()\n",
    "\n",
    "# Rerank the documents\n",
    "run = ReRank_object.rerank_documents(df=run, top_n=5)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 129,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "The run file is normalized outside the TIRA sandbox, I will store it at \"runs/bm25_and_discounted_pagerank_reranked_run.txt\".\n",
      "Done. run file is stored under \"runs/bm25_and_discounted_pagerank_reranked_run.txt\".\n"
     ]
    }
   ],
   "source": [
    "persist_and_normalize_run(run, system_name='BM25_and_discounted_pagerank_reranked', default_output='runs/bm25_and_discounted_pagerank_reranked_run.txt')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Evaluate results"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 130,
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.metrics import classification_report, confusion_matrix\n",
    "from scipy.optimize import minimize\n",
    "from sklearn.metrics import accuracy_score, classification_report"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 131,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Accuracy: 0.5714285714285714\n",
      "[[101 159]\n",
      " [ 66 199]]\n",
      "              precision    recall  f1-score   support\n",
      "\n",
      "           0       0.60      0.39      0.47       260\n",
      "           1       0.56      0.75      0.64       265\n",
      "\n",
      "    accuracy                           0.57       525\n",
      "   macro avg       0.58      0.57      0.56       525\n",
      "weighted avg       0.58      0.57      0.56       525\n",
      "\n"
     ]
    }
   ],
   "source": [
    "y_pred = model.predict(X_test)\n",
    "\n",
    "print(\"Accuracy:\", accuracy_score(y_test, y_pred))\n",
    "\n",
    "print(confusion_matrix(y_test, y_pred))\n",
    "print(classification_report(y_test, y_pred))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Try freezing the bm_25_weight\n",
    "\n",
    "This always leads to the pagerank weight being set to 0."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 132,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Accuracy: 0.5047619047619047\n",
      "              precision    recall  f1-score   support\n",
      "\n",
      "           0       0.00      0.00      0.00       260\n",
      "           1       0.50      1.00      0.67       265\n",
      "\n",
      "    accuracy                           0.50       525\n",
      "   macro avg       0.25      0.50      0.34       525\n",
      "weighted avg       0.25      0.50      0.34       525\n",
      "\n",
      "Optimized coefficients: [0.0001 0.    ]\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/usr/local/lib/python3.10/dist-packages/sklearn/metrics/_classification.py:1471: UndefinedMetricWarning: Precision and F-score are ill-defined and being set to 0.0 in labels with no predicted samples. Use `zero_division` parameter to control this behavior.\n",
      "  _warn_prf(average, modifier, msg_start, len(result))\n",
      "/usr/local/lib/python3.10/dist-packages/sklearn/metrics/_classification.py:1471: UndefinedMetricWarning: Precision and F-score are ill-defined and being set to 0.0 in labels with no predicted samples. Use `zero_division` parameter to control this behavior.\n",
      "  _warn_prf(average, modifier, msg_start, len(result))\n",
      "/usr/local/lib/python3.10/dist-packages/sklearn/metrics/_classification.py:1471: UndefinedMetricWarning: Precision and F-score are ill-defined and being set to 0.0 in labels with no predicted samples. Use `zero_division` parameter to control this behavior.\n",
      "  _warn_prf(average, modifier, msg_start, len(result))\n"
     ]
    }
   ],
   "source": [
    "def logistic_loss(beta, X, y):\n",
    "    z = np.dot(X, beta)\n",
    "    prob = 1 / (1 + np.exp(-z))\n",
    "    loss = -np.mean(y * np.log(prob) + (1 - y) * np.log(1 - prob))\n",
    "    return loss\n",
    "\n",
    "def frozen_loss(beta, X, y, freeze_index, freeze_bm25_weight):\n",
    "    beta_with_freeze = np.copy(beta)\n",
    "    beta_with_freeze[freeze_index] = freeze_bm25_weight\n",
    "    return logistic_loss(beta_with_freeze, X, y)\n",
    "\n",
    "freeze_bm25_weight = 0.0001  \n",
    "\n",
    "if freeze_bm25_weight != 0:\n",
    "    initial_beta = np.zeros(X_train.shape[1])\n",
    "    freeze_index = 0\n",
    "\n",
    "    result = minimize(frozen_loss, initial_beta, args=(X_train, y_train, freeze_index, freeze_bm25_weight), method='BFGS')\n",
    "\n",
    "    # Extract the optimized coefficients\n",
    "    optimized_beta = result.x\n",
    "    optimized_beta[freeze_index] = freeze_bm25_weight \n",
    "\n",
    "    y_pred_prob = 1 / (1 + np.exp(-np.dot(X_test, optimized_beta)))\n",
    "    y_pred = (y_pred_prob >= 0.5).astype(int)\n",
    "\n",
    "    print(\"Accuracy:\", accuracy_score(y_test, y_pred))\n",
    "    print(classification_report(y_test, y_pred))\n",
    "    print(\"Optimized coefficients:\", optimized_beta)"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.10.12"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}