4 - Evaluation procedure.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2021-03-26T02:22:07.014878Z",
     "start_time": "2021-03-26T02:22:06.547979Z"
    }
   },
   "outputs": [],
   "source": [
    "import os\n",
    "import torch\n",
    "import numpy as np\n",
    "import pandas as pd\n",
    "from skimage import io\n",
    "from tqdm.auto import tqdm\n",
    "\n",
    "from models import VBPR, DVBPR, ACF, CuratorNet\n",
    "from datasets.user_mode_img import ToTensor\n",
    "from utils.data import extract_embedding\n",
    "from utils.metrics import (\n",
    "    auc_exact,\n",
    "    nDCG,\n",
    "    precision,\n",
    "    recall,\n",
    "    reciprocal_rank,\n",
    ")\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Evaluation procedure"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2021-03-26T02:22:08.555918Z",
     "start_time": "2021-03-26T02:22:08.553818Z"
    }
   },
   "outputs": [],
   "source": [
    "# Dataset\n",
    "DATASET = \"Wikimedia\"\n",
    "assert DATASET in [\"UGallery\", \"Wikimedia\"]\n",
    "\n",
    "# Model\n",
    "MODEL = \"VBPR\"\n",
    "assert MODEL in [\"VBPR\", \"DVBPR\", \"CuratorNet\", \"ACF\"]\n",
    "\n",
    "FEATURE_EXTRACTOR = \"resnet50\"\n",
    "assert FEATURE_EXTRACTOR in [\"alexnet\", \"vgg16\", \"resnet50\"]\n",
    "\n",
    "FEATURE_LAYER = \"layer4\"\n",
    "FEATURE_LAYER = FEATURE_LAYER if MODEL == \"ACF\" else \"\" "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2021-03-26T02:22:10.377057Z",
     "start_time": "2021-03-26T02:22:10.375132Z"
    }
   },
   "outputs": [],
   "source": [
    "# Mode\n",
    "# Use 'MODE_PROFILE = True' for CuratorNet-like training \n",
    "# Use 'MODE_PROFILE = False' for VBPR-like training\n",
    "MODE_PROFILE = MODEL in [\"CuratorNet\"]\n",
    "MODE_PROFILE = \"profile\" if MODE_PROFILE else \"user\"\n",
    "\n",
    "# Checkpoint (ex. 'VBPR_wikimedia')\n",
    "CHECKPOINT = f\"{MODEL}_{DATASET.lower()}\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2021-03-26T02:22:11.286344Z",
     "start_time": "2021-03-26T02:22:11.283714Z"
    }
   },
   "outputs": [],
   "source": [
    "# Paths (general)\n",
    "CHECKPOINT_EXT = \"pt\" if MODEL == \"ACF\" else \"tar\" \n",
    "CHECKPOINT_PATH = os.path.join(\"checkpoints\", f\"{CHECKPOINT}.{CHECKPOINT_EXT}\")\n",
    "FEATURE_EXTRACTOR = f\"{FEATURE_EXTRACTOR}-{FEATURE_LAYER}\" if FEATURE_LAYER else FEATURE_EXTRACTOR\n",
    "EMBEDDING_PATH = os.path.join(\"data\", DATASET, f\"embedding-{FEATURE_EXTRACTOR}.npy\")\n",
    "EVALUATION_PATH = os.path.join(\"data\", DATASET, f\"naive-{MODE_PROFILE}-evaluation.csv\")\n",
    "\n",
    "# Paths (images)\n",
    "IMAGES_DIR = None\n",
    "if DATASET == \"Wikimedia\":\n",
    "    IMAGES_DIR = os.path.join(\"/\", \"mnt\", \"data2\", \"wikimedia\", \"mini-images-224-224-v2\")  # IMAGES_DIR = os.path.join(\"data\", \"mini-images-224-224-v2\", \"mini-images-224-224-v2\")\n",
    "elif DATASET == \"UGallery\":\n",
    "    IMAGES_DIR = os.path.join(\"/\", \"mnt\", \"workspace\", \"Ugallery\", \"mini-images-224-224-v2\")\n",
    "\n",
    "# General constants\n",
    "RNG_SEED = 0\n",
    "USE_GPU = True"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2021-03-26T02:22:12.084777Z",
     "start_time": "2021-03-26T02:22:12.082033Z"
    }
   },
   "outputs": [],
   "source": [
    "# Freezing RNG seed if needed\n",
    "if RNG_SEED is not None:\n",
    "    print(f\"\\nUsing random seed... ({RNG_SEED})\")\n",
    "    torch.manual_seed(RNG_SEED)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2021-03-26T02:22:13.288058Z",
     "start_time": "2021-03-26T02:22:12.722614Z"
    }
   },
   "outputs": [],
   "source": [
    "# Load embedding from file\n",
    "print(f\"\\nLoading embedding from file... ({EMBEDDING_PATH})\")\n",
    "embedding = np.load(EMBEDDING_PATH, allow_pickle=True)\n",
    "\n",
    "# Extract features and \"id2index\" mapping\n",
    "print(\"\\nExtracting data into variables...\")\n",
    "features, id2index, item_index2fn = extract_embedding(embedding, verbose=True)\n",
    "print(f\">> Features shape: {features.shape}\")\n",
    "del embedding  # Release some memory"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2021-03-26T02:22:14.006894Z",
     "start_time": "2021-03-26T02:22:13.987811Z"
    }
   },
   "outputs": [],
   "source": [
    "# Load evaluation dataframe\n",
    "print(\"\\nLoad evaluation dataframe\")\n",
    "evaluation_df = pd.read_csv(EVALUATION_PATH)\n",
    "# Transform lists from str to int\n",
    "string_to_list = lambda s: list(map(int, s.split()))\n",
    "evaluation_df[\"profile\"] = evaluation_df[\"profile\"].apply(\n",
    "    lambda s: string_to_list(s) if isinstance(s, str) else s,\n",
    ")\n",
    "evaluation_df[\"predict\"] = evaluation_df[\"predict\"].apply(\n",
    "    lambda s: string_to_list(s) if isinstance(s, str) else s,\n",
    ")\n",
    "# Group evaluations by profile and user\n",
    "evaluation_df[\"profile\"] = evaluation_df[\"profile\"].map(tuple)\n",
    "evaluation_df = evaluation_df.groupby([\"profile\", \"user_id\"]).agg({\"predict\": sum}).reset_index()\n",
    "evaluation_df[\"profile\"] = evaluation_df[\"profile\"].map(list)\n",
    "print(f\">> Evaluation: {evaluation_df.shape}\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2021-03-26T02:25:06.857895Z",
     "start_time": "2021-03-26T02:25:04.884129Z"
    }
   },
   "outputs": [],
   "source": [
    "# Create device instance\n",
    "print(\"\\nDevice initialization\")\n",
    "device = torch.device(\"cuda:0\" if torch.cuda.is_available() and USE_GPU else \"cpu\")\n",
    "if torch.cuda.is_available() != USE_GPU:\n",
    "    print((f\"\\nNotice: Not using GPU - \"\n",
    "           f\"Cuda available ({torch.cuda.is_available()}) \"\n",
    "           f\"does not match USE_GPU ({USE_GPU})\"\n",
    "    ))\n",
    "\n",
    "# Loading checkpoint\n",
    "if CHECKPOINT is not None:\n",
    "    print(\"\\nLoading checkpoint\")\n",
    "    checkpoint = torch.load(CHECKPOINT_PATH, map_location=torch.device(\"cpu\"))\n",
    "    if 'epoch' in checkpoint and 'accuracy' in checkpoint:\n",
    "        print(f\">> Best epoch: {checkpoint['epoch']} | Best accuracy: {checkpoint['accuracy']}\")\n",
    "    elif 'epoch' in checkpoint and 'loss' in checkpoint:\n",
    "        print(f\">> Best epoch: {checkpoint['epoch']} | Best Loss: {checkpoint['loss']}\")\n",
    "\n",
    "# Model initialization\n",
    "print(\"\\nModel initialization\")\n",
    "model = None\n",
    "checkpoint_loaded = False\n",
    "if MODEL == \"VBPR\":\n",
    "    n_users = checkpoint[\"model\"][\"gamma_users.weight\"].size(0)\n",
    "    n_items = checkpoint[\"model\"][\"gamma_items.weight\"].size(0)\n",
    "    dim_gamma = checkpoint[\"model\"][\"gamma_users.weight\"].size(1)\n",
    "    dim_theta = checkpoint[\"model\"][\"theta_users.weight\"].size(1)\n",
    "    model = VBPR(\n",
    "        n_users, n_items,  # Number of users and items\n",
    "        torch.Tensor(features),  # Pretrained visual features\n",
    "        dim_gamma, dim_theta,  # Size of internal spaces\n",
    "    ).to(device)\n",
    "elif MODEL == \"CuratorNet\":\n",
    "    model = CuratorNet(\n",
    "        torch.Tensor(features),\n",
    "        input_size=features.shape[1],\n",
    "    ).to(device)\n",
    "elif MODEL == \"ACF\":\n",
    "    model = ACF.from_checkpoint(checkpoint, device=device)\n",
    "    checkpoint_loaded = True\n",
    "elif MODEL == \"DVBPR\":\n",
    "    n_users = checkpoint[\"model\"][\"theta_users.weight\"].size(0)\n",
    "    n_items = checkpoint[\"model\"][\"gamma_items.weight\"].size(0)\n",
    "    K = checkpoint[\"model\"][\"theta_users.weight\"].size(1)\n",
    "    model = DVBPR(n_users, n_items, K=K).to(device)\n",
    "    \n",
    "# Load state dict\n",
    "if not checkpoint_loaded and CHECKPOINT is not None:\n",
    "    model.load_state_dict(checkpoint[\"model\"])\n",
    "    print('loaded')\n",
    "\n",
    "# Change model mode to eval\n",
    "print(\"\\nChanging model mode to eval\")\n",
    "model.eval()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2021-03-26T02:25:12.155616Z",
     "start_time": "2021-03-26T02:25:12.153969Z"
    }
   },
   "outputs": [],
   "source": [
    "# Predict all\n",
    "# If True, ranks every item including already consumed items\n",
    "# If False, ranks ALL - PROFILE (consumed) + PREDICT (ground truth)\n",
    "PREDICT_ALL = False"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2021-03-26T02:25:18.043466Z",
     "start_time": "2021-03-26T02:25:15.699736Z"
    }
   },
   "outputs": [],
   "source": [
    "%%time\n",
    "# Metrics\n",
    "N_EVALS = len(evaluation_df.index)\n",
    "# Area Under the Curve (AUC)\n",
    "AUC = torch.zeros([N_EVALS], dtype=torch.float64, device=device)\n",
    "# Reciprocal Rank (RR)\n",
    "RR = torch.zeros([N_EVALS], dtype=torch.float64, device=device)\n",
    "# Recall\n",
    "R20 = torch.zeros([N_EVALS], dtype=torch.float64, device=device)\n",
    "R100 = torch.zeros([N_EVALS], dtype=torch.float64, device=device)\n",
    "R200 = torch.zeros([N_EVALS], dtype=torch.float64, device=device)\n",
    "# Precision\n",
    "P20 = torch.zeros([N_EVALS], dtype=torch.float64, device=device)\n",
    "P100 = torch.zeros([N_EVALS], dtype=torch.float64, device=device)\n",
    "P200 = torch.zeros([N_EVALS], dtype=torch.float64, device=device)\n",
    "# Normalized discounted cumulative gain (nDCG)\n",
    "N20 = torch.zeros([N_EVALS], dtype=torch.float64, device=device)\n",
    "N100 = torch.zeros([N_EVALS], dtype=torch.float64, device=device)\n",
    "N200 = torch.zeros([N_EVALS], dtype=torch.float64, device=device)\n",
    "PROFILE_SIZES = torch.zeros([N_EVALS], dtype=int, device=device)\n",
    "N_ITEMS = len(features)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2021-03-26T02:25:18.043466Z",
     "start_time": "2021-03-26T02:25:15.699736Z"
    }
   },
   "outputs": [],
   "source": [
    "if MODEL in (\"VBPR\", \"CuratorNet\"):\n",
    "    cache = model.generate_cache()\n",
    "elif MODEL == \"DVBPR\":\n",
    "    def getimg(path, tensorizer):\n",
    "        img = io.imread(path)\n",
    "        return tensorizer(img)\n",
    "    \n",
    "    imglist = {}\n",
    "    for path in tqdm(os.listdir(IMAGES_DIR)):\n",
    "        if path in item_index2fn.values():\n",
    "            img = getimg(os.path.join(IMAGES_DIR, path), ToTensor()) \n",
    "            name = path.split('.')[0]\n",
    "            imglist[id2index[name]] = img\n",
    "\n",
    "    assert len(imglist) == N_ITEMS\n",
    "    print('images loaded:', N_ITEMS)\n",
    "    cache = model.generate_cache(imglist, device=device)\n",
    "    print('generated cache: ', cache.shape)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2021-03-26T02:25:18.043466Z",
     "start_time": "2021-03-26T02:25:15.699736Z"
    }
   },
   "outputs": [],
   "source": [
    "evaluation_df[\"profile\"] = evaluation_df[\"profile\"].map(tuple)\n",
    "grouped_evals = evaluation_df.groupby([\"profile\", \"user_id\"]).agg({\"predict\": sum}).reset_index()\n",
    "for i, row in tqdm(enumerate(evaluation_df.itertuples()), total=len(evaluation_df.index)):\n",
    "    # Load data into tensors\n",
    "    profile = torch.tensor(row.profile).to(device, non_blocking=True).unsqueeze(0)\n",
    "    user_id = torch.tensor([int(row.user_id)]).to(device, non_blocking=True)\n",
    "    predict = torch.tensor(row.predict).to(device, non_blocking=True)\n",
    "    # Prediction\n",
    "    if MODEL == \"ACF\":\n",
    "        acf_profile = profile + 1 # In ACF items are indexed starting at 1\n",
    "        scores = model.recommend_all(user_id, acf_profile).squeeze()\n",
    "    elif MODEL == 'DVBPR':\n",
    "        scores = model.recommend_all(user_id, imglist, cache=cache)\n",
    "    elif MODE_PROFILE == \"profile\":\n",
    "        scores = model.recommend_all(profile, cache=cache)\n",
    "    elif MODE_PROFILE == \"user\":\n",
    "        scores = model.recommend_all(user_id, cache=cache).squeeze()\n",
    "    \n",
    "    # Ranking\n",
    "    pos_of_evals = (torch.argsort(scores, descending=True)[..., None] == predict).any(-1).nonzero().flatten()\n",
    "    if not PREDICT_ALL:\n",
    "        pos_of_profi = (torch.argsort(scores, descending=True)[..., None] == profile).any(-1).nonzero().flatten()\n",
    "        # Relevant dimensions\n",
    "        _a, _b = pos_of_evals.size(0), pos_of_profi.size(0)\n",
    "        # Calculate shift for each eval item\n",
    "        shift = (pos_of_profi.expand(_a, _b) < pos_of_evals.reshape(_a, 1).expand(_a, _b)).sum(1)\n",
    "        # Apply shift\n",
    "        pos_of_evals -= shift.squeeze(0)\n",
    "    # Store metrics\n",
    "    AUC[i] = auc_exact(pos_of_evals, N_ITEMS)\n",
    "    RR[i] = reciprocal_rank(pos_of_evals)\n",
    "    R20[i] = recall(pos_of_evals, 20)\n",
    "    P20[i] = precision(pos_of_evals, 20)\n",
    "    N20[i] = nDCG(pos_of_evals, 20)\n",
    "    R100[i] = recall(pos_of_evals, 100)\n",
    "    P100[i] = precision(pos_of_evals, 100)\n",
    "    N100[i] = nDCG(pos_of_evals, 100)\n",
    "    R200[i] = recall(pos_of_evals, 200)\n",
    "    P200[i] = precision(pos_of_evals, 200)\n",
    "    N200[i] = nDCG(pos_of_evals, 200)\n",
    "    PROFILE_SIZES[i] = len(row.profile)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2021-03-26T02:25:19.777232Z",
     "start_time": "2021-03-26T02:25:19.768835Z"
    }
   },
   "outputs": [],
   "source": [
    "# Display stats\n",
    "print(f\"AVG AUC = {AUC.mean()}\")\n",
    "print(f\"AVG RR = {RR.mean()}\")\n",
    "print(f\"AVG R20 = {R20.mean()}\")\n",
    "print(f\"AVG P20 = {P20.mean()}\")\n",
    "print(f\"AVG NDCG20 = {N20.mean()}\")\n",
    "print(f\"AVG R100 = {R100.mean()}\")\n",
    "print(f\"AVG P100 = {P100.mean()}\")\n",
    "print(f\"AVG NDCG100 = {N100.mean()}\")\n",
    "print(f\"AVG R200 = {R200.mean()}\")\n",
    "print(f\"AVG P200 = {P200.mean()}\")\n",
    "print(f\"AVG NDCG200 = {N200.mean()}\")\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Relevant plots"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2021-03-26T02:25:46.376657Z",
     "start_time": "2021-03-26T02:25:46.371906Z"
    }
   },
   "outputs": [],
   "source": [
    "import numpy as np\n",
    "\n",
    "\n",
    "def smart_group(value):\n",
    "    if value == 0:\n",
    "        return 0\n",
    "    digits = int(np.log10(value)) + 1\n",
    "    return (10**(digits - 1)) * (value // (10**(digits - 1)))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2021-03-26T02:25:46.978857Z",
     "start_time": "2021-03-26T02:25:46.880712Z"
    }
   },
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "\n",
    "\n",
    "metrics_data = [\n",
    "    [\n",
    "        PROFILE_SIZES[i].item(), AUC[i].item(), RR[i].item(),\n",
    "        R20[i].item(), P20[i].item(), N20[i].item(),\n",
    "        R100[i].item(), P100[i].item(), N100[i].item(),\n",
    "    ]\n",
    "    for i in range(N_EVALS)\n",
    "]\n",
    "metrics_df = pd.DataFrame(metrics_data, columns=[\n",
    "    \"PROFILE_SIZES\", \"AUC\", \"RR\",\n",
    "    \"R20\", \"P20\", \"N20\",\n",
    "    \"R100\", \"P100\", \"N100\",\n",
    "])\n",
    "metrics_df[\"PROFILE_SIZES_STEPS\"] = metrics_df[\"PROFILE_SIZES\"].map(smart_group)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2021-03-26T02:25:48.819523Z",
     "start_time": "2021-03-26T02:25:47.492939Z"
    }
   },
   "outputs": [],
   "source": [
    "import matplotlib.pyplot as plt\n",
    "import seaborn as sns\n",
    "\n",
    "\n",
    "# Metric\n",
    "METRIC = \"AUC\"\n",
    "# Profile size range\n",
    "metrics_df_plot = metrics_df.copy()\n",
    "metrics_df_plot = metrics_df_plot[\n",
    "    (metrics_df_plot[\"PROFILE_SIZES_STEPS\"] >= 0) & (metrics_df_plot[\"PROFILE_SIZES_STEPS\"] < 100)\n",
    "]\n",
    "# Plot METRIC distribution across users grouped by profile size\n",
    "plt.figure(figsize=(24, 9))\n",
    "ax = sns.violinplot(x=\"PROFILE_SIZES_STEPS\", y=METRIC, data=metrics_df_plot, inner=None)\n",
    "if DATASET != \"Pinterest\":\n",
    "    ax = sns.swarmplot(x=\"PROFILE_SIZES_STEPS\", y=METRIC, data=metrics_df_plot, color=\"black\", edgecolor=\"gray\")\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2021-03-26T02:25:50.930112Z",
     "start_time": "2021-03-26T02:25:50.836694Z"
    }
   },
   "outputs": [],
   "source": [
    "# Area Under the Curve distribution across users\n",
    "metrics_df[\"AUC\"].plot.box(sym=\"r+\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2021-03-26T02:25:53.368655Z",
     "start_time": "2021-03-26T02:25:53.263595Z"
    }
   },
   "outputs": [],
   "source": [
    "# First relevant item position (1 / reciprocal_rank) distribution across users\n",
    "# Line marks the 10% of the dataset\n",
    "graph = (1 / metrics_df[\"RR\"]).plot.box(sym=\"r+\")\n",
    "plt.ylim(0, features.shape[0])\n",
    "graph.axhline(features.shape[0] / 10, color=\"red\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2021-03-26T02:25:53.924416Z",
     "start_time": "2021-03-26T02:25:53.786511Z"
    }
   },
   "outputs": [],
   "source": [
    "# First relevant item position (1 / reciprocal_rank) histogram\n",
    "graph = (1 / metrics_df[\"RR\"]).plot.hist(bins=50)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Results inspection"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2021-03-26T02:25:54.837633Z",
     "start_time": "2021-03-26T02:25:54.834283Z"
    }
   },
   "outputs": [],
   "source": [
    "ROW = 0"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2021-03-26T02:25:55.556066Z",
     "start_time": "2021-03-26T02:25:55.538691Z"
    }
   },
   "outputs": [],
   "source": [
    "# Row in evaluation dataframe\n",
    "row = evaluation_df.iloc[ROW]\n",
    "\n",
    "# Load data into tensors\n",
    "profile = torch.tensor(row.profile).to(device, non_blocking=True).unsqueeze(0)\n",
    "user_id = torch.tensor([int(row.user_id)]).to(device, non_blocking=True)\n",
    "predict = torch.tensor(row.predict).to(device, non_blocking=True)\n",
    "# Prediction\n",
    "if MODEL == \"ACF\":\n",
    "    acf_profile = profile + 1\n",
    "    scores = model.recommend_all(user_id, acf_profile).squeeze()\n",
    "elif MODEL == 'DVBPR':\n",
    "    scores = model.recommend_all(user_id, imglist, cache=cache).squeeze()\n",
    "elif MODE_PROFILE == \"profile\":\n",
    "    scores = model.recommend_all(profile)\n",
    "elif MODE_PROFILE == \"user\":\n",
    "    scores = model.recommend_all(user_id).squeeze()\n",
    "# Ranking\n",
    "pos_of_evals = (torch.argsort(scores, descending=True)[..., None] == predict).any(-1).nonzero().flatten()\n",
    "if not PREDICT_ALL:\n",
    "    pos_of_profi = (torch.argsort(scores, descending=True)[..., None] == profile).any(-1).nonzero().flatten()\n",
    "    pos_of_evals -= (pos_of_profi < pos_of_evals).sum()\n",
    "\n",
    "# Display metrics\n",
    "print(f\"| {'-' * 15} | {'-' * 7} |\")\n",
    "print(f\"| {'Metric':^15} | {'Score':^7} |\")\n",
    "print(f\"| {'-' * 15} | {'-' * 7} |\")\n",
    "print(f\"| {'AUC':^15} | {auc_exact(pos_of_evals, N_ITEMS):.5f} |\")\n",
    "print(f\"| {'RR':^15} | {reciprocal_rank(pos_of_evals):.5f} |\")\n",
    "for k in [20, 100, 500]:\n",
    "    print(f\"| {'-' * 15} | {'-' * 7} |\")\n",
    "    print(f\"| {f'Recall@{k}':^15} | {recall(pos_of_evals, k):.5f} |\")\n",
    "    print(f\"| {f'Precision@{k}':^15} | {precision(pos_of_evals, k):.5f} |\")\n",
    "    print(f\"| {f'nDCG@{k}':^15} | {nDCG(pos_of_evals, k):.5f} |\")\n",
    "print(f\"| {'-' * 15} | {'-' * 7} |\")\n",
    "\n",
    "# Profile and prediction\n",
    "profile = profile.cpu().numpy().flatten()\n",
    "predict = predict.cpu().numpy().flatten()\n",
    "# Ranking\n",
    "K = 20\n",
    "ranking = torch.argsort(scores, descending=True).cpu().numpy().flatten()\n",
    "if not PREDICT_ALL:\n",
    "    ranking = ranking[(~np.isin(ranking, profile)) | (np.isin(ranking, predict))]\n",
    "ranking = ranking[:K]\n",
    "print()\n",
    "print(f\"Size of profile: {profile.size}\")\n",
    "print(f\"Position of actual items: {pos_of_evals.cpu().numpy()}\")\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2021-03-26T02:26:02.034249Z",
     "start_time": "2021-03-26T02:26:00.729988Z"
    },
    "scrolled": false
   },
   "outputs": [],
   "source": [
    "import matplotlib.image as mpimg\n",
    "import matplotlib.pyplot as plt\n",
    "\n",
    "\n",
    "COLUMNS = 10\n",
    "ELEMENTS = {\n",
    "    \"Consumed\": profile,\n",
    "    \"Recommendation\": ranking,\n",
    "    \"Ground truth\": predict,\n",
    "}\n",
    "SHOW_FILENAME = False\n",
    "\n",
    "for label, items in ELEMENTS.items():\n",
    "    n_rows = ((len(items) - 1) // COLUMNS + 1)\n",
    "    fig = plt.figure(figsize=(COLUMNS * 2, 4 * n_rows))\n",
    "    plt.title(f\"{label.title()} (n={len(items)})\")\n",
    "    plt.axis(\"off\")\n",
    "    for i, img_id in enumerate(items, start=1):\n",
    "        img_fn = item_index2fn[img_id]\n",
    "        image = mpimg.imread(os.path.join(IMAGES_DIR, img_fn))\n",
    "        ax = fig.add_subplot(n_rows, COLUMNS, i)\n",
    "        if SHOW_FILENAME:\n",
    "            ax.set_title(img_fn)\n",
    "        if label == \"Recommendation\":\n",
    "            if img_id in predict:\n",
    "                ax.patch.set_edgecolor(\"green\")\n",
    "                ax.patch.set_linewidth(\"5\")\n",
    "                if SHOW_FILENAME:\n",
    "                    ax.set_title(img_fn, color=\"green\")\n",
    "                else:\n",
    "                    ax.set_title(\"Ground truth\", color=\"green\")\n",
    "            elif img_id in profile:\n",
    "                ax.patch.set_edgecolor(\"red\")\n",
    "                ax.patch.set_linewidth(\"5\")\n",
    "                if SHOW_FILENAME:\n",
    "                    ax.set_title(img_fn, color=\"red\")\n",
    "                else:\n",
    "                    ax.set_title(\"Consumed\", color=\"red\")\n",
    "        plt.xticks([])\n",
    "        plt.yticks([])\n",
    "        plt.imshow(image)\n"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.5"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}