oxford-test.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "import os\n",
    "import numpy as np\n",
    "import torch\n",
    "import torchvision\n",
    "import matplotlib.pyplot as plt\n",
    "import math\n",
    "import time\n",
    "\n",
    "from KittiDataset import KittiDataset\n",
    "from Model import EnDeWithPooling, EnDeConvLSTM_ws, SkipLSTMEnDe\n",
    "from torchvision import transforms\n",
    "from PIL import Image"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "def saveTransformedImages(imageTensor):\n",
    "    to_pil = torchvision.transforms.ToPILImage()\n",
    "    im = to_pil(imageTensor)\n",
    "    mn, mx = np.min(im), np.max(im)\n",
    "    im = (im - mn) / (mx - mn)\n",
    "    print(im)\n",
    "    plt.imshow(im, cmap='gray')\n",
    "    plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "def plotTrajectory(xValsGT, yValsGT, xValsPred, yValsPred, seqLen, im_path, numFrames=None):\n",
    "    fig = plt.figure(figsize=(8, 8))\n",
    "    plt.scatter(yValsGT, xValsGT, c='r', marker='o', label='Ground Truth')\n",
    "    plt.scatter(yValsPred, xValsPred, c='g', marker='x', label='Prediction')\n",
    "    axes = plt.gca()\n",
    "    axes.set_xlim([1, 512])\n",
    "    axes.set_ylim([1, 512])\n",
    "    plt.xlabel('X-Axis')\n",
    "    plt.ylabel('Y-Axis')\n",
    "    plt.legend(loc='upper right')\n",
    "    if numFrames == None:\n",
    "        plt.title('Trajectory')\n",
    "    else:\n",
    "        plot_title = 'Trajectory (' + str(numFrames // 10 - 1) + \"s)\"\n",
    "        plt.title(plot_title)\n",
    "    plt.savefig(im_path)\n",
    "    plt.close()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "def heatmapAccuracy(outputMap, labelMap, thr=1.5):\n",
    "    pred = np.unravel_index(outputMap.argmax(), outputMap.shape)\n",
    "    gt = np.unravel_index(labelMap.argmax(), labelMap.shape)\n",
    "\n",
    "    dist = math.sqrt((pred[0] - gt[0]) ** 2 + (pred[1] - gt[1]) ** 2)\n",
    "    if dist <= thr:\n",
    "        return 1, dist, (pred[0], pred[1]), (gt[0], gt[1])\n",
    "    return 0, dist, (pred[0], pred[1]), (gt[0], gt[1])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "def largest_indices(ary, n):\n",
    "    \"\"\"Returns the n largest indices from a numpy array.\"\"\"\n",
    "    flat = ary.flatten()\n",
    "    indices = np.argpartition(flat, -n)[-n:]\n",
    "    indices = indices[np.argsort(-flat[indices])]\n",
    "    return np.unravel_index(indices, ary.shape)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "def multiAccuracy(outputMap, labelMap, topK=5):\n",
    "    pred = largest_indices(outputMap, topK)\n",
    "    gt = np.unravel_index(labelMap.argmax(), labelMap.shape)\n",
    "    dist_arr = []\n",
    "    for i in range(len(pred[0])):\n",
    "        dist = math.sqrt((pred[0][i] - gt[0]) ** 2 + (pred[1][i] - gt[1]) ** 2)\n",
    "        dist_arr.append(dist)\n",
    "    \n",
    "    min_val = np.min(dist_arr)\n",
    "    min_idx = np.argmin(dist_arr)\n",
    "    within_radius = 0\n",
    "    if min_val <= 4:\n",
    "        within_radius = 1\n",
    "    return 0, min_val, (pred[0][min_idx], pred[1][min_idx]), (gt[0], gt[1]), within_radius"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "cuda:0\n"
     ]
    }
   ],
   "source": [
    "device = torch.device(\"cuda:0\" if torch.cuda.is_available() else \"cpu\")\n",
    "print(device)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "torch.set_default_tensor_type(torch.cuda.FloatTensor)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Future Prediction (Final, Validation)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Main Model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "metadata": {},
   "outputs": [],
   "source": [
    "repo_dir = \"/home/fbd/rrc/submission/INFER-code\"\n",
    "checkpoint_path = os.path.join(repo_dir, \"models\", \"oxford-transfer\", \"infer-transfer\", \"checkpoint_future.tar\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "metadata": {},
   "outputs": [],
   "source": [
    "checkpoint = torch.load(checkpoint_path)\n",
    "model = SkipLSTMEnDe(activation=\"relu\", initType=\"default\", numChannels=5, imageHeight=256, imageWidth=256, batchnorm=False, softmax=False)\n",
    "model.load_state_dict(checkpoint[\"model_state_dict\"])\n",
    "model = model.cuda()\n",
    "model.convlstm = model.convlstm.cuda()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 34,
   "metadata": {},
   "outputs": [],
   "source": [
    "data_dir = \"/home/fbd/rrc/submission/INFER-datasets/oxford\"\n",
    "val_dir = os.path.join(data_dir, \"test.csv\")\n",
    "\n",
    "val_dataset = KittiDataset(data_dir, height=256, width=256, train=False, infoPath=val_dir, augmentation=False, groundTruth=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 35,
   "metadata": {},
   "outputs": [],
   "source": [
    "upsample_512 = torch.nn.Upsample(scale_factor=2, mode='bilinear')\n",
    "labelTransform = transforms.Compose([\n",
    "    transforms.ToTensor()\n",
    "])\n",
    "targetGTDir = os.path.join(data_dir, 'targetGT')\n",
    "valLoss1, valLoss2, valLoss3, valLoss4, valLoss = [], [], [], [], []\n",
    "futureFrames = 32\n",
    "topK = 5"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 36,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/fbd/anaconda3/envs/torchenv/lib/python3.5/site-packages/torch/nn/functional.py:1749: UserWarning: Default upsampling behavior when mode=bilinear is changed to align_corners=False since 0.4.0. Please specify align_corners=True if the old behavior is desired. See the documentation of nn.Upsample for details.\n",
      "  \"See the documentation of nn.Upsample for details.\".format(mode))\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "SeqNum: 1, KittiSeqNum: 0, VehicleNum: 658, numFrames: 97, loss: 13.979432602794336, len(seqLoss): 32\n",
      "----------------------------------------------------------------------------------------------------\n",
      "\n",
      "SeqNum: 2, KittiSeqNum: 0, VehicleNum: 658, numFrames: 97, loss: 2.0127222703097907, len(seqLoss): 32\n",
      "----------------------------------------------------------------------------------------------------\n",
      "\n",
      "SeqNum: 3, KittiSeqNum: 0, VehicleNum: 658, numFrames: 97, loss: 5.2777926660947685, len(seqLoss): 32\n",
      "----------------------------------------------------------------------------------------------------\n",
      "\n",
      "SeqNum: 4, KittiSeqNum: 0, VehicleNum: 658, numFrames: 97, loss: 6.266770641614745, len(seqLoss): 32\n",
      "----------------------------------------------------------------------------------------------------\n",
      "\n",
      "SeqNum: 5, KittiSeqNum: 0, VehicleNum: 658, numFrames: 97, loss: 2.4804873828133003, len(seqLoss): 32\n",
      "----------------------------------------------------------------------------------------------------\n"
     ]
    }
   ],
   "source": [
    "debug, prevOut, state = True, None, None\n",
    "prevChannels = None\n",
    "xValsGT, yValsGT, xValsPred, yValsPred = [], [], [], []\n",
    "seqLoss, seqVals = [], []\n",
    "seqNum, seqLen = 0, 0\n",
    "\n",
    "start_time = time.time()\n",
    "model.eval()\n",
    "\n",
    "for i in range(len(val_dataset)):\n",
    "    grid, kittiSeqNum, vehicleId, frame1, frame2, endOfSequence, offset, numFrames, augmentation = val_dataset[i]\n",
    "    \n",
    "    if endOfSequence is False:\n",
    "        if int(offset) % 2 == 0:\n",
    "            continue\n",
    "\n",
    "    # The Last Channel is the target frame and first n - 1 are source frames\n",
    "    inp = grid[:-1, :].unsqueeze(0).to(device)\n",
    "    currLabel = grid[-1:, :].unsqueeze(0).to(device)\n",
    "    \n",
    "    if offset < futureFrames:\n",
    "        prevChannels = inp\n",
    "\n",
    "    if offset >= futureFrames:\n",
    "        new_inp = inp.clone().squeeze(0)\n",
    "        mn, mx = torch.min(prevOut), torch.max(prevOut)\n",
    "        prevOut = (prevOut - mn) / (mx - mn)\n",
    "        new_inp[0] = prevOut\n",
    "        new_inp[4] = prevChannels[0, 4, :, :]\n",
    "        inp = new_inp.unsqueeze(0).cuda()\n",
    "\n",
    "    # Forward the input and obtain the result\n",
    "    out = model.forward(inp, state)\n",
    "    state = (model.h, model.c, model.h1, model.c1, model.h2, model.c2)\n",
    "    currOutputMap = out.clone()\n",
    "    newOutputMap = upsample_512(currOutputMap)\n",
    "    nextTargetImg = Image.open(os.path.join(targetGTDir, str(kittiSeqNum).zfill(4), \n",
    "                                            str(frame2).zfill(6), str(vehicleId).zfill(6) + '.png'))\n",
    "    \n",
    "    nextTargetTensor = labelTransform(nextTargetImg).unsqueeze(0)\n",
    "    \n",
    "    prevOut = currOutputMap.detach().cpu().squeeze(0).squeeze(0)\n",
    "    currOutputMap = currOutputMap.detach().cpu().numpy().squeeze(0).squeeze(0)\n",
    "    currLabel = currLabel.detach().cpu().numpy().squeeze(0).squeeze(0)\n",
    "    _, dist, predCoordinates, gtCoordinates = heatmapAccuracy(currOutputMap, currLabel)\n",
    "    \n",
    "    # Upsampled outputs and inputs\n",
    "    currOutputMap1 = newOutputMap.detach().cpu().numpy().squeeze(0).squeeze(0)\n",
    "    currLabel1 = nextTargetTensor.detach().cpu().numpy().squeeze(0).squeeze(0)\n",
    "    \n",
    "    _, dist1, predCoordinates1, gtCoordinates1 = heatmapAccuracy(currOutputMap1, currLabel1)\n",
    "    _, dist2, predCoordinates2, gtCoordinates2, within_radius = multiAccuracy(currOutputMap1, currLabel1, topK=topK)\n",
    "    \n",
    "    if offset >= futureFrames:\n",
    "        seqLoss.append(dist2)\n",
    "    \n",
    "    seqLen += 1\n",
    "    xValsGT.append(gtCoordinates1[0])\n",
    "    yValsGT.append(gtCoordinates1[1])\n",
    "    xValsPred.append(predCoordinates1[0])\n",
    "    yValsPred.append(predCoordinates1[1])\n",
    "    \n",
    "    if endOfSequence:\n",
    "        seqVals.append(seqLen)\n",
    "        xValsGT, yValsGT, xValsPred, yValsPred = [], [], [], []\n",
    "        seqNum += 1\n",
    "        state = None\n",
    "        valLoss.append(np.mean(seqLoss))\n",
    "        valLoss1.append(np.mean(seqLoss[:8]))\n",
    "        valLoss2.append(np.mean(seqLoss[:16]))\n",
    "        valLoss3.append(np.mean(seqLoss[:24]))\n",
    "        valLoss4.append(np.mean(seqLoss[:32]))        \n",
    "        print()\n",
    "        print(\"SeqNum: {}, KittiSeqNum: {}, VehicleNum: {}, numFrames: {}, loss: {}, len(seqLoss): {}\".format(seqNum, kittiSeqNum, vehicleId, numFrames, np.mean(seqLoss), len(seqLoss)))\n",
    "        print(\"-\"*100)\n",
    "        seqLoss = []\n",
    "\n",
    "end_time = time.time()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 37,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Avg Loss: 6.003441112725389\n",
      "Avg Loss in m: 1.5008602781813472\n",
      "Num Seq: 5\n"
     ]
    }
   ],
   "source": [
    "print(\"Avg Loss: {}\".format(np.mean(valLoss)))\n",
    "print(\"Avg Loss in m: {}\".format(np.mean(valLoss) * 0.25))\n",
    "print(\"Num Seq: {}\".format(len(valLoss)))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 38,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "1s: 0.8525580539170416, 2s: 1.1473824358816835, 3s: 1.2965484780935983, 4s: 1.5008602781813472\n"
     ]
    }
   ],
   "source": [
    "print(\"1s: {}, 2s: {}, 3s: {}, 4s: {}\".format(np.mean(valLoss1) * 0.25, np.mean(valLoss2) * 0.25, np.mean(valLoss3) * 0.25, np.mean(valLoss4) * 0.25))"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}