Dataset_cleaner.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import argparse, random\n",
    "import torch\n",
    "import torch.nn as nn\n",
    "import torch.backends.cudnn as cudnn\n",
    "import torch.nn.functional as F\n",
    "import torch.optim as optim\n",
    "from torch.utils.data import DataLoader,Dataset\n",
    "from torchvision import datasets, transforms, utils, models\n",
    "from skimage import io, transform, img_as_float\n",
    "from torch.autograd import Variable\n",
    "from collections import Counter\n",
    "import pandas as pd\n",
    "import matplotlib.pyplot as plt\n",
    "import math\n",
    "import random\n",
    "from imblearn.over_sampling import RandomOverSampler\n",
    "import pandas as pd\n",
    "from lifelines.statistics import logrank_test\n",
    "from lifelines.utils import concordance_index\n",
    "import tables\n",
    "import csv\n",
    "import numpy as np\n",
    "import json\n",
    "from tqdm import tqdm\n",
    "import gc\n",
    "import copy\n",
    "import os\n",
    "from PIL import *\n",
    "from PIL import ImageFile, Image\n",
    "ImageFile.LOAD_TRUNCATED_IMAGES = False"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/yuming/anaconda3/lib/python3.6/site-packages/IPython/core/interactiveshell.py:3049: DtypeWarning: Columns (10,12) have mixed types. Specify dtype option on import or set low_memory=False.\n",
      "  interactivity=interactivity, compiler=compiler, result=result)\n"
     ]
    }
   ],
   "source": [
    "csv_file='/deepdata/adib/prognostic_study/pytorch_code/Patch_clinical_new.csv'\n",
    "files_list = pd.read_csv(csv_file)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "975143"
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(files_list)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0.0%\n",
      "1.0254906203500411%\n",
      "Bad file: F:\\Prognostic study\\Patches\\deceased\\All_A5_21600_rot180_res1.jpeg\n",
      "Bad file: F:\\Prognostic study\\Patches\\deceased\\All_A5_31500_rot180_res1.jpeg\n",
      "2.050985447232759%\n",
      "Bad file: F:\\Prognostic study\\Patches\\deceased\\All_A9_215628_rot180_res1.jpeg\n",
      "Bad file: F:\\Prognostic study\\Patches\\deceased\\All_A9_217440_rot180_res1.jpeg\n",
      "Bad file: F:\\Prognostic study\\Patches\\deceased\\All_A9_218950_rot180_res1.jpeg\n",
      "Bad file: F:\\Prognostic study\\Patches\\deceased\\All_A9_219856_rot180_res1.jpeg\n",
      "Bad file: F:\\Prognostic study\\Patches\\deceased\\All_A9_221970_rot180_res1.jpeg\n",
      "Bad file: F:\\Prognostic study\\Patches\\deceased\\All_A9_226500_rot180_res1.jpeg\n",
      "3.076497100401483%\n",
      "4.101996133868644%\n",
      "5.127495167335805%\n",
      "Bad file: F:\\Prognostic study\\Patches\\deceased\\All_C17_670680_rot180_res1.jpeg\n",
      "Bad file: F:\\Prognostic study\\Patches\\deceased\\All_C17_860220_rot180_res1.jpeg\n",
      "Bad file: F:\\Prognostic study\\Patches\\deceased\\All_C17_864432_rot180_res1.jpeg\n",
      "6.15301313053002%\n",
      "Bad file: F:\\Prognostic study\\Patches\\deceased\\All_C9_86400_rot180_res1.jpeg\n",
      "7.178522680542409%\n",
      "Bad file: F:\\Prognostic study\\Patches\\survival\\All_D3_89650_rot180_res1.jpeg\n",
      "Bad file: F:\\Prognostic study\\Patches\\survival\\All_D3_74328_rot180_res1.jpeg\n",
      "8.204042747164735%\n",
      "9.229548090560327%\n",
      "10.255053433955917%\n",
      "11.28055877735151%\n",
      "12.3060641207471%\n",
      "13.331569464142692%\n",
      "14.357074807538284%\n",
      "15.382580150933878%\n",
      "16.40808549432947%\n",
      "17.43359083772506%\n",
      "18.459096181120653%\n",
      "19.484601524516243%\n",
      "20.510106867911833%\n",
      "21.535612211307427%\n",
      "22.56111755470302%\n",
      "23.58662289809861%\n",
      "24.6121282414942%\n",
      "25.637633584889798%\n",
      "Bad file: F:\\Prognostic study\\Patches\\deceased\\HPan-Ade170Sur-01-151_H13_120060_rot180_res1.jpeg\n",
      "Bad file: F:\\Prognostic study\\Patches\\deceased\\HPan-Ade170Sur-01-151_H13_120582_rot180_res1.jpeg\n",
      "Bad file: F:\\Prognostic study\\Patches\\deceased\\HPan-Ade170Sur-01-151_H9_131054_rot180_res1.jpeg\n",
      "26.663220958112078%\n",
      "27.688729456501004%\n",
      "28.71423795488993%\n",
      "29.73974645327886%\n",
      "30.765254951667785%\n",
      "31.79076345005671%\n",
      "32.81627194844564%\n",
      "33.84178044683456%\n",
      "34.86728894522349%\n",
      "35.892797443612416%\n",
      "36.91830594200134%\n",
      "37.94381444039027%\n",
      "38.96932293877919%\n",
      "39.99483143716812%\n",
      "41.02033993555705%\n",
      "42.04584843394597%\n",
      "43.0713569323349%\n",
      "44.096865430723824%\n",
      "45.122373929112754%\n",
      "46.14788242750167%\n",
      "47.1733909258906%\n",
      "48.19889942427953%\n",
      "49.224407922668455%\n",
      "50.249916421057385%\n",
      "51.27542491944631%\n",
      "52.30093341783524%\n",
      "53.326441916224155%\n",
      "54.35195041461308%\n",
      "55.37745891300201%\n",
      "56.40296741139094%\n",
      "57.42847590977986%\n",
      "58.45398440816879%\n",
      "59.47949290655772%\n",
      "60.50500140494665%\n",
      "61.53050990333557%\n",
      "62.5560184017245%\n",
      "63.58152690011342%\n",
      "64.60703539850235%\n",
      "65.63254389689128%\n",
      "66.6580523952802%\n",
      "67.68356089366912%\n",
      "68.70906939205805%\n",
      "Bad file: F:\\Prognostic study\\Patches\\survival\\HPan-Ade170Sur-01_115_I3_33660_rot180_res1.jpeg\n",
      "Bad file: F:\\Prognostic study\\Patches\\survival\\HPan-Ade170Sur-01_115_I3_34155_rot180_res1.jpeg\n",
      "Bad file: F:\\Prognostic study\\Patches\\survival\\HPan-Ade170Sur-01_115_I3_34485_rot180_res1.jpeg\n",
      "Bad file: F:\\Prognostic study\\Patches\\survival\\HPan-Ade170Sur-01_115_I3_54450_rot180_res1.jpeg\n",
      "Bad file: F:\\Prognostic study\\Patches\\survival\\HPan-Ade170Sur-01_115_I3_57420_rot180_res1.jpeg\n",
      "Bad file: F:\\Prognostic study\\Patches\\survival\\HPan-Ade170Sur-01_115_I3_75240_rot180_res1.jpeg\n",
      "Bad file: F:\\Prognostic study\\Patches\\survival\\HPan-Ade170Sur-01_115_I3_57750_rot180_res1.jpeg\n",
      "Bad file: F:\\Prognostic study\\Patches\\survival\\HPan-Ade170Sur-01_115_I3_58080_rot180_res1.jpeg\n",
      "Bad file: F:\\Prognostic study\\Patches\\survival\\HPan-Ade170Sur-01_115_I3_35640_rot180_res1.jpeg\n",
      "Bad file: F:\\Prognostic study\\Patches\\survival\\HPan-Ade170Sur-01_115_I3_145200_rot180_res1.jpeg\n",
      "Bad file: F:\\Prognostic study\\Patches\\survival\\HPan-Ade170Sur-01_115_I3_58905_rot180_res1.jpeg\n",
      "Bad file: F:\\Prognostic study\\Patches\\survival\\HPan-Ade170Sur-01_115_I3_36300_rot180_res1.jpeg\n",
      "Bad file: F:\\Prognostic study\\Patches\\survival\\HPan-Ade170Sur-01_115_I3_59400_rot180_res1.jpeg\n",
      "Bad file: F:\\Prognostic study\\Patches\\survival\\HPan-Ade170Sur-01_115_I3_60060_rot180_res1.jpeg\n",
      "Bad file: F:\\Prognostic study\\Patches\\survival\\HPan-Ade170Sur-01_115_I3_57420_rot180_res1.jpeg\n",
      "Bad file: F:\\Prognostic study\\Patches\\survival\\HPan-Ade170Sur-01_115_I3_38115_rot180_res1.jpeg\n",
      "Bad file: F:\\Prognostic study\\Patches\\survival\\HPan-Ade170Sur-01_115_I3_79365_rot180_res1.jpeg\n",
      "Bad file: F:\\Prognostic study\\Patches\\survival\\HPan-Ade170Sur-01_115_I3_332310_rot180_res1.jpeg\n",
      "Bad file: F:\\Prognostic study\\Patches\\survival\\HPan-Ade170Sur-01_115_I3_62370_rot180_res1.jpeg\n",
      "Bad file: F:\\Prognostic study\\Patches\\survival\\HPan-Ade170Sur-01_115_I3_148995_rot180_res1.jpeg\n",
      "Bad file: F:\\Prognostic study\\Patches\\survival\\HPan-Ade170Sur-01_115_I3_63360_rot180_res1.jpeg\n",
      "Bad file: F:\\Prognostic study\\Patches\\survival\\HPan-Ade170Sur-01_115_I3_334620_rot180_res1.jpeg\n",
      "Bad file: F:\\Prognostic study\\Patches\\survival\\HPan-Ade170Sur-01_115_I5_390432_rot180_res1.jpeg\n",
      "69.73622273749542%\n",
      "Bad file: F:\\Prognostic study\\Patches\\survival\\HPan-Ade170Sur-01_115_I5_392590_rot180_res1.jpeg\n",
      "70.76182799337916%\n",
      "71.78736173241363%\n",
      "72.81289547144812%\n",
      "73.83842921048259%\n",
      "74.86396294951707%\n",
      "75.88949668855156%\n",
      "76.91503042758605%\n",
      "77.94056416662052%\n",
      "78.96609790565499%\n",
      "79.99163164468948%\n",
      "81.01716538372395%\n",
      "82.04269912275845%\n",
      "83.06823286179292%\n",
      "84.09376660082741%\n",
      "85.11930033986188%\n",
      "86.14483407889637%\n",
      "87.17036781793084%\n",
      "88.19590155696531%\n",
      "89.22143529599981%\n",
      "90.24696903503428%\n",
      "91.27250277406877%\n",
      "92.29803651310324%\n",
      "93.32357025213773%\n",
      "94.3491039911722%\n",
      "95.37463773020669%\n",
      "96.40017146924117%\n",
      "97.42570520827564%\n",
      "98.45123894731013%\n",
      "99.4767726863446%\n"
     ]
    },
    {
     "ename": "IndexError",
     "evalue": "single positional indexer is out-of-bounds",
     "output_type": "error",
     "traceback": [
      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[0;31mIndexError\u001b[0m                                Traceback (most recent call last)",
      "\u001b[0;32m<ipython-input-8-3d533aa63554>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m\u001b[0m\n\u001b[1;32m      4\u001b[0m     \u001b[0;32mif\u001b[0m \u001b[0mi\u001b[0m\u001b[0;34m%\u001b[0m\u001b[0;36m10000\u001b[0m \u001b[0;34m==\u001b[0m \u001b[0;36m0\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m      5\u001b[0m         \u001b[0mprint\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mstr\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mi\u001b[0m\u001b[0;34m/\u001b[0m\u001b[0mlen\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mfiles_list\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m*\u001b[0m\u001b[0;36m100\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m+\u001b[0m\u001b[0;34m\"%\"\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m----> 6\u001b[0;31m     \u001b[0mhe_name\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mos\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mpath\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mjoin\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mfiles_list\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0miloc\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0mi\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;36m0\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m      7\u001b[0m     \u001b[0mhe_name\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mhe_name\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mreplace\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m'Teresa_TMA_patch'\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m'Patches'\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m      8\u001b[0m     \u001b[0mindex_sub\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mhe_name\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mfind\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m'Patches'\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/anaconda3/lib/python3.6/site-packages/pandas/core/indexing.py\u001b[0m in \u001b[0;36m__getitem__\u001b[0;34m(self, key)\u001b[0m\n\u001b[1;32m   1492\u001b[0m             \u001b[0;32mexcept\u001b[0m \u001b[0;34m(\u001b[0m\u001b[0mKeyError\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mIndexError\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mAttributeError\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1493\u001b[0m                 \u001b[0;32mpass\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m-> 1494\u001b[0;31m             \u001b[0;32mreturn\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_getitem_tuple\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mkey\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m   1495\u001b[0m         \u001b[0;32melse\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1496\u001b[0m             \u001b[0;31m# we by definition only have the 0th axis\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/anaconda3/lib/python3.6/site-packages/pandas/core/indexing.py\u001b[0m in \u001b[0;36m_getitem_tuple\u001b[0;34m(self, tup)\u001b[0m\n\u001b[1;32m   2141\u001b[0m     \u001b[0;32mdef\u001b[0m \u001b[0m_getitem_tuple\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mtup\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   2142\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m-> 2143\u001b[0;31m         \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_has_valid_tuple\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mtup\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m   2144\u001b[0m         \u001b[0;32mtry\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   2145\u001b[0m             \u001b[0;32mreturn\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_getitem_lowerdim\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mtup\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/anaconda3/lib/python3.6/site-packages/pandas/core/indexing.py\u001b[0m in \u001b[0;36m_has_valid_tuple\u001b[0;34m(self, key)\u001b[0m\n\u001b[1;32m    221\u001b[0m                 \u001b[0;32mraise\u001b[0m \u001b[0mIndexingError\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m'Too many indexers'\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    222\u001b[0m             \u001b[0;32mtry\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 223\u001b[0;31m                 \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_validate_key\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mk\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mi\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    224\u001b[0m             \u001b[0;32mexcept\u001b[0m \u001b[0mValueError\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    225\u001b[0m                 raise ValueError(\"Location based indexing can only have \"\n",
      "\u001b[0;32m~/anaconda3/lib/python3.6/site-packages/pandas/core/indexing.py\u001b[0m in \u001b[0;36m_validate_key\u001b[0;34m(self, key, axis)\u001b[0m\n\u001b[1;32m   2068\u001b[0m             \u001b[0;32mreturn\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   2069\u001b[0m         \u001b[0;32melif\u001b[0m \u001b[0mis_integer\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mkey\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m-> 2070\u001b[0;31m             \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_validate_integer\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mkey\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0maxis\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m   2071\u001b[0m         \u001b[0;32melif\u001b[0m \u001b[0misinstance\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mkey\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mtuple\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   2072\u001b[0m             \u001b[0;31m# a tuple should already have been caught by this point\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/anaconda3/lib/python3.6/site-packages/pandas/core/indexing.py\u001b[0m in \u001b[0;36m_validate_integer\u001b[0;34m(self, key, axis)\u001b[0m\n\u001b[1;32m   2137\u001b[0m         \u001b[0mlen_axis\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mlen\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mobj\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_get_axis\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0maxis\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   2138\u001b[0m         \u001b[0;32mif\u001b[0m \u001b[0mkey\u001b[0m \u001b[0;34m>=\u001b[0m \u001b[0mlen_axis\u001b[0m \u001b[0;32mor\u001b[0m \u001b[0mkey\u001b[0m \u001b[0;34m<\u001b[0m \u001b[0;34m-\u001b[0m\u001b[0mlen_axis\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m-> 2139\u001b[0;31m             \u001b[0;32mraise\u001b[0m \u001b[0mIndexError\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m\"single positional indexer is out-of-bounds\"\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m   2140\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   2141\u001b[0m     \u001b[0;32mdef\u001b[0m \u001b[0m_getitem_tuple\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mtup\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;31mIndexError\u001b[0m: single positional indexer is out-of-bounds"
     ]
    }
   ],
   "source": [
    "from PIL import Image\n",
    "count = 0\n",
    "for i in range(len(files_list)):\n",
    "    if i%10000 == 0:\n",
    "        print(str(i/len(files_list)*100)+\"%\")\n",
    "    he_name = os.path.join(files_list.iloc[i, 0])\n",
    "    he_name = he_name.replace('Teresa_TMA_patch', 'Patches')\n",
    "    index_sub = he_name.find('Patches')\n",
    "    he_name = os.path.join('/deepdata/adib/prognostic_study', he_name[index_sub:])\n",
    "    he_name = he_name.replace('\\\\', '/')  \n",
    "    try:\n",
    "        img = Image.open(he_name) # open the image file\n",
    "        img.verify() # verify that it is, in fact an image\n",
    "    except (IOError, SyntaxError) as e:\n",
    "        count = count + 1\n",
    "        if os.path.isfile(he_name):\n",
    "            os.remove(he_name)\n",
    "        files_list.drop(files_list.index[[i]], inplace=True)           \n",
    "        print('Bad file:', files_list.iloc[i, 0]) # print out the names of corrupt files\n",
    "print(count)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "975102"
      ]
     },
     "execution_count": 9,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(files_list)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [],
   "source": [
    "files_list.to_csv(r'/deepdata/adib/prognostic_study/pytorch_code/Patch_clinical_clean_v2.csv', encoding='utf-8', index=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "cwd = os.getcwd()\n",
    "surPath = os.path.join(cwd, '../Patches', 'survival')\n",
    "decPath = os.path.join(cwd, '../Patches', 'deceased')\n",
    "fileListSur = [name for name in os.listdir(surPath) if \n",
    "                    os.path.isfile(os.path.join(surPath, name))]\n",
    "fileListDec = [name for name in os.listdir(decPath) if \n",
    "                    os.path.isfile(os.path.join(decPath, name))]  "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "1186514"
      ]
     },
     "execution_count": 11,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(fileListSur)\n",
    "len(fileListDec)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0.0%\n",
      "3.056524303953003%\n",
      "Bad file: All_D13_248912_rot90_res1.jpeg\n",
      "6.113067292644757%\n",
      "Bad file: All_D13_63552_rot180_res1.jpeg\n",
      "9.169628966246595%\n",
      "12.226171954995461%\n",
      "15.282714943744327%\n",
      "18.33925793249319%\n",
      "21.395800921242056%\n",
      "24.452343909990923%\n",
      "27.50888689873979%\n",
      "30.565429887488655%\n",
      "33.621972876237514%\n",
      "36.67851586498638%\n",
      "39.735058853735254%\n",
      "42.79160184248411%\n",
      "45.84814483123298%\n",
      "48.904687819981845%\n",
      "51.961230808730704%\n",
      "55.01777379747958%\n",
      "58.074316786228444%\n",
      "61.13085977497731%\n",
      "64.18740276372617%\n",
      "67.24394575247503%\n",
      "70.3004887412239%\n",
      "73.35703172997276%\n",
      "76.41357471872163%\n",
      "79.47011770747051%\n",
      "Bad file: HPan-Ade170Sur-01_115_I3_12375_rot90_res0.5.jpeg\n",
      "Bad file: HPan-Ade170Sur-01_115_I3_141900_rot45_res1.jpeg\n",
      "Bad file: HPan-Ade170Sur-01_115_I3_18810_rot90_res1.jpeg\n",
      "Bad file: HPan-Ade170Sur-01_115_I3_190080_rot180_res1.jpeg\n",
      "Bad file: HPan-Ade170Sur-01_115_I3_190575_rot90_res1.jpeg\n",
      "Bad file: HPan-Ade170Sur-01_115_I3_217140_rot180_res1.jpeg\n",
      "Bad file: HPan-Ade170Sur-01_115_I3_21780_rot180_res0.5.jpeg\n",
      "Bad file: HPan-Ade170Sur-01_115_I3_224895_rot180_res1.jpeg\n",
      "Bad file: HPan-Ade170Sur-01_115_I3_232650_rot180_res1.jpeg\n",
      "Bad file: HPan-Ade170Sur-01_115_I3_23595_rot90_res0.5.jpeg\n",
      "Bad file: HPan-Ade170Sur-01_115_I3_238095_rot180_res1.jpeg\n",
      "Bad file: HPan-Ade170Sur-01_115_I3_24420_rot180_res1.jpeg\n",
      "Bad file: HPan-Ade170Sur-01_115_I3_2475_rot90_res0.5.jpeg\n",
      "Bad file: HPan-Ade170Sur-01_115_I3_248820_rot180_res1.jpeg\n",
      "Bad file: HPan-Ade170Sur-01_115_I3_25410_rot180_res0.5.jpeg\n",
      "Bad file: HPan-Ade170Sur-01_115_I3_27720_rot180_res0.5.jpeg\n",
      "Bad file: HPan-Ade170Sur-01_115_I3_27885_rot90_res0.5.jpeg\n",
      "Bad file: HPan-Ade170Sur-01_115_I3_28710_rot180_res1.jpeg\n",
      "Bad file: HPan-Ade170Sur-01_115_I3_323400_rot180_res1.jpeg\n",
      "Bad file: HPan-Ade170Sur-01_115_I3_327360_rot45_res1.jpeg\n",
      "Bad file: HPan-Ade170Sur-01_115_I3_332640_rot180_res1.jpeg\n",
      "Bad file: HPan-Ade170Sur-01_115_I3_33495_rot180_res1.jpeg\n",
      "Bad file: HPan-Ade170Sur-01_115_I3_35640_rot180_res1.jpeg\n",
      "Bad file: HPan-Ade170Sur-01_115_I3_37125_rot180_res1.jpeg\n",
      "Bad file: HPan-Ade170Sur-01_115_I3_413820_rot45_res1.jpeg\n",
      "Bad file: HPan-Ade170Sur-01_115_I3_44550_rot180_res1.jpeg\n",
      "Bad file: HPan-Ade170Sur-01_115_I3_51975_rot180_res1.jpeg\n",
      "Bad file: HPan-Ade170Sur-01_115_I3_53295_rot180_res1.jpeg\n",
      "Bad file: HPan-Ade170Sur-01_115_I3_57750_rot180_res0.5.jpeg\n",
      "Bad file: HPan-Ade170Sur-01_115_I3_6270_rot180_res0.5.jpeg\n",
      "Bad file: HPan-Ade170Sur-01_115_I3_67320_rot180_res1.jpeg\n",
      "Bad file: HPan-Ade170Sur-01_115_I3_7425_rot90_res0.5.jpeg\n",
      "Bad file: HPan-Ade170Sur-01_115_I3_83490_rot45_res1.jpeg\n",
      "Bad file: HPan-Ade170Sur-01_115_I3_85470_rot180_res1.jpeg\n",
      "Bad file: HPan-Ade170Sur-01_115_I3_89100_rot45_res1.jpeg\n",
      "Bad file: HPan-Ade170Sur-01_115_I5_132634_rot90_res1.jpeg\n",
      "Bad file: HPan-Ade170Sur-01_115_I5_207500_rot45_res1.jpeg\n",
      "Bad file: HPan-Ade170Sur-01_115_I5_57270_rot180_res0.5.jpeg\n",
      "82.53624716854819%\n",
      "85.59314521182775%\n",
      "88.65004325510732%\n",
      "91.70694129838688%\n",
      "94.76383934166644%\n",
      "97.820737384946%\n",
      "0.0%\n",
      "0.8427887204528809%\n",
      "Bad file: All_A5_102600_rot180_res1.jpeg\n",
      "Bad file: All_A7_235683_rot90_res0.5.jpeg\n",
      "1.6855802820818604%\n",
      "Bad file: All_A9_670440_rot45_res1.jpeg\n",
      "Bad file: All_A9_727216_rot180_res1.jpeg\n",
      "Bad file: All_A9_847110_rot45_res1.jpeg\n",
      "2.5283768157959496%\n",
      "Bad file: All_B3_232232_rot90_res1.jpeg\n",
      "3.371171928925582%\n",
      "4.213964911156977%\n",
      "Bad file: All_C17_267624_rot180_res1.jpeg\n",
      "Bad file: All_C17_446148_rot180_res1.jpeg\n",
      "Bad file: All_C17_873180_rot90_res1.jpeg\n",
      "5.056770678820896%\n",
      "Bad file: All_C3_681550_rot45_res1.jpeg\n",
      "5.899570764087121%\n",
      "Bad file: All_C9_1066240_rot45_res1.jpeg\n",
      "Bad file: All_C9_117760_rot180_res1.jpeg\n",
      "Bad file: All_C9_467840_rot45_res1.jpeg\n",
      "Bad file: All_C9_78400_rot180_res1.jpeg\n",
      "6.742389317358366%\n",
      "7.585187982028161%\n",
      "Bad file: All_D1_213200_rot45_res1.jpeg\n",
      "Bad file: All_D1_52000_rot180_res0.5.jpeg\n",
      "8.428000852913687%\n",
      "Bad file: All_E1_164996_rot90_res1.jpeg\n",
      "Bad file: All_E1_208750_rot90_res1.jpeg\n",
      "Bad file: All_E1_265530_rot90_res1.jpeg\n",
      "Bad file: All_E1_33066_rot180_res1.jpeg\n",
      "Bad file: All_E1_48430_rot180_res1.jpeg\n",
      "9.270840005528793%\n",
      "10.113643642395045%\n",
      "10.9564472792613%\n",
      "11.799250916127553%\n",
      "12.642054552993805%\n",
      "13.484858189860061%\n",
      "14.327661826726315%\n",
      "15.170465463592569%\n",
      "16.01326910045882%\n",
      "16.856072737325075%\n",
      "17.69887637419133%\n",
      "18.541680011057586%\n",
      "19.384483647923837%\n",
      "20.22728728479009%\n",
      "21.070090921656345%\n",
      "21.9128945585226%\n",
      "22.755698195388852%\n",
      "Bad file: HPan-Ade170Sur-01-151_H11_9100_rot180_res1.jpeg\n",
      "Bad file: HPan-Ade170Sur-01-151_H13_83520_rot180_res1.jpeg\n",
      "23.598541610128493%\n",
      "24.44134666763308%\n",
      "25.284151725137672%\n",
      "26.126956782642264%\n",
      "26.969761840146848%\n",
      "27.812566897651443%\n",
      "28.655371955156028%\n",
      "29.49817701266062%\n",
      "30.340982070165207%\n",
      "31.18378712766979%\n",
      "32.02659218517439%\n",
      "32.86939724267897%\n",
      "33.71220230018356%\n",
      "34.55500735768815%\n",
      "35.39781241519274%\n",
      "36.24061747269733%\n",
      "37.08342253020192%\n",
      "37.92622758770651%\n",
      "38.7690326452111%\n",
      "39.61183770271568%\n",
      "40.454642760220274%\n",
      "41.297447817724866%\n",
      "42.14025287522946%\n",
      "42.98305793273404%\n",
      "43.82586299023863%\n",
      "44.66866804774322%\n",
      "45.51147310524781%\n",
      "46.3542781627524%\n",
      "47.197083220256985%\n",
      "48.03988827776158%\n",
      "48.88269333526616%\n",
      "49.72549839277075%\n",
      "50.568303450275344%\n",
      "51.411108507779936%\n",
      "52.25391356528453%\n",
      "53.096718622789105%\n",
      "53.939523680293696%\n",
      "54.782328737798295%\n",
      "55.62513379530289%\n",
      "56.467938852807464%\n",
      "57.310743910312056%\n",
      "58.15354896781665%\n",
      "58.99635402532124%\n",
      "59.83915908282582%\n",
      "60.681964140330415%\n",
      "61.524769197835006%\n",
      "62.36757425533958%\n",
      "63.210379312844175%\n",
      "64.05318437034877%\n",
      "64.89598942785337%\n",
      "65.73879448535794%\n",
      "66.58159954286253%\n",
      "67.42440460036713%\n",
      "68.26720965787172%\n",
      "69.1100147153763%\n",
      "69.95281977288089%\n",
      "70.79562483038548%\n",
      "71.63842988789007%\n",
      "72.48123494539466%\n",
      "73.32404000289925%\n",
      "74.16684506040384%\n",
      "75.00965011790844%\n",
      "75.85245517541301%\n",
      "76.6952602329176%\n",
      "77.5380652904222%\n",
      "78.38087034792677%\n",
      "79.22367540543136%\n",
      "80.06648046293596%\n",
      "80.90928552044055%\n",
      "81.75209057794514%\n",
      "82.59489563544973%\n",
      "83.43770069295432%\n",
      "84.28050575045891%\n",
      "85.12331080796349%\n",
      "85.96611586546808%\n",
      "86.80892092297267%\n",
      "87.65172598047727%\n",
      "88.49453103798184%\n",
      "89.33733609548644%\n",
      "90.18014115299103%\n",
      "91.02294621049562%\n",
      "91.86575126800021%\n",
      "92.7085563255048%\n",
      "93.5513613830094%\n",
      "94.39416644051397%\n",
      "95.23697149801856%\n",
      "96.07977655552315%\n",
      "96.92258161302775%\n",
      "97.76538667053232%\n",
      "98.60819172803691%\n",
      "99.4509967855415%\n"
     ]
    }
   ],
   "source": [
    "count_1 = 0\n",
    "for index, item in enumerate(fileListSur):\n",
    "    if index%10000 == 0:\n",
    "        print(str(index/len(fileListSur)*100)+\"%\")\n",
    "    try:\n",
    "        img = io.imread(os.path.join(surPath, item)) # open the image file\n",
    "#         img.verify() # verify that it is, in fact an image\n",
    "    except (IOError, SyntaxError, ValueError) as e:\n",
    "        fileListSur.pop(index)\n",
    "        count_1 = count_1 + 1\n",
    "        print('Bad file:', item) # print out the names of corrupt files\n",
    "\n",
    "count_2 = 0\n",
    "for index, item in enumerate(fileListDec):\n",
    "    if index%10000 == 0:\n",
    "        print(str(index/len(fileListDec)*100)+\"%\")\n",
    "    try:\n",
    "        img = io.imread(os.path.join(decPath, item)) # open the image file\n",
    "#         img.verify() # verify that it is, in fact an image\n",
    "    except (IOError, SyntaxError, ValueError) as e:\n",
    "        fileListDec.pop(index)\n",
    "        count_2 = count_2 + 1\n",
    "        print('Bad file:', item) # print out the names of corrupt files\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "63\n"
     ]
    }
   ],
   "source": [
    "print(count_1+count_2)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "1186514\n",
      "('All_A11_1001112_rot45_res1.jpeg', 'All_A13_1000160_rot45_res1.jpeg')\n"
     ]
    }
   ],
   "source": [
    "from itertools import cycle\n",
    "csvFilePath = os.path.join(cwd, 'image_files.csv')\n",
    "\n",
    "fileListSur.sort()\n",
    "fileListDec.sort()\n",
    "\n",
    "numOfSur = len(fileListSur)\n",
    "numOfDec = len(fileListDec)\n",
    "numOfPatch = max(numOfSur, numOfDec)\n",
    "zipList = zip(cycle(fileListSur), fileListDec)\n",
    "zipList = list(zipList)\n",
    "print(numOfPatch)\n",
    "print(zipList[0])\n",
    "\n",
    "with open(csvFilePath, 'w', newline='') as csvfile:\n",
    "    filewriter = csv.writer(csvfile, delimiter=',', quotechar='|', \n",
    "                            quoting=csv.QUOTE_MINIMAL)\n",
    "    for i in range(numOfPatch):\n",
    "\n",
    "        sur = os.path.join(surPath, str(zipList[i][0]))\n",
    "        dec = os.path.join(decPath, str(zipList[i][1]))\n",
    "        filewriter.writerow([sur, dec])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 35,
   "metadata": {},
   "outputs": [],
   "source": [
    "class LambdaLR:\n",
    "    def __init__(self, n_epochs, offset, decay_start_epoch):\n",
    "        assert (n_epochs - decay_start_epoch) > 0, \"Decay must start before the training session ends!\"\n",
    "        self.n_epochs = n_epochs\n",
    "        self.offset = offset\n",
    "        self.decay_start_epoch = decay_start_epoch\n",
    "\n",
    "    def step(self, epoch):\n",
    "        print(epoch)\n",
    "        print(\"test\")\n",
    "        return 1.0 - max(0, epoch + self.offset - self.decay_start_epoch) / (self.n_epochs - self.decay_start_epoch)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 37,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0\n",
      "test\n"
     ]
    }
   ],
   "source": [
    "import torch\n",
    "from torchvision import datasets, transforms, utils, models\n",
    "model_ft = models.resnet101()  \n",
    "optimizer = torch.optim.Adam(model_ft.parameters(), lr=0.002)\n",
    "lr = torch.optim.lr_scheduler.LambdaLR(\n",
    "    optimizer, lr_lambda=LambdaLR(200, 0, 5).step\n",
    ")"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.7"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}