Exploring.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "author=\"Jumabek Alikhanov\"\n",
    "date = 'Nov 17,2019'"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "This is Final Project material for the \"Deep Learning\" class I took  "
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Download the dataset from https://www.unb.ca/cic/datasets/ids-2017.html\n",
    "We only need CSV files that is preprocessed and labeled for ML "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "#imports\n",
    "import os\n",
    "from os.path import join\n",
    "import glob\n",
    "import pandas as pd\n",
    "import numpy as np\n",
    "import time\n",
    "%load_ext autoreload"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "# load data\n",
    "dataroot = 'MachineLearningCVE/'\n",
    "SEED =2 \n",
    "np.random.seed(SEED)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "MachineLearningCVE/*.pcap_ISCX.csv\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/juma/.local/lib/python3.6/site-packages/IPython/core/interactiveshell.py:3326: DtypeWarning: Columns (14,15) have mixed types. Specify dtype option on import or set low_memory=False.\n",
      "  exec(code_obj, self.user_global_ns, self.user_ns)\n",
      "/home/juma/.local/lib/python3.6/site-packages/IPython/core/interactiveshell.py:3326: DtypeWarning: Columns (14) have mixed types. Specify dtype option on import or set low_memory=False.\n",
      "  exec(code_obj, self.user_global_ns, self.user_ns)\n"
     ]
    }
   ],
   "source": [
    "from preprocessing import read_data\n",
    "data = read_data(dataroot,'*.pcap_ISCX.csv')\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "2830743 flow records read which has 79 feature dimension\n"
     ]
    }
   ],
   "source": [
    "num_records,num_features = data.shape\n",
    "print(\"{} flow records read which has {} feature dimension\".format(num_records,num_features))\n",
    "# there is white spaces in columns names e.g. ' Destination Port'\n",
    "# So strip the whitespace from  column names\n",
    "data = data.rename(columns=lambda x: x.strip())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "\n",
    "df_label = data['Label']\n",
    "# dropping bad columns, they coause issues due to large numbers\n",
    "data = data.drop(columns=['Flow Packets/s','Flow Bytes/s','Label'])\n",
    "#filling missing features\n",
    "data.fillna(data.mean(), inplace=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "#convert data to numeric\n",
    "data = data.astype(float).apply(pd.to_numeric)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0"
      ]
     },
     "execution_count": 9,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# lets count if there is NaN values in our dataframe( AKA missing features)\n",
    "#we should see `0`\n",
    "data.isnull().sum().sum()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Destination Port</th>\n",
       "      <th>Flow Duration</th>\n",
       "      <th>Total Fwd Packets</th>\n",
       "      <th>Total Backward Packets</th>\n",
       "      <th>Total Length of Fwd Packets</th>\n",
       "      <th>Total Length of Bwd Packets</th>\n",
       "      <th>Fwd Packet Length Max</th>\n",
       "      <th>Fwd Packet Length Min</th>\n",
       "      <th>Fwd Packet Length Mean</th>\n",
       "      <th>Fwd Packet Length Std</th>\n",
       "      <th>...</th>\n",
       "      <th>act_data_pkt_fwd</th>\n",
       "      <th>min_seg_size_forward</th>\n",
       "      <th>Active Mean</th>\n",
       "      <th>Active Std</th>\n",
       "      <th>Active Max</th>\n",
       "      <th>Active Min</th>\n",
       "      <th>Idle Mean</th>\n",
       "      <th>Idle Std</th>\n",
       "      <th>Idle Max</th>\n",
       "      <th>Idle Min</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <td>0</td>\n",
       "      <td>22.0</td>\n",
       "      <td>1266342.0</td>\n",
       "      <td>41.0</td>\n",
       "      <td>44.0</td>\n",
       "      <td>2664.0</td>\n",
       "      <td>6954.0</td>\n",
       "      <td>456.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>64.975610</td>\n",
       "      <td>109.864573</td>\n",
       "      <td>...</td>\n",
       "      <td>24.0</td>\n",
       "      <td>32.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>1</td>\n",
       "      <td>22.0</td>\n",
       "      <td>1319353.0</td>\n",
       "      <td>41.0</td>\n",
       "      <td>44.0</td>\n",
       "      <td>2664.0</td>\n",
       "      <td>6954.0</td>\n",
       "      <td>456.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>64.975610</td>\n",
       "      <td>109.864573</td>\n",
       "      <td>...</td>\n",
       "      <td>24.0</td>\n",
       "      <td>32.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>2</td>\n",
       "      <td>22.0</td>\n",
       "      <td>160.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>...</td>\n",
       "      <td>0.0</td>\n",
       "      <td>32.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>3</td>\n",
       "      <td>22.0</td>\n",
       "      <td>1303488.0</td>\n",
       "      <td>41.0</td>\n",
       "      <td>42.0</td>\n",
       "      <td>2728.0</td>\n",
       "      <td>6634.0</td>\n",
       "      <td>456.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>66.536585</td>\n",
       "      <td>110.129945</td>\n",
       "      <td>...</td>\n",
       "      <td>24.0</td>\n",
       "      <td>32.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>4</td>\n",
       "      <td>35396.0</td>\n",
       "      <td>77.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>2.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>...</td>\n",
       "      <td>0.0</td>\n",
       "      <td>32.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>5 rows × 76 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "   Destination Port  Flow Duration  Total Fwd Packets  Total Backward Packets  \\\n",
       "0              22.0      1266342.0               41.0                    44.0   \n",
       "1              22.0      1319353.0               41.0                    44.0   \n",
       "2              22.0          160.0                1.0                     1.0   \n",
       "3              22.0      1303488.0               41.0                    42.0   \n",
       "4           35396.0           77.0                1.0                     2.0   \n",
       "\n",
       "   Total Length of Fwd Packets  Total Length of Bwd Packets  \\\n",
       "0                       2664.0                       6954.0   \n",
       "1                       2664.0                       6954.0   \n",
       "2                          0.0                          0.0   \n",
       "3                       2728.0                       6634.0   \n",
       "4                          0.0                          0.0   \n",
       "\n",
       "   Fwd Packet Length Max  Fwd Packet Length Min  Fwd Packet Length Mean  \\\n",
       "0                  456.0                    0.0               64.975610   \n",
       "1                  456.0                    0.0               64.975610   \n",
       "2                    0.0                    0.0                0.000000   \n",
       "3                  456.0                    0.0               66.536585   \n",
       "4                    0.0                    0.0                0.000000   \n",
       "\n",
       "   Fwd Packet Length Std  ...  act_data_pkt_fwd  min_seg_size_forward  \\\n",
       "0             109.864573  ...              24.0                  32.0   \n",
       "1             109.864573  ...              24.0                  32.0   \n",
       "2               0.000000  ...               0.0                  32.0   \n",
       "3             110.129945  ...              24.0                  32.0   \n",
       "4               0.000000  ...               0.0                  32.0   \n",
       "\n",
       "   Active Mean  Active Std  Active Max  Active Min  Idle Mean  Idle Std  \\\n",
       "0          0.0         0.0         0.0         0.0        0.0       0.0   \n",
       "1          0.0         0.0         0.0         0.0        0.0       0.0   \n",
       "2          0.0         0.0         0.0         0.0        0.0       0.0   \n",
       "3          0.0         0.0         0.0         0.0        0.0       0.0   \n",
       "4          0.0         0.0         0.0         0.0        0.0       0.0   \n",
       "\n",
       "   Idle Max  Idle Min  \n",
       "0       0.0       0.0  \n",
       "1       0.0       0.0  \n",
       "2       0.0       0.0  \n",
       "3       0.0       0.0  \n",
       "4       0.0       0.0  \n",
       "\n",
       "[5 rows x 76 columns]"
      ]
     },
     "execution_count": 10,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "data.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "BENIGN                        2273097\n",
       "DoS Hulk                       231073\n",
       "PortScan                       158930\n",
       "DDoS                           128027\n",
       "DoS GoldenEye                   10293\n",
       "FTP-Patator                      7938\n",
       "SSH-Patator                      5897\n",
       "DoS slowloris                    5796\n",
       "DoS Slowhttptest                 5499\n",
       "Bot                              1966\n",
       "Web Attack � Brute Force         1507\n",
       "Web Attack � XSS                  652\n",
       "Infiltration                       36\n",
       "Web Attack � Sql Injection         21\n",
       "Heartbleed                         11\n",
       "Name: Label, dtype: int64"
      ]
     },
     "execution_count": 11,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# print label distribution\n",
    "pd.DataFrame(df_label.value_counts()).to_csv(join(dataroot,'class_distribution.csv'))\n",
    "df_label.value_counts()"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.8"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}