Solution - Part 1.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "_cell_guid": "b1076dfc-b9ad-4769-8c92-a6c4dae69d19",
    "_uuid": "8f2839f25d086af736a60e9eeb907d3b93b6e0e5"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "/kaggle/input/av-wns-2019/data_wns/test.csv\n",
      "/kaggle/input/av-wns-2019/data_wns/sample_submission.csv\n",
      "/kaggle/input/av-wns-2019/data_wns/train_NA17Sgz/train.csv\n",
      "/kaggle/input/av-wns-2019/data_wns/train_NA17Sgz/view_log.csv\n",
      "/kaggle/input/av-wns-2019/data_wns/train_NA17Sgz/item_data.csv\n"
     ]
    }
   ],
   "source": [
    "# This Python 3 environment comes with many helpful analytics libraries installed\n",
    "# It is defined by the kaggle/python docker image: https://github.com/kaggle/docker-python\n",
    "# For example, here's several helpful packages to load in \n",
    "\n",
    "import numpy as np # linear algebra\n",
    "import pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\n",
    "\n",
    "# Input data files are available in the \"../input/\" directory.\n",
    "# For example, running this (by clicking run or pressing Shift+Enter) will list all files under the input directory\n",
    "\n",
    "import os\n",
    "for dirname, _, filenames in os.walk('/kaggle/input'):\n",
    "    for filename in filenames:\n",
    "        print(os.path.join(dirname, filename))        \n",
    "\n",
    "# Any results you write to the current directory are saved as output."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "_cell_guid": "79c7e3d0-c299-4dcb-8224-4455121ee9b0",
    "_uuid": "d629ff2d2480ee46fbb7e2d37f6b5fab8052498a"
   },
   "outputs": [],
   "source": [
    "train=pd.read_csv('/kaggle/input/av-wns-2019/data_wns/train_NA17Sgz/train.csv')\n",
    "test=pd.read_csv('/kaggle/input/av-wns-2019/data_wns/test.csv')\n",
    "view=pd.read_csv('/kaggle/input/av-wns-2019/data_wns/train_NA17Sgz/view_log.csv')\n",
    "item=pd.read_csv('/kaggle/input/av-wns-2019/data_wns/train_NA17Sgz/item_data.csv')\n",
    "s=pd.read_csv('/kaggle/input/av-wns-2019/data_wns/sample_submission.csv')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(90675, 6)"
      ]
     },
     "execution_count": 3,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "test.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>impression_id</th>\n",
       "      <th>impression_time</th>\n",
       "      <th>user_id</th>\n",
       "      <th>app_code</th>\n",
       "      <th>os_version</th>\n",
       "      <th>is_4G</th>\n",
       "      <th>is_click</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>c4ca4238a0b923820dcc509a6f75849b</td>\n",
       "      <td>2018-11-15 00:00:00</td>\n",
       "      <td>87862</td>\n",
       "      <td>422</td>\n",
       "      <td>old</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>45c48cce2e2d7fbdea1afc51c7c6ad26</td>\n",
       "      <td>2018-11-15 00:01:00</td>\n",
       "      <td>63410</td>\n",
       "      <td>467</td>\n",
       "      <td>latest</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>70efdf2ec9b086079795c442636b55fb</td>\n",
       "      <td>2018-11-15 00:02:00</td>\n",
       "      <td>71748</td>\n",
       "      <td>259</td>\n",
       "      <td>intermediate</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>8e296a067a37563370ded05f5a3bf3ec</td>\n",
       "      <td>2018-11-15 00:02:00</td>\n",
       "      <td>69209</td>\n",
       "      <td>244</td>\n",
       "      <td>latest</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>182be0c5cdcd5072bb1864cdee4d3d6e</td>\n",
       "      <td>2018-11-15 00:02:00</td>\n",
       "      <td>62873</td>\n",
       "      <td>473</td>\n",
       "      <td>latest</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                      impression_id      impression_time  user_id  app_code  \\\n",
       "0  c4ca4238a0b923820dcc509a6f75849b  2018-11-15 00:00:00    87862       422   \n",
       "1  45c48cce2e2d7fbdea1afc51c7c6ad26  2018-11-15 00:01:00    63410       467   \n",
       "2  70efdf2ec9b086079795c442636b55fb  2018-11-15 00:02:00    71748       259   \n",
       "3  8e296a067a37563370ded05f5a3bf3ec  2018-11-15 00:02:00    69209       244   \n",
       "4  182be0c5cdcd5072bb1864cdee4d3d6e  2018-11-15 00:02:00    62873       473   \n",
       "\n",
       "     os_version  is_4G  is_click  \n",
       "0           old      0         0  \n",
       "1        latest      1         1  \n",
       "2  intermediate      1         0  \n",
       "3        latest      1         0  \n",
       "4        latest      0         0  "
      ]
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "train.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "impression_id      0\n",
       "impression_time    0\n",
       "user_id            0\n",
       "app_code           0\n",
       "os_version         0\n",
       "is_4G              0\n",
       "is_click           0\n",
       "dtype: int64"
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "train.isnull().sum()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(74723, 34079)"
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "train.user_id.nunique(),test.user_id.nunique()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "34079\n",
      "19645\n"
     ]
    }
   ],
   "source": [
    "len(np.intersect1d(train.user_id,view.user_id))\n",
    "print(len(np.intersect1d(test.user_id,view.user_id)))\n",
    "print(len(np.intersect1d(test.user_id,train.user_id)))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(3118622, 5)\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>server_time</th>\n",
       "      <th>device_type</th>\n",
       "      <th>session_id</th>\n",
       "      <th>user_id</th>\n",
       "      <th>item_id</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>2018-10-15 08:58:00</td>\n",
       "      <td>android</td>\n",
       "      <td>112333</td>\n",
       "      <td>4557</td>\n",
       "      <td>32970</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>2018-10-15 08:58:00</td>\n",
       "      <td>android</td>\n",
       "      <td>503590</td>\n",
       "      <td>74788</td>\n",
       "      <td>7640</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>2018-10-15 08:58:00</td>\n",
       "      <td>android</td>\n",
       "      <td>573960</td>\n",
       "      <td>23628</td>\n",
       "      <td>128855</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>2018-10-15 08:58:00</td>\n",
       "      <td>android</td>\n",
       "      <td>121691</td>\n",
       "      <td>2430</td>\n",
       "      <td>12774</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>2018-10-15 08:58:00</td>\n",
       "      <td>android</td>\n",
       "      <td>218564</td>\n",
       "      <td>19227</td>\n",
       "      <td>28296</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "           server_time device_type  session_id  user_id  item_id\n",
       "0  2018-10-15 08:58:00     android      112333     4557    32970\n",
       "1  2018-10-15 08:58:00     android      503590    74788     7640\n",
       "2  2018-10-15 08:58:00     android      573960    23628   128855\n",
       "3  2018-10-15 08:58:00     android      121691     2430    12774\n",
       "4  2018-10-15 08:58:00     android      218564    19227    28296"
      ]
     },
     "execution_count": 8,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "print(view.shape)\n",
    "view.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "server_time    0\n",
       "device_type    0\n",
       "session_id     0\n",
       "user_id        0\n",
       "item_id        0\n",
       "dtype: int64"
      ]
     },
     "execution_count": 9,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "view.isnull().sum()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "item_id         0\n",
       "item_price      0\n",
       "category_1      0\n",
       "category_2      0\n",
       "category_3      0\n",
       "product_type    0\n",
       "dtype: int64"
      ]
     },
     "execution_count": 10,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "item.isnull().sum()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "item_id 132761\n",
      "item_price 13531\n",
      "category_1 17\n",
      "category_2 79\n",
      "category_3 335\n",
      "product_type 7959\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>item_id</th>\n",
       "      <th>item_price</th>\n",
       "      <th>category_1</th>\n",
       "      <th>category_2</th>\n",
       "      <th>category_3</th>\n",
       "      <th>product_type</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>26880</td>\n",
       "      <td>4602</td>\n",
       "      <td>11</td>\n",
       "      <td>35</td>\n",
       "      <td>20</td>\n",
       "      <td>3040</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>54939</td>\n",
       "      <td>3513</td>\n",
       "      <td>12</td>\n",
       "      <td>57</td>\n",
       "      <td>85</td>\n",
       "      <td>6822</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>40383</td>\n",
       "      <td>825</td>\n",
       "      <td>17</td>\n",
       "      <td>8</td>\n",
       "      <td>279</td>\n",
       "      <td>1619</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>8777</td>\n",
       "      <td>2355</td>\n",
       "      <td>13</td>\n",
       "      <td>58</td>\n",
       "      <td>189</td>\n",
       "      <td>5264</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>113705</td>\n",
       "      <td>1267</td>\n",
       "      <td>17</td>\n",
       "      <td>39</td>\n",
       "      <td>151</td>\n",
       "      <td>10239</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   item_id  item_price  category_1  category_2  category_3  product_type\n",
       "0    26880        4602          11          35          20          3040\n",
       "1    54939        3513          12          57          85          6822\n",
       "2    40383         825          17           8         279          1619\n",
       "3     8777        2355          13          58         189          5264\n",
       "4   113705        1267          17          39         151         10239"
      ]
     },
     "execution_count": 11,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "for k in item.columns:\n",
    "    print(k,item[k].nunique())\n",
    "    \n",
    "item.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [],
   "source": [
    "item=pd.get_dummies(item,columns=['category_1','category_2'],drop_first=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>server_time</th>\n",
       "      <th>device_type</th>\n",
       "      <th>session_id</th>\n",
       "      <th>user_id</th>\n",
       "      <th>item_id</th>\n",
       "      <th>item_price</th>\n",
       "      <th>category_3</th>\n",
       "      <th>product_type</th>\n",
       "      <th>category_1_1</th>\n",
       "      <th>category_1_2</th>\n",
       "      <th>...</th>\n",
       "      <th>category_2_70</th>\n",
       "      <th>category_2_71</th>\n",
       "      <th>category_2_72</th>\n",
       "      <th>category_2_73</th>\n",
       "      <th>category_2_74</th>\n",
       "      <th>category_2_75</th>\n",
       "      <th>category_2_76</th>\n",
       "      <th>category_2_77</th>\n",
       "      <th>category_2_78</th>\n",
       "      <th>category_2_79</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>2018-10-15 08:58:00</td>\n",
       "      <td>android</td>\n",
       "      <td>112333</td>\n",
       "      <td>4557</td>\n",
       "      <td>32970</td>\n",
       "      <td>54685.0</td>\n",
       "      <td>253.0</td>\n",
       "      <td>3184.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>...</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>2018-10-15 08:58:00</td>\n",
       "      <td>android</td>\n",
       "      <td>503590</td>\n",
       "      <td>74788</td>\n",
       "      <td>7640</td>\n",
       "      <td>1376.0</td>\n",
       "      <td>228.0</td>\n",
       "      <td>545.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>...</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>2018-10-15 08:58:00</td>\n",
       "      <td>android</td>\n",
       "      <td>573960</td>\n",
       "      <td>23628</td>\n",
       "      <td>128855</td>\n",
       "      <td>4544.0</td>\n",
       "      <td>62.0</td>\n",
       "      <td>5609.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>...</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>2018-10-15 08:58:00</td>\n",
       "      <td>android</td>\n",
       "      <td>121691</td>\n",
       "      <td>2430</td>\n",
       "      <td>12774</td>\n",
       "      <td>904.0</td>\n",
       "      <td>252.0</td>\n",
       "      <td>2740.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>...</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>2018-10-15 08:58:00</td>\n",
       "      <td>android</td>\n",
       "      <td>218564</td>\n",
       "      <td>19227</td>\n",
       "      <td>28296</td>\n",
       "      <td>2304.0</td>\n",
       "      <td>54.0</td>\n",
       "      <td>7422.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>...</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>5 rows × 102 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "           server_time device_type  session_id  user_id  item_id  item_price  \\\n",
       "0  2018-10-15 08:58:00     android      112333     4557    32970     54685.0   \n",
       "1  2018-10-15 08:58:00     android      503590    74788     7640      1376.0   \n",
       "2  2018-10-15 08:58:00     android      573960    23628   128855      4544.0   \n",
       "3  2018-10-15 08:58:00     android      121691     2430    12774       904.0   \n",
       "4  2018-10-15 08:58:00     android      218564    19227    28296      2304.0   \n",
       "\n",
       "   category_3  product_type  category_1_1  category_1_2  ...  category_2_70  \\\n",
       "0       253.0        3184.0           0.0           0.0  ...            0.0   \n",
       "1       228.0         545.0           0.0           0.0  ...            0.0   \n",
       "2        62.0        5609.0           0.0           0.0  ...            0.0   \n",
       "3       252.0        2740.0           0.0           0.0  ...            0.0   \n",
       "4        54.0        7422.0           0.0           0.0  ...            0.0   \n",
       "\n",
       "   category_2_71  category_2_72  category_2_73  category_2_74  category_2_75  \\\n",
       "0            0.0            0.0            0.0            0.0            0.0   \n",
       "1            1.0            0.0            0.0            0.0            0.0   \n",
       "2            0.0            0.0            0.0            0.0            0.0   \n",
       "3            0.0            0.0            0.0            0.0            0.0   \n",
       "4            0.0            0.0            0.0            0.0            0.0   \n",
       "\n",
       "   category_2_76  category_2_77  category_2_78  category_2_79  \n",
       "0            0.0            0.0            0.0            0.0  \n",
       "1            0.0            0.0            0.0            0.0  \n",
       "2            0.0            0.0            0.0            0.0  \n",
       "3            0.0            0.0            0.0            0.0  \n",
       "4            0.0            0.0            0.0            0.0  \n",
       "\n",
       "[5 rows x 102 columns]"
      ]
     },
     "execution_count": 13,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "view_item=view.merge(item,on='item_id',how='left')\n",
    "view_item.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "Index(['server_time', 'device_type', 'session_id', 'user_id', 'item_id',\n",
       "       'item_price', 'category_3', 'product_type', 'category_1_1',\n",
       "       'category_1_2',\n",
       "       ...\n",
       "       'category_2_70', 'category_2_71', 'category_2_72', 'category_2_73',\n",
       "       'category_2_74', 'category_2_75', 'category_2_76', 'category_2_77',\n",
       "       'category_2_78', 'category_2_79'],\n",
       "      dtype='object', length=102)"
      ]
     },
     "execution_count": 15,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "view_item.columns"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [],
   "source": [
    "view_item['server_time']=pd.to_datetime(view_item['server_time'],format='%Y-%m-%d %H:%M:%S')\n",
    "\n",
    "view_item.sort_values(['user_id',\"server_time\"],ascending=True,inplace=True)\n",
    "view_item['cumcount_1']=view_item.groupby(\"user_id\")[\"session_id\"].cumcount() + 1\n",
    "\n",
    "view_item.sort_values(['user_id','item_id',\"server_time\"],ascending=True,inplace=True)\n",
    "view_item['cumcount_2']=view_item.groupby([\"user_id\",'item_id'])[\"session_id\"].cumcount() + 1\n",
    "\n",
    "view_item.sort_values(['user_id','session_id','item_id',\"server_time\"],ascending=True,inplace=True)\n",
    "view_item['cumcount_3']=view_item.groupby([\"user_id\",'session_id','item_id'])[\"session_id\"].cumcount() + 1\n",
    "\n",
    "view_item['device_type']=view_item['device_type'].astype('category')\n",
    "view_item['session_id']=view_item['session_id'].astype('category')\n",
    "view_item['item_id']=view_item['item_id'].astype('category')\n",
    "view_item['device_type']=view_item['device_type'].astype('category')\n",
    "\n",
    "view_item.drop(['server_time'],axis=1,inplace=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "Index(['device_type', 'session_id', 'user_id', 'item_id', 'item_price',\n",
       "       'category_3', 'product_type', 'category_1_1', 'category_1_2',\n",
       "       'category_1_4',\n",
       "       ...\n",
       "       'category_2_73', 'category_2_74', 'category_2_75', 'category_2_76',\n",
       "       'category_2_77', 'category_2_78', 'category_2_79', 'cumcount_1',\n",
       "       'cumcount_2', 'cumcount_3'],\n",
       "      dtype='object', length=104)"
      ]
     },
     "execution_count": 17,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "view_item.columns"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>device_type</th>\n",
       "      <th>session_id</th>\n",
       "      <th>user_id</th>\n",
       "      <th>item_id</th>\n",
       "      <th>item_price</th>\n",
       "      <th>category_3</th>\n",
       "      <th>product_type</th>\n",
       "      <th>category_1_1</th>\n",
       "      <th>category_1_2</th>\n",
       "      <th>category_1_4</th>\n",
       "      <th>...</th>\n",
       "      <th>category_2_73</th>\n",
       "      <th>category_2_74</th>\n",
       "      <th>category_2_75</th>\n",
       "      <th>category_2_76</th>\n",
       "      <th>category_2_77</th>\n",
       "      <th>category_2_78</th>\n",
       "      <th>category_2_79</th>\n",
       "      <th>cumcount_1</th>\n",
       "      <th>cumcount_2</th>\n",
       "      <th>cumcount_3</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>2627295</th>\n",
       "      <td>android</td>\n",
       "      <td>86373</td>\n",
       "      <td>0</td>\n",
       "      <td>116073</td>\n",
       "      <td>4181.0</td>\n",
       "      <td>151.0</td>\n",
       "      <td>124.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>...</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>33</td>\n",
       "      <td>4</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1355387</th>\n",
       "      <td>android</td>\n",
       "      <td>143955</td>\n",
       "      <td>0</td>\n",
       "      <td>2715</td>\n",
       "      <td>1601.0</td>\n",
       "      <td>159.0</td>\n",
       "      <td>7093.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>...</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>18</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1355488</th>\n",
       "      <td>android</td>\n",
       "      <td>143955</td>\n",
       "      <td>0</td>\n",
       "      <td>2715</td>\n",
       "      <td>1601.0</td>\n",
       "      <td>159.0</td>\n",
       "      <td>7093.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>...</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>19</td>\n",
       "      <td>2</td>\n",
       "      <td>2</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1355056</th>\n",
       "      <td>android</td>\n",
       "      <td>143955</td>\n",
       "      <td>0</td>\n",
       "      <td>10379</td>\n",
       "      <td>1238.0</td>\n",
       "      <td>272.0</td>\n",
       "      <td>60.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>...</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>12</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1355084</th>\n",
       "      <td>android</td>\n",
       "      <td>143955</td>\n",
       "      <td>0</td>\n",
       "      <td>10379</td>\n",
       "      <td>1238.0</td>\n",
       "      <td>272.0</td>\n",
       "      <td>60.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>...</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>13</td>\n",
       "      <td>2</td>\n",
       "      <td>2</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>5 rows × 104 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "        device_type session_id  user_id item_id  item_price  category_3  \\\n",
       "2627295     android      86373        0  116073      4181.0       151.0   \n",
       "1355387     android     143955        0    2715      1601.0       159.0   \n",
       "1355488     android     143955        0    2715      1601.0       159.0   \n",
       "1355056     android     143955        0   10379      1238.0       272.0   \n",
       "1355084     android     143955        0   10379      1238.0       272.0   \n",
       "\n",
       "         product_type  category_1_1  category_1_2  category_1_4  ...  \\\n",
       "2627295         124.0           0.0           0.0           0.0  ...   \n",
       "1355387        7093.0           0.0           0.0           0.0  ...   \n",
       "1355488        7093.0           0.0           0.0           0.0  ...   \n",
       "1355056          60.0           0.0           0.0           0.0  ...   \n",
       "1355084          60.0           0.0           0.0           0.0  ...   \n",
       "\n",
       "         category_2_73  category_2_74  category_2_75  category_2_76  \\\n",
       "2627295            0.0            0.0            0.0            0.0   \n",
       "1355387            0.0            0.0            0.0            0.0   \n",
       "1355488            0.0            0.0            0.0            0.0   \n",
       "1355056            0.0            0.0            0.0            0.0   \n",
       "1355084            0.0            0.0            0.0            0.0   \n",
       "\n",
       "         category_2_77  category_2_78  category_2_79  cumcount_1  cumcount_2  \\\n",
       "2627295            0.0            0.0            0.0          33           4   \n",
       "1355387            0.0            0.0            0.0          18           1   \n",
       "1355488            0.0            0.0            0.0          19           2   \n",
       "1355056            0.0            0.0            0.0          12           1   \n",
       "1355084            0.0            0.0            0.0          13           2   \n",
       "\n",
       "         cumcount_3  \n",
       "2627295           1  \n",
       "1355387           1  \n",
       "1355488           2  \n",
       "1355056           1  \n",
       "1355084           2  \n",
       "\n",
       "[5 rows x 104 columns]"
      ]
     },
     "execution_count": 18,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "view_item.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>device_type</th>\n",
       "      <th>session_id</th>\n",
       "      <th>user_id</th>\n",
       "      <th>item_id</th>\n",
       "      <th>item_price</th>\n",
       "      <th>category_3</th>\n",
       "      <th>product_type</th>\n",
       "      <th>category_1_1</th>\n",
       "      <th>category_1_2</th>\n",
       "      <th>category_1_4</th>\n",
       "      <th>...</th>\n",
       "      <th>category_2_73</th>\n",
       "      <th>category_2_74</th>\n",
       "      <th>category_2_75</th>\n",
       "      <th>category_2_76</th>\n",
       "      <th>category_2_77</th>\n",
       "      <th>category_2_78</th>\n",
       "      <th>category_2_79</th>\n",
       "      <th>cumcount_1</th>\n",
       "      <th>cumcount_2</th>\n",
       "      <th>cumcount_3</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>1591211</th>\n",
       "      <td>android</td>\n",
       "      <td>67746</td>\n",
       "      <td>23628</td>\n",
       "      <td>1403</td>\n",
       "      <td>441.0</td>\n",
       "      <td>170.0</td>\n",
       "      <td>734.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>...</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>33</td>\n",
       "      <td>2</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1591213</th>\n",
       "      <td>android</td>\n",
       "      <td>67746</td>\n",
       "      <td>23628</td>\n",
       "      <td>1403</td>\n",
       "      <td>441.0</td>\n",
       "      <td>170.0</td>\n",
       "      <td>734.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>...</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>34</td>\n",
       "      <td>3</td>\n",
       "      <td>2</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>19692</th>\n",
       "      <td>android</td>\n",
       "      <td>79533</td>\n",
       "      <td>23628</td>\n",
       "      <td>86127</td>\n",
       "      <td>2067.0</td>\n",
       "      <td>62.0</td>\n",
       "      <td>5609.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>...</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>3</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>19814</th>\n",
       "      <td>android</td>\n",
       "      <td>79533</td>\n",
       "      <td>23628</td>\n",
       "      <td>86127</td>\n",
       "      <td>2067.0</td>\n",
       "      <td>62.0</td>\n",
       "      <td>5609.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>...</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>4</td>\n",
       "      <td>2</td>\n",
       "      <td>2</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>20124</th>\n",
       "      <td>android</td>\n",
       "      <td>79533</td>\n",
       "      <td>23628</td>\n",
       "      <td>86127</td>\n",
       "      <td>2067.0</td>\n",
       "      <td>62.0</td>\n",
       "      <td>5609.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>...</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>5</td>\n",
       "      <td>3</td>\n",
       "      <td>3</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>5 rows × 104 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "        device_type session_id  user_id item_id  item_price  category_3  \\\n",
       "1591211     android      67746    23628    1403       441.0       170.0   \n",
       "1591213     android      67746    23628    1403       441.0       170.0   \n",
       "19692       android      79533    23628   86127      2067.0        62.0   \n",
       "19814       android      79533    23628   86127      2067.0        62.0   \n",
       "20124       android      79533    23628   86127      2067.0        62.0   \n",
       "\n",
       "         product_type  category_1_1  category_1_2  category_1_4  ...  \\\n",
       "1591211         734.0           0.0           0.0           0.0  ...   \n",
       "1591213         734.0           0.0           0.0           0.0  ...   \n",
       "19692          5609.0           0.0           0.0           1.0  ...   \n",
       "19814          5609.0           0.0           0.0           1.0  ...   \n",
       "20124          5609.0           0.0           0.0           1.0  ...   \n",
       "\n",
       "         category_2_73  category_2_74  category_2_75  category_2_76  \\\n",
       "1591211            0.0            0.0            0.0            0.0   \n",
       "1591213            0.0            0.0            0.0            0.0   \n",
       "19692              0.0            0.0            0.0            0.0   \n",
       "19814              0.0            0.0            0.0            0.0   \n",
       "20124              0.0            0.0            0.0            0.0   \n",
       "\n",
       "         category_2_77  category_2_78  category_2_79  cumcount_1  cumcount_2  \\\n",
       "1591211            0.0            0.0            0.0          33           2   \n",
       "1591213            0.0            0.0            0.0          34           3   \n",
       "19692              0.0            0.0            0.0           3           1   \n",
       "19814              0.0            0.0            0.0           4           2   \n",
       "20124              0.0            0.0            0.0           5           3   \n",
       "\n",
       "         cumcount_3  \n",
       "1591211           1  \n",
       "1591213           2  \n",
       "19692             1  \n",
       "19814             2  \n",
       "20124             3  \n",
       "\n",
       "[5 rows x 104 columns]"
      ]
     },
     "execution_count": 19,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "view_item[view_item.user_id==23628].head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "Index(['device_type', 'session_id', 'user_id', 'item_id', 'item_price',\n",
       "       'category_3', 'product_type', 'category_1_1', 'category_1_2',\n",
       "       'category_1_4',\n",
       "       ...\n",
       "       'category_2_73', 'category_2_74', 'category_2_75', 'category_2_76',\n",
       "       'category_2_77', 'category_2_78', 'category_2_79', 'cumcount_1',\n",
       "       'cumcount_2', 'cumcount_3'],\n",
       "      dtype='object', length=104)"
      ]
     },
     "execution_count": 20,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "view_item.columns"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'device_type': ['count', 'nunique'],\n",
       " 'session_id': ['count', 'nunique'],\n",
       " 'item_id': ['count', 'nunique'],\n",
       " 'item_price': ['min', 'mean', 'max', 'sum'],\n",
       " 'category_3': ['count', 'nunique', 'mean'],\n",
       " 'product_type': ['count', 'nunique', 'mean'],\n",
       " 'category_1_1': ['sum', 'mean'],\n",
       " 'category_1_2': ['sum', 'mean'],\n",
       " 'category_1_4': ['sum', 'mean'],\n",
       " 'category_1_5': ['sum', 'mean'],\n",
       " 'category_1_6': ['sum', 'mean'],\n",
       " 'category_1_7': ['sum', 'mean'],\n",
       " 'category_1_8': ['sum', 'mean'],\n",
       " 'category_1_9': ['sum', 'mean'],\n",
       " 'category_1_10': ['sum', 'mean'],\n",
       " 'category_1_11': ['sum', 'mean'],\n",
       " 'category_1_12': ['sum', 'mean'],\n",
       " 'category_1_13': ['sum', 'mean'],\n",
       " 'category_1_14': ['sum', 'mean'],\n",
       " 'category_1_15': ['sum', 'mean'],\n",
       " 'category_1_16': ['sum', 'mean'],\n",
       " 'category_1_17': ['sum', 'mean'],\n",
       " 'category_2_1': ['sum', 'mean'],\n",
       " 'category_2_2': ['sum', 'mean'],\n",
       " 'category_2_3': ['sum', 'mean'],\n",
       " 'category_2_4': ['sum', 'mean'],\n",
       " 'category_2_5': ['sum', 'mean'],\n",
       " 'category_2_6': ['sum', 'mean'],\n",
       " 'category_2_7': ['sum', 'mean'],\n",
       " 'category_2_8': ['sum', 'mean'],\n",
       " 'category_2_9': ['sum', 'mean'],\n",
       " 'category_2_10': ['sum', 'mean'],\n",
       " 'category_2_11': ['sum', 'mean'],\n",
       " 'category_2_12': ['sum', 'mean'],\n",
       " 'category_2_13': ['sum', 'mean'],\n",
       " 'category_2_14': ['sum', 'mean'],\n",
       " 'category_2_15': ['sum', 'mean'],\n",
       " 'category_2_16': ['sum', 'mean'],\n",
       " 'category_2_17': ['sum', 'mean'],\n",
       " 'category_2_18': ['sum', 'mean'],\n",
       " 'category_2_19': ['sum', 'mean'],\n",
       " 'category_2_20': ['sum', 'mean'],\n",
       " 'category_2_21': ['sum', 'mean'],\n",
       " 'category_2_22': ['sum', 'mean'],\n",
       " 'category_2_23': ['sum', 'mean'],\n",
       " 'category_2_24': ['sum', 'mean'],\n",
       " 'category_2_25': ['sum', 'mean'],\n",
       " 'category_2_26': ['sum', 'mean'],\n",
       " 'category_2_27': ['sum', 'mean'],\n",
       " 'category_2_28': ['sum', 'mean'],\n",
       " 'category_2_29': ['sum', 'mean'],\n",
       " 'category_2_30': ['sum', 'mean'],\n",
       " 'category_2_31': ['sum', 'mean'],\n",
       " 'category_2_33': ['sum', 'mean'],\n",
       " 'category_2_34': ['sum', 'mean'],\n",
       " 'category_2_35': ['sum', 'mean'],\n",
       " 'category_2_36': ['sum', 'mean'],\n",
       " 'category_2_37': ['sum', 'mean'],\n",
       " 'category_2_38': ['sum', 'mean'],\n",
       " 'category_2_39': ['sum', 'mean'],\n",
       " 'category_2_40': ['sum', 'mean'],\n",
       " 'category_2_41': ['sum', 'mean'],\n",
       " 'category_2_42': ['sum', 'mean'],\n",
       " 'category_2_43': ['sum', 'mean'],\n",
       " 'category_2_44': ['sum', 'mean'],\n",
       " 'category_2_45': ['sum', 'mean'],\n",
       " 'category_2_46': ['sum', 'mean'],\n",
       " 'category_2_47': ['sum', 'mean'],\n",
       " 'category_2_48': ['sum', 'mean'],\n",
       " 'category_2_49': ['sum', 'mean'],\n",
       " 'category_2_50': ['sum', 'mean'],\n",
       " 'category_2_51': ['sum', 'mean'],\n",
       " 'category_2_52': ['sum', 'mean'],\n",
       " 'category_2_53': ['sum', 'mean'],\n",
       " 'category_2_54': ['sum', 'mean'],\n",
       " 'category_2_55': ['sum', 'mean'],\n",
       " 'category_2_56': ['sum', 'mean'],\n",
       " 'category_2_57': ['sum', 'mean'],\n",
       " 'category_2_58': ['sum', 'mean'],\n",
       " 'category_2_59': ['sum', 'mean'],\n",
       " 'category_2_60': ['sum', 'mean'],\n",
       " 'category_2_61': ['sum', 'mean'],\n",
       " 'category_2_62': ['sum', 'mean'],\n",
       " 'category_2_63': ['sum', 'mean'],\n",
       " 'category_2_64': ['sum', 'mean'],\n",
       " 'category_2_65': ['sum', 'mean'],\n",
       " 'category_2_66': ['sum', 'mean'],\n",
       " 'category_2_67': ['sum', 'mean'],\n",
       " 'category_2_68': ['sum', 'mean'],\n",
       " 'category_2_69': ['sum', 'mean'],\n",
       " 'category_2_70': ['sum', 'mean'],\n",
       " 'category_2_71': ['sum', 'mean'],\n",
       " 'category_2_72': ['sum', 'mean'],\n",
       " 'category_2_73': ['sum', 'mean'],\n",
       " 'category_2_74': ['sum', 'mean'],\n",
       " 'category_2_75': ['sum', 'mean'],\n",
       " 'category_2_76': ['sum', 'mean'],\n",
       " 'category_2_77': ['sum', 'mean'],\n",
       " 'category_2_78': ['sum', 'mean'],\n",
       " 'category_2_79': ['sum', 'mean'],\n",
       " 'cumcount_1': ['min', 'mean', 'max', 'sum'],\n",
       " 'cumcount_2': ['min', 'mean', 'max', 'sum'],\n",
       " 'cumcount_3': ['min', 'mean', 'max', 'sum']}"
      ]
     },
     "execution_count": 21,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "cat_agg=['count','nunique']\n",
    "num_agg=['min','mean','max','sum']\n",
    "agg_col={\n",
    "    'device_type':cat_agg, 'session_id':cat_agg, 'item_id':cat_agg,'item_price':num_agg,\n",
    "       'category_3':['count','nunique','mean'], 'product_type':['count','nunique','mean']\n",
    "}\n",
    "\n",
    "for k in view_item.columns:\n",
    "    if k.startswith('category_1') or k.startswith('category_2'):\n",
    "        agg_col[k]=['sum','mean']\n",
    "    elif k.startswith('server'):\n",
    "        agg_col[k]=cat_agg\n",
    "    elif k.startswith('cumcount'):\n",
    "        agg_col[k]=num_agg\n",
    "agg_col"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {},
   "outputs": [],
   "source": [
    "view_item1=view_item.groupby('user_id').agg(agg_col)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead tr th {\n",
       "        text-align: left;\n",
       "    }\n",
       "\n",
       "    .dataframe thead tr:last-of-type th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr>\n",
       "      <th></th>\n",
       "      <th colspan=\"2\" halign=\"left\">device_type</th>\n",
       "      <th colspan=\"2\" halign=\"left\">session_id</th>\n",
       "      <th colspan=\"2\" halign=\"left\">item_id</th>\n",
       "      <th colspan=\"4\" halign=\"left\">item_price</th>\n",
       "      <th>...</th>\n",
       "      <th colspan=\"2\" halign=\"left\">cumcount_1</th>\n",
       "      <th colspan=\"4\" halign=\"left\">cumcount_2</th>\n",
       "      <th colspan=\"4\" halign=\"left\">cumcount_3</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th></th>\n",
       "      <th>count</th>\n",
       "      <th>nunique</th>\n",
       "      <th>count</th>\n",
       "      <th>nunique</th>\n",
       "      <th>count</th>\n",
       "      <th>nunique</th>\n",
       "      <th>min</th>\n",
       "      <th>mean</th>\n",
       "      <th>max</th>\n",
       "      <th>sum</th>\n",
       "      <th>...</th>\n",
       "      <th>max</th>\n",
       "      <th>sum</th>\n",
       "      <th>min</th>\n",
       "      <th>mean</th>\n",
       "      <th>max</th>\n",
       "      <th>sum</th>\n",
       "      <th>min</th>\n",
       "      <th>mean</th>\n",
       "      <th>max</th>\n",
       "      <th>sum</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>user_id</th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>42</td>\n",
       "      <td>1</td>\n",
       "      <td>42</td>\n",
       "      <td>11</td>\n",
       "      <td>42</td>\n",
       "      <td>18</td>\n",
       "      <td>332.0</td>\n",
       "      <td>9395.666667</td>\n",
       "      <td>92160.0</td>\n",
       "      <td>394618.0</td>\n",
       "      <td>...</td>\n",
       "      <td>42</td>\n",
       "      <td>903</td>\n",
       "      <td>1</td>\n",
       "      <td>2.190476</td>\n",
       "      <td>6</td>\n",
       "      <td>92</td>\n",
       "      <td>1</td>\n",
       "      <td>1.690476</td>\n",
       "      <td>6</td>\n",
       "      <td>71</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>8</td>\n",
       "      <td>1</td>\n",
       "      <td>8</td>\n",
       "      <td>3</td>\n",
       "      <td>8</td>\n",
       "      <td>8</td>\n",
       "      <td>383.0</td>\n",
       "      <td>3946.750000</td>\n",
       "      <td>12595.0</td>\n",
       "      <td>31574.0</td>\n",
       "      <td>...</td>\n",
       "      <td>8</td>\n",
       "      <td>36</td>\n",
       "      <td>1</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>1</td>\n",
       "      <td>8</td>\n",
       "      <td>1</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>1</td>\n",
       "      <td>8</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>165</td>\n",
       "      <td>1</td>\n",
       "      <td>165</td>\n",
       "      <td>37</td>\n",
       "      <td>165</td>\n",
       "      <td>130</td>\n",
       "      <td>128.0</td>\n",
       "      <td>14809.509091</td>\n",
       "      <td>281536.0</td>\n",
       "      <td>2443569.0</td>\n",
       "      <td>...</td>\n",
       "      <td>165</td>\n",
       "      <td>13695</td>\n",
       "      <td>1</td>\n",
       "      <td>1.333333</td>\n",
       "      <td>6</td>\n",
       "      <td>220</td>\n",
       "      <td>1</td>\n",
       "      <td>1.072727</td>\n",
       "      <td>4</td>\n",
       "      <td>177</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>8</td>\n",
       "      <td>1</td>\n",
       "      <td>8</td>\n",
       "      <td>1</td>\n",
       "      <td>8</td>\n",
       "      <td>3</td>\n",
       "      <td>537.0</td>\n",
       "      <td>7257.375000</td>\n",
       "      <td>16640.0</td>\n",
       "      <td>58059.0</td>\n",
       "      <td>...</td>\n",
       "      <td>8</td>\n",
       "      <td>36</td>\n",
       "      <td>1</td>\n",
       "      <td>1.875000</td>\n",
       "      <td>3</td>\n",
       "      <td>15</td>\n",
       "      <td>1</td>\n",
       "      <td>1.875000</td>\n",
       "      <td>3</td>\n",
       "      <td>15</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>2</td>\n",
       "      <td>1</td>\n",
       "      <td>2</td>\n",
       "      <td>1</td>\n",
       "      <td>2</td>\n",
       "      <td>2</td>\n",
       "      <td>1977.0</td>\n",
       "      <td>30114.500000</td>\n",
       "      <td>58252.0</td>\n",
       "      <td>60229.0</td>\n",
       "      <td>...</td>\n",
       "      <td>2</td>\n",
       "      <td>3</td>\n",
       "      <td>1</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>1</td>\n",
       "      <td>2</td>\n",
       "      <td>1</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>1</td>\n",
       "      <td>2</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>5 rows × 216 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "        device_type         session_id         item_id         item_price  \\\n",
       "              count nunique      count nunique   count nunique        min   \n",
       "user_id                                                                     \n",
       "0                42       1         42      11      42      18      332.0   \n",
       "1                 8       1          8       3       8       8      383.0   \n",
       "2               165       1        165      37     165     130      128.0   \n",
       "3                 8       1          8       1       8       3      537.0   \n",
       "4                 2       1          2       1       2       2     1977.0   \n",
       "\n",
       "                                            ... cumcount_1        cumcount_2  \\\n",
       "                 mean       max        sum  ...        max    sum        min   \n",
       "user_id                                     ...                                \n",
       "0         9395.666667   92160.0   394618.0  ...         42    903          1   \n",
       "1         3946.750000   12595.0    31574.0  ...          8     36          1   \n",
       "2        14809.509091  281536.0  2443569.0  ...        165  13695          1   \n",
       "3         7257.375000   16640.0    58059.0  ...          8     36          1   \n",
       "4        30114.500000   58252.0    60229.0  ...          2      3          1   \n",
       "\n",
       "                           cumcount_3                     \n",
       "             mean max  sum        min      mean max  sum  \n",
       "user_id                                                   \n",
       "0        2.190476   6   92          1  1.690476   6   71  \n",
       "1        1.000000   1    8          1  1.000000   1    8  \n",
       "2        1.333333   6  220          1  1.072727   4  177  \n",
       "3        1.875000   3   15          1  1.875000   3   15  \n",
       "4        1.000000   1    2          1  1.000000   1    2  \n",
       "\n",
       "[5 rows x 216 columns]"
      ]
     },
     "execution_count": 23,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "view_item1.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>user_id</th>\n",
       "      <th>J_device_type_count</th>\n",
       "      <th>J_device_type_nunique</th>\n",
       "      <th>J_session_id_count</th>\n",
       "      <th>J_session_id_nunique</th>\n",
       "      <th>J_item_id_count</th>\n",
       "      <th>J_item_id_nunique</th>\n",
       "      <th>J_item_price_min</th>\n",
       "      <th>J_item_price_mean</th>\n",
       "      <th>J_item_price_max</th>\n",
       "      <th>...</th>\n",
       "      <th>J_cumcount_1_max</th>\n",
       "      <th>J_cumcount_1_sum</th>\n",
       "      <th>J_cumcount_2_min</th>\n",
       "      <th>J_cumcount_2_mean</th>\n",
       "      <th>J_cumcount_2_max</th>\n",
       "      <th>J_cumcount_2_sum</th>\n",
       "      <th>J_cumcount_3_min</th>\n",
       "      <th>J_cumcount_3_mean</th>\n",
       "      <th>J_cumcount_3_max</th>\n",
       "      <th>J_cumcount_3_sum</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>0</td>\n",
       "      <td>42</td>\n",
       "      <td>1</td>\n",
       "      <td>42</td>\n",
       "      <td>11</td>\n",
       "      <td>42</td>\n",
       "      <td>18</td>\n",
       "      <td>332.0</td>\n",
       "      <td>9395.666667</td>\n",
       "      <td>92160.0</td>\n",
       "      <td>...</td>\n",
       "      <td>42</td>\n",
       "      <td>903</td>\n",
       "      <td>1</td>\n",
       "      <td>2.190476</td>\n",
       "      <td>6</td>\n",
       "      <td>92</td>\n",
       "      <td>1</td>\n",
       "      <td>1.690476</td>\n",
       "      <td>6</td>\n",
       "      <td>71</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>1</td>\n",
       "      <td>8</td>\n",
       "      <td>1</td>\n",
       "      <td>8</td>\n",
       "      <td>3</td>\n",
       "      <td>8</td>\n",
       "      <td>8</td>\n",
       "      <td>383.0</td>\n",
       "      <td>3946.750000</td>\n",
       "      <td>12595.0</td>\n",
       "      <td>...</td>\n",
       "      <td>8</td>\n",
       "      <td>36</td>\n",
       "      <td>1</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>1</td>\n",
       "      <td>8</td>\n",
       "      <td>1</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>1</td>\n",
       "      <td>8</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>2</td>\n",
       "      <td>165</td>\n",
       "      <td>1</td>\n",
       "      <td>165</td>\n",
       "      <td>37</td>\n",
       "      <td>165</td>\n",
       "      <td>130</td>\n",
       "      <td>128.0</td>\n",
       "      <td>14809.509091</td>\n",
       "      <td>281536.0</td>\n",
       "      <td>...</td>\n",
       "      <td>165</td>\n",
       "      <td>13695</td>\n",
       "      <td>1</td>\n",
       "      <td>1.333333</td>\n",
       "      <td>6</td>\n",
       "      <td>220</td>\n",
       "      <td>1</td>\n",
       "      <td>1.072727</td>\n",
       "      <td>4</td>\n",
       "      <td>177</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>3</td>\n",
       "      <td>8</td>\n",
       "      <td>1</td>\n",
       "      <td>8</td>\n",
       "      <td>1</td>\n",
       "      <td>8</td>\n",
       "      <td>3</td>\n",
       "      <td>537.0</td>\n",
       "      <td>7257.375000</td>\n",
       "      <td>16640.0</td>\n",
       "      <td>...</td>\n",
       "      <td>8</td>\n",
       "      <td>36</td>\n",
       "      <td>1</td>\n",
       "      <td>1.875000</td>\n",
       "      <td>3</td>\n",
       "      <td>15</td>\n",
       "      <td>1</td>\n",
       "      <td>1.875000</td>\n",
       "      <td>3</td>\n",
       "      <td>15</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>4</td>\n",
       "      <td>2</td>\n",
       "      <td>1</td>\n",
       "      <td>2</td>\n",
       "      <td>1</td>\n",
       "      <td>2</td>\n",
       "      <td>2</td>\n",
       "      <td>1977.0</td>\n",
       "      <td>30114.500000</td>\n",
       "      <td>58252.0</td>\n",
       "      <td>...</td>\n",
       "      <td>2</td>\n",
       "      <td>3</td>\n",
       "      <td>1</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>1</td>\n",
       "      <td>2</td>\n",
       "      <td>1</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>1</td>\n",
       "      <td>2</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>5 rows × 217 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "   user_id  J_device_type_count  J_device_type_nunique  J_session_id_count  \\\n",
       "0        0                   42                      1                  42   \n",
       "1        1                    8                      1                   8   \n",
       "2        2                  165                      1                 165   \n",
       "3        3                    8                      1                   8   \n",
       "4        4                    2                      1                   2   \n",
       "\n",
       "   J_session_id_nunique  J_item_id_count  J_item_id_nunique  J_item_price_min  \\\n",
       "0                    11               42                 18             332.0   \n",
       "1                     3                8                  8             383.0   \n",
       "2                    37              165                130             128.0   \n",
       "3                     1                8                  3             537.0   \n",
       "4                     1                2                  2            1977.0   \n",
       "\n",
       "   J_item_price_mean  J_item_price_max  ...  J_cumcount_1_max  \\\n",
       "0        9395.666667           92160.0  ...                42   \n",
       "1        3946.750000           12595.0  ...                 8   \n",
       "2       14809.509091          281536.0  ...               165   \n",
       "3        7257.375000           16640.0  ...                 8   \n",
       "4       30114.500000           58252.0  ...                 2   \n",
       "\n",
       "   J_cumcount_1_sum  J_cumcount_2_min  J_cumcount_2_mean  J_cumcount_2_max  \\\n",
       "0               903                 1           2.190476                 6   \n",
       "1                36                 1           1.000000                 1   \n",
       "2             13695                 1           1.333333                 6   \n",
       "3                36                 1           1.875000                 3   \n",
       "4                 3                 1           1.000000                 1   \n",
       "\n",
       "   J_cumcount_2_sum  J_cumcount_3_min  J_cumcount_3_mean  J_cumcount_3_max  \\\n",
       "0                92                 1           1.690476                 6   \n",
       "1                 8                 1           1.000000                 1   \n",
       "2               220                 1           1.072727                 4   \n",
       "3                15                 1           1.875000                 3   \n",
       "4                 2                 1           1.000000                 1   \n",
       "\n",
       "   J_cumcount_3_sum  \n",
       "0                71  \n",
       "1                 8  \n",
       "2               177  \n",
       "3                15  \n",
       "4                 2  \n",
       "\n",
       "[5 rows x 217 columns]"
      ]
     },
     "execution_count": 24,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "view_item1.columns=['J_' + '_'.join(col).strip() for col in view_item1.columns.values]\n",
    "view_item1.reset_index(inplace=True)\n",
    "view_item1.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(132761, (89157, 217))"
      ]
     },
     "execution_count": 25,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "item.item_id.nunique(),view_item1.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "67"
      ]
     },
     "execution_count": 26,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "del view_item\n",
    "import gc \n",
    "gc.collect()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>impression_id</th>\n",
       "      <th>is_click</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>a9e7126a585a69a32bc7414e9d0c0ada</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>caac14a5bf2ba283db7708bb34855760</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>13f10ba306a19ce7bec2f3cae507b698</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>39c4b4dc0e9701b55a0a4f072008fb3f</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>bf5a572cca75f5fc67f4b14e58b11d70</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                      impression_id  is_click\n",
       "0  a9e7126a585a69a32bc7414e9d0c0ada         0\n",
       "1  caac14a5bf2ba283db7708bb34855760         0\n",
       "2  13f10ba306a19ce7bec2f3cae507b698         0\n",
       "3  39c4b4dc0e9701b55a0a4f072008fb3f         0\n",
       "4  bf5a572cca75f5fc67f4b14e58b11d70         0"
      ]
     },
     "execution_count": 27,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "s.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/opt/conda/lib/python3.6/site-packages/pandas/core/frame.py:7116: FutureWarning: Sorting because non-concatenation axis is not aligned. A future version\n",
      "of pandas will change to not sort by default.\n",
      "\n",
      "To accept the future behavior, pass 'sort=False'.\n",
      "\n",
      "To retain the current behavior and silence the warning, pass 'sort=True'.\n",
      "\n",
      "  sort=sort,\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>app_code</th>\n",
       "      <th>impression_id</th>\n",
       "      <th>impression_time</th>\n",
       "      <th>is_4G</th>\n",
       "      <th>is_click</th>\n",
       "      <th>os_version</th>\n",
       "      <th>user_id</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>422</td>\n",
       "      <td>c4ca4238a0b923820dcc509a6f75849b</td>\n",
       "      <td>2018-11-15 00:00:00</td>\n",
       "      <td>0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>old</td>\n",
       "      <td>87862</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>467</td>\n",
       "      <td>45c48cce2e2d7fbdea1afc51c7c6ad26</td>\n",
       "      <td>2018-11-15 00:01:00</td>\n",
       "      <td>1</td>\n",
       "      <td>1.0</td>\n",
       "      <td>latest</td>\n",
       "      <td>63410</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>259</td>\n",
       "      <td>70efdf2ec9b086079795c442636b55fb</td>\n",
       "      <td>2018-11-15 00:02:00</td>\n",
       "      <td>1</td>\n",
       "      <td>0.0</td>\n",
       "      <td>intermediate</td>\n",
       "      <td>71748</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>244</td>\n",
       "      <td>8e296a067a37563370ded05f5a3bf3ec</td>\n",
       "      <td>2018-11-15 00:02:00</td>\n",
       "      <td>1</td>\n",
       "      <td>0.0</td>\n",
       "      <td>latest</td>\n",
       "      <td>69209</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>473</td>\n",
       "      <td>182be0c5cdcd5072bb1864cdee4d3d6e</td>\n",
       "      <td>2018-11-15 00:02:00</td>\n",
       "      <td>0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>latest</td>\n",
       "      <td>62873</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   app_code                     impression_id      impression_time  is_4G  \\\n",
       "0       422  c4ca4238a0b923820dcc509a6f75849b  2018-11-15 00:00:00      0   \n",
       "1       467  45c48cce2e2d7fbdea1afc51c7c6ad26  2018-11-15 00:01:00      1   \n",
       "2       259  70efdf2ec9b086079795c442636b55fb  2018-11-15 00:02:00      1   \n",
       "3       244  8e296a067a37563370ded05f5a3bf3ec  2018-11-15 00:02:00      1   \n",
       "4       473  182be0c5cdcd5072bb1864cdee4d3d6e  2018-11-15 00:02:00      0   \n",
       "\n",
       "   is_click    os_version  user_id  \n",
       "0       0.0           old    87862  \n",
       "1       1.0        latest    63410  \n",
       "2       0.0  intermediate    71748  \n",
       "3       0.0        latest    69209  \n",
       "4       0.0        latest    62873  "
      ]
     },
     "execution_count": 28,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df=train.append(test,ignore_index=True)\n",
    "df.shape\n",
    "df.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>app_code</th>\n",
       "      <th>impression_id</th>\n",
       "      <th>impression_time</th>\n",
       "      <th>is_4G</th>\n",
       "      <th>is_click</th>\n",
       "      <th>os_version</th>\n",
       "      <th>user_id</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>422</td>\n",
       "      <td>c4ca4238a0b923820dcc509a6f75849b</td>\n",
       "      <td>2018-11-15 00:00:00</td>\n",
       "      <td>0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>old</td>\n",
       "      <td>87862</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>467</td>\n",
       "      <td>45c48cce2e2d7fbdea1afc51c7c6ad26</td>\n",
       "      <td>2018-11-15 00:01:00</td>\n",
       "      <td>1</td>\n",
       "      <td>1.0</td>\n",
       "      <td>latest</td>\n",
       "      <td>63410</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>259</td>\n",
       "      <td>70efdf2ec9b086079795c442636b55fb</td>\n",
       "      <td>2018-11-15 00:02:00</td>\n",
       "      <td>1</td>\n",
       "      <td>0.0</td>\n",
       "      <td>intermediate</td>\n",
       "      <td>71748</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>244</td>\n",
       "      <td>8e296a067a37563370ded05f5a3bf3ec</td>\n",
       "      <td>2018-11-15 00:02:00</td>\n",
       "      <td>1</td>\n",
       "      <td>0.0</td>\n",
       "      <td>latest</td>\n",
       "      <td>69209</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>473</td>\n",
       "      <td>182be0c5cdcd5072bb1864cdee4d3d6e</td>\n",
       "      <td>2018-11-15 00:02:00</td>\n",
       "      <td>0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>latest</td>\n",
       "      <td>62873</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   app_code                     impression_id      impression_time  is_4G  \\\n",
       "0       422  c4ca4238a0b923820dcc509a6f75849b  2018-11-15 00:00:00      0   \n",
       "1       467  45c48cce2e2d7fbdea1afc51c7c6ad26  2018-11-15 00:01:00      1   \n",
       "2       259  70efdf2ec9b086079795c442636b55fb  2018-11-15 00:02:00      1   \n",
       "3       244  8e296a067a37563370ded05f5a3bf3ec  2018-11-15 00:02:00      1   \n",
       "4       473  182be0c5cdcd5072bb1864cdee4d3d6e  2018-11-15 00:02:00      0   \n",
       "\n",
       "   is_click    os_version  user_id  \n",
       "0       0.0           old    87862  \n",
       "1       1.0        latest    63410  \n",
       "2       0.0  intermediate    71748  \n",
       "3       0.0        latest    69209  \n",
       "4       0.0        latest    62873  "
      ]
     },
     "execution_count": 29,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "metadata": {},
   "outputs": [],
   "source": [
    "df=pd.get_dummies(df,columns=['os_version'],drop_first=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>app_code</th>\n",
       "      <th>impression_id</th>\n",
       "      <th>is_4G</th>\n",
       "      <th>is_click</th>\n",
       "      <th>user_id</th>\n",
       "      <th>os_version_latest</th>\n",
       "      <th>os_version_old</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>422</td>\n",
       "      <td>c4ca4238a0b923820dcc509a6f75849b</td>\n",
       "      <td>0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>87862</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>467</td>\n",
       "      <td>45c48cce2e2d7fbdea1afc51c7c6ad26</td>\n",
       "      <td>1</td>\n",
       "      <td>1.0</td>\n",
       "      <td>63410</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>259</td>\n",
       "      <td>70efdf2ec9b086079795c442636b55fb</td>\n",
       "      <td>1</td>\n",
       "      <td>0.0</td>\n",
       "      <td>71748</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>244</td>\n",
       "      <td>8e296a067a37563370ded05f5a3bf3ec</td>\n",
       "      <td>1</td>\n",
       "      <td>0.0</td>\n",
       "      <td>69209</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>473</td>\n",
       "      <td>182be0c5cdcd5072bb1864cdee4d3d6e</td>\n",
       "      <td>0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>62873</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "  app_code                     impression_id is_4G  is_click  user_id  \\\n",
       "0      422  c4ca4238a0b923820dcc509a6f75849b     0       0.0    87862   \n",
       "1      467  45c48cce2e2d7fbdea1afc51c7c6ad26     1       1.0    63410   \n",
       "2      259  70efdf2ec9b086079795c442636b55fb     1       0.0    71748   \n",
       "3      244  8e296a067a37563370ded05f5a3bf3ec     1       0.0    69209   \n",
       "4      473  182be0c5cdcd5072bb1864cdee4d3d6e     0       0.0    62873   \n",
       "\n",
       "   os_version_latest  os_version_old  \n",
       "0                  0               1  \n",
       "1                  1               0  \n",
       "2                  0               0  \n",
       "3                  1               0  \n",
       "4                  1               0  "
      ]
     },
     "execution_count": 31,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "#preprocessing for df\n",
    "df['impression_time']=pd.to_datetime(df['impression_time'],format='%Y-%m-%d %H:%M:%S')\n",
    "df['is_4G']=df['is_4G'].astype('category')\n",
    "df['app_code']=df['app_code'].astype('category')\n",
    "\n",
    "df.drop(['impression_time'],axis=1,inplace=True)\n",
    "\n",
    "df.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>app_code</th>\n",
       "      <th>impression_id</th>\n",
       "      <th>is_4G</th>\n",
       "      <th>is_click</th>\n",
       "      <th>user_id</th>\n",
       "      <th>os_version_latest</th>\n",
       "      <th>os_version_old</th>\n",
       "      <th>J_device_type_count</th>\n",
       "      <th>J_device_type_nunique</th>\n",
       "      <th>J_session_id_count</th>\n",
       "      <th>...</th>\n",
       "      <th>J_cumcount_1_max</th>\n",
       "      <th>J_cumcount_1_sum</th>\n",
       "      <th>J_cumcount_2_min</th>\n",
       "      <th>J_cumcount_2_mean</th>\n",
       "      <th>J_cumcount_2_max</th>\n",
       "      <th>J_cumcount_2_sum</th>\n",
       "      <th>J_cumcount_3_min</th>\n",
       "      <th>J_cumcount_3_mean</th>\n",
       "      <th>J_cumcount_3_max</th>\n",
       "      <th>J_cumcount_3_sum</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>422</td>\n",
       "      <td>c4ca4238a0b923820dcc509a6f75849b</td>\n",
       "      <td>0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>87862</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>...</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>467</td>\n",
       "      <td>45c48cce2e2d7fbdea1afc51c7c6ad26</td>\n",
       "      <td>1</td>\n",
       "      <td>1.0</td>\n",
       "      <td>63410</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>12</td>\n",
       "      <td>1</td>\n",
       "      <td>12</td>\n",
       "      <td>...</td>\n",
       "      <td>12</td>\n",
       "      <td>78</td>\n",
       "      <td>1</td>\n",
       "      <td>1.916667</td>\n",
       "      <td>5</td>\n",
       "      <td>23</td>\n",
       "      <td>1</td>\n",
       "      <td>1.166667</td>\n",
       "      <td>2</td>\n",
       "      <td>14</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>259</td>\n",
       "      <td>70efdf2ec9b086079795c442636b55fb</td>\n",
       "      <td>1</td>\n",
       "      <td>0.0</td>\n",
       "      <td>71748</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>2</td>\n",
       "      <td>1</td>\n",
       "      <td>2</td>\n",
       "      <td>...</td>\n",
       "      <td>2</td>\n",
       "      <td>3</td>\n",
       "      <td>1</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>1</td>\n",
       "      <td>2</td>\n",
       "      <td>1</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>1</td>\n",
       "      <td>2</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>244</td>\n",
       "      <td>8e296a067a37563370ded05f5a3bf3ec</td>\n",
       "      <td>1</td>\n",
       "      <td>0.0</td>\n",
       "      <td>69209</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>18</td>\n",
       "      <td>1</td>\n",
       "      <td>18</td>\n",
       "      <td>...</td>\n",
       "      <td>18</td>\n",
       "      <td>171</td>\n",
       "      <td>1</td>\n",
       "      <td>1.388889</td>\n",
       "      <td>3</td>\n",
       "      <td>25</td>\n",
       "      <td>1</td>\n",
       "      <td>1.277778</td>\n",
       "      <td>3</td>\n",
       "      <td>23</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>473</td>\n",
       "      <td>182be0c5cdcd5072bb1864cdee4d3d6e</td>\n",
       "      <td>0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>62873</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>46</td>\n",
       "      <td>1</td>\n",
       "      <td>46</td>\n",
       "      <td>...</td>\n",
       "      <td>46</td>\n",
       "      <td>1081</td>\n",
       "      <td>1</td>\n",
       "      <td>1.456522</td>\n",
       "      <td>5</td>\n",
       "      <td>67</td>\n",
       "      <td>1</td>\n",
       "      <td>1.130435</td>\n",
       "      <td>2</td>\n",
       "      <td>52</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>5 rows × 223 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "  app_code                     impression_id is_4G  is_click  user_id  \\\n",
       "0      422  c4ca4238a0b923820dcc509a6f75849b     0       0.0    87862   \n",
       "1      467  45c48cce2e2d7fbdea1afc51c7c6ad26     1       1.0    63410   \n",
       "2      259  70efdf2ec9b086079795c442636b55fb     1       0.0    71748   \n",
       "3      244  8e296a067a37563370ded05f5a3bf3ec     1       0.0    69209   \n",
       "4      473  182be0c5cdcd5072bb1864cdee4d3d6e     0       0.0    62873   \n",
       "\n",
       "   os_version_latest  os_version_old  J_device_type_count  \\\n",
       "0                  0               1                    1   \n",
       "1                  1               0                   12   \n",
       "2                  0               0                    2   \n",
       "3                  1               0                   18   \n",
       "4                  1               0                   46   \n",
       "\n",
       "   J_device_type_nunique  J_session_id_count  ...  J_cumcount_1_max  \\\n",
       "0                      1                   1  ...                 1   \n",
       "1                      1                  12  ...                12   \n",
       "2                      1                   2  ...                 2   \n",
       "3                      1                  18  ...                18   \n",
       "4                      1                  46  ...                46   \n",
       "\n",
       "   J_cumcount_1_sum  J_cumcount_2_min  J_cumcount_2_mean  J_cumcount_2_max  \\\n",
       "0                 1                 1           1.000000                 1   \n",
       "1                78                 1           1.916667                 5   \n",
       "2                 3                 1           1.000000                 1   \n",
       "3               171                 1           1.388889                 3   \n",
       "4              1081                 1           1.456522                 5   \n",
       "\n",
       "   J_cumcount_2_sum  J_cumcount_3_min  J_cumcount_3_mean  J_cumcount_3_max  \\\n",
       "0                 1                 1           1.000000                 1   \n",
       "1                23                 1           1.166667                 2   \n",
       "2                 2                 1           1.000000                 1   \n",
       "3                25                 1           1.277778                 3   \n",
       "4                67                 1           1.130435                 2   \n",
       "\n",
       "   J_cumcount_3_sum  \n",
       "0                 1  \n",
       "1                14  \n",
       "2                 2  \n",
       "3                23  \n",
       "4                52  \n",
       "\n",
       "[5 rows x 223 columns]"
      ]
     },
     "execution_count": 32,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df=df.merge(view_item1,on='user_id',how='left')\n",
    "\n",
    "df.shape\n",
    "df.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "metadata": {},
   "outputs": [],
   "source": [
    "df['user_id']=df['user_id'].astype('category')\n",
    "df['is_4G']=df['is_4G'].astype('category')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 34,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "app_code                 0\n",
       "impression_id            0\n",
       "is_4G                    0\n",
       "is_click             90675\n",
       "user_id                  0\n",
       "                     ...  \n",
       "J_cumcount_2_sum         0\n",
       "J_cumcount_3_min         0\n",
       "J_cumcount_3_mean        0\n",
       "J_cumcount_3_max         0\n",
       "J_cumcount_3_sum         0\n",
       "Length: 223, dtype: int64"
      ]
     },
     "execution_count": 34,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df.isnull().sum()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 35,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(237609, 223) (90675, 223)\n"
     ]
    }
   ],
   "source": [
    "df_train=df[df['is_click'].isnull()==False].copy()\n",
    "df_test=df[df['is_click'].isnull()==True].copy()\n",
    "\n",
    "print(df_train.shape,df_test.shape)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 37,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>app_code</th>\n",
       "      <th>impression_id</th>\n",
       "      <th>is_4G</th>\n",
       "      <th>is_click</th>\n",
       "      <th>user_id</th>\n",
       "      <th>os_version_latest</th>\n",
       "      <th>os_version_old</th>\n",
       "      <th>J_device_type_count</th>\n",
       "      <th>J_device_type_nunique</th>\n",
       "      <th>J_session_id_count</th>\n",
       "      <th>...</th>\n",
       "      <th>J_cumcount_1_max</th>\n",
       "      <th>J_cumcount_1_sum</th>\n",
       "      <th>J_cumcount_2_min</th>\n",
       "      <th>J_cumcount_2_mean</th>\n",
       "      <th>J_cumcount_2_max</th>\n",
       "      <th>J_cumcount_2_sum</th>\n",
       "      <th>J_cumcount_3_min</th>\n",
       "      <th>J_cumcount_3_mean</th>\n",
       "      <th>J_cumcount_3_max</th>\n",
       "      <th>J_cumcount_3_sum</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>422</td>\n",
       "      <td>c4ca4238a0b923820dcc509a6f75849b</td>\n",
       "      <td>0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>87862</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>...</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>467</td>\n",
       "      <td>45c48cce2e2d7fbdea1afc51c7c6ad26</td>\n",
       "      <td>1</td>\n",
       "      <td>1.0</td>\n",
       "      <td>63410</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>12</td>\n",
       "      <td>1</td>\n",
       "      <td>12</td>\n",
       "      <td>...</td>\n",
       "      <td>12</td>\n",
       "      <td>78</td>\n",
       "      <td>1</td>\n",
       "      <td>1.916667</td>\n",
       "      <td>5</td>\n",
       "      <td>23</td>\n",
       "      <td>1</td>\n",
       "      <td>1.166667</td>\n",
       "      <td>2</td>\n",
       "      <td>14</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>259</td>\n",
       "      <td>70efdf2ec9b086079795c442636b55fb</td>\n",
       "      <td>1</td>\n",
       "      <td>0.0</td>\n",
       "      <td>71748</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>2</td>\n",
       "      <td>1</td>\n",
       "      <td>2</td>\n",
       "      <td>...</td>\n",
       "      <td>2</td>\n",
       "      <td>3</td>\n",
       "      <td>1</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>1</td>\n",
       "      <td>2</td>\n",
       "      <td>1</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>1</td>\n",
       "      <td>2</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>244</td>\n",
       "      <td>8e296a067a37563370ded05f5a3bf3ec</td>\n",
       "      <td>1</td>\n",
       "      <td>0.0</td>\n",
       "      <td>69209</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>18</td>\n",
       "      <td>1</td>\n",
       "      <td>18</td>\n",
       "      <td>...</td>\n",
       "      <td>18</td>\n",
       "      <td>171</td>\n",
       "      <td>1</td>\n",
       "      <td>1.388889</td>\n",
       "      <td>3</td>\n",
       "      <td>25</td>\n",
       "      <td>1</td>\n",
       "      <td>1.277778</td>\n",
       "      <td>3</td>\n",
       "      <td>23</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>473</td>\n",
       "      <td>182be0c5cdcd5072bb1864cdee4d3d6e</td>\n",
       "      <td>0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>62873</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>46</td>\n",
       "      <td>1</td>\n",
       "      <td>46</td>\n",
       "      <td>...</td>\n",
       "      <td>46</td>\n",
       "      <td>1081</td>\n",
       "      <td>1</td>\n",
       "      <td>1.456522</td>\n",
       "      <td>5</td>\n",
       "      <td>67</td>\n",
       "      <td>1</td>\n",
       "      <td>1.130435</td>\n",
       "      <td>2</td>\n",
       "      <td>52</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>5 rows × 223 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "  app_code                     impression_id is_4G  is_click user_id  \\\n",
       "0      422  c4ca4238a0b923820dcc509a6f75849b     0       0.0   87862   \n",
       "1      467  45c48cce2e2d7fbdea1afc51c7c6ad26     1       1.0   63410   \n",
       "2      259  70efdf2ec9b086079795c442636b55fb     1       0.0   71748   \n",
       "3      244  8e296a067a37563370ded05f5a3bf3ec     1       0.0   69209   \n",
       "4      473  182be0c5cdcd5072bb1864cdee4d3d6e     0       0.0   62873   \n",
       "\n",
       "   os_version_latest  os_version_old  J_device_type_count  \\\n",
       "0                  0               1                    1   \n",
       "1                  1               0                   12   \n",
       "2                  0               0                    2   \n",
       "3                  1               0                   18   \n",
       "4                  1               0                   46   \n",
       "\n",
       "   J_device_type_nunique  J_session_id_count  ...  J_cumcount_1_max  \\\n",
       "0                      1                   1  ...                 1   \n",
       "1                      1                  12  ...                12   \n",
       "2                      1                   2  ...                 2   \n",
       "3                      1                  18  ...                18   \n",
       "4                      1                  46  ...                46   \n",
       "\n",
       "   J_cumcount_1_sum  J_cumcount_2_min  J_cumcount_2_mean  J_cumcount_2_max  \\\n",
       "0                 1                 1           1.000000                 1   \n",
       "1                78                 1           1.916667                 5   \n",
       "2                 3                 1           1.000000                 1   \n",
       "3               171                 1           1.388889                 3   \n",
       "4              1081                 1           1.456522                 5   \n",
       "\n",
       "   J_cumcount_2_sum  J_cumcount_3_min  J_cumcount_3_mean  J_cumcount_3_max  \\\n",
       "0                 1                 1           1.000000                 1   \n",
       "1                23                 1           1.166667                 2   \n",
       "2                 2                 1           1.000000                 1   \n",
       "3                25                 1           1.277778                 3   \n",
       "4                67                 1           1.130435                 2   \n",
       "\n",
       "   J_cumcount_3_sum  \n",
       "0                 1  \n",
       "1                14  \n",
       "2                 2  \n",
       "3                23  \n",
       "4                52  \n",
       "\n",
       "[5 rows x 223 columns]"
      ]
     },
     "execution_count": 37,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# del df\n",
    "gc.collect()\n",
    "\n",
    "df_train.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 40,
   "metadata": {},
   "outputs": [],
   "source": [
    "df_train=df_train.merge(df_train.groupby(['user_id']).count()[['impression_id']].reset_index().rename(columns={'impression_id':'user_count'}),on='user_id',how='left')\n",
    "df_test=df_test.merge(df_test.groupby(['user_id']).count()[['impression_id']].reset_index().rename(columns={'impression_id':'user_count'}),on='user_id',how='left')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 41,
   "metadata": {},
   "outputs": [],
   "source": [
    "from catboost import CatBoostClassifier,Pool, cv\n",
    "from lightgbm import LGBMClassifier\n",
    "from sklearn.model_selection import StratifiedKFold,train_test_split\n",
    "from sklearn.linear_model import LogisticRegression\n",
    "from sklearn.ensemble import RandomForestClassifier\n",
    "from sklearn.metrics import accuracy_score,confusion_matrix,roc_auc_score"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 42,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(237609, 222) (90675, 222)\n"
     ]
    }
   ],
   "source": [
    "X,y=df_train.drop(['impression_id','is_click'],axis=1),df_train['is_click']\n",
    "Xtest=df_test.drop(['impression_id','is_click'],axis=1)\n",
    "\n",
    "print(X.shape,Xtest.shape)\n",
    "X_train,X_val,y_train,y_val = train_test_split(X,y,test_size=0.25,random_state = 1994,stratify=y)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 43,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0    0.954286\n",
       "1    0.045714\n",
       "Name: is_click, dtype: float64"
      ]
     },
     "execution_count": 43,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "train['is_click'].value_counts()/train.shape[0]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 46,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "app_code             0\n",
       "is_4G                0\n",
       "user_id              0\n",
       "os_version_latest    0\n",
       "os_version_old       0\n",
       "                    ..\n",
       "J_cumcount_3_min     0\n",
       "J_cumcount_3_mean    0\n",
       "J_cumcount_3_max     0\n",
       "J_cumcount_3_sum     0\n",
       "user_count           0\n",
       "Length: 222, dtype: int64"
      ]
     },
     "execution_count": 46,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Xtest.head()\n",
    "X.isnull().sum()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 49,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Training until validation scores don't improve for 200 rounds.\n",
      "[200]\tvalid_0's binary_logloss: 0.168698\tvalid_0's auc: 0.746438\n",
      "[400]\tvalid_0's binary_logloss: 0.167574\tvalid_0's auc: 0.749888\n",
      "[600]\tvalid_0's binary_logloss: 0.167415\tvalid_0's auc: 0.750645\n",
      "Early stopping, best iteration is:\n",
      "[525]\tvalid_0's binary_logloss: 0.167256\tvalid_0's auc: 0.751311\n",
      "err:  0.7513112264925058\n",
      "Training until validation scores don't improve for 200 rounds.\n",
      "[200]\tvalid_0's binary_logloss: 0.170478\tvalid_0's auc: 0.735446\n",
      "[400]\tvalid_0's binary_logloss: 0.169739\tvalid_0's auc: 0.738919\n",
      "[600]\tvalid_0's binary_logloss: 0.169722\tvalid_0's auc: 0.740304\n",
      "Early stopping, best iteration is:\n",
      "[528]\tvalid_0's binary_logloss: 0.169609\tvalid_0's auc: 0.740465\n",
      "err:  0.7404650733485545\n",
      "Training until validation scores don't improve for 200 rounds.\n",
      "[200]\tvalid_0's binary_logloss: 0.168564\tvalid_0's auc: 0.745053\n",
      "[400]\tvalid_0's binary_logloss: 0.167726\tvalid_0's auc: 0.747611\n",
      "[600]\tvalid_0's binary_logloss: 0.167639\tvalid_0's auc: 0.749005\n",
      "Early stopping, best iteration is:\n",
      "[502]\tvalid_0's binary_logloss: 0.167526\tvalid_0's auc: 0.749219\n",
      "err:  0.7492190472709699\n",
      "Training until validation scores don't improve for 200 rounds.\n",
      "[200]\tvalid_0's binary_logloss: 0.168293\tvalid_0's auc: 0.746348\n",
      "[400]\tvalid_0's binary_logloss: 0.167559\tvalid_0's auc: 0.746954\n",
      "[600]\tvalid_0's binary_logloss: 0.167745\tvalid_0's auc: 0.746265\n",
      "Early stopping, best iteration is:\n",
      "[508]\tvalid_0's binary_logloss: 0.167476\tvalid_0's auc: 0.747519\n",
      "err:  0.7475194365087583\n",
      "Training until validation scores don't improve for 200 rounds.\n",
      "[200]\tvalid_0's binary_logloss: 0.168911\tvalid_0's auc: 0.747868\n",
      "[400]\tvalid_0's binary_logloss: 0.167786\tvalid_0's auc: 0.75152\n",
      "[600]\tvalid_0's binary_logloss: 0.167698\tvalid_0's auc: 0.751676\n",
      "Early stopping, best iteration is:\n",
      "[484]\tvalid_0's binary_logloss: 0.167638\tvalid_0's auc: 0.751853\n",
      "err:  0.7518532754248215\n",
      "Training until validation scores don't improve for 200 rounds.\n",
      "[200]\tvalid_0's binary_logloss: 0.16677\tvalid_0's auc: 0.755928\n",
      "[400]\tvalid_0's binary_logloss: 0.165211\tvalid_0's auc: 0.76145\n",
      "[600]\tvalid_0's binary_logloss: 0.164819\tvalid_0's auc: 0.763055\n",
      "[800]\tvalid_0's binary_logloss: 0.164882\tvalid_0's auc: 0.763161\n",
      "Early stopping, best iteration is:\n",
      "[613]\tvalid_0's binary_logloss: 0.164762\tvalid_0's auc: 0.763393\n",
      "err:  0.7633934144296153\n",
      "Training until validation scores don't improve for 200 rounds.\n",
      "[200]\tvalid_0's binary_logloss: 0.169565\tvalid_0's auc: 0.738628\n",
      "[400]\tvalid_0's binary_logloss: 0.168572\tvalid_0's auc: 0.743103\n",
      "[600]\tvalid_0's binary_logloss: 0.16851\tvalid_0's auc: 0.74435\n",
      "Early stopping, best iteration is:\n",
      "[484]\tvalid_0's binary_logloss: 0.16848\tvalid_0's auc: 0.743688\n",
      "err:  0.7436882158614906\n",
      "Training until validation scores don't improve for 200 rounds.\n",
      "[200]\tvalid_0's binary_logloss: 0.168709\tvalid_0's auc: 0.748216\n",
      "[400]\tvalid_0's binary_logloss: 0.167448\tvalid_0's auc: 0.752274\n",
      "[600]\tvalid_0's binary_logloss: 0.167042\tvalid_0's auc: 0.754599\n",
      "Early stopping, best iteration is:\n",
      "[543]\tvalid_0's binary_logloss: 0.167006\tvalid_0's auc: 0.754556\n",
      "err:  0.7545564150434918\n",
      "Training until validation scores don't improve for 200 rounds.\n",
      "[200]\tvalid_0's binary_logloss: 0.169281\tvalid_0's auc: 0.740119\n",
      "[400]\tvalid_0's binary_logloss: 0.168729\tvalid_0's auc: 0.740322\n",
      "Early stopping, best iteration is:\n",
      "[293]\tvalid_0's binary_logloss: 0.168745\tvalid_0's auc: 0.741119\n",
      "err:  0.7411194842552564\n",
      "Training until validation scores don't improve for 200 rounds.\n",
      "[200]\tvalid_0's binary_logloss: 0.169126\tvalid_0's auc: 0.74353\n",
      "[400]\tvalid_0's binary_logloss: 0.168223\tvalid_0's auc: 0.747395\n",
      "[600]\tvalid_0's binary_logloss: 0.16821\tvalid_0's auc: 0.748223\n",
      "Early stopping, best iteration is:\n",
      "[516]\tvalid_0's binary_logloss: 0.167996\tvalid_0's auc: 0.749049\n",
      "err:  0.7490488330798404\n"
     ]
    }
   ],
   "source": [
    "err=[]\n",
    "y_pred_tot=[]\n",
    "from sklearn.model_selection import KFold,StratifiedKFold\n",
    "fold=StratifiedKFold(n_splits=10,shuffle=True,random_state=1994)\n",
    "i=1\n",
    "for train_index, test_index in fold.split(X,y):\n",
    "    X_train, X_test = X.iloc[train_index], X.iloc[test_index]\n",
    "    y_train, y_test = y[train_index], y[test_index]\n",
    "    m=LGBMClassifier(n_estimators=3000,random_state=1994,learning_rate=0.03,colsample_bytree=0.2,objective='binary',scale_pos_weight=1)\n",
    "    m.fit(X_train,y_train,eval_set=[(X_test, y_test)],eval_metric='auc', early_stopping_rounds=200,verbose=200)\n",
    "    preds=m.predict_proba(X_test)[:,-1]\n",
    "    print(\"err: \",roc_auc_score(y_test,preds))\n",
    "    err.append(roc_auc_score(y_test,preds))\n",
    "    p = m.predict_proba(Xtest)[:,-1]\n",
    "    i=i+1\n",
    "    y_pred_tot.append(p)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 51,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.7492174421715305"
      ]
     },
     "execution_count": 51,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "np.mean(err,0)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 52,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>impression_id</th>\n",
       "      <th>is_click</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>a9e7126a585a69a32bc7414e9d0c0ada</td>\n",
       "      <td>0.013531</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>caac14a5bf2ba283db7708bb34855760</td>\n",
       "      <td>0.010222</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>13f10ba306a19ce7bec2f3cae507b698</td>\n",
       "      <td>0.062582</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>39c4b4dc0e9701b55a0a4f072008fb3f</td>\n",
       "      <td>0.014803</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>bf5a572cca75f5fc67f4b14e58b11d70</td>\n",
       "      <td>0.190164</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                      impression_id  is_click\n",
       "0  a9e7126a585a69a32bc7414e9d0c0ada  0.013531\n",
       "1  caac14a5bf2ba283db7708bb34855760  0.010222\n",
       "2  13f10ba306a19ce7bec2f3cae507b698  0.062582\n",
       "3  39c4b4dc0e9701b55a0a4f072008fb3f  0.014803\n",
       "4  bf5a572cca75f5fc67f4b14e58b11d70  0.190164"
      ]
     },
     "execution_count": 52,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "s['is_click']=np.mean(y_pred_tot,0)\n",
    "s.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 53,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/opt/conda/lib/python3.6/site-packages/tensorflow/python/framework/dtypes.py:516: FutureWarning: Passing (type, 1) or '1type' as a synonym of type is deprecated; in a future version of numpy, it will be understood as (type, (1,)) / '(1,)type'.\n",
      "  _np_qint8 = np.dtype([(\"qint8\", np.int8, 1)])\n",
      "/opt/conda/lib/python3.6/site-packages/tensorflow/python/framework/dtypes.py:517: FutureWarning: Passing (type, 1) or '1type' as a synonym of type is deprecated; in a future version of numpy, it will be understood as (type, (1,)) / '(1,)type'.\n",
      "  _np_quint8 = np.dtype([(\"quint8\", np.uint8, 1)])\n",
      "/opt/conda/lib/python3.6/site-packages/tensorflow/python/framework/dtypes.py:518: FutureWarning: Passing (type, 1) or '1type' as a synonym of type is deprecated; in a future version of numpy, it will be understood as (type, (1,)) / '(1,)type'.\n",
      "  _np_qint16 = np.dtype([(\"qint16\", np.int16, 1)])\n",
      "/opt/conda/lib/python3.6/site-packages/tensorflow/python/framework/dtypes.py:519: FutureWarning: Passing (type, 1) or '1type' as a synonym of type is deprecated; in a future version of numpy, it will be understood as (type, (1,)) / '(1,)type'.\n",
      "  _np_quint16 = np.dtype([(\"quint16\", np.uint16, 1)])\n",
      "/opt/conda/lib/python3.6/site-packages/tensorflow/python/framework/dtypes.py:520: FutureWarning: Passing (type, 1) or '1type' as a synonym of type is deprecated; in a future version of numpy, it will be understood as (type, (1,)) / '(1,)type'.\n",
      "  _np_qint32 = np.dtype([(\"qint32\", np.int32, 1)])\n",
      "/opt/conda/lib/python3.6/site-packages/tensorflow/python/framework/dtypes.py:525: FutureWarning: Passing (type, 1) or '1type' as a synonym of type is deprecated; in a future version of numpy, it will be understood as (type, (1,)) / '(1,)type'.\n",
      "  np_resource = np.dtype([(\"resource\", np.ubyte, 1)])\n",
      "/opt/conda/lib/python3.6/site-packages/tensorboard/compat/tensorflow_stub/dtypes.py:541: FutureWarning: Passing (type, 1) or '1type' as a synonym of type is deprecated; in a future version of numpy, it will be understood as (type, (1,)) / '(1,)type'.\n",
      "  _np_qint8 = np.dtype([(\"qint8\", np.int8, 1)])\n",
      "/opt/conda/lib/python3.6/site-packages/tensorboard/compat/tensorflow_stub/dtypes.py:542: FutureWarning: Passing (type, 1) or '1type' as a synonym of type is deprecated; in a future version of numpy, it will be understood as (type, (1,)) / '(1,)type'.\n",
      "  _np_quint8 = np.dtype([(\"quint8\", np.uint8, 1)])\n",
      "/opt/conda/lib/python3.6/site-packages/tensorboard/compat/tensorflow_stub/dtypes.py:543: FutureWarning: Passing (type, 1) or '1type' as a synonym of type is deprecated; in a future version of numpy, it will be understood as (type, (1,)) / '(1,)type'.\n",
      "  _np_qint16 = np.dtype([(\"qint16\", np.int16, 1)])\n",
      "/opt/conda/lib/python3.6/site-packages/tensorboard/compat/tensorflow_stub/dtypes.py:544: FutureWarning: Passing (type, 1) or '1type' as a synonym of type is deprecated; in a future version of numpy, it will be understood as (type, (1,)) / '(1,)type'.\n",
      "  _np_quint16 = np.dtype([(\"quint16\", np.uint16, 1)])\n",
      "/opt/conda/lib/python3.6/site-packages/tensorboard/compat/tensorflow_stub/dtypes.py:545: FutureWarning: Passing (type, 1) or '1type' as a synonym of type is deprecated; in a future version of numpy, it will be understood as (type, (1,)) / '(1,)type'.\n",
      "  _np_qint32 = np.dtype([(\"qint32\", np.int32, 1)])\n",
      "/opt/conda/lib/python3.6/site-packages/tensorboard/compat/tensorflow_stub/dtypes.py:550: FutureWarning: Passing (type, 1) or '1type' as a synonym of type is deprecated; in a future version of numpy, it will be understood as (type, (1,)) / '(1,)type'.\n",
      "  np_resource = np.dtype([(\"resource\", np.ubyte, 1)])\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "(90675, 2)"
      ]
     },
     "execution_count": 53,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "s.to_csv('AV_WNS_forkkv2_lgb_folds.csv',index=False)\n",
    "s.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 54,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(237609, 222) (90675, 222)\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "array([0, 1, 2])"
      ]
     },
     "execution_count": 54,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "print(X.shape,Xtest.shape)\n",
    "X_train,X_val,y_train,y_val = train_test_split(X,y,test_size=0.25,random_state = 1994,stratify=y)\n",
    "categorical_features_indices = np.where(X_train.dtypes =='category')[0]\n",
    "categorical_features_indices"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 55,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0:\ttest: 0.5418666\tbest: 0.5418666 (0)\ttotal: 370ms\tremaining: 15m 23s\n",
      "200:\ttest: 0.7364161\tbest: 0.7364161 (200)\ttotal: 1m\tremaining: 11m 29s\n",
      "400:\ttest: 0.7390786\tbest: 0.7390786 (400)\ttotal: 2m\tremaining: 10m 30s\n",
      "600:\ttest: 0.7398497\tbest: 0.7399070 (589)\ttotal: 2m 58s\tremaining: 9m 25s\n",
      "800:\ttest: 0.7405001\tbest: 0.7405083 (799)\ttotal: 3m 57s\tremaining: 8m 24s\n",
      "1000:\ttest: 0.7414331\tbest: 0.7414480 (995)\ttotal: 4m 57s\tremaining: 7m 25s\n",
      "1200:\ttest: 0.7416168\tbest: 0.7416217 (1194)\ttotal: 5m 56s\tremaining: 6m 25s\n",
      "1400:\ttest: 0.7415737\tbest: 0.7416946 (1277)\ttotal: 6m 55s\tremaining: 5m 25s\n",
      "1600:\ttest: 0.7416015\tbest: 0.7417861 (1448)\ttotal: 7m 55s\tremaining: 4m 26s\n",
      "Stopped by overfitting detector  (200 iterations wait)\n",
      "\n",
      "bestTest = 0.7417861135\n",
      "bestIteration = 1448\n",
      "\n",
      "Shrink model to first 1449 iterations.\n",
      "0.7417861135403558\n"
     ]
    }
   ],
   "source": [
    "m=CatBoostClassifier(n_estimators=2500,random_state=1994,learning_rate=0.03,eval_metric='AUC')\n",
    "m.fit(X_train,y_train,eval_set=[(X_val, y_val.values)], early_stopping_rounds=200,verbose=200,cat_features=categorical_features_indices)\n",
    "p=m.predict_proba(X_val)[:,-1]\n",
    "print(roc_auc_score(y_val,p))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 58,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0:\ttest: 0.5144795\tbest: 0.5144795 (0)\ttotal: 332ms\tremaining: 13m 49s\n",
      "200:\ttest: 0.7436090\tbest: 0.7436090 (200)\ttotal: 1m 7s\tremaining: 12m 47s\n",
      "400:\ttest: 0.7464037\tbest: 0.7464189 (397)\ttotal: 2m 14s\tremaining: 11m 42s\n",
      "600:\ttest: 0.7471002\tbest: 0.7471002 (600)\ttotal: 3m 19s\tremaining: 10m 29s\n",
      "800:\ttest: 0.7484396\tbest: 0.7484861 (797)\ttotal: 4m 24s\tremaining: 9m 21s\n",
      "1000:\ttest: 0.7489092\tbest: 0.7490326 (885)\ttotal: 5m 30s\tremaining: 8m 14s\n",
      "Stopped by overfitting detector  (200 iterations wait)\n",
      "\n",
      "bestTest = 0.7490325983\n",
      "bestIteration = 885\n",
      "\n",
      "Shrink model to first 886 iterations.\n",
      "err_cb:  0.7490325983432549\n",
      "0:\ttest: 0.5255802\tbest: 0.5255802 (0)\ttotal: 353ms\tremaining: 14m 41s\n",
      "200:\ttest: 0.7349605\tbest: 0.7349727 (196)\ttotal: 1m 11s\tremaining: 13m 41s\n",
      "400:\ttest: 0.7367486\tbest: 0.7367954 (398)\ttotal: 2m 22s\tremaining: 12m 27s\n",
      "600:\ttest: 0.7380834\tbest: 0.7381583 (588)\ttotal: 3m 31s\tremaining: 11m 9s\n",
      "800:\ttest: 0.7385614\tbest: 0.7385799 (798)\ttotal: 4m 41s\tremaining: 9m 56s\n",
      "1000:\ttest: 0.7390868\tbest: 0.7390968 (998)\ttotal: 5m 51s\tremaining: 8m 46s\n",
      "1200:\ttest: 0.7395140\tbest: 0.7395239 (1192)\ttotal: 7m 2s\tremaining: 7m 36s\n",
      "1400:\ttest: 0.7399729\tbest: 0.7400785 (1373)\ttotal: 8m 11s\tremaining: 6m 25s\n",
      "1600:\ttest: 0.7401355\tbest: 0.7401405 (1580)\ttotal: 9m 22s\tremaining: 5m 15s\n",
      "1800:\ttest: 0.7406562\tbest: 0.7406607 (1799)\ttotal: 10m 31s\tremaining: 4m 5s\n",
      "2000:\ttest: 0.7406412\tbest: 0.7407321 (1977)\ttotal: 11m 42s\tremaining: 2m 55s\n",
      "2200:\ttest: 0.7407028\tbest: 0.7408802 (2079)\ttotal: 12m 53s\tremaining: 1m 45s\n",
      "Stopped by overfitting detector  (200 iterations wait)\n",
      "\n",
      "bestTest = 0.7408802435\n",
      "bestIteration = 2079\n",
      "\n",
      "Shrink model to first 2080 iterations.\n",
      "err_cb:  0.740880243511318\n",
      "0:\ttest: 0.5699077\tbest: 0.5699077 (0)\ttotal: 406ms\tremaining: 16m 53s\n",
      "200:\ttest: 0.7484476\tbest: 0.7485515 (195)\ttotal: 1m 9s\tremaining: 13m 14s\n",
      "400:\ttest: 0.7499270\tbest: 0.7499463 (398)\ttotal: 2m 18s\tremaining: 12m 3s\n",
      "600:\ttest: 0.7511684\tbest: 0.7511878 (579)\ttotal: 3m 26s\tremaining: 10m 52s\n",
      "800:\ttest: 0.7522715\tbest: 0.7523015 (797)\ttotal: 4m 34s\tremaining: 9m 41s\n",
      "1000:\ttest: 0.7530256\tbest: 0.7531171 (955)\ttotal: 5m 42s\tremaining: 8m 32s\n",
      "1200:\ttest: 0.7532913\tbest: 0.7534078 (1133)\ttotal: 6m 50s\tremaining: 7m 23s\n",
      "1400:\ttest: 0.7535164\tbest: 0.7535470 (1321)\ttotal: 7m 59s\tremaining: 6m 15s\n",
      "1600:\ttest: 0.7538200\tbest: 0.7538463 (1594)\ttotal: 9m 7s\tremaining: 5m 7s\n",
      "1800:\ttest: 0.7540321\tbest: 0.7540744 (1795)\ttotal: 10m 15s\tremaining: 3m 58s\n",
      "2000:\ttest: 0.7545968\tbest: 0.7546311 (1984)\ttotal: 11m 23s\tremaining: 2m 50s\n",
      "2200:\ttest: 0.7549419\tbest: 0.7549912 (2189)\ttotal: 12m 30s\tremaining: 1m 41s\n",
      "2400:\ttest: 0.7545905\tbest: 0.7550024 (2223)\ttotal: 13m 39s\tremaining: 33.8s\n",
      "Stopped by overfitting detector  (200 iterations wait)\n",
      "\n",
      "bestTest = 0.7550023655\n",
      "bestIteration = 2223\n",
      "\n",
      "Shrink model to first 2224 iterations.\n",
      "err_cb:  0.7550023654774305\n",
      "0:\ttest: 0.5525951\tbest: 0.5525951 (0)\ttotal: 351ms\tremaining: 14m 38s\n",
      "200:\ttest: 0.7439602\tbest: 0.7439894 (199)\ttotal: 1m 7s\tremaining: 12m 53s\n",
      "400:\ttest: 0.7458887\tbest: 0.7458887 (400)\ttotal: 2m 13s\tremaining: 11m 38s\n",
      "600:\ttest: 0.7467823\tbest: 0.7468110 (599)\ttotal: 3m 18s\tremaining: 10m 25s\n",
      "800:\ttest: 0.7475678\tbest: 0.7476593 (786)\ttotal: 4m 23s\tremaining: 9m 19s\n",
      "1000:\ttest: 0.7481178\tbest: 0.7481585 (998)\ttotal: 5m 29s\tremaining: 8m 13s\n",
      "1200:\ttest: 0.7483658\tbest: 0.7483658 (1200)\ttotal: 6m 34s\tremaining: 7m 7s\n",
      "1400:\ttest: 0.7480768\tbest: 0.7483658 (1200)\ttotal: 7m 40s\tremaining: 6m\n",
      "Stopped by overfitting detector  (200 iterations wait)\n",
      "\n",
      "bestTest = 0.7483658104\n",
      "bestIteration = 1200\n",
      "\n",
      "Shrink model to first 1201 iterations.\n",
      "err_cb:  0.7483658104247505\n",
      "0:\ttest: 0.5295316\tbest: 0.5295316 (0)\ttotal: 341ms\tremaining: 14m 12s\n",
      "200:\ttest: 0.7419186\tbest: 0.7419224 (198)\ttotal: 1m 7s\tremaining: 12m 51s\n",
      "400:\ttest: 0.7447588\tbest: 0.7447588 (400)\ttotal: 2m 13s\tremaining: 11m 40s\n",
      "600:\ttest: 0.7464010\tbest: 0.7464042 (599)\ttotal: 3m 19s\tremaining: 10m 29s\n",
      "800:\ttest: 0.7470388\tbest: 0.7470816 (786)\ttotal: 4m 23s\tremaining: 9m 19s\n",
      "1000:\ttest: 0.7479356\tbest: 0.7479492 (982)\ttotal: 5m 28s\tremaining: 8m 11s\n",
      "1200:\ttest: 0.7482608\tbest: 0.7483751 (1151)\ttotal: 6m 33s\tremaining: 7m 5s\n",
      "1400:\ttest: 0.7484390\tbest: 0.7486519 (1269)\ttotal: 7m 38s\tremaining: 5m 59s\n",
      "1600:\ttest: 0.7486441\tbest: 0.7486764 (1572)\ttotal: 8m 43s\tremaining: 4m 54s\n",
      "1800:\ttest: 0.7489762\tbest: 0.7490161 (1799)\ttotal: 9m 49s\tremaining: 3m 48s\n",
      "2000:\ttest: 0.7493574\tbest: 0.7494478 (1927)\ttotal: 10m 55s\tremaining: 2m 43s\n",
      "2200:\ttest: 0.7493936\tbest: 0.7494826 (2110)\ttotal: 11m 59s\tremaining: 1m 37s\n",
      "2400:\ttest: 0.7498119\tbest: 0.7498972 (2358)\ttotal: 13m 5s\tremaining: 32.4s\n",
      "2499:\ttest: 0.7498882\tbest: 0.7499641 (2478)\ttotal: 13m 37s\tremaining: 0us\n",
      "\n",
      "bestTest = 0.7499641016\n",
      "bestIteration = 2478\n",
      "\n",
      "Shrink model to first 2479 iterations.\n",
      "err_cb:  0.7499641015957328\n",
      "0:\ttest: 0.5570438\tbest: 0.5570438 (0)\ttotal: 403ms\tremaining: 16m 46s\n",
      "200:\ttest: 0.7486009\tbest: 0.7486074 (199)\ttotal: 1m 7s\tremaining: 12m 46s\n",
      "400:\ttest: 0.7506760\tbest: 0.7506760 (400)\ttotal: 2m 15s\tremaining: 11m 48s\n",
      "600:\ttest: 0.7515125\tbest: 0.7516139 (568)\ttotal: 3m 20s\tremaining: 10m 33s\n",
      "800:\ttest: 0.7532311\tbest: 0.7532333 (798)\ttotal: 4m 25s\tremaining: 9m 23s\n",
      "1000:\ttest: 0.7537186\tbest: 0.7537824 (947)\ttotal: 5m 30s\tremaining: 8m 15s\n",
      "1200:\ttest: 0.7541483\tbest: 0.7541535 (1199)\ttotal: 6m 37s\tremaining: 7m 9s\n",
      "1400:\ttest: 0.7544343\tbest: 0.7546010 (1256)\ttotal: 7m 43s\tremaining: 6m 3s\n",
      "Stopped by overfitting detector  (200 iterations wait)\n",
      "\n",
      "bestTest = 0.7546009653\n",
      "bestIteration = 1256\n",
      "\n",
      "Shrink model to first 1257 iterations.\n",
      "err_cb:  0.7546009652772279\n",
      "0:\ttest: 0.5590386\tbest: 0.5590386 (0)\ttotal: 399ms\tremaining: 16m 37s\n",
      "200:\ttest: 0.7360658\tbest: 0.7360749 (199)\ttotal: 1m 6s\tremaining: 12m 45s\n",
      "400:\ttest: 0.7385525\tbest: 0.7385525 (400)\ttotal: 2m 11s\tremaining: 11m 30s\n",
      "600:\ttest: 0.7402099\tbest: 0.7402099 (600)\ttotal: 3m 16s\tremaining: 10m 19s\n",
      "800:\ttest: 0.7408576\tbest: 0.7409951 (774)\ttotal: 4m 19s\tremaining: 9m 11s\n",
      "1000:\ttest: 0.7412622\tbest: 0.7412838 (996)\ttotal: 5m 23s\tremaining: 8m 5s\n",
      "1200:\ttest: 0.7419062\tbest: 0.7419062 (1200)\ttotal: 6m 28s\tremaining: 7m\n",
      "1400:\ttest: 0.7420229\tbest: 0.7420326 (1392)\ttotal: 7m 33s\tremaining: 5m 55s\n",
      "1600:\ttest: 0.7426657\tbest: 0.7427121 (1593)\ttotal: 8m 39s\tremaining: 4m 51s\n",
      "1800:\ttest: 0.7429967\tbest: 0.7430180 (1799)\ttotal: 9m 43s\tremaining: 3m 46s\n",
      "2000:\ttest: 0.7434697\tbest: 0.7434927 (1992)\ttotal: 10m 47s\tremaining: 2m 41s\n",
      "2200:\ttest: 0.7436493\tbest: 0.7437120 (2135)\ttotal: 11m 52s\tremaining: 1m 36s\n",
      "2400:\ttest: 0.7438011\tbest: 0.7438741 (2282)\ttotal: 12m 58s\tremaining: 32.1s\n",
      "2499:\ttest: 0.7437049\tbest: 0.7438786 (2432)\ttotal: 13m 30s\tremaining: 0us\n",
      "\n",
      "bestTest = 0.7438786114\n",
      "bestIteration = 2432\n",
      "\n",
      "Shrink model to first 2433 iterations.\n",
      "err_cb:  0.7438786114139869\n",
      "0:\ttest: 0.5587621\tbest: 0.5587621 (0)\ttotal: 365ms\tremaining: 15m 12s\n",
      "200:\ttest: 0.7479608\tbest: 0.7479945 (192)\ttotal: 1m 6s\tremaining: 12m 37s\n",
      "400:\ttest: 0.7507254\tbest: 0.7507254 (400)\ttotal: 2m 12s\tremaining: 11m 31s\n",
      "600:\ttest: 0.7519246\tbest: 0.7520294 (593)\ttotal: 3m 16s\tremaining: 10m 20s\n",
      "800:\ttest: 0.7521744\tbest: 0.7522218 (739)\ttotal: 4m 21s\tremaining: 9m 14s\n",
      "1000:\ttest: 0.7532134\tbest: 0.7533053 (975)\ttotal: 5m 27s\tremaining: 8m 10s\n",
      "Stopped by overfitting detector  (200 iterations wait)\n",
      "\n",
      "bestTest = 0.7533052964\n",
      "bestIteration = 975\n",
      "\n",
      "Shrink model to first 976 iterations.\n",
      "err_cb:  0.7533052964177498\n",
      "0:\ttest: 0.5604210\tbest: 0.5604210 (0)\ttotal: 404ms\tremaining: 16m 49s\n",
      "200:\ttest: 0.7361066\tbest: 0.7361066 (200)\ttotal: 1m 9s\tremaining: 13m 11s\n",
      "400:\ttest: 0.7385754\tbest: 0.7385830 (399)\ttotal: 2m 18s\tremaining: 12m 6s\n",
      "600:\ttest: 0.7403040\tbest: 0.7403500 (590)\ttotal: 3m 27s\tremaining: 10m 55s\n",
      "800:\ttest: 0.7417078\tbest: 0.7417528 (792)\ttotal: 4m 36s\tremaining: 9m 46s\n",
      "1000:\ttest: 0.7421358\tbest: 0.7421992 (975)\ttotal: 5m 45s\tremaining: 8m 37s\n",
      "1200:\ttest: 0.7423636\tbest: 0.7425127 (1158)\ttotal: 6m 53s\tremaining: 7m 27s\n",
      "1400:\ttest: 0.7427913\tbest: 0.7429467 (1315)\ttotal: 8m 4s\tremaining: 6m 20s\n",
      "1600:\ttest: 0.7431347\tbest: 0.7431689 (1578)\ttotal: 9m 14s\tremaining: 5m 11s\n",
      "1800:\ttest: 0.7431388\tbest: 0.7432008 (1626)\ttotal: 10m 23s\tremaining: 4m 2s\n",
      "Stopped by overfitting detector  (200 iterations wait)\n",
      "\n",
      "bestTest = 0.7432008104\n",
      "bestIteration = 1626\n",
      "\n",
      "Shrink model to first 1627 iterations.\n",
      "err_cb:  0.743200810397546\n",
      "0:\ttest: 0.5633069\tbest: 0.5633069 (0)\ttotal: 369ms\tremaining: 15m 22s\n",
      "200:\ttest: 0.7405152\tbest: 0.7405219 (199)\ttotal: 1m 8s\tremaining: 12m 58s\n",
      "400:\ttest: 0.7428690\tbest: 0.7428690 (400)\ttotal: 2m 15s\tremaining: 11m 48s\n",
      "600:\ttest: 0.7445805\tbest: 0.7446532 (583)\ttotal: 3m 19s\tremaining: 10m 31s\n",
      "800:\ttest: 0.7454955\tbest: 0.7455142 (796)\ttotal: 4m 24s\tremaining: 9m 21s\n",
      "1000:\ttest: 0.7462670\tbest: 0.7462670 (1000)\ttotal: 5m 29s\tremaining: 8m 13s\n",
      "1200:\ttest: 0.7461326\tbest: 0.7462814 (1009)\ttotal: 6m 35s\tremaining: 7m 7s\n",
      "Stopped by overfitting detector  (200 iterations wait)\n",
      "\n",
      "bestTest = 0.7462813867\n",
      "bestIteration = 1009\n",
      "\n",
      "Shrink model to first 1010 iterations.\n",
      "err_cb:  0.7462813867011826\n"
     ]
    }
   ],
   "source": [
    "errCB=[]\n",
    "y_pred_tot_cb=[]\n",
    "from sklearn.model_selection import KFold,StratifiedKFold\n",
    "fold=StratifiedKFold(n_splits=10,shuffle=True,random_state=1994)\n",
    "i=1\n",
    "for train_index, test_index in fold.split(X,y):\n",
    "    X_train, X_test = X.iloc[train_index], X.iloc[test_index]\n",
    "    y_train, y_test = y[train_index], y[test_index]\n",
    "    m=CatBoostClassifier(n_estimators=2500,random_state=1994,eval_metric='AUC',learning_rate=0.03)\n",
    "    m.fit(X_train,y_train,eval_set=[(X_test, y_test)], early_stopping_rounds=200,verbose=200,cat_features=categorical_features_indices)\n",
    "    preds=m.predict_proba(X_test)[:,-1]\n",
    "    print(\"err_cb: \",roc_auc_score(y_test,preds))\n",
    "    errCB.append(roc_auc_score(y_test,preds))\n",
    "    p = m.predict_proba(Xtest)[:,-1]\n",
    "    i=i+1\n",
    "    y_pred_tot_cb.append(p)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 59,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.7484512189560181"
      ]
     },
     "execution_count": 59,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "np.mean(errCB,0)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 60,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>impression_id</th>\n",
       "      <th>is_click</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>a9e7126a585a69a32bc7414e9d0c0ada</td>\n",
       "      <td>0.016591</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>caac14a5bf2ba283db7708bb34855760</td>\n",
       "      <td>0.031968</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>13f10ba306a19ce7bec2f3cae507b698</td>\n",
       "      <td>0.077346</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>39c4b4dc0e9701b55a0a4f072008fb3f</td>\n",
       "      <td>0.012104</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>bf5a572cca75f5fc67f4b14e58b11d70</td>\n",
       "      <td>0.141255</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                      impression_id  is_click\n",
       "0  a9e7126a585a69a32bc7414e9d0c0ada  0.016591\n",
       "1  caac14a5bf2ba283db7708bb34855760  0.031968\n",
       "2  13f10ba306a19ce7bec2f3cae507b698  0.077346\n",
       "3  39c4b4dc0e9701b55a0a4f072008fb3f  0.012104\n",
       "4  bf5a572cca75f5fc67f4b14e58b11d70  0.141255"
      ]
     },
     "execution_count": 60,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "s['is_click']=np.mean(y_pred_tot_cb,0)\n",
    "s.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 61,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(90675, 2)"
      ]
     },
     "execution_count": 61,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "s.to_csv('AV_WNS_forkkv2_cb_folds.csv',index=False)\n",
    "s.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 64,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>impression_id</th>\n",
       "      <th>is_click</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>a9e7126a585a69a32bc7414e9d0c0ada</td>\n",
       "      <td>0.015520</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>caac14a5bf2ba283db7708bb34855760</td>\n",
       "      <td>0.024357</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>13f10ba306a19ce7bec2f3cae507b698</td>\n",
       "      <td>0.072179</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>39c4b4dc0e9701b55a0a4f072008fb3f</td>\n",
       "      <td>0.013049</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>bf5a572cca75f5fc67f4b14e58b11d70</td>\n",
       "      <td>0.158373</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                      impression_id  is_click\n",
       "0  a9e7126a585a69a32bc7414e9d0c0ada  0.015520\n",
       "1  caac14a5bf2ba283db7708bb34855760  0.024357\n",
       "2  13f10ba306a19ce7bec2f3cae507b698  0.072179\n",
       "3  39c4b4dc0e9701b55a0a4f072008fb3f  0.013049\n",
       "4  bf5a572cca75f5fc67f4b14e58b11d70  0.158373"
      ]
     },
     "execution_count": 64,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "s['is_click']=np.mean(y_pred_tot_cb,0)*0.65 + np.mean(y_pred_tot,0)*0.35\n",
    "s.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 65,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(90675, 2)"
      ]
     },
     "execution_count": 65,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "s.to_csv('AV_WNS_forkkv2_CBstack_folds.csv',index=False)\n",
    "s.shape"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.1"
  },
  "toc": {
   "base_numbering": 1,
   "nav_menu": {},
   "number_sections": true,
   "sideBar": true,
   "skip_h1_title": false,
   "title_cell": "Table of Contents",
   "title_sidebar": "Contents",
   "toc_cell": false,
   "toc_position": {},
   "toc_section_display": true,
   "toc_window_display": false
  }
 },
 "nbformat": 4,
 "nbformat_minor": 1
}