Model.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 9,
   "id": "50418ecf",
   "metadata": {},
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "import matplotlib.pyplot as plt\n",
    "import seaborn as sns\n",
    "from sklearn.linear_model import LinearRegression,Lasso,Ridge\n",
    "from sklearn.ensemble import RandomForestRegressor\n",
    "from sklearn.model_selection import GridSearchCV\n",
    "from sklearn.preprocessing import StandardScaler\n",
    "from sklearn.model_selection import train_test_split\n",
    "import numpy as np\n",
    "from sklearn.base import BaseEstimator, TransformerMixin\n",
    "import statsmodels.api as sm\n",
    "from sklearn.pipeline import Pipeline\n",
    "from sklearn.neural_network import MLPRegressor\n",
    "from sklearn.model_selection import cross_val_score"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "ee1532b5",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>bedrooms</th>\n",
       "      <th>bathrooms</th>\n",
       "      <th>area</th>\n",
       "      <th>prices</th>\n",
       "      <th>locations</th>\n",
       "      <th>property_types</th>\n",
       "      <th>pool</th>\n",
       "      <th>balcony</th>\n",
       "      <th>furnished</th>\n",
       "      <th>maid</th>\n",
       "      <th>gym</th>\n",
       "      <th>brand_new</th>\n",
       "      <th>burj_view</th>\n",
       "      <th>sea_view</th>\n",
       "      <th>beach</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>352.0</td>\n",
       "      <td>21500.0</td>\n",
       "      <td>Academic City</td>\n",
       "      <td>Apartment</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>400.0</td>\n",
       "      <td>21500.0</td>\n",
       "      <td>Academic City</td>\n",
       "      <td>Apartment</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>400.0</td>\n",
       "      <td>21500.0</td>\n",
       "      <td>Academic City</td>\n",
       "      <td>Apartment</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>410.0</td>\n",
       "      <td>21500.0</td>\n",
       "      <td>Academic City</td>\n",
       "      <td>Apartment</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>390.0</td>\n",
       "      <td>20500.0</td>\n",
       "      <td>Academic City</td>\n",
       "      <td>Apartment</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>157375</th>\n",
       "      <td>3</td>\n",
       "      <td>3</td>\n",
       "      <td>1806.0</td>\n",
       "      <td>152000.0</td>\n",
       "      <td>World Trade Centre</td>\n",
       "      <td>Hotel Apartment</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>157376</th>\n",
       "      <td>3</td>\n",
       "      <td>4</td>\n",
       "      <td>1806.0</td>\n",
       "      <td>152000.0</td>\n",
       "      <td>World Trade Centre</td>\n",
       "      <td>Apartment</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>157377</th>\n",
       "      <td>3</td>\n",
       "      <td>4</td>\n",
       "      <td>1806.0</td>\n",
       "      <td>152000.0</td>\n",
       "      <td>World Trade Centre</td>\n",
       "      <td>Apartment</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>157378</th>\n",
       "      <td>3</td>\n",
       "      <td>3</td>\n",
       "      <td>1806.0</td>\n",
       "      <td>152000.0</td>\n",
       "      <td>World Trade Centre</td>\n",
       "      <td>Hotel Apartment</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>157379</th>\n",
       "      <td>3</td>\n",
       "      <td>4</td>\n",
       "      <td>2786.0</td>\n",
       "      <td>299900.0</td>\n",
       "      <td>World Trade Centre</td>\n",
       "      <td>Apartment</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>157380 rows × 15 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "        bedrooms  bathrooms    area    prices           locations  \\\n",
       "0              0          1   352.0   21500.0       Academic City   \n",
       "1              0          1   400.0   21500.0       Academic City   \n",
       "2              0          1   400.0   21500.0       Academic City   \n",
       "3              0          1   410.0   21500.0       Academic City   \n",
       "4              0          1   390.0   20500.0       Academic City   \n",
       "...          ...        ...     ...       ...                 ...   \n",
       "157375         3          3  1806.0  152000.0  World Trade Centre   \n",
       "157376         3          4  1806.0  152000.0  World Trade Centre   \n",
       "157377         3          4  1806.0  152000.0  World Trade Centre   \n",
       "157378         3          3  1806.0  152000.0  World Trade Centre   \n",
       "157379         3          4  2786.0  299900.0  World Trade Centre   \n",
       "\n",
       "         property_types  pool  balcony  furnished  maid  gym  brand_new  \\\n",
       "0             Apartment     1        0          0     0    1          0   \n",
       "1             Apartment     1        1          1     0    1          0   \n",
       "2             Apartment     0        0          1     0    0          0   \n",
       "3             Apartment     1        1          1     1    1          0   \n",
       "4             Apartment     1        1          0     0    1          0   \n",
       "...                 ...   ...      ...        ...   ...  ...        ...   \n",
       "157375  Hotel Apartment     1        1          0     0    1          0   \n",
       "157376        Apartment     0        1          1     0    0          0   \n",
       "157377        Apartment     1        1          1     0    1          0   \n",
       "157378  Hotel Apartment     0        1          1     0    0          0   \n",
       "157379        Apartment     1        1          0     0    1          0   \n",
       "\n",
       "        burj_view  sea_view  beach  \n",
       "0               0         0      0  \n",
       "1               0         0      0  \n",
       "2               0         0      0  \n",
       "3               1         0      0  \n",
       "4               0         0      0  \n",
       "...           ...       ...    ...  \n",
       "157375          0         0      0  \n",
       "157376          0         0      0  \n",
       "157377          0         0      0  \n",
       "157378          0         0      0  \n",
       "157379          0         0      0  \n",
       "\n",
       "[157380 rows x 15 columns]"
      ]
     },
     "execution_count": 2,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df = pd.read_csv('properties_cleaned_engineered.csv')\n",
    "df"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "9aea1288",
   "metadata": {},
   "source": [
    "#### We need to create dummy variables for categorical variables"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "5e7bf112",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Now we have 130 features!\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>bedrooms</th>\n",
       "      <th>bathrooms</th>\n",
       "      <th>area</th>\n",
       "      <th>prices</th>\n",
       "      <th>pool</th>\n",
       "      <th>balcony</th>\n",
       "      <th>furnished</th>\n",
       "      <th>maid</th>\n",
       "      <th>gym</th>\n",
       "      <th>brand_new</th>\n",
       "      <th>...</th>\n",
       "      <th>locations_Umm Suqeim</th>\n",
       "      <th>locations_Wadi Al Safa 2</th>\n",
       "      <th>locations_Wadi Al Shabak</th>\n",
       "      <th>locations_Wasl Gate</th>\n",
       "      <th>locations_World Trade Centre</th>\n",
       "      <th>property_types_Apartment</th>\n",
       "      <th>property_types_Hotel Apartment</th>\n",
       "      <th>property_types_Penthouse</th>\n",
       "      <th>property_types_Townhouse</th>\n",
       "      <th>property_types_Villa</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>352.0</td>\n",
       "      <td>21500.0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>400.0</td>\n",
       "      <td>21500.0</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>400.0</td>\n",
       "      <td>21500.0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>410.0</td>\n",
       "      <td>21500.0</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>390.0</td>\n",
       "      <td>20500.0</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>157375</th>\n",
       "      <td>3</td>\n",
       "      <td>3</td>\n",
       "      <td>1806.0</td>\n",
       "      <td>152000.0</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>157376</th>\n",
       "      <td>3</td>\n",
       "      <td>4</td>\n",
       "      <td>1806.0</td>\n",
       "      <td>152000.0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>157377</th>\n",
       "      <td>3</td>\n",
       "      <td>4</td>\n",
       "      <td>1806.0</td>\n",
       "      <td>152000.0</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>157378</th>\n",
       "      <td>3</td>\n",
       "      <td>3</td>\n",
       "      <td>1806.0</td>\n",
       "      <td>152000.0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>157379</th>\n",
       "      <td>3</td>\n",
       "      <td>4</td>\n",
       "      <td>2786.0</td>\n",
       "      <td>299900.0</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>157380 rows × 130 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "        bedrooms  bathrooms    area    prices  pool  balcony  furnished  maid  \\\n",
       "0              0          1   352.0   21500.0     1        0          0     0   \n",
       "1              0          1   400.0   21500.0     1        1          1     0   \n",
       "2              0          1   400.0   21500.0     0        0          1     0   \n",
       "3              0          1   410.0   21500.0     1        1          1     1   \n",
       "4              0          1   390.0   20500.0     1        1          0     0   \n",
       "...          ...        ...     ...       ...   ...      ...        ...   ...   \n",
       "157375         3          3  1806.0  152000.0     1        1          0     0   \n",
       "157376         3          4  1806.0  152000.0     0        1          1     0   \n",
       "157377         3          4  1806.0  152000.0     1        1          1     0   \n",
       "157378         3          3  1806.0  152000.0     0        1          1     0   \n",
       "157379         3          4  2786.0  299900.0     1        1          0     0   \n",
       "\n",
       "        gym  brand_new  ...  locations_Umm Suqeim  locations_Wadi Al Safa 2  \\\n",
       "0         1          0  ...                     0                         0   \n",
       "1         1          0  ...                     0                         0   \n",
       "2         0          0  ...                     0                         0   \n",
       "3         1          0  ...                     0                         0   \n",
       "4         1          0  ...                     0                         0   \n",
       "...     ...        ...  ...                   ...                       ...   \n",
       "157375    1          0  ...                     0                         0   \n",
       "157376    0          0  ...                     0                         0   \n",
       "157377    1          0  ...                     0                         0   \n",
       "157378    0          0  ...                     0                         0   \n",
       "157379    1          0  ...                     0                         0   \n",
       "\n",
       "        locations_Wadi Al Shabak  locations_Wasl Gate  \\\n",
       "0                              0                    0   \n",
       "1                              0                    0   \n",
       "2                              0                    0   \n",
       "3                              0                    0   \n",
       "4                              0                    0   \n",
       "...                          ...                  ...   \n",
       "157375                         0                    0   \n",
       "157376                         0                    0   \n",
       "157377                         0                    0   \n",
       "157378                         0                    0   \n",
       "157379                         0                    0   \n",
       "\n",
       "        locations_World Trade Centre  property_types_Apartment  \\\n",
       "0                                  0                         1   \n",
       "1                                  0                         1   \n",
       "2                                  0                         1   \n",
       "3                                  0                         1   \n",
       "4                                  0                         1   \n",
       "...                              ...                       ...   \n",
       "157375                             1                         0   \n",
       "157376                             1                         1   \n",
       "157377                             1                         1   \n",
       "157378                             1                         0   \n",
       "157379                             1                         1   \n",
       "\n",
       "        property_types_Hotel Apartment  property_types_Penthouse  \\\n",
       "0                                    0                         0   \n",
       "1                                    0                         0   \n",
       "2                                    0                         0   \n",
       "3                                    0                         0   \n",
       "4                                    0                         0   \n",
       "...                                ...                       ...   \n",
       "157375                               1                         0   \n",
       "157376                               0                         0   \n",
       "157377                               0                         0   \n",
       "157378                               1                         0   \n",
       "157379                               0                         0   \n",
       "\n",
       "        property_types_Townhouse  property_types_Villa  \n",
       "0                              0                     0  \n",
       "1                              0                     0  \n",
       "2                              0                     0  \n",
       "3                              0                     0  \n",
       "4                              0                     0  \n",
       "...                          ...                   ...  \n",
       "157375                         0                     0  \n",
       "157376                         0                     0  \n",
       "157377                         0                     0  \n",
       "157378                         0                     0  \n",
       "157379                         0                     0  \n",
       "\n",
       "[157380 rows x 130 columns]"
      ]
     },
     "execution_count": 3,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df = pd.get_dummies(df)\n",
    "print(f\"Now we have {df.shape[-1]} features!\")\n",
    "df"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "da45f2ad",
   "metadata": {},
   "source": [
    "### Split into Train/Test splits"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "360e85a7",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "X: (157380, 129), Y: (157380,)\n",
      " Size of train: 125904 \n",
      " Size of test: 31476\n"
     ]
    }
   ],
   "source": [
    "X = df.drop('prices',axis=1)\n",
    "y = df.prices.values\n",
    "print(f\"X: {X.shape}, Y: {y.shape}\")\n",
    "\n",
    "X_train, X_test, y_train, y_test = train_test_split(X,y, test_size=0.2, random_state=42)\n",
    "print(f\" Size of train: {X_train.shape[0]} \\n Size of test: {X_test.shape[0]}\")"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "b38c8f63",
   "metadata": {},
   "source": [
    "#### Multiple Linear Regression (using statsmodels)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "baef0d1b",
   "metadata": {},
   "source": [
    "#### Multiple Linear Regression using sklearn"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "id": "fe10b337",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Mean of CV R2 score: 0.7727074609999142\n"
     ]
    }
   ],
   "source": [
    "lr = LinearRegression()\n",
    "score = cross_val_score(lr,X_train,y_train,scoring='r2',cv=3,n_jobs=3)\n",
    "print(f\"Mean of CV R2 score: {np.mean(score)}\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "id": "ced04288",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Test R2 score is: 0.7943860590045717\n"
     ]
    }
   ],
   "source": [
    "lr.fit(X_train,y_train)\n",
    "print(f\"Test R2 score is: {lr.score(X_test,y_test)}\") "
   ]
  },
  {
   "cell_type": "markdown",
   "id": "3c6b18f6",
   "metadata": {},
   "source": [
    "### Multiple Linear Regression using Lasso regularizer"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "f3374750",
   "metadata": {},
   "source": [
    "AKA Linear Regression model with L1 regularizer.\n",
    "\n",
    "Let's first begin by scaling our input variables. We have to be careful and only scale our continous features, which area: bedrooms, bathrooms and area. These represent the first three columns in our input matrix."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "id": "ec31bd8b",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "First example, first 5 features:\n",
      "[3.000e+00 5.000e+00 1.784e+03 1.000e+00 0.000e+00]   ---scale--->   [ 0.80270613  1.63426775 -0.00395507  1.          0.        ]\n"
     ]
    }
   ],
   "source": [
    "X_train_scaled = X_train.copy(deep=True).values\n",
    "X_test_scaled = X_test.copy(deep=True).values\n",
    "\n",
    "scaler = StandardScaler(with_mean=True)\n",
    "scaler.fit(X_train_scaled[:,:3])\n",
    "\n",
    "X_train_scaled[:,:3] = scaler.transform(X_train_scaled[:,:3])\n",
    "X_test_scaled[:,:3] = scaler.transform(X_test_scaled[:,:3])\n",
    "\n",
    "print(\"First example, first 5 features:\")\n",
    "print(f\"{X_train.values[0,:5]}   ---scale--->   {X_train_scaled[0,:5]}\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "id": "9b856efe",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/Users/ahmed/miniconda3/envs/ds/lib/python3.10/site-packages/sklearn/linear_model/_coordinate_descent.py:648: ConvergenceWarning: Objective did not converge. You might want to increase the number of iterations, check the scale of the features or consider increasing regularisation. Duality gap: 3.681e+12, tolerance: 3.161e+11\n",
      "  model = cd_fast.enet_coordinate_descent(\n",
      "/Users/ahmed/miniconda3/envs/ds/lib/python3.10/site-packages/sklearn/linear_model/_coordinate_descent.py:648: ConvergenceWarning: Objective did not converge. You might want to increase the number of iterations, check the scale of the features or consider increasing regularisation. Duality gap: 5.340e+12, tolerance: 3.162e+11\n",
      "  model = cd_fast.enet_coordinate_descent(\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Best parameters: {'alpha': 1.0}\n",
      "Best score: 0.7727032907018364\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/Users/ahmed/miniconda3/envs/ds/lib/python3.10/site-packages/sklearn/linear_model/_coordinate_descent.py:648: ConvergenceWarning: Objective did not converge. You might want to increase the number of iterations, check the scale of the features or consider increasing regularisation. Duality gap: 6.672e+12, tolerance: 4.819e+11\n",
      "  model = cd_fast.enet_coordinate_descent(\n"
     ]
    }
   ],
   "source": [
    "reg_lasso = Lasso(random_state=42)\n",
    "\n",
    "parameters = {\n",
    "    'alpha': np.arange(1, 20, 0.5)\n",
    "}\n",
    "\n",
    "gs =  GridSearchCV(reg_lasso, parameters ,cv=3,scoring='r2')\n",
    "gs.fit(X_train_scaled,y_train)\n",
    "\n",
    "best_lasso_model = gs.best_estimator_\n",
    "best_lasso_score = gs.best_score_\n",
    "print(f\"Best parameters: {gs.best_params_}\\nBest score: {best_lasso_score}\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "id": "2279952f",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Test R2 score is: 0.7943886741938103\n"
     ]
    }
   ],
   "source": [
    "print(f\"Test R2 score is: {best_lasso_model.score(X_test_scaled,y_test)}\") "
   ]
  },
  {
   "cell_type": "markdown",
   "id": "a9103d83",
   "metadata": {},
   "source": [
    "### Random Forest"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "b476340d",
   "metadata": {},
   "source": [
    "Let's begin by fitting a simple Random Forest using default parameters, and evaluate the score"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "id": "be2957d2",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Mean of CV R2 score: 0.9819410063337961\n"
     ]
    }
   ],
   "source": [
    "rf = RandomForestRegressor()\n",
    "score = cross_val_score(rf,X_train,y_train,scoring='r2',cv=3,n_jobs=3)\n",
    "print(f\"Mean of CV R2 score: {np.mean(score)}\")"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "dfb8f327",
   "metadata": {},
   "source": [
    "Not too bad! This is already much better than our linear regression models. Maybe we can improve the score? Let's perform a hyperparameter search.. "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "id": "c2a97259",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Best parameters: {'criterion': 'squared_error', 'n_estimators': 100}\n",
      "Best score: 0.9818594141941768\n"
     ]
    }
   ],
   "source": [
    "parameters = {\n",
    "    \"n_estimators\": [10,100],\n",
    "    'criterion':['squared_error'], \n",
    "#     'max_features':[1.0,'sqrt','log']\n",
    "}\n",
    "\n",
    "gs = GridSearchCV(rf,parameters,scoring='r2',cv=3)\n",
    "gs.fit(X_train,y_train)\n",
    "\n",
    "best_rf_model = gs.best_estimator_\n",
    "best_rf_score = gs.best_score_\n",
    "print(f\"Best parameters: {gs.best_params_}\\nBest score: {gs.best_score_}\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "id": "140b64f0",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Test R2 score is: 0.9868367179351583\n"
     ]
    }
   ],
   "source": [
    "print(f\"Test R2 score is: {best_rf_model.score(X_test,y_test)}\") "
   ]
  },
  {
   "cell_type": "markdown",
   "id": "1d9da519",
   "metadata": {},
   "source": [
    "### Neural Network"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "6ac7b498",
   "metadata": {},
   "source": [
    "Let's define a hyperparameter space for the hyperparameters search"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "id": "27036927",
   "metadata": {},
   "outputs": [],
   "source": [
    "parameters = {\n",
    "                \"learning_rate_init\" : [0.001,0.001,0.01],\n",
    "                \"hidden_layer_sizes\" : [(129,100,10,),(129,100,50,10,),(129,100,50,30,10,),(129,100,50,30,10,3,)], # why 129? poorly chosen \"rule of thumb\" where the first layer = n_features - 1 \n",
    "                \"alpha\": [0.0001,0.001,0.01,0.1,1]\n",
    "}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "id": "775c5e7a",
   "metadata": {},
   "outputs": [],
   "source": [
    "regr = MLPRegressor(random_state=42,\n",
    "                    learning_rate='adaptive',\n",
    "                    solver='adam',\n",
    "                    max_iter=1000,\n",
    "                    early_stopping= True\n",
    "                   )"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "id": "b0cf4758",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Best Parameters: {'alpha': 0.1, 'hidden_layer_sizes': (129, 100, 50, 10), 'learning_rate_init': 0.01}\n",
      "Best Score: 0.9499625656411718\n"
     ]
    }
   ],
   "source": [
    "gs = GridSearchCV(regr,parameters,scoring='r2',cv=3,n_jobs=3)\n",
    "gs.fit(X_train_scaled,y_train)\n",
    "r2 = gs.best_score_\n",
    "best_nn_model = gs.best_estimator_\n",
    "print(f\"Best Parameters: {gs.best_params_}\\nBest Score: {r2}\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "id": "277d8e2d",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Test R2 score is: 0.9468102158376087\n"
     ]
    }
   ],
   "source": [
    "print(f\"Test R2 score is: {best_nn_model.score(X_test_scaled,y_test)}\") "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "id": "3ec576e6",
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<style type=\"text/css\">\n",
       "</style>\n",
       "<table id=\"T_81d25\">\n",
       "  <caption>Summary table of our Test R2 results using different models.</caption>\n",
       "  <thead>\n",
       "    <tr>\n",
       "      <th class=\"blank level0\" >&nbsp;</th>\n",
       "      <th id=\"T_81d25_level0_col0\" class=\"col_heading level0 col0\" >Linear Regression</th>\n",
       "      <th id=\"T_81d25_level0_col1\" class=\"col_heading level0 col1\" >Lasso Regression</th>\n",
       "      <th id=\"T_81d25_level0_col2\" class=\"col_heading level0 col2\" >Random Forest</th>\n",
       "      <th id=\"T_81d25_level0_col3\" class=\"col_heading level0 col3\" >Neural Network</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th id=\"T_81d25_level0_row0\" class=\"row_heading level0 row0\" >0</th>\n",
       "      <td id=\"T_81d25_row0_col0\" class=\"data row0 col0\" >0.794386</td>\n",
       "      <td id=\"T_81d25_row0_col1\" class=\"data row0 col1\" >0.794389</td>\n",
       "      <td id=\"T_81d25_row0_col2\" class=\"data row0 col2\" >0.986837</td>\n",
       "      <td id=\"T_81d25_row0_col3\" class=\"data row0 col3\" >0.946810</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n"
      ],
      "text/plain": [
       "<pandas.io.formats.style.Styler at 0x7f86a0ea5e10>"
      ]
     },
     "execution_count": 32,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "pd.DataFrame({\n",
    "    \"Linear Regression\": [lr.score(X_test,y_test)],\n",
    "    \"Lasso Regression\" : [best_lasso_model.score(X_test_scaled,y_test)],\n",
    "     \"Random Forest\": [best_rf_model.score(X_test,y_test)],\n",
    "     \"Neural Network\": [best_nn_model.score(X_test_scaled,y_test)]\n",
    "},).style.set_caption(\"Summary table of our Test R2 results using different models.\")\n"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "bb8ccf6f",
   "metadata": {},
   "source": [
    "Clearly, our Random Forest regressor performed the best! Let's go ahead and pickle this model "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 66,
   "id": "b298bf9d",
   "metadata": {},
   "outputs": [],
   "source": [
    "import pickle\n",
    "\n",
    "filename = 'finalized_model.p'\n",
    "pickle.dump(best_rf_model, open(filename, 'wb'))"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "207fc0b2",
   "metadata": {},
   "source": [
    "And a sanity check.."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 67,
   "id": "a1967697",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0.9868367179351583\n"
     ]
    }
   ],
   "source": [
    "# load the model from disk\n",
    "loaded_model = pickle.load(open(filename, 'rb'))\n",
    "result = loaded_model.score(X_test, y_test)\n",
    "print(result)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "92b757b4",
   "metadata": {},
   "source": [
    "Great! 😃"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "ds",
   "language": "python",
   "name": "ds"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.10.8"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}