MachineLearning_FinalProj.ipynb

{
  "cells": [
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "view-in-github",
        "colab_type": "text"
      },
      "source": [
        "<a href=\"https://colab.research.google.com/github/tjvilliard/MahineLearning/blob/main/MachineLearning_FinalProj.ipynb\" target=\"_parent\"><img src=\"https://colab.research.google.com/assets/colab-badge.svg\" alt=\"Open In Colab\"/></a>"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "P1IVqbkuAktL"
      },
      "outputs": [],
      "source": [
        "from google.colab import files\n",
        "files.upload()"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 1,
      "metadata": {
        "id": "1MrGU2PUAhqr"
      },
      "outputs": [],
      "source": [
        "import pandas as pd\n",
        "import numpy as np\n",
        "from sklearn import svm\n",
        "from sklearn.ensemble import RandomForestClassifier\n",
        "from sklearn.ensemble import AdaBoostClassifier\n",
        "from sklearn.neural_network import MLPClassifier\n",
        "from sklearn.neighbors import KNeighborsClassifier\n",
        "from sklearn.naive_bayes import GaussianNB, MultinomialNB\n",
        "from sklearn.preprocessing import OneHotEncoder\n",
        "from sklearn.model_selection import train_test_split"
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "def prediction_error(actual, prediction):\n",
        "    if len(actual) != len(prediction):\n",
        "        print(\"arrays not equal length\")\n",
        "        return None\n",
        "    error = np.equal(actual, prediction)\n",
        "    error_rate = np.round(1 - (sum(error) / len(actual)), 4)\n",
        "    return error_rate"
      ],
      "metadata": {
        "id": "wdG5EG3zvT-2"
      },
      "execution_count": 2,
      "outputs": []
    },
    {
      "cell_type": "code",
      "execution_count": 3,
      "metadata": {
        "id": "dnpumorgCrAr"
      },
      "outputs": [],
      "source": [
        "train_df = pd.read_csv(\"train_final.csv\")\n",
        "test_df = pd.read_csv(\"test_final.csv\").set_index(\"ID\")"
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "#EDA and Preprocessing"
      ],
      "metadata": {
        "id": "_DBPhYO6lVer"
      }
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 357
        },
        "id": "X-o7V5N2LL29",
        "outputId": "25f824cb-17f8-40d2-cae0-e97ad0fcc481"
      },
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "   age         workclass  fnlwgt    education  education.num  \\\n",
              "0   53  Self-emp-not-inc   93449  Prof-school             15   \n",
              "1   33  Self-emp-not-inc  123424    Bachelors             13   \n",
              "2   47           Private  144844      HS-grad              9   \n",
              "3   40           Private  114580      HS-grad              9   \n",
              "4   39           Private  115618      HS-grad              9   \n",
              "\n",
              "       marital.status        occupation    relationship                race  \\\n",
              "0  Married-civ-spouse    Prof-specialty         Husband  Asian-Pac-Islander   \n",
              "1  Married-civ-spouse   Exec-managerial         Husband               White   \n",
              "2  Married-civ-spouse      Craft-repair         Husband               White   \n",
              "3            Divorced      Craft-repair  Other-relative               White   \n",
              "4  Married-civ-spouse  Transport-moving         Husband               White   \n",
              "\n",
              "      sex  capital.gain  capital.loss  hours.per.week native.country  \\\n",
              "0    Male             0             0              40          India   \n",
              "1    Male             0             0              40  United-States   \n",
              "2    Male             0             0              40  United-States   \n",
              "3  Female             0             0              40        Vietnam   \n",
              "4    Male             0             0              50  United-States   \n",
              "\n",
              "   income>50K  \n",
              "0           1  \n",
              "1           1  \n",
              "2           0  \n",
              "3           0  \n",
              "4           0  "
            ],
            "text/html": [
              "\n",
              "  <div id=\"df-9bfbb3f0-c5d2-4543-9410-c110f7e0f18d\">\n",
              "    <div class=\"colab-df-container\">\n",
              "      <div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>age</th>\n",
              "      <th>workclass</th>\n",
              "      <th>fnlwgt</th>\n",
              "      <th>education</th>\n",
              "      <th>education.num</th>\n",
              "      <th>marital.status</th>\n",
              "      <th>occupation</th>\n",
              "      <th>relationship</th>\n",
              "      <th>race</th>\n",
              "      <th>sex</th>\n",
              "      <th>capital.gain</th>\n",
              "      <th>capital.loss</th>\n",
              "      <th>hours.per.week</th>\n",
              "      <th>native.country</th>\n",
              "      <th>income&gt;50K</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>0</th>\n",
              "      <td>53</td>\n",
              "      <td>Self-emp-not-inc</td>\n",
              "      <td>93449</td>\n",
              "      <td>Prof-school</td>\n",
              "      <td>15</td>\n",
              "      <td>Married-civ-spouse</td>\n",
              "      <td>Prof-specialty</td>\n",
              "      <td>Husband</td>\n",
              "      <td>Asian-Pac-Islander</td>\n",
              "      <td>Male</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>40</td>\n",
              "      <td>India</td>\n",
              "      <td>1</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1</th>\n",
              "      <td>33</td>\n",
              "      <td>Self-emp-not-inc</td>\n",
              "      <td>123424</td>\n",
              "      <td>Bachelors</td>\n",
              "      <td>13</td>\n",
              "      <td>Married-civ-spouse</td>\n",
              "      <td>Exec-managerial</td>\n",
              "      <td>Husband</td>\n",
              "      <td>White</td>\n",
              "      <td>Male</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>40</td>\n",
              "      <td>United-States</td>\n",
              "      <td>1</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>2</th>\n",
              "      <td>47</td>\n",
              "      <td>Private</td>\n",
              "      <td>144844</td>\n",
              "      <td>HS-grad</td>\n",
              "      <td>9</td>\n",
              "      <td>Married-civ-spouse</td>\n",
              "      <td>Craft-repair</td>\n",
              "      <td>Husband</td>\n",
              "      <td>White</td>\n",
              "      <td>Male</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>40</td>\n",
              "      <td>United-States</td>\n",
              "      <td>0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>3</th>\n",
              "      <td>40</td>\n",
              "      <td>Private</td>\n",
              "      <td>114580</td>\n",
              "      <td>HS-grad</td>\n",
              "      <td>9</td>\n",
              "      <td>Divorced</td>\n",
              "      <td>Craft-repair</td>\n",
              "      <td>Other-relative</td>\n",
              "      <td>White</td>\n",
              "      <td>Female</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>40</td>\n",
              "      <td>Vietnam</td>\n",
              "      <td>0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>4</th>\n",
              "      <td>39</td>\n",
              "      <td>Private</td>\n",
              "      <td>115618</td>\n",
              "      <td>HS-grad</td>\n",
              "      <td>9</td>\n",
              "      <td>Married-civ-spouse</td>\n",
              "      <td>Transport-moving</td>\n",
              "      <td>Husband</td>\n",
              "      <td>White</td>\n",
              "      <td>Male</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>50</td>\n",
              "      <td>United-States</td>\n",
              "      <td>0</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "</div>\n",
              "      <button class=\"colab-df-convert\" onclick=\"convertToInteractive('df-9bfbb3f0-c5d2-4543-9410-c110f7e0f18d')\"\n",
              "              title=\"Convert this dataframe to an interactive table.\"\n",
              "              style=\"display:none;\">\n",
              "        \n",
              "  <svg xmlns=\"http://www.w3.org/2000/svg\" height=\"24px\"viewBox=\"0 0 24 24\"\n",
              "       width=\"24px\">\n",
              "    <path d=\"M0 0h24v24H0V0z\" fill=\"none\"/>\n",
              "    <path d=\"M18.56 5.44l.94 2.06.94-2.06 2.06-.94-2.06-.94-.94-2.06-.94 2.06-2.06.94zm-11 1L8.5 8.5l.94-2.06 2.06-.94-2.06-.94L8.5 2.5l-.94 2.06-2.06.94zm10 10l.94 2.06.94-2.06 2.06-.94-2.06-.94-.94-2.06-.94 2.06-2.06.94z\"/><path d=\"M17.41 7.96l-1.37-1.37c-.4-.4-.92-.59-1.43-.59-.52 0-1.04.2-1.43.59L10.3 9.45l-7.72 7.72c-.78.78-.78 2.05 0 2.83L4 21.41c.39.39.9.59 1.41.59.51 0 1.02-.2 1.41-.59l7.78-7.78 2.81-2.81c.8-.78.8-2.07 0-2.86zM5.41 20L4 18.59l7.72-7.72 1.47 1.35L5.41 20z\"/>\n",
              "  </svg>\n",
              "      </button>\n",
              "      \n",
              "  <style>\n",
              "    .colab-df-container {\n",
              "      display:flex;\n",
              "      flex-wrap:wrap;\n",
              "      gap: 12px;\n",
              "    }\n",
              "\n",
              "    .colab-df-convert {\n",
              "      background-color: #E8F0FE;\n",
              "      border: none;\n",
              "      border-radius: 50%;\n",
              "      cursor: pointer;\n",
              "      display: none;\n",
              "      fill: #1967D2;\n",
              "      height: 32px;\n",
              "      padding: 0 0 0 0;\n",
              "      width: 32px;\n",
              "    }\n",
              "\n",
              "    .colab-df-convert:hover {\n",
              "      background-color: #E2EBFA;\n",
              "      box-shadow: 0px 1px 2px rgba(60, 64, 67, 0.3), 0px 1px 3px 1px rgba(60, 64, 67, 0.15);\n",
              "      fill: #174EA6;\n",
              "    }\n",
              "\n",
              "    [theme=dark] .colab-df-convert {\n",
              "      background-color: #3B4455;\n",
              "      fill: #D2E3FC;\n",
              "    }\n",
              "\n",
              "    [theme=dark] .colab-df-convert:hover {\n",
              "      background-color: #434B5C;\n",
              "      box-shadow: 0px 1px 3px 1px rgba(0, 0, 0, 0.15);\n",
              "      filter: drop-shadow(0px 1px 2px rgba(0, 0, 0, 0.3));\n",
              "      fill: #FFFFFF;\n",
              "    }\n",
              "  </style>\n",
              "\n",
              "      <script>\n",
              "        const buttonEl =\n",
              "          document.querySelector('#df-9bfbb3f0-c5d2-4543-9410-c110f7e0f18d button.colab-df-convert');\n",
              "        buttonEl.style.display =\n",
              "          google.colab.kernel.accessAllowed ? 'block' : 'none';\n",
              "\n",
              "        async function convertToInteractive(key) {\n",
              "          const element = document.querySelector('#df-9bfbb3f0-c5d2-4543-9410-c110f7e0f18d');\n",
              "          const dataTable =\n",
              "            await google.colab.kernel.invokeFunction('convertToInteractive',\n",
              "                                                     [key], {});\n",
              "          if (!dataTable) return;\n",
              "\n",
              "          const docLinkHtml = 'Like what you see? Visit the ' +\n",
              "            '<a target=\"_blank\" href=https://colab.research.google.com/notebooks/data_table.ipynb>data table notebook</a>'\n",
              "            + ' to learn more about interactive tables.';\n",
              "          element.innerHTML = '';\n",
              "          dataTable['output_type'] = 'display_data';\n",
              "          await google.colab.output.renderOutput(dataTable, element);\n",
              "          const docLink = document.createElement('div');\n",
              "          docLink.innerHTML = docLinkHtml;\n",
              "          element.appendChild(docLink);\n",
              "        }\n",
              "      </script>\n",
              "    </div>\n",
              "  </div>\n",
              "  "
            ]
          },
          "metadata": {},
          "execution_count": 41
        }
      ],
      "source": [
        "train_df.head()"
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "train_df.iloc[24999]"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "u_5Hilxw6HNI",
        "outputId": "ea07199b-668e-4d04-ee1c-f0e74499bbf3"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "age                          18\n",
              "workclass                     ?\n",
              "fnlwgt                   192321\n",
              "education          Some-college\n",
              "education.num                10\n",
              "marital.status    Never-married\n",
              "occupation                    ?\n",
              "relationship          Own-child\n",
              "race                      White\n",
              "sex                        Male\n",
              "capital.gain                  0\n",
              "capital.loss                  0\n",
              "hours.per.week               40\n",
              "native.country    United-States\n",
              "income>50K                    0\n",
              "Name: 24999, dtype: object"
            ]
          },
          "metadata": {},
          "execution_count": 45
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "train_df.where(train_df.isin([\"?\"])).any()"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 311
        },
        "id": "DUZPqUFP3PHO",
        "outputId": "b6d0e951-dbc1-4593-d046-8dfa3002d50c"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "error",
          "ename": "TypeError",
          "evalue": "ignored",
          "traceback": [
            "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
            "\u001b[0;31mTypeError\u001b[0m                                 Traceback (most recent call last)",
            "\u001b[0;32m<ipython-input-51-8d1f728e3b91>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m\u001b[0m\n\u001b[0;32m----> 1\u001b[0;31m \u001b[0mtrain_df\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mwhere\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mtrain_df\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0misin\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;34m\"?\"\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0maxis\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;36m1\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0many\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m",
            "\u001b[0;32m/usr/local/lib/python3.8/dist-packages/pandas/util/_decorators.py\u001b[0m in \u001b[0;36mwrapper\u001b[0;34m(*args, **kwargs)\u001b[0m\n\u001b[1;32m    309\u001b[0m                     \u001b[0mstacklevel\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mstacklevel\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    310\u001b[0m                 )\n\u001b[0;32m--> 311\u001b[0;31m             \u001b[0;32mreturn\u001b[0m \u001b[0mfunc\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m*\u001b[0m\u001b[0margs\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m**\u001b[0m\u001b[0mkwargs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    312\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    313\u001b[0m         \u001b[0;32mreturn\u001b[0m \u001b[0mwrapper\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;32m/usr/local/lib/python3.8/dist-packages/pandas/core/frame.py\u001b[0m in \u001b[0;36mwhere\u001b[0;34m(self, cond, other, inplace, axis, level, errors, try_cast)\u001b[0m\n\u001b[1;32m  10734\u001b[0m         \u001b[0mtry_cast\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mlib\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mno_default\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m  10735\u001b[0m     ):\n\u001b[0;32m> 10736\u001b[0;31m         \u001b[0;32mreturn\u001b[0m \u001b[0msuper\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mwhere\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mcond\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mother\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0minplace\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0maxis\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mlevel\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0merrors\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mtry_cast\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m  10737\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m  10738\u001b[0m     @deprecate_nonkeyword_arguments(\n",
            "\u001b[0;32m/usr/local/lib/python3.8/dist-packages/pandas/core/generic.py\u001b[0m in \u001b[0;36mwhere\u001b[0;34m(self, cond, other, inplace, axis, level, errors, try_cast)\u001b[0m\n\u001b[1;32m   9030\u001b[0m             )\n\u001b[1;32m   9031\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m-> 9032\u001b[0;31m         \u001b[0;32mreturn\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_where\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mcond\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mother\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0minplace\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0maxis\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mlevel\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0merrors\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0merrors\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m   9033\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   9034\u001b[0m     \u001b[0;34m@\u001b[0m\u001b[0mfinal\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;32m/usr/local/lib/python3.8/dist-packages/pandas/core/generic.py\u001b[0m in \u001b[0;36m_where\u001b[0;34m(self, cond, other, inplace, axis, level, errors)\u001b[0m\n\u001b[1;32m   8871\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   8872\u001b[0m         \u001b[0;32melse\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m-> 8873\u001b[0;31m             new_data = self._mgr.where(\n\u001b[0m\u001b[1;32m   8874\u001b[0m                 \u001b[0mother\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mother\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   8875\u001b[0m                 \u001b[0mcond\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mcond\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;32m/usr/local/lib/python3.8/dist-packages/pandas/core/internals/managers.py\u001b[0m in \u001b[0;36mwhere\u001b[0;34m(self, other, cond, align, errors)\u001b[0m\n\u001b[1;32m    344\u001b[0m             \u001b[0mother\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mextract_array\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mother\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mextract_numpy\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;32mTrue\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    345\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 346\u001b[0;31m         return self.apply(\n\u001b[0m\u001b[1;32m    347\u001b[0m             \u001b[0;34m\"where\"\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    348\u001b[0m             \u001b[0malign_keys\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0malign_keys\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;32m/usr/local/lib/python3.8/dist-packages/pandas/core/internals/managers.py\u001b[0m in \u001b[0;36mapply\u001b[0;34m(self, f, align_keys, ignore_failures, **kwargs)\u001b[0m\n\u001b[1;32m    319\u001b[0m                     \u001b[0;32melse\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    320\u001b[0m                         \u001b[0;31m# otherwise we have an ndarray\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 321\u001b[0;31m                         \u001b[0mkwargs\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0mk\u001b[0m\u001b[0;34m]\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mobj\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0mb\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mmgr_locs\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mindexer\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    322\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    323\u001b[0m             \u001b[0;32mtry\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;31mTypeError\u001b[0m: 'float' object is not subscriptable"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 388
        },
        "id": "eyNJISg3LNML",
        "outputId": "05a432a0-da9b-4665-a33b-20210e0596c9"
      },
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "    age         workclass  fnlwgt     education  education.num  \\\n",
              "ID                                                               \n",
              "1    33  Self-emp-not-inc  222162       HS-grad              9   \n",
              "2    68           Private   29240       HS-grad              9   \n",
              "3    34           Private  103596       HS-grad              9   \n",
              "4    57           Private  103403       5th-6th              3   \n",
              "5    48           Private  152915  Some-college             10   \n",
              "\n",
              "        marital.status         occupation   relationship   race     sex  \\\n",
              "ID                                                                        \n",
              "1   Married-civ-spouse       Craft-repair        Husband  White    Male   \n",
              "2              Widowed     Prof-specialty  Not-in-family  White  Female   \n",
              "3   Married-civ-spouse  Handlers-cleaners        Husband  White    Male   \n",
              "4   Married-civ-spouse   Transport-moving        Husband  White    Male   \n",
              "5        Never-married       Adm-clerical  Not-in-family  White  Female   \n",
              "\n",
              "    capital.gain  capital.loss  hours.per.week native.country  \n",
              "ID                                                             \n",
              "1              0             0              40  United-States  \n",
              "2              0             0              12  United-States  \n",
              "3              0             0              40  United-States  \n",
              "4              0             0              40  United-States  \n",
              "5              0             0              40  United-States  "
            ],
            "text/html": [
              "\n",
              "  <div id=\"df-46ee409b-8525-45e9-bb15-b84f57d23a31\">\n",
              "    <div class=\"colab-df-container\">\n",
              "      <div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>age</th>\n",
              "      <th>workclass</th>\n",
              "      <th>fnlwgt</th>\n",
              "      <th>education</th>\n",
              "      <th>education.num</th>\n",
              "      <th>marital.status</th>\n",
              "      <th>occupation</th>\n",
              "      <th>relationship</th>\n",
              "      <th>race</th>\n",
              "      <th>sex</th>\n",
              "      <th>capital.gain</th>\n",
              "      <th>capital.loss</th>\n",
              "      <th>hours.per.week</th>\n",
              "      <th>native.country</th>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>ID</th>\n",
              "      <th></th>\n",
              "      <th></th>\n",
              "      <th></th>\n",
              "      <th></th>\n",
              "      <th></th>\n",
              "      <th></th>\n",
              "      <th></th>\n",
              "      <th></th>\n",
              "      <th></th>\n",
              "      <th></th>\n",
              "      <th></th>\n",
              "      <th></th>\n",
              "      <th></th>\n",
              "      <th></th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>1</th>\n",
              "      <td>33</td>\n",
              "      <td>Self-emp-not-inc</td>\n",
              "      <td>222162</td>\n",
              "      <td>HS-grad</td>\n",
              "      <td>9</td>\n",
              "      <td>Married-civ-spouse</td>\n",
              "      <td>Craft-repair</td>\n",
              "      <td>Husband</td>\n",
              "      <td>White</td>\n",
              "      <td>Male</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>40</td>\n",
              "      <td>United-States</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>2</th>\n",
              "      <td>68</td>\n",
              "      <td>Private</td>\n",
              "      <td>29240</td>\n",
              "      <td>HS-grad</td>\n",
              "      <td>9</td>\n",
              "      <td>Widowed</td>\n",
              "      <td>Prof-specialty</td>\n",
              "      <td>Not-in-family</td>\n",
              "      <td>White</td>\n",
              "      <td>Female</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>12</td>\n",
              "      <td>United-States</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>3</th>\n",
              "      <td>34</td>\n",
              "      <td>Private</td>\n",
              "      <td>103596</td>\n",
              "      <td>HS-grad</td>\n",
              "      <td>9</td>\n",
              "      <td>Married-civ-spouse</td>\n",
              "      <td>Handlers-cleaners</td>\n",
              "      <td>Husband</td>\n",
              "      <td>White</td>\n",
              "      <td>Male</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>40</td>\n",
              "      <td>United-States</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>4</th>\n",
              "      <td>57</td>\n",
              "      <td>Private</td>\n",
              "      <td>103403</td>\n",
              "      <td>5th-6th</td>\n",
              "      <td>3</td>\n",
              "      <td>Married-civ-spouse</td>\n",
              "      <td>Transport-moving</td>\n",
              "      <td>Husband</td>\n",
              "      <td>White</td>\n",
              "      <td>Male</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>40</td>\n",
              "      <td>United-States</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>5</th>\n",
              "      <td>48</td>\n",
              "      <td>Private</td>\n",
              "      <td>152915</td>\n",
              "      <td>Some-college</td>\n",
              "      <td>10</td>\n",
              "      <td>Never-married</td>\n",
              "      <td>Adm-clerical</td>\n",
              "      <td>Not-in-family</td>\n",
              "      <td>White</td>\n",
              "      <td>Female</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>40</td>\n",
              "      <td>United-States</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "</div>\n",
              "      <button class=\"colab-df-convert\" onclick=\"convertToInteractive('df-46ee409b-8525-45e9-bb15-b84f57d23a31')\"\n",
              "              title=\"Convert this dataframe to an interactive table.\"\n",
              "              style=\"display:none;\">\n",
              "        \n",
              "  <svg xmlns=\"http://www.w3.org/2000/svg\" height=\"24px\"viewBox=\"0 0 24 24\"\n",
              "       width=\"24px\">\n",
              "    <path d=\"M0 0h24v24H0V0z\" fill=\"none\"/>\n",
              "    <path d=\"M18.56 5.44l.94 2.06.94-2.06 2.06-.94-2.06-.94-.94-2.06-.94 2.06-2.06.94zm-11 1L8.5 8.5l.94-2.06 2.06-.94-2.06-.94L8.5 2.5l-.94 2.06-2.06.94zm10 10l.94 2.06.94-2.06 2.06-.94-2.06-.94-.94-2.06-.94 2.06-2.06.94z\"/><path d=\"M17.41 7.96l-1.37-1.37c-.4-.4-.92-.59-1.43-.59-.52 0-1.04.2-1.43.59L10.3 9.45l-7.72 7.72c-.78.78-.78 2.05 0 2.83L4 21.41c.39.39.9.59 1.41.59.51 0 1.02-.2 1.41-.59l7.78-7.78 2.81-2.81c.8-.78.8-2.07 0-2.86zM5.41 20L4 18.59l7.72-7.72 1.47 1.35L5.41 20z\"/>\n",
              "  </svg>\n",
              "      </button>\n",
              "      \n",
              "  <style>\n",
              "    .colab-df-container {\n",
              "      display:flex;\n",
              "      flex-wrap:wrap;\n",
              "      gap: 12px;\n",
              "    }\n",
              "\n",
              "    .colab-df-convert {\n",
              "      background-color: #E8F0FE;\n",
              "      border: none;\n",
              "      border-radius: 50%;\n",
              "      cursor: pointer;\n",
              "      display: none;\n",
              "      fill: #1967D2;\n",
              "      height: 32px;\n",
              "      padding: 0 0 0 0;\n",
              "      width: 32px;\n",
              "    }\n",
              "\n",
              "    .colab-df-convert:hover {\n",
              "      background-color: #E2EBFA;\n",
              "      box-shadow: 0px 1px 2px rgba(60, 64, 67, 0.3), 0px 1px 3px 1px rgba(60, 64, 67, 0.15);\n",
              "      fill: #174EA6;\n",
              "    }\n",
              "\n",
              "    [theme=dark] .colab-df-convert {\n",
              "      background-color: #3B4455;\n",
              "      fill: #D2E3FC;\n",
              "    }\n",
              "\n",
              "    [theme=dark] .colab-df-convert:hover {\n",
              "      background-color: #434B5C;\n",
              "      box-shadow: 0px 1px 3px 1px rgba(0, 0, 0, 0.15);\n",
              "      filter: drop-shadow(0px 1px 2px rgba(0, 0, 0, 0.3));\n",
              "      fill: #FFFFFF;\n",
              "    }\n",
              "  </style>\n",
              "\n",
              "      <script>\n",
              "        const buttonEl =\n",
              "          document.querySelector('#df-46ee409b-8525-45e9-bb15-b84f57d23a31 button.colab-df-convert');\n",
              "        buttonEl.style.display =\n",
              "          google.colab.kernel.accessAllowed ? 'block' : 'none';\n",
              "\n",
              "        async function convertToInteractive(key) {\n",
              "          const element = document.querySelector('#df-46ee409b-8525-45e9-bb15-b84f57d23a31');\n",
              "          const dataTable =\n",
              "            await google.colab.kernel.invokeFunction('convertToInteractive',\n",
              "                                                     [key], {});\n",
              "          if (!dataTable) return;\n",
              "\n",
              "          const docLinkHtml = 'Like what you see? Visit the ' +\n",
              "            '<a target=\"_blank\" href=https://colab.research.google.com/notebooks/data_table.ipynb>data table notebook</a>'\n",
              "            + ' to learn more about interactive tables.';\n",
              "          element.innerHTML = '';\n",
              "          dataTable['output_type'] = 'display_data';\n",
              "          await google.colab.output.renderOutput(dataTable, element);\n",
              "          const docLink = document.createElement('div');\n",
              "          docLink.innerHTML = docLinkHtml;\n",
              "          element.appendChild(docLink);\n",
              "        }\n",
              "      </script>\n",
              "    </div>\n",
              "  </div>\n",
              "  "
            ]
          },
          "metadata": {},
          "execution_count": 8
        }
      ],
      "source": [
        "test_df.head()"
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "test_df.info()"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "mogtzilx78LO",
        "outputId": "3aa4d14e-15b8-414b-ca65-f334a10d50e3"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "<class 'pandas.core.frame.DataFrame'>\n",
            "Int64Index: 23842 entries, 1 to 23842\n",
            "Data columns (total 14 columns):\n",
            " #   Column          Non-Null Count  Dtype \n",
            "---  ------          --------------  ----- \n",
            " 0   age             23842 non-null  int64 \n",
            " 1   workclass       23842 non-null  object\n",
            " 2   fnlwgt          23842 non-null  int64 \n",
            " 3   education       23842 non-null  object\n",
            " 4   education.num   23842 non-null  int64 \n",
            " 5   marital.status  23842 non-null  object\n",
            " 6   occupation      23842 non-null  object\n",
            " 7   relationship    23842 non-null  object\n",
            " 8   race            23842 non-null  object\n",
            " 9   sex             23842 non-null  object\n",
            " 10  capital.gain    23842 non-null  int64 \n",
            " 11  capital.loss    23842 non-null  int64 \n",
            " 12  hours.per.week  23842 non-null  int64 \n",
            " 13  native.country  23842 non-null  object\n",
            "dtypes: int64(6), object(8)\n",
            "memory usage: 2.7+ MB\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 4,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "VmcDyXJqMfLW",
        "outputId": "eee1fc4e-2778-44e4-d3d4-062f9d7948b4"
      },
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Private\n",
            "HS-grad\n",
            "Married-civ-spouse\n",
            "Prof-specialty\n",
            "Husband\n",
            "White\n",
            "Male\n",
            "United-States\n"
          ]
        }
      ],
      "source": [
        "# Fill missing values in test_df with column mode or median \n",
        "train_df_dropna = train_df.replace([\"?\"], np.nan).dropna()\n",
        "\n",
        "for (columnName, columnData) in test_df.iteritems():\n",
        "  if test_df[columnName].dtype == \"object\":\n",
        "    most_common = test_df[columnName].mode()[0]\n",
        "    print(most_common)\n",
        "    test_df[columnName].fillna(most_common, inplace=True)\n",
        "    test_df[columnName] = test_df[columnName].replace([\"?\"], most_common)\n",
        "  else:\n",
        "    fill_val = test_df[columnName].mean()\n",
        "    test_df[columnName].fillna(fill_val, inplace=True)\n",
        "    test_df[columnName] = test_df[columnName].replace([\"?\"], fill_val)\n",
        "\n"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 5,
      "metadata": {
        "id": "hHU5WoNaDnfM"
      },
      "outputs": [],
      "source": [
        "# declare encoder\n",
        "encoder = OneHotEncoder(handle_unknown='ignore')\n",
        "\n",
        "# categorical columns\n",
        "cat_col = ['workclass', 'education', 'marital.status', 'occupation', 'relationship', 'race', 'sex', 'native.country']\n",
        "\n",
        "# fit the encoder to the training data categorical columns\n",
        "encoder.fit(train_df[cat_col])\n",
        "\n",
        "# encode the categorical columns of test and train df\n",
        "train_encoding = encoder.transform(train_df[cat_col])\n",
        "test_encoding = encoder.transform(test_df[cat_col])\n"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 6,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "g0JYpd51SwJg",
        "outputId": "d7f98a00-ead8-4848-a3f4-6b0fb70242f0"
      },
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "(23842, 101)\n",
            "(25000, 101)\n"
          ]
        }
      ],
      "source": [
        "# get the df of encoded cat variable columns\n",
        "encoded_train_df = pd.DataFrame(train_encoding.toarray())\n",
        "\n",
        "encoded_test_df = pd.DataFrame(test_encoding.toarray())\n",
        "# align the index of encoded df\n",
        "encoded_test_df.index = test_df.index\n",
        "\n",
        "print(encoded_test_df.shape)\n",
        "print(encoded_train_df.shape)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 7,
      "metadata": {
        "id": "PpnRSkKKAw5D"
      },
      "outputs": [],
      "source": [
        "# drop original cat columns and join encoded df\n",
        "target_col = train_df['income>50K']\n",
        "encoded_train_df = encoded_train_df.join(target_col)\n",
        "drop_col = cat_col.copy()\n",
        "drop_col.append(\"income>50K\")\n",
        "\n",
        "train_df.drop(columns=drop_col, axis=1, inplace=True)\n",
        "encoded_train_df = train_df.join(encoded_train_df)\n",
        "\n"
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "reduced_test_df = test_df.drop(cat_col, axis=1)\n",
        "encoded_test_df = reduced_test_df.join(encoded_test_df)"
      ],
      "metadata": {
        "id": "sCwn86Klly-T"
      },
      "execution_count": 8,
      "outputs": []
    },
    {
      "cell_type": "code",
      "execution_count": 9,
      "metadata": {
        "id": "KXSyBXyXHX1A"
      },
      "outputs": [],
      "source": [
        "# convert df to np array and split x and y\n",
        "train_array = encoded_train_df.to_numpy()\n",
        "x_train, y_train = train_array[:, :-1], train_array[:, -1]\n",
        "\n",
        "# prepare encoded test data\n",
        "test_array = encoded_test_df.to_numpy()"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 108,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "GiOUaMe5KQA0",
        "outputId": "8293e841-4ef2-415e-ae4b-28ee75da858c"
      },
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "23842"
            ]
          },
          "metadata": {},
          "execution_count": 108
        }
      ],
      "source": [
        "len(test_array)"
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "x, x_holdout, y, y_holdout = train_test_split(x_train, y_train, test_size=0.10)\n",
        "\n",
        "# training svm on more than 5-10% of the data resulted in unreasonable training times\n",
        "# performance was still comparable to other classifiers\n",
        "x_svm, x_holdout_svm, y_svm, y_holdout_svm = train_test_split(x_train, y_train, test_size=0.95)"
      ],
      "metadata": {
        "id": "2dA4hPnNv3Og"
      },
      "execution_count": 10,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "source": [
        "#SVM\n"
      ],
      "metadata": {
        "id": "V6NPw7fax49y"
      }
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "rVNLXHrKIacS",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "101f584b-3870-4fcb-e4b6-ef64cf1105d9"
      },
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Train Error:  0.2152\n",
            "Test Error:  0.2056\n",
            "\n",
            "\n",
            "Train Error:  0.2184\n",
            "Test Error:  0.2059\n",
            "\n",
            "\n",
            "Train Error:  0.2144\n",
            "Test Error:  0.2051\n",
            "\n",
            "\n",
            "Train Error:  0.2176\n",
            "Test Error:  0.2052\n",
            "\n",
            "\n",
            "Train Error:  0.2168\n",
            "Test Error:  0.2052\n",
            "\n",
            "\n",
            "Train Error:  0.2224\n",
            "Test Error:  0.2081\n",
            "\n",
            "\n"
          ]
        }
      ],
      "source": [
        "#SVM\n",
        "# linear performed best experimentally\n",
        "# kernels = ['linear', 'poly', 'rbf', 'sigmoid']\n",
        "regulizers = [0.5, 1, 2, 5, 10, 50]\n",
        "for c in regulizers:\n",
        "  clf = svm.SVC(kernel='linear', C=c)\n",
        "  clf.fit(x_svm, y_svm)\n",
        "\n",
        "  train_pred = clf.predict(x_svm)\n",
        "  holdout_pred = clf.predict(x_holdout_svm)\n",
        "\n",
        "  # print(\"Kernel: \", k)\n",
        "  print(\"Train Error: \", prediction_error(y_svm, train_pred))\n",
        "  print(\"Test Error: \", prediction_error(y_holdout_svm, holdout_pred))\n",
        "  print(\"\\n\")"
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "\n",
        "svm_test_series = pd.Series(svm_test, name='Prediction')\n",
        "svm_test_series.index = test_df.index\n",
        "svm_test_series.to_csv(\"svm_submission.csv\")"
      ],
      "metadata": {
        "id": "qzvboqAFvTgU",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 200
        },
        "outputId": "b0c06157-1ae6-41c5-a0f4-96c7508d50e2"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "error",
          "ename": "NameError",
          "evalue": "ignored",
          "traceback": [
            "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
            "\u001b[0;31mNameError\u001b[0m                                 Traceback (most recent call last)",
            "\u001b[0;32m<ipython-input-17-0ca931bb347f>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m\u001b[0m\n\u001b[0;32m----> 1\u001b[0;31m \u001b[0msvm_test_series\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mpd\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mSeries\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0msvm_test\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mname\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;34m'Prediction'\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m      2\u001b[0m \u001b[0msvm_test_series\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mindex\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mtest_df\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mindex\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m      3\u001b[0m \u001b[0msvm_test_series\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mto_csv\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m\"svm_submission.csv\"\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;31mNameError\u001b[0m: name 'svm_test' is not defined"
          ]
        }
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "#Ensemble Methods\n"
      ],
      "metadata": {
        "id": "9Q5pktmP0PvT"
      }
    },
    {
      "cell_type": "markdown",
      "source": [
        "## Random Forest\n"
      ],
      "metadata": {
        "id": "-n-PwXKAswyy"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "depths =[1, 10, 20, 50, 100]\n",
        "n_estimators = [100, 500, 1000]\n",
        "\n",
        "for max_depth in depths:\n",
        "  print(\"max_depth: \", max_depth)\n",
        "  for n in n_estimators:\n",
        "    print(\"n_estimators: \", n)\n",
        "    rf_clf = RandomForestClassifier(n_estimators=n,max_depth=max_depth).fit(x,y)\n",
        "    print(\"Train Score: \", rf_clf.score(x, y))\n",
        "    print(\"Test Score: \", rf_clf.score(x_holdout, y_holdout))\n",
        "    print('\\n')\n",
        "\n"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "Uwhsu83L0lEg",
        "outputId": "ba035f47-6a64-443f-8e47-80dccb679501"
      },
      "execution_count": 11,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "max_depth:  1\n",
            "n_estimators:  100\n",
            "Train Score:  0.7588444444444444\n",
            "Test Score:  0.764\n",
            "\n",
            "\n",
            "n_estimators:  500\n",
            "Train Score:  0.7588444444444444\n",
            "Test Score:  0.764\n",
            "\n",
            "\n",
            "n_estimators:  1000\n",
            "Train Score:  0.7588444444444444\n",
            "Test Score:  0.764\n",
            "\n",
            "\n",
            "max_depth:  10\n",
            "n_estimators:  100\n",
            "Train Score:  0.8655555555555555\n",
            "Test Score:  0.8628\n",
            "\n",
            "\n",
            "n_estimators:  500\n",
            "Train Score:  0.8671555555555556\n",
            "Test Score:  0.8624\n",
            "\n",
            "\n",
            "n_estimators:  1000\n",
            "Train Score:  0.8665333333333334\n",
            "Test Score:  0.8616\n",
            "\n",
            "\n",
            "max_depth:  20\n",
            "n_estimators:  100\n",
            "Train Score:  0.9257777777777778\n",
            "Test Score:  0.8704\n",
            "\n",
            "\n",
            "n_estimators:  500\n",
            "Train Score:  0.9250222222222222\n",
            "Test Score:  0.8708\n",
            "\n",
            "\n",
            "n_estimators:  1000\n",
            "Train Score:  0.9256888888888889\n",
            "Test Score:  0.8704\n",
            "\n",
            "\n",
            "max_depth:  50\n",
            "n_estimators:  100\n",
            "Train Score:  0.9999555555555556\n",
            "Test Score:  0.8592\n",
            "\n",
            "\n",
            "n_estimators:  500\n",
            "Train Score:  0.9999555555555556\n",
            "Test Score:  0.8636\n",
            "\n",
            "\n",
            "n_estimators:  1000\n",
            "Train Score:  0.9999555555555556\n",
            "Test Score:  0.864\n",
            "\n",
            "\n",
            "max_depth:  100\n",
            "n_estimators:  100\n",
            "Train Score:  0.9999555555555556\n",
            "Test Score:  0.8588\n",
            "\n",
            "\n",
            "n_estimators:  500\n",
            "Train Score:  0.9999555555555556\n",
            "Test Score:  0.8608\n",
            "\n",
            "\n",
            "n_estimators:  1000\n",
            "Train Score:  0.9999555555555556\n",
            "Test Score:  0.862\n",
            "\n",
            "\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "rf_test = RandomForestClassifier(n_estimators=1000, max_depth=10).fit(x_train, y_train).predict(test_array)"
      ],
      "metadata": {
        "id": "3YoPqMym01q2"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "rf_test_series = pd.Series(rf_test, name=\"Prediction\")\n",
        "rf_test_series.index = test_df.index\n",
        "rf_test_series.to_csv(\"rf_submission.csv\")"
      ],
      "metadata": {
        "id": "r7j0sBh_05VK"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "rf_test_series"
      ],
      "metadata": {
        "id": "SzjgRgvJ4zjo"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "source": [
        "## AdaBoost "
      ],
      "metadata": {
        "id": "TnNbs6b_szgR"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "# learning rate of 1 conistently outperforms others\n",
        "# learning_rates = [0.5, 0.75, 1, 1.5, 2, 5]\n",
        "n_estimators = [1000, 2000, 5000, 10000]\n",
        "\n",
        "for n in n_estimators:\n",
        "    print(\"n_estimators: \", n)\n",
        "  # for lr in learning_rates:\n",
        "    clf = AdaBoostClassifier(n_estimators=n, learning_rate=1)\n",
        "\n",
        "    clf.fit(x, y)\n",
        "\n",
        "    train_pred = clf.predict(x)\n",
        "    holdout_pred = clf.predict(x_holdout)\n",
        "\n",
        "\n",
        "    print(\"Train Error: \", prediction_error(y, train_pred))\n",
        "    print(\"Test Error: \", prediction_error(y_holdout, holdout_pred))\n",
        "    print(\"\\n\")\n",
        "\n",
        "\n"
      ],
      "metadata": {
        "id": "NYJQFB3Rs1Ot",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 588
        },
        "outputId": "9690f843-388f-472f-8374-79e203193736"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "n_estimators:  1000\n",
            "Train Error:  0.1225\n",
            "Test Error:  0.1308\n",
            "\n",
            "\n",
            "n_estimators:  2000\n",
            "Train Error:  0.1198\n",
            "Test Error:  0.1328\n",
            "\n",
            "\n",
            "n_estimators:  5000\n"
          ]
        },
        {
          "output_type": "error",
          "ename": "KeyboardInterrupt",
          "evalue": "ignored",
          "traceback": [
            "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
            "\u001b[0;31mKeyboardInterrupt\u001b[0m                         Traceback (most recent call last)",
            "\u001b[0;32m<ipython-input-30-932793493601>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m\u001b[0m\n\u001b[1;32m      8\u001b[0m     \u001b[0mclf\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mAdaBoostClassifier\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mn_estimators\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mn\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mlearning_rate\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;36m1\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m      9\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 10\u001b[0;31m     \u001b[0mclf\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mfit\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mx\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0my\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m     11\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     12\u001b[0m     \u001b[0mtrain_pred\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mclf\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mpredict\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mx\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;32m/usr/local/lib/python3.8/dist-packages/sklearn/ensemble/_weight_boosting.py\u001b[0m in \u001b[0;36mfit\u001b[0;34m(self, X, y, sample_weight)\u001b[0m\n\u001b[1;32m    484\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    485\u001b[0m         \u001b[0;31m# Fit\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 486\u001b[0;31m         \u001b[0;32mreturn\u001b[0m \u001b[0msuper\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mfit\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mX\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0my\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0msample_weight\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    487\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    488\u001b[0m     \u001b[0;32mdef\u001b[0m \u001b[0m_validate_estimator\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;32m/usr/local/lib/python3.8/dist-packages/sklearn/ensemble/_weight_boosting.py\u001b[0m in \u001b[0;36mfit\u001b[0;34m(self, X, y, sample_weight)\u001b[0m\n\u001b[1;32m    143\u001b[0m         \u001b[0;32mfor\u001b[0m \u001b[0miboost\u001b[0m \u001b[0;32min\u001b[0m \u001b[0mrange\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mn_estimators\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    144\u001b[0m             \u001b[0;31m# Boosting step\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 145\u001b[0;31m             sample_weight, estimator_weight, estimator_error = self._boost(\n\u001b[0m\u001b[1;32m    146\u001b[0m                 \u001b[0miboost\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mX\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0my\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0msample_weight\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mrandom_state\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    147\u001b[0m             )\n",
            "\u001b[0;32m/usr/local/lib/python3.8/dist-packages/sklearn/ensemble/_weight_boosting.py\u001b[0m in \u001b[0;36m_boost\u001b[0;34m(self, iboost, X, y, sample_weight, random_state)\u001b[0m\n\u001b[1;32m    546\u001b[0m         \"\"\"\n\u001b[1;32m    547\u001b[0m         \u001b[0;32mif\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0malgorithm\u001b[0m \u001b[0;34m==\u001b[0m \u001b[0;34m\"SAMME.R\"\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 548\u001b[0;31m             \u001b[0;32mreturn\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_boost_real\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0miboost\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mX\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0my\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0msample_weight\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mrandom_state\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    549\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    550\u001b[0m         \u001b[0;32melse\u001b[0m\u001b[0;34m:\u001b[0m  \u001b[0;31m# elif self.algorithm == \"SAMME\":\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;32m/usr/local/lib/python3.8/dist-packages/sklearn/ensemble/_weight_boosting.py\u001b[0m in \u001b[0;36m_boost_real\u001b[0;34m(self, iboost, X, y, sample_weight, random_state)\u001b[0m\n\u001b[1;32m    555\u001b[0m         \u001b[0mestimator\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_make_estimator\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mrandom_state\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mrandom_state\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    556\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 557\u001b[0;31m         \u001b[0mestimator\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mfit\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mX\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0my\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0msample_weight\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0msample_weight\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    558\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    559\u001b[0m         \u001b[0my_predict_proba\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mestimator\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mpredict_proba\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mX\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;32m/usr/local/lib/python3.8/dist-packages/sklearn/tree/_classes.py\u001b[0m in \u001b[0;36mfit\u001b[0;34m(self, X, y, sample_weight, check_input, X_idx_sorted)\u001b[0m\n\u001b[1;32m    935\u001b[0m         \"\"\"\n\u001b[1;32m    936\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 937\u001b[0;31m         super().fit(\n\u001b[0m\u001b[1;32m    938\u001b[0m             \u001b[0mX\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    939\u001b[0m             \u001b[0my\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;32m/usr/local/lib/python3.8/dist-packages/sklearn/tree/_classes.py\u001b[0m in \u001b[0;36mfit\u001b[0;34m(self, X, y, sample_weight, check_input, X_idx_sorted)\u001b[0m\n\u001b[1;32m    163\u001b[0m             \u001b[0mcheck_X_params\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mdict\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mdtype\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mDTYPE\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0maccept_sparse\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;34m\"csc\"\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    164\u001b[0m             \u001b[0mcheck_y_params\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mdict\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mensure_2d\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;32mFalse\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mdtype\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;32mNone\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 165\u001b[0;31m             X, y = self._validate_data(\n\u001b[0m\u001b[1;32m    166\u001b[0m                 \u001b[0mX\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0my\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mvalidate_separately\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mcheck_X_params\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mcheck_y_params\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    167\u001b[0m             )\n",
            "\u001b[0;32m/usr/local/lib/python3.8/dist-packages/sklearn/base.py\u001b[0m in \u001b[0;36m_validate_data\u001b[0;34m(self, X, y, reset, validate_separately, **check_params)\u001b[0m\n\u001b[1;32m    576\u001b[0m                 \u001b[0;31m# :(\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    577\u001b[0m                 \u001b[0mcheck_X_params\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mcheck_y_params\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mvalidate_separately\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 578\u001b[0;31m                 \u001b[0mX\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mcheck_array\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mX\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m**\u001b[0m\u001b[0mcheck_X_params\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    579\u001b[0m                 \u001b[0my\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mcheck_array\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0my\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m**\u001b[0m\u001b[0mcheck_y_params\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    580\u001b[0m             \u001b[0;32melse\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;32m/usr/local/lib/python3.8/dist-packages/sklearn/utils/validation.py\u001b[0m in \u001b[0;36mcheck_array\u001b[0;34m(array, accept_sparse, accept_large_sparse, dtype, order, copy, force_all_finite, ensure_2d, allow_nd, ensure_min_samples, ensure_min_features, estimator)\u001b[0m\n\u001b[1;32m    744\u001b[0m                     \u001b[0marray\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0marray\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mastype\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mdtype\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mcasting\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;34m\"unsafe\"\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mcopy\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;32mFalse\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    745\u001b[0m                 \u001b[0;32melse\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 746\u001b[0;31m                     \u001b[0marray\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mnp\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0masarray\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0marray\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0morder\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0morder\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mdtype\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mdtype\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    747\u001b[0m             \u001b[0;32mexcept\u001b[0m \u001b[0mComplexWarning\u001b[0m \u001b[0;32mas\u001b[0m \u001b[0mcomplex_warning\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    748\u001b[0m                 raise ValueError(\n",
            "\u001b[0;31mKeyboardInterrupt\u001b[0m: "
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "clf = AdaBoostClassifier(n_estimators=1000, learning_rate=1)\n",
        "clf.fit(x_train, y_train)\n",
        "\n",
        "ada_final_pred = clf.predict(test_array)\n",
        "\n",
        "ada_test_series = pd.Series(ada_final_pred, name=\"Prediction\")\n",
        "ada_test_series.index = test_df.index\n",
        "ada_test_series.to_csv(\"ada_submission.csv\")"
      ],
      "metadata": {
        "id": "n9M5-xabuP3u"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "source": [
        "# Nearest Neighbors"
      ],
      "metadata": {
        "id": "eRiSMElG4jU3"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "k_neighbors = [3, 5, 10, 50, 100]\n",
        "for k in range(5, 15):\n",
        "  knc_uniform = KNeighborsClassifier(k)\n",
        "  knc_distance = KNeighborsClassifier(k, weights=\"distance\")\n",
        "\n",
        "  knc_uniform.fit(x, y)\n",
        "  knc_distance.fit(x, y)\n",
        "\n",
        "  uni_train_pred = knc_uniform.predict(x)\n",
        "  uni_holdout_pred = knc_uniform.predict(x_holdout)\n",
        "\n",
        "  # dist_train_pred = knc_distance.predict(x)\n",
        "  # dist_holdout_pred = knc_distance.predict(x_holdout)\n",
        "\n",
        "  print(\"k: \", k)\n",
        "  print('w: Uniform========')\n",
        "  print(\"Train Error: \", prediction_error(y, uni_train_pred))\n",
        "  print(\"Test Error: \", prediction_error(y_holdout, uni_holdout_pred))\n",
        "  # print('w: Distance=======')\n",
        "  # print(\"Train Error: \", prediction_error(y, uni_train_pred))\n",
        "  # print(\"Test Error: \", prediction_error(y_holdout, uni_holdout_pred))\n",
        "  # print('\\n')"
      ],
      "metadata": {
        "id": "Yzx6CKmq4mac"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "source": [
        "# MLP"
      ],
      "metadata": {
        "id": "4RCTkEuBZ4HN"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "alpha = [.0001, .001, .1, 1, 10]\n",
        "for a in alpha:\n",
        "  mlpc_logistic = MLPClassifier(alpha=a,  max_iter=1000, activation=\"logistic\")\n",
        "  mlpc_relu = MLPClassifier(alpha=a,  max_iter=1000, activation=\"relu\")\n",
        "\n",
        "  mlpc_logistic.fit(x, y)\n",
        "  mlpc_relu.fit(x, y)\n",
        "\n",
        "  log_train_pred = mlpc_logistic.predict(x)\n",
        "  log_holdout_pred = mlpc_logistic.predict(x_holdout)\n",
        "\n",
        "  relu_train_pred = mlpc_relu.predict(x)\n",
        "  relu_holdout_pred = mlpc_relu.predict(x_holdout)\n",
        "\n",
        "  print(\"alpha: \", a)\n",
        "  print('Logistic ========')\n",
        "  print(\"Train Error: \", prediction_error(y, log_train_pred))\n",
        "  print(\"Test Error: \", prediction_error(y_holdout, log_holdout_pred))\n",
        "  print('Relu ========')\n",
        "  print(\"Train Error: \", prediction_error(y, relu_train_pred))\n",
        "  print(\"Test Error: \", prediction_error(y_holdout, relu_holdout_pred))\n",
        "  print(\"\\n\")"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "S_dr_9q7Z5-d",
        "outputId": "99512f80-e8a9-4802-8170-2d4dae46cdbb"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "alpha:  0.0001\n",
            "Logistic ========\n",
            "Train Error:  0.2328\n",
            "Test Error:  0.24\n",
            "Relu ========\n",
            "Train Error:  0.2016\n",
            "Test Error:  0.2098\n",
            "\n",
            "\n",
            "alpha:  0.001\n",
            "Logistic ========\n",
            "Train Error:  0.2333\n",
            "Test Error:  0.2404\n",
            "Relu ========\n",
            "Train Error:  0.2147\n",
            "Test Error:  0.22\n",
            "\n",
            "\n",
            "alpha:  0.1\n",
            "Logistic ========\n",
            "Train Error:  0.239\n",
            "Test Error:  0.244\n",
            "Relu ========\n",
            "Train Error:  0.2018\n",
            "Test Error:  0.2101\n",
            "\n",
            "\n",
            "alpha:  1\n",
            "Logistic ========\n",
            "Train Error:  0.239\n",
            "Test Error:  0.244\n",
            "Relu ========\n",
            "Train Error:  0.214\n",
            "Test Error:  0.2195\n",
            "\n",
            "\n",
            "alpha:  10\n",
            "Logistic ========\n",
            "Train Error:  0.239\n",
            "Test Error:  0.244\n",
            "Relu ========\n",
            "Train Error:  0.761\n",
            "Test Error:  0.756\n",
            "\n",
            "\n"
          ]
        }
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "# Naive Bays"
      ],
      "metadata": {
        "id": "0BUz0AowklrM"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "gmnb = GaussianNB().fit(x, y)\n",
        "mnb = MultinomialNB().fit(x, y)\n",
        "\n",
        "gmnb_train_pred = gmnb.predict(x)\n",
        "gmnb_holdout_pred = gmnb.predict(x_holdout)\n",
        "\n",
        "mnb_train_pred = mnb.predict(x)\n",
        "mnb_holdout_pred = mnb.predict(x_holdout)\n",
        "\n",
        "# print('Gaussian ========')\n",
        "# print(\"Train Error: \", prediction_error(y, gmnb_train_pred))\n",
        "# print(\"Test Error: \", prediction_error(y_holdout, gmnb_holdout_pred))\n",
        "# print('Multinomial ========')\n",
        "# print(\"Train Error: \", prediction_error(y, mnb_train_pred))\n",
        "# print(\"Test Error: \", prediction_error(y_holdout, mnb_holdout_pred))\n",
        "# print(\"\\n\")\n",
        "\n",
        "print('Gaussian ========')\n",
        "print(\"Train Error: \", gmnb.score(x, y))\n",
        "print(\"Test Error: \", gmnb.score(x_holdout, y_holdout))\n",
        "print('Multinomial ========')\n",
        "print(\"Train Error: \", mnb.score(x, y))\n",
        "print(\"Test Error: \", mnb.score(x_holdout, y_holdout))\n",
        "print(\"\\n\")"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "T-5gX29_krjR",
        "outputId": "721b9353-a2e3-4ec6-bac6-7aade2994183"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Gaussian ========\n",
            "Train Error:  0.7949777777777778\n",
            "Test Error:  0.808\n",
            "Multinomial ========\n",
            "Train Error:  0.7822666666666667\n",
            "Test Error:  0.7976\n",
            "\n",
            "\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "# for submission\n",
        "gnb_final = GaussianNB().fit(x_train, y_train).predict(test_array)\n",
        "gnb_test_series = pd.Series(gnb_final, name=\"Prediction\")\n",
        "gnb_test_series.index = test_df.index\n",
        "gnb_test_series.to_csv(\"gnb_submission.csv\")"
      ],
      "metadata": {
        "id": "LT2jlT0Xs6ES"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "source": [
        "#Keras\n",
        "\n"
      ],
      "metadata": {
        "id": "GSQGRaLmAX1Z"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "!pip install tensorflow"
      ],
      "metadata": {
        "id": "d6ZCYrGXCkmh"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "import tensorflow as tf\n",
        "from tensorflow import keras\n",
        "\n",
        "model = keras.Sequential([\n",
        "    keras.layers.Flatten(input_shape=(x.shape[1],)),\n",
        "    keras.layers.Dense(10, activation=tf.nn.relu),\n",
        "\t  keras.layers.Dense(10, activation=tf.nn.relu),\n",
        "    keras.layers.Dense(1, activation=tf.nn.sigmoid),\n",
        "])\n",
        "\n",
        "model.compile(optimizer='adam', loss='binary_crossentropy', metrics=['accuracy'])\n",
        "\n",
        "model.fit(x, y, epochs=100, batch_size=32)\n",
        "\n",
        "print(model.evaluate(x_holdout, y_holdout))\n"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "CVLBm9OXBrQw",
        "outputId": "1fa93363-55d4-4333-fe91-154b3b7fe562"
      },
      "execution_count": 13,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Epoch 1/100\n",
            "704/704 [==============================] - 2s 3ms/step - loss: 257.2299 - accuracy: 0.6966\n",
            "Epoch 2/100\n",
            "704/704 [==============================] - 2s 3ms/step - loss: 26.7021 - accuracy: 0.6861\n",
            "Epoch 3/100\n",
            "704/704 [==============================] - 2s 2ms/step - loss: 25.7120 - accuracy: 0.6794\n",
            "Epoch 4/100\n",
            "704/704 [==============================] - 2s 2ms/step - loss: 21.7602 - accuracy: 0.6877\n",
            "Epoch 5/100\n",
            "704/704 [==============================] - 2s 2ms/step - loss: 20.7429 - accuracy: 0.6880\n",
            "Epoch 6/100\n",
            "704/704 [==============================] - 2s 2ms/step - loss: 21.7505 - accuracy: 0.6898\n",
            "Epoch 7/100\n",
            "704/704 [==============================] - 2s 3ms/step - loss: 22.2401 - accuracy: 0.6903\n",
            "Epoch 8/100\n",
            "704/704 [==============================] - 3s 4ms/step - loss: 19.2705 - accuracy: 0.6935\n",
            "Epoch 9/100\n",
            "704/704 [==============================] - 2s 2ms/step - loss: 16.5417 - accuracy: 0.6955\n",
            "Epoch 10/100\n",
            "704/704 [==============================] - 2s 3ms/step - loss: 15.3633 - accuracy: 0.7035\n",
            "Epoch 11/100\n",
            "704/704 [==============================] - 4s 5ms/step - loss: 15.9344 - accuracy: 0.7046\n",
            "Epoch 12/100\n",
            "704/704 [==============================] - 3s 4ms/step - loss: 16.9003 - accuracy: 0.7028\n",
            "Epoch 13/100\n",
            "704/704 [==============================] - 2s 3ms/step - loss: 13.2000 - accuracy: 0.7072\n",
            "Epoch 14/100\n",
            "704/704 [==============================] - 2s 3ms/step - loss: 19.0698 - accuracy: 0.7009\n",
            "Epoch 15/100\n",
            "704/704 [==============================] - 2s 2ms/step - loss: 15.0760 - accuracy: 0.7070\n",
            "Epoch 16/100\n",
            "704/704 [==============================] - 2s 2ms/step - loss: 19.6780 - accuracy: 0.7033\n",
            "Epoch 17/100\n",
            "704/704 [==============================] - 2s 2ms/step - loss: 13.0397 - accuracy: 0.7116\n",
            "Epoch 18/100\n",
            "704/704 [==============================] - 2s 2ms/step - loss: 9.8905 - accuracy: 0.7242\n",
            "Epoch 19/100\n",
            "704/704 [==============================] - 2s 2ms/step - loss: 10.5915 - accuracy: 0.7216\n",
            "Epoch 20/100\n",
            "704/704 [==============================] - 2s 2ms/step - loss: 12.3012 - accuracy: 0.7258\n",
            "Epoch 21/100\n",
            "704/704 [==============================] - 2s 2ms/step - loss: 13.2120 - accuracy: 0.7151\n",
            "Epoch 22/100\n",
            "704/704 [==============================] - 2s 2ms/step - loss: 11.5570 - accuracy: 0.7226\n",
            "Epoch 23/100\n",
            "704/704 [==============================] - 2s 2ms/step - loss: 9.9885 - accuracy: 0.7345\n",
            "Epoch 24/100\n",
            "704/704 [==============================] - 2s 2ms/step - loss: 11.8168 - accuracy: 0.7288\n",
            "Epoch 25/100\n",
            "704/704 [==============================] - 2s 2ms/step - loss: 7.5053 - accuracy: 0.7427\n",
            "Epoch 26/100\n",
            "704/704 [==============================] - 2s 2ms/step - loss: 9.0075 - accuracy: 0.7356\n",
            "Epoch 27/100\n",
            "704/704 [==============================] - 3s 4ms/step - loss: 10.2327 - accuracy: 0.7362\n",
            "Epoch 28/100\n",
            "704/704 [==============================] - 3s 4ms/step - loss: 9.4943 - accuracy: 0.7340\n",
            "Epoch 29/100\n",
            "704/704 [==============================] - 4s 5ms/step - loss: 9.7794 - accuracy: 0.7349\n",
            "Epoch 30/100\n",
            "704/704 [==============================] - 2s 3ms/step - loss: 7.7048 - accuracy: 0.7449\n",
            "Epoch 31/100\n",
            "704/704 [==============================] - 2s 2ms/step - loss: 7.4418 - accuracy: 0.7480\n",
            "Epoch 32/100\n",
            "704/704 [==============================] - 2s 2ms/step - loss: 8.3577 - accuracy: 0.7456\n",
            "Epoch 33/100\n",
            "704/704 [==============================] - 2s 3ms/step - loss: 7.7907 - accuracy: 0.7446\n",
            "Epoch 34/100\n",
            "704/704 [==============================] - 2s 2ms/step - loss: 6.8998 - accuracy: 0.7449\n",
            "Epoch 35/100\n",
            "704/704 [==============================] - 2s 2ms/step - loss: 8.2584 - accuracy: 0.7448\n",
            "Epoch 36/100\n",
            "704/704 [==============================] - 2s 2ms/step - loss: 6.4718 - accuracy: 0.7540\n",
            "Epoch 37/100\n",
            "704/704 [==============================] - 2s 2ms/step - loss: 7.2018 - accuracy: 0.7508\n",
            "Epoch 38/100\n",
            "704/704 [==============================] - 1s 2ms/step - loss: 7.6142 - accuracy: 0.7435\n",
            "Epoch 39/100\n",
            "704/704 [==============================] - 2s 2ms/step - loss: 6.7665 - accuracy: 0.7544\n",
            "Epoch 40/100\n",
            "704/704 [==============================] - 2s 2ms/step - loss: 7.3534 - accuracy: 0.7507\n",
            "Epoch 41/100\n",
            "704/704 [==============================] - 2s 2ms/step - loss: 7.0175 - accuracy: 0.7570\n",
            "Epoch 42/100\n",
            "704/704 [==============================] - 2s 2ms/step - loss: 7.1225 - accuracy: 0.7516\n",
            "Epoch 43/100\n",
            "704/704 [==============================] - 2s 3ms/step - loss: 5.5504 - accuracy: 0.7614\n",
            "Epoch 44/100\n",
            "704/704 [==============================] - 3s 4ms/step - loss: 4.7031 - accuracy: 0.7668\n",
            "Epoch 45/100\n",
            "704/704 [==============================] - 3s 4ms/step - loss: 5.3035 - accuracy: 0.7541\n",
            "Epoch 46/100\n",
            "704/704 [==============================] - 2s 3ms/step - loss: 6.7764 - accuracy: 0.7526\n",
            "Epoch 47/100\n",
            "704/704 [==============================] - 2s 3ms/step - loss: 6.0040 - accuracy: 0.7530\n",
            "Epoch 48/100\n",
            "704/704 [==============================] - 2s 2ms/step - loss: 4.6098 - accuracy: 0.7632\n",
            "Epoch 49/100\n",
            "704/704 [==============================] - 2s 2ms/step - loss: 5.3298 - accuracy: 0.7650\n",
            "Epoch 50/100\n",
            "704/704 [==============================] - 2s 2ms/step - loss: 4.9939 - accuracy: 0.7615\n",
            "Epoch 51/100\n",
            "704/704 [==============================] - 2s 2ms/step - loss: 4.3571 - accuracy: 0.7685\n",
            "Epoch 52/100\n",
            "704/704 [==============================] - 2s 2ms/step - loss: 4.2213 - accuracy: 0.7728\n",
            "Epoch 53/100\n",
            "704/704 [==============================] - 2s 2ms/step - loss: 4.7248 - accuracy: 0.7615\n",
            "Epoch 54/100\n",
            "704/704 [==============================] - 2s 2ms/step - loss: 3.8023 - accuracy: 0.7748\n",
            "Epoch 55/100\n",
            "704/704 [==============================] - 2s 3ms/step - loss: 4.1722 - accuracy: 0.7682\n",
            "Epoch 56/100\n",
            "704/704 [==============================] - 2s 2ms/step - loss: 5.4321 - accuracy: 0.7603\n",
            "Epoch 57/100\n",
            "704/704 [==============================] - 3s 4ms/step - loss: 4.0410 - accuracy: 0.7718\n",
            "Epoch 58/100\n",
            "704/704 [==============================] - 2s 2ms/step - loss: 4.6388 - accuracy: 0.7673\n",
            "Epoch 59/100\n",
            "704/704 [==============================] - 3s 4ms/step - loss: 3.9108 - accuracy: 0.7708\n",
            "Epoch 60/100\n",
            "704/704 [==============================] - 3s 5ms/step - loss: 3.3012 - accuracy: 0.7811\n",
            "Epoch 61/100\n",
            "704/704 [==============================] - 3s 4ms/step - loss: 4.0320 - accuracy: 0.7759\n",
            "Epoch 62/100\n",
            "704/704 [==============================] - 2s 3ms/step - loss: 3.2438 - accuracy: 0.7801\n",
            "Epoch 63/100\n",
            "704/704 [==============================] - 2s 2ms/step - loss: 3.2514 - accuracy: 0.7829\n",
            "Epoch 64/100\n",
            "704/704 [==============================] - 2s 2ms/step - loss: 4.0051 - accuracy: 0.7724\n",
            "Epoch 65/100\n",
            "704/704 [==============================] - 2s 3ms/step - loss: 3.5977 - accuracy: 0.7728\n",
            "Epoch 66/100\n",
            "704/704 [==============================] - 2s 2ms/step - loss: 3.6081 - accuracy: 0.7737\n",
            "Epoch 67/100\n",
            "704/704 [==============================] - 2s 2ms/step - loss: 3.5706 - accuracy: 0.7778\n",
            "Epoch 68/100\n",
            "704/704 [==============================] - 2s 2ms/step - loss: 3.4228 - accuracy: 0.7776\n",
            "Epoch 69/100\n",
            "704/704 [==============================] - 2s 2ms/step - loss: 2.6969 - accuracy: 0.7827\n",
            "Epoch 70/100\n",
            "704/704 [==============================] - 2s 2ms/step - loss: 2.8519 - accuracy: 0.7837\n",
            "Epoch 71/100\n",
            "704/704 [==============================] - 2s 2ms/step - loss: 3.1830 - accuracy: 0.7792\n",
            "Epoch 72/100\n",
            "704/704 [==============================] - 2s 2ms/step - loss: 3.1153 - accuracy: 0.7811\n",
            "Epoch 73/100\n",
            "704/704 [==============================] - 2s 3ms/step - loss: 3.0014 - accuracy: 0.7847\n",
            "Epoch 74/100\n",
            "704/704 [==============================] - 2s 2ms/step - loss: 2.5783 - accuracy: 0.7862\n",
            "Epoch 75/100\n",
            "704/704 [==============================] - 2s 3ms/step - loss: 2.9201 - accuracy: 0.7797\n",
            "Epoch 76/100\n",
            "704/704 [==============================] - 3s 4ms/step - loss: 2.2902 - accuracy: 0.7907\n",
            "Epoch 77/100\n",
            "704/704 [==============================] - 4s 5ms/step - loss: 2.6138 - accuracy: 0.7896\n",
            "Epoch 78/100\n",
            "704/704 [==============================] - 2s 3ms/step - loss: 4.1774 - accuracy: 0.7708\n",
            "Epoch 79/100\n",
            "704/704 [==============================] - 2s 2ms/step - loss: 2.5674 - accuracy: 0.7866\n",
            "Epoch 80/100\n",
            "704/704 [==============================] - 2s 2ms/step - loss: 2.5906 - accuracy: 0.7831\n",
            "Epoch 81/100\n",
            "704/704 [==============================] - 2s 2ms/step - loss: 2.3871 - accuracy: 0.7855\n",
            "Epoch 82/100\n",
            "704/704 [==============================] - 2s 3ms/step - loss: 2.7589 - accuracy: 0.7824\n",
            "Epoch 83/100\n",
            "704/704 [==============================] - 2s 3ms/step - loss: 2.9791 - accuracy: 0.7784\n",
            "Epoch 84/100\n",
            "704/704 [==============================] - 2s 2ms/step - loss: 2.3627 - accuracy: 0.7875\n",
            "Epoch 85/100\n",
            "704/704 [==============================] - 2s 2ms/step - loss: 2.3382 - accuracy: 0.7858\n",
            "Epoch 86/100\n",
            "704/704 [==============================] - 2s 2ms/step - loss: 2.5166 - accuracy: 0.7836\n",
            "Epoch 87/100\n",
            "704/704 [==============================] - 2s 3ms/step - loss: 2.3023 - accuracy: 0.7860\n",
            "Epoch 88/100\n",
            "704/704 [==============================] - 3s 4ms/step - loss: 2.9816 - accuracy: 0.7716\n",
            "Epoch 89/100\n",
            "704/704 [==============================] - 2s 3ms/step - loss: 2.2441 - accuracy: 0.7843\n",
            "Epoch 90/100\n",
            "704/704 [==============================] - 2s 2ms/step - loss: 1.9633 - accuracy: 0.7924\n",
            "Epoch 91/100\n",
            "704/704 [==============================] - 3s 4ms/step - loss: 2.5244 - accuracy: 0.7803\n",
            "Epoch 92/100\n",
            "704/704 [==============================] - 4s 5ms/step - loss: 2.1467 - accuracy: 0.7842\n",
            "Epoch 93/100\n",
            "704/704 [==============================] - 4s 6ms/step - loss: 2.2935 - accuracy: 0.7847\n",
            "Epoch 94/100\n",
            "704/704 [==============================] - 3s 5ms/step - loss: 1.9771 - accuracy: 0.7869\n",
            "Epoch 95/100\n",
            "704/704 [==============================] - 2s 2ms/step - loss: 1.9426 - accuracy: 0.7873\n",
            "Epoch 96/100\n",
            "704/704 [==============================] - 2s 3ms/step - loss: 1.6114 - accuracy: 0.7960\n",
            "Epoch 97/100\n",
            "704/704 [==============================] - 2s 3ms/step - loss: 2.0414 - accuracy: 0.7852\n",
            "Epoch 98/100\n",
            "704/704 [==============================] - 2s 2ms/step - loss: 1.7804 - accuracy: 0.7919\n",
            "Epoch 99/100\n",
            "704/704 [==============================] - 2s 3ms/step - loss: 2.4563 - accuracy: 0.7747\n",
            "Epoch 100/100\n",
            "704/704 [==============================] - 4s 5ms/step - loss: 2.0841 - accuracy: 0.7797\n",
            "79/79 [==============================] - 1s 4ms/step - loss: 0.7317 - accuracy: 0.8204\n",
            "Test accuracy: 0.8203999996185303\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "keras_predictions = model.predict(test_array).flatten()\n",
        "\n"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "7LNHho0YJFkF",
        "outputId": "12a750ef-a1a7-4d1d-c5bb-841577f5c320"
      },
      "execution_count": 16,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "746/746 [==============================] - 1s 1ms/step\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "keras_predictions"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "GsssqUzwJoAi",
        "outputId": "cef51ac2-892c-4678-8693-45186ab17d64"
      },
      "execution_count": 17,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "array([6.1113659e-02, 5.4381764e-04, 2.2346221e-02, ..., 1.0000000e+00,\n",
              "       5.5828369e-03, 9.9896216e-01], dtype=float32)"
            ]
          },
          "metadata": {},
          "execution_count": 17
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "keras_test_series = pd.Series(keras_predictions, name=\"Prediction\")\n",
        "keras_test_series.index = test_df.index\n",
        "keras_test_series.to_csv(\"keras_submission.csv\")"
      ],
      "metadata": {
        "id": "th4-84wlJkKo"
      },
      "execution_count": 18,
      "outputs": []
    }
  ],
  "metadata": {
    "colab": {
      "provenance": [],
      "authorship_tag": "ABX9TyOtfssRmUwfVQrcY2BCA3eF",
      "include_colab_link": true
    },
    "kernelspec": {
      "display_name": "Python 3",
      "name": "python3"
    },
    "language_info": {
      "name": "python"
    }
  },
  "nbformat": 4,
  "nbformat_minor": 0
}