embeddings_classification-tests.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "id": "c83d4779",
   "metadata": {},
   "source": [
    "# Setup enviorment"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "ba7a62fd",
   "metadata": {},
   "outputs": [],
   "source": [
    "#!pip install keras-tuner"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "3c3b3134",
   "metadata": {},
   "outputs": [],
   "source": [
    "# Data reading in Dataframe format and data preprocessing\n",
    "import pandas as pd\n",
    "from pandas import read_csv\n",
    "from pandas import DataFrame\n",
    "from pandas import concat\n",
    "\n",
    "# Data Visualization\n",
    "import matplotlib.pyplot as plt\n",
    "import seaborn as sns\n",
    "\n",
    "# Linear algebra operations\n",
    "import numpy as np\n",
    "\n",
    "# Machine learning models and preprocessing\n",
    "from sklearn.preprocessing import MinMaxScaler\n",
    "from sklearn.preprocessing import StandardScaler\n",
    "from sklearn import preprocessing\n",
    "\n",
    "# Deep learning\n",
    "import tensorflow as tf\n",
    "from tensorflow import keras\n",
    "from tensorflow.keras.utils import to_categorical\n",
    "from tensorflow.keras import Sequential, layers, callbacks\n",
    "from tensorflow.keras.layers import Dense, LSTM, Dropout, GRU, Bidirectional\n",
    "import keras_tuner\n",
    "\n",
    "# Epiweek\n",
    "from epiweeks import Week, Year\n",
    "\n",
    "# Date\n",
    "from datetime import date as convert_to_date"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "9191b0f3",
   "metadata": {},
   "outputs": [],
   "source": [
    "import warnings\n",
    "warnings.filterwarnings('ignore')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "437962b9",
   "metadata": {},
   "outputs": [],
   "source": [
    "embeddings = 'Embeddings/embeddings_autoencoder_1024features.csv'\n",
    "labels = 'Tabular_data/Label_CSV_All_Municipality.csv'\n",
    "MUNICIPALITY = 'Ibagué'"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "4abeea5d",
   "metadata": {},
   "source": [
    "# Read Data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "id": "d3f62b4f",
   "metadata": {},
   "outputs": [],
   "source": [
    "def epiweek_from_date(image_date):\n",
    "    date = image_date.split('-')\n",
    "    \n",
    "    # Get year as int\n",
    "    year = ''.join(filter(str.isdigit, date[0]))\n",
    "    year = int(year)\n",
    "    \n",
    "    # Get month as int\n",
    "    month = ''.join(filter(str.isdigit, date[1]))\n",
    "    month = int(month)\n",
    "    \n",
    "    # Get day as int\n",
    "    day = ''.join(filter(str.isdigit, date[2]))\n",
    "    day = int(day)\n",
    "    \n",
    "    # Get epiweek:\n",
    "    date = convert_to_date(year, month, day)\n",
    "    epiweek = str(Week.fromdate(date))\n",
    "    epiweek = int(epiweek)\n",
    "    \n",
    "    return epiweek"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "56d8ee55",
   "metadata": {},
   "source": [
    "### 1. Features"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "id": "b600edc8",
   "metadata": {},
   "outputs": [],
   "source": [
    "def read_features(path, Municipality = None):\n",
    "    df = pd.read_csv(path)\n",
    "    #df.Date = pd.to_datetime(df.Date)\n",
    "    \n",
    "    if Municipality:\n",
    "        print('Obtaining dataframe for the city of Medellin only...')\n",
    "        df = df[df['Municipality Code'] == Municipality]\n",
    "        \n",
    "    df.Date = df.Date.apply(epiweek_from_date)\n",
    "    \n",
    "    df = df.sort_values(by=['Date'])\n",
    "    \n",
    "    df = df.set_index('Date')\n",
    "    \n",
    "    if Municipality:\n",
    "        df.drop(columns=['Municipality Code'], inplace=True)\n",
    "        \n",
    "    df.index.name = None\n",
    "    return df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "id": "c9a985c1",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Obtaining dataframe for the city of Medellin only...\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>0</th>\n",
       "      <th>1</th>\n",
       "      <th>2</th>\n",
       "      <th>3</th>\n",
       "      <th>4</th>\n",
       "      <th>5</th>\n",
       "      <th>6</th>\n",
       "      <th>7</th>\n",
       "      <th>8</th>\n",
       "      <th>9</th>\n",
       "      <th>...</th>\n",
       "      <th>1014</th>\n",
       "      <th>1015</th>\n",
       "      <th>1016</th>\n",
       "      <th>1017</th>\n",
       "      <th>1018</th>\n",
       "      <th>1019</th>\n",
       "      <th>1020</th>\n",
       "      <th>1021</th>\n",
       "      <th>1022</th>\n",
       "      <th>1023</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>201544</th>\n",
       "      <td>0.0</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>...</td>\n",
       "      <td>0.0</td>\n",
       "      <td>172.602520</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.00000</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>9.285753</td>\n",
       "      <td>68.562750</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>201545</th>\n",
       "      <td>0.0</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>...</td>\n",
       "      <td>0.0</td>\n",
       "      <td>172.602520</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.00000</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>9.285753</td>\n",
       "      <td>68.562750</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>201546</th>\n",
       "      <td>0.0</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.0</td>\n",
       "      <td>12.446627</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>...</td>\n",
       "      <td>0.0</td>\n",
       "      <td>274.144930</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.00000</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>15.671944</td>\n",
       "      <td>2.534543</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>201547</th>\n",
       "      <td>0.0</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.0</td>\n",
       "      <td>12.446627</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>...</td>\n",
       "      <td>0.0</td>\n",
       "      <td>274.144930</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.00000</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>15.671944</td>\n",
       "      <td>2.534543</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>201548</th>\n",
       "      <td>0.0</td>\n",
       "      <td>3.889202</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>...</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.00000</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>4.791192</td>\n",
       "      <td>3.796963</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>201848</th>\n",
       "      <td>0.0</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.0</td>\n",
       "      <td>21.014330</td>\n",
       "      <td>0.0</td>\n",
       "      <td>20.102703</td>\n",
       "      <td>0.469093</td>\n",
       "      <td>...</td>\n",
       "      <td>0.0</td>\n",
       "      <td>177.102700</td>\n",
       "      <td>0.0</td>\n",
       "      <td>4.91719</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>5.318927</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>201849</th>\n",
       "      <td>0.0</td>\n",
       "      <td>16.103472</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>...</td>\n",
       "      <td>0.0</td>\n",
       "      <td>31.717075</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.00000</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>21.635908</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>201850</th>\n",
       "      <td>0.0</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>...</td>\n",
       "      <td>0.0</td>\n",
       "      <td>80.470200</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.00000</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>22.293388</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>201851</th>\n",
       "      <td>0.0</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>...</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.00000</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>27.029966</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>201852</th>\n",
       "      <td>0.0</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>7.444656</td>\n",
       "      <td>0.0</td>\n",
       "      <td>37.199368</td>\n",
       "      <td>0.0</td>\n",
       "      <td>35.591797</td>\n",
       "      <td>10.594391</td>\n",
       "      <td>...</td>\n",
       "      <td>0.0</td>\n",
       "      <td>151.715740</td>\n",
       "      <td>0.0</td>\n",
       "      <td>5.35182</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>12.584043</td>\n",
       "      <td>14.189192</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>165 rows × 1024 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "          0          1    2    3         4    5          6    7          8  \\\n",
       "201544  0.0   0.000000  0.0  0.0  0.000000  0.0   0.000000  0.0   0.000000   \n",
       "201545  0.0   0.000000  0.0  0.0  0.000000  0.0   0.000000  0.0   0.000000   \n",
       "201546  0.0   0.000000  0.0  0.0  0.000000  0.0   0.000000  0.0  12.446627   \n",
       "201547  0.0   0.000000  0.0  0.0  0.000000  0.0   0.000000  0.0  12.446627   \n",
       "201548  0.0   3.889202  0.0  0.0  0.000000  0.0   0.000000  0.0   0.000000   \n",
       "...     ...        ...  ...  ...       ...  ...        ...  ...        ...   \n",
       "201848  0.0   0.000000  0.0  0.0  0.000000  0.0  21.014330  0.0  20.102703   \n",
       "201849  0.0  16.103472  0.0  0.0  0.000000  0.0   0.000000  0.0   0.000000   \n",
       "201850  0.0   0.000000  0.0  0.0  0.000000  0.0   0.000000  0.0   0.000000   \n",
       "201851  0.0   0.000000  0.0  0.0  0.000000  0.0   0.000000  0.0   0.000000   \n",
       "201852  0.0   0.000000  0.0  0.0  7.444656  0.0  37.199368  0.0  35.591797   \n",
       "\n",
       "                9  ...  1014        1015  1016     1017  1018  1019  \\\n",
       "201544   0.000000  ...   0.0  172.602520   0.0  0.00000   0.0   0.0   \n",
       "201545   0.000000  ...   0.0  172.602520   0.0  0.00000   0.0   0.0   \n",
       "201546   0.000000  ...   0.0  274.144930   0.0  0.00000   0.0   0.0   \n",
       "201547   0.000000  ...   0.0  274.144930   0.0  0.00000   0.0   0.0   \n",
       "201548   0.000000  ...   0.0    0.000000   0.0  0.00000   0.0   0.0   \n",
       "...           ...  ...   ...         ...   ...      ...   ...   ...   \n",
       "201848   0.469093  ...   0.0  177.102700   0.0  4.91719   0.0   0.0   \n",
       "201849   0.000000  ...   0.0   31.717075   0.0  0.00000   0.0   0.0   \n",
       "201850   0.000000  ...   0.0   80.470200   0.0  0.00000   0.0   0.0   \n",
       "201851   0.000000  ...   0.0    0.000000   0.0  0.00000   0.0   0.0   \n",
       "201852  10.594391  ...   0.0  151.715740   0.0  5.35182   0.0   0.0   \n",
       "\n",
       "             1020       1021       1022  1023  \n",
       "201544   9.285753  68.562750   0.000000   0.0  \n",
       "201545   9.285753  68.562750   0.000000   0.0  \n",
       "201546  15.671944   2.534543   0.000000   0.0  \n",
       "201547  15.671944   2.534543   0.000000   0.0  \n",
       "201548   4.791192   3.796963   0.000000   0.0  \n",
       "...           ...        ...        ...   ...  \n",
       "201848   0.000000   0.000000   5.318927   0.0  \n",
       "201849   0.000000  21.635908   0.000000   0.0  \n",
       "201850   0.000000  22.293388   0.000000   0.0  \n",
       "201851   0.000000  27.029966   0.000000   0.0  \n",
       "201852   0.000000  12.584043  14.189192   0.0  \n",
       "\n",
       "[165 rows x 1024 columns]"
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "features_df = read_features(path=embeddings, Municipality='Medellín')\n",
    "features_df"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "449df8de",
   "metadata": {},
   "source": [
    "### 2. Labels"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "id": "d86d29cf",
   "metadata": {},
   "outputs": [],
   "source": [
    "def get_epiweek(name):\n",
    "    \n",
    "    # Get week\n",
    "    week = name.split('/')[1]\n",
    "    week = week.replace('w','')\n",
    "    week = int(week)\n",
    "    \n",
    "    # Year\n",
    "    year = name.split('/')[0]\n",
    "    year = int(year)\n",
    "    \n",
    "    epiweek = Week(year, week)\n",
    "    \n",
    "    epiweek = str(epiweek)\n",
    "    epiweek = int(epiweek)\n",
    "\n",
    "    return epiweek"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "id": "d612bff1",
   "metadata": {},
   "outputs": [],
   "source": [
    "def read_labels(path, Municipality = None):\n",
    "    df = pd.read_csv(path)\n",
    "    if df.shape[1] > 678:\n",
    "        df = pd.concat([df[['Municipality code', 'Municipality']], df.iloc[:,-676:]], axis=1)\n",
    "        cols = df.iloc[:, 2:].columns\n",
    "        new_cols = df.iloc[:, 2:].columns.to_series().apply(get_epiweek)\n",
    "        df = df.rename(columns=dict(zip(cols, new_cols))) \n",
    "        \n",
    "    if 'Label_CSV_All_Municipality' in path:\n",
    "        # Get Columns\n",
    "        df = df[['epiweek', 'Municipality code', 'Municipality', 'final_cases_label']]\n",
    "        \n",
    "        # change epiweek format\n",
    "        df.epiweek = df.epiweek.apply(get_epiweek)\n",
    "        \n",
    "        # Remove duplicates\n",
    "        df = df[df.duplicated(['epiweek','Municipality code','Municipality']) == False]\n",
    "        \n",
    "        # Replace Increase, decrease, stable to numerical:\n",
    "        \"\"\"\n",
    "        - Stable = 0\n",
    "        - Increased = 1 \n",
    "        - Decreased = 2\n",
    "        \"\"\"\n",
    "        df.final_cases_label = df.final_cases_label.replace({'Stable': 0, 'Increased': 1, 'Decreased': 2})\n",
    "        \n",
    "        # Create table\n",
    "        df = df.pivot(index=['Municipality code', 'Municipality'], columns='epiweek', values='final_cases_label')\n",
    "\n",
    "        # Reset Index:\n",
    "        df = df.reset_index()\n",
    "    \n",
    "    if Municipality:\n",
    "        df = df[df['Municipality'] == Municipality]\n",
    "        df.drop(columns=['Municipality code'], inplace=True)\n",
    "        df.rename(columns={'Municipality': 'Municipality Code'}, inplace=True)\n",
    "    \n",
    "        df = df.set_index('Municipality Code')\n",
    "        df = df.T\n",
    "\n",
    "        df.columns.name = None\n",
    "        df.index.name = None\n",
    "        \n",
    "        df.columns = ['Labels']\n",
    "    \n",
    "    return df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "id": "4eb01426",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>0</th>\n",
       "      <th>1</th>\n",
       "      <th>2</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>201601</th>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>201602</th>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>201603</th>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>201604</th>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>201605</th>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>201848</th>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>201849</th>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>201850</th>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>201851</th>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>201852</th>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>156 rows × 3 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "        0  1  2\n",
       "201601  1  0  0\n",
       "201602  0  1  0\n",
       "201603  0  0  1\n",
       "201604  1  0  0\n",
       "201605  1  0  0\n",
       "...    .. .. ..\n",
       "201848  1  0  0\n",
       "201849  0  0  1\n",
       "201850  0  1  0\n",
       "201851  1  0  0\n",
       "201852  1  0  0\n",
       "\n",
       "[156 rows x 3 columns]"
      ]
     },
     "execution_count": 10,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "labels_df = read_labels(path=labels, Municipality='Medellín')\n",
    "labels_df_orig = labels_df\n",
    "labels_df = pd.get_dummies(labels_df['Labels'])\n",
    "labels_df"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "b65bb863",
   "metadata": {},
   "source": [
    "# Data preparation"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "id": "0e68c251",
   "metadata": {},
   "outputs": [],
   "source": [
    "n_labels = labels_df.shape[1]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "id": "aa080ae2",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>0</th>\n",
       "      <th>1</th>\n",
       "      <th>2</th>\n",
       "      <th>3</th>\n",
       "      <th>4</th>\n",
       "      <th>5</th>\n",
       "      <th>6</th>\n",
       "      <th>7</th>\n",
       "      <th>8</th>\n",
       "      <th>9</th>\n",
       "      <th>...</th>\n",
       "      <th>1017</th>\n",
       "      <th>1018</th>\n",
       "      <th>1019</th>\n",
       "      <th>1020</th>\n",
       "      <th>1021</th>\n",
       "      <th>1022</th>\n",
       "      <th>1023</th>\n",
       "      <th>0</th>\n",
       "      <th>1</th>\n",
       "      <th>2</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>201601</th>\n",
       "      <td>0.0</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.0</td>\n",
       "      <td>8.161131</td>\n",
       "      <td>0.0</td>\n",
       "      <td>10.415243</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>...</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>7.435628</td>\n",
       "      <td>12.417972</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>201602</th>\n",
       "      <td>0.0</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.0</td>\n",
       "      <td>8.161131</td>\n",
       "      <td>0.0</td>\n",
       "      <td>10.415243</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>...</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>7.435628</td>\n",
       "      <td>12.417972</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>201603</th>\n",
       "      <td>0.0</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>17.168774</td>\n",
       "      <td>0.0</td>\n",
       "      <td>43.288970</td>\n",
       "      <td>0.0</td>\n",
       "      <td>43.422768</td>\n",
       "      <td>15.177656</td>\n",
       "      <td>...</td>\n",
       "      <td>21.384554</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.990702</td>\n",
       "      <td>9.009418</td>\n",
       "      <td>26.557423</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>201604</th>\n",
       "      <td>0.0</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.034701</td>\n",
       "      <td>0.0</td>\n",
       "      <td>7.108294</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>...</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>2.045272</td>\n",
       "      <td>12.347998</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>201605</th>\n",
       "      <td>0.0</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>...</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>15.193535</td>\n",
       "      <td>6.474056</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>201848</th>\n",
       "      <td>0.0</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.0</td>\n",
       "      <td>21.014330</td>\n",
       "      <td>0.0</td>\n",
       "      <td>20.102703</td>\n",
       "      <td>0.469093</td>\n",
       "      <td>...</td>\n",
       "      <td>4.917190</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>5.318927</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>201849</th>\n",
       "      <td>0.0</td>\n",
       "      <td>16.103472</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>...</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>21.635908</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>201850</th>\n",
       "      <td>0.0</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>...</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>22.293388</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>201851</th>\n",
       "      <td>0.0</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>...</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>27.029966</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>201852</th>\n",
       "      <td>0.0</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>7.444656</td>\n",
       "      <td>0.0</td>\n",
       "      <td>37.199368</td>\n",
       "      <td>0.0</td>\n",
       "      <td>35.591797</td>\n",
       "      <td>10.594391</td>\n",
       "      <td>...</td>\n",
       "      <td>5.351820</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>12.584043</td>\n",
       "      <td>14.189192</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>156 rows × 1027 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "          0          1    2    3          4    5          6    7          8  \\\n",
       "201601  0.0   0.000000  0.0  0.0   0.000000  0.0   8.161131  0.0  10.415243   \n",
       "201602  0.0   0.000000  0.0  0.0   0.000000  0.0   8.161131  0.0  10.415243   \n",
       "201603  0.0   0.000000  0.0  0.0  17.168774  0.0  43.288970  0.0  43.422768   \n",
       "201604  0.0   0.000000  0.0  0.0   0.000000  0.0   1.034701  0.0   7.108294   \n",
       "201605  0.0   0.000000  0.0  0.0   0.000000  0.0   0.000000  0.0   0.000000   \n",
       "...     ...        ...  ...  ...        ...  ...        ...  ...        ...   \n",
       "201848  0.0   0.000000  0.0  0.0   0.000000  0.0  21.014330  0.0  20.102703   \n",
       "201849  0.0  16.103472  0.0  0.0   0.000000  0.0   0.000000  0.0   0.000000   \n",
       "201850  0.0   0.000000  0.0  0.0   0.000000  0.0   0.000000  0.0   0.000000   \n",
       "201851  0.0   0.000000  0.0  0.0   0.000000  0.0   0.000000  0.0   0.000000   \n",
       "201852  0.0   0.000000  0.0  0.0   7.444656  0.0  37.199368  0.0  35.591797   \n",
       "\n",
       "                9  ...       1017  1018  1019       1020       1021  \\\n",
       "201601   0.000000  ...   0.000000   0.0   0.0   7.435628  12.417972   \n",
       "201602   0.000000  ...   0.000000   0.0   0.0   7.435628  12.417972   \n",
       "201603  15.177656  ...  21.384554   0.0   0.0   1.990702   9.009418   \n",
       "201604   0.000000  ...   0.000000   0.0   0.0   2.045272  12.347998   \n",
       "201605   0.000000  ...   0.000000   0.0   0.0  15.193535   6.474056   \n",
       "...           ...  ...        ...   ...   ...        ...        ...   \n",
       "201848   0.469093  ...   4.917190   0.0   0.0   0.000000   0.000000   \n",
       "201849   0.000000  ...   0.000000   0.0   0.0   0.000000  21.635908   \n",
       "201850   0.000000  ...   0.000000   0.0   0.0   0.000000  22.293388   \n",
       "201851   0.000000  ...   0.000000   0.0   0.0   0.000000  27.029966   \n",
       "201852  10.594391  ...   5.351820   0.0   0.0   0.000000  12.584043   \n",
       "\n",
       "             1022  1023  0  1  2  \n",
       "201601   0.000000   0.0  1  0  0  \n",
       "201602   0.000000   0.0  0  1  0  \n",
       "201603  26.557423   0.0  0  0  1  \n",
       "201604   0.000000   0.0  1  0  0  \n",
       "201605   0.000000   0.0  1  0  0  \n",
       "...           ...   ... .. .. ..  \n",
       "201848   5.318927   0.0  1  0  0  \n",
       "201849   0.000000   0.0  0  0  1  \n",
       "201850   0.000000   0.0  0  1  0  \n",
       "201851   0.000000   0.0  1  0  0  \n",
       "201852  14.189192   0.0  1  0  0  \n",
       "\n",
       "[156 rows x 1027 columns]"
      ]
     },
     "execution_count": 12,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Merge the two dataframes based on the date values\n",
    "dengue_df = features_df.merge(labels_df, how='inner', left_index=True, right_index=True)\n",
    "dengue_df"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "f5694381",
   "metadata": {},
   "source": [
    "### Train Test split"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "id": "83cce148",
   "metadata": {},
   "outputs": [],
   "source": [
    "def train_test_split(df, train_percentage = 80):\n",
    "    # We need a sequence so we can't split randomly\n",
    "    # To divide into Train and test we have to calculate the train percentage of the dataset:\n",
    "    size = df.shape[0]\n",
    "    split = int(size*(train_percentage/100))\n",
    "    \n",
    "    \"\"\" Train \"\"\"\n",
    "    # We will train with 1st percentage % of data and test with the rest\n",
    "    train_df = df.iloc[:split,:] ## percentage % train\n",
    "    \n",
    "    \"\"\" Test \"\"\"\n",
    "    test_df = df.iloc[split:,:] # 100 - percentage % test\n",
    "    \n",
    "    print(f'The train shape is: {train_df.shape}')\n",
    "    print(f'The test shape is: {test_df.shape}')\n",
    "    \n",
    "    return train_df, test_df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "id": "374059b0",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "The train shape is: (124, 1027)\n",
      "The test shape is: (32, 1027)\n"
     ]
    }
   ],
   "source": [
    "train_df, test_df = train_test_split(dengue_df, train_percentage = 80)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "11fdca19",
   "metadata": {},
   "source": [
    "### Normalize features"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "id": "0e81b163",
   "metadata": {},
   "outputs": [],
   "source": [
    "# Normalize train data and create the scaler\n",
    "def normalize_train_features(df, feature_range=(-1, 1), n_labels=None):\n",
    "    \n",
    "    if n_labels:\n",
    "        n_features = df.shape[1] - n_labels\n",
    "    \n",
    "    scalers = {}\n",
    "    # For each column in the dataframe\n",
    "    for i, column in enumerate(df.columns):\n",
    "        if n_labels:\n",
    "            if i >= n_features:\n",
    "                break\n",
    "        # Get values of the column\n",
    "        values = df[column].values.reshape(-1,1)\n",
    "        # Generate a new scaler\n",
    "        scaler = MinMaxScaler(feature_range=feature_range)\n",
    "        # Fit the scaler just for that column\n",
    "        scaled_column = scaler.fit_transform(values)\n",
    "        # Add the scaled column to the dataframe\n",
    "        scaled_column = np.reshape(scaled_column, len(scaled_column))\n",
    "        df[column] = scaled_column\n",
    "        \n",
    "        # Save the scaler of the column\n",
    "        scalers['scaler_' + column] = scaler\n",
    "        \n",
    "    print(f' Min values are: ')\n",
    "    print(df.min())\n",
    "    print(f' Max values are: ')\n",
    "    print(df.max())\n",
    "        \n",
    "    return df, scalers\n",
    "\n",
    "\n",
    "\"\"\" If you want to use the same scaler used in train, you can use this function\"\"\"\n",
    "def normalize_test_features(df, scalers=None, n_labels=None):\n",
    "    \n",
    "    if not scalers:\n",
    "        raise TypeError(\"You should provide a list of scalers.\")\n",
    "    \n",
    "    if n_labels:\n",
    "        n_features = df.shape[1] - n_labels\n",
    "    \n",
    "    for i, column in enumerate(df.columns):\n",
    "        if n_labels:\n",
    "            if i >= n_features:\n",
    "                break\n",
    "        # Get values of the column\n",
    "        values = df[column].values.reshape(-1,1)\n",
    "        # Take the scaler of that column\n",
    "        scaler = scalers['scaler_' + column]\n",
    "        # Scale values\n",
    "        scaled_column = scaler.transform(values)\n",
    "        scaled_column = np.reshape(scaled_column,len(scaled_column))\n",
    "        # Add the scaled values to the df\n",
    "        df[column] = scaled_column\n",
    "        \n",
    "    print(f' Min values are: ')\n",
    "    print(df.min())\n",
    "    print(f' Max values are: ')\n",
    "    print(df.max())\n",
    "        \n",
    "    return df "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "id": "e35beceb",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      " Min values are: \n",
      "0      -1.0\n",
      "1      -1.0\n",
      "2      -1.0\n",
      "3      -1.0\n",
      "4      -1.0\n",
      "       ... \n",
      "1022   -1.0\n",
      "1023   -1.0\n",
      "0       0.0\n",
      "1       0.0\n",
      "2       0.0\n",
      "Length: 1027, dtype: float64\n",
      " Max values are: \n",
      "0      -1.0\n",
      "1       1.0\n",
      "2      -1.0\n",
      "3      -1.0\n",
      "4       1.0\n",
      "       ... \n",
      "1022    1.0\n",
      "1023   -1.0\n",
      "0       1.0\n",
      "1       1.0\n",
      "2       1.0\n",
      "Length: 1027, dtype: float64\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>0</th>\n",
       "      <th>1</th>\n",
       "      <th>2</th>\n",
       "      <th>3</th>\n",
       "      <th>4</th>\n",
       "      <th>5</th>\n",
       "      <th>6</th>\n",
       "      <th>7</th>\n",
       "      <th>8</th>\n",
       "      <th>9</th>\n",
       "      <th>...</th>\n",
       "      <th>1017</th>\n",
       "      <th>1018</th>\n",
       "      <th>1019</th>\n",
       "      <th>1020</th>\n",
       "      <th>1021</th>\n",
       "      <th>1022</th>\n",
       "      <th>1023</th>\n",
       "      <th>0</th>\n",
       "      <th>1</th>\n",
       "      <th>2</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>201601</th>\n",
       "      <td>-1.0</td>\n",
       "      <td>-1.0</td>\n",
       "      <td>-1.0</td>\n",
       "      <td>-1.0</td>\n",
       "      <td>-1.000000</td>\n",
       "      <td>-1.0</td>\n",
       "      <td>-0.870525</td>\n",
       "      <td>-1.0</td>\n",
       "      <td>-0.814017</td>\n",
       "      <td>-1.000000</td>\n",
       "      <td>...</td>\n",
       "      <td>-1.000000</td>\n",
       "      <td>-1.0</td>\n",
       "      <td>-1.0</td>\n",
       "      <td>-0.849838</td>\n",
       "      <td>-0.843321</td>\n",
       "      <td>-1.000000</td>\n",
       "      <td>-1.0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>201602</th>\n",
       "      <td>-1.0</td>\n",
       "      <td>-1.0</td>\n",
       "      <td>-1.0</td>\n",
       "      <td>-1.0</td>\n",
       "      <td>-1.000000</td>\n",
       "      <td>-1.0</td>\n",
       "      <td>-0.870525</td>\n",
       "      <td>-1.0</td>\n",
       "      <td>-0.814017</td>\n",
       "      <td>-1.000000</td>\n",
       "      <td>...</td>\n",
       "      <td>-1.000000</td>\n",
       "      <td>-1.0</td>\n",
       "      <td>-1.0</td>\n",
       "      <td>-0.849838</td>\n",
       "      <td>-0.843321</td>\n",
       "      <td>-1.000000</td>\n",
       "      <td>-1.0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>201603</th>\n",
       "      <td>-1.0</td>\n",
       "      <td>-1.0</td>\n",
       "      <td>-1.0</td>\n",
       "      <td>-1.0</td>\n",
       "      <td>-0.518855</td>\n",
       "      <td>-1.0</td>\n",
       "      <td>-0.313230</td>\n",
       "      <td>-1.0</td>\n",
       "      <td>-0.224607</td>\n",
       "      <td>-0.379313</td>\n",
       "      <td>...</td>\n",
       "      <td>-0.398361</td>\n",
       "      <td>-1.0</td>\n",
       "      <td>-1.0</td>\n",
       "      <td>-0.959798</td>\n",
       "      <td>-0.886327</td>\n",
       "      <td>-0.281606</td>\n",
       "      <td>-1.0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>201604</th>\n",
       "      <td>-1.0</td>\n",
       "      <td>-1.0</td>\n",
       "      <td>-1.0</td>\n",
       "      <td>-1.0</td>\n",
       "      <td>-1.000000</td>\n",
       "      <td>-1.0</td>\n",
       "      <td>-0.983585</td>\n",
       "      <td>-1.0</td>\n",
       "      <td>-0.873068</td>\n",
       "      <td>-1.000000</td>\n",
       "      <td>...</td>\n",
       "      <td>-1.000000</td>\n",
       "      <td>-1.0</td>\n",
       "      <td>-1.0</td>\n",
       "      <td>-0.958696</td>\n",
       "      <td>-0.844204</td>\n",
       "      <td>-1.000000</td>\n",
       "      <td>-1.0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>201605</th>\n",
       "      <td>-1.0</td>\n",
       "      <td>-1.0</td>\n",
       "      <td>-1.0</td>\n",
       "      <td>-1.0</td>\n",
       "      <td>-1.000000</td>\n",
       "      <td>-1.0</td>\n",
       "      <td>-1.000000</td>\n",
       "      <td>-1.0</td>\n",
       "      <td>-1.000000</td>\n",
       "      <td>-1.000000</td>\n",
       "      <td>...</td>\n",
       "      <td>-1.000000</td>\n",
       "      <td>-1.0</td>\n",
       "      <td>-1.0</td>\n",
       "      <td>-0.693167</td>\n",
       "      <td>-0.918316</td>\n",
       "      <td>-1.000000</td>\n",
       "      <td>-1.0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>5 rows × 1027 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "          0    1    2    3         4    5         6    7         8         9  \\\n",
       "201601 -1.0 -1.0 -1.0 -1.0 -1.000000 -1.0 -0.870525 -1.0 -0.814017 -1.000000   \n",
       "201602 -1.0 -1.0 -1.0 -1.0 -1.000000 -1.0 -0.870525 -1.0 -0.814017 -1.000000   \n",
       "201603 -1.0 -1.0 -1.0 -1.0 -0.518855 -1.0 -0.313230 -1.0 -0.224607 -0.379313   \n",
       "201604 -1.0 -1.0 -1.0 -1.0 -1.000000 -1.0 -0.983585 -1.0 -0.873068 -1.000000   \n",
       "201605 -1.0 -1.0 -1.0 -1.0 -1.000000 -1.0 -1.000000 -1.0 -1.000000 -1.000000   \n",
       "\n",
       "        ...      1017  1018  1019      1020      1021      1022  1023  0  1  2  \n",
       "201601  ... -1.000000  -1.0  -1.0 -0.849838 -0.843321 -1.000000  -1.0  1  0  0  \n",
       "201602  ... -1.000000  -1.0  -1.0 -0.849838 -0.843321 -1.000000  -1.0  0  1  0  \n",
       "201603  ... -0.398361  -1.0  -1.0 -0.959798 -0.886327 -0.281606  -1.0  0  0  1  \n",
       "201604  ... -1.000000  -1.0  -1.0 -0.958696 -0.844204 -1.000000  -1.0  1  0  0  \n",
       "201605  ... -1.000000  -1.0  -1.0 -0.693167 -0.918316 -1.000000  -1.0  1  0  0  \n",
       "\n",
       "[5 rows x 1027 columns]"
      ]
     },
     "execution_count": 16,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "feature_range = (-1, 1)\n",
    "\n",
    "# Scale train:\n",
    "train_df, scalers = normalize_train_features(train_df, feature_range=feature_range, n_labels=n_labels)\n",
    "\n",
    "#print(f'The scalers are: {scalers}')\n",
    "\n",
    "train_df.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "id": "7161c9c2",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      " Min values are: \n",
      "0      -1.0\n",
      "1      -1.0\n",
      "2      -1.0\n",
      "3      -1.0\n",
      "4      -1.0\n",
      "       ... \n",
      "1022   -1.0\n",
      "1023   -1.0\n",
      "0       0.0\n",
      "1       0.0\n",
      "2       0.0\n",
      "Length: 1027, dtype: float64\n",
      " Max values are: \n",
      "0      -1.000000\n",
      "1      -0.012869\n",
      "2      -1.000000\n",
      "3      -1.000000\n",
      "4      -0.053137\n",
      "          ...   \n",
      "1022    0.305472\n",
      "1023   -1.000000\n",
      "0       1.000000\n",
      "1       1.000000\n",
      "2       1.000000\n",
      "Length: 1027, dtype: float64\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>0</th>\n",
       "      <th>1</th>\n",
       "      <th>2</th>\n",
       "      <th>3</th>\n",
       "      <th>4</th>\n",
       "      <th>5</th>\n",
       "      <th>6</th>\n",
       "      <th>7</th>\n",
       "      <th>8</th>\n",
       "      <th>9</th>\n",
       "      <th>...</th>\n",
       "      <th>1017</th>\n",
       "      <th>1018</th>\n",
       "      <th>1019</th>\n",
       "      <th>1020</th>\n",
       "      <th>1021</th>\n",
       "      <th>1022</th>\n",
       "      <th>1023</th>\n",
       "      <th>0</th>\n",
       "      <th>1</th>\n",
       "      <th>2</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>201821</th>\n",
       "      <td>-1.0</td>\n",
       "      <td>-1.0</td>\n",
       "      <td>-1.0</td>\n",
       "      <td>-1.0</td>\n",
       "      <td>-0.658391</td>\n",
       "      <td>-1.0</td>\n",
       "      <td>-0.300150</td>\n",
       "      <td>-1.0</td>\n",
       "      <td>-0.298219</td>\n",
       "      <td>-0.342267</td>\n",
       "      <td>...</td>\n",
       "      <td>-0.498781</td>\n",
       "      <td>-1.0</td>\n",
       "      <td>-1.0</td>\n",
       "      <td>-1.000000</td>\n",
       "      <td>-0.861126</td>\n",
       "      <td>-0.400018</td>\n",
       "      <td>-1.0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>201822</th>\n",
       "      <td>-1.0</td>\n",
       "      <td>-1.0</td>\n",
       "      <td>-1.0</td>\n",
       "      <td>-1.0</td>\n",
       "      <td>-1.000000</td>\n",
       "      <td>-1.0</td>\n",
       "      <td>-1.000000</td>\n",
       "      <td>-1.0</td>\n",
       "      <td>-1.000000</td>\n",
       "      <td>-1.000000</td>\n",
       "      <td>...</td>\n",
       "      <td>-1.000000</td>\n",
       "      <td>-1.0</td>\n",
       "      <td>-1.0</td>\n",
       "      <td>-1.000000</td>\n",
       "      <td>-0.971721</td>\n",
       "      <td>-1.000000</td>\n",
       "      <td>-1.0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>201823</th>\n",
       "      <td>-1.0</td>\n",
       "      <td>-1.0</td>\n",
       "      <td>-1.0</td>\n",
       "      <td>-1.0</td>\n",
       "      <td>-1.000000</td>\n",
       "      <td>-1.0</td>\n",
       "      <td>-1.000000</td>\n",
       "      <td>-1.0</td>\n",
       "      <td>-1.000000</td>\n",
       "      <td>-1.000000</td>\n",
       "      <td>...</td>\n",
       "      <td>-1.000000</td>\n",
       "      <td>-1.0</td>\n",
       "      <td>-1.0</td>\n",
       "      <td>-1.000000</td>\n",
       "      <td>-0.659987</td>\n",
       "      <td>-1.000000</td>\n",
       "      <td>-1.0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>201824</th>\n",
       "      <td>-1.0</td>\n",
       "      <td>-1.0</td>\n",
       "      <td>-1.0</td>\n",
       "      <td>-1.0</td>\n",
       "      <td>-1.000000</td>\n",
       "      <td>-1.0</td>\n",
       "      <td>-1.000000</td>\n",
       "      <td>-1.0</td>\n",
       "      <td>-1.000000</td>\n",
       "      <td>-1.000000</td>\n",
       "      <td>...</td>\n",
       "      <td>-1.000000</td>\n",
       "      <td>-1.0</td>\n",
       "      <td>-1.0</td>\n",
       "      <td>-1.000000</td>\n",
       "      <td>-1.000000</td>\n",
       "      <td>-1.000000</td>\n",
       "      <td>-1.0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>201825</th>\n",
       "      <td>-1.0</td>\n",
       "      <td>-1.0</td>\n",
       "      <td>-1.0</td>\n",
       "      <td>-1.0</td>\n",
       "      <td>-0.268753</td>\n",
       "      <td>-1.0</td>\n",
       "      <td>-0.075702</td>\n",
       "      <td>-1.0</td>\n",
       "      <td>-0.065823</td>\n",
       "      <td>0.093075</td>\n",
       "      <td>...</td>\n",
       "      <td>-0.416461</td>\n",
       "      <td>-1.0</td>\n",
       "      <td>-1.0</td>\n",
       "      <td>-0.930679</td>\n",
       "      <td>-1.000000</td>\n",
       "      <td>-0.030467</td>\n",
       "      <td>-1.0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>5 rows × 1027 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "          0    1    2    3         4    5         6    7         8         9  \\\n",
       "201821 -1.0 -1.0 -1.0 -1.0 -0.658391 -1.0 -0.300150 -1.0 -0.298219 -0.342267   \n",
       "201822 -1.0 -1.0 -1.0 -1.0 -1.000000 -1.0 -1.000000 -1.0 -1.000000 -1.000000   \n",
       "201823 -1.0 -1.0 -1.0 -1.0 -1.000000 -1.0 -1.000000 -1.0 -1.000000 -1.000000   \n",
       "201824 -1.0 -1.0 -1.0 -1.0 -1.000000 -1.0 -1.000000 -1.0 -1.000000 -1.000000   \n",
       "201825 -1.0 -1.0 -1.0 -1.0 -0.268753 -1.0 -0.075702 -1.0 -0.065823  0.093075   \n",
       "\n",
       "        ...      1017  1018  1019      1020      1021      1022  1023  0  1  2  \n",
       "201821  ... -0.498781  -1.0  -1.0 -1.000000 -0.861126 -0.400018  -1.0  1  0  0  \n",
       "201822  ... -1.000000  -1.0  -1.0 -1.000000 -0.971721 -1.000000  -1.0  1  0  0  \n",
       "201823  ... -1.000000  -1.0  -1.0 -1.000000 -0.659987 -1.000000  -1.0  1  0  0  \n",
       "201824  ... -1.000000  -1.0  -1.0 -1.000000 -1.000000 -1.000000  -1.0  1  0  0  \n",
       "201825  ... -0.416461  -1.0  -1.0 -0.930679 -1.000000 -0.030467  -1.0  1  0  0  \n",
       "\n",
       "[5 rows x 1027 columns]"
      ]
     },
     "execution_count": 17,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Scale test:\n",
    "test_df = normalize_test_features(test_df, scalers=scalers, n_labels=n_labels)\n",
    "test_df.head()"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "c7b69d05",
   "metadata": {},
   "source": [
    "### Features and Labels Set"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "id": "83c65b6b",
   "metadata": {},
   "outputs": [],
   "source": [
    "def features_labels_set(timeseries_data, original_df, n_labels):\n",
    "    \n",
    "    \"\"\" Features \"\"\"\n",
    "    # We define the number of features as (features and labels)\n",
    "    n_features = original_df.shape[1]\n",
    "\n",
    "    # The features to train the model will be all except the values of the actual week \n",
    "    # We can't use other variables in week t because whe need to resample a a 3D Array\n",
    "    features_set = DataFrame(timeseries_data.values[:,:-n_labels])\n",
    "    # Convert pandas data frame to np.array to reshape as 3D Array\n",
    "    features_set = features_set.to_numpy()\n",
    "    print(f'The shape of the features is {features_set.shape}')\n",
    "    \n",
    "    \"\"\" Labels \"\"\"\n",
    "    # We will use labels in last week \n",
    "    labels_set = DataFrame(timeseries_data.values[:,-n_labels:])\n",
    "    # Convert pandas data frame to np.array\n",
    "    labels_set = labels_set.to_numpy()\n",
    "    print(f'The shape of the labels is {labels_set.shape}')\n",
    "    \n",
    "    return features_set, labels_set, n_features"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "id": "d740621e",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Train:\n",
      "The shape of the features is (124, 1024)\n",
      "The shape of the labels is (124, 3)\n",
      "Test:\n",
      "The shape of the features is (32, 1024)\n",
      "The shape of the labels is (32, 3)\n"
     ]
    }
   ],
   "source": [
    "# Train features and labels set\n",
    "print('Train:')\n",
    "train_X, train_y, n_features = features_labels_set(timeseries_data=train_df, original_df=dengue_df, n_labels=n_labels)\n",
    "\n",
    "# Test features and labels set\n",
    "print('Test:')\n",
    "test_X, test_y, n_features = features_labels_set(timeseries_data=test_df, original_df=dengue_df, n_labels=n_labels)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "3564b9e0",
   "metadata": {},
   "source": [
    "# Modeling"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "7fcc4426",
   "metadata": {},
   "source": [
    "# Define the Model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "id": "eafbffbe",
   "metadata": {},
   "outputs": [],
   "source": [
    "# Set Seed\n",
    "#tf.random.set_seed(0)\n",
    "\n",
    "def create_model():\n",
    "    \n",
    "    # design network\n",
    "    model = Sequential()\n",
    "    \n",
    "    # Choose an optimal value between 32-512\n",
    "    #hp_units = hp.Int('units', min_value=512, max_value=1024, step=64)\n",
    "    model.add(Dense(units=1024))\n",
    "    \n",
    "    # Choose an optimal value between 32-512\n",
    "    #hp_units_2 = hp.Int('units', min_value=32, max_value=512, step=32)\n",
    "    model.add(Dense(units=512))\n",
    "    \n",
    "    # Out\n",
    "    model.add(Dense(3, activation='softmax'))\n",
    "    \n",
    "    # Compile the model:\n",
    "    \n",
    "    # Choose an optimal value from 0.01, 0.001, or 0.0001\n",
    "    #hp_learning_rate = hp.Choice('learning_rate', values=[1e-2, 1e-3, 1e-4])\n",
    "    opt = keras.optimizers.Adam()\n",
    "    metrics = [\n",
    "        tf.keras.metrics.AUC(name='auc', multi_label=False), #num_labels=3),\n",
    "        tf.keras.metrics.CategoricalAccuracy(name='acc'),\n",
    "        tf.keras.metrics.CategoricalCrossentropy(name='entropy')\n",
    "    ]\n",
    "    \n",
    "    model.compile(loss='categorical_crossentropy', optimizer=opt, metrics=metrics)\n",
    "\n",
    "    return model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "id": "737fcc82",
   "metadata": {},
   "outputs": [],
   "source": [
    "def tune_model():\n",
    "    tuner = keras_tuner.RandomSearch(\n",
    "        create_model,\n",
    "        objective='val_loss',\n",
    "        max_trials=15\n",
    "    )\n",
    "    \n",
    "    tuner.search(\n",
    "        train_X,\n",
    "        train_y,\n",
    "        epochs=1,\n",
    "        batch_size=16,\n",
    "        validation_data=(test_X, test_y)\n",
    "    )\n",
    "\n",
    "    return tuner"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "ea682cb3",
   "metadata": {},
   "source": [
    "### Train the model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "id": "e7ea54da",
   "metadata": {},
   "outputs": [],
   "source": [
    "from tensorflow.keras.callbacks import EarlyStopping\n",
    "\n",
    "# EarlyStopping:\n",
    "monitor = EarlyStopping(monitor='val_loss', min_delta=1e-3, patience=20, \n",
    "        verbose=1, mode='auto', restore_best_weights=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "id": "7b93c7c3",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "zeros: 113, ones: 20, twos: 23, total: 156\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "{0: 1.3805309734513274, 1: 7.8, 2: 6.782608695652174}"
      ]
     },
     "execution_count": 23,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "### Imbalanced data\n",
    "n_zeros = (labels_df_orig.to_numpy() == 0).sum()\n",
    "n_ones = (labels_df_orig.to_numpy() == 1).sum()\n",
    "n_twos = (labels_df_orig.to_numpy() == 2).sum()\n",
    "n_total = n_zeros + n_ones + n_twos\n",
    "\n",
    "weights = {0: n_total/n_zeros, 1: n_total/n_ones, 2: n_total/n_twos}\n",
    "print(f'zeros: {n_zeros}, ones: {n_ones}, twos: {n_twos}, total: {n_total}')\n",
    "weights"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "id": "49f54a92",
   "metadata": {},
   "outputs": [],
   "source": [
    "# fit network\n",
    "def train_model(model, monitor, weights, plot=None, epochs=100):\n",
    "    if monitor and weights:\n",
    "        history = model.fit(train_X, train_y, epochs=epochs, batch_size=16, validation_data=(test_X, test_y), verbose=2, shuffle=False, callbacks=[monitor], class_weight=weights)\n",
    "    elif monitor:\n",
    "        history = model.fit(train_X, train_y, epochs=epochs, batch_size=16, validation_data=(test_X, test_y), verbose=2, shuffle=False, callbacks=[monitor])\n",
    "    elif weights:\n",
    "        history = model.fit(train_X, train_y, epochs=epochs, batch_size=16, validation_data=(test_X, test_y), verbose=2, shuffle=False, class_weight=weights)\n",
    "    else:\n",
    "        history = model.fit(train_X, train_y, epochs=epochs, batch_size=16, validation_data=(test_X, test_y), verbose=2, shuffle=False)\n",
    "    \n",
    "    if plot:\n",
    "        # plot history\n",
    "        plt.plot(history.history['loss'], label='train')\n",
    "        plt.plot(history.history['val_loss'], label='validation')\n",
    "        plt.legend()\n",
    "        plt.show()\n",
    "        "
   ]
  },
  {
   "cell_type": "markdown",
   "id": "86494c93",
   "metadata": {},
   "source": [
    "# Test the model"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "fdf8f5c9",
   "metadata": {},
   "source": [
    "# AUC"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "id": "8eb1c027",
   "metadata": {},
   "outputs": [],
   "source": [
    "# You can also evaluate or predict on a dataset.\n",
    "def evaluate(model, verbose = None):\n",
    "    if verbose:\n",
    "        print('Evaluate: ')\n",
    "    result = model.evaluate(test_X, test_y)\n",
    "    stored_results = {}\n",
    "    for i, metric in enumerate(model.metrics_names):\n",
    "        stored_results[metric] = result[i]\n",
    "        if verbose:\n",
    "            print(f'{metric}: {result[i]}')\n",
    "    return stored_results"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "feb3098a",
   "metadata": {},
   "source": [
    "# Calculate Mean and SD"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "id": "b828015f",
   "metadata": {},
   "outputs": [],
   "source": [
    "def calculate_mean_std(weights):\n",
    "    \n",
    "    metrics = {\n",
    "    \"auc\": [],\n",
    "    \"acc\": [],\n",
    "    \"entropy\": []\n",
    "    }\n",
    "\n",
    "    for i in range(5):\n",
    "        model = create_model()\n",
    "        train_model(model=model, monitor=monitor, weights=weights)\n",
    "        stored_results = evaluate(model=model)\n",
    "        \n",
    "        for key in metrics.keys():\n",
    "            metrics[key].append(stored_results[key])\n",
    "            \n",
    "    for key in metrics.keys():\n",
    "        results = metrics[key]\n",
    "        print(key, f\": average={np.average(results):.3f}, std={np.std(results):.3f}\")\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "id": "b356fdf4",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 1/100\n",
      "8/8 - 1s - loss: 19.3899 - auc: 0.5508 - acc: 0.3710 - entropy: 19.3899 - val_loss: 7.4349 - val_auc: 0.9297 - val_acc: 0.9062 - val_entropy: 7.4349\n",
      "Epoch 2/100\n",
      "8/8 - 0s - loss: 21.9682 - auc: 0.4850 - acc: 0.3710 - entropy: 21.9682 - val_loss: 3.6793 - val_auc: 0.9297 - val_acc: 0.9062 - val_entropy: 3.6793\n",
      "Epoch 3/100\n",
      "8/8 - 0s - loss: 10.4733 - auc: 0.6881 - acc: 0.5161 - entropy: 10.4733 - val_loss: 0.6420 - val_auc: 0.9387 - val_acc: 0.8750 - val_entropy: 0.6420\n",
      "Epoch 4/100\n",
      "8/8 - 0s - loss: 2.3921 - auc: 0.7110 - acc: 0.5161 - entropy: 2.3921 - val_loss: 1.3573 - val_auc: 0.9297 - val_acc: 0.9062 - val_entropy: 1.3573\n",
      "Epoch 5/100\n",
      "8/8 - 0s - loss: 5.9310 - auc: 0.7567 - acc: 0.5403 - entropy: 5.9310 - val_loss: 2.2851 - val_auc: 0.9080 - val_acc: 0.9062 - val_entropy: 2.2851\n",
      "Epoch 6/100\n",
      "8/8 - 0s - loss: 5.1703 - auc: 0.7890 - acc: 0.6371 - entropy: 5.1703 - val_loss: 0.8890 - val_auc: 0.9253 - val_acc: 0.9062 - val_entropy: 0.8890\n",
      "Epoch 7/100\n",
      "8/8 - 0s - loss: 2.4207 - auc: 0.5709 - acc: 0.3952 - entropy: 2.4207 - val_loss: 0.8577 - val_auc: 0.9058 - val_acc: 0.9062 - val_entropy: 0.8577\n",
      "Epoch 8/100\n",
      "8/8 - 0s - loss: 2.0198 - auc: 0.7335 - acc: 0.6694 - entropy: 2.0198 - val_loss: 0.4752 - val_auc: 0.9041 - val_acc: 0.9062 - val_entropy: 0.4752\n",
      "Epoch 9/100\n",
      "8/8 - 0s - loss: 1.0991 - auc: 0.7240 - acc: 0.5645 - entropy: 1.0991 - val_loss: 0.5725 - val_auc: 0.9006 - val_acc: 0.9062 - val_entropy: 0.5725\n",
      "Epoch 10/100\n",
      "8/8 - 0s - loss: 1.3270 - auc: 0.7380 - acc: 0.6371 - entropy: 1.3270 - val_loss: 0.4700 - val_auc: 0.9041 - val_acc: 0.9062 - val_entropy: 0.4700\n",
      "Epoch 11/100\n",
      "8/8 - 0s - loss: 1.0214 - auc: 0.7237 - acc: 0.5565 - entropy: 1.0214 - val_loss: 0.4785 - val_auc: 0.9119 - val_acc: 0.9062 - val_entropy: 0.4785\n",
      "Epoch 12/100\n",
      "8/8 - 0s - loss: 0.8183 - auc: 0.8078 - acc: 0.6855 - entropy: 0.8183 - val_loss: 0.5024 - val_auc: 0.9055 - val_acc: 0.9062 - val_entropy: 0.5024\n",
      "Epoch 13/100\n",
      "8/8 - 0s - loss: 0.8562 - auc: 0.7826 - acc: 0.6774 - entropy: 0.8562 - val_loss: 0.4973 - val_auc: 0.9084 - val_acc: 0.9062 - val_entropy: 0.4973\n",
      "Epoch 14/100\n",
      "8/8 - 0s - loss: 0.8181 - auc: 0.8093 - acc: 0.6694 - entropy: 0.8181 - val_loss: 0.4965 - val_auc: 0.9077 - val_acc: 0.9062 - val_entropy: 0.4965\n",
      "Epoch 15/100\n",
      "8/8 - 0s - loss: 0.7892 - auc: 0.8276 - acc: 0.6855 - entropy: 0.7892 - val_loss: 0.4819 - val_auc: 0.9041 - val_acc: 0.9062 - val_entropy: 0.4819\n",
      "Epoch 16/100\n",
      "8/8 - 0s - loss: 0.8178 - auc: 0.8062 - acc: 0.6855 - entropy: 0.8178 - val_loss: 0.5069 - val_auc: 0.9097 - val_acc: 0.9062 - val_entropy: 0.5069\n",
      "Epoch 17/100\n",
      "8/8 - 0s - loss: 0.7872 - auc: 0.8296 - acc: 0.6855 - entropy: 0.7872 - val_loss: 0.4965 - val_auc: 0.9075 - val_acc: 0.9062 - val_entropy: 0.4965\n",
      "Epoch 18/100\n",
      "8/8 - 0s - loss: 0.7900 - auc: 0.8232 - acc: 0.6855 - entropy: 0.7900 - val_loss: 0.4995 - val_auc: 0.9080 - val_acc: 0.8750 - val_entropy: 0.4995\n",
      "Epoch 19/100\n",
      "8/8 - 0s - loss: 0.7937 - auc: 0.8211 - acc: 0.6855 - entropy: 0.7937 - val_loss: 0.5153 - val_auc: 0.9067 - val_acc: 0.8750 - val_entropy: 0.5153\n",
      "Epoch 20/100\n",
      "8/8 - 0s - loss: 0.7754 - auc: 0.8338 - acc: 0.6855 - entropy: 0.7754 - val_loss: 0.5094 - val_auc: 0.9065 - val_acc: 0.8750 - val_entropy: 0.5094\n",
      "Epoch 21/100\n",
      "8/8 - 0s - loss: 0.7855 - auc: 0.8253 - acc: 0.6774 - entropy: 0.7855 - val_loss: 0.5240 - val_auc: 0.9077 - val_acc: 0.8750 - val_entropy: 0.5240\n",
      "Epoch 22/100\n",
      "8/8 - 0s - loss: 0.7738 - auc: 0.8339 - acc: 0.6774 - entropy: 0.7738 - val_loss: 0.5274 - val_auc: 0.9070 - val_acc: 0.8750 - val_entropy: 0.5274\n",
      "Epoch 23/100\n",
      "8/8 - 0s - loss: 0.7792 - auc: 0.8303 - acc: 0.6774 - entropy: 0.7792 - val_loss: 0.5411 - val_auc: 0.9053 - val_acc: 0.8750 - val_entropy: 0.5411\n",
      "Epoch 24/100\n",
      "8/8 - 0s - loss: 0.7727 - auc: 0.8335 - acc: 0.6774 - entropy: 0.7727 - val_loss: 0.5465 - val_auc: 0.9043 - val_acc: 0.8750 - val_entropy: 0.5465\n",
      "Epoch 25/100\n",
      "8/8 - 0s - loss: 0.7789 - auc: 0.8298 - acc: 0.6774 - entropy: 0.7789 - val_loss: 0.5608 - val_auc: 0.8987 - val_acc: 0.8750 - val_entropy: 0.5608\n",
      "Epoch 26/100\n",
      "8/8 - 0s - loss: 0.7796 - auc: 0.8298 - acc: 0.6774 - entropy: 0.7796 - val_loss: 0.5738 - val_auc: 0.8899 - val_acc: 0.8750 - val_entropy: 0.5738\n",
      "Epoch 27/100\n",
      "8/8 - 0s - loss: 0.7909 - auc: 0.8249 - acc: 0.6774 - entropy: 0.7909 - val_loss: 0.5993 - val_auc: 0.8857 - val_acc: 0.8750 - val_entropy: 0.5993\n",
      "Epoch 28/100\n",
      "8/8 - 0s - loss: 0.8032 - auc: 0.8216 - acc: 0.6774 - entropy: 0.8032 - val_loss: 0.6346 - val_auc: 0.8801 - val_acc: 0.8750 - val_entropy: 0.6346\n",
      "Epoch 29/100\n",
      "8/8 - 0s - loss: 0.8221 - auc: 0.8156 - acc: 0.6774 - entropy: 0.8221 - val_loss: 0.6697 - val_auc: 0.8750 - val_acc: 0.8750 - val_entropy: 0.6697\n",
      "Epoch 30/100\n",
      "8/8 - 0s - loss: 0.8174 - auc: 0.8171 - acc: 0.6855 - entropy: 0.8174 - val_loss: 0.6351 - val_auc: 0.8760 - val_acc: 0.8750 - val_entropy: 0.6351\n",
      "Restoring model weights from the end of the best epoch.\n",
      "Epoch 00030: early stopping\n",
      "1/1 [==============================] - 0s 151ms/step - loss: 0.4700 - auc: 0.9041 - acc: 0.9062 - entropy: 0.4700\n",
      "Epoch 1/100\n",
      "8/8 - 1s - loss: 16.2350 - auc: 0.5460 - acc: 0.4113 - entropy: 16.2350 - val_loss: 4.4275 - val_auc: 0.9297 - val_acc: 0.9062 - val_entropy: 4.4275\n",
      "Epoch 2/100\n",
      "8/8 - 0s - loss: 13.0607 - auc: 0.4635 - acc: 0.2984 - entropy: 13.0607 - val_loss: 2.2660 - val_auc: 0.9297 - val_acc: 0.9062 - val_entropy: 2.2660\n",
      "Epoch 3/100\n",
      "8/8 - 0s - loss: 7.7563 - auc: 0.6657 - acc: 0.5081 - entropy: 7.7563 - val_loss: 3.2264 - val_auc: 0.5132 - val_acc: 0.1250 - val_entropy: 3.2264\n",
      "Epoch 4/100\n",
      "8/8 - 0s - loss: 1.8890 - auc: 0.7761 - acc: 0.6774 - entropy: 1.8890 - val_loss: 1.1753 - val_auc: 0.8738 - val_acc: 0.8125 - val_entropy: 1.1753\n",
      "Epoch 5/100\n",
      "8/8 - 0s - loss: 2.0432 - auc: 0.6558 - acc: 0.5242 - entropy: 2.0432 - val_loss: 0.6063 - val_auc: 0.8950 - val_acc: 0.8125 - val_entropy: 0.6063\n",
      "Epoch 6/100\n",
      "8/8 - 0s - loss: 1.1547 - auc: 0.7380 - acc: 0.6452 - entropy: 1.1547 - val_loss: 0.6110 - val_auc: 0.8940 - val_acc: 0.8125 - val_entropy: 0.6110\n",
      "Epoch 7/100\n",
      "8/8 - 0s - loss: 0.9142 - auc: 0.7714 - acc: 0.5887 - entropy: 0.9142 - val_loss: 0.4632 - val_auc: 0.9143 - val_acc: 0.8750 - val_entropy: 0.4632\n",
      "Epoch 8/100\n",
      "8/8 - 0s - loss: 1.0645 - auc: 0.7159 - acc: 0.6694 - entropy: 1.0645 - val_loss: 0.6845 - val_auc: 0.8525 - val_acc: 0.7812 - val_entropy: 0.6845\n",
      "Epoch 9/100\n",
      "8/8 - 0s - loss: 0.8359 - auc: 0.8044 - acc: 0.6855 - entropy: 0.8359 - val_loss: 0.6235 - val_auc: 0.8872 - val_acc: 0.7812 - val_entropy: 0.6235\n",
      "Epoch 10/100\n",
      "8/8 - 0s - loss: 0.8768 - auc: 0.7782 - acc: 0.6774 - entropy: 0.8768 - val_loss: 0.6152 - val_auc: 0.8962 - val_acc: 0.7812 - val_entropy: 0.6152\n",
      "Epoch 11/100\n",
      "8/8 - 0s - loss: 0.7864 - auc: 0.8283 - acc: 0.6855 - entropy: 0.7864 - val_loss: 0.5726 - val_auc: 0.8999 - val_acc: 0.7812 - val_entropy: 0.5726\n",
      "Epoch 12/100\n",
      "8/8 - 0s - loss: 0.8251 - auc: 0.7985 - acc: 0.6774 - entropy: 0.8251 - val_loss: 0.6372 - val_auc: 0.8796 - val_acc: 0.7812 - val_entropy: 0.6372\n",
      "Epoch 13/100\n",
      "8/8 - 0s - loss: 0.8131 - auc: 0.8102 - acc: 0.6774 - entropy: 0.8131 - val_loss: 0.6491 - val_auc: 0.8838 - val_acc: 0.7812 - val_entropy: 0.6491\n",
      "Epoch 14/100\n",
      "8/8 - 0s - loss: 0.8270 - auc: 0.8017 - acc: 0.6774 - entropy: 0.8270 - val_loss: 0.6655 - val_auc: 0.8708 - val_acc: 0.7812 - val_entropy: 0.6655\n",
      "Epoch 15/100\n",
      "8/8 - 0s - loss: 0.8156 - auc: 0.8107 - acc: 0.6774 - entropy: 0.8156 - val_loss: 0.6517 - val_auc: 0.8765 - val_acc: 0.7812 - val_entropy: 0.6517\n",
      "Epoch 16/100\n",
      "8/8 - 0s - loss: 0.8178 - auc: 0.8076 - acc: 0.6855 - entropy: 0.8178 - val_loss: 0.6493 - val_auc: 0.8721 - val_acc: 0.7812 - val_entropy: 0.6493\n",
      "Epoch 17/100\n",
      "8/8 - 0s - loss: 0.8055 - auc: 0.8151 - acc: 0.6855 - entropy: 0.8055 - val_loss: 0.6271 - val_auc: 0.8804 - val_acc: 0.7812 - val_entropy: 0.6271\n",
      "Epoch 18/100\n",
      "8/8 - 0s - loss: 0.8092 - auc: 0.8136 - acc: 0.6855 - entropy: 0.8092 - val_loss: 0.6193 - val_auc: 0.8855 - val_acc: 0.7812 - val_entropy: 0.6193\n",
      "Epoch 19/100\n",
      "8/8 - 0s - loss: 0.8168 - auc: 0.8138 - acc: 0.6935 - entropy: 0.8168 - val_loss: 0.6266 - val_auc: 0.8821 - val_acc: 0.7812 - val_entropy: 0.6266\n",
      "Epoch 20/100\n",
      "8/8 - 0s - loss: 0.8516 - auc: 0.8066 - acc: 0.6694 - entropy: 0.8516 - val_loss: 0.7023 - val_auc: 0.8674 - val_acc: 0.7812 - val_entropy: 0.7023\n",
      "Epoch 21/100\n",
      "8/8 - 0s - loss: 0.9400 - auc: 0.7776 - acc: 0.6452 - entropy: 0.9400 - val_loss: 0.7197 - val_auc: 0.8701 - val_acc: 0.7812 - val_entropy: 0.7197\n",
      "Epoch 22/100\n",
      "8/8 - 0s - loss: 0.9218 - auc: 0.7927 - acc: 0.6855 - entropy: 0.9218 - val_loss: 0.5250 - val_auc: 0.9155 - val_acc: 0.8750 - val_entropy: 0.5250\n",
      "Epoch 23/100\n",
      "8/8 - 0s - loss: 0.8450 - auc: 0.7958 - acc: 0.6532 - entropy: 0.8450 - val_loss: 0.4862 - val_auc: 0.8975 - val_acc: 0.8750 - val_entropy: 0.4862\n",
      "Epoch 24/100\n",
      "8/8 - 0s - loss: 0.8897 - auc: 0.7641 - acc: 0.6774 - entropy: 0.8897 - val_loss: 0.6232 - val_auc: 0.8884 - val_acc: 0.8438 - val_entropy: 0.6232\n",
      "Epoch 25/100\n",
      "8/8 - 0s - loss: 0.7507 - auc: 0.8549 - acc: 0.6855 - entropy: 0.7507 - val_loss: 0.5525 - val_auc: 0.8984 - val_acc: 0.8750 - val_entropy: 0.5525\n",
      "Epoch 26/100\n",
      "8/8 - 0s - loss: 0.8365 - auc: 0.7938 - acc: 0.6613 - entropy: 0.8365 - val_loss: 0.6698 - val_auc: 0.8691 - val_acc: 0.8125 - val_entropy: 0.6698\n",
      "Epoch 27/100\n",
      "8/8 - 0s - loss: 0.7462 - auc: 0.8510 - acc: 0.6855 - entropy: 0.7462 - val_loss: 0.5944 - val_auc: 0.8889 - val_acc: 0.8125 - val_entropy: 0.5944\n",
      "Restoring model weights from the end of the best epoch.\n",
      "Epoch 00027: early stopping\n",
      "1/1 [==============================] - 0s 148ms/step - loss: 0.4632 - auc: 0.9143 - acc: 0.8750 - entropy: 0.4632\n",
      "Epoch 1/100\n",
      "8/8 - 1s - loss: 14.3032 - auc: 0.7426 - acc: 0.6129 - entropy: 14.3032 - val_loss: 5.2380 - val_auc: 0.9297 - val_acc: 0.9062 - val_entropy: 5.2380\n",
      "Epoch 2/100\n",
      "8/8 - 0s - loss: 22.9950 - auc: 0.4637 - acc: 0.2581 - entropy: 22.9950 - val_loss: 3.4828 - val_auc: 0.9297 - val_acc: 0.9062 - val_entropy: 3.4828\n",
      "Epoch 3/100\n",
      "8/8 - 0s - loss: 13.7845 - auc: 0.6236 - acc: 0.4516 - entropy: 13.7845 - val_loss: 1.8385 - val_auc: 0.9297 - val_acc: 0.9062 - val_entropy: 1.8385\n",
      "Epoch 4/100\n",
      "8/8 - 0s - loss: 5.1552 - auc: 0.6706 - acc: 0.5403 - entropy: 5.1552 - val_loss: 1.6995 - val_auc: 0.9297 - val_acc: 0.9062 - val_entropy: 1.6995\n",
      "Epoch 5/100\n",
      "8/8 - 0s - loss: 6.2556 - auc: 0.6087 - acc: 0.4032 - entropy: 6.2556 - val_loss: 1.2470 - val_auc: 0.9297 - val_acc: 0.9062 - val_entropy: 1.2470\n",
      "Epoch 6/100\n",
      "8/8 - 0s - loss: 3.6372 - auc: 0.6120 - acc: 0.3952 - entropy: 3.6372 - val_loss: 0.7033 - val_auc: 0.9241 - val_acc: 0.8438 - val_entropy: 0.7033\n",
      "Epoch 7/100\n",
      "8/8 - 0s - loss: 2.1793 - auc: 0.7808 - acc: 0.6613 - entropy: 2.1793 - val_loss: 0.5178 - val_auc: 0.9194 - val_acc: 0.9062 - val_entropy: 0.5178\n",
      "Epoch 8/100\n",
      "8/8 - 0s - loss: 1.7383 - auc: 0.5041 - acc: 0.3387 - entropy: 1.7383 - val_loss: 0.5200 - val_auc: 0.9094 - val_acc: 0.9062 - val_entropy: 0.5200\n",
      "Epoch 9/100\n",
      "8/8 - 0s - loss: 1.0598 - auc: 0.7435 - acc: 0.6613 - entropy: 1.0598 - val_loss: 0.4822 - val_auc: 0.9158 - val_acc: 0.9062 - val_entropy: 0.4822\n",
      "Epoch 10/100\n",
      "8/8 - 0s - loss: 1.1097 - auc: 0.7223 - acc: 0.5726 - entropy: 1.1097 - val_loss: 0.5237 - val_auc: 0.9089 - val_acc: 0.9062 - val_entropy: 0.5237\n",
      "Epoch 11/100\n",
      "8/8 - 0s - loss: 0.9416 - auc: 0.7537 - acc: 0.6613 - entropy: 0.9416 - val_loss: 0.4802 - val_auc: 0.9019 - val_acc: 0.9062 - val_entropy: 0.4802\n",
      "Epoch 12/100\n",
      "8/8 - 0s - loss: 0.9306 - auc: 0.7365 - acc: 0.6613 - entropy: 0.9306 - val_loss: 0.5374 - val_auc: 0.9031 - val_acc: 0.9062 - val_entropy: 0.5374\n",
      "Epoch 13/100\n",
      "8/8 - 0s - loss: 0.8189 - auc: 0.8110 - acc: 0.6694 - entropy: 0.8189 - val_loss: 0.4898 - val_auc: 0.9080 - val_acc: 0.9062 - val_entropy: 0.4898\n",
      "Epoch 14/100\n",
      "8/8 - 0s - loss: 0.8546 - auc: 0.7852 - acc: 0.6613 - entropy: 0.8546 - val_loss: 0.5530 - val_auc: 0.9121 - val_acc: 0.9062 - val_entropy: 0.5530\n",
      "Epoch 15/100\n",
      "8/8 - 0s - loss: 0.8054 - auc: 0.8218 - acc: 0.6694 - entropy: 0.8054 - val_loss: 0.5048 - val_auc: 0.9084 - val_acc: 0.9062 - val_entropy: 0.5048\n",
      "Epoch 16/100\n",
      "8/8 - 0s - loss: 0.8242 - auc: 0.8031 - acc: 0.6694 - entropy: 0.8242 - val_loss: 0.5324 - val_auc: 0.9080 - val_acc: 0.9062 - val_entropy: 0.5324\n",
      "Epoch 17/100\n",
      "8/8 - 0s - loss: 0.7969 - auc: 0.8236 - acc: 0.6774 - entropy: 0.7969 - val_loss: 0.5183 - val_auc: 0.9080 - val_acc: 0.9062 - val_entropy: 0.5183\n",
      "Epoch 18/100\n",
      "8/8 - 0s - loss: 0.8084 - auc: 0.8129 - acc: 0.6694 - entropy: 0.8084 - val_loss: 0.5361 - val_auc: 0.9072 - val_acc: 0.8750 - val_entropy: 0.5361\n",
      "Epoch 19/100\n",
      "8/8 - 0s - loss: 0.7966 - auc: 0.8215 - acc: 0.6694 - entropy: 0.7966 - val_loss: 0.5334 - val_auc: 0.9070 - val_acc: 0.8750 - val_entropy: 0.5334\n",
      "Epoch 20/100\n",
      "8/8 - 0s - loss: 0.8016 - auc: 0.8167 - acc: 0.6774 - entropy: 0.8016 - val_loss: 0.5437 - val_auc: 0.9067 - val_acc: 0.8750 - val_entropy: 0.5437\n",
      "Epoch 21/100\n",
      "8/8 - 0s - loss: 0.7965 - auc: 0.8202 - acc: 0.6774 - entropy: 0.7965 - val_loss: 0.5478 - val_auc: 0.9048 - val_acc: 0.8750 - val_entropy: 0.5478\n",
      "Epoch 22/100\n",
      "8/8 - 0s - loss: 0.8016 - auc: 0.8175 - acc: 0.6774 - entropy: 0.8016 - val_loss: 0.5603 - val_auc: 0.9048 - val_acc: 0.8750 - val_entropy: 0.5603\n",
      "Epoch 23/100\n",
      "8/8 - 0s - loss: 0.8041 - auc: 0.8178 - acc: 0.6774 - entropy: 0.8041 - val_loss: 0.5731 - val_auc: 0.9048 - val_acc: 0.8750 - val_entropy: 0.5731\n",
      "Epoch 24/100\n",
      "8/8 - 0s - loss: 0.8148 - auc: 0.8144 - acc: 0.6774 - entropy: 0.8148 - val_loss: 0.5946 - val_auc: 0.9006 - val_acc: 0.8750 - val_entropy: 0.5946\n",
      "Epoch 25/100\n",
      "8/8 - 0s - loss: 0.8233 - auc: 0.8111 - acc: 0.6774 - entropy: 0.8233 - val_loss: 0.6115 - val_auc: 0.8955 - val_acc: 0.8750 - val_entropy: 0.6115\n",
      "Epoch 26/100\n",
      "8/8 - 0s - loss: 0.8242 - auc: 0.8113 - acc: 0.6694 - entropy: 0.8242 - val_loss: 0.6004 - val_auc: 0.8965 - val_acc: 0.8750 - val_entropy: 0.6004\n",
      "Epoch 27/100\n",
      "8/8 - 0s - loss: 0.7952 - auc: 0.8228 - acc: 0.6774 - entropy: 0.7952 - val_loss: 0.5491 - val_auc: 0.9004 - val_acc: 0.8750 - val_entropy: 0.5491\n",
      "Epoch 28/100\n",
      "8/8 - 0s - loss: 0.7609 - auc: 0.8389 - acc: 0.6935 - entropy: 0.7609 - val_loss: 0.5008 - val_auc: 0.9102 - val_acc: 0.8750 - val_entropy: 0.5008\n",
      "Epoch 29/100\n",
      "8/8 - 0s - loss: 0.7745 - auc: 0.8311 - acc: 0.6855 - entropy: 0.7745 - val_loss: 0.4895 - val_auc: 0.9124 - val_acc: 0.9062 - val_entropy: 0.4895\n",
      "Epoch 30/100\n",
      "8/8 - 0s - loss: 0.8017 - auc: 0.8204 - acc: 0.6855 - entropy: 0.8017 - val_loss: 0.4962 - val_auc: 0.9136 - val_acc: 0.9062 - val_entropy: 0.4962\n",
      "Epoch 31/100\n",
      "8/8 - 0s - loss: 0.7902 - auc: 0.8305 - acc: 0.7097 - entropy: 0.7902 - val_loss: 0.5214 - val_auc: 0.9038 - val_acc: 0.8750 - val_entropy: 0.5214\n",
      "Restoring model weights from the end of the best epoch.\n",
      "Epoch 00031: early stopping\n",
      "1/1 [==============================] - 0s 152ms/step - loss: 0.4802 - auc: 0.9019 - acc: 0.9062 - entropy: 0.4802\n",
      "Epoch 1/100\n",
      "8/8 - 1s - loss: 11.9359 - auc: 0.7426 - acc: 0.5565 - entropy: 11.9359 - val_loss: 6.3397 - val_auc: 0.9297 - val_acc: 0.9062 - val_entropy: 6.3397\n",
      "Epoch 2/100\n",
      "8/8 - 0s - loss: 17.1687 - auc: 0.6431 - acc: 0.5242 - entropy: 17.1687 - val_loss: 4.4763 - val_auc: 0.9297 - val_acc: 0.9062 - val_entropy: 4.4763\n",
      "Epoch 3/100\n",
      "8/8 - 0s - loss: 17.7689 - auc: 0.6324 - acc: 0.4677 - entropy: 17.7689 - val_loss: 3.3382 - val_auc: 0.9297 - val_acc: 0.9062 - val_entropy: 3.3382\n",
      "Epoch 4/100\n",
      "8/8 - 0s - loss: 8.5671 - auc: 0.6448 - acc: 0.5161 - entropy: 8.5671 - val_loss: 2.0903 - val_auc: 0.9297 - val_acc: 0.9062 - val_entropy: 2.0903\n",
      "Epoch 5/100\n",
      "8/8 - 0s - loss: 8.1920 - auc: 0.5569 - acc: 0.3952 - entropy: 8.1920 - val_loss: 1.2270 - val_auc: 0.9297 - val_acc: 0.9062 - val_entropy: 1.2270\n",
      "Epoch 6/100\n",
      "8/8 - 0s - loss: 3.8794 - auc: 0.6078 - acc: 0.4758 - entropy: 3.8794 - val_loss: 0.4155 - val_auc: 0.9263 - val_acc: 0.9062 - val_entropy: 0.4155\n",
      "Epoch 7/100\n",
      "8/8 - 0s - loss: 1.9419 - auc: 0.7274 - acc: 0.5242 - entropy: 1.9419 - val_loss: 0.6773 - val_auc: 0.9282 - val_acc: 0.9062 - val_entropy: 0.6773\n",
      "Epoch 8/100\n",
      "8/8 - 0s - loss: 2.0455 - auc: 0.5861 - acc: 0.4113 - entropy: 2.0455 - val_loss: 0.8707 - val_auc: 0.8857 - val_acc: 0.8125 - val_entropy: 0.8707\n",
      "Epoch 9/100\n",
      "8/8 - 0s - loss: 0.8252 - auc: 0.8270 - acc: 0.7016 - entropy: 0.8252 - val_loss: 0.4450 - val_auc: 0.9080 - val_acc: 0.9062 - val_entropy: 0.4450\n",
      "Epoch 10/100\n",
      "8/8 - 0s - loss: 1.0603 - auc: 0.7042 - acc: 0.5968 - entropy: 1.0603 - val_loss: 0.5377 - val_auc: 0.9089 - val_acc: 0.9062 - val_entropy: 0.5377\n",
      "Epoch 11/100\n",
      "8/8 - 0s - loss: 0.8749 - auc: 0.7789 - acc: 0.6774 - entropy: 0.8749 - val_loss: 0.5027 - val_auc: 0.9124 - val_acc: 0.9062 - val_entropy: 0.5027\n",
      "Epoch 12/100\n",
      "8/8 - 0s - loss: 0.8762 - auc: 0.7685 - acc: 0.6613 - entropy: 0.8762 - val_loss: 0.4816 - val_auc: 0.9155 - val_acc: 0.9062 - val_entropy: 0.4816\n",
      "Epoch 13/100\n",
      "8/8 - 0s - loss: 0.8199 - auc: 0.8076 - acc: 0.6774 - entropy: 0.8199 - val_loss: 0.5151 - val_auc: 0.9092 - val_acc: 0.9062 - val_entropy: 0.5151\n",
      "Epoch 14/100\n",
      "8/8 - 0s - loss: 0.8308 - auc: 0.7964 - acc: 0.6694 - entropy: 0.8308 - val_loss: 0.5176 - val_auc: 0.9084 - val_acc: 0.9062 - val_entropy: 0.5176\n",
      "Epoch 15/100\n",
      "8/8 - 0s - loss: 0.7972 - auc: 0.8271 - acc: 0.6774 - entropy: 0.7972 - val_loss: 0.4994 - val_auc: 0.9041 - val_acc: 0.9062 - val_entropy: 0.4994\n",
      "Epoch 16/100\n",
      "8/8 - 0s - loss: 0.8225 - auc: 0.8016 - acc: 0.6774 - entropy: 0.8225 - val_loss: 0.5289 - val_auc: 0.9028 - val_acc: 0.9062 - val_entropy: 0.5289\n",
      "Epoch 17/100\n",
      "8/8 - 0s - loss: 0.7971 - auc: 0.8254 - acc: 0.6774 - entropy: 0.7971 - val_loss: 0.5063 - val_auc: 0.9065 - val_acc: 0.9062 - val_entropy: 0.5063\n",
      "Epoch 18/100\n",
      "8/8 - 0s - loss: 0.8129 - auc: 0.8093 - acc: 0.6694 - entropy: 0.8129 - val_loss: 0.5312 - val_auc: 0.9043 - val_acc: 0.9062 - val_entropy: 0.5312\n",
      "Epoch 19/100\n",
      "8/8 - 0s - loss: 0.7941 - auc: 0.8243 - acc: 0.6694 - entropy: 0.7941 - val_loss: 0.5190 - val_auc: 0.9048 - val_acc: 0.9062 - val_entropy: 0.5190\n",
      "Epoch 20/100\n",
      "8/8 - 0s - loss: 0.8025 - auc: 0.8168 - acc: 0.6694 - entropy: 0.8025 - val_loss: 0.5344 - val_auc: 0.9033 - val_acc: 0.8750 - val_entropy: 0.5344\n",
      "Epoch 21/100\n",
      "8/8 - 0s - loss: 0.7917 - auc: 0.8247 - acc: 0.6694 - entropy: 0.7917 - val_loss: 0.5330 - val_auc: 0.9009 - val_acc: 0.8750 - val_entropy: 0.5330\n",
      "Epoch 22/100\n",
      "8/8 - 0s - loss: 0.7956 - auc: 0.8204 - acc: 0.6774 - entropy: 0.7956 - val_loss: 0.5426 - val_auc: 0.9019 - val_acc: 0.8750 - val_entropy: 0.5426\n",
      "Epoch 23/100\n",
      "8/8 - 0s - loss: 0.7890 - auc: 0.8257 - acc: 0.6774 - entropy: 0.7890 - val_loss: 0.5459 - val_auc: 0.8999 - val_acc: 0.8750 - val_entropy: 0.5459\n",
      "Epoch 24/100\n",
      "8/8 - 0s - loss: 0.7924 - auc: 0.8226 - acc: 0.6774 - entropy: 0.7924 - val_loss: 0.5576 - val_auc: 0.8989 - val_acc: 0.8750 - val_entropy: 0.5576\n",
      "Epoch 25/100\n",
      "8/8 - 0s - loss: 0.7908 - auc: 0.8237 - acc: 0.6774 - entropy: 0.7908 - val_loss: 0.5660 - val_auc: 0.8982 - val_acc: 0.8750 - val_entropy: 0.5660\n",
      "Epoch 26/100\n",
      "8/8 - 0s - loss: 0.7949 - auc: 0.8218 - acc: 0.6774 - entropy: 0.7949 - val_loss: 0.5786 - val_auc: 0.8977 - val_acc: 0.8438 - val_entropy: 0.5786\n",
      "Restoring model weights from the end of the best epoch.\n",
      "Epoch 00026: early stopping\n",
      "1/1 [==============================] - 0s 147ms/step - loss: 0.4155 - auc: 0.9263 - acc: 0.9062 - entropy: 0.4155\n",
      "Epoch 1/100\n",
      "8/8 - 1s - loss: 11.5991 - auc: 0.7394 - acc: 0.6129 - entropy: 11.5991 - val_loss: 3.0042 - val_auc: 0.9297 - val_acc: 0.9062 - val_entropy: 3.0042\n",
      "Epoch 2/100\n",
      "8/8 - 0s - loss: 15.8917 - auc: 0.6069 - acc: 0.4758 - entropy: 15.8917 - val_loss: 10.1042 - val_auc: 0.9297 - val_acc: 0.9062 - val_entropy: 10.1042\n",
      "Epoch 3/100\n",
      "8/8 - 0s - loss: 27.8978 - auc: 0.6684 - acc: 0.5081 - entropy: 27.8978 - val_loss: 5.7218 - val_auc: 0.3096 - val_acc: 0.1562 - val_entropy: 5.7218\n",
      "Epoch 4/100\n",
      "8/8 - 0s - loss: 2.5176 - auc: 0.7308 - acc: 0.5887 - entropy: 2.5176 - val_loss: 0.9785 - val_auc: 0.9297 - val_acc: 0.9062 - val_entropy: 0.9785\n",
      "Epoch 5/100\n",
      "8/8 - 0s - loss: 3.1115 - auc: 0.5548 - acc: 0.2823 - entropy: 3.1115 - val_loss: 1.1034 - val_auc: 0.9219 - val_acc: 0.9062 - val_entropy: 1.1034\n",
      "Epoch 6/100\n",
      "8/8 - 0s - loss: 2.8068 - auc: 0.6927 - acc: 0.6129 - entropy: 2.8068 - val_loss: 1.1788 - val_auc: 0.5654 - val_acc: 0.2812 - val_entropy: 1.1788\n",
      "Epoch 7/100\n",
      "8/8 - 0s - loss: 0.8193 - auc: 0.8169 - acc: 0.5968 - entropy: 0.8193 - val_loss: 0.4429 - val_auc: 0.9072 - val_acc: 0.9062 - val_entropy: 0.4429\n",
      "Epoch 8/100\n",
      "8/8 - 0s - loss: 1.1574 - auc: 0.6914 - acc: 0.6774 - entropy: 1.1574 - val_loss: 0.6952 - val_auc: 0.8975 - val_acc: 0.8438 - val_entropy: 0.6952\n",
      "Epoch 9/100\n",
      "8/8 - 0s - loss: 0.8157 - auc: 0.8225 - acc: 0.6855 - entropy: 0.8157 - val_loss: 0.4742 - val_auc: 0.9111 - val_acc: 0.9062 - val_entropy: 0.4742\n",
      "Epoch 10/100\n",
      "8/8 - 0s - loss: 0.8456 - auc: 0.7786 - acc: 0.6774 - entropy: 0.8456 - val_loss: 0.5435 - val_auc: 0.9070 - val_acc: 0.9062 - val_entropy: 0.5435\n",
      "Epoch 11/100\n",
      "8/8 - 0s - loss: 0.8308 - auc: 0.7898 - acc: 0.6774 - entropy: 0.8308 - val_loss: 0.5681 - val_auc: 0.9036 - val_acc: 0.9062 - val_entropy: 0.5681\n",
      "Epoch 12/100\n",
      "8/8 - 0s - loss: 0.7771 - auc: 0.8426 - acc: 0.6774 - entropy: 0.7771 - val_loss: 0.5058 - val_auc: 0.9062 - val_acc: 0.9062 - val_entropy: 0.5058\n",
      "Epoch 13/100\n",
      "8/8 - 0s - loss: 0.8220 - auc: 0.7979 - acc: 0.6774 - entropy: 0.8220 - val_loss: 0.5571 - val_auc: 0.9058 - val_acc: 0.8750 - val_entropy: 0.5571\n",
      "Epoch 14/100\n",
      "8/8 - 0s - loss: 0.7917 - auc: 0.8256 - acc: 0.6855 - entropy: 0.7917 - val_loss: 0.5347 - val_auc: 0.9058 - val_acc: 0.8750 - val_entropy: 0.5347\n",
      "Epoch 15/100\n",
      "8/8 - 0s - loss: 0.8061 - auc: 0.8122 - acc: 0.6855 - entropy: 0.8061 - val_loss: 0.5497 - val_auc: 0.9021 - val_acc: 0.8750 - val_entropy: 0.5497\n",
      "Epoch 16/100\n",
      "8/8 - 0s - loss: 0.7978 - auc: 0.8185 - acc: 0.6774 - entropy: 0.7978 - val_loss: 0.5472 - val_auc: 0.9070 - val_acc: 0.8750 - val_entropy: 0.5472\n",
      "Epoch 17/100\n",
      "8/8 - 0s - loss: 0.8099 - auc: 0.8112 - acc: 0.6774 - entropy: 0.8099 - val_loss: 0.5653 - val_auc: 0.9023 - val_acc: 0.8750 - val_entropy: 0.5653\n",
      "Epoch 18/100\n",
      "8/8 - 0s - loss: 0.8150 - auc: 0.8106 - acc: 0.6774 - entropy: 0.8150 - val_loss: 0.5792 - val_auc: 0.8967 - val_acc: 0.8750 - val_entropy: 0.5792\n",
      "Epoch 19/100\n",
      "8/8 - 0s - loss: 0.8260 - auc: 0.8064 - acc: 0.6774 - entropy: 0.8260 - val_loss: 0.5985 - val_auc: 0.8899 - val_acc: 0.8750 - val_entropy: 0.5985\n",
      "Epoch 20/100\n",
      "8/8 - 0s - loss: 0.8309 - auc: 0.8050 - acc: 0.6774 - entropy: 0.8309 - val_loss: 0.6075 - val_auc: 0.8875 - val_acc: 0.8750 - val_entropy: 0.6075\n",
      "Epoch 21/100\n",
      "8/8 - 0s - loss: 0.8229 - auc: 0.8088 - acc: 0.6774 - entropy: 0.8229 - val_loss: 0.5910 - val_auc: 0.8909 - val_acc: 0.8750 - val_entropy: 0.5910\n",
      "Epoch 22/100\n",
      "8/8 - 0s - loss: 0.7907 - auc: 0.8225 - acc: 0.6855 - entropy: 0.7907 - val_loss: 0.5495 - val_auc: 0.9016 - val_acc: 0.8750 - val_entropy: 0.5495\n",
      "Epoch 23/100\n",
      "8/8 - 0s - loss: 0.7716 - auc: 0.8321 - acc: 0.6935 - entropy: 0.7716 - val_loss: 0.5188 - val_auc: 0.9067 - val_acc: 0.8750 - val_entropy: 0.5188\n",
      "Epoch 24/100\n",
      "8/8 - 0s - loss: 0.7842 - auc: 0.8261 - acc: 0.6935 - entropy: 0.7842 - val_loss: 0.5173 - val_auc: 0.9111 - val_acc: 0.8750 - val_entropy: 0.5173\n",
      "Epoch 25/100\n",
      "8/8 - 0s - loss: 0.7975 - auc: 0.8211 - acc: 0.6935 - entropy: 0.7975 - val_loss: 0.5249 - val_auc: 0.9114 - val_acc: 0.8438 - val_entropy: 0.5249\n",
      "Epoch 26/100\n",
      "8/8 - 0s - loss: 0.7960 - auc: 0.8257 - acc: 0.6935 - entropy: 0.7960 - val_loss: 0.5285 - val_auc: 0.9089 - val_acc: 0.8438 - val_entropy: 0.5285\n",
      "Epoch 27/100\n",
      "8/8 - 0s - loss: 0.8306 - auc: 0.8243 - acc: 0.7016 - entropy: 0.8306 - val_loss: 0.6393 - val_auc: 0.9033 - val_acc: 0.8750 - val_entropy: 0.6393\n",
      "Restoring model weights from the end of the best epoch.\n",
      "Epoch 00027: early stopping\n",
      "1/1 [==============================] - 0s 147ms/step - loss: 0.4429 - auc: 0.9072 - acc: 0.9062 - entropy: 0.4429\n",
      "auc : average=0.911, std=0.009\n",
      "acc : average=0.900, std=0.012\n",
      "entropy : average=0.454, std=0.023\n"
     ]
    }
   ],
   "source": [
    "calculate_mean_std(weights=None)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "id": "8ad6c7b0",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 1/100\n",
      "8/8 - 1s - loss: 87.2224 - auc: 0.4824 - acc: 0.3145 - entropy: 28.0897 - val_loss: 23.1546 - val_auc: 0.2734 - val_acc: 0.0312 - val_entropy: 23.1546\n",
      "Epoch 2/100\n",
      "8/8 - 0s - loss: 100.9792 - auc: 0.3770 - acc: 0.1694 - entropy: 27.6703 - val_loss: 18.8518 - val_auc: 0.2969 - val_acc: 0.0625 - val_entropy: 18.8518\n",
      "Epoch 3/100\n",
      "8/8 - 0s - loss: 11.3218 - auc: 0.7844 - acc: 0.6613 - entropy: 2.8465 - val_loss: 1.0183 - val_auc: 0.9033 - val_acc: 0.8750 - val_entropy: 1.0183\n",
      "Epoch 4/100\n",
      "8/8 - 0s - loss: 19.0414 - auc: 0.2893 - acc: 0.1532 - entropy: 7.5693 - val_loss: 3.2131 - val_auc: 0.4094 - val_acc: 0.0625 - val_entropy: 3.2131\n",
      "Epoch 5/100\n",
      "8/8 - 0s - loss: 8.3975 - auc: 0.4397 - acc: 0.2419 - entropy: 3.2393 - val_loss: 2.6930 - val_auc: 0.4622 - val_acc: 0.0625 - val_entropy: 2.6930\n",
      "Epoch 6/100\n",
      "8/8 - 0s - loss: 7.1721 - auc: 0.4382 - acc: 0.1613 - entropy: 2.0218 - val_loss: 0.7654 - val_auc: 0.8845 - val_acc: 0.8125 - val_entropy: 0.7654\n",
      "Epoch 7/100\n",
      "8/8 - 0s - loss: 5.2491 - auc: 0.3475 - acc: 0.2339 - entropy: 2.1673 - val_loss: 1.0669 - val_auc: 0.5364 - val_acc: 0.2188 - val_entropy: 1.0669\n",
      "Epoch 8/100\n",
      "8/8 - 0s - loss: 4.7574 - auc: 0.4369 - acc: 0.2742 - entropy: 1.3864 - val_loss: 1.4753 - val_auc: 0.3376 - val_acc: 0.0938 - val_entropy: 1.4753\n",
      "Epoch 9/100\n",
      "8/8 - 0s - loss: 3.2702 - auc: 0.5732 - acc: 0.3387 - entropy: 1.1049 - val_loss: 0.7859 - val_auc: 0.8972 - val_acc: 0.8750 - val_entropy: 0.7859\n",
      "Epoch 10/100\n",
      "8/8 - 0s - loss: 4.4601 - auc: 0.4171 - acc: 0.2984 - entropy: 1.3947 - val_loss: 1.5544 - val_auc: 0.2278 - val_acc: 0.1875 - val_entropy: 1.5544\n",
      "Epoch 11/100\n",
      "8/8 - 0s - loss: 3.2219 - auc: 0.5828 - acc: 0.4516 - entropy: 1.1037 - val_loss: 0.8403 - val_auc: 0.8882 - val_acc: 0.8750 - val_entropy: 0.8403\n",
      "Epoch 12/100\n",
      "8/8 - 0s - loss: 3.6525 - auc: 0.4260 - acc: 0.2823 - entropy: 1.1910 - val_loss: 1.2359 - val_auc: 0.2542 - val_acc: 0.2188 - val_entropy: 1.2359\n",
      "Epoch 13/100\n",
      "8/8 - 0s - loss: 3.3176 - auc: 0.5605 - acc: 0.3871 - entropy: 1.0881 - val_loss: 1.0919 - val_auc: 0.5754 - val_acc: 0.2812 - val_entropy: 1.0919\n",
      "Epoch 14/100\n",
      "8/8 - 0s - loss: 3.3012 - auc: 0.5126 - acc: 0.3387 - entropy: 1.1324 - val_loss: 1.1770 - val_auc: 0.3618 - val_acc: 0.2188 - val_entropy: 1.1770\n",
      "Epoch 15/100\n",
      "8/8 - 0s - loss: 3.2913 - auc: 0.5887 - acc: 0.4194 - entropy: 1.0727 - val_loss: 1.1467 - val_auc: 0.4070 - val_acc: 0.2500 - val_entropy: 1.1467\n",
      "Epoch 16/100\n",
      "8/8 - 0s - loss: 3.2339 - auc: 0.5561 - acc: 0.4355 - entropy: 1.1012 - val_loss: 1.1456 - val_auc: 0.4397 - val_acc: 0.1875 - val_entropy: 1.1456\n",
      "Epoch 17/100\n",
      "8/8 - 0s - loss: 3.3038 - auc: 0.5759 - acc: 0.4113 - entropy: 1.0881 - val_loss: 1.2045 - val_auc: 0.4148 - val_acc: 0.1562 - val_entropy: 1.2045\n",
      "Epoch 18/100\n",
      "8/8 - 0s - loss: 3.3074 - auc: 0.5801 - acc: 0.4113 - entropy: 1.0867 - val_loss: 1.1791 - val_auc: 0.4746 - val_acc: 0.1562 - val_entropy: 1.1791\n",
      "Epoch 19/100\n",
      "8/8 - 0s - loss: 3.3286 - auc: 0.5741 - acc: 0.4032 - entropy: 1.0933 - val_loss: 1.2162 - val_auc: 0.4231 - val_acc: 0.1250 - val_entropy: 1.2162\n",
      "Epoch 20/100\n",
      "8/8 - 0s - loss: 3.2905 - auc: 0.5808 - acc: 0.4032 - entropy: 1.0869 - val_loss: 1.1956 - val_auc: 0.4609 - val_acc: 0.1562 - val_entropy: 1.1956\n",
      "Epoch 21/100\n",
      "8/8 - 0s - loss: 3.3156 - auc: 0.5759 - acc: 0.4032 - entropy: 1.0935 - val_loss: 1.2370 - val_auc: 0.4180 - val_acc: 0.1250 - val_entropy: 1.2370\n",
      "Epoch 22/100\n",
      "8/8 - 0s - loss: 3.2822 - auc: 0.5865 - acc: 0.4032 - entropy: 1.0839 - val_loss: 1.2147 - val_auc: 0.4602 - val_acc: 0.1250 - val_entropy: 1.2147\n",
      "Epoch 23/100\n",
      "8/8 - 0s - loss: 3.3258 - auc: 0.5802 - acc: 0.4032 - entropy: 1.0924 - val_loss: 1.2603 - val_auc: 0.4270 - val_acc: 0.0938 - val_entropy: 1.2603\n",
      "Epoch 24/100\n",
      "8/8 - 0s - loss: 3.2907 - auc: 0.5964 - acc: 0.4113 - entropy: 1.0786 - val_loss: 1.2208 - val_auc: 0.4568 - val_acc: 0.1250 - val_entropy: 1.2208\n",
      "Epoch 25/100\n",
      "8/8 - 0s - loss: 3.2966 - auc: 0.5876 - acc: 0.4113 - entropy: 1.0859 - val_loss: 1.2485 - val_auc: 0.4106 - val_acc: 0.1250 - val_entropy: 1.2485\n",
      "Epoch 26/100\n",
      "8/8 - 0s - loss: 3.1959 - auc: 0.6028 - acc: 0.4032 - entropy: 1.0693 - val_loss: 1.1932 - val_auc: 0.4697 - val_acc: 0.1875 - val_entropy: 1.1932\n",
      "Restoring model weights from the end of the best epoch.\n",
      "Epoch 00026: early stopping\n",
      "1/1 [==============================] - 0s 146ms/step - loss: 0.7654 - auc: 0.8845 - acc: 0.8125 - entropy: 0.7654\n",
      "Epoch 1/100\n",
      "8/8 - 1s - loss: 69.8751 - auc: 0.3310 - acc: 0.1694 - entropy: 20.1187 - val_loss: 3.7360 - val_auc: 0.9297 - val_acc: 0.9062 - val_entropy: 3.7360\n",
      "Epoch 2/100\n",
      "8/8 - 0s - loss: 60.8629 - auc: 0.3277 - acc: 0.1452 - entropy: 17.9217 - val_loss: 16.2656 - val_auc: 0.2515 - val_acc: 0.0312 - val_entropy: 16.2656\n",
      "Epoch 3/100\n",
      "8/8 - 0s - loss: 18.4561 - auc: 0.7395 - acc: 0.6532 - entropy: 3.8477 - val_loss: 0.9344 - val_auc: 0.9189 - val_acc: 0.9062 - val_entropy: 0.9344\n",
      "Epoch 4/100\n",
      "8/8 - 0s - loss: 22.4454 - auc: 0.2705 - acc: 0.1452 - entropy: 9.1305 - val_loss: 3.9023 - val_auc: 0.0981 - val_acc: 0.0312 - val_entropy: 3.9023\n",
      "Epoch 5/100\n",
      "8/8 - 0s - loss: 6.0365 - auc: 0.6097 - acc: 0.3790 - entropy: 1.5308 - val_loss: 0.4645 - val_auc: 0.9229 - val_acc: 0.9062 - val_entropy: 0.4645\n",
      "Epoch 6/100\n",
      "8/8 - 0s - loss: 7.1770 - auc: 0.2803 - acc: 0.2097 - entropy: 3.3473 - val_loss: 2.4411 - val_auc: 0.1672 - val_acc: 0.1250 - val_entropy: 2.4411\n",
      "Epoch 7/100\n",
      "8/8 - 0s - loss: 4.4418 - auc: 0.6904 - acc: 0.5323 - entropy: 1.1009 - val_loss: 0.5057 - val_auc: 0.9167 - val_acc: 0.9062 - val_entropy: 0.5057\n",
      "Epoch 8/100\n",
      "8/8 - 0s - loss: 5.1108 - auc: 0.2514 - acc: 0.2177 - entropy: 2.2493 - val_loss: 2.3746 - val_auc: 0.0669 - val_acc: 0.0312 - val_entropy: 2.3746\n",
      "Epoch 9/100\n",
      "8/8 - 0s - loss: 3.0654 - auc: 0.6791 - acc: 0.5323 - entropy: 1.0659 - val_loss: 0.5895 - val_auc: 0.9138 - val_acc: 0.8750 - val_entropy: 0.5895\n",
      "Epoch 10/100\n",
      "8/8 - 0s - loss: 4.2204 - auc: 0.4241 - acc: 0.2823 - entropy: 1.2805 - val_loss: 1.6184 - val_auc: 0.0801 - val_acc: 0.0312 - val_entropy: 1.6184\n",
      "Epoch 11/100\n",
      "8/8 - 0s - loss: 3.1698 - auc: 0.4326 - acc: 0.2500 - entropy: 1.2194 - val_loss: 0.8778 - val_auc: 0.8442 - val_acc: 0.7812 - val_entropy: 0.8778\n",
      "Epoch 12/100\n",
      "8/8 - 0s - loss: 3.8159 - auc: 0.4667 - acc: 0.3548 - entropy: 1.2401 - val_loss: 1.6305 - val_auc: 0.2476 - val_acc: 0.1250 - val_entropy: 1.6305\n",
      "Epoch 13/100\n",
      "8/8 - 0s - loss: 3.3317 - auc: 0.5285 - acc: 0.3952 - entropy: 1.1645 - val_loss: 1.1512 - val_auc: 0.4915 - val_acc: 0.1250 - val_entropy: 1.1512\n",
      "Epoch 14/100\n",
      "8/8 - 0s - loss: 3.3860 - auc: 0.6105 - acc: 0.4032 - entropy: 1.0646 - val_loss: 1.0976 - val_auc: 0.5413 - val_acc: 0.2188 - val_entropy: 1.0976\n",
      "Epoch 15/100\n",
      "8/8 - 0s - loss: 3.2221 - auc: 0.5790 - acc: 0.3790 - entropy: 1.0768 - val_loss: 1.2265 - val_auc: 0.3899 - val_acc: 0.2188 - val_entropy: 1.2265\n",
      "Epoch 16/100\n",
      "8/8 - 0s - loss: 3.1815 - auc: 0.5798 - acc: 0.3629 - entropy: 1.0871 - val_loss: 1.2304 - val_auc: 0.4614 - val_acc: 0.1562 - val_entropy: 1.2304\n",
      "Epoch 17/100\n",
      "8/8 - 0s - loss: 3.3289 - auc: 0.6015 - acc: 0.3952 - entropy: 1.0771 - val_loss: 1.2426 - val_auc: 0.4670 - val_acc: 0.0938 - val_entropy: 1.2426\n",
      "Epoch 18/100\n",
      "8/8 - 0s - loss: 3.5191 - auc: 0.5807 - acc: 0.4274 - entropy: 1.1124 - val_loss: 1.2351 - val_auc: 0.4719 - val_acc: 0.1562 - val_entropy: 1.2351\n",
      "Epoch 19/100\n",
      "8/8 - 0s - loss: 3.3140 - auc: 0.5998 - acc: 0.4274 - entropy: 1.0814 - val_loss: 1.1179 - val_auc: 0.5327 - val_acc: 0.3125 - val_entropy: 1.1179\n",
      "Epoch 20/100\n",
      "8/8 - 0s - loss: 3.0750 - auc: 0.5922 - acc: 0.3952 - entropy: 1.0713 - val_loss: 1.0726 - val_auc: 0.6252 - val_acc: 0.3750 - val_entropy: 1.0726\n",
      "Epoch 21/100\n",
      "8/8 - 0s - loss: 3.1666 - auc: 0.5747 - acc: 0.3952 - entropy: 1.1004 - val_loss: 1.2027 - val_auc: 0.4631 - val_acc: 0.1875 - val_entropy: 1.2027\n",
      "Epoch 22/100\n",
      "8/8 - 0s - loss: 3.2529 - auc: 0.6145 - acc: 0.3629 - entropy: 1.0714 - val_loss: 1.2565 - val_auc: 0.4751 - val_acc: 0.1562 - val_entropy: 1.2565\n",
      "Epoch 23/100\n",
      "8/8 - 0s - loss: 3.7325 - auc: 0.5931 - acc: 0.3952 - entropy: 1.1316 - val_loss: 1.1841 - val_auc: 0.4878 - val_acc: 0.1250 - val_entropy: 1.1841\n",
      "Epoch 24/100\n",
      "8/8 - 0s - loss: 3.7604 - auc: 0.5471 - acc: 0.2984 - entropy: 1.1826 - val_loss: 1.0507 - val_auc: 0.6458 - val_acc: 0.3750 - val_entropy: 1.0507\n",
      "Epoch 25/100\n",
      "8/8 - 0s - loss: 3.2267 - auc: 0.5415 - acc: 0.3226 - entropy: 1.1609 - val_loss: 1.0577 - val_auc: 0.6128 - val_acc: 0.2812 - val_entropy: 1.0577\n",
      "Restoring model weights from the end of the best epoch.\n",
      "Epoch 00025: early stopping\n",
      "1/1 [==============================] - 0s 149ms/step - loss: 0.4645 - auc: 0.9229 - acc: 0.9062 - entropy: 0.4645\n",
      "Epoch 1/100\n",
      "8/8 - 1s - loss: 63.2005 - auc: 0.3090 - acc: 0.1613 - entropy: 18.5819 - val_loss: 1.7295 - val_auc: 0.5315 - val_acc: 0.2188 - val_entropy: 1.7295\n",
      "Epoch 2/100\n",
      "8/8 - 0s - loss: 23.9083 - auc: 0.3405 - acc: 0.1532 - entropy: 6.0152 - val_loss: 4.4888 - val_auc: 0.4495 - val_acc: 0.0312 - val_entropy: 4.4888\n",
      "Epoch 3/100\n",
      "8/8 - 0s - loss: 14.4859 - auc: 0.4077 - acc: 0.2500 - entropy: 3.4620 - val_loss: 1.9857 - val_auc: 0.1843 - val_acc: 0.1250 - val_entropy: 1.9857\n",
      "Epoch 4/100\n",
      "8/8 - 0s - loss: 3.6304 - auc: 0.6977 - acc: 0.5726 - entropy: 1.0278 - val_loss: 0.5259 - val_auc: 0.9297 - val_acc: 0.9062 - val_entropy: 0.5259\n",
      "Epoch 5/100\n",
      "8/8 - 0s - loss: 6.1870 - auc: 0.2723 - acc: 0.1774 - entropy: 2.7337 - val_loss: 0.9998 - val_auc: 0.6860 - val_acc: 0.3438 - val_entropy: 0.9998\n",
      "Epoch 6/100\n",
      "8/8 - 0s - loss: 5.1143 - auc: 0.5069 - acc: 0.2903 - entropy: 1.3734 - val_loss: 0.9573 - val_auc: 0.6670 - val_acc: 0.4062 - val_entropy: 0.9573\n",
      "Epoch 7/100\n",
      "8/8 - 0s - loss: 4.1788 - auc: 0.3438 - acc: 0.2177 - entropy: 1.4527 - val_loss: 1.0738 - val_auc: 0.5923 - val_acc: 0.3125 - val_entropy: 1.0738\n",
      "Epoch 8/100\n",
      "8/8 - 0s - loss: 3.4670 - auc: 0.5436 - acc: 0.3629 - entropy: 1.1754 - val_loss: 1.0511 - val_auc: 0.6272 - val_acc: 0.3438 - val_entropy: 1.0511\n",
      "Epoch 9/100\n",
      "8/8 - 0s - loss: 3.9815 - auc: 0.4855 - acc: 0.3065 - entropy: 1.2138 - val_loss: 1.2287 - val_auc: 0.4277 - val_acc: 0.2500 - val_entropy: 1.2287\n",
      "Epoch 10/100\n",
      "8/8 - 0s - loss: 3.2097 - auc: 0.5405 - acc: 0.3145 - entropy: 1.1384 - val_loss: 0.8921 - val_auc: 0.8032 - val_acc: 0.6562 - val_entropy: 0.8921\n",
      "Epoch 11/100\n",
      "8/8 - 0s - loss: 3.8623 - auc: 0.4337 - acc: 0.2984 - entropy: 1.2258 - val_loss: 1.2667 - val_auc: 0.2395 - val_acc: 0.0938 - val_entropy: 1.2667\n",
      "Epoch 12/100\n",
      "8/8 - 0s - loss: 3.2491 - auc: 0.5037 - acc: 0.3226 - entropy: 1.1808 - val_loss: 0.8894 - val_auc: 0.8442 - val_acc: 0.7812 - val_entropy: 0.8894\n",
      "Epoch 13/100\n",
      "8/8 - 0s - loss: 3.7889 - auc: 0.4732 - acc: 0.3790 - entropy: 1.2836 - val_loss: 1.4670 - val_auc: 0.3032 - val_acc: 0.1562 - val_entropy: 1.4670\n",
      "Epoch 14/100\n",
      "8/8 - 0s - loss: 3.4244 - auc: 0.6175 - acc: 0.4355 - entropy: 1.0678 - val_loss: 0.8681 - val_auc: 0.8032 - val_acc: 0.6562 - val_entropy: 0.8681\n",
      "Epoch 15/100\n",
      "8/8 - 0s - loss: 3.9472 - auc: 0.4258 - acc: 0.2823 - entropy: 1.2818 - val_loss: 1.1759 - val_auc: 0.4343 - val_acc: 0.2500 - val_entropy: 1.1759\n",
      "Epoch 16/100\n",
      "8/8 - 0s - loss: 3.3045 - auc: 0.5408 - acc: 0.3468 - entropy: 1.1258 - val_loss: 1.0068 - val_auc: 0.7656 - val_acc: 0.5938 - val_entropy: 1.0068\n",
      "Epoch 17/100\n",
      "8/8 - 0s - loss: 3.5623 - auc: 0.4122 - acc: 0.2823 - entropy: 1.2867 - val_loss: 1.3034 - val_auc: 0.3831 - val_acc: 0.1875 - val_entropy: 1.3034\n",
      "Epoch 18/100\n",
      "8/8 - 0s - loss: 3.3951 - auc: 0.6718 - acc: 0.4435 - entropy: 1.0155 - val_loss: 0.9720 - val_auc: 0.6316 - val_acc: 0.3438 - val_entropy: 0.9720\n",
      "Epoch 19/100\n",
      "8/8 - 0s - loss: 3.9801 - auc: 0.4636 - acc: 0.3145 - entropy: 1.2895 - val_loss: 1.2167 - val_auc: 0.3538 - val_acc: 0.2500 - val_entropy: 1.2167\n",
      "Epoch 20/100\n",
      "8/8 - 0s - loss: 3.1298 - auc: 0.5872 - acc: 0.3790 - entropy: 1.0721 - val_loss: 0.9706 - val_auc: 0.8074 - val_acc: 0.6562 - val_entropy: 0.9706\n",
      "Epoch 21/100\n",
      "8/8 - 0s - loss: 3.4137 - auc: 0.4873 - acc: 0.3468 - entropy: 1.1783 - val_loss: 1.1790 - val_auc: 0.4417 - val_acc: 0.2812 - val_entropy: 1.1790\n",
      "Epoch 22/100\n",
      "8/8 - 0s - loss: 3.2541 - auc: 0.6171 - acc: 0.3629 - entropy: 1.0603 - val_loss: 1.1385 - val_auc: 0.5271 - val_acc: 0.1562 - val_entropy: 1.1385\n",
      "Epoch 23/100\n",
      "8/8 - 0s - loss: 3.8976 - auc: 0.5440 - acc: 0.3468 - entropy: 1.2137 - val_loss: 0.8912 - val_auc: 0.7559 - val_acc: 0.5000 - val_entropy: 0.8912\n",
      "Epoch 24/100\n",
      "8/8 - 0s - loss: 3.8827 - auc: 0.4674 - acc: 0.3226 - entropy: 1.2060 - val_loss: 1.0223 - val_auc: 0.6846 - val_acc: 0.4062 - val_entropy: 1.0223\n",
      "Restoring model weights from the end of the best epoch.\n",
      "Epoch 00024: early stopping\n",
      "1/1 [==============================] - 0s 147ms/step - loss: 0.5259 - auc: 0.9297 - acc: 0.9062 - entropy: 0.5259\n",
      "Epoch 1/100\n",
      "8/8 - 1s - loss: 65.9737 - auc: 0.4791 - acc: 0.3226 - entropy: 14.5593 - val_loss: 5.5958 - val_auc: 0.9297 - val_acc: 0.9062 - val_entropy: 5.5958\n",
      "Epoch 2/100\n",
      "8/8 - 0s - loss: 103.4753 - auc: 0.3499 - acc: 0.1613 - entropy: 32.4078 - val_loss: 31.1509 - val_auc: 0.2734 - val_acc: 0.0312 - val_entropy: 31.1509\n",
      "Epoch 3/100\n",
      "8/8 - 0s - loss: 37.4865 - auc: 0.7165 - acc: 0.6129 - entropy: 7.2697 - val_loss: 3.8728 - val_auc: 0.5139 - val_acc: 0.0625 - val_entropy: 3.8728\n",
      "Epoch 4/100\n",
      "8/8 - 0s - loss: 32.0653 - auc: 0.4171 - acc: 0.2581 - entropy: 7.8914 - val_loss: 5.1764 - val_auc: 0.4849 - val_acc: 0.0312 - val_entropy: 5.1764\n",
      "Epoch 5/100\n",
      "8/8 - 0s - loss: 34.6335 - auc: 0.3498 - acc: 0.1935 - entropy: 9.8503 - val_loss: 0.8143 - val_auc: 0.9268 - val_acc: 0.9062 - val_entropy: 0.8143\n",
      "Epoch 6/100\n",
      "8/8 - 0s - loss: 23.0968 - auc: 0.2892 - acc: 0.1613 - entropy: 10.8535 - val_loss: 15.4342 - val_auc: 0.2734 - val_acc: 0.0312 - val_entropy: 15.4342\n",
      "Epoch 7/100\n",
      "8/8 - 0s - loss: 9.9895 - auc: 0.5650 - acc: 0.4194 - entropy: 3.5990 - val_loss: 0.6819 - val_auc: 0.9216 - val_acc: 0.9062 - val_entropy: 0.6819\n",
      "Epoch 8/100\n",
      "8/8 - 0s - loss: 13.5765 - auc: 0.2678 - acc: 0.1774 - entropy: 6.1439 - val_loss: 9.1648 - val_auc: 0.0605 - val_acc: 0.0312 - val_entropy: 9.1648\n",
      "Epoch 9/100\n",
      "8/8 - 0s - loss: 5.3130 - auc: 0.4425 - acc: 0.3065 - entropy: 2.7937 - val_loss: 0.4741 - val_auc: 0.9304 - val_acc: 0.9062 - val_entropy: 0.4741\n",
      "Epoch 10/100\n",
      "8/8 - 0s - loss: 8.0857 - auc: 0.3574 - acc: 0.2097 - entropy: 2.2402 - val_loss: 2.6065 - val_auc: 0.1523 - val_acc: 0.0312 - val_entropy: 2.6065\n",
      "Epoch 11/100\n",
      "8/8 - 0s - loss: 3.7294 - auc: 0.7006 - acc: 0.5484 - entropy: 1.0533 - val_loss: 0.4180 - val_auc: 0.9236 - val_acc: 0.9062 - val_entropy: 0.4180\n",
      "Epoch 12/100\n",
      "8/8 - 0s - loss: 6.3463 - auc: 0.2799 - acc: 0.2016 - entropy: 2.5286 - val_loss: 3.2695 - val_auc: 0.0593 - val_acc: 0.0312 - val_entropy: 3.2695\n",
      "Epoch 13/100\n",
      "8/8 - 0s - loss: 3.3331 - auc: 0.6088 - acc: 0.4516 - entropy: 1.2443 - val_loss: 0.4646 - val_auc: 0.9207 - val_acc: 0.9062 - val_entropy: 0.4646\n",
      "Epoch 14/100\n",
      "8/8 - 0s - loss: 5.1636 - auc: 0.3940 - acc: 0.2339 - entropy: 1.5082 - val_loss: 1.6868 - val_auc: 0.1362 - val_acc: 0.0312 - val_entropy: 1.6868\n",
      "Epoch 15/100\n",
      "8/8 - 0s - loss: 2.9365 - auc: 0.6924 - acc: 0.4919 - entropy: 1.0021 - val_loss: 0.5160 - val_auc: 0.9199 - val_acc: 0.9062 - val_entropy: 0.5160\n",
      "Epoch 16/100\n",
      "8/8 - 0s - loss: 4.6878 - auc: 0.3577 - acc: 0.2661 - entropy: 1.6935 - val_loss: 2.4927 - val_auc: 0.1396 - val_acc: 0.0312 - val_entropy: 2.4927\n",
      "Epoch 17/100\n",
      "8/8 - 0s - loss: 3.6088 - auc: 0.4296 - acc: 0.2984 - entropy: 1.4613 - val_loss: 1.1099 - val_auc: 0.5369 - val_acc: 0.1875 - val_entropy: 1.1099\n",
      "Epoch 18/100\n",
      "8/8 - 0s - loss: 3.2661 - auc: 0.6602 - acc: 0.4194 - entropy: 1.0175 - val_loss: 1.0350 - val_auc: 0.6702 - val_acc: 0.4688 - val_entropy: 1.0350\n",
      "Epoch 19/100\n",
      "8/8 - 0s - loss: 3.2129 - auc: 0.5372 - acc: 0.2903 - entropy: 1.1026 - val_loss: 1.2258 - val_auc: 0.4331 - val_acc: 0.0625 - val_entropy: 1.2258\n",
      "Epoch 20/100\n",
      "8/8 - 0s - loss: 3.2860 - auc: 0.5650 - acc: 0.3710 - entropy: 1.1081 - val_loss: 1.1576 - val_auc: 0.5107 - val_acc: 0.1875 - val_entropy: 1.1576\n",
      "Epoch 21/100\n",
      "8/8 - 0s - loss: 3.2981 - auc: 0.6032 - acc: 0.4032 - entropy: 1.0686 - val_loss: 1.2006 - val_auc: 0.4329 - val_acc: 0.1250 - val_entropy: 1.2006\n",
      "Epoch 22/100\n",
      "8/8 - 0s - loss: 3.1106 - auc: 0.5995 - acc: 0.3710 - entropy: 1.0653 - val_loss: 1.2200 - val_auc: 0.4326 - val_acc: 0.1250 - val_entropy: 1.2200\n",
      "Epoch 23/100\n",
      "8/8 - 0s - loss: 3.1005 - auc: 0.6261 - acc: 0.3790 - entropy: 1.0482 - val_loss: 1.2369 - val_auc: 0.4697 - val_acc: 0.1250 - val_entropy: 1.2369\n",
      "Epoch 24/100\n",
      "8/8 - 0s - loss: 3.2197 - auc: 0.6230 - acc: 0.3952 - entropy: 1.0603 - val_loss: 1.2999 - val_auc: 0.4529 - val_acc: 0.1250 - val_entropy: 1.2999\n",
      "Epoch 25/100\n",
      "8/8 - 0s - loss: 3.2764 - auc: 0.6152 - acc: 0.4113 - entropy: 1.0736 - val_loss: 1.2792 - val_auc: 0.4617 - val_acc: 0.1250 - val_entropy: 1.2792\n",
      "Epoch 26/100\n",
      "8/8 - 0s - loss: 3.2403 - auc: 0.6185 - acc: 0.4355 - entropy: 1.0670 - val_loss: 1.2543 - val_auc: 0.4512 - val_acc: 0.0938 - val_entropy: 1.2543\n",
      "Epoch 27/100\n",
      "8/8 - 0s - loss: 3.1065 - auc: 0.6229 - acc: 0.4355 - entropy: 1.0517 - val_loss: 1.2295 - val_auc: 0.4634 - val_acc: 0.1562 - val_entropy: 1.2295\n",
      "Epoch 28/100\n",
      "8/8 - 0s - loss: 3.0468 - auc: 0.6283 - acc: 0.4194 - entropy: 1.0444 - val_loss: 1.2373 - val_auc: 0.4668 - val_acc: 0.1562 - val_entropy: 1.2373\n",
      "Epoch 29/100\n",
      "8/8 - 0s - loss: 3.0914 - auc: 0.6254 - acc: 0.4113 - entropy: 1.0521 - val_loss: 1.2873 - val_auc: 0.4609 - val_acc: 0.0938 - val_entropy: 1.2873\n",
      "Epoch 30/100\n",
      "8/8 - 0s - loss: 3.2250 - auc: 0.6253 - acc: 0.3952 - entropy: 1.0635 - val_loss: 1.3594 - val_auc: 0.4475 - val_acc: 0.0625 - val_entropy: 1.3594\n",
      "Epoch 31/100\n",
      "8/8 - 0s - loss: 3.4840 - auc: 0.6186 - acc: 0.3871 - entropy: 1.0928 - val_loss: 1.3661 - val_auc: 0.4497 - val_acc: 0.0625 - val_entropy: 1.3661\n",
      "Restoring model weights from the end of the best epoch.\n",
      "Epoch 00031: early stopping\n",
      "1/1 [==============================] - 0s 148ms/step - loss: 0.4180 - auc: 0.9236 - acc: 0.9062 - entropy: 0.4180\n",
      "Epoch 1/100\n",
      "8/8 - 1s - loss: 58.8803 - auc: 0.5153 - acc: 0.3468 - entropy: 17.9762 - val_loss: 8.6230 - val_auc: 0.9297 - val_acc: 0.9062 - val_entropy: 8.6230\n",
      "Epoch 2/100\n",
      "8/8 - 0s - loss: 151.4703 - auc: 0.3624 - acc: 0.1774 - entropy: 40.2663 - val_loss: 30.1465 - val_auc: 0.2734 - val_acc: 0.0312 - val_entropy: 30.1465\n",
      "Epoch 3/100\n",
      "8/8 - 0s - loss: 49.4394 - auc: 0.4461 - acc: 0.3306 - entropy: 10.9968 - val_loss: 3.0234 - val_auc: 0.9297 - val_acc: 0.9062 - val_entropy: 3.0234\n",
      "Epoch 4/100\n",
      "8/8 - 0s - loss: 84.5207 - auc: 0.3284 - acc: 0.1371 - entropy: 31.7708 - val_loss: 42.7472 - val_auc: 0.2734 - val_acc: 0.0312 - val_entropy: 42.7472\n",
      "Epoch 5/100\n",
      "8/8 - 0s - loss: 26.8688 - auc: 0.6071 - acc: 0.4355 - entropy: 9.0903 - val_loss: 2.1178 - val_auc: 0.9297 - val_acc: 0.9062 - val_entropy: 2.1178\n",
      "Epoch 6/100\n",
      "8/8 - 0s - loss: 42.8602 - auc: 0.3285 - acc: 0.1613 - entropy: 13.0903 - val_loss: 19.1639 - val_auc: 0.2734 - val_acc: 0.0312 - val_entropy: 19.1639\n",
      "Epoch 7/100\n",
      "8/8 - 0s - loss: 15.4036 - auc: 0.5375 - acc: 0.3710 - entropy: 5.0783 - val_loss: 1.1805 - val_auc: 0.9297 - val_acc: 0.9062 - val_entropy: 1.1805\n",
      "Epoch 8/100\n",
      "8/8 - 0s - loss: 23.5373 - auc: 0.3085 - acc: 0.1613 - entropy: 6.3599 - val_loss: 7.6301 - val_auc: 0.0945 - val_acc: 0.0312 - val_entropy: 7.6301\n",
      "Epoch 9/100\n",
      "8/8 - 0s - loss: 6.3768 - auc: 0.5670 - acc: 0.3790 - entropy: 2.1293 - val_loss: 0.5014 - val_auc: 0.9307 - val_acc: 0.9062 - val_entropy: 0.5014\n",
      "Epoch 10/100\n",
      "8/8 - 0s - loss: 9.3811 - auc: 0.3740 - acc: 0.1935 - entropy: 2.5319 - val_loss: 2.4306 - val_auc: 0.1072 - val_acc: 0.0312 - val_entropy: 2.4306\n",
      "Epoch 11/100\n",
      "8/8 - 0s - loss: 3.7138 - auc: 0.5240 - acc: 0.3387 - entropy: 1.3150 - val_loss: 0.6576 - val_auc: 0.9106 - val_acc: 0.8750 - val_entropy: 0.6576\n",
      "Epoch 12/100\n",
      "8/8 - 0s - loss: 4.1739 - auc: 0.5490 - acc: 0.3871 - entropy: 1.1955 - val_loss: 1.1213 - val_auc: 0.5225 - val_acc: 0.3125 - val_entropy: 1.1213\n",
      "Epoch 13/100\n",
      "8/8 - 0s - loss: 3.2378 - auc: 0.5968 - acc: 0.3790 - entropy: 1.0711 - val_loss: 1.1478 - val_auc: 0.5122 - val_acc: 0.1875 - val_entropy: 1.1478\n",
      "Epoch 14/100\n",
      "8/8 - 0s - loss: 3.6915 - auc: 0.4043 - acc: 0.2661 - entropy: 1.3019 - val_loss: 1.3662 - val_auc: 0.2534 - val_acc: 0.2188 - val_entropy: 1.3662\n",
      "Epoch 15/100\n",
      "8/8 - 0s - loss: 3.0235 - auc: 0.7071 - acc: 0.6371 - entropy: 0.9901 - val_loss: 0.8231 - val_auc: 0.8843 - val_acc: 0.8750 - val_entropy: 0.8231\n",
      "Epoch 16/100\n",
      "8/8 - 0s - loss: 3.6911 - auc: 0.5233 - acc: 0.4355 - entropy: 1.1784 - val_loss: 1.3970 - val_auc: 0.2380 - val_acc: 0.1875 - val_entropy: 1.3970\n",
      "Epoch 17/100\n",
      "8/8 - 0s - loss: 3.0138 - auc: 0.6152 - acc: 0.4113 - entropy: 1.0614 - val_loss: 1.0102 - val_auc: 0.6057 - val_acc: 0.2188 - val_entropy: 1.0102\n",
      "Epoch 18/100\n",
      "8/8 - 0s - loss: 3.5748 - auc: 0.5425 - acc: 0.3710 - entropy: 1.1426 - val_loss: 1.2596 - val_auc: 0.3982 - val_acc: 0.2812 - val_entropy: 1.2596\n",
      "Epoch 19/100\n",
      "8/8 - 0s - loss: 3.0371 - auc: 0.6619 - acc: 0.4355 - entropy: 1.0176 - val_loss: 1.0136 - val_auc: 0.6025 - val_acc: 0.2500 - val_entropy: 1.0136\n",
      "Epoch 20/100\n",
      "8/8 - 0s - loss: 3.5305 - auc: 0.5533 - acc: 0.4032 - entropy: 1.1299 - val_loss: 1.2717 - val_auc: 0.3916 - val_acc: 0.2812 - val_entropy: 1.2717\n",
      "Epoch 21/100\n",
      "8/8 - 0s - loss: 3.0408 - auc: 0.6309 - acc: 0.4274 - entropy: 1.0485 - val_loss: 1.0650 - val_auc: 0.5754 - val_acc: 0.2500 - val_entropy: 1.0650\n",
      "Epoch 22/100\n",
      "8/8 - 0s - loss: 3.4371 - auc: 0.6036 - acc: 0.4113 - entropy: 1.0778 - val_loss: 1.1774 - val_auc: 0.5249 - val_acc: 0.2812 - val_entropy: 1.1774\n",
      "Epoch 23/100\n",
      "8/8 - 0s - loss: 3.0942 - auc: 0.6010 - acc: 0.4032 - entropy: 1.0787 - val_loss: 1.1481 - val_auc: 0.5220 - val_acc: 0.2188 - val_entropy: 1.1481\n",
      "Epoch 24/100\n",
      "8/8 - 0s - loss: 3.3550 - auc: 0.6363 - acc: 0.4355 - entropy: 1.0517 - val_loss: 1.0969 - val_auc: 0.6113 - val_acc: 0.3438 - val_entropy: 1.0969\n",
      "Epoch 25/100\n",
      "8/8 - 0s - loss: 3.1309 - auc: 0.5838 - acc: 0.3790 - entropy: 1.0968 - val_loss: 1.2005 - val_auc: 0.4978 - val_acc: 0.2188 - val_entropy: 1.2005\n",
      "Epoch 26/100\n",
      "8/8 - 0s - loss: 3.3564 - auc: 0.6380 - acc: 0.4355 - entropy: 1.0550 - val_loss: 1.0575 - val_auc: 0.6467 - val_acc: 0.3750 - val_entropy: 1.0575\n",
      "Epoch 27/100\n",
      "8/8 - 0s - loss: 3.1346 - auc: 0.5858 - acc: 0.3790 - entropy: 1.1011 - val_loss: 1.2021 - val_auc: 0.5017 - val_acc: 0.1875 - val_entropy: 1.2021\n",
      "Epoch 28/100\n",
      "8/8 - 0s - loss: 3.4115 - auc: 0.6227 - acc: 0.4274 - entropy: 1.0765 - val_loss: 1.0510 - val_auc: 0.6492 - val_acc: 0.3750 - val_entropy: 1.0510\n",
      "Epoch 29/100\n",
      "8/8 - 0s - loss: 3.1250 - auc: 0.5989 - acc: 0.3710 - entropy: 1.0921 - val_loss: 1.1853 - val_auc: 0.5122 - val_acc: 0.1875 - val_entropy: 1.1853\n",
      "Restoring model weights from the end of the best epoch.\n",
      "Epoch 00029: early stopping\n",
      "1/1 [==============================] - 0s 146ms/step - loss: 0.5014 - auc: 0.9307 - acc: 0.9062 - entropy: 0.5014\n",
      "auc : average=0.918, std=0.017\n",
      "acc : average=0.887, std=0.038\n",
      "entropy : average=0.535, std=0.121\n"
     ]
    }
   ],
   "source": [
    "calculate_mean_std(weights=weights)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "87feafaf",
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python [conda env:generalml_p37_gpu_v1]",
   "language": "python",
   "name": "conda-env-generalml_p37_gpu_v1-py"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.12"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}