main_covid19.py

# -*- coding: utf-8 -*-
"""main_covid19.ipynb

Automatically generated by Colaboratory.

Original file is located at
    https://colab.research.google.com/drive/1XjAMst3rLws10_wQap2Rpp-VwHFkTZeN
"""

# from google.colab import drive
# drive.mount('/content/drive')

# predict the cases(new),30 days winsize.
#%% Module
import os
import pickle
import datetime
import numpy as np
import pandas as pd

from classes import EDA
from classes import MAPE
from classes import ModelEvaluation
from classes import ModelDevelopment

from sklearn.preprocessing import MinMaxScaler
from sklearn.metrics import mean_absolute_percentage_error

from tensorflow.keras.utils import plot_model
from tensorflow.keras.callbacks import TensorBoard

#%% Constant
CSV_PATH_TRAIN = os.path.join(os.getcwd(),'Datasets',
                              'cases_malaysia_train.csv')
CSV_PATH_TEST = os.path.join(os.getcwd(),'Datasets',
                             'cases_malaysia_test.csv')
MMS_SAVE_PATH = os.path.join(os.getcwd(),'models', 'mms_train_transform.pkl')
MMS_SAVE_PATH = os.path.join(os.getcwd(),'models', 'mms_test_transform.pkl')
LOGS_PATH = os.path.join(os.getcwd(),'logs',datetime.datetime.now()
                          .strftime('%Y%m%d-%H%M%S'))

#%% Step 1) Data Loading
df_train = pd.read_csv(CSV_PATH_TRAIN)
df_test = pd.read_csv(CSV_PATH_TEST)

# From the dataset, the are a few rows in df[cases_new] have empty space and ?.
# To change cases_new into numeric and change ? and empty space into NaNs
df_train['cases_new'] = pd.to_numeric(df_train['cases_new'], errors='coerce')

#%% Step 2) Data Inspection
# Train dataset
df_train.head()
df_train.info()
df_train.describe().T

# Time Series Data
con_col_train = df_train.columns[(df_train.dtypes=='float64') | 
                                 (df_train.dtypes=='int64')]
print(con_col_train)

# time Series Data Visualization (Line Plot)
eda = EDA()
eda.lineplot_graph(con_col_train,df_train)

# To check NaNs value
df_train.isna().sum()
# From this train dataset there are 12 in cases_new and 
# 342 NaNs in cluster_import,cluster_religious,cluster_community,
# cluster_highRisk,cluster_education,cluster_detentionCentre, 
# and cluster_workplace.
# This is because there are not yet this clusters in the begining of Covid-19.

# Test Dataset
df_test.head()
df_test.info()
df_test.describe().T

# Time Series Data
con_col_test = df_test.columns[(df_test.dtypes=='float64') | 
                               (df_test.dtypes=='int64')]
print(con_col_test)

# time Series Data Visualization (Line Plot)

eda = EDA()
eda.lineplot_graph(con_col_test,df_test) 

# To check NaNs value
df_test.isna().sum()
# From this test dataset there are 1 NaNs in cases_new

#%% Step 3: Data Cleaning
# Do not clean time series data unless REALLY necessary
# But this data have NaNs, so we need to used intepolate.
# Both train and test dataset have NaNs in cases_new column.

# Train Dataset
df_train['cases_new'] = df_train['cases_new'].interpolate()
# To check the NaNs
df_train.isna().sum()
# NaNs in cases_new in train dataset have been interpolate

# Test Dataset
df_test['cases_new'] = df_test['cases_new'].interpolate()
# To check the NaNs
df_test.isna().sum()
# NaNs in cases_new in test dataset have been interpolate

#%% Step 4: Features Selection

# cases_new - Only 1 feature 
# Train Dataset
X = df_train['cases_new']

# Method 1: MinMaxScaler
mms = MinMaxScaler()
X = mms.fit_transform(np.expand_dims(X,axis=-1))

# Save Train Min Max Scaler
with open(MMS_SAVE_PATH, 'wb') as file:
    pickle.dump(mms.transform,file)

win_size = 30
X_train = []
y_train = []

# produce list
for i in range(win_size,len(X)):
  X_train.append(X[i-win_size:i])
  y_train.append(X[i])

# change list to rank 3 array by using np.array
X_train = np.array(X_train) 
y_train = np.array(y_train)

#%% Test Dataset

# We need 30 days of data to predict the number of covid19 cases
# The test data only have 100 sample so we need to concatenate together with 
# train data

#Must to concatenate
dataset_cat = pd.concat((df_train['cases_new'],df_test['cases_new']))

# Method 1
length_days = len(dataset_cat)-len(df_test)-win_size
tot_input = dataset_cat[length_days:] 

Xtest = mms.transform(np.expand_dims(tot_input,axis=-1))

# Save Test Min Max Scaler
with open(MMS_SAVE_PATH, 'wb') as file:
    pickle.dump(mms.transform,file)

X_test = []
y_test = []

# Produce list
for i in range(win_size,len(Xtest)):
  X_test.append(Xtest[i-win_size:i])
  y_test.append(Xtest[i])

# Change list to rank 3 array by using np.array
X_test = np.array(X_test) 
y_test = np.array(y_test)

#%% Step 5: Data Preprocessing
#%% Model Development

md = ModelDevelopment()

input_shape = np.shape(X_train)[1:]
nb_class = 1
nb_node = 32
dropout_rate = 0.2
activation = 'relu'

model = md.simple_dl_model(input_shape,nb_class,nb_node,
                           dropout_rate,activation)

# Visualization the model
plot_model(model,show_shapes=True,show_layer_names=True)

#%% Model Training
model.compile(optimizer='adam',loss='mse',
              metrics=['mean_absolute_percentage_error','mse'])

#Callbacks
#Tensorboard Callbacks
tensorboard_callback = TensorBoard(log_dir=LOGS_PATH,histogram_freq=1)

hist = model.fit(X_train,y_train,
                  epochs=500,
                  validation_data=(X_test,y_test),
                  callbacks=(tensorboard_callback))

print(hist.history.keys())

#%% Plot Graph
me = ModelEvaluation()
me.plot_hist_graphy(hist)

predicted_new_cases= model.predict(X_test)

# Plot line graph
me.plot_line_graph(y_test, predicted_new_cases)

# Plot inverse graph
actual_cases = mms.inverse_transform(y_test)
predicted_cases = mms.inverse_transform(predicted_new_cases)
me.inverse_line_graph(actual_cases, predicted_cases)

# Mean Absolute Percentage Error
print(mean_absolute_percentage_error(actual_cases,predicted_cases))
# Mean Absolute Percentage Error in percentage
mp = MAPE()
print("Mean_Absolute_Percentage_Error: {}".format(mp.mape(actual_cases,
                                                       predicted_cases)))

#%%
# Commented out IPython magic to ensure Python compatibility.
# %reload_ext tensorboard

# Commented out IPython magic to ensure Python compatibility.
# %tensorboard --logdir logs