BASE_1H_Close.ipynb

{"nbformat": 4, "nbformat_minor": 0, "metadata": {"colab": {"name": "BASE_1H_Close.ipynb", "provenance": [], "collapsed_sections": [], "machine_shape": "hm", "include_colab_link": true}, "kernelspec": {"name": "python3", "display_name": "Python 3"}, "accelerator": "GPU"}, "cells": [{"cell_type": "markdown", "metadata": {"id": "view-in-github", "colab_type": "text"}, "source": ["<a href=\"https://colab.research.google.com/github/ahmedengu/feature_importance/blob/master/BASE_1H_Close.ipynb\" target=\"_parent\"><img src=\"https://colab.research.google.com/assets/colab-badge.svg\" alt=\"Open In Colab\"/></a>"]}, {"cell_type": "code", "metadata": {"id": "tfoR871kPEFI", "colab_type": "code", "colab": {}}, "source": ["from IPython.display import clear_output\n", "import pandas as pd\n", "\n", "try:\n", "  from findatapy.market import Market, MarketDataRequest, MarketDataGenerator\n", "except:\n", "  # install\n", "  !apt-get install default-jre build-essential swig\n", "  !pip  install h2o==3.24.0.5 yfinance==0.1.43\n", "  !pip  install git+https://github.com/ahmedengu/findatapy.git\n", "  clear_output(wait=True)\n", "\n", "  !wget http://prdownloads.sourceforge.net/ta-lib/ta-lib-0.4.0-src.tar.gz\n", "  !tar -xzvf ta-lib-0.4.0-src.tar.gz\n", "  %cd ta-lib\n", "  !./configure --prefix=/usr\n", "  clear_output(wait=True)\n", "  !make\n", "  clear_output(wait=True)\n", "  !make install\n", "  clear_output(wait=True)\n", "  !pip install Ta-Lib==0.4.17\n", "  clear_output(wait=True)"], "execution_count": 0, "outputs": []}, {"cell_type": "code", "metadata": {"id": "xthkIDUZJQDN", "colab_type": "code", "colab": {}}, "source": ["# Experiment Config\n", "import pandas as pd\n", "from IPython.display import display\n", "from IPython.core.debugger import set_trace\n", "\n", "data_ticker = \"\"\n", "data_resample = '1H'\n", "data_label = 'Close'\n", "data_label_name = 'y'\n", "start_date = '1 Jan 2018'\n", "finish_date = '1 Jan 2019'\n", "is_duka = True\n", "is_yahoo = False\n", "kaggle_file = 'bitstamp' # default None\n", "kaggle_folder = '1 Day/Stocks'\n", "\n", "folds = 0 # should be 0, corss validation cause overfitting\n", "\n", "models_summary = {}\n", "for metric in ['accuracy','F1','auc','logloss','mean_per_class_error','rmse','mse']:\n", "  models_summary.update({metric:pd.DataFrame()})\n", "\n", "best_models = {}\n", "ensemble_model_threshold = 0.05\n", "split_percentages = [0.2,0.1] # validation size, test size\n", "\n", "# feature elimination config\n", "model_selected_features = 50\n", "PCA_k = 50\n", "HCF_threshold = 0.90\n", "LVF_threshold = 0.1\n", "GLRM_k = 20\n", "select_k = 50\n", "select_percentile = 10\n", "select_alpha = 0.1\n", "select_rfe = 50\n", "cv_rfe = 5\n", "\n", "features_to_include = ['All', 'DLF', 'XGBF', 'RFE', 'Fwe', 'Super_learner'] # True for all , ['All','Fwe','RFE','RFECV','Percentile','KBest','LVF','HCF','PCA','XGBF','GLMF','DLF','GLRM'] \n", "algos_to_include = [None, 'GLM' ,'XGBoost' ,'DeepLearning'] # [None, 'DRF', 'GLM' ,'XGBoost' ,'GBM' ,'DeepLearning' ,'StackedEnsemble'], None for all\n", "is_EDA = True\n", "is_minmax = True # use minmax\n", "max_models_num = 10\n", "\n", "# pandas options\n", "pd.set_option('display.max_columns', 500)\n", "pd.set_option('display.width', 1000)"], "execution_count": 0, "outputs": []}, {"cell_type": "code", "metadata": {"id": "YcArITcU2ieo", "colab_type": "code", "colab": {}}, "source": ["# random seed\n", "from numpy.random import seed\n", "seed(1)\n", "from tensorflow import set_random_seed\n", "set_random_seed(1)\n", "import pandas as pd\n", "import gc\n", "import os\n", "\n", "resample_how= {'Open': 'first', 'High': 'max', 'Low': 'min', 'Close': 'last',\n", "                             'Volume': 'sum', 'Date': 'first'}\n", "\n", "if is_duka:\n", "  from findatapy.util import DataConstants\n", "\n", "  DataConstants.market_thread_no.update({'dukascopy': 64})\n", "\n", "  # Download dataset\n", "  from findatapy.market import Market, MarketDataRequest, MarketDataGenerator\n", "\n", "  market = Market(market_data_generator=MarketDataGenerator(log_every_day=30))\n", "\n", "  md_request = MarketDataRequest(start_date=start_date, finish_date=finish_date,\n", "                                       fields=['temp', 'ask', 'bid', 'askv', 'bidv'], vendor_fields=['temp', 'ask', 'bid', 'askv', 'bidv'],\n", "                                       freq='tick', data_source='dukascopy',\n", "                                       tickers=[data_ticker], vendor_tickers=[data_ticker])\n", "\n", "  temp_dataset = market.fetch_market(md_request)\n", "  full_dataset = pd.DataFrame(temp_dataset[data_ticker + '.bid'].resample(data_resample).ohlc())\n", "  full_dataset.columns = ['Open', 'High', 'Low', 'Close']\n", "  temp_dataset = pd.DataFrame(temp_dataset[data_ticker + '.bidv'].resample(data_resample).sum())\n", "  temp_dataset.columns = ['Volume']\n", "  full_dataset['Volume'] = temp_dataset['Volume']\n", "\n", "  del temp_dataset, md_request, market, DataConstants, Market, MarketDataRequest, MarketDataGenerator\n", "\n", "  gc.collect()\n", "\n", "elif is_yahoo:\n", "  import yfinance as yf\n", "  from datetime import datetime\n", "\n", "  full_dataset = yf.download(data_ticker, start=str(datetime.strptime(start_date, '%d %b %Y')).split(' ')[0], end=str(datetime.strptime(finish_date, '%d %b %Y')).split(' ')[0])\n", "  full_dataset['Date']=full_dataset.index\n", "  del yf,datetime\n", "\n", "elif data_ticker == 'EURUSD':\n", "  !wget -q https://github.com/philipperemy/FX-1-Minute-Data/raw/master/2000-Jun2019/eurusd/DAT_ASCII_EURUSD_M1_2014.zip\n", "  !wget -q https://github.com/philipperemy/FX-1-Minute-Data/raw/master/2000-Jun2019/eurusd/DAT_ASCII_EURUSD_M1_2015.zip\n", "  !wget -q https://github.com/philipperemy/FX-1-Minute-Data/raw/master/2000-Jun2019/eurusd/DAT_ASCII_EURUSD_M1_2016.zip\n", "  !wget -q https://github.com/philipperemy/FX-1-Minute-Data/raw/master/2000-Jun2019/eurusd/DAT_ASCII_EURUSD_M1_2017.zip\n", "  !wget -q https://github.com/philipperemy/FX-1-Minute-Data/raw/master/2000-Jun2019/eurusd/DAT_ASCII_EURUSD_M1_2018.zip\n", "  !unzip -qq -o \\*.zip\n", "  full_dataset = pd.read_csv('DAT_ASCII_EURUSD_M1_2014.csv',names=['Date','Open', 'High', 'Low', 'Close','Volume'], delimiter=';')\n", "  full_dataset = full_dataset.append(pd.read_csv('DAT_ASCII_EURUSD_M1_2015.csv',names=['Date','Open', 'High', 'Low', 'Close','Volume'], delimiter=';'))\n", "  full_dataset = full_dataset.append(pd.read_csv('DAT_ASCII_EURUSD_M1_2016.csv',names=['Date','Open', 'High', 'Low', 'Close','Volume'], delimiter=';'))\n", "  full_dataset = full_dataset.append(pd.read_csv('DAT_ASCII_EURUSD_M1_2017.csv',names=['Date','Open', 'High', 'Low', 'Close','Volume'], delimiter=';'))\n", "  full_dataset = full_dataset.append(pd.read_csv('DAT_ASCII_EURUSD_M1_2018.csv',names=['Date','Open', 'High', 'Low', 'Close','Volume'], delimiter=';'))\n", "  full_dataset['Date'] = pd.to_datetime(full_dataset['Date'])\n", "  full_dataset.index = full_dataset['Date']\n", "  full_dataset = full_dataset.dropna()\n", "  full_dataset = full_dataset.sort_index()\n", "  full_dataset = full_dataset.resample(data_resample, how=resample_how).dropna()\n", "  full_dataset = full_dataset[full_dataset.index > start_date]\n", "  full_dataset = full_dataset[full_dataset.index < finish_date]\n", "\n", "\n", "    \n", "elif data_ticker == 'APPLUSUSD':\n", "  full_dataset=pd.read_csv('https://github.com/kyleconroy/apple-stock/raw/master/apple_stock_data.csv',index_col =0,parse_dates =[0])\n", "\n", "  full_dataset = full_dataset.astype(float)\n", "  full_dataset['Date']=full_dataset.index\n", "  full_dataset = full_dataset[full_dataset.index > '2007-04-01']\n", "\n", "\n", "elif data_ticker == 'COMI':\n", "  full_dataset=pd.read_csv('https://static.mubasher.info/File.MubasherCharts/File.Historical_Stock_Charts_Dir/44507230b2b03e2da37352abf1a659545b44.csv',index_col =0,names =['Open', 'High', 'Low', 'Close','Volume'],parse_dates =[0])\n", "\n", "  full_dataset = full_dataset.astype(float)\n", "  full_dataset['Volume'] = full_dataset['Volume'] * full_dataset['Close'] \n", "  full_dataset = full_dataset.drop_duplicates()\n", "  full_dataset['Date']=full_dataset.index\n", "  full_dataset = full_dataset[full_dataset.index > start_date]\n", "  full_dataset = full_dataset[full_dataset.index < finish_date]\n", "  \n", "elif kaggle_file is not None:\n", "  if not os.path.isfile('kaggle.json'):\n", "    from google.colab import files\n", "    files.upload()\n", "    \n", "    import time\n", "    time.sleep(10)\n", "    !ls\n", "    !mkdir -p ~/.kaggle\n", "    !cp kaggle.json ~/.kaggle/\n", "    !ls ~/.kaggle\n", "\n", "    !chmod 600 /root/.kaggle/kaggle.json\n", "    del files\n", "    \n", "  !kaggle datasets download $data_ticker\n", "  !unzip -qq -o \\*.zip\n", "  !unzip -qq -o \\*.zip\n", "\n", "  if data_ticker == 'mczielinski/bitcoin-historical-data':\n", "    \n", "    full_dataset = pd.read_csv([file for file in os.listdir() if kaggle_file in file][0])\n", "    full_dataset.columns= ['Date', 'Open', 'High', 'Low', 'Close', 'Volume', 'Volume_(Currency)','Weighted_Price']\n", "    full_dataset.index=pd.to_datetime(full_dataset['Date'], unit='s')\n", "\n", "    resample_how.update({'Volume_(Currency)': 'sum', 'Weighted_Price': 'last'})\n", "\n", "    full_dataset = full_dataset.resample(data_resample, how=resample_how).dropna()[1:-1]\n", "    full_dataset = full_dataset[full_dataset.index > start_date]\n", "    full_dataset = full_dataset[full_dataset.index < finish_date]\n", "  \n", "  elif data_ticker == 'borismarjanovic/daily-and-intraday-stock-price-data':\n", "    \n", "    full_dataset = pd.read_csv(kaggle_folder + '/' + [file for file in os.listdir(kaggle_folder) if kaggle_file in file][0])\n", "    if 'Time' in full_dataset.columns:\n", "      full_dataset['Date'] = pd.to_datetime(full_dataset['Date'] + ' ' + full_dataset['Time'])\n", "      full_dataset = full_dataset.drop('Time',axis=1)\n", "    full_dataset.columns= ['Date', 'Open', 'High', 'Low', 'Close', 'Volume', 'OpenInt']\n", "    full_dataset.index=pd.to_datetime(full_dataset['Date'])\n", "\n", "    full_dataset = full_dataset[full_dataset.index > start_date]\n", "    full_dataset = full_dataset[full_dataset.index < finish_date]\n", "\n", "  \n", "gc.collect()\n", "full_dataset = full_dataset.dropna()\n", "\n", "full_dataset = full_dataset.drop('Date',axis=1, errors='ignore')\n", "full_dataset = full_dataset.sort_index()\n", "full_dataset = full_dataset[-100000:]\n", "display(full_dataset)\n", "display(full_dataset.describe())\n", "full_dataset.to_pickle('full_dataset')\n", "del full_dataset\n", "gc.collect()"], "execution_count": 0, "outputs": []}, {"cell_type": "code", "metadata": {"id": "pNRF63LWfK95", "colab_type": "code", "colab": {}}, "source": ["# OHLC EDA\n", "if is_EDA:\n", "  import pandas_profiling as pp\n", "  display(pp.ProfileReport(pd.read_pickle('full_dataset')))\n", "  del pp"], "execution_count": 0, "outputs": []}, {"cell_type": "code", "metadata": {"id": "unAs3siDNorC", "colab_type": "code", "colab": {}}, "source": ["# features\n", "from talib._ta_lib import *\n", "import talib\n", "import numpy as np\n", "import os\n", "\n", "def fe(last_index = None):\n", "  features_dataset = pd.read_pickle('full_dataset').sort_index()\n", "  if last_index is not None:\n", "    features_dataset = features_dataset[:-last_index]\n", "\n", "\n", "  periods = np.resize([2, 3, 3, 2], len(features_dataset['Close'])).astype(float)\n", "  timeperiods = [8, 12, 14, 20, 24, 30]\n", "  matypes = [0,1,2,3,4,5,6,8]\n", "  nbdevs = [2, 3, 4, 5]\n", "\n", "  # date features\n", "  features_dataset['day'] = features_dataset.index.day\n", "  features_dataset['dayofweek'] = features_dataset.index.dayofweek\n", "  features_dataset['dayofyear'] = features_dataset.index.dayofyear\n", "  features_dataset['days_in_month'] = features_dataset.index.days_in_month\n", "  features_dataset['hour'] = features_dataset.index.hour\n", "  features_dataset['is_month_end'] = pd.factorize(features_dataset.index.is_month_end)[0]\n", "  features_dataset['is_month_start'] = pd.factorize(features_dataset.index.is_month_start)[0]\n", "  features_dataset['is_quarter_end'] = pd.factorize(features_dataset.index.is_quarter_end)[0]\n", "  features_dataset['is_quarter_start'] = pd.factorize(features_dataset.index.is_quarter_start)[0]\n", "  features_dataset['minute'] = features_dataset.index.minute\n", "  features_dataset['month'] = features_dataset.index.month\n", "  features_dataset['quarter'] = features_dataset.index.quarter\n", "  features_dataset['second'] = features_dataset.index.second\n", "  features_dataset['week'] = features_dataset.index.week\n", "  features_dataset['weekday'] = features_dataset.index.weekday\n", "  features_dataset['weekday_name'] = pd.factorize(features_dataset.index.weekday_name)[0]\n", "  features_dataset['weekofyear'] = features_dataset.index.weekofyear\n", "\n", "  # Overlap Studies\n", "  features_dataset['HT_TRENDLINE'] = HT_TRENDLINE(features_dataset['Close'])\n", "  features_dataset['mama'], features_dataset['fama'] = MAMA(features_dataset['Close'])\n", "  features_dataset['SAREXT'] = SAREXT(features_dataset['High'], features_dataset['Low'], startvalue=-4e37,\n", "                                      offsetonreverse=-4e37, accelerationinitlong=-4e37,\n", "                                      accelerationlong=-4e37, accelerationmaxlong=-4e37, accelerationinitshort=-4e37,\n", "                                      accelerationshort=-4e37, accelerationmaxshort=-4e37)\n", "\n", "  for timeperiod in timeperiods:\n", "      features_dataset['DEMA_' + str(timeperiod)] = DEMA(features_dataset['Close'], timeperiod=timeperiod)\n", "      features_dataset['EMA_' + str(timeperiod)] = EMA(features_dataset['Close'], timeperiod=timeperiod)\n", "      features_dataset['KAMA_' + str(timeperiod)] = KAMA(features_dataset['Close'], timeperiod=timeperiod)\n", "      features_dataset['MIDPOINT_' + str(timeperiod)] = MIDPOINT(features_dataset['Close'], timeperiod=timeperiod)\n", "      features_dataset['MIDPRICE_' + str(timeperiod)] = MIDPRICE(features_dataset['High'], features_dataset['Low'],\n", "                                                                timeperiod=timeperiod)\n", "      features_dataset['SAR_' + str(timeperiod / 100)] = SAR(features_dataset['High'], features_dataset['Low'],\n", "                                                            acceleration=timeperiod / 100,\n", "                                                            maximum=timeperiod / 100)\n", "      features_dataset['SMA_' + str(timeperiod)] = SMA(features_dataset['Close'], timeperiod=timeperiod)\n", "      features_dataset['TEMA_' + str(timeperiod)] = TEMA(features_dataset['Close'], timeperiod=timeperiod)\n", "      features_dataset['TRIMA_' + str(timeperiod)] = TRIMA(features_dataset['Close'], timeperiod=timeperiod)\n", "      features_dataset['WMA_' + str(timeperiod)] = WMA(features_dataset['Close'], timeperiod=timeperiod)\n", "\n", "      for matype in matypes:\n", "          features_dataset['MA_' + str(timeperiod) + '_' + str(matype)] = MA(features_dataset['Close'],\n", "                                                                            timeperiod=timeperiod, matype=matype)\n", "          features_dataset['MAVP_' + str(timeperiod) + '_' + str(matype)] = MAVP(features_dataset['Close'], periods,\n", "                                                                                minperiod=2,\n", "                                                                                maxperiod=timeperiod, matype=matype)\n", "          features_dataset['T3_' + str(timeperiod) + '_' + str(matype)] = T3(features_dataset['Close'],\n", "                                                                            timeperiod=timeperiod,\n", "                                                                            vfactor=matype / 10)\n", "\n", "          for nbdev in nbdevs:\n", "              features_dataset['BBANDS_upperband_' + str(timeperiod) + '_' + str(matype) + '_' + str(nbdev)], \\\n", "              features_dataset['BBANDS_middleband_' + str(timeperiod) + '_' + str(matype) + '_' + str(nbdev)], \\\n", "              features_dataset['BBANDS_lowerband_' + str(timeperiod) + '_' + str(matype) + '_' + str(nbdev)] = BBANDS(\n", "                  features_dataset['Close'], timeperiod=timeperiod, nbdevup=nbdev, nbdevdn=nbdev, matype=matype)\n", "\n", "              \n", "  gc.collect()\n", "  # Momentum Indicators\n", "  features_dataset['BOP'] = BOP(features_dataset['Open'], features_dataset['High'], features_dataset['Low'],\n", "                                features_dataset['Close'])\n", "\n", "  for timeperiod in timeperiods:\n", "      features_dataset['ADX_' + str(timeperiod)] = ADX(features_dataset['High'], features_dataset['Low'],\n", "                                                      features_dataset['Close'], timeperiod=timeperiod)\n", "      features_dataset['ADXR_' + str(timeperiod)] = ADXR(features_dataset['High'], features_dataset['Low'],\n", "                                                        features_dataset['Close'], timeperiod=timeperiod)\n", "      features_dataset['aroondown_' + str(timeperiod)], features_dataset['aroonup_' + str(timeperiod)] = AROON(\n", "          features_dataset['High'],\n", "          features_dataset['Low'],\n", "          timeperiod=timeperiod)\n", "      features_dataset['AROONOSC_' + str(timeperiod)] = AROONOSC(features_dataset['High'], features_dataset['Low'],\n", "                                                                timeperiod=timeperiod)\n", "      features_dataset['CCI_' + str(timeperiod)] = CCI(features_dataset['High'], features_dataset['Low'],\n", "                                                      features_dataset['Close'], timeperiod=timeperiod)\n", "      features_dataset['CMO_' + str(timeperiod)] = CMO(features_dataset['Close'], timeperiod=timeperiod)\n", "      features_dataset['DX_' + str(timeperiod)] = DX(features_dataset['High'], features_dataset['Low'],\n", "                                                    features_dataset['Close'], timeperiod=timeperiod)\n", "      features_dataset['MACDFIX_' + str(timeperiod)], features_dataset['MACDFIX_signal_' + str(timeperiod)], features_dataset[\n", "          'MACDFIX_hist_' + str(timeperiod)] = MACDFIX(features_dataset['Close'], signalperiod=timeperiod)\n", "      features_dataset['MFI_' + str(timeperiod)] = MFI(features_dataset['High'], features_dataset['Low'],\n", "                                                      features_dataset['Close'], features_dataset['Volume'],\n", "                                                      timeperiod=timeperiod)\n", "      features_dataset['MINUS_DI_' + str(timeperiod)] = MINUS_DI(features_dataset['High'], features_dataset['Low'],\n", "                                                                features_dataset['Close'], timeperiod=timeperiod)\n", "      features_dataset['MINUS_DM_' + str(timeperiod)] = MINUS_DM(features_dataset['High'], features_dataset['Low'],\n", "                                                                timeperiod=timeperiod)\n", "      features_dataset['MOM_' + str(timeperiod)] = MOM(features_dataset['Close'], timeperiod=timeperiod)\n", "      features_dataset['PLUS_DI_' + str(timeperiod)] = PLUS_DI(features_dataset['High'], features_dataset['Low'],\n", "                                                              features_dataset['Close'], timeperiod=timeperiod)\n", "      features_dataset['PLUS_DM_' + str(timeperiod)] = PLUS_DM(features_dataset['High'], features_dataset['Low'],\n", "                                                              timeperiod=timeperiod)\n", "      features_dataset['ROC_' + str(timeperiod)] = ROC(features_dataset['Close'], timeperiod=timeperiod)\n", "      features_dataset['ROCP_' + str(timeperiod)] = ROCP(features_dataset['Close'], timeperiod=timeperiod)\n", "      features_dataset['ROCR_' + str(timeperiod)] = ROCR(features_dataset['Close'], timeperiod=timeperiod)\n", "      features_dataset['ROCR100_' + str(timeperiod)] = ROCR100(features_dataset['Close'], timeperiod=timeperiod)\n", "      features_dataset['RSI_' + str(timeperiod)] = RSI(features_dataset['Close'], timeperiod=timeperiod)\n", "      features_dataset['TRIX_' + str(timeperiod)] = TRIX(features_dataset['Close'], timeperiod=timeperiod)\n", "      features_dataset['WILLR_' + str(timeperiod)] = WILLR(features_dataset['High'], features_dataset['Low'],\n", "                                                          features_dataset['Close'], timeperiod=timeperiod)\n", "\n", "      \n", "  gc.collect()\n", "  features_dataset['APO'] = APO(features_dataset['Close'], fastperiod=12, slowperiod=26, matype=0)\n", "\n", "  features_dataset['MACD'], features_dataset['MACD_signal'], features_dataset['MACD_hist'] = MACD(\n", "      features_dataset['Close'], fastperiod=12,\n", "      slowperiod=26,\n", "      signalperiod=9)\n", "  features_dataset['MACDEXT'], features_dataset['MACDEXT_signal'], features_dataset['MACDEXT_hist'] = MACDEXT(\n", "      features_dataset['Close'],\n", "      fastperiod=12,\n", "      fastmatype=0,\n", "      slowperiod=26,\n", "      slowmatype=0,\n", "      signalperiod=9,\n", "      signalmatype=0)\n", "\n", "  features_dataset['PPO'] = PPO(features_dataset['Close'], fastperiod=12, slowperiod=26, matype=0)\n", "\n", "  features_dataset['STOCH_slowk'], features_dataset['STOCH_slowd'] = STOCH(features_dataset['High'],\n", "                                                                          features_dataset['Low'],\n", "                                                                          features_dataset['Close'], fastk_period=5,\n", "                                                                          slowk_period=3, slowk_matype=0,\n", "                                                                          slowd_period=3, slowd_matype=0)\n", "  features_dataset['STOCHF_fastk'], features_dataset['STOCHF_fastd'] = STOCHF(features_dataset['High'],\n", "                                                                              features_dataset['Low'],\n", "                                                                              features_dataset['Close'], fastk_period=5,\n", "                                                                              fastd_period=3, fastd_matype=0)\n", "  features_dataset['STOCHRSI_fastk'], features_dataset['STOCHRSI_fastd'] = STOCHRSI(features_dataset['Close'],\n", "                                                                                    timeperiod=14,\n", "                                                                                    fastk_period=5, fastd_period=3,\n", "                                                                                    fastd_matype=0)\n", "  features_dataset['ULTOSC'] = ULTOSC(features_dataset['High'], features_dataset['Low'], features_dataset['Close'],\n", "                                      timeperiod1=7, timeperiod2=14, timeperiod3=28)\n", "\n", "  # Volume Indicators\n", "  features_dataset['AD'] = AD(features_dataset['High'], features_dataset['Low'], features_dataset['Close'],\n", "                              features_dataset['Volume'])\n", "  features_dataset['ADOSC'] = ADOSC(features_dataset['High'], features_dataset['Low'], features_dataset['Close'],\n", "                                    features_dataset['Volume'], fastperiod=3, slowperiod=10)\n", "  features_dataset['OBV'] = OBV(features_dataset['Close'], features_dataset['Volume'])\n", "\n", "  gc.collect()\n", "  # Volatility Indicators\n", "  for timeperiod in timeperiods:\n", "      features_dataset['ATR_' + str(timeperiod)] = ATR(features_dataset['High'], features_dataset['Low'],\n", "                                                      features_dataset['Close'], timeperiod=timeperiod)\n", "      features_dataset['NATR_' + str(timeperiod)] = NATR(features_dataset['High'], features_dataset['Low'],\n", "                                                        features_dataset['Close'], timeperiod=timeperiod)\n", "\n", "  features_dataset['TRANGE'] = TRANGE(features_dataset['High'], features_dataset['Low'], features_dataset['Close'])\n", "\n", "  # Price Transform\n", "  features_dataset['AVGPRICE'] = AVGPRICE(features_dataset['Open'], features_dataset['High'], features_dataset['Low'],\n", "                                          features_dataset['Close'])\n", "  features_dataset['MEDPRICE'] = MEDPRICE(features_dataset['High'], features_dataset['Low'])\n", "  features_dataset['TYPPRICE'] = TYPPRICE(features_dataset['High'], features_dataset['Low'], features_dataset['Close'])\n", "  features_dataset['WCLPRICE'] = WCLPRICE(features_dataset['High'], features_dataset['Low'], features_dataset['Close'])\n", "\n", "  gc.collect()\n", "  # Cycle Indicators\n", "  features_dataset['HT_DCPERIOD'] = HT_DCPERIOD(features_dataset['Close'])\n", "  features_dataset['HT_DCPHASE'] = HT_DCPHASE(features_dataset['Close'])\n", "  features_dataset['HT_PHASOR_inphase'], features_dataset['HT_PHASOR_quadrature'] = HT_PHASOR(features_dataset['Close'])\n", "  features_dataset['HT_SINE_sine'], features_dataset['HT_SINE_leadsine'] = HT_SINE(features_dataset['Close'])\n", "  features_dataset['HT_TRENDMODE'] = HT_TRENDMODE(features_dataset['Close'])\n", "\n", "  # Pattern Recognition\n", "  features_dataset['CDL2CROWS'] = CDL2CROWS(features_dataset['Open'], features_dataset['High'], features_dataset['Low'],\n", "                                            features_dataset['Close'])\n", "  features_dataset['CDL3BLACKCROWS'] = CDL3BLACKCROWS(features_dataset['Open'], features_dataset['High'],\n", "                                                      features_dataset['Low'], features_dataset['Close'])\n", "  features_dataset['CDL3INSIDE'] = CDL3INSIDE(features_dataset['Open'], features_dataset['High'], features_dataset['Low'],\n", "                                              features_dataset['Close'])\n", "  features_dataset['CDL3LINESTRIKE'] = CDL3LINESTRIKE(features_dataset['Open'], features_dataset['High'],\n", "                                                      features_dataset['Low'], features_dataset['Close'])\n", "  features_dataset['CDL3OUTSIDE'] = CDL3OUTSIDE(features_dataset['Open'], features_dataset['High'],\n", "                                                features_dataset['Low'], features_dataset['Close'])\n", "  features_dataset['CDL3STARSINSOUTH'] = CDL3STARSINSOUTH(features_dataset['Open'], features_dataset['High'],\n", "                                                          features_dataset['Low'], features_dataset['Close'])\n", "  features_dataset['CDL3WHITESOLDIERS'] = CDL3WHITESOLDIERS(features_dataset['Open'], features_dataset['High'],\n", "                                                            features_dataset['Low'], features_dataset['Close'])\n", "  features_dataset['CDLABANDONEDBABY'] = CDLABANDONEDBABY(features_dataset['Open'], features_dataset['High'],\n", "                                                          features_dataset['Low'], features_dataset['Close'],\n", "                                                          penetration=0)\n", "  features_dataset['CDLADVANCEBLOCK'] = CDLADVANCEBLOCK(features_dataset['Open'], features_dataset['High'],\n", "                                                        features_dataset['Low'], features_dataset['Close'])\n", "  features_dataset['CDLBELTHOLD'] = CDLBELTHOLD(features_dataset['Open'], features_dataset['High'],\n", "                                                features_dataset['Low'], features_dataset['Close'])\n", "  features_dataset['CDLBREAKAWAY'] = CDLBREAKAWAY(features_dataset['Open'], features_dataset['High'],\n", "                                                  features_dataset['Low'], features_dataset['Close'])\n", "  features_dataset['CDLCLOSINGMARUBOZU'] = CDLCLOSINGMARUBOZU(features_dataset['Open'], features_dataset['High'],\n", "                                                              features_dataset['Low'], features_dataset['Close'])\n", "  features_dataset['CDLCONCEALBABYSWALL'] = CDLCONCEALBABYSWALL(features_dataset['Open'], features_dataset['High'],\n", "                                                                features_dataset['Low'], features_dataset['Close'])\n", "  features_dataset['CDLCOUNTERATTACK'] = CDLCOUNTERATTACK(features_dataset['Open'], features_dataset['High'],\n", "                                                          features_dataset['Low'], features_dataset['Close'])\n", "  features_dataset['CDLDARKCLOUDCOVER'] = CDLDARKCLOUDCOVER(features_dataset['Open'], features_dataset['High'],\n", "                                                            features_dataset['Low'], features_dataset['Close'],\n", "                                                            penetration=0)\n", "  features_dataset['CDLDOJI'] = CDLDOJI(features_dataset['Open'], features_dataset['High'], features_dataset['Low'],\n", "                                        features_dataset['Close'])\n", "  features_dataset['CDLDOJISTAR'] = CDLDOJISTAR(features_dataset['Open'], features_dataset['High'],\n", "                                                features_dataset['Low'], features_dataset['Close'])\n", "  features_dataset['CDLDRAGONFLYDOJI'] = CDLDRAGONFLYDOJI(features_dataset['Open'], features_dataset['High'],\n", "                                                          features_dataset['Low'], features_dataset['Close'])\n", "  features_dataset['CDLENGULFING'] = CDLENGULFING(features_dataset['Open'], features_dataset['High'],\n", "                                                  features_dataset['Low'], features_dataset['Close'])\n", "  features_dataset['CDLEVENINGDOJISTAR'] = CDLEVENINGDOJISTAR(features_dataset['Open'], features_dataset['High'],\n", "                                                              features_dataset['Low'], features_dataset['Close'],\n", "                                                              penetration=0)\n", "  features_dataset['CDLEVENINGSTAR'] = CDLEVENINGSTAR(features_dataset['Open'], features_dataset['High'],\n", "                                                      features_dataset['Low'], features_dataset['Close'], penetration=0)\n", "  features_dataset['CDLGAPSIDESIDEWHITE'] = CDLGAPSIDESIDEWHITE(features_dataset['Open'], features_dataset['High'],\n", "                                                                features_dataset['Low'], features_dataset['Close'])\n", "  features_dataset['CDLGRAVESTONEDOJI'] = CDLGRAVESTONEDOJI(features_dataset['Open'], features_dataset['High'],\n", "                                                            features_dataset['Low'], features_dataset['Close'])\n", "  features_dataset['CDLHAMMER'] = CDLHAMMER(features_dataset['Open'], features_dataset['High'], features_dataset['Low'],\n", "                                            features_dataset['Close'])\n", "  features_dataset['CDLHANGINGMAN'] = CDLHANGINGMAN(features_dataset['Open'], features_dataset['High'],\n", "                                                    features_dataset['Low'], features_dataset['Close'])\n", "  features_dataset['CDLHARAMI'] = CDLHARAMI(features_dataset['Open'], features_dataset['High'], features_dataset['Low'],\n", "                                            features_dataset['Close'])\n", "  features_dataset['CDLHARAMICROSS'] = CDLHARAMICROSS(features_dataset['Open'], features_dataset['High'],\n", "                                                      features_dataset['Low'], features_dataset['Close'])\n", "  features_dataset['CDLHIGHWAVE'] = CDLHIGHWAVE(features_dataset['Open'], features_dataset['High'],\n", "                                                features_dataset['Low'], features_dataset['Close'])\n", "  features_dataset['CDLHIKKAKE'] = CDLHIKKAKE(features_dataset['Open'], features_dataset['High'], features_dataset['Low'],\n", "                                              features_dataset['Close'])\n", "  features_dataset['CDLHIKKAKEMOD'] = CDLHIKKAKEMOD(features_dataset['Open'], features_dataset['High'],\n", "                                                    features_dataset['Low'], features_dataset['Close'])\n", "  features_dataset['CDLHOMINGPIGEON'] = CDLHOMINGPIGEON(features_dataset['Open'], features_dataset['High'],\n", "                                                        features_dataset['Low'], features_dataset['Close'])\n", "  features_dataset['CDLIDENTICAL3CROWS'] = CDLIDENTICAL3CROWS(features_dataset['Open'], features_dataset['High'],\n", "                                                              features_dataset['Low'], features_dataset['Close'])\n", "  features_dataset['CDLINNECK'] = CDLINNECK(features_dataset['Open'], features_dataset['High'], features_dataset['Low'],\n", "                                            features_dataset['Close'])\n", "  features_dataset['CDLINVERTEDHAMMER'] = CDLINVERTEDHAMMER(features_dataset['Open'], features_dataset['High'],\n", "                                                            features_dataset['Low'], features_dataset['Close'])\n", "  features_dataset['CDLKICKING'] = CDLKICKING(features_dataset['Open'], features_dataset['High'], features_dataset['Low'],\n", "                                              features_dataset['Close'])\n", "  features_dataset['CDLKICKINGBYLENGTH'] = CDLKICKINGBYLENGTH(features_dataset['Open'], features_dataset['High'],\n", "                                                              features_dataset['Low'], features_dataset['Close'])\n", "  features_dataset['CDLLADDERBOTTOM'] = CDLLADDERBOTTOM(features_dataset['Open'], features_dataset['High'],\n", "                                                        features_dataset['Low'], features_dataset['Close'])\n", "  features_dataset['CDLLONGLEGGEDDOJI'] = CDLLONGLEGGEDDOJI(features_dataset['Open'], features_dataset['High'],\n", "                                                            features_dataset['Low'], features_dataset['Close'])\n", "  features_dataset['CDLLONGLINE'] = CDLLONGLINE(features_dataset['Open'], features_dataset['High'],\n", "                                                features_dataset['Low'], features_dataset['Close'])\n", "  features_dataset['CDLMARUBOZU'] = CDLMARUBOZU(features_dataset['Open'], features_dataset['High'],\n", "                                                features_dataset['Low'], features_dataset['Close'])\n", "  features_dataset['CDLMATCHINGLOW'] = CDLMATCHINGLOW(features_dataset['Open'], features_dataset['High'],\n", "                                                      features_dataset['Low'], features_dataset['Close'])\n", "  features_dataset['CDLMATHOLD'] = CDLMATHOLD(features_dataset['Open'], features_dataset['High'], features_dataset['Low'],\n", "                                              features_dataset['Close'], penetration=0)\n", "  features_dataset['CDLMORNINGDOJISTAR'] = CDLMORNINGDOJISTAR(features_dataset['Open'], features_dataset['High'],\n", "                                                              features_dataset['Low'], features_dataset['Close'],\n", "                                                              penetration=0)\n", "  features_dataset['CDLMORNINGSTAR'] = CDLMORNINGSTAR(features_dataset['Open'], features_dataset['High'],\n", "                                                      features_dataset['Low'], features_dataset['Close'], penetration=0)\n", "  features_dataset['CDLONNECK'] = CDLONNECK(features_dataset['Open'], features_dataset['High'], features_dataset['Low'],\n", "                                            features_dataset['Close'])\n", "  features_dataset['CDLPIERCING'] = CDLPIERCING(features_dataset['Open'], features_dataset['High'],\n", "                                                features_dataset['Low'], features_dataset['Close'])\n", "  features_dataset['CDLRICKSHAWMAN'] = CDLRICKSHAWMAN(features_dataset['Open'], features_dataset['High'],\n", "                                                      features_dataset['Low'], features_dataset['Close'])\n", "  features_dataset['CDLRISEFALL3METHODS'] = CDLRISEFALL3METHODS(features_dataset['Open'], features_dataset['High'],\n", "                                                                features_dataset['Low'], features_dataset['Close'])\n", "  features_dataset['CDLSEPARATINGLINES'] = CDLSEPARATINGLINES(features_dataset['Open'], features_dataset['High'],\n", "                                                              features_dataset['Low'], features_dataset['Close'])\n", "  features_dataset['CDLSHOOTINGSTAR'] = CDLSHOOTINGSTAR(features_dataset['Open'], features_dataset['High'],\n", "                                                        features_dataset['Low'], features_dataset['Close'])\n", "  features_dataset['CDLSHORTLINE'] = CDLSHORTLINE(features_dataset['Open'], features_dataset['High'],\n", "                                                  features_dataset['Low'], features_dataset['Close'])\n", "  features_dataset['CDLSPINNINGTOP'] = CDLSPINNINGTOP(features_dataset['Open'], features_dataset['High'],\n", "                                                      features_dataset['Low'], features_dataset['Close'])\n", "  features_dataset['CDLSTALLEDPATTERN'] = CDLSTALLEDPATTERN(features_dataset['Open'], features_dataset['High'],\n", "                                                            features_dataset['Low'], features_dataset['Close'])\n", "  features_dataset['CDLSTICKSANDWICH'] = CDLSTICKSANDWICH(features_dataset['Open'], features_dataset['High'],\n", "                                                          features_dataset['Low'], features_dataset['Close'])\n", "  features_dataset['CDLTAKURI'] = CDLTAKURI(features_dataset['Open'], features_dataset['High'], features_dataset['Low'],\n", "                                            features_dataset['Close'])\n", "  features_dataset['CDLTASUKIGAP'] = CDLTASUKIGAP(features_dataset['Open'], features_dataset['High'],\n", "                                                  features_dataset['Low'], features_dataset['Close'])\n", "  features_dataset['CDLTHRUSTING'] = CDLTHRUSTING(features_dataset['Open'], features_dataset['High'],\n", "                                                  features_dataset['Low'], features_dataset['Close'])\n", "  features_dataset['CDLUNIQUE3RIVER'] = CDLUNIQUE3RIVER(features_dataset['Open'], features_dataset['High'],\n", "                                                        features_dataset['Low'], features_dataset['Close'])\n", "  features_dataset['CDLUPSIDEGAP2CROWS'] = CDLUPSIDEGAP2CROWS(features_dataset['Open'], features_dataset['High'],\n", "                                                              features_dataset['Low'], features_dataset['Close'])\n", "  features_dataset['CDLXSIDEGAP3METHODS'] = CDLXSIDEGAP3METHODS(features_dataset['Open'], features_dataset['High'],\n", "                                                                features_dataset['Low'], features_dataset['Close'])\n", "  gc.collect()\n", "\n", "  # Statistic Functions\n", "  for timeperiod in timeperiods:\n", "      features_dataset['BETA_' + str(timeperiod)] = BETA(features_dataset['High'], features_dataset['Low'],\n", "                                                        timeperiod=timeperiod)\n", "      features_dataset['CORREL_' + str(timeperiod)] = CORREL(features_dataset['High'], features_dataset['Low'],\n", "                                                            timeperiod=timeperiod)\n", "      features_dataset['LINEARREG_' + str(timeperiod)] = LINEARREG(features_dataset['Close'], timeperiod=timeperiod)\n", "      features_dataset['LINEARREG_ANGLE_' + str(timeperiod)] = LINEARREG_ANGLE(features_dataset['Close'],\n", "                                                                              timeperiod=timeperiod)\n", "      features_dataset['LINEARREG_INTERCEPT_' + str(timeperiod)] = LINEARREG_INTERCEPT(features_dataset['Close'],\n", "                                                                                      timeperiod=timeperiod)\n", "      features_dataset['LINEARREG_SLOPE_' + str(timeperiod)] = LINEARREG_SLOPE(features_dataset['Close'],\n", "                                                                              timeperiod=timeperiod)\n", "      features_dataset['STDDEV_' + str(timeperiod)] = STDDEV(features_dataset['Close'], timeperiod=timeperiod, nbdev=1)\n", "      features_dataset['TSF_' + str(timeperiod)] = TSF(features_dataset['Close'], timeperiod=timeperiod)\n", "      features_dataset['VAR_' + str(timeperiod)] = VAR(features_dataset['Close'], timeperiod=timeperiod, nbdev=1)\n", "\n", "      \n", "  gc.collect()\n", "  # Math Transform\n", "  features_dataset['ACOS'] = ACOS(features_dataset['Close'])\n", "  features_dataset['ASIN'] = ASIN(features_dataset['Close'])\n", "  features_dataset['ATAN'] = ATAN(features_dataset['Close'])\n", "  features_dataset['CEIL'] = CEIL(features_dataset['Close'])\n", "  features_dataset['COS'] = COS(features_dataset['Close'])\n", "  features_dataset['COSH'] = COSH(features_dataset['Close'])\n", "  features_dataset['EXP'] = EXP(features_dataset['Close'])\n", "  features_dataset['FLOOR'] = FLOOR(features_dataset['Close'])\n", "  features_dataset['LN'] = LN(features_dataset['Close'])\n", "  features_dataset['LOG10'] = LOG10(features_dataset['Close'])\n", "  features_dataset['SIN'] = SIN(features_dataset['Close'])\n", "  features_dataset['SINH'] = SINH(features_dataset['Close'])\n", "  features_dataset['SQRT'] = SQRT(features_dataset['Close'])\n", "  features_dataset['TAN'] = TAN(features_dataset['Close'])\n", "  features_dataset['TANH'] = TANH(features_dataset['Close'])\n", "\n", "  gc.collect()\n", "  # Math Operator\n", "  features_dataset['ADD'] = ADD(features_dataset['High'], features_dataset['Low'])\n", "  features_dataset['DIV'] = DIV(features_dataset['High'], features_dataset['Low'])\n", "  features_dataset['MULT'] = MULT(features_dataset['High'], features_dataset['Low'])\n", "  features_dataset['SUB'] = SUB(features_dataset['High'], features_dataset['Low'])\n", "\n", "  for timeperiod in timeperiods:\n", "      features_dataset['MAX_' + str(timeperiod)] = MAX(features_dataset['Close'], timeperiod=timeperiod)\n", "      features_dataset['MIN_' + str(timeperiod)] = MIN(features_dataset['Close'], timeperiod=timeperiod)\n", "      features_dataset['MINMAX_min_' + str(timeperiod)], features_dataset['MINMAX_max_' + str(timeperiod)] = MINMAX(\n", "          features_dataset['Close'], timeperiod=timeperiod)\n", "      features_dataset['SUM_' + str(timeperiod)] = SUM(features_dataset['Close'], timeperiod=timeperiod)\n", "      \n", "  gc.collect()\n", "\n", "\n", "  features_dataset = features_dataset.drop(features_dataset.std()[(features_dataset.std() == 0)].index, axis=1)\n", "  features_dataset = pd.concat([features_dataset[50:-50].replace([np.inf, -np.inf], np.nan),\n", "                                features_dataset[-50:].replace([np.inf, -np.inf, np.nan], 0)]).dropna(axis=1)\n", "  features_dataset.to_pickle('features_dataset' if last_index is None else 'features_dataset_' + str(last_index))\n", "  if last_index is None:\n", "    display(features_dataset)\n", "    display(features_dataset.describe())\n", "\n", "  del periods, timeperiods, matypes, nbdevs, features_dataset\n", "  gc.collect()\n", "\n", "fe()"], "execution_count": 0, "outputs": []}, {"cell_type": "code", "metadata": {"id": "hBDCj4XGlOfy", "colab_type": "code", "colab": {}}, "source": ["# features EDA\n", "if is_EDA:\n", "  import pandas_profiling as pp\n", "  display(pp.ProfileReport(pd.read_pickle('features_dataset')))\n", "  del pp"], "execution_count": 0, "outputs": []}, {"cell_type": "code", "metadata": {"id": "pYArZ18zL1yA", "colab_type": "code", "colab": {}}, "source": ["# check features data leakage\n", "fe(100)\n", "long_df = pd.read_pickle('features_dataset')[:-100]\n", "short_df =  pd.read_pickle('features_dataset_100')\n", "if False in (long_df.index.values == short_df.index.values):\n", "  print('mismatch INDEX', long_df.index.values == short_df.index.values)\n", "\n", "for aCol in long_df.columns.values:\n", "  try:\n", "    if False in (long_df[aCol].values == short_df[aCol].values):\n", "      print('mismatch, ', aCol, long_df[aCol].values == short_df[aCol].values)\n", "  except:\n", "    print('mismatch exception, ', aCol)\n", "\n", "for name in dir():\n", "  if any( n in name for n in talib.get_functions()):\n", "    del globals()[name]\n", "\n", "del talib, long_df, short_df\n", "gc.collect()\n", "print('Nothing should be printed above')"], "execution_count": 0, "outputs": []}, {"cell_type": "code", "metadata": {"id": "P5wsqm6_R_uU", "colab_type": "code", "colab": {}}, "source": ["# label and scale\n", "import numpy as np\n", "from sklearn.preprocessing import minmax_scale\n", "import os\n", "import pandas as pd\n", "\n", "features_dataset = pd.read_pickle('features_dataset')\n", "label_df = features_dataset[[data_label]]\n", "longs = label_df[data_label].values[1:] > label_df[data_label].values[:-1] \n", "shorts = label_df[data_label].values[1:] <= label_df[data_label].values[:-1] \n", "label_df = label_df[:-1]\n", "label_df[data_label_name] = (longs*1 + shorts*0).astype(int)\n", "scaled_df = features_dataset\n", "scaled_df = pd.DataFrame((minmax_scale(scaled_df, axis=0, feature_range=(0, 1)) if is_minmax else scaled_df), columns=scaled_df.columns,\n", "                         index=scaled_df.index)\n", "categorical_columns = features_dataset.select_dtypes(exclude=['float64', 'float32', 'float16', 'float_']).columns\n", "\n", "scaled_df[categorical_columns] = features_dataset[categorical_columns]\n", "scaled_df = scaled_df[:-1]\n", "scaled_df[data_label_name] = label_df[data_label_name]\n", "\n", "display(scaled_df)\n", "scaled_df.to_pickle('scaled_df') \n", "\n", "label_df[-50:].to_pickle('predict_df')\n", "del features_dataset, categorical_columns, scaled_df, label_df, minmax_scale, shorts, longs\n", "gc.collect()"], "execution_count": 0, "outputs": []}, {"cell_type": "code", "metadata": {"id": "gMSXB016_JM6", "colab_type": "code", "colab": {}}, "source": ["# load data into h2o\n", "import h2o\n", "import pandas as pd\n", "from h2o.automl import H2OAutoML\n", "import matplotlib.pyplot as plt\n", "import matplotlib as mpl\n", "import time\n", "import gc\n", "import numpy as np\n", "\n", "def load_h2o():\n", "  global train_hf, index_splits\n", "  try:\n", "    h2o.cluster().shutdown()\n", "    time.sleep(5)\n", "  except:\n", "    pass\n", "\n", "  h2o.init(min_mem_size=\"9g\",nthreads=-1)\n", "  h2o.remove_all()\n", "  train_hf = h2o.H2OFrame(pd.read_pickle('scaled_df'))\n", "  \n", "  gc.collect()\n", "  train_hf[data_label_name] = train_hf[data_label_name].ascharacter().asfactor()\n", "  index_splits = [int(len(train_hf)*(1-(split_percentages[0]+split_percentages[1]))),int(len(train_hf)*(1-split_percentages[1]))]\n", "  gc.collect()\n", "\n", "\n", "try:\n", "  load_h2o()\n", "except:\n", "  pass"], "execution_count": 0, "outputs": []}, {"cell_type": "code", "metadata": {"id": "ZDe_U1iPwvTJ", "colab_type": "code", "colab": {}}, "source": ["# define the model\n", "%matplotlib inline\n", "from h2o.exceptions import H2OJobCancelled\n", "from sklearn.metrics import accuracy_score\n", "from numpy.random import seed\n", "from tensorflow import set_random_seed\n", "\n", "def model_train(feature_name='All', model_algo = None,features_list=None,training_frame=None, \n", "                nfolds=0,max_models=None):\n", "  \n", "  global train_hf, models_summary, features_to_include, max_models_num,best_models\n", "  if features_to_include is not True and feature_name not in features_to_include:\n", "    return []\n", "  \n", "  seed(1)\n", "  set_random_seed(1)\n", "\n", "  nfolds = nfolds or folds\n", "  gc.collect()\n", "  model_algo = [model_algo] if isinstance(model_algo,str) else model_algo\n", "  for _ in range(3):\n", "    try:\n", "      aml = H2OAutoML(seed=1, nfolds=nfolds, max_models=(max_models or max_models_num),\n", "                      include_algos=model_algo, keep_cross_validation_predictions=True)\n", "      t_hf = (training_frame or train_hf)\n", "      validation_frame =  t_hf[index_splits[0]:index_splits[1],:] if split_percentages[0] != 0 else None\n", "\n", "      aml.train(x=(features_list or list(t_hf.columns[:-1])), y=data_label_name, training_frame=t_hf[:index_splits[0],:], validation_frame=validation_frame, leaderboard_frame=t_hf[index_splits[1]:,:])\n", "      if aml.leader is None:\n", "        return None\n", "      break\n", "    except (KeyboardInterrupt, H2OJobCancelled):\n", "      raise Exception('Stopped')\n", "    except Exception as e:\n", "      print(e)\n", "      load_h2o()\n", "      \n", "  \n", "  important_features = None\n", "  try:\n", "    display(aml.leaderboard.as_data_frame())\n", "    important_features = aml.leader.varimp(use_pandas=True)\n", "    display(important_features)\n", "    important_features = list(important_features['variable'].values)\n", "    aml.leader.varimp_plot()\n", "    display(aml.leader)\n", "    model_performance = aml.leader.model_performance(test_data=t_hf[index_splits[1]:,:])\n", "    for k in models_summary:\n", "      result = getattr(model_performance, k)()\n", "      result = result[0][1] if isinstance(result,list) else result\n", "      models_summary[k] = models_summary[k].append(pd.DataFrame({(model_algo[0] if model_algo is not None else 'All'):[result]},index=[feature_name]))\n", "      display((k,result))\n", "  except Exception as e:\n", "    print(e)\n", "  \n", "  try:\n", "    best_models[str(feature_name) + '_' + str(model_algo)] = {'id':aml.leader.model_id ,'AUC':getattr(model_performance, \"auc\")(),\n", "                                                              'path': h2o.save_model(aml.leader,force=True)}\n", "    predicted = aml.leader.predict(t_hf).as_data_frame()\n", "\n", "    # predicted['predict'] = (predicted['p1'] >= aml.leader.find_threshold_by_max_metric('f1', train=False, valid=True, xval=False)).astype(int)\n", "    predicted.to_csv(str(feature_name) + '_' + str(model_algo) + '.csv', )\n", "    predicted = predicted[['predict']]\n", "    print(predicted)\n", "    start_idx = max(-index_splits[1],-20)\n", "    predicted = predicted[start_idx:]\n", "    predict_df = pd.read_pickle('predict_df')[start_idx:]\n", "\n", "    predicted.index = predict_df.index\n", "    predicted[data_label] = predict_df[data_label]\n", "\n", "    up_mask = predicted['predict'] == 1\n", "    down_mask = predicted['predict'] == 0\n", "\n", "    plt.plot(predicted[data_label], color='tab:blue', label='Actual')\n", "    plt.scatter(predicted.index[up_mask], predicted[data_label][up_mask], marker=mpl.markers.CARETUPBASE, color='tab:green', s=100, label='Up')\n", "    plt.scatter(predicted.index[down_mask], predicted[data_label][down_mask],marker=mpl.markers.CARETDOWNBASE, color='tab:red', s=100, label='Down')\n", "    plt.xticks(rotation = '60')\n", "    \n", "    plt.legend(loc='upper left')\n", "    plt.show()\n", "  except Exception as e:\n", "    print(e)\n", "  \n", "  try:\n", "    display(model_performance)\n", "    model_performance.plot()\n", "  except Exception as e:\n", "    print(e)\n", "\n", "  try:\n", "    aml.leader.std_coef_plot()\n", "  except Exception as e:\n", "    print(e)\n", "  \n", "  try:\n", "    aml.leader.plot()\n", "  except Exception as e:\n", "    print(e)\n", "  \n", "  import json\n", "  from PIL import Image\n", "  try:\n", "    model_path = aml.leader.download_mojo(get_genmodel_jar=True)\n", "\n", "    try:\n", "      !wget -c https://h2o-release.s3.amazonaws.com/h2o/rel-yates/5/h2o-3.24.0.5.zip && unzip -n h2o-3.24.0.5.zip \n", "\n", "      !java -Xmx6g -cp h2o-3.24.0.5/h2o.jar hex.genmodel.tools.PrintMojo --tree 0 -i $model_path -o model.gv --levels 3 -f 9\n", "      !dot -Tpng model.gv -o model.png\n", "\n", "      display(Image.open('model.png'))\n", "    except Exception as e:\n", "      pass\n", "\n", "    try:\n", "      !unzip -o -qq $model_path\n", "\n", "      with open('experimental/modelDetails.json') as f:\n", "          display(json.load(f)['parameters'])\n", "    except:\n", "      pass\n", "  \n", "  except Exception as e:\n", "    print(e)\n", "\n", "  try:\n", "    del aml, predicted, Image, json, model_performance\n", "  except Exception as e:\n", "    print(e)\n", "\n", "  gc.collect()\n", "\n", "  return important_features"], "execution_count": 0, "outputs": []}, {"cell_type": "code", "metadata": {"id": "P4Py1z0BXpY7", "colab_type": "code", "colab": {}}, "source": ["# DeepLearning\n", "features_list = model_train(feature_name='All',model_algo=\"DeepLearning\")\n", "display(len(features_list),features_list[:model_selected_features])"], "execution_count": 0, "outputs": []}, {"cell_type": "code", "metadata": {"id": "3CMuojtBvBwQ", "colab_type": "code", "colab": {}}, "source": ["# DeepLearning with top DeepLearning features\n", "model_train(feature_name='DLF',model_algo=\"DeepLearning\",features_list = features_list[:model_selected_features])"], "execution_count": 0, "outputs": []}, {"cell_type": "code", "metadata": {"id": "X46i9vgxvFY8", "colab_type": "code", "colab": {}}, "source": ["# GLM with top DeepLearning features\n", "model_train(feature_name='DLF',model_algo=\"GLM\",features_list = features_list[:model_selected_features])"], "execution_count": 0, "outputs": []}, {"cell_type": "code", "metadata": {"id": "COShDwMYvF2s", "colab_type": "code", "colab": {}}, "source": ["# XGBoost with top DeepLearning features\n", "model_train(feature_name='DLF',model_algo=\"XGBoost\",features_list = features_list[:model_selected_features])"], "execution_count": 0, "outputs": []}, {"cell_type": "code", "metadata": {"id": "tOSznN3fVvUE", "colab_type": "code", "colab": {}}, "source": ["# DRF with top DeepLearning features\n", "model_train(feature_name='DLF',model_algo=\"DRF\",features_list = features_list[:model_selected_features])"], "execution_count": 0, "outputs": []}, {"cell_type": "code", "metadata": {"id": "cEC0n6f8V2sN", "colab_type": "code", "colab": {}}, "source": ["# GBM with top DeepLearning features\n", "model_train(feature_name='DLF',model_algo=\"GBM\",features_list = features_list[:model_selected_features])"], "execution_count": 0, "outputs": []}, {"cell_type": "code", "metadata": {"id": "w909eD7DV96v", "colab_type": "code", "colab": {}}, "source": ["# StackedEnsemble with top DeepLearning features\n", "model_train(feature_name='DLF',model_algo=\"StackedEnsemble\",features_list = features_list[:model_selected_features])"], "execution_count": 0, "outputs": []}, {"cell_type": "code", "metadata": {"id": "9bK--Kg_V-oF", "colab_type": "code", "colab": {}}, "source": ["# All algos with top DeepLearning features\n", "model_train(feature_name='DLF',model_algo=None,features_list = features_list[:model_selected_features])"], "execution_count": 0, "outputs": []}, {"cell_type": "code", "metadata": {"id": "XB-CunrHYBd3", "colab_type": "code", "colab": {}}, "source": ["# GLM\n", "features_list = model_train(feature_name='All',model_algo=\"GLM\")\n", "display(features_list[:model_selected_features])\n", "display(len(features_list),features_list[:model_selected_features])"], "execution_count": 0, "outputs": []}, {"cell_type": "code", "metadata": {"id": "yGIszZBqvQHH", "colab_type": "code", "colab": {}}, "source": ["# DeepLearning with top GLM features\n", "model_train(feature_name='GLMF',model_algo=\"DeepLearning\",features_list = features_list[:model_selected_features])"], "execution_count": 0, "outputs": []}, {"cell_type": "code", "metadata": {"id": "pLY75bXsvP1x", "colab_type": "code", "colab": {}}, "source": ["# GLM with top GLM features\n", "model_train(feature_name='GLMF',model_algo=\"GLM\",features_list = features_list[:model_selected_features])"], "execution_count": 0, "outputs": []}, {"cell_type": "code", "metadata": {"id": "9nI_jaETvKDp", "colab_type": "code", "colab": {}}, "source": ["# XGBoost with top GLM features\n", "model_train(feature_name='GLMF',model_algo=\"XGBoost\",features_list = features_list[:model_selected_features])"], "execution_count": 0, "outputs": []}, {"cell_type": "code", "metadata": {"colab_type": "code", "id": "dd8GUIFpWObe", "colab": {}}, "source": ["# DRF with top GLM features\n", "model_train(feature_name='GLMF',model_algo=\"DRF\",features_list = features_list[:model_selected_features])"], "execution_count": 0, "outputs": []}, {"cell_type": "code", "metadata": {"colab_type": "code", "id": "i9oawXqRWObg", "colab": {}}, "source": ["# GBM with top GLM features\n", "model_train(feature_name='GLMF',model_algo=\"GBM\",features_list = features_list[:model_selected_features])"], "execution_count": 0, "outputs": []}, {"cell_type": "code", "metadata": {"colab_type": "code", "id": "yYz8GunzWObh", "colab": {}}, "source": ["# StackedEnsemble with top GLM features\n", "model_train(feature_name='GLMF',model_algo=\"StackedEnsemble\",features_list = features_list[:model_selected_features])"], "execution_count": 0, "outputs": []}, {"cell_type": "code", "metadata": {"colab_type": "code", "id": "2FU7pQDzWObj", "colab": {}}, "source": ["# All algos with top GLM features\n", "model_train(feature_name='GLMF',model_algo=None,features_list = features_list[:model_selected_features])"], "execution_count": 0, "outputs": []}, {"cell_type": "code", "metadata": {"id": "N4eiWe6DYaWG", "colab_type": "code", "colab": {}}, "source": ["# XGBoost\n", "features_list = model_train(feature_name='All',model_algo=\"XGBoost\")\n", "display(len(features_list),features_list[:model_selected_features])"], "execution_count": 0, "outputs": []}, {"cell_type": "code", "metadata": {"id": "80hhlYXGZ30d", "colab_type": "code", "colab": {}}, "source": ["# DeepLearning with top XGBoost features\n", "model_train(feature_name='XGBF',model_algo=\"DeepLearning\",features_list = features_list[:model_selected_features])"], "execution_count": 0, "outputs": []}, {"cell_type": "code", "metadata": {"id": "u87xRJn7aFqe", "colab_type": "code", "colab": {}}, "source": ["# GLM with top XGBoost features\n", "model_train(feature_name='XGBF',model_algo=\"GLM\",features_list = features_list[:model_selected_features])"], "execution_count": 0, "outputs": []}, {"cell_type": "code", "metadata": {"id": "DYVVSw2MaGLg", "colab_type": "code", "colab": {}}, "source": ["# XGBoost with top XGBoost features\n", "model_train(feature_name='XGBF',model_algo=\"XGBoost\",features_list = features_list[:model_selected_features])"], "execution_count": 0, "outputs": []}, {"cell_type": "code", "metadata": {"colab_type": "code", "id": "aTFYzyYBWR56", "colab": {}}, "source": ["# DRF with top XGBoost features\n", "model_train(feature_name='XGBF',model_algo=\"DRF\",features_list = features_list[:model_selected_features])"], "execution_count": 0, "outputs": []}, {"cell_type": "code", "metadata": {"colab_type": "code", "id": "CKXZIHzWWR6B", "colab": {}}, "source": ["# GBM with top XGBoost features\n", "model_train(feature_name='XGBF',model_algo=\"GBM\",features_list = features_list[:model_selected_features])"], "execution_count": 0, "outputs": []}, {"cell_type": "code", "metadata": {"colab_type": "code", "id": "zoBHQBCNWR6C", "colab": {}}, "source": ["# StackedEnsemble with top XGBoost features\n", "model_train(feature_name='XGBF',model_algo=\"StackedEnsemble\",features_list = features_list[:model_selected_features])"], "execution_count": 0, "outputs": []}, {"cell_type": "code", "metadata": {"colab_type": "code", "id": "HyWclQjZWR6F", "colab": {}}, "source": ["# All algos with top XGBoost features\n", "model_train(feature_name='XGBF',model_algo=None,features_list = features_list[:model_selected_features])"], "execution_count": 0, "outputs": []}, {"cell_type": "code", "metadata": {"id": "qJLupNSSbiFB", "colab_type": "code", "colab": {}}, "source": ["# PCA\n", "try:\n", "  del features_list\n", "except:\n", "  pass\n", "gc.collect()\n", "\n", "if features_to_include is True or 'PCA' in features_to_include:\n", "  for _ in range(3):\n", "    try:\n", "      from h2o.estimators.pca import H2OPrincipalComponentAnalysisEstimator\n", "      h2o_model = H2OPrincipalComponentAnalysisEstimator(seed=1, k=PCA_k, pca_method=\"GramSVD\", max_iterations=10000)\n", "      h2o_model.train(x=list(train_hf.columns[:-1]), training_frame=train_hf[:index_splits[0],:])\n", "\n", "      display(h2o_model.varimp(use_pandas=True))\n", "      display(h2o_model)\n", "\n", "      training_frame = h2o_model.predict(train_hf)\n", "      del h2o_model, H2OPrincipalComponentAnalysisEstimator\n", "      gc.collect()\n", "\n", "      training_frame = training_frame.cbind(train_hf[data_label_name])\n", "      break\n", "    except KeyboardInterrupt:\n", "      break\n", "    except Exception as e:\n", "      print(e)\n", "      load_h2o()\n", "else:\n", "  training_frame = []"], "execution_count": 0, "outputs": []}, {"cell_type": "code", "metadata": {"id": "UXmkstTszwTj", "colab_type": "code", "colab": {}}, "source": ["# DeepLearning with PCA\n", "model_train(feature_name='PCA',model_algo=\"DeepLearning\",training_frame=training_frame)"], "execution_count": 0, "outputs": []}, {"cell_type": "code", "metadata": {"id": "DmAYwNZJ1etk", "colab_type": "code", "colab": {}}, "source": ["# GLM with PCA\n", "model_train(feature_name='PCA',model_algo=\"GLM\",training_frame=training_frame)"], "execution_count": 0, "outputs": []}, {"cell_type": "code", "metadata": {"id": "_dXIpWK81gZ3", "colab_type": "code", "colab": {}}, "source": ["# XGBoost with PCA\n", "model_train(feature_name='PCA',model_algo=\"XGBoost\",training_frame=training_frame)"], "execution_count": 0, "outputs": []}, {"cell_type": "code", "metadata": {"colab_type": "code", "id": "PD3MJ-26WTX5", "colab": {}}, "source": ["# DRF with top PCA features\n", "model_train(feature_name='PCA',model_algo=\"DRF\",training_frame=training_frame)"], "execution_count": 0, "outputs": []}, {"cell_type": "code", "metadata": {"colab_type": "code", "id": "uM-CiGR1WTX6", "colab": {}}, "source": ["# GBM with top PCA features\n", "model_train(feature_name='PCA',model_algo=\"GBM\",training_frame=training_frame)"], "execution_count": 0, "outputs": []}, {"cell_type": "code", "metadata": {"colab_type": "code", "id": "p6UOL2hiWTX8", "colab": {}}, "source": ["# StackedEnsemble with top PCA features\n", "model_train(feature_name='PCA',model_algo=\"StackedEnsemble\",training_frame=training_frame)"], "execution_count": 0, "outputs": []}, {"cell_type": "code", "metadata": {"colab_type": "code", "id": "bPPaZwCeWTX9", "colab": {}}, "source": ["# All algos with top PCA features\n", "model_train(feature_name='PCA',model_algo=None,training_frame=training_frame)"], "execution_count": 0, "outputs": []}, {"cell_type": "code", "metadata": {"id": "-KkyfIacplS-", "colab_type": "code", "colab": {}}, "source": ["# Generalized Low Rank\n", "try:\n", "  del training_frame\n", "except:\n", "  pass\n", "gc.collect()\n", "\n", "if features_to_include is True or 'GLRM' in features_to_include:\n", "  for _ in range(3):\n", "    try:\n", "      from h2o.estimators.glrm import H2OGeneralizedLowRankEstimator\n", "\n", "      h2o_model = H2OGeneralizedLowRankEstimator(seed=1, k=GLRM_k)\n", "      h2o_model.train(x=list(train_hf.columns[:-1]), training_frame=train_hf[:index_splits[0],:])\n", "\n", "      display(h2o_model.varimp(use_pandas=True))\n", "      display(h2o_model)\n", "\n", "      training_frame = h2o_model.predict(train_hf)\n", "      del h2o_model, H2OGeneralizedLowRankEstimator\n", "      gc.collect()\n", "\n", "      training_frame = training_frame.cbind(train_hf[data_label_name])\n", "      break\n", "    except KeyboardInterrupt:\n", "      break\n", "    except Exception as e:\n", "      print(e)\n", "      load_h2o()\n", "else:\n", "  training_frame = []"], "execution_count": 0, "outputs": []}, {"cell_type": "code", "metadata": {"id": "EWvSFRoIqdEh", "colab_type": "code", "colab": {}}, "source": ["# DeepLearning with Generalized Low Rank\n", "model_train(feature_name='GLRM',model_algo=\"DeepLearning\",training_frame=training_frame)"], "execution_count": 0, "outputs": []}, {"cell_type": "code", "metadata": {"id": "j3EA71bhqdc_", "colab_type": "code", "colab": {}}, "source": ["# GLM with Generalized Low Rank\n", "model_train(feature_name='GLRM',model_algo=\"GLM\",training_frame=training_frame)"], "execution_count": 0, "outputs": []}, {"cell_type": "code", "metadata": {"id": "v3Ny9TfBqd93", "colab_type": "code", "colab": {}}, "source": ["# XGBoost with Generalized Low Rank\n", "model_train(feature_name='GLRM',model_algo=\"XGBoost\",training_frame=training_frame)"], "execution_count": 0, "outputs": []}, {"cell_type": "code", "metadata": {"colab_type": "code", "id": "qvKVlLtIWUJo", "colab": {}}, "source": ["# DRF with Generalized Low Rank\n", "model_train(feature_name='GLRM',model_algo=\"DRF\",training_frame=training_frame)"], "execution_count": 0, "outputs": []}, {"cell_type": "code", "metadata": {"colab_type": "code", "id": "zgln4Li-WUJq", "colab": {}}, "source": ["# GBM with Generalized Low Rank\n", "model_train(feature_name='GLRM',model_algo=\"GBM\",training_frame=training_frame)"], "execution_count": 0, "outputs": []}, {"cell_type": "code", "metadata": {"colab_type": "code", "id": "e8xFn6TOWUJs", "colab": {}}, "source": ["# StackedEnsemble with Generalized Low Rank\n", "model_train(feature_name='GLRM',model_algo=\"StackedEnsemble\",training_frame=training_frame)"], "execution_count": 0, "outputs": []}, {"cell_type": "code", "metadata": {"colab_type": "code", "id": "iYLop1T5WUJv", "colab": {}}, "source": ["# All algos with Generalized Low Rank\n", "model_train(feature_name='GLRM',model_algo=None,training_frame=training_frame)"], "execution_count": 0, "outputs": []}, {"cell_type": "code", "metadata": {"id": "6slV9HNA1-oE", "colab_type": "code", "colab": {}}, "source": ["# High Correlation Filter \n", "try:\n", "  del training_frame\n", "except:\n", "  pass\n", "gc.collect()\n", "\n", "if features_to_include is True or 'HCF' in features_to_include:\n", "  corr_matrix = pd.read_pickle('scaled_df')[:index_splits[0]].drop(data_label_name,axis=1).corr().abs()\n", "\n", "  upper = corr_matrix.where(np.triu(np.ones(corr_matrix.shape), k=1).astype(np.bool))\n", "\n", "  features_list = [column for column in upper.columns if not any(upper[column] > HCF_threshold)]\n", "\n", "  del corr_matrix, upper\n", "  gc.collect()\n", "\n", "  display(len(features_list),features_list)\n", "else:\n", "  features_list = []"], "execution_count": 0, "outputs": []}, {"cell_type": "code", "metadata": {"id": "TEF_Uok64Cc3", "colab_type": "code", "colab": {}}, "source": ["# DeepLearning with low correlation features\n", "model_train(feature_name='HCF',model_algo=\"DeepLearning\",features_list=features_list)"], "execution_count": 0, "outputs": []}, {"cell_type": "code", "metadata": {"id": "bvG4Zax94Nl3", "colab_type": "code", "colab": {}}, "source": ["# GLM with low correlation features\n", "model_train(feature_name='HCF',model_algo=\"GLM\",features_list=features_list)"], "execution_count": 0, "outputs": []}, {"cell_type": "code", "metadata": {"id": "Sc5SD2v94OAV", "colab_type": "code", "colab": {}}, "source": ["# XGBoost with low correlation features\n", "model_train(feature_name='HCF',model_algo=\"XGBoost\",features_list=features_list)"], "execution_count": 0, "outputs": []}, {"cell_type": "code", "metadata": {"colab_type": "code", "id": "zYE_JcmrWVb_", "colab": {}}, "source": ["# DRF with low correlation features\n", "model_train(feature_name='HCF',model_algo=\"DRF\",features_list=features_list)"], "execution_count": 0, "outputs": []}, {"cell_type": "code", "metadata": {"colab_type": "code", "id": "ZK4OWSL6WVcA", "colab": {}}, "source": ["# GBM with low correlation features\n", "model_train(feature_name='HCF',model_algo=\"GBM\",features_list=features_list)"], "execution_count": 0, "outputs": []}, {"cell_type": "code", "metadata": {"colab_type": "code", "id": "HdQm91ftWVcC", "colab": {}}, "source": ["# StackedEnsemble with low correlation features\n", "model_train(feature_name='HCF',model_algo=\"StackedEnsemble\",features_list=features_list)"], "execution_count": 0, "outputs": []}, {"cell_type": "code", "metadata": {"colab_type": "code", "id": "_shoe1ZZWVcD", "colab": {}}, "source": ["# All algos with low correlation features\n", "model_train(feature_name='HCF',model_algo=None,features_list=features_list)"], "execution_count": 0, "outputs": []}, {"cell_type": "code", "metadata": {"id": "rQn_MKnK5Dfj", "colab_type": "code", "colab": {}}, "source": ["#\tLow Variance Filter  \n", "try:\n", "  del features_list\n", "except:\n", "  pass\n", "gc.collect()\n", "\n", "if features_to_include is True or 'LVF' in features_to_include:\n", "  df_variance = pd.read_pickle('scaled_df')[:index_splits[0]].drop(data_label_name,axis=1).var()\n", "  features_list = [column for column in df_variance.index if df_variance[column]  > LVF_threshold]\n", "\n", "  del df_variance\n", "  gc.collect()\n", "\n", "  display(len(features_list),features_list)\n", "else:\n", "  features_list = []"], "execution_count": 0, "outputs": []}, {"cell_type": "code", "metadata": {"id": "FggEgScI5bGS", "colab_type": "code", "colab": {}}, "source": ["# DeepLearning with Low Variance features  \n", "model_train(feature_name='LVF',model_algo=\"DeepLearning\",features_list=features_list)"], "execution_count": 0, "outputs": []}, {"cell_type": "code", "metadata": {"id": "E1kUR5lF9J_-", "colab_type": "code", "colab": {}}, "source": ["# GLM with Low Variance features  \n", "model_train(feature_name='LVF',model_algo=\"GLM\",features_list=features_list)"], "execution_count": 0, "outputs": []}, {"cell_type": "code", "metadata": {"id": "gkn-Ir4p9Kr5", "colab_type": "code", "colab": {}}, "source": ["# XGBoost with Low Variance features  \n", "model_train(feature_name='LVF',model_algo=\"XGBoost\",features_list=features_list)"], "execution_count": 0, "outputs": []}, {"cell_type": "code", "metadata": {"colab_type": "code", "id": "mx_AtDawWWOU", "colab": {}}, "source": ["# DRF with Low Variance features\n", "model_train(feature_name='LVF',model_algo=\"DRF\",features_list=features_list)"], "execution_count": 0, "outputs": []}, {"cell_type": "code", "metadata": {"colab_type": "code", "id": "nJPu-7Y-WWOW", "colab": {}}, "source": ["# GBM with Low Variance features\n", "model_train(feature_name='LVF',model_algo=\"GBM\",features_list=features_list)"], "execution_count": 0, "outputs": []}, {"cell_type": "code", "metadata": {"colab_type": "code", "id": "1t359_jRWWOX", "colab": {}}, "source": ["# StackedEnsemble with Low Variance features\n", "model_train(feature_name='LVF',model_algo=\"StackedEnsemble\",features_list=features_list)"], "execution_count": 0, "outputs": []}, {"cell_type": "code", "metadata": {"colab_type": "code", "id": "A0UIWzstWWOa", "colab": {}}, "source": ["# All algos with Low Variance features\n", "model_train(feature_name='LVF',model_algo=None,features_list=features_list)"], "execution_count": 0, "outputs": []}, {"cell_type": "code", "metadata": {"id": "YF_X6jJNxLVZ", "colab_type": "code", "colab": {}}, "source": ["# Select k best features\n", "try:\n", "  del features_list\n", "except:\n", "  pass\n", "gc.collect()\n", "\n", "if features_to_include is True or 'KBest' in features_to_include:\n", "  from sklearn.feature_selection import SelectKBest\n", "  from sklearn.feature_selection import f_classif\n", "\n", "  bestfeatures = SelectKBest(score_func=f_classif, k=select_k).fit(pd.read_pickle('scaled_df')[:index_splits[0]].drop(data_label_name,axis=1)\n", "                                                              ,pd.read_pickle('scaled_df')[:index_splits[0]][data_label_name])\n", "\n", "  features_list = list(np.array(train_hf.columns)[:-1][bestfeatures.get_support()])\n", "\n", "  del bestfeatures, SelectKBest, f_classif\n", "  gc.collect()\n", "  display(len(features_list),features_list)\n", "else:\n", "  features_list=[]"], "execution_count": 0, "outputs": []}, {"cell_type": "code", "metadata": {"id": "_SO0IMW6zVus", "colab_type": "code", "colab": {}}, "source": ["# DeepLearning with Select k best features \n", "model_train(feature_name='KBest',model_algo=\"DeepLearning\",features_list=features_list)"], "execution_count": 0, "outputs": []}, {"cell_type": "code", "metadata": {"id": "SlJfVpr3zWIz", "colab_type": "code", "colab": {}}, "source": ["# GLM with Select k best features\n", "model_train(feature_name='KBest',model_algo=\"GLM\",features_list=features_list)"], "execution_count": 0, "outputs": []}, {"cell_type": "code", "metadata": {"id": "k8VwTsZTzcbG", "colab_type": "code", "colab": {}}, "source": ["# XGBoost with Select k best features\n", "model_train(feature_name='KBest',model_algo=\"XGBoost\",features_list=features_list)"], "execution_count": 0, "outputs": []}, {"cell_type": "code", "metadata": {"colab_type": "code", "id": "OCJ6iqdQWW3E", "colab": {}}, "source": ["# DRF with Select k best features\n", "model_train(feature_name='KBest',model_algo=\"DRF\",features_list=features_list)"], "execution_count": 0, "outputs": []}, {"cell_type": "code", "metadata": {"colab_type": "code", "id": "7yDkUBP8WW3F", "colab": {}}, "source": ["# GBM with Select k best features\n", "model_train(feature_name='KBest',model_algo=\"GBM\",features_list=features_list)"], "execution_count": 0, "outputs": []}, {"cell_type": "code", "metadata": {"colab_type": "code", "id": "ipBBvU2BWW3G", "colab": {}}, "source": ["# StackedEnsemble with Select k best features\n", "model_train(feature_name='KBest',model_algo=\"StackedEnsemble\",features_list=features_list)"], "execution_count": 0, "outputs": []}, {"cell_type": "code", "metadata": {"colab_type": "code", "id": "6upvZCSEWW3I", "colab": {}}, "source": ["# All algos with Select k best features\n", "model_train(feature_name='KBest',model_algo=None,features_list=features_list)"], "execution_count": 0, "outputs": []}, {"cell_type": "code", "metadata": {"id": "jqEimsVT4G27", "colab_type": "code", "colab": {}}, "source": ["# Select Percentile\n", "try:\n", "  del features_list\n", "except:\n", "  pass\n", "gc.collect()\n", "\n", "if features_to_include is True or 'Percentile' in features_to_include:\n", "  from sklearn.feature_selection import SelectPercentile\n", "  from sklearn.feature_selection import f_classif\n", "\n", "  bestfeatures = SelectPercentile(score_func=f_classif, percentile=select_percentile).fit(pd.read_pickle('scaled_df')[:index_splits[0]].drop(data_label_name,axis=1)\n", "                                                              ,pd.read_pickle('scaled_df')[:index_splits[0]][data_label_name])\n", "\n", "  features_list = list(np.array(train_hf.columns)[:-1][bestfeatures.get_support()])\n", "\n", "  del bestfeatures, SelectPercentile, f_classif\n", "  gc.collect()\n", "  display(len(features_list),features_list)\n", "else:\n", "  features_list=[]"], "execution_count": 0, "outputs": []}, {"cell_type": "code", "metadata": {"id": "l59Cc1ky43a_", "colab_type": "code", "colab": {}}, "source": ["# DeepLearning with Select Percentile features  \n", "model_train(feature_name='Percentile',model_algo=\"DeepLearning\",features_list=features_list)"], "execution_count": 0, "outputs": []}, {"cell_type": "code", "metadata": {"id": "YjX48fiQ5EHz", "colab_type": "code", "colab": {}}, "source": ["# GLM with Select Percentile features  \n", "model_train(feature_name='Percentile',model_algo=\"GLM\",features_list=features_list)"], "execution_count": 0, "outputs": []}, {"cell_type": "code", "metadata": {"id": "-UWubLNi5EhT", "colab_type": "code", "colab": {}}, "source": ["# XGBoost with Select Percentile features  \n", "model_train(feature_name='Percentile',model_algo=\"XGBoost\",features_list=features_list)"], "execution_count": 0, "outputs": []}, {"cell_type": "code", "metadata": {"colab_type": "code", "id": "iicj4vzFWXf1", "colab": {}}, "source": ["# DRF with Select Percentile features\n", "model_train(feature_name='Percentile',model_algo=\"DRF\",features_list=features_list)"], "execution_count": 0, "outputs": []}, {"cell_type": "code", "metadata": {"colab_type": "code", "id": "NYfSiibuWXf2", "colab": {}}, "source": ["# GBM with Select Percentile features\n", "model_train(feature_name='Percentile',model_algo=\"GBM\",features_list=features_list)"], "execution_count": 0, "outputs": []}, {"cell_type": "code", "metadata": {"colab_type": "code", "id": "u5WNIrIyWXf6", "colab": {}}, "source": ["# StackedEnsemble with Select Percentile features\n", "model_train(feature_name='Percentile',model_algo=\"StackedEnsemble\",features_list=features_list)"], "execution_count": 0, "outputs": []}, {"cell_type": "code", "metadata": {"colab_type": "code", "id": "XFpkprKfWXf7", "colab": {}}, "source": ["# All algos with Select Percentile features\n", "model_train(feature_name='Percentile',model_algo=None,features_list=features_list)"], "execution_count": 0, "outputs": []}, {"cell_type": "code", "metadata": {"id": "eaGqQhms546a", "colab_type": "code", "colab": {}}, "source": ["# Select Family-wise error\n", "try:\n", "  del features_list\n", "except:\n", "  pass\n", "gc.collect()\n", "\n", "if features_to_include is True or 'Fwe' in features_to_include:\n", "  from sklearn.feature_selection import SelectFwe\n", "  from sklearn.feature_selection import f_classif\n", "\n", "  bestfeatures = SelectFwe(score_func=f_classif, alpha=select_alpha).fit(pd.read_pickle('scaled_df')[:index_splits[0]].drop(data_label_name,axis=1)\n", "                                                              ,pd.read_pickle('scaled_df')[:index_splits[0]][data_label_name])\n", "\n", "  features_list = list(np.array(train_hf.columns)[:-1][bestfeatures.get_support()])\n", "\n", "  del bestfeatures, SelectFwe, f_classif\n", "  gc.collect()\n", "  display(len(features_list),features_list)\n", "else:\n", "  features_list=[]"], "execution_count": 0, "outputs": []}, {"cell_type": "code", "metadata": {"id": "2mZLkDUL6H3L", "colab_type": "code", "colab": {}}, "source": ["# DeepLearning with Select Family-wise error features\n", "model_train(feature_name='Fwe',model_algo=\"DeepLearning\",features_list=features_list)"], "execution_count": 0, "outputs": []}, {"cell_type": "code", "metadata": {"id": "ByFoBNYn6fZq", "colab_type": "code", "colab": {}}, "source": ["# GLM with Select Family-wise error features\n", "model_train(feature_name='Fwe',model_algo=\"GLM\",features_list=features_list)"], "execution_count": 0, "outputs": []}, {"cell_type": "code", "metadata": {"id": "n-AxJiWn6f5-", "colab_type": "code", "colab": {}}, "source": ["# XGBoost with Select Family-wise error features\n", "model_train(feature_name='Fwe',model_algo=\"XGBoost\",features_list=features_list)"], "execution_count": 0, "outputs": []}, {"cell_type": "code", "metadata": {"colab_type": "code", "id": "ePs7yCxtWYO5", "colab": {}}, "source": ["# DRF with Select Family-wise error features\n", "model_train(feature_name='Fwe',model_algo=\"DRF\",features_list=features_list)"], "execution_count": 0, "outputs": []}, {"cell_type": "code", "metadata": {"colab_type": "code", "id": "Ketw4xyxWYO6", "colab": {}}, "source": ["# GBM with Select Family-wise error features\n", "model_train(feature_name='Fwe',model_algo=\"GBM\",features_list=features_list)"], "execution_count": 0, "outputs": []}, {"cell_type": "code", "metadata": {"colab_type": "code", "id": "Ln4HbMhLWYO7", "colab": {}}, "source": ["# StackedEnsemble with Select Family-wise error features\n", "model_train(feature_name='Fwe',model_algo=\"StackedEnsemble\",features_list=features_list)"], "execution_count": 0, "outputs": []}, {"cell_type": "code", "metadata": {"colab_type": "code", "id": "Mh_Q-gnbWYO8", "colab": {}}, "source": ["# All algos with Select Family-wise error features\n", "model_train(feature_name='Fwe',model_algo=None,features_list=features_list)"], "execution_count": 0, "outputs": []}, {"cell_type": "code", "metadata": {"id": "4i2agU_6_zsO", "colab_type": "code", "colab": {}}, "source": ["# recursive feature elimination\n", "try:\n", "  del features_list\n", "except:\n", "  pass\n", "gc.collect()\n", "\n", "if features_to_include is True or 'RFE' in features_to_include:\n", "  from sklearn.feature_selection import RFE\n", "  from sklearn.svm import LinearSVC\n", "  from sklearn.tree import DecisionTreeClassifier,ExtraTreeClassifier\n", "  from sklearn.linear_model import LogisticRegression, LinearRegression\n", "\n", "  bestfeatures = RFE(ExtraTreeClassifier(random_state=1,criterion='entropy'), n_features_to_select=select_rfe, step=1).fit(pd.read_pickle('scaled_df')[:index_splits[0]].drop(data_label_name,axis=1)\n", "                                                              ,pd.read_pickle('scaled_df')[:index_splits[0]][data_label_name])\n", "\n", "  features_list = list(np.array(train_hf.columns)[:-1][bestfeatures.get_support()])\n", "\n", "  del bestfeatures, RFE, LinearSVC, DecisionTreeClassifier,ExtraTreeClassifier, LogisticRegression, LinearRegression\n", "  gc.collect()\n", "  display(len(features_list),features_list)\n", "else:\n", "  features_list = []"], "execution_count": 0, "outputs": []}, {"cell_type": "code", "metadata": {"id": "8ZyJS55CAUNO", "colab_type": "code", "colab": {}}, "source": ["# DeepLearning with recursive feature elimination\n", "model_train(feature_name='RFE',model_algo=\"DeepLearning\",features_list=features_list)"], "execution_count": 0, "outputs": []}, {"cell_type": "code", "metadata": {"id": "XY2m8aOjAj57", "colab_type": "code", "colab": {}}, "source": ["# GLM with recursive feature elimination\n", "model_train(feature_name='RFE',model_algo=\"GLM\",features_list=features_list)"], "execution_count": 0, "outputs": []}, {"cell_type": "code", "metadata": {"id": "5P3yJwbQAkgD", "colab_type": "code", "colab": {}}, "source": ["# XGBoost with recursive feature elimination\n", "model_train(feature_name='RFE',model_algo=\"XGBoost\",features_list=features_list)"], "execution_count": 0, "outputs": []}, {"cell_type": "code", "metadata": {"colab_type": "code", "id": "KcW993NFWZJ_", "colab": {}}, "source": ["# DRF with recursive feature elimination\n", "model_train(feature_name='RFE',model_algo=\"DRF\",features_list=features_list)"], "execution_count": 0, "outputs": []}, {"cell_type": "code", "metadata": {"colab_type": "code", "id": "MZjrUhblWZKA", "colab": {}}, "source": ["# GBM with recursive feature elimination\n", "model_train(feature_name='RFE',model_algo=\"GBM\",features_list=features_list)"], "execution_count": 0, "outputs": []}, {"cell_type": "code", "metadata": {"colab_type": "code", "id": "uP70r7bEWZKB", "colab": {}}, "source": ["# StackedEnsemble with recursive feature elimination\n", "model_train(feature_name='RFE',model_algo=\"StackedEnsemble\",features_list=features_list)"], "execution_count": 0, "outputs": []}, {"cell_type": "code", "metadata": {"colab_type": "code", "id": "pw45bMJ5WZKC", "colab": {}}, "source": ["# All algos with recursive feature elimination\n", "model_train(feature_name='RFE',model_algo=None,features_list=features_list)"], "execution_count": 0, "outputs": []}, {"cell_type": "code", "metadata": {"colab_type": "code", "id": "MbyzzruAxfA6", "colab": {}}, "source": ["# recursive feature elimination with cross validation\n", "try:\n", "  del features_list\n", "except:\n", "  pass\n", "gc.collect()\n", "\n", "if features_to_include is True or 'RFECV' in features_to_include:\n", "  from sklearn.feature_selection import RFECV\n", "  from sklearn.svm import LinearSVC\n", "  from sklearn.tree import DecisionTreeClassifier,ExtraTreeClassifier\n", "  from sklearn.linear_model import LogisticRegression, LinearRegression\n", "\n", "  bestfeatures = RFECV(ExtraTreeClassifier(random_state=1,criterion='entropy'), min_features_to_select=select_rfe, step=1,cv=cv_rfe).fit(pd.read_pickle('scaled_df')[:index_splits[0]].drop(data_label_name,axis=1)\n", "                                                              ,pd.read_pickle('scaled_df')[:index_splits[0]][data_label_name])\n", "\n", "  features_list = list(np.array(train_hf.columns)[:-1][bestfeatures.get_support()])\n", "\n", "  del bestfeatures, RFECV, LinearSVC, DecisionTreeClassifier,ExtraTreeClassifier, LogisticRegression, LinearRegression\n", "  gc.collect()\n", "  display(len(features_list),features_list)\n", "else:\n", "  features_list = []"], "execution_count": 0, "outputs": []}, {"cell_type": "code", "metadata": {"colab_type": "code", "id": "SZnCUwpjxfA8", "colab": {}}, "source": ["# DeepLearning with recursive feature elimination with cross validation\n", "model_train(feature_name='RFECV',model_algo=\"DeepLearning\",features_list=features_list)"], "execution_count": 0, "outputs": []}, {"cell_type": "code", "metadata": {"colab_type": "code", "id": "txQpRlQ6xfA9", "colab": {}}, "source": ["# GLM with recursive feature elimination with cross validation\n", "model_train(feature_name='RFECV',model_algo=\"GLM\",features_list=features_list)"], "execution_count": 0, "outputs": []}, {"cell_type": "code", "metadata": {"colab_type": "code", "id": "kqnROUyMxfA_", "colab": {}}, "source": ["# XGBoost with recursive feature elimination with cross validation\n", "model_train(feature_name='RFECV',model_algo=\"XGBoost\",features_list=features_list)"], "execution_count": 0, "outputs": []}, {"cell_type": "code", "metadata": {"colab_type": "code", "id": "yaTn2nYMxfBB", "colab": {}}, "source": ["# DRF with recursive feature elimination with cross validation\n", "model_train(feature_name='RFECV',model_algo=\"DRF\",features_list=features_list)"], "execution_count": 0, "outputs": []}, {"cell_type": "code", "metadata": {"colab_type": "code", "id": "qV3krljMxfBC", "colab": {}}, "source": ["# GBM with recursive feature elimination with cross validation\n", "model_train(feature_name='RFECV',model_algo=\"GBM\",features_list=features_list)"], "execution_count": 0, "outputs": []}, {"cell_type": "code", "metadata": {"colab_type": "code", "id": "_ljllbsuxfBD", "colab": {}}, "source": ["# StackedEnsemble with recursive feature elimination with cross validation\n", "model_train(feature_name='RFECV',model_algo=\"StackedEnsemble\",features_list=features_list)"], "execution_count": 0, "outputs": []}, {"cell_type": "code", "metadata": {"colab_type": "code", "id": "xjPeUDTwxfBF", "colab": {}}, "source": ["# All algos with recursive feature elimination with cross validation\n", "model_train(feature_name='RFECV',model_algo=None,features_list=features_list)"], "execution_count": 0, "outputs": []}, {"cell_type": "code", "metadata": {"id": "rKB66_0oqor-", "colab_type": "code", "colab": {}}, "source": ["# Stacked ensemble\n", "\n", "## filter models based on ensemble_model_threshold\n", "models_arr = []\n", "for model_name in best_models:\n", "  if best_models[max(best_models.keys(), key=(lambda k: best_models[k]['AUC']))]['AUC'] - best_models[model_name]['AUC'] <= ensemble_model_threshold:\n", "    models_arr.append(best_models[model_name]['id'])\n", "\n", "from h2o.estimators.stackedensemble import H2OStackedEnsembleEstimator\n", "\n", "try:\n", "  ensemble = H2OStackedEnsembleEstimator(seed=1,\n", "                                        base_models=models_arr)\n", "\n", "  ensemble.train(y=data_label_name, training_frame=train_hf[:index_splits[0],:], blending_frame =train_hf[index_splits[0]:index_splits[1],:])\n", "except:\n", "  pass\n", "\n", "try:\n", "  display(ensemble)\n", "  model_performance = ensemble.model_performance(test_data=train_hf[index_splits[1]:,:])\n", "  display(model_performance)\n", "  for k in models_summary:\n", "    result = getattr(model_performance, k)()\n", "    result = result[0][1] if isinstance(result,list) else result\n", "    models_summary[k] = models_summary[k].append(pd.DataFrame({'StackedEnsemble':[result]},index=['All']))\n", "    display((k,result))\n", "except Exception as e:\n", "  print(e)"], "execution_count": 0, "outputs": []}, {"cell_type": "code", "metadata": {"id": "zSSibE6tqtvz", "colab_type": "code", "colab": {}}, "source": ["# Super learner\n", "models_df = pd.DataFrame()\n", "for model_name in best_models:\n", "  if best_models[max(best_models.keys(), key=(lambda k: best_models[k]['AUC']))]['AUC'] - best_models[model_name]['AUC'] <= ensemble_model_threshold:\n", "    models_df[model_name] = pd.read_csv(model_name+\".csv\")['p1']\n", "    models_df[model_name+'predict'] = pd.read_csv(model_name+\".csv\")['predict']\n", "\n", "models_df[data_label_name] = pd.read_pickle('scaled_df')[[data_label_name]].values[-len(models_df):]\n", "\n", "training_frame = h2o.H2OFrame(models_df)\n", "training_frame[data_label_name] = training_frame[data_label_name].ascharacter().asfactor()\n", "\n", "model_train(feature_name='Super_learner',model_algo=None,training_frame= training_frame,max_models=500)"], "execution_count": 0, "outputs": []}, {"cell_type": "code", "metadata": {"id": "7IKBoik5qzsZ", "colab_type": "code", "colab": {}}, "source": ["# Majority Voting\n", "models1_df = pd.DataFrame()\n", "for model_name in best_models:\n", "  if best_models[max(best_models.keys(), key=(lambda k: best_models[k]['AUC']))]['AUC'] - best_models[model_name]['AUC'] <= ensemble_model_threshold:\n", "    models1_df[model_name] = pd.read_csv(model_name+\".csv\")['predict']\n", "\n", "model_accuracy = accuracy_score(models1_df.mode(1)[0].values[index_splits[1]:],pd.read_pickle('scaled_df')[[data_label_name]].values[index_splits[1]:])\n", "models_summary['accuracy'] = models_summary['accuracy'].append(pd.DataFrame({'MajorityVoting':[model_accuracy]},index=['All']))\n", "display('MajorityVoting', model_accuracy)"], "execution_count": 0, "outputs": []}, {"cell_type": "code", "metadata": {"id": "JvQJYi4Pq79h", "colab_type": "code", "colab": {}}, "source": ["# LogReg SuperLearner\n", "from sklearn.linear_model import LogisticRegression\n", "models2_df = pd.DataFrame()\n", "for model_name in best_models:\n", "  if best_models[max(best_models.keys(), key=(lambda k: best_models[k]['AUC']))]['AUC'] - best_models[model_name]['AUC'] <= ensemble_model_threshold:\n", "    models2_df[model_name] = pd.read_csv(model_name+\".csv\")['predict']\n", "    models2_df[model_name+'p1'] = pd.read_csv(model_name+\".csv\")['p1']\n", "\n", "clf = LogisticRegression(random_state=0, solver='lbfgs',\n", "                         multi_class='multinomial').fit(models2_df.values[index_splits[0]:index_splits[1]], pd.read_pickle('scaled_df')[[data_label_name]].values[index_splits[0]:index_splits[1]])\n", "\n", "model_accuracy = clf.score(models2_df.values[index_splits[1]:], pd.read_pickle('scaled_df')[[data_label_name]].values[index_splits[1]:])\n", "models_summary['accuracy'] = models_summary['accuracy'].append(pd.DataFrame({'LogReg':[model_accuracy]},index=['All']))\n", "model_accuracy"], "execution_count": 0, "outputs": []}, {"cell_type": "code", "metadata": {"id": "-e7AOxoeq97L", "colab_type": "code", "colab": {}}, "source": ["# DTree SuperLearner\n", "from sklearn.tree import DecisionTreeClassifier\n", "clf = DecisionTreeClassifier(random_state=0).fit(models2_df.values[index_splits[0]:index_splits[1]], pd.read_pickle('scaled_df')[[data_label_name]].values[index_splits[0]:index_splits[1]])\n", "\n", "model_accuracy = clf.score(models2_df.values[index_splits[1]:], pd.read_pickle('scaled_df')[[data_label_name]].values[index_splits[1]:])\n", "models_summary['accuracy'] = models_summary['accuracy'].append(pd.DataFrame({'DTree':[model_accuracy]},index=['All']))\n", "model_accuracy"], "execution_count": 0, "outputs": []}, {"cell_type": "code", "metadata": {"id": "MggFqqvarFrr", "colab_type": "code", "colab": {}}, "source": ["# Experiment Summary\n", "pd.set_option('precision', 3) \n", "\n", "for k in models_summary:\n", "  print(k)\n", "  display(models_summary[k].max())\n", "  print('\\n'*3)\n", "  display(models_summary[k].groupby(models_summary[k].index).max())\n", "  print('\\n'*3)\n", "  \n", "display(models_summary)\n", "\n", "pd.set_option('precision', 6) "], "execution_count": 0, "outputs": []}]}