surrogates.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "from scipy.io import arff\n",
    "import numpy as np\n",
    "import pandas as pd\n",
    "import os\n",
    "from sklearn.pipeline import Pipeline\n",
    "from sklearn.ensemble import RandomForestRegressor\n",
    "from sklearn.model_selection import train_test_split\n",
    "from sklearn.model_selection import cross_val_score, cross_val_predict\n",
    "from sklearn.metrics import make_scorer\n",
    "from collections import defaultdict\n",
    "from scipy.stats import rankdata\n",
    "import matplotlib.pyplot as plt\n",
    "from scipy.stats import pearsonr\n",
    "import sklearn\n",
    "from sklearn.linear_model import LinearRegression\n",
    "from sklearn.model_selection import RandomizedSearchCV\n",
    "from sklearn.preprocessing import PolynomialFeatures\n",
    "import pickle\n",
    "from sklearn.model_selection import KFold\n",
    "from sklearn.base import clone"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "# utility functions\n",
    "def get_data(data, relevant_params):\n",
    "    task_ids = data['task_id'].unique()\n",
    "    data = data.loc[data['svc__kernel'] == b'rbf']\n",
    "    data = data[relevant_params + ['task_id'] + ['predictive_accuracy']]\n",
    "    data_dict = defaultdict()\n",
    "    for task_id in task_ids:\n",
    "        X_task = data.loc[data['task_id'] == task_id]\n",
    "        y_task = np.array(X_task['predictive_accuracy'], dtype=np.float)\n",
    "        X_task.drop(['predictive_accuracy', 'task_id'], 1, inplace=True)\n",
    "        categorical_names = X_task.select_dtypes(include=['object']).columns\n",
    "        categorical_ids = [X_task.columns.get_loc(colname) for colname in categorical_names]\n",
    "        data_dict[task_id] = (X_task.as_matrix(), y_task, categorical_ids)\n",
    "    return data_dict\n",
    "\n",
    "def precatn(task_id, model, X, y, kfolds=5, topn=5):\n",
    "    kf = KFold(n_splits=kfolds, random_state=42, shuffle=True)\n",
    "    splits = kf.split(X)\n",
    "    scores = []\n",
    "    for train_idx, test_idx in splits:\n",
    "        train_x, train_y = X[train_idx], y[train_idx]\n",
    "        test_x, test_y = X[test_idx], y[test_idx]\n",
    "        new_model = clone(model)\n",
    "        new_model.fit(train_x, train_y)\n",
    "        y_hat = new_model.predict(test_x)\n",
    "        y_hat_ranks = rankdata(y_hat, method='average')\n",
    "        test_y_ranks = rankdata(test_y, method='average')\n",
    "        y_hat_maxargs = y_hat_ranks.argsort()\n",
    "        test_y_maxargs = test_y_ranks.argsort()\n",
    "        cnt = 0\n",
    "        for entry in y_hat_maxargs[:topn]:\n",
    "            if entry in test_y_maxargs[:topn]:\n",
    "                cnt += 1\n",
    "        scores.append(cnt / topn)\n",
    "    mean_score = np.mean(scores)\n",
    "    print('Task %d; Precision at %d Score: %0.4f' %(task_id, topn, mean_score))\n",
    "    return mean_score\n",
    "\n",
    "def custom_scorer(y, y_hat):\n",
    "    return pearsonr(y, y_hat)[0]\n",
    "\n",
    "def pearsonscore(task_id, model, X, y, kfolds=5):\n",
    "    # y_hat = cross_val_predict(clf, X, y, cv=kfolds)\n",
    "    # score = pearsonr(y, y_hat)[0]\n",
    "    scores = cross_val_score(model, X, y, cv=kfolds, scoring=make_scorer(custom_scorer))\n",
    "    score = scores.mean()\n",
    "    print('Task %d; Pearson Spearman Correlation: %0.4f (+/- %0.4f)' %(task_id, score, scores.std() * 2))\n",
    "    return score\n",
    "\n",
    "def get_noise(task_id, model, X, y, kfolds=5):\n",
    "    kf = KFold(n_splits=kfolds, random_state=42, shuffle=True)\n",
    "    splits = kf.split(X)\n",
    "    y_hat_all = []\n",
    "    for train_idx, test_idx in splits:\n",
    "        train_x, train_y = X[train_idx], y[train_idx]\n",
    "        test_x, test_y = X[test_idx], y[test_idx]\n",
    "        new_model = clone(model)\n",
    "        new_model.fit(train_x, train_y)\n",
    "        y_hat = new_model.predict(test_x)\n",
    "        y_hat_all.append(y_hat)\n",
    "    scores = []\n",
    "    for y_hat, i in enumerate(y_hat_all):\n",
    "        for j in range(i + 1, topn):\n",
    "            scores.append(pearsonr(y_hat, y_hat_all[j]))\n",
    "    return np.mean(scores)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/Users/prernakashyap/anaconda3/envs/gensim_env/lib/python3.6/site-packages/pandas/core/frame.py:3697: SettingWithCopyWarning: \n",
      "A value is trying to be set on a copy of a slice from a DataFrame\n",
      "\n",
      "See the caveats in the documentation: http://pandas.pydata.org/pandas-docs/stable/indexing.html#indexing-view-versus-copy\n",
      "  errors=errors)\n",
      "/Users/prernakashyap/anaconda3/envs/gensim_env/lib/python3.6/site-packages/ipykernel_launcher.py:13: FutureWarning: Method .as_matrix will be removed in a future version. Use .values instead.\n",
      "  del sys.path[0]\n"
     ]
    }
   ],
   "source": [
    "filename = './data/svc.arff'\n",
    "relevant_params = ['svc__C', 'svc__gamma']\n",
    "data, meta = arff.loadarff(filename)\n",
    "df = pd.DataFrame(data)\n",
    "data_dict = get_data(df, relevant_params)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "class surrogate:\n",
    "\n",
    "    def __init__(self, ntrees=16):\n",
    "        self.models = {}\n",
    "        self.ntrees = ntrees\n",
    "\n",
    "    def train_model_rs(self, X, y, categoricals, task_id):\n",
    "        if task_id in self.models:\n",
    "            return self.models[task_id]\n",
    "        param_dist = {'max_depth': np.arange(3, 11),\n",
    "              'max_features': [1, 2],\n",
    "              'n_estimators' : [100],\n",
    "              'min_samples_split': np.arange(2, 11)}\n",
    "        clf = Pipeline(\n",
    "            steps=[('encoder', sklearn.preprocessing.OneHotEncoder(\n",
    "                categorical_features=list(categoricals), handle_unknown='ignore')),\n",
    "                    ('random search', RandomizedSearchCV(RandomForestRegressor(n_estimators=self.ntrees), \n",
    "                                                         param_distributions=param_dist,\n",
    "                                                         n_iter=20, cv=5))])\n",
    "        clf.fit(X, y)\n",
    "        self.models[task_id] = clf\n",
    "        return clf\n",
    "    \n",
    "    def train_model(self, X, y, categoricals, task_id):\n",
    "        if task_id in self.models:\n",
    "            return self.models[task_id]\n",
    "        clf = Pipeline(\n",
    "            steps=[('encoder', sklearn.preprocessing.OneHotEncoder(\n",
    "                categorical_features=list(categoricals), handle_unknown='ignore')),\n",
    "                    ('classifier', RandomForestRegressor(n_estimators=self.ntrees))])\n",
    "        clf.fit(X, y)\n",
    "        self.models[task_id] = clf\n",
    "        return clf\n",
    "\n",
    "    def train_surrogate(self, data_dict):\n",
    "        task_ids = list(data_dict.keys())\n",
    "        scores = []\n",
    "        for task_id in task_ids:\n",
    "            X, y, categoricals = data_dict[task_id]\n",
    "            clf = self.train_model_rs(X, y, categoricals, task_id)\n",
    "            precatn_score = precatn(task_id, clf.named_steps['random search'].best_estimator_, X, y, topn=100)\n",
    "            pearson_score = pearsonscore(task_id, clf.named_steps['random search'].best_estimator_, X, y)\n",
    "            scores.append(np.array([precatn_score, pearson_score]))\n",
    "        return np.array(scores)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "300.0"
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "np.mean(np.sum([len(data_dict[task_id]) for task_id in list(data_dict.keys())]))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "s = surrogate()\n",
    "surrogate_scores = s.train_surrogate(data_dict)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/Users/prernakashyap/anaconda3/lib/python3.6/site-packages/sklearn/base.py:311: UserWarning: Trying to unpickle estimator OneHotEncoder from version 0.19.0 when using version 0.19.1. This might lead to breaking code or invalid results. Use at your own risk.\n",
      "  UserWarning)\n",
      "/Users/prernakashyap/anaconda3/lib/python3.6/site-packages/sklearn/base.py:311: UserWarning: Trying to unpickle estimator DecisionTreeRegressor from version 0.19.0 when using version 0.19.1. This might lead to breaking code or invalid results. Use at your own risk.\n",
      "  UserWarning)\n",
      "/Users/prernakashyap/anaconda3/lib/python3.6/site-packages/sklearn/base.py:311: UserWarning: Trying to unpickle estimator RandomForestRegressor from version 0.19.0 when using version 0.19.1. This might lead to breaking code or invalid results. Use at your own risk.\n",
      "  UserWarning)\n",
      "/Users/prernakashyap/anaconda3/lib/python3.6/site-packages/sklearn/base.py:311: UserWarning: Trying to unpickle estimator RandomizedSearchCV from version 0.19.0 when using version 0.19.1. This might lead to breaking code or invalid results. Use at your own risk.\n",
      "  UserWarning)\n",
      "/Users/prernakashyap/anaconda3/lib/python3.6/site-packages/sklearn/base.py:311: UserWarning: Trying to unpickle estimator Pipeline from version 0.19.0 when using version 0.19.1. This might lead to breaking code or invalid results. Use at your own risk.\n",
      "  UserWarning)\n"
     ]
    }
   ],
   "source": [
    "surrogates = pickle.load(open('/Users/prernakashyap/Documents/autoML research/models/sm2.pkl', 'rb'))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Task 3; Precision at 100 Score: 0.9820\n",
      "Task 3; Pearson Spearman Correlation: 0.9865 (+/- 0.0086)\n",
      "Task 6; Precision at 100 Score: 0.9840\n",
      "Task 6; Pearson Spearman Correlation: 0.9927 (+/- 0.0029)\n",
      "Task 11; Precision at 100 Score: 0.9800\n",
      "Task 11; Pearson Spearman Correlation: 0.9631 (+/- 0.0297)\n",
      "Task 12; Precision at 100 Score: 0.9820\n",
      "Task 12; Pearson Spearman Correlation: 0.9987 (+/- 0.0015)\n",
      "Task 14; Precision at 100 Score: 0.9580\n",
      "Task 14; Pearson Spearman Correlation: 0.9962 (+/- 0.0067)\n",
      "Task 15; Precision at 100 Score: 0.8880\n",
      "Task 15; Pearson Spearman Correlation: 0.9661 (+/- 0.0306)\n",
      "Task 16; Precision at 100 Score: 0.9740\n",
      "Task 16; Pearson Spearman Correlation: 0.9963 (+/- 0.0031)\n",
      "Task 18; Precision at 100 Score: 0.9780\n",
      "Task 18; Pearson Spearman Correlation: 0.9895 (+/- 0.0111)\n",
      "Task 20; Precision at 100 Score: 0.9620\n",
      "Task 20; Pearson Spearman Correlation: 0.9980 (+/- 0.0022)\n",
      "Task 21; Precision at 100 Score: 0.9800\n",
      "Task 21; Pearson Spearman Correlation: 0.9915 (+/- 0.0043)\n",
      "Task 22; Precision at 100 Score: 0.9660\n",
      "Task 22; Pearson Spearman Correlation: 0.9983 (+/- 0.0012)\n",
      "Task 23; Precision at 100 Score: 0.9660\n",
      "Task 23; Pearson Spearman Correlation: 0.9750 (+/- 0.0206)\n",
      "Task 24; Precision at 100 Score: 0.8260\n",
      "Task 24; Pearson Spearman Correlation: 0.9677 (+/- 0.0193)\n",
      "Task 28; Precision at 100 Score: 0.9740\n",
      "Task 28; Pearson Spearman Correlation: 0.9859 (+/- 0.0128)\n",
      "Task 29; Precision at 100 Score: 0.8960\n",
      "Task 29; Pearson Spearman Correlation: 0.9847 (+/- 0.0111)\n",
      "Task 31; Precision at 100 Score: 0.9660\n",
      "Task 31; Pearson Spearman Correlation: 0.9733 (+/- 0.0186)\n",
      "Task 32; Precision at 100 Score: 0.9520\n",
      "Task 32; Pearson Spearman Correlation: 0.9814 (+/- 0.0171)\n",
      "Task 36; Precision at 100 Score: 0.9780\n",
      "Task 36; Pearson Spearman Correlation: 0.9941 (+/- 0.0024)\n",
      "Task 37; Precision at 100 Score: 0.9520\n",
      "Task 37; Pearson Spearman Correlation: 0.9761 (+/- 0.0156)\n",
      "Task 41; Precision at 100 Score: 0.8520\n",
      "Task 41; Pearson Spearman Correlation: 0.9909 (+/- 0.0093)\n",
      "Task 43; Precision at 100 Score: 0.9400\n",
      "Task 43; Pearson Spearman Correlation: 0.9859 (+/- 0.0098)\n",
      "Task 45; Precision at 100 Score: 0.9060\n",
      "Task 45; Pearson Spearman Correlation: 0.9763 (+/- 0.0265)\n",
      "Task 49; Precision at 100 Score: 0.9620\n",
      "Task 49; Pearson Spearman Correlation: 0.9894 (+/- 0.0037)\n",
      "Task 53; Precision at 100 Score: 0.9820\n",
      "Task 53; Pearson Spearman Correlation: 0.9917 (+/- 0.0050)\n",
      "Task 58; Precision at 100 Score: 0.9580\n",
      "Task 58; Pearson Spearman Correlation: 0.9913 (+/- 0.0147)\n",
      "Task 219; Precision at 100 Score: 0.9920\n",
      "Task 219; Pearson Spearman Correlation: 0.9872 (+/- 0.0095)\n",
      "Task 2074; Precision at 100 Score: 0.9760\n",
      "Task 2074; Pearson Spearman Correlation: 0.9899 (+/- 0.0043)\n",
      "Task 2079; Precision at 100 Score: 0.9360\n",
      "Task 2079; Pearson Spearman Correlation: 0.9895 (+/- 0.0093)\n",
      "Task 3021; Precision at 100 Score: 0.9680\n",
      "Task 3021; Pearson Spearman Correlation: 0.9924 (+/- 0.0070)\n",
      "Task 3022; Precision at 100 Score: 0.9820\n",
      "Task 3022; Pearson Spearman Correlation: 0.9954 (+/- 0.0040)\n",
      "Task 3481; Precision at 100 Score: 0.9940\n",
      "Task 3481; Pearson Spearman Correlation: 0.9907 (+/- 0.0131)\n",
      "Task 3485; Precision at 100 Score: 0.9940\n",
      "Task 3485; Pearson Spearman Correlation: 0.9952 (+/- 0.0042)\n",
      "Task 3492; Precision at 100 Score: 0.8580\n",
      "Task 3492; Pearson Spearman Correlation: 0.9721 (+/- 0.0181)\n",
      "Task 3493; Precision at 100 Score: 0.9500\n",
      "Task 3493; Pearson Spearman Correlation: 0.9820 (+/- 0.0119)\n",
      "Task 3494; Precision at 100 Score: 0.9100\n",
      "Task 3494; Pearson Spearman Correlation: 0.9798 (+/- 0.0187)\n",
      "Task 3510; Precision at 100 Score: 0.9700\n",
      "Task 3510; Pearson Spearman Correlation: 0.9871 (+/- 0.0153)\n",
      "Task 3512; Precision at 100 Score: 0.9180\n",
      "Task 3512; Pearson Spearman Correlation: 0.9830 (+/- 0.0219)\n",
      "Task 3543; Precision at 100 Score: 0.8420\n",
      "Task 3543; Pearson Spearman Correlation: 0.9757 (+/- 0.0273)\n",
      "Task 3549; Precision at 100 Score: 0.8960\n",
      "Task 3549; Pearson Spearman Correlation: 0.9909 (+/- 0.0055)\n",
      "Task 3560; Precision at 100 Score: 0.9140\n",
      "Task 3560; Pearson Spearman Correlation: 0.8383 (+/- 0.0208)\n",
      "Task 3561; Precision at 100 Score: 0.9700\n",
      "Task 3561; Pearson Spearman Correlation: 0.9837 (+/- 0.0097)\n",
      "Task 3567; Precision at 100 Score: 0.9740\n",
      "Task 3567; Pearson Spearman Correlation: 0.9953 (+/- 0.0022)\n"
     ]
    }
   ],
   "source": [
    "surrogate_scores = []\n",
    "task_ids = list(data_dict.keys())\n",
    "for task_id in task_ids:\n",
    "    X, y, categoricals = data_dict[task_id]\n",
    "    clf = surrogates.models[task_id].named_steps['random search'].best_estimator_\n",
    "    precatn_score = precatn(task_id, clf, X, y, topn=100)\n",
    "    pearson_score = pearsonscore(task_id, clf, X, y)\n",
    "    surrogate_scores.append(np.array([precatn_score, pearson_score]))\n",
    "surrogate_scores = np.array(surrogate_scores)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [],
   "source": [
    "class simplified_surrogate:\n",
    "\n",
    "    def __init__(self):\n",
    "        self.models = {}\n",
    "\n",
    "    def train_model(self, data_dict):\n",
    "        scores = []\n",
    "        for task_id in data_dict:\n",
    "            X, y, _ = data_dict[task_id]\n",
    "            poly = PolynomialFeatures(2)\n",
    "            X = poly.fit_transform(X)\n",
    "            X = np.delete(X, 0, 1)\n",
    "            model = LinearRegression()\n",
    "            model.fit(X, y)\n",
    "            self.models[task_id] = model\n",
    "            precatn_score = precatn(task_id, model, X, y, topn=100)\n",
    "            pearson_score = pearsonscore(task_id, model, X, y)\n",
    "            scores.append(np.array([precatn_score, pearson_score]))\n",
    "        return np.array(scores)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Task 3; Precision at 100 Score: 0.8440\n",
      "Task 3; Pearson Spearman Correlation: 0.6233 (+/- 0.0700)\n",
      "Task 6; Precision at 100 Score: 0.8040\n",
      "Task 6; Pearson Spearman Correlation: 0.4578 (+/- 0.0803)\n",
      "Task 11; Precision at 100 Score: 0.8500\n",
      "Task 11; Pearson Spearman Correlation: 0.3715 (+/- 0.0800)\n",
      "Task 12; Precision at 100 Score: 0.8000\n",
      "Task 12; Pearson Spearman Correlation: 0.7769 (+/- 0.0531)\n",
      "Task 14; Precision at 100 Score: 0.8840\n",
      "Task 14; Pearson Spearman Correlation: 0.7832 (+/- 0.1274)\n",
      "Task 15; Precision at 100 Score: 0.7000\n",
      "Task 15; Pearson Spearman Correlation: 0.2724 (+/- 0.2563)\n",
      "Task 16; Precision at 100 Score: 0.8200\n",
      "Task 16; Pearson Spearman Correlation: 0.7921 (+/- 0.0599)\n",
      "Task 18; Precision at 100 Score: 0.7640\n",
      "Task 18; Pearson Spearman Correlation: 0.2892 (+/- 0.1161)\n",
      "Task 20; Precision at 100 Score: 0.8800\n",
      "Task 20; Pearson Spearman Correlation: 0.5511 (+/- 0.0667)\n",
      "Task 21; Precision at 100 Score: 0.8340\n",
      "Task 21; Pearson Spearman Correlation: 0.5419 (+/- 0.0810)\n",
      "Task 22; Precision at 100 Score: 0.8100\n",
      "Task 22; Pearson Spearman Correlation: 0.9260 (+/- 0.0216)\n",
      "Task 23; Precision at 100 Score: 0.7740\n",
      "Task 23; Pearson Spearman Correlation: 0.3569 (+/- 0.1800)\n",
      "Task 24; Precision at 100 Score: 0.7960\n",
      "Task 24; Pearson Spearman Correlation: 0.7524 (+/- 0.0688)\n",
      "Task 28; Precision at 100 Score: 0.8000\n",
      "Task 28; Pearson Spearman Correlation: 0.8548 (+/- 0.0657)\n",
      "Task 29; Precision at 100 Score: 0.8220\n",
      "Task 29; Pearson Spearman Correlation: 0.6670 (+/- 0.1606)\n",
      "Task 31; Precision at 100 Score: 0.8380\n",
      "Task 31; Pearson Spearman Correlation: 0.4376 (+/- 0.0302)\n",
      "Task 32; Precision at 100 Score: 0.8260\n",
      "Task 32; Pearson Spearman Correlation: 0.4837 (+/- 0.1025)\n",
      "Task 36; Precision at 100 Score: 0.8440\n",
      "Task 36; Pearson Spearman Correlation: 0.4415 (+/- 0.2249)\n",
      "Task 37; Precision at 100 Score: 0.8060\n",
      "Task 37; Pearson Spearman Correlation: 0.6045 (+/- 0.0673)\n",
      "Task 41; Precision at 100 Score: 0.8260\n",
      "Task 41; Pearson Spearman Correlation: 0.6171 (+/- 0.0266)\n",
      "Task 43; Precision at 100 Score: 0.8180\n",
      "Task 43; Pearson Spearman Correlation: 0.5977 (+/- 0.0664)\n",
      "Task 45; Precision at 100 Score: 0.7320\n",
      "Task 45; Pearson Spearman Correlation: 0.5673 (+/- 0.0744)\n",
      "Task 49; Precision at 100 Score: 0.8380\n",
      "Task 49; Pearson Spearman Correlation: 0.5510 (+/- 0.0648)\n",
      "Task 53; Precision at 100 Score: 0.8820\n",
      "Task 53; Pearson Spearman Correlation: 0.6104 (+/- 0.0698)\n",
      "Task 58; Precision at 100 Score: 0.8060\n",
      "Task 58; Pearson Spearman Correlation: 0.7686 (+/- 0.0441)\n",
      "Task 219; Precision at 100 Score: 0.8840\n",
      "Task 219; Pearson Spearman Correlation: 0.4366 (+/- 0.1707)\n",
      "Task 2074; Precision at 100 Score: 0.7720\n",
      "Task 2074; Pearson Spearman Correlation: 0.7767 (+/- 0.1222)\n",
      "Task 2079; Precision at 100 Score: 0.8240\n",
      "Task 2079; Pearson Spearman Correlation: 0.6083 (+/- 0.0509)\n",
      "Task 3021; Precision at 100 Score: 0.7840\n",
      "Task 3021; Pearson Spearman Correlation: 0.4583 (+/- 0.0597)\n",
      "Task 3022; Precision at 100 Score: 0.7720\n",
      "Task 3022; Pearson Spearman Correlation: 0.4606 (+/- 0.0543)\n",
      "Task 3481; Precision at 100 Score: 0.8820\n",
      "Task 3481; Pearson Spearman Correlation: 0.3409 (+/- 0.1096)\n",
      "Task 3485; Precision at 100 Score: 0.8740\n",
      "Task 3485; Pearson Spearman Correlation: 0.5015 (+/- 0.0860)\n",
      "Task 3492; Precision at 100 Score: 0.8000\n",
      "Task 3492; Pearson Spearman Correlation: 0.4438 (+/- 0.0891)\n",
      "Task 3493; Precision at 100 Score: 0.7940\n",
      "Task 3493; Pearson Spearman Correlation: 0.3481 (+/- 0.2385)\n",
      "Task 3494; Precision at 100 Score: 0.8440\n",
      "Task 3494; Pearson Spearman Correlation: 0.3879 (+/- 0.0546)\n",
      "Task 3510; Precision at 100 Score: 0.8040\n",
      "Task 3510; Pearson Spearman Correlation: 0.6189 (+/- 0.1617)\n",
      "Task 3512; Precision at 100 Score: 0.7880\n",
      "Task 3512; Pearson Spearman Correlation: 0.8197 (+/- 0.0963)\n",
      "Task 3543; Precision at 100 Score: 0.7580\n",
      "Task 3543; Pearson Spearman Correlation: 0.2750 (+/- 0.0698)\n",
      "Task 3549; Precision at 100 Score: 0.8480\n",
      "Task 3549; Pearson Spearman Correlation: 0.6368 (+/- 0.0517)\n",
      "Task 3560; Precision at 100 Score: 0.7620\n",
      "Task 3560; Pearson Spearman Correlation: 0.2140 (+/- 0.1021)\n",
      "Task 3561; Precision at 100 Score: 0.8580\n",
      "Task 3561; Pearson Spearman Correlation: 0.5530 (+/- 0.0788)\n",
      "Task 3567; Precision at 100 Score: 0.9000\n",
      "Task 3567; Pearson Spearman Correlation: 0.6187 (+/- 0.0653)\n",
      "Task 3573; Precision at 100 Score: 0.6640\n",
      "Task 3573; Pearson Spearman Correlation: 0.5152 (+/- 0.1383)\n",
      "Task 3889; Precision at 100 Score: 0.8680\n",
      "Task 3889; Pearson Spearman Correlation: 0.5209 (+/- 0.0479)\n",
      "Task 3891; Precision at 100 Score: 0.8620\n",
      "Task 3891; Pearson Spearman Correlation: 0.5088 (+/- 0.0332)\n",
      "Task 3896; Precision at 100 Score: 0.8220\n",
      "Task 3896; Pearson Spearman Correlation: 0.5993 (+/- 0.0528)\n",
      "Task 3899; Precision at 100 Score: 0.8800\n",
      "Task 3899; Pearson Spearman Correlation: 0.4625 (+/- 0.0846)\n",
      "Task 3902; Precision at 100 Score: 0.8480\n",
      "Task 3902; Pearson Spearman Correlation: 0.5550 (+/- 0.0651)\n",
      "Task 3903; Precision at 100 Score: 0.8260\n",
      "Task 3903; Pearson Spearman Correlation: 0.6385 (+/- 0.1248)\n",
      "Task 3904; Precision at 100 Score: 0.6400\n",
      "Task 3904; Pearson Spearman Correlation: 0.7504 (+/- 0.0948)\n",
      "Task 3913; Precision at 100 Score: 0.7700\n",
      "Task 3913; Pearson Spearman Correlation: 0.5216 (+/- 0.1350)\n",
      "Task 3917; Precision at 100 Score: 0.7160\n",
      "Task 3917; Pearson Spearman Correlation: 0.5667 (+/- 0.1130)\n",
      "Task 3918; Precision at 100 Score: 0.8200\n",
      "Task 3918; Pearson Spearman Correlation: 0.5599 (+/- 0.1621)\n",
      "Task 3946; Precision at 100 Score: 0.4000\n",
      "Task 3946; Pearson Spearman Correlation: 0.2974 (+/- 0.4390)\n",
      "Task 3948; Precision at 100 Score: 0.5840\n",
      "Task 3948; Pearson Spearman Correlation: 0.2765 (+/- 0.1261)\n",
      "Task 3954; Precision at 100 Score: 0.7620\n",
      "Task 3954; Pearson Spearman Correlation: 0.3711 (+/- 0.1768)\n",
      "Task 7592; Precision at 100 Score: 0.8220\n",
      "Task 7592; Pearson Spearman Correlation: 0.6995 (+/- 0.0555)\n",
      "Task 9914; Precision at 100 Score: 0.8280\n",
      "Task 9914; Pearson Spearman Correlation: 0.4345 (+/- 0.1501)\n",
      "Task 9946; Precision at 100 Score: 0.8260\n",
      "Task 9946; Pearson Spearman Correlation: 0.7875 (+/- 0.0405)\n",
      "Task 9950; Precision at 100 Score: 0.8680\n",
      "Task 9950; Pearson Spearman Correlation: 0.4455 (+/- 0.0447)\n",
      "Task 9952; Precision at 100 Score: 0.9000\n",
      "Task 9952; Pearson Spearman Correlation: 0.6499 (+/- 0.0693)\n",
      "Task 9954; Precision at 100 Score: 0.8300\n",
      "Task 9954; Pearson Spearman Correlation: 0.6794 (+/- 0.0473)\n",
      "Task 9955; Precision at 100 Score: 0.8780\n",
      "Task 9955; Pearson Spearman Correlation: 0.4922 (+/- 0.1036)\n",
      "Task 9956; Precision at 100 Score: 0.8760\n",
      "Task 9956; Pearson Spearman Correlation: 0.7050 (+/- 0.0476)\n",
      "Task 9957; Precision at 100 Score: 0.8220\n",
      "Task 9957; Pearson Spearman Correlation: 0.6027 (+/- 0.1326)\n",
      "Task 9960; Precision at 100 Score: 0.8260\n",
      "Task 9960; Pearson Spearman Correlation: 0.3885 (+/- 0.0771)\n",
      "Task 9964; Precision at 100 Score: 0.8380\n",
      "Task 9964; Pearson Spearman Correlation: 0.5635 (+/- 0.0496)\n",
      "Task 9967; Precision at 100 Score: 0.8660\n",
      "Task 9967; Pearson Spearman Correlation: 0.6882 (+/- 0.0928)\n",
      "Task 9968; Precision at 100 Score: 0.7500\n",
      "Task 9968; Pearson Spearman Correlation: 0.2279 (+/- 0.0443)\n",
      "Task 9970; Precision at 100 Score: 0.8580\n",
      "Task 9970; Pearson Spearman Correlation: 0.6431 (+/- 0.1465)\n",
      "Task 9971; Precision at 100 Score: 0.8180\n",
      "Task 9971; Pearson Spearman Correlation: 0.3819 (+/- 0.1956)\n",
      "Task 9976; Precision at 100 Score: 0.7660\n",
      "Task 9976; Pearson Spearman Correlation: 0.7833 (+/- 0.0756)\n",
      "Task 9977; Precision at 100 Score: 0.8120\n",
      "Task 9977; Pearson Spearman Correlation: 0.9217 (+/- 0.0273)\n",
      "Task 9978; Precision at 100 Score: 0.7280\n",
      "Task 9978; Pearson Spearman Correlation: 0.3515 (+/- 0.1813)\n",
      "Task 9979; Precision at 100 Score: 0.8460\n",
      "Task 9979; Pearson Spearman Correlation: 0.6755 (+/- 0.1837)\n",
      "Task 9980; Precision at 100 Score: 0.8020\n",
      "Task 9980; Pearson Spearman Correlation: 0.5673 (+/- 0.0424)\n",
      "Task 9981; Precision at 100 Score: 0.8720\n",
      "Task 9981; Pearson Spearman Correlation: 0.6603 (+/- 0.0307)\n",
      "Task 9983; Precision at 100 Score: 0.9040\n",
      "Task 9983; Pearson Spearman Correlation: 0.7296 (+/- 0.0196)\n",
      "Task 9985; Precision at 100 Score: 0.7560\n",
      "Task 9985; Pearson Spearman Correlation: 0.4339 (+/- 0.1074)\n",
      "Task 9986; Precision at 100 Score: 0.8740\n",
      "Task 9986; Pearson Spearman Correlation: 0.5100 (+/- 0.1169)\n",
      "Task 10093; Precision at 100 Score: 0.8520\n",
      "Task 10093; Pearson Spearman Correlation: 0.3034 (+/- 0.0780)\n",
      "Task 10101; Precision at 100 Score: 0.7260\n",
      "Task 10101; Pearson Spearman Correlation: 0.6625 (+/- 0.1153)\n",
      "Task 14964; Precision at 100 Score: 0.8660\n",
      "Task 14964; Pearson Spearman Correlation: 0.6230 (+/- 0.0947)\n",
      "Task 14965; Precision at 100 Score: 0.6460\n",
      "Task 14965; Pearson Spearman Correlation: 0.5520 (+/- 0.0463)\n",
      "Task 14966; Precision at 100 Score: 0.8120\n",
      "Task 14966; Pearson Spearman Correlation: 0.5609 (+/- 0.0535)\n",
      "Task 14967; Precision at 100 Score: 0.8140\n",
      "Task 14967; Pearson Spearman Correlation: 0.3347 (+/- 0.0809)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Task 14968; Precision at 100 Score: 0.8320\n",
      "Task 14968; Pearson Spearman Correlation: 0.5714 (+/- 0.0244)\n",
      "Task 14969; Precision at 100 Score: 0.7420\n",
      "Task 14969; Pearson Spearman Correlation: 0.2913 (+/- 0.0394)\n",
      "Task 14970; Precision at 100 Score: 0.9120\n",
      "Task 14970; Pearson Spearman Correlation: 0.6471 (+/- 0.0748)\n",
      "Task 34536; Precision at 100 Score: 0.8140\n",
      "Task 34536; Pearson Spearman Correlation: 0.4070 (+/- 0.0957)\n",
      "Task 34537; Precision at 100 Score: 0.7860\n",
      "Task 34537; Pearson Spearman Correlation: 0.3721 (+/- 0.0706)\n",
      "Task 34538; Precision at 100 Score: 0.8680\n",
      "Task 34538; Pearson Spearman Correlation: 0.7680 (+/- 0.0786)\n",
      "Task 34539; Precision at 100 Score: 0.6460\n",
      "Task 34539; Pearson Spearman Correlation: 0.2004 (+/- 0.1390)\n",
      "Task 125920; Precision at 100 Score: 0.7340\n",
      "Task 125920; Pearson Spearman Correlation: 0.4132 (+/- 0.0616)\n",
      "Task 125921; Precision at 100 Score: 0.6680\n",
      "Task 125921; Pearson Spearman Correlation: 0.2866 (+/- 0.0859)\n",
      "Task 125922; Precision at 100 Score: 0.8840\n",
      "Task 125922; Pearson Spearman Correlation: 0.6021 (+/- 0.2363)\n",
      "Task 125923; Precision at 100 Score: 0.7820\n",
      "Task 125923; Pearson Spearman Correlation: 0.7229 (+/- 0.1303)\n",
      "Task 146195; Precision at 100 Score: 0.9400\n",
      "Task 146195; Pearson Spearman Correlation: 0.2839 (+/- 0.0772)\n",
      "Task 146606; Precision at 100 Score: 0.9800\n",
      "Task 146606; Pearson Spearman Correlation: 0.7212 (+/- 0.0831)\n",
      "Task 146607; Precision at 100 Score: 0.7820\n",
      "Task 146607; Pearson Spearman Correlation: 0.3672 (+/- 0.0419)\n"
     ]
    }
   ],
   "source": [
    "s_ = simplified_surrogate()\n",
    "simplified_surrogate_scores = s_.train_model(data_dict)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Creating new dataset"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [],
   "source": [
    "meta_filename = './data/metafeatures.arff'\n",
    "meta, _ = arff.loadarff(meta_filename)\n",
    "df_meta = pd.DataFrame(meta)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/Users/prernakashyap/anaconda3/envs/gensim_env/lib/python3.6/site-packages/ipykernel_launcher.py:27: FutureWarning: Method .as_matrix will be removed in a future version. Use .values instead.\n"
     ]
    }
   ],
   "source": [
    "model_params = defaultdict(list)\n",
    "task_ids = list(data_dict.keys())\n",
    "task_ids.remove(34536.0)\n",
    "modified_data_dict = defaultdict(list)\n",
    "for i, task_id in enumerate(task_ids):\n",
    "    params = s_.models[task_id].coef_\n",
    "    model_params['a'].append(params[0]) #C\n",
    "    model_params['b'].append(params[1]) #gamma\n",
    "    model_params['c'].append(params[2]) #C^2\n",
    "    model_params['d'].append(params[3]) #C.gamma\n",
    "    model_params['e'].append(params[4]) #gamma^2\n",
    "    model_params['f'].append(s_.models[task_id].intercept_) #intercept\n",
    "    X_task, y_task, _ = data_dict[task_id]\n",
    "    X_temp = np.empty([0,22])\n",
    "    for j in range(len(X_task)):\n",
    "        modified_data_dict['task_id'].append(task_id)\n",
    "        modified_data_dict['C'].append(X_task[j,0])\n",
    "        modified_data_dict['gamma'].append(X_task[j,1])\n",
    "        modified_data_dict['y'].append(y_task[j])\n",
    "        modified_data_dict['a'].append(model_params['a'][-1])\n",
    "        modified_data_dict['b'].append(model_params['b'][-1])\n",
    "        modified_data_dict['c'].append(model_params['c'][-1])\n",
    "        modified_data_dict['d'].append(model_params['d'][-1])\n",
    "        modified_data_dict['e'].append(model_params['e'][-1])\n",
    "        modified_data_dict['f'].append(model_params['f'][-1])\n",
    "        for cidx, column in enumerate(df_meta.columns):\n",
    "            modified_data_dict[column].append(df_meta.iloc[i].as_matrix()[cidx])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [],
   "source": [
    "new_data = pd.DataFrame(modified_data_dict)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {},
   "outputs": [],
   "source": [
    "import arff # pip's not scipy's\n",
    "arff.dump('./data/new_data.arff'\n",
    "      , new_data.values\n",
    "      , relation='c, gamma, metafeatures and predictive accuracies for all tasks'\n",
    "      , names=new_data.columns)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Boxplot for precision at 100 measure"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [],
   "source": [
    "def draw_plot(data, offset,edge_color, fill_color):\n",
    "    pos = np.arange(1) + offset \n",
    "    bp = ax.boxplot(data, positions= pos, widths=0.3, patch_artist=True, manage_xticks=False)\n",
    "    for element in ['boxes', 'whiskers', 'fliers', 'medians', 'caps']:\n",
    "        plt.setp(bp[element], color=edge_color)\n",
    "    for patch in bp['boxes']:\n",
    "        patch.set(facecolor=fill_color)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAX0AAAD8CAYAAACb4nSYAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDIuMi4yLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvhp/UCwAAFAxJREFUeJzt3X+sX/V93/HnC4NhW0tqsJMlGLBTOdUlbgTTnTMpbhMvBRymhbSJNDvKSiYjlLUwKW2lObpRiBxZJVOltmMsKY1RQqTapXQ/rI6BCDHrrCarL4OwGctgWFtujcIl5Me0JgGT9/64x82Xy7Xvufj6fu+9n+dD+srnfM7n3Pv6+lqv7/H5nu89qSokSW04Z9gBJEkLx9KXpIZY+pLUEEtfkhpi6UtSQyx9SWqIpS9JDbH0Jakhlr4kNeTcYQeYbvXq1bVu3bphx5CkJeWRRx55oarWzDZv0ZX+unXrGB8fH3YMSVpSkvxln3me3pGkhsxa+knuSvJ8kv99iu1J8m+THEvyeJJ/MLDthiRPdY8b5jO4JGnu+hzpfxHYeprt7wM2dI+bgM8BJLkIuBV4J7AJuDXJqjMJK0k6M7OWflX9KfDiaaZcD9xdU74O/FSSNwPXAg9W1YtV9W3gQU7/4iFJOsvm45z+JcCzA+sT3dipxl8jyU1JxpOMT05OzkMkSdJM5qP0M8NYnWb8tYNVd1bVaFWNrlkz6xVHkqTXaT5KfwK4dGB9LXD8NOOSpCGZj9LfD/xydxXPPwK+W1XPAQ8A1yRZ1b2Be003Jkkaklk/nJVkL/AeYHWSCaauyDkPoKo+D9wHXAccA/4G+BfdtheTfAY41H2pXVV1ujeEl4cbfa8agC/cP+wEkmaQxXZj9NHR0fITuWfoxq2WrtSYJI9U1ehs8/xEriQ1xNKXpIZY+pLUEEtfkhpi6UtSQyx9SWqIpS9JDbH0Jakhlr4kNWTR3SP3jP3rX4ZvPT/sFMPX+q+DuPiN8Nm7h51CWnSWX+l/63l/BYF80ZNOwdM7ktQQS1+SGmLpS1JDLH1JaoilL0kNsfQlqSHL75JN8HI9STqF5Vn6XqcvX/ilGXl6R5IaYulLUkMsfUlqiKUvSQ2x9CWpIb1KP8nWJEeTHEuyc4btlyd5KMnjSR5OsnZg2ytJHuse++czvCRpbma9ZDPJCuAO4GpgAjiUZH9VPTEw7beAu6vqS0n+MfCbwD/vtn2/qq6c59ySpNehz5H+JuBYVT1TVS8B+4Drp825AnioWz4ww3ZJ0iLQp/QvAZ4dWJ/oxgZ9A/hgt/yLwE8mubhbvyDJeJKvJ/nAGaWVJJ2RPqWfGcZq2vpvAO9O8ijwbuCvgRPdtsuqahT4MPA7SX76Nd8gual7YRifnJzsn16SNCd9Sn8CuHRgfS1wfHBCVR2vql+qqquAsW7suye3dX8+AzwMXDX9G1TVnVU1WlWja9aseT3PQ5LUQ5/SPwRsSLI+yUpgG/Cqq3CSrE5y8mt9ArirG1+V5PyTc4B3AYNvAEuSFtCspV9VJ4CbgQeAI8A9VXU4ya4k7++mvQc4muRJ4E3A7m58BBhP8g2m3uC9bdpVP5KkBdTrt2xW1X3AfdPGPjWwfC9w7wz7/Rnws2eYUZI0T/xEriQ1xNKXpIZY+pLUEEtfkhpi6UtSQyx9SWqIpS9JDbH0Jakhlr4kNcTSl6SGWPqS1BBLX5IaYulLUkN6/ZbNJeXiN8KNW4edQsN28RuHnUBalJZf6X/27mEnGL4bt8IX7h92CkmLkKd3JKkhlr4kNcTSl6SGWPqS1BBLX5IaYulLUkMsfUlqiKUvSQ2x9CWpIZa+JDWkV+kn2ZrkaJJjSXbOsP3yJA8leTzJw0nWDmy7IclT3eOG+QwvSZqbWUs/yQrgDuB9wBXA9iRXTJv2W8DdVfUOYBfwm92+FwG3Au8ENgG3Jlk1f/ElSXPR50h/E3Csqp6pqpeAfcD10+ZcATzULR8Y2H4t8GBVvVhV3wYeBPwVmJI0JH1+y+YlwLMD6xNMHbkP+gbwQeB3gV8EfjLJxafY95Lp3yDJTcBNAJdddlnf7JLmyW2PvjDsCIvCzqtWDzvCWden9DPDWE1b/w3g3yX5KPCnwF8DJ3ruS1XdCdwJMDo6+prtks6uxVB2tz36wqLIsdz1Kf0J4NKB9bXA8cEJVXUc+CWAJD8BfLCqvptkAnjPtH0fPoO8kqQz0Oec/iFgQ5L1SVYC24D9gxOSrE5y8mt9ArirW34AuCbJqu4N3Gu6MUnSEMxa+lV1AriZqbI+AtxTVYeT7Ery/m7ae4CjSZ4E3gTs7vZ9EfgMUy8ch4Bd3ZgkaQh63S6xqu4D7ps29qmB5XuBe0+x7138+MhfkjREfiJXkhpi6UtSQyx9SWqIpS9JDbH0Jakhlr4kNcTSl6SGWPqS1BBLX5IaYulLUkMsfUlqiKUvSQ2x9CWpIZa+JDXE0pekhlj6ktQQS1+SGmLpS1JDLH1JaoilL0kNsfQlqSGWviQ1xNKXpIZY+pLUkF6ln2RrkqNJjiXZOcP2y5IcSPJokseTXNeNr0vy/SSPdY/Pz/cTkCT1d+5sE5KsAO4ArgYmgENJ9lfVEwPTPgncU1WfS3IFcB+wrtv2dFVdOb+xJUmvR58j/U3Asap6pqpeAvYB10+bU8CF3fIbgOPzF1GSNF/6lP4lwLMD6xPd2KBPAx9JMsHUUf4tA9vWd6d9/luSnzuTsJKkM9On9DPDWE1b3w58sarWAtcBX05yDvAccFlVXQX8GvAHSS6cti9JbkoynmR8cnJybs9AktRbn9KfAC4dWF/La0/f7ADuAaiqrwEXAKur6odV9a1u/BHgaeBt079BVd1ZVaNVNbpmzZq5PwtJUi99Sv8QsCHJ+iQrgW3A/mlz/gp4L0CSEaZKfzLJmu6NYJK8FdgAPDNf4SVJczPr1TtVdSLJzcADwArgrqo6nGQXMF5V+4FfB34/yceZOvXz0aqqJD8P7EpyAngF+FhVvXjWno0k6bRmLX2AqrqPqTdoB8c+NbD8BPCuGfb7Y+CPzzCjtOz9+8Mv8r2XfjTsGEN326MvDDvCUF248hx+5e0XndXv0av0NQc3bh12ginDzvGF+4f7/ZeY7730I3ZetXrYMTRkC/GiZ+nPN8tO0iLm796RpIZY+pLUEEtfkhpi6UtSQyx9SWqIpS9JDbH0l5G9e/eyceNGVqxYwcaNG9m7d++wI0laZLxOf5nYu3cvY2Nj7Nmzh82bN3Pw4EF27NgBwPbt24ecTtJi4ZH+MrF792727NnDli1bOO+889iyZQt79uxh9+7dw44maRGx9JeJI0eOsHnz5leNbd68mSNHjgwpkaTFyNJfJkZGRjh48OCrxg4ePMjIyMiQEklajCz9ZWJsbIwdO3Zw4MABXn75ZQ4cOMCOHTsYGxsbdjRJi4hv5C4TJ9+sveWWWzhy5AgjIyPs3r3bN3ElvYqlv4xs377dkpd0Wpa+tEi0fgMRLQxLX1okvImKFuKF3zdyJakhlr4kNcTSl6SGWPqS1BBLX5IaYulLUkMsfUlqSK/ST7I1ydEkx5LsnGH7ZUkOJHk0yeNJrhvY9oluv6NJrp3P8JKkuZn1w1lJVgB3AFcDE8ChJPur6omBaZ8E7qmqzyW5ArgPWNctbwPeDrwF+EqSt1XVK/P9RCRJs+tzpL8JOFZVz1TVS8A+4Pppcwq4sFt+A3C8W74e2FdVP6yq/wMc676eJGkI+pT+JcCzA+sT3digTwMfSTLB1FH+LXPYV5K0QPqUfmYYq2nr24EvVtVa4Drgy0nO6bkvSW5KMp5kfHJyskckSdLr0af0J4BLB9bX8uPTNyftAO4BqKqvARcAq3vuS1XdWVWjVTW6Zs2a/uklSXPSp/QPARuSrE+ykqk3ZvdPm/NXwHsBkowwVfqT3bxtSc5Psh7YAPz5fIWXJM3NrFfvVNWJJDcDDwArgLuq6nCSXcB4Ve0Hfh34/SQfZ+r0zUerqoDDSe4BngBOAL/qlTvSa1248hx/n764cOXZ/+hUprp58RgdHa3x8fFhx5C0wG579AXvKXAGkjxSVaOzzfMTuZLUEEtfkhpi6UtSQyx9SWqIpS9JDbH0Jakhlr4kNcTSl6SGWPqS1BBLX5IaYulLUkMsfUlqiKUvSQ2x9CWpIZa+JDXE0pekhlj6ktQQS1+SGmLpS1JDLH1JaoilL0kNsfQlqSGWviQ1xNKXpIZY+pLUkF6ln2RrkqNJjiXZOcP2307yWPd4Msl3Bra9MrBt/3yGlyTNzbmzTUiyArgDuBqYAA4l2V9VT5ycU1UfH5h/C3DVwJf4flVdOX+RJUmvV58j/U3Asap6pqpeAvYB159m/nZg73yEkyTNrz6lfwnw7MD6RDf2GkkuB9YDXx0YviDJeJKvJ/nAKfa7qZszPjk52TO6JGmu+pR+ZhirU8zdBtxbVa8MjF1WVaPAh4HfSfLTr/liVXdW1WhVja5Zs6ZHJEnS69Gn9CeASwfW1wLHTzF3G9NO7VTV8e7PZ4CHefX5fknSAupT+oeADUnWJ1nJVLG/5iqcJD8DrAK+NjC2Ksn53fJq4F3AE9P3lSQtjFmv3qmqE0luBh4AVgB3VdXhJLuA8ao6+QKwHdhXVYOnfkaA30vyI6ZeYG4bvOpHkrSwZi19gKq6D7hv2tinpq1/eob9/gz42TPIJ0maR34iV5IaYulLUkMsfUlqiKUvSQ2x9CWpIZa+JDWk1yWbkpa32x59YdgRgOHn2HnV6qF+/4Vg6Utqouw0xdM7ktQQS1+SGmLpS1JDLH1JaoilL0kNsfQlqSGWviQ1xNKXpIZY+pLUEEtfkhpi6UtSQyx9SWqIpS9JDbH0Jakhlr4kNcTSl6SG9Cr9JFuTHE1yLMnOGbb/dpLHuseTSb4zsO2GJE91jxvmM7wkaW5mLf0kK4A7gPcBVwDbk1wxOKeqPl5VV1bVlcDtwH/o9r0IuBV4J7AJuDXJqvl9CpKWsr1797Jx40ZWrFjBxo0b2bt377AjLWt9jvQ3Aceq6pmqegnYB1x/mvnbgZM/tWuBB6vqxar6NvAgsPVMAktaPvbu3cvY2Bi33347P/jBD7j99tsZGxuz+M+iPqV/CfDswPpEN/YaSS4H1gNfneu+ktqze/du9uzZw5YtWzjvvPPYsmULe/bsYffu3cOOtmz1Kf3MMFanmLsNuLeqXpnLvkluSjKeZHxycrJHJEnLwZEjR9i8efOrxjZv3syRI0eGlGj561P6E8ClA+trgeOnmLuNH5/a6b1vVd1ZVaNVNbpmzZoekSQtByMjIxw8ePBVYwcPHmRkZGRIiZa/PqV/CNiQZH2SlUwV+/7pk5L8DLAK+NrA8APANUlWdW/gXtONSRJjY2Ps2LGDAwcO8PLLL3PgwAF27NjB2NjYsKMtW+fONqGqTiS5mamyXgHcVVWHk+wCxqvq5AvAdmBfVdXAvi8m+QxTLxwAu6rqxfl9CpKWqu3btwNwyy23cOTIEUZGRti9e/ffjmv+ZaCjF4XR0dEaHx8fdgxJWlKSPFJVo7PN8xO5ktQQS1+SGmLpS1JDLH1JaoilL0kNWXRX7ySZBP5y2DmmWQ28MOwQc7CU8i6lrLC08i6lrLC08i7GrJdX1ayfbl10pb8YJRnvcynUYrGU8i6lrLC08i6lrLC08i6lrNN5ekeSGmLpS1JDLP1+7hx2gDlaSnmXUlZYWnmXUlZYWnmXUtZX8Zy+JDXEI31JaoilP4MkFyV5sLuZ+4Mz3dc3yeVJHuluBn84yceGkbXL0ifvlUm+1mV9PMk/W6xZu3n3J/lOkj9Z6Izd99+a5GiSY0l2zrD9/CR/2G3/H0nWLXzKv80yW9afT/I/k5xI8qFhZBzIMlvWX0vyRPdv9KHubnxD0yPvx5L8r64HDk6/f/iiVFU+pj2AfwPs7JZ3Ap+dYc5K4Pxu+SeAvwDesojzvg3Y0C2/BXgO+KnFmLXb9l7gnwJ/MoSMK4Cngbd2P+dvAFdMm/MrwOe75W3AHw7pZ98n6zrgHcDdwIeGkXMOWbcAf7db/pfD+nudQ94LB5bfD9w/rLx9Hx7pz+x64Evd8peAD0yfUFUvVdUPu9XzGe7/mvrkfbKqnuqWjwPPA8O4TdmsWQGq6iHg/y5UqGk2Aceq6pmqegnYx1TuQYPP417gvUlmuj3o2TZr1qr6i6p6HPjREPIN6pP1QFX9Tbf6dabutjcsffJ+b2D173HqW8kuGpb+zN5UVc8BdH++caZJSS5N8jhTN3//bFemw9Ar70lJNjF15PL0AmSbbk5Zh+QSpn6mJ010YzPOqaoTwHeBixck3SlydGbKuljMNesO4L+e1USn1ytvkl9N8jRT/4v9VwuU7XWb9c5Zy1WSrwB/f4ZNve/TVlXPAu9I8hbgPyW5t6q+OV8ZB81H3u7rvBn4MnBDVZ2VI7/5yjpEMx2xTz+C6zNnISyWHH30zprkI8Ao8O6zmuj0euWtqjuAO5J8GPgkcMPZDnYmmi39qvqFU21L8s0kb66q57qSfH6Wr3U8yWHg55j6r/68m4+8SS4E/gvwyar6+tnICfP7dzskE8ClA+trgen/izs5ZyLJucAbgGHcCrRP1sWiV9Ykv8DUAcK7B06hDsNc/273AZ87q4nmgad3ZrafH79a3wD85+kTkqxN8ne65VXAu4CjC5bw1frkXQn8R+DuqvqjBcw23axZF4FDwIYk67u/t21M5R40+Dw+BHy1unfzFlifrIvFrFmTXAX8HvD+qhr2AUGfvBsGVv8J8NQC5nt9hv1O8mJ8MHVu9iGmfoAPARd146PAF7rlq4HHmXpH/3HgpkWe9yPAy8BjA48rF2PWbv2/A5PA95k64rp2gXNeBzzJ1PseY93YLqbKCOAC4I+AY8CfA28d4s9/tqz/sPs7/H/At4DDizjrV4BvDvwb3T+srD3z/i5wuMt6AHj7MPP2efiJXElqiKd3JKkhlr4kNcTSl6SGWPqS1BBLX5IaYulLUkMsfUlqiKUvSQ35/0a1A71WkqwwAAAAAElFTkSuQmCC\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "fig, ax = plt.subplots()\n",
    "draw_plot(surrogate_scores[:,0], -0.2, 'tomato', 'white')\n",
    "draw_plot(simplified_surrogate_scores[:,0], +0.2, 'skyblue', 'white')\n",
    "plt.show()\n",
    "plt.close()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Boxplot for pearson correlation measure"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAXcAAAD8CAYAAACMwORRAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDIuMi4yLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvhp/UCwAAEz5JREFUeJzt3X+s3Xd93/HnCzcmW0taiG868I/Ynczk4CG83aWb8AYuRBiqOd2KNntig83UouAwjW6qI6MMebIa6LSqQ247D6OSSrUJSGvvmIc1UlerJ9L5Zg4pjmW4ZHS+MyI34dc0Rm3De3+ck3Byc+zzvfa599zz9fMhHeX746Pvefn65uXv/Z77/X5SVUiS2uUlow4gSRo+y12SWshyl6QWstwlqYUsd0lqIctdklrIcpekFrLcJamFLHdJaqEfGdUbr1q1qtavXz+qt5eksfTYY489U1UTg8aNrNzXr1/P9PT0qN5eksZSkj9tMs7LMpLUQpa7JLWQ5S5JLWS5S1ILDSz3JB9P8nSSL15lf5L82yQzSZ5I8leGH1OStBBNztx/G9h+jf1vBTZ2X3uA37zxWJKkGzGw3KvqvwLfuMaQe4GHquNR4CeSvHJYASVJCzeMa+6rgQs967PdbZKkERnGTUzps63vxKxJ9tC5dMO6deuG8NYj9O5rXam6iXzss6NOIKmPYZT7LLC2Z30NcLHfwKo6DBwGmJycHO+ZuZdDqb17+/LIIWnZGUa5TwF7kxwDfhr4dlV9bQjHvbpf/kfw7NOL+hZj42b/CeL2O+DDD406hbTsDCz3JEeBNwKrkswC/xK4BaCqfgs4DrwNmAG+C/zjxQr7PItdz/F7QeprYLlX1a4B+wt439ASNeXlCIE/uUhXMbKnQt6Q2+/wf2p13H7HqBNIy9J4lrvXWDv8QFXSVfhsGUlqIctdklpoPC/LLAfL5Zr/qHN4WUhaliz362WpSVrGvCwjSS1kuUtSC1nuktRClrsktZDlLkktZLlLWnRHjx5l8+bNrFixgs2bN3P06NFRR2o9fxVS0qI6evQo+/fv58iRI2zdupVTp06xe/duAHbtuuZzCXUD0nmo49KbnJys6enpkby3pKWzefNmPvrRj7Jt27bnt508eZL77ruPL37xiyNMNp6SPFZVkwPHWe6SFtOKFSv43ve+xy233PL8tsuXL3Prrbfy/e9/f4TJxlPTcveau6RFtWnTJk6dOvWCbadOnWLTpk0jSnRzaFTuSbYnOZ9kJsm+PvvvTPJIkieS/GGSNcOPKmkc7d+/n927d3Py5EkuX77MyZMn2b17N/v37x91tFZrMs3eCuAQcA+dybBPJ5mqqid7hv1r4KGq+kSSnwF+BfiHixFY0nh57kPT++67j3PnzrFp0yYOHjzoh6mLbOA19yR/A/hQVb2lu34/QFX9Ss+Ys8Bbqmo2SehMkn3btY7rNXdJWrhhXnNfDVzoWZ/tbuv1BeDnu8t/B3hZktubBJUkDV+Tck+fbfNP9/858IYkZ4A3AP8buPKiAyV7kkwnmZ6bm1twWElSM03KfRZY27O+BrjYO6CqLlbV362qLcD+7rZvzz9QVR2uqsmqmpyYmLiB2JKka2lS7qeBjUk2JFkJ7ASmegckWZXkuWPdD3x8uDElSQsxsNyr6gqwFzgBnAMerqqzSQ4k2dEd9kbgfJIvAT8JHFykvJKkBrxDVZLGiHeoStJNzHKXpBay3CWphSx3SWohy12SWshyl6QWstwlqYWcQ1W6iTx45plRR1gW9m1ZNeoIi85yH0NHjx7l4MGDzz8be//+/T4bW42MutQePPPMyDPcLCz3MeNM8pKa8Jr7mDl48CBHjhxh27Zt3HLLLWzbto0jR45w8KCP85H0Q5b7mDl37hxbt259wbatW7dy7ty5ESWStBxZ7mPGmeQlNWG5jxlnkpfUhB+ojhlnkpfUhOU+hnbt2mWZS7omL8tIUgs1Kvck25OcTzKTZF+f/euSnExyJskTSd42/KiSpKYGlnuSFcAh4K3AXcCuJHfNG/ZBOnOrbqEzgfZvDDuoJKm5JmfudwMzVfVUVV0CjgH3zhtTwG3d5R8HLg4voiRpoZqU+2rgQs/6bHdbrw8B70gyCxwH7ut3oCR7kkwnmZ6bm7uOuJKkJpqUe/psq3nru4Dfrqo1wNuA30nyomNX1eGqmqyqyYmJiYWnlSQ10qTcZ4G1PetrePFll93AwwBV9XngVsBHv0nSiDQp99PAxiQbkqyk84Hp1Lwx/wt4E0CSTXTK3esukjQiA8u9qq4Ae4ETwDk6vxVzNsmBJDu6w34J+IUkXwCOAu+qqvmXbiRJS6TRHapVdZzOB6W92x7oWX4SeP1wo0mSrpd3qEpSC1nuktRClrsktZDlLkktZLlLUgtZ7pLUQpa7JLWQ5S5JLWS5S1ILWe6S1EKWuyS1kOUuSS1kuUtSC1nuktRClrsktZDlLkkt1Kjck2xPcj7JTJJ9ffb/WpLHu68vJfnW8KNKkpoaOBNTkhXAIeAeOpNln04y1Z19CYCq+mc94+8DtixCVklSQ03O3O8GZqrqqaq6BBwD7r3G+F105lGVJI1Ik3JfDVzoWZ/tbnuRJHcCG4A/uMr+PUmmk0zPzc0tNKskqaEm5Z4+2+oqY3cCn66q7/fbWVWHq2qyqiYnJiaaZpQkLVCTcp8F1vasrwEuXmXsTrwkI0kj16TcTwMbk2xIspJOgU/NH5TkLwEvBz4/3IiSpIUaWO5VdQXYC5wAzgEPV9XZJAeS7OgZugs4VlVXu2QjSVoiA38VEqCqjgPH5217YN76h4YXS5J0I7xDVZJayHKXpBay3CWphSx3SWohy12SWshyl6QWstwlqYUsd0lqIctdklrIcpekFrLcJamFGj1bRtKN+42z3+A7l34w6hgj9+CZZ0YdYeRuW/kS3vuaVyzqe1ju0hL5zqUfsG/LqlHH0DKwFP/AeVlGklrIcpekFrLcJamFGpV7ku1JzieZSbLvKmP+XpInk5xN8rvDjSlJWoiBH6gmWQEcAu6hM1n26SRTVfVkz5iNwP3A66vqm0nuWKzAkqTBmpy53w3MVNVTVXUJOAbcO2/MLwCHquqbAFX19HBjSpIWokm5rwYu9KzPdrf1ejXw6iT/LcmjSbYPK6AkaeGa/J57+myrPsfZCLwRWAP8UZLNVfWtFxwo2QPsAVi3bt2Cw0qSmmly5j4LrO1ZXwNc7DPm96vqclX9T+A8nbJ/gao6XFWTVTU5MTFxvZklSQM0KffTwMYkG5KsBHYCU/PG/B6wDSDJKjqXaZ4aZlBJUnMDy72qrgB7gRPAOeDhqjqb5ECSHd1hJ4BnkzwJnAT+RVU9u1ihJUnX1ujZMlV1HDg+b9sDPcsFfKD7kiSNmHeoSlILWe6S1EKWuyS1kOUuSS1kuUtSC1nuktRClrsktZBzqEpLyMmhtVQsd2kJOUG2wAmyJUnXyXKXpBay3CWphSx3SWohy12SWshyl6QWstwlqYUalXuS7UnOJ5lJsq/P/nclmUvyePf17uFHlSQ1NfAmpiQrgEPAPXQmwj6dZKqqnpw39JNVtXcRMkqSFqjJmfvdwExVPVVVl4BjwL2LG0uSdCOalPtq4ELP+mx323w/n+SJJJ9OsnYo6SRJ16VJuafPtpq3/h+B9VX1WuBzwCf6HijZk2Q6yfTc3NzCkkqSGmtS7rNA75n4GuBi74Cqeraq/qy7+u+Bv9rvQFV1uKomq2pyYmLievJKkhpoUu6ngY1JNiRZCewEpnoHJHllz+oO4NzwIkqSFmrgb8tU1ZUke4ETwArg41V1NskBYLqqpoD3J9kBXAG+AbxrETNLkgZo9Dz3qjoOHJ+37YGe5fuB+4cbTZJ0vbxDVZJayHKXpBay3CWphSx3SWohy12SWshyl6QWstwlqYUsd0lqIctdklrIcpekFmr0+AFJN+62lS/hwTPPjDqGloHbVi7+ebXlLi2R977mFaOOMHIPnnmGfVtWjTrGTcHLMpLUQpa7JLWQ5S5JLWS5S1ILNSr3JNuTnE8yk2TfNca9PUklmRxeREnSQg0s9yQrgEPAW4G7gF1J7uoz7mXA+4E/HnZISdLCNDlzvxuYqaqnquoScAy4t8+4fwV8BPjeEPNJkq5Dk3JfDVzoWZ/tbnteki3A2qr6zBCzSZKuU5NyT59t9fzO5CXArwG/NPBAyZ4k00mm5+bmmqeUJC1Ik3KfBdb2rK8BLvasvwzYDPxhkq8Cfx2Y6vehalUdrqrJqpqcmJi4/tSSpGtqUu6ngY1JNiRZCewEpp7bWVXfrqpVVbW+qtYDjwI7qmp6URJLkgYaWO5VdQXYC5wAzgEPV9XZJAeS7FjsgJKkhWv04LCqOg4cn7ftgauMfeONx5Ik3QjvUJWkFrLcJamFLHdJaiHLXZJayHKXpBay3CWphSx3SWohy12SWshyl6QWstwlqYUsd0lqIctdklrIcpekFrLcJamFLHdJaiHLXZJayHKXpBZqVO5Jtic5n2Qmyb4++9+T5E+SPJ7kVJK7hh9VktTUwHJPsgI4BLwVuAvY1ae8f7eq/nJVvQ74CPBvhp5UktRYkzP3u4GZqnqqqi4Bx4B7ewdU1Xd6Vn8UqOFFlCQtVJMJslcDF3rWZ4Gfnj8oyfuADwArgZ/pd6Ake4A9AOvWrVtoVklSQ03O3NNn24vOzKvqUFX9ReCXgQ/2O1BVHa6qyaqanJiYWFhSSVJjTcp9Fljbs74GuHiN8ceAn7uRUJKkG9Ok3E8DG5NsSLIS2AlM9Q5IsrFn9WeBLw8voiRpoQZec6+qK0n2AieAFcDHq+pskgPAdFVNAXuTvBm4DHwTeOdihpYkXVuTD1SpquPA8XnbHuhZ/qdDziVJugHeoSpJLWS5S1ILWe6S1EKWuyS1kOUuSS1kuUtSC1nuktRClrsktZDlLkktZLlLUgtZ7pLUQpa7JLWQ5S5JLWS5S1ILWe6S1EKWuyS1UKPJOpJsB36dzkxMH6uqB+ft/wDwbuAKMAf8k6r60yFnlXSDHjzzzKgjLIsM+7asGnWERTew3JOsAA4B99CZLPt0kqmqerJn2Blgsqq+m+QXgY8Af38xAku6fjdDqamjyWWZu4GZqnqqqi4Bx4B7ewdU1cmq+m539VFgzXBjSpIWokm5rwYu9KzPdrddzW7gP/fbkWRPkukk03Nzc81TSpIWpEm5p8+26jsweQcwCfxqv/1VdbiqJqtqcmJionlKSdKCNPlAdRZY27O+Brg4f1CSNwP7gTdU1Z8NJ54k6Xo0OXM/DWxMsiHJSmAnMNU7IMkW4N8BO6rq6eHHlCQtxMByr6orwF7gBHAOeLiqziY5kGRHd9ivAj8GfCrJ40mmrnI4SdISaPR77lV1HDg+b9sDPctvHnIuSdIN8A5VSWqhVPX9xZfFf+NkDlhud7GuAkZ/+1xz45TXrItnnPKOU1ZYnnnvrKqBv244snJfjpJMV9XkqHM0NU55zbp4xinvOGWF8cvby8syktRClrsktZDl/kKHRx1ggcYpr1kXzzjlHaesMH55n+c1d0lqIc/cJamFbupyT/KKJP8lyZe7/315nzF3Jnmse+ft2STvWcZZX5fk892cTyQZ2TP1m+Ttjvtskm8l+cwIMm5Pcj7JTJJ9ffa/NMknu/v/OMn6pc44L8+gvH8ryf9IciXJ20eRsSfLoKwfSPJk9/v0kSR3jiJnT55Bed+T5E+6PXAqyV2jyLkgVXXTvuhMKrKvu7wP+HCfMSuBl3aXfwz4KvCqZZr11cDG7vKrgK8BP7Fcv7bdfW8C/jbwmSXOtwL4CvBT3b/jLwB3zRvzXuC3uss7gU+O4mu5gLzrgdcCDwFvX+ZZtwF/vrv8i2Pwtb2tZ3kH8NlR5W36uqnP3OlMOvKJ7vIngJ+bP6CqLtUPn3L5Ukb3006TrF+qqi93ly8CTwOjerbywLwAVfUI8H+WKlSPgZPQ8MI/w6eBNyXp9wjspdBk0pyvVtUTwA9GEbDHuE3w0yTvd3pWf5SrPPZ8ObnZy/0nq+prAN3/3tFvUJK1SZ6gM2nJh7vFudQaZX1OkrvpnIV8ZQmy9bOgvCPQZBKa58dU5wF63wZuX5J0L7bQSXNGaWgT/CyRRnmTvC/JV+j8VPr+Jcp23Ro9OGycJfkc8Bf67Nrf9BhVdQF4bZJXAb+X5NNV9fVhZXzOMLJ2j/NK4HeAd1bVop3FDSvviDSZhKbxRDVLYDllGeR6Jvh5w6ImurZGeavqEHAoyT8APgi8c7GD3YjWl3td44mVSb6e5JVV9bVuIV7zWfRVdTHJWeBv0vkxfaiGkTXJbcB/Aj5YVY8OO2OvYX5tR6DJJDTPjZlN8iPAjwPfWJp4L9Jo0pxlYtwm+Fno1/YY8JuLmmgIbvbLMlP88F/fdwK/P39AkjVJ/lx3+eXA64HzS5bwh5pkXQn8B+ChqvrUEmbrZ2DeERs4CQ0v/DO8HfiD6n6iNgJN8i4X4zbBT5O8G3tWfxb48hLmuz6j/kR3lC86108fofMX9Qjwiu72SeBj3eV7gCfofIL+BLBnGWd9B3AZeLzn9brlmre7/kfAHPD/6JxBvWUJM74N+BKdzyX2d7cdoFM4ALcCnwJmgP8O/NSIv18H5f1r3a/h/wWeBc4u46yfA77e8306tcy/tr8OnO1mPQm8ZpR5m7y8Q1WSWuhmvywjSa1kuUtSC1nuktRClrsktZDlLkktZLlLUgtZ7pLUQpa7JLXQ/wfi19kGQJqHxQAAAABJRU5ErkJggg==\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "fig, ax = plt.subplots()\n",
    "draw_plot(surrogate_scores[:,1], -0.2, 'tomato', 'white')\n",
    "draw_plot(simplified_surrogate_scores[:,1], +0.2, 'skyblue', 'white')\n",
    "plt.show()\n",
    "plt.close()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.6"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}