Test_MINE_Upper_bound.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "from pytorchtools import EarlyStopping\n",
    "import torch\n",
    "import torch.nn as nn\n",
    "import torch.nn.functional as F\n",
    "import torch.optim as optim\n",
    "import torch.autograd as autograd\n",
    "\n",
    "class Mine(nn.Module):\n",
    "    def __init__(self, input_size=2, hidden_size=100):\n",
    "        super().__init__()\n",
    "        self.fc1 = nn.Linear(input_size, hidden_size)\n",
    "        self.fc2 = nn.Linear(hidden_size, hidden_size)\n",
    "        self.fc3 = nn.Linear(hidden_size, 1)\n",
    "        nn.init.normal_(self.fc1.weight,std=0.02)\n",
    "        nn.init.constant_(self.fc1.bias, 0)\n",
    "        nn.init.normal_(self.fc2.weight,std=0.02)\n",
    "        nn.init.constant_(self.fc2.bias, 0)\n",
    "        nn.init.normal_(self.fc3.weight,std=0.02)\n",
    "        nn.init.constant_(self.fc3.bias, 0)\n",
    "        \n",
    "    def forward(self, input):\n",
    "        output = F.elu(self.fc1(input))\n",
    "        output = F.elu(self.fc2(output))\n",
    "        output = self.fc3(output)\n",
    "        return output\n",
    "\n",
    "def mutual_information(joint, marginal, mine_net):\n",
    "    t = mine_net(joint)\n",
    "    et = torch.exp(mine_net(marginal))\n",
    "    mi_lb = torch.mean(t) - torch.log(torch.mean(et))\n",
    "    return mi_lb, t, et\n",
    "\n",
    "def learn_mine(batch, mine_net, mine_net_optim,  ma_et, ma_rate=0.01):\n",
    "    # batch is a tuple of (joint, marginal)\n",
    "    joint , marginal = batch\n",
    "    joint = torch.autograd.Variable(torch.FloatTensor(joint))\n",
    "    marginal = torch.autograd.Variable(torch.FloatTensor(marginal))\n",
    "    mi_lb , t, et = mutual_information(joint, marginal, mine_net)\n",
    "    ma_et = (1-ma_rate)*ma_et + ma_rate*torch.mean(et)\n",
    "    \n",
    "    # unbiasing use moving average\n",
    "    loss = -(torch.mean(t) - (1/ma_et.mean()).detach()*torch.mean(et))\n",
    "    # use biased estimator\n",
    "#     loss = - mi_lb\n",
    "    \n",
    "    mine_net_optim.zero_grad()\n",
    "    autograd.backward(loss)\n",
    "    mine_net_optim.step()\n",
    "    return mi_lb, ma_et\n",
    "\n",
    "def valid_mine(batch, mine_net):\n",
    "    joint , marginal = batch\n",
    "    joint = torch.autograd.Variable(torch.FloatTensor(joint))\n",
    "    marginal = torch.autograd.Variable(torch.FloatTensor(marginal))\n",
    "    mi_lb , t, et = mutual_information(joint, marginal, mine_net)\n",
    "    return mi_lb\n",
    "    \n",
    "\n",
    "def create_dataset(data, batch_size=100):\n",
    "    if data.shape[0] >= batch_size * 2:\n",
    "        partSize = int(data.shape[0]/2)\n",
    "        indices = list(range(data.shape[0]))\n",
    "        np.random.shuffle(indices)\n",
    "        valid_idx = indices[:partSize]\n",
    "        train_idx = indices[partSize:]\n",
    "        train_data = data[train_idx]\n",
    "        valid_data = data[valid_idx]\n",
    "        return train_data, valid_data\n",
    "    \n",
    "def sample_batch(data, resp, cond, batch_size=100, sample_mode='joint'):\n",
    "#     if sample_mode == 'joint':\n",
    "#         index = np.random.choice(range(data.shape[0]), size=batch_size, replace=False)\n",
    "#         batch = data[index]\n",
    "#     else:\n",
    "#         joint_index = np.random.choice(range(data.shape[0]), size=batch_size, replace=False)\n",
    "#         marginal_index = np.random.choice(range(data.shape[0]), size=batch_size, replace=False)\n",
    "#         batch = np.concatenate([data[joint_index][:,0].reshape(-1,1),\n",
    "#                                          data[marginal_index][:,1].reshape(-1,1)],\n",
    "#                                        axis=1)\n",
    "    index = np.random.choice(range(data.shape[0]), size=batch_size, replace=False)\n",
    "    batch_joint = data[index]\n",
    "    marginal_index = np.random.choice(range(batch_joint.shape[0]), size=batch_size, replace=False)\n",
    "    \n",
    "#     print (batch_joint[:,0].reshape(-1,1).shape)\n",
    "#     print (batch_joint[marginal_index][:,[1,2]].reshape(-1,2).shape)\n",
    "    if data.shape[1] == 2:\n",
    "        batch_mar = np.concatenate([batch_joint[:,0].reshape(-1,1),\n",
    "                                     batch_joint[marginal_index][:,1].reshape(-1,1)],\n",
    "                                   axis=1)\n",
    "    else:\n",
    "        batch_mar = np.concatenate([batch_joint[:,resp].reshape(-1,1),\n",
    "                                     batch_joint[marginal_index][:,cond].reshape(-1,data.shape[1]-1)],\n",
    "                                   axis=1)\n",
    "    return batch_joint, batch_mar\n",
    "\n",
    "def train(data, mine_net,mine_net_optim, resp=0, cond=1, batch_size=100\\\n",
    "          , iter_num=int(5e+4), log_freq=int(1e+3)\\\n",
    "          , avg_freq=int(1e+2), verbose=True, patience=20):\n",
    "    # data is x or y\n",
    "    result = list()\n",
    "    ma_et = 1.\n",
    "    \n",
    "    #Early Stopping\n",
    "    train_losses = []\n",
    "    valid_losses = []\n",
    "    avg_train_losses = []\n",
    "    avg_valid_losses = []\n",
    "    \n",
    "    earlyStop = EarlyStopping(patience=patience, verbose=True)\n",
    "    trainData, validData = create_dataset(data, batch_size)\n",
    "    for i in range(iter_num):\n",
    "        #get train data\n",
    "        batchTrain = sample_batch(trainData,resp, cond, batch_size=batch_size)\n",
    "        mi_lb, ma_et = learn_mine(batchTrain, mine_net, mine_net_optim, ma_et)\n",
    "        result.append(mi_lb.detach().cpu().numpy())\n",
    "        train_losses.append(result[-1].item())\n",
    "        if verbose and (i+1)%(log_freq)==0:\n",
    "            print(result[-1])\n",
    "        \n",
    "        batchValid = sample_batch(validData,resp, cond, batch_size=batch_size)\n",
    "        mi_lb_valid = valid_mine(batchValid, mine_net)\n",
    "        valid_losses.append(mi_lb_valid.item())\n",
    "        \n",
    "        if (i+1)%(avg_freq)==0:\n",
    "            train_loss = np.average(train_losses)\n",
    "            valid_loss = np.average(valid_losses)\n",
    "            avg_train_losses.append(train_loss)\n",
    "            avg_valid_losses.append(valid_loss)\n",
    "\n",
    "            iter_len = len(str(iter_num))\n",
    "            print_msg = (f'[{i:>{iter_len}}/{iter_num:>{iter_len}}] ' +\n",
    "                         f'train_loss: {train_loss:.5f} ' +\n",
    "                         f'valid_loss: {valid_loss:.5f}')\n",
    "            print (print_msg)\n",
    "\n",
    "            train_losses = []\n",
    "            valid_losses = []\n",
    "\n",
    "            earlyStop(valid_loss, mine_net)\n",
    "            if (earlyStop.early_stop):\n",
    "                print(\"Early stopping\")\n",
    "                break\n",
    "            \n",
    "    mine_net.load_state_dict(torch.load('checkpoint.pt'))\n",
    "    return mine_net, avg_train_losses, avg_valid_losses\n",
    "\n",
    "def ma(a, window_size=100):\n",
    "    return [np.mean(a[i:i+window_size]) for i in range(0,len(a)-window_size)]\n",
    "\n",
    "def visualizeAndSave(train_loss, valid_loss, figName=''):\n",
    "    # visualize the loss as the network trained\n",
    "    fig = plt.figure(figsize=(10,8))\n",
    "    plt.plot(range(1,len(train_loss)+1),train_loss, label='Training Loss')\n",
    "    plt.plot(range(1,len(valid_loss)+1),valid_loss,label='Validation Loss')\n",
    "\n",
    "    # find position of lowest validation loss\n",
    "    minposs = valid_loss.index(max(valid_loss))+1 \n",
    "    plt.axvline(minposs, linestyle='--', color='r',label='Early Stopping Checkpoint')\n",
    "\n",
    "    plt.xlabel('epochs')\n",
    "    plt.ylabel('loss')\n",
    "    plt.ylim(0, 0.5) # consistent scale\n",
    "    plt.xlim(0, len(train_loss)+1) # consistent scale\n",
    "    plt.grid(True)\n",
    "    plt.legend()\n",
    "    plt.tight_layout()\n",
    "    plt.show()\n",
    "    if figName != '':\n",
    "        fig.savefig('loss_plot.png', bbox_inches='tight')\n",
    "    else:\n",
    "        fig.savefig(figName, bbox_inches='tight')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "import numpy as np\n",
    "import pandas as pd\n",
    "import matplotlib.pyplot as plt\n",
    "import math\n",
    "from scipy.stats import randint\n",
    "import DiscreteCondEnt as DC\n",
    "\n",
    "from sklearn.linear_model import LinearRegression\n",
    "linReg = LinearRegression()\n",
    "\n",
    "def varEntropy(y):\n",
    "    return np.log(np.var(y)*3.14159*2)/2\n",
    "\n",
    "from sklearn.metrics import mean_squared_error\n",
    "def MSEscorer(clf, X, y):\n",
    "    y_est = clf.predict(X)\n",
    "    return np.log(mean_squared_error(y, y_est)*3.14159*2)/2"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[   99/50000] train_loss: 0.19385 valid_loss: 0.19692\n",
      "Validation loss increased (inf --> 0.196917).  Saving model ...\n",
      "[  199/50000] train_loss: 1.00269 valid_loss: 0.97874\n",
      "Validation loss increased (0.196917 --> 0.978740).  Saving model ...\n",
      "[  299/50000] train_loss: 1.73158 valid_loss: 1.67348\n",
      "Validation loss increased (0.978740 --> 1.673481).  Saving model ...\n",
      "[  399/50000] train_loss: 2.11705 valid_loss: 2.01460\n",
      "Validation loss increased (1.673481 --> 2.014597).  Saving model ...\n",
      "[  499/50000] train_loss: 2.32003 valid_loss: 2.29414\n",
      "Validation loss increased (2.014597 --> 2.294142).  Saving model ...\n",
      "[  599/50000] train_loss: 2.55645 valid_loss: 2.50580\n",
      "Validation loss increased (2.294142 --> 2.505803).  Saving model ...\n",
      "[  699/50000] train_loss: 2.68313 valid_loss: 2.62667\n",
      "Validation loss increased (2.505803 --> 2.626669).  Saving model ...\n",
      "[  799/50000] train_loss: 2.78927 valid_loss: 2.74499\n",
      "Validation loss increased (2.626669 --> 2.744992).  Saving model ...\n",
      "[  899/50000] train_loss: 2.98365 valid_loss: 2.88667\n",
      "Validation loss increased (2.744992 --> 2.886668).  Saving model ...\n",
      "[  999/50000] train_loss: 2.96682 valid_loss: 2.93435\n",
      "Validation loss increased (2.886668 --> 2.934352).  Saving model ...\n",
      "[ 1099/50000] train_loss: 3.02042 valid_loss: 3.04332\n",
      "Validation loss increased (2.934352 --> 3.043316).  Saving model ...\n",
      "[ 1199/50000] train_loss: 3.17129 valid_loss: 3.09364\n",
      "Validation loss increased (3.043316 --> 3.093645).  Saving model ...\n",
      "[ 1299/50000] train_loss: 3.23382 valid_loss: 3.03108\n",
      "EarlyStopping counter: 1 out of 20\n",
      "[ 1399/50000] train_loss: 3.23005 valid_loss: 3.20499\n",
      "Validation loss increased (3.093645 --> 3.204989).  Saving model ...\n",
      "[ 1499/50000] train_loss: 3.34581 valid_loss: 3.33541\n",
      "Validation loss increased (3.204989 --> 3.335414).  Saving model ...\n",
      "[ 1599/50000] train_loss: 3.42916 valid_loss: 3.27544\n",
      "EarlyStopping counter: 1 out of 20\n",
      "[ 1699/50000] train_loss: 3.38092 valid_loss: 3.37416\n",
      "Validation loss increased (3.335414 --> 3.374156).  Saving model ...\n",
      "[ 1799/50000] train_loss: 3.25918 valid_loss: 3.27011\n",
      "EarlyStopping counter: 1 out of 20\n",
      "[ 1899/50000] train_loss: 3.45002 valid_loss: 3.33816\n",
      "EarlyStopping counter: 2 out of 20\n",
      "[ 1999/50000] train_loss: 3.60538 valid_loss: 3.45091\n",
      "Validation loss increased (3.374156 --> 3.450908).  Saving model ...\n",
      "[ 2099/50000] train_loss: 3.46561 valid_loss: 3.48203\n",
      "Validation loss increased (3.450908 --> 3.482029).  Saving model ...\n",
      "[ 2199/50000] train_loss: 3.47977 valid_loss: 3.54694\n",
      "Validation loss increased (3.482029 --> 3.546936).  Saving model ...\n",
      "[ 2299/50000] train_loss: 3.55294 valid_loss: 3.47100\n",
      "EarlyStopping counter: 1 out of 20\n",
      "[ 2399/50000] train_loss: 3.63509 valid_loss: 3.56430\n",
      "Validation loss increased (3.546936 --> 3.564300).  Saving model ...\n",
      "[ 2499/50000] train_loss: 3.59842 valid_loss: 3.53500\n",
      "EarlyStopping counter: 1 out of 20\n",
      "[ 2599/50000] train_loss: 3.62307 valid_loss: 3.66771\n",
      "Validation loss increased (3.564300 --> 3.667714).  Saving model ...\n",
      "[ 2699/50000] train_loss: 3.68552 valid_loss: 3.76278\n",
      "Validation loss increased (3.667714 --> 3.762778).  Saving model ...\n",
      "[ 2799/50000] train_loss: 3.47087 valid_loss: 3.67719\n",
      "EarlyStopping counter: 1 out of 20\n",
      "[ 2899/50000] train_loss: 3.79206 valid_loss: 3.68221\n",
      "EarlyStopping counter: 2 out of 20\n",
      "[ 2999/50000] train_loss: 3.59493 valid_loss: 3.78709\n",
      "Validation loss increased (3.762778 --> 3.787095).  Saving model ...\n",
      "[ 3099/50000] train_loss: 3.79554 valid_loss: 3.72883\n",
      "EarlyStopping counter: 1 out of 20\n",
      "[ 3199/50000] train_loss: 3.61304 valid_loss: 3.59148\n",
      "EarlyStopping counter: 2 out of 20\n",
      "[ 3299/50000] train_loss: 3.74489 valid_loss: 3.73405\n",
      "EarlyStopping counter: 3 out of 20\n",
      "[ 3399/50000] train_loss: 3.72506 valid_loss: 3.83352\n",
      "Validation loss increased (3.787095 --> 3.833518).  Saving model ...\n",
      "[ 3499/50000] train_loss: 3.79078 valid_loss: 3.91931\n",
      "Validation loss increased (3.833518 --> 3.919311).  Saving model ...\n",
      "[ 3599/50000] train_loss: 3.68652 valid_loss: 3.72134\n",
      "EarlyStopping counter: 1 out of 20\n",
      "[ 3699/50000] train_loss: 3.82896 valid_loss: 3.72565\n",
      "EarlyStopping counter: 2 out of 20\n",
      "[ 3799/50000] train_loss: 3.82040 valid_loss: 3.82042\n",
      "EarlyStopping counter: 3 out of 20\n",
      "[ 3899/50000] train_loss: 3.88108 valid_loss: 3.89719\n",
      "EarlyStopping counter: 4 out of 20\n",
      "[ 3999/50000] train_loss: 3.73897 valid_loss: 3.87097\n",
      "EarlyStopping counter: 5 out of 20\n",
      "[ 4099/50000] train_loss: 3.82527 valid_loss: 3.81708\n",
      "EarlyStopping counter: 6 out of 20\n",
      "[ 4199/50000] train_loss: 4.04387 valid_loss: 3.82534\n",
      "EarlyStopping counter: 7 out of 20\n",
      "[ 4299/50000] train_loss: 3.84281 valid_loss: 3.86539\n",
      "EarlyStopping counter: 8 out of 20\n",
      "[ 4399/50000] train_loss: 4.01020 valid_loss: 3.85428\n",
      "EarlyStopping counter: 9 out of 20\n",
      "[ 4499/50000] train_loss: 3.91024 valid_loss: 3.93619\n",
      "Validation loss increased (3.919311 --> 3.936194).  Saving model ...\n",
      "[ 4599/50000] train_loss: 4.11522 valid_loss: 3.82413\n",
      "EarlyStopping counter: 1 out of 20\n",
      "[ 4699/50000] train_loss: 4.03582 valid_loss: 4.00363\n",
      "Validation loss increased (3.936194 --> 4.003629).  Saving model ...\n",
      "[ 4799/50000] train_loss: 3.89461 valid_loss: 4.07504\n",
      "Validation loss increased (4.003629 --> 4.075036).  Saving model ...\n",
      "[ 4899/50000] train_loss: 4.06508 valid_loss: 4.21569\n",
      "Validation loss increased (4.075036 --> 4.215685).  Saving model ...\n",
      "[ 4999/50000] train_loss: 4.09375 valid_loss: 4.01067\n",
      "EarlyStopping counter: 1 out of 20\n",
      "[ 5099/50000] train_loss: 4.20838 valid_loss: 4.16051\n",
      "EarlyStopping counter: 2 out of 20\n",
      "[ 5199/50000] train_loss: 4.16069 valid_loss: 4.11903\n",
      "EarlyStopping counter: 3 out of 20\n",
      "[ 5299/50000] train_loss: 3.90714 valid_loss: 3.90593\n",
      "EarlyStopping counter: 4 out of 20\n",
      "[ 5399/50000] train_loss: 4.20528 valid_loss: 4.11813\n",
      "EarlyStopping counter: 5 out of 20\n",
      "[ 5499/50000] train_loss: 4.30854 valid_loss: 4.19219\n",
      "EarlyStopping counter: 6 out of 20\n",
      "[ 5599/50000] train_loss: 4.24262 valid_loss: 3.86791\n",
      "EarlyStopping counter: 7 out of 20\n",
      "[ 5699/50000] train_loss: 3.73153 valid_loss: 4.04058\n",
      "EarlyStopping counter: 8 out of 20\n",
      "[ 5799/50000] train_loss: 3.93897 valid_loss: 4.21845\n",
      "Validation loss increased (4.215685 --> 4.218445).  Saving model ...\n",
      "[ 5899/50000] train_loss: 4.21388 valid_loss: 4.18761\n",
      "EarlyStopping counter: 1 out of 20\n",
      "[ 5999/50000] train_loss: 4.31175 valid_loss: 4.25073\n",
      "Validation loss increased (4.218445 --> 4.250725).  Saving model ...\n",
      "[ 6099/50000] train_loss: 4.28054 valid_loss: 4.25241\n",
      "Validation loss increased (4.250725 --> 4.252412).  Saving model ...\n",
      "[ 6199/50000] train_loss: 4.14779 valid_loss: 4.32889\n",
      "Validation loss increased (4.252412 --> 4.328887).  Saving model ...\n",
      "[ 6299/50000] train_loss: 4.27595 valid_loss: 4.16275\n",
      "EarlyStopping counter: 1 out of 20\n",
      "[ 6399/50000] train_loss: 4.18127 valid_loss: 4.10533\n",
      "EarlyStopping counter: 2 out of 20\n",
      "[ 6499/50000] train_loss: 4.34402 valid_loss: 3.90784\n",
      "EarlyStopping counter: 3 out of 20\n",
      "[ 6599/50000] train_loss: 3.91328 valid_loss: 4.16871\n",
      "EarlyStopping counter: 4 out of 20\n",
      "[ 6699/50000] train_loss: 4.26617 valid_loss: 4.15221\n",
      "EarlyStopping counter: 5 out of 20\n",
      "[ 6799/50000] train_loss: 4.26251 valid_loss: 4.28010\n",
      "EarlyStopping counter: 6 out of 20\n",
      "[ 6899/50000] train_loss: 4.35757 valid_loss: 4.24983\n",
      "EarlyStopping counter: 7 out of 20\n",
      "[ 6999/50000] train_loss: 4.38734 valid_loss: 4.24232\n",
      "EarlyStopping counter: 8 out of 20\n",
      "[ 7099/50000] train_loss: 4.17608 valid_loss: 4.36395\n",
      "Validation loss increased (4.328887 --> 4.363949).  Saving model ...\n",
      "[ 7199/50000] train_loss: 4.27927 valid_loss: 4.36843\n",
      "Validation loss increased (4.363949 --> 4.368430).  Saving model ...\n",
      "[ 7299/50000] train_loss: 4.29675 valid_loss: 4.22651\n",
      "EarlyStopping counter: 1 out of 20\n",
      "[ 7399/50000] train_loss: 4.19408 valid_loss: 3.99277\n",
      "EarlyStopping counter: 2 out of 20\n",
      "[ 7499/50000] train_loss: 3.99637 valid_loss: 4.35738\n",
      "EarlyStopping counter: 3 out of 20\n",
      "[ 7599/50000] train_loss: 4.25431 valid_loss: 3.99530\n",
      "EarlyStopping counter: 4 out of 20\n",
      "[ 7699/50000] train_loss: 3.76952 valid_loss: 3.89540\n",
      "EarlyStopping counter: 5 out of 20\n",
      "[ 7799/50000] train_loss: 4.29682 valid_loss: 4.23387\n",
      "EarlyStopping counter: 6 out of 20\n",
      "[ 7899/50000] train_loss: 4.44151 valid_loss: 4.22231\n",
      "EarlyStopping counter: 7 out of 20\n",
      "[ 7999/50000] train_loss: 4.25506 valid_loss: 4.28451\n",
      "EarlyStopping counter: 8 out of 20\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[ 8099/50000] train_loss: 4.50906 valid_loss: 4.20589\n",
      "EarlyStopping counter: 9 out of 20\n",
      "[ 8199/50000] train_loss: 4.21210 valid_loss: 4.31026\n",
      "EarlyStopping counter: 10 out of 20\n",
      "[ 8299/50000] train_loss: 4.29520 valid_loss: 4.24936\n",
      "EarlyStopping counter: 11 out of 20\n",
      "[ 8399/50000] train_loss: 4.22947 valid_loss: 4.31413\n",
      "EarlyStopping counter: 12 out of 20\n",
      "[ 8499/50000] train_loss: 4.00646 valid_loss: 4.47435\n",
      "Validation loss increased (4.368430 --> 4.474353).  Saving model ...\n",
      "[ 8599/50000] train_loss: 4.86815 valid_loss: 4.19463\n",
      "EarlyStopping counter: 1 out of 20\n",
      "[ 8699/50000] train_loss: 4.20702 valid_loss: 4.47294\n",
      "EarlyStopping counter: 2 out of 20\n",
      "[ 8799/50000] train_loss: 4.46777 valid_loss: 4.24372\n",
      "EarlyStopping counter: 3 out of 20\n",
      "[ 8899/50000] train_loss: 4.31515 valid_loss: 4.47699\n",
      "Validation loss increased (4.474353 --> 4.476994).  Saving model ...\n",
      "[ 8999/50000] train_loss: 4.26842 valid_loss: 4.19093\n",
      "EarlyStopping counter: 1 out of 20\n",
      "[ 9099/50000] train_loss: 4.36579 valid_loss: 4.36330\n",
      "EarlyStopping counter: 2 out of 20\n",
      "[ 9199/50000] train_loss: 4.72110 valid_loss: 4.33583\n",
      "EarlyStopping counter: 3 out of 20\n",
      "[ 9299/50000] train_loss: 4.42493 valid_loss: 4.20261\n",
      "EarlyStopping counter: 4 out of 20\n",
      "[ 9399/50000] train_loss: 4.71959 valid_loss: 4.49659\n",
      "Validation loss increased (4.476994 --> 4.496589).  Saving model ...\n",
      "[ 9499/50000] train_loss: 4.37879 valid_loss: 3.98983\n",
      "EarlyStopping counter: 1 out of 20\n",
      "[ 9599/50000] train_loss: 4.42144 valid_loss: 4.36086\n",
      "EarlyStopping counter: 2 out of 20\n",
      "[ 9699/50000] train_loss: 4.24592 valid_loss: 4.71100\n",
      "Validation loss increased (4.496589 --> 4.710998).  Saving model ...\n",
      "[ 9799/50000] train_loss: 4.55367 valid_loss: 4.64364\n",
      "EarlyStopping counter: 1 out of 20\n",
      "[ 9899/50000] train_loss: 4.42298 valid_loss: 4.60031\n",
      "EarlyStopping counter: 2 out of 20\n",
      "[ 9999/50000] train_loss: 4.69003 valid_loss: 4.54585\n",
      "EarlyStopping counter: 3 out of 20\n",
      "[10099/50000] train_loss: 4.01777 valid_loss: 4.57756\n",
      "EarlyStopping counter: 4 out of 20\n",
      "[10199/50000] train_loss: 4.48117 valid_loss: 4.06575\n",
      "EarlyStopping counter: 5 out of 20\n",
      "[10299/50000] train_loss: 4.83006 valid_loss: 4.81120\n",
      "Validation loss increased (4.710998 --> 4.811200).  Saving model ...\n",
      "[10399/50000] train_loss: 4.45845 valid_loss: 4.40700\n",
      "EarlyStopping counter: 1 out of 20\n",
      "[10499/50000] train_loss: 4.38093 valid_loss: 4.31321\n",
      "EarlyStopping counter: 2 out of 20\n",
      "[10599/50000] train_loss: 4.51706 valid_loss: 4.60432\n",
      "EarlyStopping counter: 3 out of 20\n",
      "[10699/50000] train_loss: 4.59913 valid_loss: 4.51318\n",
      "EarlyStopping counter: 4 out of 20\n",
      "[10799/50000] train_loss: 4.90670 valid_loss: 4.69060\n",
      "EarlyStopping counter: 5 out of 20\n",
      "[10899/50000] train_loss: 4.50451 valid_loss: 4.47193\n",
      "EarlyStopping counter: 6 out of 20\n",
      "[10999/50000] train_loss: 4.34643 valid_loss: 4.93724\n",
      "Validation loss increased (4.811200 --> 4.937241).  Saving model ...\n",
      "[11099/50000] train_loss: 4.27810 valid_loss: 4.41090\n",
      "EarlyStopping counter: 1 out of 20\n",
      "[11199/50000] train_loss: 4.32849 valid_loss: 4.43403\n",
      "EarlyStopping counter: 2 out of 20\n",
      "[11299/50000] train_loss: 4.74798 valid_loss: 4.67304\n",
      "EarlyStopping counter: 3 out of 20\n",
      "[11399/50000] train_loss: 4.33816 valid_loss: 4.57582\n",
      "EarlyStopping counter: 4 out of 20\n",
      "[11499/50000] train_loss: 4.59884 valid_loss: 4.33415\n",
      "EarlyStopping counter: 5 out of 20\n",
      "[11599/50000] train_loss: 4.50598 valid_loss: 4.64006\n",
      "EarlyStopping counter: 6 out of 20\n",
      "[11699/50000] train_loss: 5.00539 valid_loss: 4.65510\n",
      "EarlyStopping counter: 7 out of 20\n",
      "[11799/50000] train_loss: 4.79542 valid_loss: 4.59369\n",
      "EarlyStopping counter: 8 out of 20\n",
      "[11899/50000] train_loss: 5.09253 valid_loss: 4.59477\n",
      "EarlyStopping counter: 9 out of 20\n",
      "[11999/50000] train_loss: 4.57433 valid_loss: 4.53938\n",
      "EarlyStopping counter: 10 out of 20\n",
      "[12099/50000] train_loss: 4.80630 valid_loss: 4.46302\n",
      "EarlyStopping counter: 11 out of 20\n",
      "[12199/50000] train_loss: 4.58254 valid_loss: 4.30281\n",
      "EarlyStopping counter: 12 out of 20\n",
      "[12299/50000] train_loss: 4.54899 valid_loss: 4.67246\n",
      "EarlyStopping counter: 13 out of 20\n",
      "[12399/50000] train_loss: 4.48905 valid_loss: 4.15719\n",
      "EarlyStopping counter: 14 out of 20\n",
      "[12499/50000] train_loss: 5.22451 valid_loss: 4.91270\n",
      "EarlyStopping counter: 15 out of 20\n",
      "[12599/50000] train_loss: 4.51397 valid_loss: 4.46257\n",
      "EarlyStopping counter: 16 out of 20\n",
      "[12699/50000] train_loss: 4.59326 valid_loss: 4.65942\n",
      "EarlyStopping counter: 17 out of 20\n",
      "[12799/50000] train_loss: 4.57843 valid_loss: 4.29671\n",
      "EarlyStopping counter: 18 out of 20\n",
      "[12899/50000] train_loss: 4.60873 valid_loss: 4.76339\n",
      "EarlyStopping counter: 19 out of 20\n",
      "[12999/50000] train_loss: 4.50132 valid_loss: 4.37924\n",
      "EarlyStopping counter: 20 out of 20\n",
      "Early stopping\n",
      "[   99/50000] train_loss: 0.18973 valid_loss: 0.19506\n",
      "Validation loss increased (inf --> 0.195058).  Saving model ...\n",
      "[  199/50000] train_loss: 0.93219 valid_loss: 0.94994\n",
      "Validation loss increased (0.195058 --> 0.949944).  Saving model ...\n",
      "[  299/50000] train_loss: 1.47824 valid_loss: 1.50714\n",
      "Validation loss increased (0.949944 --> 1.507140).  Saving model ...\n",
      "[  399/50000] train_loss: 1.84448 valid_loss: 1.84798\n",
      "Validation loss increased (1.507140 --> 1.847978).  Saving model ...\n",
      "[  499/50000] train_loss: 2.13095 valid_loss: 2.11422\n",
      "Validation loss increased (1.847978 --> 2.114216).  Saving model ...\n",
      "[  599/50000] train_loss: 2.29751 valid_loss: 2.27759\n",
      "Validation loss increased (2.114216 --> 2.277586).  Saving model ...\n",
      "[  699/50000] train_loss: 2.29597 valid_loss: 2.31707\n",
      "Validation loss increased (2.277586 --> 2.317075).  Saving model ...\n",
      "[  799/50000] train_loss: 2.47025 valid_loss: 2.48746\n",
      "Validation loss increased (2.317075 --> 2.487463).  Saving model ...\n",
      "[  899/50000] train_loss: 2.62579 valid_loss: 2.56699\n",
      "Validation loss increased (2.487463 --> 2.566990).  Saving model ...\n",
      "[  999/50000] train_loss: 2.63964 valid_loss: 2.70835\n",
      "Validation loss increased (2.566990 --> 2.708349).  Saving model ...\n",
      "[ 1099/50000] train_loss: 2.75311 valid_loss: 2.70080\n",
      "EarlyStopping counter: 1 out of 20\n",
      "[ 1199/50000] train_loss: 2.81817 valid_loss: 2.80291\n",
      "Validation loss increased (2.708349 --> 2.802906).  Saving model ...\n",
      "[ 1299/50000] train_loss: 2.80230 valid_loss: 2.91699\n",
      "Validation loss increased (2.802906 --> 2.916986).  Saving model ...\n",
      "[ 1399/50000] train_loss: 2.92143 valid_loss: 2.89730\n",
      "EarlyStopping counter: 1 out of 20\n",
      "[ 1499/50000] train_loss: 3.03446 valid_loss: 2.98673\n",
      "Validation loss increased (2.916986 --> 2.986734).  Saving model ...\n",
      "[ 1599/50000] train_loss: 3.08933 valid_loss: 3.00877\n",
      "Validation loss increased (2.986734 --> 3.008765).  Saving model ...\n",
      "[ 1699/50000] train_loss: 3.07093 valid_loss: 3.10306\n",
      "Validation loss increased (3.008765 --> 3.103060).  Saving model ...\n",
      "[ 1799/50000] train_loss: 3.09426 valid_loss: 3.07753\n",
      "EarlyStopping counter: 1 out of 20\n",
      "[ 1899/50000] train_loss: 3.24984 valid_loss: 3.17967\n",
      "Validation loss increased (3.103060 --> 3.179672).  Saving model ...\n",
      "[ 1999/50000] train_loss: 3.19124 valid_loss: 3.22106\n",
      "Validation loss increased (3.179672 --> 3.221056).  Saving model ...\n",
      "[ 2099/50000] train_loss: 3.32237 valid_loss: 3.25474\n",
      "Validation loss increased (3.221056 --> 3.254738).  Saving model ...\n",
      "[ 2199/50000] train_loss: 3.34421 valid_loss: 3.36249\n",
      "Validation loss increased (3.254738 --> 3.362488).  Saving model ...\n",
      "[ 2299/50000] train_loss: 3.30812 valid_loss: 3.23197\n",
      "EarlyStopping counter: 1 out of 20\n",
      "[ 2399/50000] train_loss: 3.50125 valid_loss: 3.40545\n",
      "Validation loss increased (3.362488 --> 3.405447).  Saving model ...\n",
      "[ 2499/50000] train_loss: 3.50054 valid_loss: 3.40515\n",
      "EarlyStopping counter: 1 out of 20\n",
      "[ 2599/50000] train_loss: 3.52001 valid_loss: 3.37878\n",
      "EarlyStopping counter: 2 out of 20\n",
      "[ 2699/50000] train_loss: 3.41518 valid_loss: 3.49444\n",
      "Validation loss increased (3.405447 --> 3.494439).  Saving model ...\n",
      "[ 2799/50000] train_loss: 3.45467 valid_loss: 3.47720\n",
      "EarlyStopping counter: 1 out of 20\n",
      "[ 2899/50000] train_loss: 3.62827 valid_loss: 3.63371\n",
      "Validation loss increased (3.494439 --> 3.633708).  Saving model ...\n",
      "[ 2999/50000] train_loss: 3.51160 valid_loss: 3.52942\n",
      "EarlyStopping counter: 1 out of 20\n",
      "[ 3099/50000] train_loss: 3.51208 valid_loss: 3.52209\n",
      "EarlyStopping counter: 2 out of 20\n",
      "[ 3199/50000] train_loss: 3.61444 valid_loss: 3.63279\n",
      "EarlyStopping counter: 3 out of 20\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[ 3299/50000] train_loss: 3.62236 valid_loss: 3.61045\n",
      "EarlyStopping counter: 4 out of 20\n",
      "[ 3399/50000] train_loss: 3.69638 valid_loss: 3.51046\n",
      "EarlyStopping counter: 5 out of 20\n",
      "[ 3499/50000] train_loss: 3.64554 valid_loss: 3.75700\n",
      "Validation loss increased (3.633708 --> 3.757001).  Saving model ...\n",
      "[ 3599/50000] train_loss: 3.62204 valid_loss: 3.57525\n",
      "EarlyStopping counter: 1 out of 20\n",
      "[ 3699/50000] train_loss: 3.73497 valid_loss: 3.68582\n",
      "EarlyStopping counter: 2 out of 20\n",
      "[ 3799/50000] train_loss: 3.72680 valid_loss: 3.80116\n",
      "Validation loss increased (3.757001 --> 3.801155).  Saving model ...\n",
      "[ 3899/50000] train_loss: 3.74850 valid_loss: 3.82755\n",
      "Validation loss increased (3.801155 --> 3.827552).  Saving model ...\n",
      "[ 3999/50000] train_loss: 3.87034 valid_loss: 3.77509\n",
      "EarlyStopping counter: 1 out of 20\n",
      "[ 4099/50000] train_loss: 3.73354 valid_loss: 3.93459\n",
      "Validation loss increased (3.827552 --> 3.934589).  Saving model ...\n",
      "[ 4199/50000] train_loss: 3.76011 valid_loss: 3.69633\n",
      "EarlyStopping counter: 1 out of 20\n",
      "[ 4299/50000] train_loss: 3.85499 valid_loss: 3.69226\n",
      "EarlyStopping counter: 2 out of 20\n",
      "[ 4399/50000] train_loss: 3.87524 valid_loss: 3.82180\n",
      "EarlyStopping counter: 3 out of 20\n",
      "[ 4499/50000] train_loss: 3.94802 valid_loss: 3.93083\n",
      "EarlyStopping counter: 4 out of 20\n",
      "[ 4599/50000] train_loss: 3.78134 valid_loss: 3.95824\n",
      "Validation loss increased (3.934589 --> 3.958239).  Saving model ...\n",
      "[ 4699/50000] train_loss: 4.05334 valid_loss: 3.91723\n",
      "EarlyStopping counter: 1 out of 20\n",
      "[ 4799/50000] train_loss: 3.87145 valid_loss: 4.07832\n",
      "Validation loss increased (3.958239 --> 4.078318).  Saving model ...\n",
      "[ 4899/50000] train_loss: 3.71433 valid_loss: 3.93597\n",
      "EarlyStopping counter: 1 out of 20\n",
      "[ 4999/50000] train_loss: 3.81364 valid_loss: 3.92410\n",
      "EarlyStopping counter: 2 out of 20\n",
      "[ 5099/50000] train_loss: 4.01533 valid_loss: 4.01337\n",
      "EarlyStopping counter: 3 out of 20\n",
      "[ 5199/50000] train_loss: 4.01640 valid_loss: 3.91882\n",
      "EarlyStopping counter: 4 out of 20\n",
      "[ 5299/50000] train_loss: 4.03268 valid_loss: 4.17332\n",
      "Validation loss increased (4.078318 --> 4.173319).  Saving model ...\n",
      "[ 5399/50000] train_loss: 4.08102 valid_loss: 3.99749\n",
      "EarlyStopping counter: 1 out of 20\n",
      "[ 5499/50000] train_loss: 3.91018 valid_loss: 4.06961\n",
      "EarlyStopping counter: 2 out of 20\n",
      "[ 5599/50000] train_loss: 3.79828 valid_loss: 3.97771\n",
      "EarlyStopping counter: 3 out of 20\n",
      "[ 5699/50000] train_loss: 3.99906 valid_loss: 4.08123\n",
      "EarlyStopping counter: 4 out of 20\n",
      "[ 5799/50000] train_loss: 3.84350 valid_loss: 4.00914\n",
      "EarlyStopping counter: 5 out of 20\n",
      "[ 5899/50000] train_loss: 4.05185 valid_loss: 3.98861\n",
      "EarlyStopping counter: 6 out of 20\n",
      "[ 5999/50000] train_loss: 4.19307 valid_loss: 3.75695\n",
      "EarlyStopping counter: 7 out of 20\n",
      "[ 6099/50000] train_loss: 3.86883 valid_loss: 4.22565\n",
      "Validation loss increased (4.173319 --> 4.225653).  Saving model ...\n",
      "[ 6199/50000] train_loss: 3.96670 valid_loss: 4.11000\n",
      "EarlyStopping counter: 1 out of 20\n",
      "[ 6299/50000] train_loss: 4.01511 valid_loss: 4.02881\n",
      "EarlyStopping counter: 2 out of 20\n",
      "[ 6399/50000] train_loss: 4.20257 valid_loss: 4.18721\n",
      "EarlyStopping counter: 3 out of 20\n",
      "[ 6499/50000] train_loss: 4.06174 valid_loss: 4.11404\n",
      "EarlyStopping counter: 4 out of 20\n",
      "[ 6599/50000] train_loss: 4.01082 valid_loss: 3.99946\n",
      "EarlyStopping counter: 5 out of 20\n",
      "[ 6699/50000] train_loss: 4.17273 valid_loss: 4.32924\n",
      "Validation loss increased (4.225653 --> 4.329243).  Saving model ...\n",
      "[ 6799/50000] train_loss: 4.32662 valid_loss: 4.16168\n",
      "EarlyStopping counter: 1 out of 20\n",
      "[ 6899/50000] train_loss: 3.92126 valid_loss: 4.05132\n",
      "EarlyStopping counter: 2 out of 20\n",
      "[ 6999/50000] train_loss: 4.37056 valid_loss: 4.13395\n",
      "EarlyStopping counter: 3 out of 20\n",
      "[ 7099/50000] train_loss: 4.30435 valid_loss: 4.13048\n",
      "EarlyStopping counter: 4 out of 20\n",
      "[ 7199/50000] train_loss: 4.14668 valid_loss: 4.15913\n",
      "EarlyStopping counter: 5 out of 20\n",
      "[ 7299/50000] train_loss: 4.13783 valid_loss: 4.25626\n",
      "EarlyStopping counter: 6 out of 20\n",
      "[ 7399/50000] train_loss: 4.08017 valid_loss: 3.90491\n",
      "EarlyStopping counter: 7 out of 20\n",
      "[ 7499/50000] train_loss: 4.42506 valid_loss: 4.37242\n",
      "Validation loss increased (4.329243 --> 4.372417).  Saving model ...\n",
      "[ 7599/50000] train_loss: 4.42173 valid_loss: 4.15818\n",
      "EarlyStopping counter: 1 out of 20\n",
      "[ 7699/50000] train_loss: 4.29868 valid_loss: 4.45338\n",
      "Validation loss increased (4.372417 --> 4.453385).  Saving model ...\n",
      "[ 7799/50000] train_loss: 4.25673 valid_loss: 4.13611\n",
      "EarlyStopping counter: 1 out of 20\n",
      "[ 7899/50000] train_loss: 4.43185 valid_loss: 4.43219\n",
      "EarlyStopping counter: 2 out of 20\n",
      "[ 7999/50000] train_loss: 4.20904 valid_loss: 4.31422\n",
      "EarlyStopping counter: 3 out of 20\n",
      "[ 8099/50000] train_loss: 4.42231 valid_loss: 4.12102\n",
      "EarlyStopping counter: 4 out of 20\n",
      "[ 8199/50000] train_loss: 4.65451 valid_loss: 4.34138\n",
      "EarlyStopping counter: 5 out of 20\n",
      "[ 8299/50000] train_loss: 4.23806 valid_loss: 4.30941\n",
      "EarlyStopping counter: 6 out of 20\n",
      "[ 8399/50000] train_loss: 4.43051 valid_loss: 4.18011\n",
      "EarlyStopping counter: 7 out of 20\n",
      "[ 8499/50000] train_loss: 4.33748 valid_loss: 4.24248\n",
      "EarlyStopping counter: 8 out of 20\n",
      "[ 8599/50000] train_loss: 4.90796 valid_loss: 4.14211\n",
      "EarlyStopping counter: 9 out of 20\n",
      "[ 8699/50000] train_loss: 4.01702 valid_loss: 4.18143\n",
      "EarlyStopping counter: 10 out of 20\n",
      "[ 8799/50000] train_loss: 4.54067 valid_loss: 4.32056\n",
      "EarlyStopping counter: 11 out of 20\n",
      "[ 8899/50000] train_loss: 4.54648 valid_loss: 4.44434\n",
      "EarlyStopping counter: 12 out of 20\n",
      "[ 8999/50000] train_loss: 4.49832 valid_loss: 4.24328\n",
      "EarlyStopping counter: 13 out of 20\n",
      "[ 9099/50000] train_loss: 4.71599 valid_loss: 4.35641\n",
      "EarlyStopping counter: 14 out of 20\n",
      "[ 9199/50000] train_loss: 4.62187 valid_loss: 4.35855\n",
      "EarlyStopping counter: 15 out of 20\n",
      "[ 9299/50000] train_loss: 4.59378 valid_loss: 4.19213\n",
      "EarlyStopping counter: 16 out of 20\n",
      "[ 9399/50000] train_loss: 4.20555 valid_loss: 4.37168\n",
      "EarlyStopping counter: 17 out of 20\n",
      "[ 9499/50000] train_loss: 4.60916 valid_loss: 4.38897\n",
      "EarlyStopping counter: 18 out of 20\n",
      "[ 9599/50000] train_loss: 4.51165 valid_loss: 4.30791\n",
      "EarlyStopping counter: 19 out of 20\n",
      "[ 9699/50000] train_loss: 4.70914 valid_loss: 4.35362\n",
      "EarlyStopping counter: 20 out of 20\n",
      "Early stopping\n"
     ]
    },
    {
     "ename": "IndexError",
     "evalue": "list index out of range",
     "output_type": "error",
     "traceback": [
      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[0;31mIndexError\u001b[0m                                Traceback (most recent call last)",
      "\u001b[0;32m<ipython-input-5-7926ba876d0f>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m\u001b[0m\n\u001b[1;32m     24\u001b[0m     \u001b[0mmine_net\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0mtl\u001b[0m \u001b[0;34m,\u001b[0m\u001b[0mvl\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mtrain\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mnp\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mtranspose\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mx\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0mmine_net\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0mmine_net_optim\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mverbose\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;32mFalse\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     25\u001b[0m     \u001b[0mresult_ma\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mma\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mvl\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 26\u001b[0;31m     \u001b[0mMINE2\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mappend\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mresult_ma\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;34m-\u001b[0m\u001b[0;36m1\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m     27\u001b[0m     \u001b[0;31m#MINE\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     28\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;31mIndexError\u001b[0m: list index out of range"
     ]
    }
   ],
   "source": [
    "MINE2 = []\n",
    "LinReg2 = []\n",
    "GT2 = []\n",
    "COV2 = []\n",
    "CVFold = 3\n",
    "for i in range(5, 10):\n",
    "    cov = 1 - 0.1**i\n",
    "    COV2.append(cov)\n",
    "    x = np.transpose(np.random.multivariate_normal( mean=[0,0],\n",
    "                                  cov=[[1,cov],[cov,1]],\n",
    "                                 size = 1000))\n",
    "    DE = DC.computeEnt(x, linReg, MSEscorer, varEntropy, CVFold)\n",
    "    MI = DE[1,0] + DE[0,0] - DE[0,1] - DE[1,1]\n",
    "    MI = MI/2\n",
    "    LinReg2.append(MI)\n",
    "    #plt.scatter(cov, MI, c='g',label='KNN-regressor')\n",
    "    groundTruth = -0.5*np.log(1-cov*cov)\n",
    "    GT2.append(groundTruth)\n",
    "    #plt.scatter(cov, groundTruth, c='r',label='ground truth')\n",
    "    \n",
    "    #MINE\n",
    "    mine_net = Mine()\n",
    "    mine_net_optim = optim.Adam(mine_net.parameters(), lr=1e-3)\n",
    "    mine_net,tl ,vl = train(np.transpose(x),mine_net,mine_net_optim, verbose=False)\n",
    "    result_ma = ma(vl)\n",
    "    MINE2.append(result_ma[-1])\n",
    "    #MINE\n",
    "\n",
    "#plt.legend()\n",
    "#plt.show()"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.0"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}