Mutual Information_Gaussian Distribution.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [],
   "source": [
    "import numpy as np\n",
    "import pandas as pd\n",
    "import matplotlib.pyplot as plt\n",
    "import math\n",
    "from scipy.stats import randint\n",
    "import DiscreteCondEnt as DC\n",
    "\n",
    "from sklearn.metrics import mean_squared_error\n",
    "def MSEscorer(clf, X, y):\n",
    "    y_est = clf.predict(X)\n",
    "    return np.log(mean_squared_error(y, y_est)*3.14159*2)/2\n",
    "\n",
    "def varEntropy(y):\n",
    "    return np.log(np.var(y)*3.14159*2)/2\n",
    "\n",
    "from sklearn.linear_model import LinearRegression\n",
    "linReg = LinearRegression()\n",
    "\n",
    "from sklearn.neighbors import KNeighborsRegressor\n",
    "regressor = KNeighborsRegressor()\n",
    "\n",
    "x = np.random.multivariate_normal( mean=[0,0],\n",
    "                                  cov=[[1,1],[1,1]],\n",
    "                                 size = 300)\n",
    "x = np.transpose(x)\n",
    "\n",
    "CVFold = 3\n",
    "DEntropy = DC.computeEnt(x, linReg, MSEscorer, varEntropy, CVFold)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {},
   "outputs": [],
   "source": [
    "x2 = np.random.multivariate_normal( mean=[0,0],\n",
    "                                  cov=[[1,0],[0,1]],\n",
    "                                 size = 300)\n",
    "x2 = np.transpose(x2)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {},
   "outputs": [],
   "source": [
    "DEntropy2 = DC.computeEnt(x2, linReg, MSEscorer, varEntropy, CVFold)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([[9.20094469e-01, 2.92948188e-32],\n",
       "       [9.20094469e-01, 2.63948274e-32]])"
      ]
     },
     "execution_count": 25,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "DEntropy"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0.9200944688639072 0.9200944688639068\n"
     ]
    }
   ],
   "source": [
    "#MI\n",
    "MI12_1 = DEntropy[0,0] - DEntropy[0,1]\n",
    "MI12_2 = DEntropy[1,0] - DEntropy[1,1]\n",
    "print (MI12_1, MI12_2)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 70,
   "metadata": {},
   "outputs": [],
   "source": [
    "import torch\n",
    "import torch.nn as nn\n",
    "import torch.nn.functional as F\n",
    "import torch.optim as optim\n",
    "import torch.autograd as autograd"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 71,
   "metadata": {},
   "outputs": [],
   "source": [
    "class Mine(nn.Module):\n",
    "    def __init__(self, input_size=2, hidden_size=100):\n",
    "        super().__init__()\n",
    "        self.fc1 = nn.Linear(input_size, hidden_size)\n",
    "        self.fc2 = nn.Linear(hidden_size, hidden_size)\n",
    "        self.fc3 = nn.Linear(hidden_size, 1)\n",
    "        nn.init.normal_(self.fc1.weight,std=0.02)\n",
    "        nn.init.constant_(self.fc1.bias, 0)\n",
    "        nn.init.normal_(self.fc2.weight,std=0.02)\n",
    "        nn.init.constant_(self.fc2.bias, 0)\n",
    "        nn.init.normal_(self.fc3.weight,std=0.02)\n",
    "        nn.init.constant_(self.fc3.bias, 0)\n",
    "        \n",
    "    def forward(self, input):\n",
    "        output = F.elu(self.fc1(input))\n",
    "        output = F.elu(self.fc2(output))\n",
    "        output = self.fc3(output)\n",
    "        return output"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 78,
   "metadata": {},
   "outputs": [],
   "source": [
    "def mutual_information(joint, marginal, mine_net):\n",
    "    t = mine_net(joint)\n",
    "    et = torch.exp(mine_net(marginal))\n",
    "    mi_lb = torch.mean(t) - torch.log(torch.mean(et))\n",
    "    return mi_lb, t, et\n",
    "\n",
    "def learn_mine(batch, mine_net, mine_net_optim,  ma_et, ma_rate=0.01):\n",
    "    # batch is a tuple of (joint, marginal)\n",
    "    joint , marginal = batch\n",
    "    joint = torch.autograd.Variable(torch.FloatTensor(joint))\n",
    "    marginal = torch.autograd.Variable(torch.FloatTensor(marginal))\n",
    "    mi_lb , t, et = mutual_information(joint, marginal, mine_net)\n",
    "    ma_et = (1-ma_rate)*ma_et + ma_rate*torch.mean(et)\n",
    "    \n",
    "    # unbiasing use moving average\n",
    "    loss = -(torch.mean(t) - (1/ma_et.mean()).detach()*torch.mean(et))\n",
    "    # use biased estimator\n",
    "#     loss = - mi_lb\n",
    "    \n",
    "    mine_net_optim.zero_grad()\n",
    "    autograd.backward(loss)\n",
    "    mine_net_optim.step()\n",
    "    return mi_lb, ma_et"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 73,
   "metadata": {},
   "outputs": [],
   "source": [
    "def sample_batch(data, batch_size=100, sample_mode='joint'):\n",
    "    if sample_mode == 'joint':\n",
    "        index = np.random.choice(range(data.shape[0]), size=batch_size, replace=False)\n",
    "        batch = data[index]\n",
    "    else:\n",
    "        joint_index = np.random.choice(range(data.shape[0]), size=batch_size, replace=False)\n",
    "        marginal_index = np.random.choice(range(data.shape[0]), size=batch_size, replace=False)\n",
    "        batch = np.concatenate([data[joint_index][:,0].reshape(-1,1),\n",
    "                                         data[marginal_index][:,1].reshape(-1,1)],\n",
    "                                       axis=1)\n",
    "    return batch"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 84,
   "metadata": {},
   "outputs": [],
   "source": [
    "def train(data, mine_net,mine_net_optim, batch_size=100, iter_num=int(5e+2), log_freq=int(1e+3), verbose=True):\n",
    "    # data is x or y\n",
    "    result = list()\n",
    "    ma_et = 1.\n",
    "    for i in range(iter_num):\n",
    "        batch = sample_batch(data,batch_size=batch_size)\\\n",
    "        , sample_batch(data,batch_size=batch_size,sample_mode='marginal')\n",
    "        mi_lb, ma_et = learn_mine(batch, mine_net, mine_net_optim, ma_et)\n",
    "        result.append(mi_lb.detach().cpu().numpy())\n",
    "        if verbose and (i+1)%(log_freq)==0:\n",
    "            print(result[-1])\n",
    "    return result"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 75,
   "metadata": {},
   "outputs": [],
   "source": [
    "def ma(a, window_size=100):\n",
    "    return [np.mean(a[i:i+window_size]) for i in range(0,len(a)-window_size)]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 86,
   "metadata": {},
   "outputs": [],
   "source": [
    "x3 = np.random.multivariate_normal( mean=[0,0], cov=[[1,0],[0,1]], size = 300)\n",
    "mine_net_indep = Mine()\n",
    "mine_net_optim_indep = optim.Adam(mine_net_indep.parameters(), lr=1e-3)\n",
    "result_indep = train(x3,mine_net_indep,mine_net_optim_indep, verbose=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 83,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "-1.0088212e-05\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "[<matplotlib.lines.Line2D at 0x11e00f390>]"
      ]
     },
     "execution_count": 83,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAZkAAAD8CAYAAACl69mTAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDMuMC4zLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvnQurowAAIABJREFUeJztnXmYFcXVuN8zK/u+yuKAoCwiLiOCu4ICMZEsajD5lBiRL1F/0ZgvCX4mmmhINBr9YhJNXFBMTHBXVBRQVHADhn2XERAZEUb2fbb6/XG77/Tt23eb23ebOe/zzDPddaurq+7trlOn6tQ5YoxBURRFUVJBXqYroCiKojReVMgoiqIoKUOFjKIoipIyVMgoiqIoKUOFjKIoipIyVMgoiqIoKUOFjKIoipIyVMgoiqIoKUOFjKIoipIyCjJdgUzTqVMnU1JSkulqKIqi5AyLFy/+yhjTOZ68TV7IlJSUUFZWlulqKIqi5Awi8lm8eXW6TFEURUkZKmQURVGUlKFCRlEURUkZKmQURVGUlOGLkBGRMSKyXkTKRWSyx+fFIvKM9fkCESlxfHarlb5eREbHKlNEnrbSV4nIVBEptNLPF5G9IrLM+rvdj7YpiqIoDSdpISMi+cDfgLHAIOBKERnkynYtsNsY0w94ALjHunYQMB4YDIwBHhKR/BhlPg0MAIYAzYGJjvvMN8acbP3dmWzbFEVRlOTwQ5MZBpQbYzYaY6qA6cA4V55xwDTr+HlgpIiIlT7dGHPUGLMJKLfKi1imMWamsQAWAj19aIOiKIqSAvwQMj2Azx3nW600zzzGmBpgL9AxyrUxy7Smya4C3nQkjxCR5SLyhogMjlRhEZkkImUiUlZZWRm7hUlypLqWFxZvRUNdK4rS1Mjlhf+HgHnGmPnW+RLgWGPMUOAvwMuRLjTGPGKMKTXGlHbuHNem1aT43etr+Nlzy/nw050pv5eiKEo24YeQqQB6Oc57WmmeeUSkAGgL7IxybdQyReQOoDNwi51mjNlnjDlgHc8ECkWkUzIN84vt+44CsP9ITYZroiiKkl78EDKLgP4i0kdEiggs5M9w5ZkBTLCOLwPmWmsqM4DxlvVZH6A/gXWWiGWKyERgNHClMabOvoGIdLPWeRCRYVbbskJ1kExXQFEUJUMk7bvMGFMjIjcCs4B8YKoxZrWI3AmUGWNmAI8D/xSRcmAXAaGBle9ZYA1QA9xgjKkF8CrTuuXfgc+AjyyZ8qJlSXYZ8GMRqQEOA+NNliyCVNXWxc6kKIrSCJEs6YczRmlpqUm1g8ySya8D8OfxJzPuZLdNhKIoSm4hIouNMaXx5M3lhf+c40h1baaroCiKklZUyKSRJq40KorSBFEhk0ZUxiiK0tRQIZNGVJNRFKWpoUImjRjVZRRFaWKokEkjqskoitLUUCGTRlTGKIrS1FAhkwY6tSoGYGC31hmuiaIoSnpRIZMGenVoDoDloUBRFKXJoEImreiEmaIoTQsVMmnA1l904V9RlKaGCpk0YE+TqYxRFKWpoUImjagmoyhKU0OFTBpp6h6vFUVpeqiQSQPBNZmM1kJRFCX9+CJkRGSMiKwXkXIRmezxebGIPGN9vkBEShyf3WqlrxeR0bHKFJGnrfRVIjJVRAqtdBGRB638K0TkVD/a5ifGQE1tHXsPVWe6KoqiKGkhaSEjIvnA34CxwCDgShEZ5Mp2LbDbGNMPeAC4x7p2EIEomYOBMcBDIpIfo8yngQHAEKA5MNFKH0sgfHN/YBLwcLJt8xuD4baXVjH0ztlUa7RMRVGaAH5oMsOAcmPMRmNMFTAdGOfKMw6YZh0/D4yUgMnVOGC6MeaoMWYTUG6VF7FMY8xMYwEsBHo67vGU9dHHQDsR6e5D+5LG3oNpDLy0rAKA2jqdPFMUpfHjh5DpAXzuON9qpXnmMcbUAHuBjlGujVmmNU12FfBmAvXIKMZAVY1qMIqiNB1yeeH/IWCeMWZ+oheKyCQRKRORssrKyhRUzXU/7H0yqr0oitK08EPIVAC9HOc9rTTPPCJSALQFdka5NmqZInIH0Bm4JcF6AGCMecQYU2qMKe3cuXOM5vnHkx9sTtu9FEVRsgE/hMwioL+I9BGRIgIL+TNceWYAE6zjy4C51prKDGC8ZX3Wh8Ci/cJoZYrIRGA0cKUxps51j6stK7PhwF5jzDYf2ucbb6/bkekqKIqipJWCZAswxtSIyI3ALCAfmGqMWS0idwJlxpgZwOPAP0WkHNhFQGhg5XsWWAPUADcYY2oBvMq0bvl34DPgI8tdy4vGmDuBmcDXCBgPHAKuSbZtqUT3ZSqK0hRIWshAwOKLQCfvTLvdcXwEuDzCtVOAKfGUaaV71tnSjG5IqOLpwsPDv67PKIrSFMjlhf/cwUOeqCajKEpTQIVMhlAZoyhKU0CFTIZQZ5mKojQFVMhkCBUxiqI0BVTIpAFd5FcUpamiQsYnPtm+n81fHYw7v86WKYrSFPDFhFmBix+YB8Dmuy+J7wIVMoqiNAFUk8kQOoWmKEpTQIVMGvCaGtPpMkVRmgIqZDKEyhhFUZoCKmQyhO6TURSlKaBCJg1U7DkclqYiRlGUpoAKmTSwbe+RsDRVZBRFaQqokFEURVFShgqZDKEmzIqiNAVUyGQKlTGKojQBfBEyIjJGRNaLSLmITPb4vFhEnrE+XyAiJY7PbrXS14vI6FhlisiNVpoRkU6O9PNFZK+ILLP+gkHTshGVMYqiNAWSdisjIvnA34CLgK3AIhGZYYxZ48h2LbDbGNNPRMYD9wDfFZFBBEIxDwaOAd4SkeOtayKV+QHwGvCuR3XmG2O+nmyb0oEu/CuK0hTwQ5MZBpQbYzYaY6qA6cA4V55xwDTr+HlgpIiIlT7dGHPUGLMJKLfKi1imMWapMWazD/XOKLomoyhKU8APIdMD+NxxvtVK88xjjKkB9gIdo1wbT5lejBCR5SLyhogMjpRJRCaJSJmIlFVWVsZRrP+oJqMoSlOgMS38LwGONcYMBf4CvBwpozHmEWNMqTGmtHPnzmmrYEgdMnJXRVGU9OKHkKkAejnOe1ppnnlEpABoC+yMcm08ZYZgjNlnjDlgHc8ECp2GAYqiKEr68UPILAL6i0gfESkisJA/w5VnBjDBOr4MmGsCzrtmAOMt67M+QH9gYZxlhiAi3ax1HkRkmNW2nT60LyWo7zJFUZoCSVuXGWNqRORGYBaQD0w1xqwWkTuBMmPMDOBx4J8iUg7sIiA0sPI9C6wBaoAbjDG1EDBVdpdppf8E+AXQDVghIjONMRMJCK8fi0gNcBgYb7K4J8/emimKoviHL5Exrempma602x3HR4DLI1w7BZgST5lW+oPAgx7pfwX+mmjdFUVRlNTRmBb+cwrVZBRFaQqokMkQuk9GUZSmgAqZDJEqTeZwVS03TV/Kjn3h4QUURVHSjQqZDJEqPea1FV/wyrIvuOfN9Sm6g6IoSvyokMkQqTJ8ywtYcVOniz6KomQBKmQaGfl5KmQURckeVMhkiNq61AgBS5FJWfmKoiiJoEImQ/xlbnlKyrWny1SRURQlG1AhkyE+2b4/JeUGZYukpHhFUZSEUCGTIVI9naUyRlGUbECFjA+s/mJvwtekamHetlqbvXp7SspXFCX1PDZ/I/fOWpfpaviCChkfKN9xIOFrPq08mIKa1FNVW5fS8hVFSR2/e30tf3vn00xXwxdUyPhAIkpJq+J6n6Sf7zqUgtooiqJkDypkfCCR9ZU2zeqFjJoZK4rS2FEh4wOJrK/k5dUvyUsKVufVdFlRlGzCFyEjImNEZL2IlIvIZI/Pi0XkGevzBSJS4vjsVit9vYiMjlWmiNxopRlneGUJ8KD12QoROdWPtsVDIh17vlPIpMAGTL07K4qSTSQtZEQkH/gbMBYYBFwpIoNc2a4Fdhtj+gEPAPdY1w4iECVzMDAGeEhE8mOU+QEwCvjMdY+xBMI39wcmAQ8n27Z4qU1AyuSLajKKojQd/NBkhgHlxpiNxpgqYDowzpVnHDDNOn4eGCkiYqVPN8YcNcZsAsqt8iKWaYxZaozZ7FGPccBTJsDHQDsR6e5D+2KSyHRZKgSLExUyiqJkE34ImR7A547zrVaaZx5jTA2wF+gY5dp4ymxIPVJCXZQFfLe35bxUSxlFUZQsokku/IvIJBEpE5GyysrKpMtLxEgsP9UL/zE+P3i0hufKPk9ZqAFFURQnfgiZCqCX47ynleaZR0QKgLbAzijXxlNmQ+oBgDHmEWNMqTGmtHPnzjGKjU206TL3R3khazLp12puf2U1P39+BWWf7U77vRVFaXr4IWQWAf1FpI+IFBFYyJ/hyjMDmGAdXwbMNYGh9AxgvGV91ofAov3COMt0MwO42rIyGw7sNcZs86F9MYmmyax3OcIsLKj/ylMhYmJpKDv2B8IyH6qqTcHdFUVRQklayFhrLDcCs4C1wLPGmNUicqeIXGplexzoKCLlwC3AZOva1cCzwBrgTeAGY0xtpDIBROQnIrKVgKayQkQes+4xE9hIwHjgUeD6ZNsWL9HWZNy7+n9+8Qmpro6iKErWUBA7S2yMMTMJdPLOtNsdx0eAyyNcOwWYEk+ZVvqDwIMe6Qa4IdG6+4FzuswYE3UarH3Lwvq8KaiLs0yvukgw3oyuySiKknqa5MK/3zgVmaM10R1TFubXf+U1KXZiuWJruHdoW+SoiFEUJR2okPGBUE0mel6nddm/F2zxvS5ODaWmTj0xK4qSWVTI+ED/Lq2Cx+7d/+7lmgKHkNmx/6jvdXHeLxPWa4qiKE5UyPjAY/M3BY/dnpWrXVNiThPmVHhhdmpVKmIURck0KmSSpHzHARZu3hU8P1oTahrcvkVRyLlzuqx1M1/sLsLqY6OajKIomUaFTJJ8WhkaFfPx9zeFnDutySBUyAzs3sb3+jzxwebgcVQRoyv/iqKkARUySZLv0hZ2H6wKOXcbAjizp9q6zEuRsdM0JICiKOlAhUySODUTiG1d5lyTqa5NbUfvFa9GJ9AURUknKmSSxK0tuMWG26+ZU8hUZUCTsfnhk2VcPXVhSu+vKIqiQsZn3JqM+9w5vXbvrPUprUusdf95nyTvgVpRFCUaKmR8xr3W4dZsJI3feCrCOyuKoiSCChm/CdNkogctS6UPMbVgVhQl06iQSTFuEeKyE6AmBRsybVTIKKnkrTXb+XLvkUxXQ8lyVMj4jFtkRAtaBlCTQgszT+sylTyKT0x8qoxvP/RBpquhZDkqZHzGLUTcYsf9eaotzBQlFdjTvF+oJqPEQIVMkhw8GupGZvAxobv43bNhYdNlKRQyXhsuVY9R/EDDESnx4ouQEZExIrJeRMpFZLLH58Ui8oz1+QIRKXF8dquVvl5ERscq0wrJvMBKf8YKz4yI/EBEKkVkmfU30Y+2xeKGfy8JOT+mXbOQ81jTZfuO1LDSI+6LH2hHoKQKfbRSxy6X15BcJ2khIyL5wN+AscAg4EoRGeTKdi2w2xjTD3gAuMe6dhAwHhgMjAEeEpH8GGXeAzxglbXbKtvmGWPMydbfY2SADz/dScWew8Fzt/WYezbtG395n2/89f2QB2vplt1c+cjHVMUIgBaLpiJk6uoMR6prY2dUfMO9yVjxjzdXfZnpKviKH5rMMKDcGLPRGFMFTAfGufKMA6ZZx88DIyWwAj0OmG6MOWqM2QSUW+V5lmldc6FVBlaZ3/ShDb7x1Eefcdbdc4PnYftkXFLmwNEaAA47OsmfPrOMjzbuZMOO/UnVxWu6bOGmXR45c5v/eX45o/9vXqar0aRQIaPEix9CpgfwueN8q5XmmccYUwPsBTpGuTZSekdgj1WG172+IyIrROR5EemVTKP8It530anxbN55CIAXl1T4eu99R6rZf7TGlSf3O4sXl1TwmfWdKemhETw2SppoTAv/rwIlxpiTgDnUa05hiMgkESkTkbLKytS6VknG2/GhKn+ngI5Wh0+/vbikglUVqVkTUhovKmSUePFDyFQATq2hp5XmmUdECoC2wM4o10ZK3wm0s8oIuZcxZqcxxo5n/BhwWqQKG2MeMcaUGmNKO3fuHGczG0gGX8Z4OoKfPbecr//lfTZ/dTD1FVIaDTpdpsSLH0JmEdDfsvoqIrCQP8OVZwYwwTq+DJhrAvM0M4DxlvVZH6A/sDBSmdY171hlYJX5CoCIdHfc71JgrQ9tS5p4N/R7vbPp3Dd5/n3vpu9mSs6jQkaJl6Tj/xpjakTkRmAWkA9MNcasFpE7gTJjzAzgceCfIlIO7CIgNLDyPQusAWqAG4wxtQBeZVq3/CUwXUR+Byy1ygb4iYhcapWzC/hBsm3zg2Smy9x7apK9d2Pf7G+MUY8GaUJFTOpobAEFfQkyb4yZCcx0pd3uOD4CXB7h2inAlHjKtNI3ErA+c6ffCtyaaN1TTW0SvslKOrYEAtE212/fz/C+HRO6/rH5m3jwylOC54198FlbZyjIVyGTDow6qkgZRzzWTnOZxrTwn5W4PQJAwCtAy6L8kDSnABhhCZPju7YG4KqpCxj/yMcJC6x31u1IsLa5TaojjSr16HRZ6rjrtTWZroKvqJBJEbZpsNfL+PpPzmHqD04Pze9QkWvq6qy0AKsq9gEkvDkzfI9OQpfnHNV1uTsCXLJld06Zk6uQUeJFhUyKmPrBZiDy3HW0tYMqa0Re59JcZq9JbCdwLnVaflCdpIeETDF79Zd8+6EPmb7o89iZs4Sm9WRljupG4EDXlzUZJZx73lhHh5aFQV9lz/73CEo6tQh+7l7Ud8qDWmtEXmcMXzhc1CQauyNW2AE3dXWGvAZaG7yweCu1xnBFaeb2wObqdNnmnQHz8Y2VBzJck/hZumVPpqvQJHjonU+5aVT/TFcjKVSTSRFVtXX89JnlwfOOrYro0rreeaZbk3FOP9izPiu27uVMh4uaDi2LEqqDW6jE0mySCaD2s+eW84vnVzT4ej/I1VGfXe2GCvhMcN1TZZmuQpPgqwNHY2fKclTIJMnIAV2ifm4LD7f3ZXd/4uzf7Wvcvsv6dm6ZUN3cppCxRMhBl8uZeHn8/U0Nus5vclfIBOqd39gXzZSEyaFxR0RUyCRJn07RO35beXA/K25NxqllfL4r4IfLbU2WaB/qjowZa7rsN6+ujp4hAk5rmH+89yklk1/PiGVbrk6X2T9zfmPoURRfaQz7vlTIJEmsbi0oZFzPSjRN5qDls2zW6u0heWKZMLunww673N/H2uRlC7dEcNfpD2+sA+D5JVsTLitZcleT8dZ2c4EcrHJOkYvPhBsVMkkSSzuwP3ZrFe6Hx54i+9E/F0e5V/SbxRJCsZZcGqIHvLDYW5gkEwtn/5HquKfunHFkpn24ucH3tJny+homTF2YdDmJYP9uuajJ5F6Nc4scfCTCUCGTJLH2C9idrXtA4j6vM4aqmjreXB3ZTLk2yYX7WEKqIev+v3jBe7F/3+HqxAuzGPKb2Qyb8lZcefccqr/PcxEEXiI8On8T732SWs/cbuxnKCeFTCMYaWczRQXeXfTBozX86uWVDV5HTScqZFKMe8rKxq3JGAPff+zjqGXF0lRiC5moH/u6r2ZBksHRDsYZ5iBVU2Tp3GNUG8E4JBfIvRrnFoX53l30I/M28q+Pt/DEB9lhdBMNFTJJEnsKK35NZtHm3VHLiqU1eW1GfGfdDso9ImyOPz35/SzJ+GXz4sNPv+LWF1cmdE2qwi6n039UrWWwkIMyJicFYy5RGMEXX1Wt3a8EPv9iz2HeWrPdM2+mUSGTJM5+9vSS9mGf3zf7EyB8WsFLk4lFrEF7lUeGa55cxKj75/Hl3iN846/vB9Pv/s5JYXkTHbw/+PaGxC6IwfceXcB/Fm5J6JpImmKyfLH3cMQYOyu27uGqxxcwf0Pi02p7D4VPI9qazN2W0UQ6ueHfS0LChSeMypiUkp/n3UXXuYxFLv/7R0x8qiwrvXyokEmCz3Ye5IPyr4Ln0344jM13XxKSJ7gm47o23Los9sMRS3PwinxpM/wPb4esX/jBxxt3Rvxs7IndfL1XJFZ/sc+3stZ9WV/WyD+9x/n3vcviz+q1yzlrtvPY/I3831sbmL/hK656fCFPL/gs7vJfWVbB0DtnM9m1juV2H5Qutu09zOsrtlGx5zA1OWqZ19hY9nmoJ4VIy3Rbdwc8geTnBdwSVVieQbLRg7MKmSQ479532WiNdsedfAzNC/Mj5g2fLnNbl8W+XyxBtGZboJP86/dOiZovEpFU80hEW6hOlzlxotNr0Xj643At6jsPfxg8vu6pMn73+lp27K9373PbS6viLv+m6csAwnyUxTLoSAV7D1Uz4g/1Gsz2/Q3bWZ6Dtgox2XOoqkHm/H5wy7PLQs7fXe+tLb++chsAX+49yiSHRWo2Oi71RciIyBgRWS8i5SIy2ePzYhF5xvp8gYiUOD671UpfLyKjY5VpRctcYKU/Y0XOjHqPVNOpVTF/Hn9KVEubWCbM8ai50R6gt9Zs50f/CjxsxQWRhZ2TzXdfwq+/Pih4PnJg17ius/nw03pN5taxA0I+S8ZFTbwcjtM4IBa1dYbpC7fwz4/j00oOHKm36GlVnLz7v395CLdUMXfddkomv85FD7wXkj4nglVj+Y79Ub9n93OdK7yyrIJtew9TVVPHlp2HQty3nHznHM754zsxy7jtpZVc8fePfK2XezbiowizBYOPaQPAKb3bhaRv23vYK3tGSfoNEZF84G/ARcBWYJGIzDDGOIMiXAvsNsb0E5HxwD3Ad0VkEIEomYOBY4C3ROR465pIZd4DPGCMmS4if7fKfjjSPZJtXzzEs/YZpsm4Po+nT442XbZ8a72a3aww+tihdbP6n71Hu+axbxwH7prVJLj7fp6H2XBtnYmqLQ28/c2Q8zGDGzZFd9z/hsXGC3Le8Z3D0jbvrB/ljhwY3a1QLLbsTO2I+dXlX9C/aysGdAt0Sj98MuBzbIdLc/nNq2v42pDudGlT71/vaE0to+6fx4UDuoSFprDJxXX/L/YcDmqVTp764bCE9nc9vcD/wUGEJZgw7LVI98Bo0ebd9OvS2u9qJYUfmswwoNwYs9EYUwVMB8a58owDplnHzwMjJTDsHwdMN8YcNcZsAsqt8jzLtK650CoDq8xvxrhHyonnJuFrMqEps6Lsj7H5VZSpmb/MLQ8ex9JkpnxrSPD44kH12ktD1wbatygMMxpIZLps35FqrvbYAFmxO/KozF3XZoV57I1zb87+I9VBj9Ze0yL/uW44d40bzNBe7TDAzJXbeGDOJ55lObWaaETSVMsrwy3//GLH/iP8v/8sZcz/zefXL6+iZPLrUfMP+/3bIefvbwisN85dt4NnFnl3qId80iYTZd4nlfzwyUURrQura+sifnZmBEOHq6cuZKLD8ecn20N/m4o9h/n5c8uTtmisrTPc9doaSia/zmzHe19XZ/h8V+CZL7b2x5zdr5NnGRsrA9P0C11bBfy2+PQDP4RMD8A5ybzVSvPMY4ypAfYCHaNcGym9I7DHKsN9r0j3SDlxibIYJszx2Lvvj3PjVXGEDVxeVXF6/m3o2sCZ/ToFXdb897l9Oatfx4Smy7a7Qhicf0JAexjlmtKxWbhpF31d2seR6rqIUws2Lyzeyg1PL2HIb2Yz/A9vc+uLK0OmRS4e1JU3bjqHEcd15KoRJbRpVsC8Tyq5/ukl/DmCJV28v8kml6WaLXT2HU7dZrp73lgfPHaOeIf16QBAvy6t2Hz3JYweXD/QsENLGGO4dlp9h/vLF1by0tL0uwpycuBoDSu37uX+2eu5eupC5q7bwYBf12uzxhgq9x9l8We76H/bGwz49Zt879GPOVRV/x3HErROLn5gHs8s2sIn2/ezdts+Rj8wj+cWb2V2gqbCCzbuDBEmd766OuhUdtI/F7Nl5yGOVNcy4Yn6gdb5J3SmR7vmdGvbLKy8aMQzhfzY/I1c/3RkzyJ+0yTjyYjIJGASQO/evZMuL569AmFrMq5pID8HIMUxpssi1behdfjT5UODL42I8EF5oLOvqqmLuGPZyc6DVSHnt31tIO+ur4w4dXHFP+KfB6+rM7y64gsuGdKdnz23POQzp7n0H79zEle49g7t2Bd5MTw/TzjzuI7M3/AVG7bvp3/XyFMUm746yIV/ChWYj87fyKRzj+PZsoYFKquqqeP4X70BBL6v687ty9Itu6mtMxzTrjnvrq/khQj+4/7xX6fR3hE24o+XDWXW6tlA5FE+wE+fWU675kVcEMPzeDy8s34H762vZPqiLXRsWRy0jurfpRUbdhxgWEkHjuvSkrP6deLV5V+E+fFzsvqLvayq2MsvXwg3Avnw050Mun0Wg7q3CRrGANw8qj/fO6M3W3YeollhPre9vIrln4fHyPEq8w8z13Lp0GOitu/jjTu589U1vPDjM/nuI4FN1i9efyavLd/GtI9Cp7jOvTd8/UcQCvMlYQOaKTPXct25faPm+d3ra4GAthdps6ef+CFkKgDn29nTSvPKs1VECoC2wM4Y13ql7wTaiUiBpa0480e6RxjGmEeARwBKS0uT7t7jUmRcmfx2696quIAD1qjaOV12Us+2rNi6N2pdbBo6XdbMYVXnLHvP4Sq6tG7GK8sqGNG3I13aNKNiz2EOV9WEzBs7QwU8dnVp1A4boGubYrZbAuCsfh15fMLpXP/0khCrL5uZq7Zx0/Rl/GFm5D0of7zsJM9ga+u3R57KeuSq04Ij/YsemMebN58TXPewqa6t4+x75gbr6uSvc8uZdO5xjB7cLcSAIh6MMZx215zg+ZSZa/l89yGecnVeNkt/fRFfHThK386tPNe42jYvjPve1zy5iIW3jUyovk6MMayq2Mc1TywKplU4AvNt2BEI3LZw8y4Wbt7FfxaGC+FTerejRVF+cDBzyYPvh+UBuPeyk3iubCsLN+8KETDzf3EBvToEAgjaMZ5eueEsamrr6HfbGzHbsC2O4IF3vLKa9dv3h6wbfvuhekvF3146mC27DoWFyRg1sCtrt+3jxgv7cfMzyzzXNuOdFo7F72eu5Y5vDPalrGj4IWQWAf1FpA+Bjn488D1XnhnABOAj4DJgrjHGiMgM4N8icj+Bhf/+wEIC/XZYmdY171hlTLfKfCXaPXxoX0yOxrEqvafWAAAgAElEQVRYGLYmE+cA4sQebVhVEXsvyNn9OgX9nhXkCT8ffQIvLNnK/VeczKj7Q0fRkcRbMuaP9lftLvtQVQ03TV9Gvy6teOuW84Ib/5z7ieY6wgK0axHa4b28tIJvnlI/+1pXZ0I67R+c2YdmhfkU5edRXRNef1sb+nJffccwamBXHrnqNL46eJQOLYooiDCaO6tfx2BH5ubYjqEhHn732lr+NfEMIPBdPPHBZt5Ytc1TwADBezbEkejIP70XNk0XScC88OMzad+yKERz8WLz3ZfEPZU0bMrbsTO52LLzEP/70kred+wrc/Lz0SdQuf8oPds3p1eHFizZsps2zQq5d9Z6igry+P23hvDtU3qEzABs23s4aIb93dJeXDCgC2+t3c7zi7fSqVUxl5f24rLTerJl1yHOu/ddLh7UlW+d0iMoYNxEeg4aQrQBCsB/DT+W/DzhV5cMZPFnu+nftTXNCvNCBoiF+XmeG6z3H0lMyBypruWu19Zw44X9mOiYAn3ig825IWSMMTUiciMwC8gHphpjVovInUCZMWYG8DjwTxEpB3YREBpY+Z4F1gA1wA3GmFoArzKtW/4SmC4ivwOWWmUT6R7pwD3d44V7iqogTinTuji+UebRmtC52Bsu6McNF/TzXKR0azJ/unwoP3tueVL7NWwlyNnOA0dqgi9N+Y4D7HO8HCWTX2fsid14+L9Oo/TY9kFfZy2KAo/khQO6MHfdDm5+ZlmIkHGP4loWB8ovLMjznFpo0yz0+3v7Z+fRt1NLRCQkUqkXZx7XiQ/KdzJyQBfetgRhn04tqamro1+XVvzqkoHBqQdn5/nPjz/jztfWhJX31i3nMur+eUBgIAABLSQWiz/bHdyv8+Q1pwf3ZgEM7N6GtY5R+hPXnM6bK7/k/fKvqNhzmNOODfdCEYmPbr2Qg0drQwYlRQV5SXnUrqmtY8hvZkf0zPDp77/mqV2NtiwFb7igX8Syu7dtHrb5+ax+HenZvjnXnNkHCEzfHtuxZVi+SEyfNJxWxQXMXbeD+yMYe8TDcZ1b8mll/e/0xDWnM3v1diaPHRCiOYoIpSUdPMsoasB0mRevr9jG0wu2pMQaLh58WZMxxswEZrrSbnccHwEuj3DtFGBKPGVa6RsJWJ+50yPeIxto6HTZzoPxbZJzalNOs+Rmhfk8ec3p/MAxPeE2uvvOaT351curEnIrY2sux3dtBcD3z+jNh59+xdVnHstf3wlYul34p/co+9Wo4DWzVoVa0L2x6ksuuO/dkEXxFkUBoXFij7ZBDWf26i+52Op0dh8KCPQ+nVrSuVUxI/oGbDsK88Vz1OfmuM6t4m7j98/ozaqKvUz51hAmTF3Iyoq93H/FUE7pHei4J57TNyhkICA4l/z6It5zbaCb9sNhnNyrHZWO6bwd+4+GGQNE4urHFwSPnb/ji9efyck92wWNIH576WAuOKELF5zQsDWT7m0Dz833z+gd7JBevfFsXlpawbVn96Fjy6Iwg4tY3Dt7fUQB8/yPRvjuebp1s0JuHnV87IwRGG49Tyf2aMtPRvYH4I2V2/jx00tCpqR3xxhYdm/bPETIDO3ZLuHfpTDfe+CU6Fgw02bmuuM/TYQv/Md3XTxGBbV1JmRe321UcL7r4fYqMU+imz9+tvMgf527IShcPtl+IOR/x1bFTJ80gi6tm3HLRfUvuXNOea5HtEy7oz3/hM48cc3plFiRRv/fhfUj2H85RmB2GXd8YxDP/mhEUGAW5edxuKo2zFTY+ZK2TnDjZLsWRTz8X6fRoWVR3C/qw++Wc3qf0JHpecd3pm3zwrBp1W/+7YO4yvz6Sd6LzKf2bk9enrD57kvYfPclTDizJL5KxmDKt4YEByq9OjRn8tgBdG5dTF6ecLtj8248/OO9jcHjeT+/IGjJtvB/R0YcwWcbY4d054PJF7Lqt8G94mFGJG4OVdVwdr9OrP/dGN646Rw6xJiu9KIgX6iuNdTVGW57aWVwmiyWdlNVU8d/Fm4JrrFmOoSECpkU4LnB0a3JuH74Gy44zrOsK0p78cB3h0a9X7TNhF54bR/KyxPPNZmXl1ZQMvl1zrv3Xe6b/Qm74/B/5tQWnC9ENGeWuw9Vh4z0CvPzeO3/nQ3A2f3qLdFtzcGeVrM5WlPHzoNVnPTb2SHpTu0mXnNjL46xRvnNXK6D5v38Avo6QnA/On9T0NHl8L4dWPrri0Lq6CTeBdwCD3c/Dem0EmH6pOE88N2hYd/z984IWGN2bl0cswxjTHDjb55A744t+MdVpWy++5KQTZ+5gP1O/+qSgYD3gMmmpraOJVv2sO9INcUF+Qzs3iZi3mjYmsyvX1nF0wu2cPKdAWMPt1siNw+9W86tL67k1RVfBMvxwq+N2LFQIZMCZt50Dje65pLDwy+HJrjXDpzXfeuUnhHv1RCLMC97iDyRYFm7DlaxwvIg4HYAaQsK28/ZPd8Zghtnp+gUMruiTDF4mY/arjN+P3Md677cF7K+5F5rsEd5+12bI6sdmtSjV5dGvH8s7rnsJO697KSwDqN3xxa0jKAhTZ80ImTBvWd775faFqYQamlls7IiYB145bB6C7iF/9twC6946NWhhedz16wwn09+Nzbm/ae8voY+t84M/h4fp7i+6WLiOeHmwe73yd6v5bbqTJSWRQWs/3I/rywLCAt7puH1FduiXmdP5dmDGC9N5uUbzuKDyRcmVb94USGTAto2L6SraxOV+2d2//DORfdvOGzwI41CynfsZ+GmXexpgDmjl1jKz5Pg4v1F97/HpX8NTOW4Y9wctja22Z13aw/hWOSos7OTj/bSPXFNuNsSp8b10tIKtjh257u/P6fQdr70tpB78prTuWhQYr7ZnLRtXsjlHmbOAOu/jG/XfpfWzdj0h6+FpL11y3mc2KNt8Hx1Rfh3ZH9vlw6tN4Dw0xIqUYoK8qL66TPG8Oj8UNPc4vz4/OnlAm/dcl7IuXuc55dN65urv+RQVW1wHcjG1iYjYf829qCxwEPIFKXx+VEhkybcL6V74d+pkQzpUT9avvTkgMApdYzcPyj/ilH3z+OKf3zET58J98EUi50HwjWKPKkXdLa1nJdfLXtTnN15ewnBSJpMJDbffUnMRdFNlQfZYK3/eOFcT3rw7XoXO/b9nR253zw6oZRhca4vuJ+Dfl1CDRGi9U/D++bGGoaXiySvKb9cxf2bfeHQPo0xntpoQ/Dymwex/QLaj1i0SY54Nkn7hQqZFPGhaz9A+D6ZyDv+6wzcd/lQ3vmf84PTaPY6x2srvuD7j9VbGy3aXO+7qG+nljz1wzDDuzBWVoRPTX11oCosCNdba8N3Wd8/5xOueWJhcK3DKzyA0zz7sfkbwz5PhHV3jQFg9prt3PDvJQC88OMRYfmqHV/gvxxTfLb5bSp3Np93fGee/dEI7hoX356DW8cO4Przj+ONm84J+8yYwLRiyeTXeXV5YJqkU6tixp7YLar2kE14bVbM9OKz3/ze4f/vT7Pr3fe8tmJbMPzEkx7aeSI89P1TPdNr6qIP3GwjI3uN1cvFUzo1mSbpViYduFXcWP2DcyReZwyXnRY6H24LpRv/vTQk3emgcNZPz42rM43UWdnO+Wy89nrU1hneWV/JpHMDhgpe93N2KC9b88lO/jz+ZEYO7EphvlAYw8zOvdAO3utXBxx7cCodHobtFywdL9VVI0oYObArS7fsiejYEOC/z/M28rDZ9FVAY3v8/U18Y+gx1NTVxbXQni14PV3pcF+STr53Rm8ef38jn1YepLrWMGv1l8zfUBkStiHevXCRaFlcELK3ysbLVL9bm2bBDcdTLT+ItqGJl9WoajKNgFN7hy5Mx4q74bTs8lrMj/WO/v2/Tov7RfZjTPn6ysiWKxV7oruvP65zK1oVF1BckB+m0cWD1zqQe/H90XkBDaraetHSNV1zTLvmXHJSd9q2iN9Vi5Mqhz8pu5M4Ul0bFLZ/vOwknojgdj9b8Nqc3Ng0GSDkd/rvfy4OiwvkR5v7dWkdEvMJoLrGhA2ariitH5Tae9faWJs+vaasEw1QmAwqZFKE2xw4libzxZ76KQavudRYmzcTeWgaMutyTv/Qkbn9Qnnd98zjIo/iIfH1EbdFlzMejs39V5zMvZedFDyfMnMt677cxy5r86bX4mc28vLSiqDRxcbKA9w3az1HquuCncoVlvuUbMYZdqIxY2/cjOSQ1q+BjXNtpqa2jpq6cMezxYX5XH5aT7q3bRb0sm2jmkwj5RsxvLS6ibVnIj+G6p3IdESfTuG73gd0a81xnVt65A7w1+95zw973bdrm2ZMcnmCtQ0XvARELCaMODbk3PYK4KRz64CvqpKO9X6pxvzffJ74YDMQeYow23DuvzhaUxf0npCr000n9UydwUWmuc3aM+Pcb+LcM+WX9uY0NKiqraO6ts4znEdBfh7VtSY4TWdr8c41maL8PL4x9BhfIrrGS24+uTnA8S5PwrH7uPoH4diO4Q78YvUxsUZNzo7ZK2u3ts1oVVwQ0V1GJE+9kUZE//u1gSHnn1YG1hnc+1ji4bun92Li2X2C59EEhtu881RXeNpsxcsbwSUndQ8eFxbkhpB08oMzS/j3dcN593/Oz3RVUkJhfh4dWhaFxKvp2Kp+X1QqtOeaOkNVjfF87wrzhZq6OpZsCWw7cK/J3P3tIXwyZSx/uTJ6qHi/USGTJmKtyTg3KnrFqoi1dhFrYdv58Hs9YPki1NQZNu+M7E/LS/jFu6Aej6eASIhImNCOxMSz+waDngG0b1HEMQkGfsoE835xQVias5NKpzWQX3xtSHdaFRcEXQU1Rory80LWYpxajZ/rUIOsKeOaWuM5XQYBQ4OaWhPcV2VbVpZb4RNGDmz4PrFkyL0nN0eJNXBYsqXerDiSEIhGrM15zs2KXkXl5wm1dSaqu//rPHY7tyhOzya7b5/ag++f0ZvZPz03ar68PAlZGH973Q6+iCP+R6Zp5ZpGzM8L9cCbK2tKTnKwygnjDCEBMMqx4TdZ6zIntoZeUxeYLnMPOooL8sKCnFXVBixP7Zg1mXqGVMikiWR/3lijolgL/7GElC1kojnkvHJY+E7jWO7y/aIgP48p3xoSl0YjIvSNsr6UjbjXXE7s0TYkMmcWhm6PSY4sg/nKxYO60duKV+OnJmMLiJrawHRZQX5eUGs6sUcbrhpxLAX5ErL+4g7RkJ+hDbEqZNJEsnOgsbwxx1wYjnH7I9W1bNhxIOroKz9PeOn6M4PnZ/SJvgP9R479IJefFtn/WioYksId/qnm5lH96dW+OZ853Oh4bYzNFiLFBswVYwu/GFbSgaKCvOBsgJ+agz1T8dWBo7y1djtrt+0Lrrn89tITKS7IZ/nne0Msyapq6qjJAm1YhUyaSIUm49ygF0vIhEyXeXz+jhUDxQ6OFQmn99xERmo/u/iEuPP6wZ3jTkzr/fzk5lHH89qKbSGbSrN5n0kkLSt7a5waqq2d+LbMTYUmY5u3A1RZgQqbW3uo3FFHp330WUg46Uw9Q0kJGRHpICJzRGSD9d8zDJ+ITLDybBCRCY7000RkpYiUi8iDYg19IpUrAR608q8QkVMdZdWKyDLrb0Yy7YqHRCM7JzuoW/zZ7rC0MVYgL4ht9x5vkLRYgb8SWdi0v6PJYweErTmkmjbNChrVmkA8cYUyRaR1vGyus19832HNaJsF29+Hr0LGY6rLfle9TPo9y/BxjSgRkr3rZOBtY0x/4G3rPAQR6QDcAZxBIKLlHQ5h9DBwHdDf+hsTo9yxjryTrOttDhtjTrb+Lk2yXTFJPDpdcg/cxxvDY807H7xY1kd+vu/2fppY6re9C79VcUHaVXURycl1DBv35lendWC2sSeC5WBTEDJTvjWEaZa/QLu9tpDxs/nu96dfl1YcqQ4IGS/XS15katCV7PByHHC+dTwNeBf4pSvPaGCOMWYXgIjMAcaIyLtAG2PMx1b6U8A3gTeilDsOeMoEhsgfi0g7EelujIkeYCEFvPdJZexMPlKUnxcW9Mo5RRZLk/FzftwOKxtrg+h/n9eX5oX5jD+9V5Obn28IPzizhOOsjXe/HDOA+RveD342/vTo7t0zyb8XbOHEHm3IEwnxRtBUfvIRfTtyRWnPYLhm+z310+zcrYXYZskALS0Lzy6ti9mxP3K49ky9g8l+C10dHfyXgJchdg/AGcptq5XWwzp2p0crN1JZAM1EpExEPhaRbzakMYlwzZP1sdad4YZThZe67FTHY1mXhYxifHrYYmknxQX5XHduXwry84J17ZJDjh7TzW8uHcxVwwPeDdxTICUee5SyhYNVNVw7rSzknYCmI2SKCvL442VD6dk+8Bvdf8VQhvZqR7sW/mmfXlNvj11dyiUndQ/68rv9G4mFxk4XMTUZEXkL6Obx0W3OE2OMERHfJygSKPdYY0yFiPQF5orISmPMp14ZRWQSgek2evdOboTYpXVxcATjpl2LwohTCYni3gfzxk3n8MbKegXOHSbXjZ9TF8d1bsmnlQcTnnN+4prTg5vK0kGeBBal3VE0cwH375mN4YpbFxew/2hNxJhBsTYgN1YuHNCVCwf4u/HReEQaGjWoa8i+nNGDvbrpzBNTkzHGjDLGnOjx9wqwXUS6A1j/vQJfVwDOkII9rbQK69idTpRyI5WFMcb+v5HA9NopUdr0iDGm1BhT2rmzd2CgeImmns6+OfrGwURwv669O7RIyIPxBY5d8PEG2IqEvXM4USFzwQld6JrGztKOweP0UJsrpGuTazLY2vXeCAOppqLJpANnSI9IZOuG3WSny2YAtrXYBOAVjzyzgItFpL214H8xMMuaDtsnIsMtq7KrHddHKncGcLVlZTYc2GuM2WaVXQwgIp2As4DwYChppiGjzz9dPtQz3T2OaVaYn5DxwVUjSoLHJ3SLvaHxu6W9+Oko72lA22QyVzqRTFnVJIOXL7Nsw9auX1xaESOnkizNCmIPOrJ13TPZt+9u4CIR2QCMss4RkVIReQzAWvC/C1hk/d1pGwEA1wOPAeXApwQW/SOWC8wENlr5H7WuBxgIlInIcuAd4G5jTMaFTEOI9zlxahGRpusaUq7Nt07twU9G9vP8rLm1XuAVcS+bsGuXi6F/s7XDcFIY04Q9TRVpAowcmN3hHaKR1HDJGLMTGOmRXgZMdJxPBaZGyBe2ay5KuQa4wSP9Q2CIOz0XidS33P3tIVw7rQwIdScO8W16i5XnjD4dWLCpPpRznkjEji6olmd5J2Lv08lF55K5QM8OLXLCL1xjIBcGHZHQty9JhvdNbn0jXkYO7Mof7aBcDXjeYj2k15xVEnLuHKQ6N2BCvbm012JkNlGvyehjngoe9ohBv3V39KioStMj+yd+s5wfxYjXnijRLHJsNzK2pZjdicbT1ceSS24hZBsVvHnzOWFOMLdbnmfXbtsfx50ziPXF5OJ0WS7QsVW4Ofr/+8/S4HG2D0KU9KBDvCTx2x9QNIXDFi52Fns6KB6XMbGyuE2c7fMB3drQoWWovf/cdYGNqJu+ihx7JhuwuzidLksdd40bzNMTzwieD+2ZG0HimhqRgg6mA337kiRen2B+3su+pe2+Ih45F2vPgruMaGVmq6lkJHKtvm46tsxelzJXjSjhrH6dguG2n/xwc2Yr1MS57pw+numZ3MyrQiZJEtmrEg/R1k5sS9x6H0nx10Fi/NKRNBkvxg7Jzk1fbmxNrzCGy51sJ1oguWzBHW5bSS3OfW9OTujmvdnZ9kaQCXL77csCfJ8ui/JZdW2gs9lo+Q5LpO+JvSYT/dzJN08OePLpnOUuYuyvpzAH98kA/Hn8yUD6AsMlS/ccCHPdWPj9t72NaVdu3RNy3qNdc+67fCj32EZDGUAX/htI/y6t2LDjgO+eZqMVt8gyMbZdfNvuPGL5LQuUG2u6LP522NZltVm+T8YmVxf+e7YPWPXV5oAmA/Cf64Zz/n3vBs9zpNo5SaT31fmel08Zi4hkPBZRbg7xsgD7t/Rfk4lcntta5zIr2uSYwd3jKDc67oe2LkpYmfpQsNFjz2Qau5OLGTU0S7E9FeSKMC/plFshr3OZSO/z10+q7wucjmkziWoyDcTulP1e+E+kuIHd27D57kt8Kdf9LEYbPdvOGyee0zeue2cKWyjHo+llI3YHURNN4mcxuapB5jLHdgwI+k5ZFH9IhUwDKbamjKp97gCivZbHdmj4SDFRj7jRRs9FBXlxC7dMYsvJXPUGbGtgOSpjOKFrbB95Svyc3a9TfYjlCI+0PTDJBg3GJjfnEbKATLh5OLNfRwCOacACa6LVbdcic3b1fnH1iEBsljbNc3MsZWsCuTJd5iaXXaFkI1N/cHrMPO1bFPKTkf1D9i5lmtx8+7KAVL320d5Lk4DJcqK422O7yc9lfnhWH8ad3MPX4FHpxF77ypWFfwiEBXZGbVT8w/naR9LORSQtQRQTQYVMQ7HjeMfI9tYt51K+I5Gd8ZFLrN98mbiQiXVJDvVjcVOQn5fW+DV+Y/tcyyVNRr0rpI5c1QxVyDQQ+7WP9cP369Kafl3in5uOVlxw82VDHGTGEIfqZyr7sF3p55SQyfGNr9lMCiKopwV9IpLE7986Wnm2DzHbdDmhcpugJpPr2Iu3dTkkZIpVyKQMCZkuyx1Uk2kgqeqUo2lGbZsXsv53Yxo0JRHrocydbqzpYO+TyfbgcE5Uk0kduTpdltQTISIdRGSOiGyw/rePkG+ClWeDiExwpJ8mIitFpFxEHrTCMEcsV0QGiMhHInJURP7HdY8xIrLeKmtyMu2Kh96Ww7mWPofJjfUYFRfkN+hhi3WN7efrmLbNeOzq0oTLV/zH9oaTS1OZue6MNFfIJYGT7LBjMvC2MaY/8LZ1HoKIdADuAM4AhgF3OITRw8B1QH/rb0yMcncBPwHuc90jH/gbMBYYBFwpIoOSbFtU7vnOSTxy1Wn06+KvFVaqnp14NZl+XVszalDX1FRCSQhbk8khRSanOj8lPSQrZMYB06zjacA3PfKMBuYYY3YZY3YDc4AxItIdaGOM+dgKq/yU43rPco0xO4wxi4Bq1z2GAeXGmI3GmCpgulVGymhVXMDFg5PzRvyrS8I916ZMyMQpZbSLyB6CmkwOLZjp85Mecul7TlbIdDXGbLOOvwS8hsA9gM8d51uttB7WsTs93nLjuYcnIjJJRMpEpKyysjJG0aljaK/0BXiKd4SpA9HswXZZlFuaTKZroGQbMRcUROQtwGvIfpvzxBhjRMT31yEV5RpjHgEeASgtLc2hVzh15NK8f1MhP0+4dOgxjD+9V6aromQZuSTMYwoZY8yoSJ+JyHYR6W6M2WZNf+3wyFYBnO847wm8a6X3dKVXWMfxlOu+h/NNdJaVtXg9J1U1mXFUZXS6LOsQER688pRMVyNB9AlSQkl2umwGYFuLTQBe8cgzC7hYRNpbC/4XA7Os6bB9IjLcsiq72nF9POU6WQT0F5E+IlIEjLfKyDlWf7EvI/ftY7lpv3BAl4zcX2kc5NIIO5fJJaevydrf3g08KyLXAp8BVwCISCnwI2PMRGPMLhG5i4AgALjTGLPLOr4eeBJoDrxh/UUrtxtQBrQB6kTkZmCQMWafiNxIQKDlA1ONMauTbFvKyaYXsm/nViy7/SLaNs99x5hK5siiR7pxk0NfdFJCxhizExjpkV4GTHScTwWmRsh3YgLlfknoFJvzs5nAzASqn5Wk2pBoWJ8OET/LVUeSSvaQTQMnJTvQHf8ZJb1v5JJfX0SLovy03lNpWpx5XCdmrd6e6Wo0enJJmKsPiCZEh5ZFNCtUIaOkDjuGj6LYqJDJILk0GlGUeNAd/+khl75lFTKKoihKylAhk0G8RiO6KVJRlFjkksaoQibLyCE3VYqiZIjcETEqZDKK12gkhwYoiqIoMVEhk2W0170qiqLEIJcGoypkMkgOPSeKoigNQoVMlqFrMoqixCKXfJepkMkgtso7pEdbJp7dB1DrMkVRGhcqZLIAg+E4K4xzr/YtMlwbRVGynVxak1HfZRnEqfKOP70X/bu04rRj22ewRoqiKP6iQiYLMCZgzlxaEtlDsqLkEu1aaMgIJYAKmQySSyqvosTLurvG6LOdYnLp+01qTUZEOojIHBHZYP33nOsRkQlWng0iMsGRfpqIrBSRchF50IqQGbFcERkgIh+JyFER+R/XPTZbZS0TkbJk2pVu1KJMaUw0K8ynuEC9fSsBkl34nwy8bYzpD7xtnYcgIh2AO4AzgGHAHQ5h9DBwHdDf+hsTo9xdwE+A+yLU5wJjzMnGmNIk26UoipK1NCUT5nHANOt4GvBNjzyjgTnGmF3GmN3AHGCMiHQH2hhjPjbGGOApx/We5RpjdhhjFgHVSdY7K7BV3jpVZRRFaaQkK2S6GmO2WcdfAl098vQAPnecb7XSeljH7vR4y3VjgNkislhEJsVZ/4zStU0zAIb37ZjhmiiKkkvk0n66mAv/IvIW0M3jo9ucJ8YYIyK+tzyBcs82xlSISBdgjoisM8bM88poCaFJAL179/axtonRqVUx839xAd3aNstYHRRFUVJJTCFjjBkV6TMR2S4i3Y0x26zprx0e2SqA8x3nPYF3rfServQK6ziect31rLD+7xCRlwis/3gKGWPMI8AjAKWlpRkdEvTqoJsvFUWJj7bNC9l7OLdWC5KdLpsB2NZiE4BXPPLMAi4WkfbWgv/FwCxrOmyfiAy3rMqudlwfT7lBRKSliLS2j617rGp4sxRFUbKPF358JreOHZBT1nvJ7pO5G3hWRK4FPgOuABCRUuBHxpiJxphdInIXsMi65k5jzC7r+HrgSaA58Ib1F63cbkAZ0AaoE5GbgUFAJ+AlywK6APi3MebNJNumKIqSVfTr0op+lguqXEFME7dsKi0tNWVlObWtRlEUJaOIyOJ4t4qog0xFURQlZaiQURRFUVKGChlFURQlZaiQURRFUVKGChlFURZBlRUAAASgSURBVBQlZaiQURRFUVKGChlFURQlZTT5fTIiUklgw2dD6AR85WN1cgVtd9NC2920iKfdxxpjOsdTWJMXMskgImVNMXaNtrtpoe1uWvjdbp0uUxRFUVKGChlFURQlZaiQSY5HMl2BDKHtblpou5sWvrZb12QURVGUlKGajKIoipIyVMg0ABEZIyLrRaRcRCZnuj7JIiJTRWSHiKxypHUQkTkissH6395KFxF50Gr7ChE51XHNBCv/BhGZ4HWvbEJEeonIOyKyRkRWi8hNVnqjbruINBORhSKy3Gr3b630PiKywGrfMyJSZKUXW+fl1ucljrJutdLXi8jozLQoMUQkX0SWishr1nlTafdmEVkpIstEpMxKS/2zbozRvwT+gHzgU6AvUAQsBwZlul5Jtulc4FRglSPtj8Bk63gycI91/DUCweUEGA4ssNI7ABut/+2t4/aZbluMdncHTrWOWwOfEAiC16jbbtW/lXVcCCyw2vMsMN5K/zvwY+v4euDv1vF44BnreJD1/BcDfaz3Ij/T7Yuj/bcA/wZes86bSrs3A51caSl/1lWTSZxhQLkxZqMxpgqYDozLcJ2SwhgzD9jlSh4HTLOOpwHfdKQ/ZQJ8DLQTke7AaGCOMWaXMWY3MAcYk/raNxxjzDZjzBLreD+wFuhBI2+7Vf8D1mmh9WeAC4HnrXR3u+3v43lgpBUyfRww3Rhz1BizCSgn8H5kLSLSE7gEeMw6F5pAu6OQ8mddhUzi9AA+d5xvtdIaG12NMdus4y+BrtZxpPbn9PdiTYWcQmBU3+jbbk0ZLQN2EOgoPgX2GGNqrCzONgTbZ32+F+hIDrYb+D/gF0Cddd6RptFuCAwkZovIYhGZZKWl/FkvSLbWSuPHGGNEpNGaIYpIK+AF4GZjzL7AYDVAY227MaYWOFlE2gEvAQMyXKWUIyJfB3YYYxaLyPmZrk8GONsYUyEiXYA5IrLO+WGqnnXVZBKnAujlOO9ppTU2tlvqMdb/HVZ6pPbn5PciIoUEBMzTxpgXreQm0XYAY8we4B1gBIEpEXvg6WxDsH3W522BneReu88CLhWRzQSmuS8E/kzjbzcAxpgK6/8OAgOLYaThWVchkziLgP6WRUoRgQXBGRmuUyqYAdiWIxOAVxzpV1vWJ8OBvZa6PQu4WETaWxYqF1tpWYs1v/44sNYYc7/jo0bddhHpbGkwiEhz4CIC61HvAJdZ2dzttr+Py4C5JrAKPAMYb1lh9QH6AwvT04rEMcbcaozpaYwpIfDezjXGfJ9G3m4AEWkpIq3tYwLP6CrS8axn2uIhF/8IWF58QmAe+7ZM18eH9vwH2AZUE5hjvZbA3PPbwAbgLaCDlVeAv1ltXwmUOsr5IYFF0HLgmky3K452n01gnnoFsMz6+1pjbztwErDUavcq4HYrvS+BzrIceA4ottKbWefl1ud9HWXdZn0f64GxmW5bAt/B+dRblzX6dlttXG79rbb7rXQ867rjX1EURUkZOl2mKIqipAwVMoqiKErKUCGjKIqipAwVMoqiKErKUCGjKIqipAwVMoqiKErKUCGjKIqipAwVMoqiKErK+P/JG1/YbP7vYAAAAABJRU5ErkJggg==\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "result_indep_ma = ma(result_indep)\n",
    "print(result_indep_ma[-1])\n",
    "plt.plot(range(len(result_indep_ma)),result_indep_ma)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 88,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0.7139768\n",
      "0.6316654\n",
      "0.7826391\n",
      "0.7373528\n",
      "0.71534747\n",
      "0.33773124\n",
      "0.49103215\n",
      "0.40218806\n",
      "0.4783399\n",
      "0.44352362\n",
      "0.2582965\n",
      "0.40100932\n",
      "0.21344906\n",
      "0.3392538\n",
      "0.3488893\n",
      "0.27446285\n",
      "0.1057293\n",
      "0.14917296\n",
      "0.2677378\n",
      "0.18660794\n",
      "0.097245224\n",
      "0.14853635\n",
      "0.21353453\n",
      "0.051186815\n",
      "0.10429295\n",
      "0.11462825\n",
      "-0.004557647\n",
      "0.055703018\n",
      "0.15233307\n",
      "0.05741871\n",
      "0.003936209\n",
      "-0.051913593\n",
      "0.026658354\n",
      "0.045222223\n",
      "0.030030468\n",
      "4.7496324e-05\n",
      "0.0023006122\n",
      "0.02391725\n",
      "0.03362205\n",
      "0.036872778\n",
      "0.006231467\n",
      "0.0013570078\n",
      "0.006279491\n",
      "-0.0018430948\n",
      "0.019196901\n",
      "-1.20903715e-05\n",
      "4.4165023e-05\n",
      "4.2699103e-06\n",
      "-3.7585269e-06\n",
      "-0.00031646813\n",
      "-0.0008236333\n",
      "8.823799e-06\n",
      "2.8283917e-05\n",
      "-9.653637e-05\n",
      "2.5724818e-05\n",
      "-6.722513e-05\n",
      "0.0014087242\n",
      "0.01146451\n",
      "0.013907462\n",
      "0.007841862\n",
      "0.0912253\n",
      "0.122383095\n",
      "0.056839854\n",
      "0.10575047\n",
      "0.0782597\n",
      "0.03139615\n",
      "0.22487295\n",
      "0.032391477\n",
      "0.058252454\n",
      "0.16545579\n",
      "0.1887874\n",
      "0.10880328\n",
      "0.21021816\n",
      "0.23530854\n",
      "0.08429229\n",
      "0.3004536\n",
      "0.21730933\n",
      "0.15396884\n",
      "0.20488712\n",
      "0.23025325\n",
      "0.60854334\n",
      "0.40914047\n",
      "0.3779331\n",
      "0.31298184\n",
      "0.47951263\n",
      "0.26784778\n",
      "0.655895\n",
      "0.39311054\n",
      "0.43213665\n",
      "0.68665373\n",
      "0.2409392\n",
      "0.5520464\n",
      "0.963186\n",
      "0.8936923\n",
      "0.61944497\n"
     ]
    }
   ],
   "source": [
    "MINE = []\n",
    "LinReg = []\n",
    "GT = []\n",
    "COV = np.linspace(-0.9, 0.9, 19)\n",
    "for cov in COV:\n",
    "    x = np.transpose(np.random.multivariate_normal( mean=[0,0],\n",
    "                                  cov=[[1,cov],[cov,1]],\n",
    "                                 size = 100))\n",
    "    DE = DC.computeEnt(x, linReg, MSEscorer, varEntropy, CVFold)\n",
    "    MI = DE[1,0] + DE[0,0] - DE[0,1] - DE[1,1]\n",
    "    MI = MI/2\n",
    "    LinReg.append(MI)\n",
    "    #plt.scatter(cov, MI, c='g',label='KNN-regressor')\n",
    "    groundTruth = -0.5*np.log(1-cov*cov)\n",
    "    GT.append(groundTruth)\n",
    "    #plt.scatter(cov, groundTruth, c='r',label='ground truth')\n",
    "    \n",
    "    #MINE\n",
    "    mine_net = Mine()\n",
    "    mine_net_optim = optim.Adam(mine_net.parameters(), lr=1e-3)\n",
    "    result = train(np.transpose(x),mine_net,mine_net_optim)\n",
    "    result_ma = ma(result)\n",
    "    MINE.append(result_ma[-1])\n",
    "    #MINE\n",
    "\n",
    "#plt.legend()\n",
    "#plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 92,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAX0AAAD8CAYAAACb4nSYAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDMuMC4zLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvnQurowAAHctJREFUeJzt3X2QVPW95/H3l4ERRwwoM5vkMkw3eM3eqIOoIz4kMQloNCaCk5t1wbaWhHhn1fi0uVKrNVXxYWvqxpisuhXivb2WS262o3iNukNFi1UQrTxgGA1CwJggYXCIKzgKiRl59Lt/nJ6xZ+hmeqZPP835vKqo7v71r7u/nO7+9JnfOed3zN0REZFoGFfuAkREpHQU+iIiEaLQFxGJEIW+iEiEKPRFRCJEoS8iEiEKfRGRCFHoi4hEiEJfRCRCxpfrhevr6z0ej5fr5UVEqtJLL730trs3jPbxZQv9eDxOV1dXuV5eRKQqmVl3IY/X8I6ISIQo9EVEIkShLyISIQp9EZEIUeiLiESIQl9EJEIU+iIiEaLQFxGJEIW+iEgRpVIQj8O4ccFlKlXeeqo29CttQYqIDJVKQVsbdHeDe3DZ1lbevKrK0E+l4Ov3puhujePfHkd3a5yv35tS8ItIRWlvh76TUnBzHG4fBzfH6TspRXt7+WqqytC/6cEUBy9ugyndYA5Tujl4cRs3PajUF5HK0f2RFFw2OKu4rC1oL5OqDP3e2e1Q2ze4sbYvaBcRqRA1F2fPqpqLy5dVVRn6TN4xsnYRkTI4PCl7JuVqL4WqDP2pE5pG1C4iUg6xydkzKVd7KVRl6N8/v4NaqxvUVmt13D+/o0wViYgcqWNeB3UTBmdV3YQ6OuaVL6vyCn0zu8TMXjOzrWZ2a5b7m8zsOTP7jZltNLNLwy/1Q4nmBA+1JolNjmEYsckxHmpNkmhOFPNlRURGJNGcIHnZ4KxKXlberDJ3P3oHsxrg98BFQA+wHljk7lsy+iSB37j7A2Z2CvCUu8eP9rwtLS2uM2eJiIyMmb3k7i2jfXw+a/pzgK3uvs3dDwCPAAuG9HHgI+nrk4E/jbYgEREpnnzOkTsNeCPjdg9wzpA+dwD/18xuAI4DLgylOhERCVVYG3IXAcvdvRG4FPixmR3x3GbWZmZdZta1e/fukF5aRETylU/o7wSmZ9xuTLdl+gbwKIC7/wqYCNQPfSJ3T7p7i7u3NDQ0jK5iEREZtXxCfz1wspnNMLNaYCHQOaTPDmAegJl9kiD0tSovIlJhhg19dz8EXA+sAl4FHnX3zWZ2l5nNT3f7R+AfzOwV4GHgaz7cbkEiIlJy+WzIxd2fAp4a0vbtjOtbgE+FW5qIiIStKo/IFRGR0VHoi4hEiEJfRCRCFPoiIhGi0BcRiRCFvohIhCj0RUQiRKEvIhIhCn0RkQhR6IuIRIhCX0QkQhT6IiIRotAXEYkQhb6ISIQo9EVEIkShLyISIQp9EZEIUeiLiESIQl9EJEIU+iIiEaLQFxGJEIW+iEiEKPRFRCJEoS8iEiEKfRGRCFHoi4hEiEJfRCRCFPoiIhGi0BcRiRCFvohIhCj0RUQiRKEvIhIhCn0RkQhR6IuIRIhCX0QkQhT6IiIRotAXEYmQvELfzC4xs9fMbKuZ3ZqjzxVmtsXMNpvZT8ItU0REwjB+uA5mVgMsAy4CeoD1Ztbp7lsy+pwM3AZ8yt3fNbN/V6yCRURk9PJZ058DbHX3be5+AHgEWDCkzz8Ay9z9XQB33xVumSIiEoZ8Qn8a8EbG7Z50W6ZPAJ8ws1+Y2TozuyTbE5lZm5l1mVnX7t27R1exiIiMWlgbcscDJwOfAxYB/9PMpgzt5O5Jd29x95aGhoaQXlpERPKVT+jvBKZn3G5Mt2XqATrd/aC7/xH4PcGPgIiIVJB8Qn89cLKZzTCzWmAh0Dmkz5MEa/mYWT3BcM+2EOsUEZEQDBv67n4IuB5YBbwKPOrum83sLjObn+62Cug1sy3Ac8BSd+8tVtEiIjI65u5leeGWlhbv6uoqy2uLiFQrM3vJ3VtG+3gdkSsiEiEKfRGRCFHoi4hEiEJfRCRCFPoiIhGi0BcRySGVgngcxo0LLlOpcldUuGFn2RQRiaJUCtraoK8vuN3dHdwGSCTKV1ehtKYvIpJFe/uHgd+vry9or2YKfRGRLHbsGFl7tVDoi4hk0dQ0svZqodAXEcmiowPq6ga31dUF7dVMoS8ikkUiAckkxGJgFlwmk9W9ERciHvqpTSni98UZd+c44vfFSW0aA/tjiUh4ZqXg5jjcPi64nFX9GRHZXTZTm1K0rWyj72Cweb57bzdtK4P9sRLNVf5TLiIFG6sZEdk1/fbV7QNvZr++g320r67y/bFEJBRjNSMiG/o79mbf7ypXu4hEy1jNiMiG/onjs+93latdRKKlaXL2LMjVXi0iG/o82wEHhuyPdaAuaBeRyOuY10HdhMEZUTehjo551Z0RkQ39d55PwMok7ImBW3C5Mhm0i0jkJZoTJC9LEpscwzBik2MkL0tW9UZciPA5cuPxYAKloWIx2L691NWIiOSn0HPkRnaXzY6OwTPowdg42k5kpA4ePEhPTw/79u0rdymSYeLEiTQ2NjJhwoRQnzeyod9/VF17ezCBUlNTEPjVfrSdyEj19PRw/PHHE4/HMbNylyOAu9Pb20tPTw8zZswI9bkjG/oQBLxCXqJu3759CvwKY2ZMnTqV3bt3h/7ckd2QKyIfUuBXnmK9Jwp9ESk7M+Oqq64auH3o0CEaGhr48pe/DMDy5cu5/vrrAbjjjjuoq6tj165dA/0nTZo0cL2mpobZs2cP/PvOd75Tov9FdYj08I6IVIbjjjuO3/72t7z//vsce+yxPPPMM0ybNi1n//r6er7//e9z9913H3Hfsccey4YNG4pZblXTmr6IjEixThZ+6aWX8rOf/QyAhx9+mEWLFuXsu2TJElasWME777wTzotHiEJfRPLWf7Lw7m5w//Bk4WEE/8KFC3nkkUfYt28fGzdu5JxzzsnZd9KkSSxZsoT777//iPvef//9QcM7K1asKLy4MUTDOyKSt6OdLLzQPeFmzZrF9u3befjhh7n00kuH7X/jjTcye/ZsbrnllkHtGt45OoW+iOSt2CcLnz9/Prfccgtr166lt7f3qH2nTJnClVdeybJly8J58YhQ6ItI3pqask9fEtbJwpcsWcKUKVNobm5m7dq1w/b/1re+xdlnn82hQ4fCKSACNKYvInkr9snCGxsbufHGG/PuX19fT2trK/v37x9oGzqmf+utt4ZT3BgR2QnXRCTw6quv8slPfjLv/qmUpi8plWzvjSZcE5GS0vQl1U3DOyIiEaLQFxGJEIW+iEiE5BX6ZnaJmb1mZlvNLOemcDP7ezNzMxv1RgYRESmeYUPfzGqAZcAXgVOARWZ2SpZ+xwM3AS+GXaSIiIQjnzX9OcBWd9/m7geAR4AFWfr9N+BuQOdcE5EReeutt7jyyiuZOXMmZ511Fueddx5PPPFEyeuIx+O8/fbbg9rOOeccZs+eTVNTEw0NDQP7/28fwcm016xZw7p16wZuX3XVVTz55JNhlT0i+eyyOQ14I+N2DzBoJiQzOxOY7u4/M7OluZ7IzNqANoCmsA7hE5Gq5u5cfvnlLF68mJ/85CcAdHd309nZeUTfQ4cOMX58afc0f/HFYPBi+fLldHV18YMf/CBrv8OHD1NTU5P1vjVr1lBfX8+5555btDrzVfCGXDMbB/x34B+H6+vuSXdvcfeWhoaGQl9aRMogtSlF/L444+4cR/y+OKlNhU2xuWbNGmpra7nmmmsG2mKxGDfccAMQhO38+fOZO3cu8+bNw91ZunQpp512Gs3NzQOzaK5du3bgpCsA119/PcuXLweCNfjbb7+dM888k+bmZn73u98B0Nvbyxe+8AVOPfVUrr76akZysOqhQ4eYMmUKN998M7NmzeLXv/41jY2N7NmzB4B169Zx4YUX8vrrr/Pggw9yzz33MHv2bH75y18C8Nxzz3H++eczc+bMkv5Vk0/o7wSmZ9xuTLf1Ox44DVhrZtuBc4FObcwVGXtSm1K0rWyje283jtO9t5u2lW0FBf/mzZs588wzj9rn5Zdf5rHHHuP555/n8ccfZ8OGDbzyyis8++yzLF26lDfffHPY16mvr+fll1/m2muv5Xvf+x4Ad955J5/+9KfZvHkzra2t7BjhzHF79+7lggsuYOPGjZx33nlZ+5x00klcffXVLF26lA0bNnD++ecDsGvXLn7xi1/w5JNPctttt43odQuRT+ivB042sxlmVgssBAb+7nL3ve5e7+5xd48D64D57q45FkTGmPbV7fQdHDy3ct/BPtpXt4f2Gt/85jc5/fTTOfvsswfaLrroIk488UQAfv7zn7No0SJqamr46Ec/ymc/+1nWr18/7PN+5StfAeCss84aGI9/4YUXBk7T+KUvfYkTTjhhRLXW1tbS2to6osf0u/zyyzEzZs2axc6dO4d/QEiGDX13PwRcD6wCXgUedffNZnaXmc0vdoEiUjl27M2+JpyrPR+nnnoqL7/88sDtZcuWsXr1anbv3j3Qdtxxxw37POPHj+eDDz4YuL1v3+B9So455hggOIduWLNyHnvssYNOYJ5Zw9DXH6q/HmBEw0qFymtM392fcvdPuPtJ7t6Rbvu2ux+xpcXdP6e1fJGxqWly9h0wcrXnY+7cuezbt48HHnhgoK1v6JlaMnzmM59hxYoVHD58mN27d/PCCy8wZ84cYrEYW7ZsYf/+/ezZs4fVq1cP+9oXXHDBwMbjp59+mnfffXfU/w8Ith289NJLAPz0pz8daD/++OP5y1/+UtBzh0VH5IpI3jrmdVA3YfDcynUT6uiYN/q5lc2MJ598kueff54ZM2YwZ84cFi9enPWk5wCtra3MmjWL008/nblz5/Ld736Xj33sY0yfPp0rrriC0047jSuuuIIzzjhj2Ne+/fbbeeGFFzj11FN5/PHHC96r8I477uC6667j7LPPpra2dqB9wYIFPProo5xxxhkDG3LLRVMri0TciKdW3pSifXU7O/buoGlyEx3zOkg0a9rNYtDUyiJSdonmhEK+iml4R0QkQhT6IiIRotAXkZLuMij5KdZ7otAvQNiHo4uUw8SJE+nt7R2Twd/b18vGtzbS9acuNr61kd6+3nKXlBd3p7e3l4kTJ4b+3NqQO0r9h6P3H53Yfzg6oI1cUlUaGxvp6ekZdDDUWPDXA3+l9/3BP2Zv2ptMPXYqx9UOf7BXuU2cOJHGxsbQn1e7bI5S/L443Xu7j2iPTY6x/ebtpS9IRAYZq9/RQnfZ1PDOKHXnOOw8V7uIlFYxpowYCxT6o1TzXvYj93K1i0hpFWPKiLFAoT9Kh1d1wIHBh6NzoC5oF5Gy65jXQa0N/o7WWmFTRowFCv1Riv05ASuTsCcGbsHlymTQLiLltzGBdw7+jnpnEjZG+zuqDbmjlEpBWxtkTgZYVwfJJCSi/ZkSqQjxOHQfuR2XWAxGcHrbiqMNuWWSSAQBH4uBWXCpwBepHLlOgjXCk2ONOdpPvwCJhEJepFI1NWVf0y9w9uSqpzV9ERmTOjqCIddMdXVBe5Qp9EVkTNIQbHYa3hGRMUtDsEfSmr6ISIQo9EVEIkShLyISIQp9EZEIUeiLiESIQl9EJEIU+iIiEaLQFxGJEIW+iEiEKPRFRCJEoS8iEiEKfRGRCFHoi4hEiEK/zFKbUsTvizPuznHE74uT2pQqd0kiFUHfjeLQ1MpllNqUom1lG30HgxPtdu/tpm1lGwCJZs0HK9Gl70bxaE2/jNpXtw98qPv1HeyjfXV7mSoSqQz6bhSPQr+MduzNfobmXO0iUaHvRvHkFfpmdomZvWZmW83s1iz3f8vMtpjZRjNbbWax8Esde5omZz9Dc652kag4cXz270CudsnfsKFvZjXAMuCLwCnAIjM7ZUi33wAt7j4LeAz4btiFjkUd8zqomzD4zM11E+romBfxMzeLPNsBB4ac1fxAXdAuBclnTX8OsNXdt7n7AeARYEFmB3d/zt37B+DWAY3hljk2JZoTLD4hSc17MXCj5r0Yi09IakOVRN47zydgZRL2BN8N9sRgZTJol4Lks/fONOCNjNs9wDlH6f8N4OlCioqKVAp+dEuCw33BB/kw8KM6+NRHdDJnibamJujelIBNg78ITRo4LlioG3LN7CqgBbgnx/1tZtZlZl27d+8O86WrUns79A3eQYG+vqBdJMo6OqBuyOhOXV3QLoXJJ/R3AtMzbjem2wYxswuBdmC+u+/P9kTunnT3FndvaWhoGE29Y8qOHDsi5GoXiYpEApJJiMXALLhMJvUXcBjyGd5ZD5xsZjMIwn4hcGVmBzM7A/gX4BJ33xV6lWNUUxN0d2dvF4m6REIhXwzDrum7+yHgemAV8CrwqLtvNrO7zGx+uts9wCTg38xsg5l1Fq3iMUR/wopIqeU1DYO7PwU8NaTt2xnXLwy5rkjoX4tpbw+GdJqagsDX2o2IFIvm3ikz/QkrIqWkaRhERCJEoS8iEiEKfRGRCFHoi4hEiEJ/DNAZhqTS6DNZubT3TpXTGYak0ugzWdm0pl/ldIYhqTT6TFY2hX6V685xJqFc7SLFprNeVTaFfpWreS/7RD252kWKTWe9qmwK/Sp3eFX2MwwdXqUJfKRMdNariqbQr3KxP2c/w1Dsz9pgJuWhs15VNu29U+U6OqCtLUFfxhmG6uqgI1nGoiTSdNaryqY1/Sqnk01IpdGU4ZVNa/pjgGbqlEqiKcMrm0JfREKnFZHKpeEdEZEIUeiLyBFSKYjHYdy44DKlqXPGDIW+aHIsGSSVgq/fm6K7NY5/exzdrXG+fm9KwT9GKPQjrn9yrO693Tg+MDmWgj+6bnowxcGL22BKN5jDlG4OXtzGTQ/qMzEWKPQjTpNjyVC9s9uhdvBngtq+oF2qnkI/4jRhmxxhco73Ple7VBWFfsRpwjYZauqE7O99rnapLgr9iNOEbTLU/fM7qLXBn4laq+P++fpMjAUK/YjThG0yVKI5wUOtSWKTYxhGbHKMh1qTOuvVGGHuXpYXbmlp8a6urrK8tnwolYK2NujL2G5XV6f5e0QqlZm95O4to3281vQjLqwJ23Qwj0h1UOgLiQRs3w4ffBBcjibwdTBP5dDBdnI0Gt6RgtV/PkXv+W2D9+0+UMfUXyZ5+zmNEZVSalOKJU+0ccA/fC9qrU5j8mOIhnek7HQwT+W4qbN9UOADHPA+burUeyEBhb4UTgfzVIzeg9mXea52iR6FvhRMB/NUkL05lnmudokchb4ULIyDebT3Tzimbsh+sN3UDTqwSgIKfSlYoQfzaO+fwQrZ++b+qxNMWDX4YLsJq5Lcf7U24kpAe+9I2Wnvnw+FsfdNKqXz045lhe69o9CXsrP/Eg/mbh9qTwy/d3upyymr+o44vYeOXBZTx8d4u3176QuSiqNdNqX6ae+fAdr7Rootr9A3s0vM7DUz22pmt2a5/xgzW5G+/0Uzi4ddqIxdYe39c90DKcYvjWN3jGP80jjXPVD6jQIFHw2rvW+kyIYNfTOrAZYBXwROARaZ2SlDun0DeNfd/xa4F7g77EJl7Apj75/rHkjxwM42Dk8KTvF3eFI3D+xsG1HwF/qj0T8en3nqySVPjOzUk9r7RootnzX9OcBWd9/m7geAR4AFQ/osAH6Uvv4YMM/MLLwyZSwLYyrf5LZ2mDDkqOAJfUF7HsL40QjjaFjtfSPFNj6PPtOANzJu9wDn5Orj7ofMbC8wFXg7jCJl7Es0JwqaG+bwcdnHvHO1D5Xc1g6Tsv9o/JD86uo9uAOyrOqMZDw+2MsmQXt7QnvfSFHkE/qhMbM2oA2gqUljlBKemr82BWvpWdrzUeiPBhCMu2fbC2mE4/GJhEJeiief4Z2dwPSM243ptqx9zGw8MBnoHfpE7p509xZ3b2loaBhdxSJZtM3sgINDxsIP1gXtecj145DvjwZoPF6qQz6hvx442cxmmFktsBDoHNKnE1icvv5VYI2X6wAAiaQfXpvg2mlJat4LxsJr3otx7bQkP7w2v1XmQn80QOPxUh3yOjjLzC4F7gNqgIfcvcPM7gK63L3TzCYCPwbOAN4BFrr7tqM9pw7Okkpz3QMpktvaOXzcDmr+2kTbzI68fzT66WhYKTYdkSsiEiE6IldERPKm0BcRiRCFvohIhCj0RUQiRKEvIhIhCn0RkQhR6IuIRIhCX0QkQsp2cJaZ7Qa6gXoqfzbOaqgRqqNO1RieaqizGmqE6qizv8aYu4968rKyhf5AAWZdhRxdVgrVUCNUR52qMTzVUGc11AjVUWdYNWp4R0QkQhT6IiIRUgmhnyx3AXmohhqhOupUjeGphjqroUaojjpDqbHsY/oiIlI6lbCmLyIiJVKS0Dez/2Bmm83sAzPLufXZzC4xs9fMbKuZ3ZrRPsPMXky3r0ifwSvsGk80s2fM7A/pyxOy9Pm8mW3I+LfPzC5P37fczP6Ycd/sctSY7nc4o47OjPaiL8d86zSz2Wb2q/TnYqOZ/ceM+4q2LHN9xjLuPya9bLaml1U8477b0u2vmdnFYdU0ihq/ZWZb0stttZnFMu7L+t6Xqc6vmdnujHquzrhvcfrz8QczWzz0sSWs8d6M+n5vZnsy7ivJsjSzh8xsl5n9Nsf9Zmb/I/1/2GhmZ2bcN/Ll6O5F/wd8Evj3wFqgJUefGuB1YCZQC7wCnJK+71GCs3EB/DNwbRFq/C5wa/r6rcDdw/Q/keAsYXXp28uBrxZ5OeZVI/BejvaiL8d86wQ+AZycvv43wJvAlGIuy6N9xjL6XAf8c/r6QmBF+vop6f7HADPSz1NTpho/n/G5u7a/xqO992Wq82vAD7I89kRgW/ryhPT1E8pR45D+NxCcGbDUy/IC4EzgtznuvxR4GjDgXODFQpZjSdb03f1Vd39tmG5zgK3uvs3dDwCPAAvMzIC5wGPpfj8CLi9CmQvSz53va3wVeNrd+4pQSy4jrXFACZcj5FGnu//e3f+Qvv4nYBcw6gNO8pT1MzakT2btjwHz0stuAfCIu+939z8CW9PPV/Ia3f25jM/dOqCxCHUMJ59lmcvFwDPu/o67vws8A1xSATUuAh4uQh1H5e4vEKxA5rIA+FcPrAOmmNnHGeVyrKQx/WnAGxm3e9JtU4E97n5oSHvYPurub6av/z/go8P0X8iRH5CO9J9f95rZMaFXmH+NE82sy8zW9Q8/UbrlOJI6ATCzOQRrYq9nNBdjWeb6jGXtk15WewmWXT6PLVWNmb5BsBbYL9t7Xwz51vn36ffxMTObPsLHlqpG0kNkM4A1Gc2lWpbDyfX/GNVyHB9WVWb2LPCxLHe1u/v/Cet1CnG0GjNvuLubWc7dmtK/ss3Aqozm2wgCrpZg16r/CtxVphpj7r7TzGYCa8xsE0F4hSbkZfljYLG7f5BuDmVZjnVmdhXQAnw2o/mI997dX8/+DEW3EnjY3feb2X8m+AtqbplqGc5C4DF3P5zRVknLMjShhb67X1jgU+wEpmfcbky39RL8OTM+vebV3x5qjWb2lpl93N3fTAfRrqM81RXAE+5+MOO5+9ds95vZ/wJuKVeN7r4zfbnNzNYCZwA/JaTlGFadZvYR4GcEKwbrMp47lGWZRa7PWLY+PWY2HphM8BnM57GlqhEzu5DgB/az7r6/vz3He1+MoBq2Tnfvzbj5IMG2nv7Hfm7IY9eGXuHI3rOFwDczG0q4LIeT6/8xquVYScM764GTLdjDpJbgTej0YIvFcwRj6ACLgWL85dCZfu58XuOIsb90uPWPnV8OZN0SX+wazeyE/uEQM6sHPgVsKeFyzLfOWuAJgrHKx4bcV6xlmfUzdpTavwqsSS+7TmChBXv3zABOBn4dUl0jqtHMzgD+BZjv7rsy2rO+90WoMd86P55xcz7wavr6KuAL6XpPAL7A4L+aS1Zjus6/I9gQ+quMtlIuy+F0Av8pvRfPucDe9IrR6JZjibZOtxKMN+0H3gJWpdv/BnhqyFbq3xP8mrZntM8k+IJtBf4NOKYINU4FVgN/AJ4FTky3twAPZvSLE/zCjhvy+DXAJoKA+t/ApHLUCJyfruOV9OU3SrkcR1DnVcBBYEPGv9nFXpbZPmMEQ0fz09cnppfN1vSympnx2Pb0414DvljE78twNT6b/h71L7fO4d77MtX5T8DmdD3PAX+X8dgl6WW8Ffh6uWpM374D+M6Qx5VsWRKsQL6Z/j70EGynuQa4Jn2/AcvS/4dNZOwBOZrlqCNyRUQipJKGd0REpMgU+iIiEaLQFxGJEIW+iEiEKPRFRCJEoS8iEiEKfRGRCFHoi4hEyP8H12BuwppPuH0AAAAASUVORK5CYII=\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "fig, ax = plt.subplots()\n",
    "\n",
    "ax.scatter(COV, MINE, c='b', label='MINE')\n",
    "#ax.scatter(COV, LinReg, c='r', label='Regressor')\n",
    "ax.scatter(COV, GT, c='g', label='Ground Truth')\n",
    "\n",
    "ax.legend()\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 91,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0.6838852\n",
      "0.91718066\n",
      "0.8484288\n",
      "0.78330123\n",
      "0.46787655\n",
      "2.4472227\n",
      "2.145537\n",
      "2.5620003\n",
      "1.7462051\n",
      "1.6097094\n",
      "3.2292209\n",
      "3.189553\n",
      "2.6548195\n",
      "3.2587643\n",
      "3.751107\n",
      "3.4347908\n",
      "3.327949\n",
      "3.7095993\n",
      "4.3756294\n",
      "4.9397445\n",
      "3.6102746\n",
      "4.189102\n",
      "4.8895807\n",
      "3.5474145\n",
      "4.2899437\n",
      "4.25937\n",
      "3.4531517\n",
      "4.8590817\n",
      "4.1522355\n",
      "3.581197\n",
      "3.686964\n",
      "4.629438\n",
      "8.012478\n",
      "4.9360285\n",
      "14.732037\n"
     ]
    }
   ],
   "source": [
    "MINE2 = []\n",
    "LinReg2 = []\n",
    "GT2 = []\n",
    "COV2 = []\n",
    "for i in range(1, 8):\n",
    "    cov = 1 - 0.1**i\n",
    "    COV2.append(cov)\n",
    "    x = np.transpose(np.random.multivariate_normal( mean=[0,0],\n",
    "                                  cov=[[1,cov],[cov,1]],\n",
    "                                 size = 10000))\n",
    "    DE = DC.computeEnt(x, linReg, MSEscorer, varEntropy, CVFold)\n",
    "    MI = DE[1,0] + DE[0,0] - DE[0,1] - DE[1,1]\n",
    "    MI = MI/2\n",
    "    LinReg2.append(MI)\n",
    "    #plt.scatter(cov, MI, c='g',label='KNN-regressor')\n",
    "    groundTruth = -0.5*np.log(1-cov*cov)\n",
    "    GT2.append(groundTruth)\n",
    "    #plt.scatter(cov, groundTruth, c='r',label='ground truth')\n",
    "    \n",
    "#     #MINE\n",
    "#     mine_net = Mine()\n",
    "#     mine_net_optim = optim.Adam(mine_net.parameters(), lr=1e-3)\n",
    "#     result = train(np.transpose(x),mine_net,mine_net_optim)\n",
    "#     result_ma = ma(result)\n",
    "#     MINE2.append(result_ma[-1])\n",
    "#     #MINE\n",
    "\n",
    "#plt.legend()\n",
    "#plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 108,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAX4AAAD8CAYAAABw1c+bAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDMuMC4zLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvnQurowAAG45JREFUeJzt3X98VPWd7/HXh5CIAQpCstoSk4Dr+qgYiBhw7SpaqD9KFcXtRTHeS0VvrqK1bK/eh7t5PMT6eOTxaLVudVfFzVWL7WNE3Kuy9oqtXJRSW1GBjYBiK3KNDXUBg1BpxAJ+9o85ickwwySZSWbOnPfz8ZhHzvme75zz+Tr4ycn3nPkcc3dERCQ6huQ6ABERGVxK/CIiEaPELyISMUr8IiIRo8QvIhIxSvwiIhGjxC8iEjFK/CIiEaPELyISMUNzHUAyZWVlXl1dneswRERCY8OGDR+6e3lv+uZl4q+urmb9+vW5DkNEJDTMrLW3fTXVIyISMUr8IiIRo8QvIhIxeTnHn8zBgwdpa2vjwIEDuQ5FAsOGDaOiooLi4uJchyIifRCaxN/W1sbIkSOprq7GzHIdTuS5O+3t7bS1tTF+/PhchyMifRCaqZ4DBw4wduxYJf08YWaMHTtWf4GJhFBoEj+gpJ9n9HlIlMQ2x6i+t5oh3xtC9b3VxDbHch1Sv4VmqkdEJFdim2M0/KyBjoMdALTua6XhZw0A1NfU5zK0fgnVGX+umRlXX3111/qhQ4coLy/n4osvBmDp0qXcdNNNANxxxx2Ulpaya9eurv4jRozoWi4qKqK2trbr9f3vf3+QRiEifdW4urEr6XfqONhB4+rGHEWUGZ3x98Hw4cPZsmULn3zyCcceeyyrVq1i3LhxKfuXlZVxzz338IMf/OCIbcceeywtLS0DGa6IZMn7+97vU3u+K9gz/lgMqqthyJD4z1iWpuNmzZrFc889B8CyZcuYN29eyr4LFixg+fLl7NmzJzsHF5GcqBxV2af2fFeQiT8Wg4YGaG0F9/jPhobsJP8rr7ySJ554ggMHDrBp0ybOPPPMlH1HjBjBggULuO+++47Y9sknn/SY6lm+fHnmwYnIgGia2URpcWmPttLiUppmNuUoosykneoxs0eBi4Fd7n5a0LYcOCXoMhrY6+61Sd77HvAxcBg45O51WYr7qBoboaPndBwdHfH2+gyvw0yaNIn33nuPZcuWMWvWrLT9b775Zmpra7nlllt6tGuqRyQ8Oi/gNq5u5P1971M5qpKmmU2hvLALvZvjXwrcD/yks8Hdr+hcNrN7gH1Hef9X3f3D/gbYH++nmHZL1d5Xs2fP5pZbbmHNmjW0t7cfte/o0aO56qqreOCBB7JzcBHJifqa+tAm+kRpE7+7rzWz6mTbLH4j91xgRnbDykxlZXx6J1l7NixYsIDRo0dTU1PDmjVr0vb/7ne/y9SpUzl06FB2AhARyUCmc/znADvd/Z0U2x14wcw2mFnD0XZkZg1mtt7M1u/evTujoJqaoLTndBylpfH2bKioqODmm2/udf+ysjLmzJnDp59+2tWWOMd/2223ZSc4EZE0zN3Td4qf8f/fzjn+bu1LgG3ufk+K941z9x1m9hfAKuDb7r423fHq6uo88UEsW7du5ctf/nLaWDvFYvE5/fffj5/pNzVlPr8vR+rr5yIiA8PMNvT2Omq/7+M3s6HA5cAZqfq4+47g5y4zewaYBqRN/NlQX69ELyKSTCZTPV8D3nb3tmQbzWy4mY3sXAYuALZkcDwREcmCtInfzJYBrwCnmFmbmV0bbLoSWJbQ90tmtjJYPR542czeAF4DnnP3n2cvdBGRwROpIm3unvSrqe7+rSRtfwBmBcvbgckZxiciknMq0iYiEjGFVqRNiV9EJA0VaYuwnTt3ctVVVzFhwgTOOOMMzjrrLJ555plBj6O6upoPP+z5ZegzzzyT2tpaKisrKS8v7/p+wHvvvdfr/b744ousW7eua/3qq69mxYoV2QpbJLRUpC2i3J3LLruM6dOns337djZs2MATTzxBW9uRNzXl4hu6r776Ki0tLdx5551cccUVtLS00NLSQnV1dY9+hw8fTrmPxMQvInGFVqStYBN/tq/Av/jii5SUlHD99dd3tVVVVfHtb38biD+EZfbs2cyYMYOZM2fi7tx6662cdtpp1NTUdFXfXLNmTdeDWwBuuukmli5dCsTP5BcvXsyUKVOoqanh7bffBqC9vZ0LLriAiRMnct1119GbL911OnToEKNHj2bRokVMmjSJ1157jYqKCvbu3QvAunXr+NrXvsa7777Lww8/zN13301tbS2/+c1vAHjppZf4yle+woQJE3Ly141IPqivqaf5kmaqRlVhGFWjqmi+pDmUF3ahQB/EMhBX4N98802mTJly1D4bN25k06ZNjBkzhqeeeoqWlhbeeOMNPvzwQ6ZOncr06dPTHqesrIyNGzfy4IMP8sMf/pCHH36Y733ve5x99tncfvvtPPfcczzyyCN9in3fvn1Mnz6de++9N2Wfk046ieuuu46ysjIWLVoEwIMPPsiuXbv49a9/zebNm5k7dy5z5szp07FFCkUhFWkryDP+wbgCf+ONNzJ58mSmTp3a1Xb++eczZswYAF5++WXmzZtHUVERxx9/POeeey6vv/562v1efvnlAJxxxhld8/Nr167teuTjN77xDY477rg+xVpSUtLvhH3ZZZdhZkyaNIkdO3b0ax8ikl8KMvEPxBX4iRMnsnHjxq71Bx54gNWrV9O9oNzw4cPT7mfo0KF89tlnXesHDhzosf2YY44B4s/kzda1gmOPPZZ4IdUjY0g8fqLOeIA+TTGJSP4qyMQ/EFfgZ8yYwYEDB1iyZElXW0fi0166Oeecc1i+fDmHDx9m9+7drF27lmnTplFVVcVbb73Fp59+yt69e1m9enXaY0+fPp3HH38cgOeff56PPvqo3+OA+LWEDRs2APDUU091tY8cOZKPP/44o32LSP4ryMQ/EFfgzYwVK1bwy1/+kvHjxzNt2jTmz5+f9EHqAHPmzGHSpElMnjyZGTNmcNddd3HCCSdw4oknMnfuXE477TTmzp3L6aefnvbYixcvZu3atUycOJGnn36aygwfLHDHHXewcOFCpk6dSklJSVf7pZdeypNPPsnpp5/edXFXRApPr8oyD7aslGXeHCuYx6TlM5VlFskPg1KWOd8V0hV4Ecm9QjqZLNjELyKSLSrSlkP5OC0VZfo8JCpUpC1Hhg0bRnt7u5JNnnB32tvbGTZsWK5DERlwhVakLTRTPRUVFbS1tZHpg9gle4YNG0ZFRUWuwxAZcJWjKmnd15q0PYxCk/iLi4sZP358rsMQkQhqmtnUY44fVKRNRKSgFVqRttDcxy8iIqn15T7+3jxs/VEz22VmW7q13WFmO8ysJXjNSvHei8zst2a2zcxu6/0QRERkoPRmqmcpcFGS9h+5e23wWpm40cyKgAeArwOnAvPM7NRMghURkcylTfzuvhbY0499TwO2uft2d/8z8ARwaT/2IyIiWZTJxd2bzGxTMBWUrED8OOD33dbbgjYREcmh/ib+JcBJQC3wAXBPpoGYWYOZrTez9bpXX0Rk4PQr8bv7Tnc/7O6fAf+b+LROoh3Aid3WK4K2VPtsdvc6d68rLy/vT1giIgMm28/xzqV+JX4z+2K31TnAliTdXgdONrPxZlYCXAk825/jiYjkUmeRttZ9rTjeVaQtrMm/N7dzLgNeAU4xszYzuxa4y8w2m9km4KvA3wV9v2RmKwHc/RBwE/ALYCvwpLu/OUDjEBEZMIVWpC1tyQZ3n5ek+ZEUff8AzOq2vhI44lZPEZEwKbQibSrZICKSxkA8xzuXlPhFRNIYiOd455ISv4hIGirSNghUpE1EpG+yWqRNREQKixK/iEjEKPGLiESMEr+ISMQo8YuIRIwSv4hILxRSkba0JRtERKKus0hbZ72eziJtQCjv5dcZv4hIGoVWpE2JX0QkDRVpExGJGBVpExGJGBVpExGJGBVpGwQq0iYi0jcq0iYiIikp8YuIRExvHrb+qJntMrMt3druNrO3zWyTmT1jZqNTvPe94KHsLWamuRsRkTzQmzP+pcBFCW2rgNPcfRLwO+Dvj/L+r7p7bW/nnkREZGClTfzuvhbYk9D2grsfClbXARUDEJuIiAyAbMzxLwCeT7HNgRfMbIOZNWThWCIiOaEibQEzawQOAan+C5zt7jvM7C+AVWb2dvAXRLJ9NQANAJWV4fw2nIgUJhVpC5jZt4CLgXpP8WUAd98R/NwFPANMS7U/d2929zp3rysvL+9vWCIiWacibYCZXQT8L2C2u3ek6DPczEZ2LgMXAFuS9RURyWeRK9JmZsuAV4BTzKzNzK4F7gdGEp++aTGzh4K+XzKzlcFbjwdeNrM3gNeA59z95wMyChGRAVRoRdrSzvG7+7wkzY+k6PsHYFawvB2YnFF0IiJ5oGlmU485flCRNhGRgqYibYNARdpERPpGRdpERCQlJX4RkYhR4hcRiRglfhGRiFHiFxGJGCV+EZFeUJE2EZEIUZE2EZGIUZE2EZGIiVyRNhGRqCu0Im1K/CIiaTTNbKK0uLRHm4q0iYgUMBVpGwQq0iYi0jcq0iYiIikp8YuIRIwSv4hIxCjxi4hEjBK/iEjE9Crxm9mjZrbLzLZ0axtjZqvM7J3g53Ep3js/6POOmc3PVuAiIoOpkIq09faMfylwUULbbcBqdz8ZWB2s92BmY4DFwJnANGBxql8QIiL5qrNIW+u+VhzvKtIW1uTfq8Tv7muBPQnNlwKPBcuPAZcleeuFwCp33+PuHwGrOPIXiIhIXlORts8d7+4fBMv/ARyfpM844Pfd1tuCNhGR0FCRtiQ8/vXfjL4CbGYNZrbezNbv3r07G2GJiGSFirR9bqeZfREg+LkrSZ8dwInd1iuCtiO4e7O717l7XXl5eQZhiYhkl4q0fe5ZoPMunfnAvyXp8wvgAjM7Lrioe0HQJiISGpEs0mZmy4DzgDJgJ/E7dVYATwKVQCsw1933mFkdcL27Xxe8dwHwD8Gumtz9x+mOpyJtIiJ905cibarOKSJSAFSdU0REUlLiFxGJGCV+EZGIUeIXEYkYJX4RkV4opCJtQ3MdgIhIvuss0tZZr6ezSBsQynv5dcYvIpKGirSJiESMirSJiESMirSJiESMirSJiERMJIu0DTbV6hER6RvV6hERkZSU+EVEIkaJX0QkYpT4RUQiRolfRCRilPhFRCJGiV9EpBcKqTpnvxO/mZ1iZi3dXn80s0UJfc4zs33d+tyeecgiIoOrszpn675WHO+qzhnW5N/vsszu/lugFsDMioAdwDNJuv7K3S/u73FERHLtaNU5w/jt3WxN9cwE3nX31iztT0Qkb6g6Z3JXAstSbDvLzN4ws+fNbGKWjiciMmhUnTOBmZUAs4F/TbJ5I1Dl7pOBfwZWHGU/DWa23szW7969O9OwRESyRtU5j/R1YKO770zc4O5/dPf9wfJKoNjMypLtxN2b3b3O3evKy8uzEJaISHYUWnXObDxzdx4ppnnM7ARgp7u7mU0j/oumPQvHFBEZVPU19aFN9IkySvxmNhw4H/gf3dquB3D3h4BvAjeY2SHgE+BKz8c60CIiEZJR4nf3PwFjE9oe6rZ8P3B/JscQEZHs0jd3RUQiRolfRCRilPhFRHohFoPqahgyJP4zFs5qDYASv4hIWrEYXPOjGK1zqvHbh9A6p5prfhQLbfJX4hcRSeM7D8c4eGEDjG4FcxjdysELG/jOw+HM/Er8IiJptNc2QknPIm2UdMTbQ0iJX0QknVEpirGlas9zSvwiImmMLU5ejC1Ve75T4hcRSeO+2U2UWM8ibSVWyn2zo1ukTUSkoNXX1HNteTNF+6vAjaL9VVxbHu0ibSIiBS0Wg8duqedwRzzRHwYeK4W/+QLUhzD364xfRCSNxkboSLipp6Mj3h5GSvwiImm0pniobKr2fKfELyISMUr8IiIRo8QvIhIxSvwiIr1RE4NF1bB4SPxnTTjr9IBu5xQRSa8mBpc0fF6vZ3RrfB2A8N3PqTN+EZF0ZiYv0sbMcN7PqcQvIpKOirT1ZGbvmdlmM2sxs/VJtpuZ/ZOZbTOzTWY2JdNjiogMqn0pirGlas9z2Trj/6q717p7XZJtXwdODl4NwJIsHVNEZHCsboI/9yzSxp9L4+0hNBhTPZcCP/G4dcBoM/viIBxXRCQ7NtfDz5phb7xIG3ur4uubw3dhF7JzV48DL5iZA//i7s0J28cBv++23ha0fdC9k5k1EP+LgMrKcP75JCIFbHN9aBN9omyc8Z/t7lOIT+ncaGbT+7MTd2929zp3rysvL89CWCIi2TF8eN/a813Gid/ddwQ/dwHPANMSuuwATuy2XhG0iYiEwrBhfWvPdxklfjMbbmYjO5eBC4AtCd2eBf5bcHfPXwP73P0DRERCYs+evrXnu0zn+I8HnjGzzn097u4/N7PrAdz9IWAlMAvYBnQA12R4TBGRQVVZmbwEc1gvR2aU+N19OzA5SftD3ZYduDGT44iI5FJTEzQ09HwYS2lpvD2MVKtHRCSN+nr48YYYq70x/m3dfZWcZU3Uh/G5i6hkg4hIWguXxFhd2hAvzmYOo1tZXdrAwiXhrNCpxC8ikkbz9kYoTijSVtwRbw8hJX4RkTQOD09ejC1Ve75T4hcRSaPoT8lv30nVnu+U+EVE0miY0AQHE4q0HSyNt4eQEr+ISBoP3lDPDeOaKdofL9JWtL+KG8Y18+AN4byrx+K32eeXuro6X7/+iNL+IiKSgpltSFEa/wg64xcRiRglfhGRiFHiFxGJGCV+EZEUYjGoroYhQ+I/Y+H8ou4RVKtHRCSJWKxnYbbW1vg6xGv3hJnO+EVEkmhshI6TYrCoGhYPgUXVdJwUozGcVRp6UOIXEUmi9QsxuKRnYTYuaYi3h5wSv4hIEkUXNkJJQmG2ko54e8gp8YuIJHF4RIrCbCnaw0SJX0QkiapRyQuwpWoPEyV+EZEkmmY2UVrcszBbaXEpTTPDWZitu34nfjM70cxeMrO3zOxNM/tOkj7nmdk+M2sJXrdnFq6IyOCor6mn+ZJmqkZVYRhVo6povqSZ+pqQ38tJZvfxHwL+p7tvNLORwAYzW+XubyX0+5W7X5zBcUREcqK+pr4gEn2ifp/xu/sH7r4xWP4Y2AqMy1ZgIiIyMLIyx29m1cDpwKtJNp9lZm+Y2fNmNjEbxxMRkf7LuGSDmY0AngIWufsfEzZvBKrcfb+ZzQJWACen2E8D0ABQWRn+q+YiIvkqozN+MysmnvRj7v504nZ3/6O77w+WVwLFZlaWbF/u3uzude5eV15enklYIiJyFJnc1WPAI8BWd//HFH1OCPphZtOC47X395giIpK5TKZ6/gb4r8BmM2sJ2v4BqARw94eAbwI3mNkh4BPgSs/HZz2KSGgtXBKjeXsjh4e/T9GfKmmY0BTaZ+EOFj1zV0RCa+GSGEt2NEBxt5o6B0tD/SD0/tIzd0UkEpq3N/ZM+gDFHfF2SUmJX0RC6/DwFIXUUrRLnBK/iIRW0Z+S3/qdql3ilPhFJLQaJjTBwZ6F1DhYGm+XlJT4RSS0HryhnhvGNVO0vwrcKNpfFckLu32lu3pERAqA7uoREZGUlPhFRCJGiV9EJGKU+EVEIkaJX0QkYpT4e2HhkhhDb63G7hjC0FurWbgkluuQRET6TYk/jc4iUIdHtII5h0e0smRHg5K/iISWEn8aKgIlIoVGiT8NFYESkUKjxJ+GikCJSKFR4k9DRaBEpNAo8aehIlAiUmhUpE1EpACoSJuIiKSUUeI3s4vM7Ldmts3Mbkuy/RgzWx5sf9XMqjM5noiIZK7fid/MioAHgK8DpwLzzOzUhG7XAh+5+18CPwJ+0N/jiYhIdmRyxj8N2Obu2939z8ATwKUJfS4FHguW/w8w08wsg2OKiEiGMkn844Dfd1tvC9qS9nH3Q8A+YGyynZlZg5mtN7P1u3fvziAsERE5mry5uOvuze5e5+515eXluQ5HRKRgDc3gvTuAE7utVwRtyfq0mdlQYBTQnm7HGzZs+NDMWjOIbaCUAR/mOogsKqTxFNJYoLDGU0hjgfwdT1VvO2aS+F8HTjaz8cQT/JXAVQl9ngXmA68A3wRe9F58ccDd8/KU38zW9/Y+2TAopPEU0ligsMZTSGOBwhhPvxO/ux8ys5uAXwBFwKPu/qaZ3Qmsd/dngUeAn5rZNmAP8V8OIiKSQ5mc8ePuK4GVCW23d1s+APyXTI4hIiLZlTcXd0OiOdcBZFkhjaeQxgKFNZ5CGgsUwHjyslaPiIgMHJ3xi4hEjBI/vao5VGVmq81sk5mtMbOKbtvmm9k7wWv+4EaeXH/HY2a1ZvaKmb0ZbLti8KM/UiafT7D9C2bWZmb3D17UyWX4b63SzF4ws61m9lY+1L7KcDx3Bf/WtprZP+X6W/1m9qiZ7TKzLSm2WxDntmA8U7pty7s8cFTuHukX8TuS3gUmACXAG8CpCX3+FZgfLM8AfhosjwG2Bz+PC5aPC/F4/go4OVj+EvABMDqs4+m2/T7gceD+MI8FWAOcHyyPAErDOh7gK8Cvg30UEb/l+7wcj2c6MAXYkmL7LOB5wIC/Bl4N2vMuD6R76Yy/dzWHTgVeDJZf6rb9QmCVu+9x94+AVcBFgxDz0fR7PO7+O3d/J1j+A7ALyPV3KjL5fDCzM4DjgRcGIdZ0+j2WoADiUHdfBeDu+929Y3DCTimTz8aBYcR/YRwDFAM7Bzzio3D3tcRvO0/lUuAnHrcOGG1mXyQ/88BRKfH3rubQG8DlwfIcYKSZje3lewdbJuPpYmbTiP9P+e4Axdlb/R6PmQ0B7gFuGfAoeyeTz+avgL1m9rSZ/buZ3R1UyM2lfo/H3V8h/ovgg+D1C3ffOsDxZirVePMxDxyVEn/v3AKca2b/DpxL/JvKh3MbUkaOOp7gLOanwDXu/lluQuyTVONZCKx097ZcBtdHqcYyFDgn2D6V+PTKt3IUY18kHY+Z/SXwZeKlXsYBM8zsnNyFGS0ZfYGrQKStORRMe1wOYGYjgL91971mtgM4L+G9awYy2F7o93iC9S8AzwGNwZ+zuZbJ53MWcI6ZLSQ+J15iZvvd/YiLkIMkk7G0AS3uvj3YtoL4PPMjgxF4CpmM578D69x9f7DteeAs4FeDEXg/pRpvPuaBo8v1RYZcv4j/8tsOjOfzC1QTE/qUAUOC5SbgTv/8os7/J35B57hgeUyIx1MCrAYW5fpzycZ4Evp8i9xf3M3ksykK+pcH6z8GbgzxeK4A/l+wj+Lg390lefDvrZrUF3e/Qc+Lu68F7XmXB9KOM9cB5MOL+NX63xGfz24M2u4EZgfL3wTeCfo8DBzT7b0LgG3B65pcjyWT8QBXAweBlm6v2rCOJ2EfOU/8Wfi3dj6wCdgMLAVKwjoe4r/I/gXYCrwF/GMejGUZ8esNB4nP018LXA9cH2w34k8dfDf4DOq6vTfv8sDRXvrmrohIxOjirohIxCjxi4hEjBK/iEjEKPGLiESMEr+ISMQo8YuIRIwSv4hIxCjxi4hEzH8CQqPOGv2xNnoAAAAASUVORK5CYII=\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "fig, ax = plt.subplots()\n",
    "\n",
    "ax.scatter(COV2, MINE2, c='b', label='MINE')\n",
    "#ax.scatter(COV2, LinReg2, c='r', label='Regressor')\n",
    "ax.scatter(COV2, GT2, c='g', label='Ground Truth')\n",
    "\n",
    "ax.legend()\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 106,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/usr/local/lib/python3.7/site-packages/ipykernel_launcher.py:2: RuntimeWarning: divide by zero encountered in log\n",
      "  \n"
     ]
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAX4AAAD8CAYAAABw1c+bAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDMuMC4zLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvnQurowAAIABJREFUeJzt3X90VPW57/H3Q4BCiAeqcNASk0Db67GQAEJQqsUfVFREflivFWNLizbHoLWWwmm9WUtoPVlXi9ZqlXhzPSyOlynS2wrH+qNVQUF7pDZyURHprVpCSb0FsfKjgZqE5/4xkzgJM8kkM5PZyXxea82a2d/9nb2f2YFn9nz33s82d0dERLJHv0wHICIiPUuJX0Qkyyjxi4hkGSV+EZEso8QvIpJllPhFRLKMEr+ISJZR4hcRyTJK/CIiWaZ/pgOIZfjw4V5UVJTpMEREeo1XX331fXcfkUjfQCb+oqIiamtrMx2GiEivYWZ1ifbVUI+ISJZR4hcRyTJK/CIiWSaQY/wiEgyNjY3s3buXY8eOZToUiRg0aBD5+fkMGDCg28tQ4heRuPbu3ctJJ51EUVERZpbpcLKeu3PgwAH27t3L6NGju70cDfWISFzHjh3jlFNOUdIPCDPjlFNOSfoXWN9M/KEQFBVBv37h51Ao0xGJ9FpK+sGSir9H3xvqCYWgvBwaGsLTdXXhaYCysszFJSISEH1vj7+yktCnGyi6Ffotg6JbIfTpBqiszHRkItINZsZ1113XOt3U1MSIESOYNWsWAKtXr+bmm28GYPny5eTm5rJv377W/nl5ea2vc3JymDBhQuvjzjvv7KFPESx9bo8/9A91lF8BDQPD03XDoPwK4Jd1aH9fpPcZMmQIO3bs4OjRowwePJhnn32WUaNGxe0/fPhw7rnnHu66664T5g0ePJjt27enM9xeoc/t8VdektOa9Fs0DAy3i0h6pevw2syZM3nyyScBWLt2LfPnz4/bd+HChaxbt44PPvggNSvvg/pc4t+T19yldhFJjZbDa3V14P7x4bVUJP9rrrmGRx99lGPHjvH6669z9tlnx+2bl5fHwoULue+++06Yd/To0TZDPevWrUs+uF6ozw31FAwtpO7gibWKCoYWZiAakexRWfnxORUtGiKH15I9r6KkpITdu3ezdu1aZs6c2Wn/W265hQkTJrBkyZI27RrqCet0j9/MVpnZPjPbEdW2zsy2Rx67zSzmlozMeyPSr0fKbVZNryJ3QG6bttwBuVRNr+qJ1YtkrT17utbeVbNnz2bJkiUdDvO0GDZsGNdeey0PPvhgalbexySyx78aeAB4pKXB3b/c8trM7gEOdvD+C939/e4G2FVlxeFdi8qNlew5uIeCoQVUTa9qbReR9CgoCA/vxGpPhYULFzJs2DCKi4t54YUXOu2/ePFiSktLaWpqSk0AfUinid/dt5hZUax5Fr6S4GrgotSGlZyy4jIlepEeVlXV9hIagNzccHsq5Ofnc8sttyTcf/jw4cybN4977723ta1ljL/FpZdempWndJq7d94pnPifcPdx7dqnAT9y98lx3vdH4K+AA//D3WsSCWry5MmuG7GIZN5bb73FmWeemXD/UCg8pr9nT3hPv6pK102mQ6y/i5m9Gi8Xt5fswd35wNoO5p/n7vVm9o/As2a2y923xOpoZuVAOUBBqn4bikiPKitTou8Nun06p5n1B64E4p4P5e71ked9wHpgSgd9a9x9srtPHjEiodtGiohINyRzHv8XgV3uvjfWTDMbYmYntbwGZgA7YvUVEZGek8jpnGuBl4EzzGyvmV0fmXUN7YZ5zOxTZvZUZHIk8JKZvQa8Ajzp7r9KXeg9I1S9iKKl/em33Cha2p9Q9aJMhyQikpREzuqJedKsu38tRtufgZmR1+8C45OML6NC1Ysor6+mIVLjqS6vmfL6aqiGsoqVmQ1ORKSb+lzJhlSqfLeGhnZ3N2sYEG4XEemtlPg7sGdInLo/cdpFJPVaSimPGzeOK664gg8//DDTIfV6SvwdKPhb7Iqe8dpFJPVa6uvs2LGDk08+OWVlGHryit7m5mDtLCrxd6BqTDm5jW3bchvD7SISQ5pvezp16lTq6+tbp1esWEFpaSklJSUsW7astf2OO+7gjDPO4LzzzmP+/PncfffdAFxwwQXceuutTJ48mfvuu4/9+/fzpS99idLSUkpLS/nNb34DwObNm1sreE6cOJHDhw/z3nvvMW3atNZfHy+++CIQLhNdXFzMuHHj+O53v9saQ15eHt/5zncYP348L7/8ckq3Q9LcPXCPSZMmeVCsWVnhhUty3JbhhUtyfM3KikyHJNJjdu7cmXjnNWvcc3Pdw1WZw4/c3HB7EoYMGeLu7k1NTX7VVVf5008/7e7uv/71r/0b3/iGHz9+3Jubm/3yyy/3zZs3+yuvvOLjx4/3o0eP+qFDh/wzn/mMr1ixwt3dzz//fK+o+Pj/8Pz58/3FF190d/e6ujr/p3/6J3d3nzVrlr/00kvu7n748GFvbGz0u+++2//1X/+1NZZDhw55fX29n3766b5v3z5vbGz0Cy+80NevX+/u7oCvW7cuqc8eT6y/C1DrCebYPleWOdXKKlZShs7gEelUmuoyt9TXqa+v58wzz+Tiiy8G4JlnnuGZZ55h4sSJABw5coQ//OEPHD58mDlz5jBo0CAGDRrEFVdc0WZ5X/5ya41JnnvuOXbu3Nk6fejQIY4cOcK5557L4sWLKSsr48orryQ/P5/S0lIWLlxIY2Mjc+fOZcKECWzatIkLLriAlotOy8rK2LJlC3PnziUnJ4cvfelL3f7c6aShHhFJjTTVZW4Z46+rq8PdW8f43Z3bbruN7du3s337dt5++22uv/76TpYWvpVji+PHj7N169bWZdTX15OXl8f3vvc9Hn74YY4ePcq5557Lrl27mDZtGlu2bGHUqFF87Wtf45FHHulgLTBo0CBycoJ5PFCJX0RSI16NrRTV3srNzeX+++/nnnvuoampiUsuuYRVq1Zx5MgRAOrr69m3bx/nnnsuv/zlLzl27BhHjhzhiSeeiLvMGTNm8JOf/KR1uuUmLe+88w7FxcV897vfpbS0lF27dlFXV8fIkSP5xje+wQ033MC2bduYMmUKmzdv5v3336e5uZm1a9dy/vnnp+TzppOGekQkNdJdlxmYOHEiJSUlrF27lq985Su89dZbTJ06FQgfTF2zZg2lpaXMnj2bkpISRo4cSXFxMUOHDo25vPvvv5+bbrqJkpISmpqamDZtGg899BA//vGPef755+nXrx9jx47lsssu49FHH2XFihUMGDCAvLw8HnnkEU477TTuvPNOLrzwQtydyy+/nDlz5qTs86ZLQmWZe5rKMosEQ1fLMgelLvORI0fIy8ujoaGBadOmUVNTw1lnndXjcaRLpssyi4h8LCB1mcvLy9m5cyfHjh1jwYIFfSrpp4ISfw8LVS+i8t0a9gxppuBvOVSNKVfdH5EU++lPf5rpEAJNB3d7UEvRt7q8Ztw+Lvqmip8i0pOU+HuQir6JSBAo8fcgFX0TkSBQ4u9BKvomIkGgxN+DVPRNpOv+8pe/cO211zJmzBgmTZrE1KlTWb9+fY/HUVRUxPvvv9+m7eyzz2bChAkUFBQwYsSI1sJuu3fvTni5mzZtYuvWra3T1113HRs2bEhV2DHprJ4eVFaxEqrRWT0iCXJ35s6dy4IFC1rP1Kmrq+Pxxx8/oW9TUxP9+/dsSvvtb38LwOrVq6mtreWBBx6I2a+5uTlu+YZNmzYxfPhwzjnnnLTF2Z72+HtYWcVKdq9o4vhyZ/eKJiV96VNCb4Qo+nER/b7fj6IfFxF6I7myzJs2bWLgwIHceOONrW2FhYV885vfBMIJd/bs2Vx00UVMnz4dd2fp0qWMGzeO4uJi1q1bB8ALL7zArFmzWpdx8803s3r1aiC8J79s2TLOOussiouL2bVrFwAHDhxgxowZjB07lhtuuIGuXOza1NTEsGHDuPXWWykpKeGVV14hPz+/9SYyW7du5Ytf/CLvvPMODz/8MCtWrGDChAn853/+JwDPP/88n//85xkzZkxaft0kcrP1VWa2z8x2RLUtN7N6M9seecyM895Lzez3Zva2mX0vlYGLSLCE3ghR/sty6g7W4Th1B+so/2V5Usn/zTff7PTiq23btvHzn/+czZs389hjj7F9+3Zee+01nnvuOZYuXcp7773X6XqGDx/Otm3bqKioaK3d//3vf5/zzjuPN998k3nz5rGni8XmDh48yLRp03j99ddby0q09+lPf5obbriBpUuXsn37dj7/+c8DsG/fPn7zm9+wYcMGbrvtti6tNxGJ7PGvBi6N0X6vu0+IPJ5qP9PMcoAHgcuAzwHzzexzyQQrIsFVubGShsa2ZZkbGhuo3FiZsnXcdNNNjB8/ntLS0ta2iy++mJNPPhmAl156ifnz55OTk8PIkSM5//zz+d3vftfpcq+88koAJk2a1Do+v2XLFq677joALr/8cj75yU92KdaBAwcyb968Lr2nxdy5czEzSkpK2tx4JlU6TfzuvgX4oBvLngK87e7vuvtHwKNA8KsXiUi37DkYe484Xnsixo4dy7Zt21qnH3zwQTZu3Mj+/ftb26LLLMfTv39/jh8/3jp97NixNvM/8YlPAOH7+6bqloyDBw/GzGLG0H797bXEA3RpiClRyYzx32xmr0eGgmJ9FY4C/hQ1vTfSJiJ9UMHQ2OWX47Un4qKLLuLYsWNUV1e3tjW0v9lLlC984QusW7eO5uZm9u/fz5YtW5gyZQqFhYXs3LmTv//973z44Yds3Lix03VPmzat9YDy008/zV//+tdufw4IH0t49dVXAfjFL37R2n7SSSdx+PDhpJbdVd1N/NXAp4EJwHvAPckGYmblZlZrZrXR3+Yi0jtUTa8id0Bum7bcAblUTe9+WWYzY8OGDWzevJnRo0czZcoUFixYwF133RWz/7x58ygpKWH8+PFcdNFF/PCHP+TUU0/l9NNP5+qrr2bcuHFcffXVrXft6siyZcvYsmULY8eO5bHHHqMgyfsKLF++nEWLFlFaWsrAgQNb2+fMmcPPfvYzJk6c2HpwN90SKstsZkXAE+4+LtF5ZjYVWO7ul0SmbwNw9//e2fpUljlxKvom6dTVssyhN0JUbqxkz8E9FAwtoGp6FWXFma/W2ddkpCyzmZ3m7i2HyucBO2J0+x3wWTMbDdQD1wDXdmd9EltL0beGvPB0S9E3qlHyl4woKy5Tou8FEjmdcy3wMnCGme01s+uBH5rZG2b2OnAh8O1I30+Z2VMA7t4E3Az8GngL+Jm7v5mmz5GVVPRNRLqj0z1+d58fo/nf4vT9MzAzavop4IRTPSU1VPRNeoK7tzk7RTIrFWf56MrdXkxF3yTdBg0axIEDB9JySqF0nbtz4MABBg0alNRyVKunF6saUx4e448a7lHRN0ml/Px89u7di860C45BgwaRn5+f1DKU+HsxFX2TdBswYACjR4/OdBiSYgmdztnTdDqniEjXdOV0To3xi4hkGSV+EZEso8QvIpJllPhFRLKMEr+ISJZR4pc2QtWLKFran37LjaKl/QlVL8p0SCKSYkr80qql6FtdXjNuHxd9U/IX6VuU+KWVir6JZAclfmmlom8i2UGJX1qp6JtIdlDil1ZVY8rJbWzbpqJvIn2PEr+0KqtYSc2oCgqP5GAOhUdyqBlVoaJvIn2MirSJiPQBKtImIiJxKfGLiGSZRG62vsrM9pnZjqi2FWa2y8xeN7P1ZjYsznt3R27Kvt3MNHYjIhIAiezxrwYubdf2LDDO3UuA/wvc1sH7L3T3CYmOPYmISHp1mvjdfQvwQbu2Z9y9KTK5FUjuBpAiItJjUjHGvxB4Os48B54xs1fNTCeDZyEVfRMJnqQSv5lVAk1AKE6X89z9LOAy4CYzm9bBssrNrNbMavfv359MWBIQKvomEkzdTvxm9jVgFlDmcS4GcPf6yPM+YD0wJd7y3L3G3Se7++QRI0Z0NywJEBV9EwmmbiV+M7sU+Bdgtrs3xOkzxMxOankNzAB2xOorfZOKvokEUyKnc64FXgbOMLO9ZnY98ABwEvBs5FTNhyJ9P2VmT0XeOhJ4ycxeA14BnnT3X6XlU0ggqeibSDD176yDu8+P0fxvcfr+GZgZef0uMD6p6KRXqxpTTnl9dZvhHhV9E8k8XbkraaOibyLBpCJtIiJ9gIq0iYhIXEr8IiJZRolfRCTLKPGLiGQZJX4RkSyjxC+9ioq+iSRPiV96DRV9E0kNJX7pNVT0TSQ1lPil11DRN5HUUOKXXkNF30RSQ4lfeo2qMeXkNrZtU9E3ka5T4pdeQ0XfRFJDRdpERPoAFWkTEZG4lPhFRLKMEr+ISJZR4hcRyTIJJX4zW2Vm+8xsR1TbyWb2rJn9IfL8yTjvXRDp8wczW5CqwEVSIhSCoiLo1y/8HAplOiKRtEt0j381cGm7tu8BG939s8DGyHQbZnYysAw4G5gCLIv3BSHS40IhQvd+naJ5dfS73SmaV0fo3q8r+Uufl1Did/ctwAftmucA/x55/e/A3BhvvQR41t0/cPe/As9y4heISEaEHv4W5Zc0UjeMcNG3YVB+SSOhh7+V6dBE0iqZMf6R7v5e5PX/A0bG6DMK+FPU9N5Im0jGVU44QMPAtm0NA8PtIn1ZSg7uevgqsKSuBDOzcjOrNbPa/fv3pyIskQ7tGdq1dpG+IpnE/xczOw0g8rwvRp964PSo6fxI2wncvcbdJ7v75BEjRiQRlkhiCgac0qV2kb4imcT/ONByls4C4D9i9Pk1MMPMPhk5qDsj0iaScVWz7yPX2o715NpAqmbfl6GIRHpGoqdzrgVeBs4ws71mdj1wJ3Cxmf0B+GJkGjObbGYPA7j7B8AdwO8ijx9E2kQyrqy4jJp5qygcWohhFA4tpGbeKsqKyzIdmkhaqUibiEgfoCJtIiISlxK/iEiWUeIXEckySvwiIllGiV8kRULViyha2p9+y42ipf0JVS/KdEgiMSnxi6RAqHoR5fXV1OU1h+v+5DVTXl+t5C+BpMQvkgKV79bQMKBtW8OAcLtI0Cjxi6TAniHNXWoXySQlfpEUKPhbTpfaRTJJiV8kBarGlJPb2LYttzHcLhI0SvwiKVBWsZKaURUUHsnBHAqP5FAzqoKyipWZDk3kBKrVIyLSB6hWj4iIxKXELyKSZZT4RUSyjBK/iEiWUeIXEckySvwiAaWib5IuSvwiAaSib5JO3U78ZnaGmW2Pehwys1vb9bnAzA5G9bk9+ZBF+j4VfZN06t/dN7r774EJAGaWA9QD62N0fdHdZ3V3PSLZSEXfJJ1SNdQzHXjH3etStDyRrKaib5JOqUr81wBr48ybamavmdnTZjY2ResT6dNU9E3SKenEb2YDgdnA/44xextQ6O7jgZ8AGzpYTrmZ1ZpZ7f79+5MNS6RXU9E3Saeki7SZ2RzgJnefkUDf3cBkd3+/o34q0iYi0jU9XaRtPnGGeczsVDOzyOspkfUdSME6RUSkm7p9Vg+AmQ0BLgb+OartRgB3fwi4CqgwsybgKHCNB7EOtIhIFkkq8bv734BT2rU9FPX6AeCBZNYhIiKppSt3RUSyjBJ/LxcKQVER9OsXfg6FMh2RiASdEn8vFgpBeTnU1YF7+Lm8XMlfYlPRN2mhxN+LVVZCQ0PbtoaGcLtINBV9k2hK/L3Ynj1da5fspaJvEk2JvxcrKOhau2QvFX2TaEr8vVhVFeTmtm3LzQ23i0RT0TeJpsTfi5WVQU0NFBaCWfi5pibcLhJNRd8kWtK1etJBtXpEUi9UvYjKd2vYM6SZgr/lUDWmXEXf+pCu1OpR4hcR6QN6ukib9CG6IKzv0t9WWijxSytdEBYsqUzU+ttKNCV+aZWNF4QFdS841Yk6G/+2Ep8Sv7RKxwVhQU2sEOy94FQnal3sJ9GU+KVVqi8IC3JihWDvBac6UetiP4mmxC+tUn1BWJATK6Q+uaby102qE3U6LvZT0bfeS4m/E0Eeqki1VF8QFvThhVQm11T/ukl1ok713zZUvYjyve2Kvu1V0bfeQufxd6DlP3P0Xmturq6OTVRRUTgBtldYCLt393Q0J0rl3zcdnzUUCv862rMn/GVUVRWcf3cF3+nPn/7hxDo/px/KYc89TRmISHQef4qkY6gim35BBL2WUCr3gtPx66asLPylcfx4+DkoSR9g70mxi7vFa5dgSTrxm9luM3vDzLab2Qm76RZ2v5m9bWavm9lZya6zp6RjDDjIBztTLR21hFL9xZmq5JptB08/dTB2cbd47RIsqdrjv9DdJ8T5mXEZ8NnIoxyoTtE60y7V/5mDfrAzHVK51xrkL86g/7pJtbFbyxn8Udu2wR+F2yX4emKoZw7wiIdtBYaZ2Wk9sN6kpfo/c9APdgZdkL84s61S6ldvXsnkJysY9WEO5jDqwxwmP1nBV29W0bdewd2TegB/BLYBrwLlMeY/AZwXNb0RmByjXzlQC9QWFBR4MtascS8sdDcLP69ZE4xlFRa6h/dV2z4KC7u/zGxiFnv7mWU6suyUyv8bkjyg1hPN24l2jLsAGBV5/kfgNWBau/kJJf7ox6RJk7r94descc/NbZsYcnOD8Y8yyLH1BvriFImvK4k/6aEed6+PPO8D1gNT2nWpB06Pms6PtKWFhgP6rmwbRxdJl6QSv5kNMbOTWl4DM4Ad7bo9Dnw1cnbPOcBBd38vmfV2JOjj6EE+RS/o9MUpkhr9k3z/SGC9mbUs66fu/iszuxHA3R8CngJmAm8DDcDXk1xnhwoKYl9I01dPq8s2ZWVK9CLJSirxu/u7wPgY7Q9FvXbgpmTW0xVVVbGvxtRwgIhIWJ+7clfDASK90+2LF5H/7XDRt/xv9+f2xar7ky6q1SMiGXf74kXcPbiaowM/bhv8ESw5WsEPfqRrAxKhWj0i0qus8po2SR/g6MBwu6SeEr+IZNyfh8Yu7havXZKjxC8iGaeibz1LiV9EMm6hxS76ttBU9C0dlPhFJON+8KOVLDnatuibDuymj87qERHpA3RWj4iIxKXELyKSZZT4RaRPyqb7W3dVskXaREQCp+U2nS01u1pu0wkq3wLa4xeRPqiyEuY0hPgjRTTTjz9SxJyGUCDuyxEE2uMXkT7n3LoQ04u/zgXTG9kzFAoO1nH7xq9jbwBol197/CLS55xT8i2+eUUjdcPADeqGwTevaOSckm9lOrRAUOIXkT7n7osO0NCu6FvDwHC7KPGLSB/0p6Fda882Svwi0ucUDDilS+3ZRolfRPqcqtn3kWttx3pybSBVs+/LUETB0u3Eb2anm9nzZrbTzN40sxOOmpjZBWZ20My2Rx63JxeuiEjnyorLqJm3isKhhRhG4dBCauatoqxYZ/RAcqdzNgHfcfdtZnYS8KqZPevuO9v1e9HdZyWxHhGRLisrLlOij6Pbe/zu/p67b4u8Pgy8BYxKVWAiIpIeKRnjN7MiYCLw2xizp5rZa2b2tJmN7WAZ5WZWa2a1+/fvT0VYIiISQ9KJ38zygF8At7r7oXaztwGF7j4e+AmwId5y3L3G3Se7++QRI0YkG5aIiMSRVOI3swGEk37I3R9rP9/dD7n7kcjrp4ABZjY8mXWKiEhykjmrx4B/A95y9x/F6XNqpB9mNiWyPl06JyK9zu2LF5H/7f70W27kf7s/ty9elOmQui2Zs3rOBb4CvGFm2yNt/w0oAHD3h4CrgAozawKOAtd4EO/1KCLSgdsXL+LuwdUcjVwaUD+smbs/qobF9Mr7AuueuyIincj/dn/qhzWf0D7qwxz23tuUgYhOpHvuioik0J+Hnpj0O2oPOiV+EZFOfOpgTpfag06JX0SkEwutnMEftW0b/FG4vTdS4hcR6cQPfrSSJUcrGPVhDubhsf0lRyt65YFd0MFdEZE+QQd3RUQkLiV+EZEso8QvIpJllPhFRLKMEr+ISJZR4hcRyYBMFn1T4hcR6WEtRd/qhzXjFin6Nri6x5K/Er+ISA9b5TWtlT5bHB0Ybu8JSvwiIj0s00XflPhFRHpYpou+KfGLiPSwTBd9U+IXEelhmS76piJtIiJ9gIq0iYhIXEklfjO71Mx+b2Zvm9n3Ysz/hJmti8z/rZkVJbM+ERFJXrcTv5nlAA8ClwGfA+ab2efadbse+Ku7fwa4F7iru+sTEZHUSGaPfwrwtru/6+4fAY8Cc9r1mQP8e+T1z4HpZmZJrFNERJKUTOIfBfwpanpvpC1mH3dvAg4Cp8RamJmVm1mtmdXu378/ibBERKQjgTm46+417j7Z3SePGDEi0+GIiPRZyST+euD0qOn8SFvMPmbWHxgKHEhinSIikqRkEv/vgM+a2WgzGwhcAzzers/jwILI66uATR7ECwdERLJIUhdwmdlM4MdADrDK3avM7AdArbs/bmaDgP8FTAQ+AK5x93cTWO5+oK7bgaXGcOD9DMeQKMWaPr0pXsWaPr0h3kJ3T2icPJBX7gaBmdUmehVcpinW9OlN8SrW9Olt8XYmMAd3RUSkZyjxi4hkGSX++HrmVjipoVjTpzfFq1jTp7fF2yGN8YuIZBnt8YuIZBkl/ihmdoeZvW5m283sGTP7VKT9AjM7GGnfbma3ZzpW6DBeM7P7I1VRXzezswIQ6woz2xWJZ72ZDYu0F5nZ0aht+1BQY43Muy2yXX9vZpdkMs4WZvZfzexNMztuZpOj2oO4bWPGGpkXuG0bzcyWm1l91PacmemYus3d9Yg8gH+Ien0L8FDk9QXAE5mOrwvxzgSeBgw4B/htAGKdAfSPvL4LuCvyugjYken4Eoz1c8BrwCeA0cA7QE4A4j0TOAN4AZgc1R7EbRsv1kBu23axLweWZDqOVDy0xx/F3Q9FTQ4BAn0ApIN45wCPeNhWYJiZndbjAUZx92c8XKgPYCvhEh+B1EGsc4BH3f3v7v5H4G3CVWozyt3fcvffZzqORHQQayC3bV+lxN+OmVWZ2Z+AMiB6SGeqmb1mZk+b2dgMhXeCOPEmUjk1kxYS/kXSYrSZ/R8z22xmX8hUUHFExxr07RpLkLdttN6ybW+ODAGuMrNPZjqY7uqf6QB6mpk9B5waY1alu//5kqvTAAAB5klEQVSHu1cClWZ2G3AzsAzYRvhy6CORcb0NwGcDHG9GdBZrpE8l0ASEIvPeAwrc/YCZTQI2mNnYdr9mghJrxiQSbwyB3bZB1VHsQDVwB+Ff1ncA9xDeMeh1si7xu/sXE+waAp4ClkX/R3H3p8xspZkNd/e01+7oTrwkVjk15TqL1cy+BswCpntk0NTd/w78PfL6VTN7B/gvQG3QYiVD2xW69O8g+j2B3LZxZGzbRks0djP7n8ATaQ4nbTTUE8XMovfi5wC7Iu2nmoXvHGZmUwhvt4yXl44XL+GqqF+NnN1zDnDQ3d/r8QCjmNmlwL8As929Iap9hIVv44mZjSH8S6rTQn7pFC9Wwtv1GgvfS3o04VhfyUSMiQjitu1A4Ldtu+Nk84AdmYolWVm3x9+JO83sDOA44eqgN0barwIqzKwJOEq4ymgQDvzGi/cpwmf2vA00AF/PTHhtPED4jI1nI9+hW939RmAa8AMzayT8OW509w8yFyYQJ1Z3f9PMfgbsJDwEdJO7N2cwTgDMbB7wE2AE8KSZbXf3Swjgto0Xa1C3bTs/NLMJhId6dgP/nNlwuk9X7oqIZBkN9YiIZBklfhGRLKPELyKSZZT4RUSyjBK/iEiWUeIXEckySvwiIllGiV9EJMv8f/R7KQPliSWCAAAAAElFTkSuQmCC\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "fig, ax = plt.subplots()\n",
    "COV22 = np.log(np.ones(len(COV2)) - COV2)\n",
    "ax.scatter(COV22, MINE2, c='b', label='MINE')\n",
    "ax.scatter(COV22, LinReg2, c='r', label='Regressor')\n",
    "ax.scatter(COV22, GT2, c='g', label='Ground Truth')\n",
    "\n",
    "ax.legend()\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 107,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "9.210340371976184"
      ]
     },
     "execution_count": 107,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "np.log(10000)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 104,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "4.1018114\n",
      "5.719031\n",
      "4.925558\n",
      "5.036795\n",
      "4.860683\n",
      "3.939142\n",
      "4.0863094\n",
      "3.7627614\n",
      "4.9275703\n",
      "7.447962\n",
      "3.5613244\n",
      "6.847351\n",
      "6.5556297\n",
      "6.4137144\n",
      "5.5157366\n",
      "4.077453\n",
      "6.1694155\n",
      "4.7401958\n",
      "4.2466965\n",
      "8.159683\n",
      "3.1441705\n",
      "12.408489\n",
      "5.5862813\n",
      "5.1209526\n",
      "5.0138783\n",
      "4.0317445\n",
      "3.7254608\n",
      "2.1680317\n",
      "5.194463\n",
      "5.2940736\n",
      "4.9103403\n",
      "2.9698486\n",
      "3.469598\n",
      "6.269966\n",
      "5.130463\n",
      "3.7743385\n",
      "5.3607063\n",
      "4.9128137\n",
      "4.5146365\n",
      "4.81076\n",
      "4.0846624\n",
      "4.2393556\n",
      "5.7824535\n",
      "5.574537\n",
      "3.5774617\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/usr/local/lib/python3.7/site-packages/ipykernel_launcher.py:12: RuntimeWarning: divide by zero encountered in log\n",
      "  if sys.path[0] == '':\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "3.295353\n",
      "4.1492805\n",
      "5.19768\n",
      "4.386994\n",
      "4.258513\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/usr/local/lib/python3.7/site-packages/ipykernel_launcher.py:12: RuntimeWarning: divide by zero encountered in log\n",
      "  if sys.path[0] == '':\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "4.225566\n",
      "3.5524883\n",
      "4.3728476\n",
      "6.1828737\n",
      "4.6994576\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/usr/local/lib/python3.7/site-packages/ipykernel_launcher.py:12: RuntimeWarning: divide by zero encountered in log\n",
      "  if sys.path[0] == '':\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "3.9178414\n",
      "3.9703667\n",
      "6.260885\n",
      "4.112125\n",
      "4.8858256\n"
     ]
    }
   ],
   "source": [
    "for i in range(8, 20):\n",
    "    cov = 1 - 0.1**i\n",
    "    COV2.append(cov)\n",
    "    x = np.transpose(np.random.multivariate_normal( mean=[0,0],\n",
    "                                  cov=[[1,cov],[cov,1]],\n",
    "                                 size = 10000))\n",
    "    DE = DC.computeEnt(x, linReg, MSEscorer, varEntropy, CVFold)\n",
    "    MI = DE[1,0] + DE[0,0] - DE[0,1] - DE[1,1]\n",
    "    MI = MI/2\n",
    "    LinReg2.append(MI)\n",
    "    #plt.scatter(cov, MI, c='g',label='KNN-regressor')\n",
    "    groundTruth = -0.5*np.log(1-cov*cov)\n",
    "    GT2.append(groundTruth)\n",
    "    #plt.scatter(cov, groundTruth, c='r',label='ground truth')\n",
    "    \n",
    "    #MINE\n",
    "    mine_net = Mine()\n",
    "    mine_net_optim = optim.Adam(mine_net.parameters(), lr=1e-3)\n",
    "    result = train(np.transpose(x),mine_net,mine_net_optim)\n",
    "    result_ma = ma(result)\n",
    "    MINE2.append(result_ma[-1])\n",
    "    #MINE\n"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.0"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}