11_deep_learning.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {
    "deletable": true,
    "editable": true
   },
   "source": [
    "**Chapter 11 – Deep Learning**"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "deletable": true,
    "editable": true
   },
   "source": [
    "_This notebook contains all the sample code and solutions to the exercices in chapter 11._"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "deletable": true,
    "editable": true
   },
   "source": [
    "# Setup"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "deletable": true,
    "editable": true
   },
   "source": [
    "First, let's make sure this notebook works well in both python 2 and 3, import a few common modules, ensure MatplotLib plots figures inline and prepare a function to save the figures:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "collapsed": true,
    "deletable": true,
    "editable": true
   },
   "outputs": [],
   "source": [
    "# To support both python 2 and python 3\n",
    "from __future__ import division, print_function, unicode_literals\n",
    "\n",
    "# Common imports\n",
    "import numpy as np\n",
    "import numpy.random as rnd\n",
    "import os\n",
    "\n",
    "# to make this notebook's output stable across runs\n",
    "rnd.seed(42)\n",
    "\n",
    "# To plot pretty figures\n",
    "%matplotlib inline\n",
    "import matplotlib\n",
    "import matplotlib.pyplot as plt\n",
    "plt.rcParams['axes.labelsize'] = 14\n",
    "plt.rcParams['xtick.labelsize'] = 12\n",
    "plt.rcParams['ytick.labelsize'] = 12\n",
    "\n",
    "# Where to save the figures\n",
    "PROJECT_ROOT_DIR = \".\"\n",
    "CHAPTER_ID = \"deep\"\n",
    "\n",
    "def save_fig(fig_id, tight_layout=True):\n",
    "    path = os.path.join(PROJECT_ROOT_DIR, \"images\", CHAPTER_ID, fig_id + \".png\")\n",
    "    print(\"Saving figure\", fig_id)\n",
    "    if tight_layout:\n",
    "        plt.tight_layout()\n",
    "    plt.savefig(path, format='png', dpi=300)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "deletable": true,
    "editable": true
   },
   "source": [
    "# Activation functions"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "collapsed": true,
    "deletable": true,
    "editable": true
   },
   "outputs": [],
   "source": [
    "def logit(z):\n",
    "    return 1 / (1 + np.exp(-z))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "collapsed": false,
    "deletable": true,
    "editable": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Saving figure sigmoid_saturation_plot\n"
     ]
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAagAAAEYCAYAAAAJeGK1AAAABHNCSVQICAgIfAhkiAAAAAlwSFlz\nAAALEgAACxIB0t1+/AAAIABJREFUeJzs3Xd4FFX3wPHvDaQXOiIKBikCkabSFDWIWJEm+FroYHtV\nlPKzIIooKIo0ewWkCFghSBEFIoKACEpRXkAlIhA6CYT05Pz+mE1INrshCbvZ3eR8nmceMrN3Z85e\nJjl7y8wYEUEppZTyNn6eDkAppZRyRBOUUkopr6QJSimllFfSBKWUUsoraYJSSinllTRBKaWU8kqa\noFSJGWNWG2Pe8HQcULRYjDHbjTHPl1ZMeY47wxgTUwrHud4Yk2WMqVoKx3rAGPOPMSbTE3VqF0t/\nY8wpT8ag3MPodVDKEWNMdeBF4FbgQiAB2A5MEJGVtjKVgQwROeOxQG2KEosxZjvwuYi86KYYrgdW\nA9VF5ESe7eFYv2su+yNqjNkLvCkik/NsqwhUFZEjrjqOk2NXBo4ATwBfAEkikuzOY+Y5djbQS0S+\nyrMtEAgXkWOlEYMqPRU9HYDyWl8BQcBA4C+gJnA9UC2ngIgkeCa0grwkFgOI7d9cInK6NA4uIplY\nicPdIoEKwBJ3J8OiEJE0IM3TcSg3EBFddMm3AJWAbOCGc5RbDbyRZ70mEAMkA3uBAVitrufzlMkG\nHgIWAmeAXUA0cBGwHEgCfgVa2R2rJ7ANSAX2AaPOEUsNYFGeWAbax+Lg81xqiyveFsdm4Ha7Mv7A\ny0CcLZY/gUeBS2yfLSvPv9Nt75kJxNh+fgA4BPjZ7fdT4OuixGH7rPmOZdsebVuvWox62ws8C7wH\nJAL/AiMLqaP+Dj5nXWAMsN1B2dN51sfY/g/+Y6u3U8DXeePN876cmA/lqce9eY6bDfxt2z4g73Fs\n2x4E9mAlrj3AELvXs4H7gc9sdfwXcJ+nf/d0yb/oGJRyJMm2dLV1nxTVLKAO1h/KbkAfrD9e9p7F\n+oPcHNgEzAM+At4GWgIHgRk5hY0xV2L9IfkCuBx4CnjGGPNIIbF8gvWH/gagO9APK4kUJgxYCnSy\nxfYF8KUxppHdZ+yD1b3VGBiM1f25D7jTVqYJVrfo47b1vP3on2F9Abgxz+cLAboCs4sYR09gPzAW\nqGU7Vs5xco9VjHp7AishtAJeBV4zxrR1UkfzgVtsP19lO/Z+B58TJ9sigbuwzo/OtmOOzxPzg1jJ\n8mOgGVYX8++2l1tjtU4H2z53ayefuwfwJjAZiAKmAe8YY263i+U5rATZHFgATDfG1HHyuZUneDpD\n6uKdC9ADOAakAD8BE4E2dmVyWy3AZVjfSlvnef1iIJOCLahxedajbNsez7PteqxvylVt63OA7+2O\nPQbY5ySWRrZ9tsvzel37WIpYD+uxtTqAhrb9dnZSNl/cebbPwNaCsq1/BXySZ70PcBIIKEoctvW9\nwPDCjl/EetsLzLUrsxu7lpbd61fajlPXbr/b7Mr1B07ZlUkGwvJsGwXszrP+LzC+kGNnAz3PcZy1\nwIcO/g/WFHIeVsBq0d9b2r9rujhftAWlHBKRr4HaQBesb/PtgQ3GmKedvOUyrD9am/PsYz9Wa8je\n9jw/H7b9u8PBtpq2f5sA6+z2sRa4yBgT5mD/jW2xbMoTyz4nseQyxoQYY14zxvxujDlhjDmN9cc4\npxXY0rbf2ML2UwRzgO7GmCDb+r3AFyKSXsQ4iqqo9bbNrsxBzta9q/0jIkmOjmWMqYHV1bvqPI/R\nBOtLVV5rgaZ223LPQxHJAo7ivs+tSkATlHJKRNJFZKWIjBORDljdLi/YZovZMw62OZOR9zBOthnO\nnp8Gx91HONlenFjymoTVTfcscB3QAivJBZznfu19g5Xoutn+KN/I2e69osRRVEWttwwHrxX3b0M2\nBevH30G5wo7lqvrN2e+5trnicys30v8MVRw7sWZ+Bjl5zQ/rmz4AxpiLsVphJZH3j8kfQAe7168F\n9ovjaeU5seSMUWCMqVuEWK4BZonIQhHZgfXtvn6e17fY9tvRyfvTbf9WKOwgtpbSF1hde/8B4kVk\nTTHiyDlWoceh+PV2Po4CF9hta1WcHYg1I/AA1tibMxmc+3PvxPHn/qM48SjP0wSlCjDGVDXGrDTG\n3GeMaWaMiTTG9Ab+D2tMI8n+PSKyG1gBvG+MaWuMaQlMx+rXL8nFdnm/TU8CrjfGjDHGNDTG3AcM\nxxrQL8AWy7e2WNrZYpmBNf5RmN1AD2NMK2NMM6xWTe4kERH5E/gc+MgY09NWLx2MMX1sRf7B+qy3\nG2OqG2NCCznWHOBmrBmNnxYnDps44FpjTG1jTLU820tcb8Vk39qJBaoaY0YZYy41xgzm7KSR4hgP\nPGGMecIWc0tjzPA8r8cBnYwxF9iux3JkItDXGPNfY0wDY8xjwD245nOrUqQJSjmShDUoPxTrD88O\nYBzWH9W785SzTzz9sQa5V2NNk56D9c06tZD3nHObiPwK9MaavbYda5r3yyLyzjli2QusxJpuPhfr\nj1thhmNdR7QGWIJVBz/alemLlVCmYX1TnwFE2OI8iDURYDzW9Og3nR3I1mI6gDVeNqcEcTyPNWPy\nL/Jf+3S+9eZsW6FlROR/wMNYU7e3YrWCxjt4X+E7FXkPeAQYYot5KfnHjkZgtWD/xWrROtrHIuAx\nrNmJv9t+flhEljqLv5BtyoP0ThLKbWzf7A8Cd9smXSilVJFpglIuY4zpCIRjffO9AOsbdCOgkYik\neDI2pZTv0VsdKVfyx+oKrIc13rMBuF6Tk1KqJLQFpZRSyit5fQvKGKMZVCmlyjgRKXAdnE/M4vP0\n7TYcLf379/d4DL6yaF0Vb/HWc94bF0+cW5lZmfx+5HePf3Zfqa+iLM74RIJSSilvUcGvAk1r2N81\nSbmDJqgSioyM9HQIPkPrSrmLnlvF42v1pQmqhKKjoz0dgs/QulLuoudW8fhafWmCUkopJ9Kz0hm2\nfBgnU056OpRyyetn8SmllCdkSzb9F/YnNTOV8MBwT4dTLnn9dVDGGPH2GJVyJWNMoTOblPuJCE8s\nf4JfD/3Kt32+Jdg/2NMhlWm2c77ANHNtQSmllJ1X173K6rjVrBm4RpOTB+kYVAnFxsZ6OgSfoXWl\n3MUd59YPcT/w/ub3Wd5nOZWDnD3Rwzf52u+itqCUUiqPay+5lvWD11MrrJanQyn3dAxKKS+jY1Cq\nvHE2BqVdfEoppbySJqgS8rW+XE/SulLuoudW8fhafWmCUkqVWydSTtD7894kpSd5OhTlgI5BKeVl\ndAyqdCRnJHPjrBvpULcDr3V+zdPhlGvOxqA0QSnlZTRBuV9GVgY9FvSgWkg1ZnSbgZ/RziRP0kkS\nLuZrfbmepHWl3KUk55aI8MA3D5At2Xx0x0flKjn52u+iXgellCpXYnbFsPPoTlb2W4l/BX9Ph6MK\noV18SnkZ7eJzLxEhOSOZ0IBQT4eibHQMSikfoQlKlTc6BuVivtaX60laV8pd9NwqHl+rL01QSqky\nTVujvku7+JTyMtrF5zq7j+9mcMxgvu/7PYEVAz0djnJCu/iUUuVK/Ol4bplzC/1b9Nfk5KM0QZWQ\nr/XlepLWlXIXZ+dWYmoit869lUGtBjHkiiGlG5QX87XfRU1QSqkyJTUzle4LutOhbgeevfZZT4ej\nzoOOQSnlZXQM6vzM2z6Pr//3NfPunEcFvwqeDkcVgV4HpZSP0AR1/rKyszQ5+ZBSmSRhjHnEGLPJ\nGJNqjJl+jrLDjDHxxpiTxpiPjDE+dc8RX+vL9SStK+Uuzs4tTU6O+drvoqvHoA4ALwEfF1bIGHMz\n8CTQEYgE6gNjXRyLUkopH+aWLj5jzEvARSIyyMnrc4G9IjLatn4DMFdELnRQVrv4VLmiXXzFk5GV\noTd99XHedh1UFLA1z/pWoKYxpoqH4lFK+aA1/6yh7UdtycrO8nQoyg08laDCgMQ864mAAcI9E07x\n+VpfridpXSl32HZ4G90mdGNi54k65lRE3vC7KAIpKXDiBBw4AH/+WWhhcfmCNQ41vZDXfwN65Vmv\nCmQBVRyUFUfLmDFjRERk9erVsnr1asnRv39/La/lfbq89WvpPfF4Y/m9J/fKRZMukuY3N/eKeMpm\n+UCBCIEL5cEH35U1a0Refnm1PPfcavngA5HXXxdp0WKKwEMC7wnMEnhB4BWJjPxb2rYVufTS1VK7\n9mq56CKRqlVF/Py+E1htW8YI9LctiDjIFZ4cg/pbRJ6zrd8AzBGR2g7KijtiVMpb6RhU4Y6eOUqH\nGR14tPWjPNb2MU+H49VErJbKsWPWvznL8ePO1xMS4NQpyMx0X1yBgRAcfHb5+2/HY1AufaKuMaYC\n4A9UACoaYwKBTBGx7yCeBcwwxnwKHAKeBWa4MhalVNm07M9l9G7au1wnp4wMq3ts/344dMj5cuSI\nVbYk/P0hIgLCw60l78/262FhEBp6NuGEhORPQHnXg4Kggl2PrCmQmmzbXflNzRgzBhiD1UTMMRYr\n+fwBNBGR/bayTwBPA0HAF8DDIlKgKr21BRUbG0t0dLSnw/AJWlfFoy2ooiur51ZiIsTFwb591vLP\nP2d/3rcPDh60WkdFEREBNWpAtWogEkvDhtFUqwZVq1pL3p+rVoUqVaykE+jG++v+9ttvDBs2jEWL\nFhEREeF0Fp9LW1AiMhbn1zOF25WdCkx15fGVUspXnDljTRDYvRv27Dn77549cPRo4e/184OLLoKL\nL4YLL4RatazlggvO/pyzHhR09n2xseDJfC4ivPnmmzz99NOkp6eTmJhIRESE0/J6qyOlvIy2oMqW\n9HTYtQu2b8+/7Nvn/D3BwVCvHtStay2XXHL257p1reTk72OXfiUkJHDPPfewZs0akpOTCQkJYdeu\nXVx88cWl04JSSilXS0xNpFJQJU+HUSQpKfDrr/Dzz7BpE2zbZiUnR+NA/v5Qvz40bAiNGuX/t3Zt\nq5VUVmzYsIFu3bqRkJBAeno6YLWm/M7xITVBlVBZ7ft2B60rVVKzts7izZ/f5OchP2McjKR78tzK\nyoKdO61klLNs22Ztt3fppdCsWf6lYUOoWMp/gUu7vrKzs5kwYQLjxo0jJSWlwOuaoJRSPmnpnqU8\n+d2TrO6/2mFyKm0ZGbBlC/zwg7WsXWtNx87Lzw+aN4fWra2lZUuIirJmuZU3R48epVevXmzevNlh\nchKRc/6/6hiUUl5Gx6Bgw/4NdJ3XlZh7Ymh3cTuPxCACv/8Oy5fD999bCenMmfxlLrkE2raFNm2s\n5YorrOnW5V1sbCw9e/YkKSmJDCfz3IOCgti3bx81atTQMSillG/YeXQn3ed3Z2b3maWenBISYOVK\nKyktX25dZ5RXw4Zw/fXWTLjrr7dm0amzsrKyeP7555kyZYrDVlNeRWlBaYIqIR1XKTqtK1UcPx/4\nmVdvfJXbGt52zrKuOLfi42HhQvjqK2sadt47KFxwAdx8s7VER1uTF3yZu38Xx4wZwyuvvFLkHgAd\ng1JK+ZT+Lfu7/Rh798KXX1pJacOGsxe9VqgA110Ht9xiLS1alK3ZdO42YsQI0tPTefvtt8nKyiIt\nLc1p2aLM4tMxKKW8jI5BuceJE/DZZzBnDqxbd3Z7UJDVQurRA+64w7qbgjo/x44d45FHHmHx4sVO\nu/oCAgI4cuQIlSpV0jEopVT5k54OS5bA7NnWv7ZLcAgJga5d4c47rZZSeZxl507Vq1fn8OHDpKam\nOi1TlBaUNl5LyBueq+IrtK6UMyLCvsRCbqlwDs7Orbg4eOYZqFMHevaEr7+2xpY6d4ZZs+DwYZg3\nD3r1Kl/JqbR+Fzdu3MimTZvy9QSEhobi7++Pv+0WGJqglFJe7fnVzzM4ZrBL9pWVBd98A7ffbl0Y\nO2GCdTfvyy+HiRPh339hxQro27d8JSVPGDFiBMnJyfm2hYSEsHv3bvr160dQUBCZmZl6HZRSvqa8\njEG99fNbvLHxDdYOWkvN0Jol3s/p0/DhhzBt2tn72wUEwF13wUMPwdVXO3+cg3K9devWcdNNN+VL\nUKGhoUydOpUhQ4YAsG/fPr7++muGDh2KMcbpGJQmKKW8THlIUJ/9/hnDvx3OjwN/pF6VeiXax6FD\n8MYb8M471uMpwLq33YMPwsCBUL26CwNWRdauXTs2btyYb1utWrXYt29fbveePWcJSrv4SkjHVYpO\n60rltfLvlTy69FGW3LukRMlp92544AHrLg6vvBJLYiJcey3ExFiv/d//aXJyxt2/iz/88AM7duzI\nty00NJQJEyY4TU6F0Vl8SqlSdfjMYT7r/RktarUo1vv++gtefNGaJp6dbXXbdegAr70G7du7KVhV\nZCLCiBEjOGN3P6jKlStz3333lWif2sWnlJcpD118xbFvH4wbBzNmWDPxKlaEAQNg5Ei47DJPR6dy\nrFq1iq5du+ZLUKGhobz//vvnTFA6BqWUj9AEZTl0CF5+Gd5/37p+yc8P+vWD55+3HuanvIeI0KpV\nK7Zu3Zpve506ddi7dy8VKlQo9P06BuViOq5SdFpXqjhSU+GVV6wbs775pvWYi3vugT/+sFpReZOT\nnlvF4676+u677/jzzz/zbQsNDWXixInnTE6F0QSllHKbtMw0th3eVqSyItb98Zo2hVGjICnJuvXQ\n1q3w6afaneetRIThw4cXGHuqWbMmvXv3Pq99axefUl6mrHTxZUs293x5DwEVApjdY3ahZX/7DZ54\nwnoQIFgP+Zs6FW68sRQCVedl6dKl3HXXXQXGnj755BPuvPPOIu1Dx6CU8hFlIUGJCEOXDWX7ke0s\n77OcoIpBDsudPg3PPgtvvWW1oKpVg5degvvvL/3HoaviExGaNGnCrl278m1v2LAhu3btKvKTkHUM\nysW077votK7Kn1fWvsKP+35k0d2LnCanmBirO+/NN60JEE88AXv2wMMPFz056blVPK6ur8WLF3Pg\nwIF820JDQ5k0aVKRk1NhXJqgjDFVjDFfG2OSjDF7jTH3OCkXYIx5zxhzyBhzzBizyBhzoStjUUp5\nxsdbPuajLR+x7L5lVAqqVOD1+HjrJq3dullPrG3dGjZvhilToEoVDwSsSiQ7O5vhw4eTlJSUb3ud\nOnXo0qWLS47h0i4+Y8w824+DgCuAJUB7EdlpV+5J4B6gM3AK+AgIEZFeDvapXXyqXPH1Lr5F/1tE\nkxpNaFStUb7tIvDBB/DUU9atiUJDYfx4ePRR60GByrd8+eWX9O/fv8DY0+eff86tt95arH25fQzK\nGBMCnASaishftm2zgP0iMsqu7DvAKRF52rZ+GzBJRJo42K8mKFWu+HqCcuTgQRg0CL791lq//Xbr\nHnp163o2LlUy2dnZ1K9fn7i4uHzbo6Ki2L59e7G790pjDKoRkJmTnGy2AlEOyn4MdDDGXGhLbPcB\nS10Yi9tp33fRaV2Vb19+Cc2aWcmpalXrOUyLF7smOem5VTyuqq+//vqLffv25XueU2hoKFOmTHHJ\n2FMOVyaoMCDRblsiEO6g7G5gH3AASAAaAy+5MBallIclJkL//tZ404kT1mPVt2+Hu+/Wx1/4uoYN\nG/L777/TpUsXgoODMcbQoEEDbnTxdQGunMiZBETYbYsATjso+x4QCFQBkoGngOVAO0c7HjBgAJGR\nkYB148GWLVsSHR0NnP1GoOvevZ7DW+Lx9vUc3hKPs/VFyxfxT8I/DL17aL7XK1aMpk8f+OefWPz9\nYcqUaP77X/jhh1h273bd8XO2eUt9ePt6zjZX7K9x48YMGzaM7t27s2zZMoYPH84PtgvZinJ+x8bG\nFugitOfqMagTQFSeMahPgAMOxqC2A6NEZLFtvRLW+FV1ETlhV1bHoFS54itjUGfSz9BpVic61evE\n+E7jAesu46+9Zl3blJ0NV15p3X28cWMPB6u8mtvHoEQkGfgKeNEYE2KMuQboCji6hHwT0M8YE2GM\n8QcewUpkJxyU9Ur233SVc1pXZU9GVga9P+9N4+qNGXfDOMDqxuvWDZ55xkpOTz0F69e7NznpuVU8\nvlZfrr5Q9xEgBDgCzAUeEpGdxpgOxphTecqNBNKAPcBh4Bagh4tjUUq5gYgwZPEQ/IwfH97xIcYY\nfv4ZrrgCvvnGupZp8WKYMAFK8Iw6pXLprY6U8jLe3sU3auUoVsetZmW/lQRXDOHtt2H4cOuu461b\nw2efgW3IWKki0XvxKeUjvD1BxcbF0qxmM0L9qnH//dYYE1gX3L7+OgQGejY+5Xv0Xnwu5mt9uZ6k\ndVW2REdGk5ZQjeuvt5JTaCjMn2/dU6+0k5OeW8Xja/WlCUopVSy//GJ15f38M1xyCfz0E/znP56O\nSjnTsWNHhg4d6ukwSkS7+JTyMt7cxTd/PgwcaD319tprrbtE1Kjh6ahc79ixYzz//PMsW7aM+Ph4\nKleuTLNmzXj66afp1KnTOd//ww8/0LFjR44dO0bVqlVLIWL45JNPePTRRzl9Ov+lpwkJCfj7+xMa\nGloqcZSEsy4+feKKUsqp3cd3s+f4Hm5tcDvPP2/d3BVgyBB4+20ICPBsfO7Ss2dPUlNTmTFjBvXr\n1+fIkSP88MMPHD9+vEjvFxGXfdHIyMjAvwjTIXOOaa9y5crnHYPHiIhXL1aI3mf16tWeDsFnaF0V\nj7ec8wdOHZDIqZHy7vrpcvfdIiDi5ycybZpIdrano7O449xKSEgQY4ysXLnSaZk5c+ZI69atJTw8\nXGrWrCm9e/eWAwcOiIhIXFycGGPEz88v99+BAweKiEh0dLQ89thj+fY1YMAAueOOO3LXo6Oj5eGH\nH5aRI0dKjRo1pE2bNiIiMnnyZGnevLmEhobKRRddJEOGDJHExEQREYmNjS1wzLFjxxY45urVqyUy\nMlLGjRsnDz74oERERMjFF18sEydOzBfT7t275brrrpOgoCBp3LixLF26VMLCwuSTTz45n6p1ynbO\nF/j7r2NQSqkCElITuHXurfRp+CgLnhnI/PkQFgZLl8LQoWX7XnphYWGEhYURExNDWlqawzIZGRm8\n+OKLbNu2jSVLlnD8+HHuvfdewHoe0pdffgnAzp07iY+PZ9q0acWKYe7cuQCsXbuWWbNmAVChQgWm\nTZvGH3/8wbx589i0aROPPfYYAFdffTVTp04lJCSEw4cPEx8fz8iRI53uf+rUqTRv3pxff/2Vp556\niieffJKNGzcCVqOle/fuBAQE8PPPPzNz5kzGjh1Lenp6sT6DSzjKWt604CXfJpUqLZ4+51MyUuS6\nGdfJwFnPyeWXZwuI1Kol8uuvHg2rVH311VdSrVo1CQoKkvbt28vIkSNl48aNTsvv3LlTjDG5rajY\n2Fjx8/OT48eP5ytX1BZUixYtzhnj8uXLJSgoKHd95syZEh4eXqCc/TEjIyPl3nvvzVemYcOGMn78\n+Nz9+vv7S3x8fO7rP/30kxhjtAWllPKsITFDCD7Rmu9GjWXHDkPjxrBhA7Rs6enISk+PHj04ePAg\n33zzDbfddhvr16+nXbt2TJgwAYAtW7bQvXt3IiMjiYiIoHXr1hhj2Ldvn0uOf+WVVxbYtmrVKm66\n6Sbq1KlDREQEPXv2JD09nUOHDhV7/82bN8+3Xrt2bY4cOQLArl27qF27NrVq1cp9vXXr1vkerVFa\nNEGVkK9dT+BJWle+5XoZw4YXJ7J/v6FDB1i3zppO7o3ceW4FBATQqVMnRo8ezdq1axk8eDAvvPAC\np06d4pZbbiEsLIw5c+bwyy+/sHz5ckTknN1gfn5+BSZOZGRkFChnP+Nu3759dOnShaioKL744gu2\nbNnC9OnTAYrV9ZZTX/aTLowxZGdnA84nW3iCzuJTSuVatAgevbch6elw553WhbhBQZ6Oyjs0adKE\nzMxMfvvtN44dO8b48eO5xJa5d+zYke+PeoBtemNWVla+fdSoUYP4+Ph827Zu3Uq9evUKPfYvv/xC\nRkYGkydPzj1OTExMvjIBAQEFjlcSTZo04cCBAxw6dCi3FbVp06bcBFaatAVVQnmfr6IKp3XlG+bO\ntZJSejr897+wYIH3Jyd3nFsnTpygU6dOzJ07l+3btxMXF8fnn3/OxIkTufHGG2natCmBgYG8+eab\n7N27lyVLlvD888/n28cll1yCMYYlS5Zw7Ngxzpw5A8ANN9zAsmXLWLx4Mbt372bEiBH8+++/54yp\nYcOGZGdnM2XKFOLi4pg3b16BiReRkZGkpqby/fffc/z4cVJSUgrspyj11blzZxo1akS/fv3Ytm0b\nGzZsYMSIEfj7+5d6y0oTlFKK996Dvn0hKwtGjYK33oIKFTwdlWeEhYXRvn173njjDaKjo7n88ssZ\nPXo0ffr0Yf78+VSvXp1Zs2axaNEioqKieOmll5gyZUq+fdSuXZuxY8fy7LPPUqtWrdzZdoMGDWLQ\noEEMHjyYDh06EB4eTs+ePfO911ESaNasGdOmTWPKlClERUUxffp0Jk2alK9M+/bteeihh7jnnnuo\nWbMmEydOdPj5HO0/7zZjDAsXLiQ9PZ22bdsycOBARo8eDUBQKX9j0TtJlFDep1KqwmldFU9p3kki\nNi6WD9+swqeTWwDWIzKeeqpUDu0Sem4VT0nra+vWrbRq1YrNmzfTqlUrl8eld5JQSuXzW/xWbh/8\nK8mrhmGMdWeIhx/2dFTKGyxcuJDQ0FAaNmzI3r17GTFiBK1atXJLciqMtqCU8jKl0YL6+8Remndf\nyZkfh1ChAsycCX36uPWQyofMnj2bcePGsX//fqpUqULHjh2ZPHkyNdx040V9HpRSPsLdCepw0hEu\nu305iWv6ERBgTYbo3t1th1PqnPR5UC6m1/YUndaV9xCBK7v/lJucvvrKt5OTnlvF42v1pWNQSpUT\nItZTbw+s7E5goPD114Zbb/V0VEo5p118SnkZd3TxZWdbyendd62n3i5cCLfc4tJDKFViOotPqXIq\nO9u68Pb9963ktGgR3Hyzp6NS6tx0DKqEfK0v15O0rjxHxHo8xvvvW3eFiIkpW8lJz63i8bX60haU\nUmWUCNw+cDvLPmlGYKCVnDp39nRUShWdjkEp5WVcNQZ139BdfPrmZVSsaE2I6NLFBcEp5QalMs3c\nGFPFGPMcdMVYAAAgAElEQVS1MSbJGLPXGHNPIWWvMMb8YIw5bYyJN8Y85spYlCrPHn32Hz598zL8\n/IRPP9XkpHyTq8eg3gFSgRpAH+BdY0wT+0LGmGrAMuBdoArQAFjh4ljcytf6cj1J66p0jZ4Qz9sv\nW4+BmDHD0Lu3hwNyIz23isfX6stlCcoYEwL0BEaLSIqIrANigL4Oig8HlovIfBHJFJEzIrLLVbEo\nVV59/LEw/pkLAesO5f36eTggpc6Dy8agjDEtgXUiEppn2wjgOhHpZld2JbAdaI3VetoAPCoiBR6M\nomNQqrwp6RjUvHlw333W5IjJk2HYMDcEp5QblMZ1UGFAot22RCDcQdmLgVbAjcAOYCIwD+jgaMcD\nBgwgMjISgMqVK9OyZcvcW8bnNFl1XdfL0nqOopZPSIimb18QiWXQIBg2zLs+j67ret71nJ/j4uIo\njKtbUGtFJCzPtuHA9Q5aUL8Bm0VksG29KnAMqCQip+3KemULKlafQ1NkWlfFU9wW1OrV1l0h0tPh\nmWfg5ZfdGJyX0XOreLy1vkpjFt9uoKIxpn6ebS2A3x2U3QbY/wYKULrPE1bKx23ZAt26WcnpkUdg\n/HhPR6SU67j0OihjzKdYieZ+rC68b4CrRWSnXbmOwBdAR2An8BpwhYhc72CfXtmCUspditqC2rMH\nWrZJIjkhjLvvhrlzwU/vDaN8UGk9buMRIAQ4AswFHhKRncaYDsaYUzmFRGQ1MApYChwCLgXudXEs\nSpVZBw9Cu+sTSU4II7pTOp98oslJlT0uPaVF5KSI9BCRMBGJFJEFtu1rRSTCruz7InKxiFQTkW4i\ncsCVsbib/WC2ck7ryrUSEqBddAIn4ivR4so0Fi8MICDA01F5hp5bxeNr9aXfuZTyISkpcG3nk/y7\npzKRDVL5fnkgYWHnfp9SvkjvxaeUl3E2BpWZCT16ZPPNN37UqJXGLxsDqVvXAwEq5WL6yHelfJgI\n3H8/fPONH1WqCLErNTmpsk8TVAn5Wl+uJ2ldnb+nn4aZMyEkBJYsMTRt6umIvIOeW8Xja/WlCUop\nL/f66/Daa1CxInzxBbRv7+mIlCodOgallJfJOwY1Y2Y2gwZa3yPnzLHutadUWVMa9+JTSrnQopgs\nBg+2fp46VZOTKn+0i6+EfK0v15O0rorvxx+FO3tnIdkVePLpTB5/3NMReSc9t4rH1+pLE5RSXieS\nm25LJSs9gH4D05nwsnZ0qPJJx6CU8iJxcVCv0UHIqM2td6QS81UQFTU/qTJOr4NSyssdOQKdb8qG\njNq0uSaFrz7T5KTKN01QJeRrfbmepHV1bqdOwa23wp97/IAPWbEkmKAgT0fl/fTcKh5fqy9NUEp5\nWFoa9OhhPdupfn2AJ6lUydNRKeV5OgallAdlZcHdd1sX4NaqBevWQf36xXuirlK+Tq+DUsrLiMCD\nD2fwxRf+VKoE334Ll15asNyJEyeYPXs2DRo0oGnTplxyySX46cOfVDmgLagSio2NJTo62tNh+ASt\nK8eeGZ3OhPEBBARm8f13Fbj2Wmu7/d3M9+/fT2RkJEFBQYgImZmZXHLJJTRr1ow2bdoQFRVFVFRU\nuUxcem4Vj7fWl7aglPIiU6ZmMWF8AMYvi88W+OUmJ0cuvvhi+vXrx5w5c8jIyABgz5497Nmzh5iY\nGEJCQsjKyiIjI4O6devmJq5rr72Wa665ppQ+kVKupy0opUrZ7DnZ9OtrtXQ+/CiLIYMr5Hvd0fOg\nDh48SP369UlNTS3SMfz8/GjZsiWbN292TdBKuZFeB6WUF1i6FAYMsJLP+AnpBZKTM7Vr12bIkCEE\nBgYWqXxQUBBz584tcZxKeQNNUCXka9cTeJLWlWXdOujVS8jOqsDQESmMeiqgWO8fM2YMFSqcO6GF\nhoYybdo0GjduXNJQfYaeW8Xja/WlCUqpUrBtG3TpAikphsGDYerE4GLvo3r16jz22GMEneMK3szM\nTK666qqShqqU19AxKKXc7O+/4Zpr4NAh6NkTFiyg0FsYORqDypGYmMjFF19MUlJSoccMDg5m3Lhx\nDBs2DGMKdO0r5VV0DEopDzh0CDp3tv694QaYO7fw5HQulSpV4qmnniI4uPAWWEpKCs899xw33HAD\nR48eLfkBlfIglyYoY0wVY8zXxpgkY8xeY8w95yjvb4z5nzFmnyvjKA2+1pfrSeW1rhISoFPnDP7+\nG666ChYuxCX31xs2bBgBAfnHr4KCggq0lJKTk1m3bh2XXXYZK1euPP8De6Hyem6VlK/Vl6tbUO8A\nqUANoA/wrjGmSSHlnwQOuTgGpTwuORluvi2NP3b4U/fSZJYuhfBw1+w7NDSUMWPGEBISAkDFihVp\n0qQJl156aYGWVUZGBidPnuSOO+5g5MiRuddRKeULXDYGZYwJAU4CTUXkL9u2WcB+ERnloHw94Btg\nOPChiNR1sl8dg1I+JSMD7uiewbdL/alU8xTbNkVQ1+HZ7VhhY1A50tLSuOiiizh+/DgRERHs3LmT\nqlWrMnToUObOnUtycnKB94SEhHDppZcSExNDvXr1ivuxlHKb0hiDagRk5iQnm61AlJPybwDPYLW4\nlCoTsrLgvj6ZfLvUn6CIJH5aHV6s5FRUgYGBjB8/HoDZs2dTu3ZtgoKC+OCDD5g3bx4RERFUtBvs\nSk5O5o8//qBZs2Z6jZTyCa5MUGFAot22RKBAx4YxpgdQQURiXHj8UuVrfbmeVF7qKjsbhgzJ5vPP\nKlIxOJnY70Jo2tR9M+gGDx7M6tWr6dq1a77tXbt25Y8//qBVq1a53YBnY8zmzJkzPPDAA9xzzz3n\nnA3o7crLueUqvlZfrrwXXxIQYbctAjidd4OtK/BV4NacTefa8YABA4iMjASgcuXKtGzZMveGhzkV\nXtrrOTx1fF9a/+2337wqHnesX399NEOHwsyZa/DzT2fFso60bePn1vMrp4WU9wageV9fv349Q4YM\nYd68eaSlpeXbb3JyMgsXLuT7779n/PjxPPDAA6VaX65a/+2337wqHm9f95b6yvk5Li6Owrh6DOoE\nEJVnDOoT4EDeMShjTAvgZ+A4VnIKACoBR4B2IrLPbr86BqW8mgg89RRMnAiBgfDNN3DjjSXfX1HG\noIpj/fr1dO/encTExAKJCqxrpsaOHcvIkSP1minlEc7GoFx6oa4x5lNAgPuBVliTIK4WkZ15yvgB\n1fO87RrgTVv5Y/bZSBOU8nZjx8ILL1jXN339tXXHiPPh6gQFkJCQQN++fVm1apXTCRStW7fms88+\no2bNmi49tlLnUloX6j4ChGC1huYCD4nITmNMB2PMKQARyRaRIzkLVqsrW0SO+lImsu+KUc6V5bqa\nONFKTn5+8Omn55+c3KVy5crExMQwbdo0QkJCHF4z9dNPP3HZZZfx3XffeSjK4ivL55Y7+Fp9uTRB\nichJEekhImEiEikiC2zb14qI/fhUznt+cDbFXClv9vbb8OST1s8zZkDv3p6N51yMMQwZMoTNmzdT\nv359h9dMJSQk0K1bN5544gm9Zkp5nN6LT6kSmDEDBg2yfh792n5e+r+LXbZvd3Tx2UtNTeWJJ55g\n9uzZDrv8goODufTSS1m0aBH169d3ayxK6b34yrCOHTsydOhQT4dRbsycCYMHWwmk/5NbXZqcSktQ\nUBDvvfce8+fPd3jNVEpKCjt37qRFixbMnj3bQ1Gqck9EvHqxQnS9o0ePysMPPyyRkZESGBgoF1xw\ngdx4443y/fffF+n9U6ZMEWOMHD9+3C3xOTJz5kwJCwsrsP3kyZOSlJRUanEU1+rVqz0dgst8/LGI\nMdkCIt3+u8Etx3DXOe/MgQMHpG3bthIaGipYk5zyLSEhIdK7d285depUqcZVFGXp3CoN3lpftnO+\nwN//ctuC6tmzJ7/88gszZsxgz549LFmyhFtvvZXjx48XeR+u6oopal+/iDicBly5cmVCQ0PPOw5V\nuI8/hiFDBBFDpwe+Y+HbbT0dkkvUrl2bdevWOb1LenJyMosXL6Zx48b6CHlVuhxlLW9acMO3yYSE\nBDHGyMqVK52WmTNnjrRu3VrCw8OlZs2a0rt3bzlw4ICIiMTFxYkxRvz8/HL/HThwoIiIREdHy2OP\nPZZvXwMGDJA77rgjdz06OloefvhhGTlypNSoUUPatGkjIiKTJ0+W5s2bS2hoqFx00UUyZMgQSUxM\nFBGR2NjYAsccO3asw2NGRkbKuHHj5MEHH5SIiAi5+OKLZeLEifli2r17t1x33XUSFBQkjRs3lqVL\nl0pYWJh88sknJa3WMu2DD0SsK55Eujzyg2RnZ7vtWO4454tq/fr1csEFF0hgYKDD1lRwcLBMmDBB\nsrKyPBajKnvQFtRZYWFhhIWFERMT4/DCRbBaNS+++CLbtm1jyZIlHD9+nHvvvReAOnXq8OWXXwKw\nc+dO4uPjmTZtWrFiyLkX2tq1a5k1axYAFSpUYNq0afzxxx/MmzePTZs28dhjjwFw9dVXM3XqVEJC\nQjh8+DDx8fGMHDnS6f6nTp1K8+bN+fXXX3nqqad48skn2bhxI2B9KenevTsBAQH8/PPPzJw5k7Fj\nx5Kenl6sz1BefPAB2G60wMSJsPit68rsBa3t2rVj165d3HzzzQVukwTW2NSLL75IdHQ0hw8f9kCE\nqlxxlLW8acFN3ya/+uorqVatmgQFBUn79u1l5MiRsnHjRqfld+7cKcaY3FbUlClTxM/Pr8AYVFFb\nUC1atDhnjMuXL5egoKDc9ZkzZ0p4eHiBco5aUPfee2++Mg0bNpTx48fn7tff31/i4+NzX//pp5/E\nGOOWFpS39nsXxTvvnG05TZpUOsd01zlfHNnZ2fLxxx9LSEiIGGMKtKT8/f2lcuXKsnz5co/G6cvn\nlid4a32hLaj8evTowcGDB/nmm2+47bbbWL9+Pe3atWPChAkAbNmyhe7duxMZGUlERAStW7fGGMO+\nfa55tuKVV15ZYNuqVau46aabqFOnDhEREfTs2ZP09HQOHSr+I7OaN2+eb7127docOXIEgF27dlG7\ndm1q1aqV+3rr1q3x8yu3p4NDr74K//2v9fPkyTB8uGfjKU3GGAYNGsSWLVto0KCB02umevToweOP\nP66tb+UW5fovUkBAAJ06dWL06NGsXbuWwYMH88ILL3Dq1CluueUWwsLCmDNnDr/88gvLly9HRHJ/\nEVu1auVwn35+fgUmTjiaBGE/qWHfvn106dKFqKgovvjiC7Zs2cL06dMBSvTL7+/vn2/dGEN2djbg\nfLKFu+TcKNJXiMCzz8LTT4MxwltvZzFsmKej8ozLLruM7du3M2DAAIcTKFJSUvjoo49o2bIlf/75\nZ6nH52vnlqf5Wn2V6wRlr0mTJmRmZvLbb79x7Ngxxo8fT4cOHWjUqBGHDx/O90c955HbWVlZ+fZR\no0YN4uPj823bunXrOY/9yy+/kJGRweTJk2nbti0NGjTgwIED+coEBAQUOF5JNGnShAMHDuRrmW3a\ntCk3gZVn2dkwdCi8/DL4Vcgm/O5HuP3efz0dlkcFBgbyzjvv8Nlnn1GpUiWHz5natWsXLVq0YMeO\nHR6KUpVF5TJBnThxgk6dOjF37ly2b99OXFwcn3/+ORMnTuTGG2+kadOmBAYG8uabb7J3716WLFnC\n888/n28fBw4cwBjDkiVLOHbsGGfOnAHghhtuYNmyZSxevJjdu3czYsQI/v333H/gGjZsSHZ2NlOm\nTCEuLo558+YVmHgRGRlJamoq33//PcePHyclJaVEn79z5840atSIfv36sW3bNjZs2MCIESPw9/d3\nS8vKV+7/lZkJAwfCW2+Bf0A2oX36893EAURWjvR0aF6hS5cu7Ny5k6uuusrhc6bCw8OpU6dOqcbk\nK+eWt/C1+iqXCSosLIz27dvzxhtvEB0dzeWXX87o0aPp06cP8+fPp3r16syaNYtFixYRFRXFSy+9\nxJQpU/Lto3r16owdO5Znn32WWrVq5c62GzRoEIMGDWLw4MF06NCB8PBwevbsme+9jpJAs2bNmDZt\nGlOmTCEqKorp06czadKkfGXat2/PQw89xD333EPNmjWZOHGiw8/naP95txljWLhwIenp6bRt25aB\nAwcyevRowLrDQHmUlgb/+Q/MmgXBIVmEDriLz0bfR5uL2ng6NK9y4YUXsm7dOp555pl8XX7BwcEs\nWrSISpUqeTA6VdbovfgUYHVDtmrVis2bNzsdXyurEhKgRw+IjYVKlbIJ7N+TSYN70ad5H4/EUxr3\n4nOFjRs30r17d06cOMHo0aN57rnnPB2S8lGl8jwod9AE5R4LFy4kNDSUhg0bsnfvXkaMGIExptzd\nKeDff+G222DHDrjwQvhi0RkOhi6jV9NeHovJVxIUQGJiIvPnz+f+++/XWaCqxPRmsS7ma3259k6f\nPs2jjz5KVFQUffv2JSoqiuXLl7vlWN5aV9u3Q/v2VnJq0gTWr4erW4d6NDn5mkqVKvHggw96LDl5\n67nlrXytviqeu4gqi/r27Uvfvn09HYbHrF4N3bvDqVNw7bWwcCFUrerpqJRSeWkXnyp35s2D/v0h\nIwN69YLZs8Gb5ob4UhefUq6gXXyq3MvOhjFj4N57reT0+OPQbfSnUDHV06EppRzQBFVCvtaX60ne\nUFdnzsBdd8GLL4KfH0yZApF3T2X82pc4k37G0+EpoF69ekyePLlY7/GGc8uX+Fp96RiUKvP++Qe6\ndYOtW6FSJZg/H05eNI8nv5/EukHrqBZSzdMhlhsDBw7k+PHjxMTEFHjtl19+0eeaqXzK7BjUqlWr\n2LBhA/fffz81atRwQ2TKF6xbBz17wpEj0LAhLF4M/1RcQd+v+7Ky30our3m5p0MsoCyPQRWWoLxF\nRkZGgXtZKvcqV2NQIsJDDz3E2LFjqVOnDr1792bTpk2eDkuVIhH46CPo2NFKTp07w8aNkF55O/d9\ndR9f9P7CK5NTeWbfxefn58eHH37IXXfdRVhYGPXr1899jlqOgwcPcvfdd1O1alWqVq1Kly5d8t20\n9u+//6Z79+5ceOGFhIWFceWVV7JkyZICxx07diyDBw+mSpUq9OnjmQu0VUFlMkFt3LiRgwcPkp6e\nTlpaGl9++SVt2rQpcPPV8+FrfbmeVNp1lZxs3VPv/vvPToZYuhSqVIEGVRuw6O5FXHvJtaUakyqZ\nl156iR49erBt2zb+85//MGjQoNx7W6akpNCuXTtCQ0P58ccf2bBhA7Vr1+bGG28kNdWa+JKUlMRt\nt93GypUr2bZtG7169eLOO+9k9+7d+Y4zZcoUmjRpwubNm3n55ZdL/XOWFp/7u+XoIVElXYAqwNdA\nErAXuMdJuZHAduAU8BcwspB9FvvhV927dy/wkLUmTZoUez+F8dYHf3mj0qyr//1P5PLLrQcMBgeL\n+OIT7EtyzvsK+4d35hUZGSmT8jwV0hgjzz77bO56ZmamhISEyNy5c0VE5OOPP5Y6derk20dmZqZU\nq1ZNPv/8c6cxtGvXLvfhnTnH7dq1a4k+j6/x1r9bOHlgoasnSbwDpAI1gCuAJcaY30Rkp4OyfYFt\nQANghTFmn4h8dr4BHD58OPfZTTnCwsJ4+umnz3fX+fjac1U8qbTqasECGDIEkpLgssvgiy/gcu3F\n82nNmjXL/blChQrUqFEj98GbW7ZsIT4+nvDw8HzvSUlJ4a+//gKsR4G88MILLFmyhPj4eDIyMkhL\nS6NFixb53nPVVVe5+ZN4B1/7u+WyBGWMCQF6Ak1FJAVYZ4yJwUpEo/KWFZHX86zuNsYsAq4BzjtB\nvfvuuwUGmI0x3HXXXee7a+Wl0tJgxAh4+21r/e674YMPwO7vlvJBhT14Mzs7m1atWrFgwYICv/NV\nbbcFGTFiBCtWrGDSpEk0aNCAkJAQ+vbtW+AhoDp70Du5cgyqEZApIn/l2bYViCrCe68Ffj/fADIy\nMnjjjTdIS0vL3RYQEMCDDz7o8sdI+Fxfrge5s662bYPWra3kFBBg/fvpp1ZyysrO4pUfXyEpPclt\nx1eec8UVV/C///2PatWqcemll+ZbKleuDMC6devo168f3bt35/LLL6d27dq5ravyyNf+brkyQYUB\niXbbEoFCv8caY8YCBphxvgEsWrSIzMxM+/0zdOjQ89218jLZ2fD661Zy2r4dGjSwppT/979gjDW2\n+ujSR/l+7/f4++mUYW9y6tQptm7dmm+Ji4sr9n7uu+8+qlSpQrdu3VizZg1xcXGsWbOGkSNH5iah\nRo0a8fXXX/Prr7+yfft2+vbtm+8LrPJurhyDSgIi7LZFAKedvcEY8yjQB+ggIhnOyg0YMIDIyEgA\nKleuTMuWLXP7UnO+EURHR/PKK69w+nT+wzVv3py//vor90mfecvreumt53DF/g4fhvfei8Z6KZY7\n7oB586IJDT1bfo1Zw8YDG3mp3kusX7ve45/fk/XlTeuHDh3ixx9/5Iorrsj3Oe+8806MMfz555/E\nxsYSHR2NMYYdO3ZQtWrV3PenpaXlTiMPDg5m2rRpfPDBB9x1110kJiZStWpVWrZsSZUqVQDo3bs3\nr7/+Otdddx1VqlShS5cuNG3aNPe4sbGx+RKWp+vH3es52zwdT87P5/pi4rILdW1jUCeAqJxuPmPM\nJ8ABERnloPwg4AXgWhH5p5D9SlFi3LFjB23atMn3GPSwsDBiYmLo2LFjcT+O8kIiMHcuPPooJCZC\nzZowfTrcfnv+ch9s/oBX173KT4N+4oKwCzwT7HkoyxfqKuWI2y/UFZFk4CvgRWNMiDHmGqArMNtB\nMPcB44HOhSWn4nj99dcLDHzm/eblavbfdJVzrqiruDgrEfXtayWnbt2s5zjZJ6cf//mRF2Jf4Ns+\n3/pkclLFo7+HxeNr9eXqC3UfAUKAI8Bc4CER2WmM6WCMOZWn3EtAVWCTMea0MeaUMeadkh40ISGB\nBQsWkJWVlbstJCSEJ598EmMKJGXlQzIzrRu7RkXBsmVQuTJ8/DF8/TU4uoNVu4vbsWbgGhpUbVD6\nwSqlXKpM3Itv8uTJPPfccyQnJ+duCw4O5vDhwwWukVC+49dfrbtB5DyF/q67YNo0qFXLs3G5m3bx\nqfKmzN6LLzs7m9dffz1fcqpYsSJ9+vTR5OSjEhJg+HBrht7mzVCnjnWT1wULyn5yUkqd5fMJ6rvv\nviswc69ixYoMHz7crcf1tb5cTypqXWVlwfvvW3cdnzLFmkr++OPw++/QpYt7Y1S+SX8Pi8fX6svn\nE9SECRNISsp/IWbLli1p3LixhyJSJbFqFbRqBQ89BMeOQYcOsGkTTJ3q/I4QSelJPLLkEX3goFJl\nlE+PQf39999ERUXl3rkYIDw8nDlz5tC1a9fSClGdh1274OmnYeFCa/2SS2DiROjVy7rg1pn0rHTu\nmHcHdSLq8OEdH5apyTA6BqXKG2djUD79RN2pU6fmm7kHEBgYyO32c4+V1/n7b+vx67NnW115oaHw\nzDPW2FNwcOHvzZZsBi0aRGCFQN7r8l6ZSk5KqbN8tosvOTmZ6dOnk5Fx9gYUwcHBDBs2jAoVKrj9\n+L7Wl+tJeetq/36rG++yy+CTT8DPz5qpt3s3PPvsuZOTiDByxUjiEuKY32s+Ff18+juWOk/6e1g8\nvlZfPvvb/emnnxbYJiI8+OCDHohGncu+fTBpkjUJIi3NSkz9+sHzz0P9+kXfz6Jdi1jx1wrWDFxD\niH+I+wJWSnmcT45BiQgNGzbMd1diPz8/evXqxYIFC0o7RFWIbdusMaX5862LbsG6numFF6BJk+Lv\nLys7i5OpJ6keUt2lcXoTHYNS5U2ZGoNav349hw4dyrctKCiIJ5980kMRqbxEIDYWXnsNli+3tvn5\nWc9pevppsHtWXLFU8KtQppOTUuosnxiDOnMm/zTiV199Nd+FuQD16tXjyiuvLLWYfK0vtzScOQMf\nfghXXgk33GAlp+Bg6NEjlj//hHnzzi85KWVPfw+Lx9fqyycSVOXKlenduzebNm0iPj6eFStWuP2R\n7qrofv/dusN47drwwAPWLYqqV4exY62xp6FDoV49T0eplPI1PjEGFRYWxpkzZwgODiYwMJDk5OR8\nz3CJiIjgyJEjBAYGejDS8uX0afjqK+vGrT/+eHb71VfDww9b1zGd70OM95/az2PLHmNBrwUEVAg4\nv535EB2DUuWNT49B+fn5ISIkJycX6NoLCAjg4Ycf1uRUCjIzYcUKmDPHurA259FbYWHWYzAeegia\nN3fNsU6mnOSWObfQr0W/cpWclFJn+UQXn/1j3PPy8/Nj3bp1rFq1qlS/dfpaX25JZWVZLaTHH4eL\nLrKevzRvnpWcrrsOPvgADh6Ed95xnpyKW1cpGSl0nd+Vm+rfxP9d/X/n/yFUmVVefg9dxdfqyyda\nUPYPIswrNTWVtWvX0q1bN6pUqcK0adPo0aNHKUZX9qSnw+rVVhfewoVw5MjZ1y67zGot3XcfREa6\n/tiZ2Znc/eXd1K1Ul9dvel3vEqFUOeYTCSo7O/ucZZKSkkhLS2P//v2lEBFue1Kvpxw+DN9+a828\nW7rUemptjksvhZ49reuXrrqq8HvkOVKcupq7bS6pmal83vtz/IxPNPCVB5W130N387X68plJEvZ3\nLLcXEhLCjBkzuOuuu0opMt+Wng4bN1oJadkya+ZdXpdfbiWlnj2trrvSasiICGlZaQRVPM8ZFj5M\nJ0mo8sanJ0kUlqD8/PwIDw/n22+/pW3btqUWU2xsrE99G0lLsxLSDz9Yy08/nZ3kANaMu44d4eab\n4bbbrGcyuUpx6soYU66TkyoeX/s99DRfqy+fSFAREREF7hwB1gy+Cy+8kNjYWCLdMSDio0Ssm7Ju\n2gQ//wwbNlhLnpn5gHWroZtvhltvhWuvPfeNWpVSqjT5RBdf27Zt2bhxY77twcHBtGzZkqVLl1K5\ncmUPRecdTpyALVusFtLPP1uLg3xOs2Zw/fXWct11ULNm6cdqT0R0IoQd7eJT5Y1Pd/FVrVo133pI\nSA094oIAAAoJSURBVAg9e/Zk+vTp+Pv7eyiq0peaCjt3wvbt+ZeDBwuWrVIFWreGNm2sf6++2rq7\ngzfZEr+F4d8OZ2W/lVTwc/8jUpRSvsUnElSNGjVyfw4ODmbUqFGMGjXKo9+83dWXK2IlnN27Yc+e\ns8uuXda/ds9nBCAkxJrI0LatlZDatLEeYeEtDRNHdfXXib/o8mkX3rrtLU1OqsR8bUzF03ytvnwi\nQdW09UWVhZl6InD0qHWPun374J9/zv7855/WYnezjFx+ftC4sdVVl3epV896zVccTjrMzXNuZsz1\nY+jZpKenw1FKeSmfGIN6/fXXefHFF1mxYkWpztQrDhFISrLGfg4ftv7NWeLjzyahffusrrrCVK8O\njRpZM+kaNjz7c+PG539/O087lXaK6JnRdLusG2Oix3g6HK+kY1CqvHE2BuXSBGWMqQJMBzoDR4FR\nIjLPSdlXgcGAANNF5Ckn5eT06dOkpaVRrVo1l8VaGBGrFXPihLUcP+7856NHzyYiZy0fe1WqQN26\n1nLJJWd/rlfPSkRVqrj383nSu5veZdvhbbxz+zs6OcIJTVCqvCmtBJWTjAYBVwBLgPYistOu3IPA\nE8ANtk3fA9NE5AMH+yzwRF172dlWqyQlJf9y5ox11+3Tp+HUqYI/O9t28mTBKdkFxQLR+bYEBcGF\nF0KtWmeXCy6w/s1JRnXqQHj4ufZdtuTt9xYRBNG7RBRCE1TR+dqYiqd5a325fRafMSYE6Ak0FZEU\nYJ0xJgboC4yyK94PmCQi8bb3TgKGAAUSFFjX6eRNPMnJ+dfP1WVWEoGBUK2atVStenbJWT96FNq3\nhxo1ziaj8HDvmZjgrYwxGLSSlFLn5rIWlDGmJbBORELzbBsBXCci3ezKJgCdRWSTbf1KYJWIVHKw\nX7F6AQsXFGRdaBocbM1qy/k3PNxaIiKK/nPVqnrRqvIcbUGp8qY0roMKAxLttiUCjjq07Msm2rY5\ncRHgD2QCWVhPCTFACpAEpJOaarWkTp4sYfTKPfyAc9/rV9nR8TmlXJugkoAIu20RwOkilI2wbXNI\n5MB5B+dq3tqX601idsXw6rpXGVdvHB07dvR0OD5DW1BFp7+HxeOt9eXsC5krR6p3AxWNMfXzbGsB\n/O6g7O+213K0dFJO+ai1+9YyOGYwU2+eqq0BpVSJuHoW36dYA0b3A62Ab4CrncziG4o1HR1gBdYs\nvg8d7POcs/iUd9lxZAedZnVido/Z3FT/Jk+H43O0BaXKG2djUK6e6/sIEAIcAeYCD4nITmNMB2PM\nqZxCIvI+sBjYDmwDFjtKTsr3/JPwD7fOvZUpN0/R5KSUOi8uTVAiclJEeohImIhEisgC2/a1IhJh\nV/ZpEakmItVF5BlXxlEaYmNjPR2CV1q8ezHD2w3n3mb35m7TulLuoudW8fhaffnEvfiU73i0zaOe\nDkEpVUb4xL34vD1GpVxJx6BUeVNaY1BKKaWUS2iCKiFf68t1l5Mp574yWutKuYueW8Xja/WlCUqV\n2JT1U+j9eW9Ph6GUKqN0DEqVyNxtc3lm5TOsHbSWupXqejqcMkXHoFR5Uxr34lPlxIq/VjB8xXBW\n9VulyUkp5TbaxVdCvtaX6yqbDmyiz1d9+Oqur4iqGVWk95TXulLup+dW8fhafWmCUsWy6eAmPur6\nEdfUvcbToSilyjgdg1LKy+gYlCpv9DoopZRSPkUTVAn5Wl+uJ2ldKXfRc6t4fK2+NEEpp7Kys/gn\n4R9Ph6GUKqd0DEo5JCI89M1DJKQlsKDXAk+HU67oGJQqb/Q6KFUsY38Yy+b4zazuv9rToSilyint\n4ishX+vLLY53N73L3O1zWXrfUsIDw897f2W5rpRn6blVPL5WX9qCUvl88ccXvLTmJX4c+CM1Q2t6\nOhylVDmmY1Aqn4+3fMwVF15BqwtbeTqUckvHoFR542wMShOUUl5GE5Qqb/RCXRf7//buJbSuKgrj\n+P/TVEws0YiPmUKlgqVYRUfSEoUi6kQwRSpVKIJaaFU01ZEIrcNm4ECqYHwgaqATtShUqBSEOlCU\niIhYKElbiiiizYM2LSbLwU3xtqY393Fyz9653w8yyM1JzmJnZa97dvZZJ7e13DJ5rGypOLcak9t4\nuUCZmVmSvMTXwabOTnFs4hhrb1hbdihWxUt81mm8xGcXODd7joF9Awz/MFx2KGZmCyqkQEnqk/SJ\npGlJY5Ieq3HsTkk/SZqUdFTSziJiaLfc1nKrzcUcWz/dSs+KHobuH1ry8+U8VpY251Zjchuvoq6g\n9gIzwPXA48Cbkm6rcfwTwDXAg8AOSY8WFEfbjI6Olh1CUyKCwS8HOTF5gpGBEbouW/pb4XIdK0uf\nc6sxuY1XywVKUg/wCPBKRJyJiMPAfipF6H8iYigiRiNiLiKOAJ8B2T397tSpU2WH0JQ93+zh4NhB\n9m/eT/eK7racM9exsvQ5txqT23gVcQV1K/BPRByteu1HoL7ngcMG4OcC4rA6rOpbxYEtB+jr7is7\nFDOzmopY31kJTFz02gSwaBM3SbsAAe8VEEdbjY+Plx1CUzat2dT2c+Y6VpY+51ZjchuvRbeZSzoE\n9AMLHXgYeA44HBFXVX3Pi0B/RDxc4+fuAF4A1kfEbzWO835bM7NlrqnHbUTEfbW+Pv8/qMsl3VK1\nzLeOGst2kp4EXgY21CpOlwrazMyWv0Ju1JX0MZUrrKeAO4HPgXsi4pcFjt0CDAH3RsSvLZ/czMyW\npaK2mW8HeoA/gI+AbeeLk6T1kiarjn0NuBb4TtLU/P1QewuKw8zMlonkWx2ZmVlncqsjMzNLkgtU\nASStlnRG0gdlx5IqSVdIGpY0LmlC0veSHig7rpQ00jKskzmXmpfbXOUCVYw3gG/LDiJxXcBxKjs3\nrwZeBfZJuqncsJLSaMuwTuVcal5Wc5ULVIskbQb+Br4qO5aURcTpiNgdESfmP/8CGAPuKjeyNDTa\nMqyTOZeak+Nc5QLVAkm9wC5gkEpHDKuTpBuB1bjN1XmttgzrWM6lxeU6V7lAtWY38HZEnCw7kJxI\n6gI+BN6fbxhsLbQM62TOpbplOVe5QF2CpEOS5iTNLvDxtaR1wEbg9bJjTcFi41V1nKhMKGeBZ0sL\nOD3TQO9Fr/UCUyXEkgXnUn0k3UGmc9XSPwwoU3W0eHoeuBk4Pv+HspJKy6c1EXF3O2JMyWLjVeUd\n4DrgoYiYXcKQcnME6GqkZZg5l+rUT6ZzlW/UbZKkK7nwHe9LVJJgW0T8VU5UaZP0FnA7sDEiTpcd\nT2oaaRnW6ZxL9ct5rvIVVJMiYobKlmAAJE0DM6n/wssyvwX4aSpj9nvljRwBPBMRI2XGlpDtwLtU\nWob9SVXLMPuPc6kxOc9VvoIyM7MkeZOEmZklyQXKzMyS5AJlZmZJcoEyM7MkuUCZmVmSXKDMzCxJ\nLlBmZpYkFygzM0vSvwqOP117EqfvAAAAAElFTkSuQmCC\n",
      "text/plain": [
       "<matplotlib.figure.Figure at 0x7fd7031117f0>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "z = np.linspace(-5, 5, 200)\n",
    "\n",
    "plt.plot([-5, 5], [0, 0], 'k-')\n",
    "plt.plot([-5, 5], [1, 1], 'k--')\n",
    "plt.plot([0, 0], [-0.2, 1.2], 'k-')\n",
    "plt.plot([-5, 5], [-3/4, 7/4], 'g--')\n",
    "plt.plot(z, logit(z), \"b-\", linewidth=2)\n",
    "props = dict(facecolor='black', shrink=0.1)\n",
    "plt.annotate('Saturating', xytext=(3.5, 0.7), xy=(5, 1), arrowprops=props, fontsize=14, ha=\"center\")\n",
    "plt.annotate('Saturating', xytext=(-3.5, 0.3), xy=(-5, 0), arrowprops=props, fontsize=14, ha=\"center\")\n",
    "plt.annotate('Linear', xytext=(2, 0.2), xy=(0, 0.5), arrowprops=props, fontsize=14, ha=\"center\")\n",
    "plt.grid(True)\n",
    "plt.title(\"Sigmoid activation function\", fontsize=14)\n",
    "plt.axis([-5, 5, -0.2, 1.2])\n",
    "\n",
    "save_fig(\"sigmoid_saturation_plot\")\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "collapsed": true,
    "deletable": true,
    "editable": true
   },
   "outputs": [],
   "source": [
    "def leaky_relu(z, alpha=0.01):\n",
    "    return np.maximum(alpha*z, z)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {
    "collapsed": false,
    "deletable": true,
    "editable": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Saving figure leaky_relu_plot\n"
     ]
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAagAAAEYCAYAAAAJeGK1AAAABHNCSVQICAgIfAhkiAAAAAlwSFlz\nAAALEgAACxIB0t1+/AAAIABJREFUeJzt3XmYFNW5x/HvCyPLsGowikZBMKIigkjUSMRRvC64sIgB\nV8AoLkEFr3uMiEbQixo3ogIRriImCIJKCC7AYIS4oIIrEhHCVSMigggM28y5f5wGh6Fn6enprqqu\n3+d5+pmu7pqut8+c7nfqPaeqzDmHiIhI2NQKOgAREZFklKBERCSUlKBERCSUlKBERCSUlKBERCSU\nlKBERCSUlKCkXGY2x8weCjqOXGBmx5tZsZntkYVtLTOza7OwnTZmNt/Miszs80xvrwrxlJhZr6Dj\nkJqjBBVRZjbOzF4IOo5UmVlh4oukxMw2m9lnZjbczOqk+Dr9zOyHCp5Pmlwr+72aUE6CmAc0d859\nV4PbGWpmHyR5qhPwp5raTgX+AGwADgJ+kYXtARX2/b2BF7MVh2ReXtABSOw44AngZqAu/ottfOLx\n31XjtaobQ1Y557YB32TipZNsa3UGtpPMgcA059z/ZWl7FXLOZaJ9JUDag8pRZtbYzEab2UozW5fY\noziy1PN7mNlEM/s/M9toZh+aWf9KXrOrma0xs0vN7Dgz22JmPy2zzl1mtrCS8DY651Y5575wzk0F\nXgFOLvM6+5jZX8zsu8RtupkdmForVI+ZjTCzxYl2WWZm95TdwzOz083sjcQ635rZ82ZW18zmAC2A\nkYm9xOLE+gWJ5T0Sf5uNZnZ6mdc8OdGmzSqLw8z6AUOBttu3Y2YXJZ7baQ/OzPYzs6mJfrDOzKaY\n2b6lnh9qZh+YWZ/EHu26xPrlliPNrAQ4HBia2PZtZtYiEUvHsutuL72VWqeXmb1sZhvM7CMzO6nM\n77RJtOlaM/vBzOaZWVszGwr0A04v9b67lN1OYvkwM3sl0X6rE3tejUs9P87MXjSzq83si0Q/e8LM\n6pX3viW7lKBy1wx8yaMb0AF4DZhlZnslnq8HvJN4/lDgAeAxMzsh2YuZ2dnAc8Alzrkxzrl/AJ8B\nF5Vax4ALgbFVDdLM2gOdga2lHqsPzMGXj44DjgG+Al7N0pfHeqA/cDBwBdCHUnt3ZnYqMA14CegI\nFABzAQN6AV8Aw/Dt3zzxay5xwzm3DpgOnF9mu+cBLznnvq1CHH8F7gM+BfZKbOev5byf54E9E3EW\nAPsAU8us0xL4NdAd+C/gCOCucl6PxHtbAtyb2Pa9pd5nVfwB3+cOB94GnjGzfAAzaw68DhQDXROx\njAJqAyOBScCr/Pi+55d98UQfmgmsw5c8ewDHAn8us+pxQNvEdn4N9ASuqeJ7kExzzukWwRswDnih\nnOdOxH8w65Z5/D3gugpe8xlgdKnlOcBDwKXAGqBrmfX/G/io1PJpQBGwewXbmANsBn4ANgEl+OTU\no9Q6FwOflvm92sC3QO/Ecj9gXSXbeSjJ4xX+XjmvdRmwpNTy68DTFay/DLi2zGPH479w90gsn4VP\nQA0Sy/WA74FfpxDHUOD9iraPTzZbgf1KPX9AIpYTS73ORqBhqXVuKb2tcuL5ALit1HKLxN+zY5n1\nSoBeZda5pNTz+yQeOzaxfFfiPdROpe+X2c72Pptf5m9QArQq9Tr/BmqVWmc08HKqn0fdMnPTHlRu\n6gg0AL5NlEd+MD8xoC3QGsDMapnZ78xsUaJE9QP+v8f9y7xWD+AR4FTn3Kwyz/0v0NrMjkksD8CP\nSaypJL6/4P9zPgb/X/8Y59y0MvG3KhP7WqDp9vgzycx6m9k/zOw/iW3/kZ3b5QhgdpqbmYFP5j0T\ny90TP3cM/lchjqo4GPjKlRoncs4tw++RHlpqvX8759aXWv4K2Kl8W8N2TO5wzn2VuLt9ex2A151z\nxWm8/sH45L2x1GPz8Qmq9Pv+2DlXUmo50+9bUqBJErmpFvA18Ct82am0dYmf1wNDgKuBD/H/zY/A\nl4JKWwS0Ay4B3iz9hHPuW/OzqS42syX4vYLTqdz3iS9JzOxC4CMzu8g592Sp+N/Dl7TKxl/VWXDr\ngCZJHm+K31NJysyOxu9JDsWX8Nbik8fIKm63Spxz28zsWXyZbwK+vPecc25TDcdhlF92K/341iTP\npfoP7PYv+h1/MzMr7zum7PYotb2yf/PqyOb7lgxRgspN7+Lr8257IkiiM/Cic27i9gfM7CB8WaS0\nZcBVwFwzG+2cG1jm+THA5MR6XyfZy6pQ4ot6OHC3mU1KfEG/C/QFVjs/XlMdn+JLjmUdmXiuPJ2B\nL5xzw7c/YGYty6zzHn7Moux4xnZb8CXJykwACs3sEODUMvFWJY6qbOdjYF8z2985tyLxOq3wZbWP\nqhBjKlYlfjYv9dgR1Xidd4HzzSzP+dmPZVX1fQ8wswbOuQ2JxzrjE9cn1YhJAqD/FKKtsZm1L3Nr\n4Zx7FV/OeN7MTjWzlmb2SzO73cw6J353CdDVzDqb2cFm9gh+bGIXzrnlwAnAqWY2usxzrwCr8f/p\nP1HN9zER/5/rVYnlp4GVifi7JOLvYmb3mlnpEl/tJO+/beK5R/FlwofM7HAzO8jMhuD3yiraC1mC\n/0I/z8wOMLMr8MmytLuAc8zsTjM7JDG7bHCpCRzLgePMz0T8Sanf22nPwDk3H1iReP+r8ONmqcSx\nHGhhZkeY2U8sybFkib7wPvC0mXU0s074xLjAOVdYQTukLPHPxRvAjWZ2qJkdi2/rVKf1/wloCDxr\nZp3MrLWZ9TWzwxPPLwcOS/xNf2JmyZLV0/hJNk8mZvN1AR4DpjjnAj+oWKpGCSrajsP/t1n6tv3L\n9zT8OMloYDF+3OcgfI0d/Cyqt/BjIYX4Et+EMq+/44sl8aEuAE4xs8fKrDcOvzc+vgoxJztuZyt+\nnOv6xH+8RUAX4HP8jK1PEttoys57ePXY9f3PSbzmssRr/BxfInsTP0urt3PupXKDc246vg3/iC9v\ndgV+X2adv+PHjk4ttc0Cfixx3QbsByxl52Ofkn1RP40fj5vonCvd3pXGAUzB//1mJbazPYGV3U53\nfkyAs/B9oCfpS/Z+BiR+voX/JyHZsW3Jfq/0e/8K/7fbDd+H3wUGAdv3psbg+8QC/PvunOQ1ioBT\ngMb4v/1U/MHSv6n8bUlYWKnPhEi1mNmfgNbOuVOCjkVEcofGoKTaEgc9tsUfC9U74HBEJMcoQUk6\nnsefqmisc25m0MGISG5RiU9EREKpRvegzEzZTkREUuac2+X4txqfxRf0qTEquvXr1y/wGKJ6U9ul\ndwv7ZyPsN/W/mm+/ESP86SEbN3YsXhxsfOXRNHMRkZj5+9/hllv8/QkToE2bYOMpT6wSVMuWLYMO\nIbLUdhIk9b/0lG6/f/0LzjsPnIM77oAzzwwursrEKkEVFBQEHUJkqe0kSOp/6dnefj/8AD16wNq1\n/ufvUr1EaJbFKkGJiMRVSQn06wcffwyHHgpPPgm1Qp4BUgrPzH5uZkVm9mTla4uISFgMHw5Tp0KT\nJjBtGjRqFHRElUvpOCgzewl//rN/O+cuSvK8S+X1ROLCzCqcrSSSSdOnw1ln/Xi/W7dg4ykr8fmo\n/jRzM+uLP1FnSpdTEBGR4Hz6KZx/vp8Ucddd4UtOFalSgkqcc20Y/hLfNXExsUAUFhYGHUJkqe0k\nSOp/1bNunZ8MsW5dIb17w003BR1Raqp6Jok78Jfl/tKs4vzUv3//HVMamzZtSocOHXbMINneyYJa\nXrhwYaDb17KWtazlbC3Pnl3I738PixcX0LIlXHxxIXPnhiO+wsJCxo8fD1R8CEGlY1Bm1gF/naAO\nzl/9dCj+0goagxKpIo1BSbbdfjsMGwa77w5vvw2tW1f6K4EpbwyqKntQxwMtgBXmd58a4q9keqhz\nrlMNxykiIml6/nmfnGrVgmeeCXdyqkhVxqAeB1oDHYD2+MsmTwdOzmBcGbF9F1NSp7aTIKn/Vd3H\nH8MFF/j7I0bAKadEt/0q3YNyzm0CNm1fNrP1wCbn3HeZDExERFKz/QwR69dDnz5w/fVBR5SeGr0e\nlMagRJLTGJRkWnGxP9Zpxgxo3x7mzYMGDYKOqmrSPg5KRETCa+hQn5z22MOfMSIqyakisUpQUa3D\nhoHaToKk/lexKVP8Qbi1asGkSXDAATs/H9X2i1WCEhHJNR9+6E8CCzByJHTtGmw8NUljUCJZoDEo\nyYQ1a+AXv4ClS/3pjJ56Cio5l0IoaQxKRCSHFBfDuef65HTEETB6dDSTU0VilaCiWocNA7WdBEn9\nb1e/+x289BI0a+YnReTnl79uVNsvVglKRCQXTJoE99wDtWvDs89CixZBR5QZGoMSyQKNQUlNWbQI\njj0WNm6EBx+Eq68OOqL0lTcGpQQlkgVKUFITVq/2kyKWLfMz98aNy41xJ02SILp12DBQ20mQ1P9g\n2zbo29cnp06d4LHHqp6cotp+sUpQIiJRdfPN8Oqr8NOfwnPPQb16QUeUeSrxiWSBSnySjokT/XFO\neXkwezYcd1zQEdUslfhERCLovffgN7/x9x98MPeSU0VilaCiWocNA7WdBCmu/W/VKn/5jE2bfJK6\n4orqvU5U2y9WCUpEJCq2bfPXdFqxAo4+GkaNyo0Ze6nQGJRIFmgMSlI1ZAg88ADsvTcsWAD77ht0\nRJmjMSgRkYh46imfnHbbzV9KI5eTU0VilaCiWocNA7WdBClO/W/BArj0Un//kUf8WSPSFdX2i1WC\nEhEJs5UroWdP2LwZBg70tzjTGJRIFmgMSiqzdSucdBK89prfa5o9G+rWDTqq7NAYlIhIiF17rU9O\n++wDkyfHJzlVJFYJKqp12DBQ20mQcr3/jRvnx5vq1PGnMWrevGZfP6rtF6sEJSISNm++CZdf7u8/\n+qg/5kk8jUGJZIHGoCSZr7+GI4+Er76CK6/0B+PGka4HJRIgJSgpa8sWOPFEmDfPn1/v1Vd9iS+O\nNEmC6NZhw0BtJ0HKxf53zTU+Of3sZ/6y7ZlMTlFtv1glKBGRMBgzxl9wsG5dPylir72CjiicVOIT\nyQKV+GS7+fOhoMAf9/S//wsXXRR0RMFTiU9EJGBffQVnn+2T09VXKzlVJlYJKqp12DBQ20mQcqH/\nbd7sk9PXX/s9qHvvzd62o9p+sUpQIiJBcA4GDYI33oD994dJk/yZyqViGoMSyQKNQcXbY4/5q+HW\nq+dn7nXsGHRE4aIxKBGRAPzjH3DVVf7+2LFKTqmIVYKKah02DNR2EqSo9r8vvoDevf3l26+9Fs4/\nP5g4otp+sUpQIiLZsmkT9OoF33wDXbvCPfcEHVH0aAxKJAs0BhUvzsHFF8P48dCypb9K7k9+EnRU\n4aUxKBGRLBk1yien+vVh2jQlp+qKVYKKah02DNR2EqQo9b+5c2HwYH//iSegfftg44FotV9psUpQ\nIiKZtGIFnHMOFBfDDTdA375BRxRtVRqDMrOngK5AA+A/wEjn3J+TrKcxKJEkNAaV+4qK4Fe/gnff\nhZNPhhkzoHbtoKOKhrSuB2VmhwCfOee2mtlBwFygm3PuvTLrKUGJJKEElduc8+fVmzABWrWCt9+G\nPfYIOqroSGuShHPuE+fc1u2vBTigdQ3GlxVRrcOGgdpOghT2/vfggz45NWjgJ0WELTmFvf3KU+Ux\nKDMbZWYbgE+Ar4AZGYtKRCQiZs+G667z98ePh3btAg0np6R0HJSZGfBLoAC4xzlXXOZ5lfhEklCJ\nLzctXw6dOsHq1XDLLXDXXUFHFE3llfjyUnmRRPaZb2YXAlcAj5Rdp3///rRs2RKApk2b0qFDBwoK\nCoAfdzO1rGUtaznqyzNnFjJoEKxeXcBpp8GJJxZSWBie+MK8XFhYyPjx4wF25ItkqnUmCTMbA6x3\nzg0p83io96AKCwt3NJakRm2XHu1BpSds/c85f169Z56Bn/8c3noLmjYNOqryha39yqr2JAkz29PM\n+phZAzOrZWanAH2BWZkIVEQk7O67zyenhg39pIgwJ6coq3QPysyaAZOBw/EJ7d/Ag865J5KsG+o9\nKJGgaA8qd7zyCpx6KpSUwHPPQc+eQUcUfWkdB5XCRpSgRJJQgsoNn3/uJ0WsWQO33QbDhgUdUW7Q\nyWKJ7rEAYaC2kyCFof9t2AA9evjkdOaZMHRo0BFVXRjarzpilaBERKrDORgwAD74ANq0gaeeglr6\n9sw4lfhEskAlvmi7+264+WZo1MjP2Dv44KAjyi0agxIJkBJUdM2cCd26+b2oF17w5T2pWRqDIrp1\n2DBQ20mQgup///oXnHuuT07DhkU3OUX18xurBCUiUlU//OAnRaxd63/eemvQEcWPSnwiWaASX7SU\nlEDv3jB1Khx6KLzxhh9/ksxQiU9EpIqGD/fJqUkTf6YIJadgxCpBRbUOGwZqOwlSNvvf9On+IFwz\nmDjRn2sv6qL6+U3pbOYiIrns00/9SWCd85fO6NYt6IjiTWNQIlmgMajwW7cOjj4aFi/240+TJvm9\nKMk8jUGJiJSjpAQuvNAnp8MOg3HjlJzCIFYJKqp12DBQ20mQMt3/7rjDH4TbtKmfFNGwYUY3l3VR\n/fzGKkGJiJT1/PP+INxateAvf4HWrYOOSLbTGJRIFmgMKpw++QSOOgrWr4d77oEbbgg6onjSufhE\nAqQEFT5r1/rk9K9/QZ8+/gq5GncKhiZJEN06bBio7SRINd3/Skrgggt8cjr8cPjzn3M7OUX18xur\nBCUiAv5ig3/7G+yxh58U0aBB0BFJMirxiWSBSnzhMWWKP86pVi146SU46aSgIxKV+EQk9j78EPr1\n8/dHjlRyCrtYJaio1mHDQG0nQaqJ/rdmjb9sxoYN/nRGQ4akH1dURPXzG6sEJSLxVFzsLzy4dCkc\ncQSMHp3bkyJyhcagRLJAY1DBuvlmuPtuaNYMFiyAFi2CjkhK03FQIgFSggrOpEn+OKfateHVV6Gg\nIOiIpCxNkiC6ddgwUNtJkKrb/xYtggED/P37749vcorq5zdWCUpE4mP1aujZEzZuhIsugquuCjoi\nSZVKfCJZoBJfdm3bBqed5kt6nTrBa69B/fpBRyXlUYlPRGLj5pt9cvrpT+G555ScoipWCSqqddgw\nUNtJkFLpfxMnwr33Ql4ePPss7Ldf5uKKiqh+fmOVoEQkt733Hlxyib//4IPQpUuw8Uh6NAYlkgUa\ng8q8Vav8eNOKFXDxxTB2rA7GjQodByUSICWozNq2DU4+GebMgaOPhsJCqFcv6KikqjRJgujWYcNA\nbSdBqqz/XX+9T0577+3PVq7ktLOofn5jlaBEJPc89RQ88ADsthtMngz77ht0RFJTVOITyQKV+DJj\nwQL41a9g82Z47DG47LKgI5LqUIlPRHLKN9/4M0Vs3gwDByo55aJYJaio1mHDQG0nQSrb/7ZuhXPO\ngS++gGOPhYceCiauqIjq5zdWCUpEcsO11/rTF+2zjx93qls36IgkEzQGJZIFGoOqOePG+eOc6tSB\nuXPhmGOCjkjSpTEoEYm8N9+Eyy/39//0JyWnXFdpgjKzOmY21syWm9n3ZvaOmZ2ajeBqWlTrsGGg\ntpMgFRYW8vXX0KsXbNkCV14Jv/lN0FFFR1Q/v1XZg8oDVgDHOeeaALcBk8xs/4xGJiKSsHUr9O4N\nX33lp5X/8Y9BRyTZUK0xKDNbBNzunJta5nGNQYkkoTGo9FxxhT/Oad994Z13YK+9go5IalKNjUGZ\n2V7Az4GPaiIwEZGKjBnjk1PdujB1qpJTnKSUoMwsD5gAjHfOLclMSJkT1TpsGKjtJAjz58NvfwtQ\nyOOPwy9+EXRE0RTVz29eVVc0M8Mnp83AVeWt179/f1q2bAlA06ZN6dChAwUFBcCPjRTU8sKFCwPd\nvpa1rOWqL0+eXMjAgbB1awG9ekGLFoUUFoYnPi1Xf7mwsJDx48cD7MgXyVR5DMrMngD2B7o557aU\ns47GoESS0BhUajZvhoICeOMNOP54eOUVfzJYyU3ljUFVaQ/KzB4DDgZOKi85iYjUBOdg0CCfnPbf\n31+2XckpnqpyHNT+wECgA7DSzH4ws3Vmdm7Go6th23cxJXVqO8mWxx/3V8OtV89PithzT/W/dEW1\n/Srdg3LOrUBnnBCRLHj9dbgqMcI9Zgx07BhsPBIsnYtPJAs0BlW5L76ATp1g5Up/Mtj77gs6IsmW\n8saglKBEskAJqmKbNkGXLvD229C1K8ycCXlVnmMsUaeTxRLdOmwYqO0kU5zzZ4p4+21o2RL+8pdd\nk5P6X3qi2n6xSlAiEj6jRsH48VC/PkybBs2aBR2RhIVKfCJZoBJfcnPn+pJecTE88wz07Rt0RBIE\nlfhEJFRWrPCXbS8uhuuvV3KSXcUqQUW1DhsGajupSUVF0LMnrFoFJ58MI0ZUvL76X3qi2n6xSlAi\nEjznYOBAePddaNXKl/Zq1w46KgkjjUGJZIHGoH70wAMwZAg0aAD//Ce0axd0RBI0HQclEiAlKG/2\nbF/SKy7259jr3TvoiCQMNEmC6NZhw0BtJ+lavhx+/WufnG6+ObXkpP6Xnqi2X6wSlIgEY+NGPyli\n9Wo47TS4886gI5IoUIlPJAviXOJzDs4/30+GOPBAf8aIpk2DjkrCRCU+EQnEfff55NSwoT9ThJKT\nVFWsElRU67BhoLaT6njlFbjxRn//ySehbdvqvY76X3qi2n6xSlAikj2ffw59+kBJCfz+934MSiQV\nGoMSyYK4jUFt2AC//CV88AGccQY8/zzU0r/DUg6NQYlIVjgHAwb45NSmDUyYoOQk1ROrbhPVOmwY\nqO2kqu65xx+E26iRnxTRpEn6r6n+l56otl+sEpSIZNbMmXDLLf7+hAlw8MHBxiPRpjEokSyIwxjU\nZ5/BL34Ba9fCsGFw221BRyRRoXPxiQQo1xPUDz/4SREffQQ9esCUKRp3kqrTJAmiW4cNA7WdlKek\nBPr188npkEP88U41nZzU/9IT1faLVYISkZo3fDhMneonQ0yb5idHiNQElfhEsiBXS3zTp8NZZ/14\nv1u3YOORaCqvxJcXRDAiEn2ffupPAusc3HWXkpPUvFiV+KJahw0DtZ2Utm6dnwyxbh2cfba/vlMm\nqf+lJ6rtF6sEJSLpKymBCy+ExYvhsMNg/HiwXYozIunTGJRIFuTSGNSwYXD77f6yGQsWQOvWQUck\nUafjoEQClCsJ6vnnfWmvVi2YMQNOOSXoiCQX6DgooluHDQO1nXzyCVxwgb8/YkR2k5P6X3qi2n6x\nSlAiUj1r10L37rB+vb/G0/XXBx2RxIFKfCJZEOUSX0mJP9bpb3+Dww+H+fOhQYOgo5JcohKfiFTL\n0KE+Oe2xhz9ThJKTZEusElRU67BhoLaLp+eegz/8wU+K+Otf4YADgolD/S89UW2/WCUoEam6Dz+E\niy7y90eOhJNOCjYeiR+NQYlkQdTGoNas8dd2WrrUn87oqad0MK5kjsagRKRKiovh3HN9cjriCBg9\nWslJghGrBBXVOmwYqO3i49Zb4aWXoFkzfxmN/PygI1L/S1dU2y9WCUpEKjZpEtx9N9Su7e+3aBF0\nRBJnVRqDMrPfAv2BdsBE59zF5aynMSiRJKIwBvX++/6y7Rs3wgMPwDXXBB2RxEW614P6ErgTOAWo\nX5OBiUjwVq/259jbuNHP3Lv66qAjEqliic85N8059wLwXYbjyaio1mHDQG2Xu7Zt85Mili2DTp3g\nscfCNylC/S89UW0/jUGJxNzNN8Mrr8BPf+oPzK2vGomERI1f8r1///60bNkSgKZNm9KhQwcKCgqA\nH7N4UMvbHwtLPFFaLigoCFU8Wq6Z5VdfhXvvLSAvD265pZClS2G//cIT3/Zl9b/car/CwkLGjx8P\nsCNfJJPSgbpmdiewryZJiKQmjJMk3nsPOneGoiJ45BH47W+DjkjiSgfqEt06bBio7XLLt99Cz54+\nOV18MVx5ZdARVUz9Lz1Rbb8qlfjMrDawG1AbyDOzusA251xxJoMTkZq3bRv8+tfw73/DUUfBqFHh\nmxQhAlU/DmooMBQovfIw59wdZdZTiU8kiTCV+IYM8cc57bUXvPMO7Ltv0BFJ3JVX4tPJYkWyICwJ\n6qmn/HFOu+0Gc+b4MSiRoGkMiujWYcNAbRd9CxbApZf6+w8/HK3kpP6Xnqi2X6wSlEhcffONnxSx\neTMMHAiXXRZ0RCKVU4lPJAuCLPFt3eovNvjaa/5ce3PmQN26gYQikpRKfCIxde21Pjk1bw5Tpig5\nSXTEKkFFtQ4bBmq7aBo3zh+EW6eOP41R8+ZBR1Q96n/piWr7xSpBicTJW2/B5Zf7+6NGwTHHBBuP\nSKo0BiWSBdkeg/r6a39m8i+/9GeJGDUqa5sWSZmOgxIJUDYT1JYtcOKJMG8e/OpXMGuWL/GJhJUm\nSRDdOmwYqO2i45prfHLad1+YPDk3kpP6X3qi2n6xSlDZdsIJJ3C1Lk0qWTRmjL/gYN26MHWqP52R\nSFTFusQ3YMAAVq9ezQsvvJCR1z/hhBNo164dDz30UEZeX6IjGyW+f/4Tjj/eH/c0bhz075/RzYnU\nGJX4RHLYV1/B2Wf75HTVVUpOkhtilaBSqcOuW7eOgQMHstdee9G4cWNOOOEE3nnnnR3Pf/fdd5x3\n3nnst99+5Ofnc9hhh+24QmR5Zs2axe67786YMWOq+Q6CE9Uadhxs3uyT03/+4/eg7rsv6Ihqnvpf\neqLafrFKUKno1q0bX3/9NTNmzGDhwoV06dKFrl27snLlSgA2bdrEkUceyYwZM/j4448ZPHgwl19+\nOXPmzEn6elOmTKFXr16MHTuWS7efsVMkTc7BoEHwxhuw//7w7LP+TOUiuUBjUEnGoGbPnk2PHj1Y\ntWoVdUudF+aII47g/PPP57rrrkv6eueeey6NGjVi9OjRwI9jUO3ateOGG25g8uTJdO3aNXNvSEIr\nU2NQjz0GV1wB9er5mXsdO9b4JkQyrrwxqCpdUTdu3n33XTZs2ECzZs12enzz5s0sXboUgJKSEkaM\nGMGkSZP48ssv2bx5M1u3bqWgoGCn35k2bRqPP/44r732GkcffXS23oLEwOuv+/Em8LP3lJwk18Sq\nxFfVOmzzP6DYAAAMyUlEQVRJSQl7770377//PosWLdpxW7x4MXfeeScAI0eO5I9//CM33ngjs2fP\nZtGiRXTv3p0tW7bs9Frt27enefPmjB07tqbfTlZFtYadq774Anr39pdvHzIELrgg6IgyS/0vPVFt\nP+1BJdGxY0dWrlyJmXHAAQckXWfevHmceeaZnHfeeTseW7JkCbvvvvtO6x1wwAE8/PDDHH/88Qwc\nOHBH+U+kujZtgl69YOVKf8aI//mfoCMSyYxY7UGVLb+Bn61Xei9p0aJFHHjggXTu3Jnu3bszc+ZM\nli9fzj//+U9uv/125s2bB8BBBx3ErFmzmDdvHosXL2bQoEEsW7Ys6XZbtmzJnDlzmDlzJgMHDszk\nW8yYZG0n2eecH3N6+21o0QL++lfIi8G/mep/6Ylq+8UqQSXzj3/8g44dO+50u+GGG5gxYwYnnngi\nAwcO5OCDD6Zv374sWbKEffbZB4Bbb72Vo446im7dulFQUEDDhg25oEydxezHMb9WrVpRWFjISy+9\nxOXbTzEtkqJRo2D8eKhfH6ZNgzLDpCI5JVaz+AoLCyP7n0TQ1HbpqYlZfHPnQteuUFwMzzwDffvW\nUHARoP6XnrC3n84kIRJhK1bAOef45HT99fFKThJfsdqDEglKOntQRUX+shnvvgsnnwwzZkDt2jUc\noEiAtAclEkHOwcCBPjm1auVLe0pOEhexSlBRPRYgDNR2wXjwQZgwAfLz/aSIPfYIOqJgqP+lJ6rt\nl3MJau3atXTv3p1zzjknq5fYFqlps2fD9rNqjR8P7doFGo5I1uXUGNQbb7xBjx49WLt2LbVr1+b+\n++/nsssuCyweke1SHYNavhw6dYLVq+Hmm2H48MzFJhK08sagciJBlZSUMHz4cIYPH05RUdGOx/Pz\n83nrrbdo27Zt1mMSKS2VBLVxI3TuDAsXwmmnwYsvatxJclvOTpJYuXIlXbp0YcSIETslJ4CNGzfS\np0+fHctRrcOGgdouO5yDSy7xyenAA+Hpp5WcQP0vXVFtv0ifJOXll1+mT58+rF+/nm3btu30XK1a\ntahfvz633XZbQNGJpO6++/xMvYYN/aSIMqd2FImVSJb4tm7dyk033cSjjz66y14T+NJey5YtefHF\nF2nVqlXG4xGpTFVKfK+8AqeeCiUl8Nxz0LNnloITCVjOXA9q+fLlnHXWWSxdujRpcqpfvz6XXHIJ\nI0eOpE6dOgFEKJK6zz+HPn18cvr975WcRCBiY1CTJ0+mXbt2fPTRR2zcuHGn5/Ly8mjSpAlTpkzh\nwQcfTJqcolqHDQO1XeZs2AA9esCaNXDGGXD77UFHFD7qf+mJavtFYg+qqKiIK6+8kkmTJu2SmMCX\n9Nq1a8fUqVNp3rx5ABGKVI9zMGAAfPABtGnjD8qtFal/G0UyJ/RjUB9//DFnnHEG//nPf9i0adMu\nz9evX58bb7yRW2+9ldqa7iQhVd4Y1N13++OcGjWCt96Cgw8OIDiRgEXuOCjnHGPHjmXw4MEUFRXt\n8uGuU6cOjRs3Ztq0aXTu3LlGtimSKckS1MyZ0K2b34t6/nk466yAghMJWKSOg1q3bh09e/Zk8ODB\nbNy4cZcPdn5+PieccAKffvppSskpqnXYMFDb1azPPoNzz/XJadgwJafKqP+lJ6rtF7oxqAULFnDm\nmWeyZs0aNm/evMvz9evX5+6772bQoEE7XbFWJCp++MFPili71v+89dagIxIJp6yX+JYuXcqYMWMY\nMWLETgmmpKSEe++9l9tvvz3p9PF69erRrFkzpk+fTvv27WssZpFs2F7icw569/bHOR1yCLzxBjRu\nHHR0IsFKq8RnZrub2VQzW29my8zs3OoGMnjwYEaOHMnDDz+847FVq1Zx4oknMmzYsHIPvO3ZsyeL\nFy9WcpJIGz7cJ6cmTfyZIpScRMpX1TGoPwGbgD2BC4BHzeyQVDf2/vvvM2vWLEpKSrjppptYuHAh\nc+bMoU2bNsyfP3+XKeRmRoMGDXj88ceZOHEiDRo0SHWTO4lqHTYM1Hbpmz7dH4Rr5s+xd9BBQUcU\nHep/6Ylq+1U6BmVm+UAv4FDnXBEwz8xeAC4EbkllY9ddd92OqeJFRUUUFBSwZcuWcs8I0aJFC158\n8UUOPPDAVDYjEkrnn+8nRfzhD3D66UFHIxJ+lY5BmVkHYJ5zrkGpx/4b6OKc615m3XLHoN555x2O\nO+64pMmorPz8fAYMGMD999+v0xVJpDkHEyfCBRcY4Dj7bHj2Wb8XJSJeOufiawh8X+ax74FGqQQw\nZMiQSpNTXl4e+fn5PP3005xxxhmpvLxIxpSUwPr1ftZdKrfvv/enL1qzxr/OySf7K+MqOYlUTVX3\noF53zjUs9di1wPHJ9qAyEqWIiOS06u5BLQHyzKy1c25p4rH2wEflbGSXx4455hjefPPNCjfStGlT\nvvzyS/Lz86sQUvUUFhZSUFCQsdfPZUG2XUkJrFtXvT2Y7T/TPZqiYUNo2rTiW5Mm5T+3226pXfJd\ndqbPbnrC3n7lHdNapeOgzGwi4IBLgSOA6cCxzrlPyqy3yxhUYWEhp59+etKTvJZWv359+vXrx6OP\nPlppPBItxcXpJZh169JPMI0aVT/BNGkCeWke0p7KJd9F4iatc/GZ2e7AE8B/Ad8CNzrn/ppkvZ0S\nlHOOjh07snDhwioFmZ+fzzPPPMNZOu9LqGzbln6CSVfjxtVPMI0bp59g0qUEJVK+QE4W+/LLL9Or\nVy82bNhQ4e/ttttu5OfnU1RUxDHHHMPcuXNrLKbSwr6bmynbtv2YLFJNLmvX+lPzQCFQUO0YKip/\nVSXBRP1E9UpQ6YnrZ7emhL39sn5FXeccgwcP3ik51a5dmwYNGlBcXMyWLVto0aIF7du356ijjqJd\nu3Ycdthh/OxnP8tUSJG1dWt6CWb9+vS2bwb5+bDnntVLMI0aRT/BiEj2ZWwPav78+XTu3Jm8vDya\nN29Ou3btOProozn88MNp27YtrVq1is31m7ZsSS/BVLIDWqlatdLbg2nUSBfRS5f2oETKl/US39at\nW/nss89o3bp15A+23bKl+sll7VqoZH5IpWrVqjy5VJRgGjZUggmaEpRI+SJ3wcKatHmzTxQvvVRI\nmzYFKSeYKpz8okK1a6efYII+uDPsNeywU4JKj/pfesLeflkfg6pJmzaltweT5ErxKcnLSy/BNGgQ\nfIIREYmajO9BOZdagkk2VpPkuoUpycuD3XevfoLJz1eCkfRoD0qkfFnbgzrllF0TzJYt6b3mbrul\nl2Dq11eCERGJmhrfg/InnNhZnTrpJZh69WomwYS9Dhtmarv0aA8qPep/6Ql7+2VtD+rvf0+eYERE\nRFIRi1l8IkHTHpRI+crbg9LRMSIiEkqxSlCFhYVBhxBZajsJkvpfeqLafrFKUCIiEh0agxLJAo1B\niZRPY1AiIhIpsUpQUa3DhoHaToKk/peeqLZfrBJUVa/sK7tS20mQ1P/SE9X2i1WCWrt2bdAhRJba\nToKk/peeqLZfrBKUiIhER6wS1PLly4MOIbLUdhIk9b/0RLX9MnCyWBERkdRk/Iq6IiIiNSVWJT4R\nEYkOJSgREQklJSgREQklJSgREQml2CYoM/u5mRWZ2ZNBxxIVZlbHzMaa2XIz+97M3jGzU4OOK8zM\nbHczm2pm681smZmdG3RMUaH+VnOi+n0X2wQFPAK8FXQQEZMHrACOc841AW4DJpnZ/sGGFWp/AjYB\newIXAI+a2SHBhhQZ6m81J5Lfd7FMUGbWF1gDzAo6lihxzm10zt3hnPu/xPLfgGXAkcFGFk5mlg/0\nAm51zhU55+YBLwAXBhtZNKi/1Ywof9/FLkGZWWNgGPDfwC4HhknVmdlewM+Bj4KOJaQOArY555aW\nemwR0DageCJN/S11Uf++i12CAu4Axjjnvgw6kCgzszxgAjDeObck6HhCqiHwfZnHvgcaBRBLpKm/\nVVukv+9yKkGZ2RwzKzGz4iS318ysPXAS8EDQsYZRZe1Xaj3Df1lsBq4KLODwWw80LvNYY+CHAGKJ\nLPW36jGzDkT8+y4v6ABqknPuhIqeN7NrgBbAikSnbwjUNrNDnXOdshFjmFXWfqX8GWgGdHPOFWcw\npKhbAuSZWetSZb72qESVKvW36jmeiH/fxepcfGZWj53/o70e/we83Dn3XTBRRYuZPQYcDpzknNsY\ndDxhZ2YTAQdcChwBTAeOdc59EmhgEaH+Vn258H2XU3tQlXHObcJP+QXAzNYDm6LyxwpaYnrvQHwb\nrvT/lOGAy5xzzwQZW4j9FngC+Ab4Fv/loORUBepv6cmF77tY7UGJiEh05NQkCRERyR1KUCIiEkpK\nUCIiEkpKUCIiEkpKUCIiEkpKUCIiEkpKUCIiEkpKUCIiEkr/DwSbeSK+TMmPAAAAAElFTkSuQmCC\n",
      "text/plain": [
       "<matplotlib.figure.Figure at 0x7fd700e40c88>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "plt.plot(z, leaky_relu(z, 0.05), \"b-\", linewidth=2)\n",
    "plt.plot([-5, 5], [0, 0], 'k-')\n",
    "plt.plot([0, 0], [-0.5, 4.2], 'k-')\n",
    "plt.grid(True)\n",
    "props = dict(facecolor='black', shrink=0.1)\n",
    "plt.annotate('Leak', xytext=(-3.5, 0.5), xy=(-5, -0.2), arrowprops=props, fontsize=14, ha=\"center\")\n",
    "plt.title(\"Leaky ReLU activation function\", fontsize=14)\n",
    "plt.axis([-5, 5, -0.5, 4.2])\n",
    "\n",
    "save_fig(\"leaky_relu_plot\")\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {
    "collapsed": false,
    "deletable": true,
    "editable": true
   },
   "outputs": [],
   "source": [
    "def elu(z, alpha=1):\n",
    "    return np.where(z<0, alpha*(np.exp(z)-1), z)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {
    "collapsed": false,
    "deletable": true,
    "editable": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Saving figure elu_plot\n"
     ]
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAagAAAEYCAYAAAAJeGK1AAAABHNCSVQICAgIfAhkiAAAAAlwSFlz\nAAALEgAACxIB0t1+/AAAIABJREFUeJzt3XucXPP9x/HXJxu5SSJBpG4RJSrasKW0ImG10SSoFCVU\nkR+P4EdFabVuP3EtLSptUf2JUOIatI24RJGVVuqnRdStdUskbnHJRSLZ3Pbz++M7K2syuzs7OzPf\nc3bez8djHjtz9uycz3z3O/OZ8/2c8z3m7oiIiCRNh9gBiIiI5KIEJSIiiaQEJSIiiaQEJSIiiaQE\nJSIiiaQEJSIiiaQEJSIiiaQEJSIiiaQEJaljZjeZ2dR2tB0zs9+b2UdmttbM9i71NpuJpSyvObOt\nXmb2vpltW47ttcTMppjZ6bHjkHVMM0m0b2Z2E3As4IA1+tVT7j7YzG4GNnb3g5r4+xnAC+4+Lmv5\nscA17t6jNJE3u+0ehL77SZq208z2DwDuBfYB5gAL3X1NKbeZ2e56r7tcrzmzrSsIfe/4Um8rH2b2\nFeAJoL+7L40dj2gPqlL8BfhC1m3/zO/a8g0lyrcbd19ajg/Qcm0HGAC85+7/5+4flCM5NaVcr9nM\nugLHAxNLva0c2+5uZveY2VaNl7v7i8CbwA/KHZPkpgRVGVa6+4eZD7+G2+JSb9TMhpvZTDNbaGYf\nm9nDZrZj1jo/NrNXzazOzOaZ2aWZ5TcR9ihOMbP6zNBXv4bfmdn9ZnZCZoioQ9Zz3m5mf8wnjjy2\nM7XR83QyswmZba4ws7+b2V6Nfj/DzK41s0vN7EMzW5DZS2iujW4CfgX0y2z/zczyWjP7Tfa6DfHk\nu63Wtm9rX3Ohrxs4AFjr7n9vYb2iMrPjgZ8AB5P7828qcGQ5Y5KmKUFJKW0IXA18jfBhuBi438w6\nApjZZcC5wKXATsBhwPzM354G/B24CegLbN7odxD23u4CegHDGhaaWTfgIODWPONoaTuNXZGJcQxQ\nDbwAPGxmfRut831gNbAncArwIzMb3UwbjQMuAt7ObH/3Rq+vJc1uq4D2fbvA19xiLDkMAZ7J9Qsz\n28bMxprZ8WaWc+i5UO5+o7tfwOeHuxt7GtjDzDoXc7tSmI6xA5CyGGlmjcfUHbjW3c8u5Ubd/b7G\njzPfXpcQPgCeB34EjHP3P2RWeRP4v8zffmJmq4Dl7v5hE8+/xMweBI4CHsksPoTwQTktnzjcfVZL\n28n8TTfgJOA4d384s+wk4JuED+TzM6u+nPkABHjdzE4AvkVIprlew9LM/2Ztc9tvQpPbMrMNKaB9\nzdZ9brfiNbf6dQPbAO9lLzSzhuc+0t1Xmdm9ZrYSeBn4cqM4egNnNv7TzE9v9Hg1cKG7r20ihlze\nBTYAtiDUAyUiJajK8AQwls9/ayzHEN8XgUuAPYA+hD12A/oRPjw6AY+3cTOTgZvMrIu71xG+yd/j\n7qvyjGNWntvZjvB++Wx9d683s78T9k4a/Cvr794FNmvVK8pfc9vaiba3b76vuaVYcukKvN94gZl1\nB24HvtHo/1dL+ALyV6Ah0eLui4Bz8n0hrbCC0De6luC5pZWUoCrDcncv9NvgJ8BGOZb3IuyFNGca\nYUjpBOAdYA3wCuGD02h6mKU1pgFrgVFm9jhhuG9YjnWaiiNf2d/QG2u8bHWO3xUylF7P+u2zQdbj\n5rZVjLbN9zW3FEsuHwG9s5YdDLzh7nMbLfsE2BF4pPGXjhLamBB7a/dmpQSUoKQl/wFG5li+W+Z3\nOZnZxoQPlv929ycyy3ZlXZ97GVhJGAZ6o4mnWQVUNRdcZhjoHsKRV30IR8PNbEUceW0HeJ3wITwE\nmJt5ng6EmsvkFv62EB8S6kKN7UL+w04vE15XW9q3udd8W55xNOU5wukPjW0OvJS1zIEqd/9cG2f+\nrz9p5vmN8EXkglYO8X0FeLeA4VYpASWoytA5R1F7rbt/lLnf08x2yfr9Ynd/C/gd4Uiv3xAOCa4j\nHIE1mnAwQlMWEb4ljzWzt4GtgF+S+abt7svM7NfAZZlayExgE2A3d78+8xxzCfWqbYBl7v5xE9ua\nDDwKbEsYIso7jny34+7Lzex3wOVm9jEhUZxBGMa6rpl2KNTjwNVm9h3CF4ETga3JM0EV0r7Awqzn\nKOVrnp553t6Z4bqGZUMaVjCzHYCBwNJMzamLu7+XiW0hpRniGwo8XILnlQIoQVWGYYSaQAMjHLHV\nL/N4KPBs1t/cCxzu7nMszGxwCeEDpAvwb+B77j69qQ26u5vZ4cBvCEd+vQ78OPO8Dc4mfCieR0gc\nC4BbGv3+SuBmwt5AFzPb1t3n5djWTDN7h7CnNDrrd/nEkdd2gJ8RvtFPIgxxPgcMd/cPGjbXVHsU\nYBIwCLgx8/g64D5CksmLu59lZnm3LyHBZ2vqNS9ovKl8Y2oU24tm9jRwBOFLEO7+vJldmTnA4lPg\nLXc/28x+BIzi80dmFsTMvk9Igk5IkH9z9+syv+tMGGbcr63bkeLQTBIiEoWZDQcmADt5Aj6IzOxk\n4CB3HxE7Fgl0HpSIRJHZA7+WsHeXBKuAU2MHIetoD0pERBJJe1AiIpJIUQ6SMDPttomIyGfcfb1z\n96LtQbl7Im/jx4+PHkPabmqzwm5Jfh8k+Zam/jZrltO1qwPOOeeo3Zq6NUVDfCIiJfDyy3DAAbBi\nBRx3HFxySeyI0kcJKsvcuXNjh5A6ajMppzT0t/nzYfhwWLQIvvMd+P3vwYox+VQbpKHdshU1QZnZ\nrWb2rpktMbN/Z2aNTpXq6urYIaSO2kzKKen9beFCGDEC3n4bBg+GO++EjgmYEiHp7ZZLUQ8zN7OB\nwOvuvjozTckTwP7u/lzWel7M7YqkkZk1O/4u6bN8OXz72/Dkk7DTTvDXv8LGG8eOKvky74XSHiTh\n7q+4e8McZ0aYTmS7Ym5DRCSJ1qyBI44IyWnrrWH6dCWntip6DSpz6edPCZczeBd4sNjbKKXa2trY\nIaSO2kzKKYn9zR1OPBHuvz8kpenTYaukzI+RkcR2a0nRR0bd/RQz+yFhSv4awiUV1jNmzBj69+8P\nQK9evaiurqampgZY15B6nI7Hs2fPTlQ8aXncICnxpOVxEvvbxIlw2201dO0KF11Uy4IFMHBgcuJr\nLAnxzJ49m8WLwzVTmzt4o6RTHWWm6n/J3a/JWq4alFQ81aDah9/+FsaNg6oq+NOf4MADY0eUPmWp\nQeXQEdWgRKSduusuOO20cH/iRCWnYitagjKzPmY22sw2NLMOman0jwAeK9Y2yiF7d1hapjaTckpK\nf3v0UTj66FB/uvxyGDMmdkTNS0q7tUYxa1AO/Dfh4mMdgLeA09x9WhG3ISIS3bPPwsEHw+rVYQ/q\npz+NHVH7FOVyG6pBiagGlVZvvBFOwP3gAzjySJg8GTpoTp42aaoGpQQlEokSVPosWBCS05tvwrBh\n8MAD0KlT7KjSL9ZBEqmTxnHa2NRmUk6x+tsnn8DIkSE57bYb3HdfupJTGt+nSlAiIi1YuTLUnJ57\nDrbfHh58EHr0iB1V+6chPpFINMSXDvX1odZ0993Qty/MmgVf/GLsqNoXDfGJiLSSezhK7+67wx7T\nww8rOZWTElSWNI7TxqY2k3IqZ3+77DK45ppQa/rznyGFV6z4TBrfp0pQIiI53HgjnHtuuNDgbbfB\nvvvGjqjyqAYlEolqUMk1dWo4KKK+Hq69Fk4+OXZE7ZtqUCIieXjySRg9OiSn//kfJaeYlKCypHGc\nNja1mZRTKfvbSy+FCV/r6mDsWLjwwpJtquzS+D5VghIRAebNg+HDYfFiGDUKrrsu1J8kHtWgRCJR\nDSo5Pv4Yhg6FV16BIUPgkUega9fYUVUO1aBERHL49NMwrPfKK/CVr4QDJJSckkEJKksax2ljU5tJ\nORWzv61eHQ6IeOop6NcvnIjbu3fRnj5R0vg+VYISkYrkHg6EeOAB2GQTmD4dttwydlTSmGpQIpGo\nBhXX2WeHK+F26waPPw5f/3rsiCqXalAiIhkTJoTkVFUF99yj5JRUSlBZ0jhOG5vaTMqprf3tjjvg\n9NPD/UmTwjWeKkEa36dKUCJSMf7yFzj22HD/iivgmGPixiPNUw1KJBLVoMrrn/8ME74uWwZnnAFX\nXRU7ImnQVA1KCUokEiWo8nntNdhrL/jwQzjqKLjlFuig8aPE0EESeUrjOG1sajMpp9b2t/feC1MY\nffhh+DlpUmUmpzS+Tyvw3yQilWLJknAQxJw5sPvu4Yi9Tp1iRyX50hCfSCQa4iuturqQnGprYcCA\ncBmNPn1iRyW5aIhPRCrG2rVw9NEhOW2+eZj8VckpfZSgsqRxnDY2tZmUU0v9zR3GjQvDeT17hvn1\n+vcvS2iJlsb3qRKUiLQrl1wSruXUuXOYmXznnWNHJIVSDUokEtWgiu+GG+CEE8JRelOmwCGHxI5I\n8qEalIi0a3/6E5x0Urh/7bVKTu2BElSWNI7TxqY2k3LK1d9mzoQjjoD6erjggnWJStZJ4/tUCUpE\nUu2FF+Cgg2DlypCYzj8/dkRSLKpBiUSiGlTbvfUWDB4M774bhvTuvjtcQkPSRXPxiSSMElTbfPQR\nDBkC//kP7L13uCJuly6xo5JClPwgCTPrZGYTzWyumS0xs2fMbESxnr9c0jhOG5vaTMqptraWTz+F\nAw4IyWnnneHPf1Zyakka36fFrEF1BOYBQ919I+B84G4z61fEbYhIhVuzBg47DJ5+OpyA+9BD0KtX\n7KikFEo6xGdmzwMXuPsfs5ZriE8qnob4Wq++HsaMgVtvhU03DfPr7bBD7Kikrcp+HpSZ9QUGAC+V\nahsiUlnOOiskpw03hAceUHJq7zqW4knNrCMwGbjZ3V/Ntc6YMWPon5kgq1evXlRXV1NTUwOsGyuN\n8bjxOG0S4knD4wkTJiTm/5emxw2SEk/SHz/zTA1XXAFmEzj//Gr22CNZ8SX9ccOyJMQze/ZsFi9e\nDMDcuXNpStGH+MzMgDuA7sAod1+bY53EDvHV1tZ+1pCSH7VZYTTEl7/Jk8Ps5ADnnFPLpZfWRI0n\njZL8Pi3bYeZmNgnoB+zv7quaWCexCUqkXJSg8jN9Ohx4YDg44qqr4IwzYkckxdZUgirqEJ+ZXQ/s\nCAxrKjmJiOTr6afh0ENDcjrzTCWnSlPM86D6AScA1cACM1tqZp+Y2ZHF2kY5ZNcHpGVqMymF//wn\nnOv06adwzDFw+eVhufpbYdLYbkXbg3L3eWhuPxEpgnffheHDw2wRI0fCxInhEhpSWTTVkUgkqkHl\ntngx7LMP/Otf8PWvw2OPhcPKpf3S9aBEJPHq6mDUqJCcvvQlmDZNyamSKUFlSeM4bWxqMymGtWvh\n+98P13baYotw9N6mm66/nvpbYdLYbkpQIhKdO5xyCvzxj2FevenTYZttYkclsakGJRKJalDrXHhh\nuBJuly7wyCMwdGjsiKScVIMSkUS6/vqQnDp0gDvvVHKSdZSgsqRxnDY2tZkU6t574eSTw/3rrw8H\nSLRE/a0waWw3JSgRieKJJ8JBEe5w8cUwdmzsiCRpVIMSiaSSa1DPPx8u0/7JJ+HgiN/+Fmy9CoRU\nirJNFptnMEpQUvEqNUHNmQODB8P778P3vhfqTlVVsaOSmHSQRJ7SOE4bm9pM8vXhh2EKo/ffh333\nDZfRaG1yUn8rTBrbTQlKRMpi2bIw+etrr0F1dTjnqXPn2FFJkmmITySSShriW7UKvvOdcI7TttvC\nrFnwhS/EjkqSQkN8IhJFfT0cd1xITn36hJ9KTpIPJagsaRynjU1tJk1xh5/8BG67Dbp3h4cegu23\nb9tzqr8VJo3tpgQlIiVz5ZVw9dWwwQZw332w226xI5I0UQ1KJJL2XoO65RY49thw/4474Igj4sYj\nyaUalIiUzYMPhroTwIQJSk5SGCWoLGkcp41NbSaNPfUUHHZYuL7TWWfBaacV9/nV3wqTxnZTghKR\nonnllXCu0/Ll8F//BT//eeyIJM1UgxKJpL3VoN55B/bcE+bPhwMPDCfiduwYOypJA9WgRKRkFi2C\nESNCctpzT7jrLiUnaTslqCxpHKeNTW1W2VasgIMOghdfhIED4f77oVu30m1P/a0waWw3JSgRKdia\nNeEIvb/9DbbaCqZPh002iR2VtBeqQYlEkvYalDuccAJMnAi9e8Nf/wpf/nLsqCSNVIMSkaIaPz4k\np65dYdo0JScpPiWoLGkcp41NbVZ5rr02XKa9qiocEDF4cPm2rf5WmDS2mxKUiLTKlClw6qnh/v/+\nb7iMhkgpqAYlEkkaa1AzZoTDyVetCifhnn127IikPWiqBqUEJRJJ2hLUc8/BPvvA0qUwblyYY8/W\n+0gRaT0dJJGnNI7TxqY2a//efBNGjgzJafTocAmNWMlJ/a0waWw3JSgRadaCBfDtb4ef3/oW/OEP\n0EGfHFIGGuITiSQNQ3xLl0JNDTz7LOy6a6hB9ewZOyppb8oyxGdmp5jZP8yszswmFfO5RaS8Vq2C\nQw4JyWm77cI1npScpJyKvaP+DnAxcGORn7ds0jhOG5varP2prw9Xw330UdhsszCFUd++saMK1N8K\nk8Z2K+p8w+7+JwAz2x3YspjPLSLl4Q6nnw533gk9esDDD4c9KJFyK0kNyswuBrZ09+Oa+L1qUFLx\nklqDuvzycH5Tp07w0EPwzW/Gjkjau6ZqUNGu2DJmzBj69+8PQK9evaiurqampgZYtyuqx3rcnh83\nSEo8NTU13HQTnH12eHzrrTV885vJik+P28fj2bNns3jxYgDmzp1LU7QHlaW2tvazhpT8qM0Kk7Q9\nqGnT4LvfhbVr4be/hR/+MHZEuam/FSbJ7aYTdUWkSbNmweGHh+R07rnJTU5SWYq6B2VmVcAGwPnA\nVsBYYI27r81aL7F7UCLlkpQ9qJdfhiFDwmXbjz8ebrhBUxhJeZVrD+o8YDnwM+CozP1zi7wNESmS\n+fNh+PCQnA46CK6/XslJkqOoCcrdL3T3Du5e1eh2UTG3UWrZBWxpmdosnRYuDDOTv/027LVXOKy8\nY7TDpvKn/laYNLabalAiFWj58nAdp5dfDlfCnTo1XBlXJEk0F59IJLFqUGvWwMEHh6P2tt46HCCx\n1VZlD0PkMzqKT0RwhxNPDMlp443DFEZKTpJUSlBZ0jhOG5vaLD3OOw8mTQrDeQ88AAMHxo6o9dTf\nCpPGdlOCEqkQv/lNuEx7VRVMmQLf+EbsiESapxqUSCTlrEHddRcceWQY4rv55jBTuUhSqAYlUqEe\nfRSOPjokp1/8QslJ0kMJKksax2ljU5sl17PPhiP2Vq+GH/0IzjwzdkRtp/5WmDS2mxKUSDv1+usw\nciQsWxaG9666SrNESLqoBiUSSSlrUO+/H2aHePNN2G+/cFh5p04l2ZRIm6kGJVIhPvkE9t8/JKev\nfQ3uvVfJSdJJCSpLGsdpY1ObJcfKlaHm9NxzMGBAONepR4/YURWX+lth0thuSlAi7cTateFovccf\nhy98IcwSsdlmsaMSKZxqUCKRFLMG5Q7jxsE110DPnvDEE1BdXZSnFik51aBE2rHLLgvJqVMn+POf\nlZykfVCCypLGcdrY1GZxTZwYLtNuBrffDjU1sSMqLfW3wqSx3ZSgRFJs6tQwOznAtdfCoYfGjUek\nmFSDEomkrTWoJ5+EYcOgrg7OPx8uvLCIwYmUUVM1KCUokUjakqBeegmGDIHFi+GEE+D66zVLhKSX\nDpLIUxrHaWNTm5XXvHkwfHhITgcfDNddV1nJSf2tMGlsNyUokRT5+OOQnN55B4YODQdFVFXFjkqk\nNDTEJxJJa4f4Pv001JyeegoGDYKZM6FXrxIGKFImGuITSbHVq2H06JCcttkGHn5YyUnaPyWoLGkc\np41NbVZa7jB2bJhXb5NNwhRGW2wRO6p41N8Kk8Z2U4ISSbizz4Y//AG6dQtJ6ktfih2RSHmoBiUS\nST41qAkT4PTToWNHuP9+GDGiTMGJlJFqUCIpc8cdITkBTJqk5CSVRwkqSxrHaWNTmxXfI4/AsceG\n+1deGS6jIYH6W2HS2G5KUCIJ849/wCGHhCP3fvzjcBOpRKpBiUSSqwb12msweDB89BH84Afh4IgO\n+hop7Zzm4hNJmOwE9d57sNdeMGdOqDdNnQobbBAxQJEy0UESeUrjOG1sarO2W7IERo4MyWmPPWDK\nFCWnpqi/FSaN7aYEJRJZXR1897vw/POwww7hXKfu3WNHJRJfUYf4zKw3MAnYD/gQOMfd78ixnob4\npOKZGWvWOKNHw733wuabw6xZ0L9/7MhEyqupIb6ORd7OdUAd0AfYFXjAzGa7+ytF3o5IuzBuXEhO\nG20U5tdTchJZp2hDfGbWDTgEOM/dV7j7k8BUIFVncKRxnDY2tVnhrrsOOncOB0TsvHPsaNJB/a0w\naWy3YtagdgDWuPsbjZY9D3y5iNsQaReuuSb87NAhzBix995x4xFJomIO8XUHlmQtWwL0yLWyVdIl\nQEWaUF9vHHJI7ChEkqmYCWoZ0DNrWU9gaa6VdZCEVKL77w+XaV+7FqB1FywUaa+a2mEp5hDfq0BH\nM9uu0bJdgJeKuI2SS+M4bWxqs/zU1sJhh4XkdM45saNJL/W3wqSx3YqWoNx9OXAfcJGZdTOzvYCD\ngFuLtQ2RtHrsMTjgAFi5Ek46CS65JHZEIslXyvOgPgJ+5u535VhP50FJxZg+PZyIW1cHY8bAxIlQ\nVZXf9aBEKoHm4hOJYNo0OPRQWLUKTjwxHFbeMPmrEpRIoLn48pTGcdrY1Ga53XNPuGzGqlVw6qnw\nu99pZvJiUH8rTBrbTW8XkRKYMAEOP3zdNZ1+/WvQmRUiraMhPpEiqq+Hn/wErr46PL7sMvjZz3In\nJw3xiQTlmotPpGLV1cExx6y7VMZNN8FRR8WOSiS9NMSXJY3jtLGpzWD+/DBd0ZQp0LNnmPhVyak0\n1N8Kk8Z20x6USBvNnBlOwP3ggzAb+dSpMGhQ7KhE0k81KJECuYdJX884A9asgWHD4M47YZNN8vt7\n1aBEAh1mLlJEixbBEUeE6zmtWQNnngkPPZR/chKRlilBZUnjOG1sldZmf/sbVFfD3XeHS7PfeSf8\n8pfQUQPmZVFp/a1Y0thuSlAieVq9Gi64APbZB+bNgz32gNmzYfTo2JGJtE+qQYnk4bnn4Pjjw08z\nOOssuPDCcDh5oVSDEgl0HpRIAerq4OKL4Re/CJfJ6N8fJk2CffeNHZlI+6chvixpHKeNrb222WOP\nwVe/Cj//eZghYtw4eOEFJafY2mt/K7U0tpv2oESyvPVWmD/v3nvD4y99CW68EfbaK25cIpVGNSiR\njGXL4KqrwnDeihXQrRuce244z6lLl+JvTzUokUA1KJEmrFoFN9wQak0LFoRlo0fDFVfA1lvHjU2k\nkqkGlSWN47SxpbXN1q6F226DgQPhhz8MyWmPPaC2NpzbpOSUTGntb7Glsd20ByUVZ9UqmDwZLr8c\nXnstLNtxR7j0Ujj4YF23SSQpVIOSirFsWbgExpVXhhNtAb74xVBnOuaY8s8EoRqUSKAalFSsOXPC\npK433ghLloRlAweGxDR6tKYoEkkq1aCypHGcNrYkttnq1eGyF6NGwfbbw69+FZLTkCHh8PEXXwzX\na1JySp8k9rc0SGO76e0p7cq//x2G8W65Bd5/PyzbYIOQjE47DXbbLW58IpI/1aAk9T74AO67Lxz4\n8OST65bvuCMcd1yoL/XtGy++pqgGJRKoBiXtyoIFISlNmQJPPBGmIoJw+YvRo8PErt/4ho7IE0kz\n1aCypHGcNrZytJk7vPJKqCXV1MDmm8PJJ8OMGVBVBQccADffDO+9BxMnwp57Kjm1V3qPFiaN7aY9\nKEmsZcvg8cfDlWofeijMkdegUyf49rfhsMPgoIOgV694cYpIaagGJYmxdCnMmhWG7GbOhKefDkfj\nNdh0UxgxAkaODHtMG20UL9ZiUA1KJFANShLFPZws+49/wFNPhYT07LNh+qEGZqGONHJkuO22G3TQ\noLRIxVCCylJbW0tNTU3sMFKlpTZzh3ffhX/9C/75z7Bn9PTT4ei7xqqqwlx4++wTbnvtpaE7WZ/e\no4VJY7spQUlRLV0aToJ94YWQkF54IdwWLVp/3Y03Dglp993DCbSDB4ej8EREQDUoKcDKlfDmm2Gi\n1ddeg9dfX3e/YY67bL17w6BBsOuuISntsUeYB6+Sj7RTDUokUA1K8rZsWUg08+ev/3POnHA0XcN5\nR9k6dQrz3A0aBDvvHH4OGgRbbFHZyUhEWk8JKksax2nz4Q6LF4cTXHPd3nsvJKD583MPxzXWoQNs\nuy0MGBBuUMv++9cwYAD07x+mFhIplfb6Hi21NLZbURKUmZ0CjAEGAbe7+3HFeF5Zn3vYw1m0CBYu\n/PzPXMsWLgwHI3zwQbgOUj46dw4X6+vXb93PhvvbbBOSU+fO69avrQ0nz4qIFFNRalBm9l2gHhgO\ndG0pQbX3GlR9PdTVhduKFet+Zt9fvjwkm6VLwy3f+00Nr7WkR48wJ11Tt4Zk1KePhuPKQTUokaCk\nNSh3/1NmI7sDW+bzN59+Gs55aXyrr19/WVt+13j5mjXhpM9Vq8LPxvfbsixXAlq5shit2rRu3cIR\ncL17h1uu+42XbbZZSEBdu5Y2LhGRYopWg0ru4cS1QE2bn6VLl3Dr2nXdrfHjhvs9eqy7de/e/P3u\n3cMtaTWeNI5tS3qpvxUmle3m7kW7ARcDk/JYz3PdNtxwvG+xhXufPjO8b98Zvt127jvs4N6z57E5\n199qq/E+dKj7LrvM8K9+dYbvt5/7iBHuW26Ze/1ddhnvY8e6jxo1ww89dIafeab7Oee477JL7vVH\njRrvt9/ufsEFM/zii2f4gw+6/+Uv7sOH517/pz8d78uXuz/22AyfMWOGNzj22Nzrjx8/3t3dZ8xI\n9/pXX311ouLR+u17ffW39rm+58gVLdagzGwGsE/mSbI96e57N1r3YmBLr/AalEg+VIMSCQquQbn7\nvqUJSUREpGlFmXrTzKrMrAtQBXQ0s85mVlWM5y63NF4zJTa1mZST+lth0thuxZob+jxgOfAz4KjM\n/XOL9NwGele/AAAFTklEQVQiIlKBNBefSCSqQYkETdWgdHUdERFJJCWoLGkcp41NbSblpP5WmDS2\nmxKUiIgkkmpQIpGoBiUSqAYlIiKpogSVJY3jtLGpzaSc1N8Kk8Z2U4ISEZFEUg1KJBLVoEQC1aBE\nRCRVlKCypHGcNja1mZST+lth0thuSlAiIpJIqkGJRKIalEigGpSIiKSKElSWNI7TxqY2k3JSfytM\nGttNCUpERBJJNSiRSFSDEglUgxIRkVRRgsqSxnHa2NRmUk7qb4VJY7spQYmISCKpBiUSiWpQIoFq\nUCIikipKUFnSOE4bm9pMykn9rTBpbDclKBERSSTVoEQiUQ1KJFANSkREUkUJKksax2ljU5tJOam/\nFSaN7aYEJSIiiaQalEgkqkGJBKpBiYhIqihBZUnjOG1sajMpJ/W3wqSx3ZSgREQkkVSDEolENSiR\nQDUoERFJlTYnKDPrZGYTzWyumS0xs2fMbEQxgoshjeO0sanNpJzU3wqTxnYrxh5UR2AeMNTdNwLO\nB+42s35FeG4REalQJalBmdnzwAXu/scmfq8alFQ81aBEgrLVoMysLzAAeKnYzy0iIpWjYzGfzMw6\nApOBm9391ebWHTNmDP379wegV69eVFdXU1NTA6wbK43xuPE4bRLiScPjCRMmJOb/l6bHDZIST1oe\nq78V9rhhWRLimT17NosXLwZg7ty5NKXFIT4zmwHsA+Ra8Ul33zuzngF3AN2BUe6+tpnnTOwQX21t\n7WcNKflRmxVGQ3yFUX8rTJLbrakhvqLVoMxsEtAP2N/dV7WwbmITlEi5KEGJBE0lqKIM8ZnZ9cCO\nwLCWkpOIiEg+inEeVD/gBKAaWGBmS83sEzM7ss3RRZBdH5CWqc2knNTfCpPGdmtzgnL3ee7ewd27\nuXuPzK2nu99RjADLbfbs2bFDSB21mZST+lth0thumuooS8ORJZI/tZmUk/pbYdLYbkpQIiKSSEpQ\nWZo7Jl9yU5tJOam/FSaN7Rbtchtl36iIiCRWSc+DEhERKSYN8YmISCIpQYmISCIpQYmISCIpQYmI\nSCIpQTXDzAaY2QozuyV2LElnZp3MbKKZzTWzJWb2jJmNiB1XEplZbzP7o5ktM7M5aZ0WrJzUv9ou\njZ9nSlDNuwZ4OnYQKdERmAcMdfeNgPOBuzNzNcrnXQfUAX2AHwC/M7OBcUNKPPWvtkvd55kSVBPM\n7AhgEfBY7FjSwN2Xu/tF7j4/8/gBYA6wW9zIksXMugGHAOe5+wp3fxKYChwdN7JkU/9qm7R+nilB\n5WBmPYELgR8D6508Ji0zs77AAOCl2LEkzA7AGnd/o9Gy54EvR4onldS/8pfmzzMlqNwuAm5w93di\nB5JGZtYRmAzc7O6vxo4nYboDS7KWLQF6RIglldS/Wi21n2cVl6DMbIaZ1ZvZ2hy3mWa2CzAMmBA7\n1iRpqd0arWeED4+VwKnRAk6uZUDPrGU9gaURYkkd9a/WMbNqUvx5VpQr6qaJu+/b3O/N7DRgG2Be\n5s3QHagys53c/WvliDGJWmq3Rm4ENgX2d/e1JQwprV4FOprZdo2G+XZBQ1X5Uv9qnX1I8eeZ5uLL\nYmZd+Pw33DMJ/+CT3H1hnKjSwcyuB3YGhrn78tjxJJWZ3Q44MBb4KjANGOzur0QNLOHUv1ov7Z9n\nFbcH1RJ3ryMcAgyAmS0D6tLwz4wpc7jvCYS2WxC+rOHAiWm9unIJnQJMAj4APiJ8WCg5NUP9qzBp\n/zzTHpSIiCRSxR0kISIi6aAEJSIiiaQEJSIiiaQEJSIiiaQEJSIiiaQEJSIiiaQEJSIiiaQEJSIi\nifT/9xF4HfJNR+MAAAAASUVORK5CYII=\n",
      "text/plain": [
       "<matplotlib.figure.Figure at 0x7fd7150d0710>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "plt.plot(z, elu(z), \"b-\", linewidth=2)\n",
    "plt.plot([-5, 5], [0, 0], 'k-')\n",
    "plt.plot([-5, 5], [-1, -1], 'k--')\n",
    "plt.plot([0, 0], [-2.2, 3.2], 'k-')\n",
    "plt.grid(True)\n",
    "props = dict(facecolor='black', shrink=0.1)\n",
    "plt.title(r\"ELU activation function ($\\alpha=1$)\", fontsize=14)\n",
    "plt.axis([-5, 5, -2.2, 3.2])\n",
    "\n",
    "save_fig(\"elu_plot\")\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {
    "collapsed": false,
    "deletable": true,
    "editable": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Extracting /tmp/data/train-images-idx3-ubyte.gz\n",
      "Extracting /tmp/data/train-labels-idx1-ubyte.gz\n",
      "Extracting /tmp/data/t10k-images-idx3-ubyte.gz\n",
      "Extracting /tmp/data/t10k-labels-idx1-ubyte.gz\n"
     ]
    }
   ],
   "source": [
    "from tensorflow.examples.tutorials.mnist import input_data\n",
    "mnist = input_data.read_data_sets(\"/tmp/data/\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {
    "collapsed": true,
    "deletable": true,
    "editable": true
   },
   "outputs": [],
   "source": [
    "def leaky_relu(z, name=None):\n",
    "  return tf.maximum(0.01 * z, z, name=name)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {
    "collapsed": true,
    "deletable": true,
    "editable": true
   },
   "outputs": [],
   "source": [
    "import tensorflow as tf"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {
    "collapsed": true,
    "deletable": true,
    "editable": true
   },
   "outputs": [],
   "source": [
    "from IPython.display import clear_output, Image, display, HTML\n",
    "\n",
    "def strip_consts(graph_def, max_const_size=32):\n",
    "    \"\"\"Strip large constant values from graph_def.\"\"\"\n",
    "    strip_def = tf.GraphDef()\n",
    "    for n0 in graph_def.node:\n",
    "        n = strip_def.node.add() \n",
    "        n.MergeFrom(n0)\n",
    "        if n.op == 'Const':\n",
    "            tensor = n.attr['value'].tensor\n",
    "            size = len(tensor.tensor_content)\n",
    "            if size > max_const_size:\n",
    "                tensor.tensor_content = b\"<stripped %d bytes>\"%size\n",
    "    return strip_def\n",
    "\n",
    "def show_graph(graph_def, max_const_size=32):\n",
    "    \"\"\"Visualize TensorFlow graph.\"\"\"\n",
    "    if hasattr(graph_def, 'as_graph_def'):\n",
    "        graph_def = graph_def.as_graph_def()\n",
    "    strip_def = strip_consts(graph_def, max_const_size=max_const_size)\n",
    "    code = \"\"\"\n",
    "        <script>\n",
    "          function load() {{\n",
    "            document.getElementById(\"{id}\").pbtxt = {data};\n",
    "          }}\n",
    "        </script>\n",
    "        <link rel=\"import\" href=\"https://tensorboard.appspot.com/tf-graph-basic.build.html\" onload=load()>\n",
    "        <div style=\"height:600px\">\n",
    "          <tf-graph-basic id=\"{id}\"></tf-graph-basic>\n",
    "        </div>\n",
    "    \"\"\".format(data=repr(str(strip_def)), id='graph'+str(np.random.rand()))\n",
    "\n",
    "    iframe = \"\"\"\n",
    "        <iframe seamless style=\"width:1200px;height:620px;border:0\" srcdoc=\"{}\"></iframe>\n",
    "    \"\"\".format(code.replace('\"', '&quot;'))\n",
    "    display(HTML(iframe))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "deletable": true,
    "editable": true
   },
   "source": [
    "Note: the book uses `tensorflow.contrib.layers.fully_connected()` rather than `tf.layers.dense()` (which did not exist when this chapter was written). It is now preferable to use `tf.layers.dense()`, because anything in the contrib module may change or be deleted without notice. The `dense()` function is almost identical to the `fully_connected()` function. The main differences relevant to this chapter are:\n",
    "* several parameters are renamed: `scope` becomes `name`, `activation_fn` becomes `activation` (and similarly the `_fn` suffix is removed from other parameters such as `normalizer_fn`), `weights_initializer` becomes `kernel_initializer`, etc.\n",
    "* the default `activation` is now `None` rather than `tf.nn.relu`.\n",
    "* it does not support `tensorflow.contrib.framework.arg_scope()` (introduced later in chapter 11).\n",
    "* it does not support regularizer params (introduced later in chapter 11)."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {
    "collapsed": false,
    "deletable": true,
    "editable": true
   },
   "outputs": [],
   "source": [
    "tf.reset_default_graph()\n",
    "\n",
    "n_inputs = 28*28  # MNIST\n",
    "n_hidden1 = 300\n",
    "n_hidden2 = 100\n",
    "n_outputs = 10\n",
    "learning_rate = 0.01\n",
    "\n",
    "X = tf.placeholder(tf.float32, shape=(None, n_inputs), name=\"X\")\n",
    "y = tf.placeholder(tf.int64, shape=(None), name=\"y\")\n",
    "\n",
    "with tf.name_scope(\"dnn\"):\n",
    "    hidden1 = tf.layers.dense(X, n_hidden1, activation=leaky_relu, name=\"hidden1\")\n",
    "    hidden2 = tf.layers.dense(hidden1, n_hidden2, activation=leaky_relu, name=\"hidden2\")\n",
    "    logits = tf.layers.dense(hidden2, n_outputs, name=\"outputs\")\n",
    "\n",
    "with tf.name_scope(\"loss\"):\n",
    "    xentropy = tf.nn.sparse_softmax_cross_entropy_with_logits(labels=y, logits=logits)\n",
    "    loss = tf.reduce_mean(xentropy, name=\"loss\")\n",
    "\n",
    "with tf.name_scope(\"train\"):\n",
    "    optimizer = tf.train.GradientDescentOptimizer(learning_rate)\n",
    "    training_op = optimizer.minimize(loss)\n",
    "\n",
    "with tf.name_scope(\"eval\"):\n",
    "    correct = tf.nn.in_top_k(logits, y, 1)\n",
    "    accuracy = tf.reduce_mean(tf.cast(correct, tf.float32))\n",
    "    \n",
    "init = tf.global_variables_initializer()\n",
    "saver = tf.train.Saver()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {
    "collapsed": false,
    "deletable": true,
    "editable": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0 Train accuracy: 0.58 Test accuracy: 0.6673\n",
      "1 Train accuracy: 0.69 Test accuracy: 0.778\n",
      "2 Train accuracy: 0.83 Test accuracy: 0.8292\n",
      "3 Train accuracy: 0.8 Test accuracy: 0.8534\n",
      "4 Train accuracy: 0.73 Test accuracy: 0.8666\n",
      "5 Train accuracy: 0.87 Test accuracy: 0.8808\n",
      "6 Train accuracy: 0.85 Test accuracy: 0.885\n",
      "7 Train accuracy: 0.88 Test accuracy: 0.8935\n",
      "8 Train accuracy: 0.9 Test accuracy: 0.8973\n",
      "9 Train accuracy: 0.9 Test accuracy: 0.9005\n",
      "10 Train accuracy: 0.88 Test accuracy: 0.9031\n",
      "11 Train accuracy: 0.87 Test accuracy: 0.9087\n",
      "12 Train accuracy: 0.91 Test accuracy: 0.9098\n",
      "13 Train accuracy: 0.89 Test accuracy: 0.9099\n",
      "14 Train accuracy: 0.95 Test accuracy: 0.9124\n",
      "15 Train accuracy: 0.92 Test accuracy: 0.9139\n",
      "16 Train accuracy: 0.87 Test accuracy: 0.9162\n",
      "17 Train accuracy: 0.94 Test accuracy: 0.9181\n",
      "18 Train accuracy: 0.95 Test accuracy: 0.9181\n",
      "19 Train accuracy: 0.88 Test accuracy: 0.9186\n"
     ]
    }
   ],
   "source": [
    "n_epochs = 20\n",
    "batch_size = 100\n",
    "\n",
    "with tf.Session() as sess:\n",
    "    init.run()\n",
    "    for epoch in range(n_epochs):\n",
    "        for iteration in range(len(mnist.test.labels)//batch_size):\n",
    "            X_batch, y_batch = mnist.train.next_batch(batch_size)\n",
    "            sess.run(training_op, feed_dict={X: X_batch, y: y_batch})\n",
    "        acc_train = accuracy.eval(feed_dict={X: X_batch, y: y_batch})\n",
    "        acc_test = accuracy.eval(feed_dict={X: mnist.test.images, y: mnist.test.labels})\n",
    "        print(epoch, \"Train accuracy:\", acc_train, \"Test accuracy:\", acc_test)\n",
    "\n",
    "    save_path = saver.save(sess, \"my_model_final.ckpt\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "deletable": true,
    "editable": true
   },
   "source": [
    "# Batch Normalization"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "deletable": true,
    "editable": true
   },
   "source": [
    "Note: the book uses `tensorflow.contrib.layers.batch_norm()` rather than `tf.layers.batch_normalization()` (which did not exist when this chapter was written). It is now preferable to use `tf.layers.batch_normalization()`, because anything in the contrib module may change or be deleted without notice. Instead of using the `batch_norm()` function as a regularizer parameter to the `fully_connected()` function, we now use `batch_normalization()` and we explicitly create a distinct layer. The parameters are a bit different, in particular:\n",
    "* `decay` is renamed to `momentum`,\n",
    "* `is_training` is renamed to `training`,\n",
    "* `updates_collections` is removed: the update operations needed by batch normalization are added to the `UPDATE_OPS` collection and you need to explicity run these operations during training (see the execution phase below),\n",
    "* we don't need to specify `scale=True`, as that is the default.\n",
    "\n",
    "Also note that in order to run batch norm just _before_ each hidden layer's activation function, we apply the ELU activation function manually, right after the batch norm layer.\n",
    "\n",
    "Note: since the `tf.layers.dense()` function is incompatible with `tf.contrib.layers.arg_scope()` (which is used in the book), we now use python's `functools.partial()` function instead. It makes it easy to create a `my_dense_layer()` function that just calls `tf.layers.dense()` with the desired parameters automatically set (unless they are overridden when calling `my_dense_layer()`). As you can see, the code remains very similar."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {
    "collapsed": false,
    "deletable": true,
    "editable": true
   },
   "outputs": [],
   "source": [
    "tf.reset_default_graph()\n",
    "\n",
    "from functools import partial\n",
    "\n",
    "n_inputs = 28 * 28  # MNIST\n",
    "n_hidden1 = 300\n",
    "n_hidden2 = 100\n",
    "n_outputs = 10\n",
    "learning_rate = 0.01\n",
    "momentum = 0.25\n",
    "\n",
    "X = tf.placeholder(tf.float32, shape=(None, n_inputs), name=\"X\")\n",
    "y = tf.placeholder(tf.int64, shape=(None), name=\"y\")\n",
    "is_training = tf.placeholder(tf.bool, shape=(), name='is_training')\n",
    "\n",
    "with tf.name_scope(\"dnn\"):\n",
    "    he_init = tf.contrib.layers.variance_scaling_initializer()\n",
    "\n",
    "    my_batch_norm_layer = partial(\n",
    "            tf.layers.batch_normalization,\n",
    "            training=is_training,\n",
    "            momentum=0.9)\n",
    "\n",
    "    my_dense_layer = partial(\n",
    "            tf.layers.dense,\n",
    "            kernel_initializer=he_init)\n",
    "\n",
    "    hidden1 = my_dense_layer(X, n_hidden1, name=\"hidden1\")\n",
    "    bn1 = tf.nn.elu(my_batch_norm_layer(hidden1))\n",
    "    hidden2 = my_dense_layer(bn1, n_hidden2, name=\"hidden2\")\n",
    "    bn2 = tf.nn.elu(my_batch_norm_layer(hidden2))\n",
    "    logits_before_bn = my_dense_layer(bn2, n_outputs, activation=None, name=\"outputs\")\n",
    "    logits = my_batch_norm_layer(logits_before_bn)\n",
    "    extra_update_ops = tf.get_collection(tf.GraphKeys.UPDATE_OPS)\n",
    "\n",
    "with tf.name_scope(\"loss\"):\n",
    "    xentropy = tf.nn.sparse_softmax_cross_entropy_with_logits(labels=y, logits=logits)\n",
    "    loss = tf.reduce_mean(xentropy, name=\"loss\")\n",
    "\n",
    "with tf.name_scope(\"train\"):\n",
    "    optimizer = tf.train.MomentumOptimizer(learning_rate, momentum)\n",
    "    training_op = optimizer.minimize(loss)\n",
    "\n",
    "with tf.name_scope(\"eval\"):\n",
    "    correct = tf.nn.in_top_k(logits, y, 1)\n",
    "    accuracy = tf.reduce_mean(tf.cast(correct, tf.float32))\n",
    "    \n",
    "init = tf.global_variables_initializer()\n",
    "saver = tf.train.Saver()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "deletable": true,
    "editable": true
   },
   "source": [
    "Note: since we are using `tf.layers.batch_normalization()` rather than `tf.contrib.layers.batch_norm()` (as in the book), we need to explicitly run the extra update operations needed by batch normalization (`sess.run([training_op, extra_update_ops],...`)."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {
    "collapsed": false,
    "deletable": true,
    "editable": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0 Train accuracy: 0.795 Test accuracy: 0.7745\n",
      "1 Train accuracy: 0.785 Test accuracy: 0.8319\n",
      "2 Train accuracy: 0.84 Test accuracy: 0.8573\n",
      "3 Train accuracy: 0.865 Test accuracy: 0.8718\n",
      "4 Train accuracy: 0.88 Test accuracy: 0.8801\n",
      "5 Train accuracy: 0.88 Test accuracy: 0.8882\n",
      "6 Train accuracy: 0.85 Test accuracy: 0.8954\n",
      "7 Train accuracy: 0.895 Test accuracy: 0.899\n",
      "8 Train accuracy: 0.91 Test accuracy: 0.9034\n",
      "9 Train accuracy: 0.88 Test accuracy: 0.9063\n",
      "10 Train accuracy: 0.935 Test accuracy: 0.9105\n",
      "11 Train accuracy: 0.915 Test accuracy: 0.9126\n",
      "12 Train accuracy: 0.92 Test accuracy: 0.9153\n",
      "13 Train accuracy: 0.93 Test accuracy: 0.9174\n",
      "14 Train accuracy: 0.945 Test accuracy: 0.9194\n",
      "15 Train accuracy: 0.93 Test accuracy: 0.9203\n",
      "16 Train accuracy: 0.91 Test accuracy: 0.921\n",
      "17 Train accuracy: 0.905 Test accuracy: 0.9245\n",
      "18 Train accuracy: 0.88 Test accuracy: 0.924\n",
      "19 Train accuracy: 0.94 Test accuracy: 0.927\n"
     ]
    }
   ],
   "source": [
    "n_epochs = 20\n",
    "batch_size = 200\n",
    "\n",
    "with tf.Session() as sess:\n",
    "    init.run()\n",
    "    for epoch in range(n_epochs):\n",
    "        for iteration in range(len(mnist.test.labels)//batch_size):\n",
    "            X_batch, y_batch = mnist.train.next_batch(batch_size)\n",
    "            sess.run([training_op, extra_update_ops], feed_dict={is_training: True, X: X_batch, y: y_batch})\n",
    "        acc_train = accuracy.eval(feed_dict={is_training: False, X: X_batch, y: y_batch})\n",
    "        acc_test = accuracy.eval(feed_dict={is_training: False, X: mnist.test.images, y: mnist.test.labels})\n",
    "        print(epoch, \"Train accuracy:\", acc_train, \"Test accuracy:\", acc_test)\n",
    "\n",
    "    save_path = saver.save(sess, \"my_model_final.ckpt\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "deletable": true,
    "editable": true
   },
   "source": [
    "Now the same model with $\\ell_1$ regularization:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {
    "collapsed": true,
    "deletable": true,
    "editable": true
   },
   "outputs": [],
   "source": [
    "tf.reset_default_graph()\n",
    "\n",
    "from functools import partial\n",
    "\n",
    "X = tf.placeholder(tf.float32, shape=(None, n_inputs), name=\"X\")\n",
    "y = tf.placeholder(tf.int64, shape=(None), name=\"y\")\n",
    "is_training = tf.placeholder(tf.bool, shape=(), name='is_training')\n",
    "\n",
    "with tf.name_scope(\"dnn\"):\n",
    "    he_init = tf.contrib.layers.variance_scaling_initializer()\n",
    "\n",
    "    my_batch_norm_layer = partial(\n",
    "            tf.layers.batch_normalization,\n",
    "            training=is_training,\n",
    "            momentum=0.9)\n",
    "\n",
    "    my_dense_layer = partial(\n",
    "            tf.layers.dense,\n",
    "            kernel_initializer=he_init,\n",
    "            kernel_regularizer=tf.contrib.layers.l1_regularizer(0.01))\n",
    "\n",
    "    hidden1 = my_dense_layer(X, n_hidden1, name=\"hidden1\")\n",
    "    bn1 = tf.nn.elu(my_batch_norm_layer(hidden1))\n",
    "    hidden2 = my_dense_layer(bn1, n_hidden2, name=\"hidden2\")\n",
    "    bn2 = tf.nn.elu(my_batch_norm_layer(hidden2))\n",
    "    logits_before_bn = my_dense_layer(bn2, n_outputs, activation=None, name=\"outputs\")\n",
    "    logits = my_batch_norm_layer(logits_before_bn)\n",
    "    extra_update_ops = tf.get_collection(tf.GraphKeys.UPDATE_OPS)\n",
    "\n",
    "with tf.name_scope(\"loss\"):\n",
    "    xentropy = tf.nn.sparse_softmax_cross_entropy_with_logits(labels=y, logits=logits)\n",
    "    reg_losses = tf.get_collection(tf.GraphKeys.REGULARIZATION_LOSSES)\n",
    "    base_loss = tf.reduce_mean(xentropy, name=\"base_loss\")\n",
    "    loss = tf.add_n([base_loss] + reg_losses, name=\"loss\")\n",
    "\n",
    "with tf.name_scope(\"train\"):\n",
    "    optimizer = tf.train.MomentumOptimizer(learning_rate, momentum)\n",
    "    training_op = optimizer.minimize(loss)\n",
    "\n",
    "with tf.name_scope(\"eval\"):\n",
    "    correct = tf.nn.in_top_k(logits, y, 1)\n",
    "    accuracy = tf.reduce_mean(tf.cast(correct, tf.float32))\n",
    "\n",
    "init = tf.global_variables_initializer()\n",
    "saver = tf.train.Saver()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {
    "collapsed": false,
    "deletable": true,
    "editable": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0 Train accuracy: 0.79 Test accuracy: 0.7826\n",
      "1 Train accuracy: 0.8 Test accuracy: 0.8398\n",
      "2 Train accuracy: 0.85 Test accuracy: 0.8619\n",
      "3 Train accuracy: 0.87 Test accuracy: 0.8757\n",
      "4 Train accuracy: 0.905 Test accuracy: 0.8843\n",
      "5 Train accuracy: 0.845 Test accuracy: 0.8945\n",
      "6 Train accuracy: 0.895 Test accuracy: 0.8998\n",
      "7 Train accuracy: 0.91 Test accuracy: 0.9041\n",
      "8 Train accuracy: 0.95 Test accuracy: 0.909\n",
      "9 Train accuracy: 0.945 Test accuracy: 0.9079\n",
      "10 Train accuracy: 0.955 Test accuracy: 0.8989\n",
      "11 Train accuracy: 0.94 Test accuracy: 0.8831\n",
      "12 Train accuracy: 0.93 Test accuracy: 0.8818\n",
      "13 Train accuracy: 0.935 Test accuracy: 0.8482\n",
      "14 Train accuracy: 0.905 Test accuracy: 0.7991\n",
      "15 Train accuracy: 0.76 Test accuracy: 0.698\n",
      "16 Train accuracy: 0.905 Test accuracy: 0.8235\n",
      "17 Train accuracy: 0.925 Test accuracy: 0.8659\n",
      "18 Train accuracy: 0.96 Test accuracy: 0.8588\n",
      "19 Train accuracy: 0.87 Test accuracy: 0.8105\n"
     ]
    }
   ],
   "source": [
    "n_epochs = 20\n",
    "batch_size = 200\n",
    "\n",
    "with tf.Session() as sess:\n",
    "    init.run()\n",
    "    for epoch in range(n_epochs):\n",
    "        for iteration in range(len(mnist.test.labels)//batch_size):\n",
    "            X_batch, y_batch = mnist.train.next_batch(batch_size)\n",
    "            sess.run([training_op, extra_update_ops], feed_dict={is_training: True, X: X_batch, y: y_batch})\n",
    "        acc_train = accuracy.eval(feed_dict={is_training: False, X: X_batch, y: y_batch})\n",
    "        acc_test = accuracy.eval(feed_dict={is_training: False, X: mnist.test.images, y: mnist.test.labels})\n",
    "        print(epoch, \"Train accuracy:\", acc_train, \"Test accuracy:\", acc_test)\n",
    "\n",
    "    save_path = saver.save(sess, \"my_model_final.ckpt\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {
    "collapsed": false,
    "deletable": true,
    "editable": true
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['hidden1/kernel:0',\n",
       " 'hidden1/bias:0',\n",
       " 'batch_normalization/beta:0',\n",
       " 'batch_normalization/gamma:0',\n",
       " 'batch_normalization/moving_mean:0',\n",
       " 'batch_normalization/moving_variance:0',\n",
       " 'hidden2/kernel:0',\n",
       " 'hidden2/bias:0',\n",
       " 'batch_normalization_1/beta:0',\n",
       " 'batch_normalization_1/gamma:0',\n",
       " 'batch_normalization_1/moving_mean:0',\n",
       " 'batch_normalization_1/moving_variance:0',\n",
       " 'outputs/kernel:0',\n",
       " 'outputs/bias:0',\n",
       " 'batch_normalization_2/beta:0',\n",
       " 'batch_normalization_2/gamma:0',\n",
       " 'batch_normalization_2/moving_mean:0',\n",
       " 'batch_normalization_2/moving_variance:0',\n",
       " 'hidden1/kernel/Momentum:0',\n",
       " 'hidden1/bias/Momentum:0',\n",
       " 'batch_normalization/beta/Momentum:0',\n",
       " 'batch_normalization/gamma/Momentum:0',\n",
       " 'hidden2/kernel/Momentum:0',\n",
       " 'hidden2/bias/Momentum:0',\n",
       " 'batch_normalization_1/beta/Momentum:0',\n",
       " 'batch_normalization_1/gamma/Momentum:0',\n",
       " 'outputs/kernel/Momentum:0',\n",
       " 'outputs/bias/Momentum:0',\n",
       " 'batch_normalization_2/beta/Momentum:0',\n",
       " 'batch_normalization_2/gamma/Momentum:0']"
      ]
     },
     "execution_count": 18,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "[v.name for v in tf.global_variables()]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "deletable": true,
    "editable": true
   },
   "source": [
    "Note: the weights variable created by the `tf.layers.dense()` function is called `\"kernel\"` (instead of `\"weights\"` when using the `tf.contrib.layers.fully_connected()`, as in the book):"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {
    "collapsed": false,
    "deletable": true,
    "editable": true
   },
   "outputs": [],
   "source": [
    "with tf.variable_scope(\"\", default_name=\"\", reuse=True):  # root scope\n",
    "    weights1 = tf.get_variable(\"hidden1/kernel\")\n",
    "    weights2 = tf.get_variable(\"hidden2/kernel\")\n",
    "    "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {
    "collapsed": false,
    "deletable": true,
    "editable": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[[   0.    0.]\n",
      " [   3.    4.]\n",
      " [  30.   40.]\n",
      " [ 300.  400.]]\n"
     ]
    }
   ],
   "source": [
    "tf.reset_default_graph()\n",
    "\n",
    "x = tf.constant([0., 0., 3., 4., 30., 40., 300., 400.], shape=(4, 2))\n",
    "c = tf.clip_by_norm(x, clip_norm=10)\n",
    "c0 = tf.clip_by_norm(x, clip_norm=350, axes=0)\n",
    "c1 = tf.clip_by_norm(x, clip_norm=10, axes=1)\n",
    "\n",
    "with tf.Session() as sess:\n",
    "    xv = x.eval()\n",
    "    cv = c.eval()\n",
    "    c0v = c0.eval()\n",
    "    c1v = c1.eval()\n",
    "\n",
    "print(xv)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {
    "collapsed": false,
    "deletable": true,
    "editable": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[[ 0.          0.        ]\n",
      " [ 0.05969927  0.07959903]\n",
      " [ 0.59699273  0.79599035]\n",
      " [ 5.96992731  7.95990324]]\n"
     ]
    }
   ],
   "source": [
    "print(cv)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {
    "collapsed": false,
    "deletable": true,
    "editable": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "10.0\n"
     ]
    }
   ],
   "source": [
    "print(np.linalg.norm(cv))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {
    "collapsed": false,
    "deletable": true,
    "editable": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[[   0.            0.        ]\n",
      " [   3.            3.48245788]\n",
      " [  30.           34.82457733]\n",
      " [ 300.          348.24578857]]\n"
     ]
    }
   ],
   "source": [
    "print(c0v)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {
    "collapsed": false,
    "deletable": true,
    "editable": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[ 301.51119995  350.        ]\n"
     ]
    }
   ],
   "source": [
    "print(np.linalg.norm(c0v, axis=0))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {
    "collapsed": false,
    "deletable": true,
    "editable": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[[ 0.          0.        ]\n",
      " [ 3.          4.        ]\n",
      " [ 6.          8.        ]\n",
      " [ 6.00000048  8.        ]]\n"
     ]
    }
   ],
   "source": [
    "print(c1v)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {
    "collapsed": false,
    "deletable": true,
    "editable": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[  0.   5.  10.  10.]\n"
     ]
    }
   ],
   "source": [
    "print(np.linalg.norm(c1v, axis=1))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {
    "collapsed": false,
    "deletable": true,
    "editable": true
   },
   "outputs": [],
   "source": [
    "tf.reset_default_graph()\n",
    "\n",
    "from functools import partial\n",
    "\n",
    "X = tf.placeholder(tf.float32, shape=(None, n_inputs), name=\"X\")\n",
    "y = tf.placeholder(tf.int64, shape=(None), name=\"y\")\n",
    "is_training = tf.placeholder(tf.bool, shape=(), name='is_training')\n",
    "\n",
    "def max_norm_regularizer(threshold, axes=1, name=\"max_norm\", collection=\"max_norm\"):\n",
    "    def max_norm(weights):\n",
    "        clip_weights = tf.assign(weights, tf.clip_by_norm(weights, clip_norm=threshold, axes=axes), name=name)\n",
    "        tf.add_to_collection(collection, clip_weights)\n",
    "        return None # there is no regularization loss term\n",
    "    return max_norm\n",
    "\n",
    "with tf.name_scope(\"dnn\"):\n",
    "    \n",
    "    my_dense_layer = partial(\n",
    "            tf.layers.dense,\n",
    "            activation=tf.nn.relu,\n",
    "            kernel_regularizer=max_norm_regularizer(1.5))\n",
    "\n",
    "    hidden1 = my_dense_layer(X, n_hidden1, name=\"hidden1\")\n",
    "    hidden2 = my_dense_layer(hidden1, n_hidden2, name=\"hidden2\")\n",
    "    logits = my_dense_layer(hidden2, n_outputs, activation=None, name=\"outputs\")\n",
    "\n",
    "clip_all_weights = tf.get_collection(\"max_norm\")\n",
    "        \n",
    "with tf.name_scope(\"loss\"):\n",
    "    xentropy = tf.nn.sparse_softmax_cross_entropy_with_logits(labels=y, logits=logits)\n",
    "    loss = tf.reduce_mean(xentropy, name=\"loss\")\n",
    "\n",
    "with tf.name_scope(\"train\"):\n",
    "    optimizer = tf.train.MomentumOptimizer(learning_rate, momentum)\n",
    "    threshold = 1.0\n",
    "    grads_and_vars = optimizer.compute_gradients(loss)\n",
    "    capped_gvs = [(tf.clip_by_value(grad, -threshold, threshold), var)\n",
    "                  for grad, var in grads_and_vars]\n",
    "    training_op = optimizer.apply_gradients(capped_gvs)\n",
    "\n",
    "with tf.name_scope(\"eval\"):\n",
    "    correct = tf.nn.in_top_k(logits, y, 1)\n",
    "    accuracy = tf.reduce_mean(tf.cast(correct, tf.float32))\n",
    "    \n",
    "init = tf.global_variables_initializer()\n",
    "saver = tf.train.Saver()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {
    "collapsed": false,
    "deletable": true,
    "editable": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0 Train accuracy: 0.86 Test accuracy: 0.8026\n",
      "1 Train accuracy: 0.9 Test accuracy: 0.8744\n",
      "2 Train accuracy: 0.9 Test accuracy: 0.8899\n",
      "3 Train accuracy: 0.94 Test accuracy: 0.9016\n",
      "4 Train accuracy: 0.9 Test accuracy: 0.9076\n",
      "5 Train accuracy: 0.94 Test accuracy: 0.9087\n",
      "6 Train accuracy: 0.88 Test accuracy: 0.9168\n",
      "7 Train accuracy: 0.92 Test accuracy: 0.9188\n",
      "8 Train accuracy: 0.9 Test accuracy: 0.9193\n",
      "9 Train accuracy: 0.92 Test accuracy: 0.9249\n",
      "10 Train accuracy: 0.94 Test accuracy: 0.9275\n",
      "11 Train accuracy: 0.94 Test accuracy: 0.9298\n",
      "12 Train accuracy: 0.96 Test accuracy: 0.929\n",
      "13 Train accuracy: 0.98 Test accuracy: 0.9318\n",
      "14 Train accuracy: 0.92 Test accuracy: 0.9323\n",
      "15 Train accuracy: 0.96 Test accuracy: 0.9347\n",
      "16 Train accuracy: 0.96 Test accuracy: 0.9365\n",
      "17 Train accuracy: 0.88 Test accuracy: 0.9359\n",
      "18 Train accuracy: 0.9 Test accuracy: 0.938\n",
      "19 Train accuracy: 0.94 Test accuracy: 0.9396\n"
     ]
    }
   ],
   "source": [
    "n_epochs = 20\n",
    "batch_size = 50\n",
    "\n",
    "with tf.Session() as sess:\n",
    "    init.run()\n",
    "    for epoch in range(n_epochs):\n",
    "        for iteration in range(len(mnist.test.labels)//batch_size):\n",
    "            X_batch, y_batch = mnist.train.next_batch(batch_size)\n",
    "            sess.run(training_op, feed_dict={is_training: True, X: X_batch, y: y_batch})\n",
    "            sess.run(clip_all_weights)\n",
    "        acc_train = accuracy.eval(feed_dict={is_training: False, X: X_batch, y: y_batch})\n",
    "        acc_test = accuracy.eval(feed_dict={is_training: False, X: mnist.test.images, y: mnist.test.labels})\n",
    "        print(epoch, \"Train accuracy:\", acc_train, \"Test accuracy:\", acc_test)\n",
    "\n",
    "    save_path = saver.save(sess, \"my_model_final.ckpt\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "metadata": {
    "collapsed": false,
    "deletable": true,
    "editable": true
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "\n",
       "        <iframe seamless style=\"width:1200px;height:620px;border:0\" srcdoc=\"\n",
       "        <script>\n",
       "          function load() {\n",
       "            document.getElementById(&quot;graph0.14695850554017442&quot;).pbtxt = 'node {\\n  name: &quot;X&quot;\\n  op: &quot;Placeholder&quot;\\n  attr {\\n    key: &quot;dtype&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;shape&quot;\\n    value {\\n      shape {\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;y&quot;\\n  op: &quot;Placeholder&quot;\\n  attr {\\n    key: &quot;dtype&quot;\\n    value {\\n      type: DT_INT64\\n    }\\n  }\\n  attr {\\n    key: &quot;shape&quot;\\n    value {\\n      shape {\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;is_training&quot;\\n  op: &quot;Placeholder&quot;\\n  attr {\\n    key: &quot;dtype&quot;\\n    value {\\n      type: DT_BOOL\\n    }\\n  }\\n  attr {\\n    key: &quot;shape&quot;\\n    value {\\n      shape {\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;hidden1/kernel/Initializer/random_uniform/shape&quot;\\n  op: &quot;Const&quot;\\n  attr {\\n    key: &quot;_class&quot;\\n    value {\\n      list {\\n        s: &quot;loc:@hidden1/kernel&quot;\\n      }\\n    }\\n  }\\n  attr {\\n    key: &quot;dtype&quot;\\n    value {\\n      type: DT_INT32\\n    }\\n  }\\n  attr {\\n    key: &quot;value&quot;\\n    value {\\n      tensor {\\n        dtype: DT_INT32\\n        tensor_shape {\\n          dim {\\n            size: 2\\n          }\\n        }\\n        tensor_content: &quot;\\\\020\\\\003\\\\000\\\\000,\\\\001\\\\000\\\\000&quot;\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;hidden1/kernel/Initializer/random_uniform/min&quot;\\n  op: &quot;Const&quot;\\n  attr {\\n    key: &quot;_class&quot;\\n    value {\\n      list {\\n        s: &quot;loc:@hidden1/kernel&quot;\\n      }\\n    }\\n  }\\n  attr {\\n    key: &quot;dtype&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;value&quot;\\n    value {\\n      tensor {\\n        dtype: DT_FLOAT\\n        tensor_shape {\\n        }\\n        float_val: -0.07439795136451721\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;hidden1/kernel/Initializer/random_uniform/max&quot;\\n  op: &quot;Const&quot;\\n  attr {\\n    key: &quot;_class&quot;\\n    value {\\n      list {\\n        s: &quot;loc:@hidden1/kernel&quot;\\n      }\\n    }\\n  }\\n  attr {\\n    key: &quot;dtype&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;value&quot;\\n    value {\\n      tensor {\\n        dtype: DT_FLOAT\\n        tensor_shape {\\n        }\\n        float_val: 0.07439795136451721\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;hidden1/kernel/Initializer/random_uniform/RandomUniform&quot;\\n  op: &quot;RandomUniform&quot;\\n  input: &quot;hidden1/kernel/Initializer/random_uniform/shape&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_INT32\\n    }\\n  }\\n  attr {\\n    key: &quot;_class&quot;\\n    value {\\n      list {\\n        s: &quot;loc:@hidden1/kernel&quot;\\n      }\\n    }\\n  }\\n  attr {\\n    key: &quot;dtype&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;seed&quot;\\n    value {\\n      i: 0\\n    }\\n  }\\n  attr {\\n    key: &quot;seed2&quot;\\n    value {\\n      i: 0\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;hidden1/kernel/Initializer/random_uniform/sub&quot;\\n  op: &quot;Sub&quot;\\n  input: &quot;hidden1/kernel/Initializer/random_uniform/max&quot;\\n  input: &quot;hidden1/kernel/Initializer/random_uniform/min&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;_class&quot;\\n    value {\\n      list {\\n        s: &quot;loc:@hidden1/kernel&quot;\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;hidden1/kernel/Initializer/random_uniform/mul&quot;\\n  op: &quot;Mul&quot;\\n  input: &quot;hidden1/kernel/Initializer/random_uniform/RandomUniform&quot;\\n  input: &quot;hidden1/kernel/Initializer/random_uniform/sub&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;_class&quot;\\n    value {\\n      list {\\n        s: &quot;loc:@hidden1/kernel&quot;\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;hidden1/kernel/Initializer/random_uniform&quot;\\n  op: &quot;Add&quot;\\n  input: &quot;hidden1/kernel/Initializer/random_uniform/mul&quot;\\n  input: &quot;hidden1/kernel/Initializer/random_uniform/min&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;_class&quot;\\n    value {\\n      list {\\n        s: &quot;loc:@hidden1/kernel&quot;\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;hidden1/kernel&quot;\\n  op: &quot;VariableV2&quot;\\n  attr {\\n    key: &quot;_class&quot;\\n    value {\\n      list {\\n        s: &quot;loc:@hidden1/kernel&quot;\\n      }\\n    }\\n  }\\n  attr {\\n    key: &quot;container&quot;\\n    value {\\n      s: &quot;&quot;\\n    }\\n  }\\n  attr {\\n    key: &quot;dtype&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;shape&quot;\\n    value {\\n      shape {\\n        dim {\\n          size: 784\\n        }\\n        dim {\\n          size: 300\\n        }\\n      }\\n    }\\n  }\\n  attr {\\n    key: &quot;shared_name&quot;\\n    value {\\n      s: &quot;&quot;\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;hidden1/kernel/Assign&quot;\\n  op: &quot;Assign&quot;\\n  input: &quot;hidden1/kernel&quot;\\n  input: &quot;hidden1/kernel/Initializer/random_uniform&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;_class&quot;\\n    value {\\n      list {\\n        s: &quot;loc:@hidden1/kernel&quot;\\n      }\\n    }\\n  }\\n  attr {\\n    key: &quot;use_locking&quot;\\n    value {\\n      b: true\\n    }\\n  }\\n  attr {\\n    key: &quot;validate_shape&quot;\\n    value {\\n      b: true\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;hidden1/kernel/read&quot;\\n  op: &quot;Identity&quot;\\n  input: &quot;hidden1/kernel&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;_class&quot;\\n    value {\\n      list {\\n        s: &quot;loc:@hidden1/kernel&quot;\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;hidden1/kernel/Regularizer/clip_by_norm/mul&quot;\\n  op: &quot;Mul&quot;\\n  input: &quot;hidden1/kernel/read&quot;\\n  input: &quot;hidden1/kernel/read&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;_class&quot;\\n    value {\\n      list {\\n        s: &quot;loc:@hidden1/kernel&quot;\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;hidden1/kernel/Regularizer/clip_by_norm/Sum/reduction_indices&quot;\\n  op: &quot;Const&quot;\\n  attr {\\n    key: &quot;_class&quot;\\n    value {\\n      list {\\n        s: &quot;loc:@hidden1/kernel&quot;\\n      }\\n    }\\n  }\\n  attr {\\n    key: &quot;dtype&quot;\\n    value {\\n      type: DT_INT32\\n    }\\n  }\\n  attr {\\n    key: &quot;value&quot;\\n    value {\\n      tensor {\\n        dtype: DT_INT32\\n        tensor_shape {\\n        }\\n        int_val: 1\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;hidden1/kernel/Regularizer/clip_by_norm/Sum&quot;\\n  op: &quot;Sum&quot;\\n  input: &quot;hidden1/kernel/Regularizer/clip_by_norm/mul&quot;\\n  input: &quot;hidden1/kernel/Regularizer/clip_by_norm/Sum/reduction_indices&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;Tidx&quot;\\n    value {\\n      type: DT_INT32\\n    }\\n  }\\n  attr {\\n    key: &quot;_class&quot;\\n    value {\\n      list {\\n        s: &quot;loc:@hidden1/kernel&quot;\\n      }\\n    }\\n  }\\n  attr {\\n    key: &quot;keep_dims&quot;\\n    value {\\n      b: true\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;hidden1/kernel/Regularizer/clip_by_norm/Rsqrt&quot;\\n  op: &quot;Rsqrt&quot;\\n  input: &quot;hidden1/kernel/Regularizer/clip_by_norm/Sum&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;_class&quot;\\n    value {\\n      list {\\n        s: &quot;loc:@hidden1/kernel&quot;\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;hidden1/kernel/Regularizer/clip_by_norm/mul_1/y&quot;\\n  op: &quot;Const&quot;\\n  attr {\\n    key: &quot;_class&quot;\\n    value {\\n      list {\\n        s: &quot;loc:@hidden1/kernel&quot;\\n      }\\n    }\\n  }\\n  attr {\\n    key: &quot;dtype&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;value&quot;\\n    value {\\n      tensor {\\n        dtype: DT_FLOAT\\n        tensor_shape {\\n        }\\n        float_val: 1.5\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;hidden1/kernel/Regularizer/clip_by_norm/mul_1&quot;\\n  op: &quot;Mul&quot;\\n  input: &quot;hidden1/kernel/read&quot;\\n  input: &quot;hidden1/kernel/Regularizer/clip_by_norm/mul_1/y&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;_class&quot;\\n    value {\\n      list {\\n        s: &quot;loc:@hidden1/kernel&quot;\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;hidden1/kernel/Regularizer/clip_by_norm/Const&quot;\\n  op: &quot;Const&quot;\\n  attr {\\n    key: &quot;_class&quot;\\n    value {\\n      list {\\n        s: &quot;loc:@hidden1/kernel&quot;\\n      }\\n    }\\n  }\\n  attr {\\n    key: &quot;dtype&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;value&quot;\\n    value {\\n      tensor {\\n        dtype: DT_FLOAT\\n        tensor_shape {\\n        }\\n        float_val: 1.0\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;hidden1/kernel/Regularizer/clip_by_norm/truediv/y&quot;\\n  op: &quot;Const&quot;\\n  attr {\\n    key: &quot;_class&quot;\\n    value {\\n      list {\\n        s: &quot;loc:@hidden1/kernel&quot;\\n      }\\n    }\\n  }\\n  attr {\\n    key: &quot;dtype&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;value&quot;\\n    value {\\n      tensor {\\n        dtype: DT_FLOAT\\n        tensor_shape {\\n        }\\n        float_val: 1.5\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;hidden1/kernel/Regularizer/clip_by_norm/truediv&quot;\\n  op: &quot;RealDiv&quot;\\n  input: &quot;hidden1/kernel/Regularizer/clip_by_norm/Const&quot;\\n  input: &quot;hidden1/kernel/Regularizer/clip_by_norm/truediv/y&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;_class&quot;\\n    value {\\n      list {\\n        s: &quot;loc:@hidden1/kernel&quot;\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;hidden1/kernel/Regularizer/clip_by_norm/Minimum&quot;\\n  op: &quot;Minimum&quot;\\n  input: &quot;hidden1/kernel/Regularizer/clip_by_norm/Rsqrt&quot;\\n  input: &quot;hidden1/kernel/Regularizer/clip_by_norm/truediv&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;_class&quot;\\n    value {\\n      list {\\n        s: &quot;loc:@hidden1/kernel&quot;\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;hidden1/kernel/Regularizer/clip_by_norm/mul_2&quot;\\n  op: &quot;Mul&quot;\\n  input: &quot;hidden1/kernel/Regularizer/clip_by_norm/mul_1&quot;\\n  input: &quot;hidden1/kernel/Regularizer/clip_by_norm/Minimum&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;_class&quot;\\n    value {\\n      list {\\n        s: &quot;loc:@hidden1/kernel&quot;\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;hidden1/kernel/Regularizer/clip_by_norm&quot;\\n  op: &quot;Identity&quot;\\n  input: &quot;hidden1/kernel/Regularizer/clip_by_norm/mul_2&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;_class&quot;\\n    value {\\n      list {\\n        s: &quot;loc:@hidden1/kernel&quot;\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;hidden1/kernel/Regularizer/max_norm&quot;\\n  op: &quot;Assign&quot;\\n  input: &quot;hidden1/kernel&quot;\\n  input: &quot;hidden1/kernel/Regularizer/clip_by_norm&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;_class&quot;\\n    value {\\n      list {\\n        s: &quot;loc:@hidden1/kernel&quot;\\n      }\\n    }\\n  }\\n  attr {\\n    key: &quot;use_locking&quot;\\n    value {\\n      b: true\\n    }\\n  }\\n  attr {\\n    key: &quot;validate_shape&quot;\\n    value {\\n      b: true\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;hidden1/bias/Initializer/Const&quot;\\n  op: &quot;Const&quot;\\n  attr {\\n    key: &quot;_class&quot;\\n    value {\\n      list {\\n        s: &quot;loc:@hidden1/bias&quot;\\n      }\\n    }\\n  }\\n  attr {\\n    key: &quot;dtype&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;value&quot;\\n    value {\\n      tensor {\\n        dtype: DT_FLOAT\\n        tensor_shape {\\n          dim {\\n            size: 300\\n          }\\n        }\\n        float_val: 0.0\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;hidden1/bias&quot;\\n  op: &quot;VariableV2&quot;\\n  attr {\\n    key: &quot;_class&quot;\\n    value {\\n      list {\\n        s: &quot;loc:@hidden1/bias&quot;\\n      }\\n    }\\n  }\\n  attr {\\n    key: &quot;container&quot;\\n    value {\\n      s: &quot;&quot;\\n    }\\n  }\\n  attr {\\n    key: &quot;dtype&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;shape&quot;\\n    value {\\n      shape {\\n        dim {\\n          size: 300\\n        }\\n      }\\n    }\\n  }\\n  attr {\\n    key: &quot;shared_name&quot;\\n    value {\\n      s: &quot;&quot;\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;hidden1/bias/Assign&quot;\\n  op: &quot;Assign&quot;\\n  input: &quot;hidden1/bias&quot;\\n  input: &quot;hidden1/bias/Initializer/Const&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;_class&quot;\\n    value {\\n      list {\\n        s: &quot;loc:@hidden1/bias&quot;\\n      }\\n    }\\n  }\\n  attr {\\n    key: &quot;use_locking&quot;\\n    value {\\n      b: true\\n    }\\n  }\\n  attr {\\n    key: &quot;validate_shape&quot;\\n    value {\\n      b: true\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;hidden1/bias/read&quot;\\n  op: &quot;Identity&quot;\\n  input: &quot;hidden1/bias&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;_class&quot;\\n    value {\\n      list {\\n        s: &quot;loc:@hidden1/bias&quot;\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;dnn/hidden1/MatMul&quot;\\n  op: &quot;MatMul&quot;\\n  input: &quot;X&quot;\\n  input: &quot;hidden1/kernel/read&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;transpose_a&quot;\\n    value {\\n      b: false\\n    }\\n  }\\n  attr {\\n    key: &quot;transpose_b&quot;\\n    value {\\n      b: false\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;dnn/hidden1/BiasAdd&quot;\\n  op: &quot;BiasAdd&quot;\\n  input: &quot;dnn/hidden1/MatMul&quot;\\n  input: &quot;hidden1/bias/read&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;data_format&quot;\\n    value {\\n      s: &quot;NHWC&quot;\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;dnn/hidden1/Relu&quot;\\n  op: &quot;Relu&quot;\\n  input: &quot;dnn/hidden1/BiasAdd&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;hidden2/kernel/Initializer/random_uniform/shape&quot;\\n  op: &quot;Const&quot;\\n  attr {\\n    key: &quot;_class&quot;\\n    value {\\n      list {\\n        s: &quot;loc:@hidden2/kernel&quot;\\n      }\\n    }\\n  }\\n  attr {\\n    key: &quot;dtype&quot;\\n    value {\\n      type: DT_INT32\\n    }\\n  }\\n  attr {\\n    key: &quot;value&quot;\\n    value {\\n      tensor {\\n        dtype: DT_INT32\\n        tensor_shape {\\n          dim {\\n            size: 2\\n          }\\n        }\\n        tensor_content: &quot;,\\\\001\\\\000\\\\000d\\\\000\\\\000\\\\000&quot;\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;hidden2/kernel/Initializer/random_uniform/min&quot;\\n  op: &quot;Const&quot;\\n  attr {\\n    key: &quot;_class&quot;\\n    value {\\n      list {\\n        s: &quot;loc:@hidden2/kernel&quot;\\n      }\\n    }\\n  }\\n  attr {\\n    key: &quot;dtype&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;value&quot;\\n    value {\\n      tensor {\\n        dtype: DT_FLOAT\\n        tensor_shape {\\n        }\\n        float_val: -0.12247448414564133\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;hidden2/kernel/Initializer/random_uniform/max&quot;\\n  op: &quot;Const&quot;\\n  attr {\\n    key: &quot;_class&quot;\\n    value {\\n      list {\\n        s: &quot;loc:@hidden2/kernel&quot;\\n      }\\n    }\\n  }\\n  attr {\\n    key: &quot;dtype&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;value&quot;\\n    value {\\n      tensor {\\n        dtype: DT_FLOAT\\n        tensor_shape {\\n        }\\n        float_val: 0.12247448414564133\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;hidden2/kernel/Initializer/random_uniform/RandomUniform&quot;\\n  op: &quot;RandomUniform&quot;\\n  input: &quot;hidden2/kernel/Initializer/random_uniform/shape&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_INT32\\n    }\\n  }\\n  attr {\\n    key: &quot;_class&quot;\\n    value {\\n      list {\\n        s: &quot;loc:@hidden2/kernel&quot;\\n      }\\n    }\\n  }\\n  attr {\\n    key: &quot;dtype&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;seed&quot;\\n    value {\\n      i: 0\\n    }\\n  }\\n  attr {\\n    key: &quot;seed2&quot;\\n    value {\\n      i: 0\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;hidden2/kernel/Initializer/random_uniform/sub&quot;\\n  op: &quot;Sub&quot;\\n  input: &quot;hidden2/kernel/Initializer/random_uniform/max&quot;\\n  input: &quot;hidden2/kernel/Initializer/random_uniform/min&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;_class&quot;\\n    value {\\n      list {\\n        s: &quot;loc:@hidden2/kernel&quot;\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;hidden2/kernel/Initializer/random_uniform/mul&quot;\\n  op: &quot;Mul&quot;\\n  input: &quot;hidden2/kernel/Initializer/random_uniform/RandomUniform&quot;\\n  input: &quot;hidden2/kernel/Initializer/random_uniform/sub&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;_class&quot;\\n    value {\\n      list {\\n        s: &quot;loc:@hidden2/kernel&quot;\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;hidden2/kernel/Initializer/random_uniform&quot;\\n  op: &quot;Add&quot;\\n  input: &quot;hidden2/kernel/Initializer/random_uniform/mul&quot;\\n  input: &quot;hidden2/kernel/Initializer/random_uniform/min&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;_class&quot;\\n    value {\\n      list {\\n        s: &quot;loc:@hidden2/kernel&quot;\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;hidden2/kernel&quot;\\n  op: &quot;VariableV2&quot;\\n  attr {\\n    key: &quot;_class&quot;\\n    value {\\n      list {\\n        s: &quot;loc:@hidden2/kernel&quot;\\n      }\\n    }\\n  }\\n  attr {\\n    key: &quot;container&quot;\\n    value {\\n      s: &quot;&quot;\\n    }\\n  }\\n  attr {\\n    key: &quot;dtype&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;shape&quot;\\n    value {\\n      shape {\\n        dim {\\n          size: 300\\n        }\\n        dim {\\n          size: 100\\n        }\\n      }\\n    }\\n  }\\n  attr {\\n    key: &quot;shared_name&quot;\\n    value {\\n      s: &quot;&quot;\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;hidden2/kernel/Assign&quot;\\n  op: &quot;Assign&quot;\\n  input: &quot;hidden2/kernel&quot;\\n  input: &quot;hidden2/kernel/Initializer/random_uniform&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;_class&quot;\\n    value {\\n      list {\\n        s: &quot;loc:@hidden2/kernel&quot;\\n      }\\n    }\\n  }\\n  attr {\\n    key: &quot;use_locking&quot;\\n    value {\\n      b: true\\n    }\\n  }\\n  attr {\\n    key: &quot;validate_shape&quot;\\n    value {\\n      b: true\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;hidden2/kernel/read&quot;\\n  op: &quot;Identity&quot;\\n  input: &quot;hidden2/kernel&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;_class&quot;\\n    value {\\n      list {\\n        s: &quot;loc:@hidden2/kernel&quot;\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;hidden2/kernel/Regularizer/clip_by_norm/mul&quot;\\n  op: &quot;Mul&quot;\\n  input: &quot;hidden2/kernel/read&quot;\\n  input: &quot;hidden2/kernel/read&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;_class&quot;\\n    value {\\n      list {\\n        s: &quot;loc:@hidden2/kernel&quot;\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;hidden2/kernel/Regularizer/clip_by_norm/Sum/reduction_indices&quot;\\n  op: &quot;Const&quot;\\n  attr {\\n    key: &quot;_class&quot;\\n    value {\\n      list {\\n        s: &quot;loc:@hidden2/kernel&quot;\\n      }\\n    }\\n  }\\n  attr {\\n    key: &quot;dtype&quot;\\n    value {\\n      type: DT_INT32\\n    }\\n  }\\n  attr {\\n    key: &quot;value&quot;\\n    value {\\n      tensor {\\n        dtype: DT_INT32\\n        tensor_shape {\\n        }\\n        int_val: 1\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;hidden2/kernel/Regularizer/clip_by_norm/Sum&quot;\\n  op: &quot;Sum&quot;\\n  input: &quot;hidden2/kernel/Regularizer/clip_by_norm/mul&quot;\\n  input: &quot;hidden2/kernel/Regularizer/clip_by_norm/Sum/reduction_indices&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;Tidx&quot;\\n    value {\\n      type: DT_INT32\\n    }\\n  }\\n  attr {\\n    key: &quot;_class&quot;\\n    value {\\n      list {\\n        s: &quot;loc:@hidden2/kernel&quot;\\n      }\\n    }\\n  }\\n  attr {\\n    key: &quot;keep_dims&quot;\\n    value {\\n      b: true\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;hidden2/kernel/Regularizer/clip_by_norm/Rsqrt&quot;\\n  op: &quot;Rsqrt&quot;\\n  input: &quot;hidden2/kernel/Regularizer/clip_by_norm/Sum&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;_class&quot;\\n    value {\\n      list {\\n        s: &quot;loc:@hidden2/kernel&quot;\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;hidden2/kernel/Regularizer/clip_by_norm/mul_1/y&quot;\\n  op: &quot;Const&quot;\\n  attr {\\n    key: &quot;_class&quot;\\n    value {\\n      list {\\n        s: &quot;loc:@hidden2/kernel&quot;\\n      }\\n    }\\n  }\\n  attr {\\n    key: &quot;dtype&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;value&quot;\\n    value {\\n      tensor {\\n        dtype: DT_FLOAT\\n        tensor_shape {\\n        }\\n        float_val: 1.5\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;hidden2/kernel/Regularizer/clip_by_norm/mul_1&quot;\\n  op: &quot;Mul&quot;\\n  input: &quot;hidden2/kernel/read&quot;\\n  input: &quot;hidden2/kernel/Regularizer/clip_by_norm/mul_1/y&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;_class&quot;\\n    value {\\n      list {\\n        s: &quot;loc:@hidden2/kernel&quot;\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;hidden2/kernel/Regularizer/clip_by_norm/Const&quot;\\n  op: &quot;Const&quot;\\n  attr {\\n    key: &quot;_class&quot;\\n    value {\\n      list {\\n        s: &quot;loc:@hidden2/kernel&quot;\\n      }\\n    }\\n  }\\n  attr {\\n    key: &quot;dtype&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;value&quot;\\n    value {\\n      tensor {\\n        dtype: DT_FLOAT\\n        tensor_shape {\\n        }\\n        float_val: 1.0\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;hidden2/kernel/Regularizer/clip_by_norm/truediv/y&quot;\\n  op: &quot;Const&quot;\\n  attr {\\n    key: &quot;_class&quot;\\n    value {\\n      list {\\n        s: &quot;loc:@hidden2/kernel&quot;\\n      }\\n    }\\n  }\\n  attr {\\n    key: &quot;dtype&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;value&quot;\\n    value {\\n      tensor {\\n        dtype: DT_FLOAT\\n        tensor_shape {\\n        }\\n        float_val: 1.5\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;hidden2/kernel/Regularizer/clip_by_norm/truediv&quot;\\n  op: &quot;RealDiv&quot;\\n  input: &quot;hidden2/kernel/Regularizer/clip_by_norm/Const&quot;\\n  input: &quot;hidden2/kernel/Regularizer/clip_by_norm/truediv/y&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;_class&quot;\\n    value {\\n      list {\\n        s: &quot;loc:@hidden2/kernel&quot;\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;hidden2/kernel/Regularizer/clip_by_norm/Minimum&quot;\\n  op: &quot;Minimum&quot;\\n  input: &quot;hidden2/kernel/Regularizer/clip_by_norm/Rsqrt&quot;\\n  input: &quot;hidden2/kernel/Regularizer/clip_by_norm/truediv&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;_class&quot;\\n    value {\\n      list {\\n        s: &quot;loc:@hidden2/kernel&quot;\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;hidden2/kernel/Regularizer/clip_by_norm/mul_2&quot;\\n  op: &quot;Mul&quot;\\n  input: &quot;hidden2/kernel/Regularizer/clip_by_norm/mul_1&quot;\\n  input: &quot;hidden2/kernel/Regularizer/clip_by_norm/Minimum&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;_class&quot;\\n    value {\\n      list {\\n        s: &quot;loc:@hidden2/kernel&quot;\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;hidden2/kernel/Regularizer/clip_by_norm&quot;\\n  op: &quot;Identity&quot;\\n  input: &quot;hidden2/kernel/Regularizer/clip_by_norm/mul_2&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;_class&quot;\\n    value {\\n      list {\\n        s: &quot;loc:@hidden2/kernel&quot;\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;hidden2/kernel/Regularizer/max_norm&quot;\\n  op: &quot;Assign&quot;\\n  input: &quot;hidden2/kernel&quot;\\n  input: &quot;hidden2/kernel/Regularizer/clip_by_norm&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;_class&quot;\\n    value {\\n      list {\\n        s: &quot;loc:@hidden2/kernel&quot;\\n      }\\n    }\\n  }\\n  attr {\\n    key: &quot;use_locking&quot;\\n    value {\\n      b: true\\n    }\\n  }\\n  attr {\\n    key: &quot;validate_shape&quot;\\n    value {\\n      b: true\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;hidden2/bias/Initializer/Const&quot;\\n  op: &quot;Const&quot;\\n  attr {\\n    key: &quot;_class&quot;\\n    value {\\n      list {\\n        s: &quot;loc:@hidden2/bias&quot;\\n      }\\n    }\\n  }\\n  attr {\\n    key: &quot;dtype&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;value&quot;\\n    value {\\n      tensor {\\n        dtype: DT_FLOAT\\n        tensor_shape {\\n          dim {\\n            size: 100\\n          }\\n        }\\n        float_val: 0.0\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;hidden2/bias&quot;\\n  op: &quot;VariableV2&quot;\\n  attr {\\n    key: &quot;_class&quot;\\n    value {\\n      list {\\n        s: &quot;loc:@hidden2/bias&quot;\\n      }\\n    }\\n  }\\n  attr {\\n    key: &quot;container&quot;\\n    value {\\n      s: &quot;&quot;\\n    }\\n  }\\n  attr {\\n    key: &quot;dtype&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;shape&quot;\\n    value {\\n      shape {\\n        dim {\\n          size: 100\\n        }\\n      }\\n    }\\n  }\\n  attr {\\n    key: &quot;shared_name&quot;\\n    value {\\n      s: &quot;&quot;\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;hidden2/bias/Assign&quot;\\n  op: &quot;Assign&quot;\\n  input: &quot;hidden2/bias&quot;\\n  input: &quot;hidden2/bias/Initializer/Const&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;_class&quot;\\n    value {\\n      list {\\n        s: &quot;loc:@hidden2/bias&quot;\\n      }\\n    }\\n  }\\n  attr {\\n    key: &quot;use_locking&quot;\\n    value {\\n      b: true\\n    }\\n  }\\n  attr {\\n    key: &quot;validate_shape&quot;\\n    value {\\n      b: true\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;hidden2/bias/read&quot;\\n  op: &quot;Identity&quot;\\n  input: &quot;hidden2/bias&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;_class&quot;\\n    value {\\n      list {\\n        s: &quot;loc:@hidden2/bias&quot;\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;dnn/hidden2/MatMul&quot;\\n  op: &quot;MatMul&quot;\\n  input: &quot;dnn/hidden1/Relu&quot;\\n  input: &quot;hidden2/kernel/read&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;transpose_a&quot;\\n    value {\\n      b: false\\n    }\\n  }\\n  attr {\\n    key: &quot;transpose_b&quot;\\n    value {\\n      b: false\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;dnn/hidden2/BiasAdd&quot;\\n  op: &quot;BiasAdd&quot;\\n  input: &quot;dnn/hidden2/MatMul&quot;\\n  input: &quot;hidden2/bias/read&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;data_format&quot;\\n    value {\\n      s: &quot;NHWC&quot;\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;dnn/hidden2/Relu&quot;\\n  op: &quot;Relu&quot;\\n  input: &quot;dnn/hidden2/BiasAdd&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;outputs/kernel/Initializer/random_uniform/shape&quot;\\n  op: &quot;Const&quot;\\n  attr {\\n    key: &quot;_class&quot;\\n    value {\\n      list {\\n        s: &quot;loc:@outputs/kernel&quot;\\n      }\\n    }\\n  }\\n  attr {\\n    key: &quot;dtype&quot;\\n    value {\\n      type: DT_INT32\\n    }\\n  }\\n  attr {\\n    key: &quot;value&quot;\\n    value {\\n      tensor {\\n        dtype: DT_INT32\\n        tensor_shape {\\n          dim {\\n            size: 2\\n          }\\n        }\\n        tensor_content: &quot;d\\\\000\\\\000\\\\000\\\\n\\\\000\\\\000\\\\000&quot;\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;outputs/kernel/Initializer/random_uniform/min&quot;\\n  op: &quot;Const&quot;\\n  attr {\\n    key: &quot;_class&quot;\\n    value {\\n      list {\\n        s: &quot;loc:@outputs/kernel&quot;\\n      }\\n    }\\n  }\\n  attr {\\n    key: &quot;dtype&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;value&quot;\\n    value {\\n      tensor {\\n        dtype: DT_FLOAT\\n        tensor_shape {\\n        }\\n        float_val: -0.23354968428611755\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;outputs/kernel/Initializer/random_uniform/max&quot;\\n  op: &quot;Const&quot;\\n  attr {\\n    key: &quot;_class&quot;\\n    value {\\n      list {\\n        s: &quot;loc:@outputs/kernel&quot;\\n      }\\n    }\\n  }\\n  attr {\\n    key: &quot;dtype&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;value&quot;\\n    value {\\n      tensor {\\n        dtype: DT_FLOAT\\n        tensor_shape {\\n        }\\n        float_val: 0.23354968428611755\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;outputs/kernel/Initializer/random_uniform/RandomUniform&quot;\\n  op: &quot;RandomUniform&quot;\\n  input: &quot;outputs/kernel/Initializer/random_uniform/shape&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_INT32\\n    }\\n  }\\n  attr {\\n    key: &quot;_class&quot;\\n    value {\\n      list {\\n        s: &quot;loc:@outputs/kernel&quot;\\n      }\\n    }\\n  }\\n  attr {\\n    key: &quot;dtype&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;seed&quot;\\n    value {\\n      i: 0\\n    }\\n  }\\n  attr {\\n    key: &quot;seed2&quot;\\n    value {\\n      i: 0\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;outputs/kernel/Initializer/random_uniform/sub&quot;\\n  op: &quot;Sub&quot;\\n  input: &quot;outputs/kernel/Initializer/random_uniform/max&quot;\\n  input: &quot;outputs/kernel/Initializer/random_uniform/min&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;_class&quot;\\n    value {\\n      list {\\n        s: &quot;loc:@outputs/kernel&quot;\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;outputs/kernel/Initializer/random_uniform/mul&quot;\\n  op: &quot;Mul&quot;\\n  input: &quot;outputs/kernel/Initializer/random_uniform/RandomUniform&quot;\\n  input: &quot;outputs/kernel/Initializer/random_uniform/sub&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;_class&quot;\\n    value {\\n      list {\\n        s: &quot;loc:@outputs/kernel&quot;\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;outputs/kernel/Initializer/random_uniform&quot;\\n  op: &quot;Add&quot;\\n  input: &quot;outputs/kernel/Initializer/random_uniform/mul&quot;\\n  input: &quot;outputs/kernel/Initializer/random_uniform/min&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;_class&quot;\\n    value {\\n      list {\\n        s: &quot;loc:@outputs/kernel&quot;\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;outputs/kernel&quot;\\n  op: &quot;VariableV2&quot;\\n  attr {\\n    key: &quot;_class&quot;\\n    value {\\n      list {\\n        s: &quot;loc:@outputs/kernel&quot;\\n      }\\n    }\\n  }\\n  attr {\\n    key: &quot;container&quot;\\n    value {\\n      s: &quot;&quot;\\n    }\\n  }\\n  attr {\\n    key: &quot;dtype&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;shape&quot;\\n    value {\\n      shape {\\n        dim {\\n          size: 100\\n        }\\n        dim {\\n          size: 10\\n        }\\n      }\\n    }\\n  }\\n  attr {\\n    key: &quot;shared_name&quot;\\n    value {\\n      s: &quot;&quot;\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;outputs/kernel/Assign&quot;\\n  op: &quot;Assign&quot;\\n  input: &quot;outputs/kernel&quot;\\n  input: &quot;outputs/kernel/Initializer/random_uniform&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;_class&quot;\\n    value {\\n      list {\\n        s: &quot;loc:@outputs/kernel&quot;\\n      }\\n    }\\n  }\\n  attr {\\n    key: &quot;use_locking&quot;\\n    value {\\n      b: true\\n    }\\n  }\\n  attr {\\n    key: &quot;validate_shape&quot;\\n    value {\\n      b: true\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;outputs/kernel/read&quot;\\n  op: &quot;Identity&quot;\\n  input: &quot;outputs/kernel&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;_class&quot;\\n    value {\\n      list {\\n        s: &quot;loc:@outputs/kernel&quot;\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;outputs/kernel/Regularizer/clip_by_norm/mul&quot;\\n  op: &quot;Mul&quot;\\n  input: &quot;outputs/kernel/read&quot;\\n  input: &quot;outputs/kernel/read&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;_class&quot;\\n    value {\\n      list {\\n        s: &quot;loc:@outputs/kernel&quot;\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;outputs/kernel/Regularizer/clip_by_norm/Sum/reduction_indices&quot;\\n  op: &quot;Const&quot;\\n  attr {\\n    key: &quot;_class&quot;\\n    value {\\n      list {\\n        s: &quot;loc:@outputs/kernel&quot;\\n      }\\n    }\\n  }\\n  attr {\\n    key: &quot;dtype&quot;\\n    value {\\n      type: DT_INT32\\n    }\\n  }\\n  attr {\\n    key: &quot;value&quot;\\n    value {\\n      tensor {\\n        dtype: DT_INT32\\n        tensor_shape {\\n        }\\n        int_val: 1\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;outputs/kernel/Regularizer/clip_by_norm/Sum&quot;\\n  op: &quot;Sum&quot;\\n  input: &quot;outputs/kernel/Regularizer/clip_by_norm/mul&quot;\\n  input: &quot;outputs/kernel/Regularizer/clip_by_norm/Sum/reduction_indices&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;Tidx&quot;\\n    value {\\n      type: DT_INT32\\n    }\\n  }\\n  attr {\\n    key: &quot;_class&quot;\\n    value {\\n      list {\\n        s: &quot;loc:@outputs/kernel&quot;\\n      }\\n    }\\n  }\\n  attr {\\n    key: &quot;keep_dims&quot;\\n    value {\\n      b: true\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;outputs/kernel/Regularizer/clip_by_norm/Rsqrt&quot;\\n  op: &quot;Rsqrt&quot;\\n  input: &quot;outputs/kernel/Regularizer/clip_by_norm/Sum&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;_class&quot;\\n    value {\\n      list {\\n        s: &quot;loc:@outputs/kernel&quot;\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;outputs/kernel/Regularizer/clip_by_norm/mul_1/y&quot;\\n  op: &quot;Const&quot;\\n  attr {\\n    key: &quot;_class&quot;\\n    value {\\n      list {\\n        s: &quot;loc:@outputs/kernel&quot;\\n      }\\n    }\\n  }\\n  attr {\\n    key: &quot;dtype&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;value&quot;\\n    value {\\n      tensor {\\n        dtype: DT_FLOAT\\n        tensor_shape {\\n        }\\n        float_val: 1.5\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;outputs/kernel/Regularizer/clip_by_norm/mul_1&quot;\\n  op: &quot;Mul&quot;\\n  input: &quot;outputs/kernel/read&quot;\\n  input: &quot;outputs/kernel/Regularizer/clip_by_norm/mul_1/y&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;_class&quot;\\n    value {\\n      list {\\n        s: &quot;loc:@outputs/kernel&quot;\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;outputs/kernel/Regularizer/clip_by_norm/Const&quot;\\n  op: &quot;Const&quot;\\n  attr {\\n    key: &quot;_class&quot;\\n    value {\\n      list {\\n        s: &quot;loc:@outputs/kernel&quot;\\n      }\\n    }\\n  }\\n  attr {\\n    key: &quot;dtype&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;value&quot;\\n    value {\\n      tensor {\\n        dtype: DT_FLOAT\\n        tensor_shape {\\n        }\\n        float_val: 1.0\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;outputs/kernel/Regularizer/clip_by_norm/truediv/y&quot;\\n  op: &quot;Const&quot;\\n  attr {\\n    key: &quot;_class&quot;\\n    value {\\n      list {\\n        s: &quot;loc:@outputs/kernel&quot;\\n      }\\n    }\\n  }\\n  attr {\\n    key: &quot;dtype&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;value&quot;\\n    value {\\n      tensor {\\n        dtype: DT_FLOAT\\n        tensor_shape {\\n        }\\n        float_val: 1.5\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;outputs/kernel/Regularizer/clip_by_norm/truediv&quot;\\n  op: &quot;RealDiv&quot;\\n  input: &quot;outputs/kernel/Regularizer/clip_by_norm/Const&quot;\\n  input: &quot;outputs/kernel/Regularizer/clip_by_norm/truediv/y&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;_class&quot;\\n    value {\\n      list {\\n        s: &quot;loc:@outputs/kernel&quot;\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;outputs/kernel/Regularizer/clip_by_norm/Minimum&quot;\\n  op: &quot;Minimum&quot;\\n  input: &quot;outputs/kernel/Regularizer/clip_by_norm/Rsqrt&quot;\\n  input: &quot;outputs/kernel/Regularizer/clip_by_norm/truediv&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;_class&quot;\\n    value {\\n      list {\\n        s: &quot;loc:@outputs/kernel&quot;\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;outputs/kernel/Regularizer/clip_by_norm/mul_2&quot;\\n  op: &quot;Mul&quot;\\n  input: &quot;outputs/kernel/Regularizer/clip_by_norm/mul_1&quot;\\n  input: &quot;outputs/kernel/Regularizer/clip_by_norm/Minimum&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;_class&quot;\\n    value {\\n      list {\\n        s: &quot;loc:@outputs/kernel&quot;\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;outputs/kernel/Regularizer/clip_by_norm&quot;\\n  op: &quot;Identity&quot;\\n  input: &quot;outputs/kernel/Regularizer/clip_by_norm/mul_2&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;_class&quot;\\n    value {\\n      list {\\n        s: &quot;loc:@outputs/kernel&quot;\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;outputs/kernel/Regularizer/max_norm&quot;\\n  op: &quot;Assign&quot;\\n  input: &quot;outputs/kernel&quot;\\n  input: &quot;outputs/kernel/Regularizer/clip_by_norm&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;_class&quot;\\n    value {\\n      list {\\n        s: &quot;loc:@outputs/kernel&quot;\\n      }\\n    }\\n  }\\n  attr {\\n    key: &quot;use_locking&quot;\\n    value {\\n      b: true\\n    }\\n  }\\n  attr {\\n    key: &quot;validate_shape&quot;\\n    value {\\n      b: true\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;outputs/bias/Initializer/Const&quot;\\n  op: &quot;Const&quot;\\n  attr {\\n    key: &quot;_class&quot;\\n    value {\\n      list {\\n        s: &quot;loc:@outputs/bias&quot;\\n      }\\n    }\\n  }\\n  attr {\\n    key: &quot;dtype&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;value&quot;\\n    value {\\n      tensor {\\n        dtype: DT_FLOAT\\n        tensor_shape {\\n          dim {\\n            size: 10\\n          }\\n        }\\n        float_val: 0.0\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;outputs/bias&quot;\\n  op: &quot;VariableV2&quot;\\n  attr {\\n    key: &quot;_class&quot;\\n    value {\\n      list {\\n        s: &quot;loc:@outputs/bias&quot;\\n      }\\n    }\\n  }\\n  attr {\\n    key: &quot;container&quot;\\n    value {\\n      s: &quot;&quot;\\n    }\\n  }\\n  attr {\\n    key: &quot;dtype&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;shape&quot;\\n    value {\\n      shape {\\n        dim {\\n          size: 10\\n        }\\n      }\\n    }\\n  }\\n  attr {\\n    key: &quot;shared_name&quot;\\n    value {\\n      s: &quot;&quot;\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;outputs/bias/Assign&quot;\\n  op: &quot;Assign&quot;\\n  input: &quot;outputs/bias&quot;\\n  input: &quot;outputs/bias/Initializer/Const&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;_class&quot;\\n    value {\\n      list {\\n        s: &quot;loc:@outputs/bias&quot;\\n      }\\n    }\\n  }\\n  attr {\\n    key: &quot;use_locking&quot;\\n    value {\\n      b: true\\n    }\\n  }\\n  attr {\\n    key: &quot;validate_shape&quot;\\n    value {\\n      b: true\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;outputs/bias/read&quot;\\n  op: &quot;Identity&quot;\\n  input: &quot;outputs/bias&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;_class&quot;\\n    value {\\n      list {\\n        s: &quot;loc:@outputs/bias&quot;\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;dnn/outputs/MatMul&quot;\\n  op: &quot;MatMul&quot;\\n  input: &quot;dnn/hidden2/Relu&quot;\\n  input: &quot;outputs/kernel/read&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;transpose_a&quot;\\n    value {\\n      b: false\\n    }\\n  }\\n  attr {\\n    key: &quot;transpose_b&quot;\\n    value {\\n      b: false\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;dnn/outputs/BiasAdd&quot;\\n  op: &quot;BiasAdd&quot;\\n  input: &quot;dnn/outputs/MatMul&quot;\\n  input: &quot;outputs/bias/read&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;data_format&quot;\\n    value {\\n      s: &quot;NHWC&quot;\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;loss/SparseSoftmaxCrossEntropyWithLogits/Shape&quot;\\n  op: &quot;Shape&quot;\\n  input: &quot;y&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_INT64\\n    }\\n  }\\n  attr {\\n    key: &quot;out_type&quot;\\n    value {\\n      type: DT_INT32\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;loss/SparseSoftmaxCrossEntropyWithLogits/SparseSoftmaxCrossEntropyWithLogits&quot;\\n  op: &quot;SparseSoftmaxCrossEntropyWithLogits&quot;\\n  input: &quot;dnn/outputs/BiasAdd&quot;\\n  input: &quot;y&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;Tlabels&quot;\\n    value {\\n      type: DT_INT64\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;loss/Const&quot;\\n  op: &quot;Const&quot;\\n  attr {\\n    key: &quot;dtype&quot;\\n    value {\\n      type: DT_INT32\\n    }\\n  }\\n  attr {\\n    key: &quot;value&quot;\\n    value {\\n      tensor {\\n        dtype: DT_INT32\\n        tensor_shape {\\n          dim {\\n            size: 1\\n          }\\n        }\\n        int_val: 0\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;loss/loss&quot;\\n  op: &quot;Mean&quot;\\n  input: &quot;loss/SparseSoftmaxCrossEntropyWithLogits/SparseSoftmaxCrossEntropyWithLogits&quot;\\n  input: &quot;loss/Const&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;Tidx&quot;\\n    value {\\n      type: DT_INT32\\n    }\\n  }\\n  attr {\\n    key: &quot;keep_dims&quot;\\n    value {\\n      b: false\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;train/gradients/Shape&quot;\\n  op: &quot;Const&quot;\\n  attr {\\n    key: &quot;dtype&quot;\\n    value {\\n      type: DT_INT32\\n    }\\n  }\\n  attr {\\n    key: &quot;value&quot;\\n    value {\\n      tensor {\\n        dtype: DT_INT32\\n        tensor_shape {\\n          dim {\\n          }\\n        }\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;train/gradients/Const&quot;\\n  op: &quot;Const&quot;\\n  attr {\\n    key: &quot;dtype&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;value&quot;\\n    value {\\n      tensor {\\n        dtype: DT_FLOAT\\n        tensor_shape {\\n        }\\n        float_val: 1.0\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;train/gradients/Fill&quot;\\n  op: &quot;Fill&quot;\\n  input: &quot;train/gradients/Shape&quot;\\n  input: &quot;train/gradients/Const&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;train/gradients/loss/loss_grad/Reshape/shape&quot;\\n  op: &quot;Const&quot;\\n  attr {\\n    key: &quot;dtype&quot;\\n    value {\\n      type: DT_INT32\\n    }\\n  }\\n  attr {\\n    key: &quot;value&quot;\\n    value {\\n      tensor {\\n        dtype: DT_INT32\\n        tensor_shape {\\n          dim {\\n            size: 1\\n          }\\n        }\\n        int_val: 1\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;train/gradients/loss/loss_grad/Reshape&quot;\\n  op: &quot;Reshape&quot;\\n  input: &quot;train/gradients/Fill&quot;\\n  input: &quot;train/gradients/loss/loss_grad/Reshape/shape&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;Tshape&quot;\\n    value {\\n      type: DT_INT32\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;train/gradients/loss/loss_grad/Shape&quot;\\n  op: &quot;Shape&quot;\\n  input: &quot;loss/SparseSoftmaxCrossEntropyWithLogits/SparseSoftmaxCrossEntropyWithLogits&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;out_type&quot;\\n    value {\\n      type: DT_INT32\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;train/gradients/loss/loss_grad/Tile&quot;\\n  op: &quot;Tile&quot;\\n  input: &quot;train/gradients/loss/loss_grad/Reshape&quot;\\n  input: &quot;train/gradients/loss/loss_grad/Shape&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;Tmultiples&quot;\\n    value {\\n      type: DT_INT32\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;train/gradients/loss/loss_grad/Shape_1&quot;\\n  op: &quot;Shape&quot;\\n  input: &quot;loss/SparseSoftmaxCrossEntropyWithLogits/SparseSoftmaxCrossEntropyWithLogits&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;out_type&quot;\\n    value {\\n      type: DT_INT32\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;train/gradients/loss/loss_grad/Shape_2&quot;\\n  op: &quot;Const&quot;\\n  attr {\\n    key: &quot;dtype&quot;\\n    value {\\n      type: DT_INT32\\n    }\\n  }\\n  attr {\\n    key: &quot;value&quot;\\n    value {\\n      tensor {\\n        dtype: DT_INT32\\n        tensor_shape {\\n          dim {\\n          }\\n        }\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;train/gradients/loss/loss_grad/Const&quot;\\n  op: &quot;Const&quot;\\n  attr {\\n    key: &quot;dtype&quot;\\n    value {\\n      type: DT_INT32\\n    }\\n  }\\n  attr {\\n    key: &quot;value&quot;\\n    value {\\n      tensor {\\n        dtype: DT_INT32\\n        tensor_shape {\\n          dim {\\n            size: 1\\n          }\\n        }\\n        int_val: 0\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;train/gradients/loss/loss_grad/Prod&quot;\\n  op: &quot;Prod&quot;\\n  input: &quot;train/gradients/loss/loss_grad/Shape_1&quot;\\n  input: &quot;train/gradients/loss/loss_grad/Const&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_INT32\\n    }\\n  }\\n  attr {\\n    key: &quot;Tidx&quot;\\n    value {\\n      type: DT_INT32\\n    }\\n  }\\n  attr {\\n    key: &quot;keep_dims&quot;\\n    value {\\n      b: false\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;train/gradients/loss/loss_grad/Const_1&quot;\\n  op: &quot;Const&quot;\\n  attr {\\n    key: &quot;dtype&quot;\\n    value {\\n      type: DT_INT32\\n    }\\n  }\\n  attr {\\n    key: &quot;value&quot;\\n    value {\\n      tensor {\\n        dtype: DT_INT32\\n        tensor_shape {\\n          dim {\\n            size: 1\\n          }\\n        }\\n        int_val: 0\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;train/gradients/loss/loss_grad/Prod_1&quot;\\n  op: &quot;Prod&quot;\\n  input: &quot;train/gradients/loss/loss_grad/Shape_2&quot;\\n  input: &quot;train/gradients/loss/loss_grad/Const_1&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_INT32\\n    }\\n  }\\n  attr {\\n    key: &quot;Tidx&quot;\\n    value {\\n      type: DT_INT32\\n    }\\n  }\\n  attr {\\n    key: &quot;keep_dims&quot;\\n    value {\\n      b: false\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;train/gradients/loss/loss_grad/Maximum/y&quot;\\n  op: &quot;Const&quot;\\n  attr {\\n    key: &quot;dtype&quot;\\n    value {\\n      type: DT_INT32\\n    }\\n  }\\n  attr {\\n    key: &quot;value&quot;\\n    value {\\n      tensor {\\n        dtype: DT_INT32\\n        tensor_shape {\\n        }\\n        int_val: 1\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;train/gradients/loss/loss_grad/Maximum&quot;\\n  op: &quot;Maximum&quot;\\n  input: &quot;train/gradients/loss/loss_grad/Prod_1&quot;\\n  input: &quot;train/gradients/loss/loss_grad/Maximum/y&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_INT32\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;train/gradients/loss/loss_grad/floordiv&quot;\\n  op: &quot;FloorDiv&quot;\\n  input: &quot;train/gradients/loss/loss_grad/Prod&quot;\\n  input: &quot;train/gradients/loss/loss_grad/Maximum&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_INT32\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;train/gradients/loss/loss_grad/Cast&quot;\\n  op: &quot;Cast&quot;\\n  input: &quot;train/gradients/loss/loss_grad/floordiv&quot;\\n  attr {\\n    key: &quot;DstT&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;SrcT&quot;\\n    value {\\n      type: DT_INT32\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;train/gradients/loss/loss_grad/truediv&quot;\\n  op: &quot;RealDiv&quot;\\n  input: &quot;train/gradients/loss/loss_grad/Tile&quot;\\n  input: &quot;train/gradients/loss/loss_grad/Cast&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;train/gradients/zeros_like&quot;\\n  op: &quot;ZerosLike&quot;\\n  input: &quot;loss/SparseSoftmaxCrossEntropyWithLogits/SparseSoftmaxCrossEntropyWithLogits:1&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;train/gradients/loss/SparseSoftmaxCrossEntropyWithLogits/SparseSoftmaxCrossEntropyWithLogits_grad/PreventGradient&quot;\\n  op: &quot;PreventGradient&quot;\\n  input: &quot;loss/SparseSoftmaxCrossEntropyWithLogits/SparseSoftmaxCrossEntropyWithLogits:1&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;train/gradients/loss/SparseSoftmaxCrossEntropyWithLogits/SparseSoftmaxCrossEntropyWithLogits_grad/ExpandDims/dim&quot;\\n  op: &quot;Const&quot;\\n  attr {\\n    key: &quot;dtype&quot;\\n    value {\\n      type: DT_INT32\\n    }\\n  }\\n  attr {\\n    key: &quot;value&quot;\\n    value {\\n      tensor {\\n        dtype: DT_INT32\\n        tensor_shape {\\n        }\\n        int_val: -1\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;train/gradients/loss/SparseSoftmaxCrossEntropyWithLogits/SparseSoftmaxCrossEntropyWithLogits_grad/ExpandDims&quot;\\n  op: &quot;ExpandDims&quot;\\n  input: &quot;train/gradients/loss/loss_grad/truediv&quot;\\n  input: &quot;train/gradients/loss/SparseSoftmaxCrossEntropyWithLogits/SparseSoftmaxCrossEntropyWithLogits_grad/ExpandDims/dim&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;Tdim&quot;\\n    value {\\n      type: DT_INT32\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;train/gradients/loss/SparseSoftmaxCrossEntropyWithLogits/SparseSoftmaxCrossEntropyWithLogits_grad/mul&quot;\\n  op: &quot;Mul&quot;\\n  input: &quot;train/gradients/loss/SparseSoftmaxCrossEntropyWithLogits/SparseSoftmaxCrossEntropyWithLogits_grad/ExpandDims&quot;\\n  input: &quot;train/gradients/loss/SparseSoftmaxCrossEntropyWithLogits/SparseSoftmaxCrossEntropyWithLogits_grad/PreventGradient&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;train/gradients/dnn/outputs/BiasAdd_grad/BiasAddGrad&quot;\\n  op: &quot;BiasAddGrad&quot;\\n  input: &quot;train/gradients/loss/SparseSoftmaxCrossEntropyWithLogits/SparseSoftmaxCrossEntropyWithLogits_grad/mul&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;data_format&quot;\\n    value {\\n      s: &quot;NHWC&quot;\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;train/gradients/dnn/outputs/BiasAdd_grad/tuple/group_deps&quot;\\n  op: &quot;NoOp&quot;\\n  input: &quot;^train/gradients/loss/SparseSoftmaxCrossEntropyWithLogits/SparseSoftmaxCrossEntropyWithLogits_grad/mul&quot;\\n  input: &quot;^train/gradients/dnn/outputs/BiasAdd_grad/BiasAddGrad&quot;\\n}\\nnode {\\n  name: &quot;train/gradients/dnn/outputs/BiasAdd_grad/tuple/control_dependency&quot;\\n  op: &quot;Identity&quot;\\n  input: &quot;train/gradients/loss/SparseSoftmaxCrossEntropyWithLogits/SparseSoftmaxCrossEntropyWithLogits_grad/mul&quot;\\n  input: &quot;^train/gradients/dnn/outputs/BiasAdd_grad/tuple/group_deps&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;_class&quot;\\n    value {\\n      list {\\n        s: &quot;loc:@train/gradients/loss/SparseSoftmaxCrossEntropyWithLogits/SparseSoftmaxCrossEntropyWithLogits_grad/mul&quot;\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;train/gradients/dnn/outputs/BiasAdd_grad/tuple/control_dependency_1&quot;\\n  op: &quot;Identity&quot;\\n  input: &quot;train/gradients/dnn/outputs/BiasAdd_grad/BiasAddGrad&quot;\\n  input: &quot;^train/gradients/dnn/outputs/BiasAdd_grad/tuple/group_deps&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;_class&quot;\\n    value {\\n      list {\\n        s: &quot;loc:@train/gradients/dnn/outputs/BiasAdd_grad/BiasAddGrad&quot;\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;train/gradients/dnn/outputs/MatMul_grad/MatMul&quot;\\n  op: &quot;MatMul&quot;\\n  input: &quot;train/gradients/dnn/outputs/BiasAdd_grad/tuple/control_dependency&quot;\\n  input: &quot;outputs/kernel/read&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;transpose_a&quot;\\n    value {\\n      b: false\\n    }\\n  }\\n  attr {\\n    key: &quot;transpose_b&quot;\\n    value {\\n      b: true\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;train/gradients/dnn/outputs/MatMul_grad/MatMul_1&quot;\\n  op: &quot;MatMul&quot;\\n  input: &quot;dnn/hidden2/Relu&quot;\\n  input: &quot;train/gradients/dnn/outputs/BiasAdd_grad/tuple/control_dependency&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;transpose_a&quot;\\n    value {\\n      b: true\\n    }\\n  }\\n  attr {\\n    key: &quot;transpose_b&quot;\\n    value {\\n      b: false\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;train/gradients/dnn/outputs/MatMul_grad/tuple/group_deps&quot;\\n  op: &quot;NoOp&quot;\\n  input: &quot;^train/gradients/dnn/outputs/MatMul_grad/MatMul&quot;\\n  input: &quot;^train/gradients/dnn/outputs/MatMul_grad/MatMul_1&quot;\\n}\\nnode {\\n  name: &quot;train/gradients/dnn/outputs/MatMul_grad/tuple/control_dependency&quot;\\n  op: &quot;Identity&quot;\\n  input: &quot;train/gradients/dnn/outputs/MatMul_grad/MatMul&quot;\\n  input: &quot;^train/gradients/dnn/outputs/MatMul_grad/tuple/group_deps&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;_class&quot;\\n    value {\\n      list {\\n        s: &quot;loc:@train/gradients/dnn/outputs/MatMul_grad/MatMul&quot;\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;train/gradients/dnn/outputs/MatMul_grad/tuple/control_dependency_1&quot;\\n  op: &quot;Identity&quot;\\n  input: &quot;train/gradients/dnn/outputs/MatMul_grad/MatMul_1&quot;\\n  input: &quot;^train/gradients/dnn/outputs/MatMul_grad/tuple/group_deps&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;_class&quot;\\n    value {\\n      list {\\n        s: &quot;loc:@train/gradients/dnn/outputs/MatMul_grad/MatMul_1&quot;\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;train/gradients/dnn/hidden2/Relu_grad/ReluGrad&quot;\\n  op: &quot;ReluGrad&quot;\\n  input: &quot;train/gradients/dnn/outputs/MatMul_grad/tuple/control_dependency&quot;\\n  input: &quot;dnn/hidden2/Relu&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;train/gradients/dnn/hidden2/BiasAdd_grad/BiasAddGrad&quot;\\n  op: &quot;BiasAddGrad&quot;\\n  input: &quot;train/gradients/dnn/hidden2/Relu_grad/ReluGrad&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;data_format&quot;\\n    value {\\n      s: &quot;NHWC&quot;\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;train/gradients/dnn/hidden2/BiasAdd_grad/tuple/group_deps&quot;\\n  op: &quot;NoOp&quot;\\n  input: &quot;^train/gradients/dnn/hidden2/Relu_grad/ReluGrad&quot;\\n  input: &quot;^train/gradients/dnn/hidden2/BiasAdd_grad/BiasAddGrad&quot;\\n}\\nnode {\\n  name: &quot;train/gradients/dnn/hidden2/BiasAdd_grad/tuple/control_dependency&quot;\\n  op: &quot;Identity&quot;\\n  input: &quot;train/gradients/dnn/hidden2/Relu_grad/ReluGrad&quot;\\n  input: &quot;^train/gradients/dnn/hidden2/BiasAdd_grad/tuple/group_deps&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;_class&quot;\\n    value {\\n      list {\\n        s: &quot;loc:@train/gradients/dnn/hidden2/Relu_grad/ReluGrad&quot;\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;train/gradients/dnn/hidden2/BiasAdd_grad/tuple/control_dependency_1&quot;\\n  op: &quot;Identity&quot;\\n  input: &quot;train/gradients/dnn/hidden2/BiasAdd_grad/BiasAddGrad&quot;\\n  input: &quot;^train/gradients/dnn/hidden2/BiasAdd_grad/tuple/group_deps&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;_class&quot;\\n    value {\\n      list {\\n        s: &quot;loc:@train/gradients/dnn/hidden2/BiasAdd_grad/BiasAddGrad&quot;\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;train/gradients/dnn/hidden2/MatMul_grad/MatMul&quot;\\n  op: &quot;MatMul&quot;\\n  input: &quot;train/gradients/dnn/hidden2/BiasAdd_grad/tuple/control_dependency&quot;\\n  input: &quot;hidden2/kernel/read&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;transpose_a&quot;\\n    value {\\n      b: false\\n    }\\n  }\\n  attr {\\n    key: &quot;transpose_b&quot;\\n    value {\\n      b: true\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;train/gradients/dnn/hidden2/MatMul_grad/MatMul_1&quot;\\n  op: &quot;MatMul&quot;\\n  input: &quot;dnn/hidden1/Relu&quot;\\n  input: &quot;train/gradients/dnn/hidden2/BiasAdd_grad/tuple/control_dependency&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;transpose_a&quot;\\n    value {\\n      b: true\\n    }\\n  }\\n  attr {\\n    key: &quot;transpose_b&quot;\\n    value {\\n      b: false\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;train/gradients/dnn/hidden2/MatMul_grad/tuple/group_deps&quot;\\n  op: &quot;NoOp&quot;\\n  input: &quot;^train/gradients/dnn/hidden2/MatMul_grad/MatMul&quot;\\n  input: &quot;^train/gradients/dnn/hidden2/MatMul_grad/MatMul_1&quot;\\n}\\nnode {\\n  name: &quot;train/gradients/dnn/hidden2/MatMul_grad/tuple/control_dependency&quot;\\n  op: &quot;Identity&quot;\\n  input: &quot;train/gradients/dnn/hidden2/MatMul_grad/MatMul&quot;\\n  input: &quot;^train/gradients/dnn/hidden2/MatMul_grad/tuple/group_deps&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;_class&quot;\\n    value {\\n      list {\\n        s: &quot;loc:@train/gradients/dnn/hidden2/MatMul_grad/MatMul&quot;\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;train/gradients/dnn/hidden2/MatMul_grad/tuple/control_dependency_1&quot;\\n  op: &quot;Identity&quot;\\n  input: &quot;train/gradients/dnn/hidden2/MatMul_grad/MatMul_1&quot;\\n  input: &quot;^train/gradients/dnn/hidden2/MatMul_grad/tuple/group_deps&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;_class&quot;\\n    value {\\n      list {\\n        s: &quot;loc:@train/gradients/dnn/hidden2/MatMul_grad/MatMul_1&quot;\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;train/gradients/dnn/hidden1/Relu_grad/ReluGrad&quot;\\n  op: &quot;ReluGrad&quot;\\n  input: &quot;train/gradients/dnn/hidden2/MatMul_grad/tuple/control_dependency&quot;\\n  input: &quot;dnn/hidden1/Relu&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;train/gradients/dnn/hidden1/BiasAdd_grad/BiasAddGrad&quot;\\n  op: &quot;BiasAddGrad&quot;\\n  input: &quot;train/gradients/dnn/hidden1/Relu_grad/ReluGrad&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;data_format&quot;\\n    value {\\n      s: &quot;NHWC&quot;\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;train/gradients/dnn/hidden1/BiasAdd_grad/tuple/group_deps&quot;\\n  op: &quot;NoOp&quot;\\n  input: &quot;^train/gradients/dnn/hidden1/Relu_grad/ReluGrad&quot;\\n  input: &quot;^train/gradients/dnn/hidden1/BiasAdd_grad/BiasAddGrad&quot;\\n}\\nnode {\\n  name: &quot;train/gradients/dnn/hidden1/BiasAdd_grad/tuple/control_dependency&quot;\\n  op: &quot;Identity&quot;\\n  input: &quot;train/gradients/dnn/hidden1/Relu_grad/ReluGrad&quot;\\n  input: &quot;^train/gradients/dnn/hidden1/BiasAdd_grad/tuple/group_deps&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;_class&quot;\\n    value {\\n      list {\\n        s: &quot;loc:@train/gradients/dnn/hidden1/Relu_grad/ReluGrad&quot;\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;train/gradients/dnn/hidden1/BiasAdd_grad/tuple/control_dependency_1&quot;\\n  op: &quot;Identity&quot;\\n  input: &quot;train/gradients/dnn/hidden1/BiasAdd_grad/BiasAddGrad&quot;\\n  input: &quot;^train/gradients/dnn/hidden1/BiasAdd_grad/tuple/group_deps&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;_class&quot;\\n    value {\\n      list {\\n        s: &quot;loc:@train/gradients/dnn/hidden1/BiasAdd_grad/BiasAddGrad&quot;\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;train/gradients/dnn/hidden1/MatMul_grad/MatMul&quot;\\n  op: &quot;MatMul&quot;\\n  input: &quot;train/gradients/dnn/hidden1/BiasAdd_grad/tuple/control_dependency&quot;\\n  input: &quot;hidden1/kernel/read&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;transpose_a&quot;\\n    value {\\n      b: false\\n    }\\n  }\\n  attr {\\n    key: &quot;transpose_b&quot;\\n    value {\\n      b: true\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;train/gradients/dnn/hidden1/MatMul_grad/MatMul_1&quot;\\n  op: &quot;MatMul&quot;\\n  input: &quot;X&quot;\\n  input: &quot;train/gradients/dnn/hidden1/BiasAdd_grad/tuple/control_dependency&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;transpose_a&quot;\\n    value {\\n      b: true\\n    }\\n  }\\n  attr {\\n    key: &quot;transpose_b&quot;\\n    value {\\n      b: false\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;train/gradients/dnn/hidden1/MatMul_grad/tuple/group_deps&quot;\\n  op: &quot;NoOp&quot;\\n  input: &quot;^train/gradients/dnn/hidden1/MatMul_grad/MatMul&quot;\\n  input: &quot;^train/gradients/dnn/hidden1/MatMul_grad/MatMul_1&quot;\\n}\\nnode {\\n  name: &quot;train/gradients/dnn/hidden1/MatMul_grad/tuple/control_dependency&quot;\\n  op: &quot;Identity&quot;\\n  input: &quot;train/gradients/dnn/hidden1/MatMul_grad/MatMul&quot;\\n  input: &quot;^train/gradients/dnn/hidden1/MatMul_grad/tuple/group_deps&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;_class&quot;\\n    value {\\n      list {\\n        s: &quot;loc:@train/gradients/dnn/hidden1/MatMul_grad/MatMul&quot;\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;train/gradients/dnn/hidden1/MatMul_grad/tuple/control_dependency_1&quot;\\n  op: &quot;Identity&quot;\\n  input: &quot;train/gradients/dnn/hidden1/MatMul_grad/MatMul_1&quot;\\n  input: &quot;^train/gradients/dnn/hidden1/MatMul_grad/tuple/group_deps&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;_class&quot;\\n    value {\\n      list {\\n        s: &quot;loc:@train/gradients/dnn/hidden1/MatMul_grad/MatMul_1&quot;\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;train/clip_by_value/Minimum/y&quot;\\n  op: &quot;Const&quot;\\n  attr {\\n    key: &quot;dtype&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;value&quot;\\n    value {\\n      tensor {\\n        dtype: DT_FLOAT\\n        tensor_shape {\\n        }\\n        float_val: 1.0\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;train/clip_by_value/Minimum&quot;\\n  op: &quot;Minimum&quot;\\n  input: &quot;train/gradients/dnn/hidden1/MatMul_grad/tuple/control_dependency_1&quot;\\n  input: &quot;train/clip_by_value/Minimum/y&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;train/clip_by_value/y&quot;\\n  op: &quot;Const&quot;\\n  attr {\\n    key: &quot;dtype&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;value&quot;\\n    value {\\n      tensor {\\n        dtype: DT_FLOAT\\n        tensor_shape {\\n        }\\n        float_val: -1.0\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;train/clip_by_value&quot;\\n  op: &quot;Maximum&quot;\\n  input: &quot;train/clip_by_value/Minimum&quot;\\n  input: &quot;train/clip_by_value/y&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;train/clip_by_value_1/Minimum/y&quot;\\n  op: &quot;Const&quot;\\n  attr {\\n    key: &quot;dtype&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;value&quot;\\n    value {\\n      tensor {\\n        dtype: DT_FLOAT\\n        tensor_shape {\\n        }\\n        float_val: 1.0\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;train/clip_by_value_1/Minimum&quot;\\n  op: &quot;Minimum&quot;\\n  input: &quot;train/gradients/dnn/hidden1/BiasAdd_grad/tuple/control_dependency_1&quot;\\n  input: &quot;train/clip_by_value_1/Minimum/y&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;train/clip_by_value_1/y&quot;\\n  op: &quot;Const&quot;\\n  attr {\\n    key: &quot;dtype&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;value&quot;\\n    value {\\n      tensor {\\n        dtype: DT_FLOAT\\n        tensor_shape {\\n        }\\n        float_val: -1.0\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;train/clip_by_value_1&quot;\\n  op: &quot;Maximum&quot;\\n  input: &quot;train/clip_by_value_1/Minimum&quot;\\n  input: &quot;train/clip_by_value_1/y&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;train/clip_by_value_2/Minimum/y&quot;\\n  op: &quot;Const&quot;\\n  attr {\\n    key: &quot;dtype&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;value&quot;\\n    value {\\n      tensor {\\n        dtype: DT_FLOAT\\n        tensor_shape {\\n        }\\n        float_val: 1.0\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;train/clip_by_value_2/Minimum&quot;\\n  op: &quot;Minimum&quot;\\n  input: &quot;train/gradients/dnn/hidden2/MatMul_grad/tuple/control_dependency_1&quot;\\n  input: &quot;train/clip_by_value_2/Minimum/y&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;train/clip_by_value_2/y&quot;\\n  op: &quot;Const&quot;\\n  attr {\\n    key: &quot;dtype&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;value&quot;\\n    value {\\n      tensor {\\n        dtype: DT_FLOAT\\n        tensor_shape {\\n        }\\n        float_val: -1.0\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;train/clip_by_value_2&quot;\\n  op: &quot;Maximum&quot;\\n  input: &quot;train/clip_by_value_2/Minimum&quot;\\n  input: &quot;train/clip_by_value_2/y&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;train/clip_by_value_3/Minimum/y&quot;\\n  op: &quot;Const&quot;\\n  attr {\\n    key: &quot;dtype&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;value&quot;\\n    value {\\n      tensor {\\n        dtype: DT_FLOAT\\n        tensor_shape {\\n        }\\n        float_val: 1.0\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;train/clip_by_value_3/Minimum&quot;\\n  op: &quot;Minimum&quot;\\n  input: &quot;train/gradients/dnn/hidden2/BiasAdd_grad/tuple/control_dependency_1&quot;\\n  input: &quot;train/clip_by_value_3/Minimum/y&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;train/clip_by_value_3/y&quot;\\n  op: &quot;Const&quot;\\n  attr {\\n    key: &quot;dtype&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;value&quot;\\n    value {\\n      tensor {\\n        dtype: DT_FLOAT\\n        tensor_shape {\\n        }\\n        float_val: -1.0\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;train/clip_by_value_3&quot;\\n  op: &quot;Maximum&quot;\\n  input: &quot;train/clip_by_value_3/Minimum&quot;\\n  input: &quot;train/clip_by_value_3/y&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;train/clip_by_value_4/Minimum/y&quot;\\n  op: &quot;Const&quot;\\n  attr {\\n    key: &quot;dtype&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;value&quot;\\n    value {\\n      tensor {\\n        dtype: DT_FLOAT\\n        tensor_shape {\\n        }\\n        float_val: 1.0\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;train/clip_by_value_4/Minimum&quot;\\n  op: &quot;Minimum&quot;\\n  input: &quot;train/gradients/dnn/outputs/MatMul_grad/tuple/control_dependency_1&quot;\\n  input: &quot;train/clip_by_value_4/Minimum/y&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;train/clip_by_value_4/y&quot;\\n  op: &quot;Const&quot;\\n  attr {\\n    key: &quot;dtype&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;value&quot;\\n    value {\\n      tensor {\\n        dtype: DT_FLOAT\\n        tensor_shape {\\n        }\\n        float_val: -1.0\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;train/clip_by_value_4&quot;\\n  op: &quot;Maximum&quot;\\n  input: &quot;train/clip_by_value_4/Minimum&quot;\\n  input: &quot;train/clip_by_value_4/y&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;train/clip_by_value_5/Minimum/y&quot;\\n  op: &quot;Const&quot;\\n  attr {\\n    key: &quot;dtype&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;value&quot;\\n    value {\\n      tensor {\\n        dtype: DT_FLOAT\\n        tensor_shape {\\n        }\\n        float_val: 1.0\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;train/clip_by_value_5/Minimum&quot;\\n  op: &quot;Minimum&quot;\\n  input: &quot;train/gradients/dnn/outputs/BiasAdd_grad/tuple/control_dependency_1&quot;\\n  input: &quot;train/clip_by_value_5/Minimum/y&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;train/clip_by_value_5/y&quot;\\n  op: &quot;Const&quot;\\n  attr {\\n    key: &quot;dtype&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;value&quot;\\n    value {\\n      tensor {\\n        dtype: DT_FLOAT\\n        tensor_shape {\\n        }\\n        float_val: -1.0\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;train/clip_by_value_5&quot;\\n  op: &quot;Maximum&quot;\\n  input: &quot;train/clip_by_value_5/Minimum&quot;\\n  input: &quot;train/clip_by_value_5/y&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;train/zeros&quot;\\n  op: &quot;Const&quot;\\n  attr {\\n    key: &quot;dtype&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;value&quot;\\n    value {\\n      tensor {\\n        dtype: DT_FLOAT\\n        tensor_shape {\\n          dim {\\n            size: 784\\n          }\\n          dim {\\n            size: 300\\n          }\\n        }\\n        float_val: 0.0\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;hidden1/kernel/Momentum&quot;\\n  op: &quot;VariableV2&quot;\\n  attr {\\n    key: &quot;_class&quot;\\n    value {\\n      list {\\n        s: &quot;loc:@hidden1/kernel&quot;\\n      }\\n    }\\n  }\\n  attr {\\n    key: &quot;container&quot;\\n    value {\\n      s: &quot;&quot;\\n    }\\n  }\\n  attr {\\n    key: &quot;dtype&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;shape&quot;\\n    value {\\n      shape {\\n        dim {\\n          size: 784\\n        }\\n        dim {\\n          size: 300\\n        }\\n      }\\n    }\\n  }\\n  attr {\\n    key: &quot;shared_name&quot;\\n    value {\\n      s: &quot;&quot;\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;hidden1/kernel/Momentum/Assign&quot;\\n  op: &quot;Assign&quot;\\n  input: &quot;hidden1/kernel/Momentum&quot;\\n  input: &quot;train/zeros&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;_class&quot;\\n    value {\\n      list {\\n        s: &quot;loc:@hidden1/kernel&quot;\\n      }\\n    }\\n  }\\n  attr {\\n    key: &quot;use_locking&quot;\\n    value {\\n      b: true\\n    }\\n  }\\n  attr {\\n    key: &quot;validate_shape&quot;\\n    value {\\n      b: true\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;hidden1/kernel/Momentum/read&quot;\\n  op: &quot;Identity&quot;\\n  input: &quot;hidden1/kernel/Momentum&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;_class&quot;\\n    value {\\n      list {\\n        s: &quot;loc:@hidden1/kernel&quot;\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;train/zeros_1&quot;\\n  op: &quot;Const&quot;\\n  attr {\\n    key: &quot;dtype&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;value&quot;\\n    value {\\n      tensor {\\n        dtype: DT_FLOAT\\n        tensor_shape {\\n          dim {\\n            size: 300\\n          }\\n        }\\n        float_val: 0.0\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;hidden1/bias/Momentum&quot;\\n  op: &quot;VariableV2&quot;\\n  attr {\\n    key: &quot;_class&quot;\\n    value {\\n      list {\\n        s: &quot;loc:@hidden1/bias&quot;\\n      }\\n    }\\n  }\\n  attr {\\n    key: &quot;container&quot;\\n    value {\\n      s: &quot;&quot;\\n    }\\n  }\\n  attr {\\n    key: &quot;dtype&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;shape&quot;\\n    value {\\n      shape {\\n        dim {\\n          size: 300\\n        }\\n      }\\n    }\\n  }\\n  attr {\\n    key: &quot;shared_name&quot;\\n    value {\\n      s: &quot;&quot;\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;hidden1/bias/Momentum/Assign&quot;\\n  op: &quot;Assign&quot;\\n  input: &quot;hidden1/bias/Momentum&quot;\\n  input: &quot;train/zeros_1&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;_class&quot;\\n    value {\\n      list {\\n        s: &quot;loc:@hidden1/bias&quot;\\n      }\\n    }\\n  }\\n  attr {\\n    key: &quot;use_locking&quot;\\n    value {\\n      b: true\\n    }\\n  }\\n  attr {\\n    key: &quot;validate_shape&quot;\\n    value {\\n      b: true\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;hidden1/bias/Momentum/read&quot;\\n  op: &quot;Identity&quot;\\n  input: &quot;hidden1/bias/Momentum&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;_class&quot;\\n    value {\\n      list {\\n        s: &quot;loc:@hidden1/bias&quot;\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;train/zeros_2&quot;\\n  op: &quot;Const&quot;\\n  attr {\\n    key: &quot;dtype&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;value&quot;\\n    value {\\n      tensor {\\n        dtype: DT_FLOAT\\n        tensor_shape {\\n          dim {\\n            size: 300\\n          }\\n          dim {\\n            size: 100\\n          }\\n        }\\n        float_val: 0.0\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;hidden2/kernel/Momentum&quot;\\n  op: &quot;VariableV2&quot;\\n  attr {\\n    key: &quot;_class&quot;\\n    value {\\n      list {\\n        s: &quot;loc:@hidden2/kernel&quot;\\n      }\\n    }\\n  }\\n  attr {\\n    key: &quot;container&quot;\\n    value {\\n      s: &quot;&quot;\\n    }\\n  }\\n  attr {\\n    key: &quot;dtype&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;shape&quot;\\n    value {\\n      shape {\\n        dim {\\n          size: 300\\n        }\\n        dim {\\n          size: 100\\n        }\\n      }\\n    }\\n  }\\n  attr {\\n    key: &quot;shared_name&quot;\\n    value {\\n      s: &quot;&quot;\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;hidden2/kernel/Momentum/Assign&quot;\\n  op: &quot;Assign&quot;\\n  input: &quot;hidden2/kernel/Momentum&quot;\\n  input: &quot;train/zeros_2&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;_class&quot;\\n    value {\\n      list {\\n        s: &quot;loc:@hidden2/kernel&quot;\\n      }\\n    }\\n  }\\n  attr {\\n    key: &quot;use_locking&quot;\\n    value {\\n      b: true\\n    }\\n  }\\n  attr {\\n    key: &quot;validate_shape&quot;\\n    value {\\n      b: true\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;hidden2/kernel/Momentum/read&quot;\\n  op: &quot;Identity&quot;\\n  input: &quot;hidden2/kernel/Momentum&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;_class&quot;\\n    value {\\n      list {\\n        s: &quot;loc:@hidden2/kernel&quot;\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;train/zeros_3&quot;\\n  op: &quot;Const&quot;\\n  attr {\\n    key: &quot;dtype&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;value&quot;\\n    value {\\n      tensor {\\n        dtype: DT_FLOAT\\n        tensor_shape {\\n          dim {\\n            size: 100\\n          }\\n        }\\n        float_val: 0.0\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;hidden2/bias/Momentum&quot;\\n  op: &quot;VariableV2&quot;\\n  attr {\\n    key: &quot;_class&quot;\\n    value {\\n      list {\\n        s: &quot;loc:@hidden2/bias&quot;\\n      }\\n    }\\n  }\\n  attr {\\n    key: &quot;container&quot;\\n    value {\\n      s: &quot;&quot;\\n    }\\n  }\\n  attr {\\n    key: &quot;dtype&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;shape&quot;\\n    value {\\n      shape {\\n        dim {\\n          size: 100\\n        }\\n      }\\n    }\\n  }\\n  attr {\\n    key: &quot;shared_name&quot;\\n    value {\\n      s: &quot;&quot;\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;hidden2/bias/Momentum/Assign&quot;\\n  op: &quot;Assign&quot;\\n  input: &quot;hidden2/bias/Momentum&quot;\\n  input: &quot;train/zeros_3&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;_class&quot;\\n    value {\\n      list {\\n        s: &quot;loc:@hidden2/bias&quot;\\n      }\\n    }\\n  }\\n  attr {\\n    key: &quot;use_locking&quot;\\n    value {\\n      b: true\\n    }\\n  }\\n  attr {\\n    key: &quot;validate_shape&quot;\\n    value {\\n      b: true\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;hidden2/bias/Momentum/read&quot;\\n  op: &quot;Identity&quot;\\n  input: &quot;hidden2/bias/Momentum&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;_class&quot;\\n    value {\\n      list {\\n        s: &quot;loc:@hidden2/bias&quot;\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;train/zeros_4&quot;\\n  op: &quot;Const&quot;\\n  attr {\\n    key: &quot;dtype&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;value&quot;\\n    value {\\n      tensor {\\n        dtype: DT_FLOAT\\n        tensor_shape {\\n          dim {\\n            size: 100\\n          }\\n          dim {\\n            size: 10\\n          }\\n        }\\n        float_val: 0.0\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;outputs/kernel/Momentum&quot;\\n  op: &quot;VariableV2&quot;\\n  attr {\\n    key: &quot;_class&quot;\\n    value {\\n      list {\\n        s: &quot;loc:@outputs/kernel&quot;\\n      }\\n    }\\n  }\\n  attr {\\n    key: &quot;container&quot;\\n    value {\\n      s: &quot;&quot;\\n    }\\n  }\\n  attr {\\n    key: &quot;dtype&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;shape&quot;\\n    value {\\n      shape {\\n        dim {\\n          size: 100\\n        }\\n        dim {\\n          size: 10\\n        }\\n      }\\n    }\\n  }\\n  attr {\\n    key: &quot;shared_name&quot;\\n    value {\\n      s: &quot;&quot;\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;outputs/kernel/Momentum/Assign&quot;\\n  op: &quot;Assign&quot;\\n  input: &quot;outputs/kernel/Momentum&quot;\\n  input: &quot;train/zeros_4&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;_class&quot;\\n    value {\\n      list {\\n        s: &quot;loc:@outputs/kernel&quot;\\n      }\\n    }\\n  }\\n  attr {\\n    key: &quot;use_locking&quot;\\n    value {\\n      b: true\\n    }\\n  }\\n  attr {\\n    key: &quot;validate_shape&quot;\\n    value {\\n      b: true\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;outputs/kernel/Momentum/read&quot;\\n  op: &quot;Identity&quot;\\n  input: &quot;outputs/kernel/Momentum&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;_class&quot;\\n    value {\\n      list {\\n        s: &quot;loc:@outputs/kernel&quot;\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;train/zeros_5&quot;\\n  op: &quot;Const&quot;\\n  attr {\\n    key: &quot;dtype&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;value&quot;\\n    value {\\n      tensor {\\n        dtype: DT_FLOAT\\n        tensor_shape {\\n          dim {\\n            size: 10\\n          }\\n        }\\n        float_val: 0.0\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;outputs/bias/Momentum&quot;\\n  op: &quot;VariableV2&quot;\\n  attr {\\n    key: &quot;_class&quot;\\n    value {\\n      list {\\n        s: &quot;loc:@outputs/bias&quot;\\n      }\\n    }\\n  }\\n  attr {\\n    key: &quot;container&quot;\\n    value {\\n      s: &quot;&quot;\\n    }\\n  }\\n  attr {\\n    key: &quot;dtype&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;shape&quot;\\n    value {\\n      shape {\\n        dim {\\n          size: 10\\n        }\\n      }\\n    }\\n  }\\n  attr {\\n    key: &quot;shared_name&quot;\\n    value {\\n      s: &quot;&quot;\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;outputs/bias/Momentum/Assign&quot;\\n  op: &quot;Assign&quot;\\n  input: &quot;outputs/bias/Momentum&quot;\\n  input: &quot;train/zeros_5&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;_class&quot;\\n    value {\\n      list {\\n        s: &quot;loc:@outputs/bias&quot;\\n      }\\n    }\\n  }\\n  attr {\\n    key: &quot;use_locking&quot;\\n    value {\\n      b: true\\n    }\\n  }\\n  attr {\\n    key: &quot;validate_shape&quot;\\n    value {\\n      b: true\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;outputs/bias/Momentum/read&quot;\\n  op: &quot;Identity&quot;\\n  input: &quot;outputs/bias/Momentum&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;_class&quot;\\n    value {\\n      list {\\n        s: &quot;loc:@outputs/bias&quot;\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;train/Momentum/learning_rate&quot;\\n  op: &quot;Const&quot;\\n  attr {\\n    key: &quot;dtype&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;value&quot;\\n    value {\\n      tensor {\\n        dtype: DT_FLOAT\\n        tensor_shape {\\n        }\\n        float_val: 0.009999999776482582\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;train/Momentum/momentum&quot;\\n  op: &quot;Const&quot;\\n  attr {\\n    key: &quot;dtype&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;value&quot;\\n    value {\\n      tensor {\\n        dtype: DT_FLOAT\\n        tensor_shape {\\n        }\\n        float_val: 0.25\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;train/Momentum/update_hidden1/kernel/ApplyMomentum&quot;\\n  op: &quot;ApplyMomentum&quot;\\n  input: &quot;hidden1/kernel&quot;\\n  input: &quot;hidden1/kernel/Momentum&quot;\\n  input: &quot;train/Momentum/learning_rate&quot;\\n  input: &quot;train/clip_by_value&quot;\\n  input: &quot;train/Momentum/momentum&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;_class&quot;\\n    value {\\n      list {\\n        s: &quot;loc:@hidden1/kernel&quot;\\n      }\\n    }\\n  }\\n  attr {\\n    key: &quot;use_locking&quot;\\n    value {\\n      b: false\\n    }\\n  }\\n  attr {\\n    key: &quot;use_nesterov&quot;\\n    value {\\n      b: false\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;train/Momentum/update_hidden1/bias/ApplyMomentum&quot;\\n  op: &quot;ApplyMomentum&quot;\\n  input: &quot;hidden1/bias&quot;\\n  input: &quot;hidden1/bias/Momentum&quot;\\n  input: &quot;train/Momentum/learning_rate&quot;\\n  input: &quot;train/clip_by_value_1&quot;\\n  input: &quot;train/Momentum/momentum&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;_class&quot;\\n    value {\\n      list {\\n        s: &quot;loc:@hidden1/bias&quot;\\n      }\\n    }\\n  }\\n  attr {\\n    key: &quot;use_locking&quot;\\n    value {\\n      b: false\\n    }\\n  }\\n  attr {\\n    key: &quot;use_nesterov&quot;\\n    value {\\n      b: false\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;train/Momentum/update_hidden2/kernel/ApplyMomentum&quot;\\n  op: &quot;ApplyMomentum&quot;\\n  input: &quot;hidden2/kernel&quot;\\n  input: &quot;hidden2/kernel/Momentum&quot;\\n  input: &quot;train/Momentum/learning_rate&quot;\\n  input: &quot;train/clip_by_value_2&quot;\\n  input: &quot;train/Momentum/momentum&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;_class&quot;\\n    value {\\n      list {\\n        s: &quot;loc:@hidden2/kernel&quot;\\n      }\\n    }\\n  }\\n  attr {\\n    key: &quot;use_locking&quot;\\n    value {\\n      b: false\\n    }\\n  }\\n  attr {\\n    key: &quot;use_nesterov&quot;\\n    value {\\n      b: false\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;train/Momentum/update_hidden2/bias/ApplyMomentum&quot;\\n  op: &quot;ApplyMomentum&quot;\\n  input: &quot;hidden2/bias&quot;\\n  input: &quot;hidden2/bias/Momentum&quot;\\n  input: &quot;train/Momentum/learning_rate&quot;\\n  input: &quot;train/clip_by_value_3&quot;\\n  input: &quot;train/Momentum/momentum&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;_class&quot;\\n    value {\\n      list {\\n        s: &quot;loc:@hidden2/bias&quot;\\n      }\\n    }\\n  }\\n  attr {\\n    key: &quot;use_locking&quot;\\n    value {\\n      b: false\\n    }\\n  }\\n  attr {\\n    key: &quot;use_nesterov&quot;\\n    value {\\n      b: false\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;train/Momentum/update_outputs/kernel/ApplyMomentum&quot;\\n  op: &quot;ApplyMomentum&quot;\\n  input: &quot;outputs/kernel&quot;\\n  input: &quot;outputs/kernel/Momentum&quot;\\n  input: &quot;train/Momentum/learning_rate&quot;\\n  input: &quot;train/clip_by_value_4&quot;\\n  input: &quot;train/Momentum/momentum&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;_class&quot;\\n    value {\\n      list {\\n        s: &quot;loc:@outputs/kernel&quot;\\n      }\\n    }\\n  }\\n  attr {\\n    key: &quot;use_locking&quot;\\n    value {\\n      b: false\\n    }\\n  }\\n  attr {\\n    key: &quot;use_nesterov&quot;\\n    value {\\n      b: false\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;train/Momentum/update_outputs/bias/ApplyMomentum&quot;\\n  op: &quot;ApplyMomentum&quot;\\n  input: &quot;outputs/bias&quot;\\n  input: &quot;outputs/bias/Momentum&quot;\\n  input: &quot;train/Momentum/learning_rate&quot;\\n  input: &quot;train/clip_by_value_5&quot;\\n  input: &quot;train/Momentum/momentum&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;_class&quot;\\n    value {\\n      list {\\n        s: &quot;loc:@outputs/bias&quot;\\n      }\\n    }\\n  }\\n  attr {\\n    key: &quot;use_locking&quot;\\n    value {\\n      b: false\\n    }\\n  }\\n  attr {\\n    key: &quot;use_nesterov&quot;\\n    value {\\n      b: false\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;train/Momentum&quot;\\n  op: &quot;NoOp&quot;\\n  input: &quot;^train/Momentum/update_hidden1/kernel/ApplyMomentum&quot;\\n  input: &quot;^train/Momentum/update_hidden1/bias/ApplyMomentum&quot;\\n  input: &quot;^train/Momentum/update_hidden2/kernel/ApplyMomentum&quot;\\n  input: &quot;^train/Momentum/update_hidden2/bias/ApplyMomentum&quot;\\n  input: &quot;^train/Momentum/update_outputs/kernel/ApplyMomentum&quot;\\n  input: &quot;^train/Momentum/update_outputs/bias/ApplyMomentum&quot;\\n}\\nnode {\\n  name: &quot;eval/InTopK&quot;\\n  op: &quot;InTopK&quot;\\n  input: &quot;dnn/outputs/BiasAdd&quot;\\n  input: &quot;y&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_INT64\\n    }\\n  }\\n  attr {\\n    key: &quot;k&quot;\\n    value {\\n      i: 1\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;eval/Cast&quot;\\n  op: &quot;Cast&quot;\\n  input: &quot;eval/InTopK&quot;\\n  attr {\\n    key: &quot;DstT&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;SrcT&quot;\\n    value {\\n      type: DT_BOOL\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;eval/Const&quot;\\n  op: &quot;Const&quot;\\n  attr {\\n    key: &quot;dtype&quot;\\n    value {\\n      type: DT_INT32\\n    }\\n  }\\n  attr {\\n    key: &quot;value&quot;\\n    value {\\n      tensor {\\n        dtype: DT_INT32\\n        tensor_shape {\\n          dim {\\n            size: 1\\n          }\\n        }\\n        int_val: 0\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;eval/Mean&quot;\\n  op: &quot;Mean&quot;\\n  input: &quot;eval/Cast&quot;\\n  input: &quot;eval/Const&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;Tidx&quot;\\n    value {\\n      type: DT_INT32\\n    }\\n  }\\n  attr {\\n    key: &quot;keep_dims&quot;\\n    value {\\n      b: false\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;init&quot;\\n  op: &quot;NoOp&quot;\\n  input: &quot;^hidden1/kernel/Assign&quot;\\n  input: &quot;^hidden1/bias/Assign&quot;\\n  input: &quot;^hidden2/kernel/Assign&quot;\\n  input: &quot;^hidden2/bias/Assign&quot;\\n  input: &quot;^outputs/kernel/Assign&quot;\\n  input: &quot;^outputs/bias/Assign&quot;\\n  input: &quot;^hidden1/kernel/Momentum/Assign&quot;\\n  input: &quot;^hidden1/bias/Momentum/Assign&quot;\\n  input: &quot;^hidden2/kernel/Momentum/Assign&quot;\\n  input: &quot;^hidden2/bias/Momentum/Assign&quot;\\n  input: &quot;^outputs/kernel/Momentum/Assign&quot;\\n  input: &quot;^outputs/bias/Momentum/Assign&quot;\\n}\\nnode {\\n  name: &quot;save/Const&quot;\\n  op: &quot;Const&quot;\\n  attr {\\n    key: &quot;dtype&quot;\\n    value {\\n      type: DT_STRING\\n    }\\n  }\\n  attr {\\n    key: &quot;value&quot;\\n    value {\\n      tensor {\\n        dtype: DT_STRING\\n        tensor_shape {\\n        }\\n        string_val: &quot;model&quot;\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;save/SaveV2/tensor_names&quot;\\n  op: &quot;Const&quot;\\n  attr {\\n    key: &quot;dtype&quot;\\n    value {\\n      type: DT_STRING\\n    }\\n  }\\n  attr {\\n    key: &quot;value&quot;\\n    value {\\n      tensor {\\n        dtype: DT_STRING\\n        tensor_shape {\\n          dim {\\n            size: 12\\n          }\\n        }\\n        string_val: &quot;hidden1/bias&quot;\\n        string_val: &quot;hidden1/bias/Momentum&quot;\\n        string_val: &quot;hidden1/kernel&quot;\\n        string_val: &quot;hidden1/kernel/Momentum&quot;\\n        string_val: &quot;hidden2/bias&quot;\\n        string_val: &quot;hidden2/bias/Momentum&quot;\\n        string_val: &quot;hidden2/kernel&quot;\\n        string_val: &quot;hidden2/kernel/Momentum&quot;\\n        string_val: &quot;outputs/bias&quot;\\n        string_val: &quot;outputs/bias/Momentum&quot;\\n        string_val: &quot;outputs/kernel&quot;\\n        string_val: &quot;outputs/kernel/Momentum&quot;\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;save/SaveV2/shape_and_slices&quot;\\n  op: &quot;Const&quot;\\n  attr {\\n    key: &quot;dtype&quot;\\n    value {\\n      type: DT_STRING\\n    }\\n  }\\n  attr {\\n    key: &quot;value&quot;\\n    value {\\n      tensor {\\n        dtype: DT_STRING\\n        tensor_shape {\\n          dim {\\n            size: 12\\n          }\\n        }\\n        string_val: &quot;&quot;\\n        string_val: &quot;&quot;\\n        string_val: &quot;&quot;\\n        string_val: &quot;&quot;\\n        string_val: &quot;&quot;\\n        string_val: &quot;&quot;\\n        string_val: &quot;&quot;\\n        string_val: &quot;&quot;\\n        string_val: &quot;&quot;\\n        string_val: &quot;&quot;\\n        string_val: &quot;&quot;\\n        string_val: &quot;&quot;\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;save/SaveV2&quot;\\n  op: &quot;SaveV2&quot;\\n  input: &quot;save/Const&quot;\\n  input: &quot;save/SaveV2/tensor_names&quot;\\n  input: &quot;save/SaveV2/shape_and_slices&quot;\\n  input: &quot;hidden1/bias&quot;\\n  input: &quot;hidden1/bias/Momentum&quot;\\n  input: &quot;hidden1/kernel&quot;\\n  input: &quot;hidden1/kernel/Momentum&quot;\\n  input: &quot;hidden2/bias&quot;\\n  input: &quot;hidden2/bias/Momentum&quot;\\n  input: &quot;hidden2/kernel&quot;\\n  input: &quot;hidden2/kernel/Momentum&quot;\\n  input: &quot;outputs/bias&quot;\\n  input: &quot;outputs/bias/Momentum&quot;\\n  input: &quot;outputs/kernel&quot;\\n  input: &quot;outputs/kernel/Momentum&quot;\\n  attr {\\n    key: &quot;dtypes&quot;\\n    value {\\n      list {\\n        type: DT_FLOAT\\n        type: DT_FLOAT\\n        type: DT_FLOAT\\n        type: DT_FLOAT\\n        type: DT_FLOAT\\n        type: DT_FLOAT\\n        type: DT_FLOAT\\n        type: DT_FLOAT\\n        type: DT_FLOAT\\n        type: DT_FLOAT\\n        type: DT_FLOAT\\n        type: DT_FLOAT\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;save/control_dependency&quot;\\n  op: &quot;Identity&quot;\\n  input: &quot;save/Const&quot;\\n  input: &quot;^save/SaveV2&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_STRING\\n    }\\n  }\\n  attr {\\n    key: &quot;_class&quot;\\n    value {\\n      list {\\n        s: &quot;loc:@save/Const&quot;\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;save/RestoreV2/tensor_names&quot;\\n  op: &quot;Const&quot;\\n  attr {\\n    key: &quot;dtype&quot;\\n    value {\\n      type: DT_STRING\\n    }\\n  }\\n  attr {\\n    key: &quot;value&quot;\\n    value {\\n      tensor {\\n        dtype: DT_STRING\\n        tensor_shape {\\n          dim {\\n            size: 1\\n          }\\n        }\\n        string_val: &quot;hidden1/bias&quot;\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;save/RestoreV2/shape_and_slices&quot;\\n  op: &quot;Const&quot;\\n  attr {\\n    key: &quot;dtype&quot;\\n    value {\\n      type: DT_STRING\\n    }\\n  }\\n  attr {\\n    key: &quot;value&quot;\\n    value {\\n      tensor {\\n        dtype: DT_STRING\\n        tensor_shape {\\n          dim {\\n            size: 1\\n          }\\n        }\\n        string_val: &quot;&quot;\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;save/RestoreV2&quot;\\n  op: &quot;RestoreV2&quot;\\n  input: &quot;save/Const&quot;\\n  input: &quot;save/RestoreV2/tensor_names&quot;\\n  input: &quot;save/RestoreV2/shape_and_slices&quot;\\n  attr {\\n    key: &quot;dtypes&quot;\\n    value {\\n      list {\\n        type: DT_FLOAT\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;save/Assign&quot;\\n  op: &quot;Assign&quot;\\n  input: &quot;hidden1/bias&quot;\\n  input: &quot;save/RestoreV2&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;_class&quot;\\n    value {\\n      list {\\n        s: &quot;loc:@hidden1/bias&quot;\\n      }\\n    }\\n  }\\n  attr {\\n    key: &quot;use_locking&quot;\\n    value {\\n      b: true\\n    }\\n  }\\n  attr {\\n    key: &quot;validate_shape&quot;\\n    value {\\n      b: true\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;save/RestoreV2_1/tensor_names&quot;\\n  op: &quot;Const&quot;\\n  attr {\\n    key: &quot;dtype&quot;\\n    value {\\n      type: DT_STRING\\n    }\\n  }\\n  attr {\\n    key: &quot;value&quot;\\n    value {\\n      tensor {\\n        dtype: DT_STRING\\n        tensor_shape {\\n          dim {\\n            size: 1\\n          }\\n        }\\n        string_val: &quot;hidden1/bias/Momentum&quot;\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;save/RestoreV2_1/shape_and_slices&quot;\\n  op: &quot;Const&quot;\\n  attr {\\n    key: &quot;dtype&quot;\\n    value {\\n      type: DT_STRING\\n    }\\n  }\\n  attr {\\n    key: &quot;value&quot;\\n    value {\\n      tensor {\\n        dtype: DT_STRING\\n        tensor_shape {\\n          dim {\\n            size: 1\\n          }\\n        }\\n        string_val: &quot;&quot;\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;save/RestoreV2_1&quot;\\n  op: &quot;RestoreV2&quot;\\n  input: &quot;save/Const&quot;\\n  input: &quot;save/RestoreV2_1/tensor_names&quot;\\n  input: &quot;save/RestoreV2_1/shape_and_slices&quot;\\n  attr {\\n    key: &quot;dtypes&quot;\\n    value {\\n      list {\\n        type: DT_FLOAT\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;save/Assign_1&quot;\\n  op: &quot;Assign&quot;\\n  input: &quot;hidden1/bias/Momentum&quot;\\n  input: &quot;save/RestoreV2_1&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;_class&quot;\\n    value {\\n      list {\\n        s: &quot;loc:@hidden1/bias&quot;\\n      }\\n    }\\n  }\\n  attr {\\n    key: &quot;use_locking&quot;\\n    value {\\n      b: true\\n    }\\n  }\\n  attr {\\n    key: &quot;validate_shape&quot;\\n    value {\\n      b: true\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;save/RestoreV2_2/tensor_names&quot;\\n  op: &quot;Const&quot;\\n  attr {\\n    key: &quot;dtype&quot;\\n    value {\\n      type: DT_STRING\\n    }\\n  }\\n  attr {\\n    key: &quot;value&quot;\\n    value {\\n      tensor {\\n        dtype: DT_STRING\\n        tensor_shape {\\n          dim {\\n            size: 1\\n          }\\n        }\\n        string_val: &quot;hidden1/kernel&quot;\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;save/RestoreV2_2/shape_and_slices&quot;\\n  op: &quot;Const&quot;\\n  attr {\\n    key: &quot;dtype&quot;\\n    value {\\n      type: DT_STRING\\n    }\\n  }\\n  attr {\\n    key: &quot;value&quot;\\n    value {\\n      tensor {\\n        dtype: DT_STRING\\n        tensor_shape {\\n          dim {\\n            size: 1\\n          }\\n        }\\n        string_val: &quot;&quot;\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;save/RestoreV2_2&quot;\\n  op: &quot;RestoreV2&quot;\\n  input: &quot;save/Const&quot;\\n  input: &quot;save/RestoreV2_2/tensor_names&quot;\\n  input: &quot;save/RestoreV2_2/shape_and_slices&quot;\\n  attr {\\n    key: &quot;dtypes&quot;\\n    value {\\n      list {\\n        type: DT_FLOAT\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;save/Assign_2&quot;\\n  op: &quot;Assign&quot;\\n  input: &quot;hidden1/kernel&quot;\\n  input: &quot;save/RestoreV2_2&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;_class&quot;\\n    value {\\n      list {\\n        s: &quot;loc:@hidden1/kernel&quot;\\n      }\\n    }\\n  }\\n  attr {\\n    key: &quot;use_locking&quot;\\n    value {\\n      b: true\\n    }\\n  }\\n  attr {\\n    key: &quot;validate_shape&quot;\\n    value {\\n      b: true\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;save/RestoreV2_3/tensor_names&quot;\\n  op: &quot;Const&quot;\\n  attr {\\n    key: &quot;dtype&quot;\\n    value {\\n      type: DT_STRING\\n    }\\n  }\\n  attr {\\n    key: &quot;value&quot;\\n    value {\\n      tensor {\\n        dtype: DT_STRING\\n        tensor_shape {\\n          dim {\\n            size: 1\\n          }\\n        }\\n        string_val: &quot;hidden1/kernel/Momentum&quot;\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;save/RestoreV2_3/shape_and_slices&quot;\\n  op: &quot;Const&quot;\\n  attr {\\n    key: &quot;dtype&quot;\\n    value {\\n      type: DT_STRING\\n    }\\n  }\\n  attr {\\n    key: &quot;value&quot;\\n    value {\\n      tensor {\\n        dtype: DT_STRING\\n        tensor_shape {\\n          dim {\\n            size: 1\\n          }\\n        }\\n        string_val: &quot;&quot;\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;save/RestoreV2_3&quot;\\n  op: &quot;RestoreV2&quot;\\n  input: &quot;save/Const&quot;\\n  input: &quot;save/RestoreV2_3/tensor_names&quot;\\n  input: &quot;save/RestoreV2_3/shape_and_slices&quot;\\n  attr {\\n    key: &quot;dtypes&quot;\\n    value {\\n      list {\\n        type: DT_FLOAT\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;save/Assign_3&quot;\\n  op: &quot;Assign&quot;\\n  input: &quot;hidden1/kernel/Momentum&quot;\\n  input: &quot;save/RestoreV2_3&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;_class&quot;\\n    value {\\n      list {\\n        s: &quot;loc:@hidden1/kernel&quot;\\n      }\\n    }\\n  }\\n  attr {\\n    key: &quot;use_locking&quot;\\n    value {\\n      b: true\\n    }\\n  }\\n  attr {\\n    key: &quot;validate_shape&quot;\\n    value {\\n      b: true\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;save/RestoreV2_4/tensor_names&quot;\\n  op: &quot;Const&quot;\\n  attr {\\n    key: &quot;dtype&quot;\\n    value {\\n      type: DT_STRING\\n    }\\n  }\\n  attr {\\n    key: &quot;value&quot;\\n    value {\\n      tensor {\\n        dtype: DT_STRING\\n        tensor_shape {\\n          dim {\\n            size: 1\\n          }\\n        }\\n        string_val: &quot;hidden2/bias&quot;\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;save/RestoreV2_4/shape_and_slices&quot;\\n  op: &quot;Const&quot;\\n  attr {\\n    key: &quot;dtype&quot;\\n    value {\\n      type: DT_STRING\\n    }\\n  }\\n  attr {\\n    key: &quot;value&quot;\\n    value {\\n      tensor {\\n        dtype: DT_STRING\\n        tensor_shape {\\n          dim {\\n            size: 1\\n          }\\n        }\\n        string_val: &quot;&quot;\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;save/RestoreV2_4&quot;\\n  op: &quot;RestoreV2&quot;\\n  input: &quot;save/Const&quot;\\n  input: &quot;save/RestoreV2_4/tensor_names&quot;\\n  input: &quot;save/RestoreV2_4/shape_and_slices&quot;\\n  attr {\\n    key: &quot;dtypes&quot;\\n    value {\\n      list {\\n        type: DT_FLOAT\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;save/Assign_4&quot;\\n  op: &quot;Assign&quot;\\n  input: &quot;hidden2/bias&quot;\\n  input: &quot;save/RestoreV2_4&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;_class&quot;\\n    value {\\n      list {\\n        s: &quot;loc:@hidden2/bias&quot;\\n      }\\n    }\\n  }\\n  attr {\\n    key: &quot;use_locking&quot;\\n    value {\\n      b: true\\n    }\\n  }\\n  attr {\\n    key: &quot;validate_shape&quot;\\n    value {\\n      b: true\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;save/RestoreV2_5/tensor_names&quot;\\n  op: &quot;Const&quot;\\n  attr {\\n    key: &quot;dtype&quot;\\n    value {\\n      type: DT_STRING\\n    }\\n  }\\n  attr {\\n    key: &quot;value&quot;\\n    value {\\n      tensor {\\n        dtype: DT_STRING\\n        tensor_shape {\\n          dim {\\n            size: 1\\n          }\\n        }\\n        string_val: &quot;hidden2/bias/Momentum&quot;\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;save/RestoreV2_5/shape_and_slices&quot;\\n  op: &quot;Const&quot;\\n  attr {\\n    key: &quot;dtype&quot;\\n    value {\\n      type: DT_STRING\\n    }\\n  }\\n  attr {\\n    key: &quot;value&quot;\\n    value {\\n      tensor {\\n        dtype: DT_STRING\\n        tensor_shape {\\n          dim {\\n            size: 1\\n          }\\n        }\\n        string_val: &quot;&quot;\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;save/RestoreV2_5&quot;\\n  op: &quot;RestoreV2&quot;\\n  input: &quot;save/Const&quot;\\n  input: &quot;save/RestoreV2_5/tensor_names&quot;\\n  input: &quot;save/RestoreV2_5/shape_and_slices&quot;\\n  attr {\\n    key: &quot;dtypes&quot;\\n    value {\\n      list {\\n        type: DT_FLOAT\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;save/Assign_5&quot;\\n  op: &quot;Assign&quot;\\n  input: &quot;hidden2/bias/Momentum&quot;\\n  input: &quot;save/RestoreV2_5&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;_class&quot;\\n    value {\\n      list {\\n        s: &quot;loc:@hidden2/bias&quot;\\n      }\\n    }\\n  }\\n  attr {\\n    key: &quot;use_locking&quot;\\n    value {\\n      b: true\\n    }\\n  }\\n  attr {\\n    key: &quot;validate_shape&quot;\\n    value {\\n      b: true\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;save/RestoreV2_6/tensor_names&quot;\\n  op: &quot;Const&quot;\\n  attr {\\n    key: &quot;dtype&quot;\\n    value {\\n      type: DT_STRING\\n    }\\n  }\\n  attr {\\n    key: &quot;value&quot;\\n    value {\\n      tensor {\\n        dtype: DT_STRING\\n        tensor_shape {\\n          dim {\\n            size: 1\\n          }\\n        }\\n        string_val: &quot;hidden2/kernel&quot;\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;save/RestoreV2_6/shape_and_slices&quot;\\n  op: &quot;Const&quot;\\n  attr {\\n    key: &quot;dtype&quot;\\n    value {\\n      type: DT_STRING\\n    }\\n  }\\n  attr {\\n    key: &quot;value&quot;\\n    value {\\n      tensor {\\n        dtype: DT_STRING\\n        tensor_shape {\\n          dim {\\n            size: 1\\n          }\\n        }\\n        string_val: &quot;&quot;\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;save/RestoreV2_6&quot;\\n  op: &quot;RestoreV2&quot;\\n  input: &quot;save/Const&quot;\\n  input: &quot;save/RestoreV2_6/tensor_names&quot;\\n  input: &quot;save/RestoreV2_6/shape_and_slices&quot;\\n  attr {\\n    key: &quot;dtypes&quot;\\n    value {\\n      list {\\n        type: DT_FLOAT\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;save/Assign_6&quot;\\n  op: &quot;Assign&quot;\\n  input: &quot;hidden2/kernel&quot;\\n  input: &quot;save/RestoreV2_6&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;_class&quot;\\n    value {\\n      list {\\n        s: &quot;loc:@hidden2/kernel&quot;\\n      }\\n    }\\n  }\\n  attr {\\n    key: &quot;use_locking&quot;\\n    value {\\n      b: true\\n    }\\n  }\\n  attr {\\n    key: &quot;validate_shape&quot;\\n    value {\\n      b: true\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;save/RestoreV2_7/tensor_names&quot;\\n  op: &quot;Const&quot;\\n  attr {\\n    key: &quot;dtype&quot;\\n    value {\\n      type: DT_STRING\\n    }\\n  }\\n  attr {\\n    key: &quot;value&quot;\\n    value {\\n      tensor {\\n        dtype: DT_STRING\\n        tensor_shape {\\n          dim {\\n            size: 1\\n          }\\n        }\\n        string_val: &quot;hidden2/kernel/Momentum&quot;\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;save/RestoreV2_7/shape_and_slices&quot;\\n  op: &quot;Const&quot;\\n  attr {\\n    key: &quot;dtype&quot;\\n    value {\\n      type: DT_STRING\\n    }\\n  }\\n  attr {\\n    key: &quot;value&quot;\\n    value {\\n      tensor {\\n        dtype: DT_STRING\\n        tensor_shape {\\n          dim {\\n            size: 1\\n          }\\n        }\\n        string_val: &quot;&quot;\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;save/RestoreV2_7&quot;\\n  op: &quot;RestoreV2&quot;\\n  input: &quot;save/Const&quot;\\n  input: &quot;save/RestoreV2_7/tensor_names&quot;\\n  input: &quot;save/RestoreV2_7/shape_and_slices&quot;\\n  attr {\\n    key: &quot;dtypes&quot;\\n    value {\\n      list {\\n        type: DT_FLOAT\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;save/Assign_7&quot;\\n  op: &quot;Assign&quot;\\n  input: &quot;hidden2/kernel/Momentum&quot;\\n  input: &quot;save/RestoreV2_7&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;_class&quot;\\n    value {\\n      list {\\n        s: &quot;loc:@hidden2/kernel&quot;\\n      }\\n    }\\n  }\\n  attr {\\n    key: &quot;use_locking&quot;\\n    value {\\n      b: true\\n    }\\n  }\\n  attr {\\n    key: &quot;validate_shape&quot;\\n    value {\\n      b: true\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;save/RestoreV2_8/tensor_names&quot;\\n  op: &quot;Const&quot;\\n  attr {\\n    key: &quot;dtype&quot;\\n    value {\\n      type: DT_STRING\\n    }\\n  }\\n  attr {\\n    key: &quot;value&quot;\\n    value {\\n      tensor {\\n        dtype: DT_STRING\\n        tensor_shape {\\n          dim {\\n            size: 1\\n          }\\n        }\\n        string_val: &quot;outputs/bias&quot;\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;save/RestoreV2_8/shape_and_slices&quot;\\n  op: &quot;Const&quot;\\n  attr {\\n    key: &quot;dtype&quot;\\n    value {\\n      type: DT_STRING\\n    }\\n  }\\n  attr {\\n    key: &quot;value&quot;\\n    value {\\n      tensor {\\n        dtype: DT_STRING\\n        tensor_shape {\\n          dim {\\n            size: 1\\n          }\\n        }\\n        string_val: &quot;&quot;\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;save/RestoreV2_8&quot;\\n  op: &quot;RestoreV2&quot;\\n  input: &quot;save/Const&quot;\\n  input: &quot;save/RestoreV2_8/tensor_names&quot;\\n  input: &quot;save/RestoreV2_8/shape_and_slices&quot;\\n  attr {\\n    key: &quot;dtypes&quot;\\n    value {\\n      list {\\n        type: DT_FLOAT\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;save/Assign_8&quot;\\n  op: &quot;Assign&quot;\\n  input: &quot;outputs/bias&quot;\\n  input: &quot;save/RestoreV2_8&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;_class&quot;\\n    value {\\n      list {\\n        s: &quot;loc:@outputs/bias&quot;\\n      }\\n    }\\n  }\\n  attr {\\n    key: &quot;use_locking&quot;\\n    value {\\n      b: true\\n    }\\n  }\\n  attr {\\n    key: &quot;validate_shape&quot;\\n    value {\\n      b: true\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;save/RestoreV2_9/tensor_names&quot;\\n  op: &quot;Const&quot;\\n  attr {\\n    key: &quot;dtype&quot;\\n    value {\\n      type: DT_STRING\\n    }\\n  }\\n  attr {\\n    key: &quot;value&quot;\\n    value {\\n      tensor {\\n        dtype: DT_STRING\\n        tensor_shape {\\n          dim {\\n            size: 1\\n          }\\n        }\\n        string_val: &quot;outputs/bias/Momentum&quot;\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;save/RestoreV2_9/shape_and_slices&quot;\\n  op: &quot;Const&quot;\\n  attr {\\n    key: &quot;dtype&quot;\\n    value {\\n      type: DT_STRING\\n    }\\n  }\\n  attr {\\n    key: &quot;value&quot;\\n    value {\\n      tensor {\\n        dtype: DT_STRING\\n        tensor_shape {\\n          dim {\\n            size: 1\\n          }\\n        }\\n        string_val: &quot;&quot;\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;save/RestoreV2_9&quot;\\n  op: &quot;RestoreV2&quot;\\n  input: &quot;save/Const&quot;\\n  input: &quot;save/RestoreV2_9/tensor_names&quot;\\n  input: &quot;save/RestoreV2_9/shape_and_slices&quot;\\n  attr {\\n    key: &quot;dtypes&quot;\\n    value {\\n      list {\\n        type: DT_FLOAT\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;save/Assign_9&quot;\\n  op: &quot;Assign&quot;\\n  input: &quot;outputs/bias/Momentum&quot;\\n  input: &quot;save/RestoreV2_9&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;_class&quot;\\n    value {\\n      list {\\n        s: &quot;loc:@outputs/bias&quot;\\n      }\\n    }\\n  }\\n  attr {\\n    key: &quot;use_locking&quot;\\n    value {\\n      b: true\\n    }\\n  }\\n  attr {\\n    key: &quot;validate_shape&quot;\\n    value {\\n      b: true\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;save/RestoreV2_10/tensor_names&quot;\\n  op: &quot;Const&quot;\\n  attr {\\n    key: &quot;dtype&quot;\\n    value {\\n      type: DT_STRING\\n    }\\n  }\\n  attr {\\n    key: &quot;value&quot;\\n    value {\\n      tensor {\\n        dtype: DT_STRING\\n        tensor_shape {\\n          dim {\\n            size: 1\\n          }\\n        }\\n        string_val: &quot;outputs/kernel&quot;\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;save/RestoreV2_10/shape_and_slices&quot;\\n  op: &quot;Const&quot;\\n  attr {\\n    key: &quot;dtype&quot;\\n    value {\\n      type: DT_STRING\\n    }\\n  }\\n  attr {\\n    key: &quot;value&quot;\\n    value {\\n      tensor {\\n        dtype: DT_STRING\\n        tensor_shape {\\n          dim {\\n            size: 1\\n          }\\n        }\\n        string_val: &quot;&quot;\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;save/RestoreV2_10&quot;\\n  op: &quot;RestoreV2&quot;\\n  input: &quot;save/Const&quot;\\n  input: &quot;save/RestoreV2_10/tensor_names&quot;\\n  input: &quot;save/RestoreV2_10/shape_and_slices&quot;\\n  attr {\\n    key: &quot;dtypes&quot;\\n    value {\\n      list {\\n        type: DT_FLOAT\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;save/Assign_10&quot;\\n  op: &quot;Assign&quot;\\n  input: &quot;outputs/kernel&quot;\\n  input: &quot;save/RestoreV2_10&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;_class&quot;\\n    value {\\n      list {\\n        s: &quot;loc:@outputs/kernel&quot;\\n      }\\n    }\\n  }\\n  attr {\\n    key: &quot;use_locking&quot;\\n    value {\\n      b: true\\n    }\\n  }\\n  attr {\\n    key: &quot;validate_shape&quot;\\n    value {\\n      b: true\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;save/RestoreV2_11/tensor_names&quot;\\n  op: &quot;Const&quot;\\n  attr {\\n    key: &quot;dtype&quot;\\n    value {\\n      type: DT_STRING\\n    }\\n  }\\n  attr {\\n    key: &quot;value&quot;\\n    value {\\n      tensor {\\n        dtype: DT_STRING\\n        tensor_shape {\\n          dim {\\n            size: 1\\n          }\\n        }\\n        string_val: &quot;outputs/kernel/Momentum&quot;\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;save/RestoreV2_11/shape_and_slices&quot;\\n  op: &quot;Const&quot;\\n  attr {\\n    key: &quot;dtype&quot;\\n    value {\\n      type: DT_STRING\\n    }\\n  }\\n  attr {\\n    key: &quot;value&quot;\\n    value {\\n      tensor {\\n        dtype: DT_STRING\\n        tensor_shape {\\n          dim {\\n            size: 1\\n          }\\n        }\\n        string_val: &quot;&quot;\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;save/RestoreV2_11&quot;\\n  op: &quot;RestoreV2&quot;\\n  input: &quot;save/Const&quot;\\n  input: &quot;save/RestoreV2_11/tensor_names&quot;\\n  input: &quot;save/RestoreV2_11/shape_and_slices&quot;\\n  attr {\\n    key: &quot;dtypes&quot;\\n    value {\\n      list {\\n        type: DT_FLOAT\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;save/Assign_11&quot;\\n  op: &quot;Assign&quot;\\n  input: &quot;outputs/kernel/Momentum&quot;\\n  input: &quot;save/RestoreV2_11&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;_class&quot;\\n    value {\\n      list {\\n        s: &quot;loc:@outputs/kernel&quot;\\n      }\\n    }\\n  }\\n  attr {\\n    key: &quot;use_locking&quot;\\n    value {\\n      b: true\\n    }\\n  }\\n  attr {\\n    key: &quot;validate_shape&quot;\\n    value {\\n      b: true\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;save/restore_all&quot;\\n  op: &quot;NoOp&quot;\\n  input: &quot;^save/Assign&quot;\\n  input: &quot;^save/Assign_1&quot;\\n  input: &quot;^save/Assign_2&quot;\\n  input: &quot;^save/Assign_3&quot;\\n  input: &quot;^save/Assign_4&quot;\\n  input: &quot;^save/Assign_5&quot;\\n  input: &quot;^save/Assign_6&quot;\\n  input: &quot;^save/Assign_7&quot;\\n  input: &quot;^save/Assign_8&quot;\\n  input: &quot;^save/Assign_9&quot;\\n  input: &quot;^save/Assign_10&quot;\\n  input: &quot;^save/Assign_11&quot;\\n}\\n';\n",
       "          }\n",
       "        </script>\n",
       "        <link rel=&quot;import&quot; href=&quot;https://tensorboard.appspot.com/tf-graph-basic.build.html&quot; onload=load()>\n",
       "        <div style=&quot;height:600px&quot;>\n",
       "          <tf-graph-basic id=&quot;graph0.14695850554017442&quot;></tf-graph-basic>\n",
       "        </div>\n",
       "    \"></iframe>\n",
       "    "
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "show_graph(tf.get_default_graph())"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "deletable": true,
    "editable": true
   },
   "source": [
    "Note: the book uses `tf.contrib.layers.dropout()` rather than `tf.layers.dropout()` (which did not exist when this chapter was written). It is now preferable to use `tf.layers.dropout()`, because anything in the contrib module may change or be deleted without notice. The `tf.layers.dropout()` function is almost identical to the `tf.contrib.layers.dropout()` function, except for a few minor differences. Most importantly:\n",
    "* you must specify the dropout rate (`rate`) rather than the keep probability (`keep_prob`), where `rate` is simply equal to `1 - keep_prob`,\n",
    "* the `is_training` parameter is renamed to `training`."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "metadata": {
    "collapsed": false,
    "deletable": true,
    "editable": true
   },
   "outputs": [],
   "source": [
    "from functools import partial\n",
    "\n",
    "tf.reset_default_graph()\n",
    "\n",
    "X = tf.placeholder(tf.float32, shape=(None, n_inputs), name=\"X\")\n",
    "y = tf.placeholder(tf.int64, shape=(None), name=\"y\")\n",
    "is_training = tf.placeholder(tf.bool, shape=(), name='is_training')\n",
    "\n",
    "initial_learning_rate = 0.1\n",
    "decay_steps = 10000\n",
    "decay_rate = 1/10\n",
    "global_step = tf.Variable(0, trainable=False)\n",
    "learning_rate = tf.train.exponential_decay(initial_learning_rate, global_step,\n",
    "                                           decay_steps, decay_rate)\n",
    "\n",
    "dropout_rate = 0.5\n",
    "\n",
    "with tf.name_scope(\"dnn\"):\n",
    "    he_init = tf.contrib.layers.variance_scaling_initializer()\n",
    "\n",
    "    my_dense_layer = partial(\n",
    "            tf.layers.dense,\n",
    "            activation=tf.nn.elu,\n",
    "            kernel_initializer=he_init)\n",
    "\n",
    "    X_drop = tf.layers.dropout(X, dropout_rate, training=is_training)\n",
    "    hidden1 = my_dense_layer(X_drop, n_hidden1, name=\"hidden1\")\n",
    "    hidden1_drop = tf.layers.dropout(hidden1, dropout_rate, training=is_training)\n",
    "    hidden2 = my_dense_layer(hidden1_drop, n_hidden2, name=\"hidden2\")\n",
    "    hidden2_drop = tf.layers.dropout(hidden2, dropout_rate, training=is_training)\n",
    "    logits = my_dense_layer(hidden2_drop, n_outputs, activation=None, name=\"outputs\")\n",
    "\n",
    "with tf.name_scope(\"loss\"):\n",
    "    xentropy = tf.nn.sparse_softmax_cross_entropy_with_logits(labels=y, logits=logits)\n",
    "    loss = tf.reduce_mean(xentropy, name=\"loss\")\n",
    "\n",
    "with tf.name_scope(\"train\"):\n",
    "    optimizer = tf.train.MomentumOptimizer(learning_rate, momentum)\n",
    "    training_op = optimizer.minimize(loss, global_step=global_step)    \n",
    "\n",
    "with tf.name_scope(\"eval\"):\n",
    "    correct = tf.nn.in_top_k(logits, y, 1)\n",
    "    accuracy = tf.reduce_mean(tf.cast(correct, tf.float32))\n",
    "    \n",
    "init = tf.global_variables_initializer()\n",
    "saver = tf.train.Saver()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "metadata": {
    "collapsed": false,
    "deletable": true,
    "editable": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0 Train accuracy: 0.92 Test accuracy: 0.8797\n",
      "1 Train accuracy: 0.94 Test accuracy: 0.897\n",
      "2 Train accuracy: 0.88 Test accuracy: 0.9064\n",
      "3 Train accuracy: 0.92 Test accuracy: 0.9134\n",
      "4 Train accuracy: 0.86 Test accuracy: 0.9167\n",
      "5 Train accuracy: 0.92 Test accuracy: 0.9189\n",
      "6 Train accuracy: 0.9 Test accuracy: 0.9187\n",
      "7 Train accuracy: 0.98 Test accuracy: 0.9256\n",
      "8 Train accuracy: 0.88 Test accuracy: 0.9284\n",
      "9 Train accuracy: 0.98 Test accuracy: 0.9181\n",
      "10 Train accuracy: 0.88 Test accuracy: 0.9259\n",
      "11 Train accuracy: 0.98 Test accuracy: 0.9306\n",
      "12 Train accuracy: 0.92 Test accuracy: 0.9351\n",
      "13 Train accuracy: 0.94 Test accuracy: 0.9327\n",
      "14 Train accuracy: 0.96 Test accuracy: 0.9304\n",
      "15 Train accuracy: 0.98 Test accuracy: 0.9349\n",
      "16 Train accuracy: 0.96 Test accuracy: 0.938\n",
      "17 Train accuracy: 0.96 Test accuracy: 0.9354\n",
      "18 Train accuracy: 0.9 Test accuracy: 0.9381\n",
      "19 Train accuracy: 0.94 Test accuracy: 0.942\n"
     ]
    }
   ],
   "source": [
    "n_epochs = 20\n",
    "batch_size = 50\n",
    "\n",
    "with tf.Session() as sess:\n",
    "    init.run()\n",
    "    for epoch in range(n_epochs):\n",
    "        for iteration in range(len(mnist.test.labels)//batch_size):\n",
    "            X_batch, y_batch = mnist.train.next_batch(batch_size)\n",
    "            sess.run(training_op, feed_dict={is_training: True, X: X_batch, y: y_batch})\n",
    "        acc_train = accuracy.eval(feed_dict={is_training: False, X: X_batch, y: y_batch})\n",
    "        acc_test = accuracy.eval(feed_dict={is_training: False, X: mnist.test.images, y: mnist.test.labels})\n",
    "        print(epoch, \"Train accuracy:\", acc_train, \"Test accuracy:\", acc_test)\n",
    "\n",
    "    save_path = saver.save(sess, \"my_model_final.ckpt\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "metadata": {
    "collapsed": true,
    "deletable": true,
    "editable": true
   },
   "outputs": [],
   "source": [
    "train_vars = tf.get_collection(tf.GraphKeys.TRAINABLE_VARIABLES,\n",
    "                               scope=\"hidden[2]|outputs\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "metadata": {
    "collapsed": false,
    "deletable": true,
    "editable": true
   },
   "outputs": [],
   "source": [
    "training_op2 = optimizer.minimize(loss, var_list=train_vars)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 34,
   "metadata": {
    "collapsed": false,
    "deletable": true,
    "editable": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Variable:0\n",
      "hidden1/kernel:0\n",
      "hidden1/bias:0\n",
      "hidden2/kernel:0\n",
      "hidden2/bias:0\n",
      "outputs/kernel:0\n",
      "outputs/bias:0\n",
      "hidden1/kernel/Momentum:0\n",
      "hidden1/bias/Momentum:0\n",
      "hidden2/kernel/Momentum:0\n",
      "hidden2/bias/Momentum:0\n",
      "outputs/kernel/Momentum:0\n",
      "outputs/bias/Momentum:0\n"
     ]
    }
   ],
   "source": [
    "for i in tf.global_variables():\n",
    "    print(i.name)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 35,
   "metadata": {
    "collapsed": false,
    "deletable": true,
    "editable": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "hidden1/kernel:0\n",
      "hidden1/bias:0\n",
      "hidden2/kernel:0\n",
      "hidden2/bias:0\n",
      "outputs/kernel:0\n",
      "outputs/bias:0\n"
     ]
    }
   ],
   "source": [
    "for i in tf.get_collection(tf.GraphKeys.TRAINABLE_VARIABLES):\n",
    "    print(i.name)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 36,
   "metadata": {
    "collapsed": false,
    "deletable": true,
    "editable": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "hidden2/kernel:0\n",
      "hidden2/bias:0\n",
      "outputs/kernel:0\n",
      "outputs/bias:0\n"
     ]
    }
   ],
   "source": [
    "for i in train_vars:\n",
    "    print(i.name)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "collapsed": true,
    "deletable": true,
    "editable": true
   },
   "source": [
    "# Exercise solutions"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "deletable": true,
    "editable": true
   },
   "source": [
    "**Coming soon**"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true,
    "deletable": true,
    "editable": true
   },
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.5.3"
  },
  "nav_menu": {
   "height": "360px",
   "width": "416px"
  },
  "toc": {
   "navigate_menu": true,
   "number_sections": true,
   "sideBar": true,
   "threshold": 6,
   "toc_cell": false,
   "toc_section_display": "block",
   "toc_window_display": false
  }
 },
 "nbformat": 4,
 "nbformat_minor": 0
}