mnist_experiments.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# MNIST with various deep-learning approaches\n",
    "\n",
    "### Possible approaches\n",
    "\n",
    "- standard fully-connected network (baseline)\n",
    "- LSTM along the pixels (possibly with attention?)\n",
    "- RL algorithm where an LSTM looks at a *window* of the image, and steers it, while building a classification?"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "import tensorflow as tf\n",
    "import numpy as np\n",
    "from read_mnist import load_dataset\n",
    "import random\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 78,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "class MNISTClassifier(object):\n",
    "    def __init__(self, train_path, test_path):\n",
    "        self.train_in, self.train_out = load_dataset('../data/train')\n",
    "        self.test_in, self.test_out = load_dataset('../data/t10k')\n",
    "        \n",
    "        self.global_step = tf.contrib.framework.get_or_create_global_step()\n",
    "        self.session = None\n",
    "        \n",
    "        with tf.variable_scope(self.name()):\n",
    "            self.build()\n",
    "    \n",
    "    def setup(self):\n",
    "        if self.session: self.session.close()\n",
    "        self.session = tf.InteractiveSession()\n",
    "        self.session.run(tf.global_variables_initializer())\n",
    "    \n",
    "    def name(self):\n",
    "        return 'mnist14'\n",
    "    \n",
    "    def build(self):\n",
    "        assert False, '{} does not implement build'.format(self)\n",
    "    \n",
    "    def train(self, epochs, max_batches_per_epoch=None):\n",
    "        for epoch in xrange(epochs):\n",
    "            for i, (input, output) in enumerate(self.iterate_batches(self.train_in, self.train_out)):\n",
    "                self.train_on_batch(input, output)\n",
    "                if max_batches_per_epoch and i >= max_batches_per_epoch: break\n",
    "            acc = self.eval_accuracy()\n",
    "            print \"Epoch {}: accuracy: {}%\".format(epoch, acc * 100)\n",
    "    \n",
    "    def iterate_batches(self, input, output):\n",
    "        batch_size = 32\n",
    "        xy = list(zip(input, output))\n",
    "        random.shuffle(xy)\n",
    "        i = 0\n",
    "        while i < len(xy):\n",
    "            batch = xy[i:min(i+batch_size, len(xy))]\n",
    "            yield [x for x,y in batch], [y for x,y in batch]\n",
    "            i += batch_size\n",
    "\n",
    "    \n",
    "    def train_on_batch(self, input, output):\n",
    "        assert False, '{} does not implement train_on_batch'.format(self)\n",
    "    \n",
    "    def eval_accuracy(self):\n",
    "        num = 0\n",
    "        denom = 0\n",
    "        for x, y in self.iterate_batches(self.test_in, self.test_out):\n",
    "            denom += len(x)\n",
    "            num += len(x) * self.accuracy_for_batch(x, y)\n",
    "        return num * 1.0 / denom\n",
    "    \n",
    "    def accuracy_for_batch(self, input, output):\n",
    "        assert False, '{} does not implement accuracy_for_batch'.format(self)\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## How well does a plain fully-connected neural network perform?"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 90,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "class MLPClassifier(MNISTClassifier):\n",
    "    def build(self):\n",
    "        self.input = tf.placeholder(tf.float32, [None, 28, 28], name='input')\n",
    "        self.target = tf.placeholder(tf.int32, [None], name='target')\n",
    "        x = self.create_logits(self.input)\n",
    "        self.loss = tf.nn.sparse_softmax_cross_entropy_with_logits(logits=x, labels=self.target)\n",
    "        self.prediction = tf.cast(tf.argmax(x, axis=1), tf.int32)\n",
    "        correct = tf.equal(self.prediction, self.target)\n",
    "        self.accuracy = tf.reduce_mean(tf.cast(correct, tf.float32))\n",
    "        self.train_op = tf.train.AdamOptimizer(1e-3).minimize(self.loss)\n",
    "    \n",
    "    def create_logits(self, x):\n",
    "        x = tf.reshape(self.input, [-1, 28 * 28])\n",
    "        for i, l in enumerate([256, 128, 10]):\n",
    "            x = tf.layers.dense(x, l, tf.nn.relu, name='fc'+str(i))\n",
    "            x = tf.nn.dropout(x, 0.7)\n",
    "        return x\n",
    "    \n",
    "    def accuracy_for_batch(self, x, y):\n",
    "        feed = {\n",
    "            self.input: x,\n",
    "            self.target: y\n",
    "        }\n",
    "        return self.session.run(self.accuracy, feed_dict=feed)\n",
    "    \n",
    "    def train_on_batch(self, input, output):\n",
    "        feed = {\n",
    "            self.input: input,\n",
    "            self.target: output\n",
    "        }\n",
    "        _, loss = self.session.run([self.train_op, self.loss], feed_dict=feed)\n",
    "        return loss"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 64,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "m = MLPClassifier('../data/train', '../data/test')\n",
    "m.setup()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 61,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 0: accuracy: 68.76%\n",
      "Epoch 1: accuracy: 70.16%\n",
      "Epoch 2: accuracy: 70.28%\n"
     ]
    }
   ],
   "source": [
    "m.train(3)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## How does a Tensorflow LSTM compare?\n",
    "\n",
    "_Let's run an LSTM over the image row-by-row and see what happens_"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "class RNNClassifier(MLPClassifier):\n",
    "    def name(self):\n",
    "        return 'rnn1'\n",
    "    \n",
    "    def make_cell(self, size):\n",
    "        return tf.contrib.rnn.BasicLSTMCell(size)\n",
    "    \n",
    "    def create_logits(self, x):\n",
    "        x = tf.reshape(x, [-1, 28, 28]) # each time step is a different row of the image\n",
    "        \n",
    "        rnn_cell = self.make_cell(128)\n",
    "        outputs, state = tf.nn.dynamic_rnn(rnn_cell, x, dtype=tf.float32, scope='lstm1')\n",
    "        x = outputs[:, -1]\n",
    "        \n",
    "        return tf.layers.dense(x, 10, tf.nn.relu, name='fc')\n",
    "\n",
    "l = LSTMClassifier('../data/train', '../data/test')\n",
    "l.setup()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "l.train(20, max_batches_per_epoch=1)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "collapsed": true
   },
   "source": [
    "## Experiment: multiple random initializations\n",
    "\n",
    "One thing I've noticed when training deep models (especially small ones) is that initial losses can vary _wildly_ between different random initializations.\n",
    "\n",
    "Before starting gradient descent, why not perform a preliminary search of the space by randomly initializing then network a couple times, training for a couple batches with a high learning rate, and keeping the initialization with the best loss?"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 102,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "class HeadStartClassifier(MLPClassifier):\n",
    "    def headstart(self, trials=10, batches=10):\n",
    "        variables = tf.get_collection(tf.GraphKeys.TRAINABLE_VARIABLES, scope=self.name())\n",
    "        \n",
    "        saved_vars = []\n",
    "        accuracies = []\n",
    "        for trial in xrange(trials):\n",
    "            self.session.run(tf.initialize_variables(variables))\n",
    "            self.train(1, max_batches_per_epoch=batches)\n",
    "            saved_vars.append(self.session.run(variables))\n",
    "            accuracies.append(self.eval_accuracy())\n",
    "            print 'Trial {} accuracy: {}'.format(trial, accuracies[-1])\n",
    "            \n",
    "        # restore the best set of variables:\n",
    "        best_idx = accuracies.index(max(accuracies))\n",
    "        assign_ops = [var.assign(value) for var, value in zip(variables, saved_vars[best_idx])]\n",
    "        self.session.run(assign_ops)\n",
    "        print \"All accuracies:\", accuracies\n",
    "    \n",
    "    def name(self):\n",
    "        return 'headstart5'\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 106,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "hc = HeadStartClassifier('../data/train', '../data/test')\n",
    "hc.setup()\n",
    "hc.headstart()\n",
    "l.train(20, max_batches_per_epoch=4)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### preliminary results:\n",
    "\n",
    "Randomly initialized the net 20 times, and then trained each for 10 batches.\n",
    "\n",
    "After this, accuracies ranged from 15% to 30%. After 1 epoch, the best initialization was trained to 85% accuracy."
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "collapsed": true
   },
   "source": [
    "# Highway networks\n",
    "\n",
    "- Performance of 3-layer MLP net: Epoch 3: accuracy: 97.78%\n",
    "- Performance of 20-layer MLP net: Epoch 3: accuracy: 64.82% (fails to improve in subsequent iterations)\n",
    "- Performance with 20-layer MLP net with relu only applied every third layer: Epoch 3: accuracy: 96.09%\n",
    "- Performance with 20-layer residual MLP net: Epoch 3: accuracy: 97.32%\n",
    "- Performance with 20-layer highway (gated) MLP net: Epoch 3: accuracy: 3.23% (generally hovers around 9% – yikes!)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 124,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "class NLayerClassifier(MLPClassifier):\n",
    "    def __init__(self, train_path, test_path, n_layers, style=None):\n",
    "        # style: None, 'residual' or 'highway'\n",
    "        self.n_layers = n_layers\n",
    "        self.style = style\n",
    "        super(NLayerClassifier, self).__init__(train_path, test_path)\n",
    "    \n",
    "    def name(self):\n",
    "        return 'n6_{}.{}'.format(self.n_layers, self.style)\n",
    "    \n",
    "    def create_logits(self, x):\n",
    "        x = tf.reshape(self.input, [-1, 28 * 28])\n",
    "        \n",
    "        layers = [256] + ([128] * (self.n_layers-1))\n",
    "        \n",
    "        shortcut = None\n",
    "        \n",
    "        def identity(x): return x\n",
    "        \n",
    "        for i, l in enumerate(layers):\n",
    "            x_orig = x\n",
    "            x = tf.layers.dense(x, l, identity, name='fc'+str(i))\n",
    "            if self.style == 'residual':\n",
    "                if i % 3 == 0:\n",
    "                    if shortcut is not None and shortcut.get_shape()[-1].value == x.get_shape()[-1].value:\n",
    "                        x += shortcut\n",
    "                    shortcut = x\n",
    "                    x = tf.nn.relu(x)\n",
    "            elif self.style == 'highway':\n",
    "                if x_orig.get_shape()[-1].value == x.get_shape()[-1].value:\n",
    "                    transform_gate = tf.layers.dense(x_orig, l, tf.nn.sigmoid, name='transform'+str(i))\n",
    "                    carry = 1 - transform_gate # can also learn carry gate independently, just like the transform gate\n",
    "                    x = x * transform_gate + x_orig * carry\n",
    "            else:\n",
    "                x = tf.nn.relu(x)\n",
    "        \n",
    "        return x\n",
    "\n",
    "# m3 = NLayerClassifier('../data/train', '../data/test', 3)\n",
    "# m3.setup()\n",
    "\n",
    "# m20 = NLayerClassifier('../data/train', '../data/test', 20)\n",
    "# m20.setup()\n",
    "\n",
    "# m20r = NLayerClassifier('../data/train', '../data/test', 20, 'residual')\n",
    "# m20r.setup()\n",
    "\n",
    "m20h = NLayerClassifier('../data/train', '../data/test', 20, 'highway')\n",
    "m20h.setup()\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 125,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 0: accuracy: 8.87%\n",
      "Epoch 1: accuracy: 9.57%\n",
      "Epoch 2: accuracy: 8.67%\n",
      "Epoch 3: accuracy: 3.23%\n"
     ]
    },
    {
     "ename": "KeyboardInterrupt",
     "evalue": "",
     "output_type": "error",
     "traceback": [
      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[0;31mKeyboardInterrupt\u001b[0m                         Traceback (most recent call last)",
      "\u001b[0;32m<ipython-input-125-9fd73607792f>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m()\u001b[0m\n\u001b[1;32m      2\u001b[0m \u001b[0;31m# m20.train(20)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m      3\u001b[0m \u001b[0;31m# m20r.train(20)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m----> 4\u001b[0;31m \u001b[0mm20h\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mtrain\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;36m20\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m",
      "\u001b[0;32m<ipython-input-78-08550db08dcd>\u001b[0m in \u001b[0;36mtrain\u001b[0;34m(self, epochs, max_batches_per_epoch)\u001b[0m\n\u001b[1;32m     24\u001b[0m         \u001b[0;32mfor\u001b[0m \u001b[0mepoch\u001b[0m \u001b[0;32min\u001b[0m \u001b[0mxrange\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mepochs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     25\u001b[0m             \u001b[0;32mfor\u001b[0m \u001b[0mi\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m(\u001b[0m\u001b[0minput\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0moutput\u001b[0m\u001b[0;34m)\u001b[0m \u001b[0;32min\u001b[0m \u001b[0menumerate\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0miterate_batches\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mtrain_in\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mtrain_out\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 26\u001b[0;31m                 \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mtrain_on_batch\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0minput\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0moutput\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m     27\u001b[0m                 \u001b[0;32mif\u001b[0m \u001b[0mmax_batches_per_epoch\u001b[0m \u001b[0;32mand\u001b[0m \u001b[0mi\u001b[0m \u001b[0;34m>=\u001b[0m \u001b[0mmax_batches_per_epoch\u001b[0m\u001b[0;34m:\u001b[0m \u001b[0;32mbreak\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     28\u001b[0m             \u001b[0macc\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0meval_accuracy\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m<ipython-input-90-7e60845ff514>\u001b[0m in \u001b[0;36mtrain_on_batch\u001b[0;34m(self, input, output)\u001b[0m\n\u001b[1;32m     29\u001b[0m             \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mtarget\u001b[0m\u001b[0;34m:\u001b[0m \u001b[0moutput\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     30\u001b[0m         }\n\u001b[0;32m---> 31\u001b[0;31m         \u001b[0m_\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mloss\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0msession\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mrun\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mtrain_op\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mloss\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mfeed_dict\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mfeed\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m     32\u001b[0m         \u001b[0;32mreturn\u001b[0m \u001b[0mloss\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/Library/Python/2.7/site-packages/tensorflow/python/client/session.pyc\u001b[0m in \u001b[0;36mrun\u001b[0;34m(self, fetches, feed_dict, options, run_metadata)\u001b[0m\n\u001b[1;32m    765\u001b[0m     \u001b[0;32mtry\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    766\u001b[0m       result = self._run(None, fetches, feed_dict, options_ptr,\n\u001b[0;32m--> 767\u001b[0;31m                          run_metadata_ptr)\n\u001b[0m\u001b[1;32m    768\u001b[0m       \u001b[0;32mif\u001b[0m \u001b[0mrun_metadata\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    769\u001b[0m         \u001b[0mproto_data\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mtf_session\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mTF_GetBuffer\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mrun_metadata_ptr\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/Library/Python/2.7/site-packages/tensorflow/python/client/session.pyc\u001b[0m in \u001b[0;36m_run\u001b[0;34m(self, handle, fetches, feed_dict, options, run_metadata)\u001b[0m\n\u001b[1;32m    963\u001b[0m     \u001b[0;32mif\u001b[0m \u001b[0mfinal_fetches\u001b[0m \u001b[0;32mor\u001b[0m \u001b[0mfinal_targets\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    964\u001b[0m       results = self._do_run(handle, final_targets, final_fetches,\n\u001b[0;32m--> 965\u001b[0;31m                              feed_dict_string, options, run_metadata)\n\u001b[0m\u001b[1;32m    966\u001b[0m     \u001b[0;32melse\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    967\u001b[0m       \u001b[0mresults\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0;34m[\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/Library/Python/2.7/site-packages/tensorflow/python/client/session.pyc\u001b[0m in \u001b[0;36m_do_run\u001b[0;34m(self, handle, target_list, fetch_list, feed_dict, options, run_metadata)\u001b[0m\n\u001b[1;32m   1013\u001b[0m     \u001b[0;32mif\u001b[0m \u001b[0mhandle\u001b[0m \u001b[0;32mis\u001b[0m \u001b[0mNone\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1014\u001b[0m       return self._do_call(_run_fn, self._session, feed_dict, fetch_list,\n\u001b[0;32m-> 1015\u001b[0;31m                            target_list, options, run_metadata)\n\u001b[0m\u001b[1;32m   1016\u001b[0m     \u001b[0;32melse\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1017\u001b[0m       return self._do_call(_prun_fn, self._session, handle, feed_dict,\n",
      "\u001b[0;32m/Library/Python/2.7/site-packages/tensorflow/python/client/session.pyc\u001b[0m in \u001b[0;36m_do_call\u001b[0;34m(self, fn, *args)\u001b[0m\n\u001b[1;32m   1020\u001b[0m   \u001b[0;32mdef\u001b[0m \u001b[0m_do_call\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mfn\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m*\u001b[0m\u001b[0margs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1021\u001b[0m     \u001b[0;32mtry\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m-> 1022\u001b[0;31m       \u001b[0;32mreturn\u001b[0m \u001b[0mfn\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m*\u001b[0m\u001b[0margs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m   1023\u001b[0m     \u001b[0;32mexcept\u001b[0m \u001b[0merrors\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mOpError\u001b[0m \u001b[0;32mas\u001b[0m \u001b[0me\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1024\u001b[0m       \u001b[0mmessage\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mcompat\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mas_text\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0me\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mmessage\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/Library/Python/2.7/site-packages/tensorflow/python/client/session.pyc\u001b[0m in \u001b[0;36m_run_fn\u001b[0;34m(session, feed_dict, fetch_list, target_list, options, run_metadata)\u001b[0m\n\u001b[1;32m   1002\u001b[0m         return tf_session.TF_Run(session, options,\n\u001b[1;32m   1003\u001b[0m                                  \u001b[0mfeed_dict\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mfetch_list\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mtarget_list\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m-> 1004\u001b[0;31m                                  status, run_metadata)\n\u001b[0m\u001b[1;32m   1005\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1006\u001b[0m     \u001b[0;32mdef\u001b[0m \u001b[0m_prun_fn\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0msession\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mhandle\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mfeed_dict\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mfetch_list\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;31mKeyboardInterrupt\u001b[0m: "
     ]
    }
   ],
   "source": [
    "# m3.train(20)\n",
    "# m20.train(20)\n",
    "# m20r.train(20)\n",
    "m20h.train(20)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 2",
   "language": "python",
   "name": "python2"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 2
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython2",
   "version": "2.7.10"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}