2nd_DNN_3.ipynb

{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "colab": {
      "name": "2nd_DNN-3.ipynb",
      "version": "0.3.2",
      "provenance": []
    },
    "kernelspec": {
      "name": "python3",
      "display_name": "Python 3"
    },
    "accelerator": "GPU"
  },
  "cells": [
    {
      "cell_type": "code",
      "metadata": {
        "id": "vZOhbujJ-3wr",
        "colab_type": "code",
        "outputId": "a2ae8d06-0329-4e31-e298-9d375d9c7082",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 35
        }
      },
      "source": [
        "# https://keras.io/\n",
        "!pip install -q keras\n",
        "import keras"
      ],
      "execution_count": 1,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Using TensorFlow backend.\n"
          ],
          "name": "stderr"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab_type": "code",
        "id": "W_kwaa8cGb4J",
        "colab": {}
      },
      "source": [
        "# https://keras.io/\n",
        "!pip install -q keras\n",
        "import keras"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab_type": "code",
        "id": "v0zVezSvGgRx",
        "colab": {}
      },
      "source": [
        "# https://keras.io/\n",
        "!pip install -q keras\n",
        "import keras"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab_type": "code",
        "id": "lfJUz7NBGgSJ",
        "colab": {}
      },
      "source": [
        "import numpy as np\n",
        "\n",
        "from keras.models import Sequential\n",
        "from keras.layers import Dense, Dropout, Activation, Flatten, Add, BatchNormalization\n",
        "from keras.layers import Convolution2D, MaxPooling2D\n",
        "from keras.utils import np_utils\n",
        "\n",
        "from keras.optimizers import Adam, SGD\n",
        "from keras.preprocessing.image import ImageDataGenerator\n",
        "\n",
        "from keras.callbacks import LearningRateScheduler, ModelCheckpoint\n",
        "from keras.initializers import VarianceScaling\n",
        "from keras.models import load_model\n",
        "\n",
        "\n",
        "from keras.datasets import mnist"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab_type": "code",
        "id": "UiiL33jSGgSV",
        "colab": {}
      },
      "source": [
        "\n",
        "#spliting train/test data\n",
        "(X_train, y_train), (X_test, y_test) = mnist.load_data()"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab_type": "code",
        "outputId": "8c7dd27a-b059-4696-e828-1f9429062b13",
        "id": "JRvD5ZsgGgSu",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 305
        }
      },
      "source": [
        "# Display image shape of image \n",
        "print (X_train.shape)\n",
        "from matplotlib import pyplot as plt\n",
        "%matplotlib inline\n",
        "plt.imshow(X_train[0])"
      ],
      "execution_count": 6,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "(60000, 28, 28)\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "<matplotlib.image.AxesImage at 0x7fda535fc400>"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 6
        },
        {
          "output_type": "display_data",
          "data": {
            "image/png": "iVBORw0KGgoAAAANSUhEUgAAAP8AAAD8CAYAAAC4nHJkAAAABHNCSVQICAgIfAhkiAAAAAlwSFlz\nAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDMuMC4zLCBo\ndHRwOi8vbWF0cGxvdGxpYi5vcmcvnQurowAADoBJREFUeJzt3X2MXOV1x/HfyXq9jo1JvHHYboiL\nHeMEiGlMOjIgLKCiuA5CMiiKiRVFDiFxmuCktK4EdavGrWjlVgmRQynS0ri2I95CAsJ/0CR0FUGi\nwpbFMeYtvJlNY7PsYjZgQ4i9Xp/+sdfRBnaeWc/cmTu75/uRVjtzz71zj6792zszz8x9zN0FIJ53\nFd0AgGIQfiAowg8ERfiBoAg/EBThB4Ii/EBQhB8IivADQU1r5M6mW5vP0KxG7hII5bd6U4f9kE1k\n3ZrCb2YrJG2W1CLpP9x9U2r9GZqls+2iWnYJIKHHuye8btVP+82sRdJNkj4h6QxJq83sjGofD0Bj\n1fKaf6mk5919j7sflnSHpJX5tAWg3moJ/8mSfjXm/t5s2e8xs7Vm1mtmvcM6VMPuAOSp7u/2u3uX\nu5fcvdSqtnrvDsAE1RL+fZLmjbn/wWwZgEmglvA/ImmRmS0ws+mSPi1pRz5tAai3qof63P2Ima2T\n9CONDvVtcfcnc+sMQF3VNM7v7vdJui+nXgA0EB/vBYIi/EBQhB8IivADQRF+ICjCDwRF+IGgCD8Q\nFOEHgiL8QFCEHwiK8ANBEX4gKMIPBEX4gaAIPxAU4QeCIvxAUIQfCIrwA0ERfiAowg8ERfiBoAg/\nEBThB4Ii/EBQhB8IivADQRF+IKiaZuk1sz5JByWNSDri7qU8mkJ+bFr6n7jl/XPruv9n/np+2drI\nzKPJbU9ZOJisz/yKJesv3zC9bG1n6c7ktvtH3kzWz75rfbJ+6l89nKw3g5rCn/kTd9+fw+MAaCCe\n9gNB1Rp+l/RjM3vUzNbm0RCAxqj1af8yd99nZidJut/MfuHuD45dIfujsFaSZmhmjbsDkJeazvzu\nvi/7PSjpHklLx1mny91L7l5qVVstuwOQo6rDb2azzGz2sduSlkt6Iq/GANRXLU/7OyTdY2bHHuc2\nd/9hLl0BqLuqw+/ueyR9LMdepqyW0xcl697Wmqy/dMF7k/W3zik/Jt3+nvR49U8/lh7vLtJ//WZ2\nsv4v/7YiWe8587aytReH30puu2ng4mT9Az/1ZH0yYKgPCIrwA0ERfiAowg8ERfiBoAg/EFQe3+oL\nb+TCjyfrN2y9KVn/cGv5r55OZcM+kqz//Y2fS9anvZkebjv3rnVla7P3HUlu27Y/PRQ4s7cnWZ8M\nOPMDQRF+ICjCDwRF+IGgCD8QFOEHgiL8QFCM8+eg7ZmXkvVHfzsvWf9w60Ce7eRqff85yfqeN9KX\n/t668Ptla68fTY/Td3z7f5L1epr8X9itjDM/EBThB4Ii/EBQhB8IivADQRF+ICjCDwRl7o0b0TzR\n2v1su6hh+2sWQ1eem6wfWJG+vHbL7hOS9ce+cuNx93TM9fv/KFl/5IL0OP7Ia68n635u+au7930t\nuakWrH4svQLeoce7dcCH0nOXZzjzA0ERfiAowg8ERfiBoAg/EBThB4Ii/EBQFcf5zWyLpEslDbr7\n4mxZu6Q7Jc2X1Cdplbv/utLOoo7zV9Iy933J+sirQ8n6i7eVH6t/8vwtyW2X/vNXk/WTbiruO/U4\nfnmP82+V9PaJ0K+T1O3uiyR1Z/cBTCIVw+/uD0p6+6lnpaRt2e1tki7LuS8AdVbta/4Od+/Pbr8s\nqSOnfgA0SM1v+PnomwZl3zgws7Vm1mtmvcM6VOvuAOSk2vAPmFmnJGW/B8ut6O5d7l5y91Kr2qrc\nHYC8VRv+HZLWZLfXSLo3n3YANErF8JvZ7ZIekvQRM9trZldJ2iTpYjN7TtKfZvcBTCIVr9vv7qvL\nlBiwz8nI/ldr2n74wPSqt/3oZ55K1l+5uSX9AEdHqt43isUn/ICgCD8QFOEHgiL8QFCEHwiK8ANB\nMUX3FHD6tc+WrV15ZnpE9j9P6U7WL/jU1cn67DsfTtbRvDjzA0ERfiAowg8ERfiBoAg/EBThB4Ii\n/EBQjPNPAalpsl/98unJbf9vx1vJ+nXXb0/W/2bV5cm6//w9ZWvz/umh5LZq4PTxEXHmB4Ii/EBQ\nhB8IivADQRF+ICjCDwRF+IGgKk7RnSem6G4+Q58/N1m/9evfSNYXTJtR9b4/un1dsr7olv5k/cie\nvqr3PVXlPUU3gCmI8ANBEX4gKMIPBEX4gaAIPxAU4QeCqjjOb2ZbJF0qadDdF2fLNkr6oqRXstU2\nuPt9lXbGOP/k4+ctSdZP3LQ3Wb/9Qz+qet+n/eQLyfpH/qH8dQwkaeS5PVXve7LKe5x/q6QV4yz/\nlrsvyX4qBh9Ac6kYfnd/UNJQA3oB0EC1vOZfZ2a7zWyLmc3JrSMADVFt+G+WtFDSEkn9kr5ZbkUz\nW2tmvWbWO6xDVe4OQN6qCr+7D7j7iLsflXSLpKWJdbvcveTupVa1VdsngJxVFX4z6xxz93JJT+TT\nDoBGqXjpbjO7XdKFkuaa2V5JX5d0oZktkeSS+iR9qY49AqgDvs+PmrR0nJSsv3TFqWVrPdduTm77\nrgpPTD/z4vJk/fVlrybrUxHf5wdQEeEHgiL8QFCEHwiK8ANBEX4gKIb6UJjv7U1P0T3Tpifrv/HD\nyfqlX72m/GPf05PcdrJiqA9ARYQfCIrwA0ERfiAowg8ERfiBoAg/EFTF7/MjtqPL0pfufuFT6Sm6\nFy/pK1urNI5fyY1DZyXrM+/trenxpzrO/EBQhB8IivADQRF+ICjCDwRF+IGgCD8QFOP8U5yVFifr\nz34tPdZ+y3nbkvXzZ6S/U1+LQz6crD88tCD9AEf7c+xm6uHMDwRF+IGgCD8QFOEHgiL8QFCEHwiK\n8ANBVRznN7N5krZL6pDkkrrcfbOZtUu6U9J8SX2SVrn7r+vXalzTFpySrL9w5QfK1jZecUdy20+e\nsL+qnvKwYaCUrD+w+Zxkfc629HX/kTaRM/8RSevd/QxJ50i62szOkHSdpG53XySpO7sPYJKoGH53\n73f3ndntg5KelnSypJWSjn38a5uky+rVJID8HddrfjObL+ksST2SOtz92OcnX9boywIAk8SEw29m\nJ0j6gaRr3P3A2JqPTvg37qR/ZrbWzHrNrHdYh2pqFkB+JhR+M2vVaPBvdfe7s8UDZtaZ1TslDY63\nrbt3uXvJ3UutasujZwA5qBh+MzNJ35H0tLvfMKa0Q9Ka7PYaSffm3x6AepnIV3rPk/RZSY+b2a5s\n2QZJmyR9z8yukvRLSavq0+LkN23+Hybrr/9xZ7J+xT/+MFn/8/fenazX0/r+9HDcQ/9efjivfev/\nJredc5ShvHqqGH53/5mkcvN9X5RvOwAahU/4AUERfiAowg8ERfiBoAg/EBThB4Li0t0TNK3zD8rW\nhrbMSm775QUPJOurZw9U1VMe1u1blqzvvDk9Rffc7z+RrLcfZKy+WXHmB4Ii/EBQhB8IivADQRF+\nICjCDwRF+IGgwozzH/6z9GWiD//lULK+4dT7ytaWv/vNqnrKy8DIW2Vr5+9Yn9z2tL/7RbLe/lp6\nnP5osopmxpkfCIrwA0ERfiAowg8ERfiBoAg/EBThB4IKM87fd1n679yzZ95Vt33f9NrCZH3zA8uT\ndRspd+X0Uadd/2LZ2qKBnuS2I8kqpjLO/EBQhB8IivADQRF+ICjCDwRF+IGgCD8QlLl7egWzeZK2\nS+qQ5JK63H2zmW2U9EVJr2SrbnD38l96l3SitfvZxqzeQL30eLcO+FD6gyGZiXzI54ik9e6+08xm\nS3rUzO7Pat9y929U2yiA4lQMv7v3S+rPbh80s6clnVzvxgDU13G95jez+ZLOknTsM6PrzGy3mW0x\nszlltllrZr1m1jusQzU1CyA/Ew6/mZ0g6QeSrnH3A5JulrRQ0hKNPjP45njbuXuXu5fcvdSqthxa\nBpCHCYXfzFo1Gvxb3f1uSXL3AXcfcfejkm6RtLR+bQLIW8Xwm5lJ+o6kp939hjHLO8esdrmk9HSt\nAJrKRN7tP0/SZyU9bma7smUbJK02syUaHf7rk/SlunQIoC4m8m7/zySNN26YHNMH0Nz4hB8QFOEH\ngiL8QFCEHwiK8ANBEX4gKMIPBEX4gaAIPxAU4QeCIvxAUIQfCIrwA0ERfiCoipfuznVnZq9I+uWY\nRXMl7W9YA8enWXtr1r4keqtWnr2d4u7vn8iKDQ3/O3Zu1uvupcIaSGjW3pq1L4neqlVUbzztB4Ii\n/EBQRYe/q+D9pzRrb83al0Rv1Sqkt0Jf8wMoTtFnfgAFKST8ZrbCzJ4xs+fN7LoieijHzPrM7HEz\n22VmvQX3ssXMBs3siTHL2s3sfjN7Lvs97jRpBfW20cz2Zcdul5ldUlBv88zsJ2b2lJk9aWZ/kS0v\n9Ngl+irkuDX8ab+ZtUh6VtLFkvZKekTSand/qqGNlGFmfZJK7l74mLCZnS/pDUnb3X1xtuxfJQ25\n+6bsD+ccd7+2SXrbKOmNomduziaU6Rw7s7SkyyR9TgUeu0Rfq1TAcSvizL9U0vPuvsfdD0u6Q9LK\nAvpoeu7+oKShty1eKWlbdnubRv/zNFyZ3pqCu/e7+87s9kFJx2aWLvTYJfoqRBHhP1nSr8bc36vm\nmvLbJf3YzB41s7VFNzOOjmzadEl6WVJHkc2Mo+LMzY30tpmlm+bYVTPjdd54w++dlrn7xyV9QtLV\n2dPbpuSjr9maabhmQjM3N8o4M0v/TpHHrtoZr/NWRPj3SZo35v4Hs2VNwd33Zb8HJd2j5pt9eODY\nJKnZ78GC+/mdZpq5ebyZpdUEx66ZZrwuIvyPSFpkZgvMbLqkT0vaUUAf72Bms7I3YmRmsyQtV/PN\nPrxD0prs9hpJ9xbYy+9plpmby80srYKPXdPNeO3uDf+RdIlG3/F/QdLfFtFDmb4+JOmx7OfJonuT\ndLtGnwYOa/S9kaskvU9St6TnJP23pPYm6u27kh6XtFujQessqLdlGn1Kv1vSruznkqKPXaKvQo4b\nn/ADguINPyAowg8ERfiBoAg/EBThB4Ii/EBQhB8IivADQf0/sEWOix6VKakAAAAASUVORK5CYII=\n",
            "text/plain": [
              "<Figure size 432x288 with 1 Axes>"
            ]
          },
          "metadata": {
            "tags": []
          }
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab_type": "code",
        "id": "ib-ZE3oFGgTl",
        "colab": {}
      },
      "source": [
        "#Reshape image size\n",
        "X_train = X_train.reshape(X_train.shape[0], 28, 28,1)\n",
        "X_test = X_test.reshape(X_test.shape[0], 28, 28,1)"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab_type": "code",
        "id": "Hw6VJtKZGgTt",
        "colab": {}
      },
      "source": [
        "#Convert numpy array to float32\n",
        "X_train = X_train.astype('float32')\n",
        "X_test = X_test.astype('float32')\n",
        "X_train /= 255\n",
        "X_test /= 255"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab_type": "code",
        "outputId": "f5fc83e5-b908-4036-9f9b-1c6342d66a70",
        "id": "BdSfimNcGgTy",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 35
        }
      },
      "source": [
        "y_train[:10]"
      ],
      "execution_count": 9,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "array([5, 0, 4, 1, 9, 2, 1, 3, 1, 4], dtype=uint8)"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 9
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab_type": "code",
        "id": "k9OO4Tp7GgT6",
        "colab": {}
      },
      "source": [
        "Y_train = keras.utils.to_categorical(y_train, 10)\n",
        "Y_test = keras.utils.to_categorical(y_test, 10)"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab_type": "code",
        "outputId": "fd02499c-a3a7-49b6-ba5e-1c1914f2db46",
        "id": "joAoPxqFGgUK",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 199
        }
      },
      "source": [
        "Y_train[:10]"
      ],
      "execution_count": 11,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "array([[0., 0., 0., 0., 0., 1., 0., 0., 0., 0.],\n",
              "       [1., 0., 0., 0., 0., 0., 0., 0., 0., 0.],\n",
              "       [0., 0., 0., 0., 1., 0., 0., 0., 0., 0.],\n",
              "       [0., 1., 0., 0., 0., 0., 0., 0., 0., 0.],\n",
              "       [0., 0., 0., 0., 0., 0., 0., 0., 0., 1.],\n",
              "       [0., 0., 1., 0., 0., 0., 0., 0., 0., 0.],\n",
              "       [0., 1., 0., 0., 0., 0., 0., 0., 0., 0.],\n",
              "       [0., 0., 0., 1., 0., 0., 0., 0., 0., 0.],\n",
              "       [0., 1., 0., 0., 0., 0., 0., 0., 0., 0.],\n",
              "       [0., 0., 0., 0., 1., 0., 0., 0., 0., 0.]], dtype=float32)"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 11
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "Wmr0pvKVGxgD",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 165
        },
        "outputId": "9132f4a2-a9bf-4056-f92c-7340b7b06b0d"
      },
      "source": [
        "#3rd sample N/W ( chaged dropout to 0.2 and #24 and #11\n",
        "#Changed mxpool to 24 added,  dropout + reducing chalnnels to 16 (parmeters) )\n",
        "\n",
        "from keras.layers import Activation\n",
        "model = Sequential()\n",
        "\n",
        "\n",
        "model.add(Convolution2D(10, 3, 3, activation='relu', input_shape=(28,28,1))) # 26\n",
        "model.add(BatchNormalization())\n",
        "\n",
        "model.add(Convolution2D(16, 3, 3, activation='relu')) # 24\n",
        "model.add(BatchNormalization())\n",
        "model.add(Dropout(0.20))\n",
        "\n",
        "\n",
        "model.add(MaxPooling2D(pool_size=(2, 2))) # 12\n",
        "model.add(Convolution2D(16, 3, 3, activation='relu')) # 10\n",
        "model.add(BatchNormalization())\n",
        "model.add(Dropout(0.20))\n",
        "\n",
        "\n",
        "model.add(Convolution2D(16, 3, 3, activation='relu')) # 8\n",
        "model.add(Dropout(0.20))\n",
        "\n",
        "model.add(Convolution2D(10, 1, activation='relu')) #8\n",
        "# model.add(BatchNormalization())\n",
        "\n",
        "model.add(Convolution2D(10, 8)) #seeing 7x7 receptive fild\n",
        "model.add(Flatten())\n",
        "model.add(Activation('softmax'))"
      ],
      "execution_count": 18,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "/usr/local/lib/python3.6/dist-packages/ipykernel_launcher.py:6: UserWarning: Update your `Conv2D` call to the Keras 2 API: `Conv2D(10, (3, 3), activation=\"relu\", input_shape=(28, 28, 1...)`\n",
            "  \n",
            "/usr/local/lib/python3.6/dist-packages/ipykernel_launcher.py:9: UserWarning: Update your `Conv2D` call to the Keras 2 API: `Conv2D(16, (3, 3), activation=\"relu\")`\n",
            "  if __name__ == '__main__':\n",
            "/usr/local/lib/python3.6/dist-packages/ipykernel_launcher.py:15: UserWarning: Update your `Conv2D` call to the Keras 2 API: `Conv2D(16, (3, 3), activation=\"relu\")`\n",
            "  from ipykernel import kernelapp as app\n",
            "/usr/local/lib/python3.6/dist-packages/ipykernel_launcher.py:20: UserWarning: Update your `Conv2D` call to the Keras 2 API: `Conv2D(16, (3, 3), activation=\"relu\")`\n"
          ],
          "name": "stderr"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "colab_type": "text",
        "id": "66ZO-Je6GgUc"
      },
      "source": [
        ""
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab_type": "code",
        "outputId": "990259fd-f8d4-4dc4-c764-e615907f8a71",
        "id": "_xDhsgR_GgUe",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 690
        }
      },
      "source": [
        "model.summary()"
      ],
      "execution_count": 19,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "_________________________________________________________________\n",
            "Layer (type)                 Output Shape              Param #   \n",
            "=================================================================\n",
            "conv2d_7 (Conv2D)            (None, 26, 26, 10)        100       \n",
            "_________________________________________________________________\n",
            "batch_normalization_4 (Batch (None, 26, 26, 10)        40        \n",
            "_________________________________________________________________\n",
            "conv2d_8 (Conv2D)            (None, 24, 24, 16)        1456      \n",
            "_________________________________________________________________\n",
            "batch_normalization_5 (Batch (None, 24, 24, 16)        64        \n",
            "_________________________________________________________________\n",
            "dropout_4 (Dropout)          (None, 24, 24, 16)        0         \n",
            "_________________________________________________________________\n",
            "max_pooling2d_2 (MaxPooling2 (None, 12, 12, 16)        0         \n",
            "_________________________________________________________________\n",
            "conv2d_9 (Conv2D)            (None, 10, 10, 16)        2320      \n",
            "_________________________________________________________________\n",
            "batch_normalization_6 (Batch (None, 10, 10, 16)        64        \n",
            "_________________________________________________________________\n",
            "dropout_5 (Dropout)          (None, 10, 10, 16)        0         \n",
            "_________________________________________________________________\n",
            "conv2d_10 (Conv2D)           (None, 8, 8, 16)          2320      \n",
            "_________________________________________________________________\n",
            "dropout_6 (Dropout)          (None, 8, 8, 16)          0         \n",
            "_________________________________________________________________\n",
            "conv2d_11 (Conv2D)           (None, 8, 8, 10)          170       \n",
            "_________________________________________________________________\n",
            "conv2d_12 (Conv2D)           (None, 1, 1, 10)          6410      \n",
            "_________________________________________________________________\n",
            "flatten_2 (Flatten)          (None, 10)                0         \n",
            "_________________________________________________________________\n",
            "activation_2 (Activation)    (None, 10)                0         \n",
            "=================================================================\n",
            "Total params: 12,944\n",
            "Trainable params: 12,860\n",
            "Non-trainable params: 84\n",
            "_________________________________________________________________\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab_type": "code",
        "id": "lSTXSov-GgUs",
        "colab": {}
      },
      "source": [
        "# def lr_schedule(epoch):\n",
        "#     lrate = 0.01\n",
        "#     if epoch > 90:\n",
        "#         lrate = 0.01\n",
        "#     elif epoch > 135:\n",
        "#         lrate = 0.001       \n",
        "#     return lrate\n",
        "# LR_scheduler = LearningRateScheduler(lr_schedule)"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab_type": "code",
        "id": "Em84syMeGgU1",
        "colab": {}
      },
      "source": [
        "model.compile(loss='categorical_crossentropy',\n",
        "              optimizer=SGD(lr=0.01, momentum=0.9, nesterov=True),\n",
        "              metrics=['accuracy'])"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab_type": "code",
        "outputId": "ed2940be-be12-40f2-8d06-75cf9ab57fca",
        "id": "EokEw3M_GgU8",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 437
        }
      },
      "source": [
        "model.fit(X_train, Y_train,  epochs=10,\n",
        "    verbose=1, validation_data=(X_test, Y_test)\n",
        ")"
      ],
      "execution_count": 22,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Train on 60000 samples, validate on 10000 samples\n",
            "Epoch 1/10\n",
            "60000/60000 [==============================] - 14s 231us/step - loss: 0.1419 - acc: 0.9558 - val_loss: 0.0686 - val_acc: 0.9778\n",
            "Epoch 2/10\n",
            "60000/60000 [==============================] - 13s 209us/step - loss: 0.0628 - acc: 0.9806 - val_loss: 0.0383 - val_acc: 0.9875\n",
            "Epoch 3/10\n",
            "60000/60000 [==============================] - 13s 209us/step - loss: 0.0513 - acc: 0.9841 - val_loss: 0.0427 - val_acc: 0.9857\n",
            "Epoch 4/10\n",
            "60000/60000 [==============================] - 13s 210us/step - loss: 0.0458 - acc: 0.9854 - val_loss: 0.0337 - val_acc: 0.9879\n",
            "Epoch 5/10\n",
            "60000/60000 [==============================] - 12s 208us/step - loss: 0.0401 - acc: 0.9872 - val_loss: 0.0372 - val_acc: 0.9891\n",
            "Epoch 6/10\n",
            "60000/60000 [==============================] - 12s 207us/step - loss: 0.0372 - acc: 0.9884 - val_loss: 0.0326 - val_acc: 0.9899\n",
            "Epoch 7/10\n",
            "60000/60000 [==============================] - 14s 234us/step - loss: 0.0351 - acc: 0.9889 - val_loss: 0.0285 - val_acc: 0.9913\n",
            "Epoch 8/10\n",
            "60000/60000 [==============================] - 12s 205us/step - loss: 0.0332 - acc: 0.9890 - val_loss: 0.0298 - val_acc: 0.9901\n",
            "Epoch 9/10\n",
            "60000/60000 [==============================] - 12s 205us/step - loss: 0.0305 - acc: 0.9900 - val_loss: 0.0295 - val_acc: 0.9911\n",
            "Epoch 10/10\n",
            "60000/60000 [==============================] - 12s 206us/step - loss: 0.0301 - acc: 0.9904 - val_loss: 0.0284 - val_acc: 0.9905\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "<keras.callbacks.History at 0x7fda401510f0>"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 22
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "3fI4d9HoKKj2",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 437
        },
        "outputId": "b356b5a4-db64-4b38-fee1-81cecf7717fb"
      },
      "source": [
        "model.fit(X_train, Y_train,  epochs=10,\n",
        "    verbose=1, validation_data=(X_test, Y_test)\n",
        ")"
      ],
      "execution_count": 23,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Train on 60000 samples, validate on 10000 samples\n",
            "Epoch 1/10\n",
            "60000/60000 [==============================] - 12s 205us/step - loss: 0.0277 - acc: 0.9912 - val_loss: 0.0258 - val_acc: 0.9912\n",
            "Epoch 2/10\n",
            "60000/60000 [==============================] - 12s 207us/step - loss: 0.0269 - acc: 0.9913 - val_loss: 0.0284 - val_acc: 0.9917\n",
            "Epoch 3/10\n",
            "60000/60000 [==============================] - 13s 219us/step - loss: 0.0251 - acc: 0.9916 - val_loss: 0.0248 - val_acc: 0.9928\n",
            "Epoch 4/10\n",
            "60000/60000 [==============================] - 13s 222us/step - loss: 0.0246 - acc: 0.9922 - val_loss: 0.0242 - val_acc: 0.9931\n",
            "Epoch 5/10\n",
            "60000/60000 [==============================] - 12s 205us/step - loss: 0.0239 - acc: 0.9923 - val_loss: 0.0253 - val_acc: 0.9924\n",
            "Epoch 6/10\n",
            "60000/60000 [==============================] - 12s 205us/step - loss: 0.0243 - acc: 0.9921 - val_loss: 0.0233 - val_acc: 0.9929\n",
            "Epoch 7/10\n",
            "60000/60000 [==============================] - 12s 205us/step - loss: 0.0226 - acc: 0.9923 - val_loss: 0.0364 - val_acc: 0.9897\n",
            "Epoch 8/10\n",
            "60000/60000 [==============================] - 12s 204us/step - loss: 0.0230 - acc: 0.9929 - val_loss: 0.0233 - val_acc: 0.9926\n",
            "Epoch 9/10\n",
            "60000/60000 [==============================] - 12s 207us/step - loss: 0.0223 - acc: 0.9923 - val_loss: 0.0267 - val_acc: 0.9916\n",
            "Epoch 10/10\n",
            "60000/60000 [==============================] - 13s 222us/step - loss: 0.0215 - acc: 0.9928 - val_loss: 0.0259 - val_acc: 0.9917\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "<keras.callbacks.History at 0x7fda44060e48>"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 23
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "EXhs0dkKKLiF",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        ""
      ],
      "execution_count": 0,
      "outputs": []
    }
  ]
}