lab-11-5-mnist-cnn-best-keras-eager.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "from __future__ import absolute_import\n",
    "from __future__ import division\n",
    "from __future__ import print_function"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Importing Libraries"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "1.12.0\n",
      "2.1.6-tf\n"
     ]
    }
   ],
   "source": [
    "import tensorflow as tf\n",
    "from tensorflow import keras\n",
    "from tensorflow.keras.utils import to_categorical\n",
    "import numpy as np\n",
    "import matplotlib.pyplot as plt\n",
    "import os\n",
    "from scipy import ndimage\n",
    "\n",
    "print(tf.__version__)\n",
    "print(keras.__version__)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Enable Eager Mode"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "tf.enable_eager_execution()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Hyper Parameters"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "learning_rate = 0.001\n",
    "training_epochs = 15\n",
    "batch_size = 100\n",
    "\n",
    "tf.set_random_seed(777)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Creating a Checkpoint Directory"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "cur_dir = os.getcwd()\n",
    "ckpt_dir_name = 'checkpoints'\n",
    "model_dir_name = 'minst_cnn_best'\n",
    "\n",
    "checkpoint_dir = os.path.join(cur_dir, ckpt_dir_name, model_dir_name)\n",
    "os.makedirs(checkpoint_dir, exist_ok=True)\n",
    "\n",
    "checkpoint_prefix = os.path.join(checkpoint_dir, model_dir_name)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## MNIST/Fashion MNIST Data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "## MNIST Dataset #########################################################\n",
    "mnist = keras.datasets.mnist\n",
    "class_names = ['0', '1', '2', '3', '4', '5', '6', '7', '8', '9']\n",
    "##########################################################################\n",
    "\n",
    "## Fashion MNIST Dataset #################################################\n",
    "#mnist = keras.datasets.fashion_mnist\n",
    "#class_names = ['T-shirt/top', 'Trouser', 'Pullover', 'Dress', 'Coat', 'Sandal', 'Shirt', 'Sneaker', 'Bag', 'Ankle boot']\n",
    "##########################################################################"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Data Augmentation"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "def data_augmentation(images, labels):\n",
    "    aug_images = []\n",
    "    aug_labels = []    \n",
    "    \n",
    "    for x, y in zip(images, labels):        \n",
    "        aug_images.append(x)\n",
    "        aug_labels.append(y)\n",
    "        \n",
    "        bg_value = np.median(x)\n",
    "        for _ in range(4):\n",
    "            angle = np.random.randint(-15, 15, 1)\n",
    "            rot_img = ndimage.rotate(x, angle, reshape=False, cval=bg_value)\n",
    "            \n",
    "            shift = np.random.randint(-2, 2, 2)\n",
    "            shift_img = ndimage.shift(rot_img, shift, cval=bg_value)            \n",
    "            \n",
    "            aug_images.append(shift_img)\n",
    "            aug_labels.append(y)\n",
    "    aug_images = np.array(aug_images)\n",
    "    aug_labels = np.array(aug_labels)\n",
    "    return aug_images, aug_labels"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Datasets"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "(train_images, train_labels), (test_images, test_labels) = mnist.load_data()\n",
    "\n",
    "train_images, train_labels = data_augmentation(train_images, train_labels)\n",
    "    \n",
    "train_images = train_images.astype(np.float32) / 255.\n",
    "test_images = test_images.astype(np.float32) / 255.\n",
    "train_images = np.expand_dims(train_images, axis=-1)\n",
    "test_images = np.expand_dims(test_images, axis=-1)\n",
    "    \n",
    "train_labels = to_categorical(train_labels, 10)\n",
    "test_labels = to_categorical(test_labels, 10)    \n",
    "    \n",
    "train_dataset = tf.data.Dataset.from_tensor_slices((train_images, train_labels)).shuffle(\n",
    "                buffer_size=500000).batch(batch_size)\n",
    "test_dataset = tf.data.Dataset.from_tensor_slices((test_images, test_labels)).batch(batch_size)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Model Class"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "class ConvBNRelu(tf.keras.Model):\n",
    "    def __init__(self, filters, kernel_size=3, strides=1, padding='SAME'):\n",
    "        super(ConvBNRelu, self).__init__()\n",
    "        self.conv = keras.layers.Conv2D(filters=filters, kernel_size=kernel_size, strides=strides, \n",
    "                                        padding=padding, kernel_initializer='glorot_normal')\n",
    "        self.batchnorm = tf.keras.layers.BatchNormalization()\n",
    "    def call(self, inputs, training=False):\n",
    "        layer = self.conv(inputs)\n",
    "        layer = self.batchnorm(layer)\n",
    "        layer = tf.nn.relu(layer)\n",
    "        return layer        "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [],
   "source": [
    "class DenseBNRelu(tf.keras.Model):\n",
    "    def __init__(self, units):\n",
    "        super(DenseBNRelu, self).__init__()\n",
    "        self.dense = keras.layers.Dense(units=units, kernel_initializer='glorot_normal')\n",
    "        self.batchnorm = tf.keras.layers.BatchNormalization()\n",
    "    def call(self, inputs, training=False):\n",
    "        layer = self.dense(inputs)\n",
    "        layer = self.batchnorm(layer)\n",
    "        layer = tf.nn.relu(layer)\n",
    "        return layer"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [],
   "source": [
    "class MNISTModel(tf.keras.Model):\n",
    "    def __init__(self):\n",
    "        super(MNISTModel, self).__init__()\n",
    "        self.conv1 = ConvBNRelu(filters=32, kernel_size=[3, 3], padding='SAME')        \n",
    "        self.pool1 = keras.layers.MaxPool2D(padding='SAME')\n",
    "        self.conv2 = ConvBNRelu(filters=64, kernel_size=[3, 3], padding='SAME')\n",
    "        self.pool2 = keras.layers.MaxPool2D(padding='SAME')\n",
    "        self.conv3 = ConvBNRelu(filters=128, kernel_size=[3, 3], padding='SAME')\n",
    "        self.pool3 = keras.layers.MaxPool2D(padding='SAME')\n",
    "        self.pool3_flat = keras.layers.Flatten()\n",
    "        self.dense4 = DenseBNRelu(units=256)\n",
    "        self.drop4 = keras.layers.Dropout(rate=0.4)\n",
    "        self.dense5 = keras.layers.Dense(units=10, kernel_initializer='glorot_normal')\n",
    "    def call(self, inputs, training=False):\n",
    "        net = self.conv1(inputs)        \n",
    "        net = self.pool1(net)\n",
    "        net = self.conv2(net)\n",
    "        net = self.pool2(net)\n",
    "        net = self.conv3(net)\n",
    "        net = self.pool3(net)\n",
    "        net = self.pool3_flat(net)\n",
    "        net = self.dense4(net)\n",
    "        net = self.drop4(net)\n",
    "        net = self.dense5(net)\n",
    "        return net"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [],
   "source": [
    "models = []\n",
    "num_models = 5\n",
    "for m in range(num_models):\n",
    "    models.append(MNISTModel())"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Loss Function"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [],
   "source": [
    "def loss_fn(model, images, labels):\n",
    "    logits = model(images, training=True)\n",
    "    loss = tf.reduce_mean(tf.nn.softmax_cross_entropy_with_logits_v2(\n",
    "            logits=logits, labels=labels))    \n",
    "    return loss   "
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Calculating Gradient"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [],
   "source": [
    "def grad(model, images, labels):\n",
    "    with tf.GradientTape() as tape:\n",
    "        loss = loss_fn(model, images, labels)\n",
    "    return tape.gradient(loss, model.variables)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Caculating Model's Accuracy"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [],
   "source": [
    "def evaluate(models, images, labels):\n",
    "    predictions = np.zeros_like(labels)\n",
    "    for model in models:\n",
    "        logits = model(images, training=False)\n",
    "        predictions += logits\n",
    "    correct_prediction = tf.equal(tf.argmax(predictions, 1), tf.argmax(labels, 1))\n",
    "    accuracy = tf.reduce_mean(tf.cast(correct_prediction, tf.float32))\n",
    "    return accuracy"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Optimizer"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [],
   "source": [
    "global_step = tf.train.get_or_create_global_step()\n",
    "lr_decay = tf.train.exponential_decay(learning_rate, global_step, \n",
    "                                      train_images.shape[0]/batch_size*num_models*5, 0.5, staircase=True)\n",
    "optimizer = tf.train.AdamOptimizer(learning_rate=lr_decay)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Creating Checkpoints"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [],
   "source": [
    "checkpoints = []\n",
    "for m in range(num_models):\n",
    "    checkpoints.append(tf.train.Checkpoint(cnn=models[m]))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Training"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {
    "scrolled": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Learning started. It takes sometime.\n",
      "Epoch: 1 loss = 0.05254757 train accuracy =  0.9863 test accuracy =  0.9947\n",
      "Epoch: 2 loss = 0.01288698 train accuracy =  0.9976 test accuracy =  0.9960\n",
      "Epoch: 3 loss = 0.00692925 train accuracy =  0.9991 test accuracy =  0.9965\n",
      "Epoch: 4 loss = 0.00416141 train accuracy =  0.9995 test accuracy =  0.9959\n",
      "Epoch: 5 loss = 0.00290176 train accuracy =  0.9998 test accuracy =  0.9962\n",
      "Epoch: 6 loss = 0.00146718 train accuracy =  0.9999 test accuracy =  0.9963\n",
      "Epoch: 7 loss = 0.00065968 train accuracy =  0.9999 test accuracy =  0.9961\n",
      "Epoch: 8 loss = 0.00049940 train accuracy =  1.0000 test accuracy =  0.9965\n",
      "Epoch: 9 loss = 0.00040568 train accuracy =  1.0000 test accuracy =  0.9961\n",
      "Epoch: 10 loss = 0.00036997 train accuracy =  1.0000 test accuracy =  0.9958\n",
      "Epoch: 11 loss = 0.00018087 train accuracy =  1.0000 test accuracy =  0.9968\n",
      "Epoch: 12 loss = 0.00006729 train accuracy =  1.0000 test accuracy =  0.9964\n",
      "Epoch: 13 loss = 0.00005760 train accuracy =  1.0000 test accuracy =  0.9962\n",
      "Epoch: 14 loss = 0.00005532 train accuracy =  1.0000 test accuracy =  0.9961\n",
      "Epoch: 15 loss = 0.00004803 train accuracy =  1.0000 test accuracy =  0.9963\n",
      "Learning Finished!\n"
     ]
    }
   ],
   "source": [
    "# train my model\n",
    "print('Learning started. It takes sometime.')\n",
    "for epoch in range(training_epochs):\n",
    "    avg_loss = 0.\n",
    "    avg_train_acc = 0.\n",
    "    avg_test_acc = 0.\n",
    "    train_step = 0\n",
    "    test_step = 0    \n",
    "    \n",
    "    for images, labels in train_dataset:\n",
    "        for model in models:\n",
    "            grads = grad(model, images, labels)                \n",
    "            optimizer.apply_gradients(zip(grads, model.variables), global_step=global_step)            \n",
    "            loss = loss_fn(model, images, labels)\n",
    "            avg_loss += loss / num_models\n",
    "        acc = evaluate(models, images, labels)\n",
    "        avg_train_acc += acc\n",
    "        train_step += 1\n",
    "    avg_loss = avg_loss / train_step\n",
    "    avg_train_acc = avg_train_acc / train_step\n",
    "    \n",
    "    for images, labels in test_dataset:        \n",
    "        acc = evaluate(models, images, labels)        \n",
    "        avg_test_acc += acc\n",
    "        test_step += 1    \n",
    "    avg_test_acc = avg_test_acc / test_step    \n",
    "\n",
    "    print('Epoch:', '{}'.format(epoch + 1), 'loss =', '{:.8f}'.format(avg_loss), \n",
    "          'train accuracy = ', '{:.4f}'.format(avg_train_acc), \n",
    "          'test accuracy = ', '{:.4f}'.format(avg_test_acc))\n",
    "    \n",
    "    \n",
    "    for idx, checkpoint in enumerate(checkpoints):\n",
    "        checkpoint.save(file_prefix=checkpoint_prefix+'-{}'.format(idx))\n",
    "\n",
    "print('Learning Finished!')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "_________________________________________________________________\n",
      "Layer (type)                 Output Shape              Param #   \n",
      "=================================================================\n",
      "conv_bn_relu (ConvBNRelu)    multiple                  448       \n",
      "_________________________________________________________________\n",
      "max_pooling2d (MaxPooling2D) multiple                  0         \n",
      "_________________________________________________________________\n",
      "conv_bn_relu_1 (ConvBNRelu)  multiple                  18752     \n",
      "_________________________________________________________________\n",
      "max_pooling2d_1 (MaxPooling2 multiple                  0         \n",
      "_________________________________________________________________\n",
      "conv_bn_relu_2 (ConvBNRelu)  multiple                  74368     \n",
      "_________________________________________________________________\n",
      "max_pooling2d_2 (MaxPooling2 multiple                  0         \n",
      "_________________________________________________________________\n",
      "flatten (Flatten)            multiple                  0         \n",
      "_________________________________________________________________\n",
      "dense_bn_relu (DenseBNRelu)  multiple                  525568    \n",
      "_________________________________________________________________\n",
      "dropout (Dropout)            multiple                  0         \n",
      "_________________________________________________________________\n",
      "dense_1 (Dense)              multiple                  2570      \n",
      "=================================================================\n",
      "Total params: 621,706\n",
      "Trainable params: 620,746\n",
      "Non-trainable params: 960\n",
      "_________________________________________________________________\n"
     ]
    }
   ],
   "source": [
    "models[0].summary()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.5.5"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}