mfcc.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Using TensorFlow backend.\n"
     ]
    }
   ],
   "source": [
    "%load_ext autoreload\n",
    "%autoreload 2\n",
    "#tensorboard doesn't work with keras model callbacks! needs tensorflow.keras\n",
    "from preprocess import *\n",
    "from tensorflow import keras\n",
    "from tensorflow.keras.models import Sequential\n",
    "#from tensorflow.keras.layers import Dense, Dropout, Flatten, Conv2D, MaxPooling2D\n",
    "from tensorflow.keras.layers import (Dense, Dropout, Flatten, Convolution2D, GlobalAveragePooling2D, BatchNormalization, Flatten,\n",
    "                          GlobalMaxPool2D, MaxPool2D, concatenate, Activation, Input)\n",
    "from tensorflow.keras import losses, models, optimizers\n",
    "from tensorflow.keras.activations import relu, softmax\n",
    "from tensorflow.keras.utils import to_categorical\n",
    "from tensorflow.keras.callbacks import TensorBoard\n",
    "import time  \n",
    "import matplotlib\n",
    "import matplotlib.pyplot as plt\n",
    "%matplotlib inline"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(['shh', 'silence', 'ttt'], array([0, 1, 2]), array([[1., 0., 0.],\n",
       "        [0., 1., 0.],\n",
       "        [0., 0., 1.]], dtype=float32))"
      ]
     },
     "execution_count": 3,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "#get_labels returns the label names, their indices and 1 hot encodings\n",
    "labels = get_labels('./data')  \n",
    "num_classes = len(labels[0])\n",
    "labels"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "16384\n"
     ]
    }
   ],
   "source": [
    "#length of longest wav data\n",
    "#all data will be padded to this length\n",
    "max_sample_length = max_sample_len()\n",
    "print(max_sample_length)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Saving vectors of label - 'shh': 100%|██████████| 260/260 [00:01<00:00, 244.78it/s]\n",
      "Saving vectors of label - 'silence': 100%|██████████| 214/214 [00:00<00:00, 214.73it/s]\n",
      "Saving vectors of label - 'ttt': 100%|██████████| 229/229 [00:00<00:00, 236.96it/s]\n"
     ]
    }
   ],
   "source": [
    "# Save data to array file first\n",
    "n_mfcc = 40\n",
    "save_data_to_array(n_mfcc=n_mfcc)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "# # Loading train set and test set\n",
    "X_train, X_test, y_train, y_test = get_train_test()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(421, 40, 33)\n",
      "(421,)\n",
      "(282, 40, 33)\n",
      "(282,)\n"
     ]
    }
   ],
   "source": [
    "print(X_train.shape)\n",
    "print(y_train.shape)\n",
    "print(X_test.shape)\n",
    "print(y_test.shape)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "X_train = X_train.reshape(X_train.shape[0], X_train.shape[1], X_train.shape[2], 1)\n",
    "X_test = X_test.reshape(X_test.shape[0], X_test.shape[1], X_test.shape[2], 1)\n",
    "\n",
    "y_train_hot = to_categorical(y_train)\n",
    "y_test_hot = to_categorical(y_test)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(421, 40, 33, 1)\n",
      "(421,)\n",
      "(282, 40, 33, 1)\n",
      "(282,)\n",
      "(421, 3)\n"
     ]
    }
   ],
   "source": [
    "print(X_train.shape)\n",
    "print(y_train.shape)\n",
    "print(X_test.shape)\n",
    "print(y_test.shape)\n",
    "print(y_train_hot.shape)#2nd dim is the number of labels"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "#kaggle standardizes data, TODO try normalizing like in the 1dconv\n",
    "mean = np.mean(X_train, axis=0)\n",
    "std = np.std(X_train, axis=0)\n",
    "\n",
    "X_train = (X_train - mean)/std\n",
    "X_test = (X_test - mean)/std"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [],
   "source": [
    "#modified kaggle tut, https://www.kaggle.com/fizzbuzz/beginner-s-guide-to-audio-data\n",
    "def get_2d_conv_model(training_shape):\n",
    "    \n",
    "    inp = Input(shape=(training_shape[1], training_shape[2], 1))\n",
    "    x = Convolution2D(16, (4,10), padding=\"same\")(inp)\n",
    "    x = BatchNormalization()(x)\n",
    "    x = Activation(\"relu\")(x)\n",
    "    x = MaxPool2D()(x)\n",
    "\n",
    "    \n",
    "    x = Flatten()(x)\n",
    "    x = Dense(64)(x)\n",
    "    x = BatchNormalization()(x)\n",
    "    x = Activation(\"relu\")(x)\n",
    "    out = Dense(num_classes, activation=softmax)(x)\n",
    "\n",
    "    model = models.Model(inputs=inp, outputs=out)\n",
    "    opt = optimizers.Adam(0.001)#learning rate\n",
    "\n",
    "    model.compile(optimizer=opt, loss=losses.categorical_crossentropy, metrics=['acc'])\n",
    "    return model"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Building The Model Then Training it"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<tensorflow.python.keras.callbacks.TensorBoard at 0x7ff4d4c7e390>"
      ]
     },
     "execution_count": 11,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "NAME = \"mfccmodel{}\".format(int(time.time()))\n",
    "tensorboard_cb = TensorBoard(log_dir=\"logs/{}\".format(NAME), histogram_freq=1)\n",
    "%load_ext tensorboard\n",
    "tensorboard_cb"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Train on 421 samples, validate on 282 samples\n",
      "Epoch 1/8\n",
      "421/421 [==============================] - 1s 2ms/sample - loss: 0.5878 - acc: 0.7601 - val_loss: 0.9476 - val_acc: 0.7021\n",
      "Epoch 2/8\n",
      "421/421 [==============================] - 0s 570us/sample - loss: 0.0751 - acc: 0.9881 - val_loss: 0.9647 - val_acc: 0.7199\n",
      "Epoch 3/8\n",
      "421/421 [==============================] - 0s 611us/sample - loss: 0.0456 - acc: 0.9952 - val_loss: 0.7799 - val_acc: 0.7270\n",
      "Epoch 4/8\n",
      "421/421 [==============================] - 0s 571us/sample - loss: 0.0318 - acc: 0.9976 - val_loss: 0.5523 - val_acc: 0.7447\n",
      "Epoch 5/8\n",
      "421/421 [==============================] - 0s 581us/sample - loss: 0.0346 - acc: 0.9929 - val_loss: 0.3936 - val_acc: 0.8014\n",
      "Epoch 6/8\n",
      "421/421 [==============================] - 0s 580us/sample - loss: 0.0192 - acc: 0.9976 - val_loss: 0.3036 - val_acc: 0.8546\n",
      "Epoch 7/8\n",
      "421/421 [==============================] - 0s 567us/sample - loss: 0.0157 - acc: 1.0000 - val_loss: 0.2370 - val_acc: 0.9291\n",
      "Epoch 8/8\n",
      "421/421 [==============================] - 0s 580us/sample - loss: 0.0151 - acc: 1.0000 - val_loss: 0.1982 - val_acc: 0.9681\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "<tensorflow.python.keras.callbacks.History at 0x7ff4b0616320>"
      ]
     },
     "execution_count": 15,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "model = get_2d_conv_model(X_train.shape)\n",
    "model.fit(X_train, y_train_hot, batch_size=100, epochs=8, verbose=True, validation_data=(X_test, y_test_hot), callbacks=[tensorboard_cb])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'mfccmodel1584667400'"
      ]
     },
     "execution_count": 16,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "#save the model and stats if you want to use it from the udp listener\n",
    "save_model(model, NAME)\n",
    "np.save(\"stats/{}.npy\".format(NAME), [std, mean]) \n",
    "NAME"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [],
   "source": [
    "%tensorboard --logdir logs/ --bind_all"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "shh\n"
     ]
    }
   ],
   "source": [
    "def prep_data(filepath, shape):\n",
    "    sample = wav2mfcc(filepath, n_mfcc=n_mfcc, max_sample_length=max_sample_length)\n",
    "    sample = sample.reshape(1, shape[1], shape[2], 1)\n",
    "    sample = (sample - mean)/std\n",
    "    return sample\n",
    "\n",
    "# Predicts one sample\n",
    "def predict(filepath, model):\n",
    "    sample_reshaped = prep_data(filepath, model.input_shape)\n",
    "    return get_labels()[0][\n",
    "            np.argmax(model.predict(sample_reshaped))\n",
    "    ]\n",
    "\n",
    "print(predict('./data/shh/W9M3W3YR15.wav', model=model))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.9"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}