uppercase.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Captal Letters Recognition"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "import os\n",
    "\n",
    "import nltk\n",
    "nltk.download('semcor')\n",
    "nltk.download('punkt')\n",
    "nltk.download('perluniprops')\n",
    "from nltk.corpus import semcor\n",
    "from nltk.tokenize.moses import MosesDetokenizer\n",
    "from keras.layers import Embedding, LSTM, GRU, Conv1D, Dense, Bidirectional\n",
    "from keras.models import Sequential\n",
    "from keras.preprocessing.sequence import pad_sequences\n",
    "from keras.utils.np_utils import to_categorical\n",
    "import numpy as np\n",
    "\n",
    "# detokenization: turnig tokens back into sentences\n",
    "MDETOK = MosesDetokenizer()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Load Brown / Semcor Corpus, select sentences with lots of capitalized words"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def filter_titled_sents(sentences, u_case_min_nb=3):\n",
    "    '''select only those sentences with at least u_case_min_nb number of words beginning with capital letter'''\n",
    "    filtered_sents = []\n",
    "    for sent in sentences:\n",
    "        nb_titles = 0\n",
    "        for token in sent:\n",
    "            if token.istitle():\n",
    "                nb_titles += 1\n",
    "        if nb_titles >= u_case_min_nb:\n",
    "            filtered_sents.append(sent)\n",
    "    return filtered_sents"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false,
    "scrolled": false
   },
   "outputs": [],
   "source": [
    "sents = semcor.sents()  # loading tokenized sentences from Semcor corpus\n",
    "print(\"number of sentences: %s\" % len(sents))\n",
    "sents = filter_titled_sents(sents, u_case_min_nb=3)\n",
    "print(\"number of sentences after filtering: %s\" % len(sents))\n",
    "print(\"sample sentence:\")\n",
    "print(sents[10000])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "# we are going to keep only shorter sentences\n",
    "MAX_SEQUENCE_LENGTH = 35\n",
    "sents = [sent for sent in sents if len(sent) <= MAX_SEQUENCE_LENGTH]\n",
    "print(\"number of sentences after filtering: %s\" % len(sents))\n",
    "\n",
    "# and clean the dataset a bit removing tokens like `` first\n",
    "filter_out_toks = {'``', '\\'\\''}\n",
    "sents =[[word for word in sent if word not in filter_out_toks] for sent in sents]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "collapsed": true
   },
   "source": [
    "## Word Level Models"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "In this task we are going to use pre-trained \"GloVe\" word embeddings that can be downloaded from https://nlp.stanford.edu/data/glove.6B.zip\n",
    "\n",
    "You can have a look at the visualization of pre-trained Word2Vec word embeddings, words with similar meaning are clustered together: http://projector.tensorflow.org/\n",
    "\n",
    "They used PCA or T-SNE to reduce the dimensionality of the vectors."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false,
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "# make sentences lowercase\n",
    "sents_lower =[[word.lower() for word in sent] for sent in sents]\n",
    "# annotate words in sentences based on their first letter case\n",
    "capitalization_sent_tags = [[word.istitle() for word in sent] for sent in sents]\n",
    "print(\"tokens anotated based on their first letter case:\")\n",
    "print(zip(sents_lower[-500], capitalization_sent_tags[-500]))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "# create a vocabulary of all words in our dataset\n",
    "words = set([])\n",
    "for sent in sents_lower:\n",
    "    words.update(sent)\n",
    "print(\"vocabulary size: %s\" % len(words))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "# create a dictionary, an index for each word\n",
    "dictionary = dict()\n",
    "for i, word in enumerate(words):\n",
    "    dictionary[word] = i\n",
    "print(\"index of `hello`: %s\" % dictionary[\"hello\"])\n",
    "\n",
    "# a mapping for indexes back into words\n",
    "idx2word = {}\n",
    "for word, i in dictionary.items():\n",
    "    idx2word[i] = word"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false,
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "# convert sentences into sequences of word indexes\n",
    "sequences = [[dictionary[word] for word in sent] for sent in sents_lower]\n",
    "print(\"sequence of word indexes for each sentence: %s\" % sequences[0][:10])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "# pad sequences with zeros to make them same length: we need it for vectorized computations\n",
    "data = pad_sequences(sequences, maxlen=MAX_SEQUENCE_LENGTH, padding='post')\n",
    "data[0]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "# labels will be converted to categories: first indicates the probability of a capitalized word, second a lowercased word \n",
    "labels = pad_sequences(capitalization_sent_tags, maxlen=MAX_SEQUENCE_LENGTH, padding='post')\n",
    "labels = to_categorical(labels)\n",
    "labels[0][:10]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false,
    "scrolled": false
   },
   "outputs": [],
   "source": [
    "# our dataset will be split into a traing part and a validation part,\n",
    "# where we measure our model's performance during training,\n",
    "# this can be done automatically during the Keras model training\n",
    "\n",
    "# we will further keep a testing part to evaluate predictions \n",
    "TEST_SPLIT = .1\n",
    "nb_test_samples = int(TEST_SPLIT * data.shape[0])\n",
    "print(\"number of test samples: %s\" % nb_test_samples)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# we want to shuffle the data a bit to split the dataset uniformly\n",
    "indices = np.arange(data.shape[0])\n",
    "np.random.shuffle(indices)\n",
    "data = data[indices]\n",
    "labels = labels[indices]\n",
    "\n",
    "x_train = data[:-nb_test_samples]\n",
    "y_train = labels[:-nb_test_samples]\n",
    "x_test = data[-nb_test_samples:]\n",
    "y_test = labels[-nb_test_samples:]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false,
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "# let's create a dictionary of embeddings from each word embedding vector in the pre-trained GloVe embeddings file\n",
    "GLOVE_DIR = \"../glove.6B/\"\n",
    "embeddings_index = {}\n",
    "f = open(os.path.join(GLOVE_DIR, 'glove.6B.100d.txt'))\n",
    "for line in f:\n",
    "    values = line.split()\n",
    "    word = values[0]\n",
    "    coefs = np.asarray(values[1:], dtype='float32')\n",
    "    embeddings_index[word] = coefs\n",
    "f.close()\n",
    "\n",
    "print('Found %s word vectors.' % len(embeddings_index))\n",
    "print(\"embedding for the word `word`:\")\n",
    "print(embeddings_index.get(\"word\"))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# let's try to extract the GloVe embeddings for each word from our dataset vocabulary\n",
    "EMBEDDING_DIM = 100\n",
    "embedding_matrix = np.zeros((len(dictionary) + 1, EMBEDDING_DIM))\n",
    "for word, i in dictionary.items():\n",
    "    embedding_vector = embeddings_index.get(word)\n",
    "    if embedding_vector is not None:\n",
    "        # words not found in embedding index will be all-zeros.\n",
    "        embedding_matrix[i] = embedding_vector\n",
    "del embeddings_index"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Checking how many words have no pre-trained GloVe word embeddings:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false,
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "oov_percentage = 100. * np.count_nonzero(np.all(embedding_matrix == 0, axis=1)) / len(dictionary)  # OOV portion\n",
    "print(\"percentage of words out of vocabulary: %s percent\" % oov_percentage)\n",
    "outta_vocab_idxs = set(np.where(np.all(embedding_matrix == 0, axis=1))[0])\n",
    "outta_vocab_words = [word for word, i in dictionary.items() if i in outta_vocab_idxs]\n",
    "print(\"examples of words without pre-trained GloVe embeddings:\")\n",
    "print(outta_vocab_words[:15])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def print_predictions(x_test, y_pred, idx2word):\n",
    "    \"\"\"\n",
    "    print and detokenize the results of our model's predictions\n",
    "    y_pred has two output channels, first one giving the probability of the lowercased word,\n",
    "    the second channel the probability of the capitalized word.\n",
    "    x_test are word indices of lowercased sentences\n",
    "    \"\"\"\n",
    "    for seq, preds in zip(x_test, y_pred):\n",
    "        sentence = []\n",
    "        for word_id, pred in zip(seq, preds):\n",
    "            if pred[0] > pred[1]:\n",
    "                sentence.append(idx2word[word_id])\n",
    "            else:\n",
    "                sentence.append(idx2word[word_id].capitalize())\n",
    "        print(MDETOK.detokenize(sentence, return_str=True).strip() + \"\\n\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Simple baseline: A single fully connected layer\n",
    "A Dense/fully connected layer won't remember the order of the words, but we might get some words right.\n",
    "\n",
    "Embeddings layer will be using the weights from the pre-trained GloVe vectors. We don't want to change them so we set `trainable=False`  \n",
    "\n",
    "We use softmax acivation, all outputs (2) of our model will sum up to one.\n",
    "\n",
    "20 % of the dataset will be used for validation.\n",
    "\n",
    "Number of epochs tells us how many times we want our model to look at all the traing samples in our traing set.\n",
    "\n",
    "If we set a number of epochs too high and our model is very complex, it will start overfitting the training data and the validation error will start rising."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false,
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "BATCH_SIZE = 32\n",
    "model = Sequential()\n",
    "model.add(Embedding(input_dim=len(dictionary) + 1, output_dim=EMBEDDING_DIM, weights=[embedding_matrix],\n",
    "                    input_length=MAX_SEQUENCE_LENGTH, trainable=False))\n",
    "model.add(Dense(2, activation='softmax'))\n",
    "model.compile(loss='categorical_crossentropy', optimizer='adam', metrics=['accuracy'])\n",
    "model.fit(x_train, y_train, validation_split=0.2, epochs=30, batch_size=BATCH_SIZE)\n",
    "score, acc = model.evaluate(x_test, y_test, batch_size=BATCH_SIZE)\n",
    "print('Test accuracy:', acc)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false,
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "# let's see what our model predicts\n",
    "y_pred = model.predict(x_test)\n",
    "print_predictions(x_test[:50], y_pred[:50], idx2word)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### RNN Model\n",
    "add a Bidirectional layer of LSTM cells or GRU cells after the Embedding layer: \n",
    "\n",
    "`Bidirectional(cells, input_shape=(MAX_SEQUENCE_LENGTH, EMBEDDING_DIM))`\n",
    "\n",
    "you can try experimenting with different `cells` (LSTM, RNN,...):\n",
    "\n",
    "Try `LSTM(HIDDEN_SIZE_LSTM, return_sequences=...)` or `GRU(HIDDEN_SIZE_LSTM, return_sequences=...)`, can you see any difference in training or accuracy when using different cells?\n",
    "\n",
    "if we set `return_sequences` True, we will get the output of the cells in each timestep of the sequence, that's what we want:)\n",
    "\n",
    "you can change the complexity of the model by setting `HIDDEN_SIZE_LSTM` which changes the number of `units`.\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "HIDDEN_SIZE_LSTM = 100\n",
    "BATCH_SIZE = 32\n",
    "model = Sequential()\n",
    "model.add(Embedding(input_dim=len(dictionary) + 1, output_dim=EMBEDDING_DIM, weights=[embedding_matrix],\n",
    "                    input_length=MAX_SEQUENCE_LENGTH, trainable=False))\n",
    "# TODO: add a recurrent layer or two\n",
    "model.add(Dense(2, activation='softmax'))\n",
    "model.compile(loss='categorical_crossentropy', optimizer='adam', metrics=['accuracy'])\n",
    "model.fit(x_train, y_train, validation_split=0.2, epochs=10, batch_size=BATCH_SIZE)\n",
    "score, acc = model.evaluate(x_test, y_test, batch_size=BATCH_SIZE)\n",
    "print('Test accuracy:', acc)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false,
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "# let's see what our model predicts\n",
    "y_pred = model.predict(x_test)\n",
    "print_predictions(x_test[:50], y_pred[:50], idx2word)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### CNN Model\n",
    "\n",
    "Use Conv1d instead of RNN layers:\n",
    "\n",
    "`Conv1D(filters=EMBEDDING_DIM, kernel_size=..., activation='relu', padding=...)`\n",
    "\n",
    "We need to preserve the sequence legth, when going from one layer to another, so we set padding='same'\n",
    "\n",
    "kernel_size (window size) is a parameter setting the scope of view for our convolutional filter, how many words we look at.\n",
    "\n",
    "We want a filter for each index of our word embedding vector\n",
    "\n",
    "Try setting padding='causal'. This will make our window (kernel) wider, but we will look only at every other word in a sequence inside the window. This is also called a dilated convolution.\n",
    "\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "BATCH_SIZE = 32\n",
    "WINDOW_SIZES = [3, 3]\n",
    "model = Sequential()\n",
    "model.add(Embedding(input_dim=len(dictionary) + 1, output_dim=EMBEDDING_DIM, weights=[embedding_matrix],\n",
    "                    input_length=MAX_SEQUENCE_LENGTH, trainable=False))\n",
    "# TODO: Try experimenting...adding some 1D Convolutinal layers here.\n",
    "model.add(Dense(2, activation='softmax'))\n",
    "model.compile(loss='categorical_crossentropy', optimizer='adam', metrics=['accuracy'])\n",
    "model.fit(x_train, y_train, validation_split=0.2, epochs=10, batch_size=BATCH_SIZE)\n",
    "score, acc = model.evaluate(x_test, y_test, batch_size=BATCH_SIZE)\n",
    "print('Test accuracy:', acc)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false,
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "# let's see what our model predicts\n",
    "y_pred = model.predict(x_test)\n",
    "print_predictions(x_test[:50], y_pred[:50], idx2word)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "collapsed": true
   },
   "source": [
    "## Character Level Models\n",
    "\n",
    "We are now going to work with models that look at each letter of the text, deciding whether it should be big or small.\n",
    "The strategy can be the same as with the words, but now we have no pre-trained vector embeddings, so we initialize them randomly and allow their training by setting trainable=True."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "HIDDEN_SIZE_LSTM = 100\n",
    "EMBEDDING_DIM = 20\n",
    "BATCH_SIZE = 32\n",
    "WIN_SIZE = 30"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "# our dataset will be split into a traing part and a validation part, where we measure our model's performance\n",
    "VALIDATION_SPLIT = .2\n",
    "nb_validation_samples = int(VALIDATION_SPLIT * len(sents))\n",
    "indices = np.arange(len(sents))\n",
    "np.random.shuffle(indices)\n",
    "train_indices = indices[:-nb_validation_samples]\n",
    "val_indices = indices[-nb_validation_samples:]\n",
    "\n",
    "\n",
    "whole_sents_train = [MDETOK.detokenize(sents[index], return_str=True) for index in train_indices]\n",
    "whole_sents_val = [MDETOK.detokenize(sents[index], return_str=True) for index in val_indices]\n",
    "whole_sents_lower_train = [sent.lower() for sent in whole_sents_train]\n",
    "whole_sents_lower_val = [sent.lower() for sent in whole_sents_val]\n",
    "\n",
    "whole_text_train = \" \".join(whole_sents_train)\n",
    "whole_text_val = \" \".join(whole_sents_val)\n",
    "whole_text_lower_train = whole_text_train.lower()\n",
    "whole_text_lower_val = whole_text_val.lower()\n",
    "\n",
    "# create a vocabulary\n",
    "chars_vocab = set(whole_text_lower_train).union(whole_text_lower_val)\n",
    "print(\"vocabulary size: %s\" % len(chars_vocab))\n",
    "\n",
    "# create a dictionary, an index for each character\n",
    "chars_dictionary = dict()\n",
    "for i, char in enumerate(chars_vocab):\n",
    "    chars_dictionary[char] = i\n",
    "\n",
    "# a mapping for indexes back into chars\n",
    "idx2char = {}\n",
    "for char, i in chars_dictionary.items():\n",
    "    idx2char[i] = char"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Looking at the whole sentence characters in a single prediction"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "capitalization_char_tags_train = [[char.isupper() for char in sent] for sent in whole_sents_train]\n",
    "capitalization_char_tags_val = [[char.isupper() for char in sent] for sent in whole_sents_val]\n",
    "\n",
    "# convert sentences into sequences of character indexes\n",
    "sequences_train = [[chars_dictionary[char] for char in sent] for sent in whole_sents_lower_train]\n",
    "sequences_val = [[chars_dictionary[char] for char in sent] for sent in whole_sents_lower_val]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "MAX_SEQUENCE_LENGTH = 200\n",
    "\n",
    "data_train = pad_sequences(sequences_train, maxlen=MAX_SEQUENCE_LENGTH, padding='post', value=chars_dictionary[\" \"])\n",
    "data_val = pad_sequences(sequences_val, maxlen=MAX_SEQUENCE_LENGTH, padding='post', value=chars_dictionary[\" \"])\n",
    "\n",
    "labels_train = pad_sequences(capitalization_char_tags_train, maxlen=MAX_SEQUENCE_LENGTH, padding='post', value=chars_dictionary[\" \"])[:,:,np.newaxis]\n",
    "labels_val = pad_sequences(capitalization_char_tags_val, maxlen=MAX_SEQUENCE_LENGTH, padding='post', value=chars_dictionary[\" \"])[:,:,np.newaxis]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### RNN Model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false,
    "scrolled": false
   },
   "outputs": [],
   "source": [
    "HIDDEN_SIZE_LSTM = 100\n",
    "BATCH_SIZE = 32\n",
    "model = Sequential()\n",
    "model.add(Embedding(input_dim=len(chars_dictionary) + 1, output_dim=EMBEDDING_DIM, input_length=MAX_SEQUENCE_LENGTH, trainable=True))\n",
    "model.add(Bidirectional(LSTM(HIDDEN_SIZE_LSTM, return_sequences=True), input_shape=(MAX_SEQUENCE_LENGTH, EMBEDDING_DIM)))\n",
    "model.add(Dense(1, activation='sigmoid'))\n",
    "model.compile(loss='binary_crossentropy', optimizer='adam', metrics=['accuracy'])\n",
    "model.fit(data_train, labels_train, validation_data=(data_val, labels_val), epochs=10, batch_size=BATCH_SIZE)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### CNN model\n",
    "let's try to build a similar model like the one we used for words above, only keeping allowing the Embeddings to be trainable this time :)."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# TODO implement a character level CNN model, notice that now we output a single value as our prediction\n",
    "# before our output was categorical, one output for lowercase, one for uppercase"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def print_char_predictions(x_test, y_pred, idx2char):\n",
    "    \"\"\"\n",
    "    print the results of our model's predictions\n",
    "    y_pred is a number between zero and one, a probability of the letter being Uppercase.\n",
    "    x_test are char indices of lowercased sentences\n",
    "    \"\"\"\n",
    "    for seq, preds in zip(x_test, y_pred):\n",
    "        sentence = []\n",
    "        for char_id, pred in zip(seq, preds):\n",
    "            if pred > .5:\n",
    "                sentence.append(idx2char[char_id].upper())\n",
    "            else:\n",
    "                sentence.append(idx2char[char_id])\n",
    "        print(\"\".join(sentence) + \"\\n\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "y_pred = model.predict(data_val)\n",
    "print_char_predictions(data_val, y_pred, idx2char)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Using a smaller sliding window on a continuous text\n",
    "up to now, the sequences of characters were quite long (hundreds of characters)...We can have a look at continuous text and create a window, that will limit our scope to say 30 characters. We are going to slide the window one character forwards and make predictions in each of them.\n",
    "\n",
    "What to do with the excess predictions from overlapping windows?\n",
    "\n",
    "Let's just take the prediction from the middle of the window. So that we cover enough context from both sides."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "capitalization_char_tags_train = [char.isupper() for char in whole_text_train]\n",
    "capitalization_char_tags_val = [char.isupper() for char in whole_text_val]\n",
    "\n",
    "# convert sentences into sequences of character indexes\n",
    "sequence_train = [chars_dictionary[char] for char in whole_text_lower_train]\n",
    "sequence_val = [chars_dictionary[char] for char in whole_text_lower_val]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "def window_batch_generator(sequence, labels, win_size=30, batch_size=32):\n",
    "    \"\"\"yield batches of sequences and lables in windows, each window sliding one step-ahead\"\"\"\n",
    "    x_batch, y_batch = [], []\n",
    "    while True:\n",
    "        for i in range(0, len(sequence) - win_size):\n",
    "            if len(x_batch) == batch_size:\n",
    "                yield np.array(x_batch), np.array(y_batch, dtype=\"float64\")[:,:,np.newaxis]\n",
    "                x_batch, y_batch = [], []\n",
    "            x_batch.append(sequence[i:i + win_size])\n",
    "            y_batch.append(labels[i:i + win_size])\n",
    "        if len(x_batch) != 0:\n",
    "            yield np.array(x_batch), np.array(y_batch, dtype=\"float64\")[:,:,np.newaxis]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### RNN Model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "data_gen_train = window_batch_generator(sequence_train, capitalization_char_tags_train)\n",
    "data_gen_val = window_batch_generator(sequence_val, capitalization_char_tags_val)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "we will now provide our Keras model with a Generator that is going to collect samples batch by batch\n",
    "\n",
    "the generator is \"bottom-less\" repeating it's iteration after we go through the whole text (EPOCH)\n",
    "we will calculate the epoch size to let our model know when to go to next epoch:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "steps_per_epoch_train = (len(sequence_train) - WIN_SIZE) / BATCH_SIZE\n",
    "steps_per_epoch_val = (len(sequence_val) - WIN_SIZE) / BATCH_SIZE"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "HIDDEN_SIZE_GRU = 10\n",
    "EMBEDDING_DIM = 20\n",
    "BATCH_SIZE = 32\n",
    "WIN_SIZE = 30\n",
    "NB_EPOCHS = 1\n",
    "model = Sequential()\n",
    "model.add(Embedding(input_dim=len(chars_dictionary) + 1, output_dim=EMBEDDING_DIM, input_length=WIN_SIZE, trainable=True))\n",
    "model.add(Bidirectional(GRU(HIDDEN_SIZE_GRU, return_sequences=True), input_shape=(WIN_SIZE, EMBEDDING_DIM)))\n",
    "model.add(Dense(1, activation='sigmoid'))\n",
    "model.compile(loss='binary_crossentropy', optimizer='adam', metrics=['accuracy'])\n",
    "model.fit_generator(data_gen_train, validation_data=data_gen_val,\n",
    "                    epochs=NB_EPOCHS, steps_per_epoch=steps_per_epoch_train, validation_steps=steps_per_epoch_val)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "collapsed": true
   },
   "source": [
    "*We can take each overlapping window prediction and extract only its middle part to make sure we cover the neighboring characters from both sides:*"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "def print_mid_window_predictions(batch_samples, model, idx2char):\n",
    "    \"\"\"\n",
    "    print the results of our model's predictions\n",
    "    take only the prediction from the middle of a window.\n",
    "    \"\"\"\n",
    "    win_size = test_samples[0].shape[1]\n",
    "    mid_win_idx = win_size / 2\n",
    "    extacted_text = []\n",
    "    for batch in test_samples:\n",
    "        predictions = model.predict_on_batch(batch)\n",
    "        for chars, preds in zip(batch, predictions):\n",
    "            if preds[mid_win_idx] > .5:\n",
    "                extacted_text.append(idx2char[chars[mid_win_idx]].upper())\n",
    "            else:\n",
    "                extacted_text.append(idx2char[chars[mid_win_idx]])\n",
    "    print(\"\".join(extacted_text))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "steps = 50\n",
    "test_samples = [data_gen_val.next()[0] for _ in range(steps)]  # we extract some test samples from our validation set "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "print_mid_window_predictions(test_samples, model, idx2char)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "####  Predicting the first letter in a window only by learning to read backwards :D...we won't be able to recognize the beginnings of sentences, but perheaps we might succeed at catching some names, lets give it a try:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def window_batch_generator_first_letter_out(sequence, labels, win_size=30, batch_size=32):\n",
    "    \"\"\"this generator only outputs a single label for each window, a label of our first character\"\"\"\n",
    "    x_batch, y_batch = [], []\n",
    "    while True:\n",
    "        for i in range(0, len(sequence) - win_size):\n",
    "            if len(x_batch) == batch_size:\n",
    "                yield np.array(x_batch), np.array(y_batch, dtype=\"float64\")[:, np.newaxis]\n",
    "                x_batch, y_batch = [], []\n",
    "            x_batch.append(sequence[i:i + win_size])\n",
    "            y_batch.append(labels[i])\n",
    "        if len(x_batch) != 0:\n",
    "            yield np.array(x_batch), np.array(y_batch, dtype=\"float64\")[:, np.newaxis]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "data_gen_train = window_batch_generator_first_letter_out(sequence_train, capitalization_char_tags_train)\n",
    "data_gen_val = window_batch_generator_first_letter_out(sequence_val, capitalization_char_tags_val)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Now we are only making a prediction from the final RNN cell's output (last timestep) by setting return_sequences=False.\n",
    "\n",
    "Setting go_backwards=True is going to revert the order of the sequence when passing it to RNN.\n",
    "\n",
    "When training, we are giving only a single example (label) for the whole sequence of characters. This is similar to the task of IMDB movie sentiment. In the models before, we had labels for each letter in the sequence. Could it more difficult to train the model now as it sees less information?"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "HIDDEN_SIZE_GRU = 10\n",
    "EMBEDDING_DIM = 20\n",
    "BATCH_SIZE = 32\n",
    "WIN_SIZE = 30\n",
    "NB_EPOCHS = 3\n",
    "model = Sequential()\n",
    "model.add(Embedding(input_dim=len(chars_dictionary) + 1, output_dim=EMBEDDING_DIM, input_length=WIN_SIZE, trainable=True))\n",
    "model.add(GRU(HIDDEN_SIZE_GRU, return_sequences=False, go_backwards=True, input_shape=(WIN_SIZE, EMBEDDING_DIM)))\n",
    "model.add(Dense(1, activation='sigmoid'))\n",
    "model.compile(loss='binary_crossentropy', optimizer='adam', metrics=['accuracy'])\n",
    "model.fit_generator(data_gen_train, validation_data=data_gen_val,\n",
    "                    epochs=NB_EPOCHS, steps_per_epoch=steps_per_epoch_train, validation_steps=steps_per_epoch_val)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def print_first_letter_predictions(batch_samples, model, idx2char):\n",
    "    \"\"\"\n",
    "    print the results of our model's predictions\n",
    "    take only the first letter prediction\n",
    "    \"\"\"\n",
    "    extacted_text = []\n",
    "    for batch in test_samples:\n",
    "        predictions = model.predict_on_batch(batch)\n",
    "        for chars, pred in zip(batch, predictions):\n",
    "            if pred > .5:\n",
    "                extacted_text.append(idx2char[chars[0]].upper())\n",
    "            else:\n",
    "                extacted_text.append(idx2char[chars[0]])\n",
    "    print(\"\".join(extacted_text))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "steps = 50\n",
    "test_samples = [data_gen_val.next()[0] for _ in range(steps)]  # we extract some test samples from our validation set\n",
    "print_first_letter_predictions(test_samples, model, idx2char)"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 2",
   "language": "python",
   "name": "python2"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 2
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython2",
   "version": "2.7.14"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 1
}