experiment2_lstm_sum.ipynb

{
  "cells": [
    {
      "cell_type": "markdown",
      "metadata": {},
      "source": [
        "# Sebastian Petrik - experiment 2 - Experiment with LSTM-based summarization model, failed."
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 1,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 17
        },
        "id": "F5Ut2Hr99wBj",
        "outputId": "d2d6c554-c3cc-4370-fb23-012ddf0e9d8d"
      },
      "outputs": [
        {
          "data": {
            "application/javascript": "\nfunction ClickConnect(){\nconsole.log(\"Working\");\ndocument.querySelector(\"colab-toolbar-button#connect\").click()\n}\nsetInterval(ClickConnect,60000)\n",
            "text/plain": [
              "<IPython.core.display.Javascript object>"
            ]
          },
          "metadata": {},
          "output_type": "display_data"
        }
      ],
      "source": [
        "# Keep Colab alive\n",
        "\n",
        "import IPython\n",
        "js_code = '''\n",
        "function ClickConnect(){\n",
        "console.log(\"Working\");\n",
        "document.querySelector(\"colab-toolbar-button#connect\").click()\n",
        "}\n",
        "setInterval(ClickConnect,60000)\n",
        "'''\n",
        "display(IPython.display.Javascript(js_code))"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 2,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "qiC6b4V36jMi",
        "outputId": "e984dec9-29fe-4955-eff6-292d7814cd82"
      },
      "outputs": [
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Looking in indexes: https://pypi.org/simple, https://us-python.pkg.dev/colab-wheels/public/simple/\n",
            "Collecting keras-self-attention\n",
            "  Downloading keras-self-attention-0.51.0.tar.gz (11 kB)\n",
            "Requirement already satisfied: numpy in /usr/local/lib/python3.7/dist-packages (from keras-self-attention) (1.21.6)\n",
            "Building wheels for collected packages: keras-self-attention\n",
            "  Building wheel for keras-self-attention (setup.py) ... \u001b[?25l\u001b[?25hdone\n",
            "  Created wheel for keras-self-attention: filename=keras_self_attention-0.51.0-py3-none-any.whl size=18913 sha256=958a58833d33da9caff25c82f56b46e9c15c885a029f46a971a13ba0d487f21d\n",
            "  Stored in directory: /root/.cache/pip/wheels/95/b1/a8/5ee00cc137940b2f6fa198212e8f45d813d0e0d9c3a04035a3\n",
            "Successfully built keras-self-attention\n",
            "Installing collected packages: keras-self-attention\n",
            "Successfully installed keras-self-attention-0.51.0\n"
          ]
        }
      ],
      "source": [
        "%pip install keras-self-attention"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 3,
      "metadata": {
        "id": "3CrTKiTn5wDs"
      },
      "outputs": [],
      "source": [
        "import pandas as pd\n",
        "import numpy as np\n",
        "import seaborn as sns\n",
        "from sklearn.model_selection import train_test_split\n",
        "from keras.preprocessing.text import Tokenizer \n",
        "from keras.utils import pad_sequences\n",
        "from keras import backend as K \n",
        "import gensim\n",
        "from numpy import *\n",
        "import numpy as np\n",
        "import pandas as pd \n",
        "import re\n",
        "from bs4 import BeautifulSoup\n",
        "from keras.preprocessing.text import Tokenizer \n",
        "from keras.utils import pad_sequences\n",
        "from nltk.corpus import stopwords\n",
        "import tensorflow as tf\n",
        "from tensorflow.keras.layers import Input, LSTM, Embedding, Dense, Concatenate, TimeDistributed\n",
        "from tensorflow.keras.models import Model\n",
        "from tensorflow.keras.callbacks import EarlyStopping\n",
        "import warnings\n",
        "\n",
        "pd.set_option(\"display.max_colwidth\", 200)\n",
        "warnings.filterwarnings(\"ignore\")"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 4,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "mwxSlLtm6nGo",
        "outputId": "7fecad21-b72c-42e2-b3d1-c67d7ef1dea5"
      },
      "outputs": [
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Mounted at /content/drive\n",
            "\u001b[0m\u001b[01;34mxsum\u001b[0m/  xsum_reduced_prepared_150_50.csv\n"
          ]
        }
      ],
      "source": [
        "from google.colab import drive\n",
        "drive.mount('/content/drive')\n",
        "%ls /content/drive/MyDrive/Datasets/"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 5,
      "metadata": {
        "id": "ipADEKGI6vUj"
      },
      "outputs": [],
      "source": [
        "# sequences config:\n",
        "max_text_len=150\n",
        "max_summary_len=50"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 6,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 659
        },
        "id": "gZhRogul61In",
        "outputId": "c18f6ca0-983c-4c34-c3c8-8f93004b2046"
      },
      "outputs": [
        {
          "data": {
            "text/html": [
              "\n",
              "  <div id=\"df-f59d3214-2127-4d97-9d3c-973518f563bd\">\n",
              "    <div class=\"colab-df-container\">\n",
              "      <div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>Unnamed: 0</th>\n",
              "      <th>text</th>\n",
              "      <th>summary</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>0</th>\n",
              "      <td>0</td>\n",
              "      <td>the national oceanic and atmospheric administration noaa forecasts 14 to 23 named storms eight to 14 of these could develop into hurricanes it says.the season could be one of the most active on re...</td>\n",
              "      <td>sostok _START_ as many as 14 hurricanes could hit the atlantic basin this year the top us climate agency says. _END_ eostok</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1</th>\n",
              "      <td>1</td>\n",
              "      <td>it was rare example of the cold war turning hot pitting the us and its allies against the ussr north korea and communist china it was marked by dramatic swings of fortune and devastating death tol...</td>\n",
              "      <td>sostok _START_ in 1950 as the international community was coming to terms with the aftermath of world war ii new conflict broke out at the edge of the asian continent. _END_ eostok</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>2</th>\n",
              "      <td>2</td>\n",
              "      <td>the singer and factor judge cited unreasonable behaviour in the action filed at the high court in london.the high profile couple split in february after claims the england star had been unfaithful...</td>\n",
              "      <td>sostok _START_ cheryl cole has filed for divorce from footballer husband ashley cole it has been confirmed. _END_ eostok</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>3</th>\n",
              "      <td>3</td>\n",
              "      <td>ofcom said they had received handful of complaints about the act which saw mr starr swallow lightbulb among other items.some viewers were concerned that it would encourage children to try dangerou...</td>\n",
              "      <td>sostok _START_ complaints about britain got talent are being looked at by the media watchdog after viewers raised concerns about glass swallower stevie starr. _END_ eostok</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>4</th>\n",
              "      <td>4</td>\n",
              "      <td>napper was linked to series of rapes and attempted rapes in south london parkland known as the green chain but was cleared for being too tall .he went on to kill rachel nickell in 1992 and samanth...</td>\n",
              "      <td>sostok _START_ serial rapist robert napper was brought to the attention of metropolitan police as early as 1989 but series of bad errors allowed the 42-year-old to slip the net report by the indep...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>...</th>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>47995</th>\n",
              "      <td>47995</td>\n",
              "      <td>the new zealand racer set the fastest time of the week so far on his superbike 124.888mph before achieving the fastest ever lightweight lap.his 118.812mph lightweight lap on 250cc honda was well w...</td>\n",
              "      <td>sostok _START_ bruce anstey again set the pace in the third qualifying session for the 2017 festival of motorcycling on wednesday. _END_ eostok</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>47996</th>\n",
              "      <td>47996</td>\n",
              "      <td>city paid their premier league counterparts £25m for the ivory coast forward two and half years ago.no fee has been agreed as yet and talks may continue into next week but it is anticipated bony w...</td>\n",
              "      <td>sostok _START_ swansea city are close to securing the return of striker wilfried bony from manchester city. _END_ eostok</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>47997</th>\n",
              "      <td>47997</td>\n",
              "      <td>media playback is not supported on this devicekeetels member of the victorious 2014 world cup team fired low past maddie hinch in the fourth quarter after two shots had been blocked.england won se...</td>\n",
              "      <td>sostok _START_ england hopes of defending their eurohockey title were thwarted at the semi-final stage as marloes keetels strike sent hosts netherlands through. _END_ eostok</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>47998</th>\n",
              "      <td>47998</td>\n",
              "      <td>the chorus of taylor new single look what you made me do follows the same rhythmic pattern as too sexy.after the song was released overnight the duo tweeted thank you taylor swift what marvellous ...</td>\n",
              "      <td>sostok _START_ right said fred have been credited as writers on taylor swift new single in one of the most bizarre musical combinations of the year. _END_ eostok</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>47999</th>\n",
              "      <td>47999</td>\n",
              "      <td>media playback is not supported on this device</td>\n",
              "      <td>sostok _START_ the floyd mayweather conor mcgregor fight brought boxing and the mixed martial arts head to head but while mayweather won on the night whose organisation is winning the battle of th...</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "<p>48000 rows × 3 columns</p>\n",
              "</div>\n",
              "      <button class=\"colab-df-convert\" onclick=\"convertToInteractive('df-f59d3214-2127-4d97-9d3c-973518f563bd')\"\n",
              "              title=\"Convert this dataframe to an interactive table.\"\n",
              "              style=\"display:none;\">\n",
              "        \n",
              "  <svg xmlns=\"http://www.w3.org/2000/svg\" height=\"24px\"viewBox=\"0 0 24 24\"\n",
              "       width=\"24px\">\n",
              "    <path d=\"M0 0h24v24H0V0z\" fill=\"none\"/>\n",
              "    <path d=\"M18.56 5.44l.94 2.06.94-2.06 2.06-.94-2.06-.94-.94-2.06-.94 2.06-2.06.94zm-11 1L8.5 8.5l.94-2.06 2.06-.94-2.06-.94L8.5 2.5l-.94 2.06-2.06.94zm10 10l.94 2.06.94-2.06 2.06-.94-2.06-.94-.94-2.06-.94 2.06-2.06.94z\"/><path d=\"M17.41 7.96l-1.37-1.37c-.4-.4-.92-.59-1.43-.59-.52 0-1.04.2-1.43.59L10.3 9.45l-7.72 7.72c-.78.78-.78 2.05 0 2.83L4 21.41c.39.39.9.59 1.41.59.51 0 1.02-.2 1.41-.59l7.78-7.78 2.81-2.81c.8-.78.8-2.07 0-2.86zM5.41 20L4 18.59l7.72-7.72 1.47 1.35L5.41 20z\"/>\n",
              "  </svg>\n",
              "      </button>\n",
              "      \n",
              "  <style>\n",
              "    .colab-df-container {\n",
              "      display:flex;\n",
              "      flex-wrap:wrap;\n",
              "      gap: 12px;\n",
              "    }\n",
              "\n",
              "    .colab-df-convert {\n",
              "      background-color: #E8F0FE;\n",
              "      border: none;\n",
              "      border-radius: 50%;\n",
              "      cursor: pointer;\n",
              "      display: none;\n",
              "      fill: #1967D2;\n",
              "      height: 32px;\n",
              "      padding: 0 0 0 0;\n",
              "      width: 32px;\n",
              "    }\n",
              "\n",
              "    .colab-df-convert:hover {\n",
              "      background-color: #E2EBFA;\n",
              "      box-shadow: 0px 1px 2px rgba(60, 64, 67, 0.3), 0px 1px 3px 1px rgba(60, 64, 67, 0.15);\n",
              "      fill: #174EA6;\n",
              "    }\n",
              "\n",
              "    [theme=dark] .colab-df-convert {\n",
              "      background-color: #3B4455;\n",
              "      fill: #D2E3FC;\n",
              "    }\n",
              "\n",
              "    [theme=dark] .colab-df-convert:hover {\n",
              "      background-color: #434B5C;\n",
              "      box-shadow: 0px 1px 3px 1px rgba(0, 0, 0, 0.15);\n",
              "      filter: drop-shadow(0px 1px 2px rgba(0, 0, 0, 0.3));\n",
              "      fill: #FFFFFF;\n",
              "    }\n",
              "  </style>\n",
              "\n",
              "      <script>\n",
              "        const buttonEl =\n",
              "          document.querySelector('#df-f59d3214-2127-4d97-9d3c-973518f563bd button.colab-df-convert');\n",
              "        buttonEl.style.display =\n",
              "          google.colab.kernel.accessAllowed ? 'block' : 'none';\n",
              "\n",
              "        async function convertToInteractive(key) {\n",
              "          const element = document.querySelector('#df-f59d3214-2127-4d97-9d3c-973518f563bd');\n",
              "          const dataTable =\n",
              "            await google.colab.kernel.invokeFunction('convertToInteractive',\n",
              "                                                     [key], {});\n",
              "          if (!dataTable) return;\n",
              "\n",
              "          const docLinkHtml = 'Like what you see? Visit the ' +\n",
              "            '<a target=\"_blank\" href=https://colab.research.google.com/notebooks/data_table.ipynb>data table notebook</a>'\n",
              "            + ' to learn more about interactive tables.';\n",
              "          element.innerHTML = '';\n",
              "          dataTable['output_type'] = 'display_data';\n",
              "          await google.colab.output.renderOutput(dataTable, element);\n",
              "          const docLink = document.createElement('div');\n",
              "          docLink.innerHTML = docLinkHtml;\n",
              "          element.appendChild(docLink);\n",
              "        }\n",
              "      </script>\n",
              "    </div>\n",
              "  </div>\n",
              "  "
            ],
            "text/plain": [
              "       Unnamed: 0  \\\n",
              "0               0   \n",
              "1               1   \n",
              "2               2   \n",
              "3               3   \n",
              "4               4   \n",
              "...           ...   \n",
              "47995       47995   \n",
              "47996       47996   \n",
              "47997       47997   \n",
              "47998       47998   \n",
              "47999       47999   \n",
              "\n",
              "                                                                                                                                                                                                          text  \\\n",
              "0      the national oceanic and atmospheric administration noaa forecasts 14 to 23 named storms eight to 14 of these could develop into hurricanes it says.the season could be one of the most active on re...   \n",
              "1      it was rare example of the cold war turning hot pitting the us and its allies against the ussr north korea and communist china it was marked by dramatic swings of fortune and devastating death tol...   \n",
              "2      the singer and factor judge cited unreasonable behaviour in the action filed at the high court in london.the high profile couple split in february after claims the england star had been unfaithful...   \n",
              "3      ofcom said they had received handful of complaints about the act which saw mr starr swallow lightbulb among other items.some viewers were concerned that it would encourage children to try dangerou...   \n",
              "4      napper was linked to series of rapes and attempted rapes in south london parkland known as the green chain but was cleared for being too tall .he went on to kill rachel nickell in 1992 and samanth...   \n",
              "...                                                                                                                                                                                                        ...   \n",
              "47995  the new zealand racer set the fastest time of the week so far on his superbike 124.888mph before achieving the fastest ever lightweight lap.his 118.812mph lightweight lap on 250cc honda was well w...   \n",
              "47996  city paid their premier league counterparts £25m for the ivory coast forward two and half years ago.no fee has been agreed as yet and talks may continue into next week but it is anticipated bony w...   \n",
              "47997  media playback is not supported on this devicekeetels member of the victorious 2014 world cup team fired low past maddie hinch in the fourth quarter after two shots had been blocked.england won se...   \n",
              "47998  the chorus of taylor new single look what you made me do follows the same rhythmic pattern as too sexy.after the song was released overnight the duo tweeted thank you taylor swift what marvellous ...   \n",
              "47999                                                                                                                                                           media playback is not supported on this device   \n",
              "\n",
              "                                                                                                                                                                                                       summary  \n",
              "0                                                                                  sostok _START_ as many as 14 hurricanes could hit the atlantic basin this year the top us climate agency says. _END_ eostok  \n",
              "1                         sostok _START_ in 1950 as the international community was coming to terms with the aftermath of world war ii new conflict broke out at the edge of the asian continent. _END_ eostok  \n",
              "2                                                                                     sostok _START_ cheryl cole has filed for divorce from footballer husband ashley cole it has been confirmed. _END_ eostok  \n",
              "3                                  sostok _START_ complaints about britain got talent are being looked at by the media watchdog after viewers raised concerns about glass swallower stevie starr. _END_ eostok  \n",
              "4      sostok _START_ serial rapist robert napper was brought to the attention of metropolitan police as early as 1989 but series of bad errors allowed the 42-year-old to slip the net report by the indep...  \n",
              "...                                                                                                                                                                                                        ...  \n",
              "47995                                                          sostok _START_ bruce anstey again set the pace in the third qualifying session for the 2017 festival of motorcycling on wednesday. _END_ eostok  \n",
              "47996                                                                                 sostok _START_ swansea city are close to securing the return of striker wilfried bony from manchester city. _END_ eostok  \n",
              "47997                            sostok _START_ england hopes of defending their eurohockey title were thwarted at the semi-final stage as marloes keetels strike sent hosts netherlands through. _END_ eostok  \n",
              "47998                                        sostok _START_ right said fred have been credited as writers on taylor swift new single in one of the most bizarre musical combinations of the year. _END_ eostok  \n",
              "47999  sostok _START_ the floyd mayweather conor mcgregor fight brought boxing and the mixed martial arts head to head but while mayweather won on the night whose organisation is winning the battle of th...  \n",
              "\n",
              "[48000 rows x 3 columns]"
            ]
          },
          "execution_count": 6,
          "metadata": {},
          "output_type": "execute_result"
        }
      ],
      "source": [
        "# load\n",
        "prepared = pd.read_csv('/content/drive/MyDrive/Datasets/xsum_reduced_prepared_150_50.csv')\n",
        "prepared"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 7,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "VvPbgx9F7WJM",
        "outputId": "bff17c39-76a3-45a2-ec25-852077b19452"
      },
      "outputs": [
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "% of rare words in vocabulary: 63.383132872431105\n",
            "Total Coverage of rare words: 1.9008429752383218\n",
            "Size of vocabulary in X = 30932\n",
            "% of rare words in vocabulary: 73.60875073386275\n",
            "Total Coverage of rare words: 4.367686488532698\n",
            "Size of vocabulary in Y = 8542\n",
            "Size of vocabulary from the w2v model = 30932\n"
          ]
        }
      ],
      "source": [
        "# data more prepare\n",
        "x_tr,x_val,y_tr,y_val=train_test_split(np.array(prepared['text']),np.array(prepared['summary']),test_size=0.1,random_state=0,shuffle=True)\n",
        "\n",
        "#prepare a tokenizer for reviews on training data\n",
        "x_tokenizer = Tokenizer() \n",
        "x_tokenizer.fit_on_texts(list(x_tr))\n",
        "\n",
        "# rare word analysis - X\n",
        "thresh=4\n",
        "cnt=0\n",
        "tot_cnt=0\n",
        "freq=0\n",
        "tot_freq=0\n",
        "\n",
        "for key,value in x_tokenizer.word_counts.items():\n",
        "    tot_cnt=tot_cnt+1\n",
        "    tot_freq=tot_freq+value\n",
        "    if(value<thresh):\n",
        "        cnt=cnt+1\n",
        "        freq=freq+value\n",
        "    \n",
        "print(\"% of rare words in vocabulary:\",(cnt/tot_cnt)*100)\n",
        "print(\"Total Coverage of rare words:\",(freq/tot_freq)*100)\n",
        "\n",
        "#prepare a tokenizer for reviews on training data - X\n",
        "x_tokenizer = Tokenizer(num_words=tot_cnt-cnt) \n",
        "x_tokenizer.fit_on_texts(list(x_tr))\n",
        "\n",
        "#convert text sequences into integer sequences (i.e one-hot encodeing all the words)\n",
        "x_tr_seq    =   x_tokenizer.texts_to_sequences(x_tr) \n",
        "x_val_seq   =   x_tokenizer.texts_to_sequences(x_val)\n",
        "\n",
        "#padding zero upto maximum length\n",
        "x_tr    =   pad_sequences(x_tr_seq,  maxlen=max_text_len, padding='post')\n",
        "x_val   =   pad_sequences(x_val_seq, maxlen=max_text_len, padding='post')\n",
        "\n",
        "#size of vocabulary ( +1 for padding token)\n",
        "x_voc   =  x_tokenizer.num_words + 1\n",
        "\n",
        "print(\"Size of vocabulary in X = {}\".format(x_voc))\n",
        "\n",
        "#prepare a tokenizer for reviews on training data - Y\n",
        "y_tokenizer = Tokenizer()   \n",
        "y_tokenizer.fit_on_texts(list(y_tr))\n",
        "\n",
        "# rare word analysis - Y\n",
        "thresh=6\n",
        "cnt=0\n",
        "tot_cnt=0\n",
        "freq=0\n",
        "tot_freq=0\n",
        "\n",
        "for key,value in y_tokenizer.word_counts.items():\n",
        "    tot_cnt=tot_cnt+1\n",
        "    tot_freq=tot_freq+value\n",
        "    if(value<thresh):\n",
        "        cnt=cnt+1\n",
        "        freq=freq+value\n",
        "    \n",
        "print(\"% of rare words in vocabulary:\",(cnt/tot_cnt)*100)\n",
        "print(\"Total Coverage of rare words:\",(freq/tot_freq)*100)\n",
        "\n",
        "#prepare a tokenizer for reviews on training data\n",
        "y_tokenizer = Tokenizer(num_words=tot_cnt-cnt) \n",
        "y_tokenizer.fit_on_texts(list(y_tr))\n",
        "\n",
        "#convert text sequences into integer sequences (i.e one hot encode the text in Y)\n",
        "y_tr_seq    =   y_tokenizer.texts_to_sequences(y_tr) \n",
        "y_val_seq   =   y_tokenizer.texts_to_sequences(y_val) \n",
        "\n",
        "#padding zero upto maximum length\n",
        "y_tr    =   pad_sequences(y_tr_seq, maxlen=max_summary_len, padding='post')\n",
        "y_val   =   pad_sequences(y_val_seq, maxlen=max_summary_len, padding='post')\n",
        "\n",
        "#size of vocabulary\n",
        "y_voc  =   y_tokenizer.num_words +1\n",
        "print(\"Size of vocabulary in Y = {}\".format(y_voc))\n",
        "\n",
        "# remove empty summaries\n",
        "ind=[]\n",
        "for i in range(len(y_tr)):\n",
        "    cnt=0\n",
        "    for j in y_tr[i]:\n",
        "        if j!=0:\n",
        "            cnt=cnt+1\n",
        "    if(cnt==2):\n",
        "        ind.append(i)\n",
        "\n",
        "y_tr=np.delete(y_tr,ind, axis=0)\n",
        "x_tr=np.delete(x_tr,ind, axis=0)\n",
        "ind=[]\n",
        "for i in range(len(y_val)):\n",
        "    cnt=0\n",
        "    for j in y_val[i]:\n",
        "        if j!=0:\n",
        "            cnt=cnt+1\n",
        "    if(cnt==2):\n",
        "        ind.append(i)\n",
        "\n",
        "y_val=np.delete(y_val,ind, axis=0)\n",
        "x_val=np.delete(x_val,ind, axis=0)\n",
        "\n",
        "print(\"Size of vocabulary from the w2v model = {}\".format(x_voc))\n"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 8,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "3adX794C9ZQx",
        "outputId": "419b9426-9b27-4a20-c014-5e6cf827c7fe"
      },
      "outputs": [
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "WARNING:tensorflow:Layer lstm will not use cuDNN kernels since it doesn't meet the criteria. It will use a generic GPU kernel as fallback when running on GPU.\n",
            "WARNING:tensorflow:Layer lstm_1 will not use cuDNN kernels since it doesn't meet the criteria. It will use a generic GPU kernel as fallback when running on GPU.\n",
            "WARNING:tensorflow:Layer lstm_2 will not use cuDNN kernels since it doesn't meet the criteria. It will use a generic GPU kernel as fallback when running on GPU.\n",
            "WARNING:tensorflow:Layer lstm_3 will not use cuDNN kernels since it doesn't meet the criteria. It will use a generic GPU kernel as fallback when running on GPU.\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Model: \"model\"\n",
            "__________________________________________________________________________________________________\n",
            " Layer (type)                   Output Shape         Param #     Connected to                     \n",
            "==================================================================================================\n",
            " input_1 (InputLayer)           [(None, 150)]        0           []                               \n",
            "                                                                                                  \n",
            " embedding (Embedding)          (None, 150, 200)     6186400     ['input_1[0][0]']                \n",
            "                                                                                                  \n",
            " lstm (LSTM)                    [(None, 150, 300),   601200      ['embedding[0][0]']              \n",
            "                                 (None, 300),                                                     \n",
            "                                 (None, 300)]                                                     \n",
            "                                                                                                  \n",
            " input_2 (InputLayer)           [(None, None)]       0           []                               \n",
            "                                                                                                  \n",
            " lstm_1 (LSTM)                  [(None, 150, 300),   721200      ['lstm[0][0]']                   \n",
            "                                 (None, 300),                                                     \n",
            "                                 (None, 300)]                                                     \n",
            "                                                                                                  \n",
            " embedding_1 (Embedding)        (None, None, 200)    1708400     ['input_2[0][0]']                \n",
            "                                                                                                  \n",
            " lstm_2 (LSTM)                  [(None, 150, 300),   721200      ['lstm_1[0][0]']                 \n",
            "                                 (None, 300),                                                     \n",
            "                                 (None, 300)]                                                     \n",
            "                                                                                                  \n",
            " lstm_3 (LSTM)                  [(None, None, 300),  601200      ['embedding_1[0][0]',            \n",
            "                                 (None, 300),                     'lstm_2[0][1]',                 \n",
            "                                 (None, 300)]                     'lstm_2[0][2]']                 \n",
            "                                                                                                  \n",
            " time_distributed (TimeDistribu  (None, None, 8542)  2571142     ['lstm_3[0][0]']                 \n",
            " ted)                                                                                             \n",
            "                                                                                                  \n",
            "==================================================================================================\n",
            "Total params: 13,110,742\n",
            "Trainable params: 13,110,742\n",
            "Non-trainable params: 0\n",
            "__________________________________________________________________________________________________\n",
            "None\n"
          ]
        }
      ],
      "source": [
        "# model setup\n",
        "\n",
        "K.clear_session()\n",
        "\n",
        "latent_dim = 300\n",
        "embedding_dim=200\n",
        "\n",
        "# Encoder\n",
        "encoder_inputs = Input(shape=(max_text_len,))\n",
        "\n",
        "#embedding layer\n",
        "enc_emb =  Embedding(x_voc, embedding_dim,trainable=True)(encoder_inputs)\n",
        "\n",
        "#encoder lstm 1\n",
        "encoder_lstm1 = LSTM(latent_dim,return_sequences=True,return_state=True,dropout=0.4,recurrent_dropout=0.4)\n",
        "encoder_output1, state_h1, state_c1 = encoder_lstm1(enc_emb)\n",
        "\n",
        "#encoder lstm 2\n",
        "encoder_lstm2 = LSTM(latent_dim,return_sequences=True,return_state=True,dropout=0.4,recurrent_dropout=0.4)\n",
        "encoder_output2, state_h2, state_c2 = encoder_lstm2(encoder_output1)\n",
        "\n",
        "#encoder lstm 3\n",
        "encoder_lstm3=LSTM(latent_dim, return_state=True, return_sequences=True,dropout=0.4,recurrent_dropout=0.4)\n",
        "encoder_outputs, state_h, state_c= encoder_lstm3(encoder_output2)\n",
        "\n",
        "# Set up the decoder, using `encoder_states` as initial state.\n",
        "decoder_inputs = Input(shape=(None,))\n",
        "\n",
        "#embedding layer\n",
        "dec_emb_layer = Embedding(y_voc, embedding_dim,trainable=True)\n",
        "dec_emb = dec_emb_layer(decoder_inputs)\n",
        "\n",
        "decoder_lstm = LSTM(latent_dim, return_sequences=True, return_state=True,dropout=0.4,recurrent_dropout=0.2)\n",
        "decoder_outputs,decoder_fwd_state, decoder_back_state = decoder_lstm(dec_emb,initial_state=[state_h, state_c])\n",
        "\n",
        "#dense layer\n",
        "decoder_dense =  TimeDistributed(Dense(y_voc, activation='softmax'))\n",
        "decoder_outputs = decoder_dense(decoder_outputs)\n",
        "\n",
        "# Define the model \n",
        "model = Model([encoder_inputs, decoder_inputs], decoder_outputs)\n",
        "\n",
        "print(model.summary())\n",
        "model.compile(optimizer='rmsprop', loss='sparse_categorical_crossentropy')"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 9,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "bnyfM4vc9jeE",
        "outputId": "f6f8060a-d6bb-4db0-bef7-336c16a01057"
      },
      "outputs": [
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Loaded model checkpoint.\n"
          ]
        }
      ],
      "source": [
        "# train setup\n",
        "es = EarlyStopping(monitor='val_loss', mode='min', verbose=1,patience=2)\n",
        "\n",
        "checkpoint_filepath = '/content/drive/MyDrive/checkpoint.best.hdf'\n",
        "model_checkpoint_callback = tf.keras.callbacks.ModelCheckpoint(\n",
        "    filepath=checkpoint_filepath,\n",
        "    save_weights_only=True,\n",
        "    monitor='val_loss',\n",
        "    mode='min',\n",
        "    save_best_only=True\n",
        ")\n",
        "\n",
        "try:\n",
        "  model.load_weights(checkpoint_filepath)\n",
        "  print('Loaded model checkpoint.')\n",
        "except:\n",
        "  print('Cannot load weights from checkpoint, it may not exist yet.')"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 12,
      "metadata": {
        "id": "QYEbF9a597bJ"
      },
      "outputs": [],
      "source": [
        "# training\n",
        "do_train = False\n",
        "\n",
        "if do_train:\n",
        "  history = model.fit(\n",
        "      [x_tr, y_tr[:, :-1]],\n",
        "      y_tr.reshape(y_tr.shape[0], y_tr.shape[1], 1)[:, 1:],\n",
        "      epochs=50,\n",
        "      callbacks=[es, model_checkpoint_callback],\n",
        "      batch_size=256,\n",
        "      validation_data=(\n",
        "          [x_val, y_val[:, :-1]],\n",
        "          y_val.reshape(y_val.shape[0], y_val.shape[1], 1)[:, 1:],\n",
        "      ),\n",
        "  )\n",
        "  # training visualisation\n",
        "  from matplotlib import pyplot\n",
        "  pyplot.plot(history.history['loss'], label='train')\n",
        "  pyplot.plot(history.history['val_loss'], label='test')\n",
        "  pyplot.legend()\n",
        "  pyplot.show()"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 12,
      "metadata": {
        "id": "f0Dfcf6R-RqS"
      },
      "outputs": [],
      "source": []
    },
    {
      "cell_type": "code",
      "execution_count": 15,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "8stx38M9-VjN",
        "outputId": "3665f934-9e49-4590-b667-eb23a2b5831f"
      },
      "outputs": [
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Review: the collision happened between junction 10 for winchester at bar end and junction at winnall just after 10 20 bst initially all three lanes of the northbound carriageway were closed by police for the rest of the afternoon although one lane has since reopened one person was taken to hospital with serious injuries following the crash motorists were told to avoid the area and delays of up to two hours were reported with traffic back to junction 13 for eastleigh surrounding roads including in and out of winchester and through have also faced disruption the hampshire and isle of wight air ambulance was called to the scene a spokesman for south central ambulance service said one patient with serious injuries was taken to southampton general hospital one patient with minor injuries to the royal hampshire county hospital winchester and third patient with minor injuries was treated at the scene \n",
            "Original summary: start a serious crash involving three lorries and car caused the closure of the northbound end \n",
            "1/1 [==============================] - 2s 2s/step\n",
            "Predicted summary:  start a car has been hit by car which killed two people in crash end\n",
            "\n",
            "\n",
            "Review: the salvage operation began late on thursday evening in hubei province as cranes slowly turned over the stricken ship relatives gathered to hold candlelit vigil nearby just 14 of the 456 passengers and crew are known to have survived authorities said the chances of finding anyone else alive were slim why did the eastern star sink \n",
            "Original summary: start chinese have fully the ship which capsized on the river killing more than 400 people end \n",
            "1/1 [==============================] - 1s 515ms/step\n",
            "Predicted summary:  start a plane carrying out on the isle of man has been killed in an explosion at the centre of the building in the north west of end\n",
            "\n",
            "\n",
            "Review: conservative jim gifford was elected council leader by 38 votes to 25 against the snp richard thomson lib dem councillor peter argyle was appointed deputy leader at thursday morning meeting fellow lib dem bill howatson was elected provost with conservative ron taking on the deputy provost role in the local elections in aberdeenshire the conservatives had 23 seats up from 14 while the snp fell eight to 21 the lib dems gained two to 14 independents fell from 12 to 10 and labour dropped from two to one in shetland independent cecil smith who has served on the council since 2007 is the council new leader shetland islands councils remained in the control of independents \n",
            "Original summary: start a coalition of conservative liberal democrats and independent councillors will run aberdeenshire council end \n",
            "1/1 [==============================] - 0s 249ms/step\n",
            "Predicted summary:  start the liberal democrats have voted to make the first of the next financial year end\n",
            "\n",
            "\n",
            "Review: of the men who died an estimated 120 known as the jersey pals were killed during the battle of the somme made from jersey granite the memorial stone was officially unveiled by bailiff william in the french village of the delegation from the island also included the lieutenant governor chief minister and the dean of jersey ian an author and historian said the memorial would be permanent reminder of the role jersey played in the great war although part of it would return to the island core has been removed from the middle of the stone about the height of person heart and that is being brought back to jersey to become part of monument in royal square he told bbc news in july hundreds of people attended ceremony to reveal the memorial stone \n",
            "Original summary: start a memorial dedicated to the jersey men who died in world war has been unveiled in the somme region of france end \n",
            "1/1 [==============================] - 0s 189ms/step\n",
            "Predicted summary:  start a memorial service has been held to mark the anniversary of the royal navy helicopter in the scottish borders end\n",
            "\n",
            "\n",
            "Review: a selection of photos from across the african continent this week \n",
            "Original summary: start images courtesy of afp ap epa and reuters end \n",
            "1/1 [==============================] - 1s 502ms/step\n",
            "Predicted summary:  start images courtesy of afp ap epa getty images and reuters end\n",
            "\n",
            "\n",
            "Review: roy hodgson side are one of the top seeds while the other three are among six nations in the pot of bottom seeds there will be six groups of four teams for next summer finals in france the winners and runners up in each group will qualify for the knockout phase as well as the four best third place sides england holders spain belgium germany portugal and hosts france complete pot one meanwhile 2012 runners up and four time world champions italy are one of the teams in pot two pot three contains sweden and poland the finals will start on friday 10 june with the final taking place at stade de france on sunday 10 july \n",
            "Original summary: start england will have 50 50 chance of being drawn against wales northern ireland or the republic of ireland in the euro 2016 draw on 12 december end \n",
            "1/1 [==============================] - 0s 192ms/step\n",
            "Predicted summary:  start wales will face their place in the women world cup in the netherlands after losing to the semi finals of the women super league end\n",
            "\n",
            "\n",
            "Review: the 19 year old central defender who was recently with irish club bohemians will initially be with the lions development squad rooney impressed after previously being invited to train with the south london club who have not disclosed the length of his contract paul came in for few days last year to show us his talent manager neil harris told millwall website he been playing in the league of ireland as young man in men division so physically he can compete not putting any pressure on him he needs to come in find his feet and enjoy himself fingers crossed it works out find all the latest football transfers on our dedicated page \n",
            "Original summary: start league one side millwall have signed irish teenager paul rooney end \n",
            "1/1 [==============================] - 0s 182ms/step\n",
            "Predicted summary:  start league one side northampton town have signed former arsenal midfielder on loan until the end of the season end\n",
            "\n",
            "\n",
            "Review: they went ahead against the run of play when david navarro turned jordi alba cross into his own net bottom of the table levante hit the woodwork through jose luis morales while giuseppe rossi forced fine save from claudio bravo luis suarez 20th league goal of the season sealed the points barcelona have one game in hand over second placed atletico madrid who had drawn level on points after beating eibar 3 1 on saturday this was far from vintage barca performance but they have now won their past 10 league and cup games they have also equalled the spanish record of 28 matches unbeaten levante meanwhile are five points from safety relive the action from the de valencia \n",
            "Original summary: start barcelona celebrated boss luis enrique 100th match in charge with win over to go three points clear at the top of la liga end \n",
            "1/1 [==============================] - 0s 191ms/step\n",
            "Predicted summary:  start borussia dortmund were denied the chance to play in the first round of the season with victory over the league two leaders end\n",
            "\n",
            "\n",
            "Review: it comes only four years after â£10 000 was spent repairing the steps in somerset when the stone monument was previously damaged by floodwater exmoor national park authority said exceptional rainfall could cause slabs to wash away every few years the 50m 164ft grade i listed bridge over the river which is still under water attracts thousands of visitors every year it is known as bridge which is believed to have derived from the latin word meaning stack or pile of stones somerset county council said the bridge is an enduring and iconic feature of the exmoor landscape that has been in use since medieval times as an ancient monument the bridge must be put back exactly the way it was the river is currently in high flow and we will have to wait until the flow reduces which may be few months before we can plan repairs \n",
            "Original summary: start part of an ancient stone footbridge has been washed away in flooding on end \n",
            "1/1 [==============================] - 0s 172ms/step\n",
            "Predicted summary:  start a new rnli has been closed to the public after it was found on the site of the river end\n",
            "\n",
            "\n",
            "Review: david lord from westerham is charged with causing valerie deakin death by dangerous driving in the kent town on christmas eve ms deakin 74 of east sussex died when mr lord audi crashed into the town centre branch of costa he was bailed to appear at maidstone crown court on august mr lord had trouble hearing chairman of the bench jackie hamilton during his three minute appearance at sevenoaks magistrates court asked if he could hear proceedings having to family members in the public gallery he said not really two men and two women were also taken to hospital after the crash another woman was treated for minor injuries \n",
            "Original summary: start an 86 year old man accused of killing woman who died when his car ploughed into kent coffee shop has made his first appearance in court end \n",
            "1/1 [==============================] - 0s 190ms/step\n",
            "Predicted summary:  start a man has admitted causing the death of woman by dangerous driving after she was hit by car in the borders end\n",
            "\n",
            "\n"
          ]
        }
      ],
      "source": [
        "# target to source dictionary setup and decoder\n",
        "\n",
        "reverse_target_word_index=y_tokenizer.index_word\n",
        "reverse_source_word_index=x_tokenizer.index_word\n",
        "target_word_index=y_tokenizer.word_index\n",
        "\n",
        "# Encode the input sequence to get the feature vector\n",
        "encoder_model = Model(inputs=encoder_inputs,outputs=[encoder_outputs, state_h, state_c])\n",
        "\n",
        "# Decoder setup\n",
        "# Below tensors will hold the states of the previous time step\n",
        "decoder_state_input_h = Input(shape=(latent_dim,))\n",
        "decoder_state_input_c = Input(shape=(latent_dim,))\n",
        "decoder_hidden_state_input = Input(shape=(max_text_len,latent_dim))\n",
        "\n",
        "# Get the embeddings of the decoder sequence\n",
        "dec_emb2= dec_emb_layer(decoder_inputs) \n",
        "# To predict the next word in the sequence, set the initial states to the states from the previous time step\n",
        "decoder_outputs2, state_h2, state_c2 = decoder_lstm(dec_emb2, initial_state=[decoder_state_input_h, decoder_state_input_c])\n",
        "\n",
        "# A dense softmax layer to generate prob dist. over the target vocabulary\n",
        "decoder_outputs2 = decoder_dense(decoder_outputs2) \n",
        "\n",
        "# Final decoder model\n",
        "decoder_model = Model(\n",
        "    [decoder_inputs] + [decoder_hidden_state_input,decoder_state_input_h, decoder_state_input_c],\n",
        "    [decoder_outputs2] + [state_h2, state_c2])\n",
        "\n",
        "def decode_sequence(input_seq):\n",
        "    # Encode the input as state vectors.\n",
        "    e_out, e_h, e_c = encoder_model.predict(input_seq)\n",
        "    \n",
        "    # Generate empty target sequence of length 1.\n",
        "    target_seq = np.zeros((1,1))\n",
        "    \n",
        "    # Populate the first word of target sequence with the start word.\n",
        "    target_seq[0, 0] = target_word_index['sostok']\n",
        "\n",
        "    stop_condition = False\n",
        "    decoded_sentence = ''\n",
        "    while not stop_condition:\n",
        "      \n",
        "        output_tokens, h, c = decoder_model.predict([target_seq] + [e_out, e_h, e_c], verbose=0)\n",
        "\n",
        "        # Sample a token\n",
        "        sampled_token_index = np.argmax(output_tokens[0, -1, :])\n",
        "        sampled_token = reverse_target_word_index[sampled_token_index]\n",
        "        \n",
        "        if(sampled_token!='eostok'):\n",
        "            decoded_sentence += ' '+sampled_token\n",
        "\n",
        "        # Exit condition: either hit max length or find stop word.\n",
        "        if (sampled_token == 'eostok'  or len(decoded_sentence.split()) >= (max_summary_len-1)):\n",
        "            stop_condition = True\n",
        "\n",
        "        # Update the target sequence (of length 1).\n",
        "        target_seq = np.zeros((1,1))\n",
        "        target_seq[0, 0] = sampled_token_index\n",
        "\n",
        "        # Update internal states\n",
        "        e_h, e_c = h, c\n",
        "\n",
        "    return decoded_sentence\n",
        "\n",
        "def seq2summary(input_seq):\n",
        "    newString=''\n",
        "    for i in input_seq:\n",
        "        if((i!=0 and i!=target_word_index['sostok']) and i!=target_word_index['eostok']):\n",
        "            newString=newString+reverse_target_word_index[i]+' '\n",
        "    return newString\n",
        "\n",
        "def seq2text(input_seq):\n",
        "    newString=''\n",
        "    for i in input_seq:\n",
        "        if(i!=0):\n",
        "            newString=newString+reverse_source_word_index[i]+' '\n",
        "    return newString\n",
        "\n",
        "# finally apply model\n",
        "for i in range(0,10):\n",
        "    print(\"Review:\",seq2text(x_tr[i]))\n",
        "    print(\"Original summary:\",seq2summary(y_tr[i]))\n",
        "    print(\"Predicted summary:\",decode_sequence(x_tr[i].reshape(1,max_text_len)))\n",
        "    print(\"\\n\")"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 31,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 53
        },
        "id": "v77L_aacHT85",
        "outputId": "6c916adc-bbaa-497b-9656-b1109f304265"
      },
      "outputs": [
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "1/1 [==============================] - 0s 71ms/step\n"
          ]
        },
        {
          "data": {
            "application/vnd.google.colaboratory.intrinsic+json": {
              "type": "string"
            },
            "text/plain": [
              "' start britain won the first round of the world cup with victory over the netherlands end'"
            ]
          },
          "execution_count": 31,
          "metadata": {},
          "output_type": "execute_result"
        }
      ],
      "source": [
        "sekv = x_tokenizer.texts_to_sequences(['sostok _START_ cheryl cole has filed for divorce from footballer husband ashley cole it has been confirmed. _END_ eostok'])\n",
        "padded = pad_sequences(sekv, maxlen=max_summary_len, padding='post')\n",
        "decode_sequence(padded)"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "2_w-Rk6ZKEEr"
      },
      "source": [
        "??? why"
      ]
    }
  ],
  "metadata": {
    "accelerator": "GPU",
    "colab": {
      "provenance": []
    },
    "gpuClass": "standard",
    "kernelspec": {
      "display_name": "Python 3",
      "name": "python3"
    },
    "language_info": {
      "name": "python"
    }
  },
  "nbformat": 4,
  "nbformat_minor": 0
}