AOBDL_data_preparation/AOBDL_data_preparation.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "CwSVBuG-E6X6"
   },
   "outputs": [],
   "source": [
    "# MODULES IMPORT \n",
    "\n",
    "import numpy as np\n",
    "import pandas as pd\n",
    "import pdb\n",
    "import os"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 35
    },
    "colab_type": "code",
    "executionInfo": {
     "elapsed": 1117,
     "status": "ok",
     "timestamp": 1559912371470,
     "user": {
      "displayName": "Elizaveta",
      "photoUrl": "https://lh3.googleusercontent.com/-3-3kSLC4Mzw/AAAAAAAAAAI/AAAAAAAAABg/Usb9K3n3cRI/s64/photo.jpg",
      "userId": "01253993997636551956"
     },
     "user_tz": -120
    },
    "id": "YB0lSfPVEpA9",
    "outputId": "34bf627b-9837-4ce5-fe52-16628f20673f"
   },
   "outputs": [],
   "source": [
    "# GOOGLE COLAB SETUP\n",
    "\n",
    "# Load the Drive helper and mount\n",
    "if os.name=='posix':\n",
    "    from google.colab import drive\n",
    "\n",
    "    # This will prompt for authorization.\n",
    "    drive.mount('/content/drive')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "7esVSpP-E5ZD"
   },
   "outputs": [],
   "source": [
    "# VARIABLES AND PATHS\n",
    "if os.name=='posix':\n",
    "    raw_data_path = 'drive/My Drive/Colab Notebooks/adaptHAN/AOBDL_code/raw_data'\n",
    "    data_path     = 'drive/My Drive/Colab Notebooks/adaptHAN/AOBDL_code/data'\n",
    "    codes_path    = 'drive/My Drive/Colab Notebooks/adaptHAN/AOBDL_code/AOBDL_data_preparation'\n",
    "else:\n",
    "    raw_data_path = '../raw_data'\n",
    "    data_path     = '../data'\n",
    "    codes_path    = '../AOBDL_data_preparation'\n",
    "\n",
    "\n",
    "#3. Read file as panda dataframe\n",
    "train         = pd.read_csv(f'{raw_data_path}/train.csv.zip') \n",
    "test          = pd.read_csv(f'{raw_data_path}/test.csv.zip') \n",
    "labels        = pd.read_csv(f'{raw_data_path}/test_labels.csv.zip')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Create target Directory if don't exist\n",
    "if not os.path.exists(data_path):\n",
    "    os.mkdir(data_path)\n",
    "    print(\"Directory \" , data_path ,  \" Created \")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "9ilY_NUAI3NV"
   },
   "outputs": [],
   "source": [
    "#Combine labels and test set\n",
    "test                = test.merge(labels, how=\"left\", on=\"id\")\n",
    "test['used_for_sc'] = test[['toxic', 'severe_toxic', 'obscene', 'threat', 'insult', 'identity_hate']].sum(axis=1) >= 0 \n",
    "test_labelled       = test[test.used_for_sc==True]\n",
    "test_unlabelled     = test[test.used_for_sc!=True]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "VhBEDM5JLots"
   },
   "outputs": [],
   "source": [
    "# Clean data completely \n",
    "def substitute_repeats_fixed_len(text, nchars, ntimes=3):\n",
    "    return re.sub(r\"(\\S{{{}}})(\\1{{{},}})\".format(nchars, ntimes-1), r\"\\1\", text)\n",
    "  \n",
    "def substitute_repeats(text, ntimes=3):\n",
    "    for nchars in range(1, 20):\n",
    "        text         = substitute_repeats_fixed_len(text, nchars, ntimes)\n",
    "    return text\n",
    "\n",
    "def text_to_wordlist(text, remove_stop_words=True, stem_words=False, with_punct_sent=False):\n",
    "    stop_words       = set(['a', 'the', \"an\", \"are\", \"as\",  'did',\n",
    "                       \"do\", \"is\", \"has\", \"have\", \"had\", \"was\", \"were\",\n",
    "                       \"will\", \"would\", \"am\", \"it\", \"for\", \"on\", \"it\", \"of\"])\n",
    "    #from string import punctuation\n",
    "    import re\n",
    "    NEG_CONTRACTIONS    = [\n",
    "                          (r'aren\\'t', 'are not'),\n",
    "                          (r'can\\'t', 'can not'),\n",
    "                          (r'couldn\\'t', 'could not'),\n",
    "                          (r'daren\\'t', 'dare not'),\n",
    "                          (r'didn\\'t', 'did not'),\n",
    "                          (r'doesn\\'t', 'does not'),\n",
    "                          (r'don\\'t', 'do not'),\n",
    "                          (r'isn\\'t', 'is not'),\n",
    "                          (r'hasn\\'t', 'has not'),\n",
    "                          (r'haven\\'t', 'have not'),\n",
    "                          (r'hadn\\'t', 'had not'),\n",
    "                          (r'mayn\\'t', 'may not'),\n",
    "                          (r'mightn\\'t', 'might not'),\n",
    "                          (r'mustn\\'t', 'must not'),\n",
    "                          (r'needn\\'t', 'need not'),\n",
    "                          (r'oughtn\\'t', 'ought not'),\n",
    "                          (r'shan\\'t', 'shall not'),\n",
    "                          (r'shouldn\\'t', 'should not'),\n",
    "                          (r'wasn\\'t', 'was not'),\n",
    "                          (r'weren\\'t', 'were not'),\n",
    "                          (r'won\\'t', 'will not'),\n",
    "                          (r'wouldn\\'t', 'would not'),\n",
    "                          (r'ain\\'t', 'am not') # not only but stopword anyway\n",
    "                          ]\n",
    "    OTHER_CONTRACTIONS = [\n",
    "                          #(r\"'m\", 'am'),\n",
    "                          (r\"'ll\", ' will'),\n",
    "                          (r\"'s\", ' has'), # or 'is' but both are stopwords\n",
    "                          (r\"'d\", ' had'), # or 'would' but both are stopwords\n",
    "                          (r\"'ve\", \" have\"),\n",
    "                           (r\"'re\", \" are\")   \n",
    "    ]\n",
    "    OTHER_RPS          = [\n",
    "                          (\"&lt;3\", \" good \"),\n",
    "                          (\":d\", \" good \"),\n",
    "                          (\":dd\", \" good \"),\n",
    "                          (\":p\", \" good \"),\n",
    "                          (\"8)\", \" good \"),\n",
    "                          (\":-)\", \" good \"),\n",
    "                          (\":)\", \" good \"),\n",
    "                          (\";)\", \" good \"),\n",
    "                          (\"(-:\", \" good \"),\n",
    "                          (\"(:\", \" good \"),\n",
    "                          (\"yay!\", \" good \"),\n",
    "                          (\"yay\", \" good \"),\n",
    "                          (\"yaay\", \" good \"),\n",
    "                          (\"yaaay\", \" good \"),\n",
    "                          (\"yaaaay\", \" good \"),\n",
    "                          (\"yaaaaay\", \" good \"),\n",
    "                          (\":/\", \" bad \"),\n",
    "                          (\":&gt;\", \" sad \"),\n",
    "                          (\":')\", \" sad \"),\n",
    "                          (\":-(\", \" bad \"),\n",
    "                          (\":(\", \" bad \"),\n",
    "                          (\":s\", \" bad \"),\n",
    "                          (\":-s\", \" bad \"),\n",
    "                          (\"&lt;3\", \" heart \"),\n",
    "                          (\":d\", \" smile \"),\n",
    "                          (\":p\", \" smile \"),\n",
    "                          (\":dd\", \" smile \"),\n",
    "                          (\"8)\", \" smile \"),\n",
    "                          (\":-)\", \" smile \"),\n",
    "                          (\":)\", \" smile \"),\n",
    "                          (\";)\", \" smile \"),\n",
    "                          (\"(-:\", \" smile \"),\n",
    "                          (\"(:\", \" smile \"),\n",
    "                          (\":/\", \" worry \"),\n",
    "                          (\":&gt;\", \" angry \"),\n",
    "                          (\":')\", \" sad \"),\n",
    "                          (\":-(\", \" sad \"),\n",
    "                          (\":(\", \" sad \"),\n",
    "                          (\":s\", \" sad \"),\n",
    "                          (\":-s\", \" sad \"),\n",
    "                          (r\"\\br\\b\", \"are\"),\n",
    "                          (r\"\\bu\\b\", \"you\"),\n",
    "                          (r\"\\bhaha\\b\", \"ha\"),\n",
    "                          (r\"\\bhahaha\\b\", \"ha\"),\n",
    "                          (r\"\\bdon't\\b\", \"do not\"),\n",
    "                          (r\"\\bdoesn't\\b\", \"does not\"),\n",
    "                          (r\"\\bdidn't\\b\", \"did not\"),\n",
    "                          (r\"\\bhasn't\\b\", \"has not\"),\n",
    "                          (r\"\\bhaven't\\b\", \"have not\"),\n",
    "                          (r\"\\bhadn't\\b\", \"had not\"),\n",
    "                          (r\"\\bwon't\\b\", \"will not\"),\n",
    "                          (r\"\\bwouldn't\\b\", \"would not\"),\n",
    "                          (r\"\\bcan't\\b\", \"can not\"),\n",
    "                          (r\"\\bcannot\\b\", \"can not\"),\n",
    "                          (r\"\\bi'm\\b\", \"i am\"),\n",
    "                          (\"m\", \"am\"),\n",
    "                          (\"r\", \"are\"),\n",
    "                          (\"u\", \"you\"),\n",
    "                          (\"haha\", \"ha\"),\n",
    "                          (\"hahaha\", \"ha\"),\n",
    "                          (\"don't\", \"do not\"),\n",
    "                          (\"doesn't\", \"does not\"),\n",
    "                          (\"didn't\", \"did not\"),\n",
    "                          (\"hasn't\", \"has not\"),\n",
    "                          (\"haven't\", \"have not\"),\n",
    "                          (\"hadn't\", \"had not\"),\n",
    "                          (\"won't\", \"will not\"),\n",
    "                          (\"wouldn't\", \"would not\"),\n",
    "                          (\"can't\", \"can not\"),\n",
    "                          (\"cannot\", \"can not\"),\n",
    "                          (\"i'm\", \"i am\"),\n",
    "                          (\"m\", \"am\"),\n",
    "                          (\"i'll\" , \"i will\"),\n",
    "                          (\"its\" , \"it is\"),\n",
    "                          (\"it's\" , \"it is\"),\n",
    "                          (\"'s\" , \" is\"),\n",
    "                          (\"that's\" , \"that is\"),\n",
    "                          (\"weren't\" , \"were not\")\n",
    "    ]\n",
    "    \n",
    "    # Clean the text, with the option to remove stop_words and to stem words.\n",
    "    text                = text.lower()\n",
    "    \n",
    "    for t in NEG_CONTRACTIONS:\n",
    "            text        = re.sub(t[0], t[1], text)\n",
    "\n",
    "\n",
    "    for t in OTHER_CONTRACTIONS:\n",
    "            text        = re.sub(t[0], t[1], text)\n",
    "    for t in OTHER_RPS:\n",
    "            #print(t)\n",
    "            text.replace(t[0], t[1])\n",
    "            #text = re.sub(t[0], t[1], text)    \n",
    "    \n",
    "    # Clean the text\n",
    "    if with_punct_sent:\n",
    "      text              = re.sub(r\"[^A-Za-z0-9!.?]\", \" \", text)\n",
    "    else: \n",
    "      text              = re.sub(r\"[^A-Za-z0-9]\", \" \", text)\n",
    "    \n",
    "    text                = re.sub(r\"what's\", \"\", text)\n",
    "    #text               = re.sub(r\"What's\", \"\", text)\n",
    "    text                = re.sub(r\"\\'s\", \" \", text)\n",
    "    text                = re.sub(r\"\\'ve\", \" have \", text)\n",
    "    text                = re.sub(r\"can\\'t\", \"cannot \", text)\n",
    "    text                = re.sub(r\"n\\'t\", \" not \", text)\n",
    "    text                = re.sub(r\"i\\'m\", \"i am\", text)\n",
    "    text                = re.sub(r\" m \", \" am \", text)\n",
    "    text                = re.sub(r\"\\'re\", \" are \", text)\n",
    "    text                = re.sub(r\"\\'d\", \" would \", text)\n",
    "    text                = re.sub(r\"\\'ll\", \" will \", text)\n",
    "    text                = re.sub('[0-9]+', '', text)\n",
    "   \n",
    "    \n",
    "    \n",
    "    #if with_punct_sent==False:\n",
    "    #    pass\n",
    "        #text = ''.join([c for c in text if c not in '!\"#$%&\\'()*+,-./:;<=>?@[\\\\]^_`{|}~'])\n",
    "    #else: \n",
    "     #   text = ''.join([c for c in text if c not in '!.?'])\n",
    "        \n",
    "    # Optionally, remove stop words\n",
    "    if remove_stop_words:\n",
    "        text           = text.split()\n",
    "        text           = [w for w in text if not w in stop_words]\n",
    "        text           = \" \".join(text)\n",
    "\n",
    "    # Optionally, shorten words to their stems\n",
    "    if stem_words:\n",
    "        text           = text.split()\n",
    "        stemmer        = SnowballStemmer('english')\n",
    "        stemmed_words  = [stemmer.stem(word) for word in text]\n",
    "        text           = \" \".join(stemmed_words)\n",
    "   \n",
    "    ltr = text.split()\n",
    "    new_data = []\n",
    "    for i in ltr:\n",
    "        arr = str(i).split()\n",
    "        xx = \"\"\n",
    "        for j in arr:\n",
    "            j = str(j).lower()\n",
    "            if j[:4] == 'http' or j[:3] == 'www':\n",
    "                continue\n",
    "            xx += j + ' '\n",
    "        new_data.append(xx)\n",
    "    text = ''.join(new_data)\n",
    "    return(text)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 85
    },
    "colab_type": "code",
    "executionInfo": {
     "elapsed": 64434,
     "status": "ok",
     "timestamp": 1559806882874,
     "user": {
      "displayName": "Elizaveta",
      "photoUrl": "https://lh3.googleusercontent.com/-3-3kSLC4Mzw/AAAAAAAAAAI/AAAAAAAAABg/Usb9K3n3cRI/s64/photo.jpg",
      "userId": "01253993997636551956"
     },
     "user_tz": -120
    },
    "id": "zKQkh-EkSZyn",
    "outputId": "1b3556cb-a324-4bf5-e29a-688d665d5a4f"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Preparing the data without any punctuation\n",
      "Train\n",
      "Test labelled\n",
      "Test unlabelled\n"
     ]
    }
   ],
   "source": [
    "print('Preparing the data without any punctuation')\n",
    "train_no_punct                            = train.copy()\n",
    "test_labelled_no_punct                    = test_labelled.copy()\n",
    "test_unlabelled_no_punct                  = test_unlabelled.copy()\n",
    "\n",
    "print('Train')\n",
    "train_no_punct.loc[:,'comment_text']            = train['comment_text'].apply(lambda x : text_to_wordlist(x, remove_stop_words=True, stem_words=False))\n",
    "print('Test labelled')\n",
    "test_labelled_no_punct.loc[:,'comment_text']    = test_labelled['comment_text'].apply(lambda x : text_to_wordlist(x, remove_stop_words=True, stem_words=False))\n",
    "print('Test unlabelled')\n",
    "test_unlabelled_no_punct.loc[:,'comment_text']  = test_unlabelled['comment_text'].apply(lambda x : text_to_wordlist(x, remove_stop_words=True, stem_words=False))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "rikI5ZSYUltl"
   },
   "outputs": [],
   "source": [
    "train_no_punct.to_csv(f'{data_path}/train_cleaned_no_punkt.csv', index=False)\n",
    "test_labelled_no_punct.drop('used_for_sc', axis=1).to_csv(f'{data_path}/test_labelled_cleaned_no_punkt.csv', index=False)\n",
    "test_unlabelled_no_punct.drop('used_for_sc', axis=1).to_csv(f'{data_path}/test_unlabelled_cleaned_no_punkt.csv', index=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 90
    },
    "colab_type": "code",
    "executionInfo": {
     "elapsed": 65498,
     "status": "ok",
     "timestamp": 1559912769910,
     "user": {
      "displayName": "Elizaveta",
      "photoUrl": "https://lh3.googleusercontent.com/-3-3kSLC4Mzw/AAAAAAAAAAI/AAAAAAAAABg/Usb9K3n3cRI/s64/photo.jpg",
      "userId": "01253993997636551956"
     },
     "user_tz": -120
    },
    "id": "LoqeUel9VWOx",
    "outputId": "8194fa73-7489-409e-915f-fc2c9bbedbeb"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Preparing the data with sentence punctuation\n",
      "Train\n",
      "Test labelled\n",
      "Test unlabelled\n"
     ]
    }
   ],
   "source": [
    "print('Preparing the data with sentence punctuation')\n",
    "train_sent_punct = train.copy()\n",
    "test_labelled_sent_punct = test_labelled.copy()\n",
    "test_unlabelled_sent_punct = test_unlabelled.copy()\n",
    "\n",
    "print('Train')\n",
    "train_sent_punct.loc[:,'comment_text']            = train['comment_text'].apply(lambda x : text_to_wordlist(x, remove_stop_words=True, stem_words=False, with_punct_sent=True))\n",
    "print('Test labelled')\n",
    "test_labelled_sent_punct.loc[:,'comment_text']    = test_labelled['comment_text'].apply(lambda x : text_to_wordlist(x, remove_stop_words=True, stem_words=False, with_punct_sent=True))\n",
    "print('Test unlabelled')\n",
    "test_unlabelled_sent_punct.loc[:,'comment_text']  = test_unlabelled['comment_text'].apply(lambda x : text_to_wordlist(x, remove_stop_words=True, stem_words=False, with_punct_sent=True))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "bGjZHk_vV7p0"
   },
   "outputs": [],
   "source": [
    "train_sent_punct.to_csv(f'{data_path}/train_cleaned_sent_punkt.csv', index=False)\n",
    "test_labelled_sent_punct.drop('used_for_sc', axis=1).to_csv(f'{data_path}/test_labelled_cleaned_sent_punkt.csv', index=False)\n",
    "test_unlabelled_sent_punct.drop('used_for_sc', axis=1).to_csv(f'{data_path}/test_unlabelled_cleaned_sent_punkt.csv', index=False)"
   ]
  }
 ],
 "metadata": {
  "colab": {
   "collapsed_sections": [],
   "name": "code_1_preparation.ipynb",
   "provenance": [],
   "version": "0.3.2"
  },
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.8"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}