AOBDL_data_preparation/AOBDL_data_preparation_twitter.ipynb

{"nbformat":4,"nbformat_minor":0,"metadata":{"colab":{"name":"AOBDL_data_preparation_twitter.ipynb","provenance":[],"collapsed_sections":[]},"kernelspec":{"display_name":"Python 3","language":"python","name":"python3"},"language_info":{"codemirror_mode":{"name":"ipython","version":3},"file_extension":".py","mimetype":"text/x-python","name":"python","nbconvert_exporter":"python","pygments_lexer":"ipython3","version":"3.6.8"}},"cells":[{"cell_type":"code","metadata":{"colab_type":"code","id":"CwSVBuG-E6X6","colab":{}},"source":["# MODULES IMPORT \n","\n","import numpy as np\n","import pandas as pd\n","import pdb\n","import os"],"execution_count":0,"outputs":[]},{"cell_type":"code","metadata":{"colab_type":"code","executionInfo":{"status":"ok","timestamp":1586617522946,"user_tz":-120,"elapsed":1281,"user":{"displayName":"Elizaveta","photoUrl":"https://lh3.googleusercontent.com/a-/AOh14GhRMtJEZoFvbzkQ0Tdx4cRRyl0XsdQGL6RF1O95=s64","userId":"01253993997636551956"}},"id":"YB0lSfPVEpA9","outputId":"479d19e4-d235-4796-904f-86f1c1b0d492","colab":{"base_uri":"https://localhost:8080/","height":34}},"source":["# GOOGLE COLAB SETUP\n","\n","# Load the Drive helper and mount\n","if os.name=='posix':\n","    from google.colab import drive\n","\n","    # This will prompt for authorization.\n","    drive.mount('/content/drive')"],"execution_count":0,"outputs":[{"output_type":"stream","text":["Drive already mounted at /content/drive; to attempt to forcibly remount, call drive.mount(\"/content/drive\", force_remount=True).\n"],"name":"stdout"}]},{"cell_type":"code","metadata":{"colab_type":"code","id":"7esVSpP-E5ZD","colab":{}},"source":["# VARIABLES AND PATHS\n","if os.name=='posix':\n","    raw_data_path = 'drive/My Drive/Colab Notebooks/adaptHAN/AOBDL_code/raw_data'\n","    data_path     = 'drive/My Drive/Colab Notebooks/adaptHAN/AOBDL_code/data'\n","    codes_path    = 'drive/My Drive/Colab Notebooks/adaptHAN/AOBDL_code/AOBDL_data_preparation'\n","else:\n","    raw_data_path = '../raw_data'\n","    data_path     = '../data'\n","    codes_path    = '../AOBDL_data_preparation'\n","\n","\n","#3. Read file as panda dataframe\n","df = pd.read_csv(f'{raw_data_path}/labeled_data_twitter_davidson.csv', index_col=0) "],"execution_count":0,"outputs":[]},{"cell_type":"code","metadata":{"id":"FrdUxsDZOXX1","colab_type":"code","colab":{}},"source":["# Create target Directory if don't exist\n","if not os.path.exists(data_path):\n","    os.mkdir(data_path)\n","    print(\"Directory \" , data_path ,  \" Created \")"],"execution_count":0,"outputs":[]},{"cell_type":"code","metadata":{"id":"HpMhjlsTyfVT","colab_type":"code","colab":{}},"source":["df['mal'] = 0\n","df.loc[df['class']==0,'mal'] = 1\n","df.loc[df['class']==1,'mal'] = 1"],"execution_count":0,"outputs":[]},{"cell_type":"code","metadata":{"colab_type":"code","id":"VhBEDM5JLots","colab":{}},"source":["# Clean data completely \n","def substitute_repeats_fixed_len(text, nchars, ntimes=3):\n","    return re.sub(r\"(\\S{{{}}})(\\1{{{},}})\".format(nchars, ntimes-1), r\"\\1\", text)\n","  \n","def substitute_repeats(text, ntimes=3):\n","    for nchars in range(1, 20):\n","        text         = substitute_repeats_fixed_len(text, nchars, ntimes)\n","    return text\n","\n","def text_to_wordlist(text, remove_stop_words=True, stem_words=False, with_punct_sent=False):\n","    stop_words       = set(['a', 'the', \"an\", \"are\", \"as\",  'did',\n","                       \"do\", \"is\", \"has\", \"have\", \"had\", \"was\", \"were\",\n","                       \"will\", \"would\", \"am\", \"it\", \"for\", \"on\", \"it\", \"of\"])\n","    #from string import punctuation\n","    import re\n","    NEG_CONTRACTIONS    = [\n","                          (r'aren\\'t', 'are not'),\n","                          (r'can\\'t', 'can not'),\n","                          (r'couldn\\'t', 'could not'),\n","                          (r'daren\\'t', 'dare not'),\n","                          (r'didn\\'t', 'did not'),\n","                          (r'doesn\\'t', 'does not'),\n","                          (r'don\\'t', 'do not'),\n","                          (r'isn\\'t', 'is not'),\n","                          (r'hasn\\'t', 'has not'),\n","                          (r'haven\\'t', 'have not'),\n","                          (r'hadn\\'t', 'had not'),\n","                          (r'mayn\\'t', 'may not'),\n","                          (r'mightn\\'t', 'might not'),\n","                          (r'mustn\\'t', 'must not'),\n","                          (r'needn\\'t', 'need not'),\n","                          (r'oughtn\\'t', 'ought not'),\n","                          (r'shan\\'t', 'shall not'),\n","                          (r'shouldn\\'t', 'should not'),\n","                          (r'wasn\\'t', 'was not'),\n","                          (r'weren\\'t', 'were not'),\n","                          (r'won\\'t', 'will not'),\n","                          (r'wouldn\\'t', 'would not'),\n","                          (r'ain\\'t', 'am not') # not only but stopword anyway\n","                          ]\n","    OTHER_CONTRACTIONS = [\n","                          #(r\"'m\", 'am'),\n","                          (r\"'ll\", ' will'),\n","                          (r\"'s\", ' has'), # or 'is' but both are stopwords\n","                          (r\"'d\", ' had'), # or 'would' but both are stopwords\n","                          (r\"'ve\", \" have\"),\n","                           (r\"'re\", \" are\")   \n","    ]\n","    OTHER_RPS          = [\n","                          (\"&lt;3\", \" good \"),\n","                          (\":d\", \" good \"),\n","                          (\":dd\", \" good \"),\n","                          (\":p\", \" good \"),\n","                          (\"8)\", \" good \"),\n","                          (\":-)\", \" good \"),\n","                          (\":)\", \" good \"),\n","                          (\";)\", \" good \"),\n","                          (\"(-:\", \" good \"),\n","                          (\"(:\", \" good \"),\n","                          (\"yay!\", \" good \"),\n","                          (\"yay\", \" good \"),\n","                          (\"yaay\", \" good \"),\n","                          (\"yaaay\", \" good \"),\n","                          (\"yaaaay\", \" good \"),\n","                          (\"yaaaaay\", \" good \"),\n","                          (\":/\", \" bad \"),\n","                          (\":&gt;\", \" sad \"),\n","                          (\":')\", \" sad \"),\n","                          (\":-(\", \" bad \"),\n","                          (\":(\", \" bad \"),\n","                          (\":s\", \" bad \"),\n","                          (\":-s\", \" bad \"),\n","                          (\"&lt;3\", \" heart \"),\n","                          (\":d\", \" smile \"),\n","                          (\":p\", \" smile \"),\n","                          (\":dd\", \" smile \"),\n","                          (\"8)\", \" smile \"),\n","                          (\":-)\", \" smile \"),\n","                          (\":)\", \" smile \"),\n","                          (\";)\", \" smile \"),\n","                          (\"(-:\", \" smile \"),\n","                          (\"(:\", \" smile \"),\n","                          (\":/\", \" worry \"),\n","                          (\":&gt;\", \" angry \"),\n","                          (\":')\", \" sad \"),\n","                          (\":-(\", \" sad \"),\n","                          (\":(\", \" sad \"),\n","                          (\":s\", \" sad \"),\n","                          (\":-s\", \" sad \"),\n","                          (r\"\\br\\b\", \"are\"),\n","                          (r\"\\bu\\b\", \"you\"),\n","                          (r\"\\bhaha\\b\", \"ha\"),\n","                          (r\"\\bhahaha\\b\", \"ha\"),\n","                          (r\"\\bdon't\\b\", \"do not\"),\n","                          (r\"\\bdoesn't\\b\", \"does not\"),\n","                          (r\"\\bdidn't\\b\", \"did not\"),\n","                          (r\"\\bhasn't\\b\", \"has not\"),\n","                          (r\"\\bhaven't\\b\", \"have not\"),\n","                          (r\"\\bhadn't\\b\", \"had not\"),\n","                          (r\"\\bwon't\\b\", \"will not\"),\n","                          (r\"\\bwouldn't\\b\", \"would not\"),\n","                          (r\"\\bcan't\\b\", \"can not\"),\n","                          (r\"\\bcannot\\b\", \"can not\"),\n","                          (r\"\\bi'm\\b\", \"i am\"),\n","                          (\"m\", \"am\"),\n","                          (\"r\", \"are\"),\n","                          (\"u\", \"you\"),\n","                          (\"haha\", \"ha\"),\n","                          (\"hahaha\", \"ha\"),\n","                          (\"don't\", \"do not\"),\n","                          (\"doesn't\", \"does not\"),\n","                          (\"didn't\", \"did not\"),\n","                          (\"hasn't\", \"has not\"),\n","                          (\"haven't\", \"have not\"),\n","                          (\"hadn't\", \"had not\"),\n","                          (\"won't\", \"will not\"),\n","                          (\"wouldn't\", \"would not\"),\n","                          (\"can't\", \"can not\"),\n","                          (\"cannot\", \"can not\"),\n","                          (\"i'm\", \"i am\"),\n","                          (\"m\", \"am\"),\n","                          (\"i'll\" , \"i will\"),\n","                          (\"its\" , \"it is\"),\n","                          (\"it's\" , \"it is\"),\n","                          (\"'s\" , \" is\"),\n","                          (\"that's\" , \"that is\"),\n","                          (\"weren't\" , \"were not\")\n","    ]\n","    \n","    # Clean the text, with the option to remove stop_words and to stem words.\n","    text                = text.lower()\n","    \n","    for t in NEG_CONTRACTIONS:\n","            text        = re.sub(t[0], t[1], text)\n","\n","\n","    for t in OTHER_CONTRACTIONS:\n","            text        = re.sub(t[0], t[1], text)\n","    for t in OTHER_RPS:\n","            #print(t)\n","            text.replace(t[0], t[1])\n","            #text = re.sub(t[0], t[1], text)    \n","    \n","    # Clean the text\n","    if with_punct_sent:\n","      text              = re.sub(r\"[^A-Za-z0-9!.?]\", \" \", text)\n","    else: \n","      text              = re.sub(r\"[^A-Za-z0-9]\", \" \", text)\n","    \n","    text                = re.sub(r\"what's\", \"\", text)\n","    #text               = re.sub(r\"What's\", \"\", text)\n","    text                = re.sub(r\"\\'s\", \" \", text)\n","    text                = re.sub(r\"\\'ve\", \" have \", text)\n","    text                = re.sub(r\"can\\'t\", \"cannot \", text)\n","    text                = re.sub(r\"n\\'t\", \" not \", text)\n","    text                = re.sub(r\"i\\'m\", \"i am\", text)\n","    text                = re.sub(r\" m \", \" am \", text)\n","    text                = re.sub(r\"\\'re\", \" are \", text)\n","    text                = re.sub(r\"\\'d\", \" would \", text)\n","    text                = re.sub(r\"\\'ll\", \" will \", text)\n","    text                = re.sub('[0-9]+', '', text)\n","   \n","    \n","    \n","    #if with_punct_sent==False:\n","    #    pass\n","        #text = ''.join([c for c in text if c not in '!\"#$%&\\'()*+,-./:;<=>?@[\\\\]^_`{|}~'])\n","    #else: \n","     #   text = ''.join([c for c in text if c not in '!.?'])\n","        \n","    # Optionally, remove stop words\n","    if remove_stop_words:\n","        text           = text.split()\n","        text           = [w for w in text if not w in stop_words]\n","        text           = \" \".join(text)\n","\n","    # Optionally, shorten words to their stems\n","    if stem_words:\n","        text           = text.split()\n","        stemmer        = SnowballStemmer('english')\n","        stemmed_words  = [stemmer.stem(word) for word in text]\n","        text           = \" \".join(stemmed_words)\n","   \n","    ltr = text.split()\n","    new_data = []\n","    for i in ltr:\n","        arr = str(i).split()\n","        xx = \"\"\n","        for j in arr:\n","            j = str(j).lower()\n","            if j[:4] == 'http' or j[:3] == 'www':\n","                continue\n","            xx += j + ' '\n","        new_data.append(xx)\n","    text = ''.join(new_data)\n","    return(text)"],"execution_count":0,"outputs":[]},{"cell_type":"code","metadata":{"colab_type":"code","executionInfo":{"status":"ok","timestamp":1586617989573,"user_tz":-120,"elapsed":2910,"user":{"displayName":"Elizaveta","photoUrl":"https://lh3.googleusercontent.com/a-/AOh14GhRMtJEZoFvbzkQ0Tdx4cRRyl0XsdQGL6RF1O95=s64","userId":"01253993997636551956"}},"id":"zKQkh-EkSZyn","outputId":"46aaae66-c475-486c-b1b7-4ee9b3595b2c","colab":{"base_uri":"https://localhost:8080/","height":51}},"source":["print('Preparing the data without any punctuation')\n","df_no_punct = df.copy()\n","\n","print('Train')\n","df_no_punct.loc[:,'tweet'] = df['tweet'].apply(lambda x : text_to_wordlist(x, remove_stop_words=True, stem_words=False))"],"execution_count":0,"outputs":[{"output_type":"stream","text":["Preparing the data without any punctuation\n","Train\n"],"name":"stdout"}]},{"cell_type":"code","metadata":{"colab_type":"code","id":"rikI5ZSYUltl","colab":{}},"source":["df_no_punct.to_csv(f'{data_path}/twitter_cleaned_no_punkt.csv', index=False)"],"execution_count":0,"outputs":[]},{"cell_type":"code","metadata":{"colab_type":"code","executionInfo":{"status":"ok","timestamp":1586618117985,"user_tz":-120,"elapsed":3074,"user":{"displayName":"Elizaveta","photoUrl":"https://lh3.googleusercontent.com/a-/AOh14GhRMtJEZoFvbzkQ0Tdx4cRRyl0XsdQGL6RF1O95=s64","userId":"01253993997636551956"}},"id":"LoqeUel9VWOx","outputId":"4a2129f4-59b8-4c4e-ba15-7939ed9126b6","colab":{"base_uri":"https://localhost:8080/","height":51}},"source":["print('Preparing the data with sentence punctuation')\n","df_sent_punct = df.copy()\n","\n","print('Train')\n","df_sent_punct.loc[:,'tweet'] = df['tweet'].apply(lambda x : text_to_wordlist(x, remove_stop_words=True, stem_words=False, with_punct_sent=True))"],"execution_count":0,"outputs":[{"output_type":"stream","text":["Preparing the data with sentence punctuation\n","Train\n"],"name":"stdout"}]},{"cell_type":"code","metadata":{"colab_type":"code","id":"bGjZHk_vV7p0","colab":{}},"source":["df_sent_punct.to_csv(f'{data_path}/twitter_cleaned_sent_punkt.csv', index=False)"],"execution_count":0,"outputs":[]},{"cell_type":"code","metadata":{"id":"mGX_1bbxz4-C","colab_type":"code","colab":{}},"source":[""],"execution_count":0,"outputs":[]}]}