Assignment-4LatentSemanticAnalysis.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[nltk_data] Downloading package stopwords to\n",
      "[nltk_data]     C:\\Users\\bansispc\\AppData\\Roaming\\nltk_data...\n",
      "[nltk_data]   Package stopwords is already up-to-date!\n"
     ]
    }
   ],
   "source": [
    "import nltk\n",
    "from nltk.tokenize import word_tokenize, sent_tokenize\n",
    "import pandas as pd\n",
    "from sklearn.feature_extraction.text import TfidfVectorizer\n",
    "from sklearn.decomposition import TruncatedSVD\n",
    "nltk.download('stopwords')\n",
    "from nltk.corpus import stopwords"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [],
   "source": [
    "from nltk.corpus import stopwords\n",
    "import pandas as pd"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0     [{'text': ['🙏', '🇺🇸', ' Peace to all politics'...\n",
       "1                                                    []\n",
       "2                                                    []\n",
       "3                                                    []\n",
       "4                                                    []\n",
       "5                                                    []\n",
       "6                                                    []\n",
       "7     [{'text': ['It was hilarious. He represents th...\n",
       "8     [{'text': ['Yes I do, but at the expense of ma...\n",
       "9     [{'text': ['Please keep seeking God’s will for...\n",
       "10    [{'text': ['Good this destruction is costing m...\n",
       "11    [{'text': [\"How can the media get away with su...\n",
       "12    [{'text': ['Thankful for you getting the news ...\n",
       "13    [{'text': ['Is this the same Fox News that you...\n",
       "14    [{'text': [\"Please sir bring in the national g...\n",
       "15    [{'text': ['PRESIDENT YOUR AWESOME.No one else...\n",
       "16                                                   []\n",
       "17                                                   []\n",
       "18                                                   []\n",
       "19                                                   []\n",
       "20    [{'text': ['Thank you for spending your late y...\n",
       "21                                                   []\n",
       "22                                                   []\n",
       "23                                                   []\n",
       "24    [{'text': ['How well has it helped you?', \" Sh...\n",
       "Name: comments, dtype: object"
      ]
     },
     "execution_count": 20,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df=pd.read_csv('profile_data.csv')\n",
    "df['comments']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>index</th>\n",
       "      <th>comments</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>0</td>\n",
       "      <td>[[{'text': ['🙏',  '🇺🇸',  ' Peace to all politi...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>1</td>\n",
       "      <td>[[]]</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>2</td>\n",
       "      <td>[[]]</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>3</td>\n",
       "      <td>[[]]</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>4</td>\n",
       "      <td>[[]]</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>5</td>\n",
       "      <td>[[]]</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6</th>\n",
       "      <td>6</td>\n",
       "      <td>[[]]</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7</th>\n",
       "      <td>7</td>\n",
       "      <td>[[{'text': ['It was hilarious. He represents t...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8</th>\n",
       "      <td>8</td>\n",
       "      <td>[[{'text': ['Yes I do,  but at the expense of ...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9</th>\n",
       "      <td>9</td>\n",
       "      <td>[[{'text': ['Please keep seeking God’s will fo...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>10</th>\n",
       "      <td>10</td>\n",
       "      <td>[[{'text': ['Good this destruction is costing ...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>11</th>\n",
       "      <td>11</td>\n",
       "      <td>[[{'text': [\"How can the media get away with s...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>12</th>\n",
       "      <td>12</td>\n",
       "      <td>[[{'text': ['Thankful for you getting the news...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>13</th>\n",
       "      <td>13</td>\n",
       "      <td>[[{'text': ['Is this the same Fox News that yo...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>14</th>\n",
       "      <td>14</td>\n",
       "      <td>[[{'text': [\"Please sir bring in the national ...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>15</th>\n",
       "      <td>15</td>\n",
       "      <td>[[{'text': ['PRESIDENT YOUR AWESOME.No one els...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>16</th>\n",
       "      <td>16</td>\n",
       "      <td>[[]]</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>17</th>\n",
       "      <td>17</td>\n",
       "      <td>[[]]</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>18</th>\n",
       "      <td>18</td>\n",
       "      <td>[[]]</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>19</th>\n",
       "      <td>19</td>\n",
       "      <td>[[]]</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "    index                                           comments\n",
       "0       0  [[{'text': ['🙏',  '🇺🇸',  ' Peace to all politi...\n",
       "1       1                                               [[]]\n",
       "2       2                                               [[]]\n",
       "3       3                                               [[]]\n",
       "4       4                                               [[]]\n",
       "5       5                                               [[]]\n",
       "6       6                                               [[]]\n",
       "7       7  [[{'text': ['It was hilarious. He represents t...\n",
       "8       8  [[{'text': ['Yes I do,  but at the expense of ...\n",
       "9       9  [[{'text': ['Please keep seeking God’s will fo...\n",
       "10     10  [[{'text': ['Good this destruction is costing ...\n",
       "11     11  [[{'text': [\"How can the media get away with s...\n",
       "12     12  [[{'text': ['Thankful for you getting the news...\n",
       "13     13  [[{'text': ['Is this the same Fox News that yo...\n",
       "14     14  [[{'text': [\"Please sir bring in the national ...\n",
       "15     15  [[{'text': ['PRESIDENT YOUR AWESOME.No one els...\n",
       "16     16                                               [[]]\n",
       "17     17                                               [[]]\n",
       "18     18                                               [[]]\n",
       "19     19                                               [[]]"
      ]
     },
     "execution_count": 21,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "comments=df['comments'].str.split(',').reset_index()\n",
    "comments.head(20)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>index</th>\n",
       "      <th>comments</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>0</td>\n",
       "      <td>[[{'text': ['🙏',  '🇺🇸',  ' Peace to all politi...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>1</td>\n",
       "      <td>[[]]</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>2</td>\n",
       "      <td>[[]]</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>3</td>\n",
       "      <td>[[]]</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>4</td>\n",
       "      <td>[[]]</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>5</td>\n",
       "      <td>[[]]</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6</th>\n",
       "      <td>6</td>\n",
       "      <td>[[]]</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7</th>\n",
       "      <td>7</td>\n",
       "      <td>[[{'text': ['It was hilarious. He represents t...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8</th>\n",
       "      <td>8</td>\n",
       "      <td>[[{'text': ['Yes I do,  but at the expense of ...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9</th>\n",
       "      <td>9</td>\n",
       "      <td>[[{'text': ['Please keep seeking God’s will fo...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>10</th>\n",
       "      <td>10</td>\n",
       "      <td>[[{'text': ['Good this destruction is costing ...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>11</th>\n",
       "      <td>11</td>\n",
       "      <td>[[{'text': [\"How can the media get away with s...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>12</th>\n",
       "      <td>12</td>\n",
       "      <td>[[{'text': ['Thankful for you getting the news...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>13</th>\n",
       "      <td>13</td>\n",
       "      <td>[[{'text': ['Is this the same Fox News that yo...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>14</th>\n",
       "      <td>14</td>\n",
       "      <td>[[{'text': [\"Please sir bring in the national ...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>15</th>\n",
       "      <td>15</td>\n",
       "      <td>[[{'text': ['PRESIDENT YOUR AWESOME.No one els...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>16</th>\n",
       "      <td>16</td>\n",
       "      <td>[[]]</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>17</th>\n",
       "      <td>17</td>\n",
       "      <td>[[]]</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>18</th>\n",
       "      <td>18</td>\n",
       "      <td>[[]]</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>19</th>\n",
       "      <td>19</td>\n",
       "      <td>[[]]</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>20</th>\n",
       "      <td>20</td>\n",
       "      <td>[[{'text': ['Thank you for spending your late ...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>21</th>\n",
       "      <td>21</td>\n",
       "      <td>[[]]</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>22</th>\n",
       "      <td>22</td>\n",
       "      <td>[[]]</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>23</th>\n",
       "      <td>23</td>\n",
       "      <td>[[]]</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>24</th>\n",
       "      <td>24</td>\n",
       "      <td>[[{'text': ['How well has it helped you?',  \" ...</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "    index                                           comments\n",
       "0       0  [[{'text': ['🙏',  '🇺🇸',  ' Peace to all politi...\n",
       "1       1                                               [[]]\n",
       "2       2                                               [[]]\n",
       "3       3                                               [[]]\n",
       "4       4                                               [[]]\n",
       "5       5                                               [[]]\n",
       "6       6                                               [[]]\n",
       "7       7  [[{'text': ['It was hilarious. He represents t...\n",
       "8       8  [[{'text': ['Yes I do,  but at the expense of ...\n",
       "9       9  [[{'text': ['Please keep seeking God’s will fo...\n",
       "10     10  [[{'text': ['Good this destruction is costing ...\n",
       "11     11  [[{'text': [\"How can the media get away with s...\n",
       "12     12  [[{'text': ['Thankful for you getting the news...\n",
       "13     13  [[{'text': ['Is this the same Fox News that yo...\n",
       "14     14  [[{'text': [\"Please sir bring in the national ...\n",
       "15     15  [[{'text': ['PRESIDENT YOUR AWESOME.No one els...\n",
       "16     16                                               [[]]\n",
       "17     17                                               [[]]\n",
       "18     18                                               [[]]\n",
       "19     19                                               [[]]\n",
       "20     20  [[{'text': ['Thank you for spending your late ...\n",
       "21     21                                               [[]]\n",
       "22     22                                               [[]]\n",
       "23     23                                               [[]]\n",
       "24     24  [[{'text': ['How well has it helped you?',  \" ..."
      ]
     },
     "execution_count": 22,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "comments.dropna()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {},
   "outputs": [],
   "source": [
    "list2=comments['comments'][7]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[\"[{'text': ['It was hilarious. He represents the Democrat party perfectly! '\", \" '🥴']\", \" 'profile_name': 'Alicia Nielsen'\", \" 'profile_url': '/alicia.nielsen.94'}\", \" {'text': ['It’s all corrupt. They are literally creating their own narrative. Everything scripted. Everything “perfect”. Those of us who are paying attention see it. God will win in the end.']\", \" 'profile_name': 'Blake Hickey'\", \" 'profile_url': '/blake.hickey.14'}\", \" {'text': ['That press conference was a sham.  It was obvious he knew the questions ahead of time.  Biden is very low energy and frail.  Trump2020']\", \" 'profile_name': 'Dar Garcia'\", \" 'profile_url': '/profile.php'}\", \" {'text': ['He will be given questions at the three debates']\", \" 'profile_name': 'Gail M. Fuller'\", \" 'profile_url': '/gail.m.fuller.7'}\", \" {'text': ['This was the only way he could answer the questions is to get them ahead of time!']\", \" 'profile_name': 'Lois Petelinkar'\", \" 'profile_url': '/lois.petelinkar'}\", ' {\\'text\\': [\"You\\'ve got my vote!! Thank you for being a great leader\"]', \" 'profile_name': 'Jenn Fah Fah'\", \" 'profile_url': '/jenfahfah'}\", \" {'text': ['You would have performed dismally.']\", \" 'profile_name': 'Philip Nyamai'\", \" 'profile_url': '/philip.nyamai.3'}\", \" {'text': ['Well that’s because they didn’t want him to tell the world he was going to beat “Donald Reagan again” '\", \" '🤣'\", \" '🤣'\", \" '🤣']\", \" 'profile_name': 'Erin Rachelle'\", \" 'profile_url': '/profile.php'}\", \" {'text': ['We have gotten an increase in our social security each year President Trump has been in office. During the Obama Biden Administration we did not get an increase in our social security each year. This information needs to become widespread.']\", \" 'profile_name': 'Claudia Bourgeois'\", \" 'profile_url': '/profile.php'}\", \" {'text': ['He said he was given a list of whom to take questions from and in what order with their pre-given questions to ask.'\", \" ' Baby Boy Biden mentioned twice he was going to be in trouble!'\", \" ' This man is being exploited and it is incredibly sad! Where the hell is his family who should step in and rescue him to spend the time he has left with dignity!!!!!']\", \" 'profile_name': 'Marti McConnell'\", \" 'profile_url': '/marti.kurthminer'}\", ' {\\'text\\': [\"It\\'s sort of sad really.   I believe the establishment has forced all of this on him and he truly is no longer capable of doing it.   They\\'re in a tough spot for sure.\"]', \" 'profile_name': 'Harry Windsor'\", \" 'profile_url': '/HarryLeeWindsor'}\", \" {'text': ['It’s sad his family is letting him run for office. Very obvious he was reading his answers!']\", \" 'profile_name': 'Susan Thompson'\", \" 'profile_url': '/profile.php'}\", \" {'text': ['It was terrible. Calling on reporters like they were 4th grade students asking a guest speaker planned 1st grade questions. Unbelievable what extremes they will resort to - actually is unconscionable -this is very sad for Joe Biden and his family.']\", \" 'profile_name': 'Shari Ann'\", \" 'profile_url': '/shacapp'}\", \" {'text': ['If he has been tested for cognitive decline\", ' then what were the results?  And if he gets tested regularly', \" then why?   Great debate question ']\", \" 'profile_name': 'Matt Reynolds'\", \" 'profile_url': '/financialmattrs'}\", \" {'text': ['Actually as I watched he appeared to look down on his paper on the podium while he was answering the questions. He also read from his paper the name of the person who was next. Made me wonder if he knew what the reporters would ask ahead of the press conference.']\", \" 'profile_name': 'Beverly Stevens'\", \" 'profile_url': '/beverly.stevens.96592'}\", \" {'text': ['Well he can not answer them on his own. His wife probably writes the answers for him! This should make everyone feel safe '\", \" '😂']\", \" 'profile_name': 'Cheryl Scully-Cappello'\", \" 'profile_url': '/cheryl.scullycappello'}\", \" {'text': ['He can’t remember his name let alone answers to questions of course they have to give him the answers he’s their puppet.']\", \" 'profile_name': 'Sarah Southern'\", \" 'profile_url': '/sarah.southern.58'}\", \" {'text': ['You know that he WILL NOT be showing for the debates. I am going to guess here that he will have contracted covid and Will not be able to hold them due to him being temporarily being sick AND of course forcing the necessity of a mandated vaccine produced by Glaxo where Gates sold his covid patent.']\", \" 'profile_name': 'Elaine Landry'\", \" 'profile_url': '/elaine.landry.378'}\", \" {'text': ['I feel sorry for the guy (Biden)His brain is deteriorating and no one is stopping him!!He definitely has dementia!!']\", \" 'profile_name': 'Miguel Nave'\", \" 'profile_url': '/miguel.nave.50'}\", \" {'text': ['Absolutely\", \" I watched it too and Biden knew exactly what questions were coming. '\", \" ' I cannot believe people will vote for this man. He’s hopeless. ']\", \" 'profile_name': 'Gary Williams'\", \" 'profile_url': '/profile.php'}]\"]\n"
     ]
    }
   ],
   "source": [
    "print(list2)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[\"[{'text': ['It was hilarious. He represents the Democrat party perfectly! '\", \" 'profile_url': '/blake.hickey.14'}\", \" 'profile_url': '/gail.m.fuller.7'}\", \" 'profile_url': '/jenfahfah'}\", \" '🤣'\", \" 'profile_url': '/profile.php'}\", ' {\\'text\\': [\"It\\'s sort of sad really.   I believe the establishment has forced all of this on him and he truly is no longer capable of doing it.   They\\'re in a tough spot for sure.\"]', \" {'text': ['It was terrible. Calling on reporters like they were 4th grade students asking a guest speaker planned 1st grade questions. Unbelievable what extremes they will resort to - actually is unconscionable -this is very sad for Joe Biden and his family.']\", \" 'profile_name': 'Matt Reynolds'\", \" '😂']\"]\n"
     ]
    }
   ],
   "source": [
    "list3=list2[0:-1:6]\n",
    "list4=list3[0:10]\n",
    "print(list4)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>comment</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>[{'text': ['It was hilarious. He represents th...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>'profile_url': '/blake.hickey.14'}</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>'profile_url': '/gail.m.fuller.7'}</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>'profile_url': '/jenfahfah'}</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>'🤣'</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>'profile_url': '/profile.php'}</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6</th>\n",
       "      <td>{'text': [\"It's sort of sad really.   I belie...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7</th>\n",
       "      <td>{'text': ['It was terrible. Calling on report...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8</th>\n",
       "      <td>'profile_name': 'Matt Reynolds'</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9</th>\n",
       "      <td>'😂']</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                                             comment\n",
       "0  [{'text': ['It was hilarious. He represents th...\n",
       "1                 'profile_url': '/blake.hickey.14'}\n",
       "2                 'profile_url': '/gail.m.fuller.7'}\n",
       "3                       'profile_url': '/jenfahfah'}\n",
       "4                                                '🤣'\n",
       "5                     'profile_url': '/profile.php'}\n",
       "6   {'text': [\"It's sort of sad really.   I belie...\n",
       "7   {'text': ['It was terrible. Calling on report...\n",
       "8                    'profile_name': 'Matt Reynolds'\n",
       "9                                               '😂']"
      ]
     },
     "execution_count": 26,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "from pandas import DataFrame\n",
    "df2=DataFrame(list4,columns=[\"comment\"])\n",
    "df2.head(10)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>comment</th>\n",
       "      <th>clean_comments</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>[{'text': ['It was hilarious. He represents th...</td>\n",
       "      <td>text was hilarious represents the democrat par...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>'profile_url': '/blake.hickey.14'}</td>\n",
       "      <td>profile url blake hickey</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>'profile_url': '/gail.m.fuller.7'}</td>\n",
       "      <td>profile url gail fuller</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>'profile_url': '/jenfahfah'}</td>\n",
       "      <td>profile url jenfahfah</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>'🤣'</td>\n",
       "      <td></td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                                             comment  \\\n",
       "0  [{'text': ['It was hilarious. He represents th...   \n",
       "1                 'profile_url': '/blake.hickey.14'}   \n",
       "2                 'profile_url': '/gail.m.fuller.7'}   \n",
       "3                       'profile_url': '/jenfahfah'}   \n",
       "4                                                '🤣'   \n",
       "\n",
       "                                      clean_comments  \n",
       "0  text was hilarious represents the democrat par...  \n",
       "1                           profile url blake hickey  \n",
       "2                            profile url gail fuller  \n",
       "3                              profile url jenfahfah  \n",
       "4                                                     "
      ]
     },
     "execution_count": 27,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Preprocessing\n",
    "df2['clean_comments'] = df2['comment'].str.replace(\"[^a-zA-Z#]\", \" \")\n",
    "df2['clean_comments'] = df2['clean_comments'].fillna('').apply(lambda x: ' '.join([w for w in x.split() if len(w)>2]))\n",
    "df2['clean_comments'] = df2['clean_comments'].fillna('').apply(lambda x: x.lower())\n",
    "\n",
    "df2.head()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# tokenization"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "['{', \"'text\", \"'\", ':', '[', \"'It\", 'was', 'terrible', '.', 'Calling', 'on', 'reporters', 'like', 'they', 'were', '4th', 'grade', 'students', 'asking', 'a', 'guest', 'speaker', 'planned', '1st', 'grade', 'questions', '.', 'Unbelievable', 'what', 'extremes', 'they', 'will', 'resort', 'to', '-', 'actually', 'is', 'unconscionable', '-this', 'is', 'very', 'sad', 'for', 'Joe', 'Biden', 'and', 'his', 'family', '.', \"'\", ']']\n"
     ]
    }
   ],
   "source": [
    "word_tokens=nltk.word_tokenize(df2[\"comment\"][7])\n",
    "print(word_tokens)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "['was', 'terrible', 'Calling', 'on', 'reporters', 'like', 'they', 'were', '4th', 'grade', 'students', 'asking', 'a', 'guest', 'speaker', 'planned', '1st', 'grade', 'questions', 'Unbelievable', 'what', 'extremes', 'they', 'will', 'resort', 'to', 'actually', 'is', 'unconscionable', 'is', 'very', 'sad', 'for', 'Joe', 'Biden', 'and', 'his', 'family']\n"
     ]
    }
   ],
   "source": [
    "word_tokens = nltk.word_tokenize(df2[\"comment\"][7])\n",
    "new_words= [word for word in word_tokens if word.isalnum()]\n",
    "\n",
    "print(new_words)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 43,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>comment</th>\n",
       "      <th>clean_comments</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>[{'text': ['It was hilarious. He represents th...</td>\n",
       "      <td>text was hilarious represents the democrat par...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>'profile_url': '/blake.hickey.14'}</td>\n",
       "      <td>profile url blake hickey</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>'profile_url': '/gail.m.fuller.7'}</td>\n",
       "      <td>profile url gail fuller</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>'profile_url': '/jenfahfah'}</td>\n",
       "      <td>profile url jenfahfah</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>'🤣'</td>\n",
       "      <td></td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                                             comment  \\\n",
       "0  [{'text': ['It was hilarious. He represents th...   \n",
       "1                 'profile_url': '/blake.hickey.14'}   \n",
       "2                 'profile_url': '/gail.m.fuller.7'}   \n",
       "3                       'profile_url': '/jenfahfah'}   \n",
       "4                                                '🤣'   \n",
       "\n",
       "                                      clean_comments  \n",
       "0  text was hilarious represents the democrat par...  \n",
       "1                           profile url blake hickey  \n",
       "2                            profile url gail fuller  \n",
       "3                              profile url jenfahfah  \n",
       "4                                                     "
      ]
     },
     "execution_count": 43,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df2.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 44,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([[0.        , 0.        , 0.        , 0.        , 0.        ,\n",
       "        0.        , 0.        , 0.42435658, 0.        , 0.        ,\n",
       "        0.        , 0.        , 0.        , 0.        , 0.        ,\n",
       "        0.        , 0.        , 0.        , 0.42435658, 0.        ,\n",
       "        0.        , 0.        , 0.        , 0.        , 0.42435658,\n",
       "        0.42435658, 0.        , 0.        , 0.        , 0.        ,\n",
       "        0.        , 0.        , 0.42435658, 0.        , 0.        ,\n",
       "        0.        , 0.        , 0.        , 0.        , 0.        ,\n",
       "        0.        , 0.        , 0.3156065 , 0.        , 0.        ,\n",
       "        0.        , 0.        , 0.        ],\n",
       "       [0.        , 0.        , 0.        , 0.        , 0.59870078,\n",
       "        0.        , 0.        , 0.        , 0.        , 0.        ,\n",
       "        0.        , 0.        , 0.        , 0.        , 0.        ,\n",
       "        0.        , 0.        , 0.59870078, 0.        , 0.        ,\n",
       "        0.        , 0.        , 0.        , 0.        , 0.        ,\n",
       "        0.        , 0.        , 0.        , 0.35552101, 0.        ,\n",
       "        0.        , 0.        , 0.        , 0.        , 0.        ,\n",
       "        0.        , 0.        , 0.        , 0.        , 0.        ,\n",
       "        0.        , 0.        , 0.        , 0.        , 0.        ,\n",
       "        0.        , 0.        , 0.39587821],\n",
       "       [0.        , 0.        , 0.        , 0.        , 0.        ,\n",
       "        0.        , 0.        , 0.        , 0.        , 0.        ,\n",
       "        0.        , 0.        , 0.        , 0.59870078, 0.59870078,\n",
       "        0.        , 0.        , 0.        , 0.        , 0.        ,\n",
       "        0.        , 0.        , 0.        , 0.        , 0.        ,\n",
       "        0.        , 0.        , 0.        , 0.35552101, 0.        ,\n",
       "        0.        , 0.        , 0.        , 0.        , 0.        ,\n",
       "        0.        , 0.        , 0.        , 0.        , 0.        ,\n",
       "        0.        , 0.        , 0.        , 0.        , 0.        ,\n",
       "        0.        , 0.        , 0.39587821],\n",
       "       [0.        , 0.        , 0.        , 0.        , 0.        ,\n",
       "        0.        , 0.        , 0.        , 0.        , 0.        ,\n",
       "        0.        , 0.        , 0.        , 0.        , 0.        ,\n",
       "        0.        , 0.        , 0.        , 0.        , 0.74746709,\n",
       "        0.        , 0.        , 0.        , 0.        , 0.        ,\n",
       "        0.        , 0.        , 0.        , 0.44386154, 0.        ,\n",
       "        0.        , 0.        , 0.        , 0.        , 0.        ,\n",
       "        0.        , 0.        , 0.        , 0.        , 0.        ,\n",
       "        0.        , 0.        , 0.        , 0.        , 0.        ,\n",
       "        0.        , 0.        , 0.49424678],\n",
       "       [0.        , 0.        , 0.        , 0.        , 0.        ,\n",
       "        0.        , 0.        , 0.        , 0.        , 0.        ,\n",
       "        0.        , 0.        , 0.        , 0.        , 0.        ,\n",
       "        0.        , 0.        , 0.        , 0.        , 0.        ,\n",
       "        0.        , 0.        , 0.        , 0.        , 0.        ,\n",
       "        0.        , 0.        , 0.        , 0.        , 0.        ,\n",
       "        0.        , 0.        , 0.        , 0.        , 0.        ,\n",
       "        0.        , 0.        , 0.        , 0.        , 0.        ,\n",
       "        0.        , 0.        , 0.        , 0.        , 0.        ,\n",
       "        0.        , 0.        , 0.        ],\n",
       "       [0.        , 0.        , 0.        , 0.        , 0.        ,\n",
       "        0.        , 0.        , 0.        , 0.        , 0.        ,\n",
       "        0.        , 0.        , 0.        , 0.        , 0.        ,\n",
       "        0.        , 0.        , 0.        , 0.        , 0.        ,\n",
       "        0.        , 0.        , 0.        , 0.        , 0.        ,\n",
       "        0.        , 0.59258634, 0.        , 0.70378025, 0.        ,\n",
       "        0.        , 0.        , 0.        , 0.        , 0.        ,\n",
       "        0.        , 0.        , 0.        , 0.        , 0.        ,\n",
       "        0.        , 0.        , 0.        , 0.        , 0.        ,\n",
       "        0.        , 0.        , 0.39183516],\n",
       "       [0.        , 0.        , 0.27445417, 0.        , 0.        ,\n",
       "        0.        , 0.27445417, 0.        , 0.27445417, 0.27445417,\n",
       "        0.        , 0.        , 0.27445417, 0.        , 0.        ,\n",
       "        0.        , 0.        , 0.        , 0.        , 0.        ,\n",
       "        0.        , 0.        , 0.27445417, 0.        , 0.        ,\n",
       "        0.        , 0.        , 0.        , 0.        , 0.        ,\n",
       "        0.27445417, 0.        , 0.        , 0.        , 0.        ,\n",
       "        0.23331112, 0.27445417, 0.        , 0.27445417, 0.        ,\n",
       "        0.27445417, 0.        , 0.20411966, 0.27445417, 0.27445417,\n",
       "        0.        , 0.        , 0.        ],\n",
       "       [0.20727542, 0.20727542, 0.        , 0.20727542, 0.        ,\n",
       "        0.20727542, 0.        , 0.        , 0.        , 0.        ,\n",
       "        0.20727542, 0.20727542, 0.        , 0.        , 0.        ,\n",
       "        0.41455083, 0.20727542, 0.        , 0.        , 0.        ,\n",
       "        0.20727542, 0.20727542, 0.        , 0.        , 0.        ,\n",
       "        0.        , 0.        , 0.20727542, 0.        , 0.20727542,\n",
       "        0.        , 0.20727542, 0.        , 0.20727542, 0.        ,\n",
       "        0.17620304, 0.        , 0.20727542, 0.        , 0.20727542,\n",
       "        0.        , 0.20727542, 0.15415684, 0.        , 0.        ,\n",
       "        0.20727542, 0.20727542, 0.        ],\n",
       "       [0.        , 0.        , 0.        , 0.        , 0.        ,\n",
       "        0.        , 0.        , 0.        , 0.        , 0.        ,\n",
       "        0.        , 0.        , 0.        , 0.        , 0.        ,\n",
       "        0.        , 0.        , 0.        , 0.        , 0.        ,\n",
       "        0.        , 0.        , 0.        , 0.6519643 , 0.        ,\n",
       "        0.        , 0.        , 0.        , 0.38714999, 0.        ,\n",
       "        0.        , 0.        , 0.        , 0.        , 0.6519643 ,\n",
       "        0.        , 0.        , 0.        , 0.        , 0.        ,\n",
       "        0.        , 0.        , 0.        , 0.        , 0.        ,\n",
       "        0.        , 0.        , 0.        ],\n",
       "       [0.        , 0.        , 0.        , 0.        , 0.        ,\n",
       "        0.        , 0.        , 0.        , 0.        , 0.        ,\n",
       "        0.        , 0.        , 0.        , 0.        , 0.        ,\n",
       "        0.        , 0.        , 0.        , 0.        , 0.        ,\n",
       "        0.        , 0.        , 0.        , 0.        , 0.        ,\n",
       "        0.        , 0.        , 0.        , 0.        , 0.        ,\n",
       "        0.        , 0.        , 0.        , 0.        , 0.        ,\n",
       "        0.        , 0.        , 0.        , 0.        , 0.        ,\n",
       "        0.        , 0.        , 0.        , 0.        , 0.        ,\n",
       "        0.        , 0.        , 0.        ]])"
      ]
     },
     "execution_count": 44,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# TF-IDF vector\n",
    "vectorizer = TfidfVectorizer(stop_words='english', smooth_idf=True)\n",
    "X = vectorizer.fit_transform(df2['clean_comments'])\n",
    "X.toarray()\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 45,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(10, 48)"
      ]
     },
     "execution_count": 45,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "X.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 46,
   "metadata": {},
   "outputs": [],
   "source": [
    "# SVD represent documents and terms in vectors \n",
    "svd_model = TruncatedSVD(n_components=2, algorithm='randomized', n_iter=100, random_state=122)\n",
    "lsa = svd_model.fit_transform(X)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 47,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>comment</th>\n",
       "      <th>topic_1</th>\n",
       "      <th>topic_2</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>text was hilarious represents the democrat par...</td>\n",
       "      <td>-0.0000000000000000</td>\n",
       "      <td>0.5754861366263894</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>profile url blake hickey</td>\n",
       "      <td>0.6609603712133006</td>\n",
       "      <td>-0.0000000000000001</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>profile url gail fuller</td>\n",
       "      <td>0.6609603712133005</td>\n",
       "      <td>0.0000000000000003</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>profile url jenfahfah</td>\n",
       "      <td>0.7488496308219693</td>\n",
       "      <td>0.0000000000000003</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td></td>\n",
       "      <td>0.0000000000000000</td>\n",
       "      <td>0.0000000000000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>profile url profile php</td>\n",
       "      <td>0.8112597365557778</td>\n",
       "      <td>-0.0000000000000003</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6</th>\n",
       "      <td>text sort sad really believe the establishment...</td>\n",
       "      <td>-0.0000000000000007</td>\n",
       "      <td>0.6523470512790432</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7</th>\n",
       "      <td>text was terrible calling reporters like they ...</td>\n",
       "      <td>0.0000000000000007</td>\n",
       "      <td>0.6062493157017649</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8</th>\n",
       "      <td>profile name matt reynolds</td>\n",
       "      <td>0.4191804544123812</td>\n",
       "      <td>-0.0000000000000005</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9</th>\n",
       "      <td></td>\n",
       "      <td>-0.0000000000000000</td>\n",
       "      <td>0.0000000000000000</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                                             comment             topic_1  \\\n",
       "0  text was hilarious represents the democrat par... -0.0000000000000000   \n",
       "1                           profile url blake hickey  0.6609603712133006   \n",
       "2                            profile url gail fuller  0.6609603712133005   \n",
       "3                              profile url jenfahfah  0.7488496308219693   \n",
       "4                                                     0.0000000000000000   \n",
       "5                            profile url profile php  0.8112597365557778   \n",
       "6  text sort sad really believe the establishment... -0.0000000000000007   \n",
       "7  text was terrible calling reporters like they ...  0.0000000000000007   \n",
       "8                         profile name matt reynolds  0.4191804544123812   \n",
       "9                                                    -0.0000000000000000   \n",
       "\n",
       "              topic_2  \n",
       "0  0.5754861366263894  \n",
       "1 -0.0000000000000001  \n",
       "2  0.0000000000000003  \n",
       "3  0.0000000000000003  \n",
       "4  0.0000000000000000  \n",
       "5 -0.0000000000000003  \n",
       "6  0.6523470512790432  \n",
       "7  0.6062493157017649  \n",
       "8 -0.0000000000000005  \n",
       "9  0.0000000000000000  "
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "#Documents - Topic vector\n",
    "pd.options.display.float_format = '{:,.16f}'.format\n",
    "topic_encoded_df = pd.DataFrame(lsa, columns = [\"topic_1\", \"topic_2\"])\n",
    "topic_encoded_df[\"comment\"] = df2['clean_comments']\n",
    "display(topic_encoded_df[[\"comment\", \"topic_1\", \"topic_2\"]])\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 48,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Features or words used as features \n",
    "dictionary = vectorizer.get_feature_names()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 49,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['actually',\n",
       " 'asking',\n",
       " 'believe',\n",
       " 'biden',\n",
       " 'blake',\n",
       " 'calling',\n",
       " 'capable',\n",
       " 'democrat',\n",
       " 'doing',\n",
       " 'establishment',\n",
       " 'extremes',\n",
       " 'family',\n",
       " 'forced',\n",
       " 'fuller',\n",
       " 'gail',\n",
       " 'grade',\n",
       " 'guest',\n",
       " 'hickey',\n",
       " 'hilarious',\n",
       " 'jenfahfah',\n",
       " 'joe',\n",
       " 'like',\n",
       " 'longer',\n",
       " 'matt',\n",
       " 'party',\n",
       " 'perfectly',\n",
       " 'php',\n",
       " 'planned',\n",
       " 'profile',\n",
       " 'questions',\n",
       " 'really',\n",
       " 'reporters',\n",
       " 'represents',\n",
       " 'resort',\n",
       " 'reynolds',\n",
       " 'sad',\n",
       " 'sort',\n",
       " 'speaker',\n",
       " 'spot',\n",
       " 'students',\n",
       " 'sure',\n",
       " 'terrible',\n",
       " 'text',\n",
       " 'tough',\n",
       " 'truly',\n",
       " 'unbelievable',\n",
       " 'unconscionable',\n",
       " 'url']"
      ]
     },
     "execution_count": 49,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "dictionary"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 50,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Term-Topic matrix\n",
    "encoding_matrix = pd.DataFrame(svd_model.components_, index = [\"topic_1\",\"topic_2\"], columns = (dictionary)).T\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 51,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>topic_1</th>\n",
       "      <th>topic_2</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>actually</th>\n",
       "      <td>0.0000000000000002</td>\n",
       "      <td>0.1117699046048708</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>asking</th>\n",
       "      <td>0.0000000000000002</td>\n",
       "      <td>0.1117699046048707</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>believe</th>\n",
       "      <td>-0.0000000000000002</td>\n",
       "      <td>0.1592481363400214</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>biden</th>\n",
       "      <td>0.0000000000000002</td>\n",
       "      <td>0.1117699046048708</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>blake</th>\n",
       "      <td>0.1744503366444961</td>\n",
       "      <td>-0.0000000000000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>calling</th>\n",
       "      <td>0.0000000000000002</td>\n",
       "      <td>0.1117699046048708</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>capable</th>\n",
       "      <td>-0.0000000000000002</td>\n",
       "      <td>0.1592481363400214</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>democrat</th>\n",
       "      <td>-0.0000000000000001</td>\n",
       "      <td>0.2172159091648397</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>doing</th>\n",
       "      <td>-0.0000000000000002</td>\n",
       "      <td>0.1592481363400214</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>establishment</th>\n",
       "      <td>-0.0000000000000002</td>\n",
       "      <td>0.1592481363400215</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>extremes</th>\n",
       "      <td>0.0000000000000002</td>\n",
       "      <td>0.1117699046048708</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>family</th>\n",
       "      <td>0.0000000000000002</td>\n",
       "      <td>0.1117699046048708</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>forced</th>\n",
       "      <td>-0.0000000000000002</td>\n",
       "      <td>0.1592481363400214</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>fuller</th>\n",
       "      <td>0.1744503366444961</td>\n",
       "      <td>0.0000000000000001</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>gail</th>\n",
       "      <td>0.1744503366444961</td>\n",
       "      <td>0.0000000000000001</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>grade</th>\n",
       "      <td>0.0000000000000004</td>\n",
       "      <td>0.2235398092097417</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>guest</th>\n",
       "      <td>0.0000000000000002</td>\n",
       "      <td>0.1117699046048708</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>hickey</th>\n",
       "      <td>0.1744503366444961</td>\n",
       "      <td>-0.0000000000000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>hilarious</th>\n",
       "      <td>-0.0000000000000001</td>\n",
       "      <td>0.2172159091648397</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>jenfahfah</th>\n",
       "      <td>0.2467591473733696</td>\n",
       "      <td>0.0000000000000001</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>joe</th>\n",
       "      <td>0.0000000000000002</td>\n",
       "      <td>0.1117699046048708</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>like</th>\n",
       "      <td>0.0000000000000002</td>\n",
       "      <td>0.1117699046048708</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>longer</th>\n",
       "      <td>-0.0000000000000002</td>\n",
       "      <td>0.1592481363400214</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>matt</th>\n",
       "      <td>0.1204790126125470</td>\n",
       "      <td>-0.0000000000000003</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>party</th>\n",
       "      <td>-0.0000000000000001</td>\n",
       "      <td>0.2172159091648397</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>perfectly</th>\n",
       "      <td>-0.0000000000000001</td>\n",
       "      <td>0.2172159091648397</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>php</th>\n",
       "      <td>0.2119327737629347</td>\n",
       "      <td>-0.0000000000000002</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>planned</th>\n",
       "      <td>0.0000000000000002</td>\n",
       "      <td>0.1117699046048708</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>profile</th>\n",
       "      <td>0.6769583672081703</td>\n",
       "      <td>-0.0000000000000003</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>questions</th>\n",
       "      <td>0.0000000000000002</td>\n",
       "      <td>0.1117699046048708</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>really</th>\n",
       "      <td>-0.0000000000000002</td>\n",
       "      <td>0.1592481363400214</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>reporters</th>\n",
       "      <td>0.0000000000000002</td>\n",
       "      <td>0.1117699046048708</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>represents</th>\n",
       "      <td>-0.0000000000000001</td>\n",
       "      <td>0.2172159091648397</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>resort</th>\n",
       "      <td>0.0000000000000002</td>\n",
       "      <td>0.1117699046048708</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>reynolds</th>\n",
       "      <td>0.1204790126125470</td>\n",
       "      <td>-0.0000000000000003</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>sad</th>\n",
       "      <td>-0.0000000000000000</td>\n",
       "      <td>0.2303900950055568</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>sort</th>\n",
       "      <td>-0.0000000000000002</td>\n",
       "      <td>0.1592481363400214</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>speaker</th>\n",
       "      <td>0.0000000000000002</td>\n",
       "      <td>0.1117699046048708</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>spot</th>\n",
       "      <td>-0.0000000000000002</td>\n",
       "      <td>0.1592481363400214</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>students</th>\n",
       "      <td>0.0000000000000002</td>\n",
       "      <td>0.1117699046048708</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>sure</th>\n",
       "      <td>-0.0000000000000002</td>\n",
       "      <td>0.1592481363400214</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>terrible</th>\n",
       "      <td>0.0000000000000002</td>\n",
       "      <td>0.1117699046048708</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>text</th>\n",
       "      <td>-0.0000000000000001</td>\n",
       "      <td>0.3631139853546652</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>tough</th>\n",
       "      <td>-0.0000000000000002</td>\n",
       "      <td>0.1592481363400214</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>truly</th>\n",
       "      <td>-0.0000000000000002</td>\n",
       "      <td>0.1592481363400214</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>unbelievable</th>\n",
       "      <td>0.0000000000000002</td>\n",
       "      <td>0.1117699046048708</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>unconscionable</th>\n",
       "      <td>0.0000000000000002</td>\n",
       "      <td>0.1117699046048708</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>url</th>\n",
       "      <td>0.5340034895746139</td>\n",
       "      <td>0.0000000000000000</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                           topic_1             topic_2\n",
       "actually        0.0000000000000002  0.1117699046048708\n",
       "asking          0.0000000000000002  0.1117699046048707\n",
       "believe        -0.0000000000000002  0.1592481363400214\n",
       "biden           0.0000000000000002  0.1117699046048708\n",
       "blake           0.1744503366444961 -0.0000000000000000\n",
       "calling         0.0000000000000002  0.1117699046048708\n",
       "capable        -0.0000000000000002  0.1592481363400214\n",
       "democrat       -0.0000000000000001  0.2172159091648397\n",
       "doing          -0.0000000000000002  0.1592481363400214\n",
       "establishment  -0.0000000000000002  0.1592481363400215\n",
       "extremes        0.0000000000000002  0.1117699046048708\n",
       "family          0.0000000000000002  0.1117699046048708\n",
       "forced         -0.0000000000000002  0.1592481363400214\n",
       "fuller          0.1744503366444961  0.0000000000000001\n",
       "gail            0.1744503366444961  0.0000000000000001\n",
       "grade           0.0000000000000004  0.2235398092097417\n",
       "guest           0.0000000000000002  0.1117699046048708\n",
       "hickey          0.1744503366444961 -0.0000000000000000\n",
       "hilarious      -0.0000000000000001  0.2172159091648397\n",
       "jenfahfah       0.2467591473733696  0.0000000000000001\n",
       "joe             0.0000000000000002  0.1117699046048708\n",
       "like            0.0000000000000002  0.1117699046048708\n",
       "longer         -0.0000000000000002  0.1592481363400214\n",
       "matt            0.1204790126125470 -0.0000000000000003\n",
       "party          -0.0000000000000001  0.2172159091648397\n",
       "perfectly      -0.0000000000000001  0.2172159091648397\n",
       "php             0.2119327737629347 -0.0000000000000002\n",
       "planned         0.0000000000000002  0.1117699046048708\n",
       "profile         0.6769583672081703 -0.0000000000000003\n",
       "questions       0.0000000000000002  0.1117699046048708\n",
       "really         -0.0000000000000002  0.1592481363400214\n",
       "reporters       0.0000000000000002  0.1117699046048708\n",
       "represents     -0.0000000000000001  0.2172159091648397\n",
       "resort          0.0000000000000002  0.1117699046048708\n",
       "reynolds        0.1204790126125470 -0.0000000000000003\n",
       "sad            -0.0000000000000000  0.2303900950055568\n",
       "sort           -0.0000000000000002  0.1592481363400214\n",
       "speaker         0.0000000000000002  0.1117699046048708\n",
       "spot           -0.0000000000000002  0.1592481363400214\n",
       "students        0.0000000000000002  0.1117699046048708\n",
       "sure           -0.0000000000000002  0.1592481363400214\n",
       "terrible        0.0000000000000002  0.1117699046048708\n",
       "text           -0.0000000000000001  0.3631139853546652\n",
       "tough          -0.0000000000000002  0.1592481363400214\n",
       "truly          -0.0000000000000002  0.1592481363400214\n",
       "unbelievable    0.0000000000000002  0.1117699046048708\n",
       "unconscionable  0.0000000000000002  0.1117699046048708\n",
       "url             0.5340034895746139  0.0000000000000000"
      ]
     },
     "execution_count": 51,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "encoding_matrix"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.6"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}