CountVectorizer/CountVectorizer.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# CountVectorizer Usage Examples"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.feature_extraction.text import CountVectorizer"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Warm Up Example"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "\n",
    "doc=[\"One Cent, Two Cents, Old Cent, New Cent: All About Money\"]\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "cv = CountVectorizer(doc)\n",
    "count_vector=cv.fit_transform(doc)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'one': 7,\n",
       " 'cent': 2,\n",
       " 'two': 8,\n",
       " 'cents': 3,\n",
       " 'old': 6,\n",
       " 'new': 5,\n",
       " 'all': 1,\n",
       " 'about': 0,\n",
       " 'money': 4}"
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# show resulting vocabulary; the numbers are not counts, they are the position in the sparse vector.\n",
    "cv.vocabulary_"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(1, 9)"
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# matrix shape. 1 documents, 9 unique words\n",
    "count_vector.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "set()"
      ]
     },
     "execution_count": 8,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# any words eliminated internally? -- nope\n",
    "cv.stop_words_"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# CountVectorizer With More Data"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Plain and Simple"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "cat_in_the_hat_docs=[\n",
    "      \"One Cent, Two Cents, Old Cent, New Cent: All About Money (Cat in the Hat's Learning Library\",\n",
    "      \"Inside Your Outside: All About the Human Body (Cat in the Hat's Learning Library)\",\n",
    "      \"Oh, The Things You Can Do That Are Good for You: All About Staying Healthy (Cat in the Hat's Learning Library)\",\n",
    "      \"On Beyond Bugs: All About Insects (Cat in the Hat's Learning Library)\",\n",
    "      \"There's No Place Like Space: All About Our Solar System (Cat in the Hat's Learning Library)\" \n",
    "     ]\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [],
   "source": [
    "cv = CountVectorizer(cat_in_the_hat_docs)\n",
    "count_vector=cv.fit_transform(cat_in_the_hat_docs)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'one': 28,\n",
       " 'cent': 8,\n",
       " 'two': 40,\n",
       " 'cents': 9,\n",
       " 'old': 26,\n",
       " 'new': 23,\n",
       " 'all': 1,\n",
       " 'about': 0,\n",
       " 'money': 22,\n",
       " 'cat': 7,\n",
       " 'in': 16,\n",
       " 'the': 37,\n",
       " 'hat': 13,\n",
       " 'learning': 19,\n",
       " 'library': 20,\n",
       " 'inside': 18,\n",
       " 'your': 42,\n",
       " 'outside': 30,\n",
       " 'human': 15,\n",
       " 'body': 4,\n",
       " 'oh': 25,\n",
       " 'things': 39,\n",
       " 'you': 41,\n",
       " 'can': 6,\n",
       " 'do': 10,\n",
       " 'that': 36,\n",
       " 'are': 2,\n",
       " 'good': 12,\n",
       " 'for': 11,\n",
       " 'staying': 34,\n",
       " 'healthy': 14,\n",
       " 'on': 27,\n",
       " 'beyond': 3,\n",
       " 'bugs': 5,\n",
       " 'insects': 17,\n",
       " 'there': 38,\n",
       " 'no': 24,\n",
       " 'place': 31,\n",
       " 'like': 21,\n",
       " 'space': 33,\n",
       " 'our': 29,\n",
       " 'solar': 32,\n",
       " 'system': 35}"
      ]
     },
     "execution_count": 11,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# show resulting vocabulary; the numbers are not counts, they are the position in the sparse vector.\n",
    "cv.vocabulary_"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(5, 43)"
      ]
     },
     "execution_count": 14,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "#shape of count vector: 5 docs (book titles) and 43 unique words\n",
    "count_vector.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "set()"
      ]
     },
     "execution_count": 15,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "#any stop words?\n",
    "cv.stop_words_"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## CountVectorizer With Custom StopWords"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(5, 40)"
      ]
     },
     "execution_count": 21,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "cv = CountVectorizer(cat_in_the_hat_docs,stop_words=[\"all\",\"in\",\"the\",\"is\",\"and\"])\n",
    "count_vector=cv.fit_transform(cat_in_the_hat_docs)\n",
    "count_vector.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['all', 'in', 'the', 'is', 'and']"
      ]
     },
     "execution_count": 22,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "#any stop words that we explicitly specified?\n",
    "cv.stop_words"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "set()"
      ]
     },
     "execution_count": 23,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "#any stop words internally stopped by countvectorizer?\n",
    "cv.stop_words_"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## CountVectorizer With Predefined StopWords"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 435,
   "metadata": {},
   "outputs": [],
   "source": [
    "cv = CountVectorizer(cat_in_the_hat_docs,stop_words=\"english\")\n",
    "count_vector=cv.fit_transform(cat_in_the_hat_docs)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 436,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(5, 24)"
      ]
     },
     "execution_count": 436,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# the shape should be smaller\n",
    "count_vector.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 437,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'english'"
      ]
     },
     "execution_count": 437,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "#any stop words that we explicitly specified?\n",
    "cv.stop_words"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 438,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "set()"
      ]
     },
     "execution_count": 438,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "#any stop words internally stopped by countvectorizer?\n",
    "cv.stop_words_"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 439,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'cent': 3,\n",
       " 'cents': 4,\n",
       " 'old': 17,\n",
       " 'new': 15,\n",
       " 'money': 14,\n",
       " 'cat': 2,\n",
       " 'hat': 6,\n",
       " 'learning': 11,\n",
       " 'library': 12,\n",
       " 'inside': 10,\n",
       " 'outside': 18,\n",
       " 'human': 8,\n",
       " 'body': 0,\n",
       " 'oh': 16,\n",
       " 'things': 23,\n",
       " 'good': 5,\n",
       " 'staying': 22,\n",
       " 'healthy': 7,\n",
       " 'bugs': 1,\n",
       " 'insects': 9,\n",
       " 'place': 19,\n",
       " 'like': 13,\n",
       " 'space': 21,\n",
       " 'solar': 20}"
      ]
     },
     "execution_count": 439,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# much smaller vocabulary with stopwords applied\n",
    "cv.vocabulary_"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## CountVectorizer with MIN_DF as StopWords"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 440,
   "metadata": {},
   "outputs": [],
   "source": [
    "# ignore terms that appeared in less than n documents (can be proportion or absolute counts)\n",
    "cv = CountVectorizer(cat_in_the_hat_docs,min_df=2)\n",
    "count_vector=cv.fit_transform(cat_in_the_hat_docs)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 441,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'are',\n",
       " 'beyond',\n",
       " 'body',\n",
       " 'bugs',\n",
       " 'can',\n",
       " 'cent',\n",
       " 'cents',\n",
       " 'do',\n",
       " 'for',\n",
       " 'good',\n",
       " 'healthy',\n",
       " 'human',\n",
       " 'insects',\n",
       " 'inside',\n",
       " 'like',\n",
       " 'money',\n",
       " 'new',\n",
       " 'no',\n",
       " 'oh',\n",
       " 'old',\n",
       " 'on',\n",
       " 'one',\n",
       " 'our',\n",
       " 'outside',\n",
       " 'place',\n",
       " 'solar',\n",
       " 'space',\n",
       " 'staying',\n",
       " 'system',\n",
       " 'that',\n",
       " 'there',\n",
       " 'things',\n",
       " 'two',\n",
       " 'you',\n",
       " 'your'}"
      ]
     },
     "execution_count": 441,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "#any stop words internally stopped by countvectorizer?\n",
    "cv.stop_words_"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 442,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(5, 8)"
      ]
     },
     "execution_count": 442,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "count_vector.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 443,
   "metadata": {},
   "outputs": [],
   "source": [
    "# use proportion here. Ignore terms that occurred in less than 25% of the documents\n",
    "cv = CountVectorizer(cat_in_the_hat_docs,min_df=0.25)\n",
    "count_vector=cv.fit_transform(cat_in_the_hat_docs)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 444,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(5, 8)"
      ]
     },
     "execution_count": 444,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "count_vector.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 445,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'all': 1,\n",
       " 'about': 0,\n",
       " 'cat': 2,\n",
       " 'in': 4,\n",
       " 'the': 7,\n",
       " 'hat': 3,\n",
       " 'learning': 5,\n",
       " 'library': 6}"
      ]
     },
     "execution_count": 445,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "cv.vocabulary_"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 446,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'are',\n",
       " 'beyond',\n",
       " 'body',\n",
       " 'bugs',\n",
       " 'can',\n",
       " 'cent',\n",
       " 'cents',\n",
       " 'do',\n",
       " 'for',\n",
       " 'good',\n",
       " 'healthy',\n",
       " 'human',\n",
       " 'insects',\n",
       " 'inside',\n",
       " 'like',\n",
       " 'money',\n",
       " 'new',\n",
       " 'no',\n",
       " 'oh',\n",
       " 'old',\n",
       " 'on',\n",
       " 'one',\n",
       " 'our',\n",
       " 'outside',\n",
       " 'place',\n",
       " 'solar',\n",
       " 'space',\n",
       " 'staying',\n",
       " 'system',\n",
       " 'that',\n",
       " 'there',\n",
       " 'things',\n",
       " 'two',\n",
       " 'you',\n",
       " 'your'}"
      ]
     },
     "execution_count": 446,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "#any stop words internally stopped by countvectorizer?\n",
    "cv.stop_words_"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## CountVectorizer with MAX_DF as StopWords"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 447,
   "metadata": {},
   "outputs": [],
   "source": [
    "# ignore terms that appeared in more than n documents (can be proportion or absolute counts)\n",
    "# use proportion here\n",
    "cv = CountVectorizer(cat_in_the_hat_docs,max_df=0.50)\n",
    "count_vector=cv.fit_transform(cat_in_the_hat_docs)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 448,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'one': 21,\n",
       " 'cent': 5,\n",
       " 'two': 32,\n",
       " 'cents': 6,\n",
       " 'old': 19,\n",
       " 'new': 16,\n",
       " 'money': 15,\n",
       " 'inside': 13,\n",
       " 'your': 34,\n",
       " 'outside': 23,\n",
       " 'human': 11,\n",
       " 'body': 2,\n",
       " 'oh': 18,\n",
       " 'things': 31,\n",
       " 'you': 33,\n",
       " 'can': 4,\n",
       " 'do': 7,\n",
       " 'that': 29,\n",
       " 'are': 0,\n",
       " 'good': 9,\n",
       " 'for': 8,\n",
       " 'staying': 27,\n",
       " 'healthy': 10,\n",
       " 'on': 20,\n",
       " 'beyond': 1,\n",
       " 'bugs': 3,\n",
       " 'insects': 12,\n",
       " 'there': 30,\n",
       " 'no': 17,\n",
       " 'place': 24,\n",
       " 'like': 14,\n",
       " 'space': 26,\n",
       " 'our': 22,\n",
       " 'solar': 25,\n",
       " 'system': 28}"
      ]
     },
     "execution_count": 448,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "cv.vocabulary_"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 449,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'about', 'all', 'cat', 'hat', 'in', 'learning', 'library', 'the'}"
      ]
     },
     "execution_count": 449,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "cv.stop_words_"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 450,
   "metadata": {},
   "outputs": [],
   "source": [
    "# ignore terms that appeared in more than n documents (can be proportion or absolute counts)\n",
    "# use absolute values here - suitable when you know number of documents ahead of time and are dealing with only a handful\n",
    "cv = CountVectorizer(cat_in_the_hat_docs,max_df=4)\n",
    "count_vector=cv.fit_transform(cat_in_the_hat_docs)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 451,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'about', 'all', 'cat', 'hat', 'in', 'learning', 'library', 'the'}"
      ]
     },
     "execution_count": 451,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "cv.stop_words_"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Custom Preprocessing "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "metadata": {},
   "outputs": [],
   "source": [
    "import re\n",
    "import nltk\n",
    "import pandas as pd\n",
    "from nltk.stem import PorterStemmer\n",
    "\n",
    "# init stemmer\n",
    "porter_stemmer=PorterStemmer()\n",
    "\n",
    "def my_cool_preprocessor(text):\n",
    "    \n",
    "    text=text.lower() \n",
    "    text=re.sub(\"\\\\W\",\" \",text) # remove special chars\n",
    "    text=re.sub(\"\\\\s+(in|the|all|for|and|on)\\\\s+\",\" _connector_ \",text) # normalize certain words\n",
    "    \n",
    "    # stem words\n",
    "    words=re.split(\"\\\\s+\",text)\n",
    "    stemmed_words=[porter_stemmer.stem(word=word) for word in words]\n",
    "    return ' '.join(stemmed_words)\n",
    "\n",
    "cv = CountVectorizer(cat_in_the_hat_docs,preprocessor=my_cool_preprocessor)\n",
    "count_vector=cv.fit_transform(cat_in_the_hat_docs)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'one': 25,\n",
       " 'cent': 8,\n",
       " 'two': 37,\n",
       " 'old': 23,\n",
       " 'new': 20,\n",
       " '_connector_': 0,\n",
       " 'about': 1,\n",
       " 'money': 19,\n",
       " 'cat': 7,\n",
       " 'the': 34,\n",
       " 'hat': 11,\n",
       " 'learn': 16,\n",
       " 'librari': 17,\n",
       " 'insid': 15,\n",
       " 'your': 39,\n",
       " 'outsid': 27,\n",
       " 'human': 13,\n",
       " 'bodi': 4,\n",
       " 'oh': 22,\n",
       " 'thing': 36,\n",
       " 'you': 38,\n",
       " 'can': 6,\n",
       " 'do': 9,\n",
       " 'that': 33,\n",
       " 'are': 2,\n",
       " 'good': 10,\n",
       " 'stay': 31,\n",
       " 'healthi': 12,\n",
       " 'on': 24,\n",
       " 'beyond': 3,\n",
       " 'bug': 5,\n",
       " 'insect': 14,\n",
       " 'there': 35,\n",
       " 'no': 21,\n",
       " 'place': 28,\n",
       " 'like': 18,\n",
       " 'space': 30,\n",
       " 'our': 26,\n",
       " 'solar': 29,\n",
       " 'system': 32}"
      ]
     },
     "execution_count": 33,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "cv.vocabulary_"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 454,
   "metadata": {},
   "outputs": [],
   "source": [
    "cv = CountVectorizer(cat_in_the_hat_docs,preprocessor=my_cool_preprocessor)\n",
    "count_vector=cv.fit_transform(cat_in_the_hat_docs)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Working With N-Grams"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 455,
   "metadata": {},
   "outputs": [],
   "source": [
    "#only bigrams, word level\n",
    "cv = CountVectorizer(cat_in_the_hat_docs,ngram_range=(2,2),preprocessor=my_cool_preprocessor)\n",
    "count_vector=cv.fit_transform(cat_in_the_hat_docs)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 456,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'one cent': 35,\n",
       " 'cent two': 19,\n",
       " 'two cent': 47,\n",
       " 'cent old': 18,\n",
       " 'old cent': 33,\n",
       " 'cent new': 17,\n",
       " 'new cent': 30,\n",
       " 'cent _connector_': 16,\n",
       " '_connector_ about': 0,\n",
       " 'about money': 7,\n",
       " 'money cat': 29,\n",
       " 'cat _connector_': 15,\n",
       " '_connector_ the': 2,\n",
       " 'the hat': 44,\n",
       " 'hat learn': 22,\n",
       " 'learn librari': 27,\n",
       " 'insid your': 26,\n",
       " 'your outsid': 50,\n",
       " 'outsid _connector_': 37,\n",
       " 'about _connector_': 5,\n",
       " '_connector_ human': 1,\n",
       " 'human bodi': 24,\n",
       " 'bodi cat': 12,\n",
       " 'oh _connector_': 32,\n",
       " '_connector_ thing': 3,\n",
       " 'thing you': 46,\n",
       " 'you can': 49,\n",
       " 'can do': 14,\n",
       " 'do that': 20,\n",
       " 'that are': 43,\n",
       " 'are good': 10,\n",
       " 'good _connector_': 21,\n",
       " '_connector_ you': 4,\n",
       " 'you _connector_': 48,\n",
       " 'about stay': 9,\n",
       " 'stay healthi': 41,\n",
       " 'healthi cat': 23,\n",
       " 'on beyond': 34,\n",
       " 'beyond bug': 11,\n",
       " 'bug _connector_': 13,\n",
       " 'about insect': 6,\n",
       " 'insect cat': 25,\n",
       " 'there no': 45,\n",
       " 'no place': 31,\n",
       " 'place like': 38,\n",
       " 'like space': 28,\n",
       " 'space _connector_': 40,\n",
       " 'about our': 8,\n",
       " 'our solar': 36,\n",
       " 'solar system': 39,\n",
       " 'system cat': 42}"
      ]
     },
     "execution_count": 456,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "cv.vocabulary_"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 457,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(5, 51)"
      ]
     },
     "execution_count": 457,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "count_vector.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 458,
   "metadata": {},
   "outputs": [],
   "source": [
    "#only bigrams and unigrams\n",
    "cv = CountVectorizer(cat_in_the_hat_docs,ngram_range=(1,2),preprocessor=my_cool_preprocessor)\n",
    "count_vector=cv.fit_transform(cat_in_the_hat_docs)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 459,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'one': 60,\n",
       " 'cent': 24,\n",
       " 'two': 84,\n",
       " 'old': 56,\n",
       " 'new': 50,\n",
       " '_connector_': 0,\n",
       " 'about': 6,\n",
       " 'money': 48,\n",
       " 'cat': 22,\n",
       " 'the': 78,\n",
       " 'hat': 33,\n",
       " 'learn': 43,\n",
       " 'librari': 45,\n",
       " 'one cent': 61,\n",
       " 'cent two': 28,\n",
       " 'two cent': 85,\n",
       " 'cent old': 27,\n",
       " 'old cent': 57,\n",
       " 'cent new': 26,\n",
       " 'new cent': 51,\n",
       " 'cent _connector_': 25,\n",
       " '_connector_ about': 1,\n",
       " 'about money': 9,\n",
       " 'money cat': 49,\n",
       " 'cat _connector_': 23,\n",
       " '_connector_ the': 3,\n",
       " 'the hat': 79,\n",
       " 'hat learn': 34,\n",
       " 'learn librari': 44,\n",
       " 'insid': 41,\n",
       " 'your': 89,\n",
       " 'outsid': 64,\n",
       " 'human': 37,\n",
       " 'bodi': 16,\n",
       " 'insid your': 42,\n",
       " 'your outsid': 90,\n",
       " 'outsid _connector_': 65,\n",
       " 'about _connector_': 7,\n",
       " '_connector_ human': 2,\n",
       " 'human bodi': 38,\n",
       " 'bodi cat': 17,\n",
       " 'oh': 54,\n",
       " 'thing': 82,\n",
       " 'you': 86,\n",
       " 'can': 20,\n",
       " 'do': 29,\n",
       " 'that': 76,\n",
       " 'are': 12,\n",
       " 'good': 31,\n",
       " 'stay': 72,\n",
       " 'healthi': 35,\n",
       " 'oh _connector_': 55,\n",
       " '_connector_ thing': 4,\n",
       " 'thing you': 83,\n",
       " 'you can': 88,\n",
       " 'can do': 21,\n",
       " 'do that': 30,\n",
       " 'that are': 77,\n",
       " 'are good': 13,\n",
       " 'good _connector_': 32,\n",
       " '_connector_ you': 5,\n",
       " 'you _connector_': 87,\n",
       " 'about stay': 11,\n",
       " 'stay healthi': 73,\n",
       " 'healthi cat': 36,\n",
       " 'on': 58,\n",
       " 'beyond': 14,\n",
       " 'bug': 18,\n",
       " 'insect': 39,\n",
       " 'on beyond': 59,\n",
       " 'beyond bug': 15,\n",
       " 'bug _connector_': 19,\n",
       " 'about insect': 8,\n",
       " 'insect cat': 40,\n",
       " 'there': 80,\n",
       " 'no': 52,\n",
       " 'place': 66,\n",
       " 'like': 46,\n",
       " 'space': 70,\n",
       " 'our': 62,\n",
       " 'solar': 68,\n",
       " 'system': 74,\n",
       " 'there no': 81,\n",
       " 'no place': 53,\n",
       " 'place like': 67,\n",
       " 'like space': 47,\n",
       " 'space _connector_': 71,\n",
       " 'about our': 10,\n",
       " 'our solar': 63,\n",
       " 'solar system': 69,\n",
       " 'system cat': 75}"
      ]
     },
     "execution_count": 459,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "cv.vocabulary_"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 460,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(5, 91)"
      ]
     },
     "execution_count": 460,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "count_vector.shape"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Working With Character N-Grams"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 461,
   "metadata": {},
   "outputs": [],
   "source": [
    "#only character level bigrams \n",
    "cv = CountVectorizer(cat_in_the_hat_docs,ngram_range=(2,2),preprocessor=my_cool_preprocessor,analyzer='char_wb')\n",
    "count_vector=cv.fit_transform(cat_in_the_hat_docs)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 462,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{' o': 11,\n",
       " 'on': 77,\n",
       " 'ne': 67,\n",
       " 'e ': 36,\n",
       " ' c': 3,\n",
       " 'ce': 30,\n",
       " 'en': 40,\n",
       " 'nt': 72,\n",
       " 't ': 96,\n",
       " ' t': 14,\n",
       " 'tw': 102,\n",
       " 'wo': 109,\n",
       " 'o ': 73,\n",
       " 'ol': 76,\n",
       " 'ld': 58,\n",
       " 'd ': 33,\n",
       " ' n': 10,\n",
       " 'ew': 42,\n",
       " 'w ': 108,\n",
       " ' _': 0,\n",
       " '_c': 17,\n",
       " 'co': 31,\n",
       " 'nn': 69,\n",
       " 'ec': 38,\n",
       " 'ct': 32,\n",
       " 'to': 100,\n",
       " 'or': 79,\n",
       " 'r_': 84,\n",
       " '_ ': 16,\n",
       " ' a': 1,\n",
       " 'ab': 18,\n",
       " 'bo': 26,\n",
       " 'ou': 80,\n",
       " 'ut': 107,\n",
       " ' m': 9,\n",
       " 'mo': 64,\n",
       " 'ey': 43,\n",
       " 'y ': 110,\n",
       " 'ca': 29,\n",
       " 'at': 23,\n",
       " 'th': 99,\n",
       " 'he': 48,\n",
       " ' h': 6,\n",
       " 'ha': 47,\n",
       " ' s': 13,\n",
       " 's ': 89,\n",
       " ' l': 8,\n",
       " 'le': 59,\n",
       " 'ea': 37,\n",
       " 'ar': 22,\n",
       " 'rn': 88,\n",
       " 'n ': 65,\n",
       " 'li': 60,\n",
       " 'ib': 52,\n",
       " 'br': 27,\n",
       " 'ra': 85,\n",
       " 'ri': 87,\n",
       " 'i ': 51,\n",
       " ' i': 7,\n",
       " 'in': 55,\n",
       " 'ns': 71,\n",
       " 'si': 91,\n",
       " 'id': 53,\n",
       " ' y': 15,\n",
       " 'yo': 111,\n",
       " 'ur': 106,\n",
       " 'r ': 83,\n",
       " 'ts': 101,\n",
       " 'hu': 50,\n",
       " 'um': 105,\n",
       " 'ma': 63,\n",
       " 'an': 21,\n",
       " ' b': 2,\n",
       " 'od': 74,\n",
       " 'di': 34,\n",
       " 'oh': 75,\n",
       " 'h ': 46,\n",
       " 'hi': 49,\n",
       " 'ng': 68,\n",
       " 'g ': 44,\n",
       " 'u ': 103,\n",
       " ' d': 4,\n",
       " 'do': 35,\n",
       " 're': 86,\n",
       " ' g': 5,\n",
       " 'go': 45,\n",
       " 'oo': 78,\n",
       " 'st': 94,\n",
       " 'ta': 97,\n",
       " 'ay': 24,\n",
       " 'al': 20,\n",
       " 'lt': 61,\n",
       " 'be': 25,\n",
       " 'nd': 66,\n",
       " 'bu': 28,\n",
       " 'ug': 104,\n",
       " 'se': 90,\n",
       " 'er': 41,\n",
       " 'no': 70,\n",
       " ' p': 12,\n",
       " 'pl': 82,\n",
       " 'la': 57,\n",
       " 'ac': 19,\n",
       " 'ik': 54,\n",
       " 'ke': 56,\n",
       " 'sp': 93,\n",
       " 'pa': 81,\n",
       " 'so': 92,\n",
       " 'sy': 95,\n",
       " 'ys': 112,\n",
       " 'te': 98,\n",
       " 'em': 39,\n",
       " 'm ': 62}"
      ]
     },
     "execution_count": 462,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "cv.vocabulary_"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 463,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(5, 113)"
      ]
     },
     "execution_count": 463,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "count_vector.shape"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Limiting Vocabulary Size"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 464,
   "metadata": {},
   "outputs": [],
   "source": [
    "#only bigrams and unigrams, limit to vocab size of 10\n",
    "cv = CountVectorizer(cat_in_the_hat_docs,ngram_range=(1,2),preprocessor=my_cool_preprocessor,analyzer='word',max_features=10)\n",
    "count_vector=cv.fit_transform(cat_in_the_hat_docs)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 465,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(5, 10)"
      ]
     },
     "execution_count": 465,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "count_vector.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 466,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'_connector_': 0,\n",
       " 'cat': 1,\n",
       " 'the': 8,\n",
       " 'hat': 3,\n",
       " 'learn': 5,\n",
       " 'librari': 7,\n",
       " 'cat _connector_': 2,\n",
       " 'the hat': 9,\n",
       " 'hat learn': 4,\n",
       " 'learn librari': 6}"
      ]
     },
     "execution_count": 466,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "cv.vocabulary_"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Extracting Counts of Words / N-Grams"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 37,
   "metadata": {},
   "outputs": [],
   "source": [
    "def sort_coo(coo_matrix):\n",
    "    tuples = zip(coo_matrix.col, coo_matrix.data)\n",
    "    return sorted(tuples, key=lambda x: (x[1], x[0]), reverse=True)\n",
    " \n",
    "def extract_topn_from_vector(feature_names, sorted_items, topn=10):\n",
    "    \"\"\"return n-gram counts in descending order of counts\"\"\"\n",
    "    \n",
    "    #use only topn items from vector\n",
    "    sorted_items = sorted_items[:topn]\n",
    " \n",
    "    score_vals = []\n",
    "    feature_vals = []\n",
    "    results=[]\n",
    "    \n",
    "    # word index, count i\n",
    "    for idx, count in sorted_items:\n",
    "        \n",
    "        # get the ngram name\n",
    "        n_gram=feature_names[idx]\n",
    "        \n",
    "        # collect as a list of tuples\n",
    "        results.append((n_gram,count))\n",
    " \n",
    "    return results"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 54,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[('cent', 4),\n",
       " ('_connector_', 2),\n",
       " ('two cent', 1),\n",
       " ('two', 1),\n",
       " ('the hat', 1),\n",
       " ('the', 1),\n",
       " ('one cent', 1),\n",
       " ('one', 1),\n",
       " ('old cent', 1),\n",
       " ('old', 1)]"
      ]
     },
     "execution_count": 54,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "cv = CountVectorizer(cat_in_the_hat_docs,ngram_range=(1,2),preprocessor=my_cool_preprocessor,max_features=100)\n",
    "count_vector=cv.fit_transform(cat_in_the_hat_docs)\n",
    "\n",
    "#sort the counts of first book title by descending order of counts\n",
    "sorted_items=sort_coo(count_vector[0].tocoo())\n",
    "\n",
    "#Get feature names (words/n-grams). It is sorted by position in sparse matrix\n",
    "feature_names=cv.get_feature_names()\n",
    "n_grams=extract_topn_from_vector(feature_names,sorted_items,10)\n",
    "n_grams"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Binary Values Instead of Counts"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 60,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "  (0, 35)\t1\n",
      "  (0, 74)\t1\n",
      "  (0, 4)\t1\n",
      "  (0, 90)\t1\n",
      "  (0, 29)\t1\n",
      "  (0, 31)\t1\n",
      "  (0, 9)\t1\n",
      "  (0, 78)\t1\n",
      "  (0, 27)\t1\n",
      "  (0, 17)\t1\n",
      "  (0, 91)\t1\n",
      "  (0, 86)\t1\n",
      "  (0, 82)\t1\n",
      "  (0, 56)\t1\n",
      "  (0, 34)\t1\n",
      "  (0, 73)\t1\n",
      "  (0, 28)\t1\n",
      "  (0, 30)\t1\n",
      "  (0, 8)\t1\n",
      "  (0, 77)\t1\n",
      "  (0, 26)\t1\n",
      "  (0, 16)\t1\n",
      "  (0, 89)\t1\n",
      "  (0, 85)\t1\n",
      "  (0, 55)\t1\n",
      "  (0, 45)\t1\n",
      "  (0, 33)\t1\n",
      "  (0, 80)\t1\n",
      "  (0, 39)\t1\n",
      "  (0, 19)\t1\n",
      "  (0, 7)\t1\n",
      "  (0, 46)\t1\n",
      "  (0, 44)\t1\n",
      "  (0, 32)\t1\n",
      "  (0, 79)\t1\n",
      "  (0, 38)\t1\n",
      "  (0, 18)\t1\n",
      "  (0, 0)\t1\n",
      "  (0, 6)\t1\n"
     ]
    }
   ],
   "source": [
    "cv = CountVectorizer(cat_in_the_hat_docs,ngram_range=(1,2),binary=True)\n",
    "count_vector=cv.fit_transform(cat_in_the_hat_docs)\n",
    "print(count_vector[2])"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Custom Tokenizer"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 78,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'one': 34, 'cent': 14, ',': 4, 'two': 47, 'cents': 15, 'old': 32, 'new': 29, ':': 5, 'all': 7, 'about': 6, 'money': 28, '(': 2, 'cat': 13, 'in': 22, 'the': 44, 'hat': 19, \"'\": 1, 's': 38, 'learning': 25, 'library': 26, 'inside': 24, 'your': 49, 'outside': 36, 'human': 21, 'body': 10, ')': 3, '': 0, 'oh': 31, 'things': 46, 'you': 48, 'can': 12, 'do': 16, 'that': 43, 'are': 8, 'good': 18, 'for': 17, 'staying': 41, 'healthy': 20, 'on': 33, 'beyond': 9, 'bugs': 11, 'insects': 23, 'there': 45, 'no': 30, 'place': 37, 'like': 27, 'space': 40, 'our': 35, 'solar': 39, 'system': 42}\n"
     ]
    }
   ],
   "source": [
    "import re\n",
    "\n",
    "def my_tokenizer(text):\n",
    "    text=re.sub(\"(\\\\W)\",\" \\\\1 \",text)\n",
    "    return re.split(\"\\\\s+\",text)\n",
    "    \n",
    "\n",
    "cv = CountVectorizer(cat_in_the_hat_docs,tokenizer=my_tokenizer)\n",
    "count_vector=cv.fit_transform(cat_in_the_hat_docs)\n",
    "print(cv.vocabulary_)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.5"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}