Text Processing Basics.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['canada', 'canada', 'canada', 'canada']"
      ]
     },
     "execution_count": 3,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "texts=[\"CANADA\",\"Canada\",\"canadA\",\"canada\"]\n",
    "lower_words=[word.lower() for word in texts]\n",
    "lower_words\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Stemming"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 44,
   "metadata": {},
   "outputs": [],
   "source": [
    "import nltk\n",
    "import pandas as pd\n",
    "import numpy as np\n",
    "from nltk.stem import PorterStemmer\n",
    "\n",
    "# init stemmer\n",
    "porter_stemmer=PorterStemmer()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>original_word</th>\n",
       "      <th>stemmed_word</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>connect</td>\n",
       "      <td>connect</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>connected</td>\n",
       "      <td>connect</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>connection</td>\n",
       "      <td>connect</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>connections</td>\n",
       "      <td>connect</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>connects</td>\n",
       "      <td>connect</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "  original_word stemmed_word\n",
       "0       connect      connect\n",
       "1     connected      connect\n",
       "2    connection      connect\n",
       "3   connections      connect\n",
       "4      connects      connect"
      ]
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# stem connect variations\n",
    "words=[\"connect\",\"connected\",\"connection\",\"connections\",\"connects\"]\n",
    "stemmed_words=[porter_stemmer.stem(word=word) for word in words]\n",
    "\n",
    "stemdf= pd.DataFrame({'original_word': words,'stemmed_word': stemmed_words})\n",
    "stemdf"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>original_word</th>\n",
       "      <th>stemmed_word</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>trouble</td>\n",
       "      <td>troubl</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>troubled</td>\n",
       "      <td>troubl</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>troubles</td>\n",
       "      <td>troubl</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>troublemsome</td>\n",
       "      <td>troublemsom</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "  original_word stemmed_word\n",
       "0       trouble       troubl\n",
       "1      troubled       troubl\n",
       "2      troubles       troubl\n",
       "3  troublemsome  troublemsom"
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# stem trouble variations\n",
    "words=[\"trouble\",\"troubled\",\"troubles\",\"troublemsome\"]\n",
    "stemmed_words=[porter_stemmer.stem(word=word) for word in words]\n",
    "\n",
    "stemdf= pd.DataFrame({'original_word': words,'stemmed_word': stemmed_words})\n",
    "stemdf"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Lemmatization"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[nltk_data] Downloading package wordnet to\n",
      "[nltk_data]     C:\\Users\\USX28939\\AppData\\Roaming\\nltk_data...\n",
      "[nltk_data]   Package wordnet is already up-to-date!\n"
     ]
    }
   ],
   "source": [
    "from nltk.stem import WordNetLemmatizer\n",
    "nltk.download('wordnet')\n",
    "\n",
    "# init lemmatizer\n",
    "lemmatizer = WordNetLemmatizer()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>original_word</th>\n",
       "      <th>lemmatized_word</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>trouble</td>\n",
       "      <td>trouble</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>troubling</td>\n",
       "      <td>trouble</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>troubled</td>\n",
       "      <td>trouble</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>troubles</td>\n",
       "      <td>trouble</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "  original_word lemmatized_word\n",
       "0       trouble         trouble\n",
       "1     troubling         trouble\n",
       "2      troubled         trouble\n",
       "3      troubles         trouble"
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "#lemmatize trouble variations\n",
    "words=[\"trouble\",\"troubling\",\"troubled\",\"troubles\",]\n",
    "lemmatized_words=[lemmatizer.lemmatize(word=word,pos='v') for word in words]\n",
    "lemmatizeddf= pd.DataFrame({'original_word': words,'lemmatized_word': lemmatized_words})\n",
    "lemmatizeddf=lemmatizeddf[['original_word','lemmatized_word']]\n",
    "lemmatizeddf"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>original_word</th>\n",
       "      <th>lemmatized_word</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>goose</td>\n",
       "      <td>goose</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>geese</td>\n",
       "      <td>goose</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "  original_word lemmatized_word\n",
       "0         goose           goose\n",
       "1         geese           goose"
      ]
     },
     "execution_count": 8,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "#lemmatize goose variations\n",
    "words=[\"goose\",\"geese\"]\n",
    "lemmatized_words=[lemmatizer.lemmatize(word=word,pos='n') for word in words]\n",
    "lemmatizeddf= pd.DataFrame({'original_word': words,'lemmatized_word': lemmatized_words})\n",
    "lemmatizeddf=lemmatizeddf[['original_word','lemmatized_word']]\n",
    "lemmatizeddf"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Stop Word Removal"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "stopwords=['this','that','and','a','we','it','to','is','of','up','need']\n",
    "text=\"this is a text full of content and we need to clean it up\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "original sentence =  this is a text full of content and we need to clean it up\n",
      "sentence with stop words removed=  W W W text full W content W W W W clean W W\n"
     ]
    }
   ],
   "source": [
    "words=text.split(\" \")\n",
    "shortlisted_words=[]\n",
    "\n",
    "#remove stop words\n",
    "for w in words:\n",
    "    if w not in stopwords:\n",
    "        shortlisted_words.append(w)\n",
    "    else:\n",
    "        shortlisted_words.append(\"W\")\n",
    "\n",
    "print(\"original sentence = \",text)    \n",
    "print(\"sentence with stop words removed= \",' '.join(shortlisted_words))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Noise Removal"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [],
   "source": [
    "import nltk\n",
    "import pandas as pd\n",
    "import re\n",
    "from nltk.stem import PorterStemmer\n",
    "\n",
    "porter_stemmer=PorterStemmer()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>raw_word</th>\n",
       "      <th>stemmed_word</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>..trouble..</td>\n",
       "      <td>..trouble..</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>trouble&lt;</td>\n",
       "      <td>trouble&lt;</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>trouble!</td>\n",
       "      <td>trouble!</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>&lt;a&gt;trouble&lt;/a&gt;</td>\n",
       "      <td>&lt;a&gt;trouble&lt;/a&gt;</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>1.trouble</td>\n",
       "      <td>1.troubl</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "         raw_word    stemmed_word\n",
       "0     ..trouble..     ..trouble..\n",
       "1        trouble<        trouble<\n",
       "2        trouble!        trouble!\n",
       "3  <a>trouble</a>  <a>trouble</a>\n",
       "4       1.trouble        1.troubl"
      ]
     },
     "execution_count": 12,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# stem raw words with noise\n",
    "raw_words=[\"..trouble..\",\"trouble<\",\"trouble!\",\"<a>trouble</a>\",'1.trouble']\n",
    "stemmed_words=[porter_stemmer.stem(word=word) for word in raw_words]\n",
    "stemdf= pd.DataFrame({'raw_word': raw_words,'stemmed_word': stemmed_words})\n",
    "stemdf"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [],
   "source": [
    "def scrub_words(text):\n",
    "    \"\"\"Basic cleaning of texts.\"\"\"\n",
    "    \n",
    "    # remove html markup\n",
    "    text=re.sub(\"(<.*?>)\",\"\",text)\n",
    "    \n",
    "    #remove non-ascii and digits\n",
    "    text=re.sub(\"(\\\\W|\\\\d)\",\" \",text)\n",
    "    \n",
    "    #remove whitespace\n",
    "    text=text.strip()\n",
    "    return text"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>raw_word</th>\n",
       "      <th>cleaned_word</th>\n",
       "      <th>stemmed_word</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>..trouble..</td>\n",
       "      <td>trouble</td>\n",
       "      <td>troubl</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>trouble&lt;</td>\n",
       "      <td>trouble</td>\n",
       "      <td>troubl</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>trouble!</td>\n",
       "      <td>trouble</td>\n",
       "      <td>troubl</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>&lt;a&gt;trouble&lt;/a&gt;</td>\n",
       "      <td>trouble</td>\n",
       "      <td>troubl</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>1.trouble</td>\n",
       "      <td>trouble</td>\n",
       "      <td>troubl</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "         raw_word cleaned_word stemmed_word\n",
       "0     ..trouble..      trouble       troubl\n",
       "1        trouble<      trouble       troubl\n",
       "2        trouble!      trouble       troubl\n",
       "3  <a>trouble</a>      trouble       troubl\n",
       "4       1.trouble      trouble       troubl"
      ]
     },
     "execution_count": 14,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# stem words already cleaned\n",
    "cleaned_words=[scrub_words(w) for w in raw_words]\n",
    "cleaned_stemmed_words=[porter_stemmer.stem(word=word) for word in cleaned_words]\n",
    "stemdf= pd.DataFrame({'raw_word': raw_words,'cleaned_word':cleaned_words,'stemmed_word': cleaned_stemmed_words})\n",
    "stemdf=stemdf[['raw_word','cleaned_word','stemmed_word']]\n",
    "stemdf"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Cosine Similarity"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 39,
   "metadata": {},
   "outputs": [],
   "source": [
    "#https://www.machinelearningplus.com/nlp/cosine-similarity/\n",
    "from sklearn.feature_extraction.text import CountVectorizer, TfidfVectorizer"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Define the documents\n",
    "doc_trump = \"Mr. Trump became president after winning the political election. Though he lost the support of some republican friends, Trump is friends with President Putin\"\n",
    "\n",
    "doc_election = \"President Trump says Putin had no political interference is the election outcome. He says it was a witchhunt by political parties. He claimed President Putin is a friend who had nothing to do with the election\"\n",
    "\n",
    "doc_putin = \"Post elections, Vladimir Putin became President of Russia. President Putin had served as the Prime Minister earlier in his political career\"\n",
    "\n",
    "documents = [doc_trump, doc_election, doc_putin]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>career</th>\n",
       "      <th>claimed</th>\n",
       "      <th>earlier</th>\n",
       "      <th>election</th>\n",
       "      <th>elections</th>\n",
       "      <th>friend</th>\n",
       "      <th>friends</th>\n",
       "      <th>interference</th>\n",
       "      <th>lost</th>\n",
       "      <th>minister</th>\n",
       "      <th>...</th>\n",
       "      <th>putin</th>\n",
       "      <th>republican</th>\n",
       "      <th>russia</th>\n",
       "      <th>says</th>\n",
       "      <th>served</th>\n",
       "      <th>support</th>\n",
       "      <th>trump</th>\n",
       "      <th>vladimir</th>\n",
       "      <th>winning</th>\n",
       "      <th>witchhunt</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>doc_trump</th>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>2</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>2</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>doc_election</th>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>2</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>2</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>2</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>doc_putin</th>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>...</td>\n",
       "      <td>2</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>3 rows × 27 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "              career  claimed  earlier  election  elections  friend  friends  \\\n",
       "doc_trump          0        0        0         1          0       0        2   \n",
       "doc_election       0        1        0         2          0       1        0   \n",
       "doc_putin          1        0        1         0          1       0        0   \n",
       "\n",
       "              interference  lost  minister    ...      putin  republican  \\\n",
       "doc_trump                0     1         0    ...          1           1   \n",
       "doc_election             1     0         0    ...          2           0   \n",
       "doc_putin                0     0         1    ...          2           0   \n",
       "\n",
       "              russia  says  served  support  trump  vladimir  winning  \\\n",
       "doc_trump          0     0       0        1      2         0        1   \n",
       "doc_election       0     2       0        0      1         0        0   \n",
       "doc_putin          1     0       1        0      0         1        0   \n",
       "\n",
       "              witchhunt  \n",
       "doc_trump             0  \n",
       "doc_election          1  \n",
       "doc_putin             0  \n",
       "\n",
       "[3 rows x 27 columns]"
      ]
     },
     "execution_count": 28,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Create the Document Term Matrix\n",
    "count_vectorizer = CountVectorizer(stop_words='english')\n",
    "sparse_matrix = count_vectorizer.fit_transform(documents)\n",
    "# OPTIONAL: Convert Sparse Matrix to Pandas Dataframe if you want to see the word frequencies.\n",
    "doc_term_matrix = sparse_matrix.todense()\n",
    "df = pd.DataFrame(doc_term_matrix, \n",
    "                  columns=count_vectorizer.get_feature_names(), \n",
    "                  index=['doc_trump', 'doc_election', 'doc_putin'])\n",
    "df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>career</th>\n",
       "      <th>claimed</th>\n",
       "      <th>earlier</th>\n",
       "      <th>election</th>\n",
       "      <th>elections</th>\n",
       "      <th>friend</th>\n",
       "      <th>friends</th>\n",
       "      <th>interference</th>\n",
       "      <th>lost</th>\n",
       "      <th>minister</th>\n",
       "      <th>...</th>\n",
       "      <th>putin</th>\n",
       "      <th>republican</th>\n",
       "      <th>russia</th>\n",
       "      <th>says</th>\n",
       "      <th>served</th>\n",
       "      <th>support</th>\n",
       "      <th>trump</th>\n",
       "      <th>vladimir</th>\n",
       "      <th>winning</th>\n",
       "      <th>witchhunt</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>doc_trump</th>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.203368</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.53481</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.267405</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>...</td>\n",
       "      <td>0.157934</td>\n",
       "      <td>0.267405</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.267405</td>\n",
       "      <td>0.406737</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.267405</td>\n",
       "      <td>0.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>doc_election</th>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.241982</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.368067</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.241982</td>\n",
       "      <td>0.00000</td>\n",
       "      <td>0.241982</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>...</td>\n",
       "      <td>0.285837</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.483963</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.184033</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.241982</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>doc_putin</th>\n",
       "      <td>0.287012</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.287012</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.287012</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.00000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.287012</td>\n",
       "      <td>...</td>\n",
       "      <td>0.339028</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.287012</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.287012</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.287012</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>3 rows × 27 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "                career   claimed   earlier  election  elections    friend  \\\n",
       "doc_trump     0.000000  0.000000  0.000000  0.203368   0.000000  0.000000   \n",
       "doc_election  0.000000  0.241982  0.000000  0.368067   0.000000  0.241982   \n",
       "doc_putin     0.287012  0.000000  0.287012  0.000000   0.287012  0.000000   \n",
       "\n",
       "              friends  interference      lost  minister    ...         putin  \\\n",
       "doc_trump     0.53481      0.000000  0.267405  0.000000    ...      0.157934   \n",
       "doc_election  0.00000      0.241982  0.000000  0.000000    ...      0.285837   \n",
       "doc_putin     0.00000      0.000000  0.000000  0.287012    ...      0.339028   \n",
       "\n",
       "              republican    russia      says    served   support     trump  \\\n",
       "doc_trump       0.267405  0.000000  0.000000  0.000000  0.267405  0.406737   \n",
       "doc_election    0.000000  0.000000  0.483963  0.000000  0.000000  0.184033   \n",
       "doc_putin       0.000000  0.287012  0.000000  0.287012  0.000000  0.000000   \n",
       "\n",
       "              vladimir   winning  witchhunt  \n",
       "doc_trump     0.000000  0.267405   0.000000  \n",
       "doc_election  0.000000  0.000000   0.241982  \n",
       "doc_putin     0.287012  0.000000   0.000000  \n",
       "\n",
       "[3 rows x 27 columns]"
      ]
     },
     "execution_count": 31,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Create the Document Term Matrix\n",
    "\n",
    "#could have used the TfidfVectorizer() instead of CountVectorizer(), \n",
    "#because it would have downweighted words that occur frequently across docuemnts.\n",
    "\n",
    "count_vectorizer = TfidfVectorizer(stop_words='english')\n",
    "sparse_matrix = count_vectorizer.fit_transform(documents)\n",
    "# OPTIONAL: Convert Sparse Matrix to Pandas Dataframe if you want to see the word frequencies.\n",
    "doc_term_matrix = sparse_matrix.todense()\n",
    "df = pd.DataFrame(doc_term_matrix, \n",
    "                  columns=count_vectorizer.get_feature_names(), \n",
    "                  index=['doc_trump', 'doc_election', 'doc_putin'])\n",
    "df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 34,
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.metrics.pairwise import cosine_similarity,cosine_distances"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[[1.         0.33027897 0.18740386]\n",
      " [0.33027897 1.         0.24226661]\n",
      " [0.18740386 0.24226661 1.        ]]\n"
     ]
    }
   ],
   "source": [
    "print(cosine_similarity(df, df))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 35,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[[0.         0.66972103 0.81259614]\n",
      " [0.66972103 0.         0.75773339]\n",
      " [0.81259614 0.75773339 0.        ]]\n"
     ]
    }
   ],
   "source": [
    "print(cosine_distances(df,df))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Soft Cosine"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "we need to consider the semantic meaning should be considered. That is, words similar in meaning should be treated as similar. For Example, ‘President’ vs ‘Prime minister’, ‘Food’ vs ‘Dish’, ‘Hi’ vs ‘Hello’ should be considered similar."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 36,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Define the documents\n",
    "doc_soup = \"Soup is a primarily liquid food, generally served warm or hot (but may be cool or cold), that is made by combining ingredients of meat or vegetables with stock, juice, water, or another liquid. \"\n",
    "\n",
    "doc_noodles = \"Noodles are a staple food in many cultures. They are made from unleavened dough which is stretched, extruded, or rolled flat and cut into one of a variety of shapes.\"\n",
    "\n",
    "doc_dosa = \"Dosa is a type of pancake from the Indian subcontinent, made from a fermented batter. It is somewhat similar to a crepe in appearance. Its main ingredients are rice and black gram.\"\n",
    "\n",
    "documents = [doc_trump, doc_election, doc_putin, doc_soup, doc_noodles, doc_dosa]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\ProgramData\\Anaconda3\\lib\\site-packages\\gensim\\utils.py:1212: UserWarning: detected Windows; aliasing chunkize to chunkize_serial\n",
      "  warnings.warn(\"detected Windows; aliasing chunkize to chunkize_serial\")\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "3.6.0\n"
     ]
    }
   ],
   "source": [
    "import gensim\n",
    "# upgrade gensim if you can't import softcossim\n",
    "from gensim.matutils import softcossim \n",
    "from gensim import corpora\n",
    "import gensim.downloader as api\n",
    "from gensim.utils import simple_preprocess\n",
    "print(gensim.__version__)\n",
    "#> '3.6.0'\n",
    "\n",
    "# Download the FastText model\n",
    "fasttext_model300 = api.load('fasttext-wiki-news-subwords-300')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "To compute soft cosines, you need the dictionary (a map of word toTo compute soft cosines, you need the dictionary (a map of word to unique id), the corpus (word counts) for each sentence and the similarity matrix. unique id), "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 38,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\ProgramData\\Anaconda3\\lib\\site-packages\\gensim\\matutils.py:737: FutureWarning: Conversion of the second argument of issubdtype from `int` to `np.signedinteger` is deprecated. In future, it will be treated as `np.int32 == np.dtype(int).type`.\n",
      "  if np.issubdtype(vec.dtype, np.int):\n"
     ]
    }
   ],
   "source": [
    "# Prepare a dictionary and a corpus.\n",
    "dictionary = corpora.Dictionary([simple_preprocess(doc) for doc in documents])\n",
    "\n",
    "# Prepare the similarity matrix\n",
    "similarity_matrix = fasttext_model300.similarity_matrix(dictionary, tfidf=None, threshold=0.0, exponent=2.0, nonzero_limit=100)\n",
    "\n",
    "# Convert the sentences into bag-of-words vectors.\n",
    "sent_1 = dictionary.doc2bow(simple_preprocess(doc_trump))\n",
    "sent_2 = dictionary.doc2bow(simple_preprocess(doc_election))\n",
    "sent_3 = dictionary.doc2bow(simple_preprocess(doc_putin))\n",
    "sent_4 = dictionary.doc2bow(simple_preprocess(doc_soup))\n",
    "sent_5 = dictionary.doc2bow(simple_preprocess(doc_noodles))\n",
    "sent_6 = dictionary.doc2bow(simple_preprocess(doc_dosa))\n",
    "\n",
    "sentences = [sent_1, sent_2, sent_3, sent_4, sent_5, sent_6]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 41,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0.5842469768347521\n"
     ]
    }
   ],
   "source": [
    "# Compute soft cosine similarity\n",
    "print(softcossim(sent_1, sent_2, similarity_matrix))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 46,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>doc_trump</th>\n",
       "      <th>doc_election</th>\n",
       "      <th>doc_putin</th>\n",
       "      <th>doc_soup</th>\n",
       "      <th>doc_noodles</th>\n",
       "      <th>doc_dosa</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>doc_trump</th>\n",
       "      <td>1.00</td>\n",
       "      <td>0.58</td>\n",
       "      <td>0.56</td>\n",
       "      <td>0.28</td>\n",
       "      <td>0.34</td>\n",
       "      <td>0.40</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>doc_election</th>\n",
       "      <td>0.58</td>\n",
       "      <td>1.00</td>\n",
       "      <td>0.54</td>\n",
       "      <td>0.25</td>\n",
       "      <td>0.31</td>\n",
       "      <td>0.43</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>doc_putin</th>\n",
       "      <td>0.56</td>\n",
       "      <td>0.54</td>\n",
       "      <td>1.00</td>\n",
       "      <td>0.19</td>\n",
       "      <td>0.25</td>\n",
       "      <td>0.36</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>doc_soup</th>\n",
       "      <td>0.28</td>\n",
       "      <td>0.25</td>\n",
       "      <td>0.19</td>\n",
       "      <td>1.00</td>\n",
       "      <td>0.50</td>\n",
       "      <td>0.38</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>doc_noodles</th>\n",
       "      <td>0.34</td>\n",
       "      <td>0.31</td>\n",
       "      <td>0.25</td>\n",
       "      <td>0.50</td>\n",
       "      <td>1.00</td>\n",
       "      <td>0.56</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>doc_dosa</th>\n",
       "      <td>0.40</td>\n",
       "      <td>0.43</td>\n",
       "      <td>0.36</td>\n",
       "      <td>0.38</td>\n",
       "      <td>0.56</td>\n",
       "      <td>1.00</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "              doc_trump  doc_election  doc_putin  doc_soup  doc_noodles  \\\n",
       "doc_trump          1.00          0.58       0.56      0.28         0.34   \n",
       "doc_election       0.58          1.00       0.54      0.25         0.31   \n",
       "doc_putin          0.56          0.54       1.00      0.19         0.25   \n",
       "doc_soup           0.28          0.25       0.19      1.00         0.50   \n",
       "doc_noodles        0.34          0.31       0.25      0.50         1.00   \n",
       "doc_dosa           0.40          0.43       0.36      0.38         0.56   \n",
       "\n",
       "              doc_dosa  \n",
       "doc_trump         0.40  \n",
       "doc_election      0.43  \n",
       "doc_putin         0.36  \n",
       "doc_soup          0.38  \n",
       "doc_noodles       0.56  \n",
       "doc_dosa          1.00  "
      ]
     },
     "execution_count": 46,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "col  = [\"doc_trump\",\"doc_election\",\"doc_putin\",\"doc_soup\",\"doc_noodles\",\"doc_dosa\"]\n",
    "\n",
    "def create_soft_cossim_matrix(sentences):\n",
    "    len_array = np.arange(len(sentences))\n",
    "    xx, yy = np.meshgrid(len_array, len_array)\n",
    "    cossim_mat = pd.DataFrame([[round(softcossim(sentences[i],sentences[j], similarity_matrix) ,2) for i, j in zip(x,y)] for y, x in zip(xx, yy)],\n",
    "                             columns=col, index = col )\n",
    "    return cossim_mat\n",
    "\n",
    "create_soft_cossim_matrix(sentences)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## TF-IDF"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 47,
   "metadata": {},
   "outputs": [],
   "source": [
    "S1 = \"The car is driven on the road.\"\n",
    "S2 = \"The truck is driven on the highway.\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 48,
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.feature_extraction.text import TfidfVectorizer"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 53,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "  (0, 6)\t0.6043795515372431\n",
      "  (0, 0)\t0.42471718586982765\n",
      "  (0, 3)\t0.30218977576862155\n",
      "  (0, 1)\t0.30218977576862155\n",
      "  (0, 4)\t0.30218977576862155\n",
      "  (0, 5)\t0.42471718586982765\n",
      "  (1, 6)\t0.6043795515372431\n",
      "  (1, 3)\t0.30218977576862155\n",
      "  (1, 1)\t0.30218977576862155\n",
      "  (1, 4)\t0.30218977576862155\n",
      "  (1, 7)\t0.42471718586982765\n",
      "  (1, 2)\t0.42471718586982765\n"
     ]
    }
   ],
   "source": [
    "vec = TfidfVectorizer()\n",
    "out = vec.fit_transform([S1,S2]) # see it pass as a list\n",
    "print(out)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 56,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['car', 'driven', 'highway', 'is', 'on', 'road', 'the', 'truck']"
      ]
     },
     "execution_count": 56,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "vec.get_feature_names()\n",
    "#so the output above is document number and word index in the get feature names"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 59,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "scipy.sparse.csr.csr_matrix"
      ]
     },
     "execution_count": 59,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "type(out)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 63,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>car</th>\n",
       "      <th>driven</th>\n",
       "      <th>highway</th>\n",
       "      <th>is</th>\n",
       "      <th>on</th>\n",
       "      <th>road</th>\n",
       "      <th>the</th>\n",
       "      <th>truck</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>S1</th>\n",
       "      <td>0.424717</td>\n",
       "      <td>0.30219</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.30219</td>\n",
       "      <td>0.30219</td>\n",
       "      <td>0.424717</td>\n",
       "      <td>0.60438</td>\n",
       "      <td>0.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>S2</th>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.30219</td>\n",
       "      <td>0.424717</td>\n",
       "      <td>0.30219</td>\n",
       "      <td>0.30219</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.60438</td>\n",
       "      <td>0.424717</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "         car   driven   highway       is       on      road      the     truck\n",
       "S1  0.424717  0.30219  0.000000  0.30219  0.30219  0.424717  0.60438  0.000000\n",
       "S2  0.000000  0.30219  0.424717  0.30219  0.30219  0.000000  0.60438  0.424717"
      ]
     },
     "execution_count": 63,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "doc_term_matrix = out.todense()\n",
    "df = pd.DataFrame(doc_term_matrix, \n",
    "                  columns=vec.get_feature_names(), \n",
    "                  index=[\"S1\",\"S2\"])\n",
    "df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "corpus = ['The sky is blue and beautiful.',\n",
    "          'Love this blue and beautiful sky!',\n",
    "   \n",
    "          #https://towardsdatascience.com/understanding-feature-engineering-part-3-traditional-methods-for-text-data-f6f7d70acd41\n",
    "]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Count Vector"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\ProgramData\\Anaconda3\\lib\\importlib\\_bootstrap.py:219: RuntimeWarning: numpy.ufunc size changed, may indicate binary incompatibility. Expected 192 from C header, got 216 from PyObject\n",
      "  return f(*args, **kwds)\n"
     ]
    }
   ],
   "source": [
    "from sklearn.feature_extraction.text import CountVectorizer\n",
    "import nltk as nk"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([[1, 0, 0, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0,\n",
       "        1, 1, 0, 0, 0, 0],\n",
       "       [1, 0, 0, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0,\n",
       "        1, 0, 1, 0, 0, 0],\n",
       "       [0, 0, 0, 0, 0, 0, 1, 0, 1, 0, 1, 0, 0, 0, 0, 1, 0, 1, 0, 1, 1, 0,\n",
       "        0, 2, 0, 0, 0, 0],\n",
       "       [1, 1, 1, 0, 0, 1, 0, 0, 0, 1, 0, 0, 1, 1, 0, 0, 1, 0, 0, 0, 0, 1,\n",
       "        0, 0, 0, 1, 0, 0],\n",
       "       [1, 1, 0, 0, 0, 0, 0, 0, 0, 1, 0, 1, 1, 0, 0, 0, 0, 0, 1, 0, 0, 1,\n",
       "        0, 0, 0, 0, 0, 0],\n",
       "       [1, 0, 0, 0, 1, 0, 1, 0, 1, 0, 1, 0, 0, 0, 2, 0, 0, 1, 0, 0, 1, 0,\n",
       "        0, 2, 0, 0, 0, 0],\n",
       "       [1, 0, 0, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0,\n",
       "        2, 2, 0, 0, 1, 2],\n",
       "       [0, 0, 0, 0, 0, 0, 1, 1, 1, 0, 1, 0, 0, 0, 2, 0, 0, 1, 0, 0, 1, 0,\n",
       "        0, 2, 0, 0, 0, 0]], dtype=int64)"
      ]
     },
     "execution_count": 3,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "cv = CountVectorizer(min_df=0., max_df=1.)\n",
    "cv_matrix = cv.fit_transform(corpus)\n",
    "cv_matrix = cv_matrix.toarray()\n",
    "cv_matrix"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "<generator object bigrams at 0x000001C874552D00>\n"
     ]
    }
   ],
   "source": [
    "t = nk.bigrams(corpus)\n",
    "print(t)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.9"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}