lda_training_offline.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Loading the models for categorizing"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "from gensim.models import LdaModel, Phrases\n",
    "from gensim.corpora import Dictionary\n",
    "\n",
    "from nltk.stem.wordnet import WordNetLemmatizer\n",
    "from nltk.tokenize import RegexpTokenizer"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Loading the previously saved models"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "dictionary = Dictionary.load('dicionario_1')\n",
    "phraser = Phrases.load('frases_1')\n",
    "model = LdaModel.load('modelo_1')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Apply the same preprocessing to incoming texts"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Text before preprocessing\n",
      " ['I am trying to learn Machine Learning']\n",
      "Text after preprocessing\n",
      " [['am', 'trying', 'to', 'learn', 'machine', 'learning']]\n"
     ]
    }
   ],
   "source": [
    "text = \"I am trying to learn Machine Learning\"\n",
    "docs = [text]\n",
    "print(\"Text before preprocessing\\n\", docs)\n",
    "\n",
    "# Tokenize the documents.\n",
    "# Split the documents into tokens.\n",
    "tokenizer = RegexpTokenizer(r'\\w+')\n",
    "for idx in range(len(docs)):\n",
    "    docs[idx] = docs[idx].lower()  # Convert to lowercase.\n",
    "    docs[idx] = tokenizer.tokenize(docs[idx])  # Split into words.\n",
    "\n",
    "# Remove numbers, but not words that contain numbers.\n",
    "docs = [[token for token in doc if not token.isnumeric()] for doc in docs]\n",
    "\n",
    "# Remove words that are only one character.\n",
    "docs = [[token for token in doc if len(token) > 1] for doc in docs]\n",
    "\n",
    "# Lemmatize all words in documents.\n",
    "lemmatizer = WordNetLemmatizer()\n",
    "docs = [[lemmatizer.lemmatize(token) for token in doc] for doc in docs]\n",
    "print(\"Text after preprocessing\\n\", docs)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Document after adding n-grams\n",
      " ['am', 'trying', 'to', 'learn', 'machine', 'learning', 'machine_learning']\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/Users/home/Desenvolvimento/anaconda3/lib/python3.6/site-packages/gensim/models/phrases.py:316: UserWarning: For a faster implementation, use the gensim.models.phrases.Phraser class\n",
      "  warnings.warn(\"For a faster implementation, use the gensim.models.phrases.Phraser class\")\n"
     ]
    }
   ],
   "source": [
    "doc = docs[0]\n",
    "for token in phraser[doc]:\n",
    "    if '_' in token:\n",
    "        # Token is a bigram, add to document.\n",
    "        doc.append(token)\n",
    "print(\"Document after adding n-grams\\n\", doc)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Use the loaded model to infer the document's topics"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def normalize_word_topic(word_from_topic):\n",
    "    if '_' in word_from_topic:\n",
    "        words = word_from_topic.split('_')\n",
    "        if words[0] == words[1]:\n",
    "            return words[0]\n",
    "        else:\n",
    "            return ' '.join(words)\n",
    "    else:\n",
    "        return word_from_topic"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "The document's learned topics\n",
      " [(1, 0.92712819089902543), (3, 0.010380412276981575), (9, 0.010418364350076953)]\n",
      "The document's category's topic words\n",
      " ['action', 'policy', 'control', 'reinforcement', 'optimal', 'reinforcement learning', 'robot', 'controller', 'dynamic', 'environment']\n"
     ]
    }
   ],
   "source": [
    "bow = dictionary.doc2bow(doc)\n",
    "# Os topics retornados aqui são tuplas de valores (tópico, score do documento no tópico)\n",
    "topics_scores = model[bow]\n",
    "print(\"The document's learned topics\\n\", topics_scores)\n",
    "best_topic = max(topics_scores, key=lambda topic_score:topic_score[1])\n",
    "topic = model.show_topic(best_topic[0])\n",
    "print(\"The document's category's topic words\\n\", [normalize_word_topic(word_probability[0]) for word_probability in topic])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.3"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 1
}