Word Embedding - sentiment analysis.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import os\n",
    "from nltk.corpus import stopwords\n",
    "import string\n",
    "from collections import Counter"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'C:\\\\Users\\\\USX28939\\\\PYTHON_CODE_BASE\\\\GitHub_Doc\\\\TensorFlow_Framework'"
      ]
     },
     "execution_count": 2,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "os.getcwd()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "os.chdir(\"C:\\\\Machine Learning\\\\Hackathons\\\\Sample Data\\\\NLP\\\\Sentiment-Analysis\\\\txt_sentoken\\\\\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "def load_doc(filename):\n",
    "\n",
    "    # open the file as read only\n",
    "    file = open(filename, 'r')\n",
    "    # read all text\n",
    "    text = file.read()\n",
    "    # close the file\n",
    "    file.close()\n",
    "    return text"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {
    "collapsed": true
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'plot : two teen couples go to a church party , drink and then drive . \\nthey get into an accident . \\none of the guys dies , but his girlfriend continues to see him in her life , and has nightmares . \\nwhat\\'s the deal ? \\nwatch the movie and \" sorta \" find out . . . \\ncritique : a mind-fuck movie for the teen generation that touches on a very cool idea , but presents it in a very bad package . \\nwhich is what makes this review an even harder one to write , since i generally applaud films which attempt to break the mold , mess with your head and such ( lost highway & memento ) , but there are good and bad ways of making all types of films , and these folks just didn\\'t snag this one correctly . \\nthey seem to have taken this pretty neat concept , but executed it terribly . \\nso what are the problems with the movie ? \\nwell , its main problem is that it\\'s simply too jumbled . \\nit starts off \" normal \" but then downshifts into this \" fantasy \" world in which you , as an audience member , have no idea what\\'s going on . \\nthere are dreams , there are characters coming back from the dead , there are others who look like the dead , there are strange apparitions , there are disappearances , there are a looooot of chase scenes , there are tons of weird things that happen , and most of it is simply not explained . \\nnow i personally don\\'t mind trying to unravel a film every now and then , but when all it does is give me the same clue over and over again , i get kind of fed up after a while , which is this film\\'s biggest problem . \\nit\\'s obviously got this big secret to hide , but it seems to want to hide it completely until its final five minutes . \\nand do they make things entertaining , thrilling or even engaging , in the meantime ? \\nnot really . \\nthe sad part is that the arrow and i both dig on flicks like this , so we actually figured most of it out by the half-way point , so all of the strangeness after that did start to make a little bit of sense , but it still didn\\'t the make the film all that more entertaining . \\ni guess the bottom line with movies like this is that you should always make sure that the audience is \" into it \" even before they are given the secret password to enter your world of understanding . \\ni mean , showing melissa sagemiller running away from visions for about 20 minutes throughout the movie is just plain lazy ! ! \\nokay , we get it . . . there \\nare people chasing her and we don\\'t know who they are . \\ndo we really need to see it over and over again ? \\nhow about giving us different scenes offering further insight into all of the strangeness going down in the movie ? \\napparently , the studio took this film away from its director and chopped it up themselves , and it shows . \\nthere might\\'ve been a pretty decent teen mind-fuck movie in here somewhere , but i guess \" the suits \" decided that turning it into a music video with little edge , would make more sense . \\nthe actors are pretty good for the most part , although wes bentley just seemed to be playing the exact same character that he did in american beauty , only in a new neighborhood . \\nbut my biggest kudos go out to sagemiller , who holds her own throughout the entire film , and actually has you feeling her character\\'s unraveling . \\noverall , the film doesn\\'t stick because it doesn\\'t entertain , it\\'s confusing , it rarely excites and it feels pretty redundant for most of its runtime , despite a pretty cool ending and explanation to all of the craziness that came before it . \\noh , and by the way , this is not a horror or teen slasher flick . . . it\\'s \\njust packaged to look that way because someone is apparently assuming that the genre is still hot with the kids . \\nit also wrapped production two years ago and has been sitting on the shelves ever since . \\nwhatever . . . skip \\nit ! \\nwhere\\'s joblo coming from ? \\na nightmare of elm street 3 ( 7/10 ) - blair witch 2 ( 7/10 ) - the crow ( 9/10 ) - the crow : salvation ( 4/10 ) - lost highway ( 10/10 ) - memento ( 10/10 ) - the others ( 9/10 ) - stir of echoes ( 8/10 ) \\n'"
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "a = load_doc(\"C:\\\\Machine Learning\\\\Hackathons\\\\Sample Data\\\\NLP\\\\Sentiment-Analysis\\\\txt_sentoken\\\\\\\\neg\\\\cv000_29416.txt\")\n",
    "a"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "string.punctuation"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "# load all docs in a directory\n",
    "def process_docs(directory, vocab, is_trian):\n",
    "# walk through all files in the folder\n",
    "    for filename in os.listdir(directory):\n",
    "        # skip any reviews in the test set\n",
    "        if is_trian and filename.startswith('cv9'):\n",
    "                continue\n",
    "        if not is_trian and not filename.startswith('cv9'):\n",
    "                continue\n",
    "        # create the full path of the file to open\n",
    "        path = directory + '/' + filename\n",
    "        # add doc to vocab\n",
    "        add_doc_to_vocab(path, vocab)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "maketrans function\n",
    "\n",
    "x - If only one argument is supplied, it must be a dictionary.The dictionary should contain 1-to-1 mapping from a single character string to its translation OR a unicode number (97 for 'a') to its translation.\n",
    "\n",
    "y - If two arguments are passed, it must be two strings with equal length.Each character in the first string is a replacement to its corresponding index in the second string.\n",
    "\n",
    "z - If three arguments are passed, each character in the third argument is mapped to None.\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "def clean_doc(doc):\n",
    "    # split into tokens by white space\n",
    "    tokens = doc.split()\n",
    "    # remove punctuation from each token\n",
    "    table = str.maketrans('', '', string.punctuation)\n",
    "    #print(\"table values are :\\n\")\n",
    "    tokens = [w.translate(table) for w in tokens]\n",
    "    # remove remaining tokens that are not alphabetic\n",
    "    tokens = [word for word in tokens if word.isalpha()]\n",
    "    # filter out stop words\n",
    "    stop_words = set(stopwords.words('english'))\n",
    "    tokens = [w for w in tokens if not w in stop_words]\n",
    "    # filter out short tokens\n",
    "    tokens = [word for word in tokens if len(word) > 1]\n",
    "    return tokens\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "# load doc and add to vocab\n",
    "def add_doc_to_vocab(filename, vocab):\n",
    "    # load doc\n",
    "    doc = load_doc(filename)\n",
    "    # clean doc\n",
    "    tokens = clean_doc(doc)\n",
    "    # update counts\n",
    "    vocab.update(tokens)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "44276\n",
      "[('film', 7983), ('one', 4946), ('movie', 4826), ('like', 3201), ('even', 2262), ('good', 2080), ('time', 2041), ('story', 1907), ('films', 1873), ('would', 1844), ('much', 1824), ('also', 1757), ('characters', 1735), ('get', 1724), ('character', 1703), ('two', 1643), ('first', 1588), ('see', 1557), ('way', 1515), ('well', 1511), ('make', 1418), ('really', 1407), ('little', 1351), ('life', 1334), ('plot', 1288), ('people', 1269), ('bad', 1248), ('could', 1248), ('scene', 1241), ('movies', 1238), ('never', 1201), ('best', 1179), ('new', 1140), ('scenes', 1135), ('man', 1131), ('many', 1130), ('doesnt', 1118), ('know', 1092), ('dont', 1086), ('hes', 1024), ('great', 1014), ('another', 992), ('action', 985), ('love', 977), ('us', 967), ('go', 952), ('director', 948), ('end', 946), ('something', 945), ('still', 936)]\n"
     ]
    }
   ],
   "source": [
    "# define vocab\n",
    "vocab = Counter()\n",
    "# add all docs to vocab\n",
    "process_docs(os.getcwd() +\"\\\\neg\" , vocab, True)\n",
    "process_docs(os.getcwd() + \"\\\\pos\", vocab, True)\n",
    "# print the size of the vocab\n",
    "print(len(vocab))\n",
    "# print the top words in the vocab\n",
    "print(vocab.most_common(50))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "25767\n"
     ]
    }
   ],
   "source": [
    "min_occurane = 2\n",
    "tokens = [k for k,c in vocab.items() if c >= min_occurane]\n",
    "print(len(tokens))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [],
   "source": [
    "# save list to file\n",
    "def save_list(lines, filename):\n",
    "    # convert lines to a single blob of text\n",
    "    data = '\\n'.join(lines)\n",
    "    # open file\n",
    "    file = open(filename, 'w')\n",
    "    # write text\n",
    "    file.write(data)\n",
    "    # close file\n",
    "    file.close()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [],
   "source": [
    "# save tokens to a vocabulary file\n",
    "save_list(tokens, 'vocab.txt')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [],
   "source": [
    "def load_doc(filename):\n",
    "    # open the file as read only\n",
    "    file = open(filename, 'r')\n",
    "    # read all text\n",
    "    text = file.read()\n",
    "    # close the file\n",
    "    file.close()\n",
    "    return text"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "25767\n"
     ]
    }
   ],
   "source": [
    "# load the vocabulary\n",
    "vocab_filename = 'vocab.txt'\n",
    "vocab = load_doc(vocab_filename)\n",
    "vocab = vocab.split()\n",
    "vocab = set(vocab)\n",
    "print(len(vocab))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Embedding Layer while Training"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [],
   "source": [
    "# turn a doc into clean tokens\n",
    "def clean_doc_emb(doc, vocab):\n",
    "    # split into tokens by white space\n",
    "    tokens = doc.split()\n",
    "    # remove punctuation from each token\n",
    "    table = str.maketrans('', '', string.punctuation)\n",
    "    tokens = [w.translate(table) for w in tokens]\n",
    "    # filter out tokens not in vocab\n",
    "    tokens = [w for w in tokens if w in vocab]\n",
    "    tokens = ' '.join(tokens)\n",
    "    return tokens"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [],
   "source": [
    "# load all docs in a directory\n",
    "def process_docs_emb(directory, vocab, is_trian):\n",
    "    documents = list()\n",
    "    # walk through all files in the folder\n",
    "    for filename in os.listdir(directory):\n",
    "        # skip any reviews in the test set\n",
    "        if is_trian and filename.startswith('cv9'):\n",
    "            continue\n",
    "        if not is_trian and not filename.startswith('cv9'):\n",
    "            continue\n",
    "        # create the full path of the file to open\n",
    "        path = directory + '/' + filename\n",
    "        # load the doc\n",
    "        #print(path)\n",
    "        doc = load_doc(path)\n",
    "        # clean doc\n",
    "        tokens = clean_doc_emb(doc, vocab)\n",
    "        # add to list\n",
    "        documents.append(tokens)\n",
    "    return documents"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [],
   "source": [
    "\n",
    "# load all training reviews\n",
    "positive_docs = process_docs_emb(os.getcwd()+\"/pos\", vocab, True)\n",
    "negative_docs = process_docs_emb(os.getcwd()+\"/neg\", vocab, True)\n",
    "train_docs = negative_docs + positive_docs"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "1800"
      ]
     },
     "execution_count": 19,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(train_docs)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\ProgramData\\Anaconda3\\lib\\site-packages\\h5py\\__init__.py:36: FutureWarning: Conversion of the second argument of issubdtype from `float` to `np.floating` is deprecated. In future, it will be treated as `np.float64 == np.dtype(float).type`.\n",
      "  from ._conv import register_converters as _register_converters\n",
      "Using TensorFlow backend.\n"
     ]
    }
   ],
   "source": [
    "from keras.preprocessing.text import Tokenizer\n",
    "from keras.preprocessing.sequence import pad_sequences\n",
    "from numpy import array"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Tokenizer develops a vocabulary of all tokens in the training dataset and develops a consistent mapping from words in the vocabulary to unique integers. "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [],
   "source": [
    "# create the tokenizer\n",
    "tokenizer = Tokenizer()\n",
    "# fit the tokenizer on the documents\n",
    "tokenizer.fit_on_texts(train_docs)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {},
   "outputs": [],
   "source": [
    "# sequence encode\n",
    "encoded_docs = tokenizer.texts_to_sequences(train_docs)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "1800"
      ]
     },
     "execution_count": 23,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(encoded_docs)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {},
   "outputs": [],
   "source": [
    "# pad sequences\n",
    "max_length = max([len(s.split()) for s in train_docs])\n",
    "Xtrain = pad_sequences(encoded_docs, maxlen=max_length, padding='post')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(1800, 1317)"
      ]
     },
     "execution_count": 25,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "Xtrain.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {},
   "outputs": [],
   "source": [
    "# define training labels\n",
    "ytrain = array([0 for _ in range(900)] + [1 for _ in range(900)])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(1800,)"
      ]
     },
     "execution_count": 27,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "ytrain.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {},
   "outputs": [],
   "source": [
    "# load all test reviews\n",
    "positive_docs_test = process_docs_emb(os.getcwd()+\"/pos\", vocab, False)\n",
    "negative_docs_test = process_docs_emb(os.getcwd()+\"/neg\", vocab, False)\n",
    "test_docs = negative_docs_test + positive_docs_test\n",
    "# sequence encode\n",
    "encoded_docs_test = tokenizer.texts_to_sequences(test_docs)\n",
    "# pad sequences\n",
    "Xtest = pad_sequences(encoded_docs_test, maxlen=max_length, padding='post')\n",
    "# define test labels\n",
    "ytest = array([0 for _ in range(100)] + [1 for _ in range(100)])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "metadata": {},
   "outputs": [],
   "source": [
    "from keras.models import Sequential\n",
    "from keras.layers import Dense\n",
    "from keras.layers import Flatten\n",
    "from keras.layers import Embedding\n",
    "from keras.layers.convolutional import Conv1D\n",
    "from keras.layers.convolutional import MaxPooling1D\n",
    "from keras.layers import LSTM"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "metadata": {},
   "outputs": [],
   "source": [
    "# define vocabulary size (largest integer value)\n",
    "vocab_size = len(tokenizer.word_index) + 1"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "1317"
      ]
     },
     "execution_count": 31,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "max_length"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\ProgramData\\Anaconda3\\lib\\site-packages\\ipykernel_launcher.py:5: UserWarning: Update your `LSTM` call to the Keras 2 API: `LSTM(200, dropout=0.2, recurrent_dropout=0.2)`\n",
      "  \"\"\"\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "_________________________________________________________________\n",
      "Layer (type)                 Output Shape              Param #   \n",
      "=================================================================\n",
      "embedding_1 (Embedding)      (None, 1317, 100)         2576800   \n",
      "_________________________________________________________________\n",
      "lstm_1 (LSTM)                (None, 200)               240800    \n",
      "_________________________________________________________________\n",
      "dense_1 (Dense)              (None, 1)                 201       \n",
      "=================================================================\n",
      "Total params: 2,817,801\n",
      "Trainable params: 2,817,801\n",
      "Non-trainable params: 0\n",
      "_________________________________________________________________\n",
      "None\n"
     ]
    }
   ],
   "source": [
    "# define model\n",
    "lstm_out = 200\n",
    "model = Sequential()\n",
    "model.add(Embedding(vocab_size, 100, input_length=max_length))\n",
    "model.add(LSTM(lstm_out, dropout_U = 0.2, dropout_W = 0.2))\n",
    "#model.add(Conv1D(filters=32, kernel_size=8, activation='relu'))\n",
    "#model.add(MaxPooling1D(pool_size=2))\n",
    "#model.add(Flatten())\n",
    "#model.add(Dense(50, activation='relu'))\n",
    "model.add(Dense(1, activation='softmax'))\n",
    "print(model.summary())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 1/5\n",
      " - 530s - loss: 7.9712 - acc: 0.5000\n",
      "Epoch 2/5\n",
      " - 453s - loss: 7.9712 - acc: 0.5000\n",
      "Epoch 3/5\n",
      " - 435s - loss: 7.9712 - acc: 0.5000\n",
      "Epoch 4/5\n",
      " - 430s - loss: 7.9712 - acc: 0.5000\n",
      "Epoch 5/5\n",
      " - 431s - loss: 7.9712 - acc: 0.5000\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "<keras.callbacks.History at 0x1394b3256d8>"
      ]
     },
     "execution_count": 33,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# compile network\n",
    "model.compile(loss='binary_crossentropy', optimizer='adam', metrics=['accuracy'])\n",
    "# fit network\n",
    "model.fit(Xtrain, ytrain, epochs=5, verbose=2)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "loss, acc = model.evaluate(Xtest, ytest, verbose=0)\n",
    "print('Test Accuracy: {} and loss is {}' .format(acc*100,loss))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Train word2vec Embedding"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# load doc into memory\n",
    "def load_doc_learn_emb(filename):\n",
    "    # open the file as read only\n",
    "    file = open(filename, 'r')\n",
    "    # read all text\n",
    "    text = file.read()\n",
    "    # close the file\n",
    "    file.close()\n",
    "    return text\n",
    " \n",
    "# load the vocabulary\n",
    "vocab_filename = 'vocab.txt'\n",
    "vocab = load_doc_learn_emb(vocab_filename)\n",
    "vocab = vocab.split()\n",
    "vocab = set(vocab)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# turn a doc into clean tokens\n",
    "def doc_to_clean_lines_learn_emb(doc, vocab):\n",
    "    clean_lines = list()\n",
    "    lines = doc.splitlines()\n",
    "    for line in lines:\n",
    "        # split into tokens by white space\n",
    "        tokens = line.split()\n",
    "        # remove punctuation from each token\n",
    "        table = str.maketrans('', '', string.punctuation)\n",
    "        tokens = [w.translate(table) for w in tokens]\n",
    "        # filter out tokens not in vocab\n",
    "        tokens = [w for w in tokens if w in vocab]\n",
    "        clean_lines.append(tokens)\n",
    "    return clean_lines"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def process_docs_learn_emb(directory, vocab, is_trian):\n",
    "    lines = list()\n",
    "    # walk through all files in the folder\n",
    "    for filename in os.listdir(directory):\n",
    "        # skip any reviews in the test set\n",
    "        if is_trian and filename.startswith('cv9'):\n",
    "            continue\n",
    "        if not is_trian and not filename.startswith('cv9'):\n",
    "            continue\n",
    "        # create the full path of the file to open\n",
    "        path = directory + '/' + filename\n",
    "        # load and clean the doc\n",
    "        doc = load_doc_learn_emb(path)\n",
    "        doc_lines = doc_to_clean_lines_learn_emb(doc, vocab)\n",
    "        # add lines to list\n",
    "        lines += doc_lines\n",
    "    return lines\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# load training data\n",
    "positive_lines_learn_emb = process_docs_learn_emb(os.getcwd()+\"/pos\", vocab, True)\n",
    "negative_lines_learn_emb = process_docs_learn_emb(os.getcwd()+\"/neg\", vocab, True)\n",
    "sentences_learn_emb = negative_lines_learn_emb + positive_lines_learn_emb\n",
    "print('Total training sentences: %d' % len(sentences_learn_emb))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "len(sentences_learn_emb)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "from gensim.models import Word2Vec"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# train word2vec model\n",
    "model = Word2Vec(sentences_learn_emb, size=100, window=5, workers=8, min_count=1)\n",
    "# summarize vocabulary size in model\n",
    "words = list(model.wv.vocab)\n",
    "print('Vocabulary size: %d' % len(words))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "model.wv.word_vec"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# save model in ASCII (word2vec) format\n",
    "filename = 'embedding_word2vec.txt'\n",
    "model.wv.save_word2vec_format(filename, binary=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "model.wv.word_vec(\"film\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "model.wv.similarity(\"movie\",\"movie\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "from numpy import asarray\n",
    "from numpy import zeros"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def load_embedding(filename):\n",
    "    # load embedding into memory, skip first line\n",
    "    file = open(filename,'r')\n",
    "    lines = file.readlines()[1:]\n",
    "    file.close()\n",
    "    # create a map of words to vectors\n",
    "    embedding = dict()\n",
    "    for line in lines:\n",
    "        parts = line.split()\n",
    "        # key is string word, value is numpy array for vector\n",
    "        embedding[parts[0]] = asarray(parts[1:], dtype='float32')\n",
    "    return embedding"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "t = load_embedding(\"embedding_word2vec.txt\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "tokenizer.word_index"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# create a weight matrix for the Embedding layer from a loaded embedding\n",
    "def get_weight_matrix(embedding, vocab):\n",
    "    # total vocabulary size plus 0 for unknown words\n",
    "    vocab_size = len(vocab) + 1\n",
    "    # define weight matrix dimensions with all 0\n",
    "    weight_matrix = zeros((vocab_size, 100))\n",
    "    # step vocab, store vectors using the Tokenizer's integer mapping\n",
    "    for word, i in vocab.items():\n",
    "        weight_matrix[i] = embedding.get(word)\n",
    "    return weight_matrix"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# load embedding from file\n",
    "raw_embedding = load_embedding('embedding_word2vec.txt')\n",
    "# get vectors in the right order\n",
    "embedding_vectors = get_weight_matrix(raw_embedding, tokenizer.word_index)\n",
    "# create the embedding layer\n",
    "embedding_layer = Embedding(vocab_size, 100, weights=[embedding_vectors], input_length=max_length, trainable=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# define model\n",
    "model = Sequential()\n",
    "model.add(embedding_layer)\n",
    "model.add(Conv1D(filters=128, kernel_size=5, activation='relu'))\n",
    "model.add(MaxPooling1D(pool_size=2))\n",
    "model.add(Flatten())\n",
    "model.add(Dense(1, activation='sigmoid'))\n",
    "print(model.summary())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# compile network\n",
    "model.compile(loss='binary_crossentropy', optimizer='adam', metrics=['accuracy'])\n",
    "# fit network\n",
    "model.fit(Xtrain, ytrain, epochs=10, verbose=2)\n",
    "# evaluate\n",
    "loss, acc = model.evaluate(Xtest, ytest, verbose=0)\n",
    "print('Test Accuracy: {} and loss is {}' .format(acc*100,loss))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Using Pre Trained Layer"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# load embedding as a dict\n",
    "def load_embedding(filename):\n",
    "    # load embedding into memory, skip first line\n",
    "    file = open(filename,'r',encoding=\"utf8\")\n",
    "    lines = file.readlines()\n",
    "    file.close()\n",
    "    # create a map of words to vectors\n",
    "    embedding = dict()\n",
    "    for line in lines:\n",
    "        parts = line.split()\n",
    "        # key is string word, value is numpy array for vector\n",
    "        embedding[parts[0]] = asarray(parts[1:], dtype='float32')\n",
    "    return embedding"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def get_weight_matrix(embedding, vocab):\n",
    "    # total vocabulary size plus 0 for unknown words\n",
    "    vocab_size = len(vocab) + 1\n",
    "    # define weight matrix dimensions with all 0\n",
    "    weight_matrix = zeros((vocab_size, 100))\n",
    "    # step vocab, store vectors using the Tokenizer's integer mapping\n",
    "    for word, i in vocab.items():\n",
    "        vector = embedding.get(word)\n",
    "        if vector is not None:\n",
    "            weight_matrix[i] = vector\n",
    "    return weight_matrix"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# load embedding from file\n",
    "raw_embedding = load_embedding('glove.6B.100d.txt')\n",
    "# get vectors in the right order\n",
    "embedding_vectors = get_weight_matrix(raw_embedding, tokenizer.word_index)\n",
    "# create the embedding layer\n",
    "embedding_layer = Embedding(vocab_size, 100, weights=[embedding_vectors], input_length=max_length, trainable=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# define model\n",
    "model = Sequential()\n",
    "model.add(embedding_layer)\n",
    "model.add(Conv1D(filters=128, kernel_size=5, activation='relu'))\n",
    "model.add(MaxPooling1D(pool_size=2))\n",
    "model.add(Flatten())\n",
    "model.add(Dense(1, activation='sigmoid'))\n",
    "print(model.summary())\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# compile network\n",
    "model.compile(loss='binary_crossentropy', optimizer='adam', metrics=['accuracy'])\n",
    "# fit network\n",
    "model.fit(Xtrain, ytrain, epochs=7, verbose=2)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# evaluate\n",
    "loss, acc = model.evaluate(Xtest, ytest, verbose=0)\n",
    "print('Test Accuracy: {} and loss is {}' .format(acc*100,loss))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "strr = [[\"message\",\"Two\"],[\"message\",\"one\",\"is\"],[\"This\",\"is\",\"message\",\"three\"]]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "t = Tokenizer()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "t.fit_on_texts(strr)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "t.word_index"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "t.index_word"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "t.index_docs"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "t.word_docs"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "for w,c in list(t.word_docs.items()):\n",
    "    print(w, \"   \", c)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.5"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}