LOTR_test.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "## Josh Blaz -- LOTR\n",
    "## CS401 -- NLP\n",
    "\n",
    "import nltk\n",
    "import re\n",
    "import urllib.request\n",
    "import lxml.html as lh\n",
    "import io\n",
    "import requests\n",
    "import os\n",
    "import glob\n",
    "\n",
    "import pandas as pd\n",
    "import matplotlib.pyplot as plt\n",
    "import numpy as np\n",
    "\n",
    "from sklearn.feature_extraction.text import CountVectorizer\n",
    "from nltk.corpus import stopwords\n",
    "from collections import Counter \n",
    "from sklearn.decomposition import LatentDirichletAllocation\n",
    "from nltk.corpus import wordnet as wn\n",
    "import mglearn \n",
    "\n",
    "#NOTE: Sentiment140 Polarity values: 0: negative, 2: neutral, 4: positive"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# Initialize stop words, tolkien stop words, names and pucntuation to be filtered out of the text later\n",
    "stop_words = set(stopwords.words('english')) \n",
    "\n",
    "# Tolkien stop words -- words that he uses a ton in the text, also broken up parts like \"'ll\", \"'i\", and \"n't\"\n",
    "tolkien_stop = [\"men\",\"great\", \"'s\", \"said\", \"went\", \"he\", \"would\", \"many\", \"one\", \"he\", \"came\", \"yet\", \"even\",\\\n",
    "                \"shall\",\"upon\", \"days\", \"looked\", \"n't\", \"back\", \"could\", \"'ll\", \"'ve\", \"come\", \"still\", \"gate\", \"'i\" ]\n",
    "    \n",
    "names = [\"gandalf\", \"merry\", \"pippin\", \"frodo\", \"sam\", \"aragorn\", \"faramir\", \"denethor\", \"gimli\",\\\n",
    "        \"legolas\", \"strider\", \"boromir\", \"jowyn\", \"jomer\", \"beregond\", \"gollum\", \"bilbo\", \"thorin\"] \n",
    "\n",
    "punctuation = \".,;!?:`'()’■''\" ## including other symbols in the text"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### NOTE: Elvish text is translated awkwardly into the .txt format    \n",
    "</br>\n",
    "##### IE:   \n",
    "</br>\n",
    "►M MPR -F+MTRX MP ft PPtK P&RMPht: P. t. The last Two runes are the initials of Thror and Thrain.**  \n",
    "</br>\n",
    "#### Same with some of the intros:\n",
    "</br>\n",
    "“THE LORD OF THE RINGS” \n",
    "\n",
    "Pjrt Thttt \n",
    "\n",
    "THE RETURN \n",
    "OF THE KING \n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "## Get all tokens\n",
    "\n",
    "## --- The Silmarillion ---\n",
    "silm_file = open('silmarillion.txt', 'r')\n",
    "silm = silm_file.read() \n",
    "silm_raw = silm[43190:-5436] ## save raw files for later\n",
    "silm = silm.lower() ## Make all words lowercase \n",
    "silm = silm[43190:-5436] ## remove HTML jargon\n",
    "silm = nltk.word_tokenize(silm) ## tokenize\n",
    "\n",
    "## --- The Hobbit ---\n",
    "hobbit_file = open('hobbit.txt', 'r')\n",
    "hobbit = hobbit_file.read()\n",
    "hobbit_raw = hobbit[43212:-8543]\n",
    "hobbit = hobbit.lower()\n",
    "hobbit = hobbit[43212:-8543]\n",
    "hobbit = nltk.word_tokenize(hobbit)\n",
    "\n",
    "## --- The Fellowship of the Ring ---\n",
    "fellowship_file = open('fellowship.txt', 'r')\n",
    "fellowship = fellowship_file.read()\n",
    "fellowship_raw = fellowship[43242:-5436]\n",
    "fellowship = fellowship.lower()\n",
    "fellowship = fellowship[43242:-5436]\n",
    "fellowship = nltk.word_tokenize(fellowship)\n",
    "\n",
    "## --- The Two Towers ---\n",
    "twotowers_file = open('twotowers.txt', 'r')\n",
    "twotowers = twotowers_file.read()\n",
    "twotowers_raw = twotowers[43302:-19245]\n",
    "twotowers = twotowers.lower()\n",
    "twotowers = twotowers[43302:-19245]\n",
    "twotowers = nltk.word_tokenize(twotowers)\n",
    "\n",
    "## --- The Return of the King ---\n",
    "ret_file = open('return.txt', 'r')\n",
    "ret = ret_file.read()\n",
    "ret_raw = ret[43252:-5434]\n",
    "ret = ret.lower()\n",
    "ret = ret[43252:-5434]\n",
    "ret = nltk.word_tokenize(ret)\n",
    "\n",
    "tokenlist = [silm, hobbit, fellowship, twotowers, ret]\n",
    "\n",
    "raw_texts = [silm_raw, hobbit_raw, fellowship_raw, twotowers_raw, ret_raw] #Text files to use with LDA topic modeling\n",
    "\n",
    "\n",
    "entirety = [] ## This is all tokens combined\n",
    "for i in range(len(tokenlist)):\n",
    "    for word in tokenlist[i]:\n",
    "        entirety.append(word)\n",
    "        "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# Used Chapterize to split books into chapters \n",
    "## https://github.com/JonathanReeve/chapterize\n",
    "### Chapterize didn't work 100% perfectly, so I had to go through and the prologues back in when it cut them out\n",
    "\n",
    "# These are lists containing strings of every chapter for each book\n",
    "silm_chapters = []\n",
    "hobbit_chapters = []\n",
    "fellowship_chapters = []\n",
    "twotowers_chapters = []\n",
    "return_chapters = []\n",
    "\n",
    "# Paths to directories storing book chapters\n",
    "list_of_paths = ['/Users/blaz/Desktop/LOTR/silmarillion-chapters', '/Users/blaz/Desktop/LOTR/hobbit-chapters',\\\n",
    "                '/Users/blaz/Desktop/LOTR/fellowship-chapters', '/Users/blaz/Desktop/LOTR/twotowers-chapters',\\\n",
    "                '/Users/blaz/Desktop/LOTR/return-chapters']\n",
    "\n",
    "for path in list_of_paths: # iterate through the list of folder paths for each book\n",
    "    for file in sorted(glob.glob(os.path.join(path,'*.txt'))): # This gives us a sorted list of the files in each directory                                                         \n",
    "        f = open(file, 'r') # open and read file               # allowing us to read in the chapters in order.\n",
    "        txt = f.read()\n",
    "        ## determine which path we're using and append it to the correct book chapter list\n",
    "        if path == '/Users/blaz/Desktop/LOTR/silmarillion-chapters': \n",
    "            # Because of an issue with 'glob', I had to create a copy of the final chapter in The Silmarillion\n",
    "            silm_chapters.append(txt)\n",
    "        elif path == '/Users/blaz/Desktop/LOTR/hobbit-chapters':\n",
    "            hobbit_chapters.append(txt)\n",
    "        elif path == '/Users/blaz/Desktop/LOTR/fellowship-chapters': \n",
    "            fellowship_chapters.append(txt)\n",
    "        elif path == '/Users/blaz/Desktop/LOTR/twotowers-chapters': \n",
    "            twotowers_chapters.append(txt)\n",
    "        elif path == '/Users/blaz/Desktop/LOTR/return-chapters': \n",
    "            return_chapters.append(txt)\n",
    "        "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# Store chapter names for use in dataframes later\n",
    "\n",
    "silm_chapter_names = [\"Ainundalë\", \"Valaquenta\", \"Of the Beginning of Days\", \"Of Aulë and Yavanna\" , \"Of the Coming of the Elves and the Captivity of Melkor\",\\\n",
    "                     \"Of Thingol and Melian\", \"Of Eldamar and the Princes of the Eldalië\", \"Of Fëanor and the Unchaining of Melkor\", \"Of the Silmarils and the Unrest of the Noldor\",\\\n",
    "                     \"Of the Darkening of Valinor\", \"Of the Flight of the Noldor\", \"Of the Sindar\", \"Of the Sun and Moon and the Hiding of Valinor\", \"Of Men\", \"Of the Return of the Noldor\",\\\n",
    "                     \"Of Beleriand and its Realms\", \"Of the Noldor in Beleriand\", \"Of Maeglin\", \"Of the Coming of Men into the West\", \"Of the Ruin of Beleriand and the Fall of Fingolfin\", \"Of Beren and Lúthien\",\\\n",
    "                     \"Of the Fifth Battle: Nirnaeth Arnoediad\", \"Of Túrin Turambar\", \"Of the Ruin of Doriath\", \"Of Tuor and the Fall of Gondolin\", \"Of the Voyage of Eärendil and the War of Wrath\", \\\n",
    "                     \"Akallabêth: The Downfall of Númenor\", \"Of the Rings of Power and the Third Age\"]\n",
    "\n",
    "hobbit_chapter_names = [\"An Unexpected Party\", \"Roast Mutton\", \"A Short Rest\", \"Over Hill and Under Hill\", \"Riddles In The Dark\", \\\n",
    "                       \"Out Of The Frying-Pan Into The Fire\", \"Queer Lodgings\", \"Flies And Spiders\", \"Barrels Out Of Bond\", \"A Warm Welcome\", \\\n",
    "                       \"On The Doorstep\", \"Inside Information\", \"Not At Home\", \"Fire And Water\", \"The Gathering Of The Clouds\", \"A Thief In The Night\", \\\n",
    "                       \"The Clouds Burst\", \"The Return Journey\", \"The Last Stage\"]\n",
    "\n",
    "fellowship_chapter_names = [\"Concerning Hobbits\", \"Concerning Pipeweed\", \"Of the Ordering of the Shire\", \"Note on the Shire Records\", \"A Long-expected Party\", \"The Shadow of the Past\", \\\n",
    "                           \"Three is Company\", \"A Short Cut to Mushrooms\", \"A Conspiracy Unmasked\", \"The Old Forest\", \"In the House of Tom Bombadil\", \"Fog on the Barrow-downs\", \"At the Sign of the Prancing Pony\",\\\n",
    "                           \"Strider\", \"A Knife in the Dark\", \"Flight to the Ford\", \"Many Meetings\", \"The Council of Elrond\", \"The Ring goes South\", \"A Journey in the Dark\", \"The Bridge of Khazad-dûm\", \\\n",
    "                           \"Lothlórien\", \"The Mirror of Galadriel\", \"Farewell to Lórien\", \"The Great River\", \"The Breaking of the Fellowship\"]\n",
    "\n",
    "twotowers_chapter_names = [\"The Departure of Boromir\", \"The Riders of Rohan\", \"The Uruk-hai\", \"Treebeard\", \"The White Rider\", \"The King of the Golden Hall\", \"Helm's Deep\", \"The Road to Isengard\", \"Flotsam and Jetsam\", \\\n",
    "                          \"The Voice of Saruman\", \"The Palantír\", \"The Taming of Smeagol\", \"The Passage of the Marshes\", \"The Black Gate is Closed\", \"Of Herbs and Stewed Rabbit\", \"The Window of the West\", \"The Forbidden Pool\", \\\n",
    "                          \"Journey to the Cross-roads\", \"The Stairs to Cirith Ungol\", \"Shelob's Lair\", \"The Choices of Master Samwise\"]\n",
    "\n",
    "return_chapter_names = [\"Minas Tirith\", \"The Passing of the Grey Company\", \"The Muster of Rohan\", \"The Siege of Gondor\", \"The Ride of the Rohirrim\", \"The Battle of the Pelennor Fields\", \"The Pyre of Denethor\",\\\n",
    "                       \"The Houses of Healing\", \"The Last Debate\", \"The Black Gate Opens\", \"The Tower of Cirith Ungol\", \"The Land of Shadow\", \"Mount Doom\", \"The Field of Cormallen\", \"The Steward and the King\", \\\n",
    "                       \"Many Partings\", \"Homeward Bound\", \"Scouring of the Shire\", \"The Grey Havens\"]\n",
    "\n",
    "chapter_name_list = [silm_chapter_names, hobbit_chapter_names, fellowship_chapter_names, twotowers_chapter_names, return_chapter_names]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Sentiment Analysis"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "\n",
    "### ** 1. Segment all chapters into page-sized objects    **    \n",
    "</br>\n",
    "### ** 2. Send all segments to Sentiment140 API by chapter    **   \n",
    "</br>\n",
    "### ** 3. Calculate polarity averages and polarity lists. **   \n",
    "</br>\n",
    "### ** 4. Store API polarity ratings and export to csv**\n",
    "</br>\n",
    "### ** 5. Plot all polarities + Averages**       "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "\"\"\"\n",
    "Function that segments given chapter into n-sized segments to be sent to the API.\n",
    "Typically using n=2940, as this is the #chars in my copy of Fellowship of the Ring.\n",
    "\n",
    "Parameters - chapter - chapter of a book to be broken into segments\n",
    "           - n - length that we segment the text with\n",
    "       \n",
    "Returns a list of (string) segments of the chapter.\n",
    "\"\"\"\n",
    "def Segmenter(chapter, n):\n",
    "    segments = []\n",
    "    # start and end indices for segmenting the text\n",
    "    start = 0\n",
    "    end = n\n",
    "    while end < len(chapter) + n:\n",
    "        segments.append(chapter[start:end])\n",
    "        start = end\n",
    "        end = end + n\n",
    "    return segments #segments of input chapter"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# Goal:\n",
    "# Create lists of lists for segments of each chapter of each book, append to them using \"Segmenter\" function, \n",
    "# storing them like this will allow for iterative querying of the API server\n",
    "\n",
    "\n",
    "# Lists of Lists of Lists storing all segments of all chapters for each book\n",
    "# [[chapter1 segment 0-2500, chap1, segmenet 2500-5000]... [chapter2 segment0-2500, ...]...]\n",
    "silm_segments = []\n",
    "hobbit_segments = []\n",
    "fellowship_segments = []\n",
    "twotowers_segments = []\n",
    "return_segments = []\n",
    "\n",
    "# List containing the lists storing each books' chapters\n",
    "list_of_books = [silm_chapters, hobbit_chapters, fellowship_chapters, twotowers_chapters, return_chapters]\n",
    "# List allowing us to access the segment lists\n",
    "list_of_segments = [silm_segments, hobbit_segments, fellowship_segments, twotowers_segments, return_segments]\n",
    "\n",
    "for i in range(len(list_of_books)):\n",
    "    for chapter in list_of_books[i]: # Segment entire chapter using Segmenter function, with 2940 character cuts\n",
    "        list_of_segments[i].append(Segmenter(chapter,2940))\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "\"\"\"\n",
    "Function that sends segments of 1 chapter through the Sentiment140 API.\n",
    "In order to do so, it creates and appends segments to a JSON file, then posts the JSON queries to the API server\n",
    "using requests module (using an HTTP Post)\n",
    "\n",
    "Parameters - chapter_segments - segments of an entire chapter of a book\n",
    "\n",
    "Returns a list of polarities for segments of the chapter, as well as the polarity average for the chapter\n",
    "\n",
    "Note: Maximum of 700,000 characters per API request, though this shouldn't be a problem\n",
    "\"\"\"\n",
    "\n",
    "def Polarity(chapter_segments): # segments of a single chapter\n",
    "    request = {'data':[]}\n",
    "    polarityList = []\n",
    "    counter = 0\n",
    "    for segment in chapter_segments: # Fill JSON\n",
    "        request['data'].append({'text':segment})\n",
    "    r = requests.post('http://www.sentiment140.com/api/bulkClassifyJson?appid=blaz_j1@denison.edu', json=request)\n",
    "    jso = r.json()\n",
    "    for i in range(len(request['data'])-1):\n",
    "        polarityList.append(jso['data'][i]['polarity'])\n",
    "    \n",
    "    polarityTotal = 0\n",
    "    for value in polarityList:\n",
    "        polarityTotal = polarityTotal + value\n",
    "    \n",
    "    polarityAVG = polarityTotal/len(polarityList)\n",
    "    return polarityList, polarityAVG"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {
    "collapsed": true,
    "scrolled": false
   },
   "outputs": [],
   "source": [
    "# store all chapter polarity averages along with all polarity ratings for each chapter\n",
    "silm_polarity_avg = []\n",
    "hobbit_polarity_avg = []\n",
    "fellowship_polarity_avg = []\n",
    "twotowers_polarity_avg = []\n",
    "return_polarity_avg = []\n",
    "\n",
    "silm_polarity_lists = []\n",
    "hobbit_polarity_lists = []\n",
    "fellowship_polarity_lists = []\n",
    "twotowers_polarity_lists = []\n",
    "return_polarity_lists = []\n",
    "\n",
    "for x in range(len(list_of_books)):\n",
    "    book = list_of_books[x]\n",
    "    segs = list_of_segments[x]\n",
    "    \n",
    "    for i in range(len(book)):\n",
    "        if x == 0:\n",
    "            temp1 = []\n",
    "            temp2 = 0.0\n",
    "            temp1,temp2 = Polarity(segs[i])\n",
    "            silm_polarity_lists.append(temp1)\n",
    "            silm_polarity_avg.append(temp2)\n",
    "        if x == 1:\n",
    "            temp1 = []\n",
    "            temp2 = 0.0\n",
    "            temp1,temp2 = Polarity(segs[i])\n",
    "            hobbit_polarity_lists.append(temp1)\n",
    "            hobbit_polarity_avg.append(temp2)\n",
    "        if x == 2:\n",
    "            temp1 = []\n",
    "            temp2 = 0.0\n",
    "            temp1,temp2 = Polarity(segs[i])\n",
    "            fellowship_polarity_lists.append(temp1)\n",
    "            fellowship_polarity_avg.append(temp2)\n",
    "        if x == 3:\n",
    "            temp1 = []\n",
    "            temp2 = 0.0\n",
    "            temp1,temp2 = Polarity(segs[i])\n",
    "            twotowers_polarity_lists.append(temp1)\n",
    "            twotowers_polarity_avg.append(temp2)\n",
    "        if x == 4:\n",
    "            temp1 = []\n",
    "            temp2 = 0.0\n",
    "            temp1,temp2 = Polarity(segs[i])\n",
    "            return_polarity_lists.append(temp1)\n",
    "            return_polarity_avg.append(temp2)\n",
    "\n",
    "all_polarity_avgs = [silm_polarity_avg, hobbit_polarity_avg, fellowship_polarity_avg, twotowers_polarity_avg, return_polarity_avg]\n",
    "\n",
    "all_polarity_lists = [silm_polarity_lists, hobbit_polarity_lists, fellowship_polarity_lists, twotowers_polarity_lists, return_polarity_lists]  \n",
    "# chapter 3 of return of the king is super dark"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'\\n# Converting Polarity AVG data into pandas dataframes\\n## These CSVs store all average chapter polarities for each book\\nsilm_df = pd.DataFrame(silm_polarity_avg, index = silm_chapter_names, columns = [\"Polarity\"])\\nsilm_df = silm_df.rename_axis(\"--- The Silmarillion ---\")\\n#silm_df.to_csv(\"silm_df.csv\")\\n\\nhobbit_df = pd.DataFrame(hobbit_polarity_avg, index = hobbit_chapter_names, columns = [\"Polarity\"])\\nhobbit_df = hobbit_df.rename_axis(\"--- The Hobbit ---\")\\n#hobbit_df.to_csv(\"hobbit_df.csv\")\\n\\nfellowship_df = pd.DataFrame(fellowship_polarity_avg, index = fellowship_chapter_names, columns = [\"Polarity\"])\\nfellowship_df = fellowship_df.rename_axis(\"--- The Fellowship of the Ring ---\")\\n# Prologue chapters have weird polarities - have solid values because they\\'re shorter\\n#fellowship_df.to_csv(\"fellowship_df.csv\")\\n\\ntwotowers_df = pd.DataFrame(twotowers_polarity_avg, index = twotowers_chapter_names, columns = [\"Polarity\"])\\ntwotowers_df = twotowers_df.rename_axis(\"--- The Two Towers ---\")\\n#twotowers_df.to_csv(\"twotowers_df.csv\")\\n\\nreturn_df = pd.DataFrame(return_polarity_avg, index = return_chapter_names, columns = [\"Polarity\"])\\nreturn_df = return_df.rename_axis(\"--- The Return of the King ---\")\\n#return_df.to_csv(\"return_df.csv\")\\n\\n# Dataframe of all Books overlaid\\nbooks_df = [silm_df, hobbit_df, fellowship_df, twotowers_df, return_df]\\nfull_df = pd.concat(books_df)\\n\\n# Export to CSV\\nfull_df.to_csv(\"full_df.csv\")\\n\\n\\n# Exported to Excel as well\\nexcel = pd.ExcelWriter(\\'LOTR1.xlsx\\')\\nsilm_df.to_excel(excel, \\'The Silmarillion\\')\\nhobbit_df.to_excel(excel, \\'The Hobbit\\')\\nfellowship_df.to_excel(excel, \\'Fellowship of the Rings\\')\\ntwotowers_df.to_excel(excel, \\'The Two Towers\\')\\nreturn_df.to_excel(excel, \\'The Return of the King\\')\\nfull_df.to_excel(excel, \\'Combined\\')\\nexcel.save()\\n'"
      ]
     },
     "execution_count": 28,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Commented this all out so I don't reset my excel work every time\n",
    "\"\"\"\n",
    "# Converting Polarity AVG data into pandas dataframes\n",
    "## These CSVs store all average chapter polarities for each book\n",
    "silm_df = pd.DataFrame(silm_polarity_avg, index = silm_chapter_names, columns = [\"Polarity\"])\n",
    "silm_df = silm_df.rename_axis(\"--- The Silmarillion ---\")\n",
    "#silm_df.to_csv(\"silm_df.csv\")\n",
    "\n",
    "hobbit_df = pd.DataFrame(hobbit_polarity_avg, index = hobbit_chapter_names, columns = [\"Polarity\"])\n",
    "hobbit_df = hobbit_df.rename_axis(\"--- The Hobbit ---\")\n",
    "#hobbit_df.to_csv(\"hobbit_df.csv\")\n",
    "\n",
    "fellowship_df = pd.DataFrame(fellowship_polarity_avg, index = fellowship_chapter_names, columns = [\"Polarity\"])\n",
    "fellowship_df = fellowship_df.rename_axis(\"--- The Fellowship of the Ring ---\")\n",
    "# Prologue chapters have weird polarities - have solid values because they're shorter\n",
    "#fellowship_df.to_csv(\"fellowship_df.csv\")\n",
    "\n",
    "twotowers_df = pd.DataFrame(twotowers_polarity_avg, index = twotowers_chapter_names, columns = [\"Polarity\"])\n",
    "twotowers_df = twotowers_df.rename_axis(\"--- The Two Towers ---\")\n",
    "#twotowers_df.to_csv(\"twotowers_df.csv\")\n",
    "\n",
    "return_df = pd.DataFrame(return_polarity_avg, index = return_chapter_names, columns = [\"Polarity\"])\n",
    "return_df = return_df.rename_axis(\"--- The Return of the King ---\")\n",
    "#return_df.to_csv(\"return_df.csv\")\n",
    "\n",
    "# Dataframe of all Books overlaid\n",
    "books_df = [silm_df, hobbit_df, fellowship_df, twotowers_df, return_df]\n",
    "full_df = pd.concat(books_df)\n",
    "\n",
    "# Export to CSV\n",
    "full_df.to_csv(\"full_df.csv\")\n",
    "\n",
    "\n",
    "# Exported to Excel as well\n",
    "excel = pd.ExcelWriter('LOTR1.xlsx')\n",
    "silm_df.to_excel(excel, 'The Silmarillion')\n",
    "hobbit_df.to_excel(excel, 'The Hobbit')\n",
    "fellowship_df.to_excel(excel, 'Fellowship of the Rings')\n",
    "twotowers_df.to_excel(excel, 'The Two Towers')\n",
    "return_df.to_excel(excel, 'The Return of the King')\n",
    "full_df.to_excel(excel, 'Combined')\n",
    "excel.save()\n",
    "\"\"\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'\\n# Converting Polarity List data into pandas dataframes\\n## These CSVs store all polarity ratings for each book, rather than average chapter polarity ratings\\n### These dataframes are absolutely unusable, the data is just too hard to viz\\nsilm_all_pol = []\\nfor i in range(len(silm_polarity_lists)):\\n    for polarity in silm_polarity_lists[i]:\\n        silm_all_pol.append(polarity)\\nsilm_all_pol = pd.DataFrame(silm_all_pol, columns = [\"Polarity\"])\\n\\nhobbit_all_pol = []\\nfor i in range(len(hobbit_polarity_lists)):\\n    for polarity in hobbit_polarity_lists[i]:\\n        hobbit_all_pol.append(polarity)\\nhobbit_all_pol = pd.DataFrame(hobbit_all_pol, columns = [\"Polarity\"])\\n\\nfellowship_all_pol = []\\nfor i in range(len(fellowship_polarity_lists)):\\n    for polarity in fellowship_polarity_lists[i]:\\n        fellowship_all_pol.append(polarity)\\nfellowship_all_pol = pd.DataFrame(fellowship_all_pol, columns = [\"Polarity\"])\\n\\ntwotowers_all_pol = []\\nfor i in range(len(twotowers_polarity_lists)):\\n    for polarity in twotowers_polarity_lists[i]:\\n        twotowers_all_pol.append(polarity)\\ntwotowers_all_pol = pd.DataFrame(twotowers_all_pol, columns = [\"Polarity\"])\\n\\nreturn_all_pol = []\\nfor i in range(len(return_polarity_lists)):\\n    for polarity in return_polarity_lists[i]:\\n        return_all_pol.append(polarity)\\nreturn_all_pol = pd.DataFrame(return_all_pol, columns = [\"Polarity\"])\\n\\nall_pol_list = [silm_all_pol, hobbit_all_pol, fellowship_all_pol, twotowers_all_pol, return_all_pol]\\nall_pol = pd.concat(all_pol_list)\\n\\nexcel2 = pd.ExcelWriter(\\'LOTR2.xlsx\\')\\nsilm_all_pol.to_excel(excel2, \\'The Silmarillion\\')\\nhobbit_all_pol.to_excel(excel2, \\'The Hobbit\\')\\nfellowship_all_pol.to_excel(excel2, \\'Fellowship of the Rings\\')\\ntwotowers_all_pol.to_excel(excel2, \\'The Two Towers\\')\\nreturn_all_pol.to_excel(excel2, \\'The Return of the King\\')\\nall_pol.to_excel(excel2, \\'Combined\\')\\n\\nexcel2.save()\\n'"
      ]
     },
     "execution_count": 29,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Again, commented out data exports\n",
    "\"\"\"\n",
    "# Converting Polarity List data into pandas dataframes\n",
    "## These CSVs store all polarity ratings for each book, rather than average chapter polarity ratings\n",
    "### These dataframes are absolutely unusable, the data is just too hard to viz\n",
    "silm_all_pol = []\n",
    "for i in range(len(silm_polarity_lists)):\n",
    "    for polarity in silm_polarity_lists[i]:\n",
    "        silm_all_pol.append(polarity)\n",
    "silm_all_pol = pd.DataFrame(silm_all_pol, columns = [\"Polarity\"])\n",
    "\n",
    "hobbit_all_pol = []\n",
    "for i in range(len(hobbit_polarity_lists)):\n",
    "    for polarity in hobbit_polarity_lists[i]:\n",
    "        hobbit_all_pol.append(polarity)\n",
    "hobbit_all_pol = pd.DataFrame(hobbit_all_pol, columns = [\"Polarity\"])\n",
    "\n",
    "fellowship_all_pol = []\n",
    "for i in range(len(fellowship_polarity_lists)):\n",
    "    for polarity in fellowship_polarity_lists[i]:\n",
    "        fellowship_all_pol.append(polarity)\n",
    "fellowship_all_pol = pd.DataFrame(fellowship_all_pol, columns = [\"Polarity\"])\n",
    "\n",
    "twotowers_all_pol = []\n",
    "for i in range(len(twotowers_polarity_lists)):\n",
    "    for polarity in twotowers_polarity_lists[i]:\n",
    "        twotowers_all_pol.append(polarity)\n",
    "twotowers_all_pol = pd.DataFrame(twotowers_all_pol, columns = [\"Polarity\"])\n",
    "\n",
    "return_all_pol = []\n",
    "for i in range(len(return_polarity_lists)):\n",
    "    for polarity in return_polarity_lists[i]:\n",
    "        return_all_pol.append(polarity)\n",
    "return_all_pol = pd.DataFrame(return_all_pol, columns = [\"Polarity\"])\n",
    "\n",
    "all_pol_list = [silm_all_pol, hobbit_all_pol, fellowship_all_pol, twotowers_all_pol, return_all_pol]\n",
    "all_pol = pd.concat(all_pol_list)\n",
    "\n",
    "excel2 = pd.ExcelWriter('LOTR2.xlsx')\n",
    "silm_all_pol.to_excel(excel2, 'The Silmarillion')\n",
    "hobbit_all_pol.to_excel(excel2, 'The Hobbit')\n",
    "fellowship_all_pol.to_excel(excel2, 'Fellowship of the Rings')\n",
    "twotowers_all_pol.to_excel(excel2, 'The Two Towers')\n",
    "return_all_pol.to_excel(excel2, 'The Return of the King')\n",
    "all_pol.to_excel(excel2, 'Combined')\n",
    "\n",
    "excel2.save()\n",
    "\"\"\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "\"\"\"\n",
    "Function that tokenizes and cleans the words of every chapter in a book.\n",
    "\n",
    "Parameters - book - a book.\n",
    "\n",
    "Returns a List of Lists storing a tokenized list for every chapter in a book.\n",
    "\"\"\"\n",
    "\n",
    "def Tokenize(book):\n",
    "    punctuation = \".,;!?:`'()’■''\" ## including other symbols\n",
    "    token_list = []\n",
    "    for chapter in book:\n",
    "        temp = []\n",
    "        words = nltk.word_tokenize(chapter)\n",
    "        for word in words:\n",
    "            word = word.lower()\n",
    "            if word not in punctuation and not word.isnumeric(): \n",
    "                temp.append(word)\n",
    "        token_list.append(temp)\n",
    "        \n",
    "    return token_list\n",
    "\n",
    "\n",
    "#tokens = (Tokenize(silm_chapters))\n",
    "#print(tokens[0]) ## Tokens of first chapter of The Silmarillion"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "\"\"\"\n",
    "Function that returns the n most common words for every chapter in a book.\n",
    "This is accomplished by using 'Counter' in the 'Collections' module.\n",
    "\n",
    "Parameters - book - a tokenized list of lists of all chapters of a book\n",
    "          - n - number of most common words in the chapter\n",
    "          \n",
    "Returns a List of Lists of the n most common words of every chapter in the book.\n",
    "\"\"\"\n",
    "\n",
    "def MostCommon(book, n): \n",
    "    common_words = []\n",
    "    for chapter in book:\n",
    "        temp = []\n",
    "        for word in chapter:\n",
    "            if word.isalpha() and word not in stop_words and word not in names and word not in tolkien_stop:\n",
    "                temp.append(word)\n",
    "                  \n",
    "        common_words.append(Counter(temp).most_common(n))\n",
    "    return common_words"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "## Find top 5 NON-STOP words per chapter\n",
    "\n",
    "silm_chapters_tokenized = []\n",
    "hobbit_chapters_tokenized = []\n",
    "fellowship_chapters_tokenized = []\n",
    "twotowers_chapters_tokenized = []\n",
    "return_chapters_tokenized = []\n",
    "\n",
    "silm_chapters_common = []\n",
    "hobbit_chapters_common = []\n",
    "fellowship_chapters_common = []\n",
    "twotowers_chapters_common = []\n",
    "return_chapters_common = []\n",
    "\n",
    "for i in range(len(list_of_books)):\n",
    "    if i == 0:\n",
    "        silm_chapters_tokenized = Tokenize(list_of_books[i])\n",
    "        silm_chapters_common = MostCommon(silm_chapters_tokenized, 5)\n",
    "    if i == 1:\n",
    "        hobbit_chapters_tokenized = Tokenize(list_of_books[i])\n",
    "        hobbit_chapters_common = MostCommon(hobbit_chapters_tokenized, 5)\n",
    "    if i == 2:\n",
    "        fellowship_chapters_tokenized = Tokenize(list_of_books[i])\n",
    "        fellowship_chapters_common = MostCommon(fellowship_chapters_tokenized, 5)\n",
    "    if i == 3:\n",
    "        twotowers_chapters_tokenized = Tokenize(list_of_books[i])\n",
    "        twotowers_chapters_common = MostCommon(twotowers_chapters_tokenized, 5)\n",
    "    if i == 4:\n",
    "        return_chapters_tokenized = Tokenize(list_of_books[i])\n",
    "        return_chapters_common = MostCommon(return_chapters_tokenized, 5)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "A Knife in the Dark\n",
      "[('last', 23), ('us', 23), ('away', 22), ('left', 22), ('road', 22)]\n",
      "The Departure of Boromir\n",
      "[('ores', 24), ('away', 12), ('long', 11), ('boat', 11), ('wind', 11)]\n",
      "The Ride of the Rohirrim\n",
      "[('wild', 25), ('king', 21), ('road', 17), ('like', 15), ('away', 14)]\n",
      "The Battle of the Pelennor Fields\n",
      "[('king', 27), ('like', 17), ('fell', 17), ('black', 16), ('city', 14)]\n",
      "The Pyre of Denethor\n",
      "[('lord', 17), ('stood', 14), ('city', 14), ('door', 12), ('away', 11)]\n",
      "The Black Gate Opens\n",
      "[('mordor', 17), ('black', 17), ('sauron', 16), ('last', 14), ('away', 13)]\n",
      "Mount Doom\n",
      "[('mountain', 27), ('master', 24), ('away', 24), ('dark', 24), ('last', 24)]\n"
     ]
    }
   ],
   "source": [
    "## Common words in Full-Negative chapters\n",
    "\n",
    "print(fellowship_chapter_names[14])\n",
    "print(fellowship_chapters_common[14])\n",
    "\n",
    "print(twotowers_chapter_names[0])\n",
    "print(twotowers_chapters_common[0])\n",
    "\n",
    "print(return_chapter_names[4])\n",
    "print(return_chapters_common[4])\n",
    "\n",
    "print(return_chapter_names[5])\n",
    "print(return_chapters_common[5])\n",
    "\n",
    "print(return_chapter_names[6])\n",
    "print(return_chapters_common[6])\n",
    "\n",
    "print(return_chapter_names[9])\n",
    "print(return_chapters_common[9])\n",
    "\n",
    "print(return_chapter_names[12])\n",
    "print(return_chapters_common[12])\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 34,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Riddles In The Dark\n",
      "[('yes', 25), ('way', 24), ('got', 23), ('goblins', 23), ('dark', 22)]\n",
      "On The Doorstep\n",
      "[('mountain', 14), ('day', 12), ('dwarves', 11), ('valley', 10), ('river', 9)]\n",
      "Not At Home\n",
      "[('light', 23), ('dwarves', 17), ('smaug', 12), ('long', 12), ('door', 11)]\n",
      "Fire And Water\n",
      "[('bard', 21), ('master', 17), ('town', 16), ('dragon', 16), ('lake', 14)]\n"
     ]
    }
   ],
   "source": [
    "# Hobbit\n",
    "# 8 Chapters with polarity below 0.5\n",
    "\n",
    "print(hobbit_chapter_names[4])\n",
    "print(hobbit_chapters_common[4])\n",
    "\n",
    "print(hobbit_chapter_names[10])\n",
    "print(hobbit_chapters_common[10])\n",
    "\n",
    "print(hobbit_chapter_names[12])\n",
    "print(hobbit_chapters_common[12])\n",
    "\n",
    "print(hobbit_chapter_names[13])\n",
    "print(hobbit_chapters_common[13])\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 35,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "## Create list of Full-Negative Chapters\n",
    "\n",
    "\"\"\"\n",
    "A Knife in the Dark\n",
    "The Departure of Boromir\n",
    "The Ride of the Rohirrim\n",
    "The Battle of the Pelennor Fields\n",
    "The Pyre of Denethor\n",
    "The Black Gate Opens\n",
    "Mount Doom\n",
    "\"\"\"\n",
    "\n",
    "negative_chapters = []\n",
    "\n",
    "negative_chapters.append(fellowship_chapters[14])\n",
    "negative_chapters.append(twotowers_chapters[0])\n",
    "negative_chapters.append(return_chapters[4])\n",
    "negative_chapters.append(return_chapters[5])\n",
    "negative_chapters.append(return_chapters[6])\n",
    "negative_chapters.append(return_chapters[9])\n",
    "negative_chapters.append(return_chapters[12])"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "collapsed": true
   },
   "source": [
    "# Topic Modeling"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "LDA - Latent Dirichlet Allocation \n",
    "- tokenize\n",
    "- remove stop words\n",
    "- lemmatize tokens\n",
    "- vectorize\n",
    "- model\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 36,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "\"\"\"\n",
    "Function that lemmatizes a word.\n",
    "\n",
    "Parameters - word - word to be lemmatized\n",
    "\n",
    "Returns the lemmatized version of the word.\n",
    "\"\"\"\n",
    "def lemma(word):\n",
    "    lemma = wn.morphy(word)\n",
    "    if lemma is None:\n",
    "        return word\n",
    "    else:\n",
    "        return lemma\n",
    "    "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 37,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "\"\"\"\n",
    "This function prepares tokens to be sent through the LDA process. In doing so, it needs to make sure the tokens are\n",
    "words, they aren't in the 'Tolkien Stop Words' and that they are lemmas.\n",
    "The Latent Dirichlet Allocation function will remove stop words and ignore punctuation.\n",
    "\n",
    "Parameters - tokens - tokens of a chapter to lematize.\n",
    "\n",
    "Returns a list of tokens that are ready to be joined back together for Latent Dirichlet Allocation.\n",
    "\"\"\"\n",
    "def LDA_prepare(tokens):\n",
    "    ret = []\n",
    "    lemmatized = []\n",
    "    \n",
    "    for word in tokens:\n",
    "        word = word.lower()\n",
    "        if word.isalpha() and word not in tolkien_stop and word not in names:\n",
    "            ret.append(word)\n",
    "            \n",
    "    for word in ret:\n",
    "        lemmatized.append(lemma(word))\n",
    "        \n",
    "    return lemmatized"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 38,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "lda.components_.shape: (100, 920)\n",
      "topic 0       topic 1       topic 2       topic 3       topic 4       \n",
      "--------      --------      --------      --------      --------      \n",
      "foul          young         young         young         young         \n",
      "young         freshen       freshen       freshen       freshen       \n",
      "freshen       foul          foul          foul          foul          \n",
      "forward       forward       forward       forward       forward       \n",
      "fortune       fortune       fortune       fortune       fortune       \n",
      "forth         forth         forth         forth         forth         \n",
      "forlorn       forlorn       forlorn       forlorn       forlorn       \n",
      "forgetfulness forgetfulness forgetfulness forgetfulness forgetfulness \n",
      "forget        forget        forget        forget        forget        \n",
      "foreboding    foreboding    foreboding    foreboding    foreboding    \n",
      "\n",
      "\n",
      "topic 5       topic 6       topic 7       topic 8       topic 9       \n",
      "--------      --------      --------      --------      --------      \n",
      "young         young         young         young         young         \n",
      "freshen       freshen       freshen       freshen       freshen       \n",
      "foul          foul          foul          foul          foul          \n",
      "forward       forward       forward       forward       forward       \n",
      "fortune       fortune       fortune       fortune       fortune       \n",
      "forth         forth         forth         forth         forth         \n",
      "forlorn       forlorn       forlorn       forlorn       forlorn       \n",
      "forgetfulness forgetfulness forgetfulness forgetfulness forgetfulness \n",
      "forget        forget        forget        forget        forget        \n",
      "foreboding    foreboding    foreboding    foreboding    foreboding    \n",
      "\n",
      "\n"
     ]
    }
   ],
   "source": [
    "\"\"\"\n",
    "Function that prepares a chapter for LDA and then performs LDA.\n",
    "\n",
    "Parameters - chapter - chapter to perform LDA on\n",
    "           - topics - number of topics in the LDA model\n",
    "           \n",
    "Prints the topics found in the model.\n",
    "\"\"\"\n",
    "\n",
    "# I created this function in order to iteratively run LDA on each chapter, but something in the LDA sklearn LDA \n",
    "# function doesn't allow this. When I tried to run it this way, LDA didn't run properly, so I just had to run a \n",
    "# new instance of the model for each chapter in the cells below.\n",
    "\n",
    "\"\"\"\n",
    "def LDA_function(chapter, topics):\n",
    "    \n",
    "    # Initialize Vectorizer and LDA model \n",
    "    vect = CountVectorizer(max_features=10000, max_df=.15, stop_words='english')\n",
    "    \n",
    "    lda = LatentDirichletAllocation(n_components=topics, learning_method=\"batch\",\n",
    "                                    max_iter=200, random_state=1)\n",
    "    \n",
    "    # Tokenize\n",
    "    tokens = nltk.word_tokenize(chapter)\n",
    "    # Prepare for LDA\n",
    "    tokens = LDA_prepare(tokens)\n",
    "    \n",
    "    # Join back together\n",
    "    join = [''.join(filter(str.isalpha, word)) for word in tokens]\n",
    "    #Vectorize\n",
    "    vec = vect.fit_transform(join)\n",
    "    document_topics = lda.fit_transform(vec)\n",
    "    \n",
    "    print(\"lda.components_.shape: {}\".format(lda.components_.shape))\n",
    "    lda.components_.shape: (10, 10000)\n",
    "\n",
    "    sorting = np.argsort(lda.components_, axis=1)[:, ::-1]\n",
    "\n",
    "    feature_names = np.array(vect.get_feature_names())\n",
    "\n",
    "    mglearn.tools.print_topics(topics=range(10), feature_names=feature_names, \n",
    "                           sorting=sorting, topics_per_chunk=5, n_words=10)\n",
    "    \n",
    "LDA_function(negative_chapters[5], 100)\n",
    "\"\"\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 40,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "lda.components_.shape: (10, 1313)\n",
      "topic 0       topic 1       topic 2       topic 3       topic 4       \n",
      "--------      --------      --------      --------      --------      \n",
      "black         round         house         make          saw           \n",
      "eyes          weathertop    run           foot          old           \n",
      "stand         sign          thought       time          hobbit        \n",
      "ask           slowly        hand          hope          day           \n",
      "enemy         felt          fell          dark          shadow        \n",
      "fear          soon          ha            ring          answer        \n",
      "near          turn          sky           line          star          \n",
      "tall          ground        look          open          right         \n",
      "path          looking       suddenly      stone         elves         \n",
      "hollow        hair          north         slope         use           \n",
      "\n",
      "\n",
      "topic 5       topic 6       topic 7       topic 8       topic 9       \n",
      "--------      --------      --------      --------      --------      \n",
      "far           hill          bree          road          wa            \n",
      "way           long          pass          away          left          \n",
      "tell          know          dell          light         like          \n",
      "pony          night         hear          land          cold          \n",
      "better        door          beren         lay           tale          \n",
      "figure        sound         little        think         begin         \n",
      "clear         world         fall          end           good          \n",
      "moon          sleep         close         ferny         country       \n",
      "spring        butterbur     short         mean          lost          \n",
      "bird          journey       voice         face          pale          \n",
      "\n",
      "\n"
     ]
    }
   ],
   "source": [
    "vect = CountVectorizer(max_features=10000, max_df=.15, stop_words='english')\n",
    "    \n",
    "lda = LatentDirichletAllocation(n_components=10, learning_method=\"batch\",\n",
    "                                    max_iter=500, random_state=1)\n",
    "\n",
    "Knife = negative_chapters[0]\n",
    "KT = nltk.word_tokenize(Knife)\n",
    "KT = LDA_prepare(KT)\n",
    "join = [''.join(filter(str.isalpha, word)) for word in KT]\n",
    "vec = vect.fit_transform(join)\n",
    "document_topics = lda.fit_transform(vec)\n",
    "\n",
    "print(\"lda.components_.shape: {}\".format(lda.components_.shape))\n",
    "lda.components_.shape: (10, 10000)\n",
    "\n",
    "sorting = np.argsort(lda.components_, axis=1)[:, ::-1]\n",
    "\n",
    "feature_names = np.array(vect.get_feature_names())\n",
    "\n",
    "mglearn.tools.print_topics(topics=range(10), feature_names=feature_names, \n",
    "                           sorting=sorting, topics_per_chunk=5, n_words=10)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 56,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "lda.components_.shape: (10, 1313)\n",
      "topic 0       topic 1       topic 2       topic 3       topic 4       \n",
      "--------      --------      --------      --------      --------      \n",
      "black         round         house         make          saw           \n",
      "eyes          weathertop    run           foot          old           \n",
      "stand         sign          fell          time          day           \n",
      "ask           slowly        hand          hope          hobbit        \n",
      "enemy         soon          thought       dark          shadow        \n",
      "fear          felt          sky           ring          answer        \n",
      "near          turn          ha            line          star          \n",
      "tall          looking       suddenly      open          right         \n",
      "path          ground        north         stone         elves         \n",
      "cut           hair          look          grey          use           \n",
      "\n",
      "\n",
      "topic 5       topic 6       topic 7       topic 8       topic 9       \n",
      "--------      --------      --------      --------      --------      \n",
      "far           hill          bree          away          wa            \n",
      "way           long          pass          road          left          \n",
      "tell          know          dell          light         like          \n",
      "pony          night         hear          land          cold          \n",
      "better        door          beren         lay           tale          \n",
      "figure        sound         little        think         good          \n",
      "clear         world         fall          end           begin         \n",
      "moon          sleep         close         ferny         country       \n",
      "spring        butterbur     reach         mean          pale          \n",
      "bird          journey       voice         shape         lost          \n",
      "\n",
      "\n"
     ]
    }
   ],
   "source": [
    "vect = CountVectorizer(max_features=10000, max_df=.15, stop_words='english')\n",
    "    \n",
    "lda = LatentDirichletAllocation(n_components=10, learning_method=\"batch\",\n",
    "                                    max_iter=50, random_state=1)\n",
    "\n",
    "chapter = negative_chapters[0]\n",
    "KT = nltk.word_tokenize(chapter)\n",
    "KT = LDA_prepare(KT)\n",
    "join = [''.join(filter(str.isalpha, word)) for word in KT]\n",
    "vec = vect.fit_transform(join)\n",
    "document_topics = lda.fit_transform(vec)\n",
    "\n",
    "print(\"lda.components_.shape: {}\".format(lda.components_.shape))\n",
    "lda.components_.shape: (10, 10000)\n",
    "\n",
    "sorting = np.argsort(lda.components_, axis=1)[:, ::-1]\n",
    "\n",
    "feature_names = np.array(vect.get_feature_names())\n",
    "\n",
    "mglearn.tools.print_topics(topics=range(10), feature_names=feature_names, \n",
    "                           sorting=sorting, topics_per_chunk=5, n_words=10)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 44,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "lda.components_.shape: (10, 676)\n",
      "topic 0       topic 1       topic 2       topic 3       topic 4       \n",
      "--------      --------      --------      --------      --------      \n",
      "did           left          ores          wind          news          \n",
      "follow        minas         long          use           white         \n",
      "towers        rauros        boat          time          hobbits       \n",
      "parth         strange       taken         water         hand          \n",
      "golden        let           river         high          grey          \n",
      "red           beneath       way           end           ask           \n",
      "choice        set           turned        far           longer        \n",
      "save          heard         fallen        read          like          \n",
      "dead          clear         slain         anduin        sea           \n",
      "answered      bent          roaring       earth         ground        \n",
      "\n",
      "\n",
      "topic 5       topic 6       topic 7       topic 8       topic 9       \n",
      "--------      --------      --------      --------      --------      \n",
      "away          dark          horn          saw           hope          \n",
      "north         mordor        think         gone          suddenly      \n",
      "tower         alas          laid          tirith        easy          \n",
      "make          know          galen         hill          bore          \n",
      "glade         lands         right         falls         saruman       \n",
      "company       sprang        south         departure     swords        \n",
      "say           late          arrows        cloven        shore         \n",
      "ing           wish          moment        ran           enemies       \n",
      "foot          swift         passed        sword         stood         \n",
      "path          eyes          dwarves       lay           seen          \n",
      "\n",
      "\n"
     ]
    }
   ],
   "source": [
    "vect = CountVectorizer(max_features=10000, max_df=.15, stop_words='english')\n",
    "    \n",
    "lda = LatentDirichletAllocation(n_components=10, learning_method=\"batch\",\n",
    "                                    max_iter=50, random_state=1)\n",
    "\n",
    "chapter = negative_chapters[1]\n",
    "KT = nltk.word_tokenize(chapter)\n",
    "KT = LDA_prepare(KT)\n",
    "join = [''.join(filter(str.isalpha, word)) for word in KT]\n",
    "vec = vect.fit_transform(join)\n",
    "document_topics = lda.fit_transform(vec)\n",
    "\n",
    "print(\"lda.components_.shape: {}\".format(lda.components_.shape))\n",
    "lda.components_.shape: (10, 10000)\n",
    "\n",
    "sorting = np.argsort(lda.components_, axis=1)[:, ::-1]\n",
    "\n",
    "feature_names = np.array(vect.get_feature_names())\n",
    "\n",
    "mglearn.tools.print_topics(topics=range(10), feature_names=feature_names, \n",
    "                           sorting=sorting, topics_per_chunk=5, n_words=10)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 45,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "lda.components_.shape: (10, 859)\n",
      "topic 0       topic 1       topic 2       topic 3       topic 4       \n",
      "--------      --------      --------      --------      --------      \n",
      "road          wild          ride          king          man           \n",
      "thjoden       ghvn          horses        old           elfhelm       \n",
      "dark          gondor        time          long          darkness      \n",
      "suddenly      dernhelm      day           hills         morning       \n",
      "spoke         taken         set           company       stone         \n",
      "felt          hand          light         tall          walls         \n",
      "night         dread         hope          seen          black         \n",
      "wind          doubt         voice         wall          hear          \n",
      "late          answered      began         nearer        hour          \n",
      "years         ago           held          left          leading       \n",
      "\n",
      "\n",
      "topic 5       topic 6       topic 7       topic 8       topic 9       \n",
      "--------      --------      --------      --------      --------      \n",
      "like          lead          heard         far           away          \n",
      "lord          eastward      soon          enemy         host          \n",
      "city          riding        grey          don           trees         \n",
      "riders        know          lay           jored         battle        \n",
      "turned        head          rode          woods         rohan         \n",
      "things        tidings       walk          gorgyn        field         \n",
      "talk          south         busy          air           look          \n",
      "wndfara       foes          white         east          end           \n",
      "halted        just          valley        words         mark          \n",
      "forgotten     way           mountain      fear          rider         \n",
      "\n",
      "\n"
     ]
    }
   ],
   "source": [
    "vect = CountVectorizer(max_features=10000, max_df=.15, stop_words='english')\n",
    "    \n",
    "lda = LatentDirichletAllocation(n_components=10, learning_method=\"batch\",\n",
    "                                    max_iter=50, random_state=1)\n",
    "\n",
    "chapter = negative_chapters[2]\n",
    "KT = nltk.word_tokenize(chapter)\n",
    "KT = LDA_prepare(KT)\n",
    "join = [''.join(filter(str.isalpha, word)) for word in KT]\n",
    "vec = vect.fit_transform(join)\n",
    "document_topics = lda.fit_transform(vec)\n",
    "\n",
    "print(\"lda.components_.shape: {}\".format(lda.components_.shape))\n",
    "lda.components_.shape: (10, 10000)\n",
    "\n",
    "sorting = np.argsort(lda.components_, axis=1)[:, ::-1]\n",
    "\n",
    "feature_names = np.array(vect.get_feature_names())\n",
    "\n",
    "mglearn.tools.print_topics(topics=range(10), feature_names=feature_names, \n",
    "                           sorting=sorting, topics_per_chunk=5, n_words=10)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 46,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "lda.components_.shape: (10, 1005)\n",
      "topic 0       topic 1       topic 2       topic 3       topic 4       \n",
      "--------      --------      --------      --------      --------      \n",
      "king          eyes          stood         like          battle        \n",
      "fell          lord          field         away          foes          \n",
      "left          gondor        turned        lay           face          \n",
      "beast         thjoden       far           rode          clear         \n",
      "snowmane      new           sea           knights       borne         \n",
      "hope          dark          shadow        long          walls         \n",
      "knew          ships         heard         river         standard      \n",
      "hosts         world         hour          red           wrath         \n",
      "corsairs      cold          thy           wept          filled        \n",
      "foe           saw           drove         day           lifted        \n",
      "\n",
      "\n",
      "topic 5       topic 6       topic 7       topic 8       topic 9       \n",
      "--------      --------      --------      --------      --------      \n",
      "wind          onset         black         city          rohirrim      \n",
      "hand          spoke         sword         cried         white         \n",
      "fallen        fields        green         death         man           \n",
      "voice         look          southrons     tears         dernhelm      \n",
      "war           soon          mind          took          midst         \n",
      "fair          moment        prince        banner        heart         \n",
      "ruin          haradrim      shield        fall          air           \n",
      "enemy         let           harlond       despair       bear          \n",
      "crown         rose          hills         passed        raised        \n",
      "slowly        body          ground        fury          living        \n",
      "\n",
      "\n"
     ]
    }
   ],
   "source": [
    "vect = CountVectorizer(max_features=10000, max_df=.15, stop_words='english')\n",
    "    \n",
    "lda = LatentDirichletAllocation(n_components=10, learning_method=\"batch\",\n",
    "                                    max_iter=50, random_state=1)\n",
    "\n",
    "chapter = negative_chapters[3]\n",
    "KT = nltk.word_tokenize(chapter)\n",
    "KT = LDA_prepare(KT)\n",
    "join = [''.join(filter(str.isalpha, word)) for word in KT]\n",
    "vec = vect.fit_transform(join)\n",
    "document_topics = lda.fit_transform(vec)\n",
    "\n",
    "print(\"lda.components_.shape: {}\".format(lda.components_.shape))\n",
    "lda.components_.shape: (10, 10000)\n",
    "\n",
    "sorting = np.argsort(lda.components_, axis=1)[:, ::-1]\n",
    "\n",
    "feature_names = np.array(vect.get_feature_names())\n",
    "\n",
    "mglearn.tools.print_topics(topics=range(10), feature_names=feature_names, \n",
    "                           sorting=sorting, topics_per_chunk=5, n_words=10)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 47,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "lda.components_.shape: (10, 699)\n",
      "topic 0       topic 1       topic 2       topic 3       topic 4       \n",
      "--------      --------      --------      --------      --------      \n",
      "set           light         away          son           servants      \n",
      "heart         guard         house         healing       eyes          \n",
      "turned        cried         enemy         thy           despair       \n",
      "power         black         way           death         grey          \n",
      "passed        brought       battle        hands         far           \n",
      "table         dark          place         madness       tower         \n",
      "bier          swiftly       fear          spoke         lies          \n",
      "saw           shadowfax     hastened      mind          pass          \n",
      "closed        drew          know          face          taken         \n",
      "fields        dead          left          dnnen         broke         \n",
      "\n",
      "\n",
      "topic 5       topic 6       topic 7       topic 8       topic 9       \n",
      "--------      --------      --------      --------      --------      \n",
      "stood         door          city          hope          lord          \n",
      "stone         heard         lay           hand          houses        \n",
      "forward       flame         wind          suddenly      long          \n",
      "steward       evil          thou          sprang        sword         \n",
      "held          white         thought       tirith        field         \n",
      "man           gondor        captain       minas         like          \n",
      "thee          seen          hearts        given         fell          \n",
      "did           strength      friend        terrible      sent          \n",
      "citadel       lifted        return        moment        west          \n",
      "soon          slain         things        command       think         \n",
      "\n",
      "\n"
     ]
    }
   ],
   "source": [
    "vect = CountVectorizer(max_features=10000, max_df=.15, stop_words='english')\n",
    "    \n",
    "lda = LatentDirichletAllocation(n_components=10, learning_method=\"batch\",\n",
    "                                    max_iter=50, random_state=1)\n",
    "\n",
    "chapter = negative_chapters[4]\n",
    "KT = nltk.word_tokenize(chapter)\n",
    "KT = LDA_prepare(KT)\n",
    "join = [''.join(filter(str.isalpha, word)) for word in KT]\n",
    "vec = vect.fit_transform(join)\n",
    "document_topics = lda.fit_transform(vec)\n",
    "\n",
    "print(\"lda.components_.shape: {}\".format(lda.components_.shape))\n",
    "lda.components_.shape: (10, 10000)\n",
    "\n",
    "sorting = np.argsort(lda.components_, axis=1)[:, ::-1]\n",
    "\n",
    "feature_names = np.array(vect.get_feature_names())\n",
    "\n",
    "mglearn.tools.print_topics(topics=range(10), feature_names=feature_names, \n",
    "                           sorting=sorting, topics_per_chunk=5, n_words=10)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 48,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "lda.components_.shape: (10, 1039)\n",
      "topic 0       topic 1       topic 2       topic 3       topic 4       \n",
      "--------      --------      --------      --------      --------      \n",
      "black         mordor        hills         away          enemy         \n",
      "day           far           long          west          ores          \n",
      "army          cried         tower         heard         destroyed     \n",
      "fear          pass          messenger     did           maybe         \n",
      "forth         north         set           soon          assault       \n",
      "left          wish          laughed       red           way           \n",
      "tall          banner        minas         east          scouts        \n",
      "south         grey          land          began         blew          \n",
      "mind          saw           march         anduin        watched       \n",
      "nazgyl        silence       remained      enemies       time          \n",
      "\n",
      "\n",
      "topic 5       topic 6       topic 7       topic 8       topic 9       \n",
      "--------      --------      --------      --------      --------      \n",
      "stood         evil          sauron        gondor        like          \n",
      "captains      end           hope          company       little        \n",
      "imrahil       thought       lay           ring          old           \n",
      "king          road          dark          mouth         eyes          \n",
      "head          host          heralds       tirith        morannon      \n",
      "hideous       thou          terms         gone          sword         \n",
      "wind          broken        guard         fair          white         \n",
      "leave         lord          rohan         sun           master        \n",
      "air           trumpets      war           let           rode          \n",
      "eastward      forward       blade         city          morgul        \n",
      "\n",
      "\n"
     ]
    }
   ],
   "source": [
    "vect = CountVectorizer(max_features=10000, max_df=.15, stop_words='english')\n",
    "    \n",
    "lda = LatentDirichletAllocation(n_components=10, learning_method=\"batch\",\n",
    "                                    max_iter=50, random_state=1)\n",
    "\n",
    "chapter = negative_chapters[5]\n",
    "KT = nltk.word_tokenize(chapter)\n",
    "KT = LDA_prepare(KT)\n",
    "join = [''.join(filter(str.isalpha, word)) for word in KT]\n",
    "vec = vect.fit_transform(join)\n",
    "document_topics = lda.fit_transform(vec)\n",
    "\n",
    "print(\"lda.components_.shape: {}\".format(lda.components_.shape))\n",
    "lda.components_.shape: (10, 10000)\n",
    "\n",
    "sorting = np.argsort(lda.components_, axis=1)[:, ::-1]\n",
    "\n",
    "feature_names = np.array(vect.get_feature_names())\n",
    "\n",
    "mglearn.tools.print_topics(topics=range(10), feature_names=feature_names, \n",
    "                           sorting=sorting, topics_per_chunk=5, n_words=10)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 49,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "lda.components_.shape: (10, 1363)\n",
      "topic 0       topic 1       topic 2       topic 3       topic 4       \n",
      "--------      --------      --------      --------      --------      \n",
      "away          mountain      dark          eyes          thing         \n",
      "day           saw           like          strength      dreadful      \n",
      "hand          turned        fell          heart         feet          \n",
      "long          road          fear          end           burden        \n",
      "stood         night         thought       path          plain         \n",
      "doom          voice         took          little        lying         \n",
      "gave          knew          miles         lay           worn          \n",
      "high          wild          clouds        rose          mordor        \n",
      "moment        set           north         west          coming        \n",
      "grew          power         effort        gone          tearing       \n",
      "\n",
      "\n",
      "topic 5       topic 6       topic 7       topic 8       topic 9       \n",
      "--------      --------      --------      --------      --------      \n",
      "passed        master        land          far           way           \n",
      "shadow        mind          drew          did           light         \n",
      "heard         deep          hands         ring          black         \n",
      "die           water         precious      going         things        \n",
      "forward       grey          suddenly      carry         felt          \n",
      "shook         just          longer        cried         ground        \n",
      "realm         breast        vast          know          slowly        \n",
      "slope         muttered      bit           knees         head          \n",
      "thinking      ca            air           words         left          \n",
      "lifted        ago           new           rising        hope          \n",
      "\n",
      "\n"
     ]
    }
   ],
   "source": [
    "vect = CountVectorizer(max_features=10000, max_df=.15, stop_words='english')\n",
    "    \n",
    "lda = LatentDirichletAllocation(n_components=10, learning_method=\"batch\",\n",
    "                                    max_iter=50, random_state=1)\n",
    "\n",
    "chapter = negative_chapters[6]\n",
    "KT = nltk.word_tokenize(chapter)\n",
    "KT = LDA_prepare(KT)\n",
    "join = [''.join(filter(str.isalpha, word)) for word in KT]\n",
    "vec = vect.fit_transform(join)\n",
    "document_topics = lda.fit_transform(vec)\n",
    "\n",
    "print(\"lda.components_.shape: {}\".format(lda.components_.shape))\n",
    "lda.components_.shape: (10, 10000)\n",
    "\n",
    "sorting = np.argsort(lda.components_, axis=1)[:, ::-1]\n",
    "\n",
    "feature_names = np.array(vect.get_feature_names())\n",
    "\n",
    "mglearn.tools.print_topics(topics=range(10), feature_names=feature_names, \n",
    "                           sorting=sorting, topics_per_chunk=5, n_words=10)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Word Search"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 41,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "## Calculate all frequencies\n",
    "\n",
    "silm_freq = {}\n",
    "hobbit_freq = {}\n",
    "fellowship_freq = {}\n",
    "twotowers_freq = {}\n",
    "return_freq = {}\n",
    "\n",
    "freq_dicts = [silm_freq, hobbit_freq, fellowship_freq, twotowers_freq, return_freq]\n",
    "\n",
    "for i in range(len(tokenlist)):\n",
    "    for word in tokenlist[i]:\n",
    "        word = word.lower()\n",
    "        if word not in punctuation and not word.isnumeric() and word.isalpha():\n",
    "            if word in freq_dicts[i]:\n",
    "                freq_dicts[i][word] += 1\n",
    "            else:\n",
    "                freq_dicts[i][word] = 1\n",
    "\n",
    "entirety_dict = {}\n",
    "\n",
    "for word in entirety:\n",
    "    word = word.lower()\n",
    "    if word not in punctuation and not word.isnumeric() and word.isalpha():\n",
    "        if word in entirety_dict:\n",
    "            entirety_dict[word] += 1\n",
    "        else:\n",
    "            entirety_dict[word] = 1\n",
    "            "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 42,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "------------------------------------------------------\n",
      "The Silmarillion: 130 Out of 148,914 Words\n",
      "------------------------------------------------------\n",
      "The Hobbit: 2 Out of 96,180 Words\n",
      "------------------------------------------------------\n",
      "The Fellowship of the Ring: 54 Out of 182,858 Words\n",
      "------------------------------------------------------\n",
      "The Two Towers: 220 Out of 155,947 Words\n",
      "------------------------------------------------------\n",
      "The Return of the King: 61 Out of 132,059 Words\n",
      "------------------------------------------------------\n",
      "Entire Corpus: 467\n",
      "------------------------------------------------------\n"
     ]
    }
   ],
   "source": [
    "\"\"\"\n",
    "Function that returns the frequency of a given word in each book:\n",
    "\n",
    "Parameters - word - input word\n",
    "\n",
    "Returns the frequency of the input word in each book.\n",
    "\"\"\"\n",
    "\n",
    "#NOTE:: For whatever reason, the words \"orc\" and \"orcs\" were converted to \"ore\" and \"ores\"\n",
    "## I suppose this is the drawback of pulling 5 books off the internet\n",
    "\n",
    "\n",
    "def WordSearch(word):\n",
    "    word = word.lower() \n",
    "    for i in range(len(tokenlist)):\n",
    "        if i == 0:\n",
    "            if word in freq_dicts[i]:\n",
    "                print(\"------------------------------------------------------\")\n",
    "                print(\"The Silmarillion:\", freq_dicts[i][word], \"Out of 148,914 Words\")\n",
    "                print(\"------------------------------------------------------\")\n",
    "            else:\n",
    "                print(\"------------------------------------------------------\")\n",
    "                print(\"The Silmarillion: 0 Out of 148,914 Words\")   \n",
    "                print(\"------------------------------------------------------\")\n",
    "        if i == 1:\n",
    "            if word in freq_dicts[i]:\n",
    "                print(\"The Hobbit:\", freq_dicts[i][word], \"Out of 96,180 Words\")\n",
    "                print(\"------------------------------------------------------\")\n",
    "            else:\n",
    "                print(\"The Hobbit: 0 Out of 96,180 Words\")\n",
    "                print(\"------------------------------------------------------\")\n",
    "        if i == 2:\n",
    "            if word in freq_dicts[i]:\n",
    "                print(\"The Fellowship of the Ring:\", freq_dicts[i][word], \"Out of 182,858 Words\")\n",
    "                print(\"------------------------------------------------------\")\n",
    "            else:\n",
    "                print(\"The Fellowship of the Ring: 0 Out of 182,858 Words\")\n",
    "                print(\"------------------------------------------------------\")\n",
    "        if i == 3:\n",
    "            if word in freq_dicts[i]:\n",
    "                print(\"The Two Towers:\", freq_dicts[i][word], \"Out of 155,947 Words\")\n",
    "                print(\"------------------------------------------------------\")\n",
    "            else:\n",
    "                print(\"The Two Towers: 0 Out of 155,947 Words\")\n",
    "                print(\"------------------------------------------------------\")\n",
    "        if i == 4:\n",
    "            if word in freq_dicts[i]:\n",
    "                print(\"The Return of the King:\", freq_dicts[i][word], \"Out of 132,059 Words\")\n",
    "                print(\"------------------------------------------------------\")\n",
    "            else:\n",
    "                print(\"The Return of the King: 0 Out of 132,059 Words\")\n",
    "                print(\"------------------------------------------------------\")\n",
    "    if word in entirety_dict:\n",
    "        print(\"Entire Corpus:\", entirety_dict[word])\n",
    "        print(\"------------------------------------------------------\")\n",
    "                \n",
    "WordSearch(\"ores\")\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Word Clouds"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 43,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "## Lists of words to go to word clouds\n",
    "silm_cloud = []\n",
    "hobbit_cloud = []\n",
    "fellowship_cloud = []\n",
    "twotowers_cloud = []\n",
    "return_cloud = []\n",
    "total_list = []\n",
    "\n",
    "cloudlist = [silm_cloud, hobbit_cloud, fellowship_cloud, twotowers_cloud, return_cloud]\n",
    "\n",
    "names = [\"gandalf\", \"merry\", \"pippin\", \"frodo\", \"sam\", \"aragorn\", \"faramir\", \"denethor\", \"gimli\",\\\n",
    "        \"legolas\", \"strider\", \"boromir\", \"jowyn\", \"jomer\", \"beregond\", \"gollum\", \"bilbo\", \"thorin\"] \n",
    "\n",
    "for i in range(len(tokenlist)):\n",
    "    for word in tokenlist[i]:\n",
    "        word=word.lower()\n",
    "        if word not in punctuation and word not in stop_words and word.isalpha() and word not in tolkien_stop and word not in names:\n",
    "            if freq_dicts[i][word] > 5: ## Only words that occur 5 times or more\n",
    "                cloudlist[i].append(word)\n",
    "                total_list.append(word)\n",
    "## Dataframes to send to Tableau\n",
    "\n",
    "silm_cloud_df = pd.DataFrame(cloudlist[0], columns = [\"The Silmarillion\"])\n",
    "hobbit_cloud_df = pd.DataFrame(cloudlist[1], columns = [\"The Hobbit\"])\n",
    "fellowship_cloud_df = pd.DataFrame(cloudlist[2], columns = [\"The Fellowship of the Ring\"])\n",
    "twotowers_cloud_df = pd.DataFrame(cloudlist[3], columns = [\"The Two Towers\"])\n",
    "return_cloud_df = pd.DataFrame(cloudlist[4], columns = [\"The Return of the King\"])\n",
    "combined = pd.DataFrame(total_list)\n",
    "\n",
    "\n",
    "cloud = pd.ExcelWriter('cloud.xlsx')\n",
    "\n",
    "silm_cloud_df.to_excel(cloud, 'The Silmarillion')\n",
    "hobbit_cloud_df.to_excel(cloud, 'The Hobbit')\n",
    "fellowship_cloud_df.to_excel(cloud, 'Fellowship of the Rings')\n",
    "twotowers_cloud_df.to_excel(cloud, 'The Two Towers')\n",
    "return_cloud_df.to_excel(cloud, 'The Return of the King')\n",
    "combined.to_excel(cloud, 'Combined')\n",
    "\n",
    "\n",
    "\n",
    "cloud.save()\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 44,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# Find common words within Negative Chapters\n",
    "negative_tokens = []\n",
    "negative_words = []\n",
    "\n",
    "for chapter in negative_chapters:\n",
    "    negative_tokens.append(nltk.word_tokenize(chapter))\n",
    "  \n",
    "for i in range(len(negative_tokens)):\n",
    "    for word in negative_tokens[i]:\n",
    "        word = word.lower()\n",
    "        if word.isalpha() and word not in stop_words and word not in tolkien_stop and word not in names:\n",
    "            negative_words.append(word)\n",
    "\n",
    "            \n",
    "neg_df = pd.DataFrame(negative_words, columns = [\"Negative Chapter Words\"])                 \n",
    "neg = pd.ExcelWriter('negative.xlsx')\n",
    "neg_df.to_excel(neg, 'Sheet 1')\n",
    "neg.save()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 45,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "### Positive Word Cloud Exports\n",
    "\n",
    "\n",
    "\n",
    "positive_chapters = [silm_chapters[15], fellowship_chapters[15], fellowship_chapters[10], fellowship_chapters[23], \\\n",
    "                    return_chapters[14], return_chapters[15], fellowship_chapters[1], silm_chapters[1], silm_chapters[2],\\\n",
    "                    silm_chapters[3], silm_chapters[7]]\n",
    "\n",
    "positive_tokens = []\n",
    "positive_words = []\n",
    "\n",
    "for chapter in positive_chapters:\n",
    "    positive_tokens.append(nltk.word_tokenize(chapter))\n",
    "  \n",
    "for i in range(len(positive_tokens)):\n",
    "    for word in positive_tokens[i]:\n",
    "        word = word.lower()\n",
    "        if word.isalpha() and word not in stop_words and word not in tolkien_stop and word not in names:\n",
    "            positive_words.append(word)\n",
    "\n",
    "            \n",
    "pos_df = pd.DataFrame(positive_words, columns = [\"Positive Chapter Words\"])                 \n",
    "pos = pd.ExcelWriter('positive.xlsx')\n",
    "pos_df.to_excel(pos, 'Sheet 1')\n",
    "pos.save()\n"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.3"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}