Model1.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import findspark\n",
    "findspark.init('/home/sdmohant/spark-3.0.0-bin-hadoop2.7')\n",
    "\n",
    "import pyspark\n",
    "import random"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "from pyspark import SparkContext\n",
    "from pyspark.sql import SparkSession\n",
    "from pyspark.sql import SQLContext\n",
    "from pyspark.sql.types import *"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/sdmohant/spark-3.0.0-bin-hadoop2.7/python/pyspark/context.py:220: DeprecationWarning: Support for Python 2 and Python 3 prior to version 3.6 is deprecated as of Spark 3.0. See also the plan for dropping Python 2 support at https://spark.apache.org/news/plan-for-dropping-python-2-support.html.\n",
      "  DeprecationWarning)\n"
     ]
    }
   ],
   "source": [
    "spark = SparkSession.builder.appName(\"Basics\").config('spark.sql.shuffle.partitions',300).getOrCreate() # Increasing SQL shuffle partitions"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "sc = spark.sparkContext"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "from pyspark.sql.functions import isnan, when, count, col"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "#Reading Paper Data\n",
    "paper_df = spark.read.options(header = True,delimiter=',', inferSchema= True).csv(\"Paper.csv\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "root\n",
      " |-- PaperID: long (nullable = true)\n",
      " |-- Rank: integer (nullable = true)\n",
      " |-- DOI: string (nullable = true)\n",
      " |-- DocType: string (nullable = true)\n",
      " |-- PaperTitle: string (nullable = true)\n",
      " |-- OriginalTitle: string (nullable = true)\n",
      " |-- PaperYear: integer (nullable = true)\n",
      " |-- PaperDate: string (nullable = true)\n",
      " |-- Publisher: string (nullable = true)\n",
      " |-- JournalID: long (nullable = true)\n",
      " |-- ConfSeriesID: long (nullable = true)\n",
      " |-- ConfInstanceID: long (nullable = true)\n",
      " |-- ReferenceCount: integer (nullable = true)\n",
      " |-- CitationCount: integer (nullable = true)\n",
      " |-- EstimatedCount: integer (nullable = true)\n",
      "\n"
     ]
    }
   ],
   "source": [
    "paper_df.printSchema()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "+----------+-----+--------------------+----------+--------------------+--------------------+---------+----------+---------+---------+------------+--------------+--------------+-------------+--------------+\n",
      "|   PaperID| Rank|                 DOI|   DocType|          PaperTitle|       OriginalTitle|PaperYear| PaperDate|Publisher|JournalID|ConfSeriesID|ConfInstanceID|ReferenceCount|CitationCount|EstimatedCount|\n",
      "+----------+-----+--------------------+----------+--------------------+--------------------+---------+----------+---------+---------+------------+--------------+--------------+-------------+--------------+\n",
      "|1819143627|24199|10.1109/PICMET.20...|Conference|technological inn...|Technological inn...|     2001|2001-01-01|     IEEE| 23074404|  2734834919|    2735348883|             0|            0|             0|\n",
      "|1963578506|22929|10.1109/ICIEEM.20...|Conference|a hybrid due date...|A hybrid due-date...|     2010|2010-10-01|     IEEE|137519996|  1141689323|     180417304|            16|            0|             0|\n",
      "+----------+-----+--------------------+----------+--------------------+--------------------+---------+----------+---------+---------+------------+--------------+--------------+-------------+--------------+\n",
      "only showing top 2 rows\n",
      "\n"
     ]
    }
   ],
   "source": [
    "paper_df.show(2)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "paper_df=paper_df.withColumnRenamed(\"Rank\",\"Paper_Rank\")\n",
    "paper_df=paper_df.withColumnRenamed(\"ReferenceCount\",\"Paper_RefCnt\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[Row(PaperID=1819143627, Paper_Rank=24199, DOI=u'10.1109/PICMET.2001.951754', DocType=u'Conference', PaperTitle=u'technological innovation system and strategy in china s western regions', OriginalTitle=u\"Technological innovation system and strategy in China's Western Regions\", PaperYear=2001, PaperDate=u'2001-01-01', Publisher=u'IEEE', JournalID=23074404, ConfSeriesID=2734834919, ConfInstanceID=2735348883, Paper_RefCnt=0, CitationCount=0, EstimatedCount=0)]"
      ]
     },
     "execution_count": 10,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "paper_df.head(1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [],
   "source": [
    "#Read Journal Data\n",
    "journal_df = spark.read.options(header = 'True',delimiter=',', inferSchema = True).csv(\"Journal.csv\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "+---------+-----+--------------------+--------------------+-----------+--------------+\n",
      "|JournalID| Rank|     Normalized_Name|        Display_Name|Paper_count|Citation_Count|\n",
      "+---------+-----+--------------------+--------------------+-----------+--------------+\n",
      "| 98941412|12196|handbooks of mana...|Handbooks of Mana...|         97|          1955|\n",
      "|108758393| 9716|progress in lipid...|Progress in Lipid...|       1048|         77361|\n",
      "+---------+-----+--------------------+--------------------+-----------+--------------+\n",
      "only showing top 2 rows\n",
      "\n"
     ]
    }
   ],
   "source": [
    "journal_df.show(2)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [],
   "source": [
    "journal_df=journal_df.withColumnRenamed(\"Rank\",\"Journal_Rank\")\n",
    "journal_df=journal_df.withColumnRenamed(\"Paper_count\",\"Journal_Pap_Cnt\")\n",
    "journal_df=journal_df.withColumnRenamed(\"Citation_Count\",\"Journal_Cit_Cnt\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "47919"
      ]
     },
     "execution_count": 14,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "journal_df.count()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [],
   "source": [
    "#Joining Paper and Journal Data\n",
    "df = paper_df.join(journal_df, on=['JournalID'], how='inner')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "+---------+----------+----------+--------------------+----------+--------------------+--------------------+---------+----------+---------+------------+--------------+------------+-------------+--------------+------------+--------------------+--------------------+---------------+---------------+\n",
      "|JournalID|   PaperID|Paper_Rank|                 DOI|   DocType|          PaperTitle|       OriginalTitle|PaperYear| PaperDate|Publisher|ConfSeriesID|ConfInstanceID|Paper_RefCnt|CitationCount|EstimatedCount|Journal_Rank|     Normalized_Name|        Display_Name|Journal_Pap_Cnt|Journal_Cit_Cnt|\n",
      "+---------+----------+----------+--------------------+----------+--------------------+--------------------+---------+----------+---------+------------+--------------+------------+-------------+--------------+------------+--------------------+--------------------+---------------+---------------+\n",
      "| 23074404|1819143627|     24199|10.1109/PICMET.20...|Conference|technological inn...|Technological inn...|     2001|2001-01-01|     IEEE|  2734834919|    2735348883|           0|            0|             0|        9728|physica e low dim...|Physica E-low-dim...|           9284|          64351|\n",
      "|137519996|1963578506|     22929|10.1109/ICIEEM.20...|Conference|a hybrid due date...|A hybrid due-date...|     2010|2010-10-01|     IEEE|  1141689323|     180417304|          16|            0|             0|        9694|  electronic markets|  Electronic Markets|           4073|          16612|\n",
      "+---------+----------+----------+--------------------+----------+--------------------+--------------------+---------+----------+---------+------------+--------------+------------+-------------+--------------+------------+--------------------+--------------------+---------------+---------------+\n",
      "only showing top 2 rows\n",
      "\n"
     ]
    }
   ],
   "source": [
    "df.show(2)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "120861"
      ]
     },
     "execution_count": 17,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df.count()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [],
   "source": [
    "#df.describe().show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [],
   "source": [
    "#Read Conference Series Data\n",
    "confSeries_df = spark.read.options(header = 'True',delimiter=',', inferSchema = True).csv(\"ConferenceSeries.csv\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "+------------+------+----------------+--------------------+------------+----------+\n",
      "|ConfSeriesID|CSRank|CSNormalizedName|       CSDisplayName|CSPaperCount|CSCitation|\n",
      "+------------+------+----------------+--------------------+------------+----------+\n",
      "|  1141689323| 11674|            IEEM|Industrial Engine...|        7218|      7851|\n",
      "|  2727030889| 13563| BigDataSecurity|International Con...|         120|       122|\n",
      "+------------+------+----------------+--------------------+------------+----------+\n",
      "only showing top 2 rows\n",
      "\n"
     ]
    }
   ],
   "source": [
    "confSeries_df.show(2)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [],
   "source": [
    "#Joining Paper, Journal with ConferenceSeries\n",
    "df1 = df.join(confSeries_df, on=['ConfSeriesID'], how='inner')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "120861"
      ]
     },
     "execution_count": 22,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df1.count()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "+------------+---------+----------+----------+--------------------+----------+--------------------+--------------------+---------+----------+---------+--------------+------------+-------------+--------------+------------+--------------------+--------------------+---------------+---------------+------+----------------+--------------------+------------+----------+\n",
      "|ConfSeriesID|JournalID|   PaperID|Paper_Rank|                 DOI|   DocType|          PaperTitle|       OriginalTitle|PaperYear| PaperDate|Publisher|ConfInstanceID|Paper_RefCnt|CitationCount|EstimatedCount|Journal_Rank|     Normalized_Name|        Display_Name|Journal_Pap_Cnt|Journal_Cit_Cnt|CSRank|CSNormalizedName|       CSDisplayName|CSPaperCount|CSCitation|\n",
      "+------------+---------+----------+----------+--------------------+----------+--------------------+--------------------+---------+----------+---------+--------------+------------+-------------+--------------+------------+--------------------+--------------------+---------------+---------------+------+----------------+--------------------+------------+----------+\n",
      "|  2734834919| 23074404|1819143627|     24199|10.1109/PICMET.20...|Conference|technological inn...|Technological inn...|     2001|2001-01-01|     IEEE|    2735348883|           0|            0|             0|        9728|physica e low dim...|Physica E-low-dim...|           9284|          64351| 11187|          PICMET|Portland Internat...|        4309|      6256|\n",
      "|  1141689323|137519996|1963578506|     22929|10.1109/ICIEEM.20...|Conference|a hybrid due date...|A hybrid due-date...|     2010|2010-10-01|     IEEE|     180417304|          16|            0|             0|        9694|  electronic markets|  Electronic Markets|           4073|          16612| 11674|            IEEM|Industrial Engine...|        7218|      7851|\n",
      "+------------+---------+----------+----------+--------------------+----------+--------------------+--------------------+---------+----------+---------+--------------+------------+-------------+--------------+------------+--------------------+--------------------+---------------+---------------+------+----------------+--------------------+------------+----------+\n",
      "only showing top 2 rows\n",
      "\n"
     ]
    }
   ],
   "source": [
    "df1.show(2)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['ConfSeriesID',\n",
       " 'JournalID',\n",
       " 'PaperID',\n",
       " 'Paper_Rank',\n",
       " 'DOI',\n",
       " 'DocType',\n",
       " 'PaperTitle',\n",
       " 'OriginalTitle',\n",
       " 'PaperYear',\n",
       " 'PaperDate',\n",
       " 'Publisher',\n",
       " 'ConfInstanceID',\n",
       " 'Paper_RefCnt',\n",
       " 'CitationCount',\n",
       " 'EstimatedCount',\n",
       " 'Journal_Rank',\n",
       " 'Normalized_Name',\n",
       " 'Display_Name',\n",
       " 'Journal_Pap_Cnt',\n",
       " 'Journal_Cit_Cnt',\n",
       " 'CSRank',\n",
       " 'CSNormalizedName',\n",
       " 'CSDisplayName',\n",
       " 'CSPaperCount',\n",
       " 'CSCitation']"
      ]
     },
     "execution_count": 24,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df1.columns"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {},
   "outputs": [],
   "source": [
    "#Read ConferenceInstance Data\n",
    "confInstances_df = spark.read.options(header = 'True',delimiter=',', inferSchema = True).csv(\"ConferenceInstances.csv\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {},
   "outputs": [],
   "source": [
    "#Join Paper, Journal, ConferenceSeries with ConferenceInstance\n",
    "df2 = df1.join(confInstances_df, on=['ConfInstanceID'], how='inner')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "+--------------+------------+----------+----------+----------+--------------------+----------+--------------------+--------------------+---------+----------+--------------------+------------+-------------+--------------+------------+--------------------+--------------------+---------------+---------------+------+----------------+--------------------+------------+----------+------+--------------+-------------+----------+--------------------+--------------------+----------+----------+---------------+---------------+-------------------+-------------------+------------+------------------+\n",
      "|ConfInstanceID|ConfSeriesID| JournalID|   PaperID|Paper_Rank|                 DOI|   DocType|          PaperTitle|       OriginalTitle|PaperYear| PaperDate|           Publisher|Paper_RefCnt|CitationCount|EstimatedCount|Journal_Rank|     Normalized_Name|        Display_Name|Journal_Pap_Cnt|Journal_Cit_Cnt|CSRank|CSNormalizedName|       CSDisplayName|CSPaperCount|CSCitation|CIRank|CINormalizedNm|  CIDisplayNm|    CICSID|            Location|       CIOfficialURL| StartDate|   EndDate|AbstractRegDate|SubDeadlineDate|NotificationDueDate|FinalVersionDueDate|CIPaperCount|CIPaperCitationCnt|\n",
      "+--------------+------------+----------+----------+----------+--------------------+----------+--------------------+--------------------+---------+----------+--------------------+------------+-------------+--------------+------------+--------------------+--------------------+---------------+---------------+------+----------------+--------------------+------------+----------+------+--------------+-------------+----------+--------------------+--------------------+----------+----------+---------------+---------------+-------------------+-------------------+------------+------------------+\n",
      "|    2735348883|  2734834919|  23074404|1819143627|     24199|10.1109/PICMET.20...|Conference|technological inn...|Technological inn...|     2001|2001-01-01|                IEEE|           0|            0|             0|        9728|physica e low dim...|Physica E-low-dim...|           9284|          64351| 11187|          PICMET|Portland Internat...|        4309|      6256| 25000|   picmet 2001|  PICMET 2001|2734834919|   Portland, OR, USA|                null|2001-07-29|      null|           null|           null|               null|               null|         405|               759|\n",
      "|     180417304|  1141689323| 137519996|1963578506|     22929|10.1109/ICIEEM.20...|Conference|a hybrid due date...|A hybrid due-date...|     2010|2010-10-01|                IEEE|          16|            0|             0|        9694|  electronic markets|  Electronic Markets|           4073|          16612| 11674|            IEEM|Industrial Engine...|        7218|      7851| 25000|     ieem 2010|    IEEM 2010|1141689323|               Macau|http://www.ieem.o...|2010-12-07|2010-12-10|     2010-09-01|     2010-06-01|         2010-08-01|               null|        1061|              1162|\n",
      "|     190938570|  1123077274|  65753830|1963617477|     20898|10.1007/s00500-01...|   Journal|a dynamic trust m...|A dynamic trust m...|     2014|2014-09-01|Springer Berlin H...|          37|            6|             6|       10088|      soft computing|      Soft Computing|           3816|          36469| 10252|            SOCO|      Soft Computing|       10917|     58129| 25000|     soco 2014|    SOCO 2014|1123077274|       Bilbao, Spain|http://soco14.deu...|2014-06-25|2014-06-27|           null|     2014-02-20|         2014-03-20|         2014-04-07|         995|              2589|\n",
      "|     185510187|  1171345118|  97070762|2095756693|     23181|10.1504/IJAHUC.20...|   Journal|distributed wirel...|Distributed wirel...|     2013|2013-02-01|Inderscience Publ...|          34|            0|             0|       10184|international jou...|International Jou...|            662|           3773| 10318|         UbiComp|Ubiquitous Computing|        5425|     91539| 25000|  ubicomp 2013| UbiComp 2013|1171345118|              Zurich|http://www.ubicom...|2013-09-08|2013-09-12|           null|     2013-03-22|         2013-06-16|         2013-06-23|         702|              6536|\n",
      "|    2734539129|  2735422400|  13478950|1963841745|     23277|10.1109/SIU.2013....|Conference|gpu based infrare...|GPU-based infrare...|     2013|2013-04-01|                IEEE|           3|            0|             0|       11312|studia informatic...|Studia Informatic...|           4740|           3710| 12687|             SIU|Signal Processing...|        5213|      3898| 25000|      siu 2013|     SIU 2013|2735422400|    Haspolat, Turkey|                null|2013-04-24|2013-04-26|           null|           null|               null|               null|         446|               395|\n",
      "|     194297192|  1137878512| 120348307|1964344681|     21077|10.1016/j.procs.2...|   Journal|genetic algorithm...|Genetic Algorithm...|     2013|2013-01-01|            Elsevier|          20|            5|             5|       10204|procedia computer...|Procedia Computer...|          12564|          32224| 10439|            ICCS|International Con...|        5506|     24198| 25000|     iccs 2013|    ICCS 2013|1137878512|       Mumbai, India|http://iccs2013.h...|2013-01-10|2013-01-12|     2012-09-04|     2012-09-07|         2012-10-04|         2012-10-15|         322|              1797|\n",
      "|    2624962461|  2624549483| 101949793|2781480885|     21051|10.1109/ICSENS.20...|Conference|environmentally f...|Environmentally f...|     2017|2017-10-01|                null|           4|            0|             0|        9156|             sensors|             Sensors|          22416|         180580|  9553|         SENSORS|        IEEE Sensors|        7654|     26479| 25000|  sensors 2017| SENSORS 2017|2624549483|Glasgow, United K...|http://ieee-senso...|2017-10-29|2017-11-01|           null|     2017-06-18|         2017-08-10|               null|         670|                48|\n",
      "|    2583557579|  1130177464| 196734849|2596752321|     21584|10.1038/s41598-01...|   Journal|porewater salinit...|Porewater salinit...|     2017|2017-01-12|Nature Publishing...|          49|            1|             1|        9175|  scientific reports|  Scientific Reports|          65856|         417105|  8303|            SODA|Symposium on Disc...|        3709|     98299| 25000|     soda 2017|    SODA 2017|1130177464|    Barcelona, Spain|http://siam.org/m...|2017-01-16|2017-01-19|     2016-07-06|           null|               null|               null|         195|               441|\n",
      "|      28432213|  1171689754|2765011064|2181387890|     21702|10.1109/ACII.2015...|Conference|the platformer ex...|The platformer ex...|     2015|2015-09-01|Institute of Elec...|          42|            4|             4|       11101|the institute of ...|The Institute of ...|            573|           2348| 11258|            ACII|Affective Computi...|         856|      8579| 25000|     acii 2015|    ACII 2015|1171689754|         Xian, China|http://www.acii20...|2015-09-21|2015-09-24|           null|     2015-04-24|         2015-06-26|         2014-07-24|         155|               369|\n",
      "|     632688938|  2757320638|  35771696|1965057804|     21986|10.1504/IJDMB.201...|   Journal|alibimotif integr...|AliBiMotif: Integ...|     2012|2012-07-01|Int J Data Min Bi...|          16|            1|             1|       12359|international jou...|International Jou...|            495|           1755| 13332|          BIOKDD|Data Mining in Bi...|         475|      1831| 25000|   biokdd 2012|  BIOKDD 2012|2757320638|      Beijing, China|http://bio.inform...|2012-08-12|2012-08-12|           null|     2012-05-15|         2012-06-06|         2012-06-15|          47|               154|\n",
      "|    2624962461|  2624549483| 101949793|2781486615|     21559|10.1109/ICSENS.20...|Conference|emission monitori...|Emission monitori...|     2017|2017-10-01|                null|           0|            0|             0|        9156|             sensors|             Sensors|          22416|         180580|  9553|         SENSORS|        IEEE Sensors|        7654|     26479| 25000|  sensors 2017| SENSORS 2017|2624549483|Glasgow, United K...|http://ieee-senso...|2017-10-29|2017-11-01|           null|     2017-06-18|         2017-08-10|               null|         670|                48|\n",
      "|     162221927|  1158167855|2597175965|2096733369|     16312|10.1109/CVPR.2015...|Conference|facenet a unified...|FaceNet: A unifie...|     2015|2015-06-01|                IEEE|          22|          709|          1066|        9072|arxiv computer vi...|arXiv: Computer V...|           9299|         121450|  7594|            CVPR|Computer Vision a...|       12789|    645856| 25000|     cvpr 2015|    CVPR 2015|1158167855|          Boston, MA|http://www.pamitc...|2015-06-07|2015-06-12|           null|     2014-11-14|               null|               null|         884|             30743|\n",
      "|     165500188|  1170695740|  76152103|1965693572|     19558|10.1109/TSMCB.201...|   Journal|tracking control ...|Tracking Control ...|     2012|2012-10-01|IEEE Trans Syst M...|          30|           34|            59|        6944|ieee transactions...|IEEE Transactions...|          10930|         546568|  8684|             SMC|Systems, Man and ...|       28495|    512260| 25000|      smc 2012|     SMC 2012|1170695740|        Seoul, Korea|http://www.smc201...|2012-10-14|2012-10-17|           null|     2012-05-13|         2012-06-15|         2012-07-31|        1003|             15868|\n",
      "|    2320869366|  1179645356|  43927114|2554458174|     23446|10.1007/s10527-01...|   Journal|an automated rang...|An Automated Rang...|     2016|2016-01-11|         Springer US|           0|            0|             0|       10469|biomedical engine...|Biomedical Engine...|           8273|           4678| 13967|          BioMED|Biomedical Engine...|        5457|      5853| 25000|   biomed 2016|  BioMed 2016|1179645356|  Innsbruck, Austria|http://www.iasted...|2016-02-15|2016-02-17|           null|     2015-12-10|               null|               null|         175|                63|\n",
      "|     186126769|  1203052477|  84614559|1966292977|     20021|10.1145/2038617.2...|   Journal|using network cal...|Using Network Cal...|     2011|2011-09-01|                 ACM|           7|           16|            16|       10903|   acm sigbed review|   ACM Sigbed Review|            425|           2723| 10862|           ECRTS|Euromicro Confere...|         869|     16272| 25000|    ecrts 2011|   ECRTS 2011|1203052477|     Porto, Portugal|http://www.cister...|2011-07-06|2011-07-08|           null|     2011-01-23|         2011-03-25|               null|          40|               694|\n",
      "|     125232756|  1124077590| 106296714|1482825550|     18456|10.1007/978-3-642...|      null|hough transform a...|Hough transform a...|     2010|2010-09-05|     Springer-Verlag|          33|          151|           262|        7265|lecture notes in ...|Lecture Notes in ...|         118940|        1509576|  8748|            ECCV|European Conferen...|        3788|    186835| 25000|     eccv 2010|    ECCV 2010|1124077590|Hersonissos, Cret...|http://www.ics.fo...|2010-09-05|2010-09-11|           null|     2010-03-17|         2010-06-10|         2010-07-01|         390|             24816|\n",
      "|     574904361|  1182309694|  11479521|1966765298|     19991|10.1016/j.dss.201...|   Journal|optimal planning ...|Optimal planning ...|     2013|2013-04-01|       North-Holland|          18|           19|            19|        9095|decision support ...|Decision Support ...|           3224|         112193| 11293|             DSS|Decision Support ...|        4006|    115833| 25000|      dss 2013|     DSS 2013|1182309694|   Algarve, Portugal|http://www.neurot...|2013-09-18|2013-09-20|           null|     2013-06-30|               null|               null|         244|              4997|\n",
      "|    2329275568|  1177288818|  65043351|2460423758|     21920|10.1504/IJCSE.201...|   Journal|a new bag of word...|A new bag-of-word...|     2016|2016-01-01|Inderscience Publ...|          11|            1|             1|       11872|international jou...|International Jou...|            542|           1568| 10453|             CSE|Computational Sci...|        3216|     20038| 25000| ieee cse 2016|IEEE CSE 2016|1177288818|       Paris, France|http://cse2016.co...|2016-08-24|2016-08-26|           null|     2016-04-29|         2016-06-23|         2016-07-15|         246|                92|\n",
      "|    2331209432|  1151146444| 176643777|2460447926|     21347|10.1145/2942358.2...|Conference|towards distribut...|Towards distribut...|     2016|2016-07-05|                 ACM|          39|            1|             1|        9313|              scopus|              Scopus|           5423|          31399|  9219|         MobiHoc|Mobile Ad Hoc Net...|         879|     54649| 25000|  mobihoc 2016| MobiHoc 2016|1151146444|  Paderborn, Germany|http://www.sigmob...|2016-07-04|2016-07-08|           null|     2016-01-01|         2016-04-01|               null|          66|               187|\n",
      "|     618576426|  1138732554| 148324379|2097747636|     20950|10.1145/2048066.2...|Conference|jet exception che...|JET: exception ch...|     2011|2011-10-22|                 ACM|          21|            7|             7|        7860|     sigplan notices|     Sigplan Notices|           9590|         326857| 10223|          OOPSLA|Conference on Obj...|        2985|     74872| 25000|   oopsla 2011|  OOPSLA 2011|1138732554|Oregon, USA - Uni...|http://www.oopsla...|2011-10-22|2011-10-22|           null|     2011-04-08|               null|               null|         145|              1621|\n",
      "+--------------+------------+----------+----------+----------+--------------------+----------+--------------------+--------------------+---------+----------+--------------------+------------+-------------+--------------+------------+--------------------+--------------------+---------------+---------------+------+----------------+--------------------+------------+----------+------+--------------+-------------+----------+--------------------+--------------------+----------+----------+---------------+---------------+-------------------+-------------------+------------+------------------+\n",
      "only showing top 20 rows\n",
      "\n"
     ]
    }
   ],
   "source": [
    "df2.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['ConfInstanceID',\n",
       " 'ConfSeriesID',\n",
       " 'JournalID',\n",
       " 'PaperID',\n",
       " 'Paper_Rank',\n",
       " 'DOI',\n",
       " 'DocType',\n",
       " 'PaperTitle',\n",
       " 'OriginalTitle',\n",
       " 'PaperYear',\n",
       " 'PaperDate',\n",
       " 'Publisher',\n",
       " 'Paper_RefCnt',\n",
       " 'CitationCount',\n",
       " 'EstimatedCount',\n",
       " 'Journal_Rank',\n",
       " 'Normalized_Name',\n",
       " 'Display_Name',\n",
       " 'Journal_Pap_Cnt',\n",
       " 'Journal_Cit_Cnt',\n",
       " 'CSRank',\n",
       " 'CSNormalizedName',\n",
       " 'CSDisplayName',\n",
       " 'CSPaperCount',\n",
       " 'CSCitation',\n",
       " 'CIRank',\n",
       " 'CINormalizedNm',\n",
       " 'CIDisplayNm',\n",
       " 'CICSID',\n",
       " 'Location',\n",
       " 'CIOfficialURL',\n",
       " 'StartDate',\n",
       " 'EndDate',\n",
       " 'AbstractRegDate',\n",
       " 'SubDeadlineDate',\n",
       " 'NotificationDueDate',\n",
       " 'FinalVersionDueDate',\n",
       " 'CIPaperCount',\n",
       " 'CIPaperCitationCnt']"
      ]
     },
     "execution_count": 30,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df2.columns"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "39"
      ]
     },
     "execution_count": 31,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(df2.columns)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "metadata": {},
   "outputs": [],
   "source": [
    "drop_cols = [ 'PaperTitle',\n",
    " 'OriginalTitle',\n",
    "'PaperYear',\n",
    " 'PaperDate',\n",
    " 'Normalized_Name',\n",
    " 'Display_Name',\n",
    " 'CSNormalizedName',\n",
    " 'CSDisplayName',\n",
    " 'CINormalizedNm',\n",
    " 'CIDisplayNm',\n",
    "'CIOfficialURL',\n",
    " 'StartDate',\n",
    " 'EndDate',\n",
    " 'AbstractRegDate',\n",
    " 'SubDeadlineDate',\n",
    " 'NotificationDueDate',\n",
    " 'FinalVersionDueDate']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "metadata": {},
   "outputs": [],
   "source": [
    "#Drop non-numeric columns\n",
    "df3 = df2.drop(*drop_cols)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 34,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "39"
      ]
     },
     "execution_count": 34,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(df2.columns)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 35,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "22"
      ]
     },
     "execution_count": 35,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(df3.columns)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 36,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "+--------------+------------+---------+----------+----------+--------------------+----------+---------+------------+-------------+--------------+------------+---------------+---------------+------+------------+----------+------+----------+-----------------+------------+------------------+\n",
      "|ConfInstanceID|ConfSeriesID|JournalID|   PaperID|Paper_Rank|                 DOI|   DocType|Publisher|Paper_RefCnt|CitationCount|EstimatedCount|Journal_Rank|Journal_Pap_Cnt|Journal_Cit_Cnt|CSRank|CSPaperCount|CSCitation|CIRank|    CICSID|         Location|CIPaperCount|CIPaperCitationCnt|\n",
      "+--------------+------------+---------+----------+----------+--------------------+----------+---------+------------+-------------+--------------+------------+---------------+---------------+------+------------+----------+------+----------+-----------------+------------+------------------+\n",
      "|    2735348883|  2734834919| 23074404|1819143627|     24199|10.1109/PICMET.20...|Conference|     IEEE|           0|            0|             0|        9728|           9284|          64351| 11187|        4309|      6256| 25000|2734834919|Portland, OR, USA|         405|               759|\n",
      "|     180417304|  1141689323|137519996|1963578506|     22929|10.1109/ICIEEM.20...|Conference|     IEEE|          16|            0|             0|        9694|           4073|          16612| 11674|        7218|      7851| 25000|1141689323|            Macau|        1061|              1162|\n",
      "+--------------+------------+---------+----------+----------+--------------------+----------+---------+------------+-------------+--------------+------------+---------------+---------------+------+------------+----------+------+----------+-----------------+------------+------------------+\n",
      "only showing top 2 rows\n",
      "\n"
     ]
    }
   ],
   "source": [
    "df3.show(2)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 37,
   "metadata": {},
   "outputs": [],
   "source": [
    "drop_cols = ['DOI','Location','CICSID']\n",
    "df3 = df3.drop(*drop_cols)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 38,
   "metadata": {},
   "outputs": [],
   "source": [
    "result = df3.describe()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 39,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "root\n",
      " |-- ConfInstanceID: long (nullable = true)\n",
      " |-- ConfSeriesID: long (nullable = true)\n",
      " |-- JournalID: long (nullable = true)\n",
      " |-- PaperID: long (nullable = true)\n",
      " |-- Paper_Rank: integer (nullable = true)\n",
      " |-- DocType: string (nullable = true)\n",
      " |-- Publisher: string (nullable = true)\n",
      " |-- Paper_RefCnt: integer (nullable = true)\n",
      " |-- CitationCount: integer (nullable = true)\n",
      " |-- EstimatedCount: integer (nullable = true)\n",
      " |-- Journal_Rank: integer (nullable = true)\n",
      " |-- Journal_Pap_Cnt: integer (nullable = true)\n",
      " |-- Journal_Cit_Cnt: integer (nullable = true)\n",
      " |-- CSRank: integer (nullable = true)\n",
      " |-- CSPaperCount: integer (nullable = true)\n",
      " |-- CSCitation: integer (nullable = true)\n",
      " |-- CIRank: integer (nullable = true)\n",
      " |-- CIPaperCount: integer (nullable = true)\n",
      " |-- CIPaperCitationCnt: integer (nullable = true)\n",
      "\n"
     ]
    }
   ],
   "source": [
    "df3.printSchema()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 40,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "+------------------+\n",
      "|     CitationCount|\n",
      "+------------------+\n",
      "|            120861|\n",
      "|13.158777438545105|\n",
      "| 60.97803587644721|\n",
      "|                 0|\n",
      "|              6770|\n",
      "+------------------+\n",
      "\n"
     ]
    }
   ],
   "source": [
    "result.select(['CitationCount']).show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 41,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "120056"
      ]
     },
     "execution_count": 41,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "cit1 = df3.filter(df3.CitationCount < 200)\n",
    "cit1.count()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 42,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "120685"
      ]
     },
     "execution_count": 42,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "cit2 = df3.filter(df3.CitationCount < 500)\n",
    "cit2.count()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 43,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "122"
      ]
     },
     "execution_count": 43,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "cit3 = df3.filter((df3.CitationCount >=500) & (df3.CitationCount <= 1000))\n",
    "cit3.count()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 44,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "53"
      ]
     },
     "execution_count": 44,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "cit4 = df3.filter((df3.CitationCount >=1000) & (df3.CitationCount <= 6000))\n",
    "cit4.count()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 45,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "2"
      ]
     },
     "execution_count": 45,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "cit5 = df3.filter(df3.CitationCount > 5000)\n",
    "#len(cit5)\n",
    "cit5.count()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## CitationsCount < 200"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 46,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(array([6.5410e+04, 1.8526e+04, 9.5090e+03, 5.9340e+03, 4.0120e+03,\n",
       "        2.8360e+03, 2.1820e+03, 1.6960e+03, 1.3850e+03, 1.1240e+03,\n",
       "        9.0000e+02, 7.6800e+02, 6.2800e+02, 5.6200e+02, 4.4700e+02,\n",
       "        4.3000e+02, 3.6600e+02, 3.1600e+02, 2.5400e+02, 2.6200e+02,\n",
       "        2.2500e+02, 2.0200e+02, 1.8500e+02, 1.4700e+02, 1.5500e+02,\n",
       "        1.3800e+02, 1.1400e+02, 1.1700e+02, 1.0500e+02, 1.0400e+02,\n",
       "        8.8000e+01, 8.2000e+01, 8.2000e+01, 6.9000e+01, 6.6000e+01,\n",
       "        7.2000e+01, 6.8000e+01, 5.2000e+01, 4.8000e+01, 5.6000e+01,\n",
       "        4.2000e+01, 4.9000e+01, 3.5000e+01, 3.8000e+01, 2.8000e+01,\n",
       "        3.3000e+01, 3.1000e+01, 1.7000e+01, 3.1000e+01, 3.0000e+01]),\n",
       " array([  0.  ,   3.98,   7.96,  11.94,  15.92,  19.9 ,  23.88,  27.86,\n",
       "         31.84,  35.82,  39.8 ,  43.78,  47.76,  51.74,  55.72,  59.7 ,\n",
       "         63.68,  67.66,  71.64,  75.62,  79.6 ,  83.58,  87.56,  91.54,\n",
       "         95.52,  99.5 , 103.48, 107.46, 111.44, 115.42, 119.4 , 123.38,\n",
       "        127.36, 131.34, 135.32, 139.3 , 143.28, 147.26, 151.24, 155.22,\n",
       "        159.2 , 163.18, 167.16, 171.14, 175.12, 179.1 , 183.08, 187.06,\n",
       "        191.04, 195.02, 199.  ]),\n",
       " <a list of 50 Patch objects>)"
      ]
     },
     "execution_count": 46,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "import matplotlib.pyplot as plt\n",
    "\n",
    "bins, counts = cit1.select(cit1['CitationCount']).rdd.flatMap(lambda x: x).histogram(50)\n",
    "\n",
    " \n",
    "plt.hist(bins[:-1], bins=bins, weights=counts)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 47,
   "metadata": {},
   "outputs": [],
   "source": [
    "cit = df3.filter(df3.CitationCount < 100)\n",
    "#len(cit1)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## CitationsCount < 100"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 48,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(array([3.2955e+04, 1.4400e+04, 1.0368e+04, 7.6870e+03, 6.0820e+03,\n",
       "        4.9550e+03, 4.1420e+03, 3.3470e+03, 2.8710e+03, 2.5720e+03,\n",
       "        2.1400e+03, 1.9260e+03, 1.7410e+03, 1.5630e+03, 1.3700e+03,\n",
       "        1.2600e+03, 1.1620e+03, 1.0230e+03, 9.9300e+02, 8.3400e+02,\n",
       "        7.8800e+02, 7.4900e+02, 6.3700e+02, 6.6200e+02, 5.8000e+02,\n",
       "        5.8300e+02, 5.2200e+02, 4.9700e+02, 4.7200e+02, 4.3100e+02,\n",
       "        4.0100e+02, 3.9200e+02, 3.8200e+02, 3.7200e+02, 3.1200e+02,\n",
       "        3.1900e+02, 2.6300e+02, 2.9000e+02, 3.0500e+02, 2.6600e+02,\n",
       "        2.3400e+02, 2.2600e+02, 2.3800e+02, 2.0200e+02, 2.1400e+02,\n",
       "        1.8100e+02, 1.7600e+02, 1.9700e+02, 1.8100e+02, 1.5600e+02,\n",
       "        1.4900e+02, 1.4200e+02, 1.4600e+02, 1.2800e+02, 1.4000e+02,\n",
       "        1.4800e+02, 1.0600e+02, 1.4200e+02, 1.0800e+02, 9.1000e+01,\n",
       "        9.9000e+01, 1.0200e+02, 1.1700e+02, 1.1200e+02, 1.0300e+02,\n",
       "        9.3000e+01, 8.1000e+01, 8.9000e+01, 8.6000e+01, 7.9000e+01,\n",
       "        6.8000e+01, 8.3000e+01, 7.4000e+01, 5.2000e+01, 6.9000e+01,\n",
       "        5.9000e+01, 7.5000e+01, 5.1000e+01, 6.2000e+01, 7.4000e+01,\n",
       "        4.5000e+01, 5.8000e+01, 7.1000e+01, 5.1000e+01, 6.2000e+01,\n",
       "        5.0000e+01, 4.8000e+01, 4.2000e+01, 5.8000e+01, 3.5000e+01,\n",
       "        5.3000e+01, 3.9000e+01, 4.5000e+01, 3.6000e+01, 4.0000e+01,\n",
       "        2.6000e+01, 2.7000e+01, 4.3000e+01, 4.3000e+01, 4.2000e+01]),\n",
       " array([ 0.  ,  0.99,  1.98,  2.97,  3.96,  4.95,  5.94,  6.93,  7.92,\n",
       "         8.91,  9.9 , 10.89, 11.88, 12.87, 13.86, 14.85, 15.84, 16.83,\n",
       "        17.82, 18.81, 19.8 , 20.79, 21.78, 22.77, 23.76, 24.75, 25.74,\n",
       "        26.73, 27.72, 28.71, 29.7 , 30.69, 31.68, 32.67, 33.66, 34.65,\n",
       "        35.64, 36.63, 37.62, 38.61, 39.6 , 40.59, 41.58, 42.57, 43.56,\n",
       "        44.55, 45.54, 46.53, 47.52, 48.51, 49.5 , 50.49, 51.48, 52.47,\n",
       "        53.46, 54.45, 55.44, 56.43, 57.42, 58.41, 59.4 , 60.39, 61.38,\n",
       "        62.37, 63.36, 64.35, 65.34, 66.33, 67.32, 68.31, 69.3 , 70.29,\n",
       "        71.28, 72.27, 73.26, 74.25, 75.24, 76.23, 77.22, 78.21, 79.2 ,\n",
       "        80.19, 81.18, 82.17, 83.16, 84.15, 85.14, 86.13, 87.12, 88.11,\n",
       "        89.1 , 90.09, 91.08, 92.07, 93.06, 94.05, 95.04, 96.03, 97.02,\n",
       "        98.01, 99.  ]),\n",
       " <a list of 100 Patch objects>)"
      ]
     },
     "execution_count": 48,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYcAAAD8CAYAAACcjGjIAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDIuMi4zLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvIxREBQAAE5FJREFUeJzt3X+sX/V93/Hnq3ZI02StTfAQs73Za6xVTqQYahFXqSYGKxhSzVTKMtBWLMTqSgUtmTItTv6hTYIE0hpWtASJBi9myuIgkg6rccssSpX1DwiXwABDEXcEhi2D3Zgf6aLBTN774/tx+Mafe7nX94e/1/c+H9JX95z3+Zzz/Xx0rPu653PO9+tUFZIkDfu5UXdAkrTwGA6SpI7hIEnqGA6SpI7hIEnqGA6SpI7hIEnqGA6SpI7hIEnqLB91B2bqnHPOqXXr1o26G5J0RnnkkUf+pqpWTdXujA2HdevWMTY2NupuSNIZJckL02nntJIkqWM4SJI6hoMkqWM4SJI6hoMkqWM4SJI6hoMkqWM4SJI6hoMkqXPGfkJ6Ntbt/M5Pl5+/+WMj7IkkLUxeOUiSOoaDJKljOEiSOoaDJKljOEiSOoaDJKljOEiSOoaDJKljOEiSOoaDJKljOEiSOoaDJKljOEiSOoaDJKkzZTgk+fkk30vyP5McSPIHrb4+yUNJxpN8M8lZrf7utj7etq8bOtZnW/2ZJJcN1be22niSnXM/TEnSqZjOlcMbwMVV9WFgE7A1yRbgFuDWqvoA8ApwXWt/HfBKq9/a2pFkI3AV8EFgK/CVJMuSLAO+DFwObASubm0lSSMyZTjUwN+21Xe1VwEXA/e0+m7gyra8ra3Ttl+SJK2+p6reqKofAOPAhe01XlXPVdWbwJ7WVpI0ItO659D+wn8MOALsB/4X8GpVHW9NDgKr2/Jq4EWAtv014P3D9ZP2maw+UT92JBlLMnb06NHpdF2SNAPTCoeqequqNgFrGPyl/yvz2qvJ+3FHVW2uqs2rVq0aRRckaUk4paeVqupV4AHg14AVSU78H9RrgENt+RCwFqBt/yXgh8P1k/aZrC5JGpHpPK20KsmKtvwe4DeApxmExMdbs+3AvW15b1unbf+LqqpWv6o9zbQe2AB8D3gY2NCefjqLwU3rvXMxOEnSzCyfugnnAbvbU0U/B9xdVX+a5ClgT5IvAo8Cd7b2dwL/Jck4cIzBL3uq6kCSu4GngOPA9VX1FkCSG4D7gGXArqo6MGcjlCSdsinDoaoeB86foP4cg/sPJ9f/L/DPJznWTcBNE9T3Afum0V9J0mngJ6QlSR3DQZLUMRwkSR3DQZLUMRwkSR3DQZLUMRwkSR3DQZLUMRwkSR3DQZLUMRwkSR3DQZLUMRwkSR3DQZLUMRwkSR3DQZLUMRwkSR3DQZLUMRwkSR3DQZLUMRwkSR3DQZLUmTIckqxN8kCSp5IcSPLJVv/9JIeSPNZeVwzt89kk40meSXLZUH1rq40n2TlUX5/koVb/ZpKz5nqgkqTpm86Vw3Hg01W1EdgCXJ9kY9t2a1Vtaq99AG3bVcAHga3AV5IsS7IM+DJwObARuHroOLe0Y30AeAW4bo7GJ0magSnDoaoOV9X32/KPgKeB1e+wyzZgT1W9UVU/AMaBC9trvKqeq6o3gT3AtiQBLgbuafvvBq6c6YAkSbN3SvcckqwDzgceaqUbkjyeZFeSla22GnhxaLeDrTZZ/f3Aq1V1/KS6JGlEph0OSd4HfAv4VFW9DtwO/DKwCTgM/OG89PBn+7AjyViSsaNHj87320nSkjWtcEjyLgbB8PWq+jZAVb1cVW9V1U+AP2YwbQRwCFg7tPuaVpus/kNgRZLlJ9U7VXVHVW2uqs2rVq2aTtclSTMwnaeVAtwJPF1VXxqqnzfU7LeAJ9vyXuCqJO9Osh7YAHwPeBjY0J5MOovBTeu9VVXAA8DH2/7bgXtnNyxJ0mwsn7oJHwV+G3giyWOt9jkGTxttAgp4HvhdgKo6kORu4CkGTzpdX1VvASS5AbgPWAbsqqoD7XifAfYk+SLwKIMwkiSNyJThUFV/BWSCTfveYZ+bgJsmqO+baL+qeo63p6UkSSPmJ6QlSR3DQZLUMRwkSR3DQZLUMRwkSR3DQZLUMRwkSR3DQZLUMRwkSR3DQZLUMRwkSR3DQZLUMRwkSR3DQZLUMRwkSR3DQZLUMRwkSR3DQZLUMRwkSR3DQZLUMRwkSR3DQZLUmTIckqxN8kCSp5IcSPLJVj87yf4kz7afK1s9SW5LMp7k8SQXDB1re2v/bJLtQ/VfTfJE2+e2JJmPwUqSpmc6Vw7HgU9X1UZgC3B9ko3ATuD+qtoA3N/WAS4HNrTXDuB2GIQJcCPwEeBC4MYTgdLa/M7QfltnPzRJ0kxNGQ5Vdbiqvt+WfwQ8DawGtgG7W7PdwJVteRtwVw08CKxIch5wGbC/qo5V1SvAfmBr2/aLVfVgVRVw19CxJEkjcEr3HJKsA84HHgLOrarDbdNLwLlteTXw4tBuB1vtneoHJ6hLkkZk2uGQ5H3At4BPVdXrw9vaX/w1x32bqA87kowlGTt69Oh8v50kLVnTCock72IQDF+vqm+38sttSoj280irHwLWDu2+ptXeqb5mgnqnqu6oqs1VtXnVqlXT6bokaQam87RSgDuBp6vqS0Ob9gInnjjaDtw7VL+mPbW0BXitTT/dB1yaZGW7EX0pcF/b9nqSLe29rhk6liRpBJZPo81Hgd8GnkjyWKt9DrgZuDvJdcALwCfatn3AFcA48GPgWoCqOpbkC8DDrd3nq+pYW/494GvAe4A/ay9J0ohMGQ5V9VfAZJ87uGSC9gVcP8mxdgG7JqiPAR+aqi+SpNPDT0hLkjqGgySpYzhIkjqGgySpYzhIkjqGgySpYzhIkjqGgySpYzhIkjqGgySpYzhIkjqGgySpYzhIkjqGgySpYzhIkjqGgySpYzhIkjqGgySpYzhIkjqGgySpYzhIkjqGgySpYzhIkjpThkOSXUmOJHlyqPb7SQ4leay9rhja9tkk40meSXLZUH1rq40n2TlUX5/koVb/ZpKz5nKAkqRTN50rh68BWyeo31pVm9prH0CSjcBVwAfbPl9JsizJMuDLwOXARuDq1hbglnasDwCvANfNZkCSpNmbMhyq6rvAsWkebxuwp6reqKofAOPAhe01XlXPVdWbwB5gW5IAFwP3tP13A1ee4hgkSXNsNvccbkjyeJt2Wtlqq4EXh9ocbLXJ6u8HXq2q4yfVJUkjNNNwuB34ZWATcBj4wznr0TtIsiPJWJKxo0ePno63lKQlaUbhUFUvV9VbVfUT4I8ZTBsBHALWDjVd02qT1X8IrEiy/KT6ZO97R1VtrqrNq1atmknXJUnTMKNwSHLe0OpvASeeZNoLXJXk3UnWAxuA7wEPAxvak0lnMbhpvbeqCngA+Hjbfztw70z6JEmaO8unapDkG8BFwDlJDgI3Ahcl2QQU8DzwuwBVdSDJ3cBTwHHg+qp6qx3nBuA+YBmwq6oOtLf4DLAnyReBR4E752x007Bu53d+uvz8zR87nW8tSQvWlOFQVVdPUJ70F3hV3QTcNEF9H7BvgvpzvD0tJUlaAPyEtCSpYzhIkjqGgySpYzhIkjqGgySpYzhIkjqGgySpYzhIkjqGgySpYzhIkjqGgySpYzhIkjqGgySpYzhIkjqGgySpYzhIkjqGgySpYzhIkjqGgySpYzhIkjqGgySpYzhIkjpThkOSXUmOJHlyqHZ2kv1Jnm0/V7Z6ktyWZDzJ40kuGNpne2v/bJLtQ/VfTfJE2+e2JJnrQUqSTs10rhy+Bmw9qbYTuL+qNgD3t3WAy4EN7bUDuB0GYQLcCHwEuBC48USgtDa/M7Tfye8lSTrNlk/VoKq+m2TdSeVtwEVteTfwl8BnWv2uqirgwSQrkpzX2u6vqmMASfYDW5P8JfCLVfVgq98FXAn82WwGNVPrdn7np8vP3/yxUXRBkhaEmd5zOLeqDrfll4Bz2/Jq4MWhdgdb7Z3qByeoS5JGaNY3pNtVQs1BX6aUZEeSsSRjR48ePR1vKUlL0kzD4eU2XUT7eaTVDwFrh9qtabV3qq+ZoD6hqrqjqjZX1eZVq1bNsOuSpKnMNBz2AieeONoO3DtUv6Y9tbQFeK1NP90HXJpkZbsRfSlwX9v2epIt7Smla4aOJUkakSlvSCf5BoMbyuckOcjgqaObgbuTXAe8AHyiNd8HXAGMAz8GrgWoqmNJvgA83Np9/sTNaeD3GDwR9R4GN6JHcjNakvS26TytdPUkmy6ZoG0B109ynF3ArgnqY8CHpuqHJOn08RPSkqSO4SBJ6hgOkqSO4SBJ6hgOkqSO4SBJ6hgOkqSO4SBJ6hgOkqTOlJ+QXqr8vx0kLWVeOUiSOoaDJKljOEiSOoaDJKljOEiSOoaDJKljOEiSOoaDJKljOEiSOoaDJKnj12dMg1+lIWmp8cpBktQxHCRJnVmFQ5LnkzyR5LEkY612dpL9SZ5tP1e2epLclmQ8yeNJLhg6zvbW/tkk22c3JEnSbM3FlcM/qapNVbW5re8E7q+qDcD9bR3gcmBDe+0AbodBmAA3Ah8BLgRuPBEokqTRmI9ppW3A7ra8G7hyqH5XDTwIrEhyHnAZsL+qjlXVK8B+YOs89EuSNE2zDYcC/nuSR5LsaLVzq+pwW34JOLctrwZeHNr3YKtNVu8k2ZFkLMnY0aNHZ9l1SdJkZvso669X1aEkfxfYn+SvhzdWVSWpWb7H8PHuAO4A2Lx585wd91T4WKukpWBWVw5Vdaj9PAL8CYN7Bi+36SLazyOt+SFg7dDua1ptsrokaURmHA5J3pvk75xYBi4FngT2AieeONoO3NuW9wLXtKeWtgCvtemn+4BLk6xsN6IvbTVJ0ojMZlrpXOBPkpw4zn+tqj9P8jBwd5LrgBeAT7T2+4ArgHHgx8C1AFV1LMkXgIdbu89X1bFZ9EuSNEszDoeqeg748AT1HwKXTFAv4PpJjrUL2DXTvkiS5pbfrTQL3pyWtFj59RmSpI7hIEnqGA6SpI7hIEnqeEN6jnhzWtJi4pWDJKljOEiSOk4rzQOnmCSd6bxykCR1vHKYZ15FSDoTeeUgSep45XAaeRUh6UzhlYMkqeOVw4h4FSFpITMcFgCDQtJCYzgsMMNBAYaFpNEwHBY4ryokjYLhcAY5+ariBEND0lwzHBYBry4kzTXDYZExKCTNBcNhEZtsGmqYASJpIgsmHJJsBf4IWAZ8tapuHnGXloTpBMgww0RaGhZEOCRZBnwZ+A3gIPBwkr1V9dRoe6aTnWqYnCrDR1oYFkQ4ABcC41X1HECSPcA2wHBYYuY7fE42HEber5HetlDCYTXw4tD6QeAjI+qLlpDJwuh0h5RmZrJwn6/jLqU/JhZKOExLkh3Ajrb6t0memeGhzgH+Zm56dcZYimOGpTnuJTPm3PIzq3M27pOOO+P6aTCTMf+D6TRaKOFwCFg7tL6m1X5GVd0B3DHbN0syVlWbZ3ucM8lSHDMszXEvxTHD0hz3fI55oXxl98PAhiTrk5wFXAXsHXGfJGnJWhBXDlV1PMkNwH0MHmXdVVUHRtwtSVqyFkQ4AFTVPmDfaXq7WU9NnYGW4phhaY57KY4Zlua4523Mqar5OrYk6Qy1UO45SJIWkCUVDkm2JnkmyXiSnaPuz3xJsjbJA0meSnIgySdb/ewk+5M8236uHHVf51qSZUkeTfKnbX19kofaOf9me+BhUUmyIsk9Sf46ydNJfm2xn+sk/7b9234yyTeS/PxiPNdJdiU5kuTJodqE5zYDt7XxP57kgtm895IJh6Gv6Lgc2AhcnWTjaHs1b44Dn66qjcAW4Po21p3A/VW1Abi/rS82nwSeHlq/Bbi1qj4AvAJcN5Jeza8/Av68qn4F+DCD8S/ac51kNfBvgM1V9SEGD7FcxeI8118Dtp5Um+zcXg5saK8dwO2zeeMlEw4MfUVHVb0JnPiKjkWnqg5X1ffb8o8Y/LJYzWC8u1uz3cCVo+nh/EiyBvgY8NW2HuBi4J7WZDGO+ZeAfwzcCVBVb1bVqyzyc83gYZr3JFkO/AJwmEV4rqvqu8Cxk8qTndttwF018CCwIsl5M33vpRQOE31Fx+oR9eW0SbIOOB94CDi3qg63TS8B546oW/PlPwL/HvhJW38/8GpVHW/ri/GcrweOAv+5Tad9Ncl7WcTnuqoOAf8B+N8MQuE14BEW/7k+YbJzO6e/45ZSOCw5Sd4HfAv4VFW9PrytBo+pLZpH1ZL8JnCkqh4ZdV9Os+XABcDtVXU+8H84aQppEZ7rlQz+Sl4P/D3gvfRTL0vCfJ7bpRQO0/qKjsUiybsYBMPXq+rbrfzyicvM9vPIqPo3Dz4K/LMkzzOYMryYwVz8ijb1AIvznB8EDlbVQ239HgZhsZjP9T8FflBVR6vq/wHfZnD+F/u5PmGyczunv+OWUjgsma/oaHPtdwJPV9WXhjbtBba35e3Avae7b/Olqj5bVWuqah2Dc/sXVfUvgQeAj7dmi2rMAFX1EvBikn/USpcw+Kr7RXuuGUwnbUnyC+3f+okxL+pzPWSyc7sXuKY9tbQFeG1o+umULakPwSW5gsG89Imv6LhpxF2aF0l+HfgfwBO8Pf/+OQb3He4G/j7wAvCJqjr5ZtcZL8lFwL+rqt9M8g8ZXEmcDTwK/KuqemOU/ZtrSTYxuAl/FvAccC2DP/wW7blO8gfAv2DwZN6jwL9mML++qM51km8AFzH49tWXgRuB/8YE57YF5X9iMMX2Y+Daqhqb8XsvpXCQJE3PUppWkiRNk+EgSeoYDpKkjuEgSeoYDpKkjuEgSeoYDpKkjuEgSer8f/iDApwnbC4YAAAAAElFTkSuQmCC\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "import matplotlib.pyplot as plt\n",
    "\n",
    "bins, counts = cit.select(cit['CitationCount']).rdd.flatMap(lambda x: x).histogram(100)\n",
    "\n",
    "\n",
    "plt.hist(bins[:-1], bins=bins, weights=counts)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## CitationsCount > 500 and < 1000"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 49,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(array([7., 5., 6., 4., 2., 2., 9., 1., 4., 3., 6., 3., 3., 2., 3., 3., 3.,\n",
       "        1., 0., 4., 3., 0., 1., 4., 3., 4., 1., 4., 1., 0., 0., 3., 2., 1.,\n",
       "        0., 3., 2., 3., 2., 2., 3., 0., 0., 1., 0., 2., 2., 2., 0., 2.]),\n",
       " array([502.  , 511.86, 521.72, 531.58, 541.44, 551.3 , 561.16, 571.02,\n",
       "        580.88, 590.74, 600.6 , 610.46, 620.32, 630.18, 640.04, 649.9 ,\n",
       "        659.76, 669.62, 679.48, 689.34, 699.2 , 709.06, 718.92, 728.78,\n",
       "        738.64, 748.5 , 758.36, 768.22, 778.08, 787.94, 797.8 , 807.66,\n",
       "        817.52, 827.38, 837.24, 847.1 , 856.96, 866.82, 876.68, 886.54,\n",
       "        896.4 , 906.26, 916.12, 925.98, 935.84, 945.7 , 955.56, 965.42,\n",
       "        975.28, 985.14, 995.  ]),\n",
       " <a list of 50 Patch objects>)"
      ]
     },
     "execution_count": 49,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAW4AAAD8CAYAAABXe05zAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDIuMi4zLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvIxREBQAADLNJREFUeJzt3X+MZfVZx/H3445Ad21gYceGAtNLoyFp+ofFiYIoMaVB7JqSGEyWpJVWm0nUVmhMmjGmafxvaxpjTYztBtoYrbS6Um26tdCfifrH6i7QssuCrDDC4lK2jVLjP4A+/nG/A8PmzsyZnTn37nPn/Upu5pxzv3vv88zZ/cz3fu+cvZGZSJLq+KFJFyBJ2hiDW5KKMbglqRiDW5KKMbglqRiDW5KKMbglqRiDW5KKMbglqZiZPh50z549ORgM+nhoSZpKR48e/V5mznYZ20twDwYDjhw50sdDS9JUioh/7zrWpRJJKsbglqRiDG5JKsbglqRiDG5JKsbglqRiDG5JKsbglqRiDG5JKqaXKyenyWDx0MjjS/v3jrkSSRpyxi1JxRjcklSMwS1JxRjcklSMwS1JxRjcklSMwS1JxRjcklSMwS1JxRjcklSMwS1JxRjcklSMwS1JxRjcklSMwS1JxRjcklSMwS1JxRjcklRMp+COiA9FxPGIOBYR90bERX0XJkkabd3gjogrgN8G5jPzrcAOYF/fhUmSRuu6VDIDvC4iZoCdwH/0V5IkaS3rBndmPgt8HHgaOA28kJkP9F2YJGm0Lkslu4FbgauBNwK7IuLdI8YtRMSRiDhy5syZra9UkgR0Wyp5B/BUZp7JzJeA+4CfOXtQZh7IzPnMnJ+dnd3qOiVJTZfgfhq4LiJ2RkQANwEn+i1LkrSaLmvch4GDwIPAI+3PHOi5LknSKma6DMrMjwIf7bkWSVIHXjkpScUY3JJUjMEtScUY3JJUjMEtScUY3JJUjMEtScUY3JJUjMEtScUY3JJUjMEtScUY3JJUjMEtScUY3JJUjMEtScUY3JJUjMEtScUY3JJUTKePLhunweKhkceX9u8dcyWSdH5yxi1JxRjcklSMwS1JxRjcklSMwS1JxRjcklSMwS1JxRjcklSMwS1JxRjcklSMwS1JxRjcklSMwS1JxRjcklSMwS1JxRjcklSMwS1JxRjcklRMp+COiEsi4mBEPBYRJyLi+r4LkySN1vUzJz8BfCUzb4uIC4CdPdYkSVrDusEdERcDNwLvBcjMF4EX+y1LkrSaLkslVwNngM9ExEMRcXdE7Oq5LknSKroslcwA1wIfzMzDEfEJYBH4yMpBEbEALADMzc1tdZ2rGiweGnl8af/esdWwWdPQg6Tx6TLjPgWcyszDbf8gwyB/jcw8kJnzmTk/Ozu7lTVKklZYN7gz8zngmYi4ph26CXi016okSavq+lslHwQ+236j5Engff2VJElaS6fgzsyHgfmea5EkdeCVk5JUjMEtScUY3JJUjMEtScUY3JJUjMEtScUY3JJUjMEtScUY3JJUjMEtScUY3JJUjMEtScUY3JJUjMEtScUY3JJUjMEtScUY3JJUjMEtScV0/czJiRssHpp0CZJ0XnDGLUnFGNySVIzBLUnFGNySVIzBLUnFGNySVIzBLUnFGNySVIzBLUnFGNySVIzBLUnFGNySVIzBLUnFGNySVIzBLUnFGNySVIzBLUnFGNySVEzn4I6IHRHxUER8qc+CJElr28iM+07gRF+FSJK66RTcEXElsBe4u99yJEnr6Trj/iPgw8D/9ViLJKmDmfUGRMQvAc9n5tGI+Pk1xi0ACwBzc3NbVuC5GiweGnl8af/eiTzvOJ77fDOO78VGz3Pf46Vx6DLjvgF4V0QsAZ8D3h4Rf3H2oMw8kJnzmTk/Ozu7xWVKkpatG9yZ+buZeWVmDoB9wDcy8929VyZJGsnf45akYtZd414pM78FfKuXSiRJnTjjlqRiDG5JKsbglqRiDG5JKsbglqRiDG5JKsbglqRiDG5JKsbglqRiDG5JKsbglqRiDG5JKsbglqRiDG5JKsbglqRiDG5JKsbglqRiDG5JKmZDH12mzRksHup1/NL+vVvyOBt9/HOxWk1b9Rxb1XPf1qqz7+/FRv++bLSevs/xduaMW5KKMbglqRiDW5KKMbglqRiDW5KKMbglqRiDW5KKMbglqRiDW5KKMbglqRiDW5KKMbglqRiDW5KKMbglqRiDW5KKMbglqRiDW5KKMbglqZh1gzsiroqIb0bEoxFxPCLuHEdhkqTRunzm5MvA72TmgxHxeuBoRHw1Mx/tuTZJ0gjrzrgz83RmPti2/xs4AVzRd2GSpNE2tMYdEQPgbcDhPoqRJK2vy1IJABHxI8DfAHdl5g9G3L8ALADMzc1tWYFbbbB4aNIl9Gaae9uOVjufS/v39vr4WzV+kraq1tW+132fm/V0mnFHxA8zDO3PZuZ9o8Zk5oHMnM/M+dnZ2a2sUZK0QpffKgngHuBEZv5h/yVJktbSZcZ9A/Ae4O0R8XC7vbPnuiRJq1h3jTsz/xGIMdQiSerAKyclqRiDW5KKMbglqRiDW5KKMbglqRiDW5KKMbglqRiDW5KKMbglqRiDW5KKMbglqRiDW5KKMbglqRiDW5KKMbglqRiDW5KKMbglqRiDW5KKWfejyzTaYPHQpEsYu9V6Xtq/d2LPPanHPx+/F+N47knZjj2vxRm3JBVjcEtSMQa3JBVjcEtSMQa3JBVjcEtSMQa3JBVjcEtSMQa3JBVjcEtSMQa3JBVjcEtSMQa3JBVjcEtSMQa3JBVjcEtSMQa3JBVjcEtSMZ2COyJuiYjHI+JkRCz2XZQkaXXrBndE7AD+BPhF4C3A7RHxlr4LkySN1mXG/VPAycx8MjNfBD4H3NpvWZKk1XQJ7iuAZ1bsn2rHJEkTEJm59oCI24BbMvP9bf89wE9n5gfOGrcALLTda4DHz6GePcD3zuHPVbYdewb73m62Y98b7flNmTnbZeBMhzHPAlet2L+yHXuNzDwAHOhU3ioi4khmzm/mMarZjj2DfU+6jnHbjn332XOXpZJ/AX48Iq6OiAuAfcAX+yhGkrS+dWfcmflyRHwAuB/YAXw6M4/3XpkkaaQuSyVk5peBL/dcC2xyqaWo7dgz2Pd2sx377q3ndd+clCSdX7zkXZKKGWtwR8RSRDwSEQ9HxJF27NKI+GpEPNG+7m7HIyL+uF1m/52IuHactW6liLgkIg5GxGMRcSIirp/mviPimnaOl28/iIi7prnnZRHxoYg4HhHHIuLeiLiovbF/uPX3+fYmPxFxYds/2e4fTLb6cxcRd7aej0fEXe3Y1J3viPh0RDwfEcdWHNtwnxFxRxv/RETcseFCMnNsN2AJ2HPWsT8AFtv2IvCxtv1O4O+BAK4DDo+z1i3u+8+A97ftC4BLtkPfrZ8dwHPAm6a9Z4YXpj0FvK7t/xXw3vZ1Xzv2SeA32vZvAp9s2/uAz0+6h3Ps+63AMWAnw/fNvgb82DSeb+BG4Frg2IpjG+oTuBR4sn3d3bZ3b6iOMTc9KrgfBy5v25cDj7ftTwG3jxpX6QZc3P4xx3bqe0X9NwP/tB165tWrjC9tAfYl4BcYXoQx08ZcD9zftu8Hrm/bM21cTKL2Tfb9K8A9K/Y/Anx4Ws83MDgruDfUJ3A78KkVx18zrstt3GvcCTwQEUfblZYAb8jM0237OeANbXtaLrW/GjgDfCYiHoqIuyNiF9Pf97J9wL1te6p7zsxngY8DTwOngReAo8B/ZebLbdjK3l7pu93/AnDZOGveIseAn4uIyyJiJ8OZ5lVM+fleYaN9brr/cQf3z2bmtQz/p8HfiogbV96Zwx8/0/ZrLjMMX1r9aWa+Dfgfhi+nXjGlfdPWct8F/PXZ901jz21t81aGP6zfCOwCbploUWOQmSeAjwEPAF8BHgb+96wxU3e+RxlXn2MN7jYjITOfB77A8H8e/G5EXA7Qvj7fhne61L6AU8CpzDzc9g8yDPJp7xuGP6AfzMzvtv1p7/kdwFOZeSYzXwLuA24ALomI5WsmVvb2St/t/ouB74+35K2Rmfdk5k9m5o3AfwL/yvSf72Ub7XPT/Y8tuCNiV0S8fnmb4drnMYaXzy+/q3oH8Hdt+4vAr7Z3Zq8DXljxcqSMzHwOeCYirmmHbgIeZcr7bm7n1WUSmP6enwaui4idERG8eq6/CdzWxpzd9/L34zbgG23GVk5E/Gj7Ogf8MvCXTP/5XrbRPu8Hbo6I3e1V2s3tWHdjXNB/M/DtdjsO/F47fhnwdeAJhu9GX9qOB8MPcPg34BFgftJvSmyi958AjgDfAf6W4TvJU903w2WC7wMXrzg21T23Xn4feIzhpOTPgQvb3/1/Bk4yXDa6sI29qO2fbPe/edL1b6Lvf2D4Q+rbwE3Ter4ZTkROAy8xfDX96+fSJ/Br7byfBN630Tq8clKSivHKSUkqxuCWpGIMbkkqxuCWpGIMbkkqxuCWpGIMbkkqxuCWpGL+HwZsJg556DM4AAAAAElFTkSuQmCC\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "import matplotlib.pyplot as plt\n",
    "\n",
    "bins, counts = cit3.select(cit3['CitationCount']).rdd.flatMap(lambda x: x).histogram(50)\n",
    "\n",
    "plt.hist(bins[:-1], bins=bins, weights=counts)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 50,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAY0AAAEWCAYAAACaBstRAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDIuMi4zLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvIxREBQAAGi5JREFUeJzt3X2QXXWd5/H3lwQJoIanlhICdmqMaHR4iCHEhaUYwoYEKVAEF2aBmPBQjuCIu6XGmaWyq1LGqalxBCWalRgywwoOOBIRgfC0yOwGkpioQAQyQKAzgDHhYQR5CPnuH/eXpI2d5Effpk83/X5VdfU5v/M753z7Vu795JzzO+dGZiJJUo2dmi5AkjR4GBqSpGqGhiSpmqEhSapmaEiSqhkakqRqhoYkqZqhIUmqZmhIkqoNb7qAvrbPPvtkZ2dn02VI0qCybNmy32Zmx476velCo7Ozk6VLlzZdhiQNKhGxuqafp6ckSdUMDUlSNUNDklTtTXdNQ9Kbw6uvvkpXVxcvvfRS06W8qYwYMYJRo0ax884792p9Q0PSgNTV1cXb3vY2Ojs7iYimy3lTyEzWrVtHV1cXo0eP7tU2PD0laUB66aWX2HvvvQ2MPhQR7L333m0dvRkakgYsA6PvtfuaGhqSpGpe0+hB58yfNF0Cj83+cNMlSANKX78va95jl156KXPmzGHcuHFcddVVfbr/wcrQkKRtuPzyy7n11lsZNWrU5rYNGzYwfPjQ/ej09JQk9eCTn/wkjzzyCFOnTmXkyJGcddZZHHnkkZx11lm89tprfO5zn+Pwww/n4IMP5jvf+Q7QGp104YUXctBBB3HcccdxwgkncO211wKtRxz99re/BWDp0qUcc8wxALzwwgvMmDGDCRMmcNhhh3H99dcDMH/+fE455RSmTJnCmDFj+PznP7+5tptuuolx48ZxyCGHMGnSJDZu3MiYMWNYu3YtABs3buTd73735vm+NHTjUpK249vf/jY33XQTd9xxB9/85jf58Y9/zN13382uu+7K3LlzGTlyJEuWLOHll1/myCOPZPLkySxfvpwHH3yQBx54gKeffpqxY8cyY8aM7e7nkksu4dhjj2XevHk8++yzTJgwgeOOOw6AFStWsHz5cnbZZRcOOuggPv3pTzNixAjOO+887rrrLkaPHs369evZaaedOPPMM7nqqqu46KKLuPXWWznkkEPo6Njh8wdftx0eaUTEvIj4TUTc161tr4hYFBEPl997lvaIiEsjYlVE/DIixnVbZ1rp/3BETOvW/sGI+FVZ59Iol/a3tQ9JasJJJ53ErrvuCsAtt9zCggULOPTQQzniiCNYt24dDz/8MHfddRdnnHEGw4YNY7/99uPYY4/d4XZvueUWZs+ezaGHHsoxxxzDSy+9xOOPPw7ApEmTGDlyJCNGjGDs2LGsXr2axYsXc/TRR2++z2KvvfYCYMaMGSxYsACAefPmMX369DfiZag6PTUfmLJV20zgtswcA9xW5gGmAmPKz/nAHGgFADALOAKYAMzqFgJzgPO6rTdlB/uQpH63++67b57OTC677DJWrFjBihUrePTRR5k8efJ21x8+fDgbN24E+IP7JDKT6667bvO2Hn/8cd73vvcBsMsuu2zuN2zYMDZs2LDN7R9wwAHsu+++3H777dx7771MnTq1V3/njuwwNDLzLmD9Vs0nA1eW6SuBj3RrX5Ati4E9IuKdwPHAosxcn5nPAIuAKWXZ2zNzcWYmsGCrbfW0D0lq1PHHH8+cOXN49dVXAXjooYd44YUXOProo7nmmmt47bXXePLJJ7njjjs2r9PZ2cmyZcsAuO666/5gW5dddhmtj0BYvnz5dvc9ceJE7rrrLh599FEA1q/f8vF87rnncuaZZ3LaaacxbNiwvvljt9Lbaxr7ZuaTZfopYN8yvT/wRLd+XaVte+1dPbRvbx9/JCLOp3Vkw4EHHvh6/xZJg8BAGoZ+7rnn8thjjzFu3Dgyk46ODn70ox/x0Y9+lNtvv52xY8dy4IEH8qEPfWjzOrNmzeKcc87h4osv3nwRHODiiy/moosu4uCDD2bjxo2MHj2aG264YZv77ujoYO7cuZxyyils3LiRd7zjHSxatAhonUKbPn36G3ZqCiA2pdt2O0V0Ajdk5gfK/LOZuUe35c9k5p4RcQMwOzPvLu23AV8AjgFGZOZXSvvFwO+BO0v/40r7fwS+kJknbmsfO6p1/Pjx2e6XMHmfhtS8lStXbj5NM1h94hOf4MQTT+TUU0/tl/0tXbqUz372s/zsZz/bbr+eXtuIWJaZ43e0j94OuX26nFqi/P5NaV8DHNCt36jStr32UT20b28fkqStzJ49m4997GN89atffUP309vQWAhsGgE1Dbi+W/vZZRTVROC5corpZmByROxZLoBPBm4uy56PiIll1NTZW22rp31I0qAwf/78fjvKmDlzJqtXr+aoo456Q/ezw2saEfF9WqeX9omILlqjoGYDP4iIc4DVwMdL9xuBE4BVwIvAdIDMXB8RXwaWlH5fysxNV28+RWuE1q7AT8sP29mHpCEiM31oYR+ruSSxPTsMjcw8YxuLJvXQN4ELtrGdecC8HtqXAh/ooX1dT/uQNDSMGDGCdevW+Xj0PrTp+zRGjBjR6214R7ikAWnUqFF0dXW9IY/CGMo2fXNfbxkakgaknXfeudffLqc3jg8slCRVMzQkSdUMDUlSNUNDklTN0JAkVTM0JEnVDA1JUjVDQ5JUzdCQJFUzNCRJ1QwNSVI1Q0OSVM3QkCRVMzQkSdUMDUlSNUNDklTN0JAkVTM0JEnVDA1JUjVDQ5JUzdCQJFUzNCRJ1QwNSVI1Q0OSVM3QkCRVMzQkSdUMDUlSNUNDklTN0JAkVWsrNCLisxFxf0TcFxHfj4gRETE6Iu6JiFURcU1EvKX03aXMryrLO7tt54ul/cGIOL5b+5TStioiZrZTqySpfb0OjYjYH/hLYHxmfgAYBpwOfA34ema+G3gGOKescg7wTGn/eulHRIwt670fmAJcHhHDImIY8C1gKjAWOKP0lSQ1pN3TU8OBXSNiOLAb8CRwLHBtWX4l8JEyfXKZpyyfFBFR2q/OzJcz81FgFTCh/KzKzEcy8xXg6tJXktSQXodGZq4B/hZ4nFZYPAcsA57NzA2lWxewf5neH3iirLuh9N+7e/tW62yr/Y9ExPkRsTQilq5du7a3f5IkaQfaOT21J63/+Y8G9gN2p3V6qd9l5tzMHJ+Z4zs6OpooQZKGhHZOTx0HPJqZazPzVeCHwJHAHuV0FcAoYE2ZXgMcAFCWjwTWdW/fap1ttUuSGtJOaDwOTIyI3cq1iUnAA8AdwKmlzzTg+jK9sMxTlt+emVnaTy+jq0YDY4B7gSXAmDIa6y20LpYvbKNeSVKbhu+4S88y856IuBb4ObABWA7MBX4CXB0RXyltV5RVrgD+ISJWAetphQCZeX9E/IBW4GwALsjM1wAi4kLgZlojs+Zl5v29rVeS1L5ehwZAZs4CZm3V/AitkU9b930JOG0b27kEuKSH9huBG9upUZLUd7wjXJJUzdCQJFUzNCRJ1QwNSVI1Q0OSVM3QkCRVMzQkSdUMDUlSNUNDklTN0JAkVTM0JEnVDA1JUjVDQ5JUzdCQJFUzNCRJ1QwNSVI1Q0OSVM3QkCRVMzQkSdUMDUlSNUNDklTN0JAkVTM0JEnVDA1JUjVDQ5JUzdCQJFUzNCRJ1QwNSVI1Q0OSVM3QkCRVMzQkSdXaCo2I2CMiro2IX0fEyoj4UETsFRGLIuLh8nvP0jci4tKIWBURv4yIcd22M630fzgipnVr/2BE/Kqsc2lERDv1SpLa0+6RxjeAmzLzvcAhwEpgJnBbZo4BbivzAFOBMeXnfGAOQETsBcwCjgAmALM2BU3pc1639aa0Wa8kqQ29Do2IGAkcDVwBkJmvZOazwMnAlaXblcBHyvTJwIJsWQzsERHvBI4HFmXm+sx8BlgETCnL3p6ZizMzgQXdtiVJakA7RxqjgbXA9yJieUR8NyJ2B/bNzCdLn6eAfcv0/sAT3dbvKm3ba+/qof2PRMT5EbE0IpauXbu2jT9JkrQ97YTGcGAcMCczDwNeYMupKADKEUK2sY8qmTk3M8dn5viOjo43eneSNGS1ExpdQFdm3lPmr6UVIk+XU0uU378py9cAB3Rbf1Rp2177qB7aJUkN6XVoZOZTwBMRcVBpmgQ8ACwENo2AmgZcX6YXAmeXUVQTgefKaaybgckRsWe5AD4ZuLksez4iJpZRU2d325YkqQHD21z/08BVEfEW4BFgOq0g+kFEnAOsBj5e+t4InACsAl4sfcnM9RHxZWBJ6felzFxfpj8FzAd2BX5afiRJDWkrNDJzBTC+h0WTeuibwAXb2M48YF4P7UuBD7RToySp73hHuCSpmqEhSapmaEiSqhkakqRqhoYkqZqhIUmqZmhIkqoZGpKkaoaGJKmaoSFJqmZoSJKqGRqSpGqGhiSpmqEhSapmaEiSqhkakqRqhoYkqZqhIUmqZmhIkqoZGpKkaoaGJKmaoSFJqmZoSJKqGRqSpGqGhiSpmqEhSapmaEiSqhkakqRqhoYkqZqhIUmqZmhIkqq1HRoRMSwilkfEDWV+dETcExGrIuKaiHhLad+lzK8qyzu7beOLpf3BiDi+W/uU0rYqIma2W6skqT19caTxGWBlt/mvAV/PzHcDzwDnlPZzgGdK+9dLPyJiLHA68H5gCnB5CaJhwLeAqcBY4IzSV5LUkLZCIyJGAR8GvlvmAzgWuLZ0uRL4SJk+ucxTlk8q/U8Grs7MlzPzUWAVMKH8rMrMRzLzFeDq0leS1JB2jzT+Hvg8sLHM7w08m5kbynwXsH+Z3h94AqAsf67039y+1Trbav8jEXF+RCyNiKVr165t80+SJG1Lr0MjIk4EfpOZy/qwnl7JzLmZOT4zx3d0dDRdjiS9aQ1vY90jgZMi4gRgBPB24BvAHhExvBxNjALWlP5rgAOArogYDowE1nVr36T7OttqlyQ1oNdHGpn5xcwclZmdtC5k356Z/wW4Azi1dJsGXF+mF5Z5yvLbMzNL++lldNVoYAxwL7AEGFNGY72l7GNhb+uVJLWvnSONbfkCcHVEfAVYDlxR2q8A/iEiVgHraYUAmXl/RPwAeADYAFyQma8BRMSFwM3AMGBeZt7/BtQrSarUJ6GRmXcCd5bpR2iNfNq6z0vAadtY/xLgkh7abwRu7IsaJUnt845wSVI1Q0OSVM3QkCRVMzQkSdUMDUlSNUNDklTN0JAkVTM0JEnVDA1JUjVDQ5JUzdCQJFUzNCRJ1QwNSVI1Q0OSVM3QkCRVMzQkSdUMDUlSNUNDklTN0JAkVTM0JEnVDA1JUjVDQ5JUzdCQJFUzNCRJ1QwNSVI1Q0OSVM3QkCRVMzQkSdUMDUlSNUNDklTN0JAkVet1aETEARFxR0Q8EBH3R8RnSvteEbEoIh4uv/cs7RERl0bEqoj4ZUSM67ataaX/wxExrVv7ByPiV2WdSyMi2vljJUntaedIYwPw3zJzLDARuCAixgIzgdsycwxwW5kHmAqMKT/nA3OgFTLALOAIYAIwa1PQlD7ndVtvShv1SpLa1OvQyMwnM/PnZfrfgZXA/sDJwJWl25XAR8r0ycCCbFkM7BER7wSOBxZl5vrMfAZYBEwpy96emYszM4EF3bYlSWpAn1zTiIhO4DDgHmDfzHyyLHoK2LdM7w880W21rtK2vfauHtp72v/5EbE0IpauXbu2rb9FkrRtbYdGRLwVuA64KDOf776sHCFku/vYkcycm5njM3N8R0fHG707SRqy2gqNiNiZVmBclZk/LM1Pl1NLlN+/Ke1rgAO6rT6qtG2vfVQP7ZKkhrQzeiqAK4CVmfl33RYtBDaNgJoGXN+t/ewyimoi8Fw5jXUzMDki9iwXwCcDN5dlz0fExLKvs7ttS5LUgOFtrHskcBbwq4hYUdr+CpgN/CAizgFWAx8vy24ETgBWAS8C0wEyc31EfBlYUvp9KTPXl+lPAfOBXYGflh9JUkN6HRqZeTewrfsmJvXQP4ELtrGtecC8HtqXAh/obY2SpL7lHeGSpGqGhiSpmqEhSapmaEiSqhkakqRqhoYkqZqhIUmqZmhIkqoZGpKkaoaGJKmaoSFJqmZoSJKqGRqSpGqGhiSpmqEhSapmaEiSqrXzzX0aAjpn/qTpEnhs9oebLkFS4ZGGJKmaoSFJqmZoSJKqGRqSpGqGhiSpmqEhSapmaEiSqhkakqRqhoYkqZqhIUmqZmhIkqr57CmpwkB4Bhf4HC41zyMNSVI1Q0OSVG3An56KiCnAN4BhwHczc3bDJUlDmqfqhrYBfaQREcOAbwFTgbHAGRExttmqJGnoGuhHGhOAVZn5CEBEXA2cDDzQaFWSxNA86orM7LedvV4RcSowJTPPLfNnAUdk5oVb9TsfOL/MHgQ82K+F/rF9gN82XMNA4Wuxha/FFr4WWwyU1+Jdmdmxo04D/UijSmbOBeY2XccmEbE0M8c3XcdA4Guxha/FFr4WWwy212JAX9MA1gAHdJsfVdokSQ0Y6KGxBBgTEaMj4i3A6cDChmuSpCFrQJ+eyswNEXEhcDOtIbfzMvP+hsuqMWBOlQ0AvhZb+Fps4WuxxaB6LQb0hXBJ0sAy0E9PSZIGEENDklTN0JAkVTM0JEnVDI0+FBF7RcReTdeh5kXEgRExokxHREyPiMsi4i8iYkCPWuwPEbGg6RrUO46ealNEHAj8DTAJeBYI4O3A7cDMzHysuer6V7d7af4tM2+NiD8H/gOwEpibma82WmA/ioj7gAmZ+WJEfA34E+BHwLEAmTmjyfr6U0RsfW9VAH9G6z1CZp7U70U1KCLeC+wP3JOZv+vWPiUzb2qusjqGRpsi4v8Bfw9cm5mvlbZhwGnARZk5scn6+lNEXEXr3p/daAXoW4Ef0grUyMxpDZbXryLigcwcW6aXAYdn5sYy/4vMPKTRAvtRRPyc1kNGvwskrdD4Pq3/YJCZ/6e56vpXRPwlcAGt/0gdCnwmM68vy36emeOarK/GkD9M7gP7ZOY13RtKeFwdEV9uqKam/GlmHlxOv6wB9svM1yLiH4FfNFxbf3siIo7NzNuBx2g9Dmd1ROzdbFmNGA98Bvhr4HOZuSIifj+UwqKb84APZubvIqITuDYiOjPzG7TCdMAzNNq3LCIuB64EnihtBwDTgOWNVdWMncopqt1pHW2MBNYDuwA7N1lYA84FFkTE/wCeA1ZExApgD+C/NllYfytHWF+PiH8qv59m6H727LTplFRmPhYRx9AKjndhaAwZZwPnAP+T1nlKgC7gx8AVTRXVkCuAX9N65MtfA/8UEY8AE4Grmyysv2XmE8CfRcT7gPcA82n9u1iy6TTVUJOZXcBpEfFh4Pmm62nI0xFxaGauAChHHCcC84A/bba0Ol7TUJ+KiP0AMvPfImIP4Djg8cy8t9nKpOZFxChgQ2Y+1cOyIzPzXxoo63UxNN5AEXFiZt7QdB0aWCJibmaev+Oeb34RcUNmnth0HarnfRpvrMObLmCgiAjDc4vvNF3AAHJe0wUMFIPlPeKRRh8o465PZss1jTXAwsxc2VxVA0tEvDMzn2y6DmmgGizvEY802hQRX6B1kTeAe8tPAN+PiJlN1jaQDIY3Q1+KiJ0iYkZE/CQifhERP4+Iq8tomSElIt4aEV+KiPsj4rmIWBsRiyPiE03XNpAMlveIRxptioiHgPdvfbdzGXp6f2aOaaay/hcRbwU+D3yM1lfzvgL8K/DtzJzfYGn9LiK+B6wGbgVOpTVa6GfAF4DrM/OyBsvrVxFxPfDPtF6Lj9Makn018N+BNZn5Vw2W16/eDO8RQ6NNEfFr4PjMXL1V+7uAWzLzoGYq639+OGwREb/MzIO7zS/OzIkRsQuwIjPf12B5/WrrO+AjYklmHh4ROwEPZOZ7GyyvX70Z3iOenmrfRcBtEfHTiJhbfm4CbqN1F+xQ0pmZ8zOzKzP/DjgpMx8GpgOnNFxbf3s1Iv4EICLG0fofJZn5Mq1HaQwlL0TEUQARcRKtGz433fQ3KG5o60OD/j3izX1tysybIuI9wAT+8EL4kk3PohpCXoiIozLz7q0/HCJiqH04fA64IyJepvU+Ox0gIjqAQTFKpg/9BfC/ImIMcD8wAza/Ft9qsrAGDPr3iKen1Gci4mBaD6Xb/OGQmQ+VD4czMvPSRgvsZxHxIVo3ci2JiLHAFODXmXljw6U1qhx1TADuy8xbmq6nP3V7j7wHuI9B+B4xNNQvImJ6Zn6v6Tr6S0TMAqbSOspYROtD8k7gPwE3Z+YlzVXXvyLi3sycUKbPBS6kdV5/MvDjzJzdZH39qTzl9p/LY2YGJUND/SIiHs/MA5uuo79ExK9oPfp6F+ApYFRmPh8Ru9L6HoWDt7uBN5GIWJ6Zh5XpJcAJmbk2InYHFmfmoHjmUl+IiOeAF2iNmPrftL5SYW2zVb0+XtNQn4mIX25rEbBvf9YyAGwo17RejIh/zcznATLz9xEx1B5YuFNE7Elr4E1s+pDMzBciYkOzpfW7R4AP0nom238GvlS+b+X7wA8z89+bLK6GoaG+tC9wPPDMVu0B/N/+L6dRr0TEbpn5Iq0PCQAiYiQw1EJjJLCM1r+D3HTnc7lnYVBc/O1DWUaN3QLcEhE70zqNeQbwt0BHk8XVMDTUl24A3rrpsc/dRcSd/V9Oo44uw2s3DS3dZGda37UyZGRm5zYWbQQ+2o+lDAR/EJLlpuCFwMKI2K2Zkl4fr2lIUj+JiPdk5kNN19EOQ0OSVM07wiVJ1QwNSVI1Q0PqQxHRGRH39dD+3XJXuDSoOXpK6geZeW7TNUh9wSMNqe8Nj4irImJlRFwbEbtFxJ0RMR4gIn4XEZeUL2daHBFD7cZHDWKGhtT3DgIuL9+Z8Tzwqa2Wb3p8xiHAXfg92RpEDA2p7z2Rmf9Spv8ROGqr5a+w5fHoy4DOfqpLapuhIfW9rW9+2nr+1dxyg9RreG1Rg4ihIfW9A8t3aQD8OXB3k8VIfcnQkPreg8AFEbES2BOY03A9Up/xMSKSpGoeaUiSqhkakqRqhoYkqZqhIUmqZmhIkqoZGpKkaoaGJKna/weBJ0CW3AjsxQAAAABJRU5ErkJggg==\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "# Doing the heavy lifting in Spark. We could leverage the `histogram` function from the RDD api\n",
    "\n",
    "import pandas as pd\n",
    "gre_histogram = cit.select('CitationCount').rdd.flatMap(lambda x: x).histogram(5)\n",
    "\n",
    "# Loading the Computed Histogram into a Pandas Dataframe for plotting\n",
    "pd.DataFrame(list(zip(*gre_histogram)), columns=['bin','frequency']).set_index('bin').plot(kind='bar');"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Count of Citations = 0 "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 51,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "32955"
      ]
     },
     "execution_count": 51,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "cit0 = df3.filter(df3.CitationCount == 0)\n",
    "\n",
    "cit0.count()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Convert Categorical variables Publisher and DocType to numerical"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 52,
   "metadata": {},
   "outputs": [],
   "source": [
    "from pyspark.ml.feature import CountVectorizer"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 53,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "+--------------+------------+---------+----------+----------+----------+---------+------------+-------------+--------------+------------+---------------+---------------+------+------------+----------+------+------------+------------------+\n",
      "|ConfInstanceID|ConfSeriesID|JournalID|   PaperID|Paper_Rank|   DocType|Publisher|Paper_RefCnt|CitationCount|EstimatedCount|Journal_Rank|Journal_Pap_Cnt|Journal_Cit_Cnt|CSRank|CSPaperCount|CSCitation|CIRank|CIPaperCount|CIPaperCitationCnt|\n",
      "+--------------+------------+---------+----------+----------+----------+---------+------------+-------------+--------------+------------+---------------+---------------+------+------------+----------+------+------------+------------------+\n",
      "|    2735348883|  2734834919| 23074404|1819143627|     24199|Conference|     IEEE|           0|            0|             0|        9728|           9284|          64351| 11187|        4309|      6256| 25000|         405|               759|\n",
      "|     180417304|  1141689323|137519996|1963578506|     22929|Conference|     IEEE|          16|            0|             0|        9694|           4073|          16612| 11674|        7218|      7851| 25000|        1061|              1162|\n",
      "+--------------+------------+---------+----------+----------+----------+---------+------------+-------------+--------------+------------+---------------+---------------+------+------------+----------+------+------------+------------------+\n",
      "only showing top 2 rows\n",
      "\n"
     ]
    }
   ],
   "source": [
    "df3.show(2)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 54,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "120861"
      ]
     },
     "execution_count": 54,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df3.count()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 55,
   "metadata": {},
   "outputs": [],
   "source": [
    "df3_fil = df3.filter(((df3[\"Publisher\"] != \"\") | (df3[\"Publisher\"].isNotNull())))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 56,
   "metadata": {},
   "outputs": [],
   "source": [
    "df3_fil = df3_fil.filter(((df3_fil[\"DocType\"] != \"\") | (df3_fil[\"DocType\"].isNotNull())))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 57,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "90728"
      ]
     },
     "execution_count": 57,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df3_fil.count()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 58,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "+--------------+------------+---------+----------+----------+----------+---------+------------+-------------+--------------+------------+---------------+---------------+------+------------+----------+------+------------+------------------+--------------+\n",
      "|ConfInstanceID|ConfSeriesID|JournalID|   PaperID|Paper_Rank|   DocType|Publisher|Paper_RefCnt|CitationCount|EstimatedCount|Journal_Rank|Journal_Pap_Cnt|Journal_Cit_Cnt|CSRank|CSPaperCount|CSCitation|CIRank|CIPaperCount|CIPaperCitationCnt|PublisherIndex|\n",
      "+--------------+------------+---------+----------+----------+----------+---------+------------+-------------+--------------+------------+---------------+---------------+------+------------+----------+------+------------+------------------+--------------+\n",
      "|    2735348883|  2734834919| 23074404|1819143627|     24199|Conference|     IEEE|           0|            0|             0|        9728|           9284|          64351| 11187|        4309|      6256| 25000|         405|               759|           0.0|\n",
      "|     180417304|  1141689323|137519996|1963578506|     22929|Conference|     IEEE|          16|            0|             0|        9694|           4073|          16612| 11674|        7218|      7851| 25000|        1061|              1162|           0.0|\n",
      "+--------------+------------+---------+----------+----------+----------+---------+------------+-------------+--------------+------------+---------------+---------------+------+------------+----------+------+------------+------------------+--------------+\n",
      "only showing top 2 rows\n",
      "\n"
     ]
    }
   ],
   "source": [
    "from pyspark.ml.feature import StringIndexer\n",
    "\n",
    "indexer = StringIndexer(inputCol=\"Publisher\", outputCol=\"PublisherIndex\")\n",
    "indexed = indexer.fit(df3_fil).transform(df3_fil)\n",
    "indexed.show(2)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 59,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "+--------------+------------+---------+----------+----------+----------+---------+------------+-------------+--------------+------------+---------------+---------------+------+------------+----------+------+------------+------------------+--------------+------------+\n",
      "|ConfInstanceID|ConfSeriesID|JournalID|   PaperID|Paper_Rank|   DocType|Publisher|Paper_RefCnt|CitationCount|EstimatedCount|Journal_Rank|Journal_Pap_Cnt|Journal_Cit_Cnt|CSRank|CSPaperCount|CSCitation|CIRank|CIPaperCount|CIPaperCitationCnt|PublisherIndex|DocTypeIndex|\n",
      "+--------------+------------+---------+----------+----------+----------+---------+------------+-------------+--------------+------------+---------------+---------------+------+------------+----------+------+------------+------------------+--------------+------------+\n",
      "|    2735348883|  2734834919| 23074404|1819143627|     24199|Conference|     IEEE|           0|            0|             0|        9728|           9284|          64351| 11187|        4309|      6256| 25000|         405|               759|           0.0|         0.0|\n",
      "|     180417304|  1141689323|137519996|1963578506|     22929|Conference|     IEEE|          16|            0|             0|        9694|           4073|          16612| 11674|        7218|      7851| 25000|        1061|              1162|           0.0|         0.0|\n",
      "+--------------+------------+---------+----------+----------+----------+---------+------------+-------------+--------------+------------+---------------+---------------+------+------------+----------+------+------------+------------------+--------------+------------+\n",
      "only showing top 2 rows\n",
      "\n"
     ]
    }
   ],
   "source": [
    "indexer = StringIndexer(inputCol=\"DocType\", outputCol=\"DocTypeIndex\")\n",
    "indexed = indexer.fit(indexed).transform(indexed)\n",
    "indexed.show(2)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 60,
   "metadata": {},
   "outputs": [],
   "source": [
    "drop_cols = [\"Publisher\",\"DocType\",\"PublisherIndex\"]\n",
    "final_df =indexed.drop(*drop_cols)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 61,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "+------------+\n",
      "|DocTypeIndex|\n",
      "+------------+\n",
      "|         1.0|\n",
      "|         0.0|\n",
      "|         2.0|\n",
      "+------------+\n",
      "\n"
     ]
    }
   ],
   "source": [
    "final_df.select(\"DocTypeIndex\").distinct().show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 62,
   "metadata": {},
   "outputs": [],
   "source": [
    "#final_df.select(\"PublisherIndex\").distinct().count()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 63,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "+--------------+------------+---------+----------+----------+------------+-------------+--------------+------------+---------------+---------------+------+------------+----------+------+------------+------------------+------------+\n",
      "|ConfInstanceID|ConfSeriesID|JournalID|   PaperID|Paper_Rank|Paper_RefCnt|CitationCount|EstimatedCount|Journal_Rank|Journal_Pap_Cnt|Journal_Cit_Cnt|CSRank|CSPaperCount|CSCitation|CIRank|CIPaperCount|CIPaperCitationCnt|DocTypeIndex|\n",
      "+--------------+------------+---------+----------+----------+------------+-------------+--------------+------------+---------------+---------------+------+------------+----------+------+------------+------------------+------------+\n",
      "|    2735348883|  2734834919| 23074404|1819143627|     24199|           0|            0|             0|        9728|           9284|          64351| 11187|        4309|      6256| 25000|         405|               759|         0.0|\n",
      "+--------------+------------+---------+----------+----------+------------+-------------+--------------+------------+---------------+---------------+------+------------+----------+------+------------+------------------+------------+\n",
      "only showing top 1 row\n",
      "\n"
     ]
    }
   ],
   "source": [
    "final_df.show(1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 64,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "We have 63430 training examples and 27298 test examples.\n"
     ]
    }
   ],
   "source": [
    "# Split 70% for training and 30% for testing\n",
    "train, test = final_df.randomSplit([0.7, 0.3])\n",
    "print(\"We have %d training examples and %d test examples.\" % (train.count(), test.count()))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 65,
   "metadata": {},
   "outputs": [],
   "source": [
    "from pyspark.ml.feature import VectorAssembler, VectorIndexer\n",
    "featuresCols = final_df.columns\n",
    "featuresCols.remove('CitationCount')\n",
    "# Concatenates all feature columns into a single feature vector in a new column \"rawFeatures\"\n",
    "vectorAssembler = VectorAssembler(inputCols=featuresCols, outputCol=\"rawFeatures\")\n",
    "# Identifies categorical features and indexes them\n",
    "vectorIndexer = VectorIndexer(inputCol=\"rawFeatures\", outputCol=\"features\", maxCategories=4)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 66,
   "metadata": {},
   "outputs": [],
   "source": [
    "from pyspark.ml.regression import GBTRegressor\n",
    "# Takes the \"features\" column and learns to predict \"cnt\"\n",
    "gbt = GBTRegressor(labelCol=\"CitationCount\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 67,
   "metadata": {},
   "outputs": [],
   "source": [
    "from pyspark.ml.tuning import CrossValidator, ParamGridBuilder\n",
    "from pyspark.ml.evaluation import RegressionEvaluator"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 68,
   "metadata": {},
   "outputs": [],
   "source": [
    "paramGrid = ParamGridBuilder()\\\n",
    "  .addGrid(gbt.maxDepth, [2, 5])\\\n",
    "  .addGrid(gbt.maxIter, [10, 100])\\\n",
    "  .build()\n",
    "evaluator = RegressionEvaluator(metricName=\"r2\", labelCol=gbt.getLabelCol(), predictionCol=gbt.getPredictionCol())\n",
    "cv = CrossValidator(estimator=gbt, evaluator=evaluator, estimatorParamMaps=paramGrid)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 69,
   "metadata": {},
   "outputs": [],
   "source": [
    "from pyspark.ml import Pipeline\n",
    "pipeline = Pipeline(stages=[vectorAssembler, vectorIndexer, cv])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 70,
   "metadata": {},
   "outputs": [],
   "source": [
    "pipelineModel = pipeline.fit(train)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 71,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "DataFrame[CitationCount: int, prediction: double, ConfInstanceID: bigint, ConfSeriesID: bigint, JournalID: bigint, PaperID: bigint, Paper_Rank: int, Paper_RefCnt: int, EstimatedCount: int, Journal_Rank: int, Journal_Pap_Cnt: int, Journal_Cit_Cnt: int, CSRank: int, CSPaperCount: int, CSCitation: int, CIRank: int, CIPaperCount: int, CIPaperCitationCnt: int, DocTypeIndex: double]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "predictions = pipelineModel.transform(test)\n",
    "display(predictions.select(\"CitationCount\", \"prediction\", *featuresCols))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 72,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "R2 on our test set: 0.466581\n"
     ]
    }
   ],
   "source": [
    "r2 = evaluator.evaluate(predictions)\n",
    "print(\"R2 on our test set: %g\" % r2)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 73,
   "metadata": {},
   "outputs": [],
   "source": [
    "paramGrid = ParamGridBuilder()\\\n",
    "  .addGrid(gbt.maxDepth, [2, 5])\\\n",
    "  .addGrid(gbt.maxIter, [10, 100])\\\n",
    "  .build()\n",
    "evaluator = RegressionEvaluator(metricName=\"rmse\", labelCol=gbt.getLabelCol(), predictionCol=gbt.getPredictionCol())\n",
    "cv = CrossValidator(estimator=gbt, evaluator=evaluator, estimatorParamMaps=paramGrid)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 74,
   "metadata": {},
   "outputs": [],
   "source": [
    "pipeline = Pipeline(stages=[vectorAssembler, vectorIndexer, cv])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 75,
   "metadata": {},
   "outputs": [],
   "source": [
    "pipelineModel = pipeline.fit(train)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 76,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "DataFrame[CitationCount: int, prediction: double, ConfInstanceID: bigint, ConfSeriesID: bigint, JournalID: bigint, PaperID: bigint, Paper_Rank: int, Paper_RefCnt: int, EstimatedCount: int, Journal_Rank: int, Journal_Pap_Cnt: int, Journal_Cit_Cnt: int, CSRank: int, CSPaperCount: int, CSCitation: int, CIRank: int, CIPaperCount: int, CIPaperCitationCnt: int, DocTypeIndex: double]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "predictions = pipelineModel.transform(test)\n",
    "display(predictions.select(\"CitationCount\", \"prediction\", *featuresCols))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 77,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "RMSE on our test set: 42.5799\n"
     ]
    }
   ],
   "source": [
    "rmse = evaluator.evaluate(predictions)\n",
    "print(\"RMSE on our test set: %g\" % rmse)"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 2",
   "language": "python",
   "name": "python2"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 2
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython2",
   "version": "2.7.15rc1"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}