W2V.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "import json\n",
    "from pyspark.mllib.feature import Word2Vec\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "with open('/home/han/.api_key/awsaccesskey.json') as f:\n",
    "    key= json.load(f)\n",
    "    \n",
    "    access= key['access-key']\n",
    "    secret = key['secret-access-key']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "from pyspark import SparkContext"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "sc = SparkContext()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# test"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {
    "collapsed": false,
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "data = sc.textFile('s3n://'+access+':'+secret +'@han.tweets.bucket/tweet/*/*/*.bz2')\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {
    "collapsed": false,
    "scrolled": true
   },
   "outputs": [
    {
     "ename": "Py4JJavaError",
     "evalue": "An error occurred while calling o17.partitions.\n: org.apache.hadoop.mapred.InvalidInputException: Input Pattern s3n://AKIAJAARWQ57D4HCBHRQ:7xdG1e4BPcEmX4EfyGIjMiSJjxyKvnSXnWGYNALm@han.tweets.bucket/tweet/*/*/*.bz2 matches 0 files\n\tat org.apache.hadoop.mapred.FileInputFormat.listStatus(FileInputFormat.java:197)\n\tat org.apache.hadoop.mapred.FileInputFormat.getSplits(FileInputFormat.java:208)\n\tat org.apache.spark.rdd.HadoopRDD.getPartitions(HadoopRDD.scala:199)\n\tat org.apache.spark.rdd.RDD$$anonfun$partitions$2.apply(RDD.scala:239)\n\tat org.apache.spark.rdd.RDD$$anonfun$partitions$2.apply(RDD.scala:237)\n\tat scala.Option.getOrElse(Option.scala:120)\n\tat org.apache.spark.rdd.RDD.partitions(RDD.scala:237)\n\tat org.apache.spark.rdd.MapPartitionsRDD.getPartitions(MapPartitionsRDD.scala:35)\n\tat org.apache.spark.rdd.RDD$$anonfun$partitions$2.apply(RDD.scala:239)\n\tat org.apache.spark.rdd.RDD$$anonfun$partitions$2.apply(RDD.scala:237)\n\tat scala.Option.getOrElse(Option.scala:120)\n\tat org.apache.spark.rdd.RDD.partitions(RDD.scala:237)\n\tat org.apache.spark.api.java.JavaRDDLike$class.partitions(JavaRDDLike.scala:64)\n\tat org.apache.spark.api.java.AbstractJavaRDDLike.partitions(JavaRDDLike.scala:46)\n\tat sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)\n\tat sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)\n\tat sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)\n\tat java.lang.reflect.Method.invoke(Method.java:497)\n\tat py4j.reflection.MethodInvoker.invoke(MethodInvoker.java:231)\n\tat py4j.reflection.ReflectionEngine.invoke(ReflectionEngine.java:381)\n\tat py4j.Gateway.invoke(Gateway.java:259)\n\tat py4j.commands.AbstractCommand.invokeMethod(AbstractCommand.java:133)\n\tat py4j.commands.CallCommand.execute(CallCommand.java:79)\n\tat py4j.GatewayConnection.run(GatewayConnection.java:209)\n\tat java.lang.Thread.run(Thread.java:745)\n",
     "output_type": "error",
     "traceback": [
      "\u001b[1;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[1;31mPy4JJavaError\u001b[0m                             Traceback (most recent call last)",
      "\u001b[1;32m<ipython-input-9-2924773d61c1>\u001b[0m in \u001b[0;36m<module>\u001b[1;34m()\u001b[0m\n\u001b[1;32m----> 1\u001b[1;33m \u001b[0mdata\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mtake\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;36m10\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m",
      "\u001b[1;32m/usr/local/spark/python/pyspark/rdd.pyc\u001b[0m in \u001b[0;36mtake\u001b[1;34m(self, num)\u001b[0m\n\u001b[0;32m   1265\u001b[0m         \"\"\"\n\u001b[0;32m   1266\u001b[0m         \u001b[0mitems\u001b[0m \u001b[1;33m=\u001b[0m \u001b[1;33m[\u001b[0m\u001b[1;33m]\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m-> 1267\u001b[1;33m         \u001b[0mtotalParts\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mgetNumPartitions\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m   1268\u001b[0m         \u001b[0mpartsScanned\u001b[0m \u001b[1;33m=\u001b[0m \u001b[1;36m0\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m   1269\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32m/usr/local/spark/python/pyspark/rdd.pyc\u001b[0m in \u001b[0;36mgetNumPartitions\u001b[1;34m(self)\u001b[0m\n\u001b[0;32m    354\u001b[0m         \u001b[1;36m2\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    355\u001b[0m         \"\"\"\n\u001b[1;32m--> 356\u001b[1;33m         \u001b[1;32mreturn\u001b[0m \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0m_jrdd\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mpartitions\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0msize\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m    357\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    358\u001b[0m     \u001b[1;32mdef\u001b[0m \u001b[0mfilter\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mself\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mf\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32m/home/han/anaconda2/lib/python2.7/site-packages/py4j/java_gateway.pyc\u001b[0m in \u001b[0;36m__call__\u001b[1;34m(self, *args)\u001b[0m\n\u001b[0;32m    833\u001b[0m         \u001b[0manswer\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mgateway_client\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0msend_command\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mcommand\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    834\u001b[0m         return_value = get_return_value(\n\u001b[1;32m--> 835\u001b[1;33m             answer, self.gateway_client, self.target_id, self.name)\n\u001b[0m\u001b[0;32m    836\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    837\u001b[0m         \u001b[1;32mfor\u001b[0m \u001b[0mtemp_arg\u001b[0m \u001b[1;32min\u001b[0m \u001b[0mtemp_args\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32m/home/han/anaconda2/lib/python2.7/site-packages/py4j/protocol.pyc\u001b[0m in \u001b[0;36mget_return_value\u001b[1;34m(answer, gateway_client, target_id, name)\u001b[0m\n\u001b[0;32m    308\u001b[0m                 raise Py4JJavaError(\n\u001b[0;32m    309\u001b[0m                     \u001b[1;34m\"An error occurred while calling {0}{1}{2}.\\n\"\u001b[0m\u001b[1;33m.\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m--> 310\u001b[1;33m                     format(target_id, \".\", name), value)\n\u001b[0m\u001b[0;32m    311\u001b[0m             \u001b[1;32melse\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    312\u001b[0m                 raise Py4JError(\n",
      "\u001b[1;31mPy4JJavaError\u001b[0m: An error occurred while calling o17.partitions.\n: org.apache.hadoop.mapred.InvalidInputException: Input Pattern s3n://AKIAJAARWQ57D4HCBHRQ:7xdG1e4BPcEmX4EfyGIjMiSJjxyKvnSXnWGYNALm@han.tweets.bucket/tweet/*/*/*.bz2 matches 0 files\n\tat org.apache.hadoop.mapred.FileInputFormat.listStatus(FileInputFormat.java:197)\n\tat org.apache.hadoop.mapred.FileInputFormat.getSplits(FileInputFormat.java:208)\n\tat org.apache.spark.rdd.HadoopRDD.getPartitions(HadoopRDD.scala:199)\n\tat org.apache.spark.rdd.RDD$$anonfun$partitions$2.apply(RDD.scala:239)\n\tat org.apache.spark.rdd.RDD$$anonfun$partitions$2.apply(RDD.scala:237)\n\tat scala.Option.getOrElse(Option.scala:120)\n\tat org.apache.spark.rdd.RDD.partitions(RDD.scala:237)\n\tat org.apache.spark.rdd.MapPartitionsRDD.getPartitions(MapPartitionsRDD.scala:35)\n\tat org.apache.spark.rdd.RDD$$anonfun$partitions$2.apply(RDD.scala:239)\n\tat org.apache.spark.rdd.RDD$$anonfun$partitions$2.apply(RDD.scala:237)\n\tat scala.Option.getOrElse(Option.scala:120)\n\tat org.apache.spark.rdd.RDD.partitions(RDD.scala:237)\n\tat org.apache.spark.api.java.JavaRDDLike$class.partitions(JavaRDDLike.scala:64)\n\tat org.apache.spark.api.java.AbstractJavaRDDLike.partitions(JavaRDDLike.scala:46)\n\tat sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)\n\tat sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)\n\tat sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)\n\tat java.lang.reflect.Method.invoke(Method.java:497)\n\tat py4j.reflection.MethodInvoker.invoke(MethodInvoker.java:231)\n\tat py4j.reflection.ReflectionEngine.invoke(ReflectionEngine.java:381)\n\tat py4j.Gateway.invoke(Gateway.java:259)\n\tat py4j.commands.AbstractCommand.invokeMethod(AbstractCommand.java:133)\n\tat py4j.commands.CallCommand.execute(CallCommand.java:79)\n\tat py4j.GatewayConnection.run(GatewayConnection.java:209)\n\tat java.lang.Thread.run(Thread.java:745)\n"
     ]
    }
   ],
   "source": [
    "data.take(10)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "word2vec = Word2Vec()\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "word_rdd = data.map(lambda x: x.split())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {
    "collapsed": false,
    "scrolled": true
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<pyspark.mllib.feature.Word2VecModel at 0x7f968a65fa50>"
      ]
     },
     "execution_count": 17,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "word2vec.fit(word_rdd)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "2"
      ]
     },
     "execution_count": 10,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "data.getNumPartitions()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "w2v_mod = _17"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {
    "collapsed": true
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[(u'mythology', 1.6330972853653365),\n",
       " (u'ancient', 1.6254558742548857),\n",
       " (u'latin', 1.5719785017791681),\n",
       " (u'koine', 1.5699189217969287),\n",
       " (u'phoenician', 1.5505149126688398),\n",
       " (u'sanskrit', 1.494935579959364),\n",
       " (u'akkadian', 1.4940735645984307),\n",
       " (u'constantinopolis', 1.4848666144920364),\n",
       " (u'persian', 1.4798156868382155),\n",
       " (u'macedonian', 1.4750030810843235)]"
      ]
     },
     "execution_count": 27,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "w2v_mod.findSynonyms('greek', 10)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# W2V tweet"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "from pyspark import SparkContext\n",
    "import json\n",
    "import nltk\n",
    "import re\n",
    "from collections import Counter, defaultdict"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "REGEX = u\"[\\U00002712\\U00002714\\U00002716\\U0000271d\\U00002721\\U00002728\\U00002733\\U00002734\\U00002744\\U00002747\\U0000274c\\U0000274e\\U00002753-\\U00002755\\U00002757\\U00002763\\U00002764\\U00002795-\\U00002797\\U000027a1\\U000027b0\\U000027bf\\U00002934\\U00002935\\U00002b05-\\U00002b07\\U00002b1b\\U00002b1c\\U00002b50\\U00002b55\\U00003030\\U0000303d\\U0001f004\\U0001f0cf\\U0001f170\\U0001f171\\U0001f17e\\U0001f17f\\U0001f18e\\U0001f191-\\U0001f19a\\U0001f201\\U0001f202\\U0001f21a\\U0001f22f\\U0001f232-\\U0001f23a\\U0001f250\\U0001f251\\U0001f300-\\U0001f321\\U0001f324-\\U0001f393\\U0001f396\\U0001f397\\U0001f399-\\U0001f39b\\U0001f39e-\\U0001f3f0\\U0001f3f3-\\U0001f3f5\\U0001f3f7-\\U0001f4fd\\U0001f4ff-\\U0001f53d\\U0001f549-\\U0001f54e\\U0001f550-\\U0001f567\\U0001f56f\\U0001f570\\U0001f573-\\U0001f579\\U0001f587\\U0001f58a-\\U0001f58d\\U0001f590\\U0001f595\\U0001f596\\U0001f5a5\\U0001f5a8\\U0001f5b1\\U0001f5b2\\U0001f5bc\\U0001f5c2-\\U0001f5c4\\U0001f5d1-\\U0001f5d3\\U0001f5dc-\\U0001f5de\\U0001f5e1\\U0001f5e3\\U0001f5ef\\U0001f5f3\\U0001f5fa-\\U0001f64f\\U0001f680-\\U0001f6c5\\U0001f6cb-\\U0001f6d0\\U0001f6e0-\\U0001f6e5\\U0001f6e9\\U0001f6eb\\U0001f6ec\\U0001f6f0\\U0001f6f3\\U0001f910-\\U0001f918\\U0001f980-\\U0001f984\\U0001f9c0\\U00003297\\U00003299\\U000000a9\\U000000ae\\U0000203c\\U00002049\\U00002122\\U00002139\\U00002194-\\U00002199\\U000021a9\\U000021aa\\U0000231a\\U0000231b\\U00002328\\U00002388\\U000023cf\\U000023e9-\\U000023f3\\U000023f8-\\U000023fa\\U000024c2\\U000025aa\\U000025ab\\U000025b6\\U000025c0\\U000025fb-\\U000025fe\\U00002600-\\U00002604\\U0000260e\\U00002611\\U00002614\\U00002615\\U00002618\\U0000261d\\U00002620\\U00002622\\U00002623\\U00002626\\U0000262a\\U0000262e\\U0000262f\\U00002638-\\U0000263a\\U00002648-\\U00002653\\U00002660\\U00002663\\U00002665\\U00002666\\U00002668\\U0000267b\\U0000267f\\U00002692-\\U00002694\\U00002696\\U00002697\\U00002699\\U0000269b\\U0000269c\\U000026a0\\U000026a1\\U000026aa\\U000026ab\\U000026b0\\U000026b1\\U000026bd\\U000026be\\U000026c4\\U000026c5\\U000026c8\\U000026ce\\U000026cf\\U000026d1\\U000026d3\\U000026d4\\U000026e9\\U000026ea\\U000026f0-\\U000026f5\\U000026f7-\\U000026fa\\U000026fd\\U00002702\\U00002705\\U00002708-\\U0000270d\\U0000270f]|[#]\\U000020e3|[*]\\U000020e3|[0]\\U000020e3|[1]\\U000020e3|[2]\\U000020e3|[3]\\U000020e3|[4]\\U000020e3|[5]\\U000020e3|[6]\\U000020e3|[7]\\U000020e3|[8]\\U000020e3|[9]\\U000020e3|\\U0001f1e6[\\U0001f1e8-\\U0001f1ec\\U0001f1ee\\U0001f1f1\\U0001f1f2\\U0001f1f4\\U0001f1f6-\\U0001f1fa\\U0001f1fc\\U0001f1fd\\U0001f1ff]|\\U0001f1e7[\\U0001f1e6\\U0001f1e7\\U0001f1e9-\\U0001f1ef\\U0001f1f1-\\U0001f1f4\\U0001f1f6-\\U0001f1f9\\U0001f1fb\\U0001f1fc\\U0001f1fe\\U0001f1ff]|\\U0001f1e8[\\U0001f1e6\\U0001f1e8\\U0001f1e9\\U0001f1eb-\\U0001f1ee\\U0001f1f0-\\U0001f1f5\\U0001f1f7\\U0001f1fa-\\U0001f1ff]|\\U0001f1e9[\\U0001f1ea\\U0001f1ec\\U0001f1ef\\U0001f1f0\\U0001f1f2\\U0001f1f4\\U0001f1ff]|\\U0001f1ea[\\U0001f1e6\\U0001f1e8\\U0001f1ea\\U0001f1ec\\U0001f1ed\\U0001f1f7-\\U0001f1fa]|\\U0001f1eb[\\U0001f1ee-\\U0001f1f0\\U0001f1f2\\U0001f1f4\\U0001f1f7]|\\U0001f1ec[\\U0001f1e6\\U0001f1e7\\U0001f1e9-\\U0001f1ee\\U0001f1f1-\\U0001f1f3\\U0001f1f5-\\U0001f1fa\\U0001f1fc\\U0001f1fe]|\\U0001f1ed[\\U0001f1f0\\U0001f1f2\\U0001f1f3\\U0001f1f7\\U0001f1f9\\U0001f1fa]|\\U0001f1ee[\\U0001f1e8-\\U0001f1ea\\U0001f1f1-\\U0001f1f4\\U0001f1f6-\\U0001f1f9]|\\U0001f1ef[\\U0001f1ea\\U0001f1f2\\U0001f1f4\\U0001f1f5]|\\U0001f1f0[\\U0001f1ea\\U0001f1ec-\\U0001f1ee\\U0001f1f2\\U0001f1f3\\U0001f1f5\\U0001f1f7\\U0001f1fc\\U0001f1fe\\U0001f1ff]|\\U0001f1f1[\\U0001f1e6-\\U0001f1e8\\U0001f1ee\\U0001f1f0\\U0001f1f7-\\U0001f1fb\\U0001f1fe]|\\U0001f1f2[\\U0001f1e6\\U0001f1e8-\\U0001f1ed\\U0001f1f0-\\U0001f1ff]|\\U0001f1f3[\\U0001f1e6\\U0001f1e8\\U0001f1ea-\\U0001f1ec\\U0001f1ee\\U0001f1f1\\U0001f1f4\\U0001f1f5\\U0001f1f7\\U0001f1fa\\U0001f1ff]|\\U0001f1f4\\U0001f1f2|\\U0001f1f5[\\U0001f1e6\\U0001f1ea-\\U0001f1ed\\U0001f1f0-\\U0001f1f3\\U0001f1f7-\\U0001f1f9\\U0001f1fc\\U0001f1fe]|\\U0001f1f6\\U0001f1e6|\\U0001f1f7[\\U0001f1ea\\U0001f1f4\\U0001f1f8\\U0001f1fa\\U0001f1fc]|\\U0001f1f8[\\U0001f1e6-\\U0001f1ea\\U0001f1ec-\\U0001f1f4\\U0001f1f7-\\U0001f1f9\\U0001f1fb\\U0001f1fd-\\U0001f1ff]|\\U0001f1f9[\\U0001f1e6\\U0001f1e8\\U0001f1e9\\U0001f1eb-\\U0001f1ed\\U0001f1ef-\\U0001f1f4\\U0001f1f7\\U0001f1f9\\U0001f1fb\\U0001f1fc\\U0001f1ff]|\\U0001f1fa[\\U0001f1e6\\U0001f1ec\\U0001f1f2\\U0001f1f8\\U0001f1fe\\U0001f1ff]|\\U0001f1fb[\\U0001f1e6\\U0001f1e8\\U0001f1ea\\U0001f1ec\\U0001f1ee\\U0001f1f3\\U0001f1fa]|\\U0001f1fc[\\U0001f1eb\\U0001f1f8]|\\U0001f1fd\\U0001f1f0|\\U0001f1fe[\\U0001f1ea\\U0001f1f9]|\\U0001f1ff[\\U0001f1e6\\U0001f1f2\\U0001f1fc]|[0-9*#]\\ufe0f\\u20e3\"\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "def tweet_process(tweet):\n",
    "    KEY = 'text'\n",
    "    try:\n",
    "        tw = json.loads(tweet.strip())\n",
    "        if KEY not in tw or tw['lang']!= 'en':\n",
    "            return None\n",
    "        return tw\n",
    "\n",
    "    except Exception as e:\n",
    "        return None\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "\n",
    "def emoji_preprocess(tweet):\n",
    "    # add space before and after space\n",
    "    for emoji in re.findall(REGEX, tweet):\n",
    "        tweet = tweet.replace(emoji, ' ' + emoji + ' ')\n",
    "\n",
    "    # tokenize and remove rt and @ and https://\n",
    "\n",
    "    tweet_token = ['<s>'] + [ wd for wd in tweet.strip('rt').split() if not wd.startswith('@') and not wd.startswith('http') and not wd.startswith('#') ] + ['</s>']\n",
    "\n",
    "    return tweet_token\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "tweets =  sc.textFile('data/twitter_dump.txt')\\\n",
    ".filter(lambda tw: len(tw)>1)\\\n",
    ".filter(lambda tw: 'created_at' in tw)\\\n",
    ".map(tweet_process)\\\n",
    ".filter(lambda tw: tw != None)\\\n",
    ".map(lambda tw: tw['text'].lower() )\\\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "tweets_tokens = tweets.map(emoji_preprocess)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "tweet_w2v = word2vec.fit(tweets_tokens)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "import cPickle as pickle"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 42,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "ename": "TypeError",
     "evalue": "can't pickle lock objects",
     "output_type": "error",
     "traceback": [
      "\u001b[1;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[1;31mTypeError\u001b[0m                                 Traceback (most recent call last)",
      "\u001b[1;32m<ipython-input-42-9f6f15ed37cd>\u001b[0m in \u001b[0;36m<module>\u001b[1;34m()\u001b[0m\n\u001b[0;32m      1\u001b[0m \u001b[1;32mwith\u001b[0m \u001b[0mopen\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;34m'tweet_w2v.pkl'\u001b[0m\u001b[1;33m,\u001b[0m \u001b[1;34m'wb'\u001b[0m\u001b[1;33m)\u001b[0m \u001b[1;32mas\u001b[0m \u001b[0mf\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m----> 2\u001b[1;33m     \u001b[0mpickle\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mdump\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mtweet_w2v\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mf\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m",
      "\u001b[1;32m/home/han/anaconda2/lib/python2.7/copy_reg.pyc\u001b[0m in \u001b[0;36m_reduce_ex\u001b[1;34m(self, proto)\u001b[0m\n\u001b[0;32m     68\u001b[0m     \u001b[1;32melse\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m     69\u001b[0m         \u001b[1;32mif\u001b[0m \u001b[0mbase\u001b[0m \u001b[1;32mis\u001b[0m \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0m__class__\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m---> 70\u001b[1;33m             \u001b[1;32mraise\u001b[0m \u001b[0mTypeError\u001b[0m\u001b[1;33m,\u001b[0m \u001b[1;34m\"can't pickle %s objects\"\u001b[0m \u001b[1;33m%\u001b[0m \u001b[0mbase\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0m__name__\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m     71\u001b[0m         \u001b[0mstate\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mbase\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mself\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m     72\u001b[0m     \u001b[0margs\u001b[0m \u001b[1;33m=\u001b[0m \u001b[1;33m(\u001b[0m\u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0m__class__\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mbase\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mstate\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;31mTypeError\u001b[0m: can't pickle lock objects"
     ]
    }
   ],
   "source": [
    "with open('tweet_w2v.pkl', 'wb') as f:\n",
    "    pickle.dump(tweet_w2v, f)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 73,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "should\n",
      "must\n",
      "would\n",
      "i'll\n",
      "won't\n",
      "you'll\n",
      "may\n",
      "wouldn't\n",
      "shall\n",
      "it'll\n",
      "we'll\n",
      "shouldn't\n",
      "can\n",
      "gotta\n",
      "let's\n"
     ]
    }
   ],
   "source": [
    "for w,s in tweet_w2v.findSynonyms('will', 15):\n",
    "    print w"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 2",
   "language": "python",
   "name": "python2"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 2
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython2",
   "version": "2.7.11"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 0
}