map_k.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "#convert the data frame to lib ffm format\n",
    "def data_frame_to_lib_ffm(frame,key,field_list): #no le currenlty\n",
    "    new_frame=pd.DataFrame()\n",
    "    \n",
    "    for column_name in frame.columns:\n",
    "        if column in field_list:\n",
    "            if column_name!=key:\n",
    "                col=[str(field_list[column_name])+':'+str(row)+':1' for row in frame[column_name]]         \n",
    "            else:\n",
    "                col=frame[column_name]\n",
    "            new_frame[column_name]=col\n",
    "    return new_frame"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "import numpy as np \n",
    "import pandas as pd \n",
    "import time\n",
    "from util import fast_mapk\n",
    "from sklearn.preprocessing import LabelEncoder\n",
    "import time\n",
    "import pickle #load encoder"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "collapsed": false,
    "scrolled": false
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/ubuntu/.local/lib/python2.7/site-packages/IPython/core/interactiveshell.py:2717: DtypeWarning: Columns (4) have mixed types. Specify dtype option on import or set low_memory=False.\n",
      "  interactivity=interactivity, compiler=compiler, result=result)\n"
     ]
    }
   ],
   "source": [
    "#get data\n",
    "\n",
    "df_train = pd.read_csv( \"clicks_train.csv\") #100,000\n",
    "df_test = pd.read_csv( \"clicks_test.csv\") #100,000\n",
    "events = pd.read_csv( \"events.csv\") #100,000\n",
    "content=pd.read_csv('promoted_content.csv')\n",
    "docs_cats_frame=pd.read_csv('documents_categories.csv')\n",
    "docs_cats_frame=docs_cats_frame.rename(columns={'document_id':'document_id_y'})"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {
    "collapsed": false,
    "scrolled": true
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>ad_id</th>\n",
       "      <th>document_id</th>\n",
       "      <th>campaign_id</th>\n",
       "      <th>advertiser_id</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>1</td>\n",
       "      <td>6614</td>\n",
       "      <td>1</td>\n",
       "      <td>7</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>2</td>\n",
       "      <td>471467</td>\n",
       "      <td>2</td>\n",
       "      <td>7</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>3</td>\n",
       "      <td>7692</td>\n",
       "      <td>3</td>\n",
       "      <td>7</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>4</td>\n",
       "      <td>471471</td>\n",
       "      <td>2</td>\n",
       "      <td>7</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>5</td>\n",
       "      <td>471472</td>\n",
       "      <td>2</td>\n",
       "      <td>7</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   ad_id  document_id  campaign_id  advertiser_id\n",
       "0      1         6614            1              7\n",
       "1      2       471467            2              7\n",
       "2      3         7692            3              7\n",
       "3      4       471471            2              7\n",
       "4      5       471472            2              7"
      ]
     },
     "execution_count": 13,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "content.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {
    "collapsed": false,
    "scrolled": true
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "185709"
      ]
     },
     "execution_count": 20,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(content.document_id.unique())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>display_id</th>\n",
       "      <th>ad_id</th>\n",
       "      <th>clicked</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>1</td>\n",
       "      <td>42337</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>1</td>\n",
       "      <td>139684</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>1</td>\n",
       "      <td>144739</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>1</td>\n",
       "      <td>156824</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>1</td>\n",
       "      <td>279295</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   display_id   ad_id  clicked\n",
       "0           1   42337        0\n",
       "1           1  139684        0\n",
       "2           1  144739        1\n",
       "3           1  156824        0\n",
       "4           1  279295        0"
      ]
     },
     "execution_count": 21,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_train.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>display_id</th>\n",
       "      <th>ad_id</th>\n",
       "      <th>clicked</th>\n",
       "      <th>uuid</th>\n",
       "      <th>document_id_x</th>\n",
       "      <th>timestamp</th>\n",
       "      <th>platform</th>\n",
       "      <th>geo_location</th>\n",
       "      <th>document_id_y</th>\n",
       "      <th>campaign_id</th>\n",
       "      <th>advertiser_id</th>\n",
       "      <th>category_id</th>\n",
       "      <th>confidence_level</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>1</td>\n",
       "      <td>42337</td>\n",
       "      <td>0</td>\n",
       "      <td>cb8c55702adb93</td>\n",
       "      <td>379743</td>\n",
       "      <td>61</td>\n",
       "      <td>3</td>\n",
       "      <td>US&gt;SC&gt;519</td>\n",
       "      <td>938164</td>\n",
       "      <td>5969</td>\n",
       "      <td>1499</td>\n",
       "      <td>2100</td>\n",
       "      <td>0.92</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>1</td>\n",
       "      <td>42337</td>\n",
       "      <td>0</td>\n",
       "      <td>cb8c55702adb93</td>\n",
       "      <td>379743</td>\n",
       "      <td>61</td>\n",
       "      <td>3</td>\n",
       "      <td>US&gt;SC&gt;519</td>\n",
       "      <td>938164</td>\n",
       "      <td>5969</td>\n",
       "      <td>1499</td>\n",
       "      <td>1408</td>\n",
       "      <td>0.07</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>320369</td>\n",
       "      <td>42337</td>\n",
       "      <td>0</td>\n",
       "      <td>aeed1d38f8e3e3</td>\n",
       "      <td>969916</td>\n",
       "      <td>29787173</td>\n",
       "      <td>1</td>\n",
       "      <td>US&gt;IL&gt;602</td>\n",
       "      <td>938164</td>\n",
       "      <td>5969</td>\n",
       "      <td>1499</td>\n",
       "      <td>2100</td>\n",
       "      <td>0.92</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>320369</td>\n",
       "      <td>42337</td>\n",
       "      <td>0</td>\n",
       "      <td>aeed1d38f8e3e3</td>\n",
       "      <td>969916</td>\n",
       "      <td>29787173</td>\n",
       "      <td>1</td>\n",
       "      <td>US&gt;IL&gt;602</td>\n",
       "      <td>938164</td>\n",
       "      <td>5969</td>\n",
       "      <td>1499</td>\n",
       "      <td>1408</td>\n",
       "      <td>0.07</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>350833</td>\n",
       "      <td>42337</td>\n",
       "      <td>1</td>\n",
       "      <td>7482d935c4ec22</td>\n",
       "      <td>1628975</td>\n",
       "      <td>31407245</td>\n",
       "      <td>3</td>\n",
       "      <td>US&gt;NC&gt;550</td>\n",
       "      <td>938164</td>\n",
       "      <td>5969</td>\n",
       "      <td>1499</td>\n",
       "      <td>2100</td>\n",
       "      <td>0.92</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>350833</td>\n",
       "      <td>42337</td>\n",
       "      <td>1</td>\n",
       "      <td>7482d935c4ec22</td>\n",
       "      <td>1628975</td>\n",
       "      <td>31407245</td>\n",
       "      <td>3</td>\n",
       "      <td>US&gt;NC&gt;550</td>\n",
       "      <td>938164</td>\n",
       "      <td>5969</td>\n",
       "      <td>1499</td>\n",
       "      <td>1408</td>\n",
       "      <td>0.07</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6</th>\n",
       "      <td>367648</td>\n",
       "      <td>42337</td>\n",
       "      <td>1</td>\n",
       "      <td>84d858a3da5fc6</td>\n",
       "      <td>1628975</td>\n",
       "      <td>32253103</td>\n",
       "      <td>3</td>\n",
       "      <td>US&gt;PA&gt;566</td>\n",
       "      <td>938164</td>\n",
       "      <td>5969</td>\n",
       "      <td>1499</td>\n",
       "      <td>2100</td>\n",
       "      <td>0.92</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7</th>\n",
       "      <td>367648</td>\n",
       "      <td>42337</td>\n",
       "      <td>1</td>\n",
       "      <td>84d858a3da5fc6</td>\n",
       "      <td>1628975</td>\n",
       "      <td>32253103</td>\n",
       "      <td>3</td>\n",
       "      <td>US&gt;PA&gt;566</td>\n",
       "      <td>938164</td>\n",
       "      <td>5969</td>\n",
       "      <td>1499</td>\n",
       "      <td>1408</td>\n",
       "      <td>0.07</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8</th>\n",
       "      <td>393030</td>\n",
       "      <td>42337</td>\n",
       "      <td>0</td>\n",
       "      <td>440adb00eae522</td>\n",
       "      <td>1459603</td>\n",
       "      <td>33522620</td>\n",
       "      <td>1</td>\n",
       "      <td>US&gt;NJ&gt;501</td>\n",
       "      <td>938164</td>\n",
       "      <td>5969</td>\n",
       "      <td>1499</td>\n",
       "      <td>2100</td>\n",
       "      <td>0.92</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9</th>\n",
       "      <td>393030</td>\n",
       "      <td>42337</td>\n",
       "      <td>0</td>\n",
       "      <td>440adb00eae522</td>\n",
       "      <td>1459603</td>\n",
       "      <td>33522620</td>\n",
       "      <td>1</td>\n",
       "      <td>US&gt;NJ&gt;501</td>\n",
       "      <td>938164</td>\n",
       "      <td>5969</td>\n",
       "      <td>1499</td>\n",
       "      <td>1408</td>\n",
       "      <td>0.07</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   display_id  ad_id  clicked            uuid  document_id_x  timestamp  \\\n",
       "0           1  42337        0  cb8c55702adb93         379743         61   \n",
       "1           1  42337        0  cb8c55702adb93         379743         61   \n",
       "2      320369  42337        0  aeed1d38f8e3e3         969916   29787173   \n",
       "3      320369  42337        0  aeed1d38f8e3e3         969916   29787173   \n",
       "4      350833  42337        1  7482d935c4ec22        1628975   31407245   \n",
       "5      350833  42337        1  7482d935c4ec22        1628975   31407245   \n",
       "6      367648  42337        1  84d858a3da5fc6        1628975   32253103   \n",
       "7      367648  42337        1  84d858a3da5fc6        1628975   32253103   \n",
       "8      393030  42337        0  440adb00eae522        1459603   33522620   \n",
       "9      393030  42337        0  440adb00eae522        1459603   33522620   \n",
       "\n",
       "  platform geo_location  document_id_y  campaign_id  advertiser_id  \\\n",
       "0        3    US>SC>519         938164         5969           1499   \n",
       "1        3    US>SC>519         938164         5969           1499   \n",
       "2        1    US>IL>602         938164         5969           1499   \n",
       "3        1    US>IL>602         938164         5969           1499   \n",
       "4        3    US>NC>550         938164         5969           1499   \n",
       "5        3    US>NC>550         938164         5969           1499   \n",
       "6        3    US>PA>566         938164         5969           1499   \n",
       "7        3    US>PA>566         938164         5969           1499   \n",
       "8        1    US>NJ>501         938164         5969           1499   \n",
       "9        1    US>NJ>501         938164         5969           1499   \n",
       "\n",
       "   category_id  confidence_level  \n",
       "0         2100              0.92  \n",
       "1         1408              0.07  \n",
       "2         2100              0.92  \n",
       "3         1408              0.07  \n",
       "4         2100              0.92  \n",
       "5         1408              0.07  \n",
       "6         2100              0.92  \n",
       "7         1408              0.07  \n",
       "8         2100              0.92  \n",
       "9         1408              0.07  "
      ]
     },
     "execution_count": 25,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "joined_train_clicks.head(10)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(1, 2999286)"
      ]
     },
     "execution_count": 14,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "min(content['document_id']),max(content['document_id'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "26137112\n",
      "11.8732459545\n"
     ]
    }
   ],
   "source": [
    "#take fraction of the data:\n",
    "t=time.time()\n",
    "uniques=len(df_train.display_id.unique())\n",
    "msk=np.random.rand(uniques)<0.3 #fraction\n",
    "msk=df_train.display_id.unique()[msk]\n",
    "train = df_train[df_train.display_id.isin([msk])]\n",
    "print len(train)\n",
    "docs_cats_frame1=docs_cats_frame.drop_duplicates(subset='document_id_y',keep='first')#remove duplicates\n",
    "print time.time()-t"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {
    "collapsed": false,
    "scrolled": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0.473602056503\n",
      "7.77174019814\n",
      "8.81528401375\n",
      "20902930 66238801\n"
     ]
    }
   ],
   "source": [
    "#split train test by display id\n",
    "uniques=len(train.display_id.unique())\n",
    "t=time.time()\n",
    "msk = np.random.rand(uniques) < 0.8\n",
    "msk=train.display_id.unique()[msk]\n",
    "print time.time()-t\n",
    "t=time.time()\n",
    "train = df_train[df_train.display_id.isin([msk])]\n",
    "print time.time()-t\n",
    "t=time.time()\n",
    "val = df_train[~df_train.display_id.isin([msk])]\n",
    "print time.time()-t\n",
    "print len(train),len(val)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "21.3521990776\n",
      "7.27861499786\n",
      "61.9763579369\n"
     ]
    }
   ],
   "source": [
    "#join page views and events - training run\n",
    "\n",
    "t=time.time()\n",
    "joined_train_clicks=train.merge(events)\n",
    "joined_train_clicks=joined_train_clicks.merge(content,on='ad_id')\n",
    "print time.time()-t\n",
    "t=time.time()\n",
    "joined_train_clicks=joined_train_clicks.merge(docs_cats_frame,on='document_id_y')\n",
    "\n",
    "print time.time()-t\n",
    "t=time.time()\n",
    "joined_val_clicks=val.merge(events)\n",
    "joined_val_clicks=joined_val_clicks.merge(content,on='ad_id')\n",
    "joined_val_clicks=joined_val_clicks.merge(docs_cats_frame,on='document_id_y')\n",
    "print time.time()-t\n",
    "\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "joined_train_clicks=joined_train_clicks[~joined_train_clicks['uuid']\\\n",
    "                                        .isin(['5cda9845a1b9be', '7e8aa06b36db6a' ,'83e9ec48908c6a', 'f4bb634c3871b9','558ba104e8a37c'])]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "#remove some uids because they are not in label encoder :0\n",
    "\n",
    "\n",
    "joined_val_clicks=joined_val_clicks[~joined_val_clicks['uuid']\\\n",
    "                                        .isin(['5cda9845a1b9be', '7e8aa06b36db6a' ,'83e9ec48908c6a', 'f4bb634c3871b9','558ba104e8a37c'])]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "#set features\n",
    "ad_ids=joined_train_clicks['ad_id'].append(joined_val_clicks['ad_id'])\n",
    "uuids=joined_train_clicks['uuid'].append(joined_val_clicks['uuid'])\n",
    "dest_doc_id=content['document_id'].unique()\n",
    "docs_cats=docs_cats_frame['category_id'].unique()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 35,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "4.0621778965\n",
      "0.000227928161621\n"
     ]
    }
   ],
   "source": [
    "#precprocess: create label encoder for uuid and document id\n",
    "from sklearn.preprocessing import LabelEncoder\n",
    "'''\n",
    "le={}\n",
    "\n",
    "t=time.time()\n",
    "le['uuid']=LabelEncoder()\n",
    "le['uuid'].fit(uuids)\n",
    "print time.time()-t\n",
    "t=time.time()\n",
    "\n",
    "le['ad_id']=LabelEncoder()\n",
    "le['ad_id'].fit(ad_ids)\n",
    "print time.time()-t\n",
    "t=time.time()\n",
    "\n",
    "le['document_id']=LabelEncoder()  #off\n",
    "le['document_id'].fit(document_ids)\n",
    "print time.time()-t\n",
    "t=time.time()\n",
    "\n",
    "le['geo_location']=LabelEncoder() #off\n",
    "le['geo_location'].fit(geo_locations)\n",
    "\n",
    "print time.time()-t\n",
    "'''\n",
    "t=time.time()\n",
    "le['dest_doc_id']=LabelEncoder() #destinatiom document\n",
    "le['dest_doc_id'].fit(dest_doc_id)\n",
    "print time.time()-t\n",
    "\n",
    "t=time.time()\n",
    "le['docs_cats']=LabelEncoder() \n",
    "le['docs_cats'].fit(docs_cats)\n",
    "print time.time()-t\n",
    "\n",
    "\n",
    "\n",
    "import pickle #saved at /datadrive\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 36,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "\n",
    "with open('le.pickle', 'wb') as f:\n",
    "    pickle.dump(le, f, -1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 97,
   "metadata": {
    "collapsed": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "623.844252825\n",
      "244.84198308\n",
      "13.5361759663\n"
     ]
    },
    {
     "ename": "ValueError",
     "evalue": "y contains new labels: [   1456   20895   20967 ..., 2996109 2996871 2996938]",
     "output_type": "error",
     "traceback": [
      "\u001b[0;31m\u001b[0m",
      "\u001b[0;31mValueError\u001b[0mTraceback (most recent call last)",
      "\u001b[0;32m<ipython-input-97-9a4690defdc4>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m()\u001b[0m\n\u001b[1;32m     18\u001b[0m \u001b[0mt\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mtime\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mtime\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     19\u001b[0m \u001b[0mtrain_dest_docs\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mle\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;34m'dest_doc_id'\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mtransform\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mjoined_train_clicks\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;34m'document_id_y'\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m+\u001b[0m\u001b[0mcolumn_diffs\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;36m1\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 20\u001b[0;31m \u001b[0mtest_dest_docs\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mle\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;34m'dest_doc_id'\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mtransform\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mjoined_test_clicks\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;34m'document_id_y'\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m+\u001b[0m\u001b[0mcolumn_diffs\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;36m1\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m     21\u001b[0m \u001b[0mcolumn_diffs\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mappend\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mmax\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mnp\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mappend\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mtrain_dest_docs\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0mtest_dest_docs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     22\u001b[0m \u001b[0;32mprint\u001b[0m \u001b[0mtime\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mtime\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m-\u001b[0m\u001b[0mt\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/home/ubuntu/.local/lib/python2.7/site-packages/sklearn/preprocessing/label.pyc\u001b[0m in \u001b[0;36mtransform\u001b[0;34m(self, y)\u001b[0m\n\u001b[1;32m    151\u001b[0m         \u001b[0;32mif\u001b[0m \u001b[0mlen\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mnp\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mintersect1d\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mclasses\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mclasses_\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m)\u001b[0m \u001b[0;34m<\u001b[0m \u001b[0mlen\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mclasses\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    152\u001b[0m             \u001b[0mdiff\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mnp\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0msetdiff1d\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mclasses\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mclasses_\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 153\u001b[0;31m             \u001b[0;32mraise\u001b[0m \u001b[0mValueError\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m\"y contains new labels: %s\"\u001b[0m \u001b[0;34m%\u001b[0m \u001b[0mstr\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mdiff\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    154\u001b[0m         \u001b[0;32mreturn\u001b[0m \u001b[0mnp\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0msearchsorted\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mclasses_\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0my\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    155\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;31mValueError\u001b[0m: y contains new labels: [   1456   20895   20967 ..., 2996109 2996871 2996938]"
     ]
    }
   ],
   "source": [
    "#convert feautres to int - do once for test run, then write\n",
    "column_diffs=[]\n",
    "t=time.time()\n",
    "train_users=le['uuid'].transform(joined_train_clicks['uuid'])\n",
    "print time.time()-t\n",
    "t=time.time()\n",
    "test_users=le['uuid'].transform(joined_test_clicks['uuid'])\n",
    "column_diffs.append(max(np.append(train_users,test_users)))\n",
    "print time.time()-t\n",
    "t=time.time()\n",
    "\n",
    "test_ads=le['ad_id'].transform(joined_test_clicks['ad_id'])+column_diffs[0]\n",
    "train_ads=le['ad_id'].transform(joined_train_clicks['ad_id'])+column_diffs[0]\n",
    "column_diffs.append(max(np.append(test_ads,train_ads)))\n",
    "print time.time()-t\n",
    "t=time.time()\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 99,
   "metadata": {
    "collapsed": false,
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "12.946434021\n",
      "10.9347259998\n"
     ]
    }
   ],
   "source": [
    "t=time.time()\n",
    "train_dest_docs=le['dest_doc_id'].transform(joined_train_clicks['document_id_y'])+column_diffs[1]\n",
    "test_dest_docs=le['dest_doc_id'].transform(joined_test_clicks['document_id_y'])+column_diffs[1]\n",
    "column_diffs.append(max(np.append(train_dest_docs,test_dest_docs)))\n",
    "print time.time()-t\n",
    "t=time.time()\n",
    "\n",
    "t=time.time()\n",
    "train_categories=le['docs_cats'].transform(joined_train_clicks['category_id'])+column_diffs[2]\n",
    "test_categories=le['docs_cats'].transform(joined_test_clicks['category_id'])+column_diffs[2]\n",
    "column_diffs.append(max(np.append(train_dest_docs,test_dest_docs)))\n",
    "print time.time()-t\n",
    "t=time.time()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 100,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "full_train_features_dict={'train_users':train_users,'train_ads':train_ads,'train_dest_docs':train_dest_docs,'train_categories':train_categories}\n",
    "full_test_features_dict={'test_users':test_users,'test_ads':test_ads,'test_dest_docs':test_dest_docs,'test_categories':test_categories}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {
    "collapsed": false,
    "scrolled": true
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0    cb8c55702adb93\n",
       "1    cb8c55702adb93\n",
       "2    aeed1d38f8e3e3\n",
       "3    aeed1d38f8e3e3\n",
       "4    7482d935c4ec22\n",
       "5    7482d935c4ec22\n",
       "6    84d858a3da5fc6\n",
       "7    84d858a3da5fc6\n",
       "8    440adb00eae522\n",
       "9    440adb00eae522\n",
       "Name: uuid, dtype: object"
      ]
     },
     "execution_count": 28,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "joined_train_clicks['uuid'][:10]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 40,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([15471843, 15471843, 13110227, 13110227,  8288505,  8288505,\n",
       "        9636615,  9636615,  4292094,  4292094])"
      ]
     },
     "execution_count": 40,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "le['uuid'].transform(joined_train_clicks['uuid'][:10])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 42,
   "metadata": {
    "collapsed": false,
    "scrolled": true
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([19931290, 19931290, 19931290, 19931290, 19931290, 19931290,\n",
       "       19931290, 19931290, 19931290, 19931290, 19931290, 19931290,\n",
       "       19931290, 19931290, 19931290, 19931290, 19931290, 19931290,\n",
       "       19931290, 19931290, 19931290, 19931290, 19931290, 19931290,\n",
       "       19931290, 19931290, 19931290, 19931290, 19931290, 19931290,\n",
       "       19931290, 19931290, 19931290, 19931290, 19931290, 19931290,\n",
       "       19931290, 19931290, 19931290, 19931290, 19931290, 19931290,\n",
       "       19931290, 19931290, 19931290, 19931290, 19931290, 19931290,\n",
       "       19931290, 19931290, 19931290, 19931290, 19931290, 19931290,\n",
       "       19931290, 19931290, 19931290, 19931290, 19931290, 19931290,\n",
       "       19931290, 19931290, 19931290, 19931290, 19931290, 19931290,\n",
       "       19931290, 19931290, 19931290, 19931290, 19931290, 19931290,\n",
       "       19931290, 19931290, 19931290, 19931290, 19931290, 19931290,\n",
       "       19931290, 19931290, 19931290, 19931290, 19931290, 19931290,\n",
       "       19931290, 19931290, 19931290, 19931290, 19931290, 19931290,\n",
       "       19931290, 19931290, 19931290, 19931290, 19931290, 19931290,\n",
       "       19931290, 19931290, 19931290, 19931290])"
      ]
     },
     "execution_count": 42,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "le['ad_id'].transform(joined_train_clicks['ad_id'][12600:12700])+19794961"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 38,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(19794961, 19794960)"
      ]
     },
     "execution_count": 38,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "max(full_test_features_dict['test_users']),max(full_train_features_dict['train_users'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 38,
   "metadata": {
    "collapsed": true
   },
   "outputs": [
    {
     "ename": "KeyboardInterrupt",
     "evalue": "",
     "output_type": "error",
     "traceback": [
      "\u001b[0;31m\u001b[0m",
      "\u001b[0;31mKeyboardInterrupt\u001b[0mTraceback (most recent call last)",
      "\u001b[0;32m<ipython-input-38-9829c8d1dfad>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m()\u001b[0m\n\u001b[1;32m      3\u001b[0m \u001b[0mt\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mtime\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mtime\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m      4\u001b[0m \u001b[0mtrain_users\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mle\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;34m'uuid'\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mtransform\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mjoined_train_clicks\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;34m'uuid'\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m----> 5\u001b[0;31m \u001b[0mtest_users\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mle\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;34m'uuid'\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mtransform\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mjoined_val_clicks\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;34m'uuid'\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m      6\u001b[0m \u001b[0mcolumn_diffs\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mappend\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mmax\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mnp\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mappend\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mtrain_users\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0mtest_users\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m      7\u001b[0m \u001b[0;32mprint\u001b[0m \u001b[0mtime\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mtime\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m-\u001b[0m\u001b[0mt\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/home/ubuntu/.local/lib/python2.7/site-packages/sklearn/preprocessing/label.pyc\u001b[0m in \u001b[0;36mtransform\u001b[0;34m(self, y)\u001b[0m\n\u001b[1;32m    152\u001b[0m             \u001b[0mdiff\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mnp\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0msetdiff1d\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mclasses\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mclasses_\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    153\u001b[0m             \u001b[0;32mraise\u001b[0m \u001b[0mValueError\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m\"y contains new labels: %s\"\u001b[0m \u001b[0;34m%\u001b[0m \u001b[0mstr\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mdiff\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 154\u001b[0;31m         \u001b[0;32mreturn\u001b[0m \u001b[0mnp\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0msearchsorted\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mclasses_\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0my\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    155\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    156\u001b[0m     \u001b[0;32mdef\u001b[0m \u001b[0minverse_transform\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0my\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/home/ubuntu/.local/lib/python2.7/site-packages/numpy/core/fromnumeric.pyc\u001b[0m in \u001b[0;36msearchsorted\u001b[0;34m(a, v, side, sorter)\u001b[0m\n\u001b[1;32m   1091\u001b[0m     \u001b[0;32mexcept\u001b[0m \u001b[0mAttributeError\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1092\u001b[0m         \u001b[0;32mreturn\u001b[0m \u001b[0m_wrapit\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0ma\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m'searchsorted'\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mv\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mside\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0msorter\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m-> 1093\u001b[0;31m     \u001b[0;32mreturn\u001b[0m \u001b[0msearchsorted\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mv\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mside\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0msorter\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m   1094\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1095\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;31mKeyboardInterrupt\u001b[0m: "
     ]
    }
   ],
   "source": [
    "#convert feautres to int\n",
    "column_diffs=[]\n",
    "t=time.time()\n",
    "train_users=le['uuid'].transform(joined_train_clicks['uuid'])\n",
    "test_users=le['uuid'].transform(joined_val_clicks['uuid'])\n",
    "column_diffs.append(max(np.append(train_users,test_users)))\n",
    "print time.time()-t\n",
    "t=time.time()\n",
    "\n",
    "test_ads=le['ad_id'].transform(joined_val_clicks['ad_id'])+column_diffs[0]\n",
    "train_ads=le['ad_id'].transform(joined_train_clicks['ad_id'])+column_diffs[0]\n",
    "column_diffs.append(max(np.append(test_ads,train_ads)))\n",
    "print time.time()-t\n",
    "t=time.time()\n",
    "\n",
    "t=time.time()\n",
    "train_dest_docs=le['dest_doc_id'].transform(joined_train_clicks['document_id_y'])+column_diffs[1]\n",
    "test_dest_docs=le['dest_doc_id'].transform(joined_val_clicks['document_id_y'])+column_diffs[1]\n",
    "column_diffs.append(max(np.append(train_dest_docs,test_dest_docs)))\n",
    "print time.time()-t\n",
    "t=time.time()\n",
    "\n",
    "t=time.time()\n",
    "train_categories=le['docs_cats'].transform(joined_train_clicks['category_id'])+column_diffs[2]\n",
    "test_categories=le['docs_cats'].transform(joined_val_clicks['category_id'])+column_diffs[2]\n",
    "column_diffs.append(max(np.append(train_dest_docs,test_dest_docs)))\n",
    "print time.time()-t\n",
    "t=time.time()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 101,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "#write the docs with pickle\n",
    "with open('full_train_features_dict.pickle', 'wb') as f:\n",
    "    pickle.dump(full_train_features_dict, f, -1)\n",
    "    \n",
    "with open('full_test_features_dict.pickle', 'wb') as f:\n",
    "    pickle.dump(full_test_features_dict, f, -1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "32192851"
      ]
     },
     "execution_count": 33,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "#this will change the previous data flow... make it better\n",
    "\n",
    "pkl_file = open('/datadrive/le.pickle', 'rb')\n",
    "le=pickle.load(pkl_file)\n",
    "\n",
    "pkl_file = open('/datadrive/full_train_features_dict.pickle', 'rb')\n",
    "full_train_features_dict=pickle.load(pkl_file)\n",
    "\n",
    "pkl_file = open('/datadrive/full_test_features_dict.pickle', 'rb')\n",
    "full_test_features_dict=pickle.load(pkl_file)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 67,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "32225162"
      ]
     },
     "execution_count": 67,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(df_test)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 63,
   "metadata": {
    "collapsed": false,
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "1.27139687538\n"
     ]
    },
    {
     "ename": "ValueError",
     "evalue": "array length 63922585 does not match index length 103540364",
     "output_type": "error",
     "traceback": [
      "\u001b[0;31m\u001b[0m",
      "\u001b[0;31mValueError\u001b[0mTraceback (most recent call last)",
      "\u001b[0;32m<ipython-input-63-a69af60dafee>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m()\u001b[0m\n\u001b[1;32m      4\u001b[0m \u001b[0;32mprint\u001b[0m \u001b[0mtime\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mtime\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m-\u001b[0m\u001b[0mt\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m      5\u001b[0m \u001b[0mt\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mtime\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mtime\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m----> 6\u001b[0;31m \u001b[0mfinal_val_frame\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mpd\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mDataFrame\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m{\u001b[0m\u001b[0;34m'uuid'\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0mtest_users\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m'ad_id'\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0mtest_ads\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m'clicked'\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0mjoined_val_clicks\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;34m'clicked'\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mastype\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mint\u001b[0m\u001b[0;34m)\u001b[0m                             \u001b[0;34m,\u001b[0m\u001b[0;34m'dest_doc_id'\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0mtest_dest_docs\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m'categories'\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0mtest_categories\u001b[0m\u001b[0;34m}\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m      7\u001b[0m \u001b[0;32mprint\u001b[0m \u001b[0mtime\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mtime\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m-\u001b[0m\u001b[0mt\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/home/ubuntu/.local/lib/python2.7/site-packages/pandas/core/frame.pyc\u001b[0m in \u001b[0;36m__init__\u001b[0;34m(self, data, index, columns, dtype, copy)\u001b[0m\n\u001b[1;32m    264\u001b[0m                                  dtype=dtype, copy=copy)\n\u001b[1;32m    265\u001b[0m         \u001b[0;32melif\u001b[0m \u001b[0misinstance\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mdata\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mdict\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 266\u001b[0;31m             \u001b[0mmgr\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_init_dict\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mdata\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mindex\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mcolumns\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mdtype\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mdtype\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    267\u001b[0m         \u001b[0;32melif\u001b[0m \u001b[0misinstance\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mdata\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mma\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mMaskedArray\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    268\u001b[0m             \u001b[0;32mimport\u001b[0m \u001b[0mnumpy\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mma\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mmrecords\u001b[0m \u001b[0;32mas\u001b[0m \u001b[0mmrecords\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/home/ubuntu/.local/lib/python2.7/site-packages/pandas/core/frame.pyc\u001b[0m in \u001b[0;36m_init_dict\u001b[0;34m(self, data, index, columns, dtype)\u001b[0m\n\u001b[1;32m    400\u001b[0m             \u001b[0marrays\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0;34m[\u001b[0m\u001b[0mdata\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0mk\u001b[0m\u001b[0;34m]\u001b[0m \u001b[0;32mfor\u001b[0m \u001b[0mk\u001b[0m \u001b[0;32min\u001b[0m \u001b[0mkeys\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    401\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 402\u001b[0;31m         \u001b[0;32mreturn\u001b[0m \u001b[0m_arrays_to_mgr\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0marrays\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mdata_names\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mindex\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mcolumns\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mdtype\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mdtype\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    403\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    404\u001b[0m     \u001b[0;32mdef\u001b[0m \u001b[0m_init_ndarray\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mvalues\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mindex\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mcolumns\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mdtype\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mNone\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mcopy\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mFalse\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/home/ubuntu/.local/lib/python2.7/site-packages/pandas/core/frame.pyc\u001b[0m in \u001b[0;36m_arrays_to_mgr\u001b[0;34m(arrays, arr_names, index, columns, dtype)\u001b[0m\n\u001b[1;32m   5396\u001b[0m     \u001b[0;31m# figure out the index, if necessary\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   5397\u001b[0m     \u001b[0;32mif\u001b[0m \u001b[0mindex\u001b[0m \u001b[0;32mis\u001b[0m \u001b[0mNone\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m-> 5398\u001b[0;31m         \u001b[0mindex\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mextract_index\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0marrays\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m   5399\u001b[0m     \u001b[0;32melse\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   5400\u001b[0m         \u001b[0mindex\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0m_ensure_index\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mindex\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/home/ubuntu/.local/lib/python2.7/site-packages/pandas/core/frame.pyc\u001b[0m in \u001b[0;36mextract_index\u001b[0;34m(data)\u001b[0m\n\u001b[1;32m   5454\u001b[0m                     msg = ('array length %d does not match index length %d' %\n\u001b[1;32m   5455\u001b[0m                            (lengths[0], len(index)))\n\u001b[0;32m-> 5456\u001b[0;31m                     \u001b[0;32mraise\u001b[0m \u001b[0mValueError\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mmsg\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m   5457\u001b[0m             \u001b[0;32melse\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   5458\u001b[0m                 \u001b[0mindex\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0m_default_index\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mlengths\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;36m0\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;31mValueError\u001b[0m: array length 63922585 does not match index length 103540364"
     ]
    }
   ],
   "source": [
    "#generate final data frames\n",
    "t=time.time()\n",
    "final_frame=pd.DataFrame({'uuid':train_users,'ad_id':train_ads,'clicked':joined_train_clicks['clicked'].astype(int)\\\n",
    "                         ,'dest_doc_id':train_dest_docs,'categories':train_categories})\n",
    "print time.time()-t\n",
    "t=time.time()\n",
    "final_val_frame=pd.DataFrame({'uuid':test_users,'ad_id':test_ads,'clicked':joined_val_clicks['clicked'].astype(int)\\\n",
    "                             ,'dest_doc_id':test_dest_docs,'categories':test_categories})\n",
    "print time.time()-t"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "#convert to libffm\n",
    "    #the slowest part - can make faster w/ apply?\n",
    "t=time.time()\n",
    "lib_train_frame=data_frame_to_lib_ffm(final_frame,'clicked',{'clicked':-1, 'uuid':0, 'ad_id':1,'dest_doc_id':2})\n",
    "print time.time()-t\n",
    "t=time.time()\n",
    "lib_val_frame=data_frame_to_lib_ffm(final_val_frame,'clicked',{'clicked':-1, 'uuid':0, 'ad_id':1,'dest_doc_id':2})\n",
    "print time.time()-t\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false,
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "#write to csv\n",
    "t=time.time()\n",
    "lib_train_frame[['clicked','uuid','ad_id','dest_doc_id']].to_csv('train_frame.csv',sep=' ',header=None,index=False)\n",
    "print time.time()-t\n",
    "t=time.time()\n",
    "lib_val_frame[['clicked','uuid','ad_id','dest_doc_id']].to_csv('val_frame.csv',sep=' ',header=None,index=False)\n",
    "print time.time()-t"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false,
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "#read output\n",
    "output_list=pd.read_csv('/datadrive/output',header=None)\n",
    "\n",
    "print len(output_list),len(lib_val_frame)\n",
    "\n",
    "test_set=joined_val_clicks[['display_id','ad_id','clicked']]#\n",
    "test_set['prediction']=output_list[0]\n",
    "#join to test\n",
    "\n",
    "# sort test set\n",
    "#test_set=test_set.sort(['display_id','prediction'],ascending=[True,False])\n",
    "test_set['prediction'].fillna(0)\n",
    "test_set=test_set.rename(columns = {'prediction':'likelihood'})\n",
    "test_set.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false,
    "scrolled": false
   },
   "outputs": [],
   "source": [
    "#test_set['p_ctr']=joined_val_clicks['likelihood']\n",
    "test_set['likelihood']=(joined_val_clicks['likelihood']+test_set['likelihood'])/2\n",
    "test_set.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false,
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "#calc mapk on validation set\n",
    "util.fast_mapk(test_set) #columns: display id, likelihood,clicked"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#add feauture\n",
    "#1 join to data\n",
    "#2 convert to int\n",
    "#3 convert to libffm"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 2",
   "language": "python",
   "name": "python2"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 2
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython2",
   "version": "2.7.12+"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}