Variable Selection using Python - Vote based approach.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# 1. Introduction"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "The document will walk you through the codes that I used to select variables using python. We use different techniques to select variables and then finally each technique votes whether they have selected the variable. Finally, the vote is counted and the variables with higher votes are used in the modeling process."
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# 2. Load Dataset"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "#import packages\n",
    "import os\n",
    "import pandas as pd\n",
    "import numpy as np\n",
    "\n",
    "df = pd.read_excel('/Users/mbagav200/Desktop/Medium/bank.xlsx') #Read excel file"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "<class 'pandas.core.frame.DataFrame'>\n",
      "RangeIndex: 4521 entries, 0 to 4520\n",
      "Data columns (total 17 columns):\n",
      "age          4521 non-null int64\n",
      "job          4521 non-null object\n",
      "marital      4521 non-null object\n",
      "education    4521 non-null object\n",
      "default      4521 non-null object\n",
      "balance      4521 non-null int64\n",
      "housing      4521 non-null object\n",
      "loan         4521 non-null object\n",
      "contact      4521 non-null object\n",
      "day          4521 non-null int64\n",
      "month        4521 non-null object\n",
      "duration     4521 non-null int64\n",
      "campaign     4521 non-null int64\n",
      "pdays        4521 non-null int64\n",
      "previous     4521 non-null int64\n",
      "poutcome     4521 non-null object\n",
      "y            4521 non-null object\n",
      "dtypes: int64(7), object(10)\n",
      "memory usage: 600.5+ KB\n"
     ]
    }
   ],
   "source": [
    "df.info()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style>\n",
       "    .dataframe thead tr:only-child th {\n",
       "        text-align: right;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: left;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>age</th>\n",
       "      <th>job</th>\n",
       "      <th>marital</th>\n",
       "      <th>education</th>\n",
       "      <th>default</th>\n",
       "      <th>balance</th>\n",
       "      <th>housing</th>\n",
       "      <th>loan</th>\n",
       "      <th>contact</th>\n",
       "      <th>day</th>\n",
       "      <th>month</th>\n",
       "      <th>duration</th>\n",
       "      <th>campaign</th>\n",
       "      <th>pdays</th>\n",
       "      <th>previous</th>\n",
       "      <th>poutcome</th>\n",
       "      <th>y</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>30</td>\n",
       "      <td>unemployed</td>\n",
       "      <td>married</td>\n",
       "      <td>primary</td>\n",
       "      <td>no</td>\n",
       "      <td>1787</td>\n",
       "      <td>no</td>\n",
       "      <td>no</td>\n",
       "      <td>cellular</td>\n",
       "      <td>19</td>\n",
       "      <td>oct</td>\n",
       "      <td>79</td>\n",
       "      <td>1</td>\n",
       "      <td>-1</td>\n",
       "      <td>0</td>\n",
       "      <td>unknown</td>\n",
       "      <td>no</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>33</td>\n",
       "      <td>services</td>\n",
       "      <td>married</td>\n",
       "      <td>secondary</td>\n",
       "      <td>no</td>\n",
       "      <td>4789</td>\n",
       "      <td>yes</td>\n",
       "      <td>yes</td>\n",
       "      <td>cellular</td>\n",
       "      <td>11</td>\n",
       "      <td>may</td>\n",
       "      <td>220</td>\n",
       "      <td>1</td>\n",
       "      <td>339</td>\n",
       "      <td>4</td>\n",
       "      <td>failure</td>\n",
       "      <td>no</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>35</td>\n",
       "      <td>management</td>\n",
       "      <td>single</td>\n",
       "      <td>tertiary</td>\n",
       "      <td>no</td>\n",
       "      <td>1350</td>\n",
       "      <td>yes</td>\n",
       "      <td>no</td>\n",
       "      <td>cellular</td>\n",
       "      <td>16</td>\n",
       "      <td>apr</td>\n",
       "      <td>185</td>\n",
       "      <td>1</td>\n",
       "      <td>330</td>\n",
       "      <td>1</td>\n",
       "      <td>failure</td>\n",
       "      <td>no</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>30</td>\n",
       "      <td>management</td>\n",
       "      <td>married</td>\n",
       "      <td>tertiary</td>\n",
       "      <td>no</td>\n",
       "      <td>1476</td>\n",
       "      <td>yes</td>\n",
       "      <td>yes</td>\n",
       "      <td>unknown</td>\n",
       "      <td>3</td>\n",
       "      <td>jun</td>\n",
       "      <td>199</td>\n",
       "      <td>4</td>\n",
       "      <td>-1</td>\n",
       "      <td>0</td>\n",
       "      <td>unknown</td>\n",
       "      <td>no</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>59</td>\n",
       "      <td>blue-collar</td>\n",
       "      <td>married</td>\n",
       "      <td>secondary</td>\n",
       "      <td>no</td>\n",
       "      <td>0</td>\n",
       "      <td>yes</td>\n",
       "      <td>no</td>\n",
       "      <td>unknown</td>\n",
       "      <td>5</td>\n",
       "      <td>may</td>\n",
       "      <td>226</td>\n",
       "      <td>1</td>\n",
       "      <td>-1</td>\n",
       "      <td>0</td>\n",
       "      <td>unknown</td>\n",
       "      <td>no</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   age          job  marital  education default  balance housing loan  \\\n",
       "0   30   unemployed  married    primary      no     1787      no   no   \n",
       "1   33     services  married  secondary      no     4789     yes  yes   \n",
       "2   35   management   single   tertiary      no     1350     yes   no   \n",
       "3   30   management  married   tertiary      no     1476     yes  yes   \n",
       "4   59  blue-collar  married  secondary      no        0     yes   no   \n",
       "\n",
       "    contact  day month  duration  campaign  pdays  previous poutcome   y  \n",
       "0  cellular   19   oct        79         1     -1         0  unknown  no  \n",
       "1  cellular   11   may       220         1    339         4  failure  no  \n",
       "2  cellular   16   apr       185         1    330         1  failure  no  \n",
       "3   unknown    3   jun       199         4     -1         0  unknown  no  \n",
       "4   unknown    5   may       226         1     -1         0  unknown  no  "
      ]
     },
     "execution_count": 3,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "df['target'] = df['y'].apply(lambda x : 1 if x == 'yes' else 0)  # Convert to numeric\n",
    "df = df.drop('y',axis=1) #drop the y column"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0    4000\n",
       "1     521\n",
       "Name: target, dtype: int64"
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df['target'].value_counts()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "I am going to use a quick way to encode the class variables (Label Encoder). Since some of the variables in the train dataframe are categories"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# Data transformation\n",
    "# Convert categorical values to numeric using label encoder\n",
    "from sklearn import preprocessing\n",
    "from collections import defaultdict\n",
    "d = defaultdict(preprocessing.LabelEncoder)\n",
    "# Encoding the categorical variable\n",
    "fit = df.select_dtypes(include=['object']).apply(lambda x: d[x.name].fit_transform(x))\n",
    "\n",
    "#Convert the categorical columns based on encoding\n",
    "for i in list(d.keys()):\n",
    "    df[i] = d[i].transform(df[i])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "features = df[df.columns.difference(['target'])]\n",
    "labels = df['target']"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# 3. Variable Selection"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 3.1 Weight of Evidence and Information Value"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# import packages\n",
    "import pandas as pd\n",
    "import numpy as np\n",
    "import pandas.core.algorithms as algos\n",
    "from pandas import Series\n",
    "import scipy.stats.stats as stats\n",
    "import re\n",
    "import traceback\n",
    "import string\n",
    "\n",
    "max_bin = 20\n",
    "force_bin = 3\n",
    "\n",
    "# define a binning function\n",
    "def mono_bin(Y, X, n = max_bin):\n",
    "    \n",
    "    df1 = pd.DataFrame({\"X\": X, \"Y\": Y})\n",
    "    justmiss = df1[['X','Y']][df1.X.isnull()]\n",
    "    notmiss = df1[['X','Y']][df1.X.notnull()]\n",
    "    r = 0\n",
    "    while np.abs(r) < 1:\n",
    "        try:\n",
    "            d1 = pd.DataFrame({\"X\": notmiss.X, \"Y\": notmiss.Y, \"Bucket\": pd.qcut(notmiss.X, n)})\n",
    "            d2 = d1.groupby('Bucket', as_index=True)\n",
    "            r, p = stats.spearmanr(d2.mean().X, d2.mean().Y)\n",
    "            n = n - 1 \n",
    "        except Exception as e:\n",
    "            n = n - 1\n",
    "\n",
    "    if len(d2) == 1:\n",
    "        n = force_bin         \n",
    "        bins = algos.quantile(notmiss.X, np.linspace(0, 1, n))\n",
    "        if len(np.unique(bins)) == 2:\n",
    "            bins = np.insert(bins, 0, 1)\n",
    "            bins[1] = bins[1]-(bins[1]/2)\n",
    "        d1 = pd.DataFrame({\"X\": notmiss.X, \"Y\": notmiss.Y, \"Bucket\": pd.cut(notmiss.X, np.unique(bins),include_lowest=True)}) \n",
    "        d2 = d1.groupby('Bucket', as_index=True)\n",
    "    \n",
    "    d3 = pd.DataFrame({},index=[])\n",
    "    d3[\"MIN_VALUE\"] = d2.min().X\n",
    "    d3[\"MAX_VALUE\"] = d2.max().X\n",
    "    d3[\"COUNT\"] = d2.count().Y\n",
    "    d3[\"EVENT\"] = d2.sum().Y\n",
    "    d3[\"NONEVENT\"] = d2.count().Y - d2.sum().Y\n",
    "    d3=d3.reset_index(drop=True)\n",
    "    \n",
    "    if len(justmiss.index) > 0:\n",
    "        d4 = pd.DataFrame({'MIN_VALUE':np.nan},index=[0])\n",
    "        d4[\"MAX_VALUE\"] = np.nan\n",
    "        d4[\"COUNT\"] = justmiss.count().Y\n",
    "        d4[\"EVENT\"] = justmiss.sum().Y\n",
    "        d4[\"NONEVENT\"] = justmiss.count().Y - justmiss.sum().Y\n",
    "        d3 = d3.append(d4,ignore_index=True)\n",
    "    \n",
    "    d3[\"EVENT_RATE\"] = d3.EVENT/d3.COUNT\n",
    "    d3[\"NON_EVENT_RATE\"] = d3.NONEVENT/d3.COUNT\n",
    "    d3[\"DIST_EVENT\"] = d3.EVENT/d3.sum().EVENT\n",
    "    d3[\"DIST_NON_EVENT\"] = d3.NONEVENT/d3.sum().NONEVENT\n",
    "    d3[\"WOE\"] = np.log(d3.DIST_EVENT/d3.DIST_NON_EVENT)\n",
    "    d3[\"IV\"] = (d3.DIST_EVENT-d3.DIST_NON_EVENT)*np.log(d3.DIST_EVENT/d3.DIST_NON_EVENT)\n",
    "    d3[\"VAR_NAME\"] = \"VAR\"\n",
    "    d3 = d3[['VAR_NAME','MIN_VALUE', 'MAX_VALUE', 'COUNT', 'EVENT', 'EVENT_RATE', 'NONEVENT', 'NON_EVENT_RATE', 'DIST_EVENT','DIST_NON_EVENT','WOE', 'IV']]       \n",
    "    d3 = d3.replace([np.inf, -np.inf], 0)\n",
    "    d3.IV = d3.IV.sum()\n",
    "    \n",
    "    return(d3)\n",
    "\n",
    "def char_bin(Y, X):\n",
    "        \n",
    "    df1 = pd.DataFrame({\"X\": X, \"Y\": Y})\n",
    "    justmiss = df1[['X','Y']][df1.X.isnull()]\n",
    "    notmiss = df1[['X','Y']][df1.X.notnull()]    \n",
    "    df2 = notmiss.groupby('X',as_index=True)\n",
    "    \n",
    "    d3 = pd.DataFrame({},index=[])\n",
    "    d3[\"COUNT\"] = df2.count().Y\n",
    "    d3[\"MIN_VALUE\"] = df2.sum().Y.index\n",
    "    d3[\"MAX_VALUE\"] = d3[\"MIN_VALUE\"]\n",
    "    d3[\"EVENT\"] = df2.sum().Y\n",
    "    d3[\"NONEVENT\"] = df2.count().Y - df2.sum().Y\n",
    "    \n",
    "    if len(justmiss.index) > 0:\n",
    "        d4 = pd.DataFrame({'MIN_VALUE':np.nan},index=[0])\n",
    "        d4[\"MAX_VALUE\"] = np.nan\n",
    "        d4[\"COUNT\"] = justmiss.count().Y\n",
    "        d4[\"EVENT\"] = justmiss.sum().Y\n",
    "        d4[\"NONEVENT\"] = justmiss.count().Y - justmiss.sum().Y\n",
    "        d3 = d3.append(d4,ignore_index=True)\n",
    "    \n",
    "    d3[\"EVENT_RATE\"] = d3.EVENT/d3.COUNT\n",
    "    d3[\"NON_EVENT_RATE\"] = d3.NONEVENT/d3.COUNT\n",
    "    d3[\"DIST_EVENT\"] = d3.EVENT/d3.sum().EVENT\n",
    "    d3[\"DIST_NON_EVENT\"] = d3.NONEVENT/d3.sum().NONEVENT\n",
    "    d3[\"WOE\"] = np.log(d3.DIST_EVENT/d3.DIST_NON_EVENT)\n",
    "    d3[\"IV\"] = (d3.DIST_EVENT-d3.DIST_NON_EVENT)*np.log(d3.DIST_EVENT/d3.DIST_NON_EVENT)\n",
    "    d3[\"VAR_NAME\"] = \"VAR\"\n",
    "    d3 = d3[['VAR_NAME','MIN_VALUE', 'MAX_VALUE', 'COUNT', 'EVENT', 'EVENT_RATE', 'NONEVENT', 'NON_EVENT_RATE', 'DIST_EVENT','DIST_NON_EVENT','WOE', 'IV']]      \n",
    "    d3 = d3.replace([np.inf, -np.inf], 0)\n",
    "    d3.IV = d3.IV.sum()\n",
    "    d3 = d3.reset_index(drop=True)\n",
    "    \n",
    "    return(d3)\n",
    "\n",
    "def data_vars(df1, target):\n",
    "    \n",
    "    stack = traceback.extract_stack()\n",
    "    filename, lineno, function_name, code = stack[-2]\n",
    "    vars_name = re.compile(r'\\((.*?)\\).*$').search(code).groups()[0]\n",
    "    final = (re.findall(r\"[\\w']+\", vars_name))[-1]\n",
    "    \n",
    "    x = df1.dtypes.index\n",
    "    count = -1\n",
    "    \n",
    "    for i in x:\n",
    "        if i.upper() not in (final.upper()):\n",
    "            if np.issubdtype(df1[i], np.number) and len(Series.unique(df1[i])) > 2:\n",
    "                conv = mono_bin(target, df1[i])\n",
    "                conv[\"VAR_NAME\"] = i\n",
    "                count = count + 1\n",
    "            else:\n",
    "                conv = char_bin(target, df1[i])\n",
    "                conv[\"VAR_NAME\"] = i            \n",
    "                count = count + 1\n",
    "                \n",
    "            if count == 0:\n",
    "                iv_df = conv\n",
    "            else:\n",
    "                iv_df = iv_df.append(conv,ignore_index=True)\n",
    "    \n",
    "    iv = pd.DataFrame({'IV':iv_df.groupby('VAR_NAME').IV.max()})\n",
    "    iv = iv.reset_index()\n",
    "    return(iv_df,iv) "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {
    "collapsed": true,
    "scrolled": false
   },
   "outputs": [],
   "source": [
    "final_iv, IV = data_vars(df,df.target)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "IV = IV.rename(columns={'VAR_NAME':'index'})"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style>\n",
       "    .dataframe thead tr:only-child th {\n",
       "        text-align: right;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: left;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>index</th>\n",
       "      <th>IV</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>6</th>\n",
       "      <td>duration</td>\n",
       "      <td>1.651501</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>contact</td>\n",
       "      <td>0.247734</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>13</th>\n",
       "      <td>pdays</td>\n",
       "      <td>0.203267</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>15</th>\n",
       "      <td>previous</td>\n",
       "      <td>0.177081</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8</th>\n",
       "      <td>housing</td>\n",
       "      <td>0.106556</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>balance</td>\n",
       "      <td>0.076208</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>10</th>\n",
       "      <td>loan</td>\n",
       "      <td>0.060791</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7</th>\n",
       "      <td>education</td>\n",
       "      <td>0.024649</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>campaign</td>\n",
       "      <td>0.023342</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>11</th>\n",
       "      <td>marital</td>\n",
       "      <td>0.019556</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>14</th>\n",
       "      <td>poutcome</td>\n",
       "      <td>0.015968</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9</th>\n",
       "      <td>job</td>\n",
       "      <td>0.008522</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>12</th>\n",
       "      <td>month</td>\n",
       "      <td>0.007580</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>day</td>\n",
       "      <td>0.004581</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>age</td>\n",
       "      <td>0.000452</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>default</td>\n",
       "      <td>0.000016</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "        index        IV\n",
       "6    duration  1.651501\n",
       "3     contact  0.247734\n",
       "13      pdays  0.203267\n",
       "15   previous  0.177081\n",
       "8     housing  0.106556\n",
       "1     balance  0.076208\n",
       "10       loan  0.060791\n",
       "7   education  0.024649\n",
       "2    campaign  0.023342\n",
       "11    marital  0.019556\n",
       "14   poutcome  0.015968\n",
       "9         job  0.008522\n",
       "12      month  0.007580\n",
       "4         day  0.004581\n",
       "0         age  0.000452\n",
       "5     default  0.000016"
      ]
     },
     "execution_count": 11,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "IV.sort_values(['IV'],ascending=0)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 3.2 Variable Importance"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0.9915947799159478\n"
     ]
    }
   ],
   "source": [
    "from sklearn.ensemble import RandomForestClassifier\n",
    "clf = RandomForestClassifier()\n",
    "\n",
    "clf.fit(features,labels)\n",
    "\n",
    "preds = clf.predict(features)\n",
    "\n",
    "from sklearn.metrics import accuracy_score\n",
    "accuracy = accuracy_score(preds,labels)\n",
    "print(accuracy)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "from pandas import DataFrame\n",
    "VI = DataFrame(clf.feature_importances_, columns = [\"RF\"], index=features.columns)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "VI = VI.reset_index()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style>\n",
       "    .dataframe thead tr:only-child th {\n",
       "        text-align: right;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: left;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>index</th>\n",
       "      <th>RF</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>6</th>\n",
       "      <td>duration</td>\n",
       "      <td>0.300387</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>balance</td>\n",
       "      <td>0.109106</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>age</td>\n",
       "      <td>0.099911</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>day</td>\n",
       "      <td>0.081872</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>12</th>\n",
       "      <td>month</td>\n",
       "      <td>0.081078</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>13</th>\n",
       "      <td>pdays</td>\n",
       "      <td>0.053539</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9</th>\n",
       "      <td>job</td>\n",
       "      <td>0.046605</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>14</th>\n",
       "      <td>poutcome</td>\n",
       "      <td>0.042045</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>campaign</td>\n",
       "      <td>0.041005</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7</th>\n",
       "      <td>education</td>\n",
       "      <td>0.032687</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>11</th>\n",
       "      <td>marital</td>\n",
       "      <td>0.029996</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>15</th>\n",
       "      <td>previous</td>\n",
       "      <td>0.025654</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8</th>\n",
       "      <td>housing</td>\n",
       "      <td>0.022490</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>contact</td>\n",
       "      <td>0.021245</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>10</th>\n",
       "      <td>loan</td>\n",
       "      <td>0.009428</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>default</td>\n",
       "      <td>0.002952</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "        index        RF\n",
       "6    duration  0.300387\n",
       "1     balance  0.109106\n",
       "0         age  0.099911\n",
       "4         day  0.081872\n",
       "12      month  0.081078\n",
       "13      pdays  0.053539\n",
       "9         job  0.046605\n",
       "14   poutcome  0.042045\n",
       "2    campaign  0.041005\n",
       "7   education  0.032687\n",
       "11    marital  0.029996\n",
       "15   previous  0.025654\n",
       "8     housing  0.022490\n",
       "3     contact  0.021245\n",
       "10       loan  0.009428\n",
       "5     default  0.002952"
      ]
     },
     "execution_count": 15,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "VI.sort_values(['RF'],ascending=0)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "collapsed": true
   },
   "source": [
    "## 3.3 Recursive Feature Elimination"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "from sklearn.feature_selection import RFE\n",
    "from sklearn.linear_model import LogisticRegression\n",
    "\n",
    "model = LogisticRegression()\n",
    "rfe = RFE(model, 20)\n",
    "fit = rfe.fit(features, labels)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "from pandas import DataFrame\n",
    "Selected = DataFrame(rfe.support_, columns = [\"RFE\"], index=features.columns)\n",
    "Selected = Selected.reset_index()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style>\n",
       "    .dataframe thead tr:only-child th {\n",
       "        text-align: right;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: left;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>index</th>\n",
       "      <th>RFE</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>age</td>\n",
       "      <td>True</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>balance</td>\n",
       "      <td>True</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>campaign</td>\n",
       "      <td>True</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>contact</td>\n",
       "      <td>True</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>day</td>\n",
       "      <td>True</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>default</td>\n",
       "      <td>True</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6</th>\n",
       "      <td>duration</td>\n",
       "      <td>True</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7</th>\n",
       "      <td>education</td>\n",
       "      <td>True</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8</th>\n",
       "      <td>housing</td>\n",
       "      <td>True</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9</th>\n",
       "      <td>job</td>\n",
       "      <td>True</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>10</th>\n",
       "      <td>loan</td>\n",
       "      <td>True</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>11</th>\n",
       "      <td>marital</td>\n",
       "      <td>True</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>12</th>\n",
       "      <td>month</td>\n",
       "      <td>True</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>13</th>\n",
       "      <td>pdays</td>\n",
       "      <td>True</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>14</th>\n",
       "      <td>poutcome</td>\n",
       "      <td>True</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>15</th>\n",
       "      <td>previous</td>\n",
       "      <td>True</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "        index   RFE\n",
       "0         age  True\n",
       "1     balance  True\n",
       "2    campaign  True\n",
       "3     contact  True\n",
       "4         day  True\n",
       "5     default  True\n",
       "6    duration  True\n",
       "7   education  True\n",
       "8     housing  True\n",
       "9         job  True\n",
       "10       loan  True\n",
       "11    marital  True\n",
       "12      month  True\n",
       "13      pdays  True\n",
       "14   poutcome  True\n",
       "15   previous  True"
      ]
     },
     "execution_count": 18,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "Selected[Selected['RFE'] == True]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 3.4 Variable Importance - Extratrees Classifier"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[0.09056561 0.08782475 0.0542887  0.02927988 0.08496376 0.00523697\n",
      " 0.24361645 0.04293258 0.0258565  0.06350107 0.01501788 0.03663597\n",
      " 0.09415388 0.04237129 0.05210472 0.03165   ]\n"
     ]
    }
   ],
   "source": [
    "from sklearn.ensemble import ExtraTreesClassifier\n",
    "\n",
    "model = ExtraTreesClassifier()\n",
    "model.fit(features, labels)\n",
    "\n",
    "print(model.feature_importances_)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "from pandas import DataFrame\n",
    "FI = DataFrame(model.feature_importances_, columns = [\"Extratrees\"], index=features.columns)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "FI = FI.reset_index()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style>\n",
       "    .dataframe thead tr:only-child th {\n",
       "        text-align: right;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: left;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>index</th>\n",
       "      <th>Extratrees</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>6</th>\n",
       "      <td>duration</td>\n",
       "      <td>0.243616</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>12</th>\n",
       "      <td>month</td>\n",
       "      <td>0.094154</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>age</td>\n",
       "      <td>0.090566</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>balance</td>\n",
       "      <td>0.087825</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>day</td>\n",
       "      <td>0.084964</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9</th>\n",
       "      <td>job</td>\n",
       "      <td>0.063501</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>campaign</td>\n",
       "      <td>0.054289</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>14</th>\n",
       "      <td>poutcome</td>\n",
       "      <td>0.052105</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7</th>\n",
       "      <td>education</td>\n",
       "      <td>0.042933</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>13</th>\n",
       "      <td>pdays</td>\n",
       "      <td>0.042371</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>11</th>\n",
       "      <td>marital</td>\n",
       "      <td>0.036636</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>15</th>\n",
       "      <td>previous</td>\n",
       "      <td>0.031650</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>contact</td>\n",
       "      <td>0.029280</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8</th>\n",
       "      <td>housing</td>\n",
       "      <td>0.025857</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>10</th>\n",
       "      <td>loan</td>\n",
       "      <td>0.015018</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>default</td>\n",
       "      <td>0.005237</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "        index  Extratrees\n",
       "6    duration    0.243616\n",
       "12      month    0.094154\n",
       "0         age    0.090566\n",
       "1     balance    0.087825\n",
       "4         day    0.084964\n",
       "9         job    0.063501\n",
       "2    campaign    0.054289\n",
       "14   poutcome    0.052105\n",
       "7   education    0.042933\n",
       "13      pdays    0.042371\n",
       "11    marital    0.036636\n",
       "15   previous    0.031650\n",
       "3     contact    0.029280\n",
       "8     housing    0.025857\n",
       "10       loan    0.015018\n",
       "5     default    0.005237"
      ]
     },
     "execution_count": 22,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "FI.sort_values(['Extratrees'],ascending=0)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 3.5 Chi Square"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "df1 = df.copy()\n",
    "\n",
    "from sklearn import preprocessing\n",
    "from collections import defaultdict\n",
    "d = defaultdict(preprocessing.LabelEncoder)\n",
    "# Encoding the categorical variable\n",
    "fit = df1.apply(lambda x: d[x.name].fit_transform(x))\n",
    "\n",
    "#Convert the categorical columns based on encoding\n",
    "for i in list(d.keys()):\n",
    "    df1[i] = d[i].transform(df1[i])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "features1 = df1[df1.columns.difference(['target'])]\n",
    "labels1 = df1['target']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "from sklearn.feature_selection import SelectKBest\n",
    "from sklearn.feature_selection import chi2\n",
    "\n",
    "model = SelectKBest(score_func=chi2, k=5)\n",
    "fit = model.fit(features1, labels1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[    45.84063034  11343.97706276     88.07290162    100.51007877\n",
      "      2.60631502      0.00754274 119483.9344331       3.80267508\n",
      "     21.50080127      8.15459672     19.04525811      0.32039715\n",
      "     12.32541175   5374.89961779     11.86870754    330.26034213]\n"
     ]
    }
   ],
   "source": [
    "import numpy\n",
    "numpy.set_printoptions(suppress=True)\n",
    "print(fit.scores_)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "from pandas import DataFrame\n",
    "pd.options.display.float_format = '{:.2f}'.format\n",
    "chi_sq = DataFrame(fit.scores_, columns = [\"Chi_Square\"], index=features1.columns)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "chi_sq = chi_sq.reset_index()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style>\n",
       "    .dataframe thead tr:only-child th {\n",
       "        text-align: right;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: left;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>index</th>\n",
       "      <th>Chi_Square</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>6</th>\n",
       "      <td>duration</td>\n",
       "      <td>119483.93</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>balance</td>\n",
       "      <td>11343.98</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>13</th>\n",
       "      <td>pdays</td>\n",
       "      <td>5374.90</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>15</th>\n",
       "      <td>previous</td>\n",
       "      <td>330.26</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>contact</td>\n",
       "      <td>100.51</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>campaign</td>\n",
       "      <td>88.07</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>age</td>\n",
       "      <td>45.84</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8</th>\n",
       "      <td>housing</td>\n",
       "      <td>21.50</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>10</th>\n",
       "      <td>loan</td>\n",
       "      <td>19.05</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>12</th>\n",
       "      <td>month</td>\n",
       "      <td>12.33</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>14</th>\n",
       "      <td>poutcome</td>\n",
       "      <td>11.87</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9</th>\n",
       "      <td>job</td>\n",
       "      <td>8.15</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7</th>\n",
       "      <td>education</td>\n",
       "      <td>3.80</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>day</td>\n",
       "      <td>2.61</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>11</th>\n",
       "      <td>marital</td>\n",
       "      <td>0.32</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>default</td>\n",
       "      <td>0.01</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "        index  Chi_Square\n",
       "6    duration   119483.93\n",
       "1     balance    11343.98\n",
       "13      pdays     5374.90\n",
       "15   previous      330.26\n",
       "3     contact      100.51\n",
       "2    campaign       88.07\n",
       "0         age       45.84\n",
       "8     housing       21.50\n",
       "10       loan       19.05\n",
       "12      month       12.33\n",
       "14   poutcome       11.87\n",
       "9         job        8.15\n",
       "7   education        3.80\n",
       "4         day        2.61\n",
       "11    marital        0.32\n",
       "5     default        0.01"
      ]
     },
     "execution_count": 29,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "chi_sq.sort_values('Chi_Square',ascending=0)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 3.6 L1 feature selection"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "from sklearn.svm import LinearSVC\n",
    "from sklearn.feature_selection import SelectFromModel"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "lsvc = LinearSVC(C=0.01, penalty=\"l1\", dual=False).fit(features, labels)\n",
    "model = SelectFromModel(lsvc,prefit=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "from pandas import DataFrame\n",
    "l1 = DataFrame(model.get_support(), columns = [\"L1\"], index=features.columns)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "l1 = l1.reset_index()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 34,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style>\n",
       "    .dataframe thead tr:only-child th {\n",
       "        text-align: right;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: left;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>index</th>\n",
       "      <th>L1</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>age</td>\n",
       "      <td>True</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>campaign</td>\n",
       "      <td>True</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>contact</td>\n",
       "      <td>True</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>day</td>\n",
       "      <td>True</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6</th>\n",
       "      <td>duration</td>\n",
       "      <td>True</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8</th>\n",
       "      <td>housing</td>\n",
       "      <td>True</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9</th>\n",
       "      <td>job</td>\n",
       "      <td>True</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>10</th>\n",
       "      <td>loan</td>\n",
       "      <td>True</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>11</th>\n",
       "      <td>marital</td>\n",
       "      <td>True</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>12</th>\n",
       "      <td>month</td>\n",
       "      <td>True</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>13</th>\n",
       "      <td>pdays</td>\n",
       "      <td>True</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>14</th>\n",
       "      <td>poutcome</td>\n",
       "      <td>True</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>15</th>\n",
       "      <td>previous</td>\n",
       "      <td>True</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "       index    L1\n",
       "0        age  True\n",
       "2   campaign  True\n",
       "3    contact  True\n",
       "4        day  True\n",
       "6   duration  True\n",
       "8    housing  True\n",
       "9        job  True\n",
       "10      loan  True\n",
       "11   marital  True\n",
       "12     month  True\n",
       "13     pdays  True\n",
       "14  poutcome  True\n",
       "15  previous  True"
      ]
     },
     "execution_count": 34,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "l1[l1['L1'] == True]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "collapsed": true
   },
   "source": [
    "## 3.7 Combine all together"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 35,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "from functools import reduce\n",
    "dfs = [IV, VI, Selected, FI, chi_sq, l1]\n",
    "final_results = reduce(lambda left,right: pd.merge(left,right,on='index'), dfs)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 36,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style>\n",
       "    .dataframe thead tr:only-child th {\n",
       "        text-align: right;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: left;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>index</th>\n",
       "      <th>IV</th>\n",
       "      <th>RF</th>\n",
       "      <th>RFE</th>\n",
       "      <th>Extratrees</th>\n",
       "      <th>Chi_Square</th>\n",
       "      <th>L1</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>age</td>\n",
       "      <td>0.00</td>\n",
       "      <td>0.10</td>\n",
       "      <td>True</td>\n",
       "      <td>0.09</td>\n",
       "      <td>45.84</td>\n",
       "      <td>True</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>balance</td>\n",
       "      <td>0.08</td>\n",
       "      <td>0.11</td>\n",
       "      <td>True</td>\n",
       "      <td>0.09</td>\n",
       "      <td>11343.98</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>campaign</td>\n",
       "      <td>0.02</td>\n",
       "      <td>0.04</td>\n",
       "      <td>True</td>\n",
       "      <td>0.05</td>\n",
       "      <td>88.07</td>\n",
       "      <td>True</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>contact</td>\n",
       "      <td>0.25</td>\n",
       "      <td>0.02</td>\n",
       "      <td>True</td>\n",
       "      <td>0.03</td>\n",
       "      <td>100.51</td>\n",
       "      <td>True</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>day</td>\n",
       "      <td>0.00</td>\n",
       "      <td>0.08</td>\n",
       "      <td>True</td>\n",
       "      <td>0.08</td>\n",
       "      <td>2.61</td>\n",
       "      <td>True</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "      index   IV   RF   RFE  Extratrees  Chi_Square     L1\n",
       "0       age 0.00 0.10  True        0.09       45.84   True\n",
       "1   balance 0.08 0.11  True        0.09    11343.98  False\n",
       "2  campaign 0.02 0.04  True        0.05       88.07   True\n",
       "3   contact 0.25 0.02  True        0.03      100.51   True\n",
       "4       day 0.00 0.08  True        0.08        2.61   True"
      ]
     },
     "execution_count": 36,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "final_results.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 37,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "Index(['index', 'IV', 'RF', 'RFE', 'Extratrees', 'Chi_Square', 'L1'], dtype='object')"
      ]
     },
     "execution_count": 37,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "final_results.columns"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "collapsed": true
   },
   "source": [
    "## 3.8 Variable Score"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 38,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "columns = ['IV', 'RF', 'Extratrees', 'Chi_Square']\n",
    "\n",
    "score_table = pd.DataFrame({},[])\n",
    "score_table['index'] = final_results['index']\n",
    "\n",
    "for i in columns:\n",
    "    score_table[i] = final_results['index'].isin(list(final_results.nlargest(5,i)['index'])).astype(int)\n",
    "    \n",
    "score_table['RFE'] = final_results['RFE'].astype(int)\n",
    "score_table['L1'] = final_results['L1'].astype(int)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 39,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "score_table['final_score'] = score_table.sum(axis=1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 40,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style>\n",
       "    .dataframe thead tr:only-child th {\n",
       "        text-align: right;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: left;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>index</th>\n",
       "      <th>IV</th>\n",
       "      <th>RF</th>\n",
       "      <th>Extratrees</th>\n",
       "      <th>Chi_Square</th>\n",
       "      <th>RFE</th>\n",
       "      <th>L1</th>\n",
       "      <th>final_score</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>6</th>\n",
       "      <td>duration</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>6</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>age</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>4</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>balance</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>4</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>contact</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>4</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>day</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>4</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>12</th>\n",
       "      <td>month</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>4</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>13</th>\n",
       "      <td>pdays</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>4</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>15</th>\n",
       "      <td>previous</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>4</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8</th>\n",
       "      <td>housing</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>3</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>campaign</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>2</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9</th>\n",
       "      <td>job</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>2</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>10</th>\n",
       "      <td>loan</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>2</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>11</th>\n",
       "      <td>marital</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>2</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>14</th>\n",
       "      <td>poutcome</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>2</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>default</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7</th>\n",
       "      <td>education</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "        index  IV  RF  Extratrees  Chi_Square  RFE  L1  final_score\n",
       "6    duration   1   1           1           1    1   1            6\n",
       "0         age   0   1           1           0    1   1            4\n",
       "1     balance   0   1           1           1    1   0            4\n",
       "3     contact   1   0           0           1    1   1            4\n",
       "4         day   0   1           1           0    1   1            4\n",
       "12      month   0   1           1           0    1   1            4\n",
       "13      pdays   1   0           0           1    1   1            4\n",
       "15   previous   1   0           0           1    1   1            4\n",
       "8     housing   1   0           0           0    1   1            3\n",
       "2    campaign   0   0           0           0    1   1            2\n",
       "9         job   0   0           0           0    1   1            2\n",
       "10       loan   0   0           0           0    1   1            2\n",
       "11    marital   0   0           0           0    1   1            2\n",
       "14   poutcome   0   0           0           0    1   1            2\n",
       "5     default   0   0           0           0    1   0            1\n",
       "7   education   0   0           0           0    1   0            1"
      ]
     },
     "execution_count": 40,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "score_table.sort_values('final_score',ascending=0)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "collapsed": true
   },
   "source": [
    "## 3.9 Multicollinearity - VIF  (Addon)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 41,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "from statsmodels.stats.outliers_influence import variance_inflation_factor"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 42,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def calculate_vif(features):\n",
    "    vif = pd.DataFrame()\n",
    "    vif[\"Features\"] = features.columns\n",
    "    vif[\"VIF\"] = [variance_inflation_factor(features.values, i) for i in range(features.shape[1])]    \n",
    "    return(vif)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 43,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/Users/mbagav200/anaconda/lib/python3.5/site-packages/ipykernel/__main__.py:4: SettingWithCopyWarning: \n",
      "A value is trying to be set on a copy of a slice from a DataFrame\n",
      "\n",
      "See the caveats in the documentation: http://pandas.pydata.org/pandas-docs/stable/indexing.html#indexing-view-versus-copy\n"
     ]
    }
   ],
   "source": [
    "vif = calculate_vif(features)\n",
    "while vif['VIF'][vif['VIF'] > 10].any():\n",
    "    remove = vif.sort_values('VIF',ascending=0)['Features'][:1]\n",
    "    features.drop(remove,axis=1,inplace=True)\n",
    "    vif = calculate_vif(features)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 45,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style>\n",
       "    .dataframe thead tr:only-child th {\n",
       "        text-align: right;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: left;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Features</th>\n",
       "      <th>VIF</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>age</td>\n",
       "      <td>7.75</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>balance</td>\n",
       "      <td>1.26</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>campaign</td>\n",
       "      <td>1.87</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>contact</td>\n",
       "      <td>1.99</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>day</td>\n",
       "      <td>4.55</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>default</td>\n",
       "      <td>1.03</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6</th>\n",
       "      <td>duration</td>\n",
       "      <td>2.00</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7</th>\n",
       "      <td>education</td>\n",
       "      <td>3.64</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8</th>\n",
       "      <td>housing</td>\n",
       "      <td>2.52</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9</th>\n",
       "      <td>job</td>\n",
       "      <td>2.88</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>10</th>\n",
       "      <td>loan</td>\n",
       "      <td>1.19</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>11</th>\n",
       "      <td>marital</td>\n",
       "      <td>4.02</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>12</th>\n",
       "      <td>month</td>\n",
       "      <td>5.23</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>13</th>\n",
       "      <td>pdays</td>\n",
       "      <td>1.85</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>14</th>\n",
       "      <td>previous</td>\n",
       "      <td>1.67</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "     Features  VIF\n",
       "0         age 7.75\n",
       "1     balance 1.26\n",
       "2    campaign 1.87\n",
       "3     contact 1.99\n",
       "4         day 4.55\n",
       "5     default 1.03\n",
       "6    duration 2.00\n",
       "7   education 3.64\n",
       "8     housing 2.52\n",
       "9         job 2.88\n",
       "10       loan 1.19\n",
       "11    marital 4.02\n",
       "12      month 5.23\n",
       "13      pdays 1.85\n",
       "14   previous 1.67"
      ]
     },
     "execution_count": 45,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "vif"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.5.2"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}