Assignment2_Q1-Q7.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "from sklearn.model_selection import train_test_split, cross_val_score\n",
    "from sklearn.metrics import classification_report, confusion_matrix, accuracy_score, f1_score, matthews_corrcoef\n",
    "from sklearn.linear_model import LogisticRegression\n",
    "from sklearn.tree import DecisionTreeClassifier\n",
    "from xgboost import XGBClassifier\n",
    "import numpy as np\n",
    "from sklearn.model_selection import GridSearchCV\n",
    "from sklearn.preprocessing import StandardScaler"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>acc_now_delinq</th>\n",
       "      <th>acc_open_past_24mths</th>\n",
       "      <th>all_util</th>\n",
       "      <th>annual_inc</th>\n",
       "      <th>avg_cur_bal</th>\n",
       "      <th>bc_open_to_buy</th>\n",
       "      <th>bc_util</th>\n",
       "      <th>chargeoff_within_12_mths</th>\n",
       "      <th>collections_12_mths_ex_med</th>\n",
       "      <th>delinq_amnt</th>\n",
       "      <th>...</th>\n",
       "      <th>total_cu_tl</th>\n",
       "      <th>total_il_high_credit_limit</th>\n",
       "      <th>total_pymnt</th>\n",
       "      <th>total_pymnt_inv</th>\n",
       "      <th>total_rec_int</th>\n",
       "      <th>total_rec_late_fee</th>\n",
       "      <th>total_rec_prncp</th>\n",
       "      <th>total_rev_hi_lim</th>\n",
       "      <th>delinq_2yrs</th>\n",
       "      <th>bad_loans</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>0.0</td>\n",
       "      <td>3.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>52000.0</td>\n",
       "      <td>25099.0</td>\n",
       "      <td>30359.0</td>\n",
       "      <td>0.5</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>...</td>\n",
       "      <td>2.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>3011.577285</td>\n",
       "      <td>3011.58</td>\n",
       "      <td>11.58</td>\n",
       "      <td>0.0</td>\n",
       "      <td>3000.00</td>\n",
       "      <td>31000.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>0.0</td>\n",
       "      <td>2.0</td>\n",
       "      <td>47.0</td>\n",
       "      <td>55000.0</td>\n",
       "      <td>2222.0</td>\n",
       "      <td>10551.0</td>\n",
       "      <td>52.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>...</td>\n",
       "      <td>1.0</td>\n",
       "      <td>27751.0</td>\n",
       "      <td>5013.306667</td>\n",
       "      <td>5013.31</td>\n",
       "      <td>13.31</td>\n",
       "      <td>0.0</td>\n",
       "      <td>5000.00</td>\n",
       "      <td>33800.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>0.0</td>\n",
       "      <td>3.0</td>\n",
       "      <td>86.0</td>\n",
       "      <td>51000.0</td>\n",
       "      <td>4541.0</td>\n",
       "      <td>2498.0</td>\n",
       "      <td>0.1</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>...</td>\n",
       "      <td>0.0</td>\n",
       "      <td>24001.0</td>\n",
       "      <td>1032.903619</td>\n",
       "      <td>1032.90</td>\n",
       "      <td>32.90</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1000.00</td>\n",
       "      <td>2500.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>0.0</td>\n",
       "      <td>2.0</td>\n",
       "      <td>41.0</td>\n",
       "      <td>62000.0</td>\n",
       "      <td>18520.0</td>\n",
       "      <td>18310.0</td>\n",
       "      <td>51.2</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>...</td>\n",
       "      <td>0.0</td>\n",
       "      <td>21976.0</td>\n",
       "      <td>20009.805556</td>\n",
       "      <td>19984.79</td>\n",
       "      <td>9.81</td>\n",
       "      <td>0.0</td>\n",
       "      <td>20000.00</td>\n",
       "      <td>39500.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>0.0</td>\n",
       "      <td>11.0</td>\n",
       "      <td>60.0</td>\n",
       "      <td>30000.0</td>\n",
       "      <td>5990.0</td>\n",
       "      <td>7898.0</td>\n",
       "      <td>7.1</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>...</td>\n",
       "      <td>5.0</td>\n",
       "      <td>62319.0</td>\n",
       "      <td>24334.736554</td>\n",
       "      <td>24334.74</td>\n",
       "      <td>334.74</td>\n",
       "      <td>0.0</td>\n",
       "      <td>24000.00</td>\n",
       "      <td>12000.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>0.0</td>\n",
       "      <td>3.0</td>\n",
       "      <td>32.0</td>\n",
       "      <td>58000.0</td>\n",
       "      <td>16129.0</td>\n",
       "      <td>10282.0</td>\n",
       "      <td>6.5</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>...</td>\n",
       "      <td>0.0</td>\n",
       "      <td>68112.0</td>\n",
       "      <td>265.850000</td>\n",
       "      <td>265.30</td>\n",
       "      <td>223.63</td>\n",
       "      <td>15.0</td>\n",
       "      <td>27.22</td>\n",
       "      <td>11000.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6</th>\n",
       "      <td>0.0</td>\n",
       "      <td>2.0</td>\n",
       "      <td>57.0</td>\n",
       "      <td>48000.0</td>\n",
       "      <td>2855.0</td>\n",
       "      <td>7459.0</td>\n",
       "      <td>25.4</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>...</td>\n",
       "      <td>0.0</td>\n",
       "      <td>27697.0</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.00</td>\n",
       "      <td>0.00</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.00</td>\n",
       "      <td>12200.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7</th>\n",
       "      <td>0.0</td>\n",
       "      <td>21.0</td>\n",
       "      <td>32.0</td>\n",
       "      <td>186500.0</td>\n",
       "      <td>2443.0</td>\n",
       "      <td>107477.0</td>\n",
       "      <td>18.8</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>...</td>\n",
       "      <td>0.0</td>\n",
       "      <td>57714.0</td>\n",
       "      <td>20117.246667</td>\n",
       "      <td>20117.25</td>\n",
       "      <td>117.25</td>\n",
       "      <td>0.0</td>\n",
       "      <td>20000.00</td>\n",
       "      <td>139400.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8</th>\n",
       "      <td>0.0</td>\n",
       "      <td>7.0</td>\n",
       "      <td>58.0</td>\n",
       "      <td>12000.0</td>\n",
       "      <td>48449.0</td>\n",
       "      <td>100.0</td>\n",
       "      <td>96.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>...</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>15424.086919</td>\n",
       "      <td>15424.09</td>\n",
       "      <td>424.09</td>\n",
       "      <td>0.0</td>\n",
       "      <td>15000.00</td>\n",
       "      <td>12400.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9</th>\n",
       "      <td>0.0</td>\n",
       "      <td>4.0</td>\n",
       "      <td>74.0</td>\n",
       "      <td>54000.0</td>\n",
       "      <td>11336.0</td>\n",
       "      <td>6635.0</td>\n",
       "      <td>44.7</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>...</td>\n",
       "      <td>3.0</td>\n",
       "      <td>34050.0</td>\n",
       "      <td>2505.296389</td>\n",
       "      <td>2505.30</td>\n",
       "      <td>5.30</td>\n",
       "      <td>0.0</td>\n",
       "      <td>2500.00</td>\n",
       "      <td>12000.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>10 rows × 71 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "   acc_now_delinq  acc_open_past_24mths  all_util  annual_inc  avg_cur_bal  \\\n",
       "0             0.0                   3.0       1.0     52000.0      25099.0   \n",
       "1             0.0                   2.0      47.0     55000.0       2222.0   \n",
       "2             0.0                   3.0      86.0     51000.0       4541.0   \n",
       "3             0.0                   2.0      41.0     62000.0      18520.0   \n",
       "4             0.0                  11.0      60.0     30000.0       5990.0   \n",
       "5             0.0                   3.0      32.0     58000.0      16129.0   \n",
       "6             0.0                   2.0      57.0     48000.0       2855.0   \n",
       "7             0.0                  21.0      32.0    186500.0       2443.0   \n",
       "8             0.0                   7.0      58.0     12000.0      48449.0   \n",
       "9             0.0                   4.0      74.0     54000.0      11336.0   \n",
       "\n",
       "   bc_open_to_buy  bc_util  chargeoff_within_12_mths  \\\n",
       "0         30359.0      0.5                       0.0   \n",
       "1         10551.0     52.0                       0.0   \n",
       "2          2498.0      0.1                       0.0   \n",
       "3         18310.0     51.2                       0.0   \n",
       "4          7898.0      7.1                       0.0   \n",
       "5         10282.0      6.5                       0.0   \n",
       "6          7459.0     25.4                       0.0   \n",
       "7        107477.0     18.8                       0.0   \n",
       "8           100.0     96.0                       0.0   \n",
       "9          6635.0     44.7                       0.0   \n",
       "\n",
       "   collections_12_mths_ex_med  delinq_amnt  ...  total_cu_tl  \\\n",
       "0                         0.0          0.0  ...          2.0   \n",
       "1                         0.0          0.0  ...          1.0   \n",
       "2                         0.0          0.0  ...          0.0   \n",
       "3                         0.0          0.0  ...          0.0   \n",
       "4                         0.0          0.0  ...          5.0   \n",
       "5                         0.0          0.0  ...          0.0   \n",
       "6                         0.0          0.0  ...          0.0   \n",
       "7                         0.0          0.0  ...          0.0   \n",
       "8                         0.0          0.0  ...          0.0   \n",
       "9                         0.0          0.0  ...          3.0   \n",
       "\n",
       "   total_il_high_credit_limit   total_pymnt  total_pymnt_inv  total_rec_int  \\\n",
       "0                         0.0   3011.577285          3011.58          11.58   \n",
       "1                     27751.0   5013.306667          5013.31          13.31   \n",
       "2                     24001.0   1032.903619          1032.90          32.90   \n",
       "3                     21976.0  20009.805556         19984.79           9.81   \n",
       "4                     62319.0  24334.736554         24334.74         334.74   \n",
       "5                     68112.0    265.850000           265.30         223.63   \n",
       "6                     27697.0      0.000000             0.00           0.00   \n",
       "7                     57714.0  20117.246667         20117.25         117.25   \n",
       "8                         0.0  15424.086919         15424.09         424.09   \n",
       "9                     34050.0   2505.296389          2505.30           5.30   \n",
       "\n",
       "   total_rec_late_fee  total_rec_prncp  total_rev_hi_lim  delinq_2yrs  \\\n",
       "0                 0.0          3000.00           31000.0          0.0   \n",
       "1                 0.0          5000.00           33800.0          0.0   \n",
       "2                 0.0          1000.00            2500.0          0.0   \n",
       "3                 0.0         20000.00           39500.0          0.0   \n",
       "4                 0.0         24000.00           12000.0          0.0   \n",
       "5                15.0            27.22           11000.0          1.0   \n",
       "6                 0.0             0.00           12200.0          1.0   \n",
       "7                 0.0         20000.00          139400.0          0.0   \n",
       "8                 0.0         15000.00           12400.0          1.0   \n",
       "9                 0.0          2500.00           12000.0          0.0   \n",
       "\n",
       "   bad_loans  \n",
       "0          0  \n",
       "1          0  \n",
       "2          0  \n",
       "3          0  \n",
       "4          0  \n",
       "5          1  \n",
       "6          1  \n",
       "7          0  \n",
       "8          0  \n",
       "9          0  \n",
       "\n",
       "[10 rows x 71 columns]"
      ]
     },
     "execution_count": 2,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "loans = pd.read_csv('Assignment2Data.csv', low_memory=False, header=0)\n",
    "#loans = pd.read_csv(\"./Assignment2Data.csv\", low_memory=False, header=0)\n",
    "loans.head(n=10)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0    2516\n",
       "1     476\n",
       "Name: bad_loans, dtype: int64"
      ]
     },
     "execution_count": 3,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Check how many good and bad loans there are\n",
    "loans['bad_loans'].value_counts()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Oversample Bad Loans so that it is proportionate"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "1    2516\n",
       "0    2516\n",
       "Name: bad_loans, dtype: int64"
      ]
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Class count\n",
    "count_class_0, count_class_1 = loans['bad_loans'].value_counts()\n",
    "\n",
    "# Divide by class\n",
    "df_class_0 = loans[loans['bad_loans'] == 0]\n",
    "df_class_1 = loans[loans['bad_loans'] == 1]\n",
    "\n",
    "df_class_1_over = df_class_1.sample(count_class_0, replace=True, random_state=1234)\n",
    "\n",
    "# Address unblanced data issue: Oversampling on data samples with rare outputs (which is '1' in this example)\n",
    "df_class_1_over = df_class_1.sample(count_class_0, replace=True, random_state=1234)\n",
    "loans = pd.concat([df_class_1_over, df_class_0], axis=0)\n",
    "\n",
    "loans['bad_loans'].value_counts()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Split Data 80/20"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "X = loans.drop('bad_loans', axis=1) # Here no need to set inplace=True\n",
    "Y = loans['bad_loans']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "validation_size = 0.2\n",
    "seed = 1234\n",
    "X_train, X_test, Y_train, Y_test = train_test_split(X, Y, test_size=validation_size, random_state=seed)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "<h1 style=\"color:blue\">FULL MODEL</h1>"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "def get_results(predictions):\n",
    "    print(\"Accuracy score is: \" + str(accuracy_score(Y_test, predictions)))\n",
    "    print(\"F1_score is: \" + str(f1_score(Y_test, predictions)))\n",
    "    print(\"MCC: \", matthews_corrcoef(Y_test, predictions))\n",
    "    print(\"Confusion Matrix\")\n",
    "    print(confusion_matrix(Y_test, predictions))\n",
    "    return"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Logistic Regression"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Accuracy score is: 0.9225422045680238\n",
      "F1_score is: 0.9277777777777779\n",
      "MCC:  0.8533127404720492\n",
      "Confusion Matrix\n",
      "[[428  74]\n",
      " [  4 501]]\n"
     ]
    }
   ],
   "source": [
    "#Logistic Regression, use default param values then tune in subsequent parts\n",
    "def logisticRegression(X_train, Y_train, X_test, Y_test):\n",
    "    lg = LogisticRegression(random_state=seed, solver='liblinear')\n",
    "    lg.fit(X_train, Y_train)\n",
    "    #print(lg.get_params) #print this to see possible params to tune\n",
    "    predictions = lg.predict(X_test)\n",
    "    get_results(predictions)\n",
    "    return\n",
    "\n",
    "logisticRegression(X_train, Y_train, X_test, Y_test)"
   ]
  },
  {
   "cell_type": "raw",
   "metadata": {},
   "source": [
    "#GridSearchCV on the 80% data. \n",
    "#DO NOT RUN THIS, time consuming! See model below with best params from tuning\n",
    "\n",
    "#Logistic Regression\n",
    "lg = LogisticRegression(random_state=seed, solver='liblinear', multi_class='ovr')\n",
    "\n",
    "penalty = ['l1', 'l2']\n",
    "C = np.logspace(0, 4, 10)\n",
    "\n",
    "# Create hyperparameter options\n",
    "hyperparameters = dict(C=C, penalty=penalty)\n",
    "lg = LogisticRegression(random_state=seed, solver='liblinear')\n",
    "clf = GridSearchCV(lg, hyperparameters, cv=5, verbose=0,n_jobs=-1) \n",
    "\n",
    "best_model = clf.fit(X_train, Y_train)\n",
    "print(best_model.best_params_)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Accuracy score is: 0.9275074478649454\n",
      "F1_score is: 0.9319664492078287\n",
      "MCC:  0.8617370451537003\n",
      "Confusion Matrix\n",
      "[[434  68]\n",
      " [  5 500]]\n"
     ]
    }
   ],
   "source": [
    "LR_clf = LogisticRegression(random_state=seed, solver='liblinear', C=464.15888336127773, penalty='l2' )\n",
    "LR_model = LR_clf.fit(X_train, Y_train)\n",
    "predictions = LR_model.predict(X_test)\n",
    "get_results(predictions)\n",
    "#same result as before"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Full Model - Results for Logistic Regression\n",
    "\n",
    "Logistic Regression without GridSearchCV: 92.3% accuracy, 0.853 MCC\n",
    "\n",
    "Logistic Regression with GridSearchCV: 92.8% accuracy, 0.862 MCC \n",
    "\n",
    "Slight increase in accuracy and MCC"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Decision Tree"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Accuracy score is: 0.9602780536246276\n",
      "F1_score is: 0.9619047619047619\n",
      "MCC:  0.9234567649066872\n",
      "Confusion Matrix\n",
      "[[462  40]\n",
      " [  0 505]]\n"
     ]
    }
   ],
   "source": [
    "def decisionTree(X_train, Y_train, X_test, Y_test):\n",
    "    dtc = DecisionTreeClassifier(random_state=seed)\n",
    "    dtc.fit(X_train, Y_train)\n",
    "    predictions = dtc.predict(X_test)\n",
    "    get_results(predictions)\n",
    "    return\n",
    "decisionTree(X_train, Y_train, X_test, Y_test)"
   ]
  },
  {
   "cell_type": "raw",
   "metadata": {},
   "source": [
    "#GridSearchCV\n",
    "#DO NOT RUN THIS\n",
    "#Decision Tree\n",
    "dtc = DecisionTreeClassifier(random_state=seed)\n",
    "\n",
    "#hyperparameters to tune\n",
    "max_depth = [i for i in range(1,11)]\n",
    "min_samples_split = [i*0.1 for i in range(1,11)]\n",
    "min_samples_leaf = [1, 5, 10, 20, 50, 100]\n",
    "criterion= ['gini', 'entropy']\n",
    "\n",
    "# Create hyperparameter options\n",
    "hyperparameters = dict(max_depth=max_depth, min_samples_leaf=min_samples_leaf,min_samples_split=min_samples_split, criterion=criterion)\n",
    "clf = GridSearchCV(dtc, hyperparameters, cv=5, verbose=0)\n",
    "\n",
    "best_model = clf.fit(X_train, Y_train)\n",
    "print(best_model.best_params_)\n",
    "predictions = best_model.best_estimator_.predict(X_test)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Accuracy score is: 0.9285004965243296\n",
      "F1_score is: 0.9319470699432891\n",
      "MCC:  0.860882316689042\n",
      "Confusion Matrix\n",
      "[[442  60]\n",
      " [ 12 493]]\n"
     ]
    }
   ],
   "source": [
    "#Tuned Model \n",
    "DT_clf = DecisionTreeClassifier(random_state=seed, max_depth = 9, min_samples_split= 0.1, \\\n",
    "                             min_samples_leaf= 1, criterion='gini')\n",
    "DT_model = DT_clf.fit(X_train, Y_train)\n",
    "predictions = DT_model.predict(X_test)\n",
    "get_results(predictions)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Full Model - Results for Decision Tree\n",
    "\n",
    "Decision Tree without GridSearchCV: 96.0% accuracy, 0.923 MCC\n",
    "\n",
    "Decision tree with GridSearchCV: 92.9% accuracy, 0.861 MCC\n",
    "\n",
    "\n",
    "The default hyperparameter values perform better than the best param found in the param grid set up for in GridSearchCV. "
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## XGBClassifier"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Accuracy score is: 0.9404170804369414\n",
      "F1_score is: 0.943609022556391\n",
      "MCC:  0.8859081552873652\n",
      "Confusion Matrix\n",
      "[[445  57]\n",
      " [  3 502]]\n"
     ]
    }
   ],
   "source": [
    "##XGBOOST\n",
    "\n",
    "def xgBoost(X_train, Y_train, X_test, Y_test):\n",
    "    xgbc = XGBClassifier(random_state=seed)\n",
    "    xgbc.fit(X_train, Y_train)\n",
    "    predictions = xgbc.predict(X_test)\n",
    "    get_results(predictions)\n",
    "    return\n",
    "\n",
    "xgBoost(X_train, Y_train, X_test, Y_test)"
   ]
  },
  {
   "cell_type": "raw",
   "metadata": {},
   "source": [
    "#GridSearchCV for XGBoost. \n",
    "#DO NOT RUN THIS \n",
    "\n",
    "xgbc = XGBClassifier(random_state=seed)\n",
    "\n",
    "max_depth = [1, 2, 3, 4, 5, 6, 7, 8, 9]\n",
    "learning_rate = [0.001, 0.01, 0.1]\n",
    "hyperparameters = dict(max_depth=max_depth, learning_rate = learning_rate)\n",
    "clf = GridSearchCV(xgbc, hyperparameters, cv=5, verbose=0)\n",
    "\n",
    "best_model = clf.fit(X_train, Y_train)\n",
    "print(best_model.best_params_)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Accuracy score is: 0.9493545183714002\n",
      "F1_score is: 0.9519321394910463\n",
      "MCC:  0.9033264058721377\n",
      "Confusion Matrix\n",
      "[[451  51]\n",
      " [  0 505]]\n"
     ]
    }
   ],
   "source": [
    "XGB_clf = XGBClassifier(random_state=seed, max_depth = 6 , learning_rate = 0.1)\n",
    "XGB_model = XGB_clf.fit(X_train, Y_train)\n",
    "predictions = XGB_model.predict(X_test)\n",
    "get_results(predictions)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Full Model - Results for XGBoost\n",
    "\n",
    "XGBoost without GridSearchCV:  94.0% accuracy, 0.886 MCC\n",
    "\n",
    "XGBoost with GridSearchCV: 94.9% accuracy, 0.903 MCC \n",
    "\n",
    "We see a slight increase in accuracy rate and MCC value. Since accuracy rates are already high to begin with before hyperparameter tuning, we would expect to see minimal increase in accuracy and MCC values as well."
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Question 3 \n",
    "\n",
    "**Which loan attributes do you believe are the most informative? Please use your knowledge and intuition to choose 10 of the attributes available, and explain why you chose these attributes. Let’s call this the REDUCED-MODEL.**\n",
    "\n",
    "\n",
    "1. Select 10 variables\n",
    "\n",
    "2. Test out model without tuning\n",
    "\n",
    "3. Test out model with hyperparameter tuning. "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [],
   "source": [
    "#cols = ['annual_inc','avg_cur_bal', 'dti', 'int_rate', 'loan_amnt',\\\n",
    "       #'max_bal_bc', 'mo_sin_old_il_acct', 'pct_tl_nvr_dlq', 'mort_acc', 'num_sats']\n",
    "\n",
    "cols= ['annual_inc' ,'avg_cur_bal', 'dti','mths_since_last_delinq', 'mort_acc',\\\n",
    "'num_sats', 'open_acc_6m','total_bc_limit','total_pymnt' ,'total_rec_prncp']\n",
    "\n",
    "X_train2 = X_train[cols]\n",
    "X_test2 = X_test[cols]    \n",
    "\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Logistic Regression"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Accuracy score is: 0.9155908639523337\n",
      "F1_score is: 0.9210770659238625\n",
      "MCC:  0.8385769519352704\n",
      "Confusion Matrix\n",
      "[[426  76]\n",
      " [  9 496]]\n"
     ]
    }
   ],
   "source": [
    "logisticRegression(X_train2, Y_train, X_test2, Y_test)"
   ]
  },
  {
   "cell_type": "raw",
   "metadata": {},
   "source": [
    "#GridSearchCV Logistic Regression\n",
    "#DO NOT RUN THIS\n",
    "\n",
    "lg = LogisticRegression(random_state=seed, solver='liblinear', multi_class='ovr')\n",
    "\n",
    "penalty = ['l1', 'l2']\n",
    "C = np.logspace(0, 4, 10)\n",
    "\n",
    "# Create hyperparameter options\n",
    "hyperparameters = dict(C=C, penalty=penalty)\n",
    "lg = LogisticRegression(random_state=seed, solver='liblinear')\n",
    "clf = GridSearchCV(lg, hyperparameters, cv=5, verbose=0,n_jobs=-1) \n",
    "\n",
    "best_model = clf.fit(X_train2, Y_train)\n",
    "print(best_model.best_params_)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Accuracy score is: 0.9155908639523337\n",
      "F1_score is: 0.9210770659238625\n",
      "MCC:  0.8385769519352704\n",
      "Confusion Matrix\n",
      "[[426  76]\n",
      " [  9 496]]\n"
     ]
    }
   ],
   "source": [
    "LR_clf = LogisticRegression(random_state=seed, solver='liblinear', C=1,  penalty='l2' )\n",
    "LR_model = LR_clf.fit(X_train2, Y_train)\n",
    "predictions = LR_model.predict(X_test2)\n",
    "get_results(predictions)\n",
    "#same result as before"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Reduced Model - Results for Logistic Regression ###\n",
    "\n",
    "Logistic Regression without GridSearchCV: 91.6% accuracy, 0.839 MCC\n",
    "\n",
    "Logistic Regression with GridSearchCV: 91.6% accuracy, 0.839 MCC\n",
    "\n",
    "Both show the same accuracy and MCC. No improvement from the GridSearchCV.\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Decision Tree"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Accuracy score is: 0.9543197616683218\n",
      "F1_score is: 0.9562737642585551\n",
      "MCC:  0.9117954268367235\n",
      "Confusion Matrix\n",
      "[[458  44]\n",
      " [  2 503]]\n"
     ]
    }
   ],
   "source": [
    "decisionTree(X_train2, Y_train, X_test2, Y_test)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'criterion': 'entropy', 'max_depth': 3, 'min_samples_leaf': 100, 'min_samples_split': 0.1}\n"
     ]
    }
   ],
   "source": [
    "#Decision Tree GridSearchCV\n",
    "#DO NOT RUN THIS, see below for best params\n",
    "dtc = DecisionTreeClassifier(random_state=seed)\n",
    "\n",
    "#hyperparameters to tune\n",
    "max_depth = [i for i in range(1,11)]\n",
    "min_samples_split = [i*0.1 for i in range(1,11)]\n",
    "min_samples_leaf = [1, 5, 10, 20, 50, 100]\n",
    "criterion= ['gini', 'entropy']\n",
    "\n",
    "# Create hyperparameter options\n",
    "hyperparameters = dict(max_depth=max_depth, min_samples_leaf=min_samples_leaf,min_samples_split=min_samples_split, criterion=criterion)\n",
    "clf = GridSearchCV(dtc, hyperparameters, cv=5, verbose=0)\n",
    "\n",
    "best_model = clf.fit(X_train2, Y_train)\n",
    "print(best_model.best_params_)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Accuracy score is: 0.8967229394240318\n",
      "F1_score is: 0.9018867924528301\n",
      "MCC:  0.7973264681686234\n",
      "Confusion Matrix\n",
      "[[425  77]\n",
      " [ 27 478]]\n"
     ]
    }
   ],
   "source": [
    "#Based on GridSearchCV\n",
    "DT_clf = DecisionTreeClassifier(random_state=seed, max_depth = 3, min_samples_split=0.1,\\\n",
    "                             min_samples_leaf= 100, criterion='entropy')\n",
    "DT_model = DT_clf.fit(X_train2, Y_train)\n",
    "predictions = DT_model.predict(X_test2)\n",
    "get_results(predictions)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Reduced Model - Results for Decision Tree ###\n",
    "\n",
    "Decision tree without GridSearchCV : 95.4% accuracy, 0.912 MCC\n",
    "\n",
    "Decision tree with GridSearchCV: 89.7% accuracy, 0.797 MCC\n",
    "\n",
    "\n",
    "The default hyperparameter values perform better than the best param found in the param grid set up for in GridSearchCV. "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Accuracy score is: 0.9304865938430983\n",
      "F1_score is: 0.9343339587242027\n",
      "MCC:  0.866304400246787\n",
      "Confusion Matrix\n",
      "[[439  63]\n",
      " [  7 498]]\n"
     ]
    }
   ],
   "source": [
    "xgBoost(X_train2, Y_train, X_test2, Y_test)"
   ]
  },
  {
   "cell_type": "raw",
   "metadata": {},
   "source": [
    "#GridSearchCV for XGBoost. \n",
    "#DO NOT RUN THIS, takes quite some time. \n",
    "\n",
    "#XGBoost\n",
    "xgbc = XGBClassifier(random_state=seed)\n",
    "\n",
    "# max depth there are a total of 70 x variables\n",
    "max_depth = [1, 2, 3, 4, 5, 6, 7, 8, 9]\n",
    "learning_rate = [0.001, 0.01, 0.1]\n",
    "#n_estimators = [100, 200] #do not include this, time consuming!\n",
    "hyperparameters = dict(max_depth=max_depth, learning_rate = learning_rate)\n",
    "clf = GridSearchCV(xgbc, hyperparameters, cv=5, verbose=0)\n",
    "\n",
    "best_model = clf.fit(X_train2, Y_train)\n",
    "print(best_model.best_params_)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Accuracy score is: 0.9553128103277061\n",
      "F1_score is: 0.957345971563981\n",
      "MCC:  0.9142618241433023\n",
      "Confusion Matrix\n",
      "[[457  45]\n",
      " [  0 505]]\n"
     ]
    }
   ],
   "source": [
    "XGB_clf = XGBClassifier(random_state=seed, max_depth = 9 , learning_rate = 0.1)\n",
    "XGB_model = XGB_clf.fit(X_train2, Y_train)\n",
    "predictions = XGB_model.predict(X_test2)\n",
    "get_results(predictions)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Reduced Model - Results for XGBoost ###\n",
    "\n",
    "XGBoost without GridSearchCV:  93.0% accuracy, 0.866 MCC\n",
    "\n",
    "XGBoost with GridSearchCV: 95.5% accuracy, 0.914 MCC \n",
    "\n",
    "We see an increase in accuracy rates and MCC value. Since accuracy rates are already high to begin with before hyperparameter tuning, we would expect to see minimal increase in accuracy and MCC values as well."
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "<h1 style=\"color:blue\">LASSO-REDUCED MODEL</h1>"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Import libraries\n",
    "from sklearn import datasets\n",
    "from sklearn.model_selection import train_test_split\n",
    "from sklearn.tree import DecisionTreeClassifier\n",
    "from sklearn.linear_model import LogisticRegression\n",
    "from sklearn.ensemble import AdaBoostClassifier\n",
    "from sklearn import tree\n",
    "from sklearn.metrics import accuracy_score, f1_score, precision_score, recall_score, classification_report, confusion_matrix\n",
    "from sklearn.model_selection import KFold, cross_val_score\n",
    "from sklearn.metrics import roc_curve, auc\n",
    "from sklearn.preprocessing import StandardScaler\n",
    "import warnings\n",
    "warnings.filterwarnings(\"ignore\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>acc_now_delinq</th>\n",
       "      <th>acc_open_past_24mths</th>\n",
       "      <th>all_util</th>\n",
       "      <th>annual_inc</th>\n",
       "      <th>avg_cur_bal</th>\n",
       "      <th>bc_open_to_buy</th>\n",
       "      <th>bc_util</th>\n",
       "      <th>chargeoff_within_12_mths</th>\n",
       "      <th>collections_12_mths_ex_med</th>\n",
       "      <th>delinq_amnt</th>\n",
       "      <th>...</th>\n",
       "      <th>total_cu_tl</th>\n",
       "      <th>total_il_high_credit_limit</th>\n",
       "      <th>total_pymnt</th>\n",
       "      <th>total_pymnt_inv</th>\n",
       "      <th>total_rec_int</th>\n",
       "      <th>total_rec_late_fee</th>\n",
       "      <th>total_rec_prncp</th>\n",
       "      <th>total_rev_hi_lim</th>\n",
       "      <th>delinq_2yrs</th>\n",
       "      <th>bad_loans</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>0.0</td>\n",
       "      <td>3.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>52000.0</td>\n",
       "      <td>25099.0</td>\n",
       "      <td>30359.0</td>\n",
       "      <td>0.5</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>...</td>\n",
       "      <td>2.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>3011.577285</td>\n",
       "      <td>3011.58</td>\n",
       "      <td>11.58</td>\n",
       "      <td>0.0</td>\n",
       "      <td>3000.0</td>\n",
       "      <td>31000.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>0.0</td>\n",
       "      <td>2.0</td>\n",
       "      <td>47.0</td>\n",
       "      <td>55000.0</td>\n",
       "      <td>2222.0</td>\n",
       "      <td>10551.0</td>\n",
       "      <td>52.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>...</td>\n",
       "      <td>1.0</td>\n",
       "      <td>27751.0</td>\n",
       "      <td>5013.306667</td>\n",
       "      <td>5013.31</td>\n",
       "      <td>13.31</td>\n",
       "      <td>0.0</td>\n",
       "      <td>5000.0</td>\n",
       "      <td>33800.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>0.0</td>\n",
       "      <td>3.0</td>\n",
       "      <td>86.0</td>\n",
       "      <td>51000.0</td>\n",
       "      <td>4541.0</td>\n",
       "      <td>2498.0</td>\n",
       "      <td>0.1</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>...</td>\n",
       "      <td>0.0</td>\n",
       "      <td>24001.0</td>\n",
       "      <td>1032.903619</td>\n",
       "      <td>1032.90</td>\n",
       "      <td>32.90</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1000.0</td>\n",
       "      <td>2500.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>0.0</td>\n",
       "      <td>2.0</td>\n",
       "      <td>41.0</td>\n",
       "      <td>62000.0</td>\n",
       "      <td>18520.0</td>\n",
       "      <td>18310.0</td>\n",
       "      <td>51.2</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>...</td>\n",
       "      <td>0.0</td>\n",
       "      <td>21976.0</td>\n",
       "      <td>20009.805556</td>\n",
       "      <td>19984.79</td>\n",
       "      <td>9.81</td>\n",
       "      <td>0.0</td>\n",
       "      <td>20000.0</td>\n",
       "      <td>39500.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>0.0</td>\n",
       "      <td>11.0</td>\n",
       "      <td>60.0</td>\n",
       "      <td>30000.0</td>\n",
       "      <td>5990.0</td>\n",
       "      <td>7898.0</td>\n",
       "      <td>7.1</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>...</td>\n",
       "      <td>5.0</td>\n",
       "      <td>62319.0</td>\n",
       "      <td>24334.736554</td>\n",
       "      <td>24334.74</td>\n",
       "      <td>334.74</td>\n",
       "      <td>0.0</td>\n",
       "      <td>24000.0</td>\n",
       "      <td>12000.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>5 rows × 71 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "   acc_now_delinq  acc_open_past_24mths  all_util  annual_inc  avg_cur_bal  \\\n",
       "0             0.0                   3.0       1.0     52000.0      25099.0   \n",
       "1             0.0                   2.0      47.0     55000.0       2222.0   \n",
       "2             0.0                   3.0      86.0     51000.0       4541.0   \n",
       "3             0.0                   2.0      41.0     62000.0      18520.0   \n",
       "4             0.0                  11.0      60.0     30000.0       5990.0   \n",
       "\n",
       "   bc_open_to_buy  bc_util  chargeoff_within_12_mths  \\\n",
       "0         30359.0      0.5                       0.0   \n",
       "1         10551.0     52.0                       0.0   \n",
       "2          2498.0      0.1                       0.0   \n",
       "3         18310.0     51.2                       0.0   \n",
       "4          7898.0      7.1                       0.0   \n",
       "\n",
       "   collections_12_mths_ex_med  delinq_amnt  ...  total_cu_tl  \\\n",
       "0                         0.0          0.0  ...          2.0   \n",
       "1                         0.0          0.0  ...          1.0   \n",
       "2                         0.0          0.0  ...          0.0   \n",
       "3                         0.0          0.0  ...          0.0   \n",
       "4                         0.0          0.0  ...          5.0   \n",
       "\n",
       "   total_il_high_credit_limit   total_pymnt  total_pymnt_inv  total_rec_int  \\\n",
       "0                         0.0   3011.577285          3011.58          11.58   \n",
       "1                     27751.0   5013.306667          5013.31          13.31   \n",
       "2                     24001.0   1032.903619          1032.90          32.90   \n",
       "3                     21976.0  20009.805556         19984.79           9.81   \n",
       "4                     62319.0  24334.736554         24334.74         334.74   \n",
       "\n",
       "   total_rec_late_fee  total_rec_prncp  total_rev_hi_lim  delinq_2yrs  \\\n",
       "0                 0.0           3000.0           31000.0          0.0   \n",
       "1                 0.0           5000.0           33800.0          0.0   \n",
       "2                 0.0           1000.0            2500.0          0.0   \n",
       "3                 0.0          20000.0           39500.0          0.0   \n",
       "4                 0.0          24000.0           12000.0          0.0   \n",
       "\n",
       "   bad_loans  \n",
       "0          0  \n",
       "1          0  \n",
       "2          0  \n",
       "3          0  \n",
       "4          0  \n",
       "\n",
       "[5 rows x 71 columns]"
      ]
     },
     "execution_count": 23,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "loans = pd.read_csv('Assignment2Data.csv', low_memory=False, header=0)\n",
    "#loans = pd.read_csv(\"./Assignment2Data.csv\", low_memory=False, header=0)\n",
    "loans.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Split the dataset\n",
    "X = loans.drop('bad_loans', axis=1) # Here no need to set inplace=True\n",
    "y = loans['bad_loans']\n",
    "\n",
    "X_train_noscale, X_valid_noscale, y_train, y_valid = train_test_split(X, y, test_size=0.3, random_state=1234)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Normalize features\n",
    "scaler = StandardScaler()\n",
    "X_train = scaler.fit_transform(X_train_noscale)\n",
    "X_valid = scaler.transform(X_valid_noscale)\n",
    "\n",
    "X_train = pd.DataFrame(X_train, index=X_train_noscale.index, columns=X_train_noscale.columns)\n",
    "X_valid = pd.DataFrame(X_valid, index=X_valid_noscale.index, columns=X_valid_noscale.columns)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Initialize logistic regression model with L1 penalty: Lasso model\n",
    "model = LogisticRegression(fit_intercept=True, penalty='l1', random_state=1234)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {},
   "outputs": [],
   "source": [
    "# A list of regularization paramaters/lambdas\n",
    "lbd = np.logspace(-3, 3, 100)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Find lambdas that limit to exactly 10 coefficients\n",
    "model = LogisticRegression(fit_intercept=True, penalty='l1', random_state=1234)\n",
    "\n",
    "coefs = [] \n",
    "parameters = []\n",
    "\n",
    "# Train the model with different regularisation strengths\n",
    "for a in lbd:\n",
    "    model.set_params(C=1/a) # Note that C is inverse value of regularization factor lambda\n",
    "    model.fit(X_train, y_train)\n",
    "    coefficients = model.coef_[0]\n",
    "    count = len(list(filter(lambda f: f!=0, coefficients)))\n",
    "    \n",
    "    if count == 10:\n",
    "        coefs.append(coefficients)\n",
    "        parameters.append(a)  "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "metadata": {},
   "outputs": [],
   "source": [
    "max_accuracy = 0\n",
    "best_lambda = 0\n",
    "\n",
    "for a in parameters:\n",
    "    model.set_params(C=1/a) # Note that C is inverse value of regularization factor lambda\n",
    "    model.fit(X_train, y_train)\n",
    "    y_pred = model.predict(X_valid)\n",
    "    accuracy = accuracy_score(y_valid, y_pred)\n",
    "    \n",
    "    if accuracy >= max_accuracy:\n",
    "        max_accuracy = accuracy\n",
    "        best_lambda = a"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "30.538555088334185"
      ]
     },
     "execution_count": 30,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "best_lambda"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Lasso model: \n",
      "\n",
      "Confusion Matrix: \n",
      " [[730  19]\n",
      " [ 98  51]]\n",
      "True negative: 730, False positive: 19, False negative: 98, True positive: 51\n",
      "Accuracy:  0.8697104677060133\n",
      "Sensitivity:  0.3422818791946309\n",
      "Precision:  0.7285714285714285\n",
      "MCC:  0.43975650303673863\n"
     ]
    }
   ],
   "source": [
    "# Logistic regression model:\n",
    "lasso = LogisticRegression(fit_intercept=True, penalty='l1', C=1/best_lambda, random_state=1234)\n",
    "lasso_model = lasso.fit(X_train, y_train)\n",
    "\n",
    "# Get predicted labels for validation data\n",
    "y_pred = lasso_model.predict(X_valid)\n",
    "\n",
    "# Create confusion matrix of validation data\n",
    "cm = confusion_matrix(y_valid, y_pred)\n",
    "TN, FP, FN, TP = cm.ravel()\n",
    "print(\"Lasso model: \\n\")\n",
    "print(\"Confusion Matrix: \\n\", cm)\n",
    "print(\"True negative: %s, False positive: %s, False negative: %s, True positive: %s\" %(TN, FP, FN, TP))\n",
    "\n",
    "# Performance of model on validation data\n",
    "print(\"Accuracy: \", accuracy_score(y_valid, y_pred))\n",
    "print(\"Sensitivity: \", recall_score(y_valid, y_pred))\n",
    "print(\"Precision: \", precision_score(y_valid, y_pred))\n",
    "print(\"MCC: \", matthews_corrcoef(y_valid, y_pred))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([ 0.        ,  0.        ,  0.        ,  0.        ,  0.        ,\n",
       "        0.        ,  0.        ,  0.        ,  0.        ,  0.        ,\n",
       "        0.        ,  0.        ,  0.        ,  0.        ,  0.        ,\n",
       "        0.28425555,  0.        ,  0.        ,  0.        ,  0.        ,\n",
       "        0.        ,  0.        ,  0.03167739,  0.        ,  0.        ,\n",
       "        0.08794408,  0.        ,  0.        ,  0.        ,  0.        ,\n",
       "        0.        ,  0.        ,  0.        ,  0.        ,  0.        ,\n",
       "        0.        ,  0.        ,  0.        ,  0.        ,  0.        ,\n",
       "        0.        ,  0.        ,  0.        ,  0.        ,  0.        ,\n",
       "        0.        ,  0.67901033,  0.        ,  0.        ,  0.        ,\n",
       "        0.        ,  0.        ,  0.        ,  0.02297276,  0.        ,\n",
       "        0.        , -0.05158867,  0.        ,  0.        ,  0.        ,\n",
       "        0.        ,  0.        ,  0.        , -0.52446019, -1.0063375 ,\n",
       "       -0.23866499,  0.        ,  0.        ,  0.        ,  0.00505925])"
      ]
     },
     "execution_count": 32,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "lasso_coefficients = lasso_model.coef_[0]\n",
    "lasso_coefficients"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[15, 22, 25, 46, 53, 56, 63, 64, 65, 69]"
      ]
     },
     "execution_count": 33,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "index = [i for i in range(len(lasso_coefficients)) if lasso_coefficients[i]!=0]\n",
    "index"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 34,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['acc_now_delinq',\n",
       " 'acc_open_past_24mths',\n",
       " 'all_util',\n",
       " 'annual_inc',\n",
       " 'avg_cur_bal',\n",
       " 'bc_open_to_buy',\n",
       " 'bc_util',\n",
       " 'chargeoff_within_12_mths',\n",
       " 'collections_12_mths_ex_med',\n",
       " 'delinq_amnt',\n",
       " 'dti',\n",
       " 'inq_fi',\n",
       " 'inq_last_12m',\n",
       " 'inq_last_6mths',\n",
       " 'installment',\n",
       " 'int_rate',\n",
       " 'last_pymnt_amnt',\n",
       " 'loan_amnt',\n",
       " 'max_bal_bc',\n",
       " 'mo_sin_old_il_acct',\n",
       " 'mo_sin_old_rev_tl_op',\n",
       " 'mo_sin_rcnt_rev_tl_op',\n",
       " 'mo_sin_rcnt_tl',\n",
       " 'mort_acc',\n",
       " 'mths_since_last_delinq',\n",
       " 'mths_since_rcnt_il',\n",
       " 'num_accts_ever_120_pd',\n",
       " 'num_actv_bc_tl',\n",
       " 'num_actv_rev_tl',\n",
       " 'num_bc_sats',\n",
       " 'num_bc_tl',\n",
       " 'num_il_tl',\n",
       " 'num_op_rev_tl',\n",
       " 'num_rev_accts',\n",
       " 'num_rev_tl_bal_gt_0',\n",
       " 'num_sats',\n",
       " 'num_tl_30dpd',\n",
       " 'num_tl_90g_dpd_24m',\n",
       " 'num_tl_op_past_12m',\n",
       " 'open_acc',\n",
       " 'open_acc_6m',\n",
       " 'open_act_il',\n",
       " 'open_il_12m',\n",
       " 'open_il_24m',\n",
       " 'open_rv_12m',\n",
       " 'open_rv_24m',\n",
       " 'out_prncp',\n",
       " 'out_prncp_inv',\n",
       " 'pct_tl_nvr_dlq',\n",
       " 'percent_bc_gt_75',\n",
       " 'pub_rec_bankruptcies',\n",
       " 'revol_bal',\n",
       " 'revol_util',\n",
       " 'tax_liens',\n",
       " 'tot_coll_amt',\n",
       " 'tot_cur_bal',\n",
       " 'tot_hi_cred_lim',\n",
       " 'total_acc',\n",
       " 'total_bal_ex_mort',\n",
       " 'total_bal_il',\n",
       " 'total_bc_limit',\n",
       " 'total_cu_tl',\n",
       " 'total_il_high_credit_limit',\n",
       " 'total_pymnt',\n",
       " 'total_pymnt_inv',\n",
       " 'total_rec_int',\n",
       " 'total_rec_late_fee',\n",
       " 'total_rec_prncp',\n",
       " 'total_rev_hi_lim',\n",
       " 'delinq_2yrs']"
      ]
     },
     "execution_count": 34,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "feature_full_list = list(loans.drop('bad_loans', axis=1).columns)\n",
    "feature_full_list"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 35,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['int_rate',\n",
       " 'mo_sin_rcnt_tl',\n",
       " 'mths_since_rcnt_il',\n",
       " 'out_prncp',\n",
       " 'tax_liens',\n",
       " 'tot_hi_cred_lim',\n",
       " 'total_pymnt',\n",
       " 'total_pymnt_inv',\n",
       " 'total_rec_int',\n",
       " 'delinq_2yrs']"
      ]
     },
     "execution_count": 35,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "selected_feature = [feature_full_list[j] for j in index]\n",
    "selected_feature"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 36,
   "metadata": {},
   "outputs": [],
   "source": [
    "acc = []\n",
    "mcc = []\n",
    "kf = KFold(n_splits=5)\n",
    "for train_index, test_index in kf.split(X_train):\n",
    "    lasso = LogisticRegression(fit_intercept=True, penalty='l1', C=1/best_lambda, random_state=1234)\n",
    "    lasso_model = lasso.fit(X_train.iloc[train_index], y_train.iloc[train_index])\n",
    "    predictions = lasso_model.predict(X_train.iloc[test_index])\n",
    "    acc.append(accuracy_score(y_train.iloc[test_index], predictions))\n",
    "    mcc.append(matthews_corrcoef(y_train.iloc[test_index], predictions))\n",
    "    "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 37,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "average accuracy is: 0.8648605131835881\n",
      "average mcc is: 0.39411665108756716\n"
     ]
    }
   ],
   "source": [
    "from statistics import mean\n",
    "print(\"average accuracy is: \" + str(mean(acc)))\n",
    "print(\"average mcc is: \" + str(mean(mcc)))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.3"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}