HCA 203 PUF Counties Aggregate - Classification.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# PUF Counties Aggregate - Classification \n",
    "\n",
    "DeSYNPuf Counties Aggregate with Elixhauser Sums per User & RWJF Factor Rankings Dataset"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [],
   "source": [
    "%matplotlib inline\n",
    "import seaborn as sb\n",
    "import numpy as np\n",
    "import pandas as pd\n",
    "import matplotlib.pyplot as plt\n",
    "\n",
    "import sklearn.preprocessing as pre"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "The file contains aggregated Medicare data from 2008-2010 (DeSYNPuf Synthetic Public Use Files), and RWJF County Health Factors and Outcomes Ranking Data from 2010.\n",
    "\n",
    "2694 out of 3012 counties will be analyzed.\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {
    "scrolled": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(2947, 73)"
      ]
     },
     "execution_count": 25,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "medicare_counties = pd.read_csv(\"/home/kylier/python/DS/data/PUF/PUF_RWJF_MERGED1.csv\")\n",
    "medicare_counties.shape\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Remove a minimal # of outliers."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(2840, 73)"
      ]
     },
     "execution_count": 26,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "medicare_counties = medicare_counties[medicare_counties[\"TOT_COST_PB\"]<200000]\n",
    "medicare_counties = medicare_counties[medicare_counties[\"CNTY_BENE_USRS\"]>0]\n",
    "\n",
    "medicare_counties.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "<class 'pandas.core.frame.DataFrame'>\n",
      "Int64Index: 2840 entries, 38 to 2946\n",
      "Data columns (total 73 columns):\n",
      " #   Column                      Non-Null Count  Dtype  \n",
      "---  ------                      --------------  -----  \n",
      " 0   FIPS_STATE_CODE             2840 non-null   int64  \n",
      " 1   FIPS_COUNTY_CODE            2840 non-null   int64  \n",
      " 2   CNTY_BENE_CT                2840 non-null   int64  \n",
      " 3   CNTY_BENE_USRS              2840 non-null   int64  \n",
      " 4   CNTY_MEAN_AGE               2840 non-null   float64\n",
      " 5   CNTY_SEX_PROP               2840 non-null   float64\n",
      " 6   TOT_CNCR_CASES              2840 non-null   int64  \n",
      " 7   TOT_STRKETIA_CASES          2840 non-null   int64  \n",
      " 8   TOT_ESRD_CASES              2840 non-null   int64  \n",
      " 9   CNTY_TOT_VSTS               2840 non-null   int64  \n",
      " 10  CNTY_TOT_IP_VSTS            2840 non-null   int64  \n",
      " 11  CNTY_TOT_OP_VSTS            2840 non-null   int64  \n",
      " 12  CNTY_TOT_CODES              2840 non-null   float64\n",
      " 13  CNTY_TOT_PROCS              2840 non-null   float64\n",
      " 14  CNTY_MEAN_IP_STAY           2840 non-null   float64\n",
      " 15  CBSA                        1071 non-null   float64\n",
      " 16  CNTY_TOT_CCS                2840 non-null   int64  \n",
      " 17  CNTY_MEAN_CCS               2840 non-null   float64\n",
      " 18  CNTY_TOT_NEW_CCS            2840 non-null   int64  \n",
      " 19  CNTY_MEAN_NEW_CCS           2840 non-null   float64\n",
      " 20  CNTY_SUM_ELIX_SCORES        2840 non-null   float64\n",
      " 21  CNTY_TOT_IP_COST            2840 non-null   int64  \n",
      " 22  CNTY_TOT_OP_COST            2840 non-null   int64  \n",
      " 23  CNTY_ALL_COSTS              2840 non-null   int64  \n",
      " 24  CNTY_PERCENT_COSTS          2840 non-null   float64\n",
      " 25  CNTY_USR_PROP               2840 non-null   float64\n",
      " 26  CNTY_IP_PROP                2840 non-null   float64\n",
      " 27  CNTY_AGE_95CIL              2687 non-null   float64\n",
      " 28  CNTY_AGE_95CIU              2687 non-null   float64\n",
      " 29  CNTY_PROP_NH_WHITE          2840 non-null   float64\n",
      " 30  CNTY_PROP_AA_BLACK          2840 non-null   float64\n",
      " 31  CNTY_PROP_OTHER             2840 non-null   float64\n",
      " 32  CNTY_PROP_NW_HISPANIC       2840 non-null   float64\n",
      " 33  TOT_CNCR_PREV               2840 non-null   float64\n",
      " 34  TOT_STRK_PREV               2840 non-null   float64\n",
      " 35  TOT_ESRD_PREV               2840 non-null   float64\n",
      " 36  TOT_CVRG_MOS_PB_PY          2840 non-null   float64\n",
      " 37  TOT_RXCVRG_MOS_PB_PY        2840 non-null   float64\n",
      " 38  TOT_VSTS_PB                 2840 non-null   float64\n",
      " 39  TOT_VSTS_PU                 2840 non-null   float64\n",
      " 40  TOT_IP_VSTS_PU              2840 non-null   float64\n",
      " 41  TOT_OP_VSTS_PU              2840 non-null   float64\n",
      " 42  CODES_PU                    2840 non-null   float64\n",
      " 43  PROCS_PU                    2840 non-null   float64\n",
      " 44  PRVDRS_PU                   2840 non-null   float64\n",
      " 45  TOT_IPDAYS_PU               2840 non-null   float64\n",
      " 46  TOT_CC_PREV                 2840 non-null   float64\n",
      " 47  TOT_CCS_PU                  2840 non-null   object \n",
      " 48  TOT_CC_INCD                 2840 non-null   float64\n",
      " 49  TOT_NEW_CC_PU               2840 non-null   object \n",
      " 50  TOT_ELIX_PU                 2840 non-null   float64\n",
      " 51  TOT_IP_COST_PB              2840 non-null   float64\n",
      " 52  TOT_OP_COST_PB              2840 non-null   float64\n",
      " 53  TOT_COST_PB                 2840 non-null   float64\n",
      " 54  FIPS                        2840 non-null   int64  \n",
      " 55  STATE                       2840 non-null   object \n",
      " 56  COUNTY                      2840 non-null   object \n",
      " 57  NUM_COUNTIES                2840 non-null   int64  \n",
      " 58  HO_RANK                     2840 non-null   int64  \n",
      " 59  HO_PROP_RANK                2840 non-null   float64\n",
      " 60  HO_QUARTILE                 2840 non-null   int64  \n",
      " 61  HF_RANK                     2840 non-null   int64  \n",
      " 62  HF_PROP_RANK                2840 non-null   float64\n",
      " 63  HF_QUARTILE                 2840 non-null   int64  \n",
      " 64  HF_HIGH                     2840 non-null   int64  \n",
      " 65  HIGH_SEV                    2840 non-null   int64  \n",
      " 66  HIGH_PROCS                  2840 non-null   int64  \n",
      " 67  DIVERSE_PROP                2840 non-null   float64\n",
      " 68  TOT_COST_PB Predicted       2840 non-null   float64\n",
      " 69  Lower 95% Mean TOT_COST_PB  2840 non-null   float64\n",
      " 70  Upper 95% Mean TOT_COST_PB  2840 non-null   float64\n",
      " 71  Residuals                   2840 non-null   float64\n",
      " 72  HIGHER_COST                 2840 non-null   float64\n",
      "dtypes: float64(45), int64(24), object(4)\n",
      "memory usage: 1.6+ MB\n"
     ]
    }
   ],
   "source": [
    "medicare_counties.info()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {},
   "outputs": [],
   "source": [
    "medicare_counties[\"SR_TOT_COST_PB\"] = np.sqrt(medicare_counties[\"TOT_COST_PB\"])\n",
    "medicare_counties[\"SR_TOT_IPDAYS\"] = np.sqrt(medicare_counties[\"TOT_IPDAYS_PU\"])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "count    2840.000000\n",
       "mean      211.937052\n",
       "std        83.796688\n",
       "min         3.162278\n",
       "25%       158.568031\n",
       "50%       215.473854\n",
       "75%       264.406390\n",
       "max       445.368013\n",
       "Name: SR_TOT_COST_PB, dtype: float64"
      ]
     },
     "execution_count": 29,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "medicare_counties[\"SR_TOT_COST_PB\"].describe()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Classification\n",
    "\n",
    "In this notebook, I will show how one can use the Random Forest Classifier to try to predict high cost counties.\n",
    "\n",
    "A high cost county will be defined as counties where SR[TOT_COST_PB] > 265 (which is the 75% level). "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.metrics import accuracy_score\n",
    "from sklearn.metrics import precision_score\n",
    "from sklearn.metrics import recall_score\n",
    "from sklearn.metrics import confusion_matrix\n",
    "from sklearn.metrics import roc_auc_score\n",
    "from sklearn.metrics import f1_score\n",
    "\n",
    "from sklearn.model_selection import train_test_split\n",
    "\n",
    "from sklearn.ensemble import RandomForestClassifier"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(2840, 19)"
      ]
     },
     "execution_count": 31,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Again, we will begin with Age, Sex and Race indicators in the model.\n",
    "# The classification model will rate the importance of the variables after it uses them to make decisions.\n",
    "\n",
    "medicare_counties.loc[medicare_counties[\"SR_TOT_COST_PB\"]>265, \"B_COST\"] = 1\n",
    "medicare_counties.loc[medicare_counties[\"SR_TOT_COST_PB\"]<=265, \"B_COST\"] = 0\n",
    "\n",
    "\n",
    "df = medicare_counties[[\"CNTY_MEAN_AGE\", \"CNTY_SEX_PROP\", \"CNTY_USR_PROP\", \"SR_TOT_IPDAYS\", \"TOT_ESRD_PREV\", \n",
    "                        \"TOT_STRK_PREV\", \"TOT_CVRG_MOS_PB_PY\", \"TOT_RXCVRG_MOS_PB_PY\",\n",
    "                        \"TOT_IP_VSTS_PU\", \"TOT_OP_VSTS_PU\", \"CODES_PU\", \"PROCS_PU\", \"PRVDRS_PU\", \n",
    "                        \"TOT_CC_PREV\", \"TOT_CC_INCD\", \"TOT_ELIX_PU\", \"HO_PROP_RANK\", \"HF_PROP_RANK\", \"B_COST\"]]\n",
    "\n",
    "df = df.dropna()\n",
    "df.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.0    2138\n",
       "1.0     702\n",
       "Name: B_COST, dtype: int64"
      ]
     },
     "execution_count": 32,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df[\"B_COST\"].value_counts()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>CNTY_MEAN_AGE</th>\n",
       "      <th>CNTY_SEX_PROP</th>\n",
       "      <th>CNTY_USR_PROP</th>\n",
       "      <th>SR_TOT_IPDAYS</th>\n",
       "      <th>TOT_ESRD_PREV</th>\n",
       "      <th>TOT_STRK_PREV</th>\n",
       "      <th>TOT_CVRG_MOS_PB_PY</th>\n",
       "      <th>TOT_RXCVRG_MOS_PB_PY</th>\n",
       "      <th>TOT_IP_VSTS_PU</th>\n",
       "      <th>TOT_OP_VSTS_PU</th>\n",
       "      <th>CODES_PU</th>\n",
       "      <th>PROCS_PU</th>\n",
       "      <th>PRVDRS_PU</th>\n",
       "      <th>TOT_CC_PREV</th>\n",
       "      <th>TOT_CC_INCD</th>\n",
       "      <th>TOT_ELIX_PU</th>\n",
       "      <th>HO_PROP_RANK</th>\n",
       "      <th>HF_PROP_RANK</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>-0.729858</td>\n",
       "      <td>-0.262895</td>\n",
       "      <td>0.747347</td>\n",
       "      <td>0.834346</td>\n",
       "      <td>0.165622</td>\n",
       "      <td>-0.039389</td>\n",
       "      <td>0.955315</td>\n",
       "      <td>0.860709</td>\n",
       "      <td>0.715453</td>\n",
       "      <td>-0.259966</td>\n",
       "      <td>0.029110</td>\n",
       "      <td>-0.416061</td>\n",
       "      <td>0.686354</td>\n",
       "      <td>0.456735</td>\n",
       "      <td>0.213309</td>\n",
       "      <td>0.012173</td>\n",
       "      <td>1.371045</td>\n",
       "      <td>0.544515</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>-1.129097</td>\n",
       "      <td>-0.011970</td>\n",
       "      <td>0.181928</td>\n",
       "      <td>0.509831</td>\n",
       "      <td>-0.484579</td>\n",
       "      <td>0.060394</td>\n",
       "      <td>-1.303923</td>\n",
       "      <td>-0.524658</td>\n",
       "      <td>-0.036027</td>\n",
       "      <td>-0.494207</td>\n",
       "      <td>-0.050537</td>\n",
       "      <td>0.018642</td>\n",
       "      <td>0.885336</td>\n",
       "      <td>0.174381</td>\n",
       "      <td>0.004361</td>\n",
       "      <td>0.267322</td>\n",
       "      <td>-1.253922</td>\n",
       "      <td>-1.426189</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>-0.615790</td>\n",
       "      <td>0.301686</td>\n",
       "      <td>0.959379</td>\n",
       "      <td>0.712985</td>\n",
       "      <td>3.091529</td>\n",
       "      <td>0.858659</td>\n",
       "      <td>-0.328884</td>\n",
       "      <td>1.281978</td>\n",
       "      <td>-0.146414</td>\n",
       "      <td>-0.763829</td>\n",
       "      <td>-0.653956</td>\n",
       "      <td>-1.077540</td>\n",
       "      <td>-0.710588</td>\n",
       "      <td>0.648007</td>\n",
       "      <td>0.437182</td>\n",
       "      <td>0.093614</td>\n",
       "      <td>-0.009056</td>\n",
       "      <td>1.204892</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>-0.558756</td>\n",
       "      <td>-0.827475</td>\n",
       "      <td>0.535315</td>\n",
       "      <td>-0.648065</td>\n",
       "      <td>0.165622</td>\n",
       "      <td>-0.338738</td>\n",
       "      <td>0.646156</td>\n",
       "      <td>0.275963</td>\n",
       "      <td>-0.857306</td>\n",
       "      <td>0.737170</td>\n",
       "      <td>0.229229</td>\n",
       "      <td>0.936890</td>\n",
       "      <td>-0.721177</td>\n",
       "      <td>0.620682</td>\n",
       "      <td>-0.010564</td>\n",
       "      <td>-0.498849</td>\n",
       "      <td>-1.319806</td>\n",
       "      <td>-1.457470</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>-6.585360</td>\n",
       "      <td>0.301686</td>\n",
       "      <td>-1.585008</td>\n",
       "      <td>-2.200046</td>\n",
       "      <td>-1.297331</td>\n",
       "      <td>-0.937437</td>\n",
       "      <td>0.955315</td>\n",
       "      <td>-3.584621</td>\n",
       "      <td>-1.691831</td>\n",
       "      <td>0.790460</td>\n",
       "      <td>-0.557957</td>\n",
       "      <td>-0.624332</td>\n",
       "      <td>1.089553</td>\n",
       "      <td>-0.007783</td>\n",
       "      <td>-2.383619</td>\n",
       "      <td>-1.636429</td>\n",
       "      <td>-1.520927</td>\n",
       "      <td>-1.582594</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2125</th>\n",
       "      <td>-0.596779</td>\n",
       "      <td>-1.109765</td>\n",
       "      <td>0.005234</td>\n",
       "      <td>-0.800672</td>\n",
       "      <td>-1.297331</td>\n",
       "      <td>1.307683</td>\n",
       "      <td>0.955315</td>\n",
       "      <td>0.376565</td>\n",
       "      <td>-0.919122</td>\n",
       "      <td>-2.318118</td>\n",
       "      <td>-1.997932</td>\n",
       "      <td>-1.866746</td>\n",
       "      <td>-2.087166</td>\n",
       "      <td>-0.212718</td>\n",
       "      <td>-0.704571</td>\n",
       "      <td>-1.408211</td>\n",
       "      <td>-1.461978</td>\n",
       "      <td>-0.706726</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2126</th>\n",
       "      <td>-0.900960</td>\n",
       "      <td>0.301686</td>\n",
       "      <td>-0.428468</td>\n",
       "      <td>1.891779</td>\n",
       "      <td>0.365116</td>\n",
       "      <td>0.695378</td>\n",
       "      <td>-0.406714</td>\n",
       "      <td>0.736672</td>\n",
       "      <td>0.317211</td>\n",
       "      <td>-0.275338</td>\n",
       "      <td>-0.026766</td>\n",
       "      <td>0.055479</td>\n",
       "      <td>0.327141</td>\n",
       "      <td>0.215782</td>\n",
       "      <td>-0.124536</td>\n",
       "      <td>0.333978</td>\n",
       "      <td>-1.063204</td>\n",
       "      <td>-0.546845</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2127</th>\n",
       "      <td>-0.368642</td>\n",
       "      <td>0.301686</td>\n",
       "      <td>-0.748039</td>\n",
       "      <td>0.080655</td>\n",
       "      <td>-0.527356</td>\n",
       "      <td>-0.228452</td>\n",
       "      <td>-0.781357</td>\n",
       "      <td>-0.407730</td>\n",
       "      <td>-0.339591</td>\n",
       "      <td>0.110459</td>\n",
       "      <td>-0.161964</td>\n",
       "      <td>-0.061730</td>\n",
       "      <td>1.049844</td>\n",
       "      <td>-0.805949</td>\n",
       "      <td>-0.262716</td>\n",
       "      <td>0.130254</td>\n",
       "      <td>0.365444</td>\n",
       "      <td>0.662687</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2128</th>\n",
       "      <td>0.201699</td>\n",
       "      <td>0.084539</td>\n",
       "      <td>1.595476</td>\n",
       "      <td>-0.074243</td>\n",
       "      <td>0.390692</td>\n",
       "      <td>0.444176</td>\n",
       "      <td>0.927874</td>\n",
       "      <td>0.420094</td>\n",
       "      <td>-0.443610</td>\n",
       "      <td>-0.593028</td>\n",
       "      <td>-0.661340</td>\n",
       "      <td>-0.645971</td>\n",
       "      <td>-0.120044</td>\n",
       "      <td>0.685841</td>\n",
       "      <td>0.199532</td>\n",
       "      <td>-0.491477</td>\n",
       "      <td>-0.969579</td>\n",
       "      <td>-1.255881</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2129</th>\n",
       "      <td>1.646563</td>\n",
       "      <td>-2.521217</td>\n",
       "      <td>1.595476</td>\n",
       "      <td>0.120554</td>\n",
       "      <td>1.140925</td>\n",
       "      <td>2.056057</td>\n",
       "      <td>0.955315</td>\n",
       "      <td>-0.189319</td>\n",
       "      <td>-0.919122</td>\n",
       "      <td>0.679440</td>\n",
       "      <td>-0.333961</td>\n",
       "      <td>-0.296147</td>\n",
       "      <td>-0.657643</td>\n",
       "      <td>1.085201</td>\n",
       "      <td>-2.383619</td>\n",
       "      <td>-1.056377</td>\n",
       "      <td>1.024286</td>\n",
       "      <td>0.294267</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>2130 rows × 18 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "      CNTY_MEAN_AGE  CNTY_SEX_PROP  CNTY_USR_PROP  SR_TOT_IPDAYS  \\\n",
       "0         -0.729858      -0.262895       0.747347       0.834346   \n",
       "1         -1.129097      -0.011970       0.181928       0.509831   \n",
       "2         -0.615790       0.301686       0.959379       0.712985   \n",
       "3         -0.558756      -0.827475       0.535315      -0.648065   \n",
       "4         -6.585360       0.301686      -1.585008      -2.200046   \n",
       "...             ...            ...            ...            ...   \n",
       "2125      -0.596779      -1.109765       0.005234      -0.800672   \n",
       "2126      -0.900960       0.301686      -0.428468       1.891779   \n",
       "2127      -0.368642       0.301686      -0.748039       0.080655   \n",
       "2128       0.201699       0.084539       1.595476      -0.074243   \n",
       "2129       1.646563      -2.521217       1.595476       0.120554   \n",
       "\n",
       "      TOT_ESRD_PREV  TOT_STRK_PREV  TOT_CVRG_MOS_PB_PY  TOT_RXCVRG_MOS_PB_PY  \\\n",
       "0          0.165622      -0.039389            0.955315              0.860709   \n",
       "1         -0.484579       0.060394           -1.303923             -0.524658   \n",
       "2          3.091529       0.858659           -0.328884              1.281978   \n",
       "3          0.165622      -0.338738            0.646156              0.275963   \n",
       "4         -1.297331      -0.937437            0.955315             -3.584621   \n",
       "...             ...            ...                 ...                   ...   \n",
       "2125      -1.297331       1.307683            0.955315              0.376565   \n",
       "2126       0.365116       0.695378           -0.406714              0.736672   \n",
       "2127      -0.527356      -0.228452           -0.781357             -0.407730   \n",
       "2128       0.390692       0.444176            0.927874              0.420094   \n",
       "2129       1.140925       2.056057            0.955315             -0.189319   \n",
       "\n",
       "      TOT_IP_VSTS_PU  TOT_OP_VSTS_PU  CODES_PU  PROCS_PU  PRVDRS_PU  \\\n",
       "0           0.715453       -0.259966  0.029110 -0.416061   0.686354   \n",
       "1          -0.036027       -0.494207 -0.050537  0.018642   0.885336   \n",
       "2          -0.146414       -0.763829 -0.653956 -1.077540  -0.710588   \n",
       "3          -0.857306        0.737170  0.229229  0.936890  -0.721177   \n",
       "4          -1.691831        0.790460 -0.557957 -0.624332   1.089553   \n",
       "...              ...             ...       ...       ...        ...   \n",
       "2125       -0.919122       -2.318118 -1.997932 -1.866746  -2.087166   \n",
       "2126        0.317211       -0.275338 -0.026766  0.055479   0.327141   \n",
       "2127       -0.339591        0.110459 -0.161964 -0.061730   1.049844   \n",
       "2128       -0.443610       -0.593028 -0.661340 -0.645971  -0.120044   \n",
       "2129       -0.919122        0.679440 -0.333961 -0.296147  -0.657643   \n",
       "\n",
       "      TOT_CC_PREV  TOT_CC_INCD  TOT_ELIX_PU  HO_PROP_RANK  HF_PROP_RANK  \n",
       "0        0.456735     0.213309     0.012173      1.371045      0.544515  \n",
       "1        0.174381     0.004361     0.267322     -1.253922     -1.426189  \n",
       "2        0.648007     0.437182     0.093614     -0.009056      1.204892  \n",
       "3        0.620682    -0.010564    -0.498849     -1.319806     -1.457470  \n",
       "4       -0.007783    -2.383619    -1.636429     -1.520927     -1.582594  \n",
       "...           ...          ...          ...           ...           ...  \n",
       "2125    -0.212718    -0.704571    -1.408211     -1.461978     -0.706726  \n",
       "2126     0.215782    -0.124536     0.333978     -1.063204     -0.546845  \n",
       "2127    -0.805949    -0.262716     0.130254      0.365444      0.662687  \n",
       "2128     0.685841     0.199532    -0.491477     -0.969579     -1.255881  \n",
       "2129     1.085201    -2.383619    -1.056377      1.024286      0.294267  \n",
       "\n",
       "[2130 rows x 18 columns]"
      ]
     },
     "execution_count": 33,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "train, test = train_test_split(df, test_size=0.25, random_state=121)\n",
    "\n",
    "train_x = train[[\"CNTY_MEAN_AGE\", \"CNTY_SEX_PROP\", \"CNTY_USR_PROP\", \"SR_TOT_IPDAYS\", \"TOT_ESRD_PREV\", \n",
    "                        \"TOT_STRK_PREV\", \"TOT_CVRG_MOS_PB_PY\", \"TOT_RXCVRG_MOS_PB_PY\",\n",
    "                        \"TOT_IP_VSTS_PU\", \"TOT_OP_VSTS_PU\", \"CODES_PU\", \"PROCS_PU\", \"PRVDRS_PU\", \n",
    "                        \"TOT_CC_PREV\", \"TOT_CC_INCD\", \"TOT_ELIX_PU\", \"HO_PROP_RANK\", \"HF_PROP_RANK\"]]\n",
    "train_y = train[\"B_COST\"]\n",
    "\n",
    "test_x = test[[\"CNTY_MEAN_AGE\", \"CNTY_SEX_PROP\", \"CNTY_USR_PROP\", \"SR_TOT_IPDAYS\", \"TOT_ESRD_PREV\", \n",
    "                        \"TOT_STRK_PREV\", \"TOT_CVRG_MOS_PB_PY\", \"TOT_RXCVRG_MOS_PB_PY\",\n",
    "                        \"TOT_IP_VSTS_PU\", \"TOT_OP_VSTS_PU\", \"CODES_PU\", \"PROCS_PU\", \"PRVDRS_PU\", \n",
    "                        \"TOT_CC_PREV\", \"TOT_CC_INCD\", \"TOT_ELIX_PU\", \"HO_PROP_RANK\", \"HF_PROP_RANK\"]]\n",
    "test_y = test[\"B_COST\"]\n",
    "\n",
    "s = pre.StandardScaler()\n",
    "train_x = s.fit_transform(train_x)\n",
    "test_x = s.fit_transform(test_x)\n",
    "\n",
    "train_x = pd.DataFrame(train_x, columns=[\"CNTY_MEAN_AGE\", \"CNTY_SEX_PROP\", \"CNTY_USR_PROP\", \"SR_TOT_IPDAYS\", \"TOT_ESRD_PREV\", \n",
    "                        \"TOT_STRK_PREV\", \"TOT_CVRG_MOS_PB_PY\", \"TOT_RXCVRG_MOS_PB_PY\",\n",
    "                        \"TOT_IP_VSTS_PU\", \"TOT_OP_VSTS_PU\", \"CODES_PU\", \"PROCS_PU\", \"PRVDRS_PU\", \n",
    "                        \"TOT_CC_PREV\", \"TOT_CC_INCD\", \"TOT_ELIX_PU\", \"HO_PROP_RANK\", \"HF_PROP_RANK\"])\n",
    "\n",
    "test_x = pd.DataFrame(test_x, columns=[\"CNTY_MEAN_AGE\", \"CNTY_SEX_PROP\", \"CNTY_USR_PROP\", \"SR_TOT_IPDAYS\", \"TOT_ESRD_PREV\", \n",
    "                        \"TOT_STRK_PREV\", \"TOT_CVRG_MOS_PB_PY\", \"TOT_RXCVRG_MOS_PB_PY\",\n",
    "                        \"TOT_IP_VSTS_PU\", \"TOT_OP_VSTS_PU\", \"CODES_PU\", \"PROCS_PU\", \"PRVDRS_PU\", \n",
    "                        \"TOT_CC_PREV\", \"TOT_CC_INCD\", \"TOT_ELIX_PU\", \"HO_PROP_RANK\", \"HF_PROP_RANK\"])\n",
    "\n",
    "\n",
    "\n",
    "train_x"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 34,
   "metadata": {},
   "outputs": [],
   "source": [
    "#The default criterion is \"gini\". We can also run with criterion \"entropy\" for information gain,\n",
    "#rather than impurity\n",
    "clf = RandomForestClassifier(min_samples_leaf=10, random_state=121)\n",
    "clf.fit(train_x, train_y)\n",
    "\n",
    "# predict on test data\n",
    "prediction = clf.predict(test_x)\n",
    "\n",
    "# compute evaluation scores\n",
    "accuracy = accuracy_score(test_y, prediction)\n",
    "precision = precision_score(test_y, prediction)\n",
    "recall = recall_score(test_y, prediction)\n",
    "cm = confusion_matrix(test_y, prediction)\n",
    "roc = roc_auc_score(test_y, prediction)\n",
    "f1 = f1_score(test_y, prediction)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 35,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.8507042253521127"
      ]
     },
     "execution_count": 35,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "accuracy"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 36,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.7851851851851852"
      ]
     },
     "execution_count": 36,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "precision"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 37,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.5792349726775956"
      ]
     },
     "execution_count": 37,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "recall"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 38,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.6666666666666666"
      ]
     },
     "execution_count": 38,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "f1"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Scores\n",
    "\n",
    "These are great scores for a simple Random Forest Classifier without any boosting.\n",
    "\n",
    "The meaning of all these scores is worth reading up on.  \n",
    "\n",
    "Now let's look at the Feature Importances of each of the variables in the model."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 39,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<AxesSubplot:>"
      ]
     },
     "execution_count": 39,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAdcAAAD4CAYAAAC+CayWAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjMuMiwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy8vihELAAAACXBIWXMAAAsTAAALEwEAmpwYAAAqq0lEQVR4nO3debwkVX3+8c8DCkLABVBAJYwQNgUclghRMOIQBIMCMQqGTWNUIsjiDxS3VwhRgwEFlEVBETGICI6AigFlCahsIwzDDMgWRgVxWFQU2Znn90edHoqe29u91XeZed6v133Rfarq1Lf7NnNunaquR7aJiIiI5iwz0QVEREQsaTK4RkRENCyDa0RERMMyuEZERDQsg2tERETDnjPRBcTksNpqq3natGkTXUZExJTx85///AHbLx5pWQbXAGDatGnMmjVrosuIiJgyJP2y07JMC0dERDQsg2tERETDMrhGREQ0LOdcA4Cb7nmIaYf/oLH+5h/19431FREx1eTIFZD0tKTZkuZKOkfSiiO0f0/SCyW9S9JZbduvJul+SctLulzSrZLmSPqFpBMkvbDDvr7XWiZpGUlfKO03SbpO0iu61Dy/rHejpIslrVHaH25b712STmju3YqIiF4yuFYetT3d9sbAE8B+I7T/DtgfmAn8XWsALv4RuMD24+X5nrY3BTYFHgfO77CvVp8AuwMvBTa1vQmwG/CHHnVvZ/vVwCzgYwO/6oiIGIoMrou7EvirEdqvAl5m+4/AFcBbasv2AM5q38D2E8CHgb+U9OpOfZbHawL32l5Ytr3b9u/7rPmKDjVHRMQEyOBaI+k5wE7ATW3tywIzgAtK01lUAyqSXgqsD1w2Up+2nwZuBDbs0ee3gbeUKePPSdpsgNJ3bq+5H5LeJ2mWpFlPP/LQoJtHREQHGVwrK0iaTTW9+ivgq23tDwKrAD8q7d8HtpH0fOAdwLllEO1EI+zrWX3avhvYAPgosBC4RNKMHnVfVvp6PvCfXdYbMbTX9im2t7S95bIrvqDHriIiol+5WrjyqO3pndolvYBqQN0f+ILtRyX9D9V50T2AQzp1XI5QNwFu6dYnQDln+0Pgh5IWALsCl3SpezvbD7TXLGm5MiUN1QDevk5ERAxRjlz7YPsh4EDgUEnPLc1nAR8CVgeuHmm7su5/Ar+2Padbn5I2L1PMSFqG6mKojrfW6uJ/gb1KPytQHVmPOGUdERHDkSPXPtm+QdKNVEeq3wAuBr4OfNV2+7TrmZIeB5YHfgzs0kef9wOnSlq+LL4WGM1XaA4CvizpQKrp6DNsX9Fro01e9gJm5bupERGN0OLjQiyNttxyS+fG/RER/ZP0c9tbjrQs08IRERENy7TwJCfpGqrp5bq9bQ/81ZuIiBgfGVwnOdtbTXQNERExmEwLR0RENCyDa0RERMMyuEZERDQs51wDaD7PtV3yXSNiaZIj14iIiIYtNYOrpFVL4sxsSb+VdE/t+V9KOl/S7ZLulHS8pOUkvam2zsMlBH22pDM67OMNkh6qbTNb0vZl2cMjrH+EpEPL4wsk7V1bdqqkw7q8nlYo+42Sfippg9I+X9JqbTV9f/TvXEREDGqpmRa2/SAwHapBDXjY9jGSBFwDnGx7l3Kj/VOAT9s+DLiobHM5cKjtXrcxutL2zqMo8UCqlJvvAa8EtgI+0GObPW3PkvQ+4GjgraPYb0RENGypOXLt4o3AY7a/BovyVw8B/lnSiuNVhO35VIP6fwEnAQfYfrLPzUcVlp4814iI4cjgCq8Cfl5vsP1HqlzXgQcsYNu2aeF1B9j2GGBHYF4/N9uveQujCEtPnmtExHAsNdPCXYiRw8Q7tfcy2mlhqGLmBGwoaRnbC3usf6akR4H5wAdL20g1J50hImIc5cgV5gHPSjWQ9HxgLeDO8SqiZLieBOwN3A78ax+b7Wl7uu1dbf+6tD0IvKi2TsLSIyLGWQZXuARYUdI+AOWCps8Bp9t+ZBzreD9wu+3LqULYPyzpxaPo53KqAbr1WvYiYekREeNqqZ8Wtm1JuwEnSfok1R8cFwIfG2WX20qaXXv+KdvnUg3gd9faP996IOklwEeArUtNv5F0PNXFTe8ecP//AZxcQtgF/A/w3702Slh6RERzEpYeQMLSIyIGlbD0iIiIcbTUTwuPhqQ3AZ9ta77L9m5D2Nd3gVe0NX/E9kVN7ysiIpqRwXUUysA2LoPbMAbsiIgYrkwLR0RENCyDa0RERMMyuEZERDQs51wDSFh6RESTcuQaERHRsK6D63gEjJf9vEbSFWXdX0j6iqRXSrq73HO3vu7ssv4RtXpulvTOtvU+VPq6qQSKf17Sc7vUMF/SlSPsa27t+TaSri39/qLkqLaWbVACzGdLukXSKV321QpVv6Gs+29t7bMlzZH043L3pk79vEvS/bX34L0lRu7s2jrPL7+f9q/zRETEkHQdXG0/WG4MPx34EnBsebwZcC5wnu31gPWBlagCxi+qbTOLZ24uv89I+5C0OnAO1Xc3NwA2orpl34PAr4Fta+tuCKxs+9rS1KpnF+DLrcFT0n7ADsDWtjcB/hq4D1ihx/uxsqS1Sh8btdW5BvBNYD/bGwLbAO+X1Jrv/EKrHtsbAV/ssa8rbW9GFRqwl6Qtau3TbW8KXAfs36Ofs8t78AbgM8D5wMslbV+WHwmcZvuuHv1ERERDRjst3GTA+P7A121fVfqy7XNtLwDOAvaorbtHaXsW27cDj/BMGszHgX+1/Yey/AnbR5Wc1m6+DexeHr+zbV/7U93M//rS5wPAh4HDy/I1gUX3DrbdV76q7T9T5ck+K/dVkoCVgd/32c99VCk+a1Ml6hwnaUtgBnD0SNskLD0iYjhGO7g2GTC+cXtfNd8GdpXUuvBqd+Bb7StJ2pwqUeY+SSsDK43ySO1c4B/K47cA36stW+w1Ux2Zv6o8Pha4VNIPJR0i6YX97FDSqlQ37J9Xmlo3/v8VsD1wWp/9rAOsA9xhew7VTS4uAQ60/cRI2yQsPSJiOEY7uDYdMD4i27+lGnRmSJoOPGl7bm2VQyTdClwDHDFSDbVzwPMlvbbHLn8H/F7SHsAtVEfDi7qiSxB5OYrfiGqK+w3A1ZKW77KvbSXdAFwMHGW7Nbi2poXXAr5GlYzTze5lMD4LeL/t35X2E4F7bCduLiJinI12cG0yYHwesEWX5a2p4ZGmhI8t52l3B86Q9LxyBP3n1gU8rXPAwFxguT7qOZtqYGrf12KvudR9c+uJ7d/YPs32LsBTVEflnVxpezPbW9j+Uod1LgBe36veMhhvZfu7tfaF5SciIsbZaL/neglwlKR9bJ+hsQWMnwBcK+kHtq8BkLQX8ONy5Podqgt1HqE617sY2zMl7QvsC3wZ+E+qTNM9bP+hnL98Xp/1fJfq/OlFwEtr7ScC10iaaXt2mc79LNUFQ0jaEbjE9pPl4qdVgXv6fxtGtA2D/7EyKslzjYhozqgG1yYDxm0vKNOwx5SvnSwErgBmluV/kHQ1sHqP86hHAt+UdCpwMrAi1WD4OPAw8FPghj7q+RMl8aYakxe131sG/VPLeV0Bx9lunZfdAThe0mPl+WHlj4NBtc65CngI+JdR9BERERMoYekBJCw9ImJQSlh6RETE+Bm3ewtrHAPGe9RxDdB+Fe/e/X4vdcB9NfKaJb0bOKit+ae2e91gIiIiJkCmhQPItHBExKAyLRwRETGOMrhGREQ0LINrREREwxKWHsDww9I7SYh6RCyJcuQaERHRsCkxuGocQtslrSjpTFXh6nMl/UTS2l32u5ykp8vjuZK+10rCkTRNzw5Zf6+k6yW9qMO+T5d0V+nrekl/M0L7jZJm1La5vPaaZks6V1XY+lVtfT9H0gJJazbwq4iIiD5MiWlh2w8C0wEkHQE8bPuYcs/ga4CTbe9S7nF8ClVo+2FU9wdG0uXAoba7fdfkIGBBCVdH0gbAb8tN/5+139YGkh6tLf86Vebrp+udStob+CDwRtvdslkPs32upB2o7o+8aVv7duW1rVfbZs/6a5K0DFVQ+jTb80vz9sBc2/d22XdERDRoShy5dtFkaPua1G60b/tW248PsP1VwMvqDZLeQRWmvkMJV+/HFYycibtY/+1sL6SKvNu91jxiwHypL2HpERFDMNUH1yZD208DPiLpKkmfkrRezy2KcsQ8gyoirmVtqsSfHQa8gf9bgJHuFrUjcF5b25m1aeGjS1srog9VebJvpkoWWkzC0iMihmNKTAt30Vhoe4mRW4cq3WZ74DpJf2P7li6brVASbKZRDfI/qi27nyp8/R3AsX2UcLSkT5Tt3tPW/l/AS4Ct27bZs32q2/Z1klYq09obAVf3mI6OiIiGTfUj1yZD27H9sO2Ztj8A/DfVUV83rXOua1MFsdfv9fsIsBOwn6Q9+9j9YSX0/O9sz623Ux2FfwL4ep8v5Vt0DpiPiIghm+pHro2Ftkt6HXCz7d9LWg54JXB5P9vafkjSgcD5kk6utd+vKkT9ckkP2L5okJpq/SyUdDywr6Q39dHPWcD5wAt49lFwRwlLj4hozpQ+cnWVOrAb8HZJtwO3AY8xitB2YF3gfyXdRBWqPosO5yo71HIDcCPlfGet/S7grcBpkrYaRV2tfgx8Cvhwrbl+zvXHtXVvpjpyvtT2n0e7z4iIGJ2k4gSQVJyIiEElFSciImIcTfVzrgObyNB2SScCr2trPr71Pd2IiFgyLHWDa7kYaFQXFjWw7/17rxUREVNdpoUjIiIalsE1IiKiYUvdtHCMbKLyXPuV3NeImEpy5BoREdGwST+4jlOW6xskPVTbZrak7cuyj0uaJ2lOad+qtLfyVG+UdJ2k6bX+5qvKhb1J0s0lCGD5Lq9xmqRHS/83S/qSpGVGaD9D0nO71VzqelNb/wdLOqmBX0dERPRh0k8Lj1OWK8CVtneuN6gKLd8Z2Nz245JWo7qHcMuetmdJejdwNPB3tWXb2X5A0kqlrlOAfbvs/07b0yU9B7gU2BW4vta+LFUwwDuAM7vUvC7VXaLqV0TvQXWP4oiIGAeT/si1iyazXDtZE3igletq+wHbvxlhvY5Zq7YfBvYDdpW0Sq8d2n4K+BltkXnl9V3baT815wI7t46UJU0DXgr8pNe+IyKiGVN5cG0yyxVg27Yp1nWBi4G1JN0m6SRJf9th25GyVtvrugvomRFb/jCYQVumq6TnAVsB/9Ot5nKkf22pCaqj1rM9wn0ulbD0iIihmPTTwl00luVaLDbFCiBpC2BbYDvgbEmH2z69LD5T0l8AywKb91FvN+uqyoY1cL7tH5ajzlb7esC5tuf0qplnAtPPL//955F2aLs1Xc3ya66Xm0xHRDRkKh+5Nprl2ontp21fbvvfgAOAt9UW7wm8AvgmcGKnPiStTBWofluXXd1Z8lw3s31EezvV0fjWkt7aR9nnATMkbQ6sYPv6PraJiIiGTOXB9RJgRUn7AGgMWa6dSNpAUn0qdzrwy/o6tp+kCjLfWtJGI/SxEnAScJ7t34+2Ftv3AocDH+1j3YepsmhPI2HpERHjbspOC9u2pN2AkyR9kuoPhQsZXZYrlPOXteefojpP+kVJLwSeAu4A3jdCLY9K+hxwKM+Ek19WrmheBvgu8B+jrKvuPOAISdt2qtn2ueXxWcBM2vJlO0lYekREc5LnGkDyXCMiBqXkuUZERIyfKTstPBqawCzXsv9NgG+0NT9ue6vx2H9ERIyPpWpwncgs17L/myh3m4qIiCVXpoUjIiIalsE1IiKiYRlcIyIiGrZUnXONziZ7WHq7hKdHxGSWI9eIiIiGLbGDq8YhZL3s5zWSrijr/kLSV1qRd5J2Kqkzt5Rlx3Tp5whJh5bHp5d6W7Fxq0maX1t3fUkXSrqj9P1tSavXAtRvKPVcIWmkG/tHRMQQLbHTwuMRsi5pdeAcYA/bV5W+3wasLGkd4ATg723/ooSgL3brxC6epkqzObltn88DfgB8yPb3Stt2wIvLKouSciRNB86T9KjtSwbYd0REjMESe+TaRZMh6/sDX7d9VenLts+1vQD4MNWA/Yuy7CnbJw3Q93HAIWVQrvsn4KrWwFr6vsz23PYObM8GjqRK81lM8lwjIoZjaRxcmwxZ37i9rz6X9eNXwE+AvcfY7/XAhiMtsH2K7S1tb7nsii8YXZUREbGYpXFwbTpkfZg+AxzG2H5PvULaIyKiYUvj4NpkyPo8YItRLOuL7TuA2cA7xtDvZsAtY6kjIiIGszQOrk2GrJ8A7Ctp0Y33Je0laQ3gaOBjktYv7ctI+tAo6v00VU5syzeB10pa9EVPSTuWUIBnkbQp8EngxFHsNyIiRmmJvVq4kyZD1m0vkLQHcIyklwALgSuAmbZ/K+lg4KxyoZSprvIddB/zJF0PbF6eP1q+XnOcpOOAJ4E5wEHAqlQB6jcAKwL3AQf2c6VwwtIjIpqTsPQAEpYeETGohKVHRESMo6VuWng0mgxZl/Rx4O1tzefY/vRo64uIiMklg2sfmgxZL4NoBtKIiCVYpoUjIiIalsE1IiKiYRlcIyIiGpZzrgFMvbD0ySxB7hGRI9eIiIiGLZGD6xQNSm/VOFfSW0dov1nSO2vbnC7prlrNP5M0TdLdkpZp63+2pNeM/Z2NiIh+LJHTwlM0KP3YUuNGwJXldor19vWAn0s61/aTZdlhts9tq+vXwLbA/5bnGwIr2762x/4jIqIhS+SRaxeTPijd9i3AU8Bqbe23A48AL+rRxVnAHrXne5S2xSQsPSJiOJa2wXXSB6WXhJ2FwP1t7ZsDt9u+r9Z8dG1a+MzS9m1g13K0DLA78K2R9pWw9IiI4Vgip4W7mMxB6YdI2gv4E7B7Se9ptb8XWAfYsW2bxaaFSxrPPGCGpAXAk7bnjkP9ERFRLG1HrpM5KP1Y29Ntb2v7yrb2DaiOQM+Q9Lw++mpNDXecEo6IiOFZ2o5cLwGOkrSP7TMaCEq/VtIPbF8DVVA68GOqoPSZkn5i+7Zy9e7Btj8/2sJtz5S0L7Av8OUeq38H+AzVOdo39tN/8lwjIpqzVB25ugqv3Q14u6TbgduAxxhlUDrVkeEx5as4t1BdpftH23OAg6mC0m8B5gJrNvASjgQ+VPuqTf2c62xJy5Xa/gBcDSywfVcD+42IiAEkLD2AhKVHRAwqYekRERHjaGk75zqwBKVHRMSgMrj2kKD0iIgYVKaFIyIiGpbBNSIiomEZXCMiIhqWc64BJCx92BKgHrF0yZFrREREwybV4DqOIefbSLq2hJj/QtL7astGDC4foY83SLqqre05khZIWlPS1pKuKf3cUvp9d63WJyTdVB4fJWl1Sd+XdGMJRr+wS/3TJD1aC1H/kqRlSk3fb1v3dEn/2P9vISIixmpSTQuPU8j5GsA3gV1tXy9pNeAiSffYbs2LLhZcbnthW1dXAC+XNM32/NK2PTDX9r2llnfYvrHUu4Htm4GvlTrmA9vZfqA8/zLwI9vHl+eb9ni77rQ9vUTLXQrsCvyuxzYRETEOJtWRaxdNh5yfbvv60tcDVOHmh7ev2Cm4vCxbCJxDlVbTUk+heQlwb6veMrB2syZwd63/Of28GNtPAT9j8DzahKVHRAzJVBlcmww5X6wvYFZpf5ZOweU1rWg3JC0PvJkqkQbgWOBWSd+V9P4+ouJOBL4q6TJJH5f00n5eTPnjYgZwUz/r1yUsPSJiOKbK4NpkyHmnbepth0iaDRxDCS4fqSPb1wErSdoA2Am42vbvy7IjqbJjLwb+CfifbkWVO0GtA5wKbAjcIOnFXTZZt9T4U+AHtn/Y4XW1v7aIiBiySXXOtYt5wNvqDWMMOd8SuKDWtgVQn7Y91vYxffb3Laqj141oCya3fSdwsqRTgfslrVrOK4/I9u+ozgd/s1yY9HqeORJud6ft6W1tDwIvamtbBXigz9cSERENmCqDa5Mh5ycC10iaaXu2pFWpbsx/5ChrOws4H3gB8J5Wo6S/By4sR73rAU8Df+jUiaQ3Uh35PiJpZWBdqmnvQdwOvFTSRrZvkbQ28Gpgdq8NE5YeEdGcKTG42rak3YCTJH2Sajr7QkYXcn6vpL2AU8sgJuA4298bZW03S3oE+LntP9cW7Q0cW5Y9BexZLsTqZAvgBElPUb2+r5Rp50Fqeby8tq+Vc7xPAv9iO1crRUSMo4SlB5Cw9IiIQSUsPSIiYhxNiWnh0VgSQs4lbQJ8o635cdtbDXO/ERExNpkWDiDTwhERg8q0cERExDjK4BoREdGwJfacawwmea4TIzmvEUumHLlGREQ0bEoMrpLWkPStkuN6s6QLJa0vyZI+WFvvBEnvknRiLeu0lXs6W9KHJZ1dW//5pc9XdNjv5ZK2rD2fJmluebyipDNLJutcST+RtFJZ9rSeyYP9nqQXdnltnbJZ29vPkPTc2nZjzqSNiIjhmPSDa8ly/S5wue11bb+S6s5MqwP3AQdJWq6+je39y31330y5B295fjRVBuv2ZdUjgdNs3zWK0g4CFtjexPbGVLc+fLIse7Tsc2OqjNX9e/TVuk/wpsArqbJZ6+2bAC8H3gHPyqTdz/aGwDbA+8stF1uOLdu+HThN0qT/XUdELCmmwj+42wFP2v5Sq8H2bODXVFFwlwD79tNRuc/vvwLHlSPSGVQD7misCdxT6/tW24+PsN5VwMv6rG/EbNZy28Rra/00kkkbERHDMRUG141ZPH+17ijg/5Wb+fdUQsgvohqUD7T9xCjrOg34iKSrJH1K0nrtK5SaZvDsBJ6OOmWzlvsEb8UzsXWNZNImLD0iYjimwuDaVZnSvZYqM7VfJwL32L6sV/ed2srR8zpUR76rANdJ2qiss0LJWn2wLPtRj/2MlM1ab38Q+FX5wwAayqRNWHpExHBMhcF1HlViTDefAT5C/69nYfnppT0f9VnZqLYftj3T9geA/6Y6xwvlnCuwNrAcfZ5ztb2Z7SPa26mmibeuXZjUyqStGymTdrrtbW1f2WP/ERHRoKkwuF4KLC/pva0GSX9NNXABYPsXVAPLzg3v+3Jgr3JRFVTndi8rNbxO0ovK4+WoLkT6ZX3jEvV2IHBo/UrfQdm+l+p86kdL04nAuyRNL/tvZdL+12j3ERERzZn0N5GoZbkeJ+lw4DFgPnBw26qfBm5oePenABsCN0oy1XnN1gC3LnByGXiXAX4AfGeE+m+QdCOwB4vfhH8Q5wFHSNrW9pVNZtJCwtIjIpqUG/cHkBv3R0QMKjfuj4iIGEeTflp4PEg6EXhdW/Pxtr/W4D6SzRoRsZTI4Ep1R6dx2MdNwPRh7yciIiZepoUjIiIalsE1IiKiYRlcIyIiGpZzrgEkLH1plbD2iOHIkWtERETDMrh20RZ6fk5Jrekahi7pVZIulXSbpNslfbJ2+0Qk7VSSaG4pIefHlPYNSjj77LLslC51vUHSQ5JuKOv+W2l/l6QT2tZ9VuB7REQMXwbX7uqh508A+43QvigMXdIKVPFyR9leH3g18FrgA2X5xsAJwF62N6KK0/u/0ucXeOZm+xsBX+xR25W2N6O6gf9eknqFG0RExDjJ4Nq/K2kLMS/qYej/BPzU9sUAth8BDuCZEPMPA58uQQPYfsr2SWXZmsDdrU7L92J7sv1nqmzXdQd6NSTPNSJiWDK49kHSc4CdWDzEvD0MfbEQc9t3AitJej7dg9+PBS6V9ENJh9SnmnvUtiqwNVUM3UCS5xoRMRwZXLtrhZ7PAn4FfLWtvT0MvVOIOV3aq4XVrRY3As4B3gBcLWn5LptsK+kG4GKqaeh5o913REQ0K4Nrd61zq9Ntf9D2E/V2Fg9DXyzEXNI6wMO2/0SP4Hfbv7F9mu1dgKeojnQ7ubKEq29h+0ulrT3cHdoC3iMiYvjyPdcxsP2QpAOB8yWdDJwJfEzS9rZ/XC5w+gLPhJgfDcyU9BPbt0laBjjY9ucl7QhcYvtJSWsAqwL3DFjSdcAJktaw/dtylfDywK97bZg814iI5mRwHaN6GLrtb0jaBfhiSdpZlioJ54Sy7hxJBwNnla/1mCpkHWAH4HhJj5Xnh9n+7YC1LJB0EHBhGbgfBt5pe+EYX2ZERAwgYekBJCw9ImJQCUuPiIgYR5kWnsQkvQn4bFvzXbZ3m4h6IiKiPxlcJzHbFwEXTXQdERExmEwLR0RENCyDa0RERMMyuEZERDQs51wDSFh6RJMSQh85co2IiGjYlB1cJX1c0jxJc0rA+FYlGPxWSTdKuk7S9C7bX1O2+5Wk+8vj2ZKmSXqBpDMk3Vl+zihtm9TW+52ku8rjH3fYxzRJc8vjTgHn9fZbJV0haee2fp4j6QFJ/1me7yDpqlYIu6RlSx2vHSR0PSIihmNKTgtL+htgZ2Bz249LWo3qBvoAe9qeJendVPfy/buR+rC9VenrXcCWtg+o9X8uMNf2PuX5vwNfsf12YHppOx34vu1zByj9Sts7S/oLYLak79fbS7/TgfMkPWr7krJ8B+BW4B2SPmb7Ykn/DLwH+ArwQeA62z+TdBFV6Pr5pb9NBqgvIiIaMFWPXNcEHrD9OIDtB2z/pm2deoh53yT9FVVyzX/Umo8EtpQ0cCD5SLoFnNueXfZ3QK35ncDxVLF3W5e2Q4CPSnpVWfcjpb3v0PWEpUdEDMdUHVwvBtaSdJukkyT97Qjr7AicN4q+XwnMtv10q6E8nk0Vhj5mfQScXw9sWNZdgSqQ/fvAWVQDLbbvBY6j+iPiU7Z/V7btO3Q9YekREcMxJQdX2w9THV2+D7gfOLtM7wKcKeluqiO5L46i+06B592C0Ps1UsB5pxpadgYus/0I8B1gN0nLlmUnAsvaPr218ihC1yMiomFT8pwrLDqavBy4XNJNwL5l0Z7AjcBRVIPPPwzY9TxgM0nLtKLaSnzbq4Fbxlj2onOrPWxW29c7gddJml+erwpsB/zY9kJJiw34ZYr8NOC0ckHVxlTT0BERMQ6m5OAqaQNgoe3bS9N04JdUgwglcPwTwJ2SNrLd96Bo+45ydPkJqnOflMfX276jqdfQiaRNgU8C/yLp+cA2wFqt88vlQq13Ap2uUB5V6HrC0iMimjMlp4WBlYCvS7pZ0hyq86RH1Few/SjwOeDQUfT/HmB9SXdIuhNYv7QNy7atr+JQHW0fWK4U/gfg0tbAWpwPvLXLVO8OwFxVAe4XMYrQ9YiIGJuEpQeQsPSIiEElLD0iImIcTclzroOSdA3QPo26d7fvgA7Y/ybAN9qaH2/dqCIiIpYuS8XgOuxBrgzS04e5j4iImDoyLRwREdGwDK4RERENy+AaERHRsKXinGv0lrD0iJgKpkoQ/aQ/cpW0ai1D9beS7qk9/0tJ50u6veSuHi9pOUlvqq3zcMlJnS3pjA77eEMr/k3Su/RMvuvNkt7bYZtpku4ut0ast8+W9JqRclW71SVpRUlnSrpJ0lxJP5G0Upf35emy7VxJ55TtF+XH1tY7QtJobqQRERGjNOmPXG0/yDMZqkcAD9s+RpKAa4CTbe9SbmZ/CvBp24dR3Z0ISZcDh9oe5A4JZ9s+QNJLgHmSLrC9oK2u+ZJ+DWwL/G/Z14bAyravHSlXtVxVPGJdkj4KLLC9SXm+AfBklxoftd16X84E9gNmDvAaIyJiSCb9kWsXbwQeKykwrRv5HwL8s6QVm9iB7fuAO4G1O6xyFrBH7fkepQ0GyFWtrb/oHsC2b2277WE3VwJ/1ee6ERExZFN5cH0VbUkvtv9IFSjeyEAjaR1gHaDTDfu/DewqqTUDsDvwrfK471zV4jTgI5KukvQpSev1WeNzgJ2AgW+IoYSlR0QMxVQeXIeZu7q7pNlUR6HvrwWRP0u5If48YIak6cCTtueWZQPlqtqeTTWQHw2sAlwnaaMuNa5QapxF9QfFV+n8ukdsT1h6RMRwTPpzrl3MA95WbygRbWtRTeWOxdm2D+hz3dbU8AKemRIGBs9VLSHwM4GZkhYCb6Zzhuyic64tkh4EXtS23irAXX2+loiIaMBUPnK9BFhR0j4A5YKmzwGn235kHOv4DtUgWJ8SRtKOkp5bHvfMVZX0OkkvKo+Xo4rR++UghZTB+V5JM0o/qwA7Aj8ZpJ+IiBibKXvkatuSdgNOkvRJqj8ULgQ+Ns51/EHS1cDqtutHiDsAx0t6rDzvlau6LnByuQp6GeAHVAP3oPYBTpT0ufL83233PJJPWHpERHOS5xpA8lwjIgaVPNeIiIhxNGWnhUdD0puAz7Y132V7tx7bvRs4qK35p7b3b7K+Efa7KtW55XYzys01IiJiElqqBlfbF1HukDTgdl8DvtZ8RT33u+juVBERMXVkWjgiIqJhGVwjIiIalsE1IiKiYUvVOdfoLHmuEbG0GWY2bI5cIyIiGpbBtQtJa0j6Vgliv1nShZLWl/QqSZdKuq0EtX+y3FmpHrZ+Q1l2kaTX1vo8XdJdtdD0n5X21SV9X9KNrX11qWuapEdrge5fkrSMaqHvbfv7x2G9RxERsbgMrh2UwfK7wOW217X9SqpbK64OXAAcZXt94NXAa4EP1DY/2/ZmttcDjqK6EX894eYw29PLT2vgPRL4ke1Xl30d3qPEO8uN+zelug/xrmN5vRER0ZwMrp1tRxUh96VWQ4mFW5/qBhIXl7ZHgAPoMBjavgw4BXhfj/21h6vP6adI208BP2MUGbbJc42IGI4Mrp11iocbKaT9TmClEnk3kuuBDWvPj65NC59Z2k4EvirpMkkfl/TSfoqUtCIwg1GEpSfPNSJiOHK18OC6hbF3alfb88Nsn/usDe2LJK1DFRG3E3CDpI1t39+hz3VLWLqB823/UNLfDlhXREQMQQbXzuYBI10INA94fb2hDIoP2/5Tua6p3WZ0Dj1fxPbvgG8C3ywXJr2ezrFzrXOudZ3C0h/ote+IiGhOpoU7uxRYXtJ7Ww2S/hq4HdhG0valbQXgC8B/jdRJOZp8H3Bqt51JemOZ4kXSylT5rr8asObbgZe2Lp6StDbVBVezB+wnIiLGIEeuHdTC2I+TdDjwGDAfOBjYBfiipBOBZYFvACfUNt9d0jbAisBdwNts149cj5b0idrz1wBbACdIeorqj56v2L5uwJofl7QX8DVJzwOeBP7Fds+rlRKWHhHRnISlB5Cw9IiIQSUsPSIiYhxlWngSk7QJ1ZRz3eO2t5qIeiIioj+ZFg4AJP0JuHWi62izGpPvSufJWBNMzrpSU/8mY12pqbe1bb94pAU5co2WWzudO5gokmalpv5MxrpSU/8mY12paWxyzjUiIqJhGVwjIiIalsE1Wk6Z6AJGkJr6NxnrSk39m4x1paYxyAVNERERDcuRa0RERMMyuEZERDQsg+sSSNKOkm6VdEe5L3L7ckn6Qlk+R9LmvbaVtIqkH0m6vfy3PX1nKDVJWqtk3N4iaZ6kg2rbHCHpnlo27psHqWksdZVl8yXdVPY9q9Y+Ue/VBrX3YrakP0o6uCwb03vVR00bSrpK0uOSDu1n27G+T2Opa5ifqzG+VxP1mer0Pg3tM9VnXXuWz/gcST+T9Ope2zbxuWqE7fwsQT9UQQJ3AusAywE3Aq9sW+fNwA+pcma3Bq7ptS1V6s/h5fHhwGfHqaY1gc3L45WB22o1HQEcOhHvVVk2H1hthH4n5L0aoZ/fUn3JfUzvVZ81vQT4a+DT9f0M6zPVQF1D+VyNpaYJ/kx1rGkYn6kB6not8KLyeCeG/G9Vkz85cl3yvAa4w/b/2X4C+BZVik/dLsAZrlwNvFDSmj223QX4enn8dWDX8ajJ9r22rwew/SeqXNyXDbDvodTVo98Jea/a1plBlfn7ywH2PeqabN/nKsXpyQG2Hcv7NKa6hvi5Gst71c1QP1N91tTkZ6rfun5m+/fl6dXAy/vYdqyfq0ZkcF3yvAz4de353Sz+j0andbptu7rte6H6h4nqL93xqGkRSdOoguevqTUfUKaMThvF9M9Y6zJwsaSfS3pfbZ0Jf6+APYCz2tpG+171s7/RbDuW92msdS3S8OdqrDVN1GeqH01+pkZT13uoZmx6bTvWz1UjMrgueTRCW/v3rTqt08+2ozGWmqqF0krAd4CDbf+xNJ9MFSo/HbgX+Nw41/U625tTTVftL+n1A+5/GDUhaTngrcA5teVjea/G8rkY1meqkb6H8Lkaa00T9Znq3kHzn6mB6pK0HdXg+pFBt50oGVyXPHcDa9Wevxz4TZ/rdNt2QWvqsfz3vnGqCUnPpfoH8EzbM1sr2F5g+2nbC4FTqaaKBjGmumy3/nsf8N3a/ifsvSp2Aq63vaDVMMb3qp+aRrPtWN6nsdY1rM/VmGqawM9UL01/pvquS9KmwFeAXWw/2Me2Y/1cNSKD65LnOmA9Sa8of23uAVzQts4FwD6qbA08VKZPum17AbBvebwvcP541CRJwFeBW2x/vr5B23nG3YC5A9Q01rr+QtLKpY6/AHao7X9C3qva8nfSNn03xveqn5pGs+1Y3qcx1TXEz9VYaprIz1QvTX+m+qpL0l8CM4G9bd/W57Zj/Vw1Y5hXS+VnYn6oria9jepquo+Xtv2A/cpjASeW5TcBW3bbtrSvClwC3F7+u8p41ARsQzXdMweYXX7eXJZ9o6w7h+p/qDXH672iukrxxvIzbzK8V2XZisCDwAva+hzTe9VHTWtQHU38EfhDefz8YX6mxlLXMD9XY6hpIj9T3X5/Q/lM9VnXV4Df135Hs7pt29Tnqomf3P4wIiKiYZkWjoiIaFgG14iIiIZlcI2IiGhYBteIiIiGZXCNiIhoWAbXiIiIhmVwjYiIaNj/B5qJkBo2R8TwAAAAAElFTkSuQmCC\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "feat_importances = pd.Series(clf.feature_importances_, index=train_x.columns)\n",
    "feat_importances.nlargest(13).plot(kind='barh')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Feature Importances\n",
    "\n",
    "CODES_PU was the most useful variable to the decision tree, in classifying counties into high cost or low cost!"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 40,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYIAAAEGCAYAAABo25JHAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjMuMiwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy8vihELAAAACXBIWXMAAAsTAAALEwEAmpwYAAAktUlEQVR4nO3deZwU1bn/8c8jS0BZDIL+CMhlvI6yykQmjrhEuIS4IleNEvRqMMmLuKBZrrtGDSaGRBIVNZfgAmoQuG4RFfVqjEFF2XSAARRRWUZJRFQCKgLy/P6o6knT9MzUwFT3dNf3/XrNi66q01VPDVBPnXOqzjF3R0REkmuPfAcgIiL5pUQgIpJwSgQiIgmnRCAiknBKBCIiCdc83wE0VMeOHb179+75DkNEpKAsWLDgQ3fvlG1bwSWC7t27M3/+/HyHISJSUMxsVW3b1DQkIpJwSgQiIgmnRCAiknBKBCIiCadEICKScLElAjO7x8w+MLOqWrabmY03sxVmtsjMDo0rFhERqV2cNYLJwHF1bD8eKA1/RgH/E2MsIiJSi9jeI3D3WWbWvY4iw4D7PBgH+1Uz29vMOrv72rhiEhFJeWDOah6rfC/fYTRIr6+147qhvRt9v/l8oawLsCZtuTpct1MiMLNRBLUGunXrlpPgRJKiEC+IjWHOux8BUFHSIc+R5F8+E4FlWZd1lhx3nwhMBCgvL9dMOlI0msJFOKkXxIqSDgwr68KZFbq5zGciqAb2T1vuCryfp1hEGlXUC3xTuAjrgij5TAQzgNFmNg2oADaof0CKxWOV77F07T/p1bldneV0EZamILZEYGZTgYFARzOrBq4DWgC4+wRgJnACsAL4DDg3rlhE4pTt7j+VBKb/aECeohKJLs6nhkbUs92BC+M6vkhjq625J1vzTq/O7RhW1iVnsYnsjoIbhlokLvW169fWnq/mHSl0SgSSSNku+vV13OqCL8VKiUASI/3in+2irwu9JJUSgRStzLv+9Iu/Lvoi/6JEIEWlrrt+XfxFslMikKKQSgC66xdpOCUCafKivKWbngB08RdpGCUCadIemLOaqx5dDNQ9DIMSgMiuUyKQ2O3OwGqpO/0bT+mri7xITJQIJFZR7+hrozt9kfgpEUisUjUB3dGLNF2avF5i88Cc1cx59yMqSjooCYg0YaoRSKPK9hy/Bl8TadqUCGSX1Tdej9r3RQqDEoHssmyTr+jiL1J4lAgksswagCZfESkOSgSyk6gTsGjyFZHioEQgO6jruX81+4gUJyUCyfqkj577F0kOJYKEqm24Zt31iySPEkGC6OIvItkoESREZtu/Lv4ikqJEkADpSUBt/yKSSWMNFTklARGpjxJBkdPonyJSHzUNFalUx/DStf/U6J8iUiclgiKTbRJ3vf0rInVRIigimU8G6akgEYlCiaBIqFNYRHaVEkGBy2wKUhIQkYZSIihw6R3CagoSkV0R6+OjZnacmb1pZivM7Ios29ub2eNmttDMlpjZuXHGU6xScwIoCYjIroitRmBmzYA7gCFANTDPzGa4+9K0YhcCS919qJl1At40synuviWuuApdbZPDiIjsqjhrBIcBK9z9nfDCPg0YllHGgbZmZkAb4CNgW4wxFbRUh3CqPwA0OYyI7L44+wi6AGvSlquBiowytwMzgPeBtsBwd9+euSMzGwWMAujWLbnNH3pLWETiEGeNwLKs84zlY4FK4GtAGXC7me3UzuHuE9293N3LO3Xq1NhxFoQH5qxmzrsf6S1hEWl0cdYIqoH905a7Etz5pzsXGOvuDqwws3eBHsDcGOMqKJmPh6oZSEQaW5yJYB5QamYlwHvAd4EzM8qsBgYDL5rZfsDBwDsxxlRw9HioiMQttkTg7tvMbDTwDNAMuMfdl5jZeeH2CcANwGQzW0zQlHS5u38YV0yFJr05aPqPBuQ7HBEpUrG+UObuM4GZGesmpH1+H/h2nDEUqvQhI9QcJCJx0pvFTYyGjBCRXFMiaEI0eqiI5IMSQROi9wREJB+UCJoAzSYmIvmkOYubgFQS0HARIpIPqhE0EakRREVEck2JII/Sm4Q0gqiI5IuahvJITUIi0hSoRpBnahISkXxTjSBPUsNHiIjkW+REYGZ7xRlI0qTeGVCTkIjkW72JwMyOMLOlwLJwuZ+Z/SH2yBJA7wyISFMQpY/gZoIJZGYAuPtCM/tmrFEVqfT5hvWkkIg0FZGahtx9TcaqL2OIpeilnhICzTUsIk1HlBrBGjM7AnAzawlcTNhMJNFpbgERaaqi1AjOAy4kmIy+mmBu4QtijKkoqXNYRJqqKDWCg939rPQVZnYk8HI8IRWPzD4BdQ6LSFMUpUZwW8R1kiY1t0DqXQH1CYhIU1VrjcDMBgBHAJ3M7Gdpm9oRzEEsddDcAiJSKOpqGmoJtAnLtE1b/0/gO3EGVSzUFCQihaDWRODufwP+ZmaT3X1VDmMSEZEcitJZ/JmZ3QT0BlqlVrr7f8QWVYHSC2MiUoiidBZPAd4ASoBfACuBeTHGVLD0wpiIFKIoNYJ93P1uM/txWnPR3+IOrNDohTERKVRREsHW8M+1ZnYi8D7QNb6QCpNeGBORQhUlEfzSzNoD/03w/kA74CdxBlWo9JSQiBSiehOBuz8RftwADIKaN4tFRKQI1NpZbGbNzGyEmV1iZn3CdSeZ2Wzg9pxFWAA025iIFLK6agR3A/sDc4HxZrYKGABc4e5/zkFsTVr6o6KpJKD+AREpRHUlgnLgEHffbmatgA+BA93977kJrWlLPSraq3M7Kko6MKysi/oHRKQg1ZUItrj7dgB332xmyxuaBMzsOOBWgrGJ7nL3sVnKDARuAVoAH7r7MQ05Rj7oUVERKSZ1JYIeZrYo/GzAv4fLBri7H1LXjs2sGXAHMIRgHoN5ZjbD3Zemldkb+ANwnLuvNrN9d/1UckePiopIMakrEfTczX0fBqxw93cAzGwaMAxYmlbmTOARd18N4O4f7OYxY5deG1BTkIgUg7oGndvdgea6AOlzHVcDFRllDgJamNkLBCOc3uru92XuyMxGAaMAunXL78VXtQERKTaRJq/fRZZlnWcsNwf6AycCxwI/N7ODdvqS+0R3L3f38k6dOjV+pA2k2oCIFJMobxbvqmqCx09TuhIMT5FZ5kN3/xT41MxmAf2A5THGJSIiaSLVCMystZkd3MB9zwNKzazEzFoC3wVmZJR5DDjazJqb2Z4ETUfLGnicnNGLYyJSjOpNBGY2FKgEng6Xy8ws84K+E3ffBowGniG4uP+vuy8xs/PM7LywzLJwv4sIXly7y92rdvFcYqf+AREpRlGahq4neALoBQB3rzSz7lF27u4zgZkZ6yZkLN8E3BRlf02B+gdEpNhEaRra5u4bYo9ERETyIkqNoMrMzgSamVkpcDEwO96wmpbUuEKaflJEilGUGsFFBPMVfwE8QDAc9U9ijKnJSU8C6h8QkWITpUZwsLtfDVwddzBNWa/O7TSukIgUpSg1gt+b2RtmdoOZ9Y49oiZGj4yKSLGrNxG4+yBgILAOmGhmi83smrgDawoemLOaqx5dDOiRUREpXpFeKHP3v7v7eOA8gncKro0zqKYgPQnceEpfPTIqIkUrygtlPc3sejOrIpiicjbBcBFFS0lARJIkSmfxJGAq8G13zxwrqCil3iBWEhCRJKg3Ebj74bkIpKnRG8QikhS1JgIz+193P8PMFrPj8NGRZigTEZHCUFeN4MfhnyflIhAREcmPWjuL3X1t+PECd1+V/gNckJvwREQkblEeHx2SZd3xjR1IU6EXyEQkaerqIzif4M7/ADNblLapLfBy3IHli+YcEJGkqauP4AHgKeDXwBVp6ze6e1HfMuuJIRFJkroSgbv7SjO7MHODmXUo9mQgIpIU9dUITgIWEDw+amnbHDggxrjyItU/UFHSId+hiIjkTK2JwN1PCv8syV04+aMB5kQkqaKMNXSkme0Vfv4vM/u9mRVdA7qGlRCRpIry+Oj/AJ+ZWT/gMmAVcH+sUeWJOolFJImiTl7vwDDgVne/leARUhERKQJRRh/daGZXAmcDR5tZM6BFvGGJiEiuRKkRDCeYuP777v53oAtwU6xR5ZjeJhaRJIsyVeXfgSlAezM7Cdjs7vfFHlkO6W1iEUmyKE8NnQHMBU4HzgDmmNl34g4s19RRLCJJFaWP4GrgG+7+AYCZdQKeAx6KMzAREcmNKH0Ee6SSQGh9xO+JiEgBiFIjeNrMniGYtxiCzuOZ8YUkIiK5FGXO4kvN7FTgKILxhia6+6OxRyYiIjlR13wEpcA44N+BxcAl7v5ergITEZHcqKut/x7gCeA0ghFIb2vozs3sODN708xWmNkVdZT7hpl9WYxPI4mINHV1NQ21dfc7w89vmtlrDdlx+AbyHQRTXVYD88xshrsvzVLuN8AzDdm/iIg0jroSQSsz+zr/moegdfqyu9eXGA4DVrj7OwBmNo1gvKKlGeUuAh4GvtHA2EVEpBHUlQjWAr9PW/572rID/1HPvrsAa9KWq4GK9AJm1gU4JdxXrYnAzEYBowC6dWvcl740GY2IJF1dE9MM2s19W5Z1nrF8C3C5u39plq14TSwTgYkA5eXlmfvYLRpeQkSSLsp7BLuqGtg/bbkr8H5GmXJgWpgEOgInmNk2d/9zjHHtRMNLiEiSxZkI5gGlZlYCvAd8FzgzvUD6NJhmNhl4ItdJQEQk6WJLBO6+zcxGEzwN1Ay4x92XmNl54fYJcR1bRESiqzcRWNBucxZwgLuPCecr/n/uPre+77r7TDKGo6gtAbj7yEgRi4hIo4oyeNwfgAHAiHB5I8H7ASIiUgSiJIIKd78Q2Azg7h8DLWONKkc0M5mISLQ+gq3h278ONfMRbI81qpg9MGc1j1W+V5ME9OioiCRZlEQwHngU2NfMfgV8B7gm1qhi9ljleyxd+08qSjowrKyLHh0VkUSLMgz1FDNbAAwmeEnsP919WeyRxaxX53ZM/9GAfIchIpJ3UZ4a6gZ8Bjyevs7dV8cZmIiI5EaUpqEnCfoHDGgFlABvAr1jjEtERHIkStNQ3/RlMzsU+FFsEYmISE41eBL6cPhpDRktIlIkovQR/CxtcQ/gUGBdbBGJiEhORekjaJv2eRtBn8HD8YQjIiK5VmciCF8ka+Pul+YoHhERybFa+wjMrLm7f0nQFCQiIkWqrhrBXIIkUGlmM4AHgU9TG939kZhjExGRHIjSR9ABWE8wr3DqfQIHlAhERIpAXYlg3/CJoSr+lQBSGnXeYBERyZ+6EkEzoA3RJqEXEZECVVciWOvuY3IWSY6k5iCoKOmQ71BERJqEut4szlYTKHiPVb4HaA4CEZGUuhLB4JxFkSPptQHNQSAiEqg1Ebh70c3hqNqAiMjOGjzoXKFTbUBEZEeJSwQiIrIjJQIRkYRTIhARSbjEJILUE0MiIrKjxCQCPTEkIpJdYhIB6IkhEZFsEpUIRERkZ0oEIiIJF2siMLPjzOxNM1thZldk2X6WmS0Kf2abWb844xERkZ3FlgjC+Y7vAI4HegEjzKxXRrF3gWPc/RDgBmBiXPGIiEh2cdYIDgNWuPs77r4FmAYMSy/g7rPd/eNw8VWga4zxiIhIFnEmgi7AmrTl6nBdbX4APJVtg5mNMrP5ZjZ/3bp1jRiiiIjEmQgiz2xmZoMIEsHl2ba7+0R3L3f38k6dOjViiCIiEmXy+l1VDeyfttwVeD+zkJkdAtwFHO/u62OMR0REsoizRjAPKDWzEjNrCXwXmJFewMy6AY8AZ7v78hhjERGRWsRWI3D3bWY2GngGaAbc4+5LzOy8cPsE4FpgH+APZgawzd3L44pJRER2FmfTEO4+E5iZsW5C2ucfAj+MMwYREamb3iwWEUk4JQIRkYRTIhARSTglAhGRhFMiEBFJOCUCEZGEUyIQEUk4JQIRkYRTIhARSTglAhGRhFMiEBFJOCUCEZGEUyIQEUk4JQIRkYRTIhARSTglAhGRhFMiEBFJOCUCEZGEUyIQEUk4JQIRkYRTIhARSbjm+Q5AitfWrVuprq5m8+bN+Q5FJDFatWpF165dadGiReTvKBFIbKqrq2nbti3du3fHzPIdjkjRc3fWr19PdXU1JSUlkb+npiGJzebNm9lnn32UBERyxMzYZ599GlwLVyKQWCkJiOTWrvyfUyIQEUk4JQIpas2aNaOsrIw+ffowdOhQPvnkk0bZ7+TJkxk9enSj7Kt79+707duXsrIyysrKmD17dqPsN1NlZSUzZ87cYd1TTz1FeXk5PXv2pEePHlxyySUAXH/99YwbN67Rjn3EEUfUfL700kvp3bs3l156KRMmTOC+++7brX2//vrr/PCHP9xh3bBhwxgwYMAO60aOHMlDDz20w7o2bdrUfF6+fDknnHACBx54ID179uSMM87gH//4x27F9tFHHzFkyBBKS0sZMmQIH3/8cdZyt956K3369KF3797ccsst9X5/8eLFjBw5crdiS6dEIEWtdevWVFZWUlVVRYcOHbjjjjvyHVJWf/3rX6msrKSysnKHi2Zdtm3b1qBjZCaCqqoqRo8ezZ/+9CeWLVtGVVUVBxxwQIP2GVV6cvvjH//Ia6+9xk033cR5553HOeecE3k/2c75xhtv5KKLLqpZ/uSTT3jttdf45JNPePfddyPtd/PmzZx44omcf/75rFixgmXLlnH++eezbt26yLFlM3bsWAYPHsxbb73F4MGDGTt27E5lqqqquPPOO5k7dy4LFy7kiSee4K233qrz+3379qW6uprVq1fvVnwpempIcuIXjy9h6fv/bNR99vpaO64b2jty+QEDBrBo0SIA5s6dy09+8hM+//xzWrduzaRJkzj44IOZPHkyM2bM4LPPPuPtt9/mlFNO4be//S0AkyZN4te//jWdO3fmoIMO4itf+QoAq1at4vvf/z7r1q2jU6dOTJo0iW7dujFy5Ehat27NG2+8wapVq5g0aRL33nsvr7zyChUVFUyePLnWWOvaZ4cOHXj99dc59NBDueCCC7jwwgtZt24de+65J3feeSc9evTgwQcf5Be/+AXNmjWjffv2PPfcc1x77bV8/vnnvPTSS1x55ZU8+eSTXH311fTo0QOA5s2bc8EFF+wUy5133snEiRPZsmULBx54IPfffz977rnnTseYNWsWS5Ys4dxzz2XLli1s376dhx9+mNLSUtq0acOmTZs4+eST+fTTT6moqODKK69k2bJltGnThksuuYS3334767lknvPvfve7mtg2btzIokWL6NevX826hx9+mKFDh7Lffvsxbdo0rrzyynr/bTzwwAMMGDCAoUOH1qwbNGhQvd+rz2OPPcYLL7wAwPe+9z0GDhzIb37zmx3KLFu2jMMPP5w999wTgGOOOYZHH32Uyy67rM7vDx06lGnTpnHZZZftdpyqEUgifPnll/zlL3/h5JNPBqBHjx7MmjWL119/nTFjxnDVVVfVlK2srGT69OksXryY6dOns2bNGtauXct1113Hyy+/zLPPPsvSpUtryo8ePZpzzjmHRYsWcdZZZ3HxxRfXbPv44495/vnnufnmmxk6dCg//elPWbJkCYsXL6aysrKm3KBBgygrK6OioqLefS5fvpznnnuO3/3ud4waNYrbbruNBQsWMG7cuJoL+ZgxY3jmmWdYuHAhM2bMoGXLlowZM4bhw4dTWVnJ8OHDqaqqon///vX+7k499VTmzZvHwoUL6dmzJ3fffXfWYwBMmDCBH//4x1RWVjJ//ny6du26w75mzJhRU0sbPnz4DttqO5fMc043f/58+vTps8O6qVOnMmLECEaMGMHUqVPrPT8g8u9i48aNNU14mT/p/yZS/vGPf9C5c2cAOnfuzAcffLBTmT59+jBr1izWr1/PZ599xsyZM1mzZk293y8vL+fFF1+MdH71UY1AcqIhd+6N6fPPP6esrIyVK1fSv39/hgwZAsCGDRv43ve+x1tvvYWZsXXr1prvDB48mPbt2wPQq1cvVq1axYcffsjAgQPp1KkTAMOHD2f58uUAvPLKKzzyyCMAnH322TvcoQ0dOhQzo2/fvuy333707dsXgN69e7Ny5UrKysqAoGmoY8eONd+ra5+nn346zZo1Y9OmTcyePZvTTz+9ZtsXX3wBwJFHHsnIkSM544wzOPXUU3frd1hVVcU111zDJ598wqZNmzj22GNrPcaAAQP41a9+RXV1NaeeeiqlpaWRjlHXuaSfc6a1a9fW/J1AcOFcsWIFRx11FGZG8+bNqaqqok+fPlmfpmnoEzZt27bdIYE3hp49e3L55ZczZMgQ2rRpQ79+/WjevP5L87777sv777/fKDHEWiMws+PM7E0zW2FmV2TZbmY2Pty+yMwOjTMeSZ7U3eeqVavYsmVLTR/Bz3/+cwYNGkRVVRWPP/74Ds9dp5p8IOhsTrVLR71opJdL7WuPPfbYYb977LFHg9r40/e51157AbB9+3b23nvvmr6FyspKli1bBgR35r/85S9Zs2YNZWVlrF+/fqd99u7dmwULFtR77JEjR3L77bezePFirrvuuprfVbZjnHnmmTV3/cceeyzPP/98pPOr61zSzzlT69atd/i7mz59Oh9//DElJSV0796dlStXMm3aNAD22WefHTprP/roo5rkG/V30dAawX777cfatWuBIGntu+++Wff7gx/8gNdee41Zs2bRoUOHmgRa1/c3b95M69at6405itgSgZk1A+4Ajgd6ASPMrFdGseOB0vBnFPA/ccUjyda+fXvGjx/PuHHj2Lp1Kxs2bKBLly4AdbbVp1RUVPDCCy+wfv16tm7dyoMPPliz7Ygjjqi52EyZMoWjjjpqt+ONss927dpRUlJSE4u7s3DhQgDefvttKioqGDNmDB07dmTNmjW0bduWjRs31nz/0ksv5cYbb6yp2Wzfvp3f//73Ox1n48aNdO7cma1btzJlypSa9dmO8c4773DAAQdw8cUXc/LJJ9f0ydSnrnOpS8+ePVmxYkXN8tSpU3n66adZuXIlK1euZMGCBTW/x4EDBzJ9+nS2bNkCBH/vqX6AM888k9mzZ/Pkk0/W7Ovpp59m8eLFOxwvVSPI9tOrV+blDU4++WTuvfdeAO69916GDRuW9TxSTT6rV6/mkUceYcSIEfV+f/ny5Ts1i+2qOGsEhwEr3P0dd98CTAMyfwvDgPs88Cqwt5l1jjEmSbCvf/3r9OvXr6aD7corr+TII4/kyy+/rPe7nTt35vrrr2fAgAF861vf4tBD/1V5HT9+PJMmTeKQQw7h/vvv59Zbb93tWKPuc8qUKdx9993069eP3r1789hjjwHBRb5v37706dOHb37zm/Tr149BgwaxdOlSysrKmD59Oocccgi33HILI0aMoGfPnvTp06fm7jPdDTfcQEVFBUOGDKnpWK7tGNOnT6dPnz6UlZXxxhtvNOiJoNrOpS49evRgw4YNbNy4kZUrV7J69WoOP/zwmu0lJSW0a9eOOXPmcNJJJ3H00UfTv39/ysrKePnll2s6Xlu3bs0TTzzBbbfdRmlpKb169WLy5Mm13sFHdcUVV/Dss89SWlrKs88+yxVXBA0j77//PieccEJNudNOO41evXoxdOhQ7rjjDr761a/W+X0ImhNPPPHE3Yovxdy9UXa0047NvgMc5+4/DJfPBircfXRamSeAse7+Urj8F+Byd5+fsa9RBDUGunXr1n/VqlUNjucXjy8B8tdWnUTLli2jZ8+e+Q5DitzNN99M27Ztd3qXoJh98cUXHHPMMbz00ktZ+xOy/d8zswXuXp5tf3HWCLI1qGZmnShlcPeJ7l7u7uXpHUMNcd3Q3koCIkXo/PPP36H/JQlWr17N2LFjI3UqRxHnU0PVwP5py12BzC7uKGVERGrVqlUrzj777HyHkVOlpaWRn8iKIs4awTyg1MxKzKwl8F1gRkaZGcA54dNDhwMb3H3nRkopWHE1PYpIdrvyfy62GoG7bzOz0cAzQDPgHndfYmbnhdsnADOBE4AVwGfAuXHFI7nXqlUr1q9fr6GoRXIkNR9Bq1atGvS92DqL41JeXu7z58+vv6DknWYoE8m92mYoq6uzWG8WS2xatGjRoFmSRCQ/NNaQiEjCKRGIiCScEoGISMIVXGexma0DGv5qcaAj8GEjhlMIdM7JoHNOht05539z96xv5BZcItgdZja/tl7zYqVzTgadczLEdc5qGhIRSTglAhGRhEtaIpiY7wDyQOecDDrnZIjlnBPVRyAiIjtLWo1AREQyKBGIiCRcUSYCMzvOzN40sxVmdkWW7WZm48Pti8zs0Gz7KSQRzvms8FwXmdlsM+uXjzgbU33nnFbuG2b2ZThrXkGLcs5mNtDMKs1siZn9LdcxNrYI/7bbm9njZrYwPOeCHsXYzO4xsw/MrKqW7Y1//XL3ovohGPL6beAAoCWwEOiVUeYE4CmCGdIOB+bkO+4cnPMRwFfDz8cn4ZzTyj1PMOT5d/Iddw7+nvcGlgLdwuV98x13Ds75KuA34edOwEdAy3zHvhvn/E3gUKCqlu2Nfv0qxhrBYcAKd3/H3bcA04BhGWWGAfd54FVgbzPrnOtAG1G95+zus93943DxVYLZ4ApZlL9ngIuAh4EPchlcTKKc85nAI+6+GsDdC/28o5yzA20tmPSiDUEi2JbbMBuPu88iOIfaNPr1qxgTQRdgTdpydbiuoWUKSUPP5wcEdxSFrN5zNrMuwCnAhBzGFacof88HAV81sxfMbIGZnZOz6OIR5ZxvB3oSTHO7GPixu2/PTXh50ejXr2KcjyDbVFiZz8hGKVNIIp+PmQ0iSARHxRpR/KKc8y3A5e7+ZZHMkBblnJsD/YHBQGvgFTN71d2Xxx1cTKKc87FAJfAfwL8Dz5rZi+7+z5hjy5dGv34VYyKoBvZPW+5KcKfQ0DKFJNL5mNkhwF3A8e6+PkexxSXKOZcD08Ik0BE4wcy2ufufcxJh44v6b/tDd/8U+NTMZgH9gEJNBFHO+VxgrAcN6CvM7F2gBzA3NyHmXKNfv4qxaWgeUGpmJWbWEvguMCOjzAzgnLD3/XBgg7uvzXWgjajeczazbsAjwNkFfHeYrt5zdvcSd+/u7t2Bh4ALCjgJQLR/248BR5tZczPbE6gAluU4zsYU5ZxXE9SAMLP9gIOBd3IaZW41+vWr6GoE7r7NzEYDzxA8cXCPuy8xs/PC7RMIniA5AVgBfEZwR1GwIp7ztcA+wB/CO+RtXsAjN0Y856IS5ZzdfZmZPQ0sArYDd7l71scQC0HEv+cbgMlmtpig2eRydy/Y4anNbCowEOhoZtXAdUALiO/6pSEmREQSrhibhkREpAGUCEREEk6JQEQk4ZQIREQSTolARCThlAikSQpHC61M++leR9lNjXC8yWb2bnis18xswC7s4y4z6xV+vipj2+zdjTHcT+r3UhWOuLl3PeXLzOyExji2FC89PipNkpltcvc2jV22jn1MBp5w94fM7NvAOHc/ZDf2t9sx1bdfM7sXWO7uv6qj/Eig3N1HN3YsUjxUI5CCYGZtzOwv4d36YjPbaaRRM+tsZrPS7piPDtd/28xeCb/7oJnVd4GeBRwYfvdn4b6qzOwn4bq9zOzJcPz7KjMbHq5/wczKzWws0DqMY0q4bVP45/T0O/SwJnKamTUzs5vMbJ4FY8z/KMKv5RXCwcbM7DAL5pl4Pfzz4PBN3DHA8DCW4WHs94THeT3b71ESKN9jb+tHP9l+gC8JBhKrBB4leAu+XbitI8Fblaka7abwz/8Grg4/NwPahmVnAXuF6y8Hrs1yvMmE8xUApwNzCAZvWwzsRTC88RLg68BpwJ1p320f/vkCwd13TUxpZVIxngLcG35uSTCKZGtgFHBNuP4rwHygJEucm9LO70HguHC5HdA8/Pwt4OHw80jg9rTv3wj8V/h5b4IxiPbK99+3fvL7U3RDTEjR+Nzdy1ILZtYCuNHMvkkwdEIXYD/g72nfmQfcE5b9s7tXmtkxQC/g5XBojZYEd9LZ3GRm1wDrCEZoHQw86sEAbpjZI8DRwNPAODP7DUFz0osNOK+ngPFm9hXgOGCWu38eNkcdYv+aRa09UAq8m/H91mZWCXQHFgDPppW/18xKCUaibFHL8b8NnGxml4TLrYBuFPZ4RLKblAikUJxFMPtUf3ffamYrCS5iNdx9VpgoTgTuN7ObgI+BZ919RIRjXOruD6UWzOxb2Qq5+3Iz608w3suvzez/3H1MlJNw981m9gLB0MnDgampwwEXufsz9ezic3cvM7P2wBPAhcB4gvF2/urup4Qd6y/U8n0DTnP3N6PEK8mgPgIpFO2BD8IkMAj4t8wCZvZvYZk7gbsJpvt7FTjSzFJt/nua2UERjzkL+M/wO3sRNOu8aGZfAz5z9z8B48LjZNoa1kyymUYwUNjRBIOpEf55fuo7ZnZQeMys3H0DcDFwSfid9sB74eaRaUU3EjSRpTwDXGRh9cjMvl7bMSQ5lAikUEwBys1sPkHt4I0sZQYClWb2OkE7/q3uvo7gwjjVzBYRJIYeUQ7o7q8R9B3MJegzuMvdXwf6AnPDJpqrgV9m+fpEYFGqszjD/xHMS/ucB9MvQjBPxFLgNQsmLf8j9dTYw1gWEgzN/FuC2snLBP0HKX8FeqU6iwlqDi3C2KrCZUk4PT4qIpJwqhGIiCScEoGISMIpEYiIJJwSgYhIwikRiIgknBKBiEjCKRGIiCTc/wcWYoXtmId2CwAAAABJRU5ErkJggg==\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "from sklearn.metrics import plot_roc_curve\n",
    "\n",
    "clf_disp = plot_roc_curve(clf, test_x, test_y)\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## AUC\n",
    "\n",
    "AUC for the holdout (test) data is 0.90!"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 41,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAWAAAAD4CAYAAADSIzzWAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjMuMiwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy8vihELAAAACXBIWXMAAAsTAAALEwEAmpwYAAAMyElEQVR4nO3df6jd913H8ecrsXaFIWvZGrIksopRlypuMMpgCsMKjT8w/aeYihokkn862EDQ1H9kYKD/OPzH/hF0LKA0BFQa+odSomWIc1mddS6NpaHF9prQuM2hgnS99779437Bsyz33HN7z73ve755PsqXc87nnO/3+/nj8sq77+/nfE+qCknSztvTPQFJulMZwJLUxACWpCYGsCQ1MYAlqcn3bfcJ3vnGay6z0Pe454M/3T0F7ULL3/n3bPUYm8mcu97/Q1s+31ZYAUtSk22vgCVpR62udM9gZgawpHFZWe6ewcwMYEmjUrXaPYWZGcCSxmXVAJakHlbAktTEi3CS1MQKWJJ6lKsgJKmJF+EkqYktCElq4kU4SWpiBSxJTbwIJ0lNvAgnST2q7AFLUg97wJLUxBaEJDWxApakJivvdM9gZgawpHGxBSFJTWxBSFITK2BJamIAS1KP8iKcJDWxByxJTWxBSFITK2BJamIFLElNrIAlqcmyN2SXpB5WwJLUxB6wJDWxApakJlbAktTECliSmrgKQpKaVHXPYGYGsKRxsQcsSU0MYElq4kU4SWqystI9g5nt6Z6AJM3V6urs2wyS7E3yT0meG17fl+T5JK8Oj/dOfPbJJNeSvJLkkY2ObQBLGpc5BzDwaeDqxOvTwKWqOgxcGl6T5AhwHHgQOAo8nWTvtAMbwJLGpVZn3zaQ5CDwC8AfTwwfA84Nz88Bj06Mn6+qt6vqdeAa8NC04xvAkkalVmvmLcmpJC9ObKduOdwfAr8NTKb1vqq6ATA83j+MHwDenPjc0jC2Li/CSRqXTSxDq6qzwNnbvZfkF4GbVfWPST45w+Fyu1NM28EAljQu81sF8Qngl5L8PPAe4AeS/CnwVpL9VXUjyX7g5vD5JeDQxP4HgevTTmALQtK4zOkiXFU9WVUHq+pDrF1c+5uq+lXgInBi+NgJ4Nnh+UXgeJK7kzwAHAYuTzuHFbCkcdn+b8I9BVxIchJ4A3gMoKquJLkAvAwsA09U1dRy3ACWNC7bcDOeqnoBeGF4/k3g4XU+dwY4M+txDWBJ4zKme0Ek+THW1rcdYO2K3nXgYlVdnbqjJHVYXZzbUU69CJfkd4DzrC2vuAx8ZXj+TJLT2z89SdqklZXZt2YbVcAngQer6p3JwSSfA66w1oz+HsNi5lMAT//B7/Obv/74HKYqSRurEbUgVoEPAv92y/h+vvubId9lcnHzO994bXH+f0DS4lugFsRGAfwZ4FKSV/n/r9j9IPDDwKe2cV6S9O6M5X7AVfVXSX6EtRtKHGCt/7sEfGWj9W2S1GJEFTBVtQr8ww7MRZK2bnlxakPXAUsal7G0ICRp4YypBSFJi2RMy9AkabFYAUtSEwNYkprsgq8Yz8oAljQqZQUsSU0MYElq4ioISWpiBSxJTQxgSepRK7YgJKmHFbAk9XAZmiR1MYAlqcnitIANYEnjUsuLk8AGsKRxWZz8NYAljYsX4SSpixWwJPWwApakLlbAktSjlrtnMDsDWNKoLNCv0hvAkkbGAJakHlbAktTEAJakJrWS7inMzACWNCqLVAHv6Z6AJM1TrWbmbZok70lyOck/J7mS5LPD+H1Jnk/y6vB478Q+Tya5luSVJI9sNFcDWNKo1Ors2wbeBn6mqn4S+AhwNMnHgdPApao6DFwaXpPkCHAceBA4CjydZO+0ExjAkkalKjNv049TVVX/M7y8a9gKOAacG8bPAY8Oz48B56vq7ap6HbgGPDTtHAawpFHZTAWc5FSSFye2U5PHSrI3yUvATeD5qvoysK+qbgAMj/cPHz8AvDmx+9Iwti4vwkkaldVNrIKoqrPA2SnvrwAfSfI+4C+T/PiUw93uxFPvDGQASxqVjS6uvatjVn07yQus9XbfSrK/qm4k2c9adQxrFe+hid0OAtenHdcWhKRRmeMqiA8MlS9J7gF+FvhX4CJwYvjYCeDZ4flF4HiSu5M8ABwGLk87hxWwpFGp+d0OeD9wbljJsAe4UFXPJfkScCHJSeAN4LG189aVJBeAl4Fl4ImhhbEuA1jSqMyrBVFVXwM+epvxbwIPr7PPGeDMrOcwgCWNykbLy3YTA1jSqKx4LwhJ6mEFLElNtmMZ2nYxgCWNyhxXQWw7A1jSqFgBS1KTldXF+X6ZASxpVGxBSFKTVVdBSFIPl6FJUhNbEBN+4sgvb/cptIB+6v4Pd09BI2ULQpKauApCkposUAfCAJY0LrYgJKmJqyAkqclq9wQ2wQCWNCp12x8n3p0MYEmjsmwLQpJ6WAFLUhN7wJLUxApYkppYAUtSkxUrYEnqsUC/SGQASxqXVStgSerhzXgkqYkX4SSpyWpsQUhSi5XuCWyCASxpVFwFIUlNXAUhSU1cBSFJTWxBSFITl6FJUpMVK2BJ6rFIFfCe7glI0jytbmKbJsmhJH+b5GqSK0k+PYzfl+T5JK8Oj/dO7PNkkmtJXknyyEZzNYAljUpl9m0Dy8BvVdWHgY8DTyQ5ApwGLlXVYeDS8JrhvePAg8BR4Okke6edwACWNCrzqoCr6kZVfXV4/t/AVeAAcAw4N3zsHPDo8PwYcL6q3q6q14FrwEPTzmEASxqVlU1ss0ryIeCjwJeBfVV1A9ZCGrh/+NgB4M2J3ZaGsXUZwJJGZTWzb0lOJXlxYjt16/GSvBf4c+AzVfVfU059u6bG1O+FuApC0qhsZhVEVZ0Fzq73fpK7WAvfP6uqvxiG30qyv6puJNkP3BzGl4BDE7sfBK5PO78VsKRRmeMqiAB/Alytqs9NvHURODE8PwE8OzF+PMndSR4ADgOXp53DCljSqMzxXhCfAH4N+JckLw1jvws8BVxIchJ4A3gMoKquJLkAvMzaCoonqmpqq9kAljQq87oXRFX9Hbfv6wI8vM4+Z4Azs57DAJY0Kt6QXZKarC7QDSkNYEmjskj3gjCAJY3K4tS/BrCkkbEClqQmy1mcGtgAljQqixO/BrCkkbEFIUlNXIYmSU0WJ34NYEkjYwtCkpqsLFANbABLGhUrYElqUlbAktTDCliSmrgMTZKaLE78GsCSRmZ5gSLYAJY0Kot0Ee5d/ypykt+Y8t6pJC8mefHb//sf7/YUkrRp8/pV5J2wlZ+l/+x6b1TV2ar6WFV97H33fGALp5CkzalN/NdtagsiydfWewvYN//pSNLW7IbKdlYb9YD3AY8A/3nLeIC/35YZSdIWrFR/ZTurjQL4OeC9VfXSrW8keWE7JiRJWzGadcBVdXLKe78y/+lI0tbsht7urFyGJmlUxtQDlqSFMpoWhCQtGlsQktRkTKsgJGmh2IKQpCZehJOkJvaAJamJLQhJalJehJOkHv4svSQ1sQUhSU1sQUhSk0WqgLfyixiStOvM8xcxknw+yc0kX58Yuy/J80leHR7vnXjvySTXkryS5JGNjm8ASxqVlaqZtxl8ATh6y9hp4FJVHQYuDa9JcgQ4Djw47PN0kr3TDm4ASxqVVWrmbSNV9UXgW7cMHwPODc/PAY9OjJ+vqrer6nXgGvDQtOMbwJJGZTMBPPkL7sN2aoZT7KuqGwDD4/3D+AHgzYnPLQ1j6/IinKRR2cwqiKo6C5yd06lzu1NM28EAljQqO7AK4q0k+6vqRpL9wM1hfAk4NPG5g8D1aQeyBSFpVOa5CmIdF4ETw/MTwLMT48eT3J3kAeAwcHnagayAJY3KSs3vhpRJngE+Cbw/yRLwe8BTwIUkJ4E3gMcAqupKkgvAy8Ay8ERVrUw7vgEsaVTm+U24qnp8nbceXufzZ4Azsx7fAJY0Kov0TTgDWNKoeEN2SWqy6s14JKmHFbAkNZnnKojtZgBLGhVbEJLUxBaEJDWxApakJlbAktRkZfq3f3cVA1jSqPijnJLUxK8iS1ITK2BJauIqCElq4ioISWriV5ElqYk9YElqYg9YkppYAUtSE9cBS1ITK2BJauIqCElq4kU4SWpiC0KSmvhNOElqYgUsSU0WqQecRfrXYtElOVVVZ7vnod3Fv4s7157uCdxhTnVPQLuSfxd3KANYkpoYwJLUxADeWfb5dDv+XdyhvAgnSU2sgCWpiQEsSU0M4B2S5GiSV5JcS3K6ez7ql+TzSW4m+Xr3XNTDAN4BSfYCfwT8HHAEeDzJkd5ZaRf4AnC0exLqYwDvjIeAa1X1WlV9BzgPHGuek5pV1ReBb3XPQ30M4J1xAHhz4vXSMCbpDmYA74zcZsz1f9IdzgDeGUvAoYnXB4HrTXORtEsYwDvjK8DhJA8k+X7gOHCxeU6SmhnAO6CqloFPAX8NXAUuVNWV3lmpW5JngC8BP5pkKcnJ7jlpZ/lVZElqYgUsSU0MYElqYgBLUhMDWJKaGMCS1MQAlqQmBrAkNfk/dSCc+Z0qshgAAAAASUVORK5CYII=\n",
      "text/plain": [
       "<Figure size 432x288 with 2 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "sb.heatmap(cm)\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 42,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([[498,  29],\n",
       "       [ 77, 106]])"
      ]
     },
     "execution_count": 42,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "cm"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Confusion Matrix\n",
    "\n",
    "The confusion matrix shows the actual positives and negatives of the target classification (in this case, HIGH_COST) on the left column, plotted against the predicted positives and negatives going across the rows.\n",
    "\n",
    "Above, the first cell contains TRUE NEGATIVES (times the true value was negative, and the classifier predicted negative).  The classifier correctly predicted 498 out of 527 negative cases.\n",
    "\n",
    "The second cell contains the FALSE NEGATIVES (times the true value was negative, and the classifier predicted positive).  The classifier incorrectly classified 29 positive cases as positive.\n",
    "\n",
    "The second row, first cell contains the FALSE NEGATIVES (times the true value was positive, and the classifier predicted negative).  The classifier incorrectly classified 77 positive cases as negative.\n",
    "\n",
    "The second row, second cell contains the TRUE POSITIVES (times the true value was positive, and the classifier predicted positive high cost).  The classifier correctly predicted 106 out of 183 positive cases.\n",
    "\n",
    "Performance could be improved if we moved to a boosted classifer.\n",
    "\n",
    "The confusion matrix values can be used to calculate a wide variety of other important model metrics:\n",
    "\n",
    "https://en.wikipedia.org/wiki/Confusion_matrix"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Now...\n",
    "\n",
    "Let's try the whole model using \"entropy\" criterion instead of \"gini\" - just for comparison!"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 43,
   "metadata": {},
   "outputs": [],
   "source": [
    "clf2 = RandomForestClassifier(criterion=\"entropy\", min_samples_leaf=10, random_state=121)\n",
    "clf2.fit(train_x, train_y)\n",
    "\n",
    "# predict on test data\n",
    "prediction2 = clf2.predict(test_x)\n",
    "\n",
    "# compute evaluation scores\n",
    "accuracy2 = accuracy_score(test_y, prediction2)\n",
    "precision2 = precision_score(test_y, prediction2)\n",
    "recall2 = recall_score(test_y, prediction2)\n",
    "cm2 = confusion_matrix(test_y, prediction2)\n",
    "roc2 = roc_auc_score(test_y, prediction2)\n",
    "f12 = f1_score(test_y, prediction2)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 44,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.8492957746478873"
      ]
     },
     "execution_count": 44,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "accuracy2"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 45,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.7753623188405797"
      ]
     },
     "execution_count": 45,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "precision2"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 46,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.5846994535519126"
      ]
     },
     "execution_count": 46,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "recall2"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 47,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.6666666666666667"
      ]
     },
     "execution_count": 47,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "f12"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 48,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<AxesSubplot:>"
      ]
     },
     "execution_count": 48,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAdcAAAD4CAYAAAC+CayWAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjMuMiwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy8vihELAAAACXBIWXMAAAsTAAALEwEAmpwYAAAqzElEQVR4nO3dd7gdZb328e8NSIl0kCoSQaqUAFE4FA8QpB0UOCqEQ7UhClJ8QRDkPRyOKEdAigSQEtqBUEKogvQISCiB7FRKyEukh14iIbTf+8c8K4wrq+/Za6+dfX+ua1+s9czMM781WeTJPDN7bkUEZmZmVpz5ersAMzOzeY0HVzMzs4J5cDUzMyuYB1czM7OCeXA1MzMr2AK9XYB1hmWXXTYGDhzY22WYmfUZjz322OsR8YVKyzy4GgADBw5k7NixvV2GmVmfIenv1ZZ5WtjMzKxgHlzNzMwK5sHVzMysYL7magBMfPEdBh7z517Z9/ST/61X9mtm1lN85gpI+kRSl6RJkq6VNKBC+82SlpR0gKQRZdsvK+k1SQtJGi3pKUkTJD0p6WxJS1bZ182lZZLmk3RWap8o6VFJX65R8/S03nhJd0haIbXPLFvvAElnF3e0zMysHg+umVkRMSgi1gM+BA6q0P4mcDAwCvhmaQBOvgvcFBGz0/u9I2IDYANgNnBjlX2V+gTYE1gJ2CAi1gd2B96uU/c2EbEhMBY4tulPbWZmPcKD69zuB75SoX0MsHJEvAvcB3wrt2woMKJ8g4j4EPgl8CVJG1brM71eEXg5Ij5N274QEW81WPN9VWo2M7Ne4ME1R9ICwE7AxLL2+YEhwE2paQTZgIqklYA1gXsr9RkRnwDjgbXr9HkN8K00ZXyapI2aKH2X8pobIelASWMljf3k/Xea3dzMzKrw4JpZRFIX2fTqc8BFZe1vAEsDd6b2W4AtJS0O7AGMTINoNaqwr3/qMyJeANYCfgV8CtwtaUiduu9NfS0O/K7GehVDeyPi/IgYHBGD5x+wRJ1dmZlZo3y3cGZWRAyq1i5pCbIB9WDgrIiYJekvZNdFhwJHVOs4naGuDzxRq0+AdM32NuA2STOA3YC7a9S9TUS8Xl6zpAXTlDRkA3j5OmZm1oN85tqAiHgHOBQ4UtLnUvMI4BfA8sBDlbZL6/4OeD4iJtTqU9LGaYoZSfOR3QxV9dFaNfwV2Cf1swjZmXXFKWszM+sZPnNtUESMkzSe7Ez1cuAO4FLgoogon3a9QtJsYCHgLmDXBvp8DbhA0kJp8SNAK79CcxjwJ0mHkk1HXxYR99XbaP2Vl2Csf9/UzKwQmntcsP5o8ODB4Qf3m5k1TtJjETG40jJPC5uZmRXM08IdTtLDZNPLeftGRNO/emNmZu3hwbXDRcSmvV2DmZk1x9PCZmZmBfPgamZmVjAPrmZmZgXzNVcDejfPFZzpambzFp+5mpmZFazm4CppmZTS0iXpFUkv5t5/SdKNkqZKmibpTEkLStoht87MFBzeJemyGvv5uqT70rpPSrpQ0rqSXkiPAsyv25XWPyFXzxRJe5Wt94vUVylQ/A+5RxdWqmG6pPsr7GtS7v2Wkh5J/T4p6cDcsrVSUHqXpCcknV9jX1tLekfSuLTuf5a1dykLW79L0nI1+jlAWUh76Rj8OCXdXJ1bZ/H051M1eN3MzIpVc3CNiDdSsPcg4Dzg9PR6I2AkcENErEEWubYocFJE3J7bZixZcPigiNiv0j4kLQ9cCxwdEWsB6wB/IUuNeR7YKrfu2sBiEfFIairVsyvZI/8+l9Y7CNge2CwFj38NeBVYpM7xWEzSKqmPdcrqXAG4EjgoItYGtgR+Iqk0n3lWqZ6IWAf4Y5193R8RGwGDgX0kbZJrH5TC1h/lszD1aq5Ox2Br4LdkwexflLRdWn4iMDwinq3Tj5mZFaTVaeFtgQ8i4mKYk1l6BPADSQOa7Otg4NKIGJP6iogYGREzyOWmJtVCyacC7wNLpabjgJ9GxNtp+YcRcXIKOq/lGmDP9Hqvsn0dDFwSEY+nPl8nC0I/Ji1fEXghV1NDD3mIiH8AjwGr59slCVgMaCgwPSJeBaYBqwI/Bc6QNJgsM/aUSts4z9XMrGe0Orh+lWxAmCMNXM8BX2myr/XK+8q5BthNWYg5ZAPfVeUrSdoYmBoRr0paDFi0xTO1kcC/p9ffAm7OLZvrM5OdmX81vT4duEfSbZKOkLRkIzuUtAywGTA5NW2VMlqfA7YDhjfYz2rAasAzKYHndrK4ukNz8XP/xHmuZmY9o9XBVVQO4K7W3pKIeIVs0BkiaRDwUURMyq1yhKSngIeBEyrVkLsGPF3S5nV2+SbwlqShZPmr7+eWVftskWq9mGxK+1qyKdqHcgk3lWwlaRxZus7JEVEaXEvTwqsAFwO/r1PznmkwHgH8JCLeTO3DgBcjwnFzZmZt1urgOpnsWuEckhYHViGbmmy2r01qLC9NDVeaEj49XafdE7hM0sLpDPofpRt4SteAgUnAgg3UczXZwFS+r7k+c6p7SulNRLwUEcMjYlfgY7Kz8mruj4iNImKTiDivyjo3Ad+oV28ajDeNiOtz7Z+mHzMza7NWB9e7gQGS9gOQND9wGtk1yfdrbjm3s4H9Jc15hq6kfdINRADXATtTZUoYICJGkU3R7p+afgecW5qaTdcvF26wnuvJzhZvL2sfBhyQzqBL07n/k9ZF0o65G6pWAJYBXmxwn9VsSfP/WDEzs17W0kMkIiIk7Q6cI+l4skH6VuDYFvqakaZhT02/dvIpcB8wKi1/W9JDwPJ1rqOeCFwp6QLgXGAA8LCy0PKZwN+AcQ3U8x7ZoEk2Js9pf1nSPmSB5ouRTROfERGl67LbA2dK+iC9PypNazerdM1VwDvAj1roo2kOSzczK47D0g1wWLqZWbPksHQzM7P2aduzhSXtQJpuzXk2InZvVw2pjraFjxf1mSV9HzisrPlvEVHvARNmZtYLPC1sgKeFzcya5WlhMzOzNvLgamZmVjAPrmZmZgVzWLoBvR+WXuLQdDObF/jM1czMrGD9ZnBVG4Lfy8LOSz/bpWUzK6x/gqQj0+ubJO2bW3aBpKNqfJ7RqZ7xkv4maa3UPl3SsmU13dL6kTMzs2b1m2nhiHgDGATZoAbMjIhT03OHHwbOjYhd03OSzycLfj+K9IxhSaOBIyOi3u+r3B8Ru7RQ4qHAvZJuBtYFNgV+VmebvSNirKQDyTJbv93Cfs3MrGD95sy1hiKD31sWEdPJBvXfA+cAh0TERw1ufh/N5+g6LN3MrId4cC02+B3Sg/dzP6s3se2pwI7A5Ii4r4ntvgU0/YQph6WbmfWMfjMtXEPRwe+tTgsDbJD2u7ak+SKiXh7rFZJmAdOBn6e2qoHuZmbWHj5zLTb4vWWS5iObDt4XmAr8tIHN9k5B6btFxPOp7Q1gqdw6SwOvF1qsmZnV5DPXLPj9ZEn7RcRl3Qx+746fAFMjYrSkp4Exkq6JiNea7Gc02QD9f9Nn2Qe4od5GznM1MytOvz9zjSy5YHfge5KmAk8DH9BC8HtSfs31u6l9gKQXcj+/KG2QQuKPBo5MNb0EnEl2c1Oz/hv4iqTxZOHwzwD/2+JnMTOzFjgVxwCn4piZNcupOGZmZm3ka64taGfwu6TrgS+XNR8dEbcXvS8zMyuGB9cWpIGtLYNbTwzYZmbWszwtbGZmVjAPrmZmZgXz4GpmZlYwX3M1oHPC0sGB6WbW9/nM1czMrGB9YnBtU9D5AElXSJooaZKkByStWmO/C0r6JL2eJOlmSUumvgZKmpTr+8eSHpe0VJV9XyLp2dTX45L+pUL7eElDctuUwtJL9YxMwehjyvpeQNIMSSsW8EdhZmYN6BPTwm0KOj8MmBER66dt1gJeiYi59lvaQNKs3PJLgYOBk/KdStqXLLFm24h4q8b+j4qIkZK2B/5ElpCTb98mfbY1ctvsnf9M6eH/X5Q0MOXDAmwHTIqIl2vs28zMCtQnzlxrKDLofEXgxdKbiHgqImY3sf0YYOV8g6Q9gGOA7SOi0WSaasHnc/VfLkXUXQvsmWseCoyotL7D0s3MekZfH1yLDDofDhwtaYyk30hao+4WSTpjHgLclGteFTibbGB9pYk6qgWf78jc6TZX5KaFT0ltI8gGVCQtBOwMXFdpRw5LNzPrGX1iWriGwoLOI6JL0mrA9mRTqY9K+peIeKLGZotI6gIGkg3yd+aWvQa8CewBnN5ACadI+nXa7odl7b8HlgM2K9tm7/Kp7oh4VNKiaVp7HeChOtPRZmZWsL5+5lpo0HlEzIyIURHxM7KYtp3rbFK65roqsCDZNdeS94GdgIMk7d3A7o9KweffjIhJ+Xays/BfA5c2+FGuIjt7rTolbGZmPaevn7kWFnQuaQtgSkS8JWlBYF2y4PG6IuIdSYcCN0o6N9f+mqQdgdGSXm/1YfsR8amkM4H9Je3QQD8jgBuBJfjns+CqHJZuZlacPn3mWnDQ+erAXyVNJAsZH0uVa5VVahkHjCdd78y1Pwt8GxguadMW6ir1E8BvgF/mmvPXXO/KrTuF7Mz5noj4R6v7NDOz1jgs3QCHpZuZNcth6WZmZm3U16+5Nq2dQecV9j0M2KKs+czS7+mamdm8od8Nru0MOq+w74Prr2VmZn2dp4XNzMwK5sHVzMysYP1uWtgq66Q812Y4+9XMOpHPXM3MzArW8YNrm7Jct5b0Tm6bLknbpWXHSZosaUJq3zS1l/JUx0t6VNKgXH/TleXCTpQ0JQUBLFTjMw6UNCv1P0XSeZLmq9B+maTP1ao51bVDWf+HSzqngD8OMzNrQMdPC7cpyxXg/ojYJd+gLLR8F2DjiJgtaVmyZwiX7B0RYyV9HzgF+GZu2TYR8bqkRVNd5wP719j/tIgYJGkB4B5gN+DxXPv8ZMEAewBX1Kh5dbKnROXviB5K9oxiMzNrg44/c62hyCzXalYEXi/lukbE6xHxUoX1qmatRsRM4CBgN0lL19thRHwMPEhZZF76fI9U20/OSGCX0pmypIHASsAD9fZtZmbF6MuDa5FZrgBblU2xrg7cAawi6WlJ50j61yrbVspaLa/rWaBuRmz6h8EQyjJdJS0MbAr8pVbN6Uz/kVQTZGetV0eF51zKYelmZj2i46eFaygsyzWZa4oVQNImwFbANsDVko6JiEvS4iskfR6YH9i4gXprWV1ZNmwAN0bEbemss9S+BjAyIibUq5nPAtNvTP/9QaUdRkRpupqFVlzDD5k2MytIXz5zLTTLtZqI+CQiRkfEfwKHAN/JLd4b+DJwJTCsWh+SFiMLVH+6xq6mpTzXjSLihPJ2srPxzSR9u4GybwCGSNoYWCQiHm9gGzMzK0hfHlzvBgZI2g9A3chyrUbSWpLyU7mDgL/n14mIj8iCzDeTtE6FPhYFzgFuiIi3Wq0lIl4GjgF+1cC6M8myaIfjsHQzs7brs9PCERGSdgfOkXQ82T8UbqW1LFdI1y9z739Ddp30j5KWBD4GngEOrFDLLEmnAUfyWTj5vemO5vmA64H/brGuvBuAEyRtVa3miBiZXo8ARlGWL1uNw9LNzIrjPFcDnOdqZtYsOc/VzMysffrstHAr1ItZrmn/6wOXlzXPjohN27F/MzNrj341uPZmlmva/0TS06bMzGze5WlhMzOzgnlwNTMzK5gHVzMzs4L1q2uuVp3D0s3MiuMzVzMzs4LNs4Or2hCynvbzdUn3pXWflHRhKfJO0k4pdeaJtOzUGv2cIOnI9PqSVG8pNm5ZSdNz664p6VZJz6S+r5G0fC5AfVyq5z5JlR7sb2ZmPWienRZuR8i6pOWBa4GhETEm9f0dYDFJqwFnA/8WEU+mEPS5Hp1YwydkaTbnlu1zYeDPwC8i4ubUtg3whbTKnKQcSYOAGyTNioi7m9i3mZl1wzx75lpDkSHrBwOXRsSY1FdExMiImAH8kmzAfjIt+zgizmmi7zOAI9KgnPcfwJjSwJr6vjciJpV3EBFdwIlkaT5zcZ6rmVnP6I+Da5Eh6+uV99XgskY8BzwA7NvNfh8H1q60ICLOj4jBETF4/gFLtFalmZnNpT8OrkWHrPek3wJH0b0/p3oh7WZmVrD+OLgWGbI+GdikhWUNiYhngC5gj270uxHwRHfqMDOz5vTHwbXIkPWzgf0lzXnwvqR9JK0AnAIcK2nN1D6fpF+0UO9JZDmxJVcCm0ua8wueknZMoQD/RNIGwPHAsBb2a2ZmLZpn7xaupsiQ9YiYIWkocKqk5YBPgfuAURHxiqTDgRHpRqkgu8u32X1MlvQ4sHF6Pyv9es0Zks4APgImAIcBy5AFqI8DBgCvAoc2cqeww9LNzIrjsHQDHJZuZtYsh6WbmZm1Ub+bFm5FkSHrko4DvlfWfG1EnNRqfWZm1lk8uDagyJD1NIh6IDUzm4d5WtjMzKxgHlzNzMwK5sHVzMysYL7makDfDUvvSxzsbtZ/+MzVzMysYPPk4NpHg9JLNU6S9O0K7VMk7ZXb5hJJz+ZqflDSQEkvSJqvrP8uSV/v/pE1M7NGzJPTwn00KP30VOM6wP3pcYr59jWAxySNjIiP0rKjImJkWV3PA1sBf03v1wYWi4hH6uzfzMwKMk+eudbQ8UHpEfEE8DGwbFn7VOB9YKk6XYwAhubeD01tc3FYuplZz+hvg2vHB6WnhJ1PgdfK2jcGpkbEq7nmU3LTwlektmuA3dLZMsCewFWV9uWwdDOznjFPTgvX0MlB6UdI2gd4D9gzpfeU2n8MrAbsWLbNXNPCKY1nMjBE0gzgo4iY1Ib6zcws6W9nrp0clH56RAyKiK0i4v6y9rXIzkAvk7RwA32VpoarTgmbmVnP6W9nrncDJ0vaLyIuKyAo/RFJf46IhyELSgfuIgtKHyXpgYh4Ot29e3hE/KHVwiNilKT9gf2BP9VZ/Trgt2TXaLdtpH/nuZqZFadfnblGFl67O/A9SVOBp4EPaDEonezM8NT0qzhPkN2l+25ETAAOJwtKfwKYBKxYwEc4EfhF7ldt8tdcuyQtmGp7G3gImBERzxawXzMza4LD0g1wWLqZWbMclm5mZtZG/e2aa9MclG5mZs3y4FqHg9LNzKxZnhY2MzMrmAdXMzOzgnlwNTMzK5ivuRrgsPTe4PB0s3mXz1zNzMwK1icGV0krSLoqhZtPkXSrpDUlhaSf59Y7W9IBkoblAsZn5Z5g9EtJV+fWXzz1+eUq+x0taXDu/UBJk9LrAZKukDQxBZw/IGnRtOyTXPD5zZKWrPHZBuZqnCLpPEnzVWi/TNLnctttKemRFMT+pKQDc8sqhq+bmVl7dPzgmkLIrwdGR8TqEbEu2eMKlwdeBQ4rPfavJCIOjohBwM7AtPRA/EFkz/z9oqTt0qonAsNbfETgYWSPF1w/ItYDfgiUQsxnpX2uB7xJlv1ay7RU3wbAusBuZe3rA18E9oDsHxvAlcBBEbE2sCXwE0n5ecbT07bfA4bnHploZmY9rC/8hbsNWWzaeaWGiOgCnifLPL2b7GH2daVnC/8UOCOdkQ4hG3BbsSLwYq7vpyJidoX1xgArN1jfx8CDlGXLplD3R3L9HEwWNvB4Wv46WUD7MRX6rBi+Dg5LNzPrKX1hcK0XPH4y8H9Swk1d6aH6t5MNyodGxIct1jUcOFrSGEm/kbRG+QqppiHATY10KGlAWn9iWfvCwKbAX1LTXKHvwNjUXt5nxfB1cFi6mVlP6QuDa01pSvcR4D+a2GwY8GJE3Fuv+2pt6ex5NbIz36WBRyWtk9ZZRFIX8EZadmed/aye1v8b8OeIuK2s/Q3gufQPA6ge7p5vOyJteyopfL1ODWZmVpC+MLg2Ejz+W+BoGv88n6afet4Alsq9Xxp4vfQmImZGxKiI+Bnwv2TXeCFdcwVWBRakwWuuEbFRRJxQ3k42TbxZ7sakuULfyY7RlNz7auHrZmbWw/rC77neA/xW0o8j4gIASV8DBpRWiIgnJU0BdiE7iy3KaGAfSXelM7/9gXtTDVsAUyLirXRD1bpp/Tki4h1JhwI3Sjo3Ij6iBRHxsqRjgF+RTTEPAx6WNCoiuiQtQxYucGJLnxKHpZuZFanjz1xzAeffTL82Mxk4AXipbNWTyO6oLdL5wHvAeEnjgUXJplkBVgf+KmkiMI7smud1FeofB4wnC1bvjhuAAZK2ioiXgX2ACyQ9SXYT1PCIuLmb+zAzswI4LN0Ah6WbmTXLYelmZmZt1BeuufY4ScOALcqaz4yIiwvcx/rA5WXNsyNi06L2YWZmncGDK9kTndqwj4nAoJ7ej5mZ9T5PC5uZmRXMg6uZmVnBPC1sgPNce5uzXc3mLT5zNTMzK1hHDa6Slsllr76SyyTtkvQlSTdKmpoeJnGmpAUl7ZBbZ6akp9Lry2rsp9tZqJK2ljSmrG0BSTMkrShpM0kPp36eSP1+P1frhykLtkvSyZKWl3SLpPEpv/XWGvVXy4DdWtItZeteIum7jf8pmJlZd3XUtHBEvEG6o1bSCcDMiDg1Zbo+DJwbEbumtJnzgZMi4iiylBskjQaOjIiqT0PIZaHuFhGPS1oWuF3SixFRmhc9Pe13HeB+SctFRPmziO8jy4YdGBHTU9t2wKT0uMLRwB4RMT7Vu1ZETAEuTnVMB7ZJcXFI+hNwZ0Scmd5vUOdwTYuIQZIWIHtE5G5k2bFmZtbLOurMtYZtgQ9Kv3ea8k2PAH6QYtqaUUgWahpsrwX2zDUPBUak18sBL5fqTQNrLSsCL+T6n1Bj3XwdFTNgzcys9/SVwXWu/NKIeBd4juYHlUKyUJMRpGcGS1qILBWn9Hzh04GnJF0v6Scpk7WWYcBFku6VdJyklRr5MNUyYBvc1mHpZmY9oK8MrtXyS6u1t9JX01moEfEosKiktYCdgIci4q207ESyWLg7yLJm/1Kpj1xft5Plw14ArA2Mk/SFGptUyoCtdiyq1e+wdDOzHtBR11xrmAx8J98gaXFgFWBaC30NJotuK6mUhXoqjbmK7Ox1HT6bEgYgIqYB50q6AHhN0jLpunJFEfEm2fXgK9ONSd+gQtJOUsp6zSvPn4WyDFozM+t5feXM9W6yuLX9ANINQqeRXTt9v8m+hgEHSBqU+iplof6+xdpGkMW/bUtuwJb0b+lGLIA1gE+At6t1Imnb0vVjSYuRRdo912QtU4GV0o1YSFoV2BDoarIfMzPrhj5x5hoRIWl34BxJx5P9o+BW4NgW+npZUikLdTGyaeIzWs1CjYgpkt4HHouIf+QW7QucnpZ9DOydbsSqZhPgbEkfk32+C9O0czO1zE6f7eJ0jfcj4EcRUfeCqsPSzcyK4zxXA5znambWLDnP1czMrH36xLRwKyTtQHYtNe/ZiNi9hb6OA75X1nxtRJzUan0N7tcZsGZmfZCnhQ3wtLCZWbM8LWxmZtZGHlzNzMwK5sHVzMysYPPsDU3WHIelWzkHuJu1zmeuZmZmBfPgWoOkT3Kh6dfmHk+Yb79Z0pK5bb4q6R5JTysLdj8+9xhEJO2UkmieSEHtp6b2tSSNzoWrn1+jrq0lvSNpXFr3P1P7AZLOLlt3tKSKd7OZmVnP8OBa26yIGBQR6wEfAgdVaH+TLCMWSYuQPV/45IhYk+y5vpsDP0vL1wPOBvaJiHWA9YD/l/o8iywwYFBa9sc6td0fERuRhRDsI2mTYj6ymZl1lwfXxt1P5ezYMcDK6fV/AH+LiDsAUqjAIXwWxP5L4KSIeDIt/zgizknLysPSG8pnTc8zfozsQf9NcZ6rmVnP8ODaAEkLkOW1Tixrn58sqLyUhlMp1H0aWebr4mRnquVB7SWnA/dIuk3SEfmp5jq1LQNsRhal1xTnuZqZ9QwPrrUtkgLJx5LFv11U1v4GWV7qnam9Vnh7zUdhRcTFZJmw1wJbAw9JWqjGJltJGkcWxn5yRExudd9mZlYsD661la6tDoqIn0fEh/l2YFVgQdI1Vz4LYp9D0mrAzIh4Ly2vem00Il6KiOERsStZTN16NWq7PyI2iohNIuK81OawdDOzDuDfc+2GiHhH0qHAjZLOBa4AjpW0XUTclW5wOovPgthPAUZJeiAinpY0H3B4RPxB0o7A3RHxkaQVgGWAF5ss6VGyTNgVIuKVdJfwQsDz9TZ0nquZWXE8uHZTRIyTNB4YGhGXS9oV+KOkYcD8ZKk2Z6d1J0g6HBiRfq0ngNKTG7YHzpT0QXp/VES80mQtMyQdBtyaBu6ZwF4R8Wk3P6aZmTXBqTgGOBXHzKxZTsUxMzNrI08Ld7AiA9/NzKx9PLh2sIi4Hbi9t+swM7PmeFrYzMysYB5czczMCubB1czMrGC+5mqAw9LNOoED6ucdPnM1MzMrWJ8dXCUdJ2mypAkpYHzTFAz+lKTxkh6VNKjG9g+n7Z6T9Fp63SVpoKQlJF0maVr6uSy1rZ9b701Jz6bXd1XZx0BJk9LragHn+fanJN0naZeyfhaQ9Lqk36X320saUwphlzR/qmPzZkLXzcysZ/TJaWFJ/wLsAmwcEbMlLUv2AH2AvSNirKTvkz3L95uV+oiITVNfBwCDI+KQXP8jgUkRsV96/1/AhRHxPWBQarsEuCUiRjZR+v0RsYukzwNdkm7Jt6d+BwE3SJoVEXen5dsDTwF7SDo2Iu6Q9APgh8CFwM+BRyPiQUm3k4Wu35j6W7+J+szMrAB99cx1ReD1iJgNEBGvR8RLZevkQ8wbJukrZMk1/51rPhEYLKnpQPJKagWcR0RX2t8huea9gDPJYu82S21HAL+S9NW07tGpveHQdYelm5n1jL46uN4BrCLpaUnnSPrXCuvsCNzQQt/rAl0R8UmpIb3uIgtD77YGAs4fB9ZO6y5CFsh+CzCCbKAlIl4GziD7R8RvIuLNtG3DoesOSzcz6xl9cnCNiJlkZ5cHAq8BV6fpXYArJL1Adib3xxa6rxZ4XisIvVGVAs6r1VCyC3BvRLwPXAfsLmn+tGwYMH9EXFJauYXQdTMzK1ifvOYKc84mRwOjJU0E9k+L9gbGAyeTDT7/3mTXk4GNJM1XimpL8W0bAk90s+w511br2Ci3r72ALSRNT++XAbYB7oqITyXNNeCnKfLhwPB0Q9V6ZNPQZmbWBn1ycJW0FvBpRExNTYOAv5MNIqTA8V8D0yStExEND4oR8Uw6u/w12bVP0uvHI+KZoj5DNZI2AI4HfiRpcWBLYJXS9eV0o9ZeQLU7lFsKXXdYuplZcfrktDCwKHCppCmSJpBdJz0hv0JEzAJOA45sof8fAmtKekbSNGDN1NZTtir9Kg7Z2fah6U7hfwfuKQ2syY3At2tM9W4PTFIW4H47LYSum5lZ9zgs3QCHpZuZNcth6WZmZm3UJ6+5NkvSw0D5NOq+tX4HtMn+1wcuL2ueXXpQhZmZ9S/9YnDt6UEuDdKDenIfZmbWd3ha2MzMrGAeXM3MzArmwdXMzKxg/eKaq9XnsHQz6wv6SqB8x5+5Sloml6H6iqQXc++/JOlGSVNT7uqZkhaUtENunZkpJ7VL0mVV9rF1Kf5N0gH6LN91iqQfV9lmoKQX0qMR8+1dkr5eKVe1Vl2SBki6QtJESZMkPSBp0RrH5ZO07SRJ16bt5+TH5tY7QVIrD9IwM7MWdfyZa0S8wWcZqicAMyPiVEkCHgbOjYhd08PszwdOioijyJ5OhKTRwJER0cwTEq6OiEMkLQdMlnRTRMwoq2u6pOeBrYC/pn2tDSwWEY9UylVNdxVXrEvSr4AZEbF+er8W8FGNGmdFROm4XAEcBIxq4jOamVkP6fgz1xq2BT5IKTClB/kfAfxA0oAidhARrwLTgFWrrDICGJp7PzS1QRO5qrn15zwDOCKeKnvsYS33A19pcF0zM+thfXlw/SplSS8R8S5ZoHghA42k1YDVgGoP7L8G2E1SaQZgT+Cq9LrhXNVkOHC0pDGSfiNpjQZrXADYCWj6gRhyWLqZWY/oy4NrT+au7impi+ws9Ce5IPJ/kh6IPxkYImkQ8FFETErLmspVjYgusoH8FGBp4FFJ69SocZFU41iyf1BcRPXPXbHdYelmZj2j46+51jAZ+E6+IUW0rUI2ldsdV0fEIQ2uW5oansFnU8JA87mqKQR+FDBK0qfAzlTPkJ1zzbVE0hvAUmXrLQ082+BnMTOzAvTlM9e7gQGS9gNINzSdBlwSEe+3sY7ryAbB/JQwknaU9Ln0um6uqqQtJC2VXi9IFqP392YKSYPzy5KGpH6WBnYEHmimHzMz654+e+YaESFpd+AcSceT/UPhVuDYNtfxtqSHgOUjIn+GuD1wpqQP0vt6uaqrA+emu6DnA/5MNnA3az9gmKTT0vv/ioi6Z/IOSzczK47zXA1wnquZWbOc52pmZtZGfXZauBWSdgD+p6z52YjYvc523wcOK2v+W0QcXGR9Ffa7DNm15XJD0sM1zMysA/WrwTUibic9IanJ7S4GLi6+orr7nfN0KjMz6zs8LWxmZlYwD65mZmYF8+BqZmZWsH51zdWqc56rmfU3PZkN6zNXMzOzgnlwrUHSCpKuSkHsUyTdKmlNSV+VdI+kp1NQ+/HpyUr5sPVxadntkjbP9XmJpGdzoekPpvblJd0iaXxpXzXqGihpVi7Q/TxJ8ykX+l62v+/21DEyM7O5eXCtIg2W1wOjI2L1iFiX7NGKywM3ASdHxJrAhsDmwM9ym18dERtFxBrAyWQP4s8n3BwVEYPST2ngPRG4MyI2TPs6pk6J09KD+zcgew7xbt35vGZmVhwPrtVtQxYhd16pIcXCrUn2AIk7Utv7wCFUGQwj4l7gfODAOvsrD1ef0EiREfEx8CAtZNg6z9XMrGd4cK2uWjxcpZD2acCiKfKukseBtXPvT8lNC1+R2oYBF0m6V9JxklZqpEhJA4AhtBCW7jxXM7Oe4buFm1crjL1au8reHxURI/9pw4jbJa1GFhG3EzBO0noR8VqVPldPYekB3BgRt0n61ybrMjOzHuDBtbrJQKUbgSYD38g3pEFxZkS8l+5rKrcR1UPP54iIN4ErgSvTjUnfoHrsXOmaa161sPTX6+3bzMyK42nh6u4BFpL041KDpK8BU4EtJW2X2hYBzgJ+X6mTdDZ5IHBBrZ1J2jZN8SJpMbJ81+earHkqsFLp5ilJq5LdcNXVZD9mZtYNPnOtIhfGfoakY4APgOnA4cCuwB8lDQPmBy4Hzs5tvqekLYEBwLPAdyIif+Z6iqRf595/HdgEOFvSx2T/6LkwIh5tsubZkvYBLpa0MPAR8KOIqHu3ksPSzcyK47B0AxyWbmbWLIelm5mZtZGnhTuYpPXJppzzZkfEpr1Rj5mZNcbTwgaApPeAp3q7jhqWpbPvenZ93eP6usf1dU+r9a0aEV+otMBnrlbyVLVrB51A0ljX1zrX1z2ur3v6Y32+5mpmZlYwD65mZmYF8+BqJef3dgF1uL7ucX3d4/q6p9/V5xuazMzMCuYzVzMzs4J5cDUzMyuYB9d5kKQdJT0l6Zn0XOTy5ZJ0Vlo+QdLG9baVtLSkOyVNTf8tT9/p8fokrZLybp+QNFnSYbltTpD0Yi4nd+d215eWTZc0MdUwNtfeCcdvrdzx6ZL0rqTD07J2Hr+1JY2RNFvSkY1s2+bjV7G+Dvr+1Tp+nfD9q3b8OuX7t3f6/2KCpAclbVhv25aOX0T4Zx76IQsSmAasBiwIjAfWLVtnZ+A2spzZzYCH621LlvpzTHp9DPA/vVDfisDG6fViwNO5+k4AjuzN45eWTQeWrdBvrx+/Cv28QvZL8O0+fssBXwNOyu+zg75/1errlO9fxfo66PtXtb4O+f5tDiyVXu9ED/395zPXec/XgWci4v9FxIfAVWQpPnm7ApdF5iFgSUkr1tl2V+DS9PpSYLd21xcRL0fE4wAR8R5ZRu7KLdZReH11+u3141e2zhCyTOC/t1hHy/VFxKuRJT591MS2bTt+1errlO9fjeNXS68fvzK9+f17MCLeSm8fAr7YwLZNHz8PrvOelYHnc+9fYO6/AKqtU2vb5SPiZcj+kiH712m765tD0kCyEPqHc82HpKme4d2Y9upufQHcIekxSQfm1umo4wcMBUaUtbXr+LWybTuPX129/P2rpRO+f43olO/fD8lmeept2/Tx8+A671GFtvLft6q2TiPbdld36ssWSosC1wGHR8S7qflcsoD5QcDLwGm9VN8WEbEx2XTTwZK+0WId1RRx/BYEvg1cm1vezuPXE9s2qtv76IDvXy2d8P2r3UGHfP8kbUM2uB7d7LaN8OA673kBWCX3/ovASw2uU2vbGaWpxfTfV3uhPiR9juwvtisiYlRphYiYERGfRMSnwAVkUzxtry8iSv99Fbg+V0dHHL9kJ+DxiJhRamjz8Wtl23Yev6o65PtXVYd8/+rp9e+fpA2AC4FdI+KNBrZt+vh5cJ33PAqsIenL6V+IQ4Gbyta5CdhPmc2Ad9JUR61tbwL2T6/3B25sd32SBFwEPBERf8hvUHZNcXdgUi/U93lJi6V6Pg9sn6uj149fbvlelE3Jtfn4tbJtO49fRR30/atWX6d8/+rp1e+fpC8Bo4B9I+LpBrdt/vjVu+PJP33vh+xu0afJ7nw7LrUdBByUXgsYlpZPBAbX2ja1LwPcDUxN/1263fUBW5JN00wAutLPzmnZ5WndCel/hBV7ob7VyO4wHA9M7rTjl5YNAN4Alijrs53HbwWys4R3gbfT68U76PtXsb4O+v5Vq69Tvn+1/nw74ft3IfBW7s9wbK1tWz1+fvyhmZlZwTwtbGZmVjAPrmZmZgXz4GpmZlYwD65mZmYF8+BqZmZWMA+uZmZmBfPgamZmVrD/D5vlg/1dYYPDAAAAAElFTkSuQmCC\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "feat_importances2 = pd.Series(clf2.feature_importances_, index=train_x.columns)\n",
    "feat_importances2.nlargest(13).plot(kind='barh')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 49,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYIAAAEGCAYAAABo25JHAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjMuMiwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy8vihELAAAACXBIWXMAAAsTAAALEwEAmpwYAAAkbklEQVR4nO3de5wU1Zn/8c8jSEC5GET9sSDLGEe5M5GJI14ihBCvwKpRgq4Gk7xYL2iyu97NqsHEkIhRUROCUVCDwHqLqKiLMQYVlYsMMIDiqFwmkoiIBFQE5Pn9UTVj0/T01DhTPdNd3/frNS+6qk5VPTVAPX3OqTrH3B0REUmuvZo6ABERaVpKBCIiCadEICKScEoEIiIJp0QgIpJwLZs6gPrq1KmTd+/evanDEBHJK4sWLfrA3Q/ItC3vEkH37t1ZuHBhU4chIpJXzGxNbdvUNCQiknBKBCIiCadEICKScEoEIiIJp0QgIpJwsSUCM7vXzN43s4patpuZTTSzSjNbamZHxBWLiIjULs4awVTgxCzbTwKKw58xwO9ijEVERGoR23sE7j7XzLpnKTICuN+DcbBfNbP9zKyzu6+PKyYRkVQPvraWx8v/1tRhRNbrX9pz/bDejX7cpnyhrAuwLmW5Kly3RyIwszEEtQa6deuWk+BEpDBku9m/9u6HAJQVdcxlSM1OUyYCy7Au4yw57j4ZmAxQWlqqmXQk0fLtW2xTy3azLyvqyIiSLpxdluwvmE2ZCKqAg1OWuwLvNVEsIjnT0Bu5vsXWj272dWvKRDALGGtmM4AyYLP6ByQJHi//GyvW/5Nendt/qf11Y5PGFlsiMLPpwCCgk5lVAdcDewO4+yRgNnAyUAl8ApwfVywiDdHYTTHVSWDmfwxstGOKNEScTw2NqmO7AxfHdX6RLyv9xt/YTTG9OrdnREmXRjmWSGPIu2GoReKQevNPv/GrKUYKnRKBJE6mpp7Um79u/JI0SgSSGNUJIFNTj27+kmRKBFLwMiUA3fRFvqBEIHmvrqd6lABEslMikGavPjf6TJQARLJTIpAmE/X5fN3oReKlRCCxq+2GH/X5fN3oReKlRCCxq21IBd3gRZoHJQLJCQ2pINJ8ac5iEZGEUyIQEUk4NQ1Jo8j2BFBDhlwWkfipRiAN9uBra7nmsWU1TwGl02ibIs2bagTSINVJAOCm0/rqCSCRPKREIJFlG7VTSUAkfykRSGSZ3gfQuwAi+U+JQCJ58LW1vPbuh5QVddT7ACIFRokg4eo73o86fUUKjxJBgqV29Gq8H5HkUiJImExz86qjVyTZlAgKULbmHs3NKyLplAgKRKZv+pmae3TzF5F0SgR5LtN8vLrZi0h9KBHkuepn+3XzF5EvS4mgAGisfxFpCA06JyKScKoR5KnqvgEN8SwiDaVEkIfSXwTT274i0hBKBHmo+jFRvQgmIo1BfQR5qqyoo5KAiDSKWBOBmZ1oZm+aWaWZXZVhewcze8LMlpjZcjM7P854CkH1KKAiIo0ltkRgZi2Au4CTgF7AKDPrlVbsYmCFu/cHBgG3mFmruGIqBNXNQuoXEJHGEmcfwZFApbu/A2BmM4ARwIqUMg60MzMD2gIfAjtjjClvpT4lpGYhEWlMcSaCLsC6lOUqoCytzJ3ALOA9oB0w0t13pR/IzMYAYwC6dUvWDTDTEBKqDYhIY4ozEViGdZ62fAJQDnwL+Bowx8xedPd/7raT+2RgMkBpaWn6MQqahpAQkbjFmQiqgINTlrsSfPNPdT4w3t0dqDSzd4EewPwY48o7GkJCROIU51NDC4BiMysKO4C/R9AMlGotMATAzA4CDgfeiTEmERFJE1uNwN13mtlY4FmgBXCvuy83swvC7ZOAG4GpZraMoCnpSnf/IK6YRERkT7G+Wezus4HZaesmpXx+D/hOnDHks+p3BuqaT1hEpCH0ZnEzpncGRCQXlAiaOb0zICJxUyJopjSUhIjkikYfbUYyTUCvZiERiZsSQTOSOtGMXiATkVxRImgG0mcb08tjIpJL6iNoBlKTgJqCRCTXVCNoYqnvCqgmICJNQTWCJpQ697BqAiLSVJQImpDmHhaR5iByIjCzfeMMJKn0wpiINLU6E4GZHW1mK4CV4XJ/M/tt7JGJiEhORKkR3EowgcxGAHdfAnwzzqCSQG8Oi0hzEalpyN3Xpa36PIZYEkOdxCLSnER5fHSdmR0NeDjBzKWEzURSf6lJQJ3EItIcRKkRXABcTDAZfRVQAlwUY0wFS0lARJqjKDWCw939nNQVZnYM8HI8IRWO1EHk4IuB5JQERKQ5iVIjuCPiOklR/e0/tUO4rKijkoCINDu11gjMbCBwNHCAmf1Xyqb2BHMQSxZ6WUxE8kW2pqFWQNuwTLuU9f8EvhtnUIVCL4uJSD6oNRG4+1+Bv5rZVHdfk8OYREQkh6J0Fn9iZjcDvYHW1Svd/VuxRZXH0ucWEBFp7qJ0Fk8D3gCKgJ8Bq4EFMcaUt1I7iDW3gIjkiyg1gv3d/R4z+3FKc9Ff4w4sn1TXAvR4qIjkoyiJYEf453ozOwV4D+gaX0j5p7opSPMMi0g+ipIIfm5mHYD/Jnh/oD3wkziDykeaa1hE8lWdicDdnww/bgYGQ82bxSIiUgBq7Sw2sxZmNsrMLjOzPuG6U81sHnBnziJs5jSctIjku2w1gnuAg4H5wEQzWwMMBK5y9z/lILa8UP0GsZ4QEpF8lS0RlAL93H2XmbUGPgAOdfe/5ya05q+6NqA3iEUkn2V7j2C7u+8CcPdtwKr6JgEzO9HM3jSzSjO7qpYyg8ys3MyW59tjqaoNiEghyFYj6GFmS8PPBnwtXDbA3b1ftgObWQvgLmAowTwGC8xslruvSCmzH/Bb4ER3X2tmB375S2kaqg2ISL7Llgh6NvDYRwKV7v4OgJnNAEYAK1LKnA086u5rAdz9/QaeMyc0jISIFJJsg841dKC5LkDqXMdVQFlamcOAvc3sBYIRTm939/vTD2RmY4AxAN26Nf2379QkoGYhEcl3UV4o+7IswzrPcP4BwBCgDfCKmb3q7qt228l9MjAZoLS0NP0YTUIvkIlIoYgzEVQRPH5arSvB8BTpZT5w94+Bj81sLtAfWEUzkzrtpJqERKSQRBl9FDNrY2aH1/PYC4BiMysys1bA94BZaWUeB44zs5Zmtg9B09HKep4nJ6qbgwA1CYlIQamzRmBmw4AJBDOWFZlZCTDO3Ydn28/dd5rZWOBZgqkt73X35WZ2Qbh9kruvNLNngKXALuAP7l7RoCuKkZqDRKQQRWkauoHgCaAXANy93My6Rzm4u88GZqetm5S2fDNwc5TjiYhI44vSNLTT3TfHHkkzpvGERKSQRakRVJjZ2UALMysGLgXmxRtW86I3iEWkkEWpEVxCMF/xZ8CDBMNR/yTGmJolvUEsIoUqSo3gcHe/Frg27mBERCT3otQIfmNmb5jZjWbWO/aImpEHX1vLyN+/UvPYqIhIIaozEbj7YGAQsAGYbGbLzOyncQfWHGgoCRFJgkhvFofDT080s78AVwDXAT+PM7DmQu8OiEihq7NGYGY9zewGM6sgmKJyHsFwESIiUgCi9BFMATYB33H34939d/kyXHRD6N0BEUmKOpuG3P2oXATSXFQPLledBNQ3ICKFrtZEYGb/6+5nmdkydh8+OtIMZfmquoO4rKgjI0q66N0BESl42WoEPw7/PDUXgTQn6iAWkSSptY/A3deHHy9y9zWpP8BFuQlPRETiFqWzeGiGdSc1diAiItI0svURXEjwzf8QM1uasqkd8HLcgYmISG5k6yN4EHga+CVwVcr6Le6u5ypFRApEtkTg7r7azC5O32BmHZUMREQKQ101glOBRQSPj1rKNgcOiTEuERHJkVoTgbufGv5ZlLtwREQk16KMNXSMme0bfv53M/uNmektKxGRAhHl8dHfAZ+YWX+CkUfXAA/EGlUT0fhCIpJEUSevd2AEcLu7307wCGnB0dzEIpJEUeYj2GJmVwPnAseZWQtg73jDajqam1hEkiZKjWAkwcT1PwgnqOkC3BxrVCIikjNRpqr8OzAN6GBmpwLb3P3+2CMTEZGciPLU0FnAfOBM4CzgNTP7btyBiYhIbkTpI7gW+Eb1rGRmdgDwHPBwnIGJiEhuROkj2CttasqNEfcTEZE8EKVG8IyZPQtMD5dHArPjC0lERHIpypzFl5vZ6cCxBOMNTXb3x2KPTEREciLbfATFwATga8Ay4DJ3/1uuAhMRkdzI1tZ/L/AkcAbBCKR31PfgZnaimb1pZpVmdlWWct8ws8+b8mkkDS8hIkmVrWmonbvfHX5+08xer8+BwzeQ7yKY6rIKWGBms9x9RYZyvwKerc/xG5uGlxCRpMqWCFqb2df5Yh6CNqnL7l5XYjgSqHT3dwDMbAbBeEUr0spdAjwCfKOesTea6tqAhpcQkSTKlgjWA79JWf57yrID36rj2F2AdSnLVUBZagEz6wKcFh6r1kRgZmOAMQDdujX+jVq1ARFJsmwT0wxu4LEtwzpPW74NuNLdPzfLVLwmlsnAZIDS0tL0YzQK1QZEJKmivEfwZVUBB6csdwXeSytTCswIk0An4GQz2+nuf4oxLhERSRFnIlgAFJtZEfA34HvA2akFUqfBNLOpwJNKAiIiuRXbUBHuvhMYS/A00Ergf919uZldYGYXxHXe+tJjoyKSdHXWCCxotzkHOMTdx4XzFf8/d59f177uPpu04SjcfVItZUdHiriRqaNYRJIuSo3gt8BAYFS4vIXg/YC8p8dGRUSi9RGUufsRZrYYwN03mVmrmOOK3YOvreWax5YBqg2ISLJFqRHsCN/+daiZj2BXrFHlQHWT0E2n9VVtQEQSLUoimAg8BhxoZr8AXgJuijWqHFGTkIhItGGop5nZImAIwUti/+buK2OPTEREciLKU0PdgE+AJ1LXufvaOAMTEZHciNJZ/BRB/4ABrYEi4E2gd4xxxSr1aSERkaSL0jTUN3XZzI4A/iO2iHJA7w6IiHyh3m8Wh8NPN9mQ0Q2ldwdERHYXpY/gv1IW9wKOADbEFlHMVBsQEdldlD6CdimfdxL0GTwSTzi5odqAiMgXsiaC8EWytu5+eY7iERGRHKu1j8DMWrr75wRNQSIiUqCy1QjmEySBcjObBTwEfFy90d0fjTk2ERHJgSh9BB2BjQTzCle/T+CAEoGISAHIlggODJ8YquCLBFAtlnmDRUQk97IlghZAW6JNQi8iInkqWyJY7+7jchaJiIg0iWxvFmeqCYiISIHJlgiG5CyKHNFE9SIie6o1Ebh7wd0xNbyEiMie6j3oXL7T8BIiIrtLXCIQEZHdKRGIiCScEoGISMIpEYiIJJwSgYhIwikRiIgknBKBiEjCKRGIiCRcrInAzE40szfNrNLMrsqw/RwzWxr+zDOz/nHFouElREQyiy0RhPMd3wWcBPQCRplZr7Ri7wLHu3s/4EZgclzxaHgJEZHM4qwRHAlUuvs77r4dmAGMSC3g7vPcfVO4+CrQNcZ4NLyEiEgGcSaCLsC6lOWqcF1tfgg8nWmDmY0xs4VmtnDDhg2NGKKIiMSZCCLPbGZmgwkSwZWZtrv7ZHcvdffSAw44oBFDFBGRKJPXf1lVwMEpy12B99ILmVk/4A/ASe6+McZ4REQkgzhrBAuAYjMrMrNWwPeAWakFzKwb8ChwrruvijEWERGpRWw1AnffaWZjgWeBFsC97r7czC4It08CrgP2B35rZgA73b00rphERGRPcTYN4e6zgdlp6yalfP4R8KM4YxARkez0ZrGISMIpEYiIJJwSgYhIwikRiIgknBKBiEjCKRGIiCScEoGISMIpEYiIJJwSgYhIwikRiIgknBKBiEjCKRGIiCScEoGISMIpEYiIJJwSgYhIwikRiIgknBKBiEjCKRGIiCScEoGISMIpEYiIJJwSgYhIwrVs6gCkcO3YsYOqqiq2bdvW1KGIJEbr1q3p2rUre++9d+R9lAgkNlVVVbRr147u3btjZk0djkjBc3c2btxIVVUVRUVFkfdT05DEZtu2bey///5KAiI5Ymbsv//+9a6FKxFIrJQERHLry/yfUyIQEUk4JQIpaC1atKCkpIQ+ffowbNgwPvroo0Y57tSpUxk7dmyjHKt79+707duXkpISSkpKmDdvXqMcN115eTmzZ8/ebd3TTz9NaWkpPXv2pEePHlx22WUA3HDDDUyYMKHRzn300UfXfL788svp3bs3l19+OZMmTeL+++9v0LEXL17Mj370o93WjRgxgoEDB+62bvTo0Tz88MO7rWvbtm3N51WrVnHyySdz6KGH0rNnT8466yz+8Y9/NCi2Dz/8kKFDh1JcXMzQoUPZtGlTxnK33347ffr0oXfv3tx222117r9s2TJGjx7doNhSKRFIQWvTpg3l5eVUVFTQsWNH7rrrrqYOKaO//OUvlJeXU15evttNM5udO3fW6xzpiaCiooKxY8fyxz/+kZUrV1JRUcEhhxxSr2NGlZrcfv/73/P6669z8803c8EFF3DeeedFPk6ma77pppu45JJLapY/+ugjXn/9dT766CPefffdSMfdtm0bp5xyChdeeCGVlZWsXLmSCy+8kA0bNkSOLZPx48czZMgQ3nrrLYYMGcL48eP3KFNRUcHdd9/N/PnzWbJkCU8++SRvvfVW1v379u1LVVUVa9eubVB81fTUkOTEz55Yzor3/tmox+z1L+25fljvyOUHDhzI0qVLAZg/fz4/+clP+PTTT2nTpg1Tpkzh8MMPZ+rUqcyaNYtPPvmEt99+m9NOO41f//rXAEyZMoVf/vKXdO7cmcMOO4yvfOUrAKxZs4Yf/OAHbNiwgQMOOIApU6bQrVs3Ro8eTZs2bXjjjTdYs2YNU6ZM4b777uOVV16hrKyMqVOn1hprtmN27NiRxYsXc8QRR3DRRRdx8cUXs2HDBvbZZx/uvvtuevTowUMPPcTPfvYzWrRoQYcOHXjuuee47rrr+PTTT3nppZe4+uqreeqpp7j22mvp0aMHAC1btuSiiy7aI5a7776byZMns337dg499FAeeOAB9tlnnz3OMXfuXJYvX87555/P9u3b2bVrF4888gjFxcW0bduWrVu3Mnz4cD7++GPKysq4+uqrWblyJW3btuWyyy7j7bffzngt6dd8yy231MS2ZcsWli5dSv/+/WvWPfLIIwwbNoyDDjqIGTNmcPXVV9f5b+PBBx9k4MCBDBs2rGbd4MGD69yvLo8//jgvvPACAN///vcZNGgQv/rVr3Yrs3LlSo466ij22WcfAI4//ngee+wxrrjiiqz7Dxs2jBkzZnDFFVc0OE7VCCQRPv/8c/785z8zfPhwAHr06MHcuXNZvHgx48aN45prrqkpW15ezsyZM1m2bBkzZ85k3bp1rF+/nuuvv56XX36ZOXPmsGLFipryY8eO5bzzzmPp0qWcc845XHrppTXbNm3axPPPP8+tt97KsGHD+M///E+WL1/OsmXLKC8vryk3ePBgSkpKKCsrq/OYq1at4rnnnuOWW25hzJgx3HHHHSxatIgJEybU3MjHjRvHs88+y5IlS5g1axatWrVi3LhxjBw5kvLyckaOHElFRQUDBgyo83d3+umns2DBApYsWULPnj255557Mp4DYNKkSfz4xz+mvLychQsX0rVr192ONWvWrJpa2siRI3fbVtu1pF9zqoULF9KnT5/d1k2fPp1Ro0YxatQopk+fXuf1AZF/F1u2bKlpwkv/Sf03Ue0f//gHnTt3BqBz5868//77e5Tp06cPc+fOZePGjXzyySfMnj2bdevW1bl/aWkpL774YqTrq4tqBJIT9fnm3pg+/fRTSkpKWL16NQMGDGDo0KEAbN68me9///u89dZbmBk7duyo2WfIkCF06NABgF69erFmzRo++OADBg0axAEHHADAyJEjWbVqFQCvvPIKjz76KADnnnvubt/Qhg0bhpnRt29fDjroIPr27QtA7969Wb16NSUlJUDQNNSpU6ea/bId88wzz6RFixZs3bqVefPmceaZZ9Zs++yzzwA45phjGD16NGeddRann356g36HFRUV/PSnP+Wjjz5i69atnHDCCbWeY+DAgfziF7+gqqqK008/neLi4kjnyHYtqdecbv369TV/JxDcOCsrKzn22GMxM1q2bElFRQV9+vTJ+DRNfZ+wadeu3W4JvDH07NmTK6+8kqFDh9K2bVv69+9Py5Z135oPPPBA3nvvvUaJIdYagZmdaGZvmlmlmV2VYbuZ2cRw+1IzOyLOeCR5qr99rlmzhu3bt9f0EfzP//wPgwcPpqKigieeeGK3566rm3wg6GyubpeOetNILVd9rL322mu34+611171auNPPea+++4LwK5du9hvv/1q+hbKy8tZuXIlEHwz//nPf866desoKSlh48aNexyzd+/eLFq0qM5zjx49mjvvvJNly5Zx/fXX1/yuMp3j7LPPrvnWf8IJJ/D8889Hur5s15J6zenatGmz29/dzJkz2bRpE0VFRXTv3p3Vq1czY8YMAPbff//dOms//PDDmuQb9XdR3xrBQQcdxPr164EgaR144IEZj/vDH/6Q119/nblz59KxY8eaBJpt/23bttGmTZs6Y44itkRgZi2Au4CTgF7AKDPrlVbsJKA4/BkD/C6ueCTZOnTowMSJE5kwYQI7duxg8+bNdOnSBSBrW321srIyXnjhBTZu3MiOHTt46KGHarYdffTRNTebadOmceyxxzY43ijHbN++PUVFRTWxuDtLliwB4O2336asrIxx48bRqVMn1q1bR7t27diyZUvN/pdffjk33XRTTc1m165d/OY3v9njPFu2bKFz587s2LGDadOm1azPdI533nmHQw45hEsvvZThw4fX9MnUJdu1ZNOzZ08qKytrlqdPn84zzzzD6tWrWb16NYsWLar5PQ4aNIiZM2eyfft2IPh7r+4HOPvss5k3bx5PPfVUzbGeeeYZli1bttv5qmsEmX569Uq/vcHw4cO57777ALjvvvsYMWJExuuobvJZu3Ytjz76KKNGjapz/1WrVu3RLPZlxVkjOBKodPd33H07MANI/y2MAO73wKvAfmbWOcaYJMG+/vWv079//5oOtquvvppjjjmGzz//vM59O3fuzA033MDAgQP59re/zRFHfFF5nThxIlOmTKFfv3488MAD3H777Q2ONeoxp02bxj333EP//v3p3bs3jz/+OBDc5Pv27UufPn345je/Sf/+/Rk8eDArVqygpKSEmTNn0q9fP2677TZGjRpFz5496dOnT823z1Q33ngjZWVlDB06tKZjubZzzJw5kz59+lBSUsIbb7xRryeCaruWbHr06MHmzZvZsmULq1evZu3atRx11FE124uKimjfvj2vvfYap556KscddxwDBgygpKSEl19+uabjtU2bNjz55JPccccdFBcX06tXL6ZOnVrrN/iorrrqKubMmUNxcTFz5szhqquChpH33nuPk08+uabcGWecQa9evRg2bBh33XUXX/3qV7PuD0Fz4imnnNKg+KqZuzfKgfY4sNl3gRPd/Ufh8rlAmbuPTSnzJDDe3V8Kl/8MXOnuC9OONYagxkC3bt0GrFmzpt7x/OyJ5UDTtVUn0cqVK+nZs2dThyEF7tZbb6Vdu3Z7vEtQyD777DOOP/54XnrppYz9CZn+75nZIncvzXS8OGsEmRpU07NOlDK4+2R3L3X30tSOofq4flhvJQGRAnThhRfu1v+SBGvXrmX8+PGROpWjiPOpoSrg4JTlrkB6F3eUMiIitWrdujXnnntuU4eRU8XFxZGfyIoizhrBAqDYzIrMrBXwPWBWWplZwHnh00NHAZvdfc9GSslbcTU9ikhmX+b/XGw1AnffaWZjgWeBFsC97r7czC4It08CZgMnA5XAJ8D5ccUjude6dWs2btyooahFcqR6PoLWrVvXa7/YOovjUlpa6gsXLqy7oDQ5zVAmknu1zVCWrbNYbxZLbPbee+96zZIkIk1DYw2JiCScEoGISMIpEYiIJFzedRab2Qag/q8WBzoBHzRiOPlA15wMuuZkaMg1/6u7Z3wjN+8SQUOY2cLaes0Lla45GXTNyRDXNatpSEQk4ZQIREQSLmmJYHJTB9AEdM3JoGtOhliuOVF9BCIisqek1QhERCSNEoGISMIVZCIwsxPN7E0zqzSzqzJsNzObGG5famZHZDpOPolwzeeE17rUzOaZWf+miLMx1XXNKeW+YWafh7Pm5bUo12xmg8ys3MyWm9lfcx1jY4vwb7uDmT1hZkvCa87rUYzN7F4ze9/MKmrZ3vj3L3cvqB+CIa/fBg4BWgFLgF5pZU4GniaYIe0o4LWmjjsH13w08NXw80lJuOaUcs8TDHn+3aaOOwd/z/sBK4Bu4fKBTR13Dq75GuBX4ecDgA+BVk0dewOu+ZvAEUBFLdsb/f5ViDWCI4FKd3/H3bcDM4ARaWVGAPd74FVgPzPrnOtAG1Gd1+zu89x9U7j4KsFscPksyt8zwCXAI8D7uQwuJlGu+WzgUXdfC+Du+X7dUa7ZgXYWTHrRliAR7MxtmI3H3ecSXENtGv3+VYiJoAuwLmW5KlxX3zL5pL7X80OCbxT5rM5rNrMuwGnApBzGFacof8+HAV81sxfMbJGZnZez6OIR5ZrvBHoSTHO7DPixu+/KTXhNotHvX4U4H0GmqbDSn5GNUiafRL4eMxtMkAiOjTWi+EW55tuAK9398wKZIS3KNbcEBgBDgDbAK2b2qruviju4mES55hOAcuBbwNeAOWb2orv/M+bYmkqj378KMRFUAQenLHcl+KZQ3zL5JNL1mFk/4A/ASe6+MUexxSXKNZcCM8Ik0Ak42cx2uvufchJh44v6b/sDd/8Y+NjM5gL9gXxNBFGu+XxgvAcN6JVm9i7QA5ifmxBzrtHvX4XYNLQAKDazIjNrBXwPmJVWZhZwXtj7fhSw2d3X5zrQRlTnNZtZN+BR4Nw8/naYqs5rdvcid+/u7t2Bh4GL8jgJQLR/248Dx5lZSzPbBygDVuY4zsYU5ZrXEtSAMLODgMOBd3IaZW41+v2r4GoE7r7TzMYCzxI8cXCvuy83swvC7ZMIniA5GagEPiH4RpG3Il7zdcD+wG/Db8g7PY9Hbox4zQUlyjW7+0ozewZYCuwC/uDuGR9DzAcR/55vBKaa2TKCZpMr3T1vh6c2s+nAIKCTmVUB1wN7Q3z3Lw0xISKScIXYNCQiIvWgRCAiknBKBCIiCadEICKScEoEIiIJp0QgzVI4Wmh5yk/3LGW3NsL5pprZu+G5XjezgV/iGH8ws17h52vSts1raIzhcap/LxXhiJv71VG+xMxOboxzS+HS46PSLJnZVndv29hlsxxjKvCkuz9sZt8BJrh7vwYcr8Ex1XVcM7sPWOXuv8hSfjRQ6u5jGzsWKRyqEUheMLO2Zvbn8Nv6MjPbY6RRM+tsZnNTvjEfF67/jpm9Eu77kJnVdYOeCxwa7vtf4bEqzOwn4bp9zeypcPz7CjMbGa5/wcxKzWw80CaMY1q4bWv458zUb+hhTeQMM2thZjeb2QILxpj/jwi/llcIBxszsyMtmGdicfjn4eGbuOOAkWEsI8PY7w3PszjT71ESqKnH3taPfjL9AJ8TDCRWDjxG8BZ8+3BbJ4K3KqtrtFvDP/8buDb83AJoF5adC+wbrr8SuC7D+aYSzlcAnAm8RjB42zJgX4LhjZcDXwfOAO5O2bdD+OcLBN++a2JKKVMd42nAfeHnVgSjSLYBxgA/Ddd/BVgIFGWIc2vK9T0EnBgutwdahp+/DTwSfh4N3Jmy/03Av4ef9yMYg2jfpv771k/T/hTcEBNSMD5195LqBTPbG7jJzL5JMHRCF+Ag4O8p+ywA7g3L/sndy83seKAX8HI4tEYrgm/SmdxsZj8FNhCM0DoEeMyDAdwws0eB44BngAlm9iuC5qQX63FdTwMTzewrwInAXHf/NGyO6mdfzKLWASgG3k3bv42ZlQPdgUXAnJTy95lZMcFIlHvXcv7vAMPN7LJwuTXQjfwej0gaSIlA8sU5BLNPDXD3HWa2muAmVsPd54aJ4hTgATO7GdgEzHH3URHOcbm7P1y9YGbfzlTI3VeZ2QCC8V5+aWb/5+7jolyEu28zsxcIhk4eCUyvPh1wibs/W8chPnX3EjPrADwJXAxMJBhv5y/uflrYsf5CLfsbcIa7vxklXkkG9RFIvugAvB8mgcHAv6YXMLN/DcvcDdxDMN3fq8AxZlbd5r+PmR0W8ZxzgX8L99mXoFnnRTP7F+ATd/8jMCE8T7odYc0kkxkEA4UdRzCYGuGfF1bvY2aHhefMyN03A5cCl4X7dAD+Fm4enVJ0C0ETWbVngUssrB6Z2ddrO4ckhxKB5ItpQKmZLSSoHbyRocwgoNzMFhO049/u7hsIbozTzWwpQWLoEeWE7v46Qd/BfII+gz+4+2KgLzA/bKK5Fvh5ht0nA0urO4vT/B/BvLTPeTD9IgTzRKwAXrdg0vLfU0eNPYxlCcHQzL8mqJ28TNB/UO0vQK/qzmKCmsPeYWwV4bIknB4fFRFJONUIREQSTolARCThlAhERBJOiUBEJOGUCEREEk6JQEQk4ZQIREQS7v8DlF9KdZnM+bIAAAAASUVORK5CYII=\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "clf_disp2 = plot_roc_curve(clf2, test_x, test_y)\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 50,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAWAAAAD4CAYAAADSIzzWAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjMuMiwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy8vihELAAAACXBIWXMAAAsTAAALEwEAmpwYAAAMz0lEQVR4nO3dX6hdZ1rH8e8vsbYDg9rSacgkkakYnUkFZ2Aog4MwWKHxD6Y3hVTUIJFz04EZEDT1RgYM9MbBG3sRdJiA0hBQaeiFUqJlEMdJo1PHSWNpaLE9JjR0xr8XdnrOebw4C9yTydlnn54/z9kr309Z7L3fvfda78Xhl6fPetfaqSokSTtvT/cEJOlOZQBLUhMDWJKaGMCS1MQAlqQm37fdB3jvndddZqHv8YEP/3T3FLQLLX3n37LZfWwkc+66/0c2fbzNsAKWpCbbXgFL0o5aWe6ewcwMYEnjsrzUPYOZGcCSRqVqpXsKMzOAJY3LigEsST2sgCWpiSfhJKmJFbAk9ShXQUhSE0/CSVITWxCS1MSTcJLUxApYkpp4Ek6SmngSTpJ6VNkDlqQe9oAlqYktCElqYgUsSU2W3+uewcwMYEnjYgtCkprYgpCkJlbAktTEAJakHuVJOElqYg9YkprYgpCkJlbAktTECliSmlgBS1KTJW/ILkk9rIAlqYk9YElqYgUsSU2sgCWpiRWwJDVxFYQkNanqnsHMDGBJ42IPWJKaGMCS1GSOTsLt6Z6AJG2p5eXZtxkk2Zvk60meH17fl+SFJK8Nj/dOfPapJNeSvJrk0fX2bQBLGpeVldm32XwOuDrx+hRwsaoOAxeH1yQ5AhwHHgKOAs8k2TttxwawpHHZwgBOchD4BeCPJoaPAWeH52eBxybGz1XVu1X1BnANeHja/g1gSeNSKzNvSRaSXJ7YFm7Z2x8AvwVMpvW+qroBMDw+MIwfAN6a+NziMLYmT8JJGpVamX0dcFWdAc7c7r0kvwjcrKp/SPKZGXaX2x1i2hcMYEnjsnXL0D4N/FKSnwfuAX4gyZ8AbyfZX1U3kuwHbg6fXwQOTXz/IHB92gFsQUgaly1aBVFVT1XVwar6CKsn1/66qn4FuACcGD52AnhueH4BOJ7k7iQPAoeBS9OOYQUsaVy2/0KMp4HzSU4CbwKPA1TVlSTngVeAJeDJqpqa8gawpHHZhgCuqheBF4fn3wIeWeNzp4HTs+7XAJY0Lt6MR5KajOleEEk+yuoC4wOsLqm4DlyoqqtTvyhJHTawDK3b1FUQSX4bOMfq+rZLwEvD82eTnNr+6UnSBm3xvSC203oV8Engoap6b3IwyReBK6yeDfwew9UkCwDP/P7v8Ru/9sQWTFWS1lcjakGsAB8G/vWW8f1896V532Xy6pL33nl9fv5/QNL8m6MWxHoB/HngYpLX+P9rnH8Y+FHgs9s4L0l6f+bofsBTA7iq/jLJj7F6R58DrPZ/F4GX1ltgLEktRlQBU1UrwN/vwFwkafOW5qc2dB2wpHEZSwtCkubOmFoQkjRPxrQMTZLmixWwJDUxgCWpyS64xHhWBrCkUdnIb8J1M4AljYsBLElNXAUhSU2sgCWpiQEsST1q2RaEJPWwApakHi5Dk6QuBrAkNZmfFrABLGlcaml+EtgAljQu85O/BrCkcfEknCR1sQKWpB5WwJLUxQpYknrUUvcMZmcASxqVOfpVegNY0sgYwJLUwwpYkpoYwJLUpJbTPYWZGcCSRmWeKuA93ROQpK1UK5l5mybJPUkuJfmnJFeSfGEYvy/JC0leGx7vnfjOU0muJXk1yaPrzdUAljQqtTL7to53gZ+pqp8EPg4cTfIp4BRwsaoOAxeH1yQ5AhwHHgKOAs8k2TvtAAawpFGpyszb9P1UVdX/DC/vGrYCjgFnh/GzwGPD82PAuap6t6reAK4BD087hgEsaVQ2UgEnWUhyeWJbmNxXkr1JXgZuAi9U1deAfVV1A2B4fGD4+AHgrYmvLw5ja/IknKRRWdnAKoiqOgOcmfL+MvDxJD8E/EWSn5iyu9sdeOqdgQxgSaOy3sm197XPqv9I8iKrvd23k+yvqhtJ9rNaHcNqxXto4msHgevT9msLQtKobOEqiA8NlS9JPgD8LPAvwAXgxPCxE8Bzw/MLwPEkdyd5EDgMXJp2DCtgSaNSW3c74P3A2WElwx7gfFU9n+SrwPkkJ4E3gcdXj1tXkpwHXgGWgCeHFsaaDGBJo7JVLYiq+gbwiduMfwt4ZI3vnAZOz3oMA1jSqKy3vGw3MYAljcqy94KQpB5WwJLUZDuWoW0XA1jSqGzhKohtZwBLGhUrYElqsrwyP9eXGcCSRsUWhCQ1WXEVhCT1cBmaJDWxBTHhyMce3+5DaA791Ic+2j0FjZQtCElq4ioISWoyRx0IA1jSuNiCkKQmroKQpCYr3RPYAANY0qjUbX+ceHcygCWNypItCEnqYQUsSU3sAUtSEytgSWpiBSxJTZatgCWpxxz9IpEBLGlcVqyAJamHN+ORpCaehJOkJiuxBSFJLZa7J7ABBrCkUXEVhCQ1cRWEJDVxFYQkNbEFIUlNXIYmSU2WrYAlqYcVsCQ1macA3tM9AUnaSpXZt2mSHEryN0muJrmS5HPD+H1JXkjy2vB478R3nkpyLcmrSR5db64GsKRRWdnAto4l4Der6mPAp4AnkxwBTgEXq+owcHF4zfDeceAh4CjwTJK90w5gAEsaleUNbNNU1Y2q+sfh+X8DV4EDwDHg7PCxs8Bjw/NjwLmqereq3gCuAQ9PO4YBLGlUVjL7lmQhyeWJbeF2+0zyEeATwNeAfVV1A1ZDGnhg+NgB4K2Jry0OY2vyJJykUdnISbiqOgOcmfaZJB8E/gz4fFX9V9a+29rt3ph6YZ4VsKRR2cIeMEnuYjV8/7Sq/nwYfjvJ/uH9/cDNYXwRODTx9YPA9Wn7N4AljUptYJsmq6XuHwNXq+qLE29dAE4Mz08Az02MH09yd5IHgcPApWnHsAUhaVS28F4QnwZ+FfjnJC8PY78DPA2cT3ISeBN4HKCqriQ5D7zC6gqKJ6tq6rk+A1jSqGzVDdmr6m+5fV8X4JE1vnMaOD3rMQxgSaOyMkc3pDSAJY3KPF2KbABLGpX5qX8NYEkjYwUsSU2WMj81sAEsaVTmJ34NYEkjYwtCkpq4DE2SmsxP/BrAkkbGFoQkNVmeoxrYAJY0KlbAktSkrIAlqYcVsCQ1cRmaJDWZn/g1gCWNzNIcRbABLGlU5ukk3Pv+Uc4kvz7lvYUkl5Nc/s//fef9HkKSNmwrfxV5u23mV5G/sNYbVXWmqj5ZVZ/8wXvu38QhJGljagP/dZvagkjyjbXeAvZt/XQkaXN2Q2U7q/V6wPuAR4F/v2U8wN9ty4wkaROWq7+yndV6Afw88MGqevnWN5K8uB0TkqTNGM064Ko6OeW9X9766UjS5uyG3u6sXIYmaVTG1AOWpLkymhaEJM0bWxCS1GRMqyAkaa7YgpCkJp6Ek6Qm9oAlqYktCElqUp6Ek6Qe/iy9JDWxBSFJTWxBSFKTeaqAN/OLGJK062zlL2Ik+VKSm0m+OTF2X5IXkrw2PN478d5TSa4leTXJo+vt3wCWNCrLVTNvM/gycPSWsVPAxao6DFwcXpPkCHAceGj4zjNJ9k7buQEsaVRWqJm39VTVV4Bv3zJ8DDg7PD8LPDYxfq6q3q2qN4BrwMPT9m8ASxqVrQzgNeyrqhsAw+MDw/gB4K2Jzy0OY2sygCWNSlXNvCVZSHJ5YlvYxKFzu+lM+4KrICSNykYq26o6A5zZ4CHeTrK/qm4k2Q/cHMYXgUMTnzsIXJ+2IytgSaOylasg1nABODE8PwE8NzF+PMndSR4EDgOXpu3ICljSqCzX1t2QMsmzwGeA+5MsAr8LPA2cT3ISeBN4HKCqriQ5D7wCLAFPVtXytP0bwJJGZSuvhKuqJ9Z465E1Pn8aOD3r/g1gSaMyT1fCGcCSRsUbsktSkxVvxiNJPayAJanJVq6C2G4GsKRRsQUhSU1sQUhSEytgSWpiBSxJTZanX/27qxjAkkbFH+WUpCZeiixJTayAJamJqyAkqYmrICSpiZciS1ITe8CS1MQesCQ1sQKWpCauA5akJlbAktTEVRCS1MSTcJLUxBaEJDXxSjhJamIFLElN5qkHnHn612LeJVmoqjPd89Du4t/FnWtP9wTuMAvdE9Cu5N/FHcoAlqQmBrAkNTGAd5Z9Pt2Ofxd3KE/CSVITK2BJamIAS1ITA3iHJDma5NUk15Kc6p6P+iX5UpKbSb7ZPRf1MIB3QJK9wB8CPwccAZ5IcqR3VtoFvgwc7Z6E+hjAO+Nh4FpVvV5V3wHOAcea56RmVfUV4Nvd81AfA3hnHADemni9OIxJuoMZwDsjtxlz/Z90hzOAd8YicGji9UHgetNcJO0SBvDOeAk4nOTBJN8PHAcuNM9JUjMDeAdU1RLwWeCvgKvA+aq60jsrdUvyLPBV4MeTLCY52T0n7SwvRZakJlbAktTEAJakJgawJDUxgCWpiQEsSU0MYElqYgBLUpP/A5ZErNDxIQE+AAAAAElFTkSuQmCC\n",
      "text/plain": [
       "<Figure size 432x288 with 2 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "sb.heatmap(cm2)\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 51,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([[496,  31],\n",
       "       [ 76, 107]])"
      ]
     },
     "execution_count": 51,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "cm2"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "The entropy model had ONE MORE CORRECT prediction of the positive class."
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Lastly...\n",
    "\n",
    "What if we tried to classify only the HIGHER THAN PREDICTED rows set using only the new FACTOR variables?\n",
    "\n",
    "This might not work, but let's see."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 53,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(975, 80)"
      ]
     },
     "execution_count": 53,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "mc = pd.read_csv(\"/home/kylier/python/DS/data/PUF/HIGHER_THAN_PREDICTED_FACTORS.csv\")\n",
    "mc.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 55,
   "metadata": {},
   "outputs": [],
   "source": [
    "mc.drop(columns=mc.columns[0], axis=1, inplace=True, errors=\"ignore\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 56,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "<class 'pandas.core.frame.DataFrame'>\n",
      "RangeIndex: 975 entries, 0 to 974\n",
      "Data columns (total 79 columns):\n",
      " #   Column                      Non-Null Count  Dtype  \n",
      "---  ------                      --------------  -----  \n",
      " 0   FIPS_STATE_CODE             975 non-null    int64  \n",
      " 1   FIPS_COUNTY_CODE            975 non-null    int64  \n",
      " 2   CNTY_BENE_CT                975 non-null    int64  \n",
      " 3   CNTY_BENE_USRS              975 non-null    int64  \n",
      " 4   CNTY_MEAN_AGE               975 non-null    float64\n",
      " 5   CNTY_SEX_PROP               975 non-null    float64\n",
      " 6   TOT_CNCR_CASES              975 non-null    int64  \n",
      " 7   TOT_STRKETIA_CASES          975 non-null    int64  \n",
      " 8   TOT_ESRD_CASES              975 non-null    int64  \n",
      " 9   CNTY_TOT_VSTS               975 non-null    int64  \n",
      " 10  CNTY_TOT_IP_VSTS            975 non-null    int64  \n",
      " 11  CNTY_TOT_OP_VSTS            975 non-null    int64  \n",
      " 12  CNTY_TOT_CODES              975 non-null    float64\n",
      " 13  CNTY_TOT_PROCS              975 non-null    float64\n",
      " 14  CNTY_MEAN_IP_STAY           975 non-null    float64\n",
      " 15  CBSA                        389 non-null    float64\n",
      " 16  CNTY_TOT_CCS                975 non-null    int64  \n",
      " 17  CNTY_MEAN_CCS               975 non-null    float64\n",
      " 18  CNTY_TOT_NEW_CCS            975 non-null    int64  \n",
      " 19  CNTY_MEAN_NEW_CCS           975 non-null    float64\n",
      " 20  CNTY_SUM_ELIX_SCORES        975 non-null    float64\n",
      " 21  CNTY_TOT_IP_COST            975 non-null    int64  \n",
      " 22  CNTY_TOT_OP_COST            975 non-null    int64  \n",
      " 23  CNTY_ALL_COSTS              975 non-null    int64  \n",
      " 24  CNTY_PERCENT_COSTS          975 non-null    float64\n",
      " 25  CNTY_USR_PROP               975 non-null    float64\n",
      " 26  CNTY_IP_PROP                975 non-null    float64\n",
      " 27  CNTY_AGE_95CIL              959 non-null    float64\n",
      " 28  CNTY_AGE_95CIU              959 non-null    float64\n",
      " 29  CNTY_PROP_NH_WHITE          975 non-null    float64\n",
      " 30  CNTY_PROP_AA_BLACK          975 non-null    float64\n",
      " 31  CNTY_PROP_OTHER             975 non-null    float64\n",
      " 32  CNTY_PROP_NW_HISPANIC       975 non-null    float64\n",
      " 33  TOT_CNCR_PREV               975 non-null    float64\n",
      " 34  TOT_STRK_PREV               975 non-null    float64\n",
      " 35  TOT_ESRD_PREV               975 non-null    float64\n",
      " 36  TOT_CVRG_MOS_PB_PY          975 non-null    float64\n",
      " 37  TOT_RXCVRG_MOS_PB_PY        975 non-null    float64\n",
      " 38  TOT_VSTS_PB                 975 non-null    float64\n",
      " 39  TOT_VSTS_PU                 975 non-null    float64\n",
      " 40  TOT_IP_VSTS_PU              975 non-null    float64\n",
      " 41  TOT_OP_VSTS_PU              975 non-null    float64\n",
      " 42  CODES_PU                    975 non-null    float64\n",
      " 43  PROCS_PU                    975 non-null    float64\n",
      " 44  PRVDRS_PU                   975 non-null    float64\n",
      " 45  TOT_IPDAYS_PU               975 non-null    float64\n",
      " 46  TOT_CC_PREV                 975 non-null    float64\n",
      " 47  TOT_CCS_PU                  975 non-null    float64\n",
      " 48  TOT_CC_INCD                 975 non-null    float64\n",
      " 49  TOT_NEW_CC_PU               975 non-null    float64\n",
      " 50  TOT_ELIX_PU                 975 non-null    float64\n",
      " 51  TOT_IP_COST_PB              975 non-null    float64\n",
      " 52  TOT_OP_COST_PB              975 non-null    float64\n",
      " 53  TOT_COST_PB                 975 non-null    float64\n",
      " 54  FIPS                        975 non-null    int64  \n",
      " 55  STATE                       975 non-null    object \n",
      " 56  COUNTY                      975 non-null    object \n",
      " 57  NUM_COUNTIES                975 non-null    int64  \n",
      " 58  HO_RANK                     975 non-null    int64  \n",
      " 59  HO_PROP_RANK                975 non-null    float64\n",
      " 60  HO_QUARTILE                 975 non-null    int64  \n",
      " 61  HF_RANK                     975 non-null    int64  \n",
      " 62  HF_PROP_RANK                975 non-null    float64\n",
      " 63  HF_QUARTILE                 975 non-null    int64  \n",
      " 64  HF_HIGH                     975 non-null    int64  \n",
      " 65  HIGH_SEV                    975 non-null    int64  \n",
      " 66  HIGH_PROCS                  975 non-null    int64  \n",
      " 67  DIVERSE_PROP                975 non-null    float64\n",
      " 68  TOT_COST_PB Predicted       975 non-null    float64\n",
      " 69  Lower 95% Mean TOT_COST_PB  975 non-null    float64\n",
      " 70  Upper 95% Mean TOT_COST_PB  975 non-null    float64\n",
      " 71  Residuals                   975 non-null    float64\n",
      " 72  HIGHER_COST                 975 non-null    int64  \n",
      " 73  SR_TOT_IPDAYS               975 non-null    float64\n",
      " 74  SR_TOT_COST_PB              975 non-null    float64\n",
      " 75  FACTORS_1                   975 non-null    float64\n",
      " 76  FACTORS_2                   975 non-null    float64\n",
      " 77  FACTORS_3                   975 non-null    float64\n",
      " 78  D_FACTOR                    975 non-null    float64\n",
      "dtypes: float64(52), int64(25), object(2)\n",
      "memory usage: 601.9+ KB\n"
     ]
    }
   ],
   "source": [
    "mc.info()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 57,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "count    975.000000\n",
       "mean     290.405097\n",
       "std       54.868432\n",
       "min      164.833249\n",
       "25%      249.160936\n",
       "50%      284.892962\n",
       "75%      324.827172\n",
       "max      445.368013\n",
       "Name: SR_TOT_COST_PB, dtype: float64"
      ]
     },
     "execution_count": 57,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "mc[\"SR_TOT_COST_PB\"].describe()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Since these counties are already \"Higher than Predicted\" cost\n",
    "\n",
    "Setting the cut-off at the 25% level."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 59,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>FACTORS_1</th>\n",
       "      <th>FACTORS_2</th>\n",
       "      <th>FACTORS_3</th>\n",
       "      <th>B_COST</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>949</th>\n",
       "      <td>-0.288873</td>\n",
       "      <td>0.289165</td>\n",
       "      <td>-0.187332</td>\n",
       "      <td>1.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>755</th>\n",
       "      <td>-0.055802</td>\n",
       "      <td>0.408960</td>\n",
       "      <td>-0.216046</td>\n",
       "      <td>1.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>127</th>\n",
       "      <td>0.883905</td>\n",
       "      <td>-0.735503</td>\n",
       "      <td>-1.095119</td>\n",
       "      <td>1.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>913</th>\n",
       "      <td>-0.434412</td>\n",
       "      <td>0.192382</td>\n",
       "      <td>0.022423</td>\n",
       "      <td>1.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>136</th>\n",
       "      <td>-0.212793</td>\n",
       "      <td>0.882763</td>\n",
       "      <td>-0.052384</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>500</th>\n",
       "      <td>-0.503050</td>\n",
       "      <td>-0.622275</td>\n",
       "      <td>0.033358</td>\n",
       "      <td>1.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>607</th>\n",
       "      <td>-0.549531</td>\n",
       "      <td>-0.276182</td>\n",
       "      <td>-0.524792</td>\n",
       "      <td>1.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>264</th>\n",
       "      <td>1.486837</td>\n",
       "      <td>0.973190</td>\n",
       "      <td>0.693359</td>\n",
       "      <td>1.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>469</th>\n",
       "      <td>-0.316642</td>\n",
       "      <td>-0.857571</td>\n",
       "      <td>-0.960919</td>\n",
       "      <td>1.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>322</th>\n",
       "      <td>-0.676772</td>\n",
       "      <td>-0.261439</td>\n",
       "      <td>-1.019328</td>\n",
       "      <td>1.0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>731 rows × 4 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "     FACTORS_1  FACTORS_2  FACTORS_3  B_COST\n",
       "949  -0.288873   0.289165  -0.187332     1.0\n",
       "755  -0.055802   0.408960  -0.216046     1.0\n",
       "127   0.883905  -0.735503  -1.095119     1.0\n",
       "913  -0.434412   0.192382   0.022423     1.0\n",
       "136  -0.212793   0.882763  -0.052384     0.0\n",
       "..         ...        ...        ...     ...\n",
       "500  -0.503050  -0.622275   0.033358     1.0\n",
       "607  -0.549531  -0.276182  -0.524792     1.0\n",
       "264   1.486837   0.973190   0.693359     1.0\n",
       "469  -0.316642  -0.857571  -0.960919     1.0\n",
       "322  -0.676772  -0.261439  -1.019328     1.0\n",
       "\n",
       "[731 rows x 4 columns]"
      ]
     },
     "execution_count": 59,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "mc.loc[mc[\"SR_TOT_COST_PB\"]>250, \"B_COST\"] = 1\n",
    "mc.loc[mc[\"SR_TOT_COST_PB\"]<=250, \"B_COST\"] = 0\n",
    "\n",
    "df = mc[[\"FACTORS_1\", \"FACTORS_2\", \"FACTORS_3\", \"B_COST\"]]\n",
    "train, test = train_test_split(df, test_size=0.25, random_state=121)\n",
    "\n",
    "train"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 60,
   "metadata": {},
   "outputs": [],
   "source": [
    "train_x = train[[\"FACTORS_1\", \"FACTORS_2\", \"FACTORS_3\"]]\n",
    "train_y = train[\"B_COST\"]\n",
    "\n",
    "test_x = test[[\"FACTORS_1\", \"FACTORS_2\", \"FACTORS_3\"]]\n",
    "test_y = test[\"B_COST\"]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 61,
   "metadata": {},
   "outputs": [],
   "source": [
    "clf3 = RandomForestClassifier(min_samples_leaf=10, random_state=121)\n",
    "clf3.fit(train_x, train_y)\n",
    "\n",
    "# predict on test data\n",
    "prediction3 = clf3.predict(test_x)\n",
    "\n",
    "# compute evaluation scores\n",
    "accuracy3 = accuracy_score(test_y, prediction3)\n",
    "precision3 = precision_score(test_y, prediction3)\n",
    "recall3 = recall_score(test_y, prediction3)\n",
    "cm3 = confusion_matrix(test_y, prediction3)\n",
    "roc3 = roc_auc_score(test_y, prediction3)\n",
    "f13 = f1_score(test_y, prediction3)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 62,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.7008196721311475"
      ]
     },
     "execution_count": 62,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "accuracy3"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 63,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.7008196721311475"
      ]
     },
     "execution_count": 63,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "precision3"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 64,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "1.0"
      ]
     },
     "execution_count": 64,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "recall3"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 65,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.8240963855421687"
      ]
     },
     "execution_count": 65,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "f13"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 66,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<AxesSubplot:>"
      ]
     },
     "execution_count": 66,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAZwAAAD4CAYAAADYU1DBAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjMuMiwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy8vihELAAAACXBIWXMAAAsTAAALEwEAmpwYAAAQhklEQVR4nO3df6zddX3H8ecLyIjA1i1Cs7Y4LmOVOZA1pRIizZLFmcGKks6RtKkJypJK4kLUoKB1C9E0gmDUCSYjJugSwL8sMWlM5B+jCTC4NS0FuRgnJYo/u7hKA2xQ3vvjfG883J3ee869l885lz4fyTec8zmfz/f7/n4offH9fL/33FQVkiS91k4adwGSpBODgSNJasLAkSQ1YeBIkpowcCRJTZwy7gIm2ZlnnllTU1PjLkOSVpR9+/Ydrqqz5rYbOPOYmppienp63GVI0oqS5JlB7S6pSZKaMHAkSU0YOJKkJgwcSVITBo4kqQkDR5LUhIEjSWrCwJEkNWHgSJKaMHAkSU0YOJKkJgwcSVITBo4kqQkDR5LUhIEjSWrCwJEkNeEvYJvHwWePMHXT3nGXIWnMDt2yZdwlvC54hSNJasLAkSQ1YeBIkpowcCRJTRg4kqQmDBxJUhMGjiSpCQNHktSEgSNJasLAkSQ1YeBIkpowcCRJTRg4kqQmDBxJUhMGjiSpiQUDJ8mxJPv7tqmu/cNJXkyyak7/K5JMJ3kyyUyS25Ps6hvfv7/ruzE7u74zSR5Jsrlvf99J8lSSA0keTbKh77NrkxxM8liSx5NcNc953Nbt/7Eke5L84cizJUlatGF+AdsLVbVhQPt24FFgK/BVgCQXAncAW6pqJskpwM6q+jKwu+tztH9/Sa4EPgBsrqrDSTYC9ye5pKp+0XXbUVXTSd4P3Aa8M8nZwC5gY1UdSXIGcNY85/EA8PGqejnJrcDHgRuHOH9J0jJY1JJakvOAM4BP0gueWR8DdlfVDEBVvdyFzXxuBD5aVYe7Md8HvgZ8cEDfh4B13evVwHPA0W7c0ap6+ngHqapvV9XL3duHgbMXqEuStIyGCZw39C2B7enatgP3Ad8Dzk+yumu/ENg3Yg0XDBgz3bXPdTlwf/f6APBL4Okkdyd51wjHvBb41qAPuuW96STTx54/MsIuJUnzWeyS2jZga1W9kuQbwNXAnctYV4Dqe39PktOBk4GNAFV1LMnlwNuAdwCfT3JxVd08746TXcDLwD2DPq+qu4C7AE5ds74G9ZEkjW7kJbUkFwHrgQeSHKIXPrPLak8AF4+4yx8MGLOxa5+1AzgXuJe+YKueR6rqM10d71mg9muAK+ndEzJMJKmhxdzD2Q7cXFVT3bYWWJfkHHo39D+R5M0ASU5K8pEF9vdZ4NYkb+zGbADeB7zq3k9VvUTvntGlSd6SZG33gMGsDcAzxztIdzV0I/Duqnp+6LOVJC2LYZbU5toGXDGnbQ+wrapuTfIh4L4kp9FbFts7386q6ptJ1gEPJil6DwK8t6p+PqDvC0k+B9wAfAq4Pcla4EXg18B18xzqDuBUeldmAA9X1Xz9JUnLKK4sHd+pa9bXmmu+MO4yJI3ZoVu2jLuEFSXJvqraNLfdbxqQJDWxmCW1iZbkTuCyOc1frKq7x1GPJKnndRc4VTXoB0YlSWPmkpokqQkDR5LUhIEjSWrCwJEkNWHgSJKaMHAkSU0YOJKkJgwcSVITBo4kqYnX3TcNLKe3rlvFtF/aJ0nLwiscSVITBo4kqQkDR5LUhIEjSWrCwJEkNWHgSJKaMHAkSU0YOJKkJgwcSVITBo4kqQkDR5LUhIEjSWrCwJEkNWHgSJKaMHAkSU0YOJKkJgwcSVITBo4kqQkDR5LUhIEjSWrCwJEkNWHgSJKaMHAkSU0YOJKkJgwcSVITBo4kqQkDR5LUhIEjSWrCwJEkNWHgSJKaMHAkSU0YOJKkJgwcSVITBo4kqQkDR5LUxCnjLmCSHXz2CFM37R13GZJWsEO3bBl3CRPDKxxJUhMGjiSpCQNHktSEgSNJasLAkSQ1YeBIkpowcCRJTRg4kqQmDBxJUhMGjiSpCQNHktSEgSNJasLAkSQ1YeBIkpowcCRJTSwYOEmOJdnft0117R9O8mKSVXP6X5FkOsmTSWaS3J5kV9/4/v1d343Z2fWdSfJIks19+/tOkqeSHEjyaJINfZ9dm+RgkseSPJ7kqnnO4+okTyR5Jcmm0adKkrQUw/wCtheqasOA9u3Ao8BW4KsASS4E7gC2VNVMklOAnVX1ZWB31+do//6SXAl8ANhcVYeTbATuT3JJVf2i67ajqqaTvB+4DXhnkrOBXcDGqjqS5AzgrHnO43Hg74F/G+KcJUnLbFFLaknOA84APkkveGZ9DNhdVTMAVfVyFzbzuRH4aFUd7sZ8H/ga8MEBfR8C1nWvVwPPAUe7cUer6unjHaSqnqyqp4Y4t53dFdr0seePLNRdkjSkYQLnDX1LYHu6tu3AfcD3gPOTrO7aLwT2jVjDBQPGTHftc10O3N+9PgD8Eng6yd1J3jXicQeqqruqalNVbTr5tFULD5AkDWWxS2rbgK1V9UqSbwBXA3cuY10Bqu/9PUlOB04GNgJU1bEklwNvA94BfD7JxVV18zLWIUlaJiMvqSW5CFgPPJDkEL3wmV1WewK4eMRd/mDAmI1d+6wdwLnAvfQFW/U8UlWf6ep4z4jHliQ1sph7ONuBm6tqqtvWAuuSnEPvhv4nkrwZIMlJST6ywP4+C9ya5I3dmA3A+4BX3fupqpfo3TO6NMlbkqztHjCYtQF4ZhHnI0lqYJgltbm2AVfMadsDbKuqW5N8CLgvyWn0lsX2zrezqvpmknXAg0mK3oMA762qnw/o+0KSzwE3AJ8Cbk+yFngR+DVw3fGOk2Qr8CV6T7LtTbK/qv52qDOWJC1ZqmrhXieoU9esrzXXfGHcZUhawQ7dsmXcJTSXZF9V/b+fd/SbBiRJTSxmSW2iJbkTuGxO8xer6u5x1CNJ6nndBU5VDfqBUUnSmLmkJklqwsCRJDVh4EiSmjBwJElNGDiSpCYMHElSEwaOJKkJA0eS1MTr7gc/l9Nb161i+gT8HiRJei14hSNJasLAkSQ1YeBIkpowcCRJTRg4kqQmDBxJUhMGjiSpCQNHktSEgSNJasLAkSQ1YeBIkpowcCRJTRg4kqQmDBxJUhMGjiSpCQNHktSEgSNJasLAkSQ1YeBIkpowcCRJTRg4kqQmDBxJUhMGjiSpCQNHktSEgSNJasLAkSQ1YeBIkpowcCRJTRg4kqQmDBxJUhMGjiSpCQNHktSEgSNJasLAkSQ1YeBIkpo4ZdwFTLKDzx5h6qa94y5Dkpo6dMuW12S/XuFIkpowcCRJTRg4kqQmDBxJUhMGjiSpCQNHktSEgSNJasLAkSQ1YeBIkpowcCRJTRg4kqQmDBxJUhMGjiSpCQNHktSEgSNJamLBwElyLMn+vm2qa/9wkheTrJrT/4ok00meTDKT5PYku/rG9+/v+m7Mzq7vTJJHkmzu2993kjyV5ECSR5Ns6Pvs2iQHkzyW5PEkV81zHp/u+u1P8u0ka0efLknSYg3zC9heqKoNA9q3A48CW4GvAiS5ELgD2FJVM0lOAXZW1ZeB3V2fo/37S3Il8AFgc1UdTrIRuD/JJVX1i67bjqqaTvJ+4DbgnUnOBnYBG6vqSJIzgLPmOY/bquqfu2NeD/wLcN0Q5y9JWgaLWlJLch5wBvBJesEz62PA7qqaAaiql7uwmc+NwEer6nA35vvA14APDuj7ELCue70aeA442o07WlVPH+8gVfXbvrenA7VAXZKkZTRM4LyhbwlsT9e2HbgP+B5wfpLVXfuFwL4Ra7hgwJjprn2uy4H7u9cHgF8CTye5O8m7FjpQkt1JfgLsoHeFM6jPzm5JcPrY80eGPAVJ0kKGCZwXqmpDt23t2rYBX6+qV4BvAFcvc13h1Vcg9yT5Kb2roS8BVNUxegH0D8APgc8nuXm+nVbVrqp6E3AP8E/H6XNXVW2qqk0nn7ZqUBdJ0iKMvKSW5CJgPfBAkkP0wmd2We0J4OIRd/mDAWM2du2zdgDnAvcCd842Vs8jVfWZro73DHnMe0foK0laBou5h7MduLmqprptLbAuyTn0buh/IsmbAZKclOQjC+zvs8CtSd7YjdkAvA941b2fqnqJ3j2jS5O8Jcna7gGDWRuAZ453kCTr+96+G5hZ8EwlSctmmKfU5toGXDGnbQ+wrapuTfIh4L4kp9FbFts7386q6ptJ1gEPJil6DwK8t6p+PqDvC0k+B9wAfAq4vXu8+UXg18z/1NktSc4HXqEXTD6hJkkNpcqHtY7n1DXra801Xxh3GZLU1KFbtixpfJJ9VbVpbrvfNCBJamIxS2oTLcmdwGVzmr9YVXePox5JUs/rLnCqatAPjEqSxswlNUlSEwaOJKkJA0eS1ISBI0lqwsCRJDVh4EiSmjBwJElNGDiSpCYMHElSE6+7bxpYTm9dt4rpJX6JnSSpxyscSVITBo4kqQkDR5LUhIEjSWrCwJEkNWHgSJKaMHAkSU0YOJKkJgwcSVITBo4kqQkDR5LUhIEjSWrCwJEkNWHgSJKaMHAkSU0YOJKkJgwcSVITqapx1zCxkjwHPDXuOoZ0JnB43EUMYaXUCSun1pVSJ6ycWldKnTCZtZ5TVWfNbfRXTM/vqaraNO4ihpFkeiXUulLqhJVT60qpE1ZOrSulTlhZtbqkJklqwsCRJDVh4MzvrnEXMIKVUutKqRNWTq0rpU5YObWulDphBdXqQwOSpCa8wpEkNWHgSJKaOCEDJ8nlSZ5K8qMkNw34PEn+tfv8sSQbhx07YbUeSnIwyf4k0xNQ658neSjJ/yS5YZSxE1TnpM3pju7f+2NJHkzyl8OOnaA6J21Or+rq3J9kOsnmYcdOUJ1N53RoVXVCbcDJwH8Cfwr8HnAA+Is5ff4O+BYQ4FLgP4YdOym1dp8dAs6coHldDbwN2A3cMMrYSahzQuf07cAfda+vGMef1aXUOaFzega/u799ETAzoXM6sM7WczrKdiJe4VwC/KiqflxV/wt8HbhqTp+rgH+vnoeBP0yyZsixk1JrawvWWlW/qqpHgZdGHTshdbY2TK0PVtVvurcPA2cPO3ZC6mxtmFqPVve3NnA6UMOOnZA6J9aJGDjrgJ/0vf9p1zZMn2HGLqel1Aq9P4DfTrIvyc7XrMqF63gtx45qqcea5Dn9R3pXu4sZuxRLqRMmcE6TbE0yA+wFrh1l7ATUCW3ndGgn4lfbZEDb3P8zOF6fYcYup6XUCnBZVf0syWrggSQzVfXdZa1wuDpey7GjWuqxJnJOk/w1vb/IZ9fxJ3JOB9QJEzinVbUH2JPkr4BPA38z7NhlspQ6oe2cDu1EvML5KfCmvvdnAz8bss8wY5fTUmqlqmb/+StgD73L9NfKUuam5bwu6ViTOKdJLgK+AlxVVf81ytgJqHMi57Svtu8C5yU5c9SxS7SUOlvP6fDGfROp9Ubvqu7HwLn87mbcBXP6bOHVN+IfGXbsBNV6OvD7fa8fBC4fZ619fW/m1Q8NNJvXJdY5cXMK/AnwI+Dtiz3PMdc5iXP6Z/zuZvxG4Nnuv69Jm9Pj1dl0Tkc6r3EXMJaT7j3Z9UN6T4Hs6tquA67rXge4s/v8ILBpvrGTWCu9p1sOdNsTE1LrH9P7P7ffAv/dvf6D1vO62DondE6/AvwG2N9t0+P4s7rYOid0Tm/satkPPARsntA5HVjnOOZ02M2vtpEkNXEi3sORJI2BgSNJasLAkSQ1YeBIkpowcCRJTRg4kqQmDBxJUhP/B5v1+gKXkjbwAAAAAElFTkSuQmCC\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "feat_importances3 = pd.Series(clf3.feature_importances_, index=train_x.columns)\n",
    "feat_importances3.nlargest(3).plot(kind='barh')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## The ROC tells the whole story :-(\n",
    "\n",
    "**This did not work.**  The Factor Analysis is good for grouping variables and explaining how they vary together, but it isn't always good at helping predict anything.  Particularly when outliers are included and outcome variable has a lot of variance!"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 67,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYIAAAEGCAYAAABo25JHAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjMuMiwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy8vihELAAAACXBIWXMAAAsTAAALEwEAmpwYAAAkrklEQVR4nO3de3xU9ZnH8c9jAEG5uAhSRFNijXIRiRCNWG2hLIooslgVxVXQuqiI1tdWqlarllqL1drKapeiAtqiuIpVqqhdaymrKAgSuVaKCphKFQEpFxGQZ/84J+kwmUzOkJwJk/m+X6+8mHN/DoHzzO9yfj9zd0REJH8d0NABiIhIw1IiEBHJc0oEIiJ5TolARCTPKRGIiOS5Jg0dQKbatWvnnTt3bugwRERyysKFCz919/aptuVcIujcuTMLFixo6DBERHKKma2paZuqhkRE8pwSgYhInlMiEBHJc0oEIiJ5TolARCTPxZYIzGyymX1iZktr2G5mNsHMVpnZYjPrFVcsIiJSszhLBFOBgWm2nwkUhz+jgP+OMRYREalBbO8RuPscM+ucZpchwGMejIP9ppkdYmYd3X1dXDGJiOSKx+et5bnyv+21rtvhrbl9cPd6v1ZDthF0Aj5MWK4I11VjZqPMbIGZLVi/fn1WghMRaUjPlf+N5ev+kZVrNeSbxZZiXcpZctx9EjAJoLS0VDPpiEhe6NaxNU9e2Sf26zRkiaACODJh+QjgowaKRUQkbzVkIpgJXBr2HjoZ2Kz2ARGR7IutasjMngD6Au3MrAK4HWgK4O4TgVnAIGAVsB24LK5YRESkZnH2Grqolu0OXBPX9UVE9lepegQlW77uH3Tr2Dor8eTcMNQiIvu72h708z7YCEBZUdsa9+nWsTVDSlJ2pKx3SgQiInWU/OCv7UFfVtSWISWdGF5WmJX4aqNEICKSodoe/Pvbg742SgQiIhmqfNmrsg4/1x78yZQIRCSvRWm4TVaZBLLxslc2aBhqEclr+zKUQzYbcrNBJQIRadRq+8bf2L7d7wslAhFpVDLtwdPYvt3vCyUCEclZqb7t53oPnoagRCAiOSu59w7owb8vlAhEJKfle/1+fVCvIRGRPKdEICKS51Q1JCI5I7lxOJsjdDZmKhGISM5IfvlLXT/rh0oEIpJT1Dhc/1QiEBHJc0oEIiJ5TolARCTPKRGIiOQ5JQIRkTynRCAikufUfVRE9htR5w6Q+qUSgYjsN2qbLUwvkMVDJQIR2a/ohbHsU4lARCTPqUQgIg1Gg8jtH1QiEJEGo0Hk9g8qEYhILGrrAQT/LAGoTaBhKRGISL1IfvAnTyKfikoA+wclAhGpF8kTyWsS+dwRayIws4HA/UAB8LC7j0/a3gb4LVAYxnKvu0+JMyYRiY+qeXJTbI3FZlYAPAicCXQDLjKzbkm7XQMsd/eeQF/g52bWLK6YRESkujh7DZ0ErHL39919JzAdGJK0jwOtzMyAlsBGYHeMMYmISJI4E0En4MOE5YpwXaIHgK7AR8AS4Lvuvif5RGY2yswWmNmC9evXxxWviEheijMRWIp1nrR8BlAOHA6UAA+YWbW3Sdx9kruXuntp+/bt6ztOEZG8FmciqACOTFg+guCbf6LLgGc8sAr4AOgSY0wiIpIkzkTwFlBsZkVhA/CFwMykfdYC/QHMrANwLPB+jDGJiEiS2LqPuvtuMxsDvEzQfXSyuy8zs6vC7ROBHwNTzWwJQVXSje7+aVwxiYhIdbG+R+Dus4BZSesmJnz+CDg9zhhEpH5o0pjGS4POiUgkmjSm8dIQEyKSUk1DROvN4cZHJQIRSUlDROcPlQhEpEYqAeQHJQIRATRbWD5TIhDJQ6l6ACXPH6CqoPyhRCCSh5LnDgDNH5DPlAhE8pTq/6WSeg2JiOQ5JQIRkTwXORGY2cFxBiIiIg2j1kRgZqeY2XJgRbjc08x+FXtkIiKSFVEai39BMIHMTAB3f8fMvhFrVCJSr/SOgKQTqWrI3T9MWvVlDLGISEw0XISkE6VE8KGZnQJ4OMHMdYTVRCKSO9RdVGoSpURwFXANwcTzFQRzC4+OMSYREcmiKCWCY9394sQVZvZ14PV4QhIRkWyKUiL4r4jrREQkB9VYIjCzPsApQHsz+8+ETa0J5iAWEZFGIF3VUDOgZbhPq4T1/wDOizMoERHJnhoTgbv/GfizmU119zVZjElERLIoSmPxdjO7B+gONK9c6e7fii0qERHJmiiNxdOAvwBFwI+A1cBbMcYkIiJZFCURHOrujwC73P3P7n45cHLMcYmISJZEqRraFf65zszOAj4CjogvJBGpK40tJJmIkgjuNLM2wPcI3h9oDVwfZ1AiUjfJU1FqbCFJp9ZE4O7Phx83A/2g6s1iEdmPaWwhiSrdC2UFwAUEYwy95O5Lzexs4AdAC+CE7IQoIiJxSlcieAQ4EpgPTDCzNUAf4CZ3fzYLsYmISBakSwSlwPHuvsfMmgOfAke7+9+zE5qIiGRDuu6jO919D4C77wBWZpoEzGygmb1rZqvM7KYa9ulrZuVmtszM/pzJ+UVEpO7SlQi6mNni8LMBXwuXDXB3Pz7dicM2hgeBAQTzGLxlZjPdfXnCPocAvwIGuvtaMzts329FRET2RbpE0LWO5z4JWOXu7wOY2XRgCLA8YZ/hwDPuvhbA3T+p4zVFRCRD6Qadq+tAc52AxLmOK4CypH2OAZqa2WyCEU7vd/fHkk9kZqOAUQCFhYV1DEtERBJFmrx+H1mKdZ603AToDZwFnAH80MyOqXaQ+yR3L3X30vbt29d/pCIieSzKm8X7qoKg+2mlIwiGp0je51N33wZsM7M5QE9gZYxxiYhIgkglAjNrYWbHZnjut4BiMysys2bAhcDMpH2eA04zsyZmdhBB1dGKDK8jIiJ1UGsiMLPBQDnwUrhcYmbJD/Rq3H03MAZ4meDh/j/uvszMrjKzq8J9VoTnXUzw4trD7r50H+9FRET2QZSqoTsIegDNBnD3cjPrHOXk7j4LmJW0bmLS8j3APVHOJyKB5NFFk2m0UclElKqh3e6+OfZIRCSyytFFa6LRRiUTUUoES81sOFBgZsXAdcDceMMSkdpodFGpL1FKBNcSzFf8BfA4wXDU18cYk4iIZFGUEsGx7n4LcEvcwYhIappxTOIUpURwn5n9xcx+bGbdY49IRKpJbhNQG4DUpygzlPUzs68QTFIzycxaA0+6+52xRyciVdQmIHGJ9EKZu//d3ScAVxG8U3BbnEGJiEj21FoiMLOuwDDgPGADMJ1gInsRqQe1vRMAahOQeEVpLJ4CPAGc7u7JYwWJSB1V1v+ne9CrTUDiFKWN4ORsBCKSL2rqAaT6f2koNSYCM/sfd7/AzJaw9/DRkWYoE5FA8oN/3gcbASgragvo2740vHQlgu+Gf56djUBEGovaHvxlRW0ZUtKJ4WWaZEn2D+lmKFsXfhzt7jcmbjOzu4Ebqx8lIsl1/nrwy/4uSmPxAKo/9M9MsU5EQqrzl1ySro3gamA0cJSZLU7Y1Ap4Pe7ARHKFhn+QXJeuRPA48CLwU+CmhPVb3H1jrFGJ5JDkqiA1/kquSZcI3N1Xm9k1yRvMrK2Sgcg/qSpIclltJYKzgYUE3UctYZsDR8UYl0iDiPKWbzJVBUmuS9dr6Ozwz6LshSPSsKK85ZtMVUGS66KMNfR1oNzdt5nZvwO9gF+6+9rYoxOJmd7yFYk2+uh/A9vNrCfwfWAN8JtYoxLJEo3zLxLtPYLd7u5mNgS4390fMbMRcQcmki0qAUi+i5IItpjZzcAlwGlmVgA0jTcskXioz79IdVESwTBgOHC5u//dzAqBe+INSyRzUXr8aMA3keqiDEP9dzObBpxoZmcD8939sfhDE8lMlB4/GvdHpLoovYYuICgBzCZ4l+C/zGysuz8dc2wiGVN9v0jmolQN3QKc6O6fAJhZe+AVQIlAGpTq+0XqR5TuowdUJoHQhojHicRKXT9F6keUEsFLZvYywbzFEDQez4ovJJHoVBUkUndRGovHmtm5wKkEbQST3P13sUcmIiJZkW4+gmLgXuBrwBLgBnfPbDQuERHZ76Wr658MPA98m2AE0v/K9ORmNtDM3jWzVWZ2U5r9TjSzL83svEyvISIidZOuaqiVuz8Ufn7XzN7O5MThG8gPEkx1WQG8ZWYz3X15iv3uBl7O5PySf9RLSCQe6RJBczM7gX/OQ9Aicdnda0sMJwGr3P19ADObDgwBliftdy0wAzgxw9glz2gmMJF4pEsE64D7Epb/nrDswLdqOXcn4MOE5QqgLHEHM+sEDA3PVWMiMLNRwCiAwkK9EZrP1EtIpP6lm5imXx3PbSnWedLyL4Eb3f1Ls1S7V8UyCZgEUFpamnwOERGpgyjvEeyrCuDIhOUjgI+S9ikFpodJoB0wyMx2u/uzMcYlIiIJ4kwEbwHFZlYE/A24kGAU0yqJ02Ca2VTgeSUBEZHsii0RuPtuMxtD0BuoAJjs7svM7Kpw+8S4ri0iItFFGX3UgIuBo9x9XDgfwVfcfX5tx7r7LJKGo6gpAbj7yEgRS95Qd1GR7IgyeNyvgD7AReHyFoL3A0RipUHlRLIjStVQmbv3MrNFAO6+ycyaxRyXCKDuoiLZEKVEsCt8+9ehaj6CPbFGJSIiWRMlEUwAfgccZmY/AV4D7oo1KhERyZoow1BPM7OFQH+Cl8T+zd1XxB6ZiIhkRZReQ4XAduD3ievcfW2cgYmISHZEaSx+gaB9wIDmQBHwLtA9xrgkD6m7qEjDiFI11CNx2cx6AVfGFpHkLY0uKtIwMn6z2N3fNjMNGS2xUHdRkeyL0kbwnwmLBwC9gPWxRSQiIlkVpUTQKuHzboI2gxnxhCMiItmWNhGEL5K1dPexWYpHRESyrMYXysysibt/SVAVJCIijVS6EsF8giRQbmYzgaeAbZUb3f2ZmGMTEZEsiNJG0BbYQDCvcOX7BA4oEcg+S35nAPTegEhDSZcIDgt7DC3lnwmgkuYNljpJfmcA9N6ASENJlwgKgJZEm4ReJGN6Z0Bk/5AuEaxz93FZi0RERBpEukSQqiQgsk80jpDI/ivdfAT9sxaFNHqadlJk/1VjicDdN2YzEGn81CYgsn+KMkOZiIg0YkoEIiJ5TolARCTPKRGIiOQ5JQIRkTynRCAikueUCERE8lzGcxaLRKE3iUVyh0oEEgu9SSySO2ItEZjZQOB+gpFMH3b38UnbLwZuDBe3Ale7+ztxxiTZozeJRXJDbIkgnO/4QWAAUAG8ZWYz3X15wm4fAN90901mdiYwCSiLKyaJj6qCRHJXnFVDJwGr3P19d98JTAeGJO7g7nPdfVO4+CZwRIzxSIxUFSSSu+KsGuoEfJiwXEH6b/vfAV5MtcHMRgGjAAoLC+srPqlnqgoSyU1xlggiz2xmZv0IEsGNqba7+yR3L3X30vbt29djiCIiEmeJoAI4MmH5COCj5J3M7HjgYeBMd98QYzwiIpJCnCWCt4BiMysys2bAhcDMxB3MrBB4BrjE3VfGGIuIiNQgthKBu+82szHAywTdRye7+zIzuyrcPhG4DTgU+JWZAex299K4YhIRkepifY/A3WcBs5LWTUz4fAVwRZwxiIhIenqzWEQkz2msIdkneoFMpPFQIpBIkh/88z7YCEBZUVtAL5CJ5DIlAqkm+aEP1R/8ZUVtGVLSieFlesFPJNcpEUg1lcNFJFb16MEv0ngpEUiN9f0aLkIkP6jXkGjAOJE8pxJBHlIJQEQSqUSQh1QCEJFEKhHkKZUARKSSEkEe0MtfIpKOqobygKqCRCQdlQjyhKqCRKQmKhGIiOQ5lQgaIbUJiEgmVCJohNQmICKZUImgkVKbgIhEpRKBiEieU4lgP5dqSOjaqE1ARDKhEsF+Lrm+Pwq1CYhIJlQiyAGq7xeROCkRSGx27dpFRUUFO3bsaOhQRPJG8+bNOeKII2jatGnkY5QIJDYVFRW0atWKzp07Y2YNHY5Io+fubNiwgYqKCoqKiiIfpzYCic2OHTs49NBDlQREssTMOPTQQzMuhatEsJ9pbG8FKwmIZNe+/J9TIohZpt0/532wEQgmiwf1ABKR+KlqKGaZdv8sK2rLXUN78OSVfap+hpcVxhhh41ZQUEBJSQnHHXccgwcP5rPPPquX806dOpUxY8bUy7k6d+5Mjx49KCkpoaSkhLlz59bLeZOVl5cza9asvda9+OKLlJaW0rVrV7p06cINN9wAwB133MG9995bb9c+5ZRTqj6PHTuW7t27M3bsWCZOnMhjjz1Wp3MvWrSIK664Yq91Q4YMoU+fvXvajRw5kqeffnqvdS1btqz6vHLlSgYNGsTRRx9N165dueCCC/j444/rFNvGjRsZMGAAxcXFDBgwgE2bNqXcL/HfQGlpaa3HL1myhJEjR9YptkQqEdSjVN/+NR9ww2rRogXl5eUAjBgxggcffJBbbrmlYYNK4U9/+hPt2rXL6Jjdu3fTpEn0/8Ll5eUsWLCAQYMGAbB06VLGjBnDCy+8QJcuXdi9ezeTJk3KKIaoEpPbr3/9a9avX8+BBx6Y8XlS3fNdd93FrbfeWrX82Wef8fbbb9OyZUs++OCDSI2mO3bs4KyzzuK+++5j8ODBQPA7Wb9+PR06dMg4zkrjx4+nf//+3HTTTYwfP57x48dz9913p9w31b+Bmo7v0aMHFRUVrF27lsLCun9RVCLIQG3VPMnVOqCqnUo/+v0yln+U2Ytxtel2eGtuH9w98v59+vRh8eLFAMyfP5/rr7+ezz//nBYtWjBlyhSOPfZYpk6dysyZM9m+fTvvvfceQ4cO5Wc/+xkAU6ZM4ac//SkdO3bkmGOOqXqQrVmzhssvv5z169fTvn17pkyZQmFhISNHjqRFixb85S9/Yc2aNUyZMoVHH32UN954g7KyMqZOnVpjrOnO2bZtWxYtWkSvXr0YPXo011xzDevXr+eggw7ioYceokuXLjz11FP86Ec/oqCggDZt2vDKK69w22238fnnn/Paa69x880388ILL3DLLbfQpUsXAJo0acLo0aOrxfLQQw8xadIkdu7cydFHH81vfvMbDjrooGrXmDNnDsuWLeOyyy5j586d7NmzhxkzZlBcXEzLli3ZunUr55xzDtu2baOsrIybb76ZFStW0LJlS2644Qbee++9lPeSfM8///nPq2LbsmULixcvpmfPnlXrZsyYweDBg+nQoQPTp0/n5ptvrvXfxuOPP06fPn2qkgBAv379aj2uNs899xyzZ88Ggi8iffv2rTERZHr84MGDmT59Ot///vfrHKcSQQYqq3lqarwtK2rLkJJOqsrZD3355Zf88Y9/5Dvf+Q4AXbp0Yc6cOTRp0oRXXnmFH/zgB8yYMQMIvjkvWrSIAw88kGOPPZZrr72WJk2acPvtt7Nw4ULatGlDv379OOGEEwAYM2YMl156KSNGjGDy5Mlcd911PPvsswBs2rSJV199lZkzZzJ48GBef/11Hn74YU488UTKy8spKSkBgodOQUEBBx54IPPmzUt7zpUrV/LKK69QUFBA//79mThxIsXFxcybN4/Ro0fz6quvMm7cOF5++WU6derEZ599RrNmzRg3bhwLFizggQceAODuu+/me9/7Xq1/d+eeey7/8R//AcCtt97KI488wrXXXlvtGgATJ07ku9/9LhdffDE7d+7kyy+/3OtcM2fOpGXLllWltDvuuKNq26hRo1LeS/I9J1qwYAHHHXfcXuueeOIJbr/9djp06MB5550XKREsXbqU3r1717rfli1bOO2001Jue/zxx+nWrdte6z7++GM6duwIQMeOHfnkk09SHmtmnH766ZgZV155JaNGjar1+NLSUsaPH69E0BBUzbNvMvnmXp8+//xzSkpKWL16Nb1792bAgAEAbN68mREjRvDXv/4VM2PXrl1Vx/Tv3582bdoA0K1bN9asWcOnn35K3759ad++PQDDhg1j5cqVALzxxhs888wzAFxyySV7/cccPHgwZkaPHj3o0KEDPXr0AKB79+6sXr26KhEkVwukO+f5559PQUEBW7duZe7cuZx//vlV27744gsAvv71rzNy5EguuOACzj333Dr9HS5dupRbb72Vzz77jK1bt3LGGWfUeI0+ffrwk5/8hIqKCs4991yKi4sjXSPdvSTec7J169ZV/U4geHCuWrWKU089FTOjSZMmLF26lOOOOy5lb5pMe9i0atWqKonVp9dff53DDz+cTz75hAEDBtClSxe+8Y1vpD3msMMO46OPPqqX68faWGxmA83sXTNbZWY3pdhuZjYh3L7YzHrFGY/kn8o2gjVr1rBz504efPBBAH74wx/Sr18/li5dyu9///u9+l0n1l0XFBSwe/duIPpDI3G/ynMdcMABe533gAMOqDpvpuc8+OCDAdizZw+HHHII5eXlVT8rVqwAgm/md955Jx9++CElJSVs2LCh2jm7d+/OwoULa732yJEjeeCBB1iyZAm333571d9VqmsMHz6cmTNn0qJFC84444yqb/S1SXcvifecrEWLFnv97p588kk2bdpEUVERnTt3ZvXq1UyfPh2AQw89dK/G2o0bN1Yl36h/F1u2bKlq1E/+Wb58ebX9O3TowLp164AgaR122GEpz3v44YcDwcN96NChzJ8/v9bjd+zYQYsWLWqNOYrYEoGZFQAPAmcC3YCLzKxb0m5nAsXhzyjgv+OKZ188Pm8tw379RtVPpoO/yf6jTZs2TJgwgXvvvZddu3axefNmOnUK2m7S1dVXKisrY/bs2WzYsIFdu3bx1FNPVW075ZRTqh4206ZN49RTT61zvFHO2bp1a4qKiqpicXfeeecdAN577z3KysoYN24c7dq148MPP6RVq1Zs2bKl6vixY8dy1113VZVs9uzZw3333VftOlu2bKFjx47s2rWLadOmVa1PdY3333+fo446iuuuu45zzjmnqk2mNunuJZ2uXbuyatWqquUnnniCl156idWrV7N69WoWLlxY9ffYt29fnnzySXbu3AkEv/fKdoDhw4czd+5cXnjhhapzvfTSSyxZsmSv61WWCFL9JFcLAZxzzjk8+uijADz66KMMGTKk2j7btm2r+r1s27aNP/zhD1XVXemOX7lyZbVqsX0VZ9XQScAqd38fwMymA0OAxLQ5BHjM3R1408wOMbOO7r6uvoPZl8ZK9elvXE444QR69uxZ1cA2YsQI7rvvPr71rW/VemzHjh2544476NOnDx07dqRXr15V9d8TJkzg8ssv55577qlq2K2rqOecNm0aV199NXfeeSe7du3iwgsvpGfPnowdO5a//vWvuDv9+/enZ8+eFBYWMn78eEpKSrj55psZNmwYv/zlL7nooovYvn07ZsZZZ51V7Ro//vGPKSsr46tf/So9evSoemilusb48eP57W9/S9OmTfnKV77CbbfdFvmea7qXdLp06cLmzZvZsmULGzZsYO3atZx88slV24uKimjdujXz5s3j7LPPZuHChfTu3ZuCggK+9rWvMXHiRCAoWTz//PNcf/31XH/99TRt2pTjjz+e+++/P3L8qdx0001ccMEFPPLIIxQWFlYluo8++ogrrriCWbNm8fHHHzN06FAg6BU1fPhwBg4cmPZ4CKoTU/2+9oUFz+D6Z2bnAQPd/Ypw+RKgzN3HJOzzPDDe3V8Ll/8I3OjuC5LONYqgxEBhYWHvNWvWZBzPvvZaUePvvluxYgVdu3Zt6DCkkfvFL35Bq1atqr1L0Jh98cUXfPOb3+S1115L2YU41f89M1vo7qXVdibeEkGqCtXkrBNlH9x9EjAJoLS0dJ8yV0M1VopIvK6++uq9vinng7Vr1zJ+/PiM3iNJJ85EUAEcmbB8BJDcxB1lHxGRGjVv3pxLLrmkocPIquLi4sg9sqKIs9fQW0CxmRWZWTPgQmBm0j4zgUvD3kMnA5vjaB+QhhNX1aOIpLYv/+diKxG4+24zGwO8DBQAk919mZldFW6fCMwCBgGrgO3AZXHFI9nXvHlzNmzYoKGoRbKkcj6C5s2bZ3RcbI3FcSktLfUFCxbUvqM0OM1QJpJ9Nc1Q1lCNxZLnmjZtmtEsSSLSMDQMtYhInlMiEBHJc0oEIiJ5Lucai81sPZD5q8WBdsCn9RhOLtA95wfdc36oyz1/1d3bp9qQc4mgLsxsQU2t5o2V7jk/6J7zQ1z3rKohEZE8p0QgIpLn8i0RxDMz9/5N95wfdM/5IZZ7zqs2AhERqS7fSgQiIpJEiUBEJM81ykRgZgPN7F0zW2VmN6XYbmY2Idy+2Mx6NUSc9SnCPV8c3utiM5trZunnAMwBtd1zwn4nmtmX4ax5OS3KPZtZXzMrN7NlZvbnbMdY3yL8225jZr83s3fCe87pUYzNbLKZfWJmS2vYXv/PL3dvVD8EQ16/BxwFNAPeAbol7TMIeJFghrSTgXkNHXcW7vkU4F/Cz2fmwz0n7PcqwZDn5zV03Fn4PR9CMC94Ybh8WEPHnYV7/gFwd/i5PbARaNbQsdfhnr8B9AKW1rC93p9fjbFEcBKwyt3fd/edwHRgSNI+Q4DHPPAmcIiZdcx2oPWo1nt297nuvilcfJNgNrhcFuX3DHAtMAP4JJvBxSTKPQ8HnnH3tQDunuv3HeWeHWhlwaQXLQkSwe7shll/3H0OwT3UpN6fX40xEXQCPkxYrgjXZbpPLsn0fr5D8I0il9V6z2bWCRgKTMxiXHGK8ns+BvgXM5ttZgvN7NKsRRePKPf8ANCVYJrbJcB33X1PdsJrEPX+/GqM8xGkmgoruY9slH1ySeT7MbN+BIng1Fgjil+Ue/4lcKO7f9lIZkiLcs9NgN5Af6AF8IaZvenuK+MOLiZR7vkMoBz4FvA14H/N7P/c/R8xx9ZQ6v351RgTQQVwZMLyEQTfFDLdJ5dEuh8zOx54GDjT3TdkKba4RLnnUmB6mATaAYPMbLe7P5uVCOtf1H/bn7r7NmCbmc0BegK5mgii3PNlwHgPKtBXmdkHQBdgfnZCzLp6f341xqqht4BiMysys2bAhcDMpH1mApeGre8nA5vdfV22A61Htd6zmRUCzwCX5PC3w0S13rO7F7l7Z3fvDDwNjM7hJADR/m0/B5xmZk3M7CCgDFiR5TjrU5R7XktQAsLMOgDHAu9nNcrsqvfnV6MrEbj7bjMbA7xM0ONgsrsvM7Orwu0TCXqQDAJWAdsJvlHkrIj3fBtwKPCr8Bvybs/hkRsj3nOjEuWe3X2Fmb0ELAb2AA+7e8puiLkg4u/5x8BUM1tCUG1yo7vn7PDUZvYE0BdoZ2YVwO1AU4jv+aUhJkRE8lxjrBoSEZEMKBGIiOQ5JQIRkTynRCAikueUCERE8pwSgeyXwtFCyxN+OqfZd2s9XG+qmX0QXuttM+uzD+d42My6hZ9/kLRtbl1jDM9T+feyNBxx85Ba9i8xs0H1cW1pvNR9VPZLZrbV3VvW975pzjEVeN7dnzaz04F73f34OpyvzjHVdl4zexRY6e4/SbP/SKDU3cfUdyzSeKhEIDnBzFqa2R/Db+tLzKzaSKNm1tHM5iR8Yz4tXH+6mb0RHvuUmdX2gJ4DHB0e+5/huZaa2fXhuoPN7IVw/PulZjYsXD/bzErNbDzQIoxjWrhta/jnk4nf0MOSyLfNrMDM7jGztywYY/7KCH8tbxAONmZmJ1kwz8Si8M9jwzdxxwHDwliGhbFPDq+zKNXfo+Shhh57Wz/6SfUDfEkwkFg58DuCt+Bbh9vaEbxVWVmi3Rr++T3glvBzAdAq3HcOcHC4/kbgthTXm0o4XwFwPjCPYPC2JcDBBMMbLwNOAL4NPJRwbJvwz9kE376rYkrYpzLGocCj4edmBKNItgBGAbeG6w8EFgBFKeLcmnB/TwEDw+XWQJPw878CM8LPI4EHEo6/C/j38PMhBGMQHdzQv2/9NOxPoxtiQhqNz929pHLBzJoCd5nZNwiGTugEdAD+nnDMW8DkcN9n3b3czL4JdANeD4fWaEbwTTqVe8zsVmA9wQit/YHfeTCAG2b2DHAa8BJwr5ndTVCd9H8Z3NeLwAQzOxAYCMxx98/D6qjj7Z+zqLUBioEPko5vYWblQGdgIfC/Cfs/ambFBCNRNq3h+qcD55jZDeFyc6CQ3B6PSOpIiUByxcUEs0/1dvddZraa4CFWxd3nhIniLOA3ZnYPsAn4X3e/KMI1xrr705ULZvavqXZy95Vm1ptgvJefmtkf3H1clJtw9x1mNptg6ORhwBOVlwOudfeXaznF5+5eYmZtgOeBa4AJBOPt/Mndh4YN67NrON6Ab7v7u1HilfygNgLJFW2AT8Ik0A/4avIOZvbVcJ+HgEcIpvt7E/i6mVXW+R9kZsdEvOYc4N/CYw4mqNb5PzM7HNju7r8F7g2vk2xXWDJJZTrBQGGnEQymRvjn1ZXHmNkx4TVTcvfNwHXADeExbYC/hZtHJuy6haCKrNLLwLUWFo/M7ISariH5Q4lAcsU0oNTMFhCUDv6SYp++QLmZLSKox7/f3dcTPBifMLPFBImhS5QLuvvbBG0H8wnaDB5290VAD2B+WEVzC3BnisMnAYsrG4uT/IFgXtpXPJh+EYJ5IpYDb1swafmvqaXEHsbyDsHQzD8jKJ28TtB+UOlPQLfKxmKCkkPTMLal4bLkOXUfFRHJcyoRiIjkOSUCEZE8p0QgIpLnlAhERPKcEoGISJ5TIhARyXNKBCIiee7/AWVJqv16kJSzAAAAAElFTkSuQmCC\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "clf_disp3 = plot_roc_curve(clf3, test_x, test_y)\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 68,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAWAAAAD4CAYAAADSIzzWAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjMuMiwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy8vihELAAAACXBIWXMAAAsTAAALEwEAmpwYAAAQhElEQVR4nO3df4xlZ13H8fenW1oERNo03Sy7NRTdgi3BYJaKErVatVWRbUyabA1mg5tMNAXBH4Gt/NGYuEmjBjVRTCaydo3QZkW0GxKRuoqNEfpDfna71G6oaYcuXbCiBpO2M/P1j3tabsaZO3fu3pmz9+n71ZzM3Oecec6TsPnwzXOfc55UFZKkrXde3wOQpBcqA1iSemIAS1JPDGBJ6okBLEk9OX/Tb3DBTpdZ6P85evGP9D0EnYN+7qsfztn28ezXvzx25rzoklef9f3OhhWwJPVk0ytgSdpSy0t9j2BsBrCktiwt9j2CsRnAkppStdz3EMZmAEtqy7IBLEn9sAKWpJ74JZwk9cQKWJL6Ua6CkKSezNCXcD4JJ6kttTz+sY4kh5OcSfLgivZ3Jnk4yYkkvzPUfkuSU92569br3wpYUlum+yXc7cAfAX/+XEOSHwX2Aq+vqqeTXNq1XwnsA64CXgn8fZIrqmrNAVkBS2rLFCvgqroHeGpF8y8Dt1XV0901Z7r2vcCdVfV0VT0KnAKuHtW/ASypLUuLYx9J5pI8MHTMjXGHK4AfSnJvkn9K8saufSfw+NB1C13bmpyCkNSWDXwJV1XzwPwG73A+cBHwJuCNwNEkrwZWe7XlyFdjGsCSmjJiynVaFoCP1mBL+fuSLAOXdO2XDV23C3hiVEdOQUhqyxTngNfwN8CPASS5ArgA+DpwDNiX5MIklwO7gftGdWQFLKktU1wHnOQO4BrgkiQLwK3AYeBwtzTtGWB/Vw2fSHIUeAhYBG4etQICDGBJrZnio8hVddMap962xvWHgEPj9m8AS2rL0rN9j2BsBrCktszQo8gGsKS2+DY0SeqJFbAk9cQAlqR+lF/CSVJPnAOWpJ44BSFJPbEClqSeWAFLUk+sgCWpJ4vuiixJ/bAClqSeOAcsST2xApaknlgBS1JPZqgCdk84SW1ZXBz/WEeSw0nOdNsPrTz3G0kqySVDbbckOZXk4STXrde/ASypLVXjH+u7Hbh+ZWOSy4CfAB4barsS2Adc1f3NB5JsG9W5ASypLcvL4x/rqKp7gKdWOfX7wHuA4RTfC9xZVU9X1aPAKeDqUf0bwJLasoEATjKX5IGhY2697pO8FfhKVX1+xamdwONDnxe6tjX5JZyktmzgS7iqmgfmx70+yUuA9wE/udrp1W4xqj8DWFJblpY2s/fvAi4HPp8EYBfwmSRXM6h4Lxu6dhfwxKjODGBJbdnEdcBV9UXg0uc+J/l3YE9VfT3JMeDDSd4PvBLYDdw3qj/ngCW1ZYpfwiW5A/gU8JokC0kOrHVtVZ0AjgIPAR8Hbq6qkeW4FbCktkzxQYyqummd869a8fkQcGjc/g1gSU2p5bHW954TDGBJbfFdEJLUk81dBTFVBrCktlgBS1JPDGBJ6sl4L9k5JxjAktrSUgWc5LUM3vKzk8FzzU8Ax6rq5CaPTZI2boaWoY18Ei7Je4E7Gbxk4j7g/u73O5Ic3PzhSdIGLS2Nf/RsvQr4AHBVVT073Ng963wCuG21P+pe6TYHkG3fwXnnvXQKQ5Wk9dUMTUGs9y6IZQYvlVhpR3duVVU1X1V7qmqP4StpSy3X+EfP1quA3w0cT/II33rR8HcC3w28YxPHJUmTmaFNOUcGcFV9PMkVDLbV2Mlg/ncBuH+9t/xIUi/Ogcp2XOuugqiqZeDTWzAWSTp7i7NTG7oOWFJbWpmCkKSZM0NTEO6IIakptbw89rGeJIeTnEny4FDb7yb5UpIvJPnrJK8YOndLklNJHk5y3Xr9G8CS2jLdZWi3A9evaLsbeF1VvR74N+AWgCRXAvuAq7q/+UCSbaM6N4AltWWKAVxV9wBPrWj7RFUtdh8/zWD3Yxi8suHOqnq6qh4FTjFYQbYmA1hSWzbwKHKSuSQPDB1zG7zbLwJ/2/2+k289LwGDJbs7R/2xX8JJaspG9oSrqnlgfpL7JHkfsAh86Lmm1W4xqg8DWFJbtmAVRJL9wFuAa6uefwHxAnDZ0GW7GLw9ck1OQUhqy/Ly+McEklwPvBd4a1X979CpY8C+JBcmuRzYzeAtkmuyApbUlilWwEnuAK4BLkmyANzKYNXDhcDdSQA+XVW/VFUnkhwFHmIwNXHzeq9sMIAltWWKAVxVN63S/MER1x8CDo3bvwEsqSm15KPIktSPGXoU2QCW1JSNLEPrmwEsqS0GsCT1ZHamgA1gSW2pxdlJYANYUltmJ38NYElt8Us4SeqLFbAk9cMKWJL6YgUsSf14fq+KGWAAS2rKDO1KbwBLaowBLEn9sAKWpJ7MUgC7JZGkptRSxj7Wk+RwkjNJHhxquzjJ3Uke6X5eNHTuliSnkjyc5Lr1+jeAJTWllsc/xnA7cP2KtoPA8araDRzvPpPkSmAfcFX3Nx9Ism1U5wawpKbUcsY+1u2r6h7gqRXNe4Ej3e9HgBuG2u+sqqer6lHgFHD1qP4NYElNmXIFvJrtVXUaoPt5ade+E3h86LqFrm1NBrCkplRl7CPJXJIHho65s7j1aiX1yOeiXQUhqSkbqWyrah6Y3+Atnkyyo6pOJ9kBnOnaF4DLhq7bBTwxqiMrYElNWV7K2MeEjgH7u9/3A3cNte9LcmGSy4HdwH2jOrICltSUcb5cG1eSO4BrgEuSLAC3ArcBR5McAB4DbgSoqhNJjgIPAYvAzVW1NKp/A1hSU6YZwFV10xqnrl3j+kPAoXH7N4AlNaVm53XABrCktkyzAt5sBrCkplQZwJLUi6XJVzdsOQNYUlOsgCWpJ84BS1JPXAUhST2xApakniwtz84bFgxgSU1xCkKSerLsKghJ6ofL0CSpJ05BSOv42Qd/u+8hqFFOQUhST1wFIUk9maEZCANYUltmaQpidmp1SRrDRnZFXk+SX01yIsmDSe5I8uIkFye5O8kj3c+LJh2rASypKcsbOEZJshP4FWBPVb0O2AbsAw4Cx6tqN3C8+zwRA1hSU4qMfYzhfODbkpwPvITBNvN7gSPd+SPADZOO1QCW1JTFythHkrkkDwwdc8/1U1VfAX6Pwc7Hp4H/qqpPANur6nR3zWng0knH6pdwkpoyZmU7uLZqHphf7Vw3t7sXuBz4BvCXSd42hSE+zwpYUlOmNQcM/DjwaFV9raqeBT4K/CDwZJIdAN3PM5OO1QCW1JQpzgE/BrwpyUuSBLgWOAkcA/Z31+wH7pp0rE5BSGrKGJXtWKrq3iQfAT4DLAKfZTBd8TLgaJIDDEL6xknvYQBLasrSBuaA11NVtwK3rmh+mkE1fNYMYElNmaEdiQxgSW1ZnmIFvNkMYElN8WU8ktSTaX0JtxUMYElNWY5TEJLUi6W+B7ABBrCkprgKQpJ64ioISeqJqyAkqSdOQUhST1yGJkk9WbIClqR+WAFLUk8MYEnqyRi7zZ8zDGBJTZmlCtgtiSQ1ZWkDx3qSvCLJR5J8KcnJJD+Q5OIkdyd5pPt50aRjNYAlNWU54x9j+EPg41X1WuB7GewJdxA4XlW7gePd54kYwJKaMq1dkZO8HPhh4IMAVfVMVX2DwVb1R7rLjgA3TDpWA1hSU6a4Lf2rga8Bf5bks0n+NMlLge1VdRqg+3nppGM1gCU1pTZwJJlL8sDQMTfU1fnA9wF/UlVvAL7JWUw3rMZVEJKaspF3QVTVPIOt5lezACxU1b3d548wCOAnk+yoqtNJdgBnJh2rFbCkpkxrFURVfRV4PMlruqZrgYeAY8D+rm0/cNekY7UCltSU5em+kPKdwIeSXAB8GXg7g8L1aJIDwGPAjZN2bgBLaso0H8Soqs8Be1Y5de00+jeAJTXFF7JLUk9m6VFkA1hSUxYzOzWwASypKbMTvwawpMY4BSFJPZnyMrRNZQBLasrsxK8BLKkxTkFIUk+WZqgGNoAlNcUKWJJ6UlbAktQPK2BJ6onL0CSpJ7MTvwawpMYszlAEG8CSmjJLX8JNvCVRkrePOPf8RnfLy9+c9BaStGFT3BV5053NnnC/tdaJqpqvqj1Vtee88156FreQpI2pDfw3jiTbum3pP9Z9vjjJ3Uke6X5eNOlYR05BJPnCWqeA7ZPeVJI2yyZUtu8CTgIv7z4fBI5X1W1JDnaf3ztJx+vNAW8HrgP+c0V7gH+Z5IaStJmWanpzwEl2AT8DHAJ+rWveC1zT/X4E+CSbFMAfA17WbUy3cmCfnOSGkrSZNrIOOMkcMDfUNF9V80Of/wB4D/DtQ23bq+o0QFWdTnLppGMdGcBVdWDEuZ+f9KaStFk2sgqiC9v51c4leQtwpqr+Nck1UxncCi5Dk9SUKc4Bvxl4a5KfBl4MvDzJXwBPJtnRVb87gDOT3uBsVkFI0jlnmRr7GKWqbqmqXVX1KmAf8A9V9TbgGLC/u2w/cNekY7UCltSULXgQ4zbgaJIDwGPAjZN2ZABLaso0V0E8p6o+yWC1A1X1H8C10+jXAJbUFN+GJkk9ORceMR6XASypKbP0Mh4DWFJTnIKQpJ7UJnwJt1kMYElNcVt6SeqJUxCS1BOnICSpJ1bAktQTl6FJUk8241HkzWIAS2qKUxCS1BMDWJJ64ioISeqJFbAk9WSWVkG4JZGkpizV8tjHKEkuS/KPSU4mOZHkXV37xUnuTvJI9/OiScdqAEtqSlWNfaxjEfj1qvoe4E3AzUmuBA4Cx6tqN3C8+zwRA1hSU6a4KefpqvpM9/v/ACeBncBe4Eh32RHghknHagBLakpt4L8kc0keGDrmVuszyauANwD3Atur6jQMQhq4dNKx+iWcpKYsb2AZWlXNA/OjrknyMuCvgHdX1X8nObsBDrECltSUjVTA60nyIgbh+6Gq+mjX/GSSHd35HcCZScdqAEtqyhRXQQT4IHCyqt4/dOoYsL/7fT9w16RjdQpCUlM2MgWxjjcDvwB8McnnurbfBG4DjiY5ADwG3DjpDQxgSU2Z1oMYVfXPwFoTvtdO4x4GsKSmTLEC3nQGsKSmzNKjyAawpKYs1VLfQxibASypKb6OUpJ64usoJaknVsCS1BNXQUhST1wFIUk9We8R43OJASypKc4BS1JPnAOWpJ5YAUtST1wHLEk9sQKWpJ64CkKSeuKXcJLUk1magnBPOElNmfKmnNcneTjJqSQHpz1WK2BJTZlWBZxkG/DHwE8AC8D9SY5V1UNTuQEGsKTGTHEO+GrgVFV9GSDJncBeYHYCePGZr6y1qd0LTpK5qprvexw6t/jvYro2kjlJ5oC5oab5of8tdgKPD51bAL7/7Ef4Lc4Bb6259S/RC5D/LnpSVfNVtWfoGP4/wtWCfKrf8BnAkrS6BeCyoc+7gCemeQMDWJJWdz+wO8nlSS4A9gHHpnkDv4TbWs7zaTX+uzgHVdVikncAfwdsAw5X1Ylp3iOztGhZklriFIQk9cQAlqSeGMBbZLMfadTsSXI4yZkkD/Y9FvXDAN4CQ480/hRwJXBTkiv7HZXOAbcD1/c9CPXHAN4azz/SWFXPAM890qgXsKq6B3iq73GoPwbw1ljtkcadPY1F0jnCAN4am/5Io6TZYwBvjU1/pFHS7DGAt8amP9IoafYYwFugqhaB5x5pPAkcnfYjjZo9Se4APgW8JslCkgN9j0lby0eRJaknVsCS1BMDWJJ6YgBLUk8MYEnqiQEsST0xgCWpJwawJPXk/wDp/zl3+9dmSAAAAABJRU5ErkJggg==\n",
      "text/plain": [
       "<Figure size 432x288 with 2 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "sb.heatmap(cm3)\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 69,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([[  0,  73],\n",
       "       [  0, 171]])"
      ]
     },
     "execution_count": 69,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "cm3"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.5"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}