K_NN_tuning_hyper_Approach2.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/anaconda3/lib/python3.6/site-packages/sklearn/cross_validation.py:41: DeprecationWarning: This module was deprecated in version 0.18 in favor of the model_selection module into which all the refactored classes and functions are moved. Also note that the interface of the new CV iterators are different from that of this module. This module will be removed in 0.20.\n",
      "  \"This module will be removed in 0.20.\", DeprecationWarning)\n",
      "/anaconda3/lib/python3.6/site-packages/sklearn/grid_search.py:42: DeprecationWarning: This module was deprecated in version 0.18 in favor of the model_selection module into which all the refactored classes and functions are moved. This module will be removed in 0.20.\n",
      "  DeprecationWarning)\n"
     ]
    }
   ],
   "source": [
    "# Data analysis packages:\n",
    "import pandas as pd\n",
    "import numpy as np\n",
    "#from datetime import datetime as dt\n",
    "\n",
    "# Visualization packages:\n",
    "import seaborn as sns\n",
    "import matplotlib.pyplot as plt\n",
    "from sklearn.linear_model import LogisticRegression #logistic regression\n",
    "from sklearn import svm #support vector Machine\n",
    "from sklearn.ensemble import RandomForestClassifier #Random Forest\n",
    "from sklearn.neighbors import KNeighborsClassifier #KNN\n",
    "from sklearn.naive_bayes import GaussianNB #Naive bayes\n",
    "from sklearn.tree import DecisionTreeClassifier #Decision Tree\n",
    "from sklearn.model_selection import train_test_split #training and testing data split\n",
    "from sklearn import metrics #accuracy measure\n",
    "from sklearn.metrics import confusion_matrix #for confusion matrix\n",
    "from sklearn.metrics import f1_score\n",
    "from time import time\n",
    "from operator import itemgetter\n",
    "from scipy.stats import randint\n",
    "from sklearn.tree import DecisionTreeClassifier\n",
    "from sklearn.tree import export_graphviz\n",
    "from sklearn.grid_search import GridSearchCV\n",
    "from sklearn.grid_search import RandomizedSearchCV\n",
    "from sklearn.cross_validation import  cross_val_score\n",
    "from __future__ import print_function\n",
    "\n",
    "import os\n",
    "import subprocess\n",
    "\n",
    "from time import time\n",
    "from operator import itemgetter\n",
    "from scipy.stats import randint\n",
    "\n",
    "import pandas as pd\n",
    "import numpy as np\n",
    "from sklearn.tree import DecisionTreeClassifier\n",
    "from sklearn.tree import export_graphviz\n",
    "from sklearn.grid_search import GridSearchCV\n",
    "from sklearn.grid_search import RandomizedSearchCV\n",
    "from sklearn.cross_validation import  cross_val_score\n",
    "\n",
    "\n",
    "filename = 'df_imputed__OHE_cat_std_num.pkl'\n",
    "df = pd.read_pickle(filename)\n",
    "\n",
    "X = df[df.columns[:-1]]\n",
    "y = pd.Series(df['heartdisease'])\n",
    "\n",
    "\n",
    "train_X,test_X,train_y,test_y =train_test_split(X,y,test_size=0.33,shuffle = True, random_state=45)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def report(grid_scores, n_top=3):\n",
    "    \"\"\"Report top n_top parameters settings, default n_top=3.\n",
    "\n",
    "    Args\n",
    "    ----\n",
    "    grid_scores -- output from grid or random search\n",
    "    n_top -- how many to report, of top models\n",
    "\n",
    "    Returns\n",
    "    -------\n",
    "    top_params -- [dict] top parameter settings found in\n",
    "                  search\n",
    "    \"\"\"\n",
    "    top_scores = sorted(grid_scores,\n",
    "                        key=itemgetter(1),\n",
    "                        reverse=True)[:n_top]\n",
    "    for i, score in enumerate(top_scores):\n",
    "        print(\"Model with rank: {0}\".format(i + 1))\n",
    "        print((\"Mean validation score: \"\n",
    "               \"{0:.3f} (std: {1:.3f})\").format(\n",
    "               score.mean_validation_score,\n",
    "               np.std(score.cv_validation_scores)))\n",
    "        print(\"Parameters: {0}\".format(score.parameters))\n",
    "        print(\"\")\n",
    "\n",
    "    return top_scores[0].parameters"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def run_gridsearch(X, y, clf, param_grid, cv=5):\n",
    "    \"\"\"Run a grid search for best Decision Tree parameters.\n",
    "\n",
    "    Args\n",
    "    ----\n",
    "    X -- features\n",
    "    y -- targets (classes)\n",
    "    cf -- scikit-learn Decision Tree\n",
    "    param_grid -- [dict] parameter settings to test\n",
    "    cv -- fold of cross-validation, default 5\n",
    "\n",
    "    Returns\n",
    "    -------\n",
    "    top_params -- [dict] from report()\n",
    "    \"\"\"\n",
    "    grid_search = GridSearchCV(clf,\n",
    "                               param_grid=param_grid,\n",
    "                               cv=cv)\n",
    "    start = time()\n",
    "    grid_search.fit(X, y)\n",
    "\n",
    "    print((\"\\nGridSearchCV took {:.2f} \"\n",
    "           \"seconds for {:d} candidate \"\n",
    "           \"parameter settings.\").format(time() - start,\n",
    "                len(grid_search.grid_scores_)))\n",
    "\n",
    "    top_params = report(grid_search.grid_scores_, 3)\n",
    "    return  top_params"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def run_randomsearch(X, y, clf, para_dist, cv=5,\n",
    "                     n_iter_search=20):\n",
    "    \"\"\"Run a random search for best Decision Tree parameters.\n",
    "\n",
    "    Args\n",
    "    ----\n",
    "    X -- features\n",
    "    y -- targets (classes)\n",
    "    cf -- scikit-learn Decision Tree\n",
    "    param_dist -- [dict] list, distributions of parameters\n",
    "                  to sample\n",
    "    cv -- fold of cross-validation, default 5\n",
    "    n_iter_search -- number of random parameter sets to try,\n",
    "                     default 20.\n",
    "\n",
    "    Returns\n",
    "    -------\n",
    "    top_params -- [dict] from report()\n",
    "    \"\"\"\n",
    "    random_search = RandomizedSearchCV(clf,\n",
    "                        param_distributions=param_dist,\n",
    "                        n_iter=n_iter_search)\n",
    "\n",
    "    start = time()\n",
    "    random_search.fit(X, y)\n",
    "    print((\"\\nRandomizedSearchCV took {:.2f} seconds \"\n",
    "           \"for {:d} candidates parameter \"\n",
    "           \"settings.\").format((time() - start),\n",
    "                               n_iter_search))\n",
    "\n",
    "    top_params = report(random_search.grid_scores_, 3)\n",
    "    return  top_params"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "-- Grid Parameter Search via 10-fold CV\n",
      "\n",
      "GridSearchCV took 8.08 seconds for 100 candidate parameter settings.\n",
      "Model with rank: 1\n",
      "Mean validation score: 0.788 (std: 0.064)\n",
      "Parameters: {'n_neighbors': 22}\n",
      "\n",
      "Model with rank: 2\n",
      "Mean validation score: 0.788 (std: 0.072)\n",
      "Parameters: {'n_neighbors': 33}\n",
      "\n",
      "Model with rank: 3\n",
      "Mean validation score: 0.788 (std: 0.071)\n",
      "Parameters: {'n_neighbors': 36}\n",
      "\n",
      "\n",
      "-- Best Parameters:\n",
      "parameter: n_neighbors          setting: 22\n",
      "\n",
      "\n",
      "-- Testing best parameters [Grid]...\n",
      "mean: 0.788 (std: 0.064)\n",
      "\n"
     ]
    }
   ],
   "source": [
    "print(\"-- Grid Parameter Search via 10-fold CV\")\n",
    "k_range = list(range(1, 101))\n",
    "# set of parameters to test\n",
    "param_grid = {\"n_neighbors\": k_range,\n",
    "              }\n",
    "\n",
    "model_KNN = KNeighborsClassifier()\n",
    "model_KNN_gs = run_gridsearch(X, y, model_KNN, param_grid, cv=10)\n",
    "\n",
    "print(\"\\n-- Best Parameters:\")\n",
    "for k, v in model_KNN_gs.items():\n",
    "    print(\"parameter: {:<20s} setting: {}\".format(k, v))\n",
    "    \n",
    "# test the retuned best parameters\n",
    "print(\"\\n\\n-- Testing best parameters [Grid]...\")\n",
    "final_gs_model = KNeighborsClassifier(**model_KNN_gs)\n",
    "scores = cross_val_score(final_gs_model, X, y, cv=10)\n",
    "print(\"mean: {:.3f} (std: {:.3f})\".format(scores.mean(),\n",
    "                                          scores.std()),\n",
    "                                          end=\"\\n\\n\" )"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "-- Random Parameter Search via 10-fold CV\n",
      "\n",
      "RandomizedSearchCV took 3.40 seconds for 100 candidates parameter settings.\n",
      "Model with rank: 1\n",
      "Mean validation score: 0.765 (std: 0.014)\n",
      "Parameters: {'n_neighbors': 11}\n",
      "\n",
      "Model with rank: 2\n",
      "Mean validation score: 0.763 (std: 0.017)\n",
      "Parameters: {'n_neighbors': 13}\n",
      "\n",
      "Model with rank: 3\n",
      "Mean validation score: 0.763 (std: 0.010)\n",
      "Parameters: {'n_neighbors': 14}\n",
      "\n",
      "\n",
      "-- Best Parameters:\n",
      "parameter: n_neighbors          setting: 11\n",
      "\n",
      "\n",
      "-- Testing best parameters [Grid]...\n",
      "mean: 0.780 (std: 0.073)\n",
      "\n"
     ]
    }
   ],
   "source": [
    "print(\"-- Random Parameter Search via 10-fold CV\")\n",
    "\n",
    "# dict of parameter list/distributions to sample\n",
    "k_range = list(range(1, 101))\n",
    "# set of parameters to test\n",
    "param_dist = {\"n_neighbors\": k_range,\n",
    "              }\n",
    "\n",
    "model_KNN = KNeighborsClassifier()\n",
    "model_KNN_rs = run_randomsearch(X, y, model_KNN, param_dist, cv=10,\n",
    "                         n_iter_search=100)\n",
    "\n",
    "print(\"\\n-- Best Parameters:\")\n",
    "for k, v in model_KNN_rs.items():\n",
    "    print(\"parameter: {:<20s} setting: {}\".format(k, v))\n",
    "    \n",
    "# test the retuned best parameters\n",
    "print(\"\\n\\n-- Testing best parameters [Grid]...\")\n",
    "final_rs_model = KNeighborsClassifier(**model_KNN_rs)\n",
    "scores = cross_val_score(final_rs_model, X, y, cv=10)\n",
    "print(\"mean: {:.3f} (std: {:.3f})\".format(scores.mean(),\n",
    "                                          scores.std()),\n",
    "                                          end=\"\\n\\n\" )\n",
    "\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.3"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}