notebook-course.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "hideCode": false,
    "hidePrompt": false
   },
   "outputs": [],
   "source": [
    "%matplotlib inline\n",
    "\n",
    "import matplotlib.pyplot as plt\n",
    "import numpy as np\n",
    "import pandas as pd"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# 1. Preprocess data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "hideCode": false,
    "hidePrompt": false
   },
   "outputs": [],
   "source": [
    "train_data = pd.read_csv(\"train.csv\")\n",
    "test_data = pd.read_csv(\"test.csv\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "hideCode": false,
    "hidePrompt": false
   },
   "outputs": [],
   "source": [
    "print train_data.shape\n",
    "print test_data.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "hideCode": false,
    "hidePrompt": false
   },
   "outputs": [],
   "source": [
    "train_data.describe()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "plt.figure(figsize=(14,6))\n",
    "train_data.boxplot()\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "hideCode": false,
    "hidePrompt": false
   },
   "outputs": [],
   "source": [
    "from sklearn import preprocessing\n",
    "\n",
    "scaler = preprocessing.StandardScaler()\n",
    "tmp = pd.DataFrame(scaler.fit_transform(train_data),columns=train_data.columns)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "hideCode": false,
    "hidePrompt": false
   },
   "outputs": [],
   "source": [
    "plt.figure(figsize=(14,6))\n",
    "tmp.boxplot()\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "hideCode": false,
    "hidePrompt": false
   },
   "outputs": [],
   "source": [
    "tmp2 = tmp.pop('eyeDetection')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "hideCode": false,
    "hidePrompt": false
   },
   "outputs": [],
   "source": [
    "from scipy import stats\n",
    "\n",
    "train_data = train_data[(np.abs(stats.zscore(train_data)) < 3).all(axis=1)]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "hideCode": false,
    "hidePrompt": false
   },
   "outputs": [],
   "source": [
    "train_set_x = train_data\n",
    "train_set_y = train_set_x.pop(\"eyeDetection\")\n",
    "test_set_x = test_data\n",
    "test_set_y = test_set_x.pop(\"eyeDetection\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "scaler = preprocessing.StandardScaler()\n",
    "\n",
    "scaler.fit(train_set_x)\n",
    "train_set_x = pd.DataFrame(scaler.transform(train_set_x),columns=train_set_x.columns)\n",
    "test_set_x = pd.DataFrame(scaler.transform(test_set_x),columns=test_set_x.columns)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "hideCode": false,
    "hidePrompt": false
   },
   "outputs": [],
   "source": [
    "plt.figure(figsize=(14,6))\n",
    "train_set_x.boxplot()\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "train_set_y.value_counts()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "base_score = float(5360) / (5360+4384)\n",
    "print base_score"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# 2. Logistic regression"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#http://scikit-learn.org/stable/modules/generated/sklearn.linear_model.LogisticRegression.html\n",
    "from sklearn import linear_model\n",
    "#http://scikit-learn.org/stable/modules/classes.html#sklearn-metrics-metrics\n",
    "from sklearn import metrics\n",
    "\n",
    "clf_logistic = linear_model.LogisticRegression(C=1000000)\n",
    "clf_logistic.fit(train_set_x,train_set_y)\n",
    "\n",
    "train_predictions = clf_logistic.predict(train_set_x)\n",
    "test_predictions = clf_logistic.predict(test_set_x)\n",
    "\n",
    "print metrics.accuracy_score(train_predictions,train_set_y)\n",
    "print metrics.accuracy_score(test_predictions,test_set_y)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "tmp = pd.DataFrame()\n",
    "tmp[\"predictions\"] = test_predictions\n",
    "tmp[\"targets\"] = test_set_y\n",
    "\n",
    "tmp.hist(column=\"predictions\",by=\"targets\",sharey=True)\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "test_predictions = clf_logistic.predict_proba(test_set_x)\n",
    "\n",
    "tmp = pd.DataFrame()\n",
    "tmp[\"predictions\"] = test_predictions[:,1]\n",
    "tmp[\"targets\"] = test_set_y\n",
    "\n",
    "tmp.hist(column=\"predictions\",by=\"targets\",sharey=True)\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# 3. Model evaluation"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.model_selection import cross_val_score\n",
    "\n",
    "cv_scores = cross_val_score(clf_logistic,train_set_x,train_set_y,cv=5)\n",
    "print cv_scores\n",
    "print np.mean(cv_scores)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.model_selection import cross_val_predict\n",
    "from sklearn import metrics\n",
    "\n",
    "predicted = cross_val_predict(clf_logistic, train_set_x, train_set_y, cv=5)\n",
    "print metrics.accuracy_score(train_set_y, predicted) "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "tmp = pd.DataFrame()\n",
    "tmp[\"predictions\"] = predicted\n",
    "tmp[\"targets\"] = train_set_y\n",
    "\n",
    "tmp.hist(column=\"predictions\",by=\"targets\",sharey=True)\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "predicted = cross_val_predict(clf_logistic, train_set_x, train_set_y, cv=5, method=\"predict_proba\")\n",
    "print predicted"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "tmp = pd.DataFrame()\n",
    "tmp[\"predictions\"] = predicted[:,1]\n",
    "tmp[\"targets\"] = train_set_y\n",
    "\n",
    "tmp.hist(column=\"predictions\",by=\"targets\",sharey=True)\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.model_selection import KFold\n",
    "\n",
    "kf = KFold(n_splits=5)\n",
    "for train, test in kf.split(train_set_x):\n",
    "    clf_logistic.fit(train_set_x.iloc[train],train_set_y.iloc[train])\n",
    "    predicted = clf_logistic.predict(train_set_x.iloc[test])\n",
    "    print metrics.accuracy_score(train_set_y.iloc[test], predicted) "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.metrics import roc_curve\n",
    "\n",
    "predicted = cross_val_predict(clf_logistic, train_set_x, train_set_y, cv=5, method=\"predict_proba\")\n",
    "\n",
    "fpr, tpr, thresholds = roc_curve(train_set_y,predicted[:,1])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def plot_roc_curve(fpr, tpr, label=None):\n",
    "    plt.plot(fpr, tpr, linewidth=2, label=label)\n",
    "    plt.plot([0, 1], [0, 1], 'k--')\n",
    "    plt.axis([0, 1, 0, 1])\n",
    "    plt.xlabel('False Positive Rate', fontsize=16)\n",
    "    plt.ylabel('True Positive Rate', fontsize=16)\n",
    "\n",
    "plt.figure(figsize=(8, 6))\n",
    "plot_roc_curve(fpr, tpr)\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "print metrics.auc(fpr, tpr)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.metrics import precision_recall_curve\n",
    "\n",
    "precisions, recalls, thresholds = precision_recall_curve(train_set_y,predicted[:,1])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def plot_precision_vs_recall(precisions, recalls):\n",
    "    plt.plot(recalls, precisions, \"b-\", linewidth=2)\n",
    "    plt.xlabel(\"Recall\", fontsize=16)\n",
    "    plt.ylabel(\"Precision\", fontsize=16)\n",
    "    plt.axis([0, 1, 0, 1])\n",
    "\n",
    "plt.figure(figsize=(8, 6))\n",
    "plot_precision_vs_recall(precisions, recalls)\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "clf_logistic.fit(train_set_x,train_set_y)\n",
    "print clf_logistic.coef_"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#Exercise: perform 5-fold cross-validation on the smaller train-set below and plot the ROC\n",
    "train_set_x_small = train_set_x.iloc[:200]\n",
    "train_set_y_small = train_set_y.iloc[:200]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#Exercise: fit a logistic regression model on the full and the small train-set \n",
    "#          and plot a ROC curve for the predictions on the test-set\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# 4. Regularization"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.model_selection import GridSearchCV\n",
    "\n",
    "params = {'C':[0.001,0.01,0.1,1,10]}\n",
    "\n",
    "GSCV = GridSearchCV(clf_logistic, params,cv=5)\n",
    "GSCV.fit(train_set_x,train_set_y)\n",
    "\n",
    "print GSCV.best_score_\n",
    "print GSCV.best_estimator_"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#Exercise: find the optimal value for C for a lgostic regression model with penalty='l1'\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#Exercise: print the model-parameters for the best model obtained above\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#Exercise: how well does the best model perform on the test-set (compute AUC)?\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#Exercise: fit a logistic regression model withe peanlty='l1' and C=0.01\n",
    "#          and print the model parameters again.\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#Exercise: how wel does this model perform on the test-set (compute the AUC)?\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# 5. Linear SVM"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#Exercise: find the best value for C on the small train-set\n",
    "\n",
    "#http://scikit-learn.org/stable/modules/generated/sklearn.svm.SVC.html\n",
    "from sklearn import svm\n",
    "\n",
    "clf_svm = svm.SVC(kernel='linear')\n",
    "\n",
    "params = {'C':[0.001,0.01,0.1,1,10,100]}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#Exercise: what is the performance (accuracy) of the best model fitted on the full train-set on the test-set?\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#Exercise: what is the AUC performance on the test-set?\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# 6. Decision tree"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#Exercise: optimze a Decision tree on the train-set\n",
    "\n",
    "#http://scikit-learn.org/stable/modules/generated/sklearn.tree.DecisionTreeClassifier.html\n",
    "from sklearn.tree import DecisionTreeClassifier\n",
    "\n",
    "clf_DT = DecisionTreeClassifier()\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#Exercise: what is the performance (accuracy) of the best model fitted on the full train-set on the test-set?\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# 7. RBF SVM"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#Exercise: what is the performance (accuracy and AUC) of an optimized RBF SVM on the test set?\n",
    "#Tip: optimize on small train-set\n",
    "\n",
    "clf_svm = svm.SVC(kernel='rbf')\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#Exercise: plot the ROC curve for the test-set\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# 8. Bagging"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.ensemble import BaggingClassifier\n",
    "\n",
    "clf_logistic_bagged = BaggingClassifier(clf_logistic,n_estimators=50,max_samples=0.7)\n",
    "scores = cross_val_score(clf_logistic_bagged, train_set_x, train_set_y, cv=5)\n",
    "print scores\n",
    "print np.mean(scores)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "params_bagged = {'n_estimators':[10,30,50], 'max_samples':[0.4,0.6,0.8]} #make them increase max_depth\n",
    "\n",
    "GSCV = GridSearchCV(clf_logistic_bagged, params_bagged,cv=5)\n",
    "GSCV.fit(train_set_x,train_set_y)\n",
    "\n",
    "print GSCV.best_score_\n",
    "print GSCV.best_estimator_"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#Exercise: what is the effect of bagging on a Decision tree classifier?\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# 9. Random Forest"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#Exercise: what is the performance (accuracy and AUC) of an optimized Random Forest on the test set?\n",
    "#http://scikit-learn.org/stable/modules/generated/sklearn.ensemble.RandomForestClassifier.html\n",
    "from sklearn.ensemble import RandomForestClassifier\n",
    "from sklearn.model_selection import RandomizedSearchCV\n",
    "from scipy.stats import randint \n",
    "from scipy.stats import uniform \n",
    "\n",
    "clf_RF = RandomForestClassifier()\n",
    "\n",
    "params_dist_RF = {\n",
    "                \"max_features\": uniform(0.1,0.6),\n",
    "                \"min_samples_split\": randint(2, 100),\n",
    "                \"n_estimators\": randint(10, 100)\n",
    "                }\n",
    "\n",
    "n_iter_search = 50\n",
    "\n",
    "RSCV = RandomizedSearchCV(clf_RF, params_dist_RF,cv=5,n_jobs=-1,n_iter=n_iter_search)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "print RSCV.best_estimator_.feature_importances_"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# 10. Extreme Gradient Boosting"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "hideCode": false,
    "hidePrompt": false
   },
   "outputs": [],
   "source": [
    "#conda install py-xgboost\n",
    "#Exercise: what is the performance (accuracy and AUC) of an optimized XGBoost on the test set?\n",
    "from xgboost import XGBClassifier \n",
    "\n",
    "clf_xgb = XGBClassifier() \n"
   ]
  }
 ],
 "metadata": {
  "hide_code_all_hidden": false,
  "kernelspec": {
   "display_name": "Python 2",
   "language": "python",
   "name": "python2"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 2
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython2",
   "version": "2.7.13"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}