Lasso Ridge.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "import numpy as np\n",
    "import matplotlib \n",
    "import matplotlib.pyplot as plt\n",
    "from mpl_toolkits.mplot3d import Axes3D\n",
    "import matplotlib.cm as cmx\n",
    "import matplotlib.colors as colors\n",
    "import pandas as pd\n",
    "%matplotlib inline"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "# Read in predictors\n",
    "x_df = pd.read_csv('datasets/predictors_filled.csv')\n",
    "\n",
    "# read in disease rates\n",
    "diabetes_df = pd.read_csv('datasets/diabetes_df.csv',index_col = 0)\n",
    "cardio_df = pd.read_csv('datasets/cardio_df.csv',index_col = 0)\n",
    "cancer_df= pd.read_csv('datasets/cancer_df.csv',index_col = 0)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 84,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def ridge_k_fold_r_squared(x_train, y_train, num_folds, param_val):\n",
    "    n_train = x_train.shape[0]\n",
    "    n = int(np.round(n_train * 1. / num_folds)) # points per fold\n",
    "\n",
    "    # Iterate over folds\n",
    "    cv_r_squared = 0\n",
    "    \n",
    "    for fold in range(1, num_folds + 1):\n",
    "        # Take k-1 folds for training \n",
    "        x_first_half = x_train.iloc[:n * (fold - 1), :]\n",
    "        x_second_half = x_train.iloc[n * fold + 1:, :]\n",
    "        x_train_cv = np.concatenate((x_first_half, x_second_half), axis=0)\n",
    "        \n",
    "        y_first_half = y_train.iloc[:n * (fold - 1)]\n",
    "        y_second_half = y_train.iloc[n * fold + 1:]\n",
    "        y_train_cv = np.concatenate((y_first_half, y_second_half), axis=0)\n",
    "        \n",
    "        # Take the middle fold for testing\n",
    "        x_test_cv = x_train.iloc[1 + n * (fold - 1):n * fold, :]\n",
    "        y_test_cv = y_train.iloc[1 + n * (fold - 1):n * fold]\n",
    "\n",
    "        # Fit Decision Tree model with parameter value on CV train set, and evaluate CV test performance\n",
    "        reg = Ridge(alpha = param_val, normalize=True)\n",
    "        reg.fit(x_train_cv, y_train_cv)\n",
    "        r_squared = reg.score(x_test_cv, y_test_cv)\n",
    "    \n",
    "        # Cummulative R^2 value across folds\n",
    "        cv_r_squared += r_squared\n",
    "\n",
    "    # Return average R^2 value across folds\n",
    "    return cv_r_squared * 1.0 / num_folds"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 115,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "def lasso_k_fold_r_squared(x_train, y_train, num_folds, param_val):\n",
    "    n_train = x_train.shape[0]\n",
    "    n = int(np.round(n_train * 1. / num_folds)) # points per fold\n",
    "\n",
    "    # Iterate over folds\n",
    "    cv_r_squared = 0\n",
    "    \n",
    "    for fold in range(1, num_folds + 1):\n",
    "        # Take k-1 folds for training \n",
    "        x_first_half = x_train.iloc[:n * (fold - 1), :]\n",
    "        x_second_half = x_train.iloc[n * fold + 1:, :]\n",
    "        x_train_cv = np.concatenate((x_first_half, x_second_half), axis=0)\n",
    "        \n",
    "        y_first_half = y_train.iloc[:n * (fold - 1)]\n",
    "        y_second_half = y_train.iloc[n * fold + 1:]\n",
    "        y_train_cv = np.concatenate((y_first_half, y_second_half), axis=0)\n",
    "        \n",
    "        # Take the middle fold for testing\n",
    "        x_test_cv = x_train.iloc[1 + n * (fold - 1):n * fold, :]\n",
    "        y_test_cv = y_train.iloc[1 + n * (fold - 1):n * fold]\n",
    "\n",
    "        # Fit Decision Tree model with parameter value on CV train set, and evaluate CV test performance\n",
    "        reg = Lasso(alpha = param_val, normalize=True)\n",
    "        reg.fit(x_train_cv, y_train_cv)\n",
    "        coefficients = reg.coef_\n",
    "        #print len([i for i, item in enumerate(coefficients) if abs(item) >0])\n",
    "        r_squared = reg.score(x_test_cv, y_test_cv)\n",
    "    \n",
    "        # Cummulative R^2 value across folds\n",
    "        cv_r_squared += r_squared\n",
    "\n",
    "    # Return average R^2 value across folds\n",
    "    return cv_r_squared * 1.0 / num_folds"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 116,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0.795989014205\n",
      "0.595066579537\n",
      "LASSO\n",
      "cancer 0.001 -8.74966599085\n",
      "cancer 0.01 -2.96325751054\n",
      "cancer 0.1 -0.190699091902\n",
      "cancer 1 0.059000481265\n",
      "cancer 3 -0.0667064097657\n",
      "cancer 5 -0.0667064097657\n",
      "cancer 10 -0.0667064097657\n",
      "cancer 11 -0.0667064097657\n",
      "cancer 12 -0.0667064097657\n",
      "cancer 13 -0.0667064097657\n",
      "cancer 14 -0.0667064097657\n",
      "cancer 15 -0.0667064097657\n",
      "cancer 16 -0.0667064097657\n",
      "cancer 17 -0.0667064097657\n",
      "cancer 18 -0.0667064097657\n",
      "cancer 19 -0.0667064097657\n",
      "cancer 20 -0.0667064097657\n",
      "cancer 25 -0.0667064097657\n",
      "cancer 100 -0.0667064097657\n",
      "cancer 200 -0.0667064097657\n",
      "cancer 500 -0.0667064097657\n",
      "cancer 1000 -0.0667064097657\n",
      "RIDGE\n",
      "cancer 0.001 -7.5846526988\n",
      "cancer 0.01 -3.04209158397\n",
      "cancer 0.1 -0.743910830295\n",
      "cancer 1 0.078166346437\n",
      "cancer 3 0.179617238894\n",
      "cancer 5 0.182894051905\n",
      "cancer 10 0.157128814375\n",
      "cancer 11 0.151574001586\n",
      "cancer 12 0.146178164685\n",
      "cancer 13 0.140966406052\n",
      "cancer 14 0.13594946692\n",
      "cancer 15 0.13112966807\n",
      "cancer 16 0.126504384414\n",
      "cancer 17 0.12206812724\n",
      "cancer 18 0.117813814989\n",
      "cancer 19 0.113733558338\n",
      "cancer 20 0.109819148041\n",
      "cancer 25 0.0924547448387\n",
      "cancer 100 -0.00297085121056\n",
      "cancer 200 -0.0313484074292\n",
      "cancer 500 -0.0515743165678\n",
      "cancer 1000 -0.0589604088889\n"
     ]
    }
   ],
   "source": [
    "from sklearn.linear_model import Lasso\n",
    "from sklearn.linear_model import LassoCV\n",
    "cancer_lasso_reg = Lasso(alpha = .01, normalize=True)\n",
    "cancer_lasso_reg.fit(x_df, cancer_df)\n",
    "print cancer_lasso_reg.score(x_df, cancer_df)\n",
    "\n",
    "cancer_lassoCV_reg = LassoCV(alphas = [.01, .1], normalize=True)\n",
    "cancer_lassoCV_reg.fit(x_df, cancer_df.values.ravel())\n",
    "print cancer_lassoCV_reg.score(x_df, cancer_df)\n",
    "\n",
    "print \"LASSO\"\n",
    "for alpha in [.001, .01, .1, 1, 3,5,10, 11,12, 13, 14, 15, 16, 17, 18, 19, 20, 25, 100, 200,500,1000]:\n",
    "    print \"cancer\",alpha, lasso_k_fold_r_squared(x_df,cancer_df,5, alpha)\n",
    "    \n",
    "print \"RIDGE\"\n",
    "\n",
    "for alpha in [.001, .01, .1, 1, 3, 5,10, 11,12, 13, 14, 15, 16, 17, 18, 19, 20, 25, 100, 200,500,1000]:\n",
    "    print \"cancer\",alpha, ridge_k_fold_r_squared(x_df,cancer_df,5, alpha)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.8042871716620823"
      ]
     },
     "execution_count": 28,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "from sklearn.linear_model import Ridge\n",
    "cancer_ridge = Ridge(alpha = .01, normalize=True)\n",
    "cancer_ridge.fit(x_df, cancer_df)\n",
    "cancer_ridge.score(x_df, cancer_df)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Cardio"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 114,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0.84267925017\n",
      "0.577876646041\n",
      "LASSO\n",
      "cardio 0.001 102\n",
      "101\n",
      "102\n",
      "99\n",
      "102\n",
      "-4.38550193519\n",
      "cardio 0.01 92\n",
      "93\n",
      "91\n",
      "96\n",
      "96\n",
      "-3.01117707597\n",
      "cardio 0.1 68\n",
      "68\n",
      "70\n",
      "66\n",
      "68\n",
      "-0.123533778277\n",
      "cardio 1 17\n",
      "20\n",
      "18\n",
      "20\n",
      "18\n",
      "0.45802637765\n",
      "cardio 10 0\n",
      "0\n",
      "0\n",
      "0\n",
      "0\n",
      "-0.0121108016136\n",
      "cardio 11 0\n",
      "0\n",
      "0\n",
      "0\n",
      "0\n",
      "-0.0121108016136\n",
      "cardio 13 0\n",
      "0\n",
      "0\n",
      "0\n",
      "0\n",
      "-0.0121108016136\n",
      "cardio 14 0\n",
      "0\n",
      "0\n",
      "0\n",
      "0\n",
      "-0.0121108016136\n",
      "cardio 15 0\n",
      "0\n",
      "0\n",
      "0\n",
      "0\n",
      "-0.0121108016136\n",
      "cardio 16 0\n",
      "0\n",
      "0\n",
      "0\n",
      "0\n",
      "-0.0121108016136\n",
      "cardio 17 0\n",
      "0\n",
      "0\n",
      "0\n",
      "0\n",
      "-0.0121108016136\n",
      "cardio 18 0\n",
      "0\n",
      "0\n",
      "0\n",
      "0\n",
      "-0.0121108016136\n",
      "cardio 19 0\n",
      "0\n",
      "0\n",
      "0\n",
      "0\n",
      "-0.0121108016136\n",
      "cardio 20 0\n",
      "0\n",
      "0\n",
      "0\n",
      "0\n",
      "-0.0121108016136\n",
      "cardio 25 0\n",
      "0\n",
      "0\n",
      "0\n",
      "0\n",
      "-0.0121108016136\n",
      "cardio 100 0\n",
      "0\n",
      "0\n",
      "0\n",
      "0\n",
      "-0.0121108016136\n",
      "cardio 1000 0\n",
      "0\n",
      "0\n",
      "0\n",
      "0\n",
      "-0.0121108016136\n",
      "RIDGE\n",
      "cardio 0.001 -3.96732782949\n",
      "cardio 0.01 -2.04659444672\n",
      "cardio 0.1 -0.0743876496175\n",
      "cardio 1 0.420913047131\n",
      "cardio 10 0.285504207166\n",
      "cardio 11 0.272129846453\n",
      "cardio 13 0.248595424699\n",
      "cardio 14 0.238207682288\n",
      "cardio 15 0.228608950052\n",
      "cardio 16 0.219716026242\n",
      "cardio 17 0.211456067161\n",
      "cardio 18 0.203765303377\n",
      "cardio 19 0.196587843237\n",
      "cardio 20 0.189874603945\n",
      "cardio 25 0.16192312357\n",
      "cardio 100 0.0449116184953\n",
      "cardio 1000 -0.00578240391232\n"
     ]
    }
   ],
   "source": [
    "cardio_reg = Lasso(alpha=.01, normalize=True)\n",
    "cardio_reg.fit(x_df, cardio_df)\n",
    "print cardio_reg.score(x_df, cardio_df)\n",
    "\n",
    "cardio_lassoCV_reg = LassoCV(alphas = [.001, .01, .1, 1], normalize=True)\n",
    "cardio_lassoCV_reg.fit(x_df, cardio_df.values.ravel())\n",
    "print cardio_lassoCV_reg.score(x_df, cardio_df)\n",
    "\n",
    "print \"LASSO\"\n",
    "for alpha in [.001, .01, .1, 1, 10, 11, 13, 14, 15, 16, 17, 18, 19, 20, 25, 100, 1000]:\n",
    "    print \"cardio\",alpha, lasso_k_fold_r_squared(x_df,cardio_df,5, alpha)\n",
    "    \n",
    "print \"RIDGE\"\n",
    "\n",
    "for alpha in [.001, .01, .1, 1, 10, 11, 13, 14, 15, 16, 17, 18, 19, 20, 25, 100, 1000]:\n",
    "    print \"cardio\",alpha, ridge_k_fold_r_squared(x_df,cardio_df,5, alpha)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Diabetes"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 98,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0.823730474505\n",
      "0.0\n",
      "diabetes 0.001 -5.5597290689\n",
      "diabetes 0.01 -2.50308003386\n",
      "diabetes 0.1 -0.259201378475\n",
      "diabetes 1 0.316918281716\n",
      "diabetes 10 0.234159110602\n",
      "diabetes 11 0.225550052854\n",
      "diabetes 13 0.210095567282\n",
      "diabetes 14 0.203127987416\n",
      "diabetes 15 0.196598615778\n",
      "diabetes 16 0.190464650311\n",
      "diabetes 17 0.184688825621\n",
      "diabetes 18 0.179238579694\n",
      "diabetes 19 0.174085345279\n",
      "diabetes 20 0.169203954677\n",
      "diabetes 25 0.148175512443\n",
      "diabetes 100 0.0430740924706\n",
      "diabetes 1000 -0.0145201142658\n"
     ]
    }
   ],
   "source": [
    "diabetes_reg = Lasso(alpha=.001, normalize=True)\n",
    "diabetes_reg.fit(x_df, diabetes_df)\n",
    "print diabetes_reg.score(x_df, diabetes_df)\n",
    "\n",
    "diabetes_lassoCV_reg = LassoCV(alphas = [.001, .01, .1, 1], normalize=True)\n",
    "diabetes_lassoCV_reg.fit(x_df, diabetes_df.values.ravel())\n",
    "print diabetes_lassoCV_reg.score(x_df, diabetes_df)\n",
    "\n",
    "for alpha in [.001, .01, .1, 1, 10, 11, 13, 14, 15, 16, 17, 18, 19, 20, 25, 100, 1000]:\n",
    "    print \"diabetes\",alpha, ridge_k_fold_r_squared(x_df,diabetes_df,4, alpha)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "def k_fold_r_squared(x_train, y_train, num_folds, param_val):\n",
    "    n_train = x_train.shape[0]\n",
    "    n = int(np.round(n_train * 1. / num_folds)) # points per fold\n",
    "\n",
    "    # Iterate over folds\n",
    "    cv_r_squared = 0\n",
    "    \n",
    "    for fold in range(1, num_folds + 1):\n",
    "        # Take k-1 folds for training \n",
    "        x_first_half = x_train.iloc[:n * (fold - 1), :]\n",
    "        x_second_half = x_train.iloc[n * fold + 1:, :]\n",
    "        x_train_cv = np.concatenate((x_first_half, x_second_half), axis=0)\n",
    "        \n",
    "        y_first_half = y_train.iloc[:n * (fold - 1)]\n",
    "        y_second_half = y_train.iloc[n * fold + 1:]\n",
    "        y_train_cv = np.concatenate((y_first_half, y_second_half), axis=0)\n",
    "        \n",
    "        # Take the middle fold for testing\n",
    "        x_test_cv = x_train.iloc[1 + n * (fold - 1):n * fold, :]\n",
    "        y_test_cv = y_train.iloc[1 + n * (fold - 1):n * fold]\n",
    "\n",
    "        # Fit Decision Tree model with parameter value on CV train set, and evaluate CV test performance\n",
    "        reg = DecisionTreeRegressor(max_depth=param_val)\n",
    "        reg.fit(x_train_cv, y_train_cv)\n",
    "        r_squared = reg.score(x_test_cv, y_test_cv)\n",
    "    \n",
    "        # Cummulative R^2 value across folds\n",
    "        cv_r_squared += r_squared\n",
    "\n",
    "    # Return average R^2 value across folds\n",
    "    return cv_r_squared * 1.0 / num_folds"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 64,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "2 0.481615672246\n",
      "3 0.680470026388\n",
      "4 0.791007766296\n",
      "5 0.872312891098\n",
      "6 0.918227247708\n",
      "7 0.950040031391\n",
      "8 0.976641337886\n",
      "9 0.993858789579\n",
      "10 0.999167360437\n",
      "20 1.0\n",
      "50 1.0\n",
      "70 1.0\n",
      "100 1.0\n"
     ]
    }
   ],
   "source": [
    "# Fit regression model\n",
    "for depth in [2, 3, 4, 5, 6, 7, 8, 9, 10, 20, 50, 70, 100]:\n",
    "    reg = DecisionTreeRegressor(max_depth=depth)\n",
    "    reg.fit(x_df, cancer_df)\n",
    "    print depth, reg.score(x_df, cancer_df)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 79,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "2 0.612719201546\n",
      "3 0.754302949283\n",
      "4 0.871314547554\n",
      "5 0.940479469018\n",
      "6 0.974736803109\n",
      "7 0.991617733605\n",
      "8 0.996587978435\n",
      "9 0.999219309243\n",
      "10 0.999922435105\n",
      "20 1.0\n",
      "50 1.0\n",
      "70 1.0\n",
      "100 1.0\n"
     ]
    }
   ],
   "source": [
    "# Fit regression model\n",
    "for depth in [2, 3, 4, 5, 6, 7, 8, 9, 10, 20, 50, 70, 100]:\n",
    "    reg = DecisionTreeRegressor(max_depth=depth)\n",
    "    reg.fit(x_df, cardio_df)\n",
    "    print depth, reg.score(x_df, cardio_df)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 78,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0.157263311113\n",
      "0.0383906647747\n",
      "-0.00942628271465\n",
      "-0.0593289349265\n",
      "-0.120257271122\n",
      "-0.22705646368\n",
      "-0.243135582395\n",
      "-0.351739864219\n",
      "-0.282642169159\n",
      "-0.431793222486\n",
      "-0.301257613118\n",
      "-0.376315252601\n",
      "-0.260990194789\n"
     ]
    }
   ],
   "source": [
    "for depth in [2, 3, 4, 5, 6, 7, 8, 9, 10, 20, 50, 70, 100]:\n",
    "    print k_fold_r_squared(x_df,diabetes_df,5, depth)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "ename": "NameError",
     "evalue": "name 'k_fold_r_squared' is not defined",
     "output_type": "error",
     "traceback": [
      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[0;31mNameError\u001b[0m                                 Traceback (most recent call last)",
      "\u001b[0;32m<ipython-input-24-df571e842646>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m()\u001b[0m\n\u001b[1;32m      1\u001b[0m \u001b[0;32mfor\u001b[0m \u001b[0mdepth\u001b[0m \u001b[0;32min\u001b[0m \u001b[0;34m[\u001b[0m\u001b[0;36m2\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;36m3\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;36m4\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;36m5\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;36m6\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;36m7\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;36m8\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;36m9\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;36m10\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;36m20\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;36m50\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;36m70\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;36m100\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m----> 2\u001b[0;31m     \u001b[0;32mprint\u001b[0m \u001b[0mk_fold_r_squared\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mx_df\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0mcardio_df\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;36m10\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mdepth\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m",
      "\u001b[0;31mNameError\u001b[0m: name 'k_fold_r_squared' is not defined"
     ]
    }
   ],
   "source": [
    "for depth in [2, 3, 4, 5, 6, 7, 8, 9, 10, 20, 50, 70, 100]:\n",
    "    print k_fold_r_squared(x_df,cardio_df,10, depth)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 77,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "-0.110776258483\n",
      "-0.385507423298\n",
      "-0.314279272363\n",
      "-0.738707373085\n",
      "-0.717019374631\n",
      "-0.893857314375\n",
      "-0.617305657296\n",
      "-0.830736099529\n",
      "-0.44166106094\n",
      "-0.753511958835\n",
      "-0.378120502515\n",
      "-0.820893373863\n",
      "-0.481570507449\n"
     ]
    }
   ],
   "source": [
    "for depth in [2, 3, 4, 5, 6, 7, 8, 9, 10, 20, 50, 70, 100]:\n",
    "    print k_fold_r_squared(x_df,cancer_df,5, depth)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 67,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "gridsearch = GridSearchCV(DecisionTreeRegressor(), {'max_depth': [2, 3, 4, 5, 6, 7, 8, 9, 10, 20, 50, 70]})"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 69,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "GridSearchCV(cv=None, error_score='raise',\n",
       "       estimator=DecisionTreeRegressor(criterion='mse', max_depth=None, max_features=None,\n",
       "           max_leaf_nodes=None, min_samples_leaf=1, min_samples_split=2,\n",
       "           min_weight_fraction_leaf=0.0, presort=False, random_state=None,\n",
       "           splitter='best'),\n",
       "       fit_params={}, iid=True, n_jobs=1,\n",
       "       param_grid={'max_depth': [2, 3, 4, 5, 6, 7, 8, 9, 10, 20, 50, 70]},\n",
       "       pre_dispatch='2*n_jobs', refit=True, scoring=None, verbose=0)"
      ]
     },
     "execution_count": 69,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "gridsearch.fit(x_df, cancer_df)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 70,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'max_depth': 2}"
      ]
     },
     "execution_count": 70,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "gridsearch.best_params_"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 71,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "-0.050020039633497548"
      ]
     },
     "execution_count": 71,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "gridsearch.best_score_"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "anaconda-cloud": {},
  "kernelspec": {
   "display_name": "Python [Root]",
   "language": "python",
   "name": "Python [Root]"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 2
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython2",
   "version": "2.7.12"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 0
}