06-grid-search.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Import libraries"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Using TensorFlow backend.\n"
     ]
    }
   ],
   "source": [
    "import os\n",
    "os.environ[\"KERAS_BACKEND\"] = \"tensorflow\"\n",
    "import pandas as pd\n",
    "import numpy as np\n",
    "from keras.models import Sequential, Model\n",
    "from keras.layers import Dense, Activation, BatchNormalization, Dropout, Input\n",
    "from IPython.display import SVG\n",
    "from keras.utils.vis_utils import model_to_dot\n",
    "import keras\n",
    "from datetime import datetime\n",
    "from sklearn import preprocessing\n",
    "from keras.callbacks import ModelCheckpoint\n",
    "from keras.utils import np_utils\n",
    "from keras.wrappers.scikit_learn import KerasClassifier\n",
    "from keras.utils import plot_model\n",
    "from sklearn.model_selection import GridSearchCV\n",
    "from sklearn.utils import shuffle"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Load training dataset"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "train_dataframe = pd.read_csv('dataset/1. istanbul/train_data.csv')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Visualize some training data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Subject ID</th>\n",
       "      <th>Jitter (local)</th>\n",
       "      <th>Jitter (local, absolute)</th>\n",
       "      <th>Jitter (rap)</th>\n",
       "      <th>Jitter (ppq5)</th>\n",
       "      <th>Jitter (ddp)</th>\n",
       "      <th>Shimmer (local)</th>\n",
       "      <th>Shimmer (local, dB)</th>\n",
       "      <th>Shimmer (apq3)</th>\n",
       "      <th>Shimmer (apq5)</th>\n",
       "      <th>...</th>\n",
       "      <th>Maximum pitch</th>\n",
       "      <th>Number of pulses</th>\n",
       "      <th>Number of periods</th>\n",
       "      <th>Mean period</th>\n",
       "      <th>Standard deviation of period</th>\n",
       "      <th>Fraction of locally unvoiced frames</th>\n",
       "      <th>Number of voice breaks</th>\n",
       "      <th>Degree of voice breaks</th>\n",
       "      <th>UPDRS</th>\n",
       "      <th>Class information</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>1</td>\n",
       "      <td>1.488</td>\n",
       "      <td>0.000090</td>\n",
       "      <td>0.900</td>\n",
       "      <td>0.794</td>\n",
       "      <td>2.699</td>\n",
       "      <td>8.334</td>\n",
       "      <td>0.779</td>\n",
       "      <td>4.517</td>\n",
       "      <td>4.609</td>\n",
       "      <td>...</td>\n",
       "      <td>187.576</td>\n",
       "      <td>160</td>\n",
       "      <td>159</td>\n",
       "      <td>0.006065</td>\n",
       "      <td>0.000416</td>\n",
       "      <td>0.000</td>\n",
       "      <td>0</td>\n",
       "      <td>0.000</td>\n",
       "      <td>23</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>1</td>\n",
       "      <td>0.728</td>\n",
       "      <td>0.000038</td>\n",
       "      <td>0.353</td>\n",
       "      <td>0.376</td>\n",
       "      <td>1.059</td>\n",
       "      <td>5.864</td>\n",
       "      <td>0.642</td>\n",
       "      <td>2.058</td>\n",
       "      <td>3.180</td>\n",
       "      <td>...</td>\n",
       "      <td>234.505</td>\n",
       "      <td>170</td>\n",
       "      <td>169</td>\n",
       "      <td>0.005181</td>\n",
       "      <td>0.000403</td>\n",
       "      <td>2.247</td>\n",
       "      <td>0</td>\n",
       "      <td>0.000</td>\n",
       "      <td>23</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>1</td>\n",
       "      <td>1.220</td>\n",
       "      <td>0.000074</td>\n",
       "      <td>0.732</td>\n",
       "      <td>0.670</td>\n",
       "      <td>2.196</td>\n",
       "      <td>8.719</td>\n",
       "      <td>0.875</td>\n",
       "      <td>4.347</td>\n",
       "      <td>5.166</td>\n",
       "      <td>...</td>\n",
       "      <td>211.442</td>\n",
       "      <td>1431</td>\n",
       "      <td>1427</td>\n",
       "      <td>0.006071</td>\n",
       "      <td>0.000474</td>\n",
       "      <td>10.656</td>\n",
       "      <td>1</td>\n",
       "      <td>0.178</td>\n",
       "      <td>23</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>1</td>\n",
       "      <td>2.502</td>\n",
       "      <td>0.000123</td>\n",
       "      <td>1.156</td>\n",
       "      <td>1.634</td>\n",
       "      <td>3.469</td>\n",
       "      <td>13.513</td>\n",
       "      <td>1.273</td>\n",
       "      <td>5.263</td>\n",
       "      <td>8.771</td>\n",
       "      <td>...</td>\n",
       "      <td>220.230</td>\n",
       "      <td>94</td>\n",
       "      <td>92</td>\n",
       "      <td>0.004910</td>\n",
       "      <td>0.000320</td>\n",
       "      <td>0.000</td>\n",
       "      <td>0</td>\n",
       "      <td>0.000</td>\n",
       "      <td>23</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>1</td>\n",
       "      <td>3.509</td>\n",
       "      <td>0.000167</td>\n",
       "      <td>1.715</td>\n",
       "      <td>1.539</td>\n",
       "      <td>5.145</td>\n",
       "      <td>9.112</td>\n",
       "      <td>1.040</td>\n",
       "      <td>3.102</td>\n",
       "      <td>4.927</td>\n",
       "      <td>...</td>\n",
       "      <td>225.162</td>\n",
       "      <td>117</td>\n",
       "      <td>114</td>\n",
       "      <td>0.004757</td>\n",
       "      <td>0.000380</td>\n",
       "      <td>18.182</td>\n",
       "      <td>1</td>\n",
       "      <td>13.318</td>\n",
       "      <td>23</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>5 rows × 29 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "   Subject ID  Jitter (local)  Jitter (local, absolute)  Jitter (rap)  \\\n",
       "0           1           1.488                  0.000090         0.900   \n",
       "1           1           0.728                  0.000038         0.353   \n",
       "2           1           1.220                  0.000074         0.732   \n",
       "3           1           2.502                  0.000123         1.156   \n",
       "4           1           3.509                  0.000167         1.715   \n",
       "\n",
       "   Jitter (ppq5)  Jitter (ddp)  Shimmer (local)  Shimmer (local, dB)  \\\n",
       "0          0.794         2.699            8.334                0.779   \n",
       "1          0.376         1.059            5.864                0.642   \n",
       "2          0.670         2.196            8.719                0.875   \n",
       "3          1.634         3.469           13.513                1.273   \n",
       "4          1.539         5.145            9.112                1.040   \n",
       "\n",
       "   Shimmer (apq3)  Shimmer (apq5)        ...          Maximum pitch  \\\n",
       "0           4.517           4.609        ...                187.576   \n",
       "1           2.058           3.180        ...                234.505   \n",
       "2           4.347           5.166        ...                211.442   \n",
       "3           5.263           8.771        ...                220.230   \n",
       "4           3.102           4.927        ...                225.162   \n",
       "\n",
       "   Number of pulses  Number of periods  Mean period  \\\n",
       "0               160                159     0.006065   \n",
       "1               170                169     0.005181   \n",
       "2              1431               1427     0.006071   \n",
       "3                94                 92     0.004910   \n",
       "4               117                114     0.004757   \n",
       "\n",
       "   Standard deviation of period  Fraction of locally unvoiced frames  \\\n",
       "0                      0.000416                                0.000   \n",
       "1                      0.000403                                2.247   \n",
       "2                      0.000474                               10.656   \n",
       "3                      0.000320                                0.000   \n",
       "4                      0.000380                               18.182   \n",
       "\n",
       "   Number of voice breaks  Degree of voice breaks  UPDRS  Class information  \n",
       "0                       0                   0.000     23                  1  \n",
       "1                       0                   0.000     23                  1  \n",
       "2                       1                   0.178     23                  1  \n",
       "3                       0                   0.000     23                  1  \n",
       "4                       1                  13.318     23                  1  \n",
       "\n",
       "[5 rows x 29 columns]"
      ]
     },
     "execution_count": 3,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "train_dataframe.head()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Prepare training data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(1040, 26)\n"
     ]
    }
   ],
   "source": [
    "train_x = train_dataframe.drop(['Subject ID', 'Class information', 'UPDRS'], axis=1)\n",
    "train_x = train_x.as_matrix()\n",
    "print(train_x.shape)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Prepare training labels"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(1040,)\n",
      "(1040, 2)\n"
     ]
    }
   ],
   "source": [
    "train_y = train_dataframe['Class information']\n",
    "train_y = train_y.as_matrix()\n",
    "print(train_y.shape)\n",
    "train_y = np_utils.to_categorical(train_y)\n",
    "print(train_y.shape)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Prepare test data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(168, 26)\n"
     ]
    }
   ],
   "source": [
    "test_dataframe = pd.read_csv('dataset/1. istanbul/test_data.csv')\n",
    "test_x = test_dataframe.drop(['Subject ID', 'Class information'], axis=1)\n",
    "test_x = test_x.as_matrix()\n",
    "print(test_x.shape)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Prepare test labels"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(168,)\n",
      "(168, 2)\n"
     ]
    }
   ],
   "source": [
    "test_y = test_dataframe['Class information']\n",
    "test_y = test_y.as_matrix()\n",
    "print(test_y.shape)\n",
    "test_y = np_utils.to_categorical(test_y)\n",
    "print(test_y.shape)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Combine train and test set"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(1208, 26)\n",
      "(1208, 2)\n"
     ]
    }
   ],
   "source": [
    "train_x = np.vstack([train_x, test_x])\n",
    "train_y = np.vstack([train_y, test_y])\n",
    "print(train_x.shape)\n",
    "print(train_y.shape)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Normalize data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>0</th>\n",
       "      <th>1</th>\n",
       "      <th>2</th>\n",
       "      <th>3</th>\n",
       "      <th>4</th>\n",
       "      <th>5</th>\n",
       "      <th>6</th>\n",
       "      <th>7</th>\n",
       "      <th>8</th>\n",
       "      <th>9</th>\n",
       "      <th>...</th>\n",
       "      <th>16</th>\n",
       "      <th>17</th>\n",
       "      <th>18</th>\n",
       "      <th>19</th>\n",
       "      <th>20</th>\n",
       "      <th>21</th>\n",
       "      <th>22</th>\n",
       "      <th>23</th>\n",
       "      <th>24</th>\n",
       "      <th>25</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>0.096023</td>\n",
       "      <td>0.110551</td>\n",
       "      <td>0.106928</td>\n",
       "      <td>0.054303</td>\n",
       "      <td>0.106923</td>\n",
       "      <td>0.186514</td>\n",
       "      <td>0.266893</td>\n",
       "      <td>0.162585</td>\n",
       "      <td>0.056981</td>\n",
       "      <td>0.142044</td>\n",
       "      <td>...</td>\n",
       "      <td>0.034343</td>\n",
       "      <td>0.193353</td>\n",
       "      <td>0.204351</td>\n",
       "      <td>0.107383</td>\n",
       "      <td>0.106783</td>\n",
       "      <td>0.378729</td>\n",
       "      <td>0.062423</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>0.042716</td>\n",
       "      <td>0.042501</td>\n",
       "      <td>0.038278</td>\n",
       "      <td>0.023294</td>\n",
       "      <td>0.038318</td>\n",
       "      <td>0.125260</td>\n",
       "      <td>0.215176</td>\n",
       "      <td>0.065922</td>\n",
       "      <td>0.037237</td>\n",
       "      <td>0.150903</td>\n",
       "      <td>...</td>\n",
       "      <td>0.049169</td>\n",
       "      <td>0.238354</td>\n",
       "      <td>0.295334</td>\n",
       "      <td>0.114094</td>\n",
       "      <td>0.113499</td>\n",
       "      <td>0.295624</td>\n",
       "      <td>0.060411</td>\n",
       "      <td>0.025488</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>0.077225</td>\n",
       "      <td>0.089595</td>\n",
       "      <td>0.085843</td>\n",
       "      <td>0.045104</td>\n",
       "      <td>0.085882</td>\n",
       "      <td>0.196062</td>\n",
       "      <td>0.303133</td>\n",
       "      <td>0.155902</td>\n",
       "      <td>0.064677</td>\n",
       "      <td>0.158903</td>\n",
       "      <td>...</td>\n",
       "      <td>0.043064</td>\n",
       "      <td>0.204329</td>\n",
       "      <td>0.250621</td>\n",
       "      <td>0.960403</td>\n",
       "      <td>0.958361</td>\n",
       "      <td>0.379296</td>\n",
       "      <td>0.071557</td>\n",
       "      <td>0.120874</td>\n",
       "      <td>0.083333</td>\n",
       "      <td>0.002575</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>0.167146</td>\n",
       "      <td>0.152809</td>\n",
       "      <td>0.139056</td>\n",
       "      <td>0.116617</td>\n",
       "      <td>0.139134</td>\n",
       "      <td>0.314949</td>\n",
       "      <td>0.453379</td>\n",
       "      <td>0.191910</td>\n",
       "      <td>0.114487</td>\n",
       "      <td>0.367528</td>\n",
       "      <td>...</td>\n",
       "      <td>0.035815</td>\n",
       "      <td>0.298746</td>\n",
       "      <td>0.267658</td>\n",
       "      <td>0.063087</td>\n",
       "      <td>0.061786</td>\n",
       "      <td>0.270109</td>\n",
       "      <td>0.047233</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>0.237778</td>\n",
       "      <td>0.209959</td>\n",
       "      <td>0.209212</td>\n",
       "      <td>0.109570</td>\n",
       "      <td>0.209245</td>\n",
       "      <td>0.205808</td>\n",
       "      <td>0.365421</td>\n",
       "      <td>0.106962</td>\n",
       "      <td>0.061375</td>\n",
       "      <td>0.278121</td>\n",
       "      <td>...</td>\n",
       "      <td>0.038016</td>\n",
       "      <td>0.299027</td>\n",
       "      <td>0.277220</td>\n",
       "      <td>0.078523</td>\n",
       "      <td>0.076561</td>\n",
       "      <td>0.255738</td>\n",
       "      <td>0.056711</td>\n",
       "      <td>0.206243</td>\n",
       "      <td>0.083333</td>\n",
       "      <td>0.192688</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>5 rows × 26 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "         0         1         2         3         4         5         6   \\\n",
       "0  0.096023  0.110551  0.106928  0.054303  0.106923  0.186514  0.266893   \n",
       "1  0.042716  0.042501  0.038278  0.023294  0.038318  0.125260  0.215176   \n",
       "2  0.077225  0.089595  0.085843  0.045104  0.085882  0.196062  0.303133   \n",
       "3  0.167146  0.152809  0.139056  0.116617  0.139134  0.314949  0.453379   \n",
       "4  0.237778  0.209959  0.209212  0.109570  0.209245  0.205808  0.365421   \n",
       "\n",
       "         7         8         9     ...           16        17        18  \\\n",
       "0  0.162585  0.056981  0.142044    ...     0.034343  0.193353  0.204351   \n",
       "1  0.065922  0.037237  0.150903    ...     0.049169  0.238354  0.295334   \n",
       "2  0.155902  0.064677  0.158903    ...     0.043064  0.204329  0.250621   \n",
       "3  0.191910  0.114487  0.367528    ...     0.035815  0.298746  0.267658   \n",
       "4  0.106962  0.061375  0.278121    ...     0.038016  0.299027  0.277220   \n",
       "\n",
       "         19        20        21        22        23        24        25  \n",
       "0  0.107383  0.106783  0.378729  0.062423  0.000000  0.000000  0.000000  \n",
       "1  0.114094  0.113499  0.295624  0.060411  0.025488  0.000000  0.000000  \n",
       "2  0.960403  0.958361  0.379296  0.071557  0.120874  0.083333  0.002575  \n",
       "3  0.063087  0.061786  0.270109  0.047233  0.000000  0.000000  0.000000  \n",
       "4  0.078523  0.076561  0.255738  0.056711  0.206243  0.083333  0.192688  \n",
       "\n",
       "[5 rows x 26 columns]"
      ]
     },
     "execution_count": 9,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "mins = np.min(train_x, axis=0)\n",
    "maxs = np.max(train_x, axis=0)\n",
    "rng = maxs - mins\n",
    "train_x = 1.0 - (((1.0 - 0.0) * (maxs - train_x)) / rng)\n",
    "test_x = 1.0 - (((1.0 - 0.0) * (maxs - test_x)) / rng)\n",
    "pd.DataFrame(train_x).head()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Shuffle data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>0</th>\n",
       "      <th>1</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "     0    1\n",
       "0  0.0  1.0\n",
       "1  0.0  1.0\n",
       "2  0.0  1.0\n",
       "3  0.0  1.0\n",
       "4  1.0  0.0"
      ]
     },
     "execution_count": 10,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "train_x, train_y = shuffle(train_x, train_y)\n",
    "pd.DataFrame(train_y).head()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Create model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [],
   "source": [
    "date = datetime.now().strftime(\"%Y-%m-%d-%H-%M-%S\")\n",
    "root_directory = 'results/results-gridsearch/'\n",
    "model_directory = root_directory + \"model/\"\n",
    "tensorboard_directory = root_directory + \"tensorboard/\"\n",
    "validation_split = 0.172;\n",
    "def create_model(layer_1_units, layer_2_units, layer_3_units, optimizer):\n",
    "    global date\n",
    "    x = Input(shape=(train_x.shape[1],))\n",
    "\n",
    "    y = Dense(units = layer_1_units, activation='relu')(x)\n",
    "    y = Dropout(0.5)(y)\n",
    "\n",
    "    if layer_2_units > 0:\n",
    "        y = Dense(units = layer_2_units, activation='relu')(y)\n",
    "        y = Dropout(0.5)(y)\n",
    "        if layer_3_units > 0:\n",
    "            y = Dense(units = layer_3_units, activation='relu')(y)\n",
    "            y = Dropout(0.5)(y)\n",
    "        else:\n",
    "            if (layer_4_units > 0 or layer_5_units > 0):\n",
    "                raise ValueError('All layers after 0 sized layer 3 should have 0 units')\n",
    "    else:\n",
    "        if (layer_3_units > 0 or layer_4_units > 0 or layer_5_units > 0):\n",
    "            raise ValueError('All layers after 0 sized layer 2 should have 0 units')\n",
    "\n",
    "    y = Dense(units = train_y.shape[1], activation='softmax')(y)\n",
    "    model = Model(x, y)\n",
    "    \n",
    "    # Create directory\n",
    "    directory = model_directory + date + '/'\n",
    "    if not os.path.exists(directory):\n",
    "        os.makedirs(directory)\n",
    "    \n",
    "    # Write model hyper-parameters\n",
    "    file = open(directory + \"params.txt\", \"a\")\n",
    "    file.write(\"optimizer: %s, layer 1: %d, layer 2: %d, layer 3: %d, validation split: %f\" % (optimizer[0], layer_1_units, layer_2_units, layer_3_units, validation_split))\n",
    "    file.close()\n",
    "    \n",
    "    # Write model summary\n",
    "    file2 = open(directory + \"summary.txt\", \"a\")\n",
    "    model.summary(print_fn=lambda x: file2.write(x + '\\n'))\n",
    "    file2.close()\n",
    "\n",
    "    # Write model diagram\n",
    "    plot_model(model, to_file=directory + 'model.png', show_shapes=True, show_layer_names=False)\n",
    "    \n",
    "    # Compile the model\n",
    "    model.compile(optimizer=optimizer[1], loss=keras.losses.categorical_crossentropy, metrics=['accuracy'])\n",
    "    \n",
    "    return model"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Prepare callbacks"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [],
   "source": [
    "class KerasClassifierTensorBoard(KerasClassifier):\n",
    "    def fit(self, x, y, **kwargs):\n",
    "        global date\n",
    "        date = datetime.now().strftime(\"%Y-%m-%d-%H-%M-%S\")\n",
    "        tensorboard_callback = keras.callbacks.TensorBoard(log_dir=tensorboard_directory + date + '/')\n",
    "        csv_logger = keras.callbacks.CSVLogger(model_directory + date + '/epochs.csv')\n",
    "        callbacks = [tensorboard_callback, csv_logger]\n",
    "        return super(KerasClassifierTensorBoard, self).fit(x, y, callbacks=callbacks, **kwargs)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Grid search hyperparameters"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "model = KerasClassifierTensorBoard(build_fn=create_model, epochs=2500, batch_size=20)\n",
    "\n",
    "units_layer_1 = [4, 6, 8, 16, 32, 64, 128]\n",
    "units = [0, 4, 6, 8, 16, 32, 64, 128]\n",
    "optimizers = [\n",
    "    ['Adam(lr=0.001)', keras.optimizers.Adam(lr=0.001)],\n",
    "    ['Adam(lr=0.0001)', keras.optimizers.Adam(lr=0.0001)],\n",
    "    ['RMSprop(lr=0.001)', keras.optimizers.RMSprop(lr=0.001)],\n",
    "    ['RMSprop(lr=0.0001)', keras.optimizers.RMSprop(lr=0.0001)]\n",
    "]\n",
    "param_grid = dict(\n",
    "    layer_1_units=units_layer_1, \n",
    "    layer_2_units=units, \n",
    "    layer_3_units=units,\n",
    "    optimizer = optimizers\n",
    ")\n",
    "\n",
    "grid = GridSearchCV(estimator = model, param_grid = param_grid, error_score=0, verbose=2, cv=[(slice(None), slice(None))], n_jobs=1, fit_params=dict(validation_split=validation_split))\n",
    "grid_result = grid.fit(train_x, train_y)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Print results"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "print(\"Best: %f using %s\" % (grid_result.best_score_, grid_result.best_params_))\n",
    "means = grid_result.cv_results_['mean_test_score']\n",
    "stds = grid_result.cv_results_['std_test_score']\n",
    "params = grid_result.cv_results_['params']\n",
    "for mean, stdev, param in zip(means, stds, params):\n",
    "    print(\"Mean %f, Std %f with: %r\" % (mean, stdev, param))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# The End"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python [default]",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.5.4"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}