10-grid-search-v3.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Import libraries"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Using TensorFlow backend.\n"
     ]
    }
   ],
   "source": [
    "import os\n",
    "os.environ[\"KERAS_BACKEND\"] = \"tensorflow\"\n",
    "import pandas as pd\n",
    "import numpy as np\n",
    "from keras.models import Sequential, Model\n",
    "from keras.layers import Dense, Activation, BatchNormalization, Dropout, Input\n",
    "from IPython.display import SVG\n",
    "from keras.utils.vis_utils import model_to_dot\n",
    "import keras\n",
    "from datetime import datetime\n",
    "from sklearn import preprocessing\n",
    "from keras.callbacks import ModelCheckpoint\n",
    "from keras.utils import np_utils\n",
    "from keras.wrappers.scikit_learn import KerasClassifier\n",
    "from keras.utils import plot_model\n",
    "from sklearn.model_selection import GridSearchCV\n",
    "from sklearn.utils import shuffle"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Load training dataset"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "train_dataframe = pd.read_csv('dataset/1. istanbul/train_data.csv')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Visualize some training data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Subject ID</th>\n",
       "      <th>Jitter (local)</th>\n",
       "      <th>Jitter (local, absolute)</th>\n",
       "      <th>Jitter (rap)</th>\n",
       "      <th>Jitter (ppq5)</th>\n",
       "      <th>Jitter (ddp)</th>\n",
       "      <th>Shimmer (local)</th>\n",
       "      <th>Shimmer (local, dB)</th>\n",
       "      <th>Shimmer (apq3)</th>\n",
       "      <th>Shimmer (apq5)</th>\n",
       "      <th>...</th>\n",
       "      <th>Maximum pitch</th>\n",
       "      <th>Number of pulses</th>\n",
       "      <th>Number of periods</th>\n",
       "      <th>Mean period</th>\n",
       "      <th>Standard deviation of period</th>\n",
       "      <th>Fraction of locally unvoiced frames</th>\n",
       "      <th>Number of voice breaks</th>\n",
       "      <th>Degree of voice breaks</th>\n",
       "      <th>UPDRS</th>\n",
       "      <th>Class information</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>1</td>\n",
       "      <td>1.488</td>\n",
       "      <td>0.000090</td>\n",
       "      <td>0.900</td>\n",
       "      <td>0.794</td>\n",
       "      <td>2.699</td>\n",
       "      <td>8.334</td>\n",
       "      <td>0.779</td>\n",
       "      <td>4.517</td>\n",
       "      <td>4.609</td>\n",
       "      <td>...</td>\n",
       "      <td>187.576</td>\n",
       "      <td>160</td>\n",
       "      <td>159</td>\n",
       "      <td>0.006065</td>\n",
       "      <td>0.000416</td>\n",
       "      <td>0.000</td>\n",
       "      <td>0</td>\n",
       "      <td>0.000</td>\n",
       "      <td>23</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>1</td>\n",
       "      <td>0.728</td>\n",
       "      <td>0.000038</td>\n",
       "      <td>0.353</td>\n",
       "      <td>0.376</td>\n",
       "      <td>1.059</td>\n",
       "      <td>5.864</td>\n",
       "      <td>0.642</td>\n",
       "      <td>2.058</td>\n",
       "      <td>3.180</td>\n",
       "      <td>...</td>\n",
       "      <td>234.505</td>\n",
       "      <td>170</td>\n",
       "      <td>169</td>\n",
       "      <td>0.005181</td>\n",
       "      <td>0.000403</td>\n",
       "      <td>2.247</td>\n",
       "      <td>0</td>\n",
       "      <td>0.000</td>\n",
       "      <td>23</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>1</td>\n",
       "      <td>1.220</td>\n",
       "      <td>0.000074</td>\n",
       "      <td>0.732</td>\n",
       "      <td>0.670</td>\n",
       "      <td>2.196</td>\n",
       "      <td>8.719</td>\n",
       "      <td>0.875</td>\n",
       "      <td>4.347</td>\n",
       "      <td>5.166</td>\n",
       "      <td>...</td>\n",
       "      <td>211.442</td>\n",
       "      <td>1431</td>\n",
       "      <td>1427</td>\n",
       "      <td>0.006071</td>\n",
       "      <td>0.000474</td>\n",
       "      <td>10.656</td>\n",
       "      <td>1</td>\n",
       "      <td>0.178</td>\n",
       "      <td>23</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>1</td>\n",
       "      <td>2.502</td>\n",
       "      <td>0.000123</td>\n",
       "      <td>1.156</td>\n",
       "      <td>1.634</td>\n",
       "      <td>3.469</td>\n",
       "      <td>13.513</td>\n",
       "      <td>1.273</td>\n",
       "      <td>5.263</td>\n",
       "      <td>8.771</td>\n",
       "      <td>...</td>\n",
       "      <td>220.230</td>\n",
       "      <td>94</td>\n",
       "      <td>92</td>\n",
       "      <td>0.004910</td>\n",
       "      <td>0.000320</td>\n",
       "      <td>0.000</td>\n",
       "      <td>0</td>\n",
       "      <td>0.000</td>\n",
       "      <td>23</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>1</td>\n",
       "      <td>3.509</td>\n",
       "      <td>0.000167</td>\n",
       "      <td>1.715</td>\n",
       "      <td>1.539</td>\n",
       "      <td>5.145</td>\n",
       "      <td>9.112</td>\n",
       "      <td>1.040</td>\n",
       "      <td>3.102</td>\n",
       "      <td>4.927</td>\n",
       "      <td>...</td>\n",
       "      <td>225.162</td>\n",
       "      <td>117</td>\n",
       "      <td>114</td>\n",
       "      <td>0.004757</td>\n",
       "      <td>0.000380</td>\n",
       "      <td>18.182</td>\n",
       "      <td>1</td>\n",
       "      <td>13.318</td>\n",
       "      <td>23</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>5 rows × 29 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "   Subject ID  Jitter (local)  Jitter (local, absolute)  Jitter (rap)  \\\n",
       "0           1           1.488                  0.000090         0.900   \n",
       "1           1           0.728                  0.000038         0.353   \n",
       "2           1           1.220                  0.000074         0.732   \n",
       "3           1           2.502                  0.000123         1.156   \n",
       "4           1           3.509                  0.000167         1.715   \n",
       "\n",
       "   Jitter (ppq5)  Jitter (ddp)  Shimmer (local)  Shimmer (local, dB)  \\\n",
       "0          0.794         2.699            8.334                0.779   \n",
       "1          0.376         1.059            5.864                0.642   \n",
       "2          0.670         2.196            8.719                0.875   \n",
       "3          1.634         3.469           13.513                1.273   \n",
       "4          1.539         5.145            9.112                1.040   \n",
       "\n",
       "   Shimmer (apq3)  Shimmer (apq5)        ...          Maximum pitch  \\\n",
       "0           4.517           4.609        ...                187.576   \n",
       "1           2.058           3.180        ...                234.505   \n",
       "2           4.347           5.166        ...                211.442   \n",
       "3           5.263           8.771        ...                220.230   \n",
       "4           3.102           4.927        ...                225.162   \n",
       "\n",
       "   Number of pulses  Number of periods  Mean period  \\\n",
       "0               160                159     0.006065   \n",
       "1               170                169     0.005181   \n",
       "2              1431               1427     0.006071   \n",
       "3                94                 92     0.004910   \n",
       "4               117                114     0.004757   \n",
       "\n",
       "   Standard deviation of period  Fraction of locally unvoiced frames  \\\n",
       "0                      0.000416                                0.000   \n",
       "1                      0.000403                                2.247   \n",
       "2                      0.000474                               10.656   \n",
       "3                      0.000320                                0.000   \n",
       "4                      0.000380                               18.182   \n",
       "\n",
       "   Number of voice breaks  Degree of voice breaks  UPDRS  Class information  \n",
       "0                       0                   0.000     23                  1  \n",
       "1                       0                   0.000     23                  1  \n",
       "2                       1                   0.178     23                  1  \n",
       "3                       0                   0.000     23                  1  \n",
       "4                       1                  13.318     23                  1  \n",
       "\n",
       "[5 rows x 29 columns]"
      ]
     },
     "execution_count": 3,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "train_dataframe.head()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Prepare training data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(1040, 26)\n"
     ]
    }
   ],
   "source": [
    "train_x = train_dataframe.drop(['Subject ID', 'Class information', 'UPDRS'], axis=1)\n",
    "train_x = train_x.as_matrix()\n",
    "print(train_x.shape)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Prepare training labels"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(1040,)\n",
      "(1040, 2)\n"
     ]
    }
   ],
   "source": [
    "train_y = train_dataframe['Class information']\n",
    "train_y = train_y.as_matrix()\n",
    "print(train_y.shape)\n",
    "train_y = np_utils.to_categorical(train_y)\n",
    "print(train_y.shape)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Prepare test data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(168, 26)\n"
     ]
    }
   ],
   "source": [
    "test_dataframe = pd.read_csv('dataset/1. istanbul/test_data.csv')\n",
    "test_x = test_dataframe.drop(['Subject ID', 'Class information'], axis=1)\n",
    "test_x = test_x.as_matrix()\n",
    "print(test_x.shape)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Prepare test labels"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(168,)\n",
      "(168, 2)\n"
     ]
    }
   ],
   "source": [
    "test_y = test_dataframe['Class information']\n",
    "test_y = test_y.as_matrix()\n",
    "print(test_y.shape)\n",
    "test_y = np_utils.to_categorical(test_y)\n",
    "print(test_y.shape)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Combine train and test set"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(1208, 26)\n",
      "(1208, 2)\n"
     ]
    }
   ],
   "source": [
    "train_x = np.vstack([train_x, test_x])\n",
    "train_y = np.vstack([train_y, test_y])\n",
    "print(train_x.shape)\n",
    "print(train_y.shape)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Normalize data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>0</th>\n",
       "      <th>1</th>\n",
       "      <th>2</th>\n",
       "      <th>3</th>\n",
       "      <th>4</th>\n",
       "      <th>5</th>\n",
       "      <th>6</th>\n",
       "      <th>7</th>\n",
       "      <th>8</th>\n",
       "      <th>9</th>\n",
       "      <th>...</th>\n",
       "      <th>16</th>\n",
       "      <th>17</th>\n",
       "      <th>18</th>\n",
       "      <th>19</th>\n",
       "      <th>20</th>\n",
       "      <th>21</th>\n",
       "      <th>22</th>\n",
       "      <th>23</th>\n",
       "      <th>24</th>\n",
       "      <th>25</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>0.096023</td>\n",
       "      <td>0.110551</td>\n",
       "      <td>0.106928</td>\n",
       "      <td>0.054303</td>\n",
       "      <td>0.106923</td>\n",
       "      <td>0.186514</td>\n",
       "      <td>0.266893</td>\n",
       "      <td>0.162585</td>\n",
       "      <td>0.056981</td>\n",
       "      <td>0.142044</td>\n",
       "      <td>...</td>\n",
       "      <td>0.034343</td>\n",
       "      <td>0.193353</td>\n",
       "      <td>0.204351</td>\n",
       "      <td>0.107383</td>\n",
       "      <td>0.106783</td>\n",
       "      <td>0.378729</td>\n",
       "      <td>0.062423</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>0.042716</td>\n",
       "      <td>0.042501</td>\n",
       "      <td>0.038278</td>\n",
       "      <td>0.023294</td>\n",
       "      <td>0.038318</td>\n",
       "      <td>0.125260</td>\n",
       "      <td>0.215176</td>\n",
       "      <td>0.065922</td>\n",
       "      <td>0.037237</td>\n",
       "      <td>0.150903</td>\n",
       "      <td>...</td>\n",
       "      <td>0.049169</td>\n",
       "      <td>0.238354</td>\n",
       "      <td>0.295334</td>\n",
       "      <td>0.114094</td>\n",
       "      <td>0.113499</td>\n",
       "      <td>0.295624</td>\n",
       "      <td>0.060411</td>\n",
       "      <td>0.025488</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>0.077225</td>\n",
       "      <td>0.089595</td>\n",
       "      <td>0.085843</td>\n",
       "      <td>0.045104</td>\n",
       "      <td>0.085882</td>\n",
       "      <td>0.196062</td>\n",
       "      <td>0.303133</td>\n",
       "      <td>0.155902</td>\n",
       "      <td>0.064677</td>\n",
       "      <td>0.158903</td>\n",
       "      <td>...</td>\n",
       "      <td>0.043064</td>\n",
       "      <td>0.204329</td>\n",
       "      <td>0.250621</td>\n",
       "      <td>0.960403</td>\n",
       "      <td>0.958361</td>\n",
       "      <td>0.379296</td>\n",
       "      <td>0.071557</td>\n",
       "      <td>0.120874</td>\n",
       "      <td>0.083333</td>\n",
       "      <td>0.002575</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>0.167146</td>\n",
       "      <td>0.152809</td>\n",
       "      <td>0.139056</td>\n",
       "      <td>0.116617</td>\n",
       "      <td>0.139134</td>\n",
       "      <td>0.314949</td>\n",
       "      <td>0.453379</td>\n",
       "      <td>0.191910</td>\n",
       "      <td>0.114487</td>\n",
       "      <td>0.367528</td>\n",
       "      <td>...</td>\n",
       "      <td>0.035815</td>\n",
       "      <td>0.298746</td>\n",
       "      <td>0.267658</td>\n",
       "      <td>0.063087</td>\n",
       "      <td>0.061786</td>\n",
       "      <td>0.270109</td>\n",
       "      <td>0.047233</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>0.237778</td>\n",
       "      <td>0.209959</td>\n",
       "      <td>0.209212</td>\n",
       "      <td>0.109570</td>\n",
       "      <td>0.209245</td>\n",
       "      <td>0.205808</td>\n",
       "      <td>0.365421</td>\n",
       "      <td>0.106962</td>\n",
       "      <td>0.061375</td>\n",
       "      <td>0.278121</td>\n",
       "      <td>...</td>\n",
       "      <td>0.038016</td>\n",
       "      <td>0.299027</td>\n",
       "      <td>0.277220</td>\n",
       "      <td>0.078523</td>\n",
       "      <td>0.076561</td>\n",
       "      <td>0.255738</td>\n",
       "      <td>0.056711</td>\n",
       "      <td>0.206243</td>\n",
       "      <td>0.083333</td>\n",
       "      <td>0.192688</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>5 rows × 26 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "         0         1         2         3         4         5         6   \\\n",
       "0  0.096023  0.110551  0.106928  0.054303  0.106923  0.186514  0.266893   \n",
       "1  0.042716  0.042501  0.038278  0.023294  0.038318  0.125260  0.215176   \n",
       "2  0.077225  0.089595  0.085843  0.045104  0.085882  0.196062  0.303133   \n",
       "3  0.167146  0.152809  0.139056  0.116617  0.139134  0.314949  0.453379   \n",
       "4  0.237778  0.209959  0.209212  0.109570  0.209245  0.205808  0.365421   \n",
       "\n",
       "         7         8         9     ...           16        17        18  \\\n",
       "0  0.162585  0.056981  0.142044    ...     0.034343  0.193353  0.204351   \n",
       "1  0.065922  0.037237  0.150903    ...     0.049169  0.238354  0.295334   \n",
       "2  0.155902  0.064677  0.158903    ...     0.043064  0.204329  0.250621   \n",
       "3  0.191910  0.114487  0.367528    ...     0.035815  0.298746  0.267658   \n",
       "4  0.106962  0.061375  0.278121    ...     0.038016  0.299027  0.277220   \n",
       "\n",
       "         19        20        21        22        23        24        25  \n",
       "0  0.107383  0.106783  0.378729  0.062423  0.000000  0.000000  0.000000  \n",
       "1  0.114094  0.113499  0.295624  0.060411  0.025488  0.000000  0.000000  \n",
       "2  0.960403  0.958361  0.379296  0.071557  0.120874  0.083333  0.002575  \n",
       "3  0.063087  0.061786  0.270109  0.047233  0.000000  0.000000  0.000000  \n",
       "4  0.078523  0.076561  0.255738  0.056711  0.206243  0.083333  0.192688  \n",
       "\n",
       "[5 rows x 26 columns]"
      ]
     },
     "execution_count": 9,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "mins = np.min(train_x, axis=0)\n",
    "maxs = np.max(train_x, axis=0)\n",
    "rng = maxs - mins\n",
    "train_x = 1.0 - (((1.0 - 0.0) * (maxs - train_x)) / rng)\n",
    "test_x = 1.0 - (((1.0 - 0.0) * (maxs - test_x)) / rng)\n",
    "pd.DataFrame(train_x).head()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Shuffle data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>0</th>\n",
       "      <th>1</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "     0    1\n",
       "0  1.0  0.0\n",
       "1  1.0  0.0\n",
       "2  0.0  1.0\n",
       "3  1.0  0.0\n",
       "4  0.0  1.0"
      ]
     },
     "execution_count": 10,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "train_x, train_y = shuffle(train_x, train_y)\n",
    "pd.DataFrame(train_y).head()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Tuning parameters"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [],
   "source": [
    "validation_split = 0.172;\n",
    "epochs = 10000"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Create model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [],
   "source": [
    "date = datetime.now().strftime(\"%Y-%m-%d-%H-%M-%S\")\n",
    "root_directory = 'results/results-gridsearch-03/'\n",
    "model_directory = root_directory + \"model/\"\n",
    "tensorboard_directory = root_directory + \"tensorboard/\"\n",
    "def create_model(param, dropout):\n",
    "    global date\n",
    "    x = Input(shape=(train_x.shape[1],))\n",
    "\n",
    "    y = Dense(units = param['layer_1'], activation='relu')(x)\n",
    "    y = Dropout(dropout)(y)\n",
    "\n",
    "    y = Dense(units = param['layer_2'], activation='relu')(y)\n",
    "    y = Dropout(dropout)(y)\n",
    "    \n",
    "    y = Dense(units = param['layer_3'], activation='relu')(y)\n",
    "    y = Dropout(dropout)(y)\n",
    "    \n",
    "    if (param['layer_4'] > 0):\n",
    "        y = Dense(units = param['layer_4'], activation='relu')(y)\n",
    "        y = Dropout(dropout)(y)\n",
    "\n",
    "    y = Dense(units = train_y.shape[1], activation='softmax')(y)\n",
    "    model = Model(x, y)\n",
    "    \n",
    "    # Create directory\n",
    "    directory = model_directory + date + '/'\n",
    "    if not os.path.exists(directory):\n",
    "        os.makedirs(directory)\n",
    "    \n",
    "    # Write model hyper-parameters\n",
    "    file = open(directory + \"params.txt\", \"a\")\n",
    "    file.write(\"optimizer: %s, layer 1: %d, layer 2: %d, layer 3: %d, layer 4: %d\" % (param['optimizer_text'], param['layer_1'], param['layer_2'], param['layer_3'], param['layer_4']))\n",
    "    file.close()\n",
    "    \n",
    "    # Write model summary\n",
    "    file2 = open(directory + \"summary.txt\", \"a\")\n",
    "    model.summary(print_fn=lambda x: file2.write(x + '\\n'))\n",
    "    file2.close()\n",
    "\n",
    "    # Write model diagram\n",
    "    plot_model(model, to_file=directory + 'model.png', show_shapes=True, show_layer_names=False)\n",
    "    \n",
    "    # Compile the model\n",
    "    model.compile(optimizer=param['optimizer'], loss=keras.losses.categorical_crossentropy, metrics=['accuracy'])\n",
    "    \n",
    "    return model"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Prepare callbacks"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [],
   "source": [
    "class KerasClassifierTensorBoard(KerasClassifier):\n",
    "    def fit(self, x, y, **kwargs):\n",
    "        global date\n",
    "        date = datetime.now().strftime(\"%Y-%m-%d-%H-%M-%S\")\n",
    "        tensorboard_callback = keras.callbacks.TensorBoard(log_dir=tensorboard_directory + date + '/')\n",
    "        csv_logger = keras.callbacks.CSVLogger(model_directory + date + '/epochs.csv')\n",
    "        callbacks = [tensorboard_callback, csv_logger]\n",
    "        return super(KerasClassifierTensorBoard, self).fit(x, y, callbacks=callbacks, **kwargs)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Grid search hyperparameters"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "1000/1000 [==============================] - 0s - loss: 0.6870 - acc: 0.5590 - val_loss: 0.6771 - val_acc: 0.6010\n",
      "Epoch 9475/10000\n",
      "1000/1000 [==============================] - 0s - loss: 0.6846 - acc: 0.5610 - val_loss: 0.6771 - val_acc: 0.6010\n",
      "Epoch 9476/10000\n",
      "1000/1000 [==============================] - 0s - loss: 0.6855 - acc: 0.5590 - val_loss: 0.6771 - val_acc: 0.6010\n",
      "Epoch 9477/10000\n",
      "1000/1000 [==============================] - 0s - loss: 0.6849 - acc: 0.5600 - val_loss: 0.6771 - val_acc: 0.6010\n",
      "Epoch 9478/10000\n",
      "1000/1000 [==============================] - 0s - loss: 0.6871 - acc: 0.5590 - val_loss: 0.6771 - val_acc: 0.6010\n",
      "Epoch 9479/10000\n",
      "1000/1000 [==============================] - 0s - loss: 0.6826 - acc: 0.5630 - val_loss: 0.6771 - val_acc: 0.6010\n",
      "Epoch 9480/10000\n",
      "1000/1000 [==============================] - 0s - loss: 0.6821 - acc: 0.5650 - val_loss: 0.6771 - val_acc: 0.6010\n",
      "Epoch 9481/10000\n",
      "1000/1000 [==============================] - 0s - loss: 0.6876 - acc: 0.5590 - val_loss: 0.6771 - val_acc: 0.6010\n",
      "Epoch 9482/10000\n",
      "1000/1000 [==============================] - 0s - loss: 0.6847 - acc: 0.5620 - val_loss: 0.6771 - val_acc: 0.60100.5\n",
      "Epoch 9483/10000\n",
      "1000/1000 [==============================] - 0s - loss: 0.6824 - acc: 0.5640 - val_loss: 0.6771 - val_acc: 0.6010\n",
      "Epoch 9484/10000\n",
      "1000/1000 [==============================] - 0s - loss: 0.6817 - acc: 0.5630 - val_loss: 0.6771 - val_acc: 0.6010\n",
      "Epoch 9485/10000\n",
      "1000/1000 [==============================] - 0s - loss: 0.6856 - acc: 0.5600 - val_loss: 0.6771 - val_acc: 0.6010\n",
      "Epoch 9486/10000\n",
      "1000/1000 [==============================] - 0s - loss: 0.6852 - acc: 0.5620 - val_loss: 0.6771 - val_acc: 0.6010\n",
      "Epoch 9487/10000\n",
      "1000/1000 [==============================] - 0s - loss: 0.6849 - acc: 0.5620 - val_loss: 0.6771 - val_acc: 0.6010\n",
      "Epoch 9488/10000\n",
      "1000/1000 [==============================] - 0s - loss: 0.6829 - acc: 0.5620 - val_loss: 0.6771 - val_acc: 0.6010\n",
      "Epoch 9489/10000\n",
      "1000/1000 [==============================] - 0s - loss: 0.6842 - acc: 0.5620 - val_loss: 0.6771 - val_acc: 0.6010\n",
      "Epoch 9490/10000\n",
      "1000/1000 [==============================] - 0s - loss: 0.6850 - acc: 0.5640 - val_loss: 0.6771 - val_acc: 0.6010\n",
      "Epoch 9491/10000\n",
      "1000/1000 [==============================] - 0s - loss: 0.6825 - acc: 0.5620 - val_loss: 0.6771 - val_acc: 0.6010\n",
      "Epoch 9492/10000\n",
      "1000/1000 [==============================] - 0s - loss: 0.6858 - acc: 0.5600 - val_loss: 0.6771 - val_acc: 0.6010\n",
      "Epoch 9493/10000\n",
      "1000/1000 [==============================] - 0s - loss: 0.6818 - acc: 0.5630 - val_loss: 0.6771 - val_acc: 0.6010\n",
      "Epoch 9494/10000\n",
      "1000/1000 [==============================] - 0s - loss: 0.6871 - acc: 0.5630 - val_loss: 0.6771 - val_acc: 0.6010\n",
      "Epoch 9495/10000\n",
      "1000/1000 [==============================] - 0s - loss: 0.6860 - acc: 0.5620 - val_loss: 0.6771 - val_acc: 0.6010\n",
      "Epoch 9496/10000\n",
      "1000/1000 [==============================] - 0s - loss: 0.6864 - acc: 0.5610 - val_loss: 0.6771 - val_acc: 0.6010\n",
      "Epoch 9497/10000\n",
      "1000/1000 [==============================] - 0s - loss: 0.6879 - acc: 0.5640 - val_loss: 0.6771 - val_acc: 0.6010\n",
      "Epoch 9498/10000\n",
      "1000/1000 [==============================] - 0s - loss: 0.6855 - acc: 0.5630 - val_loss: 0.6771 - val_acc: 0.6010\n",
      "Epoch 9499/10000\n",
      "1000/1000 [==============================] - 0s - loss: 0.6875 - acc: 0.5610 - val_loss: 0.6771 - val_acc: 0.6010\n",
      "Epoch 9500/10000\n",
      "1000/1000 [==============================] - 0s - loss: 0.6876 - acc: 0.5610 - val_loss: 0.6771 - val_acc: 0.6010\n",
      "Epoch 9501/10000\n",
      "1000/1000 [==============================] - 0s - loss: 0.6858 - acc: 0.5620 - val_loss: 0.6771 - val_acc: 0.6010\n",
      "Epoch 9502/10000\n",
      "1000/1000 [==============================] - 0s - loss: 0.6857 - acc: 0.5620 - val_loss: 0.6771 - val_acc: 0.6010\n",
      "Epoch 9503/10000\n",
      "1000/1000 [==============================] - 0s - loss: 0.6868 - acc: 0.5610 - val_loss: 0.6771 - val_acc: 0.6010\n",
      "Epoch 9504/10000\n",
      "1000/1000 [==============================] - 0s - loss: 0.6873 - acc: 0.5630 - val_loss: 0.6771 - val_acc: 0.6010\n",
      "Epoch 9505/10000\n",
      "1000/1000 [==============================] - 0s - loss: 0.6858 - acc: 0.5610 - val_loss: 0.6771 - val_acc: 0.6010\n",
      "Epoch 9506/10000\n",
      "1000/1000 [==============================] - 0s - loss: 0.6844 - acc: 0.5630 - val_loss: 0.6771 - val_acc: 0.6010\n",
      "Epoch 9507/10000\n",
      "1000/1000 [==============================] - 0s - loss: 0.6859 - acc: 0.5600 - val_loss: 0.6771 - val_acc: 0.6010\n",
      "Epoch 9508/10000\n",
      "1000/1000 [==============================] - 0s - loss: 0.6836 - acc: 0.5620 - val_loss: 0.6771 - val_acc: 0.6010\n",
      "Epoch 9509/10000\n",
      "1000/1000 [==============================] - 0s - loss: 0.6844 - acc: 0.5620 - val_loss: 0.6771 - val_acc: 0.6010\n",
      "Epoch 9510/10000\n",
      "1000/1000 [==============================] - 0s - loss: 0.6850 - acc: 0.5630 - val_loss: 0.6771 - val_acc: 0.6010\n",
      "Epoch 9511/10000\n",
      "1000/1000 [==============================] - 0s - loss: 0.6860 - acc: 0.5580 - val_loss: 0.6771 - val_acc: 0.6010\n",
      "Epoch 9512/10000\n",
      "1000/1000 [==============================] - 0s - loss: 0.6845 - acc: 0.5650 - val_loss: 0.6771 - val_acc: 0.6010\n",
      "Epoch 9513/10000\n",
      "1000/1000 [==============================] - 0s - loss: 0.6853 - acc: 0.5610 - val_loss: 0.6771 - val_acc: 0.6010\n",
      "Epoch 9514/10000\n",
      "1000/1000 [==============================] - 0s - loss: 0.6853 - acc: 0.5630 - val_loss: 0.6771 - val_acc: 0.6010\n",
      "Epoch 9515/10000\n",
      "1000/1000 [==============================] - 0s - loss: 0.6835 - acc: 0.5630 - val_loss: 0.6771 - val_acc: 0.6010\n",
      "Epoch 9516/10000\n",
      "1000/1000 [==============================] - 0s - loss: 0.6833 - acc: 0.5620 - val_loss: 0.6771 - val_acc: 0.6010\n",
      "Epoch 9517/10000\n",
      "1000/1000 [==============================] - 0s - loss: 0.6842 - acc: 0.5650 - val_loss: 0.6771 - val_acc: 0.6010\n",
      "Epoch 9518/10000\n",
      "1000/1000 [==============================] - 0s - loss: 0.6856 - acc: 0.5600 - val_loss: 0.6771 - val_acc: 0.6010\n",
      "Epoch 9519/10000\n",
      "1000/1000 [==============================] - 0s - loss: 0.6829 - acc: 0.5620 - val_loss: 0.6771 - val_acc: 0.6010\n",
      "Epoch 9520/10000\n",
      "1000/1000 [==============================] - 0s - loss: 0.6864 - acc: 0.5600 - val_loss: 0.6771 - val_acc: 0.6010\n",
      "Epoch 9521/10000\n",
      " 900/1000 [==========================>...] - ETA: 0s - loss: 0.6827 - acc: 0.5544"
     ]
    },
    {
     "ename": "KeyboardInterrupt",
     "evalue": "",
     "output_type": "error",
     "traceback": [
      "\u001b[1;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[1;31mKeyboardInterrupt\u001b[0m                         Traceback (most recent call last)",
      "\u001b[1;32m<ipython-input-14-a9bfb8b80fc5>\u001b[0m in \u001b[0;36m<module>\u001b[1;34m()\u001b[0m\n\u001b[0;32m     16\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m     17\u001b[0m \u001b[0mgrid\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mGridSearchCV\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mestimator\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mmodel\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mparam_grid\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mparam_grid\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0merror_score\u001b[0m\u001b[1;33m=\u001b[0m\u001b[1;36m0\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mverbose\u001b[0m\u001b[1;33m=\u001b[0m\u001b[1;36m2\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mcv\u001b[0m\u001b[1;33m=\u001b[0m\u001b[1;33m[\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mslice\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;32mNone\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mslice\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;32mNone\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m]\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mn_jobs\u001b[0m\u001b[1;33m=\u001b[0m\u001b[1;36m1\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mfit_params\u001b[0m\u001b[1;33m=\u001b[0m\u001b[0mdict\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mvalidation_split\u001b[0m\u001b[1;33m=\u001b[0m\u001b[0mvalidation_split\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m---> 18\u001b[1;33m \u001b[0mgrid_result\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mgrid\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mfit\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mtrain_x\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mtrain_y\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m",
      "\u001b[1;32m~\\AppData\\Roaming\\Python\\Python35\\site-packages\\sklearn\\model_selection\\_search.py\u001b[0m in \u001b[0;36mfit\u001b[1;34m(self, X, y, groups)\u001b[0m\n\u001b[0;32m    943\u001b[0m             \u001b[0mtrain\u001b[0m\u001b[1;33m/\u001b[0m\u001b[0mtest\u001b[0m \u001b[0mset\u001b[0m\u001b[1;33m.\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    944\u001b[0m         \"\"\"\n\u001b[1;32m--> 945\u001b[1;33m         \u001b[1;32mreturn\u001b[0m \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0m_fit\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mX\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0my\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mgroups\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mParameterGrid\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mparam_grid\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m    946\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    947\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32m~\\AppData\\Roaming\\Python\\Python35\\site-packages\\sklearn\\model_selection\\_search.py\u001b[0m in \u001b[0;36m_fit\u001b[1;34m(self, X, y, groups, parameter_iterable)\u001b[0m\n\u001b[0;32m    562\u001b[0m                                   \u001b[0mreturn_times\u001b[0m\u001b[1;33m=\u001b[0m\u001b[1;32mTrue\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mreturn_parameters\u001b[0m\u001b[1;33m=\u001b[0m\u001b[1;32mTrue\u001b[0m\u001b[1;33m,\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    563\u001b[0m                                   error_score=self.error_score)\n\u001b[1;32m--> 564\u001b[1;33m           \u001b[1;32mfor\u001b[0m \u001b[0mparameters\u001b[0m \u001b[1;32min\u001b[0m \u001b[0mparameter_iterable\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m    565\u001b[0m           for train, test in cv_iter)\n\u001b[0;32m    566\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32m~\\AppData\\Roaming\\Python\\Python35\\site-packages\\sklearn\\externals\\joblib\\parallel.py\u001b[0m in \u001b[0;36m__call__\u001b[1;34m(self, iterable)\u001b[0m\n\u001b[0;32m    756\u001b[0m             \u001b[1;31m# was dispatched. In particular this covers the edge\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    757\u001b[0m             \u001b[1;31m# case of Parallel used with an exhausted iterator.\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m--> 758\u001b[1;33m             \u001b[1;32mwhile\u001b[0m \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mdispatch_one_batch\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0miterator\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m    759\u001b[0m                 \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0m_iterating\u001b[0m \u001b[1;33m=\u001b[0m \u001b[1;32mTrue\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    760\u001b[0m             \u001b[1;32melse\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32m~\\AppData\\Roaming\\Python\\Python35\\site-packages\\sklearn\\externals\\joblib\\parallel.py\u001b[0m in \u001b[0;36mdispatch_one_batch\u001b[1;34m(self, iterator)\u001b[0m\n\u001b[0;32m    606\u001b[0m                 \u001b[1;32mreturn\u001b[0m \u001b[1;32mFalse\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    607\u001b[0m             \u001b[1;32melse\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m--> 608\u001b[1;33m                 \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0m_dispatch\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mtasks\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m    609\u001b[0m                 \u001b[1;32mreturn\u001b[0m \u001b[1;32mTrue\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    610\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32m~\\AppData\\Roaming\\Python\\Python35\\site-packages\\sklearn\\externals\\joblib\\parallel.py\u001b[0m in \u001b[0;36m_dispatch\u001b[1;34m(self, batch)\u001b[0m\n\u001b[0;32m    569\u001b[0m         \u001b[0mdispatch_timestamp\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mtime\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mtime\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    570\u001b[0m         \u001b[0mcb\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mBatchCompletionCallBack\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mdispatch_timestamp\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mlen\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mbatch\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mself\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m--> 571\u001b[1;33m         \u001b[0mjob\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0m_backend\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mapply_async\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mbatch\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mcallback\u001b[0m\u001b[1;33m=\u001b[0m\u001b[0mcb\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m    572\u001b[0m         \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0m_jobs\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mappend\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mjob\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    573\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32m~\\AppData\\Roaming\\Python\\Python35\\site-packages\\sklearn\\externals\\joblib\\_parallel_backends.py\u001b[0m in \u001b[0;36mapply_async\u001b[1;34m(self, func, callback)\u001b[0m\n\u001b[0;32m    107\u001b[0m     \u001b[1;32mdef\u001b[0m \u001b[0mapply_async\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mself\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mfunc\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mcallback\u001b[0m\u001b[1;33m=\u001b[0m\u001b[1;32mNone\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    108\u001b[0m         \u001b[1;34m\"\"\"Schedule a func to be run\"\"\"\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m--> 109\u001b[1;33m         \u001b[0mresult\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mImmediateResult\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mfunc\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m    110\u001b[0m         \u001b[1;32mif\u001b[0m \u001b[0mcallback\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    111\u001b[0m             \u001b[0mcallback\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mresult\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32m~\\AppData\\Roaming\\Python\\Python35\\site-packages\\sklearn\\externals\\joblib\\_parallel_backends.py\u001b[0m in \u001b[0;36m__init__\u001b[1;34m(self, batch)\u001b[0m\n\u001b[0;32m    324\u001b[0m         \u001b[1;31m# Don't delay the application, to avoid keeping the input\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    325\u001b[0m         \u001b[1;31m# arguments in memory\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m--> 326\u001b[1;33m         \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mresults\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mbatch\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m    327\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    328\u001b[0m     \u001b[1;32mdef\u001b[0m \u001b[0mget\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mself\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32m~\\AppData\\Roaming\\Python\\Python35\\site-packages\\sklearn\\externals\\joblib\\parallel.py\u001b[0m in \u001b[0;36m__call__\u001b[1;34m(self)\u001b[0m\n\u001b[0;32m    129\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    130\u001b[0m     \u001b[1;32mdef\u001b[0m \u001b[0m__call__\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mself\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m--> 131\u001b[1;33m         \u001b[1;32mreturn\u001b[0m \u001b[1;33m[\u001b[0m\u001b[0mfunc\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;33m*\u001b[0m\u001b[0margs\u001b[0m\u001b[1;33m,\u001b[0m \u001b[1;33m**\u001b[0m\u001b[0mkwargs\u001b[0m\u001b[1;33m)\u001b[0m \u001b[1;32mfor\u001b[0m \u001b[0mfunc\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0margs\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mkwargs\u001b[0m \u001b[1;32min\u001b[0m \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mitems\u001b[0m\u001b[1;33m]\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m    132\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    133\u001b[0m     \u001b[1;32mdef\u001b[0m \u001b[0m__len__\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mself\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32m~\\AppData\\Roaming\\Python\\Python35\\site-packages\\sklearn\\externals\\joblib\\parallel.py\u001b[0m in \u001b[0;36m<listcomp>\u001b[1;34m(.0)\u001b[0m\n\u001b[0;32m    129\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    130\u001b[0m     \u001b[1;32mdef\u001b[0m \u001b[0m__call__\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mself\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m--> 131\u001b[1;33m         \u001b[1;32mreturn\u001b[0m \u001b[1;33m[\u001b[0m\u001b[0mfunc\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;33m*\u001b[0m\u001b[0margs\u001b[0m\u001b[1;33m,\u001b[0m \u001b[1;33m**\u001b[0m\u001b[0mkwargs\u001b[0m\u001b[1;33m)\u001b[0m \u001b[1;32mfor\u001b[0m \u001b[0mfunc\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0margs\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mkwargs\u001b[0m \u001b[1;32min\u001b[0m \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mitems\u001b[0m\u001b[1;33m]\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m    132\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    133\u001b[0m     \u001b[1;32mdef\u001b[0m \u001b[0m__len__\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mself\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32m~\\AppData\\Roaming\\Python\\Python35\\site-packages\\sklearn\\model_selection\\_validation.py\u001b[0m in \u001b[0;36m_fit_and_score\u001b[1;34m(estimator, X, y, scorer, train, test, verbose, parameters, fit_params, return_train_score, return_parameters, return_n_test_samples, return_times, error_score)\u001b[0m\n\u001b[0;32m    236\u001b[0m             \u001b[0mestimator\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mfit\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mX_train\u001b[0m\u001b[1;33m,\u001b[0m \u001b[1;33m**\u001b[0m\u001b[0mfit_params\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    237\u001b[0m         \u001b[1;32melse\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m--> 238\u001b[1;33m             \u001b[0mestimator\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mfit\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mX_train\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0my_train\u001b[0m\u001b[1;33m,\u001b[0m \u001b[1;33m**\u001b[0m\u001b[0mfit_params\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m    239\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    240\u001b[0m     \u001b[1;32mexcept\u001b[0m \u001b[0mException\u001b[0m \u001b[1;32mas\u001b[0m \u001b[0me\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32m<ipython-input-13-8c94561eb6ef>\u001b[0m in \u001b[0;36mfit\u001b[1;34m(self, x, y, **kwargs)\u001b[0m\n\u001b[0;32m      6\u001b[0m         \u001b[0mcsv_logger\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mkeras\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mcallbacks\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mCSVLogger\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mmodel_directory\u001b[0m \u001b[1;33m+\u001b[0m \u001b[0mdate\u001b[0m \u001b[1;33m+\u001b[0m \u001b[1;34m'/epochs.csv'\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m      7\u001b[0m         \u001b[0mcallbacks\u001b[0m \u001b[1;33m=\u001b[0m \u001b[1;33m[\u001b[0m\u001b[0mtensorboard_callback\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mcsv_logger\u001b[0m\u001b[1;33m]\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m----> 8\u001b[1;33m         \u001b[1;32mreturn\u001b[0m \u001b[0msuper\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mKerasClassifierTensorBoard\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mself\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mfit\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mx\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0my\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mcallbacks\u001b[0m\u001b[1;33m=\u001b[0m\u001b[0mcallbacks\u001b[0m\u001b[1;33m,\u001b[0m \u001b[1;33m**\u001b[0m\u001b[0mkwargs\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m",
      "\u001b[1;32mC:\\ProgramData\\Anaconda3\\envs\\tensorflow\\lib\\site-packages\\keras\\wrappers\\scikit_learn.py\u001b[0m in \u001b[0;36mfit\u001b[1;34m(self, x, y, **kwargs)\u001b[0m\n\u001b[0;32m    201\u001b[0m             \u001b[1;32mraise\u001b[0m \u001b[0mValueError\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;34m'Invalid shape for y: '\u001b[0m \u001b[1;33m+\u001b[0m \u001b[0mstr\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0my\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mshape\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    202\u001b[0m         \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mn_classes_\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mlen\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mclasses_\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m--> 203\u001b[1;33m         \u001b[1;32mreturn\u001b[0m \u001b[0msuper\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mKerasClassifier\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mself\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mfit\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mx\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0my\u001b[0m\u001b[1;33m,\u001b[0m \u001b[1;33m**\u001b[0m\u001b[0mkwargs\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m    204\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    205\u001b[0m     \u001b[1;32mdef\u001b[0m \u001b[0mpredict\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mself\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mx\u001b[0m\u001b[1;33m,\u001b[0m \u001b[1;33m**\u001b[0m\u001b[0mkwargs\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32mC:\\ProgramData\\Anaconda3\\envs\\tensorflow\\lib\\site-packages\\keras\\wrappers\\scikit_learn.py\u001b[0m in \u001b[0;36mfit\u001b[1;34m(self, x, y, **kwargs)\u001b[0m\n\u001b[0;32m    145\u001b[0m         \u001b[0mfit_args\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mupdate\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mkwargs\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    146\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m--> 147\u001b[1;33m         \u001b[0mhistory\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mmodel\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mfit\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mx\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0my\u001b[0m\u001b[1;33m,\u001b[0m \u001b[1;33m**\u001b[0m\u001b[0mfit_args\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m    148\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    149\u001b[0m         \u001b[1;32mreturn\u001b[0m \u001b[0mhistory\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32mC:\\ProgramData\\Anaconda3\\envs\\tensorflow\\lib\\site-packages\\keras\\engine\\training.py\u001b[0m in \u001b[0;36mfit\u001b[1;34m(self, x, y, batch_size, epochs, verbose, callbacks, validation_split, validation_data, shuffle, class_weight, sample_weight, initial_epoch, **kwargs)\u001b[0m\n\u001b[0;32m   1428\u001b[0m                               \u001b[0mval_f\u001b[0m\u001b[1;33m=\u001b[0m\u001b[0mval_f\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mval_ins\u001b[0m\u001b[1;33m=\u001b[0m\u001b[0mval_ins\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mshuffle\u001b[0m\u001b[1;33m=\u001b[0m\u001b[0mshuffle\u001b[0m\u001b[1;33m,\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m   1429\u001b[0m                               \u001b[0mcallback_metrics\u001b[0m\u001b[1;33m=\u001b[0m\u001b[0mcallback_metrics\u001b[0m\u001b[1;33m,\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m-> 1430\u001b[1;33m                               initial_epoch=initial_epoch)\n\u001b[0m\u001b[0;32m   1431\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m   1432\u001b[0m     \u001b[1;32mdef\u001b[0m \u001b[0mevaluate\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mself\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mx\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0my\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mbatch_size\u001b[0m\u001b[1;33m=\u001b[0m\u001b[1;36m32\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mverbose\u001b[0m\u001b[1;33m=\u001b[0m\u001b[1;36m1\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0msample_weight\u001b[0m\u001b[1;33m=\u001b[0m\u001b[1;32mNone\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32mC:\\ProgramData\\Anaconda3\\envs\\tensorflow\\lib\\site-packages\\keras\\engine\\training.py\u001b[0m in \u001b[0;36m_fit_loop\u001b[1;34m(self, f, ins, out_labels, batch_size, epochs, verbose, callbacks, val_f, val_ins, shuffle, callback_metrics, initial_epoch)\u001b[0m\n\u001b[0;32m   1097\u001b[0m                         \u001b[1;32mfor\u001b[0m \u001b[0ml\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mo\u001b[0m \u001b[1;32min\u001b[0m \u001b[0mzip\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mout_labels\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mval_outs\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m   1098\u001b[0m                             \u001b[0mepoch_logs\u001b[0m\u001b[1;33m[\u001b[0m\u001b[1;34m'val_'\u001b[0m \u001b[1;33m+\u001b[0m \u001b[0ml\u001b[0m\u001b[1;33m]\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mo\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m-> 1099\u001b[1;33m             \u001b[0mcallbacks\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mon_epoch_end\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mepoch\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mepoch_logs\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m   1100\u001b[0m             \u001b[1;32mif\u001b[0m \u001b[0mcallback_model\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mstop_training\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m   1101\u001b[0m                 \u001b[1;32mbreak\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32mC:\\ProgramData\\Anaconda3\\envs\\tensorflow\\lib\\site-packages\\keras\\callbacks.py\u001b[0m in \u001b[0;36mon_epoch_end\u001b[1;34m(self, epoch, logs)\u001b[0m\n\u001b[0;32m     75\u001b[0m         \u001b[0mlogs\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mlogs\u001b[0m \u001b[1;32mor\u001b[0m \u001b[1;33m{\u001b[0m\u001b[1;33m}\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m     76\u001b[0m         \u001b[1;32mfor\u001b[0m \u001b[0mcallback\u001b[0m \u001b[1;32min\u001b[0m \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mcallbacks\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m---> 77\u001b[1;33m             \u001b[0mcallback\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mon_epoch_end\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mepoch\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mlogs\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m     78\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m     79\u001b[0m     \u001b[1;32mdef\u001b[0m \u001b[0mon_batch_begin\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mself\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mbatch\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mlogs\u001b[0m\u001b[1;33m=\u001b[0m\u001b[1;32mNone\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32mC:\\ProgramData\\Anaconda3\\envs\\tensorflow\\lib\\site-packages\\keras\\callbacks.py\u001b[0m in \u001b[0;36mon_epoch_end\u001b[1;34m(self, epoch, logs)\u001b[0m\n\u001b[0;32m    784\u001b[0m             \u001b[0msummary_value\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mtag\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mname\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    785\u001b[0m             \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mwriter\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0madd_summary\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0msummary\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mepoch\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m--> 786\u001b[1;33m         \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mwriter\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mflush\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m    787\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    788\u001b[0m     \u001b[1;32mdef\u001b[0m \u001b[0mon_train_end\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mself\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0m_\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32mC:\\ProgramData\\Anaconda3\\envs\\tensorflow\\lib\\site-packages\\tensorflow\\python\\summary\\writer\\writer.py\u001b[0m in \u001b[0;36mflush\u001b[1;34m(self)\u001b[0m\n\u001b[0;32m    355\u001b[0m     \u001b[0mdisk\u001b[0m\u001b[1;33m.\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    356\u001b[0m     \"\"\"\n\u001b[1;32m--> 357\u001b[1;33m     \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mevent_writer\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mflush\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m    358\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    359\u001b[0m   \u001b[1;32mdef\u001b[0m \u001b[0mclose\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mself\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32mC:\\ProgramData\\Anaconda3\\envs\\tensorflow\\lib\\site-packages\\tensorflow\\python\\summary\\writer\\event_file_writer.py\u001b[0m in \u001b[0;36mflush\u001b[1;34m(self)\u001b[0m\n\u001b[0;32m    117\u001b[0m     \"\"\"\n\u001b[0;32m    118\u001b[0m     \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0m_event_queue\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mjoin\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m--> 119\u001b[1;33m     \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0m_ev_writer\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mFlush\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m    120\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    121\u001b[0m   \u001b[1;32mdef\u001b[0m \u001b[0mclose\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mself\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;31mKeyboardInterrupt\u001b[0m: "
     ]
    }
   ],
   "source": [
    "model = KerasClassifierTensorBoard(build_fn=create_model, epochs=epochs, batch_size=20)\n",
    "\n",
    "param = [\n",
    "    {'optimizer': keras.optimizers.Adam(lr=0.0001, decay=0.0001), 'optimizer_text': 'Adam, lr=0.0001, decay=0.0001', 'layer_1': 4, 'layer_2': 8, 'layer_3': 16, 'layer_4': 0},\n",
    "    {'optimizer': keras.optimizers.Adam(lr=0.000001, decay=0.0001), 'optimizer_text': 'Adam, lr=0.000001, decay=0.0001', 'layer_1': 4, 'layer_2': 8, 'layer_3': 16, 'layer_4': 0},\n",
    "    {'optimizer': keras.optimizers.RMSprop(lr=0.0001, decay=0.0001), 'optimizer_text': 'RMSprop, lr=0.0001, decay=0.0001', 'layer_1': 4, 'layer_2': 8, 'layer_3': 16, 'layer_4': 0},\n",
    "    {'optimizer': keras.optimizers.Adam(lr=0.0001, decay=0.0001), 'optimizer_text': 'Adam, lr=0.0001, decay=0.0001', 'layer_1': 4, 'layer_2': 64, 'layer_3': 8, 'layer_4': 0},\n",
    "    {'optimizer': keras.optimizers.Adam(lr=0.0001, decay=0.0001), 'optimizer_text': 'Adam, lr=0.0001, decay=0.0001', 'layer_1': 6, 'layer_2': 6, 'layer_3': 32, 'layer_4': 0},\n",
    "    {'optimizer': keras.optimizers.Adam(lr=0.0001, decay=0.0001), 'optimizer_text': 'Adam, lr=0.0001, decay=0.0001', 'layer_1': 6, 'layer_2': 6, 'layer_3': 32, 'layer_4': 8},\n",
    "]\n",
    "dropouts = [0.5, 0.4, 0.6]\n",
    "param_grid = dict(\n",
    "    param=param, \n",
    "    dropout = dropouts\n",
    ")\n",
    "\n",
    "grid = GridSearchCV(estimator = model, param_grid = param_grid, error_score=0, verbose=2, cv=[(slice(None), slice(None))], n_jobs=1, fit_params=dict(validation_split=validation_split))\n",
    "grid_result = grid.fit(train_x, train_y)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Print results"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "print(\"Best: %f using %s\" % (grid_result.best_score_, grid_result.best_params_))\n",
    "means = grid_result.cv_results_['mean_test_score']\n",
    "stds = grid_result.cv_results_['std_test_score']\n",
    "params = grid_result.cv_results_['params']\n",
    "for mean, stdev, param in zip(means, stds, params):\n",
    "    print(\"Mean %f, Std %f with: %r\" % (mean, stdev, param))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# The End"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python [default]",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.5.4"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}