29-split-data.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Import libraries"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Using TensorFlow backend.\n"
     ]
    }
   ],
   "source": [
    "import os\n",
    "os.environ[\"KERAS_BACKEND\"] = \"tensorflow\"\n",
    "import pandas as pd\n",
    "import numpy as np\n",
    "from IPython.display import SVG, display, Markdown\n",
    "from datetime import datetime\n",
    "from sklearn import preprocessing\n",
    "from sklearn.utils import shuffle\n",
    "from sklearn.neighbors import KNeighborsClassifier\n",
    "from keras.utils import np_utils\n",
    "from sklearn.model_selection import StratifiedShuffleSplit\n",
    "from sklearn.metrics import accuracy_score\n",
    "import pickle"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Load training dataset"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "train_dataframe = pd.read_csv('dataset/1. istanbul/train_data.csv')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Visualize some training data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "train_dataframe.head()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Prepare training data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "train_x = train_dataframe.drop(['Subject ID', 'Class information', 'UPDRS'], axis=1)\n",
    "train_x = train_x.as_matrix()\n",
    "print(train_x.shape)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Prepare training labels"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "train_y = train_dataframe['Class information']\n",
    "train_y = train_y.as_matrix()\n",
    "print(train_y.shape)\n",
    "train_y = np_utils.to_categorical(train_y)\n",
    "print(train_y.shape)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Prepare test data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "test_dataframe = pd.read_csv('dataset/1. istanbul/test_data.csv')\n",
    "test_x = test_dataframe.drop(['Subject ID', 'Class information'], axis=1)\n",
    "test_x = test_x.as_matrix()\n",
    "print(test_x.shape)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Prepare test labels"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "test_y = test_dataframe['Class information']\n",
    "test_y = test_y.as_matrix()\n",
    "print(test_y.shape)\n",
    "test_y = np_utils.to_categorical(test_y)\n",
    "print(test_y.shape)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Combine train and test set"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "x = np.vstack([train_x, test_x])\n",
    "y = np.vstack([train_y, test_y])\n",
    "print(x.shape)\n",
    "print(y.shape)\n",
    "pd.DataFrame(x).head()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Normalize data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "mins = np.min(x, axis=0)\n",
    "maxs = np.max(x, axis=0)\n",
    "rng = maxs - mins\n",
    "x = 1.0 - (((1.0 - 0.0) * (maxs - x)) / rng)\n",
    "pd.DataFrame(x).head()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Shuffle data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "x, y = shuffle(x, y)\n",
    "pd.DataFrame(y).head()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Split data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "sss = StratifiedShuffleSplit(n_splits=1, test_size=0.5, random_state=1)\n",
    "for train_index, test_index in sss.split(x, y):\n",
    "    X_train, X_test = x[train_index], x[test_index]\n",
    "    Y_train, Y_test = y[train_index], y[test_index]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Tuning parameters"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Folder paths to store outputs.\n",
    "root_directory = 'dataset/4. normalized-50/'\n",
    "pickle.dump(X_train, open(root_directory + 'x_train.p', \"wb\"))\n",
    "pickle.dump(X_test, open(root_directory + 'x_test.p', \"wb\"))\n",
    "pickle.dump(Y_train, open(root_directory + 'y_train.p', \"wb\"))\n",
    "pickle.dump(Y_test, open(root_directory + 'y_test.p', \"wb\"))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# The End"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python [default]",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.5.4"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}