Riemannian Models, Exploration, Tuning 1.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 57,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Time Series methods - actually trying to use the direct time data or a processed version of it as input to models\n",
    "import mne\n",
    "import numpy as np\n",
    "import pandas as pd\n",
    "import sklearn\n",
    "from sklearn import svm \n",
    "from sklearn.svm import SVC \n",
    "from sklearn.svm import NuSVC\n",
    "from sklearn.pipeline import Pipeline\n",
    "from sklearn.metrics import accuracy_score\n",
    "from sklearn.pipeline import make_pipeline\n",
    "from sklearn.preprocessing import StandardScaler\n",
    "from sklearn.model_selection import train_test_split\n",
    "from sklearn.linear_model import LogisticRegression\n",
    "from sklearn.ensemble import VotingClassifier as VC\n",
    "from sklearn.neighbors import KNeighborsClassifier as sknn\n",
    "from sklearn.metrics import confusion_matrix as cm\n",
    "from sklearn.ensemble import StackingClassifier as SC\n",
    "\n",
    "from mne.externals.pymatreader import read_mat\n",
    "\n",
    "from pyriemann.utils.distance import distance_riemann\n",
    "from pyriemann.classification import MDM\n",
    "from pyriemann.classification import KNearestNeighbor as knn\n",
    "from pyriemann.classification import TSclassifier as tsc\n",
    "from pyriemann.classification import FgMDM\n",
    "from pyriemann.estimation import Covariances\n",
    "from pyriemann.estimation import (XdawnCovariances, HankelCovariances,\n",
    "                                  CospCovariances, ERPCovariances)\n",
    "from pyriemann.spatialfilters import Xdawn, CSP\n",
    "from pyriemann.tangentspace import TangentSpace\n",
    "\n",
    "\n",
    "import xgboost as xgb\n",
    "\n",
    "import matplotlib.pyplot as plt\n",
    "%matplotlib qt\n",
    "\n",
    "import joblib\n",
    "import os\n",
    "import time\n",
    "\n",
    "mne.set_log_level('ERROR')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Different inputs for the TSclassifier, maybe including the other Coherence methods\n",
    "from sklearn.base import BaseEstimator, TransformerMixin\n",
    "\n",
    "class EpochsVectorizer(BaseEstimator, TransformerMixin):\n",
    "    \"\"\"Vectorize epochs.\"\"\"\n",
    "\n",
    "    def __init__(self):\n",
    "        \"\"\"Init.\"\"\"\n",
    "\n",
    "    def fit(self, X, y):\n",
    "        return self\n",
    "\n",
    "    def transform(self, X):\n",
    "        X2 = np.array([x.flatten() for x in X])\n",
    "        return X2\n",
    "\n",
    "\n",
    "class CospBoostingClassifier(BaseEstimator, TransformerMixin):\n",
    "    \"\"\"Cospectral matrice bagging.\"\"\"\n",
    "\n",
    "    def __init__(self, baseclf):\n",
    "        \"\"\"Init.\"\"\"\n",
    "        self.baseclf = baseclf\n",
    "\n",
    "    def fit(self, X, y):\n",
    "        self.clfs_ = []\n",
    "        for i in range(X.shape[-1]):\n",
    "            clf = deepcopy(self.baseclf)\n",
    "            self.clfs_.append(clf.fit(X[:, :, :, i], y))\n",
    "        return self\n",
    "\n",
    "    def predict_proba(self, X):\n",
    "        proba = []\n",
    "        for i in range(X.shape[-1]):\n",
    "            proba.append(self.clfs_[i].predict_proba(X[:, :, :, i]))\n",
    "        proba = np.mean(proba, axis=0)\n",
    "        return proba\n",
    "\n",
    "    def transform(self, X):\n",
    "        proba = []\n",
    "        for i in range(X.shape[-1]):\n",
    "            proba.append(self.clfs_[i].predict_proba(X[:, :, :, i]))\n",
    "        proba = np.concatenate(proba, 1)\n",
    "        return proba"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "%run funcs.ipynb"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(447, 61, 500)\n",
      "(447,)\n",
      "['Fp1', 'Fz', 'F3', 'F7', 'FT9', 'FC5', 'FC1', 'C3', 'T7', 'CP5', 'CP1', 'Pz', 'P3', 'P7', 'O1', 'Oz', 'O2', 'P4', 'P8', 'CP6', 'CP2', 'FCz', 'C4', 'T8', 'FT8', 'FC6', 'FC2', 'F4', 'F8', 'Fp2', 'AF7', 'AF3', 'AFz', 'F1', 'F5', 'FT7', 'FC3', 'C1', 'C5', 'TP7', 'CP3', 'P1', 'P5', 'PO7', 'PO3', 'POz', 'PO4', 'PO8', 'P6', 'P2', 'CPz', 'CP4', 'TP8', 'C6', 'C2', 'FC4', 'FT10', 'F6', 'AF8', 'AF4', 'F2']\n"
     ]
    }
   ],
   "source": [
    "# Extract the X_train - 1st subject, 1st session data\n",
    "# get the epochs data of first subject first trial\n",
    "cwd = os.getcwd()\n",
    "data_path = cwd\n",
    "\n",
    "sub_n= 0\n",
    "session_n = 0\n",
    "\n",
    "# epochs = {}\n",
    "epochs_data = []\n",
    "labels = []\n",
    "i = 0\n",
    "chans = []\n",
    "diff = ['MATBeasy', 'MATBmed', 'MATBdiff']\n",
    "for lab_idx, level in enumerate(diff):\n",
    "    sub = 'P{0:02d}'.format(sub_n+1)\n",
    "    sess = f'S{session_n+1}'\n",
    "    path = os.path.join(os.path.join(data_path, sub), sess) + f'/eeg/alldata_sbj{str(sub_n+1).zfill(2)}_sess{session_n+1}_{level}.set'\n",
    "    # Read the epoched data with MNE\n",
    "    epochs = mne.io.read_epochs_eeglab(path, verbose=False)\n",
    "    if i == 0:\n",
    "        chans = epochs.ch_names\n",
    "        i = i+1\n",
    "    # You could add some pre-processing here with MNE\n",
    "    tmp = epochs.get_data()\n",
    "    epochs_data.extend(tmp)\n",
    "    labels.extend([lab_idx]*len(tmp))\n",
    "\n",
    "X1 = np.array(epochs_data)\n",
    "y1 = np.array(labels)\n",
    "print(X1.shape)\n",
    "print(y1.shape)\n",
    "print(chans) \n",
    "# Just use this as a reference - the features will generated as range(n_channels) \n",
    "# - Should be in this order\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(447, 61, 500)\n",
      "(447,)\n"
     ]
    }
   ],
   "source": [
    "# first subject second session\n",
    "cwd = os.getcwd()\n",
    "data_path = cwd\n",
    "\n",
    "sub_n= 0\n",
    "session_n = 1\n",
    "\n",
    "# epochs1 = {}\n",
    "epochs_data1 = []\n",
    "labels1 = []\n",
    "diff = ['MATBeasy', 'MATBmed', 'MATBdiff']\n",
    "for lab_idx, level in enumerate(diff):\n",
    "    sub = 'P{0:02d}'.format(sub_n+1)\n",
    "    sess = f'S{session_n+1}'\n",
    "    path = os.path.join(os.path.join(data_path, sub), sess) + f'/eeg/alldata_sbj{str(sub_n+1).zfill(2)}_sess{session_n+1}_{level}.set'\n",
    "    # Read the epoched data with MNE\n",
    "    epochs = mne.io.read_epochs_eeglab(path, verbose=False)\n",
    "    tmp = epochs.get_data()\n",
    "    epochs_data1.extend(tmp)\n",
    "    labels1.extend([lab_idx]*len(tmp))\n",
    "# \n",
    "X2 = np.array(epochs_data1)\n",
    "y2 = np.array(labels1)\n",
    "print(X2.shape)\n",
    "print(y2.shape)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Average feature-wise mean: 0.00000\n",
      "Average feature-wise standard deviation: 1.00000\n"
     ]
    }
   ],
   "source": [
    "# direct load and vectorize and give to LDA\n",
    "# vectorize\n",
    "vx1 = X1.reshape(X1.shape[0],-1)\n",
    "vx2 = X2.reshape(X2.shape[0],-1)\n",
    "\n",
    "scaler = StandardScaler()\n",
    "sx1 = scaler.fit_transform(vx1)\n",
    "sx2 = scaler.transform(vx2)\n",
    "\n",
    "print(f'Average feature-wise mean: {sx1.mean():0.5f}')\n",
    "print('Average feature-wise standard deviation: '\n",
    "      f'{sx1.std(axis=0).mean():0.5f}')\n",
    "# np.mean(sx1,axis=0)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "Accuracy of vect + log reg: 38.03%\n",
      "\n"
     ]
    }
   ],
   "source": [
    "log_reg = LogisticRegression(random_state=87,max_iter = 1000).fit(sx1, y1)\n",
    "test_acc = log_reg.score(sx2, y2)\n",
    "\n",
    "print(f'\\nAccuracy of vect + log reg: {test_acc * 100:0.2f}%\\n')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "Accuracy of vect + log reg: 36.02%\n",
      "\n"
     ]
    }
   ],
   "source": [
    "log_reg = LogisticRegression(random_state=87,max_iter = 1000).fit(sx2, y2)\n",
    "test_acc = log_reg.score(sx1, y1)\n",
    "\n",
    "print(f'\\nAccuracy of vect + log reg: {test_acc * 100:0.2f}%\\n')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Training set covariances have shape: (447, 61, 61)\n"
     ]
    }
   ],
   "source": [
    "# Riemannian Geometry method\n",
    "# remove DC offset - Already done in the dataset\n",
    "# print(X1[0,0,:])\n",
    "# nx1 = X1 - X1.mean(axis=-1, keepdims=True)\n",
    "# print(nx1[0,0,:])\n",
    "\n",
    "# create covariance matrix - transpose the CxT to TxC and multiply the matrices\n",
    "covs_train = (X1 @ X1.transpose(0, 2, 1)) / (\n",
    "    X1.shape[-1] - 1)\n",
    "print(f'Training set covariances have shape: {covs_train.shape}')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Viz the cov mat\n",
    "ind = 30\n",
    "# ch_names = [f'ch{i}' for i in range(1, 5)]\n",
    "max_ampl = np.abs(covs_train[ind]).max()  # find max abs(value) for plotting\n",
    "\n",
    "fig, ax = plt.subplots()\n",
    "im = ax.imshow(covs_train[ind], cmap='bwr', vmin=-max_ampl, vmax=max_ampl)\n",
    "ax.set_yticks(range(61))\n",
    "ax.set_yticklabels(chans)\n",
    "ax.set_xticks(range(61))\n",
    "ax.set_xticklabels(chans)\n",
    "fig.colorbar(im, ax=ax, fraction=0.05)\n",
    "ax.set_title('A covariance matrix\\nin our dataset')\n",
    "fig.tight_layout()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 44,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Euclidean distance between an example and class 0: 0.00\n",
      "Euclidean distance between an example and class 1: 0.00\n",
      "Euclidean distance between an example and class 2: 0.00\n",
      "Geometric distance between an example and class 0: 81.83\n",
      "Geometric distance between an example and class 1: 83.96\n",
      "Geometric distance between an example and class 2: 84.28\n"
     ]
    }
   ],
   "source": [
    "# Min distance to Mean - compare the covariance matrix of that window to the\n",
    "# average covariance matrix for each class :math:`\\Sigma_{class~k}` - whichever one\n",
    "# has the smallest distance is then used as label prediction\n",
    "\n",
    "class0_cov = covs_train[y1 == 0].mean(axis=0)\n",
    "class1_cov = covs_train[y1 == 1].mean(axis=0)\n",
    "class2_cov = covs_train[y1 == 2].mean(axis=0)\n",
    "\n",
    "eucl_dist0 = np.sum(np.abs(covs_train[0] - class0_cov))\n",
    "eucl_dist1 = np.sum(np.abs(covs_train[0] - class1_cov))\n",
    "eucl_dist2 = np.sum(np.abs(covs_train[0] - class2_cov))\n",
    "\n",
    "print(f'Euclidean distance between an example and class 0: {eucl_dist0:0.2f}')\n",
    "print(f'Euclidean distance between an example and class 1: {eucl_dist1:0.2f}')\n",
    "print(f'Euclidean distance between an example and class 2: {eucl_dist2:0.2f}')\n",
    "\n",
    "rd0 = distance_riemann(covs_train[0], class0_cov)\n",
    "rd1 = distance_riemann(covs_train[0], class1_cov)\n",
    "rd2 = distance_riemann(covs_train[0], class2_cov)\n",
    "\n",
    "print(f'Geometric distance between an example and class 0: {rd0:0.2f}')\n",
    "print(f'Geometric distance between an example and class 1: {rd1:0.2f}')\n",
    "print(f'Geometric distance between an example and class 2: {rd2:0.2f}')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Cal simple np covariance mats\n",
    "\n",
    "cov1 = Covariances('oas').fit_transform(X1)\n",
    "cov2 = Covariances('oas').fit_transform(X2)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(447, 3)\n"
     ]
    }
   ],
   "source": [
    "# Simple MDM based classifier\n",
    "\n",
    "mdmc = MDM(n_jobs=-1)\n",
    "mdmc.fit(cov1,y1)\n",
    "# test_acc = mdmc.score(cov2, y2)\n",
    "pm = mdmc.predict_proba(cov2)\n",
    "print(pm.shape)\n",
    "# print(y2)\n",
    "\n",
    "# print(f'\\nAccuracy of MDM Classifier: {test_acc * 100:0.2f}%\\n')\n",
    "\n",
    "# mdmc2 = MDM(n_jobs=-1)\n",
    "# mdmc2.fit(cov2,y2)\n",
    "# test_acc2 = mdmc2.score(cov1, y1)\n",
    "\n",
    "# print(f'\\nAccuracy of MDM Classifier 2: {test_acc2 * 100:0.2f}%\\n')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "Accuracy of FgMDM Classifier: 35.57%\n",
      "\n",
      "\n",
      "Accuracy of FgMDM Classifier 2: 33.33%\n",
      "\n"
     ]
    }
   ],
   "source": [
    "\n",
    "Fgmdmc = FgMDM(n_jobs=-1)\n",
    "Fgmdmc.fit(cov1,y1)\n",
    "# test_acc = Fgmdmc.score(cov2, y2)\n",
    "\n",
    "# print(f'\\nAccuracy of FgMDM Classifier: {test_acc * 100:0.2f}%\\n')\n",
    "pfgm = \n",
    "\n",
    "Fgmdmc2 = FgMDM(n_jobs=-1)\n",
    "Fgmdmc2.fit(cov2,y2)\n",
    "test_acc2 = Fgmdmc2.score(cov1, y1)\n",
    "\n",
    "print(f'\\nAccuracy of FgMDM Classifier 2: {test_acc2 * 100:0.2f}%\\n')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "training score  1.0\n",
      "test score  0.2751677852348993\n"
     ]
    }
   ],
   "source": [
    "Fgmdmc3 = FgMDM(n_jobs=-1,tsupdate=True)\n",
    "Fgmdmc3.fit(cov2,y2)\n",
    "# test_acc2 = Fgmdmc2.score(cov1, y1)\n",
    "print(\"training score \",Fgmdmc3.score(cov2,y2))\n",
    "print(\"test score \",Fgmdmc3.score(cov1,y1))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "Accuracy of tsc Classifier: 36.02%\n",
      "\n",
      "\n",
      "Accuracy of tsc Classifier 2: 40.27%\n",
      "\n"
     ]
    }
   ],
   "source": [
    "# Simply put, this projects to TS and applies LogReg\n",
    "tsc1 = tsc()\n",
    "tsc1.fit(cov1,y1)\n",
    "test_acc = tsc1.score(cov2, y2)\n",
    "\n",
    "print(f'\\nAccuracy of tsc Classifier: {test_acc * 100:0.2f}%\\n')\n",
    "\n",
    "tsc2 = tsc()\n",
    "tsc2.fit(cov2,y2)\n",
    "test_acc2 = tsc2.score(cov1, y1)\n",
    "\n",
    "print(f'\\nAccuracy of tsc Classifier 2: {test_acc2 * 100:0.2f}%\\n')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "Accuracy of tsc Classifier: 45.41%\n",
      "\n",
      "\n",
      "Accuracy of tsc Classifier 2: 52.57%\n",
      "\n"
     ]
    }
   ],
   "source": [
    "# This enables covariate shift based on test data.. Not applicable for online eval\n",
    "tsc1 = tsc(tsupdate=True)\n",
    "tsc1.fit(cov1,y1)\n",
    "test_acc = tsc1.score(cov2, y2)\n",
    "\n",
    "print(f'\\nAccuracy of tsc Classifier: {test_acc * 100:0.2f}%\\n')\n",
    "\n",
    "tsc2 = tsc(tsupdate=True)\n",
    "tsc2.fit(cov2,y2)\n",
    "test_acc2 = tsc2.score(cov1, y1)\n",
    "\n",
    "print(f'\\nAccuracy of tsc Classifier 2: {test_acc2 * 100:0.2f}%\\n')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "Accuracy of knn Classifier: 39.15%\n",
      "\n",
      "\n",
      "Accuracy of knn Classifier 2: 54.14%\n",
      "\n"
     ]
    }
   ],
   "source": [
    "\n",
    "knn1 = knn(n_jobs=-1)\n",
    "knn1.fit(cov1,y1)\n",
    "test_acc = knn1.score(cov2, y2)\n",
    "\n",
    "print(f'\\nAccuracy of knn Classifier: {test_acc * 100:0.2f}%\\n')\n",
    "\n",
    "knn2 = knn(n_jobs=-1)\n",
    "knn2.fit(cov2,y2)\n",
    "test_acc2 = knn2.score(cov1, y1)\n",
    "\n",
    "print(f'\\nAccuracy of knn Classifier 2: {test_acc2 * 100:0.2f}%\\n')\n",
    "\n",
    "# knn pyriemann doesn't have predict_proba for itself "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 44,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(447, 447)"
      ]
     },
     "execution_count": 44,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "knn2 = knn(n_jobs=-1)\n",
    "knn2.fit(cov2,y2)\n",
    "p = knn2.predict_proba(cov1)\n",
    "p.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 47,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(447, 61, 61)"
      ]
     },
     "execution_count": 47,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "cov2.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 46,
   "metadata": {},
   "outputs": [],
   "source": [
    "# y2\n",
    "knn3 = sknn(n_jobs=-1,metric = distance_riemann)\n",
    "knn3.fit(cov2,y2)\n",
    "print(\"sk knn train = \",knn3.score(cov2,y2),\" test = \",knn3.score(X1,y1))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {
    "collapsed": true,
    "jupyter": {
     "outputs_hidden": true
    },
    "tags": []
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "VotingClassifier(estimators=[('mdm', MDM(n_jobs=-1)),\n",
       "                             ('Fgmdm', FgMDM(n_jobs=-1)),\n",
       "                             ('tsc', TSclassifier(tsupdate=True))],\n",
       "                 n_jobs=-1, voting='soft')"
      ]
     },
     "execution_count": 14,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Now do a voting classifier based on these 4/5 (depending on tsupdate as second)\n",
    "\n",
    "\n",
    "mdmc = MDM(n_jobs=-1)\n",
    "Fgmdmc = FgMDM(n_jobs=-1)\n",
    "tsc1 = tsc(tsupdate=True)\n",
    "knn1 = knn(n_jobs=-1)\n",
    "\n",
    "svoc = VC(estimators = [('mdm',mdmc),('Fgmdm',Fgmdmc),('tsc',tsc1)], voting='soft',n_jobs=-1)\n",
    "svoc.fit(cov1,y1)\n",
    "# print(svoc.predict(cov1))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {
    "collapsed": true,
    "jupyter": {
     "outputs_hidden": true
    },
    "tags": []
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "training score  1.0\n",
      "test score  0.4519015659955257\n"
     ]
    }
   ],
   "source": [
    "print(\"training score \",svoc.score(cov1,y1))\n",
    "\n",
    "print(\"test score \",svoc.score(cov2,y2))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "mdmc = MDM(n_jobs=-1)\n",
    "Fgmdmc = FgMDM(n_jobs=-1)\n",
    "tsc1 = tsc(tsupdate=True)\n",
    "knn1 = knn(n_jobs=-1)\n",
    "\n",
    "svoc = VC(estimators = [('mdm',mdmc),('Fgmdm',Fgmdmc),('tsc',tsc1)], voting='soft',n_jobs=-1)\n",
    "svoc.fit(cov1,y1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {
    "collapsed": true,
    "jupyter": {
     "outputs_hidden": true
    },
    "tags": []
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "VotingClassifier(estimators=[('mdm', MDM(n_jobs=-1)),\n",
       "                             ('Fgmdm', FgMDM(n_jobs=-1)),\n",
       "                             ('tsc', TSclassifier(tsupdate=True))],\n",
       "                 n_jobs=-1, voting='soft')"
      ]
     },
     "execution_count": 21,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "svoc2 = VC(estimators = [('mdm',mdmc),('Fgmdm',Fgmdmc),('tsc',tsc1)], voting='soft',n_jobs=-1,flatten_transform = False)\n",
    "svoc2.fit(cov2,y2)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {
    "collapsed": true,
    "jupyter": {
     "outputs_hidden": true
    },
    "tags": []
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "training score  1.0\n",
      "test score  0.5480984340044742\n"
     ]
    }
   ],
   "source": [
    "print(\"training score \",svoc2.score(cov2,y2))\n",
    "\n",
    "print(\"test score \",svoc2.score(cov1,y1))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "import seaborn as sns\n",
    "\n",
    "# View the individual prediction to see if I can possibly weight some classifier accordingly\n",
    "p1 = svoc2.transform(cov1)\n",
    "p2 = svoc2.transform(cov2) #trained on\n",
    "\n",
    "%matplotlib qt\n",
    "sns.heatmap(p1)\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {
    "collapsed": true,
    "jupyter": {
     "outputs_hidden": true
    },
    "tags": []
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(447, 9)"
      ]
     },
     "execution_count": 25,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "p1.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "VotingClassifier(estimators=[('mdm', MDM(n_jobs=-1)),\n",
       "                             ('Fgmdm', FgMDM(n_jobs=-1)),\n",
       "                             ('tsc', TSclassifier(tsupdate=True))],\n",
       "                 n_jobs=-1)"
      ]
     },
     "execution_count": 26,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "svoc3 = VC(estimators = [('mdm',mdmc),('Fgmdm',Fgmdmc),('tsc',tsc1)], voting='hard',n_jobs=-1)\n",
    "svoc3.fit(cov2,y2)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "training score  1.0\n",
      "test score  0.5816554809843401\n"
     ]
    }
   ],
   "source": [
    "print(\"training score \",svoc3.score(cov2,y2))\n",
    "\n",
    "print(\"test score \",svoc3.score(cov1,y1))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {
    "collapsed": true,
    "jupyter": {
     "outputs_hidden": true
    },
    "tags": []
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "VotingClassifier(estimators=[('mdm', MDM(n_jobs=-1)),\n",
       "                             ('Fgmdm', FgMDM(n_jobs=-1)),\n",
       "                             ('tsc', TSclassifier())],\n",
       "                 n_jobs=-1)"
      ]
     },
     "execution_count": 28,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "tsc3 = tsc()\n",
    "svoc4 = VC(estimators = [('mdm',mdmc),('Fgmdm',Fgmdmc),('tsc',tsc3)], voting='hard',n_jobs=-1)\n",
    "svoc4.fit(cov2,y2)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "metadata": {
    "collapsed": true,
    "jupyter": {
     "outputs_hidden": true
    },
    "tags": []
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "training score  1.0\n",
      "test score  0.4272930648769575\n"
     ]
    }
   ],
   "source": [
    "print(\"training score \",svoc4.score(cov2,y2))\n",
    "print(\"test score \",svoc4.score(cov1,y1))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 50,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "svoc hard train =  1.0  test =  0.5816554809843401\n"
     ]
    }
   ],
   "source": [
    "mdmc = make_pipeline(Covariances(estimator='oas'),MDM(n_jobs=-1))\n",
    "Fgmdmc = make_pipeline(Covariances(estimator='oas'),FgMDM(n_jobs=-1))\n",
    "tsc1 = make_pipeline(Covariances(estimator='oas'),tsc(tsupdate=True))\n",
    "knn1 = make_pipeline(Covariances(estimator='oas'),knn(n_jobs=-1))\n",
    "\n",
    "svoc = VC(estimators = [('mdm',mdmc),('Fgmdm',Fgmdmc),('tsc',tsc1)], voting='hard',n_jobs=-1)\n",
    "svoc.fit(X2,y2)\n",
    "\n",
    "print(\"svoc hard train = \",svoc.score(X2,y2),\" test = \",svoc.score(X1,y1))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 60,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "svoc2 hard train =  1.0  test =  0.4317673378076063\n"
     ]
    }
   ],
   "source": [
    "svoc2 = VC(estimators = [('mdm',mdmc),('Fgmdm',Fgmdmc),('tsc',tsc1)], voting='hard',n_jobs=-1)\n",
    "svoc2.fit(X1,y1)\n",
    "\n",
    "print(\"svoc2 hard train = \",svoc2.score(X1,y1),\" test = \",svoc2.score(X2,y2))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 58,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "tsc-sknn train =  0.8926174496644296  test =  0.5167785234899329\n"
     ]
    }
   ],
   "source": [
    "tsc2 = make_pipeline(Covariances(estimator='oas'),tsc(tsupdate=True,clf=sknn(n_jobs=-1)))\n",
    "tsc2.fit(X2,y2)\n",
    "print(\"tsc-sknn train = \",tsc2.score(X2,y2),\" test = \",tsc2.score(X1,y1))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 63,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "D:\\Softwares\\Anaconda3\\envs\\neuroerg\\lib\\site-packages\\xgboost\\sklearn.py:1146: UserWarning: The use of label encoder in XGBClassifier is deprecated and will be removed in a future release. To remove this warning, do the following: 1) Pass option use_label_encoder=False when constructing XGBClassifier object; and 2) Encode your labels (y) as integers starting with 0, i.e. 0, 1, 2, ..., [num_class - 1].\n",
      "  warnings.warn(label_encoder_deprecation_msg, UserWarning)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[00:28:06] WARNING: C:/Users/Administrator/workspace/xgboost-win64_release_1.4.0/src/learner.cc:1095: Starting in XGBoost 1.3.0, the default evaluation metric used with the objective 'multi:softprob' was changed from 'merror' to 'mlogloss'. Explicitly set eval_metric if you'd like to restore the old behavior.\n",
      "train =  1.0  test =  0.5167785234899329\n"
     ]
    }
   ],
   "source": [
    "tsc3 = make_pipeline(Covariances(estimator='oas'),tsc(tsupdate=True,clf=xgb.XGBClassifier(n_jobs=-1)))\n",
    "tsc3.fit(X2,y2)\n",
    "p2 = tsc3.predict(X2)\n",
    "p1 = tsc3.predict(X1)\n",
    "\n",
    "# print(\"tsc-sknn train = \",tsc3.score(X2,y2),\" test = \",tsc3.score(X1,y1))\n",
    "print(\"train = \",accuracy_score(y2,p2),\" test = \",accuracy_score(y1,p1))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 67,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "svoc3 hard train =  1.0  test =  0.5906040268456376\n"
     ]
    }
   ],
   "source": [
    "svoc3 = VC(estimators = [('mdm',mdmc),('Fgmdm',Fgmdmc),('tsc',tsc3)], voting='hard',n_jobs=-1)\n",
    "svoc3.fit(X2,y2)\n",
    "\n",
    "print(\"svoc3 hard train = \",svoc3.score(X2,y2),\" test = \",svoc3.score(X1,y1))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 108,
   "metadata": {
    "tags": []
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "svoc3 hard train =  1.0  test =  0.5973154362416108\n"
     ]
    }
   ],
   "source": [
    "tscx = make_pipeline(Covariances(estimator='oas'),tsc(tsupdate=True,clf=xgb.XGBClassifier(n_estimators = 1000,n_jobs=-1)))\n",
    "svoc3 = VC(estimators = [('mdm',mdmc),('Fgmdm',Fgmdmc),('tsc',tscx)], voting='hard',n_jobs=-1)\n",
    "svoc3.fit(X2,y2)\n",
    "p1 = svoc3.predict(X1)\n",
    "p2 = svoc3.predict(X2)\n",
    "print(\"svoc3 hard train = \",accuracy_score(y2,p2),\" test = \",accuracy_score(y1,p1))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 109,
   "metadata": {
    "tags": []
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "svoc3 hard train =  1.0  test =  0.5995525727069351\n"
     ]
    }
   ],
   "source": [
    "tscx = make_pipeline(Covariances(estimator='oas'),tsc(tsupdate=True,clf=xgb.XGBClassifier(n_estimators = 1000,max_depth = 10,n_jobs=-1)))\n",
    "svoc3 = VC(estimators = [('mdm',mdmc),('Fgmdm',Fgmdmc),('tsc',tscx)], voting='hard',n_jobs=-1)\n",
    "svoc3.fit(X2,y2)\n",
    "p1 = svoc3.predict(X1)\n",
    "p2 = svoc3.predict(X2)\n",
    "print(\"svoc3 hard train = \",accuracy_score(y2,p2),\" test = \",accuracy_score(y1,p1))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 110,
   "metadata": {
    "tags": []
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "svoc3 hard train =  1.0  test =  0.6129753914988815\n"
     ]
    }
   ],
   "source": [
    "tscx = make_pipeline(Covariances(estimator='oas'),tsc(tsupdate=True,clf=xgb.XGBClassifier(n_estimators = 1000,max_depth = 10,learning_rate= 0.6,n_jobs=-1)))\n",
    "svoc3 = VC(estimators = [('mdm',mdmc),('Fgmdm',Fgmdmc),('tsc',tscx)], voting='hard',n_jobs=-1)\n",
    "svoc3.fit(X2,y2)\n",
    "p1 = svoc3.predict(X1)\n",
    "p2 = svoc3.predict(X2)\n",
    "print(\"svoc3 hard train = \",accuracy_score(y2,p2),\" test = \",accuracy_score(y1,p1))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 118,
   "metadata": {
    "tags": []
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([[0.59, 0.17, 0.23],\n",
       "       [0.05, 0.78, 0.17],\n",
       "       [0.13, 0.4 , 0.47]])"
      ]
     },
     "execution_count": 118,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "\n",
    "\n",
    "cx = np.around(cm(y1,p1,normalize='true'),2)\n",
    "cx"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 124,
   "metadata": {
    "tags": []
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "svoc3 hard train =  1.0  test =  0.5883668903803132\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "array([[0.54, 0.13, 0.33],\n",
       "       [0.07, 0.76, 0.17],\n",
       "       [0.15, 0.38, 0.46]])"
      ]
     },
     "execution_count": 124,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "tscx = make_pipeline(Covariances(estimator='oas'),tsc(tsupdate=True,clf=xgb.XGBClassifier(n_estimators = 1500,max_depth = 15,learning_rate= 0.8,n_jobs=-1)))\n",
    "svoc3 = VC(estimators = [('mdm',mdmc),('Fgmdm',Fgmdmc),('tsc',tscx)], voting='hard',n_jobs=-1)\n",
    "svoc3.fit(X2,y2)\n",
    "p1 = svoc3.predict(X1)\n",
    "p2 = svoc3.predict(X2)\n",
    "print(\"svoc3 hard train = \",accuracy_score(y2,p2),\" test = \",accuracy_score(y1,p1))\n",
    "cx = np.around(cm(y1,p1,normalize='true'),2)\n",
    "cx"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 125,
   "metadata": {
    "tags": []
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "svoc3 hard train =  1.0  test =  0.6152125279642058\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "array([[0.62, 0.13, 0.24],\n",
       "       [0.07, 0.76, 0.17],\n",
       "       [0.17, 0.36, 0.46]])"
      ]
     },
     "execution_count": 125,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "tscx = make_pipeline(Covariances(estimator='oas'),tsc(tsupdate=True,clf=xgb.XGBClassifier(n_estimators = 1500,max_depth = 15,learning_rate= 0.5,n_jobs=-1)))\n",
    "svoc3 = VC(estimators = [('mdm',mdmc),('Fgmdm',Fgmdmc),('tsc',tscx)], voting='hard',n_jobs=-1)\n",
    "svoc3.fit(X2,y2)\n",
    "p1 = svoc3.predict(X1)\n",
    "p2 = svoc3.predict(X2)\n",
    "print(\"svoc3 hard train = \",accuracy_score(y2,p2),\" test = \",accuracy_score(y1,p1))\n",
    "cx = np.around(cm(y1,p1,normalize='true'),2)\n",
    "cx"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 128,
   "metadata": {
    "tags": []
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "svoc3 hard train =  1.0  test =  0.6174496644295302\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "array([[0.61, 0.14, 0.25],\n",
       "       [0.07, 0.77, 0.16],\n",
       "       [0.15, 0.38, 0.47]])"
      ]
     },
     "execution_count": 128,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "tscx = make_pipeline(Covariances(estimator='oas'),tsc(tsupdate=True,clf=xgb.XGBClassifier(n_estimators = 1500,max_depth = 15,learning_rate= 0.3,n_jobs=-1)))\n",
    "svoc3 = VC(estimators = [('mdm',mdmc),('Fgmdm',Fgmdmc),('tsc',tscx)], voting='hard',n_jobs=-1)\n",
    "svoc3.fit(X2,y2)\n",
    "p1 = svoc3.predict(X1)\n",
    "p2 = svoc3.predict(X2)\n",
    "print(\"svoc3 hard train = \",accuracy_score(y2,p2),\" test = \",accuracy_score(y1,p1))\n",
    "cx = np.around(cm(y1,p1,normalize='true'),2)\n",
    "cx"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 127,
   "metadata": {
    "tags": []
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "svoc3 hard train =  1.0  test =  0.6442953020134228\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "array([[0.67, 0.12, 0.21],\n",
       "       [0.05, 0.77, 0.17],\n",
       "       [0.13, 0.38, 0.49]])"
      ]
     },
     "execution_count": 127,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "tscx = make_pipeline(Covariances(estimator='oas'),tsc(tsupdate=True,clf=xgb.XGBClassifier(n_estimators = 1500,max_depth = 15,learning_rate= 0.3,reg_lambda = 5,n_jobs=-1)))\n",
    "svoc3 = VC(estimators = [('mdm',mdmc),('Fgmdm',Fgmdmc),('tsc',tscx)], voting='hard',n_jobs=-1)\n",
    "svoc3.fit(X2,y2)\n",
    "p1 = svoc3.predict(X1)\n",
    "p2 = svoc3.predict(X2)\n",
    "print(\"svoc3 hard train = \",accuracy_score(y2,p2),\" test = \",accuracy_score(y1,p1))\n",
    "cx = np.around(cm(y1,p1,normalize='true'),2)\n",
    "cx"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 129,
   "metadata": {
    "tags": []
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "svoc3 hard train =  1.0  test =  0.6420581655480985\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "array([[0.68, 0.15, 0.17],\n",
       "       [0.06, 0.76, 0.18],\n",
       "       [0.13, 0.38, 0.49]])"
      ]
     },
     "execution_count": 129,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "tscx = make_pipeline(Covariances(estimator='oas'),tsc(tsupdate=True,clf=xgb.XGBClassifier(n_estimators = 1500,max_depth = 15,learning_rate= 0.1,reg_lambda = 5,n_jobs=-1)))\n",
    "svoc3 = VC(estimators = [('mdm',mdmc),('Fgmdm',Fgmdmc),('tsc',tscx)], voting='hard',n_jobs=-1)\n",
    "svoc3.fit(X2,y2)\n",
    "p1 = svoc3.predict(X1)\n",
    "p2 = svoc3.predict(X2)\n",
    "print(\"svoc3 hard train = \",accuracy_score(y2,p2),\" test = \",accuracy_score(y1,p1))\n",
    "cx = np.around(cm(y1,p1,normalize='true'),2)\n",
    "cx"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 130,
   "metadata": {
    "tags": []
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "svoc3 hard train =  1.0  test =  0.5928411633109619\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "array([[0.56, 0.13, 0.3 ],\n",
       "       [0.07, 0.76, 0.17],\n",
       "       [0.14, 0.4 , 0.46]])"
      ]
     },
     "execution_count": 130,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "tscx = make_pipeline(Covariances(estimator='oas'),tsc(tsupdate=True,clf=xgb.XGBClassifier(n_estimators = 1500,max_depth = 15,learning_rate= 0.3,reg_lambda = 5,gamma = 0.3,n_jobs=-1)))\n",
    "svoc3 = VC(estimators = [('mdm',mdmc),('Fgmdm',Fgmdmc),('tsc',tscx)], voting='hard',n_jobs=-1)\n",
    "svoc3.fit(X2,y2)\n",
    "p1 = svoc3.predict(X1)\n",
    "p2 = svoc3.predict(X2)\n",
    "print(\"svoc3 hard train = \",accuracy_score(y2,p2),\" test = \",accuracy_score(y1,p1))\n",
    "cx = np.around(cm(y1,p1,normalize='true'),2)\n",
    "cx"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 131,
   "metadata": {
    "tags": []
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "svoc3 hard train =  1.0  test =  0.6085011185682326\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "array([[0.61, 0.07, 0.32],\n",
       "       [0.08, 0.72, 0.2 ],\n",
       "       [0.13, 0.37, 0.5 ]])"
      ]
     },
     "execution_count": 131,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "tscx = make_pipeline(Covariances(estimator='oas'),tsc(tsupdate=True,clf=xgb.XGBClassifier(n_estimators = 1500,max_depth = 15,learning_rate= 0.3,reg_lambda = 5,colsample_bytree = 0.7,n_jobs=-1)))\n",
    "svoc3 = VC(estimators = [('mdm',mdmc),('Fgmdm',Fgmdmc),('tsc',tscx)], voting='hard',n_jobs=-1)\n",
    "svoc3.fit(X2,y2)\n",
    "p1 = svoc3.predict(X1)\n",
    "p2 = svoc3.predict(X2)\n",
    "print(\"svoc3 hard train = \",accuracy_score(y2,p2),\" test = \",accuracy_score(y1,p1))\n",
    "cx = np.around(cm(y1,p1,normalize='true'),2)\n",
    "cx"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 70,
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "tscx = make_pipeline(Covariances(estimator='oas'),tsc(tsupdate=True,clf=xgb.XGBClassifier(n_estimators = 1500,max_depth = 15,learning_rate= 0.3,reg_lambda = 5,n_jobs=-1)))\n",
    "svoc3 = VC(estimators = [('mdm',mdmc),('Fgmdm',Fgmdmc),('tsc',tscx)], voting='hard',n_jobs=-1)\n",
    "svoc3.fit(X2,y2)\n",
    "p1 = svoc3.predict(X1)\n",
    "p2 = svoc3.predict(X2)\n",
    "print(\"svoc3 hard train = \",accuracy_score(y2,p2),\" test = \",accuracy_score(y1,p1))\n",
    "cx = np.around(cm(y1,p1,normalize='true'),2)\n",
    "cx"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 70,
   "metadata": {
    "tags": []
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "svoc3 hard train =  0.9284116331096197  test =  0.6174496644295302\n"
     ]
    }
   ],
   "source": [
    "tsc2 = make_pipeline(Covariances(estimator='oas'),tsc(tsupdate=True,clf=sknn(n_jobs=-1)))\n",
    "svoc3 = VC(estimators = [('mdm',mdmc),('Fgmdm',Fgmdmc),('tsc',tsc2)], voting='hard',n_jobs=-1)\n",
    "svoc3.fit(X2,y2)\n",
    "p1 = svoc3.predict(X1)\n",
    "p2 = svoc3.predict(X2)\n",
    "print(\"svoc3 hard train = \",accuracy_score(y2,p2),\" test = \",accuracy_score(y1,p1))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 73,
   "metadata": {
    "collapsed": true,
    "jupyter": {
     "outputs_hidden": true
    },
    "tags": []
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([2, 2, 0, 2, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0,\n",
       "       0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 2, 0, 0, 0, 0,\n",
       "       0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,\n",
       "       0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,\n",
       "       2, 0, 0, 2, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0,\n",
       "       0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,\n",
       "       0, 0, 0, 2, 0, 1, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 0, 1, 1,\n",
       "       1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 2, 1, 1, 1,\n",
       "       1, 1, 1, 1, 1, 0, 1, 2, 1, 1, 1, 1, 1, 1, 0, 1, 1, 0, 1, 1, 1, 1,\n",
       "       1, 1, 1, 2, 1, 1, 1, 1, 1, 1, 1, 1, 0, 1, 2, 2, 1, 0, 0, 1, 1, 1,\n",
       "       1, 1, 1, 1, 1, 0, 1, 1, 1, 1, 2, 1, 0, 2, 1, 1, 1, 0, 1, 1, 1, 1,\n",
       "       1, 1, 1, 1, 0, 0, 1, 1, 0, 1, 1, 1, 1, 1, 2, 1, 2, 1, 1, 1, 1, 1,\n",
       "       0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 2, 1, 1, 2, 1, 2, 0, 1, 1, 1, 1, 1,\n",
       "       1, 1, 1, 1, 1, 1, 1, 2, 0, 0, 1, 1, 0, 0, 2, 0, 0, 0, 1, 1, 1, 0,\n",
       "       2, 1, 0, 2, 0, 0, 0, 0, 1, 1, 1, 1, 1, 0, 0, 1, 0, 1, 0, 1, 0, 1,\n",
       "       0, 0, 0, 1, 2, 0, 1, 1, 1, 2, 1, 2, 2, 1, 0, 1, 2, 1, 1, 1, 0, 1,\n",
       "       0, 2, 1, 1, 2, 0, 2, 1, 0, 0, 0, 1, 1, 1, 1, 0, 2, 1, 1, 0, 1, 2,\n",
       "       0, 0, 2, 0, 0, 0, 0, 0, 0, 1, 0, 1, 0, 0, 1, 1, 1, 1, 2, 2, 0, 0,\n",
       "       1, 1, 1, 0, 0, 1, 0, 1, 2, 2, 0, 0, 0, 0, 0, 2, 0, 1, 1, 0, 2, 0,\n",
       "       0, 0, 1, 1, 2, 1, 1, 2, 1, 1, 1, 2, 2, 1, 0, 1, 1, 1, 1, 0, 2, 1,\n",
       "       1, 1, 1, 1, 1, 2, 0])"
      ]
     },
     "execution_count": 73,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "p1"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 74,
   "metadata": {
    "tags": []
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "svoc4 hard train =  0.9038031319910514  test =  0.6465324384787472\n"
     ]
    }
   ],
   "source": [
    "tsck1 = make_pipeline(Covariances(estimator='oas'),tsc(tsupdate=True,clf=sknn(n_neighbors = 10,n_jobs=-1)))\n",
    "svoc4 = VC(estimators = [('mdm',mdmc),('Fgmdm',Fgmdmc),('tsc',tsck1)], voting='hard',n_jobs=-1)\n",
    "svoc4.fit(X2,y2)\n",
    "p1 = svoc4.predict(X1)\n",
    "p2 = svoc4.predict(X2)\n",
    "print(\"svoc4 hard train = \",accuracy_score(y2,p2),\" test = \",accuracy_score(y1,p1))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 79,
   "metadata": {
    "tags": []
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "svock2 hard train =  1.0  test =  0.6532438478747203\n"
     ]
    }
   ],
   "source": [
    "tsck2 = make_pipeline(Covariances(estimator='oas'),tsc(tsupdate=True,clf=sknn(n_neighbors = 10,n_jobs=-1,weights='distance')))\n",
    "svock2 = VC(estimators = [('mdm',mdmc),('Fgmdm',Fgmdmc),('tsc',tsck2)], voting='hard',n_jobs=-1)\n",
    "svock2.fit(X2,y2)\n",
    "p1 = svock2.predict(X1)\n",
    "p2 = svock2.predict(X2)\n",
    "print(\"svock2 hard train = \",accuracy_score(y2,p2),\" test = \",accuracy_score(y1,p1))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 80,
   "metadata": {
    "collapsed": true,
    "jupyter": {
     "outputs_hidden": true
    },
    "tags": []
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "svock2 hard train =  1.0  test =  0.6420581655480985\n"
     ]
    }
   ],
   "source": [
    "tsck2 = make_pipeline(Covariances(estimator='oas'),tsc(tsupdate=True,clf=sknn(n_neighbors = 22,n_jobs=-1,weights='distance')))\n",
    "svock2 = VC(estimators = [('mdm',mdmc),('Fgmdm',Fgmdmc),('tsc',tsck2)], voting='hard',n_jobs=-1)\n",
    "svock2.fit(X2,y2)\n",
    "p1 = svock2.predict(X1)\n",
    "p2 = svock2.predict(X2)\n",
    "print(\"svock2 hard train = \",accuracy_score(y2,p2),\" test = \",accuracy_score(y1,p1))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 84,
   "metadata": {
    "collapsed": true,
    "jupyter": {
     "outputs_hidden": true
    },
    "tags": []
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "svock2 hard train =  1.0  test =  0.6420581655480985\n"
     ]
    }
   ],
   "source": [
    "tsck2 = make_pipeline(Covariances(estimator='oas'),tsc(tsupdate=True,clf=sknn(n_neighbors = 14,n_jobs=-1,weights='distance')))\n",
    "svock2 = VC(estimators = [('mdm',mdmc),('Fgmdm',Fgmdmc),('tsc',tsck2)], voting='hard',n_jobs=-1)\n",
    "svock2.fit(X2,y2)\n",
    "p1 = svock2.predict(X1)\n",
    "p2 = svock2.predict(X2)\n",
    "print(\"svock2 hard train = \",accuracy_score(y2,p2),\" test = \",accuracy_score(y1,p1))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 86,
   "metadata": {
    "collapsed": true,
    "jupyter": {
     "outputs_hidden": true
    },
    "tags": []
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([2, 2, 0, 2, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0,\n",
       "       0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 2, 0, 0, 0, 0,\n",
       "       0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1,\n",
       "       0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,\n",
       "       2, 0, 0, 2, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,\n",
       "       0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,\n",
       "       0, 0, 0, 2, 0, 1, 0, 0, 1, 0, 0, 0, 0, 0, 0, 2, 0, 1, 1, 1, 1, 1,\n",
       "       1, 1, 1, 1, 1, 1, 2, 1, 1, 1, 1, 1, 1, 2, 1, 0, 1, 1, 1, 1, 1, 1,\n",
       "       1, 2, 1, 1, 1, 1, 1, 2, 1, 1, 1, 1, 1, 1, 0, 1, 1, 1, 2, 1, 1, 1,\n",
       "       1, 1, 1, 2, 1, 1, 1, 1, 1, 1, 1, 1, 0, 1, 1, 1, 1, 0, 0, 1, 1, 1,\n",
       "       1, 1, 1, 1, 1, 1, 1, 1, 1, 2, 2, 1, 2, 2, 1, 1, 1, 1, 1, 2, 1, 1,\n",
       "       1, 1, 1, 1, 1, 0, 1, 1, 0, 1, 1, 1, 2, 1, 1, 1, 2, 1, 1, 1, 1, 1,\n",
       "       1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 2, 1, 1, 2, 1, 2, 1, 1, 1, 1, 1, 1,\n",
       "       1, 1, 1, 2, 1, 1, 1, 2, 1, 1, 1, 1, 0, 0, 2, 1, 0, 1, 1, 2, 1, 0,\n",
       "       2, 1, 0, 2, 1, 2, 0, 1, 1, 1, 1, 2, 1, 0, 0, 1, 0, 2, 0, 1, 0, 1,\n",
       "       1, 0, 1, 2, 2, 1, 1, 1, 1, 2, 1, 1, 2, 2, 0, 1, 2, 1, 1, 1, 1, 1,\n",
       "       1, 2, 1, 1, 2, 1, 2, 1, 2, 1, 1, 1, 2, 1, 1, 1, 1, 1, 1, 1, 1, 2,\n",
       "       0, 1, 2, 0, 1, 2, 1, 0, 0, 0, 1, 1, 0, 0, 1, 1, 1, 1, 2, 2, 1, 1,\n",
       "       1, 1, 1, 0, 0, 1, 0, 2, 2, 2, 1, 0, 1, 1, 0, 1, 0, 1, 1, 1, 2, 1,\n",
       "       0, 1, 1, 0, 2, 1, 1, 2, 2, 1, 1, 2, 2, 1, 2, 1, 1, 1, 1, 1, 2, 1,\n",
       "       1, 1, 1, 1, 1, 1, 1])"
      ]
     },
     "execution_count": 86,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "p1"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 77,
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "# tsck4 = make_pipeline(Covariances(estimator='oas'),tsc(tsupdate=True,clf=sknn(n_neighbors = 14,n_jobs=-1,weights='distance',metric=)))\n",
    "# svock4 = VC(estimators = [('mdm',mdmc),('Fgmdm',Fgmdmc),('tsc',tsck4)], voting='hard',n_jobs=-1)\n",
    "# svock4.fit(X2,y2)\n",
    "# p1 = svock4.predict(X1)\n",
    "# p2 = svock4.predict(X2)\n",
    "# print(\"svock4 hard train = \",accuracy_score(y2,p2),\" test = \",accuracy_score(y1,p1))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 77,
   "metadata": {
    "tags": []
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "svoc5 hard train =  1.0  test =  0.6353467561521253\n"
     ]
    }
   ],
   "source": [
    "\n",
    "tscs1 = make_pipeline(Covariances(estimator='oas'),tsc(tsupdate=True,clf=SVC()))\n",
    "svoc5 = VC(estimators = [('mdm',mdmc),('Fgmdm',Fgmdmc),('tsc',tscs1)], voting='hard',n_jobs=-1)\n",
    "svoc5.fit(X2,y2)\n",
    "p1 = svoc5.predict(X1)\n",
    "p2 = svoc5.predict(X2)\n",
    "print(\"svoc5 hard train = \",accuracy_score(y2,p2),\" test = \",accuracy_score(y1,p1))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 92,
   "metadata": {
    "tags": []
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "svocs2 hard train =  1.0  test =  0.6241610738255033\n"
     ]
    }
   ],
   "source": [
    "tscs2 = make_pipeline(Covariances(estimator='oas'),tsc(tsupdate=True,clf=SVC(C=10)))\n",
    "svocs2 = VC(estimators = [('mdm',mdmc),('Fgmdm',Fgmdmc),('tsc',tscs2)], voting='hard',n_jobs=-1)\n",
    "svocs2.fit(X2,y2)\n",
    "p1 = svocs2.predict(X1)\n",
    "p2 = svocs2.predict(X2)\n",
    "print(\"svocs2 hard train = \",accuracy_score(y2,p2),\" test = \",accuracy_score(y1,p1))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 93,
   "metadata": {
    "tags": []
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "svocs2 hard train =  1.0  test =  0.6241610738255033\n"
     ]
    }
   ],
   "source": [
    "tscs2 = make_pipeline(Covariances(estimator='oas'),tsc(tsupdate=True,clf=SVC(C=1000)))\n",
    "svocs2 = VC(estimators = [('mdm',mdmc),('Fgmdm',Fgmdmc),('tsc',tscs2)], voting='hard',n_jobs=-1)\n",
    "svocs2.fit(X2,y2)\n",
    "p1 = svocs2.predict(X1)\n",
    "p2 = svocs2.predict(X2)\n",
    "print(\"svocs2 hard train = \",accuracy_score(y2,p2),\" test = \",accuracy_score(y1,p1))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 103,
   "metadata": {
    "tags": []
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "svocs2 hard train =  0.901565995525727  test =  0.7002237136465325\n"
     ]
    }
   ],
   "source": [
    "tscs2 = make_pipeline(Covariances(estimator='oas'),tsc(tsupdate=True,clf=SVC(gamma='auto')))\n",
    "svocs2 = VC(estimators = [('mdm',mdmc),('Fgmdm',Fgmdmc),('tsc',tscs2)], voting='hard',n_jobs=-1)\n",
    "svocs2.fit(X2,y2)\n",
    "p1 = svocs2.predict(X1)\n",
    "p2 = svocs2.predict(X2)\n",
    "print(\"svocs2 hard train = \",accuracy_score(y2,p2),\" test = \",accuracy_score(y1,p1))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 104,
   "metadata": {
    "collapsed": true,
    "jupyter": {
     "outputs_hidden": true
    },
    "tags": []
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([2, 2, 0, 2, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0,\n",
       "       0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 2, 1, 0, 0, 0,\n",
       "       0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,\n",
       "       0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,\n",
       "       2, 0, 0, 2, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,\n",
       "       0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,\n",
       "       0, 0, 0, 2, 0, 1, 0, 0, 1, 0, 0, 0, 0, 0, 0, 1, 0, 1, 2, 0, 1, 1,\n",
       "       1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 1, 1, 1, 1, 1, 1,\n",
       "       1, 1, 1, 1, 1, 0, 1, 1, 1, 1, 1, 1, 1, 1, 0, 1, 1, 1, 1, 1, 1, 1,\n",
       "       1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 1, 1, 2, 1, 0, 0, 1, 1, 1,\n",
       "       1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 2, 1, 2, 2, 1, 1, 1, 1, 1, 1, 1, 1,\n",
       "       1, 1, 1, 1, 1, 1, 1, 1, 0, 1, 1, 1, 1, 1, 1, 1, 2, 1, 1, 1, 1, 1,\n",
       "       1, 1, 1, 1, 1, 1, 2, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,\n",
       "       1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 2, 2, 2, 0, 2, 2, 1, 2, 1, 0,\n",
       "       2, 0, 0, 2, 0, 2, 1, 1, 1, 1, 1, 2, 1, 2, 1, 1, 2, 2, 1, 1, 0, 1,\n",
       "       1, 1, 1, 1, 2, 1, 1, 1, 1, 2, 1, 1, 2, 2, 1, 1, 2, 1, 1, 1, 1, 1,\n",
       "       1, 2, 1, 1, 2, 1, 1, 1, 2, 2, 1, 1, 1, 1, 2, 1, 2, 1, 1, 1, 1, 1,\n",
       "       1, 1, 2, 1, 2, 2, 2, 2, 0, 0, 0, 1, 1, 0, 1, 1, 1, 1, 2, 2, 2, 0,\n",
       "       1, 1, 1, 0, 0, 1, 1, 2, 2, 2, 0, 0, 0, 1, 0, 2, 0, 1, 1, 2, 2, 2,\n",
       "       2, 1, 2, 1, 2, 1, 2, 2, 2, 1, 1, 2, 2, 1, 1, 1, 1, 1, 1, 1, 2, 1,\n",
       "       1, 1, 1, 1, 1, 1, 1])"
      ]
     },
     "execution_count": 104,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "p1"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 98,
   "metadata": {
    "tags": []
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "svocs2 hard train =  0.9955257270693513  test =  0.5682326621923938\n"
     ]
    }
   ],
   "source": [
    "tscs2 = make_pipeline(Covariances(estimator='oas'),tsc(tsupdate=True,clf=SVC(gamma='auto',C=10)))\n",
    "svocs2 = VC(estimators = [('mdm',mdmc),('Fgmdm',Fgmdmc),('tsc',tscs2)], voting='hard',n_jobs=-1)\n",
    "svocs2.fit(X2,y2)\n",
    "p1 = svocs2.predict(X1)\n",
    "p2 = svocs2.predict(X2)\n",
    "print(\"svocs2 hard train = \",accuracy_score(y2,p2),\" test = \",accuracy_score(y1,p1))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 101,
   "metadata": {
    "tags": []
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "svocs2 hard train =  0.8657718120805369  test =  0.6666666666666666\n"
     ]
    }
   ],
   "source": [
    "tscs2 = make_pipeline(Covariances(estimator='oas'),tsc(tsupdate=True,clf=SVC(gamma='auto',C=0.1)))\n",
    "svocs2 = VC(estimators = [('mdm',mdmc),('Fgmdm',Fgmdmc),('tsc',tscs2)], voting='hard',n_jobs=-1)\n",
    "svocs2.fit(X2,y2)\n",
    "p1 = svocs2.predict(X1)\n",
    "p2 = svocs2.predict(X2)\n",
    "print(\"svocs2 hard train = \",accuracy_score(y2,p2),\" test = \",accuracy_score(y1,p1))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 102,
   "metadata": {
    "tags": []
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "svocs2 hard train =  0.8747203579418344  test =  0.6756152125279642\n"
     ]
    }
   ],
   "source": [
    "tscs2 = make_pipeline(Covariances(estimator='oas'),tsc(tsupdate=True,clf=SVC(gamma='auto',C=0.5)))\n",
    "svocs2 = VC(estimators = [('mdm',mdmc),('Fgmdm',Fgmdmc),('tsc',tscs2)], voting='hard',n_jobs=-1)\n",
    "svocs2.fit(X2,y2)\n",
    "p1 = svocs2.predict(X1)\n",
    "p2 = svocs2.predict(X2)\n",
    "print(\"svocs2 hard train = \",accuracy_score(y2,p2),\" test = \",accuracy_score(y1,p1))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 99,
   "metadata": {
    "tags": []
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "svocs2 hard train =  1.0  test =  0.5637583892617449\n"
     ]
    }
   ],
   "source": [
    "tscs2 = make_pipeline(Covariances(estimator='oas'),tsc(tsupdate=True,clf=SVC(gamma='auto',C=100)))\n",
    "svocs2 = VC(estimators = [('mdm',mdmc),('Fgmdm',Fgmdmc),('tsc',tscs2)], voting='hard',n_jobs=-1)\n",
    "svocs2.fit(X2,y2)\n",
    "p1 = svocs2.predict(X1)\n",
    "p2 = svocs2.predict(X2)\n",
    "print(\"svocs2 hard train = \",accuracy_score(y2,p2),\" test = \",accuracy_score(y1,p1))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 100,
   "metadata": {
    "tags": []
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "svocs2 hard train =  1.0  test =  0.5637583892617449\n"
     ]
    }
   ],
   "source": [
    "tscs2 = make_pipeline(Covariances(estimator='oas'),tsc(tsupdate=True,clf=SVC(gamma='auto',C=1000)))\n",
    "svocs2 = VC(estimators = [('mdm',mdmc),('Fgmdm',Fgmdmc),('tsc',tscs2)], voting='hard',n_jobs=-1)\n",
    "svocs2.fit(X2,y2)\n",
    "p1 = svocs2.predict(X1)\n",
    "p2 = svocs2.predict(X2)\n",
    "print(\"svocs2 hard train = \",accuracy_score(y2,p2),\" test = \",accuracy_score(y1,p1))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 95,
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 95,
   "metadata": {
    "tags": []
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "svocs3 hard train =  1.0  test =  0.6756152125279642\n"
     ]
    }
   ],
   "source": [
    "tscs3 = make_pipeline(Covariances(estimator='oas'),tsc(tsupdate=True,clf=NuSVC()))\n",
    "svocs3 = VC(estimators = [('mdm',mdmc),('Fgmdm',Fgmdmc),('tsc',tscs3)], voting='hard',n_jobs=-1)\n",
    "svocs3.fit(X2,y2)\n",
    "p1 = svocs3.predict(X1)\n",
    "p2 = svocs3.predict(X2)\n",
    "print(\"svocs3 hard train = \",accuracy_score(y2,p2),\" test = \",accuracy_score(y1,p1))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 97,
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 97,
   "metadata": {
    "collapsed": true,
    "jupyter": {
     "outputs_hidden": true
    },
    "tags": []
   },
   "outputs": [
    {
     "ename": "TypeError",
     "evalue": "__init__() got an unexpected keyword argument 'C'",
     "output_type": "error",
     "traceback": [
      "\u001b[1;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[1;31mTypeError\u001b[0m                                 Traceback (most recent call last)",
      "\u001b[1;32m<ipython-input-97-25c06f02dbe1>\u001b[0m in \u001b[0;36m<module>\u001b[1;34m\u001b[0m\n\u001b[1;32m----> 1\u001b[1;33m \u001b[0mtscs3\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mmake_pipeline\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mCovariances\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mestimator\u001b[0m\u001b[1;33m=\u001b[0m\u001b[1;34m'oas'\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m,\u001b[0m\u001b[0mtsc\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mtsupdate\u001b[0m\u001b[1;33m=\u001b[0m\u001b[1;32mTrue\u001b[0m\u001b[1;33m,\u001b[0m\u001b[0mclf\u001b[0m\u001b[1;33m=\u001b[0m\u001b[0mNuSVC\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mgamma\u001b[0m\u001b[1;33m=\u001b[0m\u001b[1;34m'auto'\u001b[0m\u001b[1;33m,\u001b[0m\u001b[0mC\u001b[0m\u001b[1;33m=\u001b[0m\u001b[1;36m100\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m      2\u001b[0m \u001b[0msvocs3\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mVC\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mestimators\u001b[0m \u001b[1;33m=\u001b[0m \u001b[1;33m[\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;34m'mdm'\u001b[0m\u001b[1;33m,\u001b[0m\u001b[0mmdmc\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m,\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;34m'Fgmdm'\u001b[0m\u001b[1;33m,\u001b[0m\u001b[0mFgmdmc\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m,\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;34m'tsc'\u001b[0m\u001b[1;33m,\u001b[0m\u001b[0mtscs3\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m]\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mvoting\u001b[0m\u001b[1;33m=\u001b[0m\u001b[1;34m'hard'\u001b[0m\u001b[1;33m,\u001b[0m\u001b[0mn_jobs\u001b[0m\u001b[1;33m=\u001b[0m\u001b[1;33m-\u001b[0m\u001b[1;36m1\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m      3\u001b[0m \u001b[0msvocs3\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mfit\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mX2\u001b[0m\u001b[1;33m,\u001b[0m\u001b[0my2\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m      4\u001b[0m \u001b[0mp1\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0msvocs3\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mpredict\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mX1\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m      5\u001b[0m \u001b[0mp2\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0msvocs3\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mpredict\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mX2\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32mD:\\Softwares\\Anaconda3\\envs\\neuroerg\\lib\\site-packages\\sklearn\\utils\\validation.py\u001b[0m in \u001b[0;36minner_f\u001b[1;34m(*args, **kwargs)\u001b[0m\n\u001b[0;32m     61\u001b[0m             \u001b[0mextra_args\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mlen\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0margs\u001b[0m\u001b[1;33m)\u001b[0m \u001b[1;33m-\u001b[0m \u001b[0mlen\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mall_args\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m     62\u001b[0m             \u001b[1;32mif\u001b[0m \u001b[0mextra_args\u001b[0m \u001b[1;33m<=\u001b[0m \u001b[1;36m0\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m---> 63\u001b[1;33m                 \u001b[1;32mreturn\u001b[0m \u001b[0mf\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;33m*\u001b[0m\u001b[0margs\u001b[0m\u001b[1;33m,\u001b[0m \u001b[1;33m**\u001b[0m\u001b[0mkwargs\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m     64\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m     65\u001b[0m             \u001b[1;31m# extra_args > 0\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;31mTypeError\u001b[0m: __init__() got an unexpected keyword argument 'C'"
     ]
    }
   ],
   "source": [
    "tscs3 = make_pipeline(Covariances(estimator='oas'),tsc(tsupdate=True,clf=NuSVC(gamma='auto',C=100)))\n",
    "svocs3 = VC(estimators = [('mdm',mdmc),('Fgmdm',Fgmdmc),('tsc',tscs3)], voting='hard',n_jobs=-1)\n",
    "svocs3.fit(X2,y2)\n",
    "p1 = svocs3.predict(X1)\n",
    "p2 = svocs3.predict(X2)\n",
    "print(\"svocs3 hard train = \",accuracy_score(y2,p2),\" test = \",accuracy_score(y1,p1))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 52,
   "metadata": {
    "collapsed": true,
    "jupyter": {
     "outputs_hidden": true
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "from sklearn.svm import NuSVC\n",
    "tscs3 = make_pipeline(Covariances(estimator='oas'),tsc(tsupdate=True,clf=NuSVC(C=100)))\n",
    "svocs3 = VC(estimators = [('mdm',mdmc),('Fgmdm',Fgmdmc),('tsc',tscs3)], voting='hard',n_jobs=-1)\n",
    "svocs3.fit(X2,y2)\n",
    "p1 = svocs3.predict(X1)\n",
    "p2 = svocs3.predict(X2)\n",
    "print(\"svocs3 hard train = \",accuracy_score(y2,p2),\" test = \",accuracy_score(y1,p1))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 52,
   "metadata": {
    "tags": []
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "stackclass test =  0.4742729306487696  train =  1.0\n"
     ]
    }
   ],
   "source": [
    "#stacking classifier\n",
    "\n",
    "sc1 = SC(estimators = [('mdm',mdmc),('Fgmdm',Fgmdmc),('tsc',tsc1)], cv = 5,n_jobs=-1)\n",
    "sc1.fit(X2,y2)\n",
    "\n",
    "print(\"stackclass test = \",sc1.score(X1,y1), \" train = \",sc1.score(X2,y2))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 53,
   "metadata": {
    "collapsed": true,
    "jupyter": {
     "outputs_hidden": true
    },
    "tags": []
   },
   "outputs": [
    {
     "ename": "ValueError",
     "evalue": "all the input arrays must have same number of dimensions, but the array at index 0 has 2 dimension(s) and the array at index 3 has 3 dimension(s)",
     "output_type": "error",
     "traceback": [
      "\u001b[1;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[1;31mValueError\u001b[0m                                Traceback (most recent call last)",
      "\u001b[1;32m<ipython-input-53-c68f24335d00>\u001b[0m in \u001b[0;36m<module>\u001b[1;34m\u001b[0m\n\u001b[0;32m      1\u001b[0m \u001b[0msc2\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mSC\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mestimators\u001b[0m \u001b[1;33m=\u001b[0m \u001b[1;33m[\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;34m'mdm'\u001b[0m\u001b[1;33m,\u001b[0m\u001b[0mmdmc\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m,\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;34m'Fgmdm'\u001b[0m\u001b[1;33m,\u001b[0m\u001b[0mFgmdmc\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m,\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;34m'tsc'\u001b[0m\u001b[1;33m,\u001b[0m\u001b[0mtsc1\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m]\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mcv\u001b[0m \u001b[1;33m=\u001b[0m \u001b[1;36m5\u001b[0m\u001b[1;33m,\u001b[0m\u001b[0mn_jobs\u001b[0m\u001b[1;33m=\u001b[0m\u001b[1;33m-\u001b[0m\u001b[1;36m1\u001b[0m\u001b[1;33m,\u001b[0m\u001b[0mpassthrough\u001b[0m\u001b[1;33m=\u001b[0m\u001b[1;32mTrue\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m----> 2\u001b[1;33m \u001b[0msc2\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mfit\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mX2\u001b[0m\u001b[1;33m,\u001b[0m\u001b[0my2\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m      3\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m      4\u001b[0m \u001b[0mprint\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;34m\"stackclass test = \"\u001b[0m\u001b[1;33m,\u001b[0m\u001b[0msc2\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mscore\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mX1\u001b[0m\u001b[1;33m,\u001b[0m\u001b[0my1\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m,\u001b[0m \u001b[1;34m\" train = \"\u001b[0m\u001b[1;33m,\u001b[0m\u001b[0msc2\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mscore\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mX2\u001b[0m\u001b[1;33m,\u001b[0m\u001b[0my2\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32mD:\\Softwares\\Anaconda3\\envs\\neuroerg\\lib\\site-packages\\sklearn\\ensemble\\_stacking.py\u001b[0m in \u001b[0;36mfit\u001b[1;34m(self, X, y, sample_weight)\u001b[0m\n\u001b[0;32m    439\u001b[0m         \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0m_le\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mLabelEncoder\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mfit\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0my\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    440\u001b[0m         \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mclasses_\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0m_le\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mclasses_\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m--> 441\u001b[1;33m         \u001b[1;32mreturn\u001b[0m \u001b[0msuper\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mfit\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mX\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0m_le\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mtransform\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0my\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0msample_weight\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m    442\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    443\u001b[0m     \u001b[1;33m@\u001b[0m\u001b[0mif_delegate_has_method\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mdelegate\u001b[0m\u001b[1;33m=\u001b[0m\u001b[1;34m'final_estimator_'\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32mD:\\Softwares\\Anaconda3\\envs\\neuroerg\\lib\\site-packages\\sklearn\\ensemble\\_stacking.py\u001b[0m in \u001b[0;36mfit\u001b[1;34m(self, X, y, sample_weight)\u001b[0m\n\u001b[0;32m    193\u001b[0m         ]\n\u001b[0;32m    194\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m--> 195\u001b[1;33m         \u001b[0mX_meta\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0m_concatenate_predictions\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mX\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mpredictions\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m    196\u001b[0m         _fit_single_estimator(self.final_estimator_, X_meta, y,\n\u001b[0;32m    197\u001b[0m                               sample_weight=sample_weight)\n",
      "\u001b[1;32mD:\\Softwares\\Anaconda3\\envs\\neuroerg\\lib\\site-packages\\sklearn\\ensemble\\_stacking.py\u001b[0m in \u001b[0;36m_concatenate_predictions\u001b[1;34m(self, X, predictions)\u001b[0m\n\u001b[0;32m     91\u001b[0m                 \u001b[1;32mreturn\u001b[0m \u001b[0msparse\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mhstack\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mX_meta\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mformat\u001b[0m\u001b[1;33m=\u001b[0m\u001b[0mX\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mformat\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m     92\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m---> 93\u001b[1;33m         \u001b[1;32mreturn\u001b[0m \u001b[0mnp\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mhstack\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mX_meta\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m     94\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m     95\u001b[0m     \u001b[1;33m@\u001b[0m\u001b[0mstaticmethod\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32m<__array_function__ internals>\u001b[0m in \u001b[0;36mhstack\u001b[1;34m(*args, **kwargs)\u001b[0m\n",
      "\u001b[1;32mD:\\Softwares\\Anaconda3\\envs\\neuroerg\\lib\\site-packages\\numpy\\core\\shape_base.py\u001b[0m in \u001b[0;36mhstack\u001b[1;34m(tup)\u001b[0m\n\u001b[0;32m    343\u001b[0m         \u001b[1;32mreturn\u001b[0m \u001b[0m_nx\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mconcatenate\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0marrs\u001b[0m\u001b[1;33m,\u001b[0m \u001b[1;36m0\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    344\u001b[0m     \u001b[1;32melse\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m--> 345\u001b[1;33m         \u001b[1;32mreturn\u001b[0m \u001b[0m_nx\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mconcatenate\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0marrs\u001b[0m\u001b[1;33m,\u001b[0m \u001b[1;36m1\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m    346\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    347\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32m<__array_function__ internals>\u001b[0m in \u001b[0;36mconcatenate\u001b[1;34m(*args, **kwargs)\u001b[0m\n",
      "\u001b[1;31mValueError\u001b[0m: all the input arrays must have same number of dimensions, but the array at index 0 has 2 dimension(s) and the array at index 3 has 3 dimension(s)"
     ]
    }
   ],
   "source": [
    "sc2 = SC(estimators = [('mdm',mdmc),('Fgmdm',Fgmdmc),('tsc',tsc1)], cv = 5,n_jobs=-1,passthrough=True)\n",
    "sc2.fit(X2,y2)\n",
    "\n",
    "print(\"stackclass test = \",sc2.score(X1,y1), \" train = \",sc2.score(X2,y2))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 55,
   "metadata": {
    "collapsed": true,
    "jupyter": {
     "outputs_hidden": true
    },
    "tags": []
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "stackclass test =  0.2841163310961969  train =  1.0\n"
     ]
    }
   ],
   "source": [
    "sc3 = SC(estimators = [('knn',knn1),('mdm',mdmc),('Fgmdm',Fgmdmc),('tsc',tsc1)], stack_method = 'predict', cv = 5,n_jobs=-1)\n",
    "sc3.fit(X2,y2)\n",
    "\n",
    "print(\"stackclass test = \",sc3.score(X1,y1), \" train = \",sc3.score(X2,y2))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 160,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "tsc-svc hard train =  1.0  test =  0.5100671140939598\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "array([[0.19, 0.62, 0.19],\n",
       "       [0.01, 0.71, 0.28],\n",
       "       [0.01, 0.37, 0.62]])"
      ]
     },
     "execution_count": 160,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "ina2 = make_pipeline(HankelCovariances(delays=[2, 4, 8, 12, 16], estimator='oas'),\n",
    "                    TangentSpace(),\n",
    "                    sknn(n_neighbors = 14,n_jobs=-1,weights='distance'))\n",
    "ina2.fit(X2,y2)\n",
    "p1 = ina2.predict(X1)\n",
    "p2 = ina2.predict(X2)\n",
    "print(\"tsc-svc hard train = \",accuracy_score(y2,p2),\" test = \",accuracy_score(y1,p1))\n",
    "\n",
    "c = np.around(cm(y1,p1,normalize='true'),2)\n",
    "c"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 161,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "D:\\Softwares\\Anaconda3\\envs\\neuroerg\\lib\\site-packages\\xgboost\\sklearn.py:1146: UserWarning: The use of label encoder in XGBClassifier is deprecated and will be removed in a future release. To remove this warning, do the following: 1) Pass option use_label_encoder=False when constructing XGBClassifier object; and 2) Encode your labels (y) as integers starting with 0, i.e. 0, 1, 2, ..., [num_class - 1].\n",
      "  warnings.warn(label_encoder_deprecation_msg, UserWarning)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[05:49:48] WARNING: C:/Users/Administrator/workspace/xgboost-win64_release_1.4.0/src/learner.cc:1095: Starting in XGBoost 1.3.0, the default evaluation metric used with the objective 'multi:softprob' was changed from 'merror' to 'mlogloss'. Explicitly set eval_metric if you'd like to restore the old behavior.\n",
      "tsc-svc hard train =  1.0  test =  0.4317673378076063\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "array([[0.43, 0.48, 0.09],\n",
       "       [0.13, 0.87, 0.01],\n",
       "       [0.37, 0.63, 0.  ]])"
      ]
     },
     "execution_count": 161,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "ina2 = make_pipeline(HankelCovariances(delays=[2, 4, 8, 12, 16], estimator='oas'),\n",
    "                    TangentSpace(metric='logeuclid'),\n",
    "                    xgb.XGBClassifier(n_estimators = 1500,max_depth = 15,learning_rate= 0.3,reg_lambda = 5,n_jobs=-1))\n",
    "ina2.fit(X2,y2)\n",
    "p1 = ina2.predict(X1)\n",
    "p2 = ina2.predict(X2)\n",
    "print(\"tsc-svc hard train = \",accuracy_score(y2,p2),\" test = \",accuracy_score(y1,p1))\n",
    "\n",
    "c = np.around(cm(y1,p1,normalize='true'),2)\n",
    "c"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 162,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "tsc-svc hard train =  1.0  test =  0.5727069351230425\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "array([[0.52, 0.15, 0.33],\n",
       "       [0.11, 0.25, 0.64],\n",
       "       [0.03, 0.01, 0.95]])"
      ]
     },
     "execution_count": 162,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "ina2 = make_pipeline(HankelCovariances(delays=[2, 4, 8, 12, 16], estimator='oas'),\n",
    "                    TangentSpace(metric='logeuclid'),\n",
    "                    NuSVC())\n",
    "ina2.fit(X2,y2)\n",
    "p1 = ina2.predict(X1)\n",
    "p2 = ina2.predict(X2)\n",
    "print(\"tsc-svc hard train = \",accuracy_score(y2,p2),\" test = \",accuracy_score(y1,p1))\n",
    "\n",
    "c = np.around(cm(y1,p1,normalize='true'),2)\n",
    "c"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 156,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "tsc-svc hard train =  1.0  test =  0.5078299776286354\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "array([[0.19, 0.72, 0.09],\n",
       "       [0.02, 0.72, 0.26],\n",
       "       [0.01, 0.39, 0.6 ]])"
      ]
     },
     "execution_count": 156,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "ina2 = make_pipeline(HankelCovariances(delays=[2, 4, 8, 12, 16], estimator='oas'),\n",
    "                    TangentSpace(metric='logeuclid'),\n",
    "                    sknn(n_neighbors = 14,n_jobs=-1,weights='distance'))\n",
    "ina2.fit(X2,y2)\n",
    "p1 = ina2.predict(X1)\n",
    "p2 = ina2.predict(X2)\n",
    "print(\"tsc-svc hard train = \",accuracy_score(y2,p2),\" test = \",accuracy_score(y1,p1))\n",
    "\n",
    "c = np.around(cm(y1,p1,normalize='true'),2)\n",
    "c"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 157,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "tsc-svc hard train =  1.0  test =  0.39373601789709173\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "array([[0.  , 0.  , 1.  ],\n",
       "       [0.04, 0.19, 0.77],\n",
       "       [0.  , 0.01, 0.99]])"
      ]
     },
     "execution_count": 157,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "ina3 = make_pipeline(HankelCovariances(delays=[2, 4, 8, 12, 16], estimator='oas'),\n",
    "                    CSP(30),\n",
    "                    sknn(n_neighbors = 14,n_jobs=-1,weights='distance'))\n",
    "ina3.fit(X2,y2)\n",
    "p1 = ina3.predict(X1)\n",
    "p2 = ina3.predict(X2)\n",
    "print(\"tsc-svc hard train = \",accuracy_score(y2,p2),\" test = \",accuracy_score(y1,p1))\n",
    "\n",
    "c = np.around(cm(y1,p1,normalize='true'),2)\n",
    "c"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 158,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "D:\\Softwares\\Anaconda3\\envs\\neuroerg\\lib\\site-packages\\xgboost\\sklearn.py:1146: UserWarning: The use of label encoder in XGBClassifier is deprecated and will be removed in a future release. To remove this warning, do the following: 1) Pass option use_label_encoder=False when constructing XGBClassifier object; and 2) Encode your labels (y) as integers starting with 0, i.e. 0, 1, 2, ..., [num_class - 1].\n",
      "  warnings.warn(label_encoder_deprecation_msg, UserWarning)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[05:35:36] WARNING: C:/Users/Administrator/workspace/xgboost-win64_release_1.4.0/src/learner.cc:1095: Starting in XGBoost 1.3.0, the default evaluation metric used with the objective 'multi:softprob' was changed from 'merror' to 'mlogloss'. Explicitly set eval_metric if you'd like to restore the old behavior.\n",
      "tsc-svc hard train =  1.0  test =  0.3959731543624161\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "array([[0.  , 0.02, 0.98],\n",
       "       [0.06, 0.21, 0.72],\n",
       "       [0.  , 0.03, 0.97]])"
      ]
     },
     "execution_count": 158,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "ina3 = make_pipeline(HankelCovariances(delays=[2, 4, 8, 12, 16], estimator='oas'),\n",
    "                    CSP(30),\n",
    "                    xgb.XGBClassifier(n_estimators = 1500,max_depth = 15,learning_rate= 0.3,reg_lambda = 5,n_jobs=-1))\n",
    "ina3.fit(X2,y2)\n",
    "p1 = ina3.predict(X1)\n",
    "p2 = ina3.predict(X2)\n",
    "print(\"tsc-svc hard train = \",accuracy_score(y2,p2),\" test = \",accuracy_score(y1,p1))\n",
    "\n",
    "c = np.around(cm(y1,p1,normalize='true'),2)\n",
    "c"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 159,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "tsc-svc hard train =  0.959731543624161  test =  0.3959731543624161\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "array([[0.  , 0.01, 0.99],\n",
       "       [0.01, 0.19, 0.8 ],\n",
       "       [0.  , 0.  , 1.  ]])"
      ]
     },
     "execution_count": 159,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "ina3 = make_pipeline(HankelCovariances(delays=[2, 4, 8, 12, 16], estimator='oas'),\n",
    "                    CSP(30),\n",
    "                    NuSVC())\n",
    "ina3.fit(X2,y2)\n",
    "p1 = ina3.predict(X1)\n",
    "p2 = ina3.predict(X2)\n",
    "print(\"tsc-svc hard train = \",accuracy_score(y2,p2),\" test = \",accuracy_score(y1,p1))\n",
    "\n",
    "c = np.around(cm(y1,p1,normalize='true'),2)\n",
    "c\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 34,
   "metadata": {},
   "outputs": [],
   "source": [
    "# erp\n",
    "erp1 = make_pipeline(XdawnCovariances(6, estimator='oas'),\n",
    "                    TangentSpace(metric='riemann',tsupdate=True),\n",
    "                    LogisticRegression(penalty='l2'))\n",
    "erp2 = make_pipeline(Xdawn(12, estimator='oas'),\n",
    "                    EpochsVectorizer(),\n",
    "                    LogisticRegression(penalty='l2'))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 39,
   "metadata": {},
   "outputs": [],
   "source": [
    "# induced activity\n",
    "# removed electrode selection from the base classifier\n",
    "baseclf = make_pipeline(TangentSpace(metric='riemann', tsupdate=True),\n",
    "                        LogisticRegression(penalty='l1',solver='liblinear'))\n",
    "ina1 = make_pipeline(CospCovariances(fs=250, window=125, overlap=0, fmax=45, fmin=0.1),\n",
    "                    CospBoostingClassifier(baseclf))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 38,
   "metadata": {},
   "outputs": [],
   "source": [
    "ina2 = make_pipeline(HankelCovariances(delays=[2, 4, 8, 12, 16], estimator='oas'),\n",
    "                    TangentSpace(metric='logeuclid'),\n",
    "                    LogisticRegression(penalty='l1',solver='liblinear'))\n",
    "\n",
    "ina3 = make_pipeline(HankelCovariances(delays=[2, 4, 8, 12, 16], estimator='oas'),\n",
    "                    CSP(30),\n",
    "                    LogisticRegression(penalty='l1',solver='liblinear'))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 42,
   "metadata": {},
   "outputs": [],
   "source": [
    "# ,(\"ina1\",ina1)\n",
    "eclf1 = VC(estimators = [(\"erp1\",erp1),(\"erp2\",erp2),(\"ina2\",ina2),(\"ina3\",ina3)],\n",
    "                         voting='hard',n_jobs=-1)\n",
    "\n",
    "eclf2 = VC(estimators = [(\"erp1\",erp1),(\"erp2\",erp2),(\"ina2\",ina2),(\"ina3\",ina3)],\n",
    "                         voting='soft',n_jobs=-1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 35,
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "erp1.fit(X2,y2)\n",
    "print(\"erp1 train = \",erp1.score(X2,y2),\" test = \",erp1.score(X1,y1))\n",
    "erp2.fit(X2,y2)\n",
    "print(\"erp2 train = \",erp2.score(X2,y2),\" test = \",erp2.score(X1,y1))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 35,
   "metadata": {
    "collapsed": true,
    "jupyter": {
     "outputs_hidden": true
    },
    "tags": []
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "erp1 train =  1.0  test =  0.31543624161073824\n",
      "erp2 train =  0.3333333333333333  test =  0.3333333333333333\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "D:\\Softwares\\Anaconda3\\envs\\neuroerg\\lib\\site-packages\\pyriemann\\utils\\base.py:14: RuntimeWarning: invalid value encountered in log\n",
      "  eigvals = np.diag(operator(eigvals))\n"
     ]
    },
    {
     "ename": "ValueError",
     "evalue": "Covariance matrices must be positive definite. Add regularization to avoid this error.",
     "output_type": "error",
     "traceback": [
      "\u001b[1;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[1;31mValueError\u001b[0m                                Traceback (most recent call last)",
      "\u001b[1;32m<ipython-input-35-a158a3183e9f>\u001b[0m in \u001b[0;36m<module>\u001b[1;34m\u001b[0m\n\u001b[0;32m      3\u001b[0m \u001b[0merp2\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mfit\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mX2\u001b[0m\u001b[1;33m,\u001b[0m\u001b[0my2\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m      4\u001b[0m \u001b[0mprint\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;34m\"erp2 train = \"\u001b[0m\u001b[1;33m,\u001b[0m\u001b[0merp2\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mscore\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mX2\u001b[0m\u001b[1;33m,\u001b[0m\u001b[0my2\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m,\u001b[0m\u001b[1;34m\" test = \"\u001b[0m\u001b[1;33m,\u001b[0m\u001b[0merp2\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mscore\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mX1\u001b[0m\u001b[1;33m,\u001b[0m\u001b[0my1\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m----> 5\u001b[1;33m \u001b[0mina1\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mfit\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mX2\u001b[0m\u001b[1;33m,\u001b[0m\u001b[0my2\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m      6\u001b[0m \u001b[0mprint\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;34m\"ina1 train = \"\u001b[0m\u001b[1;33m,\u001b[0m\u001b[0mina1\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mscore\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mX2\u001b[0m\u001b[1;33m,\u001b[0m\u001b[0my2\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m,\u001b[0m\u001b[1;34m\" test = \"\u001b[0m\u001b[1;33m,\u001b[0m\u001b[0mina1\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mscore\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mX1\u001b[0m\u001b[1;33m,\u001b[0m\u001b[0my1\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m      7\u001b[0m \u001b[0mina2\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mfit\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mX2\u001b[0m\u001b[1;33m,\u001b[0m\u001b[0my2\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32mD:\\Softwares\\Anaconda3\\envs\\neuroerg\\lib\\site-packages\\sklearn\\pipeline.py\u001b[0m in \u001b[0;36mfit\u001b[1;34m(self, X, y, **fit_params)\u001b[0m\n\u001b[0;32m    344\u001b[0m             \u001b[1;32mif\u001b[0m \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0m_final_estimator\u001b[0m \u001b[1;33m!=\u001b[0m \u001b[1;34m'passthrough'\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    345\u001b[0m                 \u001b[0mfit_params_last_step\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mfit_params_steps\u001b[0m\u001b[1;33m[\u001b[0m\u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0msteps\u001b[0m\u001b[1;33m[\u001b[0m\u001b[1;33m-\u001b[0m\u001b[1;36m1\u001b[0m\u001b[1;33m]\u001b[0m\u001b[1;33m[\u001b[0m\u001b[1;36m0\u001b[0m\u001b[1;33m]\u001b[0m\u001b[1;33m]\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m--> 346\u001b[1;33m                 \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0m_final_estimator\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mfit\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mXt\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0my\u001b[0m\u001b[1;33m,\u001b[0m \u001b[1;33m**\u001b[0m\u001b[0mfit_params_last_step\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m    347\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    348\u001b[0m         \u001b[1;32mreturn\u001b[0m \u001b[0mself\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32m<ipython-input-32-4e835b13c79f>\u001b[0m in \u001b[0;36mfit\u001b[1;34m(self, X, y)\u001b[0m\n\u001b[0;32m     38\u001b[0m         \u001b[1;32mfor\u001b[0m \u001b[0mi\u001b[0m \u001b[1;32min\u001b[0m \u001b[0mrange\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mX\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mshape\u001b[0m\u001b[1;33m[\u001b[0m\u001b[1;33m-\u001b[0m\u001b[1;36m1\u001b[0m\u001b[1;33m]\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m     39\u001b[0m             \u001b[0mclf\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mdeepcopy\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mbaseclf\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m---> 40\u001b[1;33m             \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mclfs_\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mappend\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mclf\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mfit\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mX\u001b[0m\u001b[1;33m[\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m,\u001b[0m \u001b[1;33m:\u001b[0m\u001b[1;33m,\u001b[0m \u001b[1;33m:\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mi\u001b[0m\u001b[1;33m]\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0my\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m     41\u001b[0m         \u001b[1;32mreturn\u001b[0m \u001b[0mself\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m     42\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32mD:\\Softwares\\Anaconda3\\envs\\neuroerg\\lib\\site-packages\\sklearn\\pipeline.py\u001b[0m in \u001b[0;36mfit\u001b[1;34m(self, X, y, **fit_params)\u001b[0m\n\u001b[0;32m    339\u001b[0m         \"\"\"\n\u001b[0;32m    340\u001b[0m         \u001b[0mfit_params_steps\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0m_check_fit_params\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;33m**\u001b[0m\u001b[0mfit_params\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m--> 341\u001b[1;33m         \u001b[0mXt\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0m_fit\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mX\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0my\u001b[0m\u001b[1;33m,\u001b[0m \u001b[1;33m**\u001b[0m\u001b[0mfit_params_steps\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m    342\u001b[0m         with _print_elapsed_time('Pipeline',\n\u001b[0;32m    343\u001b[0m                                  self._log_message(len(self.steps) - 1)):\n",
      "\u001b[1;32mD:\\Softwares\\Anaconda3\\envs\\neuroerg\\lib\\site-packages\\sklearn\\pipeline.py\u001b[0m in \u001b[0;36m_fit\u001b[1;34m(self, X, y, **fit_params_steps)\u001b[0m\n\u001b[0;32m    305\u001b[0m                 \u001b[0mmessage_clsname\u001b[0m\u001b[1;33m=\u001b[0m\u001b[1;34m'Pipeline'\u001b[0m\u001b[1;33m,\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    306\u001b[0m                 \u001b[0mmessage\u001b[0m\u001b[1;33m=\u001b[0m\u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0m_log_message\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mstep_idx\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m,\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m--> 307\u001b[1;33m                 **fit_params_steps[name])\n\u001b[0m\u001b[0;32m    308\u001b[0m             \u001b[1;31m# Replace the transformer of the step with the fitted\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    309\u001b[0m             \u001b[1;31m# transformer. This is necessary when loading the transformer\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32mD:\\Softwares\\Anaconda3\\envs\\neuroerg\\lib\\site-packages\\joblib\\memory.py\u001b[0m in \u001b[0;36m__call__\u001b[1;34m(self, *args, **kwargs)\u001b[0m\n\u001b[0;32m    350\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    351\u001b[0m     \u001b[1;32mdef\u001b[0m \u001b[0m__call__\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mself\u001b[0m\u001b[1;33m,\u001b[0m \u001b[1;33m*\u001b[0m\u001b[0margs\u001b[0m\u001b[1;33m,\u001b[0m \u001b[1;33m**\u001b[0m\u001b[0mkwargs\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m--> 352\u001b[1;33m         \u001b[1;32mreturn\u001b[0m \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mfunc\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;33m*\u001b[0m\u001b[0margs\u001b[0m\u001b[1;33m,\u001b[0m \u001b[1;33m**\u001b[0m\u001b[0mkwargs\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m    353\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    354\u001b[0m     \u001b[1;32mdef\u001b[0m \u001b[0mcall_and_shelve\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mself\u001b[0m\u001b[1;33m,\u001b[0m \u001b[1;33m*\u001b[0m\u001b[0margs\u001b[0m\u001b[1;33m,\u001b[0m \u001b[1;33m**\u001b[0m\u001b[0mkwargs\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32mD:\\Softwares\\Anaconda3\\envs\\neuroerg\\lib\\site-packages\\sklearn\\pipeline.py\u001b[0m in \u001b[0;36m_fit_transform_one\u001b[1;34m(transformer, X, y, weight, message_clsname, message, **fit_params)\u001b[0m\n\u001b[0;32m    752\u001b[0m     \u001b[1;32mwith\u001b[0m \u001b[0m_print_elapsed_time\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mmessage_clsname\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mmessage\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    753\u001b[0m         \u001b[1;32mif\u001b[0m \u001b[0mhasattr\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mtransformer\u001b[0m\u001b[1;33m,\u001b[0m \u001b[1;34m'fit_transform'\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m--> 754\u001b[1;33m             \u001b[0mres\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mtransformer\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mfit_transform\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mX\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0my\u001b[0m\u001b[1;33m,\u001b[0m \u001b[1;33m**\u001b[0m\u001b[0mfit_params\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m    755\u001b[0m         \u001b[1;32melse\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    756\u001b[0m             \u001b[0mres\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mtransformer\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mfit\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mX\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0my\u001b[0m\u001b[1;33m,\u001b[0m \u001b[1;33m**\u001b[0m\u001b[0mfit_params\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mtransform\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mX\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32mD:\\Softwares\\Anaconda3\\envs\\neuroerg\\lib\\site-packages\\pyriemann\\tangentspace.py\u001b[0m in \u001b[0;36mfit_transform\u001b[1;34m(self, X, y, sample_weight)\u001b[0m\n\u001b[0;32m    167\u001b[0m         \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0m_check_reference_points\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mX\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    168\u001b[0m         self.reference_ = mean_covariance(X, metric=self.metric,\n\u001b[1;32m--> 169\u001b[1;33m                                           sample_weight=sample_weight)\n\u001b[0m\u001b[0;32m    170\u001b[0m         \u001b[1;32mreturn\u001b[0m \u001b[0mtangent_space\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mX\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mreference_\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    171\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32mD:\\Softwares\\Anaconda3\\envs\\neuroerg\\lib\\site-packages\\pyriemann\\utils\\mean.py\u001b[0m in \u001b[0;36mmean_covariance\u001b[1;34m(covmats, metric, sample_weight, *args)\u001b[0m\n\u001b[0;32m    333\u001b[0m         \u001b[0mC\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mmetric\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mcovmats\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0msample_weight\u001b[0m\u001b[1;33m=\u001b[0m\u001b[0msample_weight\u001b[0m\u001b[1;33m,\u001b[0m \u001b[1;33m*\u001b[0m\u001b[0margs\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    334\u001b[0m     \u001b[1;32melse\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m--> 335\u001b[1;33m         \u001b[0mC\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mmean_methods\u001b[0m\u001b[1;33m[\u001b[0m\u001b[0mmetric\u001b[0m\u001b[1;33m]\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mcovmats\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0msample_weight\u001b[0m\u001b[1;33m=\u001b[0m\u001b[0msample_weight\u001b[0m\u001b[1;33m,\u001b[0m \u001b[1;33m*\u001b[0m\u001b[0margs\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m    336\u001b[0m     \u001b[1;32mreturn\u001b[0m \u001b[0mC\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    337\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32mD:\\Softwares\\Anaconda3\\envs\\neuroerg\\lib\\site-packages\\pyriemann\\utils\\mean.py\u001b[0m in \u001b[0;36mmean_riemann\u001b[1;34m(covmats, tol, maxiter, init, sample_weight)\u001b[0m\n\u001b[0;32m     63\u001b[0m         \u001b[0mcrit\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mnp\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mlinalg\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mnorm\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mJ\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mord\u001b[0m\u001b[1;33m=\u001b[0m\u001b[1;34m'fro'\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m     64\u001b[0m         \u001b[0mh\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mnu\u001b[0m \u001b[1;33m*\u001b[0m \u001b[0mcrit\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m---> 65\u001b[1;33m         \u001b[0mC\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mnp\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mdot\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mnp\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mdot\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mC12\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mexpm\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mnu\u001b[0m \u001b[1;33m*\u001b[0m \u001b[0mJ\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mC12\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m     66\u001b[0m         \u001b[1;32mif\u001b[0m \u001b[0mh\u001b[0m \u001b[1;33m<\u001b[0m \u001b[0mtau\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m     67\u001b[0m             \u001b[0mnu\u001b[0m \u001b[1;33m=\u001b[0m \u001b[1;36m0.95\u001b[0m \u001b[1;33m*\u001b[0m \u001b[0mnu\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32mD:\\Softwares\\Anaconda3\\envs\\neuroerg\\lib\\site-packages\\pyriemann\\utils\\base.py\u001b[0m in \u001b[0;36mexpm\u001b[1;34m(Ci)\u001b[0m\n\u001b[0;32m     62\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m     63\u001b[0m     \"\"\"\n\u001b[1;32m---> 64\u001b[1;33m     \u001b[1;32mreturn\u001b[0m \u001b[0m_matrix_operator\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mCi\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mnp\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mexp\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m     65\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m     66\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32mD:\\Softwares\\Anaconda3\\envs\\neuroerg\\lib\\site-packages\\pyriemann\\utils\\base.py\u001b[0m in \u001b[0;36m_matrix_operator\u001b[1;34m(Ci, operator)\u001b[0m\n\u001b[0;32m      9\u001b[0m     \u001b[1;32mif\u001b[0m \u001b[0mCi\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mdtype\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mchar\u001b[0m \u001b[1;32min\u001b[0m \u001b[0mtypecodes\u001b[0m\u001b[1;33m[\u001b[0m\u001b[1;34m'AllFloat'\u001b[0m\u001b[1;33m]\u001b[0m \u001b[1;32mand\u001b[0m \u001b[1;32mnot\u001b[0m \u001b[0mnp\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0misfinite\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mCi\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mall\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m     10\u001b[0m         raise ValueError(\n\u001b[1;32m---> 11\u001b[1;33m             \u001b[1;34m\"Covariance matrices must be positive definite. Add \"\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m     12\u001b[0m             \"regularization to avoid this error.\")\n\u001b[0;32m     13\u001b[0m     \u001b[0meigvals\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0meigvects\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mscipy\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mlinalg\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0meigh\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mCi\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mcheck_finite\u001b[0m\u001b[1;33m=\u001b[0m\u001b[1;32mFalse\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;31mValueError\u001b[0m: Covariance matrices must be positive definite. Add regularization to avoid this error."
     ]
    }
   ],
   "source": [
    "ina1.fit(X2,y2)\n",
    "print(\"ina1 train = \",ina1.score(X2,y2),\" test = \",ina1.score(X1,y1))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 41,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "ina2 train =  0.9977628635346756  test =  0.30648769574944074\n",
      "ina3 train =  0.9686800894854586  test =  0.38926174496644295\n"
     ]
    }
   ],
   "source": [
    "ina2.fit(X2,y2)\n",
    "print(\"ina2 train = \",ina2.score(X2,y2),\" test = \",ina2.score(X1,y1))\n",
    "ina3.fit(X2,y2)\n",
    "print(\"ina3 train = \",ina3.score(X2,y2),\" test = \",ina3.score(X1,y1))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 43,
   "metadata": {
    "tags": []
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "eclf1 train =  0.9821029082774049  test =  0.2595078299776286\n",
      "eclf2 train =  1.0  test =  0.38926174496644295\n"
     ]
    }
   ],
   "source": [
    "eclf1.fit(X2,y2)\n",
    "print(\"eclf1 train = \",eclf1.score(X2,y2),\" test = \",eclf1.score(X1,y1))\n",
    "eclf2.fit(X2,y2)\n",
    "print(\"eclf2 train = \",eclf2.score(X2,y2),\" test = \",eclf2.score(X1,y1))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [],
   "source": [
    "#XdawnCovariances - for better generalization capability set applyfilters=False"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(447, 61, 61)"
      ]
     },
     "execution_count": 18,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# %run funcs.ipynb\n",
    "cov1.shape\n",
    "# y1.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 48,
   "metadata": {},
   "outputs": [],
   "source": [
    "# can I get the class covariances for the 2 sessions to see if they are similar or different?"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 48,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(447, 61, 61)\n"
     ]
    }
   ],
   "source": [
    "# print(cov2.shape)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 36,
   "metadata": {
    "jupyter": {
     "source_hidden": true
    }
   },
   "outputs": [],
   "source": [
    "# Misc\n",
    "# covs_train[0].shape\n",
    "# y1\n",
    "# class1_cov.shape\n",
    "\n",
    "# a = [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,\n",
    "#        0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,\n",
    "#        0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,\n",
    "#        0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,\n",
    "#        0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,\n",
    "#        0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,\n",
    "#        0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]\n",
    "# len(a)\n",
    "\n",
    "# y1.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 37,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Best of knn, svc and gradboost"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 147,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "tsc-svc hard train =  0.8903803131991052  test =  0.6241610738255033\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "array([[0.95, 0.03, 0.01],\n",
       "       [0.36, 0.6 , 0.05],\n",
       "       [0.25, 0.43, 0.32]])"
      ]
     },
     "execution_count": 147,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "tscs2 = make_pipeline(Covariances(estimator='oas'),tsc(tsupdate=True,clf=SVC(gamma='auto')))\n",
    "tscs2.fit(X2,y2)\n",
    "p1 = tscs2.predict(X1)\n",
    "p2 = tscs2.predict(X2)\n",
    "print(\"tsc-svc hard train = \",accuracy_score(y2,p2),\" test = \",accuracy_score(y1,p1))\n",
    "\n",
    "c = np.around(cm(y1,p1,normalize='true'),2)\n",
    "c"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 107,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "svocs2 hard train =  0.901565995525727  test =  0.7002237136465325\n"
     ]
    }
   ],
   "source": [
    "tscs2 = make_pipeline(Covariances(estimator='oas'),tsc(tsupdate=True,clf=SVC(gamma='auto')))\n",
    "svocs2 = VC(estimators = [('mdm',mdmc),('Fgmdm',Fgmdmc),('tsc',tscs2)], voting='hard',n_jobs=-1)\n",
    "svocs2.fit(X2,y2)\n",
    "ps1 = svocs2.predict(X1)\n",
    "ps2 = svocs2.predict(X2)\n",
    "print(\"svocs2 hard train = \",accuracy_score(y2,ps2),\" test = \",accuracy_score(y1,ps1))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 120,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([[0.88, 0.03, 0.09],\n",
       "       [0.05, 0.9 , 0.05],\n",
       "       [0.12, 0.56, 0.32]])"
      ]
     },
     "execution_count": 120,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "csx = np.around(cm(y1,ps1,normalize='true'),2)\n",
    "csx"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 123,
   "metadata": {
    "tags": []
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([[0.91, 0.08, 0.01],\n",
       "       [0.19, 0.8 , 0.01],\n",
       "       [0.  , 0.  , 1.  ]])"
      ]
     },
     "execution_count": 123,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "csxt = np.around(cm(y1,ps2,normalize='true'),2)\n",
    "csxt"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 148,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "tsc-nusvc hard train =  1.0  test =  0.6241610738255033\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "array([[0.91, 0.07, 0.01],\n",
       "       [0.29, 0.65, 0.06],\n",
       "       [0.31, 0.38, 0.31]])"
      ]
     },
     "execution_count": 148,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "tscs3 = make_pipeline(Covariances(estimator='oas'),tsc(tsupdate=True,clf=NuSVC()))\n",
    "tscs3.fit(X2,y2)\n",
    "p1 = tscs3.predict(X1)\n",
    "p2 = tscs3.predict(X2)\n",
    "print(\"tsc-nusvc hard train = \",accuracy_score(y2,p2),\" test = \",accuracy_score(y1,p1))\n",
    "\n",
    "c = np.around(cm(y1,p1,normalize='true'),2)\n",
    "c"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 105,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "svocs3 hard train =  1.0  test =  0.6756152125279642\n"
     ]
    }
   ],
   "source": [
    "tscs3 = make_pipeline(Covariances(estimator='oas'),tsc(tsupdate=True,clf=NuSVC()))\n",
    "svocs3 = VC(estimators = [('mdm',mdmc),('Fgmdm',Fgmdmc),('tsc',tscs3)], voting='hard',n_jobs=-1)\n",
    "svocs3.fit(X2,y2)\n",
    "pn1 = svocs3.predict(X1)\n",
    "pn2 = svocs3.predict(X2)\n",
    "print(\"svocs3 hard train = \",accuracy_score(y2,pn2),\" test = \",accuracy_score(y1,pn1))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 119,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([[0.84, 0.07, 0.09],\n",
       "       [0.06, 0.88, 0.06],\n",
       "       [0.16, 0.53, 0.31]])"
      ]
     },
     "execution_count": 119,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "cnx = np.around(cm(y1,pn1,normalize='true'),2)\n",
    "cnx"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 150,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "tsc-skknn hard train =  1.0  test =  0.5794183445190156\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "array([[0.95, 0.03, 0.02],\n",
       "       [0.36, 0.54, 0.11],\n",
       "       [0.39, 0.36, 0.25]])"
      ]
     },
     "execution_count": 150,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "tsck2 = make_pipeline(Covariances(estimator='oas'),tsc(tsupdate=True,clf=sknn(n_neighbors = 14,n_jobs=-1,weights='distance')))\n",
    "tsck2.fit(X2,y2)\n",
    "p1 = tsck2.predict(X1)\n",
    "p2 = tsck2.predict(X2)\n",
    "print(\"tsc-skknn hard train = \",accuracy_score(y2,p2),\" test = \",accuracy_score(y1,p1))\n",
    "\n",
    "c = np.around(cm(y1,p1,normalize='true'),2)\n",
    "c"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 121,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "svock2 hard train =  1.0  test =  0.6554809843400448\n"
     ]
    }
   ],
   "source": [
    "tsck2 = make_pipeline(Covariances(estimator='oas'),tsc(tsupdate=True,clf=sknn(n_neighbors = 14,n_jobs=-1,weights='distance')))\n",
    "svock2 = VC(estimators = [('mdm',mdmc),('Fgmdm',Fgmdmc),('tsc',tsck2)], voting='hard',n_jobs=-1)\n",
    "svock2.fit(X2,y2)\n",
    "pk1 = svock2.predict(X1)\n",
    "pk2 = svock2.predict(X2)\n",
    "print(\"svock2 hard train = \",accuracy_score(y2,pk2),\" test = \",accuracy_score(y1,pk1))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 122,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([[0.87, 0.03, 0.1 ],\n",
       "       [0.05, 0.85, 0.11],\n",
       "       [0.19, 0.56, 0.25]])"
      ]
     },
     "execution_count": 122,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "ckx = np.around(cm(y1,pk1,normalize='true'),2)\n",
    "ckx"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 149,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "D:\\Softwares\\Anaconda3\\envs\\neuroerg\\lib\\site-packages\\xgboost\\sklearn.py:1146: UserWarning: The use of label encoder in XGBClassifier is deprecated and will be removed in a future release. To remove this warning, do the following: 1) Pass option use_label_encoder=False when constructing XGBClassifier object; and 2) Encode your labels (y) as integers starting with 0, i.e. 0, 1, 2, ..., [num_class - 1].\n",
      "  warnings.warn(label_encoder_deprecation_msg, UserWarning)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[04:43:56] WARNING: C:/Users/Administrator/workspace/xgboost-win64_release_1.4.0/src/learner.cc:1095: Starting in XGBoost 1.3.0, the default evaluation metric used with the objective 'multi:softprob' was changed from 'merror' to 'mlogloss'. Explicitly set eval_metric if you'd like to restore the old behavior.\n",
      "tscx-nusvc hard train =  1.0  test =  0.5883668903803132\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "array([[0.66, 0.18, 0.15],\n",
       "       [0.23, 0.47, 0.3 ],\n",
       "       [0.22, 0.15, 0.63]])"
      ]
     },
     "execution_count": 149,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "tscx = make_pipeline(Covariances(estimator='oas'),tsc(tsupdate=True,clf=xgb.XGBClassifier(n_estimators = 1500,max_depth = 15,learning_rate= 0.3,reg_lambda = 5,n_jobs=-1)))\n",
    "tscx.fit(X2,y2)\n",
    "p1 = tscx.predict(X1)\n",
    "p2 = tscx.predict(X2)\n",
    "print(\"tscx-nusvc hard train = \",accuracy_score(y2,p2),\" test = \",accuracy_score(y1,p1))\n",
    "\n",
    "c = np.around(cm(y1,p1,normalize='true'),2)\n",
    "c"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 132,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "svoc3 hard train =  1.0  test =  0.6442953020134228\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "array([[0.67, 0.12, 0.21],\n",
       "       [0.05, 0.77, 0.17],\n",
       "       [0.13, 0.38, 0.49]])"
      ]
     },
     "execution_count": 132,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# xgb gives much better result for 3rd class\n",
    "tscx = make_pipeline(Covariances(estimator='oas'),tsc(tsupdate=True,clf=xgb.XGBClassifier(n_estimators = 1500,max_depth = 15,learning_rate= 0.3,reg_lambda = 5,n_jobs=-1)))\n",
    "svoc3 = VC(estimators = [('mdm',mdmc),('Fgmdm',Fgmdmc),('tsc',tscx)], voting='hard',n_jobs=-1)\n",
    "svoc3.fit(X2,y2)\n",
    "p1 = svoc3.predict(X1)\n",
    "p2 = svoc3.predict(X2)\n",
    "print(\"svoc3 hard train = \",accuracy_score(y2,p2),\" test = \",accuracy_score(y1,p1))\n",
    "cx = np.around(cm(y1,p1,normalize='true'),2)\n",
    "cx"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 133,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "svocs2 hard train =  0.9373601789709173  test =  0.639821029082774\n"
     ]
    }
   ],
   "source": [
    "tscx = make_pipeline(Covariances(estimator='oas'),tsc(tsupdate=True,clf=xgb.XGBClassifier(n_estimators = 1500,max_depth = 15,learning_rate= 0.3,reg_lambda = 5,n_jobs=-1)))\n",
    "tscs2 = make_pipeline(Covariances(estimator='oas'),tsc(tsupdate=True,clf=SVC(gamma='auto')))\n",
    "svocsx2 = VC(estimators = [('mdm',mdmc),('Fgmdm',Fgmdmc),('tsc',tscs2),('tscx',tscx)], voting='hard',n_jobs=-1)\n",
    "svocsx2.fit(X2,y2)\n",
    "psx1 = svocsx2.predict(X1)\n",
    "psx2 = svocsx2.predict(X2)\n",
    "print(\"svocs2 hard train = \",accuracy_score(y2,psx2),\" test = \",accuracy_score(y1,psx1))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 134,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([[0.77, 0.13, 0.11],\n",
       "       [0.16, 0.79, 0.05],\n",
       "       [0.11, 0.53, 0.36]])"
      ]
     },
     "execution_count": 134,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "csx = np.around(cm(y1,psx1,normalize='true'),2)\n",
    "csx"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 151,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "ms hard train =  0.9105145413870246  test =  0.6174496644295302\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "array([[0.77, 0.19, 0.03],\n",
       "       [0.23, 0.59, 0.17],\n",
       "       [0.24, 0.27, 0.49]])"
      ]
     },
     "execution_count": 151,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "ms = VC(estimators = [('mdm',mdmc),('tscx',tscx)],voting='hard',n_jobs=-1)\n",
    "ms.fit(X2,y2)\n",
    "p1 = ms.predict(X1)\n",
    "p2 = ms.predict(X2)\n",
    "print(\"ms hard train = \",accuracy_score(y2,p2),\" test = \",accuracy_score(y1,p1))\n",
    "\n",
    "cf = np.around(cm(y1,p1,normalize='true'),2)\n",
    "cf"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 152,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "ms hard train =  0.9485458612975392  test =  0.5995525727069351\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "array([[0.46, 0.41, 0.13],\n",
       "       [0.05, 0.57, 0.38],\n",
       "       [0.06, 0.17, 0.77]])"
      ]
     },
     "execution_count": 152,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "ms = VC(estimators = [('mdm',mdmc),('tscx',tscx)],voting='soft',n_jobs=-1)\n",
    "ms.fit(X2,y2)\n",
    "p1 = ms.predict(X1)\n",
    "p2 = ms.predict(X2)\n",
    "print(\"ms hard train = \",accuracy_score(y2,p2),\" test = \",accuracy_score(y1,p1))\n",
    "\n",
    "cf = np.around(cm(y1,p1,normalize='true'),2)\n",
    "cf"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 153,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "ms hard train =  1.0  test =  0.6510067114093959\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "array([[0.83, 0.14, 0.03],\n",
       "       [0.22, 0.57, 0.21],\n",
       "       [0.23, 0.21, 0.56]])"
      ]
     },
     "execution_count": 153,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "ms = VC(estimators = [('mdm',mdmc),('tscx',tscx),('NuSVC',tscs3)],voting='hard',n_jobs=-1)\n",
    "ms.fit(X2,y2)\n",
    "p1 = ms.predict(X1)\n",
    "p2 = ms.predict(X2)\n",
    "print(\"ms hard train = \",accuracy_score(y2,p2),\" test = \",accuracy_score(y1,p1))\n",
    "\n",
    "cf = np.around(cm(y1,p1,normalize='true'),2)\n",
    "cf"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 155,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "ms hard train =  1.0  test =  0.6666666666666666\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "array([[0.7 , 0.23, 0.07],\n",
       "       [0.09, 0.67, 0.24],\n",
       "       [0.11, 0.26, 0.63]])"
      ]
     },
     "execution_count": 155,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "tscs3 = make_pipeline(Covariances(estimator='oas'),tsc(tsupdate=True,clf=NuSVC(probability=True)))\n",
    "ms = VC(estimators = [('mdm',mdmc),('tscx',tscx),('NuSVC',tscs3)],voting='soft',n_jobs=-1)\n",
    "ms.fit(X2,y2)\n",
    "p1 = ms.predict(X1)\n",
    "p2 = ms.predict(X2)\n",
    "print(\"ms hard train = \",accuracy_score(y2,p2),\" test = \",accuracy_score(y1,p1))\n",
    "\n",
    "cf = np.around(cm(y1,p1,normalize='true'),2)\n",
    "cf"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 135,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "svocnsx2 hard train =  0.9261744966442953  test =  0.5861297539149888\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "array([[0.96, 0.03, 0.01],\n",
       "       [0.43, 0.53, 0.04],\n",
       "       [0.38, 0.35, 0.27]])"
      ]
     },
     "execution_count": 135,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "tscx = make_pipeline(Covariances(estimator='oas'),tsc(tsupdate=True,clf=xgb.XGBClassifier(n_estimators = 1500,max_depth = 15,learning_rate= 0.3,reg_lambda = 5,n_jobs=-1)))\n",
    "tscs2 = make_pipeline(Covariances(estimator='oas'),tsc(tsupdate=True,clf=SVC(gamma='auto')))\n",
    "svocnsx2 = VC(estimators = [('tsc',tscs2),('tscx',tscx)], voting='hard',n_jobs=-1)\n",
    "svocnsx2.fit(X2,y2)\n",
    "pnsx1 = svocnsx2.predict(X1)\n",
    "pnsx2 = svocnsx2.predict(X2)\n",
    "print(\"svocnsx2 hard train = \",accuracy_score(y2,pnsx2),\" test = \",accuracy_score(y1,pnsx1))\n",
    "csx = np.around(cm(y1,pnsx1,normalize='true'),2)\n",
    "csx"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 142,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "sf1 hard train =  0.8747203579418344  test =  0.6599552572706935\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "array([[0.77, 0.19, 0.04],\n",
       "       [0.16, 0.62, 0.21],\n",
       "       [0.11, 0.3 , 0.59]])"
      ]
     },
     "execution_count": 142,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "tscx = make_pipeline(Covariances(estimator='oas'),tsc(tsupdate=True,clf=xgb.XGBClassifier(n_estimators = 1500,max_depth = 15,learning_rate= 0.3,reg_lambda = 5,n_jobs=-1)))\n",
    "tscs2 = make_pipeline(Covariances(estimator='oas'),tsc(tsupdate=True,clf=SVC(gamma='auto')))\n",
    "mdmc = make_pipeline(Covariances(estimator='oas'),MDM(n_jobs=-1))\n",
    "sf1 = VC(estimators = [('tsc',tscs2),('tscx',tscx),('mdm',mdmc)],weights = [1,1,2], voting='hard',n_jobs=-1)\n",
    "sf1.fit(X2,y2)\n",
    "p1 = sf1.predict(X1)\n",
    "p2 = sf1.predict(X2)\n",
    "print(\"sf1 hard train = \",accuracy_score(y2,p2),\" test = \",accuracy_score(y1,p1))\n",
    "cf = np.around(cm(y1,p1,normalize='true'),2)\n",
    "cf"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 144,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "sf1 hard train =  0.843400447427293  test =  0.5257270693512305\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "array([[0.28, 0.63, 0.09],\n",
       "       [0.03, 0.6 , 0.38],\n",
       "       [0.03, 0.27, 0.7 ]])"
      ]
     },
     "execution_count": 144,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "tscx = make_pipeline(Covariances(estimator='oas'),tsc(tsupdate=True,clf=xgb.XGBClassifier(n_estimators = 1500,max_depth = 15,learning_rate= 0.3,reg_lambda = 5,n_jobs=-1)))\n",
    "tscs2 = make_pipeline(Covariances(estimator='oas'),tsc(tsupdate=True,clf=SVC(gamma='auto')))\n",
    "mdmc = make_pipeline(Covariances(estimator='oas'),MDM(n_jobs=-1))\n",
    "tsck2 = make_pipeline(Covariances(estimator='oas'),tsc(tsupdate=True,clf=sknn(n_neighbors = 14,n_jobs=-1,weights='distance')))\n",
    "\n",
    "sf1 = VC(estimators = [('tsc',tscs2),('tscx',tscx),('tskc',tsck2),('mdm',mdmc)],weights = [1,1,1,4], voting='hard',n_jobs=-1)\n",
    "sf1.fit(X2,y2)\n",
    "p1 = sf1.predict(X1)\n",
    "p2 = sf1.predict(X2)\n",
    "print(\"sf1 hard train = \",accuracy_score(y2,p2),\" test = \",accuracy_score(y1,p1))\n",
    "cf = np.around(cm(y1,p1,normalize='true'),2)\n",
    "cf"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 146,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "sf1 hard train =  0.8814317673378076  test =  0.5749440715883669\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "array([[0.42, 0.49, 0.09],\n",
       "       [0.03, 0.62, 0.34],\n",
       "       [0.05, 0.27, 0.68]])"
      ]
     },
     "execution_count": 146,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "tscx = make_pipeline(Covariances(estimator='oas'),tsc(tsupdate=True,clf=xgb.XGBClassifier(n_estimators = 1500,max_depth = 15,learning_rate= 0.3,reg_lambda = 5,n_jobs=-1)))\n",
    "tscs2 = make_pipeline(Covariances(estimator='oas'),tsc(tsupdate=True,clf=SVC(gamma='auto',probability=True)))\n",
    "mdmc = make_pipeline(Covariances(estimator='oas'),MDM(n_jobs=-1))\n",
    "tsck2 = make_pipeline(Covariances(estimator='oas'),tsc(tsupdate=True,clf=sknn(n_neighbors = 14,n_jobs=-1,weights='distance')))\n",
    "\n",
    "sf1 = VC(estimators = [('tsc',tscs2),('tscx',tscx),('tskc',tsck2),('mdm',mdmc)],weights = [1,1,1,4], voting='soft',n_jobs=-1)\n",
    "sf1.fit(X2,y2)\n",
    "p1 = sf1.predict(X1)\n",
    "p2 = sf1.predict(X2)\n",
    "print(\"sf1 hard train = \",accuracy_score(y2,p2),\" test = \",accuracy_score(y1,p1))\n",
    "cf = np.around(cm(y1,p1,normalize='true'),2)\n",
    "cf "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 137,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "indiv hard train =  0.843400447427293  test =  0.5257270693512305\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "array([[0.28, 0.63, 0.09],\n",
       "       [0.03, 0.6 , 0.38],\n",
       "       [0.03, 0.27, 0.7 ]])"
      ]
     },
     "execution_count": 137,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "mdmc = make_pipeline(Covariances(estimator='oas'),MDM(n_jobs=-1))\n",
    "mdmc.fit(X2,y2)\n",
    "p1 = mdmc.predict(X1)\n",
    "p2 = mdmc.predict(X2)\n",
    "print(\"indiv hard train = \",accuracy_score(y2,p2),\" test = \",accuracy_score(y1,p1))\n",
    "c = np.around(cm(y1,p1,normalize='true'),2)\n",
    "c"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 138,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "indiv hard train =  1.0  test =  0.3333333333333333\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "array([[0.  , 0.01, 0.99],\n",
       "       [0.  , 1.  , 0.  ],\n",
       "       [0.01, 0.99, 0.  ]])"
      ]
     },
     "execution_count": 138,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "Fgmdmc = make_pipeline(Covariances(estimator='oas'),FgMDM(n_jobs=-1))\n",
    "Fgmdmc.fit(X2,y2)\n",
    "p1 = Fgmdmc.predict(X1)\n",
    "p2 = Fgmdmc.predict(X2)\n",
    "print(\"indiv hard train = \",accuracy_score(y2,p2),\" test = \",accuracy_score(y1,p1))\n",
    "c = np.around(cm(y1,p1,normalize='true'),2)\n",
    "c"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 139,
   "metadata": {
    "tags": []
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "indiv hard train =  1.0  test =  0.2751677852348993\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "array([[0.25, 0.08, 0.67],\n",
       "       [0.42, 0.58, 0.  ],\n",
       "       [0.5 , 0.5 , 0.  ]])"
      ]
     },
     "execution_count": 139,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "Fgmdmc = make_pipeline(Covariances(estimator='oas'),FgMDM(tsupdate=True,n_jobs=-1))\n",
    "Fgmdmc.fit(X2,y2)\n",
    "p1 = Fgmdmc.predict(X1)\n",
    "p2 = Fgmdmc.predict(X2)\n",
    "print(\"indiv hard train = \",accuracy_score(y2,p2),\" test = \",accuracy_score(y1,p1))\n",
    "c = np.around(cm(y1,p1,normalize='true'),2)\n",
    "c"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 140,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "indiv hard train =  1.0  test =  0.5257270693512305\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "array([[0.68, 0.14, 0.17],\n",
       "       [0.26, 0.72, 0.01],\n",
       "       [0.36, 0.47, 0.17]])"
      ]
     },
     "execution_count": 140,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "tsc1 = make_pipeline(Covariances(estimator='oas'),tsc(tsupdate=True))\n",
    "tsc1.fit(X2,y2)\n",
    "p1 = tsc1.predict(X1)\n",
    "p2 = tsc1.predict(X2)\n",
    "print(\"indiv hard train = \",accuracy_score(y2,p2),\" test = \",accuracy_score(y1,p1))\n",
    "c = np.around(cm(y1,p1,normalize='true'),2)\n",
    "c"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 141,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "indiv hard train =  0.9038031319910514  test =  0.5413870246085011\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "array([[0.79, 0.17, 0.05],\n",
       "       [0.26, 0.68, 0.07],\n",
       "       [0.35, 0.49, 0.16]])"
      ]
     },
     "execution_count": 141,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "knn1 = make_pipeline(Covariances(estimator='oas'),knn(n_jobs=-1))\n",
    "knn1.fit(X2,y2)\n",
    "p1 = knn1.predict(X1)\n",
    "p2 = knn1.predict(X2)\n",
    "print(\"indiv hard train = \",accuracy_score(y2,p2),\" test = \",accuracy_score(y1,p1))\n",
    "c = np.around(cm(y1,p1,normalize='true'),2)\n",
    "c"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Stacking classifier\n",
    "sc1 = SC(estimators = [])"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.3"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}