vorace_test.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### TODO\n",
    "Note that the comparison in the end is to be made between the assembler using voting rules\n",
    "and the accuracy of the grid search model battery."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 284,
   "metadata": {
    "id": "5EUcdmIf0Avv"
   },
   "outputs": [],
   "source": [
    "# load packages\n",
    "import os\n",
    "import json\n",
    "import joblib\n",
    "\n",
    "from corankco.dataset import Dataset\n",
    "from corankco.scoringscheme import ScoringScheme\n",
    "from corankco.kemrankagg import KemRankAgg\n",
    "from corankco.algorithms.enumeration import Algorithm\n",
    "\n",
    "import random \n",
    "import pandas as pd\n",
    "import numpy as np\n",
    "import seaborn as sb\n",
    "import matplotlib.pyplot as plt\n",
    "from itertools import cycle\n",
    "\n",
    "from numba import jit\n",
    "from sklearn.preprocessing import LabelBinarizer\n",
    "\n",
    "from sklearn.pipeline import Pipeline\n",
    "from sklearn.preprocessing import MinMaxScaler\n",
    "from sklearn.model_selection import GridSearchCV\n",
    "from sklearn.ensemble import VotingClassifier\n",
    "from sklearn.model_selection import train_test_split\n",
    "from sklearn.preprocessing import StandardScaler\n",
    "\n",
    "from sklearn.ensemble import GradientBoostingClassifier\n",
    "from sklearn.tree import DecisionTreeClassifier\n",
    "from sklearn import svm\n",
    "from sklearn.neighbors import KNeighborsClassifier\n",
    "\n",
    "from sklearn.linear_model import LogisticRegression\n",
    "from sklearn.pipeline import make_pipeline\n",
    "\n",
    "from sklearn.metrics import ConfusionMatrixDisplay, RocCurveDisplay\n",
    "from sklearn.model_selection import learning_curve, ShuffleSplit\n",
    "from joblib import dump, load\n",
    "\n",
    "from sklearn.ensemble import RandomForestClassifier\n",
    "from sklearn.naive_bayes import ComplementNB, BernoulliNB \n",
    "from sklearn.metrics import roc_auc_score, accuracy_score , classification_report, ConfusionMatrixDisplay,precision_score,recall_score, f1_score,roc_auc_score,roc_curve\n",
    "from sklearn import metrics\n",
    "\n",
    "from sklearn.datasets import load_iris\n",
    "from sklearn.metrics import confusion_matrix\n",
    "from sklearn.model_selection import LearningCurveDisplay\n",
    "\n",
    "import warnings\n",
    "warnings.filterwarnings('ignore')\n",
    "\n",
    "from IPython.display import display\n",
    "pd.set_option('display.max_columns', None)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "GAU5a767AKz5"
   },
   "source": [
    "# FITTING PIPELINE"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "write = True\n",
    "\n",
    "def fit_models(X, Y, X_t, Y_t, target_names, models, params, run=\"\", folder=\"\", gs=1):\n",
    "    \n",
    "    if folder == \"\":\n",
    "        raise Exception(\"Please provide a folder...\")\n",
    "        \n",
    "    print(\"Fitting for parameters...\")\n",
    "    print(\"X:\", X)\n",
    "    print(\"-\"*50)\n",
    "    print(\"Y:\", Y)\n",
    "    print(\"-\"*50)\n",
    "    print(\"X_t:\", X_t)\n",
    "    print(\"-\"*50)\n",
    "    print(\"Y_t:\", Y_t)\n",
    "    print(\"-\"*50)\n",
    "    print(\"models:\", models)\n",
    "    print(\"-\"*50)\n",
    "    print(\"params:\", params)\n",
    "    print(\"-\"*50)\n",
    "    print(\"run:\", run)\n",
    "    print(\"-\"*50)\n",
    "    print(\"folder:\", folder)    \n",
    "            \n",
    "    @jit(target_backend='cuda')\n",
    "    def results(X,Y,model_name, model, data_type):\n",
    "        model_accuracy = accuracy_score(Y, model.predict(X)) \n",
    "        model_f1 = f1_score(Y, model.predict(X), average='weighted') \n",
    "        model_precision = precision_score(Y, model.predict(X),average='weighted') \n",
    "        model_recall = recall_score(Y, model.predict(X),average='weighted')\n",
    "\n",
    "        print('Model performance for ' + data_type + ' ' + model_name)\n",
    "        print(\"- Accuracy: {:.4f}\".format(model_accuracy))\n",
    "        print('- F1 score: {:4f}'.format(model_f1))\n",
    "        print('- Precision: {:4f}'.format(model_precision))\n",
    "        print('- Recall: {:4f}'.format(model_recall))\n",
    "\n",
    "        print('----------------------------------')\n",
    "        print('='*35)\n",
    "        \n",
    "        if write:\n",
    "            if os.path.isfile(\"./models/\" + folder + \"/summaries/\" + model_name + \".txt\"):\n",
    "                f = open(\"./models/\" + folder + \"/summaries/\" + model_name + \".txt\", 'a')\n",
    "            else:\n",
    "                f = open(\"./models/\" + folder + \"/summaries/\" + model_name + \".txt\", 'w')\n",
    "\n",
    "            f.write(\"Model performance for \" + data_type + \" \" + model_name + \"\\n\")\n",
    "            f.write(\"- Accuracy: \" + str(round(model_accuracy,4)) + \"\\n\")\n",
    "            f.write(\"- F1 score: \" + str(round(model_f1,4)) + \"\\n\")\n",
    "            f.write(\"- Precision: \" + str(round(model_precision,4)) + \"\\n\")\n",
    "            f.write(\"- Recall: \" + str(round(model_recall,4)) + \"\\n\")\n",
    "\n",
    "            f.write('----------------------------------\\n')\n",
    "            f.write('='*35 + \"\\n\")\n",
    "            f.close()\n",
    "    \n",
    "    @jit(target_backend='cuda')\n",
    "    def fit(model, X, Y):\n",
    "\n",
    "        model.fit(X,Y)\n",
    "        return model\n",
    "\n",
    "    def plots(model, X, Y, X_t, Y_t, name):\n",
    "        label_binarizer = LabelBinarizer().fit(Y)\n",
    "        y_onehot_test = label_binarizer.transform(Y_t)\n",
    "        y_score = model.predict_proba(X_t)\n",
    "        n_classes = len(np.unique(Y))\n",
    "        \n",
    "        colors = plt.cm.get_cmap('tab10', n_classes)\n",
    "\n",
    "        # Plot ROC curve for each class\n",
    "        for class_id in range(n_classes):\n",
    "            if n_classes > 2:  # Multiclass classification\n",
    "                y_onehot = np.zeros(y_onehot_test.shape[0])\n",
    "                y_onehot[y_onehot_test[:, class_id] == 1] = 1\n",
    "            else:  # Binary classification\n",
    "                y_onehot = y_onehot_test[:, 0]\n",
    "\n",
    "            RocCurveDisplay.from_predictions(\n",
    "                y_onehot,\n",
    "                y_score[:, class_id],\n",
    "                name=f\"ROC curve for {target_names[class_id]}\",\n",
    "                color=colors(class_id),\n",
    "                ax=ax[0][1]\n",
    "            )\n",
    "        \n",
    "#         colors = cycle([\"aqua\", \"darkorange\", \"cornflowerblue\"])\n",
    "#         for class_id, color in zip(range(n_classes), colors):\n",
    "#             RocCurveDisplay.from_predictions(\n",
    "#                 y_onehot_test[:, class_id],\n",
    "#                 y_score[:, class_id],\n",
    "#                 name=f\"ROC curve for {target_names[class_id]}\",                    # SPECIFIC TO IRIS\n",
    "#                 color=color,\n",
    "#                 ax=ax[0][1],\n",
    "#             )\n",
    "        ax[0][1].plot([0, 1], [0, 1], \"k--\", label=\"chance level (AUC = 0.5)\")\n",
    "        ax[0][1].set_xlabel(\"False Positive Rate\")\n",
    "        ax[0][1].set_ylabel(\"True Positive Rate\")\n",
    "        ax[0][1].set_title(\"Micro-averaged One-vs-Rest\\nReceiver Operating Characteristic\")\n",
    "\n",
    "        ConfusionMatrixDisplay.from_estimator(model, X, Y, ax=ax[1][0])\n",
    "        _ = ax[1][0].set_title(\n",
    "            f\"Confusion Matrix on train set for {name}\"\n",
    "        )\n",
    "\n",
    "        ConfusionMatrixDisplay.from_estimator(model, X_t, Y_t, ax=ax[1][1])\n",
    "        _ = ax[1][1].set_title(\n",
    "            f\"Confusion Matrix on test set for {name}\"\n",
    "        )\n",
    "\n",
    "    params_plot = {\n",
    "        \"X\": X,\n",
    "        \"y\": Y,\n",
    "        \"score_type\": \"both\",\n",
    "        \"n_jobs\": -2,\n",
    "        \"line_kw\": {\"marker\": \"o\"},\n",
    "        \"std_display_style\": \"fill_between\",\n",
    "        \"score_name\": \"accuracy\"\n",
    "    }\n",
    "    \n",
    "    dic_models = {}\n",
    "    models_ensemble = []\n",
    "    for i in range(len(list(models))):\n",
    "        models_ensemble.append((list(models.keys())[i], list(models.values())[i]))\n",
    "        dic_models[list(models.keys())[i]] = list(models.values())[i]\n",
    "        \n",
    "    for i in range(len(list(models))):\n",
    "        print(list(models.keys())[i] + \"\\n\")\n",
    "        print(\"#\"*50)\n",
    "\n",
    "        fig, ax = plt.subplots(nrows=2, ncols=2, figsize=(10, 10))\n",
    "        \n",
    "        if gs:\n",
    "            # FOR TRAINING MODEL BATTERY USING GRIDSEARCH             \n",
    "            model = make_pipeline(StandardScaler(), GridSearchCV(list(models.values())[i], list(params.values())[i],\n",
    "                                                                 scoring='accuracy', refit=True, n_jobs=-2))\n",
    "        else:\n",
    "            # FOR TRAINING MODEL BATTERY WITHOUT GRIDSEARCH            \n",
    "            model = make_pipeline(StandardScaler(), list(models.values())[i])\n",
    "\n",
    "        LearningCurveDisplay.from_estimator(model, **params_plot, ax=ax[0][0])\n",
    "        handles, label = ax[0][0].get_legend_handles_labels()\n",
    "        ax[0][0].legend(handles[:2], [\"Training Score\", \"Test Score\"])\n",
    "        ax[0][0].set_title(f\"Learning Curve for {list(models.keys())[i]}\")\n",
    "\n",
    "        model = fit(model, X, Y)\n",
    "        if write:\n",
    "            dump(model, \"./models/\" + folder + \"/\" + run + \"_\" + list(models.keys())[i] + \".joblib\")\n",
    "\n",
    "        plots(model, X, Y, X_t, Y_t, list(models.keys())[i])\n",
    "\n",
    "        fig.tight_layout()\n",
    "        if write:\n",
    "            plt.savefig(\"./models/\" + folder +  \"/result_images/\" + run + \"_\" + list(models.keys())[i] + \".png\", bbox_inches='tight')\n",
    "            plt.close()\n",
    "        plt.show()\n",
    "\n",
    "        results(X,Y, list(models.keys())[i] + \"_\" + run, model, \"Training set\")\n",
    "        results(X_t,Y_t, list(models.keys())[i] + \"_\" + run, model, \"Test set\")\n",
    "        saved_models.append(model)\n",
    "        \n",
    "    print(\"################################################\")\n",
    "    print(\"################### ENSEMBLE ###################\")\n",
    "\n",
    "    dic_models[\"ensemble\"] = VotingClassifier(models_ensemble, voting='soft', n_jobs=-2, verbose=True)\n",
    "\n",
    "    fig, ax = plt.subplots(nrows=2, ncols=2, figsize=(10, 10))\n",
    "\n",
    "    LearningCurveDisplay.from_estimator(dic_models[\"ensemble\"], **params_plot, ax=ax[0][0])\n",
    "    handles, label = ax[0][0].get_legend_handles_labels()\n",
    "    ax[0][0].legend(handles[:2], [\"Training Score\", \"Test Score\"])\n",
    "    ax[0][0].set_title(f\"Learning Curve for ENSEMBLE\")\n",
    "\n",
    "    dic_models[\"ensemble\"] = fit(dic_models[\"ensemble\"], X, Y)\n",
    "\n",
    "    results(X, Y, \"ENSEMBLE\"  + \"_\" + run, dic_models[\"ensemble\"], \"train set\")\n",
    "    results(X_t, Y_t, \"ENSEMBLE\"  + \"_\" + run, dic_models[\"ensemble\"], \"test set\")\n",
    "    \n",
    "    plots(dic_models[\"ensemble\"], X, Y, X_t, Y_t, \"ENSEMBLE\")\n",
    "    if write:\n",
    "        dump(dic_models[\"ensemble\"], \"./models/\" + folder + \"/\" + run + \"_\" + \"ensembler.joblib\")\n",
    "\n",
    "    fig.tight_layout()\n",
    "    if write:\n",
    "        plt.savefig(\"./models/\" + folder + \"/result_images/\" + run + \"_\" + \"ensembler.png\", bbox_inches='tight')\n",
    "        plt.close()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 42,
   "metadata": {},
   "outputs": [],
   "source": [
    "models = {\n",
    "    \"Logistic Regression\"      :LogisticRegression(),\n",
    "    \"Decision Tree\"            :DecisionTreeClassifier(),\n",
    "    \"Random Forest\"            :RandomForestClassifier(random_state=42),\n",
    "    \"Gradient Boosting\"        :GradientBoostingClassifier(),\n",
    "    \"K-Nearest Neighbors\"      :KNeighborsClassifier(),\n",
    "    \"SVM\"                      :svm.SVC(probability=True)\n",
    "}\n",
    "\n",
    "params = {\n",
    "    \"Logistic Regression\"     :{\n",
    "        'solver': ['newton-cholesky', 'liblinear', 'newton-cg', 'sag', 'saga'],\n",
    "        'penalty':['l2'],\n",
    "        'C':[0.001, 0.01, 0.1, 1, 100],\n",
    "        'max_iter':[10000],\n",
    "    },\n",
    "    \"Decision Tree\"            :{\n",
    "        'criterion':['gini','entropy'],\n",
    "        'max_depth':[4,6,8,10,15,50,90]\n",
    "    },\n",
    "    \"Random Forest\"            :{ \n",
    "        'n_estimators': [200, 500],\n",
    "        'max_features': [None, 'sqrt', 'log2'],\n",
    "        'max_depth' : [4,5,6,7,8],\n",
    "        'criterion' :['gini', 'entropy']\n",
    "    },\n",
    "    \"Gradient Boosting\"        :{\n",
    "        \"loss\":[\"log_loss\"],\n",
    "        \"learning_rate\": [0.1, 0.01],\n",
    "        \"max_depth\":[3,5,8],\n",
    "        \"max_features\":[\"log2\",\"sqrt\"],\n",
    "        \"criterion\": [\"friedman_mse\", \"squared_error\"],\n",
    "        \"subsample\":[0.5, 0.8, 1.0],\n",
    "        \"n_estimators\":[10, 100, 1000]\n",
    "    },\n",
    "    \"K-Nearest Neighbors\"      :{\n",
    "         'weights': ['uniform', 'distance'],'leaf_size': [5, 15, 25]\n",
    "    },\n",
    "    \"SVM\"                      :{\n",
    "    \"kernel\":['rbf'], \"C\":[0.1, 1, 10, 100, 1000], \"degree\":[1,5], \n",
    "        'gamma': [1, 0.1, 0.01, 0.001]\n",
    "    }\n",
    "}"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# FITTING THE MODEL BATTERY"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# WINE DATASET"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 46,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "     Cultivars  Alcohol  Malic acid   Ash  Alcalinity of ash  Magnesium  \\\n",
      "0            1    14.23        1.71  2.43               15.6        127   \n",
      "1            1    13.20        1.78  2.14               11.2        100   \n",
      "2            1    13.16        2.36  2.67               18.6        101   \n",
      "3            1    14.37        1.95  2.50               16.8        113   \n",
      "4            1    13.24        2.59  2.87               21.0        118   \n",
      "..         ...      ...         ...   ...                ...        ...   \n",
      "173          3    13.71        5.65  2.45               20.5         95   \n",
      "174          3    13.40        3.91  2.48               23.0        102   \n",
      "175          3    13.27        4.28  2.26               20.0        120   \n",
      "176          3    13.17        2.59  2.37               20.0        120   \n",
      "177          3    14.13        4.10  2.74               24.5         96   \n",
      "\n",
      "     Total Phenols  Flavanoids  Nonflavanoid phenols  Proanthocyanins  \\\n",
      "0             2.80        3.06                  0.28             2.29   \n",
      "1             2.65        2.76                  0.26             1.28   \n",
      "2             2.80        3.24                  0.30             2.81   \n",
      "3             3.85        3.49                  0.24             2.18   \n",
      "4             2.80        2.69                  0.39             1.82   \n",
      "..             ...         ...                   ...              ...   \n",
      "173           1.68        0.61                  0.52             1.06   \n",
      "174           1.80        0.75                  0.43             1.41   \n",
      "175           1.59        0.69                  0.43             1.35   \n",
      "176           1.65        0.68                  0.53             1.46   \n",
      "177           2.05        0.76                  0.56             1.35   \n",
      "\n",
      "     Color intensity   Hue  OD280/OD315 of diluted wines  Proline  \n",
      "0               5.64  1.04                          3.92     1065  \n",
      "1               4.38  1.05                          3.40     1050  \n",
      "2               5.68  1.03                          3.17     1185  \n",
      "3               7.80  0.86                          3.45     1480  \n",
      "4               4.32  1.04                          2.93      735  \n",
      "..               ...   ...                           ...      ...  \n",
      "173             7.70  0.64                          1.74      740  \n",
      "174             7.30  0.70                          1.56      750  \n",
      "175            10.20  0.59                          1.56      835  \n",
      "176             9.30  0.60                          1.62      840  \n",
      "177             9.20  0.61                          1.60      560  \n",
      "\n",
      "[178 rows x 14 columns]\n"
     ]
    }
   ],
   "source": [
    "data_wine=pd.read_csv('./data/wine.data',names=['Cultivars','Alcohol','Malic acid','Ash','Alcalinity of ash',\n",
    "'Magnesium','Total Phenols','Flavanoids','Nonflavanoid phenols',\n",
    "'Proanthocyanins','Color intensity','Hue','OD280/OD315 of diluted wines','Proline'])\n",
    "data_wine_target_names = np.array(['1','2','3'])\n",
    "print(data_wine)\n",
    "x=data_wine.drop(['Cultivars'], axis=1)\n",
    "y=data_wine['Cultivars']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 47,
   "metadata": {},
   "outputs": [],
   "source": [
    "X_train, X_test, y_train, y_test = train_test_split(x, y, test_size=0.20, random_state=42)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 48,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Fitting for parameters...\n",
      "X:      Alcohol  Malic acid   Ash  Alcalinity of ash  Magnesium  Total Phenols  \\\n",
      "158    14.34        1.68  2.70               25.0         98           2.80   \n",
      "137    12.53        5.51  2.64               25.0         96           1.79   \n",
      "98     12.37        1.07  2.10               18.5         88           3.52   \n",
      "159    13.48        1.67  2.64               22.5         89           2.60   \n",
      "38     13.07        1.50  2.10               15.5         98           2.40   \n",
      "..       ...         ...   ...                ...        ...            ...   \n",
      "71     13.86        1.51  2.67               25.0         86           2.95   \n",
      "106    12.25        1.73  2.12               19.0         80           1.65   \n",
      "14     14.38        1.87  2.38               12.0        102           3.30   \n",
      "92     12.69        1.53  2.26               20.7         80           1.38   \n",
      "102    12.34        2.45  2.46               21.0         98           2.56   \n",
      "\n",
      "     Flavanoids  Nonflavanoid phenols  Proanthocyanins  Color intensity   Hue  \\\n",
      "158        1.31                  0.53             2.70            13.00  0.57   \n",
      "137        0.60                  0.63             1.10             5.00  0.82   \n",
      "98         3.75                  0.24             1.95             4.50  1.04   \n",
      "159        1.10                  0.52             2.29            11.75  0.57   \n",
      "38         2.64                  0.28             1.37             3.70  1.18   \n",
      "..          ...                   ...              ...              ...   ...   \n",
      "71         2.86                  0.21             1.87             3.38  1.36   \n",
      "106        2.03                  0.37             1.63             3.40  1.00   \n",
      "14         3.64                  0.29             2.96             7.50  1.20   \n",
      "92         1.46                  0.58             1.62             3.05  0.96   \n",
      "102        2.11                  0.34             1.31             2.80  0.80   \n",
      "\n",
      "     OD280/OD315 of diluted wines  Proline  \n",
      "158                          1.96      660  \n",
      "137                          1.69      515  \n",
      "98                           2.77      660  \n",
      "159                          1.78      620  \n",
      "38                           2.69     1020  \n",
      "..                            ...      ...  \n",
      "71                           3.16      410  \n",
      "106                          3.17      510  \n",
      "14                           3.00     1547  \n",
      "92                           2.06      495  \n",
      "102                          3.38      438  \n",
      "\n",
      "[142 rows x 13 columns]\n",
      "--------------------------------------------------\n",
      "Y: 158    3\n",
      "137    3\n",
      "98     2\n",
      "159    3\n",
      "38     1\n",
      "      ..\n",
      "71     2\n",
      "106    2\n",
      "14     1\n",
      "92     2\n",
      "102    2\n",
      "Name: Cultivars, Length: 142, dtype: int64\n",
      "--------------------------------------------------\n",
      "X_t:      Alcohol  Malic acid   Ash  Alcalinity of ash  Magnesium  Total Phenols  \\\n",
      "19     13.64        3.10  2.56               15.2        116           2.70   \n",
      "45     14.21        4.04  2.44               18.9        111           2.85   \n",
      "140    12.93        2.81  2.70               21.0         96           1.54   \n",
      "30     13.73        1.50  2.70               22.5        101           3.00   \n",
      "67     12.37        1.17  1.92               19.6         78           2.11   \n",
      "16     14.30        1.92  2.72               20.0        120           2.80   \n",
      "119    12.00        3.43  2.00               19.0         87           2.00   \n",
      "174    13.40        3.91  2.48               23.0        102           1.80   \n",
      "109    11.61        1.35  2.70               20.0         94           2.74   \n",
      "141    13.36        2.56  2.35               20.0         89           1.40   \n",
      "24     13.50        1.81  2.61               20.0         96           2.53   \n",
      "150    13.50        3.12  2.62               24.0        123           1.40   \n",
      "41     13.41        3.84  2.12               18.8         90           2.45   \n",
      "118    12.77        3.43  1.98               16.0         80           1.63   \n",
      "15     13.63        1.81  2.70               17.2        112           2.85   \n",
      "111    12.52        2.43  2.17               21.0         88           2.55   \n",
      "113    11.41        0.74  2.50               21.0         88           2.48   \n",
      "82     12.08        1.13  2.51               24.0         78           2.00   \n",
      "9      13.86        1.35  2.27               16.0         98           2.98   \n",
      "114    12.08        1.39  2.50               22.5         84           2.56   \n",
      "18     14.19        1.59  2.48               16.5        108           3.30   \n",
      "66     13.11        1.01  1.70               15.0         78           2.98   \n",
      "60     12.33        1.10  2.28               16.0        101           2.05   \n",
      "169    13.40        4.60  2.86               25.0        112           1.98   \n",
      "171    12.77        2.39  2.28               19.5         86           1.39   \n",
      "164    13.78        2.76  2.30               22.0         90           1.35   \n",
      "117    12.42        1.61  2.19               22.5        108           2.00   \n",
      "65     12.37        1.21  2.56               18.1         98           2.42   \n",
      "90     12.08        1.83  2.32               18.5         81           1.60   \n",
      "55     13.56        1.73  2.46               20.5        116           2.96   \n",
      "29     14.02        1.68  2.21               16.0         96           2.65   \n",
      "128    12.37        1.63  2.30               24.5         88           2.22   \n",
      "145    13.16        3.57  2.15               21.0        102           1.50   \n",
      "31     13.58        1.66  2.36               19.1        106           2.86   \n",
      "12     13.75        1.73  2.41               16.0         89           2.60   \n",
      "42     13.88        1.89  2.59               15.0        101           3.25   \n",
      "\n",
      "     Flavanoids  Nonflavanoid phenols  Proanthocyanins  Color intensity   Hue  \\\n",
      "19         3.03                  0.17             1.66         5.100000  0.96   \n",
      "45         2.65                  0.30             1.25         5.240000  0.87   \n",
      "140        0.50                  0.53             0.75         4.600000  0.77   \n",
      "30         3.25                  0.29             2.38         5.700000  1.19   \n",
      "67         2.00                  0.27             1.04         4.680000  1.12   \n",
      "16         3.14                  0.33             1.97         6.200000  1.07   \n",
      "119        1.64                  0.37             1.87         1.280000  0.93   \n",
      "174        0.75                  0.43             1.41         7.300000  0.70   \n",
      "109        2.92                  0.29             2.49         2.650000  0.96   \n",
      "141        0.50                  0.37             0.64         5.600000  0.70   \n",
      "24         2.61                  0.28             1.66         3.520000  1.12   \n",
      "150        1.57                  0.22             1.25         8.600000  0.59   \n",
      "41         2.68                  0.27             1.48         4.280000  0.91   \n",
      "118        1.25                  0.43             0.83         3.400000  0.70   \n",
      "15         2.91                  0.30             1.46         7.300000  1.28   \n",
      "111        2.27                  0.26             1.22         2.000000  0.90   \n",
      "113        2.01                  0.42             1.44         3.080000  1.10   \n",
      "82         1.58                  0.40             1.40         2.200000  1.31   \n",
      "9          3.15                  0.22             1.85         7.220000  1.01   \n",
      "114        2.29                  0.43             1.04         2.900000  0.93   \n",
      "18         3.93                  0.32             1.86         8.700000  1.23   \n",
      "66         3.18                  0.26             2.28         5.300000  1.12   \n",
      "60         1.09                  0.63             0.41         3.270000  1.25   \n",
      "169        0.96                  0.27             1.11         8.500000  0.67   \n",
      "171        0.51                  0.48             0.64         9.899999  0.57   \n",
      "164        0.68                  0.41             1.03         9.580000  0.70   \n",
      "117        2.09                  0.34             1.61         2.060000  1.06   \n",
      "65         2.65                  0.37             2.08         4.600000  1.19   \n",
      "90         1.50                  0.52             1.64         2.400000  1.08   \n",
      "55         2.78                  0.20             2.45         6.250000  0.98   \n",
      "29         2.33                  0.26             1.98         4.700000  1.04   \n",
      "128        2.45                  0.40             1.90         2.120000  0.89   \n",
      "145        0.55                  0.43             1.30         4.000000  0.60   \n",
      "31         3.19                  0.22             1.95         6.900000  1.09   \n",
      "12         2.76                  0.29             1.81         5.600000  1.15   \n",
      "42         3.56                  0.17             1.70         5.430000  0.88   \n",
      "\n",
      "     OD280/OD315 of diluted wines  Proline  \n",
      "19                           3.36      845  \n",
      "45                           3.33     1080  \n",
      "140                          2.31      600  \n",
      "30                           2.71     1285  \n",
      "67                           3.48      510  \n",
      "16                           2.65     1280  \n",
      "119                          3.05      564  \n",
      "174                          1.56      750  \n",
      "109                          3.26      680  \n",
      "141                          2.47      780  \n",
      "24                           3.82      845  \n",
      "150                          1.30      500  \n",
      "41                           3.00     1035  \n",
      "118                          2.12      372  \n",
      "15                           2.88     1310  \n",
      "111                          2.78      325  \n",
      "113                          2.31      434  \n",
      "82                           2.72      630  \n",
      "9                            3.55     1045  \n",
      "114                          3.19      385  \n",
      "18                           2.82     1680  \n",
      "66                           3.18      502  \n",
      "60                           1.67      680  \n",
      "169                          1.92      630  \n",
      "171                          1.63      470  \n",
      "164                          1.68      615  \n",
      "117                          2.96      345  \n",
      "65                           2.30      678  \n",
      "90                           2.27      480  \n",
      "55                           3.03     1120  \n",
      "29                           3.59     1035  \n",
      "128                          2.78      342  \n",
      "145                          1.68      830  \n",
      "31                           2.88     1515  \n",
      "12                           2.90     1320  \n",
      "42                           3.56     1095  \n",
      "--------------------------------------------------\n",
      "Y_t: 19     1\n",
      "45     1\n",
      "140    3\n",
      "30     1\n",
      "67     2\n",
      "16     1\n",
      "119    2\n",
      "174    3\n",
      "109    2\n",
      "141    3\n",
      "24     1\n",
      "150    3\n",
      "41     1\n",
      "118    2\n",
      "15     1\n",
      "111    2\n",
      "113    2\n",
      "82     2\n",
      "9      1\n",
      "114    2\n",
      "18     1\n",
      "66     2\n",
      "60     2\n",
      "169    3\n",
      "171    3\n",
      "164    3\n",
      "117    2\n",
      "65     2\n",
      "90     2\n",
      "55     1\n",
      "29     1\n",
      "128    2\n",
      "145    3\n",
      "31     1\n",
      "12     1\n",
      "42     1\n",
      "Name: Cultivars, dtype: int64\n",
      "--------------------------------------------------\n",
      "models: {'Logistic Regression': LogisticRegression(), 'Decision Tree': DecisionTreeClassifier(), 'Random Forest': RandomForestClassifier(random_state=42), 'Gradient Boosting': GradientBoostingClassifier(), 'K-Nearest Neighbors': KNeighborsClassifier(), 'SVM': SVC(probability=True)}\n",
      "--------------------------------------------------\n",
      "params: {'Logistic Regression': {'solver': ['newton-cholesky', 'liblinear', 'newton-cg', 'sag', 'saga'], 'penalty': ['l2'], 'C': [0.001, 0.01, 0.1, 1, 100], 'max_iter': [10000]}, 'Decision Tree': {'criterion': ['gini', 'entropy'], 'max_depth': [4, 6, 8, 10, 15, 50, 90]}, 'Random Forest': {'n_estimators': [200, 500], 'max_features': [None, 'sqrt', 'log2'], 'max_depth': [4, 5, 6, 7, 8], 'criterion': ['gini', 'entropy']}, 'Gradient Boosting': {'loss': ['log_loss'], 'learning_rate': [0.1, 0.01], 'max_depth': [3, 5, 8], 'max_features': ['log2', 'sqrt'], 'criterion': ['friedman_mse', 'squared_error'], 'subsample': [0.5, 0.8, 1.0], 'n_estimators': [10, 100, 1000]}, 'K-Nearest Neighbors': {'weights': ['uniform', 'distance'], 'leaf_size': [5, 15, 25]}, 'SVM': {'kernel': ['rbf'], 'C': [0.1, 1, 10, 100, 1000], 'degree': [1, 5], 'gamma': [1, 0.1, 0.01, 0.001]}}\n",
      "--------------------------------------------------\n",
      "run: wine_Data_gridsearch\n",
      "--------------------------------------------------\n",
      "folder: experiment_wine_gridsearch\n",
      "Logistic Regression\n",
      "\n",
      "##################################################\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/potatosalad/.local/share/virtualenvs/CSC_homework_3_group4-nat9sqnt/lib/python3.10/site-packages/sklearn/model_selection/_split.py:700: UserWarning: The least populated class in y has only 2 members, which is less than n_splits=5.\n",
      "  warnings.warn(\n",
      "/home/potatosalad/.local/share/virtualenvs/CSC_homework_3_group4-nat9sqnt/lib/python3.10/site-packages/sklearn/model_selection/_split.py:700: UserWarning: The least populated class in y has only 2 members, which is less than n_splits=5.\n",
      "  warnings.warn(\n",
      "/home/potatosalad/.local/share/virtualenvs/CSC_homework_3_group4-nat9sqnt/lib/python3.10/site-packages/sklearn/model_selection/_split.py:700: UserWarning: The least populated class in y has only 2 members, which is less than n_splits=5.\n",
      "  warnings.warn(\n",
      "/home/potatosalad/.local/share/virtualenvs/CSC_homework_3_group4-nat9sqnt/lib/python3.10/site-packages/sklearn/model_selection/_split.py:700: UserWarning: The least populated class in y has only 2 members, which is less than n_splits=5.\n",
      "  warnings.warn(\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Model performance for Training set Logistic Regression_wine_Data_gridsearch\n",
      "- Accuracy: 0.9930\n",
      "- F1 score: 0.992970\n",
      "- Precision: 0.993130\n",
      "- Recall: 0.992958\n",
      "----------------------------------\n",
      "===================================\n",
      "Model performance for Test set Logistic Regression_wine_Data_gridsearch\n",
      "- Accuracy: 1.0000\n",
      "- F1 score: 1.000000\n",
      "- Precision: 1.000000\n",
      "- Recall: 1.000000\n",
      "----------------------------------\n",
      "===================================\n",
      "Decision Tree\n",
      "\n",
      "##################################################\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/potatosalad/.local/share/virtualenvs/CSC_homework_3_group4-nat9sqnt/lib/python3.10/site-packages/sklearn/model_selection/_split.py:700: UserWarning: The least populated class in y has only 2 members, which is less than n_splits=5.\n",
      "  warnings.warn(\n",
      "/home/potatosalad/.local/share/virtualenvs/CSC_homework_3_group4-nat9sqnt/lib/python3.10/site-packages/sklearn/model_selection/_split.py:700: UserWarning: The least populated class in y has only 2 members, which is less than n_splits=5.\n",
      "  warnings.warn(\n",
      "/home/potatosalad/.local/share/virtualenvs/CSC_homework_3_group4-nat9sqnt/lib/python3.10/site-packages/sklearn/model_selection/_split.py:700: UserWarning: The least populated class in y has only 2 members, which is less than n_splits=5.\n",
      "  warnings.warn(\n",
      "/home/potatosalad/.local/share/virtualenvs/CSC_homework_3_group4-nat9sqnt/lib/python3.10/site-packages/sklearn/model_selection/_split.py:700: UserWarning: The least populated class in y has only 2 members, which is less than n_splits=5.\n",
      "  warnings.warn(\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Model performance for Training set Decision Tree_wine_Data_gridsearch\n",
      "- Accuracy: 1.0000\n",
      "- F1 score: 1.000000\n",
      "- Precision: 1.000000\n",
      "- Recall: 1.000000\n",
      "----------------------------------\n",
      "===================================\n",
      "Model performance for Test set Decision Tree_wine_Data_gridsearch\n",
      "- Accuracy: 0.9167\n",
      "- F1 score: 0.909382\n",
      "- Precision: 0.925463\n",
      "- Recall: 0.916667\n",
      "----------------------------------\n",
      "===================================\n",
      "Random Forest\n",
      "\n",
      "##################################################\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/potatosalad/.local/share/virtualenvs/CSC_homework_3_group4-nat9sqnt/lib/python3.10/site-packages/sklearn/model_selection/_split.py:700: UserWarning: The least populated class in y has only 2 members, which is less than n_splits=5.\n",
      "  warnings.warn(\n",
      "/home/potatosalad/.local/share/virtualenvs/CSC_homework_3_group4-nat9sqnt/lib/python3.10/site-packages/sklearn/model_selection/_split.py:700: UserWarning: The least populated class in y has only 2 members, which is less than n_splits=5.\n",
      "  warnings.warn(\n",
      "/home/potatosalad/.local/share/virtualenvs/CSC_homework_3_group4-nat9sqnt/lib/python3.10/site-packages/sklearn/model_selection/_split.py:700: UserWarning: The least populated class in y has only 2 members, which is less than n_splits=5.\n",
      "  warnings.warn(\n",
      "/home/potatosalad/.local/share/virtualenvs/CSC_homework_3_group4-nat9sqnt/lib/python3.10/site-packages/sklearn/model_selection/_split.py:700: UserWarning: The least populated class in y has only 2 members, which is less than n_splits=5.\n",
      "  warnings.warn(\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Model performance for Training set Random Forest_wine_Data_gridsearch\n",
      "- Accuracy: 1.0000\n",
      "- F1 score: 1.000000\n",
      "- Precision: 1.000000\n",
      "- Recall: 1.000000\n",
      "----------------------------------\n",
      "===================================\n",
      "Model performance for Test set Random Forest_wine_Data_gridsearch\n",
      "- Accuracy: 1.0000\n",
      "- F1 score: 1.000000\n",
      "- Precision: 1.000000\n",
      "- Recall: 1.000000\n",
      "----------------------------------\n",
      "===================================\n",
      "Gradient Boosting\n",
      "\n",
      "##################################################\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/potatosalad/.local/share/virtualenvs/CSC_homework_3_group4-nat9sqnt/lib/python3.10/site-packages/sklearn/model_selection/_split.py:700: UserWarning: The least populated class in y has only 2 members, which is less than n_splits=5.\n",
      "  warnings.warn(\n",
      "/home/potatosalad/.local/share/virtualenvs/CSC_homework_3_group4-nat9sqnt/lib/python3.10/site-packages/sklearn/model_selection/_split.py:700: UserWarning: The least populated class in y has only 2 members, which is less than n_splits=5.\n",
      "  warnings.warn(\n",
      "/home/potatosalad/.local/share/virtualenvs/CSC_homework_3_group4-nat9sqnt/lib/python3.10/site-packages/sklearn/model_selection/_split.py:700: UserWarning: The least populated class in y has only 2 members, which is less than n_splits=5.\n",
      "  warnings.warn(\n",
      "/home/potatosalad/.local/share/virtualenvs/CSC_homework_3_group4-nat9sqnt/lib/python3.10/site-packages/sklearn/model_selection/_split.py:700: UserWarning: The least populated class in y has only 2 members, which is less than n_splits=5.\n",
      "  warnings.warn(\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Model performance for Training set Gradient Boosting_wine_Data_gridsearch\n",
      "- Accuracy: 1.0000\n",
      "- F1 score: 1.000000\n",
      "- Precision: 1.000000\n",
      "- Recall: 1.000000\n",
      "----------------------------------\n",
      "===================================\n",
      "Model performance for Test set Gradient Boosting_wine_Data_gridsearch\n",
      "- Accuracy: 1.0000\n",
      "- F1 score: 1.000000\n",
      "- Precision: 1.000000\n",
      "- Recall: 1.000000\n",
      "----------------------------------\n",
      "===================================\n",
      "K-Nearest Neighbors\n",
      "\n",
      "##################################################\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/potatosalad/.local/share/virtualenvs/CSC_homework_3_group4-nat9sqnt/lib/python3.10/site-packages/sklearn/model_selection/_split.py:700: UserWarning: The least populated class in y has only 2 members, which is less than n_splits=5.\n",
      "  warnings.warn(\n",
      "/home/potatosalad/.local/share/virtualenvs/CSC_homework_3_group4-nat9sqnt/lib/python3.10/site-packages/sklearn/model_selection/_split.py:700: UserWarning: The least populated class in y has only 2 members, which is less than n_splits=5.\n",
      "  warnings.warn(\n",
      "/home/potatosalad/.local/share/virtualenvs/CSC_homework_3_group4-nat9sqnt/lib/python3.10/site-packages/sklearn/model_selection/_split.py:700: UserWarning: The least populated class in y has only 2 members, which is less than n_splits=5.\n",
      "  warnings.warn(\n",
      "/home/potatosalad/.local/share/virtualenvs/CSC_homework_3_group4-nat9sqnt/lib/python3.10/site-packages/sklearn/model_selection/_split.py:700: UserWarning: The least populated class in y has only 2 members, which is less than n_splits=5.\n",
      "  warnings.warn(\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Model performance for Training set K-Nearest Neighbors_wine_Data_gridsearch\n",
      "- Accuracy: 0.9859\n",
      "- F1 score: 0.985872\n",
      "- Precision: 0.986240\n",
      "- Recall: 0.985915\n",
      "----------------------------------\n",
      "===================================\n",
      "Model performance for Test set K-Nearest Neighbors_wine_Data_gridsearch\n",
      "- Accuracy: 0.9444\n",
      "- F1 score: 0.943604\n",
      "- Precision: 0.949383\n",
      "- Recall: 0.944444\n",
      "----------------------------------\n",
      "===================================\n",
      "SVM\n",
      "\n",
      "##################################################\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/potatosalad/.local/share/virtualenvs/CSC_homework_3_group4-nat9sqnt/lib/python3.10/site-packages/sklearn/model_selection/_split.py:700: UserWarning: The least populated class in y has only 2 members, which is less than n_splits=5.\n",
      "  warnings.warn(\n",
      "/home/potatosalad/.local/share/virtualenvs/CSC_homework_3_group4-nat9sqnt/lib/python3.10/site-packages/sklearn/model_selection/_split.py:700: UserWarning: The least populated class in y has only 2 members, which is less than n_splits=5.\n",
      "  warnings.warn(\n",
      "/home/potatosalad/.local/share/virtualenvs/CSC_homework_3_group4-nat9sqnt/lib/python3.10/site-packages/sklearn/model_selection/_split.py:700: UserWarning: The least populated class in y has only 2 members, which is less than n_splits=5.\n",
      "  warnings.warn(\n",
      "/home/potatosalad/.local/share/virtualenvs/CSC_homework_3_group4-nat9sqnt/lib/python3.10/site-packages/sklearn/model_selection/_split.py:700: UserWarning: The least populated class in y has only 2 members, which is less than n_splits=5.\n",
      "  warnings.warn(\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Model performance for Training set SVM_wine_Data_gridsearch\n",
      "- Accuracy: 0.9789\n",
      "- F1 score: 0.978912\n",
      "- Precision: 0.979418\n",
      "- Recall: 0.978873\n",
      "----------------------------------\n",
      "===================================\n",
      "Model performance for Test set SVM_wine_Data_gridsearch\n",
      "- Accuracy: 1.0000\n",
      "- F1 score: 1.000000\n",
      "- Precision: 1.000000\n",
      "- Recall: 1.000000\n",
      "----------------------------------\n",
      "===================================\n",
      "################################################\n",
      "################### ENSEMBLE ###################\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/potatosalad/.local/share/virtualenvs/CSC_homework_3_group4-nat9sqnt/lib/python3.10/site-packages/sklearn/linear_model/_logistic.py:458: ConvergenceWarning: lbfgs failed to converge (status=1):\n",
      "STOP: TOTAL NO. of ITERATIONS REACHED LIMIT.\n",
      "\n",
      "Increase the number of iterations (max_iter) or scale the data as shown in:\n",
      "    https://scikit-learn.org/stable/modules/preprocessing.html\n",
      "Please also refer to the documentation for alternative solver options:\n",
      "    https://scikit-learn.org/stable/modules/linear_model.html#logistic-regression\n",
      "  n_iter_i = _check_optimize_result(\n",
      "/home/potatosalad/.local/share/virtualenvs/CSC_homework_3_group4-nat9sqnt/lib/python3.10/site-packages/sklearn/linear_model/_logistic.py:458: ConvergenceWarning: lbfgs failed to converge (status=1):\n",
      "STOP: TOTAL NO. of ITERATIONS REACHED LIMIT.\n",
      "\n",
      "Increase the number of iterations (max_iter) or scale the data as shown in:\n",
      "    https://scikit-learn.org/stable/modules/preprocessing.html\n",
      "Please also refer to the documentation for alternative solver options:\n",
      "    https://scikit-learn.org/stable/modules/linear_model.html#logistic-regression\n",
      "  n_iter_i = _check_optimize_result(\n",
      "/home/potatosalad/.local/share/virtualenvs/CSC_homework_3_group4-nat9sqnt/lib/python3.10/site-packages/sklearn/linear_model/_logistic.py:458: ConvergenceWarning: lbfgs failed to converge (status=1):\n",
      "STOP: TOTAL NO. of ITERATIONS REACHED LIMIT.\n",
      "\n",
      "Increase the number of iterations (max_iter) or scale the data as shown in:\n",
      "    https://scikit-learn.org/stable/modules/preprocessing.html\n",
      "Please also refer to the documentation for alternative solver options:\n",
      "    https://scikit-learn.org/stable/modules/linear_model.html#logistic-regression\n",
      "  n_iter_i = _check_optimize_result(\n",
      "/home/potatosalad/.local/share/virtualenvs/CSC_homework_3_group4-nat9sqnt/lib/python3.10/site-packages/sklearn/linear_model/_logistic.py:458: ConvergenceWarning: lbfgs failed to converge (status=1):\n",
      "STOP: TOTAL NO. of ITERATIONS REACHED LIMIT.\n",
      "\n",
      "Increase the number of iterations (max_iter) or scale the data as shown in:\n",
      "    https://scikit-learn.org/stable/modules/preprocessing.html\n",
      "Please also refer to the documentation for alternative solver options:\n",
      "    https://scikit-learn.org/stable/modules/linear_model.html#logistic-regression\n",
      "  n_iter_i = _check_optimize_result(\n",
      "/home/potatosalad/.local/share/virtualenvs/CSC_homework_3_group4-nat9sqnt/lib/python3.10/site-packages/sklearn/linear_model/_logistic.py:458: ConvergenceWarning: lbfgs failed to converge (status=1):\n",
      "STOP: TOTAL NO. of ITERATIONS REACHED LIMIT.\n",
      "\n",
      "Increase the number of iterations (max_iter) or scale the data as shown in:\n",
      "    https://scikit-learn.org/stable/modules/preprocessing.html\n",
      "Please also refer to the documentation for alternative solver options:\n",
      "    https://scikit-learn.org/stable/modules/linear_model.html#logistic-regression\n",
      "  n_iter_i = _check_optimize_result(\n",
      "/home/potatosalad/.local/share/virtualenvs/CSC_homework_3_group4-nat9sqnt/lib/python3.10/site-packages/sklearn/linear_model/_logistic.py:458: ConvergenceWarning: lbfgs failed to converge (status=1):\n",
      "STOP: TOTAL NO. of ITERATIONS REACHED LIMIT.\n",
      "\n",
      "Increase the number of iterations (max_iter) or scale the data as shown in:\n",
      "    https://scikit-learn.org/stable/modules/preprocessing.html\n",
      "Please also refer to the documentation for alternative solver options:\n",
      "    https://scikit-learn.org/stable/modules/linear_model.html#logistic-regression\n",
      "  n_iter_i = _check_optimize_result(\n",
      "/home/potatosalad/.local/share/virtualenvs/CSC_homework_3_group4-nat9sqnt/lib/python3.10/site-packages/sklearn/linear_model/_logistic.py:458: ConvergenceWarning: lbfgs failed to converge (status=1):\n",
      "STOP: TOTAL NO. of ITERATIONS REACHED LIMIT.\n",
      "\n",
      "Increase the number of iterations (max_iter) or scale the data as shown in:\n",
      "    https://scikit-learn.org/stable/modules/preprocessing.html\n",
      "Please also refer to the documentation for alternative solver options:\n",
      "    https://scikit-learn.org/stable/modules/linear_model.html#logistic-regression\n",
      "  n_iter_i = _check_optimize_result(\n",
      "/home/potatosalad/.local/share/virtualenvs/CSC_homework_3_group4-nat9sqnt/lib/python3.10/site-packages/sklearn/linear_model/_logistic.py:458: ConvergenceWarning: lbfgs failed to converge (status=1):\n",
      "STOP: TOTAL NO. of ITERATIONS REACHED LIMIT.\n",
      "\n",
      "Increase the number of iterations (max_iter) or scale the data as shown in:\n",
      "    https://scikit-learn.org/stable/modules/preprocessing.html\n",
      "Please also refer to the documentation for alternative solver options:\n",
      "    https://scikit-learn.org/stable/modules/linear_model.html#logistic-regression\n",
      "  n_iter_i = _check_optimize_result(\n",
      "/home/potatosalad/.local/share/virtualenvs/CSC_homework_3_group4-nat9sqnt/lib/python3.10/site-packages/sklearn/linear_model/_logistic.py:458: ConvergenceWarning: lbfgs failed to converge (status=1):\n",
      "STOP: TOTAL NO. of ITERATIONS REACHED LIMIT.\n",
      "\n",
      "Increase the number of iterations (max_iter) or scale the data as shown in:\n",
      "    https://scikit-learn.org/stable/modules/preprocessing.html\n",
      "Please also refer to the documentation for alternative solver options:\n",
      "    https://scikit-learn.org/stable/modules/linear_model.html#logistic-regression\n",
      "  n_iter_i = _check_optimize_result(\n",
      "/home/potatosalad/.local/share/virtualenvs/CSC_homework_3_group4-nat9sqnt/lib/python3.10/site-packages/sklearn/linear_model/_logistic.py:458: ConvergenceWarning: lbfgs failed to converge (status=1):\n",
      "STOP: TOTAL NO. of ITERATIONS REACHED LIMIT.\n",
      "\n",
      "Increase the number of iterations (max_iter) or scale the data as shown in:\n",
      "    https://scikit-learn.org/stable/modules/preprocessing.html\n",
      "Please also refer to the documentation for alternative solver options:\n",
      "    https://scikit-learn.org/stable/modules/linear_model.html#logistic-regression\n",
      "  n_iter_i = _check_optimize_result(\n",
      "/home/potatosalad/.local/share/virtualenvs/CSC_homework_3_group4-nat9sqnt/lib/python3.10/site-packages/sklearn/linear_model/_logistic.py:458: ConvergenceWarning: lbfgs failed to converge (status=1):\n",
      "STOP: TOTAL NO. of ITERATIONS REACHED LIMIT.\n",
      "\n",
      "Increase the number of iterations (max_iter) or scale the data as shown in:\n",
      "    https://scikit-learn.org/stable/modules/preprocessing.html\n",
      "Please also refer to the documentation for alternative solver options:\n",
      "    https://scikit-learn.org/stable/modules/linear_model.html#logistic-regression\n",
      "  n_iter_i = _check_optimize_result(\n",
      "/home/potatosalad/.local/share/virtualenvs/CSC_homework_3_group4-nat9sqnt/lib/python3.10/site-packages/sklearn/linear_model/_logistic.py:458: ConvergenceWarning: lbfgs failed to converge (status=1):\n",
      "STOP: TOTAL NO. of ITERATIONS REACHED LIMIT.\n",
      "\n",
      "Increase the number of iterations (max_iter) or scale the data as shown in:\n",
      "    https://scikit-learn.org/stable/modules/preprocessing.html\n",
      "Please also refer to the documentation for alternative solver options:\n",
      "    https://scikit-learn.org/stable/modules/linear_model.html#logistic-regression\n",
      "  n_iter_i = _check_optimize_result(\n",
      "/home/potatosalad/.local/share/virtualenvs/CSC_homework_3_group4-nat9sqnt/lib/python3.10/site-packages/sklearn/linear_model/_logistic.py:458: ConvergenceWarning: lbfgs failed to converge (status=1):\n",
      "STOP: TOTAL NO. of ITERATIONS REACHED LIMIT.\n",
      "\n",
      "Increase the number of iterations (max_iter) or scale the data as shown in:\n",
      "    https://scikit-learn.org/stable/modules/preprocessing.html\n",
      "Please also refer to the documentation for alternative solver options:\n",
      "    https://scikit-learn.org/stable/modules/linear_model.html#logistic-regression\n",
      "  n_iter_i = _check_optimize_result(\n",
      "/home/potatosalad/.local/share/virtualenvs/CSC_homework_3_group4-nat9sqnt/lib/python3.10/site-packages/sklearn/linear_model/_logistic.py:458: ConvergenceWarning: lbfgs failed to converge (status=1):\n",
      "STOP: TOTAL NO. of ITERATIONS REACHED LIMIT.\n",
      "\n",
      "Increase the number of iterations (max_iter) or scale the data as shown in:\n",
      "    https://scikit-learn.org/stable/modules/preprocessing.html\n",
      "Please also refer to the documentation for alternative solver options:\n",
      "    https://scikit-learn.org/stable/modules/linear_model.html#logistic-regression\n",
      "  n_iter_i = _check_optimize_result(\n",
      "/home/potatosalad/.local/share/virtualenvs/CSC_homework_3_group4-nat9sqnt/lib/python3.10/site-packages/sklearn/linear_model/_logistic.py:458: ConvergenceWarning: lbfgs failed to converge (status=1):\n",
      "STOP: TOTAL NO. of ITERATIONS REACHED LIMIT.\n",
      "\n",
      "Increase the number of iterations (max_iter) or scale the data as shown in:\n",
      "    https://scikit-learn.org/stable/modules/preprocessing.html\n",
      "Please also refer to the documentation for alternative solver options:\n",
      "    https://scikit-learn.org/stable/modules/linear_model.html#logistic-regression\n",
      "  n_iter_i = _check_optimize_result(\n",
      "/home/potatosalad/.local/share/virtualenvs/CSC_homework_3_group4-nat9sqnt/lib/python3.10/site-packages/sklearn/linear_model/_logistic.py:458: ConvergenceWarning: lbfgs failed to converge (status=1):\n",
      "STOP: TOTAL NO. of ITERATIONS REACHED LIMIT.\n",
      "\n",
      "Increase the number of iterations (max_iter) or scale the data as shown in:\n",
      "    https://scikit-learn.org/stable/modules/preprocessing.html\n",
      "Please also refer to the documentation for alternative solver options:\n",
      "    https://scikit-learn.org/stable/modules/linear_model.html#logistic-regression\n",
      "  n_iter_i = _check_optimize_result(\n",
      "/home/potatosalad/.local/share/virtualenvs/CSC_homework_3_group4-nat9sqnt/lib/python3.10/site-packages/sklearn/linear_model/_logistic.py:458: ConvergenceWarning: lbfgs failed to converge (status=1):\n",
      "STOP: TOTAL NO. of ITERATIONS REACHED LIMIT.\n",
      "\n",
      "Increase the number of iterations (max_iter) or scale the data as shown in:\n",
      "    https://scikit-learn.org/stable/modules/preprocessing.html\n",
      "Please also refer to the documentation for alternative solver options:\n",
      "    https://scikit-learn.org/stable/modules/linear_model.html#logistic-regression\n",
      "  n_iter_i = _check_optimize_result(\n",
      "/home/potatosalad/.local/share/virtualenvs/CSC_homework_3_group4-nat9sqnt/lib/python3.10/site-packages/sklearn/linear_model/_logistic.py:458: ConvergenceWarning: lbfgs failed to converge (status=1):\n",
      "STOP: TOTAL NO. of ITERATIONS REACHED LIMIT.\n",
      "\n",
      "Increase the number of iterations (max_iter) or scale the data as shown in:\n",
      "    https://scikit-learn.org/stable/modules/preprocessing.html\n",
      "Please also refer to the documentation for alternative solver options:\n",
      "    https://scikit-learn.org/stable/modules/linear_model.html#logistic-regression\n",
      "  n_iter_i = _check_optimize_result(\n",
      "/home/potatosalad/.local/share/virtualenvs/CSC_homework_3_group4-nat9sqnt/lib/python3.10/site-packages/sklearn/linear_model/_logistic.py:458: ConvergenceWarning: lbfgs failed to converge (status=1):\n",
      "STOP: TOTAL NO. of ITERATIONS REACHED LIMIT.\n",
      "\n",
      "Increase the number of iterations (max_iter) or scale the data as shown in:\n",
      "    https://scikit-learn.org/stable/modules/preprocessing.html\n",
      "Please also refer to the documentation for alternative solver options:\n",
      "    https://scikit-learn.org/stable/modules/linear_model.html#logistic-regression\n",
      "  n_iter_i = _check_optimize_result(\n",
      "/home/potatosalad/.local/share/virtualenvs/CSC_homework_3_group4-nat9sqnt/lib/python3.10/site-packages/sklearn/linear_model/_logistic.py:458: ConvergenceWarning: lbfgs failed to converge (status=1):\n",
      "STOP: TOTAL NO. of ITERATIONS REACHED LIMIT.\n",
      "\n",
      "Increase the number of iterations (max_iter) or scale the data as shown in:\n",
      "    https://scikit-learn.org/stable/modules/preprocessing.html\n",
      "Please also refer to the documentation for alternative solver options:\n",
      "    https://scikit-learn.org/stable/modules/linear_model.html#logistic-regression\n",
      "  n_iter_i = _check_optimize_result(\n",
      "/home/potatosalad/.local/share/virtualenvs/CSC_homework_3_group4-nat9sqnt/lib/python3.10/site-packages/sklearn/linear_model/_logistic.py:458: ConvergenceWarning: lbfgs failed to converge (status=1):\n",
      "STOP: TOTAL NO. of ITERATIONS REACHED LIMIT.\n",
      "\n",
      "Increase the number of iterations (max_iter) or scale the data as shown in:\n",
      "    https://scikit-learn.org/stable/modules/preprocessing.html\n",
      "Please also refer to the documentation for alternative solver options:\n",
      "    https://scikit-learn.org/stable/modules/linear_model.html#logistic-regression\n",
      "  n_iter_i = _check_optimize_result(\n",
      "/home/potatosalad/.local/share/virtualenvs/CSC_homework_3_group4-nat9sqnt/lib/python3.10/site-packages/sklearn/linear_model/_logistic.py:458: ConvergenceWarning: lbfgs failed to converge (status=1):\n",
      "STOP: TOTAL NO. of ITERATIONS REACHED LIMIT.\n",
      "\n",
      "Increase the number of iterations (max_iter) or scale the data as shown in:\n",
      "    https://scikit-learn.org/stable/modules/preprocessing.html\n",
      "Please also refer to the documentation for alternative solver options:\n",
      "    https://scikit-learn.org/stable/modules/linear_model.html#logistic-regression\n",
      "  n_iter_i = _check_optimize_result(\n",
      "/home/potatosalad/.local/share/virtualenvs/CSC_homework_3_group4-nat9sqnt/lib/python3.10/site-packages/sklearn/linear_model/_logistic.py:458: ConvergenceWarning: lbfgs failed to converge (status=1):\n",
      "STOP: TOTAL NO. of ITERATIONS REACHED LIMIT.\n",
      "\n",
      "Increase the number of iterations (max_iter) or scale the data as shown in:\n",
      "    https://scikit-learn.org/stable/modules/preprocessing.html\n",
      "Please also refer to the documentation for alternative solver options:\n",
      "    https://scikit-learn.org/stable/modules/linear_model.html#logistic-regression\n",
      "  n_iter_i = _check_optimize_result(\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/potatosalad/.local/share/virtualenvs/CSC_homework_3_group4-nat9sqnt/lib/python3.10/site-packages/sklearn/linear_model/_logistic.py:458: ConvergenceWarning: lbfgs failed to converge (status=1):\n",
      "STOP: TOTAL NO. of ITERATIONS REACHED LIMIT.\n",
      "\n",
      "Increase the number of iterations (max_iter) or scale the data as shown in:\n",
      "    https://scikit-learn.org/stable/modules/preprocessing.html\n",
      "Please also refer to the documentation for alternative solver options:\n",
      "    https://scikit-learn.org/stable/modules/linear_model.html#logistic-regression\n",
      "  n_iter_i = _check_optimize_result(\n",
      "/home/potatosalad/.local/share/virtualenvs/CSC_homework_3_group4-nat9sqnt/lib/python3.10/site-packages/sklearn/linear_model/_logistic.py:458: ConvergenceWarning: lbfgs failed to converge (status=1):\n",
      "STOP: TOTAL NO. of ITERATIONS REACHED LIMIT.\n",
      "\n",
      "Increase the number of iterations (max_iter) or scale the data as shown in:\n",
      "    https://scikit-learn.org/stable/modules/preprocessing.html\n",
      "Please also refer to the documentation for alternative solver options:\n",
      "    https://scikit-learn.org/stable/modules/linear_model.html#logistic-regression\n",
      "  n_iter_i = _check_optimize_result(\n",
      "/home/potatosalad/.local/share/virtualenvs/CSC_homework_3_group4-nat9sqnt/lib/python3.10/site-packages/sklearn/linear_model/_logistic.py:458: ConvergenceWarning: lbfgs failed to converge (status=1):\n",
      "STOP: TOTAL NO. of ITERATIONS REACHED LIMIT.\n",
      "\n",
      "Increase the number of iterations (max_iter) or scale the data as shown in:\n",
      "    https://scikit-learn.org/stable/modules/preprocessing.html\n",
      "Please also refer to the documentation for alternative solver options:\n",
      "    https://scikit-learn.org/stable/modules/linear_model.html#logistic-regression\n",
      "  n_iter_i = _check_optimize_result(\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Model performance for train set ENSEMBLE_wine_Data_gridsearch\n",
      "- Accuracy: 1.0000\n",
      "- F1 score: 1.000000\n",
      "- Precision: 1.000000\n",
      "- Recall: 1.000000\n",
      "----------------------------------\n",
      "===================================\n",
      "Model performance for test set ENSEMBLE_wine_Data_gridsearch\n",
      "- Accuracy: 1.0000\n",
      "- F1 score: 1.000000\n",
      "- Precision: 1.000000\n",
      "- Recall: 1.000000\n",
      "----------------------------------\n",
      "===================================\n"
     ]
    }
   ],
   "source": [
    "# GRIDSEARCH\n",
    "fit_models(X_train, y_train, X_test, y_test, data_wine_target_names, models, params, \"wine_Data_gridsearch\", \"experiment_wine_gridsearch\",1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 50,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Fitting for parameters...\n",
      "X:      Alcohol  Malic acid   Ash  Alcalinity of ash  Magnesium  Total Phenols  \\\n",
      "158    14.34        1.68  2.70               25.0         98           2.80   \n",
      "137    12.53        5.51  2.64               25.0         96           1.79   \n",
      "98     12.37        1.07  2.10               18.5         88           3.52   \n",
      "159    13.48        1.67  2.64               22.5         89           2.60   \n",
      "38     13.07        1.50  2.10               15.5         98           2.40   \n",
      "..       ...         ...   ...                ...        ...            ...   \n",
      "71     13.86        1.51  2.67               25.0         86           2.95   \n",
      "106    12.25        1.73  2.12               19.0         80           1.65   \n",
      "14     14.38        1.87  2.38               12.0        102           3.30   \n",
      "92     12.69        1.53  2.26               20.7         80           1.38   \n",
      "102    12.34        2.45  2.46               21.0         98           2.56   \n",
      "\n",
      "     Flavanoids  Nonflavanoid phenols  Proanthocyanins  Color intensity   Hue  \\\n",
      "158        1.31                  0.53             2.70            13.00  0.57   \n",
      "137        0.60                  0.63             1.10             5.00  0.82   \n",
      "98         3.75                  0.24             1.95             4.50  1.04   \n",
      "159        1.10                  0.52             2.29            11.75  0.57   \n",
      "38         2.64                  0.28             1.37             3.70  1.18   \n",
      "..          ...                   ...              ...              ...   ...   \n",
      "71         2.86                  0.21             1.87             3.38  1.36   \n",
      "106        2.03                  0.37             1.63             3.40  1.00   \n",
      "14         3.64                  0.29             2.96             7.50  1.20   \n",
      "92         1.46                  0.58             1.62             3.05  0.96   \n",
      "102        2.11                  0.34             1.31             2.80  0.80   \n",
      "\n",
      "     OD280/OD315 of diluted wines  Proline  \n",
      "158                          1.96      660  \n",
      "137                          1.69      515  \n",
      "98                           2.77      660  \n",
      "159                          1.78      620  \n",
      "38                           2.69     1020  \n",
      "..                            ...      ...  \n",
      "71                           3.16      410  \n",
      "106                          3.17      510  \n",
      "14                           3.00     1547  \n",
      "92                           2.06      495  \n",
      "102                          3.38      438  \n",
      "\n",
      "[142 rows x 13 columns]\n",
      "--------------------------------------------------\n",
      "Y: 158    3\n",
      "137    3\n",
      "98     2\n",
      "159    3\n",
      "38     1\n",
      "      ..\n",
      "71     2\n",
      "106    2\n",
      "14     1\n",
      "92     2\n",
      "102    2\n",
      "Name: Cultivars, Length: 142, dtype: int64\n",
      "--------------------------------------------------\n",
      "X_t:      Alcohol  Malic acid   Ash  Alcalinity of ash  Magnesium  Total Phenols  \\\n",
      "19     13.64        3.10  2.56               15.2        116           2.70   \n",
      "45     14.21        4.04  2.44               18.9        111           2.85   \n",
      "140    12.93        2.81  2.70               21.0         96           1.54   \n",
      "30     13.73        1.50  2.70               22.5        101           3.00   \n",
      "67     12.37        1.17  1.92               19.6         78           2.11   \n",
      "16     14.30        1.92  2.72               20.0        120           2.80   \n",
      "119    12.00        3.43  2.00               19.0         87           2.00   \n",
      "174    13.40        3.91  2.48               23.0        102           1.80   \n",
      "109    11.61        1.35  2.70               20.0         94           2.74   \n",
      "141    13.36        2.56  2.35               20.0         89           1.40   \n",
      "24     13.50        1.81  2.61               20.0         96           2.53   \n",
      "150    13.50        3.12  2.62               24.0        123           1.40   \n",
      "41     13.41        3.84  2.12               18.8         90           2.45   \n",
      "118    12.77        3.43  1.98               16.0         80           1.63   \n",
      "15     13.63        1.81  2.70               17.2        112           2.85   \n",
      "111    12.52        2.43  2.17               21.0         88           2.55   \n",
      "113    11.41        0.74  2.50               21.0         88           2.48   \n",
      "82     12.08        1.13  2.51               24.0         78           2.00   \n",
      "9      13.86        1.35  2.27               16.0         98           2.98   \n",
      "114    12.08        1.39  2.50               22.5         84           2.56   \n",
      "18     14.19        1.59  2.48               16.5        108           3.30   \n",
      "66     13.11        1.01  1.70               15.0         78           2.98   \n",
      "60     12.33        1.10  2.28               16.0        101           2.05   \n",
      "169    13.40        4.60  2.86               25.0        112           1.98   \n",
      "171    12.77        2.39  2.28               19.5         86           1.39   \n",
      "164    13.78        2.76  2.30               22.0         90           1.35   \n",
      "117    12.42        1.61  2.19               22.5        108           2.00   \n",
      "65     12.37        1.21  2.56               18.1         98           2.42   \n",
      "90     12.08        1.83  2.32               18.5         81           1.60   \n",
      "55     13.56        1.73  2.46               20.5        116           2.96   \n",
      "29     14.02        1.68  2.21               16.0         96           2.65   \n",
      "128    12.37        1.63  2.30               24.5         88           2.22   \n",
      "145    13.16        3.57  2.15               21.0        102           1.50   \n",
      "31     13.58        1.66  2.36               19.1        106           2.86   \n",
      "12     13.75        1.73  2.41               16.0         89           2.60   \n",
      "42     13.88        1.89  2.59               15.0        101           3.25   \n",
      "\n",
      "     Flavanoids  Nonflavanoid phenols  Proanthocyanins  Color intensity   Hue  \\\n",
      "19         3.03                  0.17             1.66         5.100000  0.96   \n",
      "45         2.65                  0.30             1.25         5.240000  0.87   \n",
      "140        0.50                  0.53             0.75         4.600000  0.77   \n",
      "30         3.25                  0.29             2.38         5.700000  1.19   \n",
      "67         2.00                  0.27             1.04         4.680000  1.12   \n",
      "16         3.14                  0.33             1.97         6.200000  1.07   \n",
      "119        1.64                  0.37             1.87         1.280000  0.93   \n",
      "174        0.75                  0.43             1.41         7.300000  0.70   \n",
      "109        2.92                  0.29             2.49         2.650000  0.96   \n",
      "141        0.50                  0.37             0.64         5.600000  0.70   \n",
      "24         2.61                  0.28             1.66         3.520000  1.12   \n",
      "150        1.57                  0.22             1.25         8.600000  0.59   \n",
      "41         2.68                  0.27             1.48         4.280000  0.91   \n",
      "118        1.25                  0.43             0.83         3.400000  0.70   \n",
      "15         2.91                  0.30             1.46         7.300000  1.28   \n",
      "111        2.27                  0.26             1.22         2.000000  0.90   \n",
      "113        2.01                  0.42             1.44         3.080000  1.10   \n",
      "82         1.58                  0.40             1.40         2.200000  1.31   \n",
      "9          3.15                  0.22             1.85         7.220000  1.01   \n",
      "114        2.29                  0.43             1.04         2.900000  0.93   \n",
      "18         3.93                  0.32             1.86         8.700000  1.23   \n",
      "66         3.18                  0.26             2.28         5.300000  1.12   \n",
      "60         1.09                  0.63             0.41         3.270000  1.25   \n",
      "169        0.96                  0.27             1.11         8.500000  0.67   \n",
      "171        0.51                  0.48             0.64         9.899999  0.57   \n",
      "164        0.68                  0.41             1.03         9.580000  0.70   \n",
      "117        2.09                  0.34             1.61         2.060000  1.06   \n",
      "65         2.65                  0.37             2.08         4.600000  1.19   \n",
      "90         1.50                  0.52             1.64         2.400000  1.08   \n",
      "55         2.78                  0.20             2.45         6.250000  0.98   \n",
      "29         2.33                  0.26             1.98         4.700000  1.04   \n",
      "128        2.45                  0.40             1.90         2.120000  0.89   \n",
      "145        0.55                  0.43             1.30         4.000000  0.60   \n",
      "31         3.19                  0.22             1.95         6.900000  1.09   \n",
      "12         2.76                  0.29             1.81         5.600000  1.15   \n",
      "42         3.56                  0.17             1.70         5.430000  0.88   \n",
      "\n",
      "     OD280/OD315 of diluted wines  Proline  \n",
      "19                           3.36      845  \n",
      "45                           3.33     1080  \n",
      "140                          2.31      600  \n",
      "30                           2.71     1285  \n",
      "67                           3.48      510  \n",
      "16                           2.65     1280  \n",
      "119                          3.05      564  \n",
      "174                          1.56      750  \n",
      "109                          3.26      680  \n",
      "141                          2.47      780  \n",
      "24                           3.82      845  \n",
      "150                          1.30      500  \n",
      "41                           3.00     1035  \n",
      "118                          2.12      372  \n",
      "15                           2.88     1310  \n",
      "111                          2.78      325  \n",
      "113                          2.31      434  \n",
      "82                           2.72      630  \n",
      "9                            3.55     1045  \n",
      "114                          3.19      385  \n",
      "18                           2.82     1680  \n",
      "66                           3.18      502  \n",
      "60                           1.67      680  \n",
      "169                          1.92      630  \n",
      "171                          1.63      470  \n",
      "164                          1.68      615  \n",
      "117                          2.96      345  \n",
      "65                           2.30      678  \n",
      "90                           2.27      480  \n",
      "55                           3.03     1120  \n",
      "29                           3.59     1035  \n",
      "128                          2.78      342  \n",
      "145                          1.68      830  \n",
      "31                           2.88     1515  \n",
      "12                           2.90     1320  \n",
      "42                           3.56     1095  \n",
      "--------------------------------------------------\n",
      "Y_t: 19     1\n",
      "45     1\n",
      "140    3\n",
      "30     1\n",
      "67     2\n",
      "16     1\n",
      "119    2\n",
      "174    3\n",
      "109    2\n",
      "141    3\n",
      "24     1\n",
      "150    3\n",
      "41     1\n",
      "118    2\n",
      "15     1\n",
      "111    2\n",
      "113    2\n",
      "82     2\n",
      "9      1\n",
      "114    2\n",
      "18     1\n",
      "66     2\n",
      "60     2\n",
      "169    3\n",
      "171    3\n",
      "164    3\n",
      "117    2\n",
      "65     2\n",
      "90     2\n",
      "55     1\n",
      "29     1\n",
      "128    2\n",
      "145    3\n",
      "31     1\n",
      "12     1\n",
      "42     1\n",
      "Name: Cultivars, dtype: int64\n",
      "--------------------------------------------------\n",
      "models: {'Logistic Regression': LogisticRegression(), 'Decision Tree': DecisionTreeClassifier(), 'Random Forest': RandomForestClassifier(random_state=42), 'Gradient Boosting': GradientBoostingClassifier(), 'K-Nearest Neighbors': KNeighborsClassifier(), 'SVM': SVC(probability=True)}\n",
      "--------------------------------------------------\n",
      "params: {'Logistic Regression': {'solver': ['newton-cholesky', 'liblinear', 'newton-cg', 'sag', 'saga'], 'penalty': ['l2'], 'C': [0.001, 0.01, 0.1, 1, 100], 'max_iter': [10000]}, 'Decision Tree': {'criterion': ['gini', 'entropy'], 'max_depth': [4, 6, 8, 10, 15, 50, 90]}, 'Random Forest': {'n_estimators': [200, 500], 'max_features': [None, 'sqrt', 'log2'], 'max_depth': [4, 5, 6, 7, 8], 'criterion': ['gini', 'entropy']}, 'Gradient Boosting': {'loss': ['log_loss'], 'learning_rate': [0.1, 0.01], 'max_depth': [3, 5, 8], 'max_features': ['log2', 'sqrt'], 'criterion': ['friedman_mse', 'squared_error'], 'subsample': [0.5, 0.8, 1.0], 'n_estimators': [10, 100, 1000]}, 'K-Nearest Neighbors': {'weights': ['uniform', 'distance'], 'leaf_size': [5, 15, 25]}, 'SVM': {'kernel': ['rbf'], 'C': [0.1, 1, 10, 100, 1000], 'degree': [1, 5], 'gamma': [1, 0.1, 0.01, 0.001]}}\n",
      "--------------------------------------------------\n",
      "run: wine_Data_no_gridsearch\n",
      "--------------------------------------------------\n",
      "folder: experiment_wine_no_gridsearch\n",
      "Logistic Regression\n",
      "\n",
      "##################################################\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Model performance for Training set Logistic Regression_wine_Data_no_gridsearch\n",
      "- Accuracy: 1.0000\n",
      "- F1 score: 1.000000\n",
      "- Precision: 1.000000\n",
      "- Recall: 1.000000\n",
      "----------------------------------\n",
      "===================================\n",
      "Model performance for Test set Logistic Regression_wine_Data_no_gridsearch\n",
      "- Accuracy: 1.0000\n",
      "- F1 score: 1.000000\n",
      "- Precision: 1.000000\n",
      "- Recall: 1.000000\n",
      "----------------------------------\n",
      "===================================\n",
      "Decision Tree\n",
      "\n",
      "##################################################\n",
      "Model performance for Training set Decision Tree_wine_Data_no_gridsearch\n",
      "- Accuracy: 1.0000\n",
      "- F1 score: 1.000000\n",
      "- Precision: 1.000000\n",
      "- Recall: 1.000000\n",
      "----------------------------------\n",
      "===================================\n",
      "Model performance for Test set Decision Tree_wine_Data_no_gridsearch\n",
      "- Accuracy: 0.9444\n",
      "- F1 score: 0.943997\n",
      "- Precision: 0.946296\n",
      "- Recall: 0.944444\n",
      "----------------------------------\n",
      "===================================\n",
      "Random Forest\n",
      "\n",
      "##################################################\n",
      "Model performance for Training set Random Forest_wine_Data_no_gridsearch\n",
      "- Accuracy: 1.0000\n",
      "- F1 score: 1.000000\n",
      "- Precision: 1.000000\n",
      "- Recall: 1.000000\n",
      "----------------------------------\n",
      "===================================\n",
      "Model performance for Test set Random Forest_wine_Data_no_gridsearch\n",
      "- Accuracy: 1.0000\n",
      "- F1 score: 1.000000\n",
      "- Precision: 1.000000\n",
      "- Recall: 1.000000\n",
      "----------------------------------\n",
      "===================================\n",
      "Gradient Boosting\n",
      "\n",
      "##################################################\n",
      "Model performance for Training set Gradient Boosting_wine_Data_no_gridsearch\n",
      "- Accuracy: 1.0000\n",
      "- F1 score: 1.000000\n",
      "- Precision: 1.000000\n",
      "- Recall: 1.000000\n",
      "----------------------------------\n",
      "===================================\n",
      "Model performance for Test set Gradient Boosting_wine_Data_no_gridsearch\n",
      "- Accuracy: 0.9444\n",
      "- F1 score: 0.943997\n",
      "- Precision: 0.946296\n",
      "- Recall: 0.944444\n",
      "----------------------------------\n",
      "===================================\n",
      "K-Nearest Neighbors\n",
      "\n",
      "##################################################\n",
      "Model performance for Training set K-Nearest Neighbors_wine_Data_no_gridsearch\n",
      "- Accuracy: 0.9859\n",
      "- F1 score: 0.985872\n",
      "- Precision: 0.986240\n",
      "- Recall: 0.985915\n",
      "----------------------------------\n",
      "===================================\n",
      "Model performance for Test set K-Nearest Neighbors_wine_Data_no_gridsearch\n",
      "- Accuracy: 0.9444\n",
      "- F1 score: 0.943604\n",
      "- Precision: 0.949383\n",
      "- Recall: 0.944444\n",
      "----------------------------------\n",
      "===================================\n",
      "SVM\n",
      "\n",
      "##################################################\n",
      "Model performance for Training set SVM_wine_Data_no_gridsearch\n",
      "- Accuracy: 1.0000\n",
      "- F1 score: 1.000000\n",
      "- Precision: 1.000000\n",
      "- Recall: 1.000000\n",
      "----------------------------------\n",
      "===================================\n",
      "Model performance for Test set SVM_wine_Data_no_gridsearch\n",
      "- Accuracy: 1.0000\n",
      "- F1 score: 1.000000\n",
      "- Precision: 1.000000\n",
      "- Recall: 1.000000\n",
      "----------------------------------\n",
      "===================================\n",
      "################################################\n",
      "################### ENSEMBLE ###################\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/potatosalad/.local/share/virtualenvs/CSC_homework_3_group4-nat9sqnt/lib/python3.10/site-packages/sklearn/linear_model/_logistic.py:458: ConvergenceWarning: lbfgs failed to converge (status=1):\n",
      "STOP: TOTAL NO. of ITERATIONS REACHED LIMIT.\n",
      "\n",
      "Increase the number of iterations (max_iter) or scale the data as shown in:\n",
      "    https://scikit-learn.org/stable/modules/preprocessing.html\n",
      "Please also refer to the documentation for alternative solver options:\n",
      "    https://scikit-learn.org/stable/modules/linear_model.html#logistic-regression\n",
      "  n_iter_i = _check_optimize_result(\n",
      "/home/potatosalad/.local/share/virtualenvs/CSC_homework_3_group4-nat9sqnt/lib/python3.10/site-packages/sklearn/linear_model/_logistic.py:458: ConvergenceWarning: lbfgs failed to converge (status=1):\n",
      "STOP: TOTAL NO. of ITERATIONS REACHED LIMIT.\n",
      "\n",
      "Increase the number of iterations (max_iter) or scale the data as shown in:\n",
      "    https://scikit-learn.org/stable/modules/preprocessing.html\n",
      "Please also refer to the documentation for alternative solver options:\n",
      "    https://scikit-learn.org/stable/modules/linear_model.html#logistic-regression\n",
      "  n_iter_i = _check_optimize_result(\n",
      "/home/potatosalad/.local/share/virtualenvs/CSC_homework_3_group4-nat9sqnt/lib/python3.10/site-packages/sklearn/linear_model/_logistic.py:458: ConvergenceWarning: lbfgs failed to converge (status=1):\n",
      "STOP: TOTAL NO. of ITERATIONS REACHED LIMIT.\n",
      "\n",
      "Increase the number of iterations (max_iter) or scale the data as shown in:\n",
      "    https://scikit-learn.org/stable/modules/preprocessing.html\n",
      "Please also refer to the documentation for alternative solver options:\n",
      "    https://scikit-learn.org/stable/modules/linear_model.html#logistic-regression\n",
      "  n_iter_i = _check_optimize_result(\n",
      "/home/potatosalad/.local/share/virtualenvs/CSC_homework_3_group4-nat9sqnt/lib/python3.10/site-packages/sklearn/linear_model/_logistic.py:458: ConvergenceWarning: lbfgs failed to converge (status=1):\n",
      "STOP: TOTAL NO. of ITERATIONS REACHED LIMIT.\n",
      "\n",
      "Increase the number of iterations (max_iter) or scale the data as shown in:\n",
      "    https://scikit-learn.org/stable/modules/preprocessing.html\n",
      "Please also refer to the documentation for alternative solver options:\n",
      "    https://scikit-learn.org/stable/modules/linear_model.html#logistic-regression\n",
      "  n_iter_i = _check_optimize_result(\n",
      "/home/potatosalad/.local/share/virtualenvs/CSC_homework_3_group4-nat9sqnt/lib/python3.10/site-packages/sklearn/linear_model/_logistic.py:458: ConvergenceWarning: lbfgs failed to converge (status=1):\n",
      "STOP: TOTAL NO. of ITERATIONS REACHED LIMIT.\n",
      "\n",
      "Increase the number of iterations (max_iter) or scale the data as shown in:\n",
      "    https://scikit-learn.org/stable/modules/preprocessing.html\n",
      "Please also refer to the documentation for alternative solver options:\n",
      "    https://scikit-learn.org/stable/modules/linear_model.html#logistic-regression\n",
      "  n_iter_i = _check_optimize_result(\n",
      "/home/potatosalad/.local/share/virtualenvs/CSC_homework_3_group4-nat9sqnt/lib/python3.10/site-packages/sklearn/linear_model/_logistic.py:458: ConvergenceWarning: lbfgs failed to converge (status=1):\n",
      "STOP: TOTAL NO. of ITERATIONS REACHED LIMIT.\n",
      "\n",
      "Increase the number of iterations (max_iter) or scale the data as shown in:\n",
      "    https://scikit-learn.org/stable/modules/preprocessing.html\n",
      "Please also refer to the documentation for alternative solver options:\n",
      "    https://scikit-learn.org/stable/modules/linear_model.html#logistic-regression\n",
      "  n_iter_i = _check_optimize_result(\n",
      "/home/potatosalad/.local/share/virtualenvs/CSC_homework_3_group4-nat9sqnt/lib/python3.10/site-packages/sklearn/linear_model/_logistic.py:458: ConvergenceWarning: lbfgs failed to converge (status=1):\n",
      "STOP: TOTAL NO. of ITERATIONS REACHED LIMIT.\n",
      "\n",
      "Increase the number of iterations (max_iter) or scale the data as shown in:\n",
      "    https://scikit-learn.org/stable/modules/preprocessing.html\n",
      "Please also refer to the documentation for alternative solver options:\n",
      "    https://scikit-learn.org/stable/modules/linear_model.html#logistic-regression\n",
      "  n_iter_i = _check_optimize_result(\n",
      "/home/potatosalad/.local/share/virtualenvs/CSC_homework_3_group4-nat9sqnt/lib/python3.10/site-packages/sklearn/linear_model/_logistic.py:458: ConvergenceWarning: lbfgs failed to converge (status=1):\n",
      "STOP: TOTAL NO. of ITERATIONS REACHED LIMIT.\n",
      "\n",
      "Increase the number of iterations (max_iter) or scale the data as shown in:\n",
      "    https://scikit-learn.org/stable/modules/preprocessing.html\n",
      "Please also refer to the documentation for alternative solver options:\n",
      "    https://scikit-learn.org/stable/modules/linear_model.html#logistic-regression\n",
      "  n_iter_i = _check_optimize_result(\n",
      "/home/potatosalad/.local/share/virtualenvs/CSC_homework_3_group4-nat9sqnt/lib/python3.10/site-packages/sklearn/linear_model/_logistic.py:458: ConvergenceWarning: lbfgs failed to converge (status=1):\n",
      "STOP: TOTAL NO. of ITERATIONS REACHED LIMIT.\n",
      "\n",
      "Increase the number of iterations (max_iter) or scale the data as shown in:\n",
      "    https://scikit-learn.org/stable/modules/preprocessing.html\n",
      "Please also refer to the documentation for alternative solver options:\n",
      "    https://scikit-learn.org/stable/modules/linear_model.html#logistic-regression\n",
      "  n_iter_i = _check_optimize_result(\n",
      "/home/potatosalad/.local/share/virtualenvs/CSC_homework_3_group4-nat9sqnt/lib/python3.10/site-packages/sklearn/linear_model/_logistic.py:458: ConvergenceWarning: lbfgs failed to converge (status=1):\n",
      "STOP: TOTAL NO. of ITERATIONS REACHED LIMIT.\n",
      "\n",
      "Increase the number of iterations (max_iter) or scale the data as shown in:\n",
      "    https://scikit-learn.org/stable/modules/preprocessing.html\n",
      "Please also refer to the documentation for alternative solver options:\n",
      "    https://scikit-learn.org/stable/modules/linear_model.html#logistic-regression\n",
      "  n_iter_i = _check_optimize_result(\n",
      "/home/potatosalad/.local/share/virtualenvs/CSC_homework_3_group4-nat9sqnt/lib/python3.10/site-packages/sklearn/linear_model/_logistic.py:458: ConvergenceWarning: lbfgs failed to converge (status=1):\n",
      "STOP: TOTAL NO. of ITERATIONS REACHED LIMIT.\n",
      "\n",
      "Increase the number of iterations (max_iter) or scale the data as shown in:\n",
      "    https://scikit-learn.org/stable/modules/preprocessing.html\n",
      "Please also refer to the documentation for alternative solver options:\n",
      "    https://scikit-learn.org/stable/modules/linear_model.html#logistic-regression\n",
      "  n_iter_i = _check_optimize_result(\n",
      "/home/potatosalad/.local/share/virtualenvs/CSC_homework_3_group4-nat9sqnt/lib/python3.10/site-packages/sklearn/linear_model/_logistic.py:458: ConvergenceWarning: lbfgs failed to converge (status=1):\n",
      "STOP: TOTAL NO. of ITERATIONS REACHED LIMIT.\n",
      "\n",
      "Increase the number of iterations (max_iter) or scale the data as shown in:\n",
      "    https://scikit-learn.org/stable/modules/preprocessing.html\n",
      "Please also refer to the documentation for alternative solver options:\n",
      "    https://scikit-learn.org/stable/modules/linear_model.html#logistic-regression\n",
      "  n_iter_i = _check_optimize_result(\n",
      "/home/potatosalad/.local/share/virtualenvs/CSC_homework_3_group4-nat9sqnt/lib/python3.10/site-packages/sklearn/linear_model/_logistic.py:458: ConvergenceWarning: lbfgs failed to converge (status=1):\n",
      "STOP: TOTAL NO. of ITERATIONS REACHED LIMIT.\n",
      "\n",
      "Increase the number of iterations (max_iter) or scale the data as shown in:\n",
      "    https://scikit-learn.org/stable/modules/preprocessing.html\n",
      "Please also refer to the documentation for alternative solver options:\n",
      "    https://scikit-learn.org/stable/modules/linear_model.html#logistic-regression\n",
      "  n_iter_i = _check_optimize_result(\n",
      "/home/potatosalad/.local/share/virtualenvs/CSC_homework_3_group4-nat9sqnt/lib/python3.10/site-packages/sklearn/linear_model/_logistic.py:458: ConvergenceWarning: lbfgs failed to converge (status=1):\n",
      "STOP: TOTAL NO. of ITERATIONS REACHED LIMIT.\n",
      "\n",
      "Increase the number of iterations (max_iter) or scale the data as shown in:\n",
      "    https://scikit-learn.org/stable/modules/preprocessing.html\n",
      "Please also refer to the documentation for alternative solver options:\n",
      "    https://scikit-learn.org/stable/modules/linear_model.html#logistic-regression\n",
      "  n_iter_i = _check_optimize_result(\n",
      "/home/potatosalad/.local/share/virtualenvs/CSC_homework_3_group4-nat9sqnt/lib/python3.10/site-packages/sklearn/linear_model/_logistic.py:458: ConvergenceWarning: lbfgs failed to converge (status=1):\n",
      "STOP: TOTAL NO. of ITERATIONS REACHED LIMIT.\n",
      "\n",
      "Increase the number of iterations (max_iter) or scale the data as shown in:\n",
      "    https://scikit-learn.org/stable/modules/preprocessing.html\n",
      "Please also refer to the documentation for alternative solver options:\n",
      "    https://scikit-learn.org/stable/modules/linear_model.html#logistic-regression\n",
      "  n_iter_i = _check_optimize_result(\n",
      "/home/potatosalad/.local/share/virtualenvs/CSC_homework_3_group4-nat9sqnt/lib/python3.10/site-packages/sklearn/linear_model/_logistic.py:458: ConvergenceWarning: lbfgs failed to converge (status=1):\n",
      "STOP: TOTAL NO. of ITERATIONS REACHED LIMIT.\n",
      "\n",
      "Increase the number of iterations (max_iter) or scale the data as shown in:\n",
      "    https://scikit-learn.org/stable/modules/preprocessing.html\n",
      "Please also refer to the documentation for alternative solver options:\n",
      "    https://scikit-learn.org/stable/modules/linear_model.html#logistic-regression\n",
      "  n_iter_i = _check_optimize_result(\n",
      "/home/potatosalad/.local/share/virtualenvs/CSC_homework_3_group4-nat9sqnt/lib/python3.10/site-packages/sklearn/linear_model/_logistic.py:458: ConvergenceWarning: lbfgs failed to converge (status=1):\n",
      "STOP: TOTAL NO. of ITERATIONS REACHED LIMIT.\n",
      "\n",
      "Increase the number of iterations (max_iter) or scale the data as shown in:\n",
      "    https://scikit-learn.org/stable/modules/preprocessing.html\n",
      "Please also refer to the documentation for alternative solver options:\n",
      "    https://scikit-learn.org/stable/modules/linear_model.html#logistic-regression\n",
      "  n_iter_i = _check_optimize_result(\n",
      "/home/potatosalad/.local/share/virtualenvs/CSC_homework_3_group4-nat9sqnt/lib/python3.10/site-packages/sklearn/linear_model/_logistic.py:458: ConvergenceWarning: lbfgs failed to converge (status=1):\n",
      "STOP: TOTAL NO. of ITERATIONS REACHED LIMIT.\n",
      "\n",
      "Increase the number of iterations (max_iter) or scale the data as shown in:\n",
      "    https://scikit-learn.org/stable/modules/preprocessing.html\n",
      "Please also refer to the documentation for alternative solver options:\n",
      "    https://scikit-learn.org/stable/modules/linear_model.html#logistic-regression\n",
      "  n_iter_i = _check_optimize_result(\n",
      "/home/potatosalad/.local/share/virtualenvs/CSC_homework_3_group4-nat9sqnt/lib/python3.10/site-packages/sklearn/linear_model/_logistic.py:458: ConvergenceWarning: lbfgs failed to converge (status=1):\n",
      "STOP: TOTAL NO. of ITERATIONS REACHED LIMIT.\n",
      "\n",
      "Increase the number of iterations (max_iter) or scale the data as shown in:\n",
      "    https://scikit-learn.org/stable/modules/preprocessing.html\n",
      "Please also refer to the documentation for alternative solver options:\n",
      "    https://scikit-learn.org/stable/modules/linear_model.html#logistic-regression\n",
      "  n_iter_i = _check_optimize_result(\n",
      "/home/potatosalad/.local/share/virtualenvs/CSC_homework_3_group4-nat9sqnt/lib/python3.10/site-packages/sklearn/linear_model/_logistic.py:458: ConvergenceWarning: lbfgs failed to converge (status=1):\n",
      "STOP: TOTAL NO. of ITERATIONS REACHED LIMIT.\n",
      "\n",
      "Increase the number of iterations (max_iter) or scale the data as shown in:\n",
      "    https://scikit-learn.org/stable/modules/preprocessing.html\n",
      "Please also refer to the documentation for alternative solver options:\n",
      "    https://scikit-learn.org/stable/modules/linear_model.html#logistic-regression\n",
      "  n_iter_i = _check_optimize_result(\n",
      "/home/potatosalad/.local/share/virtualenvs/CSC_homework_3_group4-nat9sqnt/lib/python3.10/site-packages/sklearn/linear_model/_logistic.py:458: ConvergenceWarning: lbfgs failed to converge (status=1):\n",
      "STOP: TOTAL NO. of ITERATIONS REACHED LIMIT.\n",
      "\n",
      "Increase the number of iterations (max_iter) or scale the data as shown in:\n",
      "    https://scikit-learn.org/stable/modules/preprocessing.html\n",
      "Please also refer to the documentation for alternative solver options:\n",
      "    https://scikit-learn.org/stable/modules/linear_model.html#logistic-regression\n",
      "  n_iter_i = _check_optimize_result(\n",
      "/home/potatosalad/.local/share/virtualenvs/CSC_homework_3_group4-nat9sqnt/lib/python3.10/site-packages/sklearn/linear_model/_logistic.py:458: ConvergenceWarning: lbfgs failed to converge (status=1):\n",
      "STOP: TOTAL NO. of ITERATIONS REACHED LIMIT.\n",
      "\n",
      "Increase the number of iterations (max_iter) or scale the data as shown in:\n",
      "    https://scikit-learn.org/stable/modules/preprocessing.html\n",
      "Please also refer to the documentation for alternative solver options:\n",
      "    https://scikit-learn.org/stable/modules/linear_model.html#logistic-regression\n",
      "  n_iter_i = _check_optimize_result(\n",
      "/home/potatosalad/.local/share/virtualenvs/CSC_homework_3_group4-nat9sqnt/lib/python3.10/site-packages/sklearn/linear_model/_logistic.py:458: ConvergenceWarning: lbfgs failed to converge (status=1):\n",
      "STOP: TOTAL NO. of ITERATIONS REACHED LIMIT.\n",
      "\n",
      "Increase the number of iterations (max_iter) or scale the data as shown in:\n",
      "    https://scikit-learn.org/stable/modules/preprocessing.html\n",
      "Please also refer to the documentation for alternative solver options:\n",
      "    https://scikit-learn.org/stable/modules/linear_model.html#logistic-regression\n",
      "  n_iter_i = _check_optimize_result(\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/potatosalad/.local/share/virtualenvs/CSC_homework_3_group4-nat9sqnt/lib/python3.10/site-packages/sklearn/linear_model/_logistic.py:458: ConvergenceWarning: lbfgs failed to converge (status=1):\n",
      "STOP: TOTAL NO. of ITERATIONS REACHED LIMIT.\n",
      "\n",
      "Increase the number of iterations (max_iter) or scale the data as shown in:\n",
      "    https://scikit-learn.org/stable/modules/preprocessing.html\n",
      "Please also refer to the documentation for alternative solver options:\n",
      "    https://scikit-learn.org/stable/modules/linear_model.html#logistic-regression\n",
      "  n_iter_i = _check_optimize_result(\n",
      "/home/potatosalad/.local/share/virtualenvs/CSC_homework_3_group4-nat9sqnt/lib/python3.10/site-packages/sklearn/linear_model/_logistic.py:458: ConvergenceWarning: lbfgs failed to converge (status=1):\n",
      "STOP: TOTAL NO. of ITERATIONS REACHED LIMIT.\n",
      "\n",
      "Increase the number of iterations (max_iter) or scale the data as shown in:\n",
      "    https://scikit-learn.org/stable/modules/preprocessing.html\n",
      "Please also refer to the documentation for alternative solver options:\n",
      "    https://scikit-learn.org/stable/modules/linear_model.html#logistic-regression\n",
      "  n_iter_i = _check_optimize_result(\n",
      "/home/potatosalad/.local/share/virtualenvs/CSC_homework_3_group4-nat9sqnt/lib/python3.10/site-packages/sklearn/linear_model/_logistic.py:458: ConvergenceWarning: lbfgs failed to converge (status=1):\n",
      "STOP: TOTAL NO. of ITERATIONS REACHED LIMIT.\n",
      "\n",
      "Increase the number of iterations (max_iter) or scale the data as shown in:\n",
      "    https://scikit-learn.org/stable/modules/preprocessing.html\n",
      "Please also refer to the documentation for alternative solver options:\n",
      "    https://scikit-learn.org/stable/modules/linear_model.html#logistic-regression\n",
      "  n_iter_i = _check_optimize_result(\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Model performance for train set ENSEMBLE_wine_Data_no_gridsearch\n",
      "- Accuracy: 1.0000\n",
      "- F1 score: 1.000000\n",
      "- Precision: 1.000000\n",
      "- Recall: 1.000000\n",
      "----------------------------------\n",
      "===================================\n",
      "Model performance for test set ENSEMBLE_wine_Data_no_gridsearch\n",
      "- Accuracy: 1.0000\n",
      "- F1 score: 1.000000\n",
      "- Precision: 1.000000\n",
      "- Recall: 1.000000\n",
      "----------------------------------\n",
      "===================================\n"
     ]
    }
   ],
   "source": [
    "# NO GRIDSEARCH\n",
    "fit_models(X_train, y_train, X_test, y_test, data_wine_target_names, models, params, \"wine_Data_no_gridsearch\", \"experiment_wine_no_gridsearch\",0)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# HEART DATASET"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 67,
   "metadata": {},
   "outputs": [],
   "source": [
    "data_heart=pd.read_csv('./data/heart.data',names=['age','sex','chest pain type (4 values)', 'resting blood pressure', 'serum cholesterol in mg/dl', 'fasting blood sugar > 120 mg/dl', 'resting electrocardiographic results (values 0,1,2)', 'maximum heart rate achieved', 'exercise induced angina','oldpeak = ST depression induced by exercise relative to rest', 'the slope of the peak exercise ST segment','number of major vessels (0-3) colored by flourosopy','thal: 3 = normal; 6 = fixed defect; 7 = reversable defect', 'absence = 1, presence = 2'], delim_whitespace=True)\n",
    "# print(data_heart)\n",
    "data_heart_target_names = np.array(['Absence','Presence'])\n",
    "x=data_heart.drop(['absence = 1, presence = 2'], axis=1)\n",
    "y=data_heart['absence = 1, presence = 2']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 68,
   "metadata": {},
   "outputs": [],
   "source": [
    "X_train, X_test, y_train, y_test = train_test_split(x, y, test_size=0.20, random_state=42)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 69,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Fitting for parameters...\n",
      "X:       age  sex  chest pain type (4 values)  resting blood pressure  \\\n",
      "115  49.0  0.0                         2.0                   134.0   \n",
      "33   59.0  1.0                         4.0                   170.0   \n",
      "184  53.0  1.0                         3.0                   130.0   \n",
      "142  50.0  1.0                         3.0                   140.0   \n",
      "197  54.0  0.0                         3.0                   110.0   \n",
      "..    ...  ...                         ...                     ...   \n",
      "20   67.0  1.0                         4.0                   120.0   \n",
      "188  62.0  0.0                         4.0                   140.0   \n",
      "71   57.0  0.0                         4.0                   120.0   \n",
      "106  51.0  1.0                         3.0                   100.0   \n",
      "102  49.0  0.0                         4.0                   130.0   \n",
      "\n",
      "     serum cholesterol in mg/dl  fasting blood sugar > 120 mg/dl  \\\n",
      "115                       271.0                              0.0   \n",
      "33                        326.0                              0.0   \n",
      "184                       246.0                              1.0   \n",
      "142                       233.0                              0.0   \n",
      "197                       214.0                              0.0   \n",
      "..                          ...                              ...   \n",
      "20                        229.0                              0.0   \n",
      "188                       394.0                              0.0   \n",
      "71                        354.0                              0.0   \n",
      "106                       222.0                              0.0   \n",
      "102                       269.0                              0.0   \n",
      "\n",
      "     resting electrocardiographic results (values 0,1,2)  \\\n",
      "115                                                0.0     \n",
      "33                                                 2.0     \n",
      "184                                                2.0     \n",
      "142                                                0.0     \n",
      "197                                                0.0     \n",
      "..                                                 ...     \n",
      "20                                                 2.0     \n",
      "188                                                2.0     \n",
      "71                                                 0.0     \n",
      "106                                                0.0     \n",
      "102                                                0.0     \n",
      "\n",
      "     maximum heart rate achieved  exercise induced angina  \\\n",
      "115                        162.0                      0.0   \n",
      "33                         140.0                      1.0   \n",
      "184                        173.0                      0.0   \n",
      "142                        163.0                      0.0   \n",
      "197                        158.0                      0.0   \n",
      "..                           ...                      ...   \n",
      "20                         129.0                      1.0   \n",
      "188                        157.0                      0.0   \n",
      "71                         163.0                      1.0   \n",
      "106                        143.0                      1.0   \n",
      "102                        163.0                      0.0   \n",
      "\n",
      "     oldpeak = ST depression induced by exercise relative to rest  \\\n",
      "115                                                0.0              \n",
      "33                                                 3.4              \n",
      "184                                                0.0              \n",
      "142                                                0.6              \n",
      "197                                                1.6              \n",
      "..                                                 ...              \n",
      "20                                                 2.6              \n",
      "188                                                1.2              \n",
      "71                                                 0.6              \n",
      "106                                                1.2              \n",
      "102                                                0.0              \n",
      "\n",
      "     the slope of the peak exercise ST segment  \\\n",
      "115                                        2.0   \n",
      "33                                         3.0   \n",
      "184                                        1.0   \n",
      "142                                        2.0   \n",
      "197                                        2.0   \n",
      "..                                         ...   \n",
      "20                                         2.0   \n",
      "188                                        2.0   \n",
      "71                                         1.0   \n",
      "106                                        2.0   \n",
      "102                                        1.0   \n",
      "\n",
      "     number of major vessels (0-3) colored by flourosopy  \\\n",
      "115                                                0.0     \n",
      "33                                                 0.0     \n",
      "184                                                3.0     \n",
      "142                                                1.0     \n",
      "197                                                0.0     \n",
      "..                                                 ...     \n",
      "20                                                 2.0     \n",
      "188                                                0.0     \n",
      "71                                                 0.0     \n",
      "106                                                0.0     \n",
      "102                                                0.0     \n",
      "\n",
      "     thal: 3 = normal; 6 = fixed defect; 7 = reversable defect  \n",
      "115                                                3.0          \n",
      "33                                                 7.0          \n",
      "184                                                3.0          \n",
      "142                                                7.0          \n",
      "197                                                3.0          \n",
      "..                                                 ...          \n",
      "20                                                 7.0          \n",
      "188                                                3.0          \n",
      "71                                                 3.0          \n",
      "106                                                3.0          \n",
      "102                                                3.0          \n",
      "\n",
      "[216 rows x 13 columns]\n",
      "--------------------------------------------------\n",
      "Y: 115    1\n",
      "33     2\n",
      "184    1\n",
      "142    2\n",
      "197    1\n",
      "      ..\n",
      "20     2\n",
      "188    1\n",
      "71     1\n",
      "106    1\n",
      "102    1\n",
      "Name: absence = 1, presence = 2, Length: 216, dtype: int64\n",
      "--------------------------------------------------\n",
      "X_t:       age  sex  chest pain type (4 values)  resting blood pressure  \\\n",
      "30   57.0  1.0                         3.0                   128.0   \n",
      "116  46.0  1.0                         4.0                   120.0   \n",
      "79   56.0  1.0                         2.0                   120.0   \n",
      "127  52.0  0.0                         3.0                   136.0   \n",
      "196  58.0  0.0                         4.0                   100.0   \n",
      "137  56.0  1.0                         4.0                   125.0   \n",
      "209  37.0  1.0                         3.0                   130.0   \n",
      "45   58.0  1.0                         3.0                   140.0   \n",
      "158  56.0  1.0                         1.0                   120.0   \n",
      "247  65.0  0.0                         3.0                   155.0   \n",
      "183  42.0  0.0                         4.0                   102.0   \n",
      "268  57.0  1.0                         4.0                   140.0   \n",
      "227  43.0  0.0                         4.0                   132.0   \n",
      "82   58.0  1.0                         4.0                   150.0   \n",
      "165  64.0  0.0                         4.0                   180.0   \n",
      "194  48.0  1.0                         3.0                   124.0   \n",
      "226  62.0  0.0                         3.0                   130.0   \n",
      "146  52.0  1.0                         4.0                   112.0   \n",
      "104  63.0  1.0                         4.0                   130.0   \n",
      "60   57.0  1.0                         3.0                   150.0   \n",
      "221  65.0  1.0                         4.0                   135.0   \n",
      "266  44.0  1.0                         2.0                   120.0   \n",
      "46   49.0  1.0                         3.0                   118.0   \n",
      "42   44.0  1.0                         3.0                   130.0   \n",
      "185  43.0  1.0                         3.0                   130.0   \n",
      "9    63.0  0.0                         4.0                   150.0   \n",
      "22   43.0  1.0                         4.0                   115.0   \n",
      "199  77.0  1.0                         4.0                   125.0   \n",
      "109  45.0  0.0                         2.0                   112.0   \n",
      "24   54.0  0.0                         2.0                   132.0   \n",
      "113  54.0  0.0                         3.0                   135.0   \n",
      "68   44.0  0.0                         3.0                   118.0   \n",
      "144  54.0  1.0                         2.0                   192.0   \n",
      "224  35.0  0.0                         4.0                   138.0   \n",
      "252  44.0  1.0                         4.0                   112.0   \n",
      "6    56.0  1.0                         3.0                   130.0   \n",
      "120  49.0  1.0                         3.0                   120.0   \n",
      "67   58.0  0.0                         2.0                   136.0   \n",
      "119  56.0  1.0                         4.0                   130.0   \n",
      "118  66.0  0.0                         1.0                   150.0   \n",
      "25   48.0  0.0                         3.0                   130.0   \n",
      "125  54.0  0.0                         3.0                   160.0   \n",
      "244  51.0  0.0                         3.0                   140.0   \n",
      "19   40.0  1.0                         1.0                   140.0   \n",
      "77   39.0  0.0                         3.0                    94.0   \n",
      "216  63.0  0.0                         3.0                   135.0   \n",
      "90   59.0  1.0                         2.0                   140.0   \n",
      "208  64.0  1.0                         3.0                   125.0   \n",
      "93   54.0  1.0                         4.0                   110.0   \n",
      "180  42.0  1.0                         3.0                   120.0   \n",
      "15   71.0  0.0                         4.0                   112.0   \n",
      "152  54.0  0.0                         3.0                   108.0   \n",
      "232  45.0  1.0                         4.0                   115.0   \n",
      "250  54.0  1.0                         4.0                   120.0   \n",
      "\n",
      "     serum cholesterol in mg/dl  fasting blood sugar > 120 mg/dl  \\\n",
      "30                        229.0                              0.0   \n",
      "116                       249.0                              0.0   \n",
      "79                        236.0                              0.0   \n",
      "127                       196.0                              0.0   \n",
      "196                       248.0                              0.0   \n",
      "137                       249.0                              1.0   \n",
      "209                       250.0                              0.0   \n",
      "45                        211.0                              1.0   \n",
      "158                       193.0                              0.0   \n",
      "247                       269.0                              0.0   \n",
      "183                       265.0                              0.0   \n",
      "268                       192.0                              0.0   \n",
      "227                       341.0                              1.0   \n",
      "82                        270.0                              0.0   \n",
      "165                       325.0                              0.0   \n",
      "194                       255.0                              1.0   \n",
      "226                       263.0                              0.0   \n",
      "146                       230.0                              0.0   \n",
      "104                       254.0                              0.0   \n",
      "60                        126.0                              1.0   \n",
      "221                       254.0                              0.0   \n",
      "266                       263.0                              0.0   \n",
      "46                        149.0                              0.0   \n",
      "42                        233.0                              0.0   \n",
      "185                       315.0                              0.0   \n",
      "9                         407.0                              0.0   \n",
      "22                        303.0                              0.0   \n",
      "199                       304.0                              0.0   \n",
      "109                       160.0                              0.0   \n",
      "24                        288.0                              1.0   \n",
      "113                       304.0                              1.0   \n",
      "68                        242.0                              0.0   \n",
      "144                       283.0                              0.0   \n",
      "224                       183.0                              0.0   \n",
      "252                       290.0                              0.0   \n",
      "6                         256.0                              1.0   \n",
      "120                       188.0                              0.0   \n",
      "67                        319.0                              1.0   \n",
      "119                       283.0                              1.0   \n",
      "118                       226.0                              0.0   \n",
      "25                        275.0                              0.0   \n",
      "125                       201.0                              0.0   \n",
      "244                       308.0                              0.0   \n",
      "19                        199.0                              0.0   \n",
      "77                        199.0                              0.0   \n",
      "216                       252.0                              0.0   \n",
      "90                        221.0                              0.0   \n",
      "208                       309.0                              0.0   \n",
      "93                        206.0                              0.0   \n",
      "180                       240.0                              1.0   \n",
      "15                        149.0                              0.0   \n",
      "152                       267.0                              0.0   \n",
      "232                       260.0                              0.0   \n",
      "250                       188.0                              0.0   \n",
      "\n",
      "     resting electrocardiographic results (values 0,1,2)  \\\n",
      "30                                                 2.0     \n",
      "116                                                2.0     \n",
      "79                                                 0.0     \n",
      "127                                                2.0     \n",
      "196                                                2.0     \n",
      "137                                                2.0     \n",
      "209                                                0.0     \n",
      "45                                                 2.0     \n",
      "158                                                2.0     \n",
      "247                                                0.0     \n",
      "183                                                2.0     \n",
      "268                                                0.0     \n",
      "227                                                2.0     \n",
      "82                                                 2.0     \n",
      "165                                                0.0     \n",
      "194                                                0.0     \n",
      "226                                                0.0     \n",
      "146                                                0.0     \n",
      "104                                                2.0     \n",
      "60                                                 0.0     \n",
      "221                                                2.0     \n",
      "266                                                0.0     \n",
      "46                                                 2.0     \n",
      "42                                                 0.0     \n",
      "185                                                0.0     \n",
      "9                                                  2.0     \n",
      "22                                                 0.0     \n",
      "199                                                2.0     \n",
      "109                                                0.0     \n",
      "24                                                 2.0     \n",
      "113                                                0.0     \n",
      "68                                                 0.0     \n",
      "144                                                2.0     \n",
      "224                                                0.0     \n",
      "252                                                2.0     \n",
      "6                                                  2.0     \n",
      "120                                                0.0     \n",
      "67                                                 2.0     \n",
      "119                                                2.0     \n",
      "118                                                0.0     \n",
      "25                                                 0.0     \n",
      "125                                                0.0     \n",
      "244                                                2.0     \n",
      "19                                                 0.0     \n",
      "77                                                 0.0     \n",
      "216                                                2.0     \n",
      "90                                                 0.0     \n",
      "208                                                0.0     \n",
      "93                                                 2.0     \n",
      "180                                                0.0     \n",
      "15                                                 0.0     \n",
      "152                                                2.0     \n",
      "232                                                2.0     \n",
      "250                                                0.0     \n",
      "\n",
      "     maximum heart rate achieved  exercise induced angina  \\\n",
      "30                         150.0                      0.0   \n",
      "116                        144.0                      0.0   \n",
      "79                         178.0                      0.0   \n",
      "127                        169.0                      0.0   \n",
      "196                        122.0                      0.0   \n",
      "137                        144.0                      1.0   \n",
      "209                        187.0                      0.0   \n",
      "45                         165.0                      0.0   \n",
      "158                        162.0                      0.0   \n",
      "247                        148.0                      0.0   \n",
      "183                        122.0                      0.0   \n",
      "268                        148.0                      0.0   \n",
      "227                        136.0                      1.0   \n",
      "82                         111.0                      1.0   \n",
      "165                        154.0                      1.0   \n",
      "194                        175.0                      0.0   \n",
      "226                         97.0                      0.0   \n",
      "146                        160.0                      0.0   \n",
      "104                        147.0                      0.0   \n",
      "60                         173.0                      0.0   \n",
      "221                        127.0                      0.0   \n",
      "266                        173.0                      0.0   \n",
      "46                         126.0                      0.0   \n",
      "42                         179.0                      1.0   \n",
      "185                        162.0                      0.0   \n",
      "9                          154.0                      0.0   \n",
      "22                         181.0                      0.0   \n",
      "199                        162.0                      1.0   \n",
      "109                        138.0                      0.0   \n",
      "24                         159.0                      1.0   \n",
      "113                        170.0                      0.0   \n",
      "68                         149.0                      0.0   \n",
      "144                        195.0                      0.0   \n",
      "224                        182.0                      0.0   \n",
      "252                        153.0                      0.0   \n",
      "6                          142.0                      1.0   \n",
      "120                        139.0                      0.0   \n",
      "67                         152.0                      0.0   \n",
      "119                        103.0                      1.0   \n",
      "118                        114.0                      0.0   \n",
      "25                         139.0                      0.0   \n",
      "125                        163.0                      0.0   \n",
      "244                        142.0                      0.0   \n",
      "19                         178.0                      1.0   \n",
      "77                         179.0                      0.0   \n",
      "216                        172.0                      0.0   \n",
      "90                         164.0                      1.0   \n",
      "208                        131.0                      1.0   \n",
      "93                         108.0                      1.0   \n",
      "180                        194.0                      0.0   \n",
      "15                         125.0                      0.0   \n",
      "152                        167.0                      0.0   \n",
      "232                        185.0                      0.0   \n",
      "250                        113.0                      0.0   \n",
      "\n",
      "     oldpeak = ST depression induced by exercise relative to rest  \\\n",
      "30                                                 0.4              \n",
      "116                                                0.8              \n",
      "79                                                 0.8              \n",
      "127                                                0.1              \n",
      "196                                                1.0              \n",
      "137                                                1.2              \n",
      "209                                                3.5              \n",
      "45                                                 0.0              \n",
      "158                                                1.9              \n",
      "247                                                0.8              \n",
      "183                                                0.6              \n",
      "268                                                0.4              \n",
      "227                                                3.0              \n",
      "82                                                 0.8              \n",
      "165                                                0.0              \n",
      "194                                                0.0              \n",
      "226                                                1.2              \n",
      "146                                                0.0              \n",
      "104                                                1.4              \n",
      "60                                                 0.2              \n",
      "221                                                2.8              \n",
      "266                                                0.0              \n",
      "46                                                 0.8              \n",
      "42                                                 0.4              \n",
      "185                                                1.9              \n",
      "9                                                  4.0              \n",
      "22                                                 1.2              \n",
      "199                                                0.0              \n",
      "109                                                0.0              \n",
      "24                                                 0.0              \n",
      "113                                                0.0              \n",
      "68                                                 0.3              \n",
      "144                                                0.0              \n",
      "224                                                1.4              \n",
      "252                                                0.0              \n",
      "6                                                  0.6              \n",
      "120                                                2.0              \n",
      "67                                                 0.0              \n",
      "119                                                1.6              \n",
      "118                                                2.6              \n",
      "25                                                 0.2              \n",
      "125                                                0.0              \n",
      "244                                                1.5              \n",
      "19                                                 1.4              \n",
      "77                                                 0.0              \n",
      "216                                                0.0              \n",
      "90                                                 0.0              \n",
      "208                                                1.8              \n",
      "93                                                 0.0              \n",
      "180                                                0.8              \n",
      "15                                                 1.6              \n",
      "152                                                0.0              \n",
      "232                                                0.0              \n",
      "250                                                1.4              \n",
      "\n",
      "     the slope of the peak exercise ST segment  \\\n",
      "30                                         2.0   \n",
      "116                                        1.0   \n",
      "79                                         1.0   \n",
      "127                                        2.0   \n",
      "196                                        2.0   \n",
      "137                                        2.0   \n",
      "209                                        3.0   \n",
      "45                                         1.0   \n",
      "158                                        2.0   \n",
      "247                                        1.0   \n",
      "183                                        2.0   \n",
      "268                                        2.0   \n",
      "227                                        2.0   \n",
      "82                                         1.0   \n",
      "165                                        1.0   \n",
      "194                                        1.0   \n",
      "226                                        2.0   \n",
      "146                                        1.0   \n",
      "104                                        2.0   \n",
      "60                                         1.0   \n",
      "221                                        2.0   \n",
      "266                                        1.0   \n",
      "46                                         1.0   \n",
      "42                                         1.0   \n",
      "185                                        1.0   \n",
      "9                                          2.0   \n",
      "22                                         2.0   \n",
      "199                                        1.0   \n",
      "109                                        2.0   \n",
      "24                                         1.0   \n",
      "113                                        1.0   \n",
      "68                                         2.0   \n",
      "144                                        1.0   \n",
      "224                                        1.0   \n",
      "252                                        1.0   \n",
      "6                                          2.0   \n",
      "120                                        2.0   \n",
      "67                                         1.0   \n",
      "119                                        3.0   \n",
      "118                                        3.0   \n",
      "25                                         1.0   \n",
      "125                                        1.0   \n",
      "244                                        1.0   \n",
      "19                                         1.0   \n",
      "77                                         1.0   \n",
      "216                                        1.0   \n",
      "90                                         1.0   \n",
      "208                                        2.0   \n",
      "93                                         2.0   \n",
      "180                                        3.0   \n",
      "15                                         2.0   \n",
      "152                                        1.0   \n",
      "232                                        1.0   \n",
      "250                                        2.0   \n",
      "\n",
      "     number of major vessels (0-3) colored by flourosopy  \\\n",
      "30                                                 1.0     \n",
      "116                                                0.0     \n",
      "79                                                 0.0     \n",
      "127                                                0.0     \n",
      "196                                                0.0     \n",
      "137                                                1.0     \n",
      "209                                                0.0     \n",
      "45                                                 0.0     \n",
      "158                                                0.0     \n",
      "247                                                0.0     \n",
      "183                                                0.0     \n",
      "268                                                0.0     \n",
      "227                                                0.0     \n",
      "82                                                 0.0     \n",
      "165                                                0.0     \n",
      "194                                                2.0     \n",
      "226                                                1.0     \n",
      "146                                                1.0     \n",
      "104                                                1.0     \n",
      "60                                                 1.0     \n",
      "221                                                1.0     \n",
      "266                                                0.0     \n",
      "46                                                 3.0     \n",
      "42                                                 0.0     \n",
      "185                                                1.0     \n",
      "9                                                  3.0     \n",
      "22                                                 0.0     \n",
      "199                                                3.0     \n",
      "109                                                0.0     \n",
      "24                                                 1.0     \n",
      "113                                                0.0     \n",
      "68                                                 1.0     \n",
      "144                                                1.0     \n",
      "224                                                0.0     \n",
      "252                                                1.0     \n",
      "6                                                  1.0     \n",
      "120                                                3.0     \n",
      "67                                                 2.0     \n",
      "119                                                0.0     \n",
      "118                                                0.0     \n",
      "25                                                 0.0     \n",
      "125                                                1.0     \n",
      "244                                                1.0     \n",
      "19                                                 0.0     \n",
      "77                                                 0.0     \n",
      "216                                                0.0     \n",
      "90                                                 0.0     \n",
      "208                                                0.0     \n",
      "93                                                 1.0     \n",
      "180                                                0.0     \n",
      "15                                                 0.0     \n",
      "152                                                0.0     \n",
      "232                                                0.0     \n",
      "250                                                1.0     \n",
      "\n",
      "     thal: 3 = normal; 6 = fixed defect; 7 = reversable defect  \n",
      "30                                                 7.0          \n",
      "116                                                7.0          \n",
      "79                                                 3.0          \n",
      "127                                                3.0          \n",
      "196                                                3.0          \n",
      "137                                                3.0          \n",
      "209                                                3.0          \n",
      "45                                                 3.0          \n",
      "158                                                7.0          \n",
      "247                                                3.0          \n",
      "183                                                3.0          \n",
      "268                                                6.0          \n",
      "227                                                7.0          \n",
      "82                                                 7.0          \n",
      "165                                                3.0          \n",
      "194                                                3.0          \n",
      "226                                                7.0          \n",
      "146                                                3.0          \n",
      "104                                                7.0          \n",
      "60                                                 7.0          \n",
      "221                                                7.0          \n",
      "266                                                7.0          \n",
      "46                                                 3.0          \n",
      "42                                                 3.0          \n",
      "185                                                3.0          \n",
      "9                                                  7.0          \n",
      "22                                                 3.0          \n",
      "199                                                3.0          \n",
      "109                                                3.0          \n",
      "24                                                 3.0          \n",
      "113                                                3.0          \n",
      "68                                                 3.0          \n",
      "144                                                7.0          \n",
      "224                                                3.0          \n",
      "252                                                3.0          \n",
      "6                                                  6.0          \n",
      "120                                                7.0          \n",
      "67                                                 3.0          \n",
      "119                                                7.0          \n",
      "118                                                3.0          \n",
      "25                                                 3.0          \n",
      "125                                                3.0          \n",
      "244                                                3.0          \n",
      "19                                                 7.0          \n",
      "77                                                 3.0          \n",
      "216                                                3.0          \n",
      "90                                                 3.0          \n",
      "208                                                7.0          \n",
      "93                                                 3.0          \n",
      "180                                                7.0          \n",
      "15                                                 3.0          \n",
      "152                                                3.0          \n",
      "232                                                3.0          \n",
      "250                                                7.0          \n",
      "--------------------------------------------------\n",
      "Y_t: 30     2\n",
      "116    2\n",
      "79     1\n",
      "127    1\n",
      "196    1\n",
      "137    2\n",
      "209    1\n",
      "45     1\n",
      "158    1\n",
      "247    1\n",
      "183    1\n",
      "268    1\n",
      "227    2\n",
      "82     2\n",
      "165    1\n",
      "194    1\n",
      "226    2\n",
      "146    2\n",
      "104    2\n",
      "60     1\n",
      "221    2\n",
      "266    1\n",
      "46     2\n",
      "42     1\n",
      "185    1\n",
      "9      2\n",
      "22     1\n",
      "199    2\n",
      "109    1\n",
      "24     1\n",
      "113    1\n",
      "68     1\n",
      "144    2\n",
      "224    1\n",
      "252    2\n",
      "6      2\n",
      "120    2\n",
      "67     2\n",
      "119    2\n",
      "118    1\n",
      "25     1\n",
      "125    1\n",
      "244    1\n",
      "19     1\n",
      "77     1\n",
      "216    1\n",
      "90     1\n",
      "208    2\n",
      "93     2\n",
      "180    1\n",
      "15     1\n",
      "152    1\n",
      "232    1\n",
      "250    2\n",
      "Name: absence = 1, presence = 2, dtype: int64\n",
      "--------------------------------------------------\n",
      "models: {'Logistic Regression': LogisticRegression(), 'Decision Tree': DecisionTreeClassifier(), 'Random Forest': RandomForestClassifier(random_state=42), 'Gradient Boosting': GradientBoostingClassifier(), 'K-Nearest Neighbors': KNeighborsClassifier(), 'SVM': SVC(probability=True)}\n",
      "--------------------------------------------------\n",
      "params: {'Logistic Regression': {'solver': ['newton-cholesky', 'liblinear', 'newton-cg', 'sag', 'saga'], 'penalty': ['l2'], 'C': [0.001, 0.01, 0.1, 1, 100], 'max_iter': [10000]}, 'Decision Tree': {'criterion': ['gini', 'entropy'], 'max_depth': [4, 6, 8, 10, 15, 50, 90]}, 'Random Forest': {'n_estimators': [200, 500], 'max_features': [None, 'sqrt', 'log2'], 'max_depth': [4, 5, 6, 7, 8], 'criterion': ['gini', 'entropy']}, 'Gradient Boosting': {'loss': ['log_loss'], 'learning_rate': [0.1, 0.01], 'max_depth': [3, 5, 8], 'max_features': ['log2', 'sqrt'], 'criterion': ['friedman_mse', 'squared_error'], 'subsample': [0.5, 0.8, 1.0], 'n_estimators': [10, 100, 1000]}, 'K-Nearest Neighbors': {'weights': ['uniform', 'distance'], 'leaf_size': [5, 15, 25]}, 'SVM': {'kernel': ['rbf'], 'C': [0.1, 1, 10, 100, 1000], 'degree': [1, 5], 'gamma': [1, 0.1, 0.01, 0.001]}}\n",
      "--------------------------------------------------\n",
      "run: heart_Data_gridsearch\n",
      "--------------------------------------------------\n",
      "folder: experiment_heart_gridsearch\n",
      "Logistic Regression\n",
      "\n",
      "##################################################\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Model performance for Training set Logistic Regression_heart_Data_gridsearch\n",
      "- Accuracy: 0.8426\n",
      "- F1 score: 0.842593\n",
      "- Precision: 0.842593\n",
      "- Recall: 0.842593\n",
      "----------------------------------\n",
      "===================================\n",
      "Model performance for Test set Logistic Regression_heart_Data_gridsearch\n",
      "- Accuracy: 0.8704\n",
      "- F1 score: 0.866176\n",
      "- Precision: 0.879203\n",
      "- Recall: 0.870370\n",
      "----------------------------------\n",
      "===================================\n",
      "Decision Tree\n",
      "\n",
      "##################################################\n",
      "Model performance for Training set Decision Tree_heart_Data_gridsearch\n",
      "- Accuracy: 0.8935\n",
      "- F1 score: 0.892362\n",
      "- Precision: 0.900207\n",
      "- Recall: 0.893519\n",
      "----------------------------------\n",
      "===================================\n",
      "Model performance for Test set Decision Tree_heart_Data_gridsearch\n",
      "- Accuracy: 0.8148\n",
      "- F1 score: 0.807099\n",
      "- Precision: 0.822792\n",
      "- Recall: 0.814815\n",
      "----------------------------------\n",
      "===================================\n",
      "Random Forest\n",
      "\n",
      "##################################################\n",
      "Model performance for Training set Random Forest_heart_Data_gridsearch\n",
      "- Accuracy: 0.9398\n",
      "- F1 score: 0.939665\n",
      "- Precision: 0.940537\n",
      "- Recall: 0.939815\n",
      "----------------------------------\n",
      "===================================\n",
      "Model performance for Test set Random Forest_heart_Data_gridsearch\n",
      "- Accuracy: 0.8704\n",
      "- F1 score: 0.866176\n",
      "- Precision: 0.879203\n",
      "- Recall: 0.870370\n",
      "----------------------------------\n",
      "===================================\n",
      "Gradient Boosting\n",
      "\n",
      "##################################################\n",
      "Model performance for Training set Gradient Boosting_heart_Data_gridsearch\n",
      "- Accuracy: 0.8935\n",
      "- F1 score: 0.893114\n",
      "- Precision: 0.894621\n",
      "- Recall: 0.893519\n",
      "----------------------------------\n",
      "===================================\n",
      "Model performance for Test set Gradient Boosting_heart_Data_gridsearch\n",
      "- Accuracy: 0.8704\n",
      "- F1 score: 0.863623\n",
      "- Precision: 0.893056\n",
      "- Recall: 0.870370\n",
      "----------------------------------\n",
      "===================================\n",
      "K-Nearest Neighbors\n",
      "\n",
      "##################################################\n",
      "Model performance for Training set K-Nearest Neighbors_heart_Data_gridsearch\n",
      "- Accuracy: 0.8657\n",
      "- F1 score: 0.865406\n",
      "- Precision: 0.866011\n",
      "- Recall: 0.865741\n",
      "----------------------------------\n",
      "===================================\n",
      "Model performance for Test set K-Nearest Neighbors_heart_Data_gridsearch\n",
      "- Accuracy: 0.8148\n",
      "- F1 score: 0.810359\n",
      "- Precision: 0.815757\n",
      "- Recall: 0.814815\n",
      "----------------------------------\n",
      "===================================\n",
      "SVM\n",
      "\n",
      "##################################################\n",
      "Model performance for Training set SVM_heart_Data_gridsearch\n",
      "- Accuracy: 0.8426\n",
      "- F1 score: 0.841881\n",
      "- Precision: 0.843524\n",
      "- Recall: 0.842593\n",
      "----------------------------------\n",
      "===================================\n",
      "Model performance for Test set SVM_heart_Data_gridsearch\n",
      "- Accuracy: 0.9259\n",
      "- F1 score: 0.924144\n",
      "- Precision: 0.933934\n",
      "- Recall: 0.925926\n",
      "----------------------------------\n",
      "===================================\n",
      "################################################\n",
      "################### ENSEMBLE ###################\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/potatosalad/.local/share/virtualenvs/CSC_homework_3_group4-nat9sqnt/lib/python3.10/site-packages/sklearn/linear_model/_logistic.py:458: ConvergenceWarning: lbfgs failed to converge (status=1):\n",
      "STOP: TOTAL NO. of ITERATIONS REACHED LIMIT.\n",
      "\n",
      "Increase the number of iterations (max_iter) or scale the data as shown in:\n",
      "    https://scikit-learn.org/stable/modules/preprocessing.html\n",
      "Please also refer to the documentation for alternative solver options:\n",
      "    https://scikit-learn.org/stable/modules/linear_model.html#logistic-regression\n",
      "  n_iter_i = _check_optimize_result(\n",
      "/home/potatosalad/.local/share/virtualenvs/CSC_homework_3_group4-nat9sqnt/lib/python3.10/site-packages/sklearn/linear_model/_logistic.py:458: ConvergenceWarning: lbfgs failed to converge (status=1):\n",
      "STOP: TOTAL NO. of ITERATIONS REACHED LIMIT.\n",
      "\n",
      "Increase the number of iterations (max_iter) or scale the data as shown in:\n",
      "    https://scikit-learn.org/stable/modules/preprocessing.html\n",
      "Please also refer to the documentation for alternative solver options:\n",
      "    https://scikit-learn.org/stable/modules/linear_model.html#logistic-regression\n",
      "  n_iter_i = _check_optimize_result(\n",
      "/home/potatosalad/.local/share/virtualenvs/CSC_homework_3_group4-nat9sqnt/lib/python3.10/site-packages/sklearn/linear_model/_logistic.py:458: ConvergenceWarning: lbfgs failed to converge (status=1):\n",
      "STOP: TOTAL NO. of ITERATIONS REACHED LIMIT.\n",
      "\n",
      "Increase the number of iterations (max_iter) or scale the data as shown in:\n",
      "    https://scikit-learn.org/stable/modules/preprocessing.html\n",
      "Please also refer to the documentation for alternative solver options:\n",
      "    https://scikit-learn.org/stable/modules/linear_model.html#logistic-regression\n",
      "  n_iter_i = _check_optimize_result(\n",
      "/home/potatosalad/.local/share/virtualenvs/CSC_homework_3_group4-nat9sqnt/lib/python3.10/site-packages/sklearn/linear_model/_logistic.py:458: ConvergenceWarning: lbfgs failed to converge (status=1):\n",
      "STOP: TOTAL NO. of ITERATIONS REACHED LIMIT.\n",
      "\n",
      "Increase the number of iterations (max_iter) or scale the data as shown in:\n",
      "    https://scikit-learn.org/stable/modules/preprocessing.html\n",
      "Please also refer to the documentation for alternative solver options:\n",
      "    https://scikit-learn.org/stable/modules/linear_model.html#logistic-regression\n",
      "  n_iter_i = _check_optimize_result(\n",
      "/home/potatosalad/.local/share/virtualenvs/CSC_homework_3_group4-nat9sqnt/lib/python3.10/site-packages/sklearn/linear_model/_logistic.py:458: ConvergenceWarning: lbfgs failed to converge (status=1):\n",
      "STOP: TOTAL NO. of ITERATIONS REACHED LIMIT.\n",
      "\n",
      "Increase the number of iterations (max_iter) or scale the data as shown in:\n",
      "    https://scikit-learn.org/stable/modules/preprocessing.html\n",
      "Please also refer to the documentation for alternative solver options:\n",
      "    https://scikit-learn.org/stable/modules/linear_model.html#logistic-regression\n",
      "  n_iter_i = _check_optimize_result(\n",
      "/home/potatosalad/.local/share/virtualenvs/CSC_homework_3_group4-nat9sqnt/lib/python3.10/site-packages/sklearn/linear_model/_logistic.py:458: ConvergenceWarning: lbfgs failed to converge (status=1):\n",
      "STOP: TOTAL NO. of ITERATIONS REACHED LIMIT.\n",
      "\n",
      "Increase the number of iterations (max_iter) or scale the data as shown in:\n",
      "    https://scikit-learn.org/stable/modules/preprocessing.html\n",
      "Please also refer to the documentation for alternative solver options:\n",
      "    https://scikit-learn.org/stable/modules/linear_model.html#logistic-regression\n",
      "  n_iter_i = _check_optimize_result(\n",
      "/home/potatosalad/.local/share/virtualenvs/CSC_homework_3_group4-nat9sqnt/lib/python3.10/site-packages/sklearn/linear_model/_logistic.py:458: ConvergenceWarning: lbfgs failed to converge (status=1):\n",
      "STOP: TOTAL NO. of ITERATIONS REACHED LIMIT.\n",
      "\n",
      "Increase the number of iterations (max_iter) or scale the data as shown in:\n",
      "    https://scikit-learn.org/stable/modules/preprocessing.html\n",
      "Please also refer to the documentation for alternative solver options:\n",
      "    https://scikit-learn.org/stable/modules/linear_model.html#logistic-regression\n",
      "  n_iter_i = _check_optimize_result(\n",
      "/home/potatosalad/.local/share/virtualenvs/CSC_homework_3_group4-nat9sqnt/lib/python3.10/site-packages/sklearn/linear_model/_logistic.py:458: ConvergenceWarning: lbfgs failed to converge (status=1):\n",
      "STOP: TOTAL NO. of ITERATIONS REACHED LIMIT.\n",
      "\n",
      "Increase the number of iterations (max_iter) or scale the data as shown in:\n",
      "    https://scikit-learn.org/stable/modules/preprocessing.html\n",
      "Please also refer to the documentation for alternative solver options:\n",
      "    https://scikit-learn.org/stable/modules/linear_model.html#logistic-regression\n",
      "  n_iter_i = _check_optimize_result(\n",
      "/home/potatosalad/.local/share/virtualenvs/CSC_homework_3_group4-nat9sqnt/lib/python3.10/site-packages/sklearn/linear_model/_logistic.py:458: ConvergenceWarning: lbfgs failed to converge (status=1):\n",
      "STOP: TOTAL NO. of ITERATIONS REACHED LIMIT.\n",
      "\n",
      "Increase the number of iterations (max_iter) or scale the data as shown in:\n",
      "    https://scikit-learn.org/stable/modules/preprocessing.html\n",
      "Please also refer to the documentation for alternative solver options:\n",
      "    https://scikit-learn.org/stable/modules/linear_model.html#logistic-regression\n",
      "  n_iter_i = _check_optimize_result(\n",
      "/home/potatosalad/.local/share/virtualenvs/CSC_homework_3_group4-nat9sqnt/lib/python3.10/site-packages/sklearn/linear_model/_logistic.py:458: ConvergenceWarning: lbfgs failed to converge (status=1):\n",
      "STOP: TOTAL NO. of ITERATIONS REACHED LIMIT.\n",
      "\n",
      "Increase the number of iterations (max_iter) or scale the data as shown in:\n",
      "    https://scikit-learn.org/stable/modules/preprocessing.html\n",
      "Please also refer to the documentation for alternative solver options:\n",
      "    https://scikit-learn.org/stable/modules/linear_model.html#logistic-regression\n",
      "  n_iter_i = _check_optimize_result(\n",
      "/home/potatosalad/.local/share/virtualenvs/CSC_homework_3_group4-nat9sqnt/lib/python3.10/site-packages/sklearn/linear_model/_logistic.py:458: ConvergenceWarning: lbfgs failed to converge (status=1):\n",
      "STOP: TOTAL NO. of ITERATIONS REACHED LIMIT.\n",
      "\n",
      "Increase the number of iterations (max_iter) or scale the data as shown in:\n",
      "    https://scikit-learn.org/stable/modules/preprocessing.html\n",
      "Please also refer to the documentation for alternative solver options:\n",
      "    https://scikit-learn.org/stable/modules/linear_model.html#logistic-regression\n",
      "  n_iter_i = _check_optimize_result(\n",
      "/home/potatosalad/.local/share/virtualenvs/CSC_homework_3_group4-nat9sqnt/lib/python3.10/site-packages/sklearn/linear_model/_logistic.py:458: ConvergenceWarning: lbfgs failed to converge (status=1):\n",
      "STOP: TOTAL NO. of ITERATIONS REACHED LIMIT.\n",
      "\n",
      "Increase the number of iterations (max_iter) or scale the data as shown in:\n",
      "    https://scikit-learn.org/stable/modules/preprocessing.html\n",
      "Please also refer to the documentation for alternative solver options:\n",
      "    https://scikit-learn.org/stable/modules/linear_model.html#logistic-regression\n",
      "  n_iter_i = _check_optimize_result(\n",
      "/home/potatosalad/.local/share/virtualenvs/CSC_homework_3_group4-nat9sqnt/lib/python3.10/site-packages/sklearn/linear_model/_logistic.py:458: ConvergenceWarning: lbfgs failed to converge (status=1):\n",
      "STOP: TOTAL NO. of ITERATIONS REACHED LIMIT.\n",
      "\n",
      "Increase the number of iterations (max_iter) or scale the data as shown in:\n",
      "    https://scikit-learn.org/stable/modules/preprocessing.html\n",
      "Please also refer to the documentation for alternative solver options:\n",
      "    https://scikit-learn.org/stable/modules/linear_model.html#logistic-regression\n",
      "  n_iter_i = _check_optimize_result(\n",
      "/home/potatosalad/.local/share/virtualenvs/CSC_homework_3_group4-nat9sqnt/lib/python3.10/site-packages/sklearn/linear_model/_logistic.py:458: ConvergenceWarning: lbfgs failed to converge (status=1):\n",
      "STOP: TOTAL NO. of ITERATIONS REACHED LIMIT.\n",
      "\n",
      "Increase the number of iterations (max_iter) or scale the data as shown in:\n",
      "    https://scikit-learn.org/stable/modules/preprocessing.html\n",
      "Please also refer to the documentation for alternative solver options:\n",
      "    https://scikit-learn.org/stable/modules/linear_model.html#logistic-regression\n",
      "  n_iter_i = _check_optimize_result(\n",
      "/home/potatosalad/.local/share/virtualenvs/CSC_homework_3_group4-nat9sqnt/lib/python3.10/site-packages/sklearn/linear_model/_logistic.py:458: ConvergenceWarning: lbfgs failed to converge (status=1):\n",
      "STOP: TOTAL NO. of ITERATIONS REACHED LIMIT.\n",
      "\n",
      "Increase the number of iterations (max_iter) or scale the data as shown in:\n",
      "    https://scikit-learn.org/stable/modules/preprocessing.html\n",
      "Please also refer to the documentation for alternative solver options:\n",
      "    https://scikit-learn.org/stable/modules/linear_model.html#logistic-regression\n",
      "  n_iter_i = _check_optimize_result(\n",
      "/home/potatosalad/.local/share/virtualenvs/CSC_homework_3_group4-nat9sqnt/lib/python3.10/site-packages/sklearn/linear_model/_logistic.py:458: ConvergenceWarning: lbfgs failed to converge (status=1):\n",
      "STOP: TOTAL NO. of ITERATIONS REACHED LIMIT.\n",
      "\n",
      "Increase the number of iterations (max_iter) or scale the data as shown in:\n",
      "    https://scikit-learn.org/stable/modules/preprocessing.html\n",
      "Please also refer to the documentation for alternative solver options:\n",
      "    https://scikit-learn.org/stable/modules/linear_model.html#logistic-regression\n",
      "  n_iter_i = _check_optimize_result(\n",
      "/home/potatosalad/.local/share/virtualenvs/CSC_homework_3_group4-nat9sqnt/lib/python3.10/site-packages/sklearn/linear_model/_logistic.py:458: ConvergenceWarning: lbfgs failed to converge (status=1):\n",
      "STOP: TOTAL NO. of ITERATIONS REACHED LIMIT.\n",
      "\n",
      "Increase the number of iterations (max_iter) or scale the data as shown in:\n",
      "    https://scikit-learn.org/stable/modules/preprocessing.html\n",
      "Please also refer to the documentation for alternative solver options:\n",
      "    https://scikit-learn.org/stable/modules/linear_model.html#logistic-regression\n",
      "  n_iter_i = _check_optimize_result(\n",
      "/home/potatosalad/.local/share/virtualenvs/CSC_homework_3_group4-nat9sqnt/lib/python3.10/site-packages/sklearn/linear_model/_logistic.py:458: ConvergenceWarning: lbfgs failed to converge (status=1):\n",
      "STOP: TOTAL NO. of ITERATIONS REACHED LIMIT.\n",
      "\n",
      "Increase the number of iterations (max_iter) or scale the data as shown in:\n",
      "    https://scikit-learn.org/stable/modules/preprocessing.html\n",
      "Please also refer to the documentation for alternative solver options:\n",
      "    https://scikit-learn.org/stable/modules/linear_model.html#logistic-regression\n",
      "  n_iter_i = _check_optimize_result(\n",
      "/home/potatosalad/.local/share/virtualenvs/CSC_homework_3_group4-nat9sqnt/lib/python3.10/site-packages/sklearn/linear_model/_logistic.py:458: ConvergenceWarning: lbfgs failed to converge (status=1):\n",
      "STOP: TOTAL NO. of ITERATIONS REACHED LIMIT.\n",
      "\n",
      "Increase the number of iterations (max_iter) or scale the data as shown in:\n",
      "    https://scikit-learn.org/stable/modules/preprocessing.html\n",
      "Please also refer to the documentation for alternative solver options:\n",
      "    https://scikit-learn.org/stable/modules/linear_model.html#logistic-regression\n",
      "  n_iter_i = _check_optimize_result(\n",
      "/home/potatosalad/.local/share/virtualenvs/CSC_homework_3_group4-nat9sqnt/lib/python3.10/site-packages/sklearn/linear_model/_logistic.py:458: ConvergenceWarning: lbfgs failed to converge (status=1):\n",
      "STOP: TOTAL NO. of ITERATIONS REACHED LIMIT.\n",
      "\n",
      "Increase the number of iterations (max_iter) or scale the data as shown in:\n",
      "    https://scikit-learn.org/stable/modules/preprocessing.html\n",
      "Please also refer to the documentation for alternative solver options:\n",
      "    https://scikit-learn.org/stable/modules/linear_model.html#logistic-regression\n",
      "  n_iter_i = _check_optimize_result(\n",
      "/home/potatosalad/.local/share/virtualenvs/CSC_homework_3_group4-nat9sqnt/lib/python3.10/site-packages/sklearn/linear_model/_logistic.py:458: ConvergenceWarning: lbfgs failed to converge (status=1):\n",
      "STOP: TOTAL NO. of ITERATIONS REACHED LIMIT.\n",
      "\n",
      "Increase the number of iterations (max_iter) or scale the data as shown in:\n",
      "    https://scikit-learn.org/stable/modules/preprocessing.html\n",
      "Please also refer to the documentation for alternative solver options:\n",
      "    https://scikit-learn.org/stable/modules/linear_model.html#logistic-regression\n",
      "  n_iter_i = _check_optimize_result(\n",
      "/home/potatosalad/.local/share/virtualenvs/CSC_homework_3_group4-nat9sqnt/lib/python3.10/site-packages/sklearn/linear_model/_logistic.py:458: ConvergenceWarning: lbfgs failed to converge (status=1):\n",
      "STOP: TOTAL NO. of ITERATIONS REACHED LIMIT.\n",
      "\n",
      "Increase the number of iterations (max_iter) or scale the data as shown in:\n",
      "    https://scikit-learn.org/stable/modules/preprocessing.html\n",
      "Please also refer to the documentation for alternative solver options:\n",
      "    https://scikit-learn.org/stable/modules/linear_model.html#logistic-regression\n",
      "  n_iter_i = _check_optimize_result(\n",
      "/home/potatosalad/.local/share/virtualenvs/CSC_homework_3_group4-nat9sqnt/lib/python3.10/site-packages/sklearn/linear_model/_logistic.py:458: ConvergenceWarning: lbfgs failed to converge (status=1):\n",
      "STOP: TOTAL NO. of ITERATIONS REACHED LIMIT.\n",
      "\n",
      "Increase the number of iterations (max_iter) or scale the data as shown in:\n",
      "    https://scikit-learn.org/stable/modules/preprocessing.html\n",
      "Please also refer to the documentation for alternative solver options:\n",
      "    https://scikit-learn.org/stable/modules/linear_model.html#logistic-regression\n",
      "  n_iter_i = _check_optimize_result(\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/potatosalad/.local/share/virtualenvs/CSC_homework_3_group4-nat9sqnt/lib/python3.10/site-packages/sklearn/linear_model/_logistic.py:458: ConvergenceWarning: lbfgs failed to converge (status=1):\n",
      "STOP: TOTAL NO. of ITERATIONS REACHED LIMIT.\n",
      "\n",
      "Increase the number of iterations (max_iter) or scale the data as shown in:\n",
      "    https://scikit-learn.org/stable/modules/preprocessing.html\n",
      "Please also refer to the documentation for alternative solver options:\n",
      "    https://scikit-learn.org/stable/modules/linear_model.html#logistic-regression\n",
      "  n_iter_i = _check_optimize_result(\n",
      "/home/potatosalad/.local/share/virtualenvs/CSC_homework_3_group4-nat9sqnt/lib/python3.10/site-packages/sklearn/linear_model/_logistic.py:458: ConvergenceWarning: lbfgs failed to converge (status=1):\n",
      "STOP: TOTAL NO. of ITERATIONS REACHED LIMIT.\n",
      "\n",
      "Increase the number of iterations (max_iter) or scale the data as shown in:\n",
      "    https://scikit-learn.org/stable/modules/preprocessing.html\n",
      "Please also refer to the documentation for alternative solver options:\n",
      "    https://scikit-learn.org/stable/modules/linear_model.html#logistic-regression\n",
      "  n_iter_i = _check_optimize_result(\n",
      "/home/potatosalad/.local/share/virtualenvs/CSC_homework_3_group4-nat9sqnt/lib/python3.10/site-packages/sklearn/linear_model/_logistic.py:458: ConvergenceWarning: lbfgs failed to converge (status=1):\n",
      "STOP: TOTAL NO. of ITERATIONS REACHED LIMIT.\n",
      "\n",
      "Increase the number of iterations (max_iter) or scale the data as shown in:\n",
      "    https://scikit-learn.org/stable/modules/preprocessing.html\n",
      "Please also refer to the documentation for alternative solver options:\n",
      "    https://scikit-learn.org/stable/modules/linear_model.html#logistic-regression\n",
      "  n_iter_i = _check_optimize_result(\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Model performance for train set ENSEMBLE_heart_Data_gridsearch\n",
      "- Accuracy: 0.9907\n",
      "- F1 score: 0.990741\n",
      "- Precision: 0.990741\n",
      "- Recall: 0.990741\n",
      "----------------------------------\n",
      "===================================\n",
      "Model performance for test set ENSEMBLE_heart_Data_gridsearch\n",
      "- Accuracy: 0.8519\n",
      "- F1 score: 0.850327\n",
      "- Precision: 0.851295\n",
      "- Recall: 0.851852\n",
      "----------------------------------\n",
      "===================================\n"
     ]
    }
   ],
   "source": [
    "# GRIDSEARCH\n",
    "fit_models(X_train, y_train, X_test, y_test,data_heart_target_names, models, params, \"heart_Data_gridsearch\", \"experiment_heart_gridsearch\",1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 70,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Fitting for parameters...\n",
      "X:       age  sex  chest pain type (4 values)  resting blood pressure  \\\n",
      "115  49.0  0.0                         2.0                   134.0   \n",
      "33   59.0  1.0                         4.0                   170.0   \n",
      "184  53.0  1.0                         3.0                   130.0   \n",
      "142  50.0  1.0                         3.0                   140.0   \n",
      "197  54.0  0.0                         3.0                   110.0   \n",
      "..    ...  ...                         ...                     ...   \n",
      "20   67.0  1.0                         4.0                   120.0   \n",
      "188  62.0  0.0                         4.0                   140.0   \n",
      "71   57.0  0.0                         4.0                   120.0   \n",
      "106  51.0  1.0                         3.0                   100.0   \n",
      "102  49.0  0.0                         4.0                   130.0   \n",
      "\n",
      "     serum cholesterol in mg/dl  fasting blood sugar > 120 mg/dl  \\\n",
      "115                       271.0                              0.0   \n",
      "33                        326.0                              0.0   \n",
      "184                       246.0                              1.0   \n",
      "142                       233.0                              0.0   \n",
      "197                       214.0                              0.0   \n",
      "..                          ...                              ...   \n",
      "20                        229.0                              0.0   \n",
      "188                       394.0                              0.0   \n",
      "71                        354.0                              0.0   \n",
      "106                       222.0                              0.0   \n",
      "102                       269.0                              0.0   \n",
      "\n",
      "     resting electrocardiographic results (values 0,1,2)  \\\n",
      "115                                                0.0     \n",
      "33                                                 2.0     \n",
      "184                                                2.0     \n",
      "142                                                0.0     \n",
      "197                                                0.0     \n",
      "..                                                 ...     \n",
      "20                                                 2.0     \n",
      "188                                                2.0     \n",
      "71                                                 0.0     \n",
      "106                                                0.0     \n",
      "102                                                0.0     \n",
      "\n",
      "     maximum heart rate achieved  exercise induced angina  \\\n",
      "115                        162.0                      0.0   \n",
      "33                         140.0                      1.0   \n",
      "184                        173.0                      0.0   \n",
      "142                        163.0                      0.0   \n",
      "197                        158.0                      0.0   \n",
      "..                           ...                      ...   \n",
      "20                         129.0                      1.0   \n",
      "188                        157.0                      0.0   \n",
      "71                         163.0                      1.0   \n",
      "106                        143.0                      1.0   \n",
      "102                        163.0                      0.0   \n",
      "\n",
      "     oldpeak = ST depression induced by exercise relative to rest  \\\n",
      "115                                                0.0              \n",
      "33                                                 3.4              \n",
      "184                                                0.0              \n",
      "142                                                0.6              \n",
      "197                                                1.6              \n",
      "..                                                 ...              \n",
      "20                                                 2.6              \n",
      "188                                                1.2              \n",
      "71                                                 0.6              \n",
      "106                                                1.2              \n",
      "102                                                0.0              \n",
      "\n",
      "     the slope of the peak exercise ST segment  \\\n",
      "115                                        2.0   \n",
      "33                                         3.0   \n",
      "184                                        1.0   \n",
      "142                                        2.0   \n",
      "197                                        2.0   \n",
      "..                                         ...   \n",
      "20                                         2.0   \n",
      "188                                        2.0   \n",
      "71                                         1.0   \n",
      "106                                        2.0   \n",
      "102                                        1.0   \n",
      "\n",
      "     number of major vessels (0-3) colored by flourosopy  \\\n",
      "115                                                0.0     \n",
      "33                                                 0.0     \n",
      "184                                                3.0     \n",
      "142                                                1.0     \n",
      "197                                                0.0     \n",
      "..                                                 ...     \n",
      "20                                                 2.0     \n",
      "188                                                0.0     \n",
      "71                                                 0.0     \n",
      "106                                                0.0     \n",
      "102                                                0.0     \n",
      "\n",
      "     thal: 3 = normal; 6 = fixed defect; 7 = reversable defect  \n",
      "115                                                3.0          \n",
      "33                                                 7.0          \n",
      "184                                                3.0          \n",
      "142                                                7.0          \n",
      "197                                                3.0          \n",
      "..                                                 ...          \n",
      "20                                                 7.0          \n",
      "188                                                3.0          \n",
      "71                                                 3.0          \n",
      "106                                                3.0          \n",
      "102                                                3.0          \n",
      "\n",
      "[216 rows x 13 columns]\n",
      "--------------------------------------------------\n",
      "Y: 115    1\n",
      "33     2\n",
      "184    1\n",
      "142    2\n",
      "197    1\n",
      "      ..\n",
      "20     2\n",
      "188    1\n",
      "71     1\n",
      "106    1\n",
      "102    1\n",
      "Name: absence = 1, presence = 2, Length: 216, dtype: int64\n",
      "--------------------------------------------------\n",
      "X_t:       age  sex  chest pain type (4 values)  resting blood pressure  \\\n",
      "30   57.0  1.0                         3.0                   128.0   \n",
      "116  46.0  1.0                         4.0                   120.0   \n",
      "79   56.0  1.0                         2.0                   120.0   \n",
      "127  52.0  0.0                         3.0                   136.0   \n",
      "196  58.0  0.0                         4.0                   100.0   \n",
      "137  56.0  1.0                         4.0                   125.0   \n",
      "209  37.0  1.0                         3.0                   130.0   \n",
      "45   58.0  1.0                         3.0                   140.0   \n",
      "158  56.0  1.0                         1.0                   120.0   \n",
      "247  65.0  0.0                         3.0                   155.0   \n",
      "183  42.0  0.0                         4.0                   102.0   \n",
      "268  57.0  1.0                         4.0                   140.0   \n",
      "227  43.0  0.0                         4.0                   132.0   \n",
      "82   58.0  1.0                         4.0                   150.0   \n",
      "165  64.0  0.0                         4.0                   180.0   \n",
      "194  48.0  1.0                         3.0                   124.0   \n",
      "226  62.0  0.0                         3.0                   130.0   \n",
      "146  52.0  1.0                         4.0                   112.0   \n",
      "104  63.0  1.0                         4.0                   130.0   \n",
      "60   57.0  1.0                         3.0                   150.0   \n",
      "221  65.0  1.0                         4.0                   135.0   \n",
      "266  44.0  1.0                         2.0                   120.0   \n",
      "46   49.0  1.0                         3.0                   118.0   \n",
      "42   44.0  1.0                         3.0                   130.0   \n",
      "185  43.0  1.0                         3.0                   130.0   \n",
      "9    63.0  0.0                         4.0                   150.0   \n",
      "22   43.0  1.0                         4.0                   115.0   \n",
      "199  77.0  1.0                         4.0                   125.0   \n",
      "109  45.0  0.0                         2.0                   112.0   \n",
      "24   54.0  0.0                         2.0                   132.0   \n",
      "113  54.0  0.0                         3.0                   135.0   \n",
      "68   44.0  0.0                         3.0                   118.0   \n",
      "144  54.0  1.0                         2.0                   192.0   \n",
      "224  35.0  0.0                         4.0                   138.0   \n",
      "252  44.0  1.0                         4.0                   112.0   \n",
      "6    56.0  1.0                         3.0                   130.0   \n",
      "120  49.0  1.0                         3.0                   120.0   \n",
      "67   58.0  0.0                         2.0                   136.0   \n",
      "119  56.0  1.0                         4.0                   130.0   \n",
      "118  66.0  0.0                         1.0                   150.0   \n",
      "25   48.0  0.0                         3.0                   130.0   \n",
      "125  54.0  0.0                         3.0                   160.0   \n",
      "244  51.0  0.0                         3.0                   140.0   \n",
      "19   40.0  1.0                         1.0                   140.0   \n",
      "77   39.0  0.0                         3.0                    94.0   \n",
      "216  63.0  0.0                         3.0                   135.0   \n",
      "90   59.0  1.0                         2.0                   140.0   \n",
      "208  64.0  1.0                         3.0                   125.0   \n",
      "93   54.0  1.0                         4.0                   110.0   \n",
      "180  42.0  1.0                         3.0                   120.0   \n",
      "15   71.0  0.0                         4.0                   112.0   \n",
      "152  54.0  0.0                         3.0                   108.0   \n",
      "232  45.0  1.0                         4.0                   115.0   \n",
      "250  54.0  1.0                         4.0                   120.0   \n",
      "\n",
      "     serum cholesterol in mg/dl  fasting blood sugar > 120 mg/dl  \\\n",
      "30                        229.0                              0.0   \n",
      "116                       249.0                              0.0   \n",
      "79                        236.0                              0.0   \n",
      "127                       196.0                              0.0   \n",
      "196                       248.0                              0.0   \n",
      "137                       249.0                              1.0   \n",
      "209                       250.0                              0.0   \n",
      "45                        211.0                              1.0   \n",
      "158                       193.0                              0.0   \n",
      "247                       269.0                              0.0   \n",
      "183                       265.0                              0.0   \n",
      "268                       192.0                              0.0   \n",
      "227                       341.0                              1.0   \n",
      "82                        270.0                              0.0   \n",
      "165                       325.0                              0.0   \n",
      "194                       255.0                              1.0   \n",
      "226                       263.0                              0.0   \n",
      "146                       230.0                              0.0   \n",
      "104                       254.0                              0.0   \n",
      "60                        126.0                              1.0   \n",
      "221                       254.0                              0.0   \n",
      "266                       263.0                              0.0   \n",
      "46                        149.0                              0.0   \n",
      "42                        233.0                              0.0   \n",
      "185                       315.0                              0.0   \n",
      "9                         407.0                              0.0   \n",
      "22                        303.0                              0.0   \n",
      "199                       304.0                              0.0   \n",
      "109                       160.0                              0.0   \n",
      "24                        288.0                              1.0   \n",
      "113                       304.0                              1.0   \n",
      "68                        242.0                              0.0   \n",
      "144                       283.0                              0.0   \n",
      "224                       183.0                              0.0   \n",
      "252                       290.0                              0.0   \n",
      "6                         256.0                              1.0   \n",
      "120                       188.0                              0.0   \n",
      "67                        319.0                              1.0   \n",
      "119                       283.0                              1.0   \n",
      "118                       226.0                              0.0   \n",
      "25                        275.0                              0.0   \n",
      "125                       201.0                              0.0   \n",
      "244                       308.0                              0.0   \n",
      "19                        199.0                              0.0   \n",
      "77                        199.0                              0.0   \n",
      "216                       252.0                              0.0   \n",
      "90                        221.0                              0.0   \n",
      "208                       309.0                              0.0   \n",
      "93                        206.0                              0.0   \n",
      "180                       240.0                              1.0   \n",
      "15                        149.0                              0.0   \n",
      "152                       267.0                              0.0   \n",
      "232                       260.0                              0.0   \n",
      "250                       188.0                              0.0   \n",
      "\n",
      "     resting electrocardiographic results (values 0,1,2)  \\\n",
      "30                                                 2.0     \n",
      "116                                                2.0     \n",
      "79                                                 0.0     \n",
      "127                                                2.0     \n",
      "196                                                2.0     \n",
      "137                                                2.0     \n",
      "209                                                0.0     \n",
      "45                                                 2.0     \n",
      "158                                                2.0     \n",
      "247                                                0.0     \n",
      "183                                                2.0     \n",
      "268                                                0.0     \n",
      "227                                                2.0     \n",
      "82                                                 2.0     \n",
      "165                                                0.0     \n",
      "194                                                0.0     \n",
      "226                                                0.0     \n",
      "146                                                0.0     \n",
      "104                                                2.0     \n",
      "60                                                 0.0     \n",
      "221                                                2.0     \n",
      "266                                                0.0     \n",
      "46                                                 2.0     \n",
      "42                                                 0.0     \n",
      "185                                                0.0     \n",
      "9                                                  2.0     \n",
      "22                                                 0.0     \n",
      "199                                                2.0     \n",
      "109                                                0.0     \n",
      "24                                                 2.0     \n",
      "113                                                0.0     \n",
      "68                                                 0.0     \n",
      "144                                                2.0     \n",
      "224                                                0.0     \n",
      "252                                                2.0     \n",
      "6                                                  2.0     \n",
      "120                                                0.0     \n",
      "67                                                 2.0     \n",
      "119                                                2.0     \n",
      "118                                                0.0     \n",
      "25                                                 0.0     \n",
      "125                                                0.0     \n",
      "244                                                2.0     \n",
      "19                                                 0.0     \n",
      "77                                                 0.0     \n",
      "216                                                2.0     \n",
      "90                                                 0.0     \n",
      "208                                                0.0     \n",
      "93                                                 2.0     \n",
      "180                                                0.0     \n",
      "15                                                 0.0     \n",
      "152                                                2.0     \n",
      "232                                                2.0     \n",
      "250                                                0.0     \n",
      "\n",
      "     maximum heart rate achieved  exercise induced angina  \\\n",
      "30                         150.0                      0.0   \n",
      "116                        144.0                      0.0   \n",
      "79                         178.0                      0.0   \n",
      "127                        169.0                      0.0   \n",
      "196                        122.0                      0.0   \n",
      "137                        144.0                      1.0   \n",
      "209                        187.0                      0.0   \n",
      "45                         165.0                      0.0   \n",
      "158                        162.0                      0.0   \n",
      "247                        148.0                      0.0   \n",
      "183                        122.0                      0.0   \n",
      "268                        148.0                      0.0   \n",
      "227                        136.0                      1.0   \n",
      "82                         111.0                      1.0   \n",
      "165                        154.0                      1.0   \n",
      "194                        175.0                      0.0   \n",
      "226                         97.0                      0.0   \n",
      "146                        160.0                      0.0   \n",
      "104                        147.0                      0.0   \n",
      "60                         173.0                      0.0   \n",
      "221                        127.0                      0.0   \n",
      "266                        173.0                      0.0   \n",
      "46                         126.0                      0.0   \n",
      "42                         179.0                      1.0   \n",
      "185                        162.0                      0.0   \n",
      "9                          154.0                      0.0   \n",
      "22                         181.0                      0.0   \n",
      "199                        162.0                      1.0   \n",
      "109                        138.0                      0.0   \n",
      "24                         159.0                      1.0   \n",
      "113                        170.0                      0.0   \n",
      "68                         149.0                      0.0   \n",
      "144                        195.0                      0.0   \n",
      "224                        182.0                      0.0   \n",
      "252                        153.0                      0.0   \n",
      "6                          142.0                      1.0   \n",
      "120                        139.0                      0.0   \n",
      "67                         152.0                      0.0   \n",
      "119                        103.0                      1.0   \n",
      "118                        114.0                      0.0   \n",
      "25                         139.0                      0.0   \n",
      "125                        163.0                      0.0   \n",
      "244                        142.0                      0.0   \n",
      "19                         178.0                      1.0   \n",
      "77                         179.0                      0.0   \n",
      "216                        172.0                      0.0   \n",
      "90                         164.0                      1.0   \n",
      "208                        131.0                      1.0   \n",
      "93                         108.0                      1.0   \n",
      "180                        194.0                      0.0   \n",
      "15                         125.0                      0.0   \n",
      "152                        167.0                      0.0   \n",
      "232                        185.0                      0.0   \n",
      "250                        113.0                      0.0   \n",
      "\n",
      "     oldpeak = ST depression induced by exercise relative to rest  \\\n",
      "30                                                 0.4              \n",
      "116                                                0.8              \n",
      "79                                                 0.8              \n",
      "127                                                0.1              \n",
      "196                                                1.0              \n",
      "137                                                1.2              \n",
      "209                                                3.5              \n",
      "45                                                 0.0              \n",
      "158                                                1.9              \n",
      "247                                                0.8              \n",
      "183                                                0.6              \n",
      "268                                                0.4              \n",
      "227                                                3.0              \n",
      "82                                                 0.8              \n",
      "165                                                0.0              \n",
      "194                                                0.0              \n",
      "226                                                1.2              \n",
      "146                                                0.0              \n",
      "104                                                1.4              \n",
      "60                                                 0.2              \n",
      "221                                                2.8              \n",
      "266                                                0.0              \n",
      "46                                                 0.8              \n",
      "42                                                 0.4              \n",
      "185                                                1.9              \n",
      "9                                                  4.0              \n",
      "22                                                 1.2              \n",
      "199                                                0.0              \n",
      "109                                                0.0              \n",
      "24                                                 0.0              \n",
      "113                                                0.0              \n",
      "68                                                 0.3              \n",
      "144                                                0.0              \n",
      "224                                                1.4              \n",
      "252                                                0.0              \n",
      "6                                                  0.6              \n",
      "120                                                2.0              \n",
      "67                                                 0.0              \n",
      "119                                                1.6              \n",
      "118                                                2.6              \n",
      "25                                                 0.2              \n",
      "125                                                0.0              \n",
      "244                                                1.5              \n",
      "19                                                 1.4              \n",
      "77                                                 0.0              \n",
      "216                                                0.0              \n",
      "90                                                 0.0              \n",
      "208                                                1.8              \n",
      "93                                                 0.0              \n",
      "180                                                0.8              \n",
      "15                                                 1.6              \n",
      "152                                                0.0              \n",
      "232                                                0.0              \n",
      "250                                                1.4              \n",
      "\n",
      "     the slope of the peak exercise ST segment  \\\n",
      "30                                         2.0   \n",
      "116                                        1.0   \n",
      "79                                         1.0   \n",
      "127                                        2.0   \n",
      "196                                        2.0   \n",
      "137                                        2.0   \n",
      "209                                        3.0   \n",
      "45                                         1.0   \n",
      "158                                        2.0   \n",
      "247                                        1.0   \n",
      "183                                        2.0   \n",
      "268                                        2.0   \n",
      "227                                        2.0   \n",
      "82                                         1.0   \n",
      "165                                        1.0   \n",
      "194                                        1.0   \n",
      "226                                        2.0   \n",
      "146                                        1.0   \n",
      "104                                        2.0   \n",
      "60                                         1.0   \n",
      "221                                        2.0   \n",
      "266                                        1.0   \n",
      "46                                         1.0   \n",
      "42                                         1.0   \n",
      "185                                        1.0   \n",
      "9                                          2.0   \n",
      "22                                         2.0   \n",
      "199                                        1.0   \n",
      "109                                        2.0   \n",
      "24                                         1.0   \n",
      "113                                        1.0   \n",
      "68                                         2.0   \n",
      "144                                        1.0   \n",
      "224                                        1.0   \n",
      "252                                        1.0   \n",
      "6                                          2.0   \n",
      "120                                        2.0   \n",
      "67                                         1.0   \n",
      "119                                        3.0   \n",
      "118                                        3.0   \n",
      "25                                         1.0   \n",
      "125                                        1.0   \n",
      "244                                        1.0   \n",
      "19                                         1.0   \n",
      "77                                         1.0   \n",
      "216                                        1.0   \n",
      "90                                         1.0   \n",
      "208                                        2.0   \n",
      "93                                         2.0   \n",
      "180                                        3.0   \n",
      "15                                         2.0   \n",
      "152                                        1.0   \n",
      "232                                        1.0   \n",
      "250                                        2.0   \n",
      "\n",
      "     number of major vessels (0-3) colored by flourosopy  \\\n",
      "30                                                 1.0     \n",
      "116                                                0.0     \n",
      "79                                                 0.0     \n",
      "127                                                0.0     \n",
      "196                                                0.0     \n",
      "137                                                1.0     \n",
      "209                                                0.0     \n",
      "45                                                 0.0     \n",
      "158                                                0.0     \n",
      "247                                                0.0     \n",
      "183                                                0.0     \n",
      "268                                                0.0     \n",
      "227                                                0.0     \n",
      "82                                                 0.0     \n",
      "165                                                0.0     \n",
      "194                                                2.0     \n",
      "226                                                1.0     \n",
      "146                                                1.0     \n",
      "104                                                1.0     \n",
      "60                                                 1.0     \n",
      "221                                                1.0     \n",
      "266                                                0.0     \n",
      "46                                                 3.0     \n",
      "42                                                 0.0     \n",
      "185                                                1.0     \n",
      "9                                                  3.0     \n",
      "22                                                 0.0     \n",
      "199                                                3.0     \n",
      "109                                                0.0     \n",
      "24                                                 1.0     \n",
      "113                                                0.0     \n",
      "68                                                 1.0     \n",
      "144                                                1.0     \n",
      "224                                                0.0     \n",
      "252                                                1.0     \n",
      "6                                                  1.0     \n",
      "120                                                3.0     \n",
      "67                                                 2.0     \n",
      "119                                                0.0     \n",
      "118                                                0.0     \n",
      "25                                                 0.0     \n",
      "125                                                1.0     \n",
      "244                                                1.0     \n",
      "19                                                 0.0     \n",
      "77                                                 0.0     \n",
      "216                                                0.0     \n",
      "90                                                 0.0     \n",
      "208                                                0.0     \n",
      "93                                                 1.0     \n",
      "180                                                0.0     \n",
      "15                                                 0.0     \n",
      "152                                                0.0     \n",
      "232                                                0.0     \n",
      "250                                                1.0     \n",
      "\n",
      "     thal: 3 = normal; 6 = fixed defect; 7 = reversable defect  \n",
      "30                                                 7.0          \n",
      "116                                                7.0          \n",
      "79                                                 3.0          \n",
      "127                                                3.0          \n",
      "196                                                3.0          \n",
      "137                                                3.0          \n",
      "209                                                3.0          \n",
      "45                                                 3.0          \n",
      "158                                                7.0          \n",
      "247                                                3.0          \n",
      "183                                                3.0          \n",
      "268                                                6.0          \n",
      "227                                                7.0          \n",
      "82                                                 7.0          \n",
      "165                                                3.0          \n",
      "194                                                3.0          \n",
      "226                                                7.0          \n",
      "146                                                3.0          \n",
      "104                                                7.0          \n",
      "60                                                 7.0          \n",
      "221                                                7.0          \n",
      "266                                                7.0          \n",
      "46                                                 3.0          \n",
      "42                                                 3.0          \n",
      "185                                                3.0          \n",
      "9                                                  7.0          \n",
      "22                                                 3.0          \n",
      "199                                                3.0          \n",
      "109                                                3.0          \n",
      "24                                                 3.0          \n",
      "113                                                3.0          \n",
      "68                                                 3.0          \n",
      "144                                                7.0          \n",
      "224                                                3.0          \n",
      "252                                                3.0          \n",
      "6                                                  6.0          \n",
      "120                                                7.0          \n",
      "67                                                 3.0          \n",
      "119                                                7.0          \n",
      "118                                                3.0          \n",
      "25                                                 3.0          \n",
      "125                                                3.0          \n",
      "244                                                3.0          \n",
      "19                                                 7.0          \n",
      "77                                                 3.0          \n",
      "216                                                3.0          \n",
      "90                                                 3.0          \n",
      "208                                                7.0          \n",
      "93                                                 3.0          \n",
      "180                                                7.0          \n",
      "15                                                 3.0          \n",
      "152                                                3.0          \n",
      "232                                                3.0          \n",
      "250                                                7.0          \n",
      "--------------------------------------------------\n",
      "Y_t: 30     2\n",
      "116    2\n",
      "79     1\n",
      "127    1\n",
      "196    1\n",
      "137    2\n",
      "209    1\n",
      "45     1\n",
      "158    1\n",
      "247    1\n",
      "183    1\n",
      "268    1\n",
      "227    2\n",
      "82     2\n",
      "165    1\n",
      "194    1\n",
      "226    2\n",
      "146    2\n",
      "104    2\n",
      "60     1\n",
      "221    2\n",
      "266    1\n",
      "46     2\n",
      "42     1\n",
      "185    1\n",
      "9      2\n",
      "22     1\n",
      "199    2\n",
      "109    1\n",
      "24     1\n",
      "113    1\n",
      "68     1\n",
      "144    2\n",
      "224    1\n",
      "252    2\n",
      "6      2\n",
      "120    2\n",
      "67     2\n",
      "119    2\n",
      "118    1\n",
      "25     1\n",
      "125    1\n",
      "244    1\n",
      "19     1\n",
      "77     1\n",
      "216    1\n",
      "90     1\n",
      "208    2\n",
      "93     2\n",
      "180    1\n",
      "15     1\n",
      "152    1\n",
      "232    1\n",
      "250    2\n",
      "Name: absence = 1, presence = 2, dtype: int64\n",
      "--------------------------------------------------\n",
      "models: {'Logistic Regression': LogisticRegression(), 'Decision Tree': DecisionTreeClassifier(), 'Random Forest': RandomForestClassifier(random_state=42), 'Gradient Boosting': GradientBoostingClassifier(), 'K-Nearest Neighbors': KNeighborsClassifier(), 'SVM': SVC(probability=True)}\n",
      "--------------------------------------------------\n",
      "params: {'Logistic Regression': {'solver': ['newton-cholesky', 'liblinear', 'newton-cg', 'sag', 'saga'], 'penalty': ['l2'], 'C': [0.001, 0.01, 0.1, 1, 100], 'max_iter': [10000]}, 'Decision Tree': {'criterion': ['gini', 'entropy'], 'max_depth': [4, 6, 8, 10, 15, 50, 90]}, 'Random Forest': {'n_estimators': [200, 500], 'max_features': [None, 'sqrt', 'log2'], 'max_depth': [4, 5, 6, 7, 8], 'criterion': ['gini', 'entropy']}, 'Gradient Boosting': {'loss': ['log_loss'], 'learning_rate': [0.1, 0.01], 'max_depth': [3, 5, 8], 'max_features': ['log2', 'sqrt'], 'criterion': ['friedman_mse', 'squared_error'], 'subsample': [0.5, 0.8, 1.0], 'n_estimators': [10, 100, 1000]}, 'K-Nearest Neighbors': {'weights': ['uniform', 'distance'], 'leaf_size': [5, 15, 25]}, 'SVM': {'kernel': ['rbf'], 'C': [0.1, 1, 10, 100, 1000], 'degree': [1, 5], 'gamma': [1, 0.1, 0.01, 0.001]}}\n",
      "--------------------------------------------------\n",
      "run: heart_Data_no_gridsearch\n",
      "--------------------------------------------------\n",
      "folder: experiment_heart_no_gridsearch\n",
      "Logistic Regression\n",
      "\n",
      "##################################################\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Model performance for Training set Logistic Regression_heart_Data_no_gridsearch\n",
      "- Accuracy: 0.8380\n",
      "- F1 score: 0.837742\n",
      "- Precision: 0.837847\n",
      "- Recall: 0.837963\n",
      "----------------------------------\n",
      "===================================\n",
      "Model performance for Test set Logistic Regression_heart_Data_no_gridsearch\n",
      "- Accuracy: 0.9074\n",
      "- F1 score: 0.906969\n",
      "- Precision: 0.907190\n",
      "- Recall: 0.907407\n",
      "----------------------------------\n",
      "===================================\n",
      "Decision Tree\n",
      "\n",
      "##################################################\n",
      "Model performance for Training set Decision Tree_heart_Data_no_gridsearch\n",
      "- Accuracy: 1.0000\n",
      "- F1 score: 1.000000\n",
      "- Precision: 1.000000\n",
      "- Recall: 1.000000\n",
      "----------------------------------\n",
      "===================================\n",
      "Model performance for Test set Decision Tree_heart_Data_no_gridsearch\n",
      "- Accuracy: 0.6852\n",
      "- F1 score: 0.689028\n",
      "- Precision: 0.704518\n",
      "- Recall: 0.685185\n",
      "----------------------------------\n",
      "===================================\n",
      "Random Forest\n",
      "\n",
      "##################################################\n",
      "Model performance for Training set Random Forest_heart_Data_no_gridsearch\n",
      "- Accuracy: 1.0000\n",
      "- F1 score: 1.000000\n",
      "- Precision: 1.000000\n",
      "- Recall: 1.000000\n",
      "----------------------------------\n",
      "===================================\n",
      "Model performance for Test set Random Forest_heart_Data_no_gridsearch\n",
      "- Accuracy: 0.7593\n",
      "- F1 score: 0.755233\n",
      "- Precision: 0.756173\n",
      "- Recall: 0.759259\n",
      "----------------------------------\n",
      "===================================\n",
      "Gradient Boosting\n",
      "\n",
      "##################################################\n",
      "Model performance for Training set Gradient Boosting_heart_Data_no_gridsearch\n",
      "- Accuracy: 0.9954\n",
      "- F1 score: 0.995368\n",
      "- Precision: 0.995410\n",
      "- Recall: 0.995370\n",
      "----------------------------------\n",
      "===================================\n",
      "Model performance for Test set Gradient Boosting_heart_Data_no_gridsearch\n",
      "- Accuracy: 0.7407\n",
      "- F1 score: 0.738072\n",
      "- Precision: 0.737510\n",
      "- Recall: 0.740741\n",
      "----------------------------------\n",
      "===================================\n",
      "K-Nearest Neighbors\n",
      "\n",
      "##################################################\n",
      "Model performance for Training set K-Nearest Neighbors_heart_Data_no_gridsearch\n",
      "- Accuracy: 0.8657\n",
      "- F1 score: 0.865406\n",
      "- Precision: 0.866011\n",
      "- Recall: 0.865741\n",
      "----------------------------------\n",
      "===================================\n",
      "Model performance for Test set K-Nearest Neighbors_heart_Data_no_gridsearch\n",
      "- Accuracy: 0.8148\n",
      "- F1 score: 0.810359\n",
      "- Precision: 0.815757\n",
      "- Recall: 0.814815\n",
      "----------------------------------\n",
      "===================================\n",
      "SVM\n",
      "\n",
      "##################################################\n",
      "Model performance for Training set SVM_heart_Data_no_gridsearch\n",
      "- Accuracy: 0.9074\n",
      "- F1 score: 0.907327\n",
      "- Precision: 0.907404\n",
      "- Recall: 0.907407\n",
      "----------------------------------\n",
      "===================================\n",
      "Model performance for Test set SVM_heart_Data_no_gridsearch\n",
      "- Accuracy: 0.8889\n",
      "- F1 score: 0.887745\n",
      "- Precision: 0.889223\n",
      "- Recall: 0.888889\n",
      "----------------------------------\n",
      "===================================\n",
      "################################################\n",
      "################### ENSEMBLE ###################\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/potatosalad/.local/share/virtualenvs/CSC_homework_3_group4-nat9sqnt/lib/python3.10/site-packages/sklearn/linear_model/_logistic.py:458: ConvergenceWarning: lbfgs failed to converge (status=1):\n",
      "STOP: TOTAL NO. of ITERATIONS REACHED LIMIT.\n",
      "\n",
      "Increase the number of iterations (max_iter) or scale the data as shown in:\n",
      "    https://scikit-learn.org/stable/modules/preprocessing.html\n",
      "Please also refer to the documentation for alternative solver options:\n",
      "    https://scikit-learn.org/stable/modules/linear_model.html#logistic-regression\n",
      "  n_iter_i = _check_optimize_result(\n",
      "/home/potatosalad/.local/share/virtualenvs/CSC_homework_3_group4-nat9sqnt/lib/python3.10/site-packages/sklearn/linear_model/_logistic.py:458: ConvergenceWarning: lbfgs failed to converge (status=1):\n",
      "STOP: TOTAL NO. of ITERATIONS REACHED LIMIT.\n",
      "\n",
      "Increase the number of iterations (max_iter) or scale the data as shown in:\n",
      "    https://scikit-learn.org/stable/modules/preprocessing.html\n",
      "Please also refer to the documentation for alternative solver options:\n",
      "    https://scikit-learn.org/stable/modules/linear_model.html#logistic-regression\n",
      "  n_iter_i = _check_optimize_result(\n",
      "/home/potatosalad/.local/share/virtualenvs/CSC_homework_3_group4-nat9sqnt/lib/python3.10/site-packages/sklearn/linear_model/_logistic.py:458: ConvergenceWarning: lbfgs failed to converge (status=1):\n",
      "STOP: TOTAL NO. of ITERATIONS REACHED LIMIT.\n",
      "\n",
      "Increase the number of iterations (max_iter) or scale the data as shown in:\n",
      "    https://scikit-learn.org/stable/modules/preprocessing.html\n",
      "Please also refer to the documentation for alternative solver options:\n",
      "    https://scikit-learn.org/stable/modules/linear_model.html#logistic-regression\n",
      "  n_iter_i = _check_optimize_result(\n",
      "/home/potatosalad/.local/share/virtualenvs/CSC_homework_3_group4-nat9sqnt/lib/python3.10/site-packages/sklearn/linear_model/_logistic.py:458: ConvergenceWarning: lbfgs failed to converge (status=1):\n",
      "STOP: TOTAL NO. of ITERATIONS REACHED LIMIT.\n",
      "\n",
      "Increase the number of iterations (max_iter) or scale the data as shown in:\n",
      "    https://scikit-learn.org/stable/modules/preprocessing.html\n",
      "Please also refer to the documentation for alternative solver options:\n",
      "    https://scikit-learn.org/stable/modules/linear_model.html#logistic-regression\n",
      "  n_iter_i = _check_optimize_result(\n",
      "/home/potatosalad/.local/share/virtualenvs/CSC_homework_3_group4-nat9sqnt/lib/python3.10/site-packages/sklearn/linear_model/_logistic.py:458: ConvergenceWarning: lbfgs failed to converge (status=1):\n",
      "STOP: TOTAL NO. of ITERATIONS REACHED LIMIT.\n",
      "\n",
      "Increase the number of iterations (max_iter) or scale the data as shown in:\n",
      "    https://scikit-learn.org/stable/modules/preprocessing.html\n",
      "Please also refer to the documentation for alternative solver options:\n",
      "    https://scikit-learn.org/stable/modules/linear_model.html#logistic-regression\n",
      "  n_iter_i = _check_optimize_result(\n",
      "/home/potatosalad/.local/share/virtualenvs/CSC_homework_3_group4-nat9sqnt/lib/python3.10/site-packages/sklearn/linear_model/_logistic.py:458: ConvergenceWarning: lbfgs failed to converge (status=1):\n",
      "STOP: TOTAL NO. of ITERATIONS REACHED LIMIT.\n",
      "\n",
      "Increase the number of iterations (max_iter) or scale the data as shown in:\n",
      "    https://scikit-learn.org/stable/modules/preprocessing.html\n",
      "Please also refer to the documentation for alternative solver options:\n",
      "    https://scikit-learn.org/stable/modules/linear_model.html#logistic-regression\n",
      "  n_iter_i = _check_optimize_result(\n",
      "/home/potatosalad/.local/share/virtualenvs/CSC_homework_3_group4-nat9sqnt/lib/python3.10/site-packages/sklearn/linear_model/_logistic.py:458: ConvergenceWarning: lbfgs failed to converge (status=1):\n",
      "STOP: TOTAL NO. of ITERATIONS REACHED LIMIT.\n",
      "\n",
      "Increase the number of iterations (max_iter) or scale the data as shown in:\n",
      "    https://scikit-learn.org/stable/modules/preprocessing.html\n",
      "Please also refer to the documentation for alternative solver options:\n",
      "    https://scikit-learn.org/stable/modules/linear_model.html#logistic-regression\n",
      "  n_iter_i = _check_optimize_result(\n",
      "/home/potatosalad/.local/share/virtualenvs/CSC_homework_3_group4-nat9sqnt/lib/python3.10/site-packages/sklearn/linear_model/_logistic.py:458: ConvergenceWarning: lbfgs failed to converge (status=1):\n",
      "STOP: TOTAL NO. of ITERATIONS REACHED LIMIT.\n",
      "\n",
      "Increase the number of iterations (max_iter) or scale the data as shown in:\n",
      "    https://scikit-learn.org/stable/modules/preprocessing.html\n",
      "Please also refer to the documentation for alternative solver options:\n",
      "    https://scikit-learn.org/stable/modules/linear_model.html#logistic-regression\n",
      "  n_iter_i = _check_optimize_result(\n",
      "/home/potatosalad/.local/share/virtualenvs/CSC_homework_3_group4-nat9sqnt/lib/python3.10/site-packages/sklearn/linear_model/_logistic.py:458: ConvergenceWarning: lbfgs failed to converge (status=1):\n",
      "STOP: TOTAL NO. of ITERATIONS REACHED LIMIT.\n",
      "\n",
      "Increase the number of iterations (max_iter) or scale the data as shown in:\n",
      "    https://scikit-learn.org/stable/modules/preprocessing.html\n",
      "Please also refer to the documentation for alternative solver options:\n",
      "    https://scikit-learn.org/stable/modules/linear_model.html#logistic-regression\n",
      "  n_iter_i = _check_optimize_result(\n",
      "/home/potatosalad/.local/share/virtualenvs/CSC_homework_3_group4-nat9sqnt/lib/python3.10/site-packages/sklearn/linear_model/_logistic.py:458: ConvergenceWarning: lbfgs failed to converge (status=1):\n",
      "STOP: TOTAL NO. of ITERATIONS REACHED LIMIT.\n",
      "\n",
      "Increase the number of iterations (max_iter) or scale the data as shown in:\n",
      "    https://scikit-learn.org/stable/modules/preprocessing.html\n",
      "Please also refer to the documentation for alternative solver options:\n",
      "    https://scikit-learn.org/stable/modules/linear_model.html#logistic-regression\n",
      "  n_iter_i = _check_optimize_result(\n",
      "/home/potatosalad/.local/share/virtualenvs/CSC_homework_3_group4-nat9sqnt/lib/python3.10/site-packages/sklearn/linear_model/_logistic.py:458: ConvergenceWarning: lbfgs failed to converge (status=1):\n",
      "STOP: TOTAL NO. of ITERATIONS REACHED LIMIT.\n",
      "\n",
      "Increase the number of iterations (max_iter) or scale the data as shown in:\n",
      "    https://scikit-learn.org/stable/modules/preprocessing.html\n",
      "Please also refer to the documentation for alternative solver options:\n",
      "    https://scikit-learn.org/stable/modules/linear_model.html#logistic-regression\n",
      "  n_iter_i = _check_optimize_result(\n",
      "/home/potatosalad/.local/share/virtualenvs/CSC_homework_3_group4-nat9sqnt/lib/python3.10/site-packages/sklearn/linear_model/_logistic.py:458: ConvergenceWarning: lbfgs failed to converge (status=1):\n",
      "STOP: TOTAL NO. of ITERATIONS REACHED LIMIT.\n",
      "\n",
      "Increase the number of iterations (max_iter) or scale the data as shown in:\n",
      "    https://scikit-learn.org/stable/modules/preprocessing.html\n",
      "Please also refer to the documentation for alternative solver options:\n",
      "    https://scikit-learn.org/stable/modules/linear_model.html#logistic-regression\n",
      "  n_iter_i = _check_optimize_result(\n",
      "/home/potatosalad/.local/share/virtualenvs/CSC_homework_3_group4-nat9sqnt/lib/python3.10/site-packages/sklearn/linear_model/_logistic.py:458: ConvergenceWarning: lbfgs failed to converge (status=1):\n",
      "STOP: TOTAL NO. of ITERATIONS REACHED LIMIT.\n",
      "\n",
      "Increase the number of iterations (max_iter) or scale the data as shown in:\n",
      "    https://scikit-learn.org/stable/modules/preprocessing.html\n",
      "Please also refer to the documentation for alternative solver options:\n",
      "    https://scikit-learn.org/stable/modules/linear_model.html#logistic-regression\n",
      "  n_iter_i = _check_optimize_result(\n",
      "/home/potatosalad/.local/share/virtualenvs/CSC_homework_3_group4-nat9sqnt/lib/python3.10/site-packages/sklearn/linear_model/_logistic.py:458: ConvergenceWarning: lbfgs failed to converge (status=1):\n",
      "STOP: TOTAL NO. of ITERATIONS REACHED LIMIT.\n",
      "\n",
      "Increase the number of iterations (max_iter) or scale the data as shown in:\n",
      "    https://scikit-learn.org/stable/modules/preprocessing.html\n",
      "Please also refer to the documentation for alternative solver options:\n",
      "    https://scikit-learn.org/stable/modules/linear_model.html#logistic-regression\n",
      "  n_iter_i = _check_optimize_result(\n",
      "/home/potatosalad/.local/share/virtualenvs/CSC_homework_3_group4-nat9sqnt/lib/python3.10/site-packages/sklearn/linear_model/_logistic.py:458: ConvergenceWarning: lbfgs failed to converge (status=1):\n",
      "STOP: TOTAL NO. of ITERATIONS REACHED LIMIT.\n",
      "\n",
      "Increase the number of iterations (max_iter) or scale the data as shown in:\n",
      "    https://scikit-learn.org/stable/modules/preprocessing.html\n",
      "Please also refer to the documentation for alternative solver options:\n",
      "    https://scikit-learn.org/stable/modules/linear_model.html#logistic-regression\n",
      "  n_iter_i = _check_optimize_result(\n",
      "/home/potatosalad/.local/share/virtualenvs/CSC_homework_3_group4-nat9sqnt/lib/python3.10/site-packages/sklearn/linear_model/_logistic.py:458: ConvergenceWarning: lbfgs failed to converge (status=1):\n",
      "STOP: TOTAL NO. of ITERATIONS REACHED LIMIT.\n",
      "\n",
      "Increase the number of iterations (max_iter) or scale the data as shown in:\n",
      "    https://scikit-learn.org/stable/modules/preprocessing.html\n",
      "Please also refer to the documentation for alternative solver options:\n",
      "    https://scikit-learn.org/stable/modules/linear_model.html#logistic-regression\n",
      "  n_iter_i = _check_optimize_result(\n",
      "/home/potatosalad/.local/share/virtualenvs/CSC_homework_3_group4-nat9sqnt/lib/python3.10/site-packages/sklearn/linear_model/_logistic.py:458: ConvergenceWarning: lbfgs failed to converge (status=1):\n",
      "STOP: TOTAL NO. of ITERATIONS REACHED LIMIT.\n",
      "\n",
      "Increase the number of iterations (max_iter) or scale the data as shown in:\n",
      "    https://scikit-learn.org/stable/modules/preprocessing.html\n",
      "Please also refer to the documentation for alternative solver options:\n",
      "    https://scikit-learn.org/stable/modules/linear_model.html#logistic-regression\n",
      "  n_iter_i = _check_optimize_result(\n",
      "/home/potatosalad/.local/share/virtualenvs/CSC_homework_3_group4-nat9sqnt/lib/python3.10/site-packages/sklearn/linear_model/_logistic.py:458: ConvergenceWarning: lbfgs failed to converge (status=1):\n",
      "STOP: TOTAL NO. of ITERATIONS REACHED LIMIT.\n",
      "\n",
      "Increase the number of iterations (max_iter) or scale the data as shown in:\n",
      "    https://scikit-learn.org/stable/modules/preprocessing.html\n",
      "Please also refer to the documentation for alternative solver options:\n",
      "    https://scikit-learn.org/stable/modules/linear_model.html#logistic-regression\n",
      "  n_iter_i = _check_optimize_result(\n",
      "/home/potatosalad/.local/share/virtualenvs/CSC_homework_3_group4-nat9sqnt/lib/python3.10/site-packages/sklearn/linear_model/_logistic.py:458: ConvergenceWarning: lbfgs failed to converge (status=1):\n",
      "STOP: TOTAL NO. of ITERATIONS REACHED LIMIT.\n",
      "\n",
      "Increase the number of iterations (max_iter) or scale the data as shown in:\n",
      "    https://scikit-learn.org/stable/modules/preprocessing.html\n",
      "Please also refer to the documentation for alternative solver options:\n",
      "    https://scikit-learn.org/stable/modules/linear_model.html#logistic-regression\n",
      "  n_iter_i = _check_optimize_result(\n",
      "/home/potatosalad/.local/share/virtualenvs/CSC_homework_3_group4-nat9sqnt/lib/python3.10/site-packages/sklearn/linear_model/_logistic.py:458: ConvergenceWarning: lbfgs failed to converge (status=1):\n",
      "STOP: TOTAL NO. of ITERATIONS REACHED LIMIT.\n",
      "\n",
      "Increase the number of iterations (max_iter) or scale the data as shown in:\n",
      "    https://scikit-learn.org/stable/modules/preprocessing.html\n",
      "Please also refer to the documentation for alternative solver options:\n",
      "    https://scikit-learn.org/stable/modules/linear_model.html#logistic-regression\n",
      "  n_iter_i = _check_optimize_result(\n",
      "/home/potatosalad/.local/share/virtualenvs/CSC_homework_3_group4-nat9sqnt/lib/python3.10/site-packages/sklearn/linear_model/_logistic.py:458: ConvergenceWarning: lbfgs failed to converge (status=1):\n",
      "STOP: TOTAL NO. of ITERATIONS REACHED LIMIT.\n",
      "\n",
      "Increase the number of iterations (max_iter) or scale the data as shown in:\n",
      "    https://scikit-learn.org/stable/modules/preprocessing.html\n",
      "Please also refer to the documentation for alternative solver options:\n",
      "    https://scikit-learn.org/stable/modules/linear_model.html#logistic-regression\n",
      "  n_iter_i = _check_optimize_result(\n",
      "/home/potatosalad/.local/share/virtualenvs/CSC_homework_3_group4-nat9sqnt/lib/python3.10/site-packages/sklearn/linear_model/_logistic.py:458: ConvergenceWarning: lbfgs failed to converge (status=1):\n",
      "STOP: TOTAL NO. of ITERATIONS REACHED LIMIT.\n",
      "\n",
      "Increase the number of iterations (max_iter) or scale the data as shown in:\n",
      "    https://scikit-learn.org/stable/modules/preprocessing.html\n",
      "Please also refer to the documentation for alternative solver options:\n",
      "    https://scikit-learn.org/stable/modules/linear_model.html#logistic-regression\n",
      "  n_iter_i = _check_optimize_result(\n",
      "/home/potatosalad/.local/share/virtualenvs/CSC_homework_3_group4-nat9sqnt/lib/python3.10/site-packages/sklearn/linear_model/_logistic.py:458: ConvergenceWarning: lbfgs failed to converge (status=1):\n",
      "STOP: TOTAL NO. of ITERATIONS REACHED LIMIT.\n",
      "\n",
      "Increase the number of iterations (max_iter) or scale the data as shown in:\n",
      "    https://scikit-learn.org/stable/modules/preprocessing.html\n",
      "Please also refer to the documentation for alternative solver options:\n",
      "    https://scikit-learn.org/stable/modules/linear_model.html#logistic-regression\n",
      "  n_iter_i = _check_optimize_result(\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/potatosalad/.local/share/virtualenvs/CSC_homework_3_group4-nat9sqnt/lib/python3.10/site-packages/sklearn/linear_model/_logistic.py:458: ConvergenceWarning: lbfgs failed to converge (status=1):\n",
      "STOP: TOTAL NO. of ITERATIONS REACHED LIMIT.\n",
      "\n",
      "Increase the number of iterations (max_iter) or scale the data as shown in:\n",
      "    https://scikit-learn.org/stable/modules/preprocessing.html\n",
      "Please also refer to the documentation for alternative solver options:\n",
      "    https://scikit-learn.org/stable/modules/linear_model.html#logistic-regression\n",
      "  n_iter_i = _check_optimize_result(\n",
      "/home/potatosalad/.local/share/virtualenvs/CSC_homework_3_group4-nat9sqnt/lib/python3.10/site-packages/sklearn/linear_model/_logistic.py:458: ConvergenceWarning: lbfgs failed to converge (status=1):\n",
      "STOP: TOTAL NO. of ITERATIONS REACHED LIMIT.\n",
      "\n",
      "Increase the number of iterations (max_iter) or scale the data as shown in:\n",
      "    https://scikit-learn.org/stable/modules/preprocessing.html\n",
      "Please also refer to the documentation for alternative solver options:\n",
      "    https://scikit-learn.org/stable/modules/linear_model.html#logistic-regression\n",
      "  n_iter_i = _check_optimize_result(\n",
      "/home/potatosalad/.local/share/virtualenvs/CSC_homework_3_group4-nat9sqnt/lib/python3.10/site-packages/sklearn/linear_model/_logistic.py:458: ConvergenceWarning: lbfgs failed to converge (status=1):\n",
      "STOP: TOTAL NO. of ITERATIONS REACHED LIMIT.\n",
      "\n",
      "Increase the number of iterations (max_iter) or scale the data as shown in:\n",
      "    https://scikit-learn.org/stable/modules/preprocessing.html\n",
      "Please also refer to the documentation for alternative solver options:\n",
      "    https://scikit-learn.org/stable/modules/linear_model.html#logistic-regression\n",
      "  n_iter_i = _check_optimize_result(\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Model performance for train set ENSEMBLE_heart_Data_no_gridsearch\n",
      "- Accuracy: 0.9907\n",
      "- F1 score: 0.990741\n",
      "- Precision: 0.990741\n",
      "- Recall: 0.990741\n",
      "----------------------------------\n",
      "===================================\n",
      "Model performance for test set ENSEMBLE_heart_Data_no_gridsearch\n",
      "- Accuracy: 0.8333\n",
      "- F1 score: 0.832545\n",
      "- Precision: 0.832353\n",
      "- Recall: 0.833333\n",
      "----------------------------------\n",
      "===================================\n"
     ]
    }
   ],
   "source": [
    "# NO GRIDSEARCH\n",
    "fit_models(X_train, y_train, X_test, y_test, data_heart_target_names, models, params, \"heart_Data_no_gridsearch\", \"experiment_heart_no_gridsearch\",0)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# LYMPHOGRAPHY DATASET"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 72,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "     class: normal find, metastases, malign lymph, fibrosis  \\\n",
      "0                                                    3        \n",
      "1                                                    2        \n",
      "2                                                    3        \n",
      "3                                                    3        \n",
      "4                                                    2        \n",
      "..                                                 ...        \n",
      "143                                                  3        \n",
      "144                                                  2        \n",
      "145                                                  3        \n",
      "146                                                  2        \n",
      "147                                                  2        \n",
      "\n",
      "     lymphatics: normal, arched, deformed, displaced  \\\n",
      "0                                                  4   \n",
      "1                                                  3   \n",
      "2                                                  3   \n",
      "3                                                  3   \n",
      "4                                                  3   \n",
      "..                                               ...   \n",
      "143                                                3   \n",
      "144                                                2   \n",
      "145                                                2   \n",
      "146                                                2   \n",
      "147                                                2   \n",
      "\n",
      "     block of affere: no, yes  bl. of lymph. c: no, yes  \\\n",
      "0                           2                         1   \n",
      "1                           2                         1   \n",
      "2                           2                         2   \n",
      "3                           1                         1   \n",
      "4                           1                         1   \n",
      "..                        ...                       ...   \n",
      "143                         2                         1   \n",
      "144                         1                         1   \n",
      "145                         2                         1   \n",
      "146                         1                         1   \n",
      "147                         2                         2   \n",
      "\n",
      "     bl. of lymph. s: no, yes  by pass: no, yes  extravasates: no, yes  \\\n",
      "0                           1                 1                      1   \n",
      "1                           1                 2                      2   \n",
      "2                           2                 2                      2   \n",
      "3                           1                 1                      2   \n",
      "4                           1                 1                      1   \n",
      "..                        ...               ...                    ...   \n",
      "143                         1                 2                      2   \n",
      "144                         1                 1                      1   \n",
      "145                         1                 1                      2   \n",
      "146                         1                 1                      1   \n",
      "147                         1                 2                      2   \n",
      "\n",
      "     regeneration of: no, yes  early uptake in: no, yes  lym.nodes dimin: 0-3  \\\n",
      "0                           1                         2                     1   \n",
      "1                           1                         2                     1   \n",
      "2                           2                         2                     1   \n",
      "3                           1                         2                     1   \n",
      "4                           1                         1                     1   \n",
      "..                        ...                       ...                   ...   \n",
      "143                         1                         2                     1   \n",
      "144                         1                         1                     1   \n",
      "145                         1                         2                     1   \n",
      "146                         1                         2                     1   \n",
      "147                         1                         2                     1   \n",
      "\n",
      "     lym.nodes enlar: 1-4  changes in lym.: bean, oval, round  \\\n",
      "0                       2                                   2   \n",
      "1                       3                                   3   \n",
      "2                       4                                   3   \n",
      "3                       3                                   3   \n",
      "4                       2                                   2   \n",
      "..                    ...                                 ...   \n",
      "143                     2                                   2   \n",
      "144                     1                                   1   \n",
      "145                     3                                   3   \n",
      "146                     2                                   2   \n",
      "147                     3                                   3   \n",
      "\n",
      "     defect in node: no, lacunar, lac. marginal, lac. central  \\\n",
      "0                                                    2          \n",
      "1                                                    2          \n",
      "2                                                    3          \n",
      "3                                                    4          \n",
      "4                                                    4          \n",
      "..                                                 ...          \n",
      "143                                                  4          \n",
      "144                                                  1          \n",
      "145                                                  3          \n",
      "146                                                  4          \n",
      "147                                                  4          \n",
      "\n",
      "     changes in node: no, lacunar, lac. margin, lac. central  \\\n",
      "0                                                    4         \n",
      "1                                                    3         \n",
      "2                                                    4         \n",
      "3                                                    4         \n",
      "4                                                    3         \n",
      "..                                                 ...         \n",
      "143                                                  3         \n",
      "144                                                  1         \n",
      "145                                                  3         \n",
      "146                                                  2         \n",
      "147                                                  3         \n",
      "\n",
      "     changes in stru: no, grainy, drop-like, coarse, diluted, reticular, stripped, faint  \\\n",
      "0                                                    8                                     \n",
      "1                                                    4                                     \n",
      "2                                                    8                                     \n",
      "3                                                    4                                     \n",
      "4                                                    5                                     \n",
      "..                                                 ...                                     \n",
      "143                                                  5                                     \n",
      "144                                                  3                                     \n",
      "145                                                  8                                     \n",
      "146                                                  2                                     \n",
      "147                                                  4                                     \n",
      "\n",
      "     special forms: no, chalices, vesicles  dislocation of: no, yes  \\\n",
      "0                                        1                        1   \n",
      "1                                        2                        2   \n",
      "2                                        3                        2   \n",
      "3                                        3                        1   \n",
      "4                                        1                        2   \n",
      "..                                     ...                      ...   \n",
      "143                                      2                        2   \n",
      "144                                      1                        2   \n",
      "145                                      3                        2   \n",
      "146                                      1                        2   \n",
      "147                                      3                        2   \n",
      "\n",
      "     exclusion of no: no, yes  \\\n",
      "0                           2   \n",
      "1                           2   \n",
      "2                           2   \n",
      "3                           2   \n",
      "4                           2   \n",
      "..                        ...   \n",
      "143                         2   \n",
      "144                         2   \n",
      "145                         2   \n",
      "146                         2   \n",
      "147                         2   \n",
      "\n",
      "     no. of nodes in: 0-9, 10-19, 20-29, 30-39, 40-49, 50-59, 60-69, >=70  \n",
      "0                                                    2                     \n",
      "1                                                    2                     \n",
      "2                                                    7                     \n",
      "3                                                    6                     \n",
      "4                                                    1                     \n",
      "..                                                 ...                     \n",
      "143                                                  4                     \n",
      "144                                                  1                     \n",
      "145                                                  4                     \n",
      "146                                                  1                     \n",
      "147                                                  6                     \n",
      "\n",
      "[148 rows x 19 columns]\n"
     ]
    }
   ],
   "source": [
    "data_lymphography=pd.read_csv('./data/lymphography.data',names=['class: normal find, metastases, malign lymph, fibrosis','lymphatics: normal, arched, deformed, displaced','block of affere: no, yes','bl. of lymph. c: no, yes','bl. of lymph. s: no, yes','by pass: no, yes','extravasates: no, yes', 'regeneration of: no, yes', 'early uptake in: no, yes', 'lym.nodes dimin: 0-3', 'lym.nodes enlar: 1-4', 'changes in lym.: bean, oval, round', 'defect in node: no, lacunar, lac. marginal, lac. central', 'changes in node: no, lacunar, lac. margin, lac. central', 'changes in stru: no, grainy, drop-like, coarse, diluted, reticular, stripped, faint', 'special forms: no, chalices, vesicles', 'dislocation of: no, yes', 'exclusion of no: no, yes', 'no. of nodes in: 0-9, 10-19, 20-29, 30-39, 40-49, 50-59, 60-69, >=70'])\n",
    "print(data_lymphography)\n",
    "data_lymphography_target_names = np.array(['normal find', 'metastases', 'malign lymph', 'fibrosis'])\n",
    "x=data_lymphography.drop(['class: normal find, metastases, malign lymph, fibrosis'], axis=1)\n",
    "y=data_lymphography['class: normal find, metastases, malign lymph, fibrosis']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 73,
   "metadata": {},
   "outputs": [],
   "source": [
    "X_train, X_test, y_train, y_test = train_test_split(x, y, test_size=0.20, random_state=42)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 74,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Fitting for parameters...\n",
      "X:      lymphatics: normal, arched, deformed, displaced  \\\n",
      "136                                                1   \n",
      "45                                                 2   \n",
      "119                                                4   \n",
      "27                                                 4   \n",
      "4                                                  3   \n",
      "..                                               ...   \n",
      "71                                                 3   \n",
      "106                                                3   \n",
      "14                                                 3   \n",
      "92                                                 3   \n",
      "102                                                2   \n",
      "\n",
      "     block of affere: no, yes  bl. of lymph. c: no, yes  \\\n",
      "136                         1                         1   \n",
      "45                          1                         1   \n",
      "119                         2                         1   \n",
      "27                          1                         1   \n",
      "4                           1                         1   \n",
      "..                        ...                       ...   \n",
      "71                          1                         1   \n",
      "106                         2                         1   \n",
      "14                          2                         2   \n",
      "92                          2                         2   \n",
      "102                         1                         1   \n",
      "\n",
      "     bl. of lymph. s: no, yes  by pass: no, yes  extravasates: no, yes  \\\n",
      "136                         1                 1                      2   \n",
      "45                          1                 1                      1   \n",
      "119                         1                 2                      2   \n",
      "27                          1                 1                      1   \n",
      "4                           1                 1                      1   \n",
      "..                        ...               ...                    ...   \n",
      "71                          1                 1                      1   \n",
      "106                         1                 1                      1   \n",
      "14                          2                 2                      2   \n",
      "92                          1                 2                      2   \n",
      "102                         1                 1                      1   \n",
      "\n",
      "     regeneration of: no, yes  early uptake in: no, yes  lym.nodes dimin: 0-3  \\\n",
      "136                         1                         2                     1   \n",
      "45                          1                         2                     1   \n",
      "119                         1                         2                     1   \n",
      "27                          1                         2                     1   \n",
      "4                           1                         1                     1   \n",
      "..                        ...                       ...                   ...   \n",
      "71                          1                         1                     1   \n",
      "106                         1                         2                     1   \n",
      "14                          2                         2                     3   \n",
      "92                          1                         2                     1   \n",
      "102                         1                         2                     1   \n",
      "\n",
      "     lym.nodes enlar: 1-4  changes in lym.: bean, oval, round  \\\n",
      "136                     2                                   2   \n",
      "45                      2                                   2   \n",
      "119                     3                                   3   \n",
      "27                      3                                   2   \n",
      "4                       2                                   2   \n",
      "..                    ...                                 ...   \n",
      "71                      2                                   2   \n",
      "106                     2                                   3   \n",
      "14                      1                                   1   \n",
      "92                      2                                   3   \n",
      "102                     2                                   2   \n",
      "\n",
      "     defect in node: no, lacunar, lac. marginal, lac. central  \\\n",
      "136                                                  1          \n",
      "45                                                   2          \n",
      "119                                                  4          \n",
      "27                                                   4          \n",
      "4                                                    4          \n",
      "..                                                 ...          \n",
      "71                                                   2          \n",
      "106                                                  3          \n",
      "14                                                   2          \n",
      "92                                                   3          \n",
      "102                                                  2          \n",
      "\n",
      "     changes in node: no, lacunar, lac. margin, lac. central  \\\n",
      "136                                                  1         \n",
      "45                                                   2         \n",
      "119                                                  2         \n",
      "27                                                   4         \n",
      "4                                                    3         \n",
      "..                                                 ...         \n",
      "71                                                   2         \n",
      "106                                                  3         \n",
      "14                                                   2         \n",
      "92                                                   4         \n",
      "102                                                  2         \n",
      "\n",
      "     changes in stru: no, grainy, drop-like, coarse, diluted, reticular, stripped, faint  \\\n",
      "136                                                  2                                     \n",
      "45                                                   8                                     \n",
      "119                                                  5                                     \n",
      "27                                                   8                                     \n",
      "4                                                    5                                     \n",
      "..                                                 ...                                     \n",
      "71                                                   5                                     \n",
      "106                                                  8                                     \n",
      "14                                                   8                                     \n",
      "92                                                   2                                     \n",
      "102                                                  4                                     \n",
      "\n",
      "     special forms: no, chalices, vesicles  dislocation of: no, yes  \\\n",
      "136                                      1                        1   \n",
      "45                                       2                        2   \n",
      "119                                      3                        2   \n",
      "27                                       3                        2   \n",
      "4                                        1                        2   \n",
      "..                                     ...                      ...   \n",
      "71                                       1                        1   \n",
      "106                                      2                        2   \n",
      "14                                       1                        2   \n",
      "92                                       2                        2   \n",
      "102                                      1                        2   \n",
      "\n",
      "     exclusion of no: no, yes  \\\n",
      "136                         1   \n",
      "45                          2   \n",
      "119                         2   \n",
      "27                          2   \n",
      "4                           2   \n",
      "..                        ...   \n",
      "71                          2   \n",
      "106                         2   \n",
      "14                          2   \n",
      "92                          1   \n",
      "102                         1   \n",
      "\n",
      "     no. of nodes in: 0-9, 10-19, 20-29, 30-39, 40-49, 50-59, 60-69, >=70  \n",
      "136                                                  2                     \n",
      "45                                                   1                     \n",
      "119                                                  3                     \n",
      "27                                                   1                     \n",
      "4                                                    1                     \n",
      "..                                                 ...                     \n",
      "71                                                   2                     \n",
      "106                                                  1                     \n",
      "14                                                   4                     \n",
      "92                                                   1                     \n",
      "102                                                  2                     \n",
      "\n",
      "[118 rows x 18 columns]\n",
      "--------------------------------------------------\n",
      "Y: 136    1\n",
      "45     3\n",
      "119    3\n",
      "27     3\n",
      "4      2\n",
      "      ..\n",
      "71     3\n",
      "106    2\n",
      "14     4\n",
      "92     2\n",
      "102    2\n",
      "Name: class: normal find, metastases, malign lymph, fibrosis, Length: 118, dtype: int64\n",
      "--------------------------------------------------\n",
      "X_t:      lymphatics: normal, arched, deformed, displaced  \\\n",
      "125                                                3   \n",
      "51                                                 3   \n",
      "139                                                3   \n",
      "19                                                 2   \n",
      "104                                                4   \n",
      "12                                                 4   \n",
      "76                                                 4   \n",
      "31                                                 2   \n",
      "81                                                 4   \n",
      "9                                                  2   \n",
      "26                                                 4   \n",
      "96                                                 3   \n",
      "144                                                2   \n",
      "67                                                 3   \n",
      "135                                                3   \n",
      "66                                                 2   \n",
      "18                                                 3   \n",
      "69                                                 2   \n",
      "124                                                4   \n",
      "30                                                 4   \n",
      "29                                                 3   \n",
      "105                                                2   \n",
      "36                                                 3   \n",
      "118                                                2   \n",
      "55                                                 4   \n",
      "22                                                 4   \n",
      "64                                                 2   \n",
      "131                                                4   \n",
      "82                                                 2   \n",
      "11                                                 4   \n",
      "\n",
      "     block of affere: no, yes  bl. of lymph. c: no, yes  \\\n",
      "125                         1                         1   \n",
      "51                          2                         1   \n",
      "139                         1                         1   \n",
      "19                          2                         1   \n",
      "104                         1                         1   \n",
      "12                          2                         1   \n",
      "76                          2                         2   \n",
      "31                          1                         1   \n",
      "81                          2                         1   \n",
      "9                           1                         1   \n",
      "26                          1                         1   \n",
      "96                          2                         1   \n",
      "144                         1                         1   \n",
      "67                          2                         2   \n",
      "135                         2                         2   \n",
      "66                          1                         1   \n",
      "18                          1                         1   \n",
      "69                          1                         1   \n",
      "124                         2                         1   \n",
      "30                          2                         1   \n",
      "29                          1                         1   \n",
      "105                         2                         1   \n",
      "36                          1                         1   \n",
      "118                         1                         1   \n",
      "55                          1                         1   \n",
      "22                          2                         1   \n",
      "64                          2                         1   \n",
      "131                         2                         2   \n",
      "82                          1                         1   \n",
      "11                          2                         1   \n",
      "\n",
      "     bl. of lymph. s: no, yes  by pass: no, yes  extravasates: no, yes  \\\n",
      "125                         1                 1                      2   \n",
      "51                          1                 1                      2   \n",
      "139                         1                 2                      2   \n",
      "19                          1                 1                      1   \n",
      "104                         1                 1                      1   \n",
      "12                          1                 1                      1   \n",
      "76                          1                 1                      1   \n",
      "31                          1                 1                      1   \n",
      "81                          1                 1                      1   \n",
      "9                           1                 1                      1   \n",
      "26                          1                 1                      2   \n",
      "96                          1                 1                      1   \n",
      "144                         1                 1                      1   \n",
      "67                          1                 2                      2   \n",
      "135                         2                 2                      2   \n",
      "66                          1                 1                      1   \n",
      "18                          1                 1                      1   \n",
      "69                          1                 1                      1   \n",
      "124                         1                 1                      2   \n",
      "30                          1                 1                      2   \n",
      "29                          1                 1                      1   \n",
      "105                         1                 1                      1   \n",
      "36                          1                 2                      2   \n",
      "118                         1                 1                      2   \n",
      "55                          1                 1                      1   \n",
      "22                          1                 2                      2   \n",
      "64                          1                 1                      1   \n",
      "131                         2                 2                      2   \n",
      "82                          1                 1                      1   \n",
      "11                          1                 1                      2   \n",
      "\n",
      "     regeneration of: no, yes  early uptake in: no, yes  lym.nodes dimin: 0-3  \\\n",
      "125                         1                         2                     1   \n",
      "51                          1                         2                     1   \n",
      "139                         2                         1                     3   \n",
      "19                          1                         2                     1   \n",
      "104                         1                         2                     1   \n",
      "12                          1                         2                     1   \n",
      "76                          1                         2                     1   \n",
      "31                          1                         2                     1   \n",
      "81                          1                         1                     1   \n",
      "9                           1                         2                     1   \n",
      "26                          1                         2                     1   \n",
      "96                          1                         1                     1   \n",
      "144                         1                         1                     1   \n",
      "67                          1                         1                     1   \n",
      "135                         1                         2                     1   \n",
      "66                          1                         2                     1   \n",
      "18                          1                         2                     1   \n",
      "69                          1                         2                     1   \n",
      "124                         1                         1                     1   \n",
      "30                          1                         2                     1   \n",
      "29                          1                         1                     1   \n",
      "105                         1                         2                     1   \n",
      "36                          2                         1                     3   \n",
      "118                         1                         2                     1   \n",
      "55                          1                         2                     1   \n",
      "22                          1                         2                     1   \n",
      "64                          1                         1                     1   \n",
      "131                         1                         2                     1   \n",
      "82                          1                         2                     1   \n",
      "11                          1                         1                     1   \n",
      "\n",
      "     lym.nodes enlar: 1-4  changes in lym.: bean, oval, round  \\\n",
      "125                     4                                   2   \n",
      "51                      2                                   2   \n",
      "139                     1                                   1   \n",
      "19                      3                                   3   \n",
      "104                     3                                   3   \n",
      "12                      2                                   2   \n",
      "76                      2                                   2   \n",
      "31                      3                                   2   \n",
      "81                      4                                   3   \n",
      "9                       2                                   2   \n",
      "26                      3                                   3   \n",
      "96                      2                                   2   \n",
      "144                     1                                   1   \n",
      "67                      3                                   3   \n",
      "135                     2                                   3   \n",
      "66                      4                                   3   \n",
      "18                      2                                   2   \n",
      "69                      3                                   2   \n",
      "124                     2                                   2   \n",
      "30                      4                                   3   \n",
      "29                      2                                   2   \n",
      "105                     4                                   3   \n",
      "36                      1                                   1   \n",
      "118                     3                                   2   \n",
      "55                      2                                   2   \n",
      "22                      4                                   3   \n",
      "64                      2                                   2   \n",
      "131                     4                                   3   \n",
      "82                      3                                   2   \n",
      "11                      3                                   3   \n",
      "\n",
      "     defect in node: no, lacunar, lac. marginal, lac. central  \\\n",
      "125                                                  2          \n",
      "51                                                   2          \n",
      "139                                                  2          \n",
      "19                                                   3          \n",
      "104                                                  2          \n",
      "12                                                   2          \n",
      "76                                                   2          \n",
      "31                                                   3          \n",
      "81                                                   3          \n",
      "9                                                    3          \n",
      "26                                                   4          \n",
      "96                                                   3          \n",
      "144                                                  1          \n",
      "67                                                   4          \n",
      "135                                                  3          \n",
      "66                                                   4          \n",
      "18                                                   2          \n",
      "69                                                   2          \n",
      "124                                                  3          \n",
      "30                                                   3          \n",
      "29                                                   4          \n",
      "105                                                  4          \n",
      "36                                                   4          \n",
      "118                                                  2          \n",
      "55                                                   3          \n",
      "22                                                   4          \n",
      "64                                                   3          \n",
      "131                                                  4          \n",
      "82                                                   2          \n",
      "11                                                   4          \n",
      "\n",
      "     changes in node: no, lacunar, lac. margin, lac. central  \\\n",
      "125                                                  2         \n",
      "51                                                   4         \n",
      "139                                                  1         \n",
      "19                                                   3         \n",
      "104                                                  2         \n",
      "12                                                   2         \n",
      "76                                                   3         \n",
      "31                                                   3         \n",
      "81                                                   3         \n",
      "9                                                    3         \n",
      "26                                                   2         \n",
      "96                                                   3         \n",
      "144                                                  1         \n",
      "67                                                   3         \n",
      "135                                                  3         \n",
      "66                                                   4         \n",
      "18                                                   3         \n",
      "69                                                   2         \n",
      "124                                                  3         \n",
      "30                                                   3         \n",
      "29                                                   2         \n",
      "105                                                  2         \n",
      "36                                                   2         \n",
      "118                                                  2         \n",
      "55                                                   3         \n",
      "22                                                   3         \n",
      "64                                                   3         \n",
      "131                                                  4         \n",
      "82                                                   2         \n",
      "11                                                   3         \n",
      "\n",
      "     changes in stru: no, grainy, drop-like, coarse, diluted, reticular, stripped, faint  \\\n",
      "125                                                  4                                     \n",
      "51                                                   8                                     \n",
      "139                                                  5                                     \n",
      "19                                                   2                                     \n",
      "104                                                  4                                     \n",
      "12                                                   3                                     \n",
      "76                                                   3                                     \n",
      "31                                                   8                                     \n",
      "81                                                   4                                     \n",
      "9                                                    5                                     \n",
      "26                                                   4                                     \n",
      "96                                                   3                                     \n",
      "144                                                  3                                     \n",
      "67                                                   4                                     \n",
      "135                                                  4                                     \n",
      "66                                                   5                                     \n",
      "18                                                   2                                     \n",
      "69                                                   6                                     \n",
      "124                                                  5                                     \n",
      "30                                                   7                                     \n",
      "29                                                   8                                     \n",
      "105                                                  7                                     \n",
      "36                                                   5                                     \n",
      "118                                                  4                                     \n",
      "55                                                   3                                     \n",
      "22                                                   4                                     \n",
      "64                                                   5                                     \n",
      "131                                                  7                                     \n",
      "82                                                   8                                     \n",
      "11                                                   8                                     \n",
      "\n",
      "     special forms: no, chalices, vesicles  dislocation of: no, yes  \\\n",
      "125                                      3                        2   \n",
      "51                                       3                        1   \n",
      "139                                      3                        1   \n",
      "19                                       2                        2   \n",
      "104                                      2                        2   \n",
      "12                                       2                        1   \n",
      "76                                       2                        2   \n",
      "31                                       3                        2   \n",
      "81                                       2                        2   \n",
      "9                                        3                        1   \n",
      "26                                       3                        2   \n",
      "96                                       3                        1   \n",
      "144                                      1                        2   \n",
      "67                                       2                        2   \n",
      "135                                      3                        2   \n",
      "66                                       3                        2   \n",
      "18                                       1                        2   \n",
      "69                                       3                        2   \n",
      "124                                      2                        1   \n",
      "30                                       3                        2   \n",
      "29                                       2                        2   \n",
      "105                                      3                        2   \n",
      "36                                       3                        1   \n",
      "118                                      3                        1   \n",
      "55                                       3                        2   \n",
      "22                                       2                        2   \n",
      "64                                       2                        2   \n",
      "131                                      3                        2   \n",
      "82                                       3                        2   \n",
      "11                                       3                        2   \n",
      "\n",
      "     exclusion of no: no, yes  \\\n",
      "125                         2   \n",
      "51                          2   \n",
      "139                         1   \n",
      "19                          1   \n",
      "104                         2   \n",
      "12                          1   \n",
      "76                          2   \n",
      "31                          2   \n",
      "81                          1   \n",
      "9                           1   \n",
      "26                          2   \n",
      "96                          1   \n",
      "144                         2   \n",
      "67                          2   \n",
      "135                         2   \n",
      "66                          2   \n",
      "18                          2   \n",
      "69                          2   \n",
      "124                         2   \n",
      "30                          2   \n",
      "29                          2   \n",
      "105                         2   \n",
      "36                          2   \n",
      "118                         2   \n",
      "55                          1   \n",
      "22                          2   \n",
      "64                          2   \n",
      "131                         2   \n",
      "82                          2   \n",
      "11                          2   \n",
      "\n",
      "     no. of nodes in: 0-9, 10-19, 20-29, 30-39, 40-49, 50-59, 60-69, >=70  \n",
      "125                                                  7                     \n",
      "51                                                   3                     \n",
      "139                                                  7                     \n",
      "19                                                   1                     \n",
      "104                                                  2                     \n",
      "12                                                   1                     \n",
      "76                                                   2                     \n",
      "31                                                   3                     \n",
      "81                                                   1                     \n",
      "9                                                    2                     \n",
      "26                                                   4                     \n",
      "96                                                   2                     \n",
      "144                                                  1                     \n",
      "67                                                   3                     \n",
      "135                                                  7                     \n",
      "66                                                   5                     \n",
      "18                                                   1                     \n",
      "69                                                   6                     \n",
      "124                                                  1                     \n",
      "30                                                   3                     \n",
      "29                                                   1                     \n",
      "105                                                  5                     \n",
      "36                                                   4                     \n",
      "118                                                  5                     \n",
      "55                                                   1                     \n",
      "22                                                   2                     \n",
      "64                                                   1                     \n",
      "131                                                  6                     \n",
      "82                                                   5                     \n",
      "11                                                   2                     \n",
      "--------------------------------------------------\n",
      "Y_t: 125    3\n",
      "51     3\n",
      "139    4\n",
      "19     2\n",
      "104    2\n",
      "12     2\n",
      "76     2\n",
      "31     3\n",
      "81     2\n",
      "9      3\n",
      "26     3\n",
      "96     2\n",
      "144    2\n",
      "67     2\n",
      "135    2\n",
      "66     3\n",
      "18     2\n",
      "69     3\n",
      "124    2\n",
      "30     3\n",
      "29     3\n",
      "105    3\n",
      "36     4\n",
      "118    3\n",
      "55     3\n",
      "22     2\n",
      "64     2\n",
      "131    3\n",
      "82     3\n",
      "11     2\n",
      "Name: class: normal find, metastases, malign lymph, fibrosis, dtype: int64\n",
      "--------------------------------------------------\n",
      "models: {'Logistic Regression': LogisticRegression(), 'Decision Tree': DecisionTreeClassifier(), 'Random Forest': RandomForestClassifier(random_state=42), 'Gradient Boosting': GradientBoostingClassifier(), 'K-Nearest Neighbors': KNeighborsClassifier(), 'SVM': SVC(probability=True)}\n",
      "--------------------------------------------------\n",
      "params: {'Logistic Regression': {'solver': ['newton-cholesky', 'liblinear', 'newton-cg', 'sag', 'saga'], 'penalty': ['l2'], 'C': [0.001, 0.01, 0.1, 1, 100], 'max_iter': [10000]}, 'Decision Tree': {'criterion': ['gini', 'entropy'], 'max_depth': [4, 6, 8, 10, 15, 50, 90]}, 'Random Forest': {'n_estimators': [200, 500], 'max_features': [None, 'sqrt', 'log2'], 'max_depth': [4, 5, 6, 7, 8], 'criterion': ['gini', 'entropy']}, 'Gradient Boosting': {'loss': ['log_loss'], 'learning_rate': [0.1, 0.01], 'max_depth': [3, 5, 8], 'max_features': ['log2', 'sqrt'], 'criterion': ['friedman_mse', 'squared_error'], 'subsample': [0.5, 0.8, 1.0], 'n_estimators': [10, 100, 1000]}, 'K-Nearest Neighbors': {'weights': ['uniform', 'distance'], 'leaf_size': [5, 15, 25]}, 'SVM': {'kernel': ['rbf'], 'C': [0.1, 1, 10, 100, 1000], 'degree': [1, 5], 'gamma': [1, 0.1, 0.01, 0.001]}}\n",
      "--------------------------------------------------\n",
      "run: lymphography_Data_gridsearch\n",
      "--------------------------------------------------\n",
      "folder: experiment_lymphography_gridsearch\n",
      "Logistic Regression\n",
      "\n",
      "##################################################\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/potatosalad/.local/share/virtualenvs/CSC_homework_3_group4-nat9sqnt/lib/python3.10/site-packages/sklearn/model_selection/_split.py:700: UserWarning: The least populated class in y has only 1 members, which is less than n_splits=5.\n",
      "  warnings.warn(\n",
      "/home/potatosalad/.local/share/virtualenvs/CSC_homework_3_group4-nat9sqnt/lib/python3.10/site-packages/sklearn/model_selection/_split.py:700: UserWarning: The least populated class in y has only 1 members, which is less than n_splits=5.\n",
      "  warnings.warn(\n",
      "/home/potatosalad/.local/share/virtualenvs/CSC_homework_3_group4-nat9sqnt/lib/python3.10/site-packages/sklearn/model_selection/_split.py:700: UserWarning: The least populated class in y has only 1 members, which is less than n_splits=5.\n",
      "  warnings.warn(\n",
      "/home/potatosalad/.local/share/virtualenvs/CSC_homework_3_group4-nat9sqnt/lib/python3.10/site-packages/sklearn/model_selection/_split.py:700: UserWarning: The least populated class in y has only 1 members, which is less than n_splits=5.\n",
      "  warnings.warn(\n",
      "/home/potatosalad/.local/share/virtualenvs/CSC_homework_3_group4-nat9sqnt/lib/python3.10/site-packages/sklearn/model_selection/_split.py:700: UserWarning: The least populated class in y has only 1 members, which is less than n_splits=5.\n",
      "  warnings.warn(\n",
      "/home/potatosalad/.local/share/virtualenvs/CSC_homework_3_group4-nat9sqnt/lib/python3.10/site-packages/sklearn/model_selection/_split.py:700: UserWarning: The least populated class in y has only 1 members, which is less than n_splits=5.\n",
      "  warnings.warn(\n",
      "/home/potatosalad/.local/share/virtualenvs/CSC_homework_3_group4-nat9sqnt/lib/python3.10/site-packages/sklearn/model_selection/_split.py:700: UserWarning: The least populated class in y has only 1 members, which is less than n_splits=5.\n",
      "  warnings.warn(\n",
      "/home/potatosalad/.local/share/virtualenvs/CSC_homework_3_group4-nat9sqnt/lib/python3.10/site-packages/sklearn/model_selection/_split.py:700: UserWarning: The least populated class in y has only 1 members, which is less than n_splits=5.\n",
      "  warnings.warn(\n",
      "/home/potatosalad/.local/share/virtualenvs/CSC_homework_3_group4-nat9sqnt/lib/python3.10/site-packages/sklearn/model_selection/_split.py:700: UserWarning: The least populated class in y has only 1 members, which is less than n_splits=5.\n",
      "  warnings.warn(\n",
      "/home/potatosalad/.local/share/virtualenvs/CSC_homework_3_group4-nat9sqnt/lib/python3.10/site-packages/sklearn/model_selection/_split.py:700: UserWarning: The least populated class in y has only 1 members, which is less than n_splits=5.\n",
      "  warnings.warn(\n",
      "/home/potatosalad/.local/share/virtualenvs/CSC_homework_3_group4-nat9sqnt/lib/python3.10/site-packages/sklearn/model_selection/_split.py:700: UserWarning: The least populated class in y has only 1 members, which is less than n_splits=5.\n",
      "  warnings.warn(\n",
      "/home/potatosalad/.local/share/virtualenvs/CSC_homework_3_group4-nat9sqnt/lib/python3.10/site-packages/sklearn/model_selection/_split.py:700: UserWarning: The least populated class in y has only 1 members, which is less than n_splits=5.\n",
      "  warnings.warn(\n",
      "/home/potatosalad/.local/share/virtualenvs/CSC_homework_3_group4-nat9sqnt/lib/python3.10/site-packages/sklearn/model_selection/_split.py:700: UserWarning: The least populated class in y has only 1 members, which is less than n_splits=5.\n",
      "  warnings.warn(\n",
      "/home/potatosalad/.local/share/virtualenvs/CSC_homework_3_group4-nat9sqnt/lib/python3.10/site-packages/sklearn/model_selection/_split.py:700: UserWarning: The least populated class in y has only 1 members, which is less than n_splits=5.\n",
      "  warnings.warn(\n",
      "/home/potatosalad/.local/share/virtualenvs/CSC_homework_3_group4-nat9sqnt/lib/python3.10/site-packages/sklearn/model_selection/_split.py:700: UserWarning: The least populated class in y has only 1 members, which is less than n_splits=5.\n",
      "  warnings.warn(\n",
      "/home/potatosalad/.local/share/virtualenvs/CSC_homework_3_group4-nat9sqnt/lib/python3.10/site-packages/sklearn/model_selection/_split.py:700: UserWarning: The least populated class in y has only 1 members, which is less than n_splits=5.\n",
      "  warnings.warn(\n",
      "/home/potatosalad/.local/share/virtualenvs/CSC_homework_3_group4-nat9sqnt/lib/python3.10/site-packages/sklearn/model_selection/_split.py:700: UserWarning: The least populated class in y has only 1 members, which is less than n_splits=5.\n",
      "  warnings.warn(\n",
      "/home/potatosalad/.local/share/virtualenvs/CSC_homework_3_group4-nat9sqnt/lib/python3.10/site-packages/sklearn/model_selection/_split.py:700: UserWarning: The least populated class in y has only 1 members, which is less than n_splits=5.\n",
      "  warnings.warn(\n",
      "/home/potatosalad/.local/share/virtualenvs/CSC_homework_3_group4-nat9sqnt/lib/python3.10/site-packages/sklearn/model_selection/_split.py:700: UserWarning: The least populated class in y has only 1 members, which is less than n_splits=5.\n",
      "  warnings.warn(\n",
      "/home/potatosalad/.local/share/virtualenvs/CSC_homework_3_group4-nat9sqnt/lib/python3.10/site-packages/sklearn/model_selection/_split.py:700: UserWarning: The least populated class in y has only 2 members, which is less than n_splits=5.\n",
      "  warnings.warn(\n",
      "/home/potatosalad/.local/share/virtualenvs/CSC_homework_3_group4-nat9sqnt/lib/python3.10/site-packages/sklearn/model_selection/_split.py:700: UserWarning: The least populated class in y has only 1 members, which is less than n_splits=5.\n",
      "  warnings.warn(\n",
      "/home/potatosalad/.local/share/virtualenvs/CSC_homework_3_group4-nat9sqnt/lib/python3.10/site-packages/sklearn/model_selection/_split.py:700: UserWarning: The least populated class in y has only 1 members, which is less than n_splits=5.\n",
      "  warnings.warn(\n",
      "/home/potatosalad/.local/share/virtualenvs/CSC_homework_3_group4-nat9sqnt/lib/python3.10/site-packages/sklearn/model_selection/_split.py:700: UserWarning: The least populated class in y has only 1 members, which is less than n_splits=5.\n",
      "  warnings.warn(\n",
      "/home/potatosalad/.local/share/virtualenvs/CSC_homework_3_group4-nat9sqnt/lib/python3.10/site-packages/sklearn/model_selection/_split.py:700: UserWarning: The least populated class in y has only 1 members, which is less than n_splits=5.\n",
      "  warnings.warn(\n",
      "/home/potatosalad/.local/share/virtualenvs/CSC_homework_3_group4-nat9sqnt/lib/python3.10/site-packages/sklearn/model_selection/_split.py:700: UserWarning: The least populated class in y has only 1 members, which is less than n_splits=5.\n",
      "  warnings.warn(\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Model performance for Training set Logistic Regression_lymphography_Data_gridsearch\n",
      "- Accuracy: 0.8729\n",
      "- F1 score: 0.863975\n",
      "- Precision: 0.863526\n",
      "- Recall: 0.872881\n",
      "----------------------------------\n",
      "===================================\n",
      "Model performance for Test set Logistic Regression_lymphography_Data_gridsearch\n",
      "- Accuracy: 0.9000\n",
      "- F1 score: 0.896169\n",
      "- Precision: 0.904444\n",
      "- Recall: 0.900000\n",
      "----------------------------------\n",
      "===================================\n",
      "Decision Tree\n",
      "\n",
      "##################################################\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/potatosalad/.local/share/virtualenvs/CSC_homework_3_group4-nat9sqnt/lib/python3.10/site-packages/sklearn/model_selection/_split.py:700: UserWarning: The least populated class in y has only 1 members, which is less than n_splits=5.\n",
      "  warnings.warn(\n",
      "/home/potatosalad/.local/share/virtualenvs/CSC_homework_3_group4-nat9sqnt/lib/python3.10/site-packages/sklearn/model_selection/_split.py:700: UserWarning: The least populated class in y has only 1 members, which is less than n_splits=5.\n",
      "  warnings.warn(\n",
      "/home/potatosalad/.local/share/virtualenvs/CSC_homework_3_group4-nat9sqnt/lib/python3.10/site-packages/sklearn/model_selection/_split.py:700: UserWarning: The least populated class in y has only 1 members, which is less than n_splits=5.\n",
      "  warnings.warn(\n",
      "/home/potatosalad/.local/share/virtualenvs/CSC_homework_3_group4-nat9sqnt/lib/python3.10/site-packages/sklearn/model_selection/_split.py:700: UserWarning: The least populated class in y has only 1 members, which is less than n_splits=5.\n",
      "  warnings.warn(\n",
      "/home/potatosalad/.local/share/virtualenvs/CSC_homework_3_group4-nat9sqnt/lib/python3.10/site-packages/sklearn/model_selection/_split.py:700: UserWarning: The least populated class in y has only 1 members, which is less than n_splits=5.\n",
      "  warnings.warn(\n",
      "/home/potatosalad/.local/share/virtualenvs/CSC_homework_3_group4-nat9sqnt/lib/python3.10/site-packages/sklearn/model_selection/_split.py:700: UserWarning: The least populated class in y has only 1 members, which is less than n_splits=5.\n",
      "  warnings.warn(\n",
      "/home/potatosalad/.local/share/virtualenvs/CSC_homework_3_group4-nat9sqnt/lib/python3.10/site-packages/sklearn/model_selection/_split.py:700: UserWarning: The least populated class in y has only 1 members, which is less than n_splits=5.\n",
      "  warnings.warn(\n",
      "/home/potatosalad/.local/share/virtualenvs/CSC_homework_3_group4-nat9sqnt/lib/python3.10/site-packages/sklearn/model_selection/_split.py:700: UserWarning: The least populated class in y has only 1 members, which is less than n_splits=5.\n",
      "  warnings.warn(\n",
      "/home/potatosalad/.local/share/virtualenvs/CSC_homework_3_group4-nat9sqnt/lib/python3.10/site-packages/sklearn/model_selection/_split.py:700: UserWarning: The least populated class in y has only 1 members, which is less than n_splits=5.\n",
      "  warnings.warn(\n",
      "/home/potatosalad/.local/share/virtualenvs/CSC_homework_3_group4-nat9sqnt/lib/python3.10/site-packages/sklearn/model_selection/_split.py:700: UserWarning: The least populated class in y has only 1 members, which is less than n_splits=5.\n",
      "  warnings.warn(\n",
      "/home/potatosalad/.local/share/virtualenvs/CSC_homework_3_group4-nat9sqnt/lib/python3.10/site-packages/sklearn/model_selection/_split.py:700: UserWarning: The least populated class in y has only 1 members, which is less than n_splits=5.\n",
      "  warnings.warn(\n",
      "/home/potatosalad/.local/share/virtualenvs/CSC_homework_3_group4-nat9sqnt/lib/python3.10/site-packages/sklearn/model_selection/_split.py:700: UserWarning: The least populated class in y has only 1 members, which is less than n_splits=5.\n",
      "  warnings.warn(\n",
      "/home/potatosalad/.local/share/virtualenvs/CSC_homework_3_group4-nat9sqnt/lib/python3.10/site-packages/sklearn/model_selection/_split.py:700: UserWarning: The least populated class in y has only 1 members, which is less than n_splits=5.\n",
      "  warnings.warn(\n",
      "/home/potatosalad/.local/share/virtualenvs/CSC_homework_3_group4-nat9sqnt/lib/python3.10/site-packages/sklearn/model_selection/_split.py:700: UserWarning: The least populated class in y has only 1 members, which is less than n_splits=5.\n",
      "  warnings.warn(\n",
      "/home/potatosalad/.local/share/virtualenvs/CSC_homework_3_group4-nat9sqnt/lib/python3.10/site-packages/sklearn/model_selection/_split.py:700: UserWarning: The least populated class in y has only 1 members, which is less than n_splits=5.\n",
      "  warnings.warn(\n",
      "/home/potatosalad/.local/share/virtualenvs/CSC_homework_3_group4-nat9sqnt/lib/python3.10/site-packages/sklearn/model_selection/_split.py:700: UserWarning: The least populated class in y has only 1 members, which is less than n_splits=5.\n",
      "  warnings.warn(\n",
      "/home/potatosalad/.local/share/virtualenvs/CSC_homework_3_group4-nat9sqnt/lib/python3.10/site-packages/sklearn/model_selection/_split.py:700: UserWarning: The least populated class in y has only 1 members, which is less than n_splits=5.\n",
      "  warnings.warn(\n",
      "/home/potatosalad/.local/share/virtualenvs/CSC_homework_3_group4-nat9sqnt/lib/python3.10/site-packages/sklearn/model_selection/_split.py:700: UserWarning: The least populated class in y has only 1 members, which is less than n_splits=5.\n",
      "  warnings.warn(\n",
      "/home/potatosalad/.local/share/virtualenvs/CSC_homework_3_group4-nat9sqnt/lib/python3.10/site-packages/sklearn/model_selection/_split.py:700: UserWarning: The least populated class in y has only 2 members, which is less than n_splits=5.\n",
      "  warnings.warn(\n",
      "/home/potatosalad/.local/share/virtualenvs/CSC_homework_3_group4-nat9sqnt/lib/python3.10/site-packages/sklearn/model_selection/_split.py:700: UserWarning: The least populated class in y has only 1 members, which is less than n_splits=5.\n",
      "  warnings.warn(\n",
      "/home/potatosalad/.local/share/virtualenvs/CSC_homework_3_group4-nat9sqnt/lib/python3.10/site-packages/sklearn/model_selection/_split.py:700: UserWarning: The least populated class in y has only 1 members, which is less than n_splits=5.\n",
      "  warnings.warn(\n",
      "/home/potatosalad/.local/share/virtualenvs/CSC_homework_3_group4-nat9sqnt/lib/python3.10/site-packages/sklearn/model_selection/_split.py:700: UserWarning: The least populated class in y has only 1 members, which is less than n_splits=5.\n",
      "  warnings.warn(\n",
      "/home/potatosalad/.local/share/virtualenvs/CSC_homework_3_group4-nat9sqnt/lib/python3.10/site-packages/sklearn/model_selection/_split.py:700: UserWarning: The least populated class in y has only 1 members, which is less than n_splits=5.\n",
      "  warnings.warn(\n",
      "/home/potatosalad/.local/share/virtualenvs/CSC_homework_3_group4-nat9sqnt/lib/python3.10/site-packages/sklearn/model_selection/_split.py:700: UserWarning: The least populated class in y has only 1 members, which is less than n_splits=5.\n",
      "  warnings.warn(\n",
      "/home/potatosalad/.local/share/virtualenvs/CSC_homework_3_group4-nat9sqnt/lib/python3.10/site-packages/sklearn/model_selection/_split.py:700: UserWarning: The least populated class in y has only 1 members, which is less than n_splits=5.\n",
      "  warnings.warn(\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Model performance for Training set Decision Tree_lymphography_Data_gridsearch\n",
      "- Accuracy: 1.0000\n",
      "- F1 score: 1.000000\n",
      "- Precision: 1.000000\n",
      "- Recall: 1.000000\n",
      "----------------------------------\n",
      "===================================\n",
      "Model performance for Test set Decision Tree_lymphography_Data_gridsearch\n",
      "- Accuracy: 0.7000\n",
      "- F1 score: 0.687901\n",
      "- Precision: 0.679808\n",
      "- Recall: 0.700000\n",
      "----------------------------------\n",
      "===================================\n",
      "Random Forest\n",
      "\n",
      "##################################################\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/potatosalad/.local/share/virtualenvs/CSC_homework_3_group4-nat9sqnt/lib/python3.10/site-packages/sklearn/model_selection/_split.py:700: UserWarning: The least populated class in y has only 1 members, which is less than n_splits=5.\n",
      "  warnings.warn(\n",
      "/home/potatosalad/.local/share/virtualenvs/CSC_homework_3_group4-nat9sqnt/lib/python3.10/site-packages/sklearn/model_selection/_split.py:700: UserWarning: The least populated class in y has only 1 members, which is less than n_splits=5.\n",
      "  warnings.warn(\n",
      "/home/potatosalad/.local/share/virtualenvs/CSC_homework_3_group4-nat9sqnt/lib/python3.10/site-packages/sklearn/model_selection/_split.py:700: UserWarning: The least populated class in y has only 1 members, which is less than n_splits=5.\n",
      "  warnings.warn(\n",
      "/home/potatosalad/.local/share/virtualenvs/CSC_homework_3_group4-nat9sqnt/lib/python3.10/site-packages/sklearn/model_selection/_split.py:700: UserWarning: The least populated class in y has only 1 members, which is less than n_splits=5.\n",
      "  warnings.warn(\n",
      "/home/potatosalad/.local/share/virtualenvs/CSC_homework_3_group4-nat9sqnt/lib/python3.10/site-packages/sklearn/model_selection/_split.py:700: UserWarning: The least populated class in y has only 1 members, which is less than n_splits=5.\n",
      "  warnings.warn(\n",
      "/home/potatosalad/.local/share/virtualenvs/CSC_homework_3_group4-nat9sqnt/lib/python3.10/site-packages/sklearn/model_selection/_split.py:700: UserWarning: The least populated class in y has only 1 members, which is less than n_splits=5.\n",
      "  warnings.warn(\n",
      "/home/potatosalad/.local/share/virtualenvs/CSC_homework_3_group4-nat9sqnt/lib/python3.10/site-packages/sklearn/model_selection/_split.py:700: UserWarning: The least populated class in y has only 1 members, which is less than n_splits=5.\n",
      "  warnings.warn(\n",
      "/home/potatosalad/.local/share/virtualenvs/CSC_homework_3_group4-nat9sqnt/lib/python3.10/site-packages/sklearn/model_selection/_split.py:700: UserWarning: The least populated class in y has only 1 members, which is less than n_splits=5.\n",
      "  warnings.warn(\n",
      "/home/potatosalad/.local/share/virtualenvs/CSC_homework_3_group4-nat9sqnt/lib/python3.10/site-packages/sklearn/model_selection/_split.py:700: UserWarning: The least populated class in y has only 1 members, which is less than n_splits=5.\n",
      "  warnings.warn(\n",
      "/home/potatosalad/.local/share/virtualenvs/CSC_homework_3_group4-nat9sqnt/lib/python3.10/site-packages/sklearn/model_selection/_split.py:700: UserWarning: The least populated class in y has only 1 members, which is less than n_splits=5.\n",
      "  warnings.warn(\n",
      "/home/potatosalad/.local/share/virtualenvs/CSC_homework_3_group4-nat9sqnt/lib/python3.10/site-packages/sklearn/model_selection/_split.py:700: UserWarning: The least populated class in y has only 1 members, which is less than n_splits=5.\n",
      "  warnings.warn(\n",
      "/home/potatosalad/.local/share/virtualenvs/CSC_homework_3_group4-nat9sqnt/lib/python3.10/site-packages/sklearn/model_selection/_split.py:700: UserWarning: The least populated class in y has only 1 members, which is less than n_splits=5.\n",
      "  warnings.warn(\n",
      "/home/potatosalad/.local/share/virtualenvs/CSC_homework_3_group4-nat9sqnt/lib/python3.10/site-packages/sklearn/model_selection/_split.py:700: UserWarning: The least populated class in y has only 1 members, which is less than n_splits=5.\n",
      "  warnings.warn(\n",
      "/home/potatosalad/.local/share/virtualenvs/CSC_homework_3_group4-nat9sqnt/lib/python3.10/site-packages/sklearn/model_selection/_split.py:700: UserWarning: The least populated class in y has only 1 members, which is less than n_splits=5.\n",
      "  warnings.warn(\n",
      "/home/potatosalad/.local/share/virtualenvs/CSC_homework_3_group4-nat9sqnt/lib/python3.10/site-packages/sklearn/model_selection/_split.py:700: UserWarning: The least populated class in y has only 1 members, which is less than n_splits=5.\n",
      "  warnings.warn(\n",
      "/home/potatosalad/.local/share/virtualenvs/CSC_homework_3_group4-nat9sqnt/lib/python3.10/site-packages/sklearn/model_selection/_split.py:700: UserWarning: The least populated class in y has only 1 members, which is less than n_splits=5.\n",
      "  warnings.warn(\n",
      "/home/potatosalad/.local/share/virtualenvs/CSC_homework_3_group4-nat9sqnt/lib/python3.10/site-packages/sklearn/model_selection/_split.py:700: UserWarning: The least populated class in y has only 1 members, which is less than n_splits=5.\n",
      "  warnings.warn(\n",
      "/home/potatosalad/.local/share/virtualenvs/CSC_homework_3_group4-nat9sqnt/lib/python3.10/site-packages/sklearn/model_selection/_split.py:700: UserWarning: The least populated class in y has only 2 members, which is less than n_splits=5.\n",
      "  warnings.warn(\n",
      "/home/potatosalad/.local/share/virtualenvs/CSC_homework_3_group4-nat9sqnt/lib/python3.10/site-packages/sklearn/model_selection/_split.py:700: UserWarning: The least populated class in y has only 1 members, which is less than n_splits=5.\n",
      "  warnings.warn(\n",
      "/home/potatosalad/.local/share/virtualenvs/CSC_homework_3_group4-nat9sqnt/lib/python3.10/site-packages/sklearn/model_selection/_split.py:700: UserWarning: The least populated class in y has only 1 members, which is less than n_splits=5.\n",
      "  warnings.warn(\n",
      "/home/potatosalad/.local/share/virtualenvs/CSC_homework_3_group4-nat9sqnt/lib/python3.10/site-packages/sklearn/model_selection/_split.py:700: UserWarning: The least populated class in y has only 1 members, which is less than n_splits=5.\n",
      "  warnings.warn(\n",
      "/home/potatosalad/.local/share/virtualenvs/CSC_homework_3_group4-nat9sqnt/lib/python3.10/site-packages/sklearn/model_selection/_split.py:700: UserWarning: The least populated class in y has only 1 members, which is less than n_splits=5.\n",
      "  warnings.warn(\n",
      "/home/potatosalad/.local/share/virtualenvs/CSC_homework_3_group4-nat9sqnt/lib/python3.10/site-packages/sklearn/model_selection/_split.py:700: UserWarning: The least populated class in y has only 1 members, which is less than n_splits=5.\n",
      "  warnings.warn(\n",
      "/home/potatosalad/.local/share/virtualenvs/CSC_homework_3_group4-nat9sqnt/lib/python3.10/site-packages/sklearn/model_selection/_split.py:700: UserWarning: The least populated class in y has only 1 members, which is less than n_splits=5.\n",
      "  warnings.warn(\n",
      "/home/potatosalad/.local/share/virtualenvs/CSC_homework_3_group4-nat9sqnt/lib/python3.10/site-packages/sklearn/model_selection/_split.py:700: UserWarning: The least populated class in y has only 1 members, which is less than n_splits=5.\n",
      "  warnings.warn(\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Model performance for Training set Random Forest_lymphography_Data_gridsearch\n",
      "- Accuracy: 1.0000\n",
      "- F1 score: 1.000000\n",
      "- Precision: 1.000000\n",
      "- Recall: 1.000000\n",
      "----------------------------------\n",
      "===================================\n",
      "Model performance for Test set Random Forest_lymphography_Data_gridsearch\n",
      "- Accuracy: 0.8333\n",
      "- F1 score: 0.804598\n",
      "- Precision: 0.777778\n",
      "- Recall: 0.833333\n",
      "----------------------------------\n",
      "===================================\n",
      "Gradient Boosting\n",
      "\n",
      "##################################################\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/potatosalad/.local/share/virtualenvs/CSC_homework_3_group4-nat9sqnt/lib/python3.10/site-packages/sklearn/model_selection/_split.py:700: UserWarning: The least populated class in y has only 1 members, which is less than n_splits=5.\n",
      "  warnings.warn(\n",
      "/home/potatosalad/.local/share/virtualenvs/CSC_homework_3_group4-nat9sqnt/lib/python3.10/site-packages/sklearn/model_selection/_split.py:700: UserWarning: The least populated class in y has only 1 members, which is less than n_splits=5.\n",
      "  warnings.warn(\n",
      "/home/potatosalad/.local/share/virtualenvs/CSC_homework_3_group4-nat9sqnt/lib/python3.10/site-packages/sklearn/model_selection/_split.py:700: UserWarning: The least populated class in y has only 1 members, which is less than n_splits=5.\n",
      "  warnings.warn(\n",
      "/home/potatosalad/.local/share/virtualenvs/CSC_homework_3_group4-nat9sqnt/lib/python3.10/site-packages/sklearn/model_selection/_split.py:700: UserWarning: The least populated class in y has only 1 members, which is less than n_splits=5.\n",
      "  warnings.warn(\n",
      "/home/potatosalad/.local/share/virtualenvs/CSC_homework_3_group4-nat9sqnt/lib/python3.10/site-packages/sklearn/model_selection/_split.py:700: UserWarning: The least populated class in y has only 1 members, which is less than n_splits=5.\n",
      "  warnings.warn(\n",
      "/home/potatosalad/.local/share/virtualenvs/CSC_homework_3_group4-nat9sqnt/lib/python3.10/site-packages/sklearn/model_selection/_split.py:700: UserWarning: The least populated class in y has only 1 members, which is less than n_splits=5.\n",
      "  warnings.warn(\n",
      "/home/potatosalad/.local/share/virtualenvs/CSC_homework_3_group4-nat9sqnt/lib/python3.10/site-packages/sklearn/model_selection/_split.py:700: UserWarning: The least populated class in y has only 1 members, which is less than n_splits=5.\n",
      "  warnings.warn(\n",
      "/home/potatosalad/.local/share/virtualenvs/CSC_homework_3_group4-nat9sqnt/lib/python3.10/site-packages/sklearn/model_selection/_split.py:700: UserWarning: The least populated class in y has only 1 members, which is less than n_splits=5.\n",
      "  warnings.warn(\n",
      "/home/potatosalad/.local/share/virtualenvs/CSC_homework_3_group4-nat9sqnt/lib/python3.10/site-packages/sklearn/model_selection/_split.py:700: UserWarning: The least populated class in y has only 1 members, which is less than n_splits=5.\n",
      "  warnings.warn(\n",
      "/home/potatosalad/.local/share/virtualenvs/CSC_homework_3_group4-nat9sqnt/lib/python3.10/site-packages/sklearn/model_selection/_split.py:700: UserWarning: The least populated class in y has only 1 members, which is less than n_splits=5.\n",
      "  warnings.warn(\n",
      "/home/potatosalad/.local/share/virtualenvs/CSC_homework_3_group4-nat9sqnt/lib/python3.10/site-packages/sklearn/model_selection/_split.py:700: UserWarning: The least populated class in y has only 1 members, which is less than n_splits=5.\n",
      "  warnings.warn(\n",
      "/home/potatosalad/.local/share/virtualenvs/CSC_homework_3_group4-nat9sqnt/lib/python3.10/site-packages/sklearn/model_selection/_split.py:700: UserWarning: The least populated class in y has only 1 members, which is less than n_splits=5.\n",
      "  warnings.warn(\n",
      "/home/potatosalad/.local/share/virtualenvs/CSC_homework_3_group4-nat9sqnt/lib/python3.10/site-packages/sklearn/model_selection/_split.py:700: UserWarning: The least populated class in y has only 1 members, which is less than n_splits=5.\n",
      "  warnings.warn(\n",
      "/home/potatosalad/.local/share/virtualenvs/CSC_homework_3_group4-nat9sqnt/lib/python3.10/site-packages/sklearn/model_selection/_split.py:700: UserWarning: The least populated class in y has only 1 members, which is less than n_splits=5.\n",
      "  warnings.warn(\n",
      "/home/potatosalad/.local/share/virtualenvs/CSC_homework_3_group4-nat9sqnt/lib/python3.10/site-packages/sklearn/model_selection/_split.py:700: UserWarning: The least populated class in y has only 1 members, which is less than n_splits=5.\n",
      "  warnings.warn(\n",
      "/home/potatosalad/.local/share/virtualenvs/CSC_homework_3_group4-nat9sqnt/lib/python3.10/site-packages/sklearn/model_selection/_split.py:700: UserWarning: The least populated class in y has only 1 members, which is less than n_splits=5.\n",
      "  warnings.warn(\n",
      "/home/potatosalad/.local/share/virtualenvs/CSC_homework_3_group4-nat9sqnt/lib/python3.10/site-packages/sklearn/model_selection/_split.py:700: UserWarning: The least populated class in y has only 1 members, which is less than n_splits=5.\n",
      "  warnings.warn(\n",
      "/home/potatosalad/.local/share/virtualenvs/CSC_homework_3_group4-nat9sqnt/lib/python3.10/site-packages/sklearn/model_selection/_split.py:700: UserWarning: The least populated class in y has only 1 members, which is less than n_splits=5.\n",
      "  warnings.warn(\n",
      "/home/potatosalad/.local/share/virtualenvs/CSC_homework_3_group4-nat9sqnt/lib/python3.10/site-packages/sklearn/model_selection/_split.py:700: UserWarning: The least populated class in y has only 1 members, which is less than n_splits=5.\n",
      "  warnings.warn(\n",
      "/home/potatosalad/.local/share/virtualenvs/CSC_homework_3_group4-nat9sqnt/lib/python3.10/site-packages/sklearn/model_selection/_split.py:700: UserWarning: The least populated class in y has only 2 members, which is less than n_splits=5.\n",
      "  warnings.warn(\n",
      "/home/potatosalad/.local/share/virtualenvs/CSC_homework_3_group4-nat9sqnt/lib/python3.10/site-packages/sklearn/model_selection/_split.py:700: UserWarning: The least populated class in y has only 1 members, which is less than n_splits=5.\n",
      "  warnings.warn(\n",
      "/home/potatosalad/.local/share/virtualenvs/CSC_homework_3_group4-nat9sqnt/lib/python3.10/site-packages/sklearn/model_selection/_split.py:700: UserWarning: The least populated class in y has only 1 members, which is less than n_splits=5.\n",
      "  warnings.warn(\n",
      "/home/potatosalad/.local/share/virtualenvs/CSC_homework_3_group4-nat9sqnt/lib/python3.10/site-packages/sklearn/model_selection/_split.py:700: UserWarning: The least populated class in y has only 1 members, which is less than n_splits=5.\n",
      "  warnings.warn(\n",
      "/home/potatosalad/.local/share/virtualenvs/CSC_homework_3_group4-nat9sqnt/lib/python3.10/site-packages/sklearn/model_selection/_split.py:700: UserWarning: The least populated class in y has only 1 members, which is less than n_splits=5.\n",
      "  warnings.warn(\n",
      "/home/potatosalad/.local/share/virtualenvs/CSC_homework_3_group4-nat9sqnt/lib/python3.10/site-packages/sklearn/model_selection/_split.py:700: UserWarning: The least populated class in y has only 1 members, which is less than n_splits=5.\n",
      "  warnings.warn(\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Model performance for Training set Gradient Boosting_lymphography_Data_gridsearch\n",
      "- Accuracy: 1.0000\n",
      "- F1 score: 1.000000\n",
      "- Precision: 1.000000\n",
      "- Recall: 1.000000\n",
      "----------------------------------\n",
      "===================================\n",
      "Model performance for Test set Gradient Boosting_lymphography_Data_gridsearch\n",
      "- Accuracy: 0.9000\n",
      "- F1 score: 0.899872\n",
      "- Precision: 0.901880\n",
      "- Recall: 0.900000\n",
      "----------------------------------\n",
      "===================================\n",
      "K-Nearest Neighbors\n",
      "\n",
      "##################################################\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/potatosalad/.local/share/virtualenvs/CSC_homework_3_group4-nat9sqnt/lib/python3.10/site-packages/sklearn/model_selection/_split.py:700: UserWarning: The least populated class in y has only 1 members, which is less than n_splits=5.\n",
      "  warnings.warn(\n",
      "/home/potatosalad/.local/share/virtualenvs/CSC_homework_3_group4-nat9sqnt/lib/python3.10/site-packages/sklearn/model_selection/_split.py:700: UserWarning: The least populated class in y has only 1 members, which is less than n_splits=5.\n",
      "  warnings.warn(\n",
      "/home/potatosalad/.local/share/virtualenvs/CSC_homework_3_group4-nat9sqnt/lib/python3.10/site-packages/sklearn/model_selection/_split.py:700: UserWarning: The least populated class in y has only 1 members, which is less than n_splits=5.\n",
      "  warnings.warn(\n",
      "/home/potatosalad/.local/share/virtualenvs/CSC_homework_3_group4-nat9sqnt/lib/python3.10/site-packages/sklearn/model_selection/_split.py:700: UserWarning: The least populated class in y has only 1 members, which is less than n_splits=5.\n",
      "  warnings.warn(\n",
      "/home/potatosalad/.local/share/virtualenvs/CSC_homework_3_group4-nat9sqnt/lib/python3.10/site-packages/sklearn/model_selection/_split.py:700: UserWarning: The least populated class in y has only 1 members, which is less than n_splits=5.\n",
      "  warnings.warn(\n",
      "/home/potatosalad/.local/share/virtualenvs/CSC_homework_3_group4-nat9sqnt/lib/python3.10/site-packages/sklearn/model_selection/_split.py:700: UserWarning: The least populated class in y has only 1 members, which is less than n_splits=5.\n",
      "  warnings.warn(\n",
      "/home/potatosalad/.local/share/virtualenvs/CSC_homework_3_group4-nat9sqnt/lib/python3.10/site-packages/sklearn/model_selection/_split.py:700: UserWarning: The least populated class in y has only 1 members, which is less than n_splits=5.\n",
      "  warnings.warn(\n",
      "/home/potatosalad/.local/share/virtualenvs/CSC_homework_3_group4-nat9sqnt/lib/python3.10/site-packages/sklearn/model_selection/_split.py:700: UserWarning: The least populated class in y has only 1 members, which is less than n_splits=5.\n",
      "  warnings.warn(\n",
      "/home/potatosalad/.local/share/virtualenvs/CSC_homework_3_group4-nat9sqnt/lib/python3.10/site-packages/sklearn/model_selection/_split.py:700: UserWarning: The least populated class in y has only 1 members, which is less than n_splits=5.\n",
      "  warnings.warn(\n",
      "/home/potatosalad/.local/share/virtualenvs/CSC_homework_3_group4-nat9sqnt/lib/python3.10/site-packages/sklearn/model_selection/_split.py:700: UserWarning: The least populated class in y has only 1 members, which is less than n_splits=5.\n",
      "  warnings.warn(\n",
      "/home/potatosalad/.local/share/virtualenvs/CSC_homework_3_group4-nat9sqnt/lib/python3.10/site-packages/sklearn/model_selection/_split.py:700: UserWarning: The least populated class in y has only 1 members, which is less than n_splits=5.\n",
      "  warnings.warn(\n",
      "/home/potatosalad/.local/share/virtualenvs/CSC_homework_3_group4-nat9sqnt/lib/python3.10/site-packages/sklearn/model_selection/_split.py:700: UserWarning: The least populated class in y has only 1 members, which is less than n_splits=5.\n",
      "  warnings.warn(\n",
      "/home/potatosalad/.local/share/virtualenvs/CSC_homework_3_group4-nat9sqnt/lib/python3.10/site-packages/sklearn/model_selection/_split.py:700: UserWarning: The least populated class in y has only 1 members, which is less than n_splits=5.\n",
      "  warnings.warn(\n",
      "/home/potatosalad/.local/share/virtualenvs/CSC_homework_3_group4-nat9sqnt/lib/python3.10/site-packages/sklearn/model_selection/_split.py:700: UserWarning: The least populated class in y has only 1 members, which is less than n_splits=5.\n",
      "  warnings.warn(\n",
      "/home/potatosalad/.local/share/virtualenvs/CSC_homework_3_group4-nat9sqnt/lib/python3.10/site-packages/sklearn/model_selection/_split.py:700: UserWarning: The least populated class in y has only 1 members, which is less than n_splits=5.\n",
      "  warnings.warn(\n",
      "/home/potatosalad/.local/share/virtualenvs/CSC_homework_3_group4-nat9sqnt/lib/python3.10/site-packages/sklearn/model_selection/_split.py:700: UserWarning: The least populated class in y has only 1 members, which is less than n_splits=5.\n",
      "  warnings.warn(\n",
      "/home/potatosalad/.local/share/virtualenvs/CSC_homework_3_group4-nat9sqnt/lib/python3.10/site-packages/sklearn/model_selection/_split.py:700: UserWarning: The least populated class in y has only 1 members, which is less than n_splits=5.\n",
      "  warnings.warn(\n",
      "/home/potatosalad/.local/share/virtualenvs/CSC_homework_3_group4-nat9sqnt/lib/python3.10/site-packages/sklearn/model_selection/_split.py:700: UserWarning: The least populated class in y has only 1 members, which is less than n_splits=5.\n",
      "  warnings.warn(\n",
      "/home/potatosalad/.local/share/virtualenvs/CSC_homework_3_group4-nat9sqnt/lib/python3.10/site-packages/sklearn/model_selection/_split.py:700: UserWarning: The least populated class in y has only 1 members, which is less than n_splits=5.\n",
      "  warnings.warn(\n",
      "/home/potatosalad/.local/share/virtualenvs/CSC_homework_3_group4-nat9sqnt/lib/python3.10/site-packages/sklearn/model_selection/_split.py:700: UserWarning: The least populated class in y has only 2 members, which is less than n_splits=5.\n",
      "  warnings.warn(\n",
      "/home/potatosalad/.local/share/virtualenvs/CSC_homework_3_group4-nat9sqnt/lib/python3.10/site-packages/sklearn/model_selection/_split.py:700: UserWarning: The least populated class in y has only 1 members, which is less than n_splits=5.\n",
      "  warnings.warn(\n",
      "/home/potatosalad/.local/share/virtualenvs/CSC_homework_3_group4-nat9sqnt/lib/python3.10/site-packages/sklearn/model_selection/_split.py:700: UserWarning: The least populated class in y has only 1 members, which is less than n_splits=5.\n",
      "  warnings.warn(\n",
      "/home/potatosalad/.local/share/virtualenvs/CSC_homework_3_group4-nat9sqnt/lib/python3.10/site-packages/sklearn/model_selection/_split.py:700: UserWarning: The least populated class in y has only 1 members, which is less than n_splits=5.\n",
      "  warnings.warn(\n",
      "/home/potatosalad/.local/share/virtualenvs/CSC_homework_3_group4-nat9sqnt/lib/python3.10/site-packages/sklearn/model_selection/_split.py:700: UserWarning: The least populated class in y has only 1 members, which is less than n_splits=5.\n",
      "  warnings.warn(\n",
      "/home/potatosalad/.local/share/virtualenvs/CSC_homework_3_group4-nat9sqnt/lib/python3.10/site-packages/sklearn/model_selection/_split.py:700: UserWarning: The least populated class in y has only 1 members, which is less than n_splits=5.\n",
      "  warnings.warn(\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Model performance for Training set K-Nearest Neighbors_lymphography_Data_gridsearch\n",
      "- Accuracy: 1.0000\n",
      "- F1 score: 1.000000\n",
      "- Precision: 1.000000\n",
      "- Recall: 1.000000\n",
      "----------------------------------\n",
      "===================================\n",
      "Model performance for Test set K-Nearest Neighbors_lymphography_Data_gridsearch\n",
      "- Accuracy: 0.8333\n",
      "- F1 score: 0.833120\n",
      "- Precision: 0.834872\n",
      "- Recall: 0.833333\n",
      "----------------------------------\n",
      "===================================\n",
      "SVM\n",
      "\n",
      "##################################################\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/potatosalad/.local/share/virtualenvs/CSC_homework_3_group4-nat9sqnt/lib/python3.10/site-packages/sklearn/model_selection/_split.py:700: UserWarning: The least populated class in y has only 1 members, which is less than n_splits=5.\n",
      "  warnings.warn(\n",
      "/home/potatosalad/.local/share/virtualenvs/CSC_homework_3_group4-nat9sqnt/lib/python3.10/site-packages/sklearn/model_selection/_split.py:700: UserWarning: The least populated class in y has only 1 members, which is less than n_splits=5.\n",
      "  warnings.warn(\n",
      "/home/potatosalad/.local/share/virtualenvs/CSC_homework_3_group4-nat9sqnt/lib/python3.10/site-packages/sklearn/model_selection/_split.py:700: UserWarning: The least populated class in y has only 1 members, which is less than n_splits=5.\n",
      "  warnings.warn(\n",
      "/home/potatosalad/.local/share/virtualenvs/CSC_homework_3_group4-nat9sqnt/lib/python3.10/site-packages/sklearn/model_selection/_split.py:700: UserWarning: The least populated class in y has only 1 members, which is less than n_splits=5.\n",
      "  warnings.warn(\n",
      "/home/potatosalad/.local/share/virtualenvs/CSC_homework_3_group4-nat9sqnt/lib/python3.10/site-packages/sklearn/model_selection/_split.py:700: UserWarning: The least populated class in y has only 1 members, which is less than n_splits=5.\n",
      "  warnings.warn(\n",
      "/home/potatosalad/.local/share/virtualenvs/CSC_homework_3_group4-nat9sqnt/lib/python3.10/site-packages/sklearn/model_selection/_split.py:700: UserWarning: The least populated class in y has only 1 members, which is less than n_splits=5.\n",
      "  warnings.warn(\n",
      "/home/potatosalad/.local/share/virtualenvs/CSC_homework_3_group4-nat9sqnt/lib/python3.10/site-packages/sklearn/model_selection/_split.py:700: UserWarning: The least populated class in y has only 1 members, which is less than n_splits=5.\n",
      "  warnings.warn(\n",
      "/home/potatosalad/.local/share/virtualenvs/CSC_homework_3_group4-nat9sqnt/lib/python3.10/site-packages/sklearn/model_selection/_split.py:700: UserWarning: The least populated class in y has only 1 members, which is less than n_splits=5.\n",
      "  warnings.warn(\n",
      "/home/potatosalad/.local/share/virtualenvs/CSC_homework_3_group4-nat9sqnt/lib/python3.10/site-packages/sklearn/model_selection/_split.py:700: UserWarning: The least populated class in y has only 1 members, which is less than n_splits=5.\n",
      "  warnings.warn(\n",
      "/home/potatosalad/.local/share/virtualenvs/CSC_homework_3_group4-nat9sqnt/lib/python3.10/site-packages/sklearn/model_selection/_split.py:700: UserWarning: The least populated class in y has only 1 members, which is less than n_splits=5.\n",
      "  warnings.warn(\n",
      "/home/potatosalad/.local/share/virtualenvs/CSC_homework_3_group4-nat9sqnt/lib/python3.10/site-packages/sklearn/model_selection/_split.py:700: UserWarning: The least populated class in y has only 1 members, which is less than n_splits=5.\n",
      "  warnings.warn(\n",
      "/home/potatosalad/.local/share/virtualenvs/CSC_homework_3_group4-nat9sqnt/lib/python3.10/site-packages/sklearn/model_selection/_split.py:700: UserWarning: The least populated class in y has only 1 members, which is less than n_splits=5.\n",
      "  warnings.warn(\n",
      "/home/potatosalad/.local/share/virtualenvs/CSC_homework_3_group4-nat9sqnt/lib/python3.10/site-packages/sklearn/model_selection/_split.py:700: UserWarning: The least populated class in y has only 1 members, which is less than n_splits=5.\n",
      "  warnings.warn(\n",
      "/home/potatosalad/.local/share/virtualenvs/CSC_homework_3_group4-nat9sqnt/lib/python3.10/site-packages/sklearn/model_selection/_split.py:700: UserWarning: The least populated class in y has only 1 members, which is less than n_splits=5.\n",
      "  warnings.warn(\n",
      "/home/potatosalad/.local/share/virtualenvs/CSC_homework_3_group4-nat9sqnt/lib/python3.10/site-packages/sklearn/model_selection/_split.py:700: UserWarning: The least populated class in y has only 1 members, which is less than n_splits=5.\n",
      "  warnings.warn(\n",
      "/home/potatosalad/.local/share/virtualenvs/CSC_homework_3_group4-nat9sqnt/lib/python3.10/site-packages/sklearn/model_selection/_split.py:700: UserWarning: The least populated class in y has only 1 members, which is less than n_splits=5.\n",
      "  warnings.warn(\n",
      "/home/potatosalad/.local/share/virtualenvs/CSC_homework_3_group4-nat9sqnt/lib/python3.10/site-packages/sklearn/model_selection/_split.py:700: UserWarning: The least populated class in y has only 1 members, which is less than n_splits=5.\n",
      "  warnings.warn(\n",
      "/home/potatosalad/.local/share/virtualenvs/CSC_homework_3_group4-nat9sqnt/lib/python3.10/site-packages/sklearn/model_selection/_split.py:700: UserWarning: The least populated class in y has only 1 members, which is less than n_splits=5.\n",
      "  warnings.warn(\n",
      "/home/potatosalad/.local/share/virtualenvs/CSC_homework_3_group4-nat9sqnt/lib/python3.10/site-packages/sklearn/model_selection/_split.py:700: UserWarning: The least populated class in y has only 1 members, which is less than n_splits=5.\n",
      "  warnings.warn(\n",
      "/home/potatosalad/.local/share/virtualenvs/CSC_homework_3_group4-nat9sqnt/lib/python3.10/site-packages/sklearn/model_selection/_split.py:700: UserWarning: The least populated class in y has only 2 members, which is less than n_splits=5.\n",
      "  warnings.warn(\n",
      "/home/potatosalad/.local/share/virtualenvs/CSC_homework_3_group4-nat9sqnt/lib/python3.10/site-packages/sklearn/model_selection/_split.py:700: UserWarning: The least populated class in y has only 1 members, which is less than n_splits=5.\n",
      "  warnings.warn(\n",
      "/home/potatosalad/.local/share/virtualenvs/CSC_homework_3_group4-nat9sqnt/lib/python3.10/site-packages/sklearn/model_selection/_split.py:700: UserWarning: The least populated class in y has only 1 members, which is less than n_splits=5.\n",
      "  warnings.warn(\n",
      "/home/potatosalad/.local/share/virtualenvs/CSC_homework_3_group4-nat9sqnt/lib/python3.10/site-packages/sklearn/model_selection/_split.py:700: UserWarning: The least populated class in y has only 1 members, which is less than n_splits=5.\n",
      "  warnings.warn(\n",
      "/home/potatosalad/.local/share/virtualenvs/CSC_homework_3_group4-nat9sqnt/lib/python3.10/site-packages/sklearn/model_selection/_split.py:700: UserWarning: The least populated class in y has only 1 members, which is less than n_splits=5.\n",
      "  warnings.warn(\n",
      "/home/potatosalad/.local/share/virtualenvs/CSC_homework_3_group4-nat9sqnt/lib/python3.10/site-packages/sklearn/model_selection/_split.py:700: UserWarning: The least populated class in y has only 1 members, which is less than n_splits=5.\n",
      "  warnings.warn(\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Model performance for Training set SVM_lymphography_Data_gridsearch\n",
      "- Accuracy: 1.0000\n",
      "- F1 score: 1.000000\n",
      "- Precision: 1.000000\n",
      "- Recall: 1.000000\n",
      "----------------------------------\n",
      "===================================\n",
      "Model performance for Test set SVM_lymphography_Data_gridsearch\n",
      "- Accuracy: 0.8333\n",
      "- F1 score: 0.833120\n",
      "- Precision: 0.834872\n",
      "- Recall: 0.833333\n",
      "----------------------------------\n",
      "===================================\n",
      "################################################\n",
      "################### ENSEMBLE ###################\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/potatosalad/.local/share/virtualenvs/CSC_homework_3_group4-nat9sqnt/lib/python3.10/site-packages/sklearn/linear_model/_logistic.py:458: ConvergenceWarning: lbfgs failed to converge (status=1):\n",
      "STOP: TOTAL NO. of ITERATIONS REACHED LIMIT.\n",
      "\n",
      "Increase the number of iterations (max_iter) or scale the data as shown in:\n",
      "    https://scikit-learn.org/stable/modules/preprocessing.html\n",
      "Please also refer to the documentation for alternative solver options:\n",
      "    https://scikit-learn.org/stable/modules/linear_model.html#logistic-regression\n",
      "  n_iter_i = _check_optimize_result(\n",
      "/home/potatosalad/.local/share/virtualenvs/CSC_homework_3_group4-nat9sqnt/lib/python3.10/site-packages/sklearn/linear_model/_logistic.py:458: ConvergenceWarning: lbfgs failed to converge (status=1):\n",
      "STOP: TOTAL NO. of ITERATIONS REACHED LIMIT.\n",
      "\n",
      "Increase the number of iterations (max_iter) or scale the data as shown in:\n",
      "    https://scikit-learn.org/stable/modules/preprocessing.html\n",
      "Please also refer to the documentation for alternative solver options:\n",
      "    https://scikit-learn.org/stable/modules/linear_model.html#logistic-regression\n",
      "  n_iter_i = _check_optimize_result(\n",
      "/home/potatosalad/.local/share/virtualenvs/CSC_homework_3_group4-nat9sqnt/lib/python3.10/site-packages/sklearn/linear_model/_logistic.py:458: ConvergenceWarning: lbfgs failed to converge (status=1):\n",
      "STOP: TOTAL NO. of ITERATIONS REACHED LIMIT.\n",
      "\n",
      "Increase the number of iterations (max_iter) or scale the data as shown in:\n",
      "    https://scikit-learn.org/stable/modules/preprocessing.html\n",
      "Please also refer to the documentation for alternative solver options:\n",
      "    https://scikit-learn.org/stable/modules/linear_model.html#logistic-regression\n",
      "  n_iter_i = _check_optimize_result(\n",
      "/home/potatosalad/.local/share/virtualenvs/CSC_homework_3_group4-nat9sqnt/lib/python3.10/site-packages/sklearn/linear_model/_logistic.py:458: ConvergenceWarning: lbfgs failed to converge (status=1):\n",
      "STOP: TOTAL NO. of ITERATIONS REACHED LIMIT.\n",
      "\n",
      "Increase the number of iterations (max_iter) or scale the data as shown in:\n",
      "    https://scikit-learn.org/stable/modules/preprocessing.html\n",
      "Please also refer to the documentation for alternative solver options:\n",
      "    https://scikit-learn.org/stable/modules/linear_model.html#logistic-regression\n",
      "  n_iter_i = _check_optimize_result(\n",
      "/home/potatosalad/.local/share/virtualenvs/CSC_homework_3_group4-nat9sqnt/lib/python3.10/site-packages/sklearn/linear_model/_logistic.py:458: ConvergenceWarning: lbfgs failed to converge (status=1):\n",
      "STOP: TOTAL NO. of ITERATIONS REACHED LIMIT.\n",
      "\n",
      "Increase the number of iterations (max_iter) or scale the data as shown in:\n",
      "    https://scikit-learn.org/stable/modules/preprocessing.html\n",
      "Please also refer to the documentation for alternative solver options:\n",
      "    https://scikit-learn.org/stable/modules/linear_model.html#logistic-regression\n",
      "  n_iter_i = _check_optimize_result(\n",
      "/home/potatosalad/.local/share/virtualenvs/CSC_homework_3_group4-nat9sqnt/lib/python3.10/site-packages/sklearn/linear_model/_logistic.py:458: ConvergenceWarning: lbfgs failed to converge (status=1):\n",
      "STOP: TOTAL NO. of ITERATIONS REACHED LIMIT.\n",
      "\n",
      "Increase the number of iterations (max_iter) or scale the data as shown in:\n",
      "    https://scikit-learn.org/stable/modules/preprocessing.html\n",
      "Please also refer to the documentation for alternative solver options:\n",
      "    https://scikit-learn.org/stable/modules/linear_model.html#logistic-regression\n",
      "  n_iter_i = _check_optimize_result(\n",
      "/home/potatosalad/.local/share/virtualenvs/CSC_homework_3_group4-nat9sqnt/lib/python3.10/site-packages/sklearn/linear_model/_logistic.py:458: ConvergenceWarning: lbfgs failed to converge (status=1):\n",
      "STOP: TOTAL NO. of ITERATIONS REACHED LIMIT.\n",
      "\n",
      "Increase the number of iterations (max_iter) or scale the data as shown in:\n",
      "    https://scikit-learn.org/stable/modules/preprocessing.html\n",
      "Please also refer to the documentation for alternative solver options:\n",
      "    https://scikit-learn.org/stable/modules/linear_model.html#logistic-regression\n",
      "  n_iter_i = _check_optimize_result(\n",
      "/home/potatosalad/.local/share/virtualenvs/CSC_homework_3_group4-nat9sqnt/lib/python3.10/site-packages/sklearn/linear_model/_logistic.py:458: ConvergenceWarning: lbfgs failed to converge (status=1):\n",
      "STOP: TOTAL NO. of ITERATIONS REACHED LIMIT.\n",
      "\n",
      "Increase the number of iterations (max_iter) or scale the data as shown in:\n",
      "    https://scikit-learn.org/stable/modules/preprocessing.html\n",
      "Please also refer to the documentation for alternative solver options:\n",
      "    https://scikit-learn.org/stable/modules/linear_model.html#logistic-regression\n",
      "  n_iter_i = _check_optimize_result(\n",
      "/home/potatosalad/.local/share/virtualenvs/CSC_homework_3_group4-nat9sqnt/lib/python3.10/site-packages/sklearn/linear_model/_logistic.py:458: ConvergenceWarning: lbfgs failed to converge (status=1):\n",
      "STOP: TOTAL NO. of ITERATIONS REACHED LIMIT.\n",
      "\n",
      "Increase the number of iterations (max_iter) or scale the data as shown in:\n",
      "    https://scikit-learn.org/stable/modules/preprocessing.html\n",
      "Please also refer to the documentation for alternative solver options:\n",
      "    https://scikit-learn.org/stable/modules/linear_model.html#logistic-regression\n",
      "  n_iter_i = _check_optimize_result(\n",
      "/home/potatosalad/.local/share/virtualenvs/CSC_homework_3_group4-nat9sqnt/lib/python3.10/site-packages/sklearn/linear_model/_logistic.py:458: ConvergenceWarning: lbfgs failed to converge (status=1):\n",
      "STOP: TOTAL NO. of ITERATIONS REACHED LIMIT.\n",
      "\n",
      "Increase the number of iterations (max_iter) or scale the data as shown in:\n",
      "    https://scikit-learn.org/stable/modules/preprocessing.html\n",
      "Please also refer to the documentation for alternative solver options:\n",
      "    https://scikit-learn.org/stable/modules/linear_model.html#logistic-regression\n",
      "  n_iter_i = _check_optimize_result(\n",
      "/home/potatosalad/.local/share/virtualenvs/CSC_homework_3_group4-nat9sqnt/lib/python3.10/site-packages/sklearn/linear_model/_logistic.py:458: ConvergenceWarning: lbfgs failed to converge (status=1):\n",
      "STOP: TOTAL NO. of ITERATIONS REACHED LIMIT.\n",
      "\n",
      "Increase the number of iterations (max_iter) or scale the data as shown in:\n",
      "    https://scikit-learn.org/stable/modules/preprocessing.html\n",
      "Please also refer to the documentation for alternative solver options:\n",
      "    https://scikit-learn.org/stable/modules/linear_model.html#logistic-regression\n",
      "  n_iter_i = _check_optimize_result(\n",
      "/home/potatosalad/.local/share/virtualenvs/CSC_homework_3_group4-nat9sqnt/lib/python3.10/site-packages/sklearn/linear_model/_logistic.py:458: ConvergenceWarning: lbfgs failed to converge (status=1):\n",
      "STOP: TOTAL NO. of ITERATIONS REACHED LIMIT.\n",
      "\n",
      "Increase the number of iterations (max_iter) or scale the data as shown in:\n",
      "    https://scikit-learn.org/stable/modules/preprocessing.html\n",
      "Please also refer to the documentation for alternative solver options:\n",
      "    https://scikit-learn.org/stable/modules/linear_model.html#logistic-regression\n",
      "  n_iter_i = _check_optimize_result(\n",
      "/home/potatosalad/.local/share/virtualenvs/CSC_homework_3_group4-nat9sqnt/lib/python3.10/site-packages/sklearn/linear_model/_logistic.py:458: ConvergenceWarning: lbfgs failed to converge (status=1):\n",
      "STOP: TOTAL NO. of ITERATIONS REACHED LIMIT.\n",
      "\n",
      "Increase the number of iterations (max_iter) or scale the data as shown in:\n",
      "    https://scikit-learn.org/stable/modules/preprocessing.html\n",
      "Please also refer to the documentation for alternative solver options:\n",
      "    https://scikit-learn.org/stable/modules/linear_model.html#logistic-regression\n",
      "  n_iter_i = _check_optimize_result(\n",
      "/home/potatosalad/.local/share/virtualenvs/CSC_homework_3_group4-nat9sqnt/lib/python3.10/site-packages/sklearn/linear_model/_logistic.py:458: ConvergenceWarning: lbfgs failed to converge (status=1):\n",
      "STOP: TOTAL NO. of ITERATIONS REACHED LIMIT.\n",
      "\n",
      "Increase the number of iterations (max_iter) or scale the data as shown in:\n",
      "    https://scikit-learn.org/stable/modules/preprocessing.html\n",
      "Please also refer to the documentation for alternative solver options:\n",
      "    https://scikit-learn.org/stable/modules/linear_model.html#logistic-regression\n",
      "  n_iter_i = _check_optimize_result(\n",
      "/home/potatosalad/.local/share/virtualenvs/CSC_homework_3_group4-nat9sqnt/lib/python3.10/site-packages/sklearn/linear_model/_logistic.py:458: ConvergenceWarning: lbfgs failed to converge (status=1):\n",
      "STOP: TOTAL NO. of ITERATIONS REACHED LIMIT.\n",
      "\n",
      "Increase the number of iterations (max_iter) or scale the data as shown in:\n",
      "    https://scikit-learn.org/stable/modules/preprocessing.html\n",
      "Please also refer to the documentation for alternative solver options:\n",
      "    https://scikit-learn.org/stable/modules/linear_model.html#logistic-regression\n",
      "  n_iter_i = _check_optimize_result(\n",
      "/home/potatosalad/.local/share/virtualenvs/CSC_homework_3_group4-nat9sqnt/lib/python3.10/site-packages/sklearn/linear_model/_logistic.py:458: ConvergenceWarning: lbfgs failed to converge (status=1):\n",
      "STOP: TOTAL NO. of ITERATIONS REACHED LIMIT.\n",
      "\n",
      "Increase the number of iterations (max_iter) or scale the data as shown in:\n",
      "    https://scikit-learn.org/stable/modules/preprocessing.html\n",
      "Please also refer to the documentation for alternative solver options:\n",
      "    https://scikit-learn.org/stable/modules/linear_model.html#logistic-regression\n",
      "  n_iter_i = _check_optimize_result(\n",
      "/home/potatosalad/.local/share/virtualenvs/CSC_homework_3_group4-nat9sqnt/lib/python3.10/site-packages/sklearn/linear_model/_logistic.py:458: ConvergenceWarning: lbfgs failed to converge (status=1):\n",
      "STOP: TOTAL NO. of ITERATIONS REACHED LIMIT.\n",
      "\n",
      "Increase the number of iterations (max_iter) or scale the data as shown in:\n",
      "    https://scikit-learn.org/stable/modules/preprocessing.html\n",
      "Please also refer to the documentation for alternative solver options:\n",
      "    https://scikit-learn.org/stable/modules/linear_model.html#logistic-regression\n",
      "  n_iter_i = _check_optimize_result(\n",
      "/home/potatosalad/.local/share/virtualenvs/CSC_homework_3_group4-nat9sqnt/lib/python3.10/site-packages/sklearn/linear_model/_logistic.py:458: ConvergenceWarning: lbfgs failed to converge (status=1):\n",
      "STOP: TOTAL NO. of ITERATIONS REACHED LIMIT.\n",
      "\n",
      "Increase the number of iterations (max_iter) or scale the data as shown in:\n",
      "    https://scikit-learn.org/stable/modules/preprocessing.html\n",
      "Please also refer to the documentation for alternative solver options:\n",
      "    https://scikit-learn.org/stable/modules/linear_model.html#logistic-regression\n",
      "  n_iter_i = _check_optimize_result(\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/potatosalad/.local/share/virtualenvs/CSC_homework_3_group4-nat9sqnt/lib/python3.10/site-packages/sklearn/linear_model/_logistic.py:458: ConvergenceWarning: lbfgs failed to converge (status=1):\n",
      "STOP: TOTAL NO. of ITERATIONS REACHED LIMIT.\n",
      "\n",
      "Increase the number of iterations (max_iter) or scale the data as shown in:\n",
      "    https://scikit-learn.org/stable/modules/preprocessing.html\n",
      "Please also refer to the documentation for alternative solver options:\n",
      "    https://scikit-learn.org/stable/modules/linear_model.html#logistic-regression\n",
      "  n_iter_i = _check_optimize_result(\n",
      "/home/potatosalad/.local/share/virtualenvs/CSC_homework_3_group4-nat9sqnt/lib/python3.10/site-packages/sklearn/linear_model/_logistic.py:458: ConvergenceWarning: lbfgs failed to converge (status=1):\n",
      "STOP: TOTAL NO. of ITERATIONS REACHED LIMIT.\n",
      "\n",
      "Increase the number of iterations (max_iter) or scale the data as shown in:\n",
      "    https://scikit-learn.org/stable/modules/preprocessing.html\n",
      "Please also refer to the documentation for alternative solver options:\n",
      "    https://scikit-learn.org/stable/modules/linear_model.html#logistic-regression\n",
      "  n_iter_i = _check_optimize_result(\n",
      "/home/potatosalad/.local/share/virtualenvs/CSC_homework_3_group4-nat9sqnt/lib/python3.10/site-packages/sklearn/linear_model/_logistic.py:458: ConvergenceWarning: lbfgs failed to converge (status=1):\n",
      "STOP: TOTAL NO. of ITERATIONS REACHED LIMIT.\n",
      "\n",
      "Increase the number of iterations (max_iter) or scale the data as shown in:\n",
      "    https://scikit-learn.org/stable/modules/preprocessing.html\n",
      "Please also refer to the documentation for alternative solver options:\n",
      "    https://scikit-learn.org/stable/modules/linear_model.html#logistic-regression\n",
      "  n_iter_i = _check_optimize_result(\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Model performance for train set ENSEMBLE_lymphography_Data_gridsearch\n",
      "- Accuracy: 1.0000\n",
      "- F1 score: 1.000000\n",
      "- Precision: 1.000000\n",
      "- Recall: 1.000000\n",
      "----------------------------------\n",
      "===================================\n",
      "Model performance for test set ENSEMBLE_lymphography_Data_gridsearch\n",
      "- Accuracy: 0.8667\n",
      "- F1 score: 0.862835\n",
      "- Precision: 0.871111\n",
      "- Recall: 0.866667\n",
      "----------------------------------\n",
      "===================================\n"
     ]
    }
   ],
   "source": [
    "# GRIDSEARCH\n",
    "fit_models(X_train, y_train, X_test, y_test,data_lymphography_target_names, models, params, \"lymphography_Data_gridsearch\", \"experiment_lymphography_gridsearch\", 1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 75,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Fitting for parameters...\n",
      "X:      lymphatics: normal, arched, deformed, displaced  \\\n",
      "136                                                1   \n",
      "45                                                 2   \n",
      "119                                                4   \n",
      "27                                                 4   \n",
      "4                                                  3   \n",
      "..                                               ...   \n",
      "71                                                 3   \n",
      "106                                                3   \n",
      "14                                                 3   \n",
      "92                                                 3   \n",
      "102                                                2   \n",
      "\n",
      "     block of affere: no, yes  bl. of lymph. c: no, yes  \\\n",
      "136                         1                         1   \n",
      "45                          1                         1   \n",
      "119                         2                         1   \n",
      "27                          1                         1   \n",
      "4                           1                         1   \n",
      "..                        ...                       ...   \n",
      "71                          1                         1   \n",
      "106                         2                         1   \n",
      "14                          2                         2   \n",
      "92                          2                         2   \n",
      "102                         1                         1   \n",
      "\n",
      "     bl. of lymph. s: no, yes  by pass: no, yes  extravasates: no, yes  \\\n",
      "136                         1                 1                      2   \n",
      "45                          1                 1                      1   \n",
      "119                         1                 2                      2   \n",
      "27                          1                 1                      1   \n",
      "4                           1                 1                      1   \n",
      "..                        ...               ...                    ...   \n",
      "71                          1                 1                      1   \n",
      "106                         1                 1                      1   \n",
      "14                          2                 2                      2   \n",
      "92                          1                 2                      2   \n",
      "102                         1                 1                      1   \n",
      "\n",
      "     regeneration of: no, yes  early uptake in: no, yes  lym.nodes dimin: 0-3  \\\n",
      "136                         1                         2                     1   \n",
      "45                          1                         2                     1   \n",
      "119                         1                         2                     1   \n",
      "27                          1                         2                     1   \n",
      "4                           1                         1                     1   \n",
      "..                        ...                       ...                   ...   \n",
      "71                          1                         1                     1   \n",
      "106                         1                         2                     1   \n",
      "14                          2                         2                     3   \n",
      "92                          1                         2                     1   \n",
      "102                         1                         2                     1   \n",
      "\n",
      "     lym.nodes enlar: 1-4  changes in lym.: bean, oval, round  \\\n",
      "136                     2                                   2   \n",
      "45                      2                                   2   \n",
      "119                     3                                   3   \n",
      "27                      3                                   2   \n",
      "4                       2                                   2   \n",
      "..                    ...                                 ...   \n",
      "71                      2                                   2   \n",
      "106                     2                                   3   \n",
      "14                      1                                   1   \n",
      "92                      2                                   3   \n",
      "102                     2                                   2   \n",
      "\n",
      "     defect in node: no, lacunar, lac. marginal, lac. central  \\\n",
      "136                                                  1          \n",
      "45                                                   2          \n",
      "119                                                  4          \n",
      "27                                                   4          \n",
      "4                                                    4          \n",
      "..                                                 ...          \n",
      "71                                                   2          \n",
      "106                                                  3          \n",
      "14                                                   2          \n",
      "92                                                   3          \n",
      "102                                                  2          \n",
      "\n",
      "     changes in node: no, lacunar, lac. margin, lac. central  \\\n",
      "136                                                  1         \n",
      "45                                                   2         \n",
      "119                                                  2         \n",
      "27                                                   4         \n",
      "4                                                    3         \n",
      "..                                                 ...         \n",
      "71                                                   2         \n",
      "106                                                  3         \n",
      "14                                                   2         \n",
      "92                                                   4         \n",
      "102                                                  2         \n",
      "\n",
      "     changes in stru: no, grainy, drop-like, coarse, diluted, reticular, stripped, faint  \\\n",
      "136                                                  2                                     \n",
      "45                                                   8                                     \n",
      "119                                                  5                                     \n",
      "27                                                   8                                     \n",
      "4                                                    5                                     \n",
      "..                                                 ...                                     \n",
      "71                                                   5                                     \n",
      "106                                                  8                                     \n",
      "14                                                   8                                     \n",
      "92                                                   2                                     \n",
      "102                                                  4                                     \n",
      "\n",
      "     special forms: no, chalices, vesicles  dislocation of: no, yes  \\\n",
      "136                                      1                        1   \n",
      "45                                       2                        2   \n",
      "119                                      3                        2   \n",
      "27                                       3                        2   \n",
      "4                                        1                        2   \n",
      "..                                     ...                      ...   \n",
      "71                                       1                        1   \n",
      "106                                      2                        2   \n",
      "14                                       1                        2   \n",
      "92                                       2                        2   \n",
      "102                                      1                        2   \n",
      "\n",
      "     exclusion of no: no, yes  \\\n",
      "136                         1   \n",
      "45                          2   \n",
      "119                         2   \n",
      "27                          2   \n",
      "4                           2   \n",
      "..                        ...   \n",
      "71                          2   \n",
      "106                         2   \n",
      "14                          2   \n",
      "92                          1   \n",
      "102                         1   \n",
      "\n",
      "     no. of nodes in: 0-9, 10-19, 20-29, 30-39, 40-49, 50-59, 60-69, >=70  \n",
      "136                                                  2                     \n",
      "45                                                   1                     \n",
      "119                                                  3                     \n",
      "27                                                   1                     \n",
      "4                                                    1                     \n",
      "..                                                 ...                     \n",
      "71                                                   2                     \n",
      "106                                                  1                     \n",
      "14                                                   4                     \n",
      "92                                                   1                     \n",
      "102                                                  2                     \n",
      "\n",
      "[118 rows x 18 columns]\n",
      "--------------------------------------------------\n",
      "Y: 136    1\n",
      "45     3\n",
      "119    3\n",
      "27     3\n",
      "4      2\n",
      "      ..\n",
      "71     3\n",
      "106    2\n",
      "14     4\n",
      "92     2\n",
      "102    2\n",
      "Name: class: normal find, metastases, malign lymph, fibrosis, Length: 118, dtype: int64\n",
      "--------------------------------------------------\n",
      "X_t:      lymphatics: normal, arched, deformed, displaced  \\\n",
      "125                                                3   \n",
      "51                                                 3   \n",
      "139                                                3   \n",
      "19                                                 2   \n",
      "104                                                4   \n",
      "12                                                 4   \n",
      "76                                                 4   \n",
      "31                                                 2   \n",
      "81                                                 4   \n",
      "9                                                  2   \n",
      "26                                                 4   \n",
      "96                                                 3   \n",
      "144                                                2   \n",
      "67                                                 3   \n",
      "135                                                3   \n",
      "66                                                 2   \n",
      "18                                                 3   \n",
      "69                                                 2   \n",
      "124                                                4   \n",
      "30                                                 4   \n",
      "29                                                 3   \n",
      "105                                                2   \n",
      "36                                                 3   \n",
      "118                                                2   \n",
      "55                                                 4   \n",
      "22                                                 4   \n",
      "64                                                 2   \n",
      "131                                                4   \n",
      "82                                                 2   \n",
      "11                                                 4   \n",
      "\n",
      "     block of affere: no, yes  bl. of lymph. c: no, yes  \\\n",
      "125                         1                         1   \n",
      "51                          2                         1   \n",
      "139                         1                         1   \n",
      "19                          2                         1   \n",
      "104                         1                         1   \n",
      "12                          2                         1   \n",
      "76                          2                         2   \n",
      "31                          1                         1   \n",
      "81                          2                         1   \n",
      "9                           1                         1   \n",
      "26                          1                         1   \n",
      "96                          2                         1   \n",
      "144                         1                         1   \n",
      "67                          2                         2   \n",
      "135                         2                         2   \n",
      "66                          1                         1   \n",
      "18                          1                         1   \n",
      "69                          1                         1   \n",
      "124                         2                         1   \n",
      "30                          2                         1   \n",
      "29                          1                         1   \n",
      "105                         2                         1   \n",
      "36                          1                         1   \n",
      "118                         1                         1   \n",
      "55                          1                         1   \n",
      "22                          2                         1   \n",
      "64                          2                         1   \n",
      "131                         2                         2   \n",
      "82                          1                         1   \n",
      "11                          2                         1   \n",
      "\n",
      "     bl. of lymph. s: no, yes  by pass: no, yes  extravasates: no, yes  \\\n",
      "125                         1                 1                      2   \n",
      "51                          1                 1                      2   \n",
      "139                         1                 2                      2   \n",
      "19                          1                 1                      1   \n",
      "104                         1                 1                      1   \n",
      "12                          1                 1                      1   \n",
      "76                          1                 1                      1   \n",
      "31                          1                 1                      1   \n",
      "81                          1                 1                      1   \n",
      "9                           1                 1                      1   \n",
      "26                          1                 1                      2   \n",
      "96                          1                 1                      1   \n",
      "144                         1                 1                      1   \n",
      "67                          1                 2                      2   \n",
      "135                         2                 2                      2   \n",
      "66                          1                 1                      1   \n",
      "18                          1                 1                      1   \n",
      "69                          1                 1                      1   \n",
      "124                         1                 1                      2   \n",
      "30                          1                 1                      2   \n",
      "29                          1                 1                      1   \n",
      "105                         1                 1                      1   \n",
      "36                          1                 2                      2   \n",
      "118                         1                 1                      2   \n",
      "55                          1                 1                      1   \n",
      "22                          1                 2                      2   \n",
      "64                          1                 1                      1   \n",
      "131                         2                 2                      2   \n",
      "82                          1                 1                      1   \n",
      "11                          1                 1                      2   \n",
      "\n",
      "     regeneration of: no, yes  early uptake in: no, yes  lym.nodes dimin: 0-3  \\\n",
      "125                         1                         2                     1   \n",
      "51                          1                         2                     1   \n",
      "139                         2                         1                     3   \n",
      "19                          1                         2                     1   \n",
      "104                         1                         2                     1   \n",
      "12                          1                         2                     1   \n",
      "76                          1                         2                     1   \n",
      "31                          1                         2                     1   \n",
      "81                          1                         1                     1   \n",
      "9                           1                         2                     1   \n",
      "26                          1                         2                     1   \n",
      "96                          1                         1                     1   \n",
      "144                         1                         1                     1   \n",
      "67                          1                         1                     1   \n",
      "135                         1                         2                     1   \n",
      "66                          1                         2                     1   \n",
      "18                          1                         2                     1   \n",
      "69                          1                         2                     1   \n",
      "124                         1                         1                     1   \n",
      "30                          1                         2                     1   \n",
      "29                          1                         1                     1   \n",
      "105                         1                         2                     1   \n",
      "36                          2                         1                     3   \n",
      "118                         1                         2                     1   \n",
      "55                          1                         2                     1   \n",
      "22                          1                         2                     1   \n",
      "64                          1                         1                     1   \n",
      "131                         1                         2                     1   \n",
      "82                          1                         2                     1   \n",
      "11                          1                         1                     1   \n",
      "\n",
      "     lym.nodes enlar: 1-4  changes in lym.: bean, oval, round  \\\n",
      "125                     4                                   2   \n",
      "51                      2                                   2   \n",
      "139                     1                                   1   \n",
      "19                      3                                   3   \n",
      "104                     3                                   3   \n",
      "12                      2                                   2   \n",
      "76                      2                                   2   \n",
      "31                      3                                   2   \n",
      "81                      4                                   3   \n",
      "9                       2                                   2   \n",
      "26                      3                                   3   \n",
      "96                      2                                   2   \n",
      "144                     1                                   1   \n",
      "67                      3                                   3   \n",
      "135                     2                                   3   \n",
      "66                      4                                   3   \n",
      "18                      2                                   2   \n",
      "69                      3                                   2   \n",
      "124                     2                                   2   \n",
      "30                      4                                   3   \n",
      "29                      2                                   2   \n",
      "105                     4                                   3   \n",
      "36                      1                                   1   \n",
      "118                     3                                   2   \n",
      "55                      2                                   2   \n",
      "22                      4                                   3   \n",
      "64                      2                                   2   \n",
      "131                     4                                   3   \n",
      "82                      3                                   2   \n",
      "11                      3                                   3   \n",
      "\n",
      "     defect in node: no, lacunar, lac. marginal, lac. central  \\\n",
      "125                                                  2          \n",
      "51                                                   2          \n",
      "139                                                  2          \n",
      "19                                                   3          \n",
      "104                                                  2          \n",
      "12                                                   2          \n",
      "76                                                   2          \n",
      "31                                                   3          \n",
      "81                                                   3          \n",
      "9                                                    3          \n",
      "26                                                   4          \n",
      "96                                                   3          \n",
      "144                                                  1          \n",
      "67                                                   4          \n",
      "135                                                  3          \n",
      "66                                                   4          \n",
      "18                                                   2          \n",
      "69                                                   2          \n",
      "124                                                  3          \n",
      "30                                                   3          \n",
      "29                                                   4          \n",
      "105                                                  4          \n",
      "36                                                   4          \n",
      "118                                                  2          \n",
      "55                                                   3          \n",
      "22                                                   4          \n",
      "64                                                   3          \n",
      "131                                                  4          \n",
      "82                                                   2          \n",
      "11                                                   4          \n",
      "\n",
      "     changes in node: no, lacunar, lac. margin, lac. central  \\\n",
      "125                                                  2         \n",
      "51                                                   4         \n",
      "139                                                  1         \n",
      "19                                                   3         \n",
      "104                                                  2         \n",
      "12                                                   2         \n",
      "76                                                   3         \n",
      "31                                                   3         \n",
      "81                                                   3         \n",
      "9                                                    3         \n",
      "26                                                   2         \n",
      "96                                                   3         \n",
      "144                                                  1         \n",
      "67                                                   3         \n",
      "135                                                  3         \n",
      "66                                                   4         \n",
      "18                                                   3         \n",
      "69                                                   2         \n",
      "124                                                  3         \n",
      "30                                                   3         \n",
      "29                                                   2         \n",
      "105                                                  2         \n",
      "36                                                   2         \n",
      "118                                                  2         \n",
      "55                                                   3         \n",
      "22                                                   3         \n",
      "64                                                   3         \n",
      "131                                                  4         \n",
      "82                                                   2         \n",
      "11                                                   3         \n",
      "\n",
      "     changes in stru: no, grainy, drop-like, coarse, diluted, reticular, stripped, faint  \\\n",
      "125                                                  4                                     \n",
      "51                                                   8                                     \n",
      "139                                                  5                                     \n",
      "19                                                   2                                     \n",
      "104                                                  4                                     \n",
      "12                                                   3                                     \n",
      "76                                                   3                                     \n",
      "31                                                   8                                     \n",
      "81                                                   4                                     \n",
      "9                                                    5                                     \n",
      "26                                                   4                                     \n",
      "96                                                   3                                     \n",
      "144                                                  3                                     \n",
      "67                                                   4                                     \n",
      "135                                                  4                                     \n",
      "66                                                   5                                     \n",
      "18                                                   2                                     \n",
      "69                                                   6                                     \n",
      "124                                                  5                                     \n",
      "30                                                   7                                     \n",
      "29                                                   8                                     \n",
      "105                                                  7                                     \n",
      "36                                                   5                                     \n",
      "118                                                  4                                     \n",
      "55                                                   3                                     \n",
      "22                                                   4                                     \n",
      "64                                                   5                                     \n",
      "131                                                  7                                     \n",
      "82                                                   8                                     \n",
      "11                                                   8                                     \n",
      "\n",
      "     special forms: no, chalices, vesicles  dislocation of: no, yes  \\\n",
      "125                                      3                        2   \n",
      "51                                       3                        1   \n",
      "139                                      3                        1   \n",
      "19                                       2                        2   \n",
      "104                                      2                        2   \n",
      "12                                       2                        1   \n",
      "76                                       2                        2   \n",
      "31                                       3                        2   \n",
      "81                                       2                        2   \n",
      "9                                        3                        1   \n",
      "26                                       3                        2   \n",
      "96                                       3                        1   \n",
      "144                                      1                        2   \n",
      "67                                       2                        2   \n",
      "135                                      3                        2   \n",
      "66                                       3                        2   \n",
      "18                                       1                        2   \n",
      "69                                       3                        2   \n",
      "124                                      2                        1   \n",
      "30                                       3                        2   \n",
      "29                                       2                        2   \n",
      "105                                      3                        2   \n",
      "36                                       3                        1   \n",
      "118                                      3                        1   \n",
      "55                                       3                        2   \n",
      "22                                       2                        2   \n",
      "64                                       2                        2   \n",
      "131                                      3                        2   \n",
      "82                                       3                        2   \n",
      "11                                       3                        2   \n",
      "\n",
      "     exclusion of no: no, yes  \\\n",
      "125                         2   \n",
      "51                          2   \n",
      "139                         1   \n",
      "19                          1   \n",
      "104                         2   \n",
      "12                          1   \n",
      "76                          2   \n",
      "31                          2   \n",
      "81                          1   \n",
      "9                           1   \n",
      "26                          2   \n",
      "96                          1   \n",
      "144                         2   \n",
      "67                          2   \n",
      "135                         2   \n",
      "66                          2   \n",
      "18                          2   \n",
      "69                          2   \n",
      "124                         2   \n",
      "30                          2   \n",
      "29                          2   \n",
      "105                         2   \n",
      "36                          2   \n",
      "118                         2   \n",
      "55                          1   \n",
      "22                          2   \n",
      "64                          2   \n",
      "131                         2   \n",
      "82                          2   \n",
      "11                          2   \n",
      "\n",
      "     no. of nodes in: 0-9, 10-19, 20-29, 30-39, 40-49, 50-59, 60-69, >=70  \n",
      "125                                                  7                     \n",
      "51                                                   3                     \n",
      "139                                                  7                     \n",
      "19                                                   1                     \n",
      "104                                                  2                     \n",
      "12                                                   1                     \n",
      "76                                                   2                     \n",
      "31                                                   3                     \n",
      "81                                                   1                     \n",
      "9                                                    2                     \n",
      "26                                                   4                     \n",
      "96                                                   2                     \n",
      "144                                                  1                     \n",
      "67                                                   3                     \n",
      "135                                                  7                     \n",
      "66                                                   5                     \n",
      "18                                                   1                     \n",
      "69                                                   6                     \n",
      "124                                                  1                     \n",
      "30                                                   3                     \n",
      "29                                                   1                     \n",
      "105                                                  5                     \n",
      "36                                                   4                     \n",
      "118                                                  5                     \n",
      "55                                                   1                     \n",
      "22                                                   2                     \n",
      "64                                                   1                     \n",
      "131                                                  6                     \n",
      "82                                                   5                     \n",
      "11                                                   2                     \n",
      "--------------------------------------------------\n",
      "Y_t: 125    3\n",
      "51     3\n",
      "139    4\n",
      "19     2\n",
      "104    2\n",
      "12     2\n",
      "76     2\n",
      "31     3\n",
      "81     2\n",
      "9      3\n",
      "26     3\n",
      "96     2\n",
      "144    2\n",
      "67     2\n",
      "135    2\n",
      "66     3\n",
      "18     2\n",
      "69     3\n",
      "124    2\n",
      "30     3\n",
      "29     3\n",
      "105    3\n",
      "36     4\n",
      "118    3\n",
      "55     3\n",
      "22     2\n",
      "64     2\n",
      "131    3\n",
      "82     3\n",
      "11     2\n",
      "Name: class: normal find, metastases, malign lymph, fibrosis, dtype: int64\n",
      "--------------------------------------------------\n",
      "models: {'Logistic Regression': LogisticRegression(), 'Decision Tree': DecisionTreeClassifier(), 'Random Forest': RandomForestClassifier(random_state=42), 'Gradient Boosting': GradientBoostingClassifier(), 'K-Nearest Neighbors': KNeighborsClassifier(), 'SVM': SVC(probability=True)}\n",
      "--------------------------------------------------\n",
      "params: {'Logistic Regression': {'solver': ['newton-cholesky', 'liblinear', 'newton-cg', 'sag', 'saga'], 'penalty': ['l2'], 'C': [0.001, 0.01, 0.1, 1, 100], 'max_iter': [10000]}, 'Decision Tree': {'criterion': ['gini', 'entropy'], 'max_depth': [4, 6, 8, 10, 15, 50, 90]}, 'Random Forest': {'n_estimators': [200, 500], 'max_features': [None, 'sqrt', 'log2'], 'max_depth': [4, 5, 6, 7, 8], 'criterion': ['gini', 'entropy']}, 'Gradient Boosting': {'loss': ['log_loss'], 'learning_rate': [0.1, 0.01], 'max_depth': [3, 5, 8], 'max_features': ['log2', 'sqrt'], 'criterion': ['friedman_mse', 'squared_error'], 'subsample': [0.5, 0.8, 1.0], 'n_estimators': [10, 100, 1000]}, 'K-Nearest Neighbors': {'weights': ['uniform', 'distance'], 'leaf_size': [5, 15, 25]}, 'SVM': {'kernel': ['rbf'], 'C': [0.1, 1, 10, 100, 1000], 'degree': [1, 5], 'gamma': [1, 0.1, 0.01, 0.001]}}\n",
      "--------------------------------------------------\n",
      "run: lymphography_Data_no_gridsearch\n",
      "--------------------------------------------------\n",
      "folder: experiment_lymphography_no_gridsearch\n",
      "Logistic Regression\n",
      "\n",
      "##################################################\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Model performance for Training set Logistic Regression_lymphography_Data_no_gridsearch\n",
      "- Accuracy: 0.8898\n",
      "- F1 score: 0.889221\n",
      "- Precision: 0.889847\n",
      "- Recall: 0.889831\n",
      "----------------------------------\n",
      "===================================\n",
      "Model performance for Test set Logistic Regression_lymphography_Data_no_gridsearch\n",
      "- Accuracy: 0.8333\n",
      "- F1 score: 0.833120\n",
      "- Precision: 0.834872\n",
      "- Recall: 0.833333\n",
      "----------------------------------\n",
      "===================================\n",
      "Decision Tree\n",
      "\n",
      "##################################################\n",
      "Model performance for Training set Decision Tree_lymphography_Data_no_gridsearch\n",
      "- Accuracy: 1.0000\n",
      "- F1 score: 1.000000\n",
      "- Precision: 1.000000\n",
      "- Recall: 1.000000\n",
      "----------------------------------\n",
      "===================================\n",
      "Model performance for Test set Decision Tree_lymphography_Data_no_gridsearch\n",
      "- Accuracy: 0.6667\n",
      "- F1 score: 0.654260\n",
      "- Precision: 0.651961\n",
      "- Recall: 0.666667\n",
      "----------------------------------\n",
      "===================================\n",
      "Random Forest\n",
      "\n",
      "##################################################\n",
      "Model performance for Training set Random Forest_lymphography_Data_no_gridsearch\n",
      "- Accuracy: 1.0000\n",
      "- F1 score: 1.000000\n",
      "- Precision: 1.000000\n",
      "- Recall: 1.000000\n",
      "----------------------------------\n",
      "===================================\n",
      "Model performance for Test set Random Forest_lymphography_Data_no_gridsearch\n",
      "- Accuracy: 0.8333\n",
      "- F1 score: 0.804598\n",
      "- Precision: 0.777778\n",
      "- Recall: 0.833333\n",
      "----------------------------------\n",
      "===================================\n",
      "Gradient Boosting\n",
      "\n",
      "##################################################\n",
      "Model performance for Training set Gradient Boosting_lymphography_Data_no_gridsearch\n",
      "- Accuracy: 1.0000\n",
      "- F1 score: 1.000000\n",
      "- Precision: 1.000000\n",
      "- Recall: 1.000000\n",
      "----------------------------------\n",
      "===================================\n",
      "Model performance for Test set Gradient Boosting_lymphography_Data_no_gridsearch\n",
      "- Accuracy: 0.8333\n",
      "- F1 score: 0.847745\n",
      "- Precision: 0.867778\n",
      "- Recall: 0.833333\n",
      "----------------------------------\n",
      "===================================\n",
      "K-Nearest Neighbors\n",
      "\n",
      "##################################################\n",
      "Model performance for Training set K-Nearest Neighbors_lymphography_Data_no_gridsearch\n",
      "- Accuracy: 0.8729\n",
      "- F1 score: 0.862220\n",
      "- Precision: 0.868086\n",
      "- Recall: 0.872881\n",
      "----------------------------------\n",
      "===================================\n",
      "Model performance for Test set K-Nearest Neighbors_lymphography_Data_no_gridsearch\n",
      "- Accuracy: 0.7667\n",
      "- F1 score: 0.741537\n",
      "- Precision: 0.724284\n",
      "- Recall: 0.766667\n",
      "----------------------------------\n",
      "===================================\n",
      "SVM\n",
      "\n",
      "##################################################\n",
      "Model performance for Training set SVM_lymphography_Data_no_gridsearch\n",
      "- Accuracy: 0.9407\n",
      "- F1 score: 0.939025\n",
      "- Precision: 0.941910\n",
      "- Recall: 0.940678\n",
      "----------------------------------\n",
      "===================================\n",
      "Model performance for Test set SVM_lymphography_Data_no_gridsearch\n",
      "- Accuracy: 0.8333\n",
      "- F1 score: 0.804598\n",
      "- Precision: 0.777778\n",
      "- Recall: 0.833333\n",
      "----------------------------------\n",
      "===================================\n",
      "################################################\n",
      "################### ENSEMBLE ###################\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/potatosalad/.local/share/virtualenvs/CSC_homework_3_group4-nat9sqnt/lib/python3.10/site-packages/sklearn/linear_model/_logistic.py:458: ConvergenceWarning: lbfgs failed to converge (status=1):\n",
      "STOP: TOTAL NO. of ITERATIONS REACHED LIMIT.\n",
      "\n",
      "Increase the number of iterations (max_iter) or scale the data as shown in:\n",
      "    https://scikit-learn.org/stable/modules/preprocessing.html\n",
      "Please also refer to the documentation for alternative solver options:\n",
      "    https://scikit-learn.org/stable/modules/linear_model.html#logistic-regression\n",
      "  n_iter_i = _check_optimize_result(\n",
      "/home/potatosalad/.local/share/virtualenvs/CSC_homework_3_group4-nat9sqnt/lib/python3.10/site-packages/sklearn/linear_model/_logistic.py:458: ConvergenceWarning: lbfgs failed to converge (status=1):\n",
      "STOP: TOTAL NO. of ITERATIONS REACHED LIMIT.\n",
      "\n",
      "Increase the number of iterations (max_iter) or scale the data as shown in:\n",
      "    https://scikit-learn.org/stable/modules/preprocessing.html\n",
      "Please also refer to the documentation for alternative solver options:\n",
      "    https://scikit-learn.org/stable/modules/linear_model.html#logistic-regression\n",
      "  n_iter_i = _check_optimize_result(\n",
      "/home/potatosalad/.local/share/virtualenvs/CSC_homework_3_group4-nat9sqnt/lib/python3.10/site-packages/sklearn/linear_model/_logistic.py:458: ConvergenceWarning: lbfgs failed to converge (status=1):\n",
      "STOP: TOTAL NO. of ITERATIONS REACHED LIMIT.\n",
      "\n",
      "Increase the number of iterations (max_iter) or scale the data as shown in:\n",
      "    https://scikit-learn.org/stable/modules/preprocessing.html\n",
      "Please also refer to the documentation for alternative solver options:\n",
      "    https://scikit-learn.org/stable/modules/linear_model.html#logistic-regression\n",
      "  n_iter_i = _check_optimize_result(\n",
      "/home/potatosalad/.local/share/virtualenvs/CSC_homework_3_group4-nat9sqnt/lib/python3.10/site-packages/sklearn/linear_model/_logistic.py:458: ConvergenceWarning: lbfgs failed to converge (status=1):\n",
      "STOP: TOTAL NO. of ITERATIONS REACHED LIMIT.\n",
      "\n",
      "Increase the number of iterations (max_iter) or scale the data as shown in:\n",
      "    https://scikit-learn.org/stable/modules/preprocessing.html\n",
      "Please also refer to the documentation for alternative solver options:\n",
      "    https://scikit-learn.org/stable/modules/linear_model.html#logistic-regression\n",
      "  n_iter_i = _check_optimize_result(\n",
      "/home/potatosalad/.local/share/virtualenvs/CSC_homework_3_group4-nat9sqnt/lib/python3.10/site-packages/sklearn/linear_model/_logistic.py:458: ConvergenceWarning: lbfgs failed to converge (status=1):\n",
      "STOP: TOTAL NO. of ITERATIONS REACHED LIMIT.\n",
      "\n",
      "Increase the number of iterations (max_iter) or scale the data as shown in:\n",
      "    https://scikit-learn.org/stable/modules/preprocessing.html\n",
      "Please also refer to the documentation for alternative solver options:\n",
      "    https://scikit-learn.org/stable/modules/linear_model.html#logistic-regression\n",
      "  n_iter_i = _check_optimize_result(\n",
      "/home/potatosalad/.local/share/virtualenvs/CSC_homework_3_group4-nat9sqnt/lib/python3.10/site-packages/sklearn/linear_model/_logistic.py:458: ConvergenceWarning: lbfgs failed to converge (status=1):\n",
      "STOP: TOTAL NO. of ITERATIONS REACHED LIMIT.\n",
      "\n",
      "Increase the number of iterations (max_iter) or scale the data as shown in:\n",
      "    https://scikit-learn.org/stable/modules/preprocessing.html\n",
      "Please also refer to the documentation for alternative solver options:\n",
      "    https://scikit-learn.org/stable/modules/linear_model.html#logistic-regression\n",
      "  n_iter_i = _check_optimize_result(\n",
      "/home/potatosalad/.local/share/virtualenvs/CSC_homework_3_group4-nat9sqnt/lib/python3.10/site-packages/sklearn/linear_model/_logistic.py:458: ConvergenceWarning: lbfgs failed to converge (status=1):\n",
      "STOP: TOTAL NO. of ITERATIONS REACHED LIMIT.\n",
      "\n",
      "Increase the number of iterations (max_iter) or scale the data as shown in:\n",
      "    https://scikit-learn.org/stable/modules/preprocessing.html\n",
      "Please also refer to the documentation for alternative solver options:\n",
      "    https://scikit-learn.org/stable/modules/linear_model.html#logistic-regression\n",
      "  n_iter_i = _check_optimize_result(\n",
      "/home/potatosalad/.local/share/virtualenvs/CSC_homework_3_group4-nat9sqnt/lib/python3.10/site-packages/sklearn/linear_model/_logistic.py:458: ConvergenceWarning: lbfgs failed to converge (status=1):\n",
      "STOP: TOTAL NO. of ITERATIONS REACHED LIMIT.\n",
      "\n",
      "Increase the number of iterations (max_iter) or scale the data as shown in:\n",
      "    https://scikit-learn.org/stable/modules/preprocessing.html\n",
      "Please also refer to the documentation for alternative solver options:\n",
      "    https://scikit-learn.org/stable/modules/linear_model.html#logistic-regression\n",
      "  n_iter_i = _check_optimize_result(\n",
      "/home/potatosalad/.local/share/virtualenvs/CSC_homework_3_group4-nat9sqnt/lib/python3.10/site-packages/sklearn/linear_model/_logistic.py:458: ConvergenceWarning: lbfgs failed to converge (status=1):\n",
      "STOP: TOTAL NO. of ITERATIONS REACHED LIMIT.\n",
      "\n",
      "Increase the number of iterations (max_iter) or scale the data as shown in:\n",
      "    https://scikit-learn.org/stable/modules/preprocessing.html\n",
      "Please also refer to the documentation for alternative solver options:\n",
      "    https://scikit-learn.org/stable/modules/linear_model.html#logistic-regression\n",
      "  n_iter_i = _check_optimize_result(\n",
      "/home/potatosalad/.local/share/virtualenvs/CSC_homework_3_group4-nat9sqnt/lib/python3.10/site-packages/sklearn/linear_model/_logistic.py:458: ConvergenceWarning: lbfgs failed to converge (status=1):\n",
      "STOP: TOTAL NO. of ITERATIONS REACHED LIMIT.\n",
      "\n",
      "Increase the number of iterations (max_iter) or scale the data as shown in:\n",
      "    https://scikit-learn.org/stable/modules/preprocessing.html\n",
      "Please also refer to the documentation for alternative solver options:\n",
      "    https://scikit-learn.org/stable/modules/linear_model.html#logistic-regression\n",
      "  n_iter_i = _check_optimize_result(\n",
      "/home/potatosalad/.local/share/virtualenvs/CSC_homework_3_group4-nat9sqnt/lib/python3.10/site-packages/sklearn/linear_model/_logistic.py:458: ConvergenceWarning: lbfgs failed to converge (status=1):\n",
      "STOP: TOTAL NO. of ITERATIONS REACHED LIMIT.\n",
      "\n",
      "Increase the number of iterations (max_iter) or scale the data as shown in:\n",
      "    https://scikit-learn.org/stable/modules/preprocessing.html\n",
      "Please also refer to the documentation for alternative solver options:\n",
      "    https://scikit-learn.org/stable/modules/linear_model.html#logistic-regression\n",
      "  n_iter_i = _check_optimize_result(\n",
      "/home/potatosalad/.local/share/virtualenvs/CSC_homework_3_group4-nat9sqnt/lib/python3.10/site-packages/sklearn/linear_model/_logistic.py:458: ConvergenceWarning: lbfgs failed to converge (status=1):\n",
      "STOP: TOTAL NO. of ITERATIONS REACHED LIMIT.\n",
      "\n",
      "Increase the number of iterations (max_iter) or scale the data as shown in:\n",
      "    https://scikit-learn.org/stable/modules/preprocessing.html\n",
      "Please also refer to the documentation for alternative solver options:\n",
      "    https://scikit-learn.org/stable/modules/linear_model.html#logistic-regression\n",
      "  n_iter_i = _check_optimize_result(\n",
      "/home/potatosalad/.local/share/virtualenvs/CSC_homework_3_group4-nat9sqnt/lib/python3.10/site-packages/sklearn/linear_model/_logistic.py:458: ConvergenceWarning: lbfgs failed to converge (status=1):\n",
      "STOP: TOTAL NO. of ITERATIONS REACHED LIMIT.\n",
      "\n",
      "Increase the number of iterations (max_iter) or scale the data as shown in:\n",
      "    https://scikit-learn.org/stable/modules/preprocessing.html\n",
      "Please also refer to the documentation for alternative solver options:\n",
      "    https://scikit-learn.org/stable/modules/linear_model.html#logistic-regression\n",
      "  n_iter_i = _check_optimize_result(\n",
      "/home/potatosalad/.local/share/virtualenvs/CSC_homework_3_group4-nat9sqnt/lib/python3.10/site-packages/sklearn/linear_model/_logistic.py:458: ConvergenceWarning: lbfgs failed to converge (status=1):\n",
      "STOP: TOTAL NO. of ITERATIONS REACHED LIMIT.\n",
      "\n",
      "Increase the number of iterations (max_iter) or scale the data as shown in:\n",
      "    https://scikit-learn.org/stable/modules/preprocessing.html\n",
      "Please also refer to the documentation for alternative solver options:\n",
      "    https://scikit-learn.org/stable/modules/linear_model.html#logistic-regression\n",
      "  n_iter_i = _check_optimize_result(\n",
      "/home/potatosalad/.local/share/virtualenvs/CSC_homework_3_group4-nat9sqnt/lib/python3.10/site-packages/sklearn/linear_model/_logistic.py:458: ConvergenceWarning: lbfgs failed to converge (status=1):\n",
      "STOP: TOTAL NO. of ITERATIONS REACHED LIMIT.\n",
      "\n",
      "Increase the number of iterations (max_iter) or scale the data as shown in:\n",
      "    https://scikit-learn.org/stable/modules/preprocessing.html\n",
      "Please also refer to the documentation for alternative solver options:\n",
      "    https://scikit-learn.org/stable/modules/linear_model.html#logistic-regression\n",
      "  n_iter_i = _check_optimize_result(\n",
      "/home/potatosalad/.local/share/virtualenvs/CSC_homework_3_group4-nat9sqnt/lib/python3.10/site-packages/sklearn/linear_model/_logistic.py:458: ConvergenceWarning: lbfgs failed to converge (status=1):\n",
      "STOP: TOTAL NO. of ITERATIONS REACHED LIMIT.\n",
      "\n",
      "Increase the number of iterations (max_iter) or scale the data as shown in:\n",
      "    https://scikit-learn.org/stable/modules/preprocessing.html\n",
      "Please also refer to the documentation for alternative solver options:\n",
      "    https://scikit-learn.org/stable/modules/linear_model.html#logistic-regression\n",
      "  n_iter_i = _check_optimize_result(\n",
      "/home/potatosalad/.local/share/virtualenvs/CSC_homework_3_group4-nat9sqnt/lib/python3.10/site-packages/sklearn/linear_model/_logistic.py:458: ConvergenceWarning: lbfgs failed to converge (status=1):\n",
      "STOP: TOTAL NO. of ITERATIONS REACHED LIMIT.\n",
      "\n",
      "Increase the number of iterations (max_iter) or scale the data as shown in:\n",
      "    https://scikit-learn.org/stable/modules/preprocessing.html\n",
      "Please also refer to the documentation for alternative solver options:\n",
      "    https://scikit-learn.org/stable/modules/linear_model.html#logistic-regression\n",
      "  n_iter_i = _check_optimize_result(\n",
      "/home/potatosalad/.local/share/virtualenvs/CSC_homework_3_group4-nat9sqnt/lib/python3.10/site-packages/sklearn/linear_model/_logistic.py:458: ConvergenceWarning: lbfgs failed to converge (status=1):\n",
      "STOP: TOTAL NO. of ITERATIONS REACHED LIMIT.\n",
      "\n",
      "Increase the number of iterations (max_iter) or scale the data as shown in:\n",
      "    https://scikit-learn.org/stable/modules/preprocessing.html\n",
      "Please also refer to the documentation for alternative solver options:\n",
      "    https://scikit-learn.org/stable/modules/linear_model.html#logistic-regression\n",
      "  n_iter_i = _check_optimize_result(\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/potatosalad/.local/share/virtualenvs/CSC_homework_3_group4-nat9sqnt/lib/python3.10/site-packages/sklearn/linear_model/_logistic.py:458: ConvergenceWarning: lbfgs failed to converge (status=1):\n",
      "STOP: TOTAL NO. of ITERATIONS REACHED LIMIT.\n",
      "\n",
      "Increase the number of iterations (max_iter) or scale the data as shown in:\n",
      "    https://scikit-learn.org/stable/modules/preprocessing.html\n",
      "Please also refer to the documentation for alternative solver options:\n",
      "    https://scikit-learn.org/stable/modules/linear_model.html#logistic-regression\n",
      "  n_iter_i = _check_optimize_result(\n",
      "/home/potatosalad/.local/share/virtualenvs/CSC_homework_3_group4-nat9sqnt/lib/python3.10/site-packages/sklearn/linear_model/_logistic.py:458: ConvergenceWarning: lbfgs failed to converge (status=1):\n",
      "STOP: TOTAL NO. of ITERATIONS REACHED LIMIT.\n",
      "\n",
      "Increase the number of iterations (max_iter) or scale the data as shown in:\n",
      "    https://scikit-learn.org/stable/modules/preprocessing.html\n",
      "Please also refer to the documentation for alternative solver options:\n",
      "    https://scikit-learn.org/stable/modules/linear_model.html#logistic-regression\n",
      "  n_iter_i = _check_optimize_result(\n",
      "/home/potatosalad/.local/share/virtualenvs/CSC_homework_3_group4-nat9sqnt/lib/python3.10/site-packages/sklearn/linear_model/_logistic.py:458: ConvergenceWarning: lbfgs failed to converge (status=1):\n",
      "STOP: TOTAL NO. of ITERATIONS REACHED LIMIT.\n",
      "\n",
      "Increase the number of iterations (max_iter) or scale the data as shown in:\n",
      "    https://scikit-learn.org/stable/modules/preprocessing.html\n",
      "Please also refer to the documentation for alternative solver options:\n",
      "    https://scikit-learn.org/stable/modules/linear_model.html#logistic-regression\n",
      "  n_iter_i = _check_optimize_result(\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Model performance for train set ENSEMBLE_lymphography_Data_no_gridsearch\n",
      "- Accuracy: 1.0000\n",
      "- F1 score: 1.000000\n",
      "- Precision: 1.000000\n",
      "- Recall: 1.000000\n",
      "----------------------------------\n",
      "===================================\n",
      "Model performance for test set ENSEMBLE_lymphography_Data_no_gridsearch\n",
      "- Accuracy: 0.8667\n",
      "- F1 score: 0.862835\n",
      "- Precision: 0.871111\n",
      "- Recall: 0.866667\n",
      "----------------------------------\n",
      "===================================\n"
     ]
    }
   ],
   "source": [
    "# NO GRIDSEARCH\n",
    "fit_models(X_train, y_train, X_test, y_test,data_lymphography_target_names, models, params, \"lymphography_Data_no_gridsearch\", \"experiment_lymphography_no_gridsearch\", 0)\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# GLASS DATASET"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 76,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "     Id number: 1 to 214  RI: refractive index  \\\n",
      "0                      1               1.52101   \n",
      "1                      2               1.51761   \n",
      "2                      3               1.51618   \n",
      "3                      4               1.51766   \n",
      "4                      5               1.51742   \n",
      "..                   ...                   ...   \n",
      "209                  210               1.51623   \n",
      "210                  211               1.51685   \n",
      "211                  212               1.52065   \n",
      "212                  213               1.51651   \n",
      "213                  214               1.51711   \n",
      "\n",
      "     Na: Sodium (unit measurement: weight percent in corresponding oxide, as are attributes 4-10)  \\\n",
      "0                                                13.64                                              \n",
      "1                                                13.89                                              \n",
      "2                                                13.53                                              \n",
      "3                                                13.21                                              \n",
      "4                                                13.27                                              \n",
      "..                                                 ...                                              \n",
      "209                                              14.14                                              \n",
      "210                                              14.92                                              \n",
      "211                                              14.36                                              \n",
      "212                                              14.38                                              \n",
      "213                                              14.23                                              \n",
      "\n",
      "     Mg: Magnesium  Al: Aluminum  Si: Silicon  K: Potassium  Ca: Calcium  \\\n",
      "0             4.49          1.10        71.78          0.06         8.75   \n",
      "1             3.60          1.36        72.73          0.48         7.83   \n",
      "2             3.55          1.54        72.99          0.39         7.78   \n",
      "3             3.69          1.29        72.61          0.57         8.22   \n",
      "4             3.62          1.24        73.08          0.55         8.07   \n",
      "..             ...           ...          ...           ...          ...   \n",
      "209           0.00          2.88        72.61          0.08         9.18   \n",
      "210           0.00          1.99        73.06          0.00         8.40   \n",
      "211           0.00          2.02        73.42          0.00         8.44   \n",
      "212           0.00          1.94        73.61          0.00         8.48   \n",
      "213           0.00          2.08        73.36          0.00         8.62   \n",
      "\n",
      "     Ba: Barium  Fe: Iron  Type of glass: (class attribute)  \n",
      "0          0.00       0.0                                 1  \n",
      "1          0.00       0.0                                 1  \n",
      "2          0.00       0.0                                 1  \n",
      "3          0.00       0.0                                 1  \n",
      "4          0.00       0.0                                 1  \n",
      "..          ...       ...                               ...  \n",
      "209        1.06       0.0                                 7  \n",
      "210        1.59       0.0                                 7  \n",
      "211        1.64       0.0                                 7  \n",
      "212        1.57       0.0                                 7  \n",
      "213        1.67       0.0                                 7  \n",
      "\n",
      "[214 rows x 11 columns]\n"
     ]
    }
   ],
   "source": [
    "data_glass=pd.read_csv('./data/glass.data',names=['Id number: 1 to 214', 'RI: refractive index', 'Na: Sodium (unit measurement: weight percent in corresponding oxide, as are attributes 4-10)', 'Mg: Magnesium', 'Al: Aluminum', 'Si: Silicon','K: Potassium', 'Ca: Calcium','Ba: Barium','Fe: Iron','Type of glass: (class attribute)'])\n",
    "print(data_glass)\n",
    "data_glass_target_names = np.array(['1 building_windows_float_processed', '2 building_windows_non_float_processed', '3 vehicle_windows_float_processed','4 vehicle_windows_non_float_processed', '5 containers', '6 tableware', '7 headlamps'])\n",
    "x=data_glass.drop(['Type of glass: (class attribute)'], axis=1)\n",
    "y=data_glass['Type of glass: (class attribute)']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 77,
   "metadata": {},
   "outputs": [],
   "source": [
    "X_train, X_test, y_train, y_test = train_test_split(x, y, test_size=0.20, random_state=42)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 79,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Fitting for parameters...\n",
      "X:      Id number: 1 to 214  RI: refractive index  \\\n",
      "79                    80               1.51590   \n",
      "161                  162               1.51934   \n",
      "109                  110               1.51818   \n",
      "127                  128               1.52081   \n",
      "95                    96               1.51860   \n",
      "..                   ...                   ...   \n",
      "106                  107               1.53125   \n",
      "14                    15               1.51763   \n",
      "92                    93               1.51588   \n",
      "179                  180               1.51852   \n",
      "102                  103               1.51820   \n",
      "\n",
      "     Na: Sodium (unit measurement: weight percent in corresponding oxide, as are attributes 4-10)  \\\n",
      "79                                               12.82                                              \n",
      "161                                              13.64                                              \n",
      "109                                              13.72                                              \n",
      "127                                              13.78                                              \n",
      "95                                               13.36                                              \n",
      "..                                                 ...                                              \n",
      "106                                              10.73                                              \n",
      "14                                               12.61                                              \n",
      "92                                               13.12                                              \n",
      "179                                              14.09                                              \n",
      "102                                              12.62                                              \n",
      "\n",
      "     Mg: Magnesium  Al: Aluminum  Si: Silicon  K: Potassium  Ca: Calcium  \\\n",
      "79            3.52          1.90        72.86          0.69         7.97   \n",
      "161           3.54          0.75        72.65          0.16         8.89   \n",
      "109           0.00          0.56        74.45          0.00        10.99   \n",
      "127           2.28          1.43        71.99          0.49         9.85   \n",
      "95            3.43          1.43        72.26          0.51         8.60   \n",
      "..             ...           ...          ...           ...          ...   \n",
      "106           0.00          2.10        69.81          0.58        13.30   \n",
      "14            3.59          1.31        73.29          0.58         8.50   \n",
      "92            3.41          1.58        73.26          0.07         8.39   \n",
      "179           2.19          1.66        72.67          0.00         9.32   \n",
      "102           2.76          0.83        73.81          0.35         9.42   \n",
      "\n",
      "     Ba: Barium  Fe: Iron  \n",
      "79         0.00      0.00  \n",
      "161        0.15      0.24  \n",
      "109        0.00      0.00  \n",
      "127        0.00      0.17  \n",
      "95         0.00      0.00  \n",
      "..          ...       ...  \n",
      "106        3.15      0.28  \n",
      "14         0.00      0.00  \n",
      "92         0.00      0.19  \n",
      "179        0.00      0.00  \n",
      "102        0.00      0.20  \n",
      "\n",
      "[171 rows x 10 columns]\n",
      "--------------------------------------------------\n",
      "Y: 79     2\n",
      "161    3\n",
      "109    2\n",
      "127    2\n",
      "95     2\n",
      "      ..\n",
      "106    2\n",
      "14     1\n",
      "92     2\n",
      "179    6\n",
      "102    2\n",
      "Name: Type of glass: (class attribute), Length: 171, dtype: int64\n",
      "--------------------------------------------------\n",
      "X_t:      Id number: 1 to 214  RI: refractive index  \\\n",
      "9                     10               1.51755   \n",
      "197                  198               1.51727   \n",
      "66                    67               1.52152   \n",
      "191                  192               1.51602   \n",
      "117                  118               1.51708   \n",
      "111                  112               1.52739   \n",
      "15                    16               1.51761   \n",
      "86                    87               1.51569   \n",
      "75                    76               1.51590   \n",
      "144                  145               1.51660   \n",
      "182                  183               1.51916   \n",
      "170                  171               1.52369   \n",
      "141                  142               1.51851   \n",
      "73                    74               1.51631   \n",
      "178                  179               1.51829   \n",
      "167                  168               1.51969   \n",
      "190                  191               1.51613   \n",
      "18                    19               1.51911   \n",
      "45                    46               1.51900   \n",
      "184                  185               1.51115   \n",
      "98                    99               1.51689   \n",
      "209                  210               1.51623   \n",
      "211                  212               1.52065   \n",
      "195                  196               1.51545   \n",
      "148                  149               1.51670   \n",
      "104                  105               1.52410   \n",
      "30                    31               1.51768   \n",
      "25                    26               1.51764   \n",
      "175                  176               1.52119   \n",
      "16                    17               1.51784   \n",
      "55                    56               1.51769   \n",
      "138                  139               1.51674   \n",
      "158                  159               1.51776   \n",
      "93                    94               1.51590   \n",
      "69                    70               1.52300   \n",
      "203                  204               1.51658   \n",
      "171                  172               1.51316   \n",
      "152                  153               1.51779   \n",
      "97                    98               1.51743   \n",
      "84                    85               1.51409   \n",
      "101                  102               1.51730   \n",
      "200                  201               1.51508   \n",
      "60                    61               1.51905   \n",
      "\n",
      "     Na: Sodium (unit measurement: weight percent in corresponding oxide, as are attributes 4-10)  \\\n",
      "9                                                13.00                                              \n",
      "197                                              14.70                                              \n",
      "66                                               13.05                                              \n",
      "191                                              14.85                                              \n",
      "117                                              13.72                                              \n",
      "111                                              11.02                                              \n",
      "15                                               12.81                                              \n",
      "86                                               13.24                                              \n",
      "75                                               13.02                                              \n",
      "144                                              12.99                                              \n",
      "182                                              14.15                                              \n",
      "170                                              13.44                                              \n",
      "141                                              13.20                                              \n",
      "73                                               13.34                                              \n",
      "178                                              14.46                                              \n",
      "167                                              12.64                                              \n",
      "190                                              13.88                                              \n",
      "18                                               13.90                                              \n",
      "45                                               13.49                                              \n",
      "184                                              17.38                                              \n",
      "98                                               12.67                                              \n",
      "209                                              14.14                                              \n",
      "211                                              14.36                                              \n",
      "195                                              14.14                                              \n",
      "148                                              13.24                                              \n",
      "104                                              13.83                                              \n",
      "30                                               12.65                                              \n",
      "25                                               12.98                                              \n",
      "175                                              12.97                                              \n",
      "16                                               12.68                                              \n",
      "55                                               12.45                                              \n",
      "138                                              12.79                                              \n",
      "158                                              13.53                                              \n",
      "93                                               13.24                                              \n",
      "69                                               13.31                                              \n",
      "203                                              14.80                                              \n",
      "171                                              13.02                                              \n",
      "152                                              13.64                                              \n",
      "97                                               12.20                                              \n",
      "84                                               14.25                                              \n",
      "101                                              12.35                                              \n",
      "200                                              15.15                                              \n",
      "60                                               13.60                                              \n",
      "\n",
      "     Mg: Magnesium  Al: Aluminum  Si: Silicon  K: Potassium  Ca: Calcium  \\\n",
      "9             3.60          1.36        72.99          0.57         8.40   \n",
      "197           0.00          2.34        73.28          0.00         8.95   \n",
      "66            3.65          0.87        72.22          0.19         9.85   \n",
      "191           0.00          2.38        73.28          0.00         8.76   \n",
      "117           3.68          1.81        72.06          0.64         7.88   \n",
      "111           0.00          0.75        73.08          0.00        14.96   \n",
      "15            3.54          1.23        73.24          0.58         8.39   \n",
      "86            3.49          1.47        73.25          0.38         8.03   \n",
      "75            3.58          1.51        73.12          0.69         7.96   \n",
      "144           3.18          1.23        72.97          0.58         8.81   \n",
      "182           0.00          2.09        72.74          0.00        10.88   \n",
      "170           0.00          1.58        72.22          0.32        12.24   \n",
      "141           3.63          1.07        72.83          0.57         8.41   \n",
      "73            3.57          1.57        72.87          0.61         7.89   \n",
      "178           2.24          1.62        72.38          0.00         9.26   \n",
      "167           0.00          1.65        73.75          0.38        11.53   \n",
      "190           1.78          1.79        73.10          0.00         8.67   \n",
      "18            3.73          1.18        72.12          0.06         8.89   \n",
      "45            3.48          1.35        71.95          0.55         9.00   \n",
      "184           0.00          0.34        75.41          0.00         6.65   \n",
      "98            2.88          1.71        73.21          0.73         8.54   \n",
      "209           0.00          2.88        72.61          0.08         9.18   \n",
      "211           0.00          2.02        73.42          0.00         8.44   \n",
      "195           0.00          2.68        73.39          0.08         9.07   \n",
      "148           3.57          1.38        72.70          0.56         8.44   \n",
      "104           2.90          1.17        71.15          0.08        10.79   \n",
      "30            3.56          1.30        73.08          0.61         8.69   \n",
      "25            3.54          1.21        73.00          0.65         8.53   \n",
      "175           0.33          1.51        73.39          0.13        11.27   \n",
      "16            3.67          1.16        73.11          0.61         8.70   \n",
      "55            2.71          1.29        73.70          0.56         9.06   \n",
      "138           3.52          1.54        73.36          0.66         7.90   \n",
      "158           3.41          1.52        72.04          0.58         8.79   \n",
      "93            3.34          1.47        73.10          0.39         8.22   \n",
      "69            3.58          0.82        71.99          0.12        10.17   \n",
      "203           0.00          1.99        73.11          0.00         8.28   \n",
      "171           0.00          3.04        70.48          6.21         6.96   \n",
      "152           3.65          0.65        73.00          0.06         8.93   \n",
      "97            3.25          1.16        73.55          0.62         8.90   \n",
      "84            3.09          2.08        72.28          1.10         7.08   \n",
      "101           2.72          1.63        72.87          0.70         9.23   \n",
      "200           0.00          2.25        73.50          0.00         8.34   \n",
      "60            3.62          1.11        72.64          0.14         8.76   \n",
      "\n",
      "     Ba: Barium  Fe: Iron  \n",
      "9          0.00      0.11  \n",
      "197        0.66      0.00  \n",
      "66         0.00      0.17  \n",
      "191        0.64      0.09  \n",
      "117        0.00      0.00  \n",
      "111        0.00      0.00  \n",
      "15         0.00      0.00  \n",
      "86         0.00      0.00  \n",
      "75         0.00      0.00  \n",
      "144        0.00      0.24  \n",
      "182        0.00      0.00  \n",
      "170        0.00      0.00  \n",
      "141        0.09      0.17  \n",
      "73         0.00      0.00  \n",
      "178        0.00      0.00  \n",
      "167        0.00      0.00  \n",
      "190        0.76      0.00  \n",
      "18         0.00      0.00  \n",
      "45         0.00      0.00  \n",
      "184        0.00      0.00  \n",
      "98         0.00      0.00  \n",
      "209        1.06      0.00  \n",
      "211        1.64      0.00  \n",
      "195        0.61      0.05  \n",
      "148        0.00      0.10  \n",
      "104        0.00      0.00  \n",
      "30         0.00      0.14  \n",
      "25         0.00      0.00  \n",
      "175        0.00      0.28  \n",
      "16         0.00      0.00  \n",
      "55         0.00      0.24  \n",
      "138        0.00      0.00  \n",
      "158        0.00      0.00  \n",
      "93         0.00      0.00  \n",
      "69         0.00      0.03  \n",
      "203        1.71      0.00  \n",
      "171        0.00      0.00  \n",
      "152        0.00      0.00  \n",
      "97         0.00      0.24  \n",
      "84         0.00      0.00  \n",
      "101        0.00      0.00  \n",
      "200        0.63      0.00  \n",
      "60         0.00      0.00  \n",
      "--------------------------------------------------\n",
      "Y_t: 9      1\n",
      "197    7\n",
      "66     1\n",
      "191    7\n",
      "117    2\n",
      "111    2\n",
      "15     1\n",
      "86     2\n",
      "75     2\n",
      "144    2\n",
      "182    6\n",
      "170    5\n",
      "141    2\n",
      "73     2\n",
      "178    6\n",
      "167    5\n",
      "190    7\n",
      "18     1\n",
      "45     1\n",
      "184    6\n",
      "98     2\n",
      "209    7\n",
      "211    7\n",
      "195    7\n",
      "148    3\n",
      "104    2\n",
      "30     1\n",
      "25     1\n",
      "175    5\n",
      "16     1\n",
      "55     1\n",
      "138    2\n",
      "158    3\n",
      "93     2\n",
      "69     1\n",
      "203    7\n",
      "171    5\n",
      "152    3\n",
      "97     2\n",
      "84     2\n",
      "101    2\n",
      "200    7\n",
      "60     1\n",
      "Name: Type of glass: (class attribute), dtype: int64\n",
      "--------------------------------------------------\n",
      "models: {'Logistic Regression': LogisticRegression(), 'Decision Tree': DecisionTreeClassifier(), 'Random Forest': RandomForestClassifier(random_state=42), 'Gradient Boosting': GradientBoostingClassifier(), 'K-Nearest Neighbors': KNeighborsClassifier(), 'SVM': SVC(probability=True)}\n",
      "--------------------------------------------------\n",
      "params: {'Logistic Regression': {'solver': ['newton-cholesky', 'liblinear', 'newton-cg', 'sag', 'saga'], 'penalty': ['l2'], 'C': [0.001, 0.01, 0.1, 1, 100], 'max_iter': [10000]}, 'Decision Tree': {'criterion': ['gini', 'entropy'], 'max_depth': [4, 6, 8, 10, 15, 50, 90]}, 'Random Forest': {'n_estimators': [200, 500], 'max_features': [None, 'sqrt', 'log2'], 'max_depth': [4, 5, 6, 7, 8], 'criterion': ['gini', 'entropy']}, 'Gradient Boosting': {'loss': ['log_loss'], 'learning_rate': [0.1, 0.01], 'max_depth': [3, 5, 8], 'max_features': ['log2', 'sqrt'], 'criterion': ['friedman_mse', 'squared_error'], 'subsample': [0.5, 0.8, 1.0], 'n_estimators': [10, 100, 1000]}, 'K-Nearest Neighbors': {'weights': ['uniform', 'distance'], 'leaf_size': [5, 15, 25]}, 'SVM': {'kernel': ['rbf'], 'C': [0.1, 1, 10, 100, 1000], 'degree': [1, 5], 'gamma': [1, 0.1, 0.01, 0.001]}}\n",
      "--------------------------------------------------\n",
      "run: glass_Data_gridsearch\n",
      "--------------------------------------------------\n",
      "folder: experiment_glass_gridsearch\n",
      "Logistic Regression\n",
      "\n",
      "##################################################\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/potatosalad/.local/share/virtualenvs/CSC_homework_3_group4-nat9sqnt/lib/python3.10/site-packages/sklearn/model_selection/_split.py:700: UserWarning: The least populated class in y has only 3 members, which is less than n_splits=5.\n",
      "  warnings.warn(\n",
      "/home/potatosalad/.local/share/virtualenvs/CSC_homework_3_group4-nat9sqnt/lib/python3.10/site-packages/sklearn/model_selection/_split.py:700: UserWarning: The least populated class in y has only 4 members, which is less than n_splits=5.\n",
      "  warnings.warn(\n",
      "/home/potatosalad/.local/share/virtualenvs/CSC_homework_3_group4-nat9sqnt/lib/python3.10/site-packages/sklearn/model_selection/_split.py:700: UserWarning: The least populated class in y has only 2 members, which is less than n_splits=5.\n",
      "  warnings.warn(\n",
      "/home/potatosalad/.local/share/virtualenvs/CSC_homework_3_group4-nat9sqnt/lib/python3.10/site-packages/sklearn/model_selection/_split.py:700: UserWarning: The least populated class in y has only 2 members, which is less than n_splits=5.\n",
      "  warnings.warn(\n",
      "/home/potatosalad/.local/share/virtualenvs/CSC_homework_3_group4-nat9sqnt/lib/python3.10/site-packages/sklearn/model_selection/_split.py:700: UserWarning: The least populated class in y has only 1 members, which is less than n_splits=5.\n",
      "  warnings.warn(\n",
      "/home/potatosalad/.local/share/virtualenvs/CSC_homework_3_group4-nat9sqnt/lib/python3.10/site-packages/sklearn/model_selection/_split.py:700: UserWarning: The least populated class in y has only 1 members, which is less than n_splits=5.\n",
      "  warnings.warn(\n",
      "/home/potatosalad/.local/share/virtualenvs/CSC_homework_3_group4-nat9sqnt/lib/python3.10/site-packages/sklearn/model_selection/_split.py:700: UserWarning: The least populated class in y has only 1 members, which is less than n_splits=5.\n",
      "  warnings.warn(\n",
      "/home/potatosalad/.local/share/virtualenvs/CSC_homework_3_group4-nat9sqnt/lib/python3.10/site-packages/sklearn/model_selection/_split.py:700: UserWarning: The least populated class in y has only 2 members, which is less than n_splits=5.\n",
      "  warnings.warn(\n",
      "/home/potatosalad/.local/share/virtualenvs/CSC_homework_3_group4-nat9sqnt/lib/python3.10/site-packages/sklearn/model_selection/_split.py:700: UserWarning: The least populated class in y has only 4 members, which is less than n_splits=5.\n",
      "  warnings.warn(\n",
      "/home/potatosalad/.local/share/virtualenvs/CSC_homework_3_group4-nat9sqnt/lib/python3.10/site-packages/sklearn/model_selection/_split.py:700: UserWarning: The least populated class in y has only 1 members, which is less than n_splits=5.\n",
      "  warnings.warn(\n",
      "/home/potatosalad/.local/share/virtualenvs/CSC_homework_3_group4-nat9sqnt/lib/python3.10/site-packages/sklearn/model_selection/_split.py:700: UserWarning: The least populated class in y has only 1 members, which is less than n_splits=5.\n",
      "  warnings.warn(\n",
      "/home/potatosalad/.local/share/virtualenvs/CSC_homework_3_group4-nat9sqnt/lib/python3.10/site-packages/sklearn/model_selection/_split.py:700: UserWarning: The least populated class in y has only 4 members, which is less than n_splits=5.\n",
      "  warnings.warn(\n",
      "/home/potatosalad/.local/share/virtualenvs/CSC_homework_3_group4-nat9sqnt/lib/python3.10/site-packages/sklearn/model_selection/_split.py:700: UserWarning: The least populated class in y has only 3 members, which is less than n_splits=5.\n",
      "  warnings.warn(\n",
      "/home/potatosalad/.local/share/virtualenvs/CSC_homework_3_group4-nat9sqnt/lib/python3.10/site-packages/sklearn/model_selection/_split.py:700: UserWarning: The least populated class in y has only 1 members, which is less than n_splits=5.\n",
      "  warnings.warn(\n",
      "/home/potatosalad/.local/share/virtualenvs/CSC_homework_3_group4-nat9sqnt/lib/python3.10/site-packages/sklearn/model_selection/_split.py:700: UserWarning: The least populated class in y has only 1 members, which is less than n_splits=5.\n",
      "  warnings.warn(\n",
      "/home/potatosalad/.local/share/virtualenvs/CSC_homework_3_group4-nat9sqnt/lib/python3.10/site-packages/sklearn/model_selection/_split.py:700: UserWarning: The least populated class in y has only 2 members, which is less than n_splits=5.\n",
      "  warnings.warn(\n",
      "/home/potatosalad/.local/share/virtualenvs/CSC_homework_3_group4-nat9sqnt/lib/python3.10/site-packages/sklearn/model_selection/_split.py:700: UserWarning: The least populated class in y has only 4 members, which is less than n_splits=5.\n",
      "  warnings.warn(\n",
      "/home/potatosalad/.local/share/virtualenvs/CSC_homework_3_group4-nat9sqnt/lib/python3.10/site-packages/sklearn/model_selection/_split.py:700: UserWarning: The least populated class in y has only 1 members, which is less than n_splits=5.\n",
      "  warnings.warn(\n",
      "/home/potatosalad/.local/share/virtualenvs/CSC_homework_3_group4-nat9sqnt/lib/python3.10/site-packages/sklearn/model_selection/_split.py:700: UserWarning: The least populated class in y has only 1 members, which is less than n_splits=5.\n",
      "  warnings.warn(\n",
      "/home/potatosalad/.local/share/virtualenvs/CSC_homework_3_group4-nat9sqnt/lib/python3.10/site-packages/sklearn/model_selection/_split.py:700: UserWarning: The least populated class in y has only 2 members, which is less than n_splits=5.\n",
      "  warnings.warn(\n",
      "/home/potatosalad/.local/share/virtualenvs/CSC_homework_3_group4-nat9sqnt/lib/python3.10/site-packages/sklearn/model_selection/_split.py:700: UserWarning: The least populated class in y has only 3 members, which is less than n_splits=5.\n",
      "  warnings.warn(\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Model performance for Training set Logistic Regression_glass_Data_gridsearch\n",
      "- Accuracy: 1.0000\n",
      "- F1 score: 1.000000\n",
      "- Precision: 1.000000\n",
      "- Recall: 1.000000\n",
      "----------------------------------\n",
      "===================================\n",
      "Model performance for Test set Logistic Regression_glass_Data_gridsearch\n",
      "- Accuracy: 1.0000\n",
      "- F1 score: 1.000000\n",
      "- Precision: 1.000000\n",
      "- Recall: 1.000000\n",
      "----------------------------------\n",
      "===================================\n",
      "Decision Tree\n",
      "\n",
      "##################################################\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/potatosalad/.local/share/virtualenvs/CSC_homework_3_group4-nat9sqnt/lib/python3.10/site-packages/sklearn/model_selection/_split.py:700: UserWarning: The least populated class in y has only 2 members, which is less than n_splits=5.\n",
      "  warnings.warn(\n",
      "/home/potatosalad/.local/share/virtualenvs/CSC_homework_3_group4-nat9sqnt/lib/python3.10/site-packages/sklearn/model_selection/_split.py:700: UserWarning: The least populated class in y has only 3 members, which is less than n_splits=5.\n",
      "  warnings.warn(\n",
      "/home/potatosalad/.local/share/virtualenvs/CSC_homework_3_group4-nat9sqnt/lib/python3.10/site-packages/sklearn/model_selection/_split.py:700: UserWarning: The least populated class in y has only 2 members, which is less than n_splits=5.\n",
      "  warnings.warn(\n",
      "/home/potatosalad/.local/share/virtualenvs/CSC_homework_3_group4-nat9sqnt/lib/python3.10/site-packages/sklearn/model_selection/_split.py:700: UserWarning: The least populated class in y has only 1 members, which is less than n_splits=5.\n",
      "  warnings.warn(\n",
      "/home/potatosalad/.local/share/virtualenvs/CSC_homework_3_group4-nat9sqnt/lib/python3.10/site-packages/sklearn/model_selection/_split.py:700: UserWarning: The least populated class in y has only 1 members, which is less than n_splits=5.\n",
      "  warnings.warn(\n",
      "/home/potatosalad/.local/share/virtualenvs/CSC_homework_3_group4-nat9sqnt/lib/python3.10/site-packages/sklearn/model_selection/_split.py:700: UserWarning: The least populated class in y has only 1 members, which is less than n_splits=5.\n",
      "  warnings.warn(\n",
      "/home/potatosalad/.local/share/virtualenvs/CSC_homework_3_group4-nat9sqnt/lib/python3.10/site-packages/sklearn/model_selection/_split.py:700: UserWarning: The least populated class in y has only 1 members, which is less than n_splits=5.\n",
      "  warnings.warn(\n",
      "/home/potatosalad/.local/share/virtualenvs/CSC_homework_3_group4-nat9sqnt/lib/python3.10/site-packages/sklearn/model_selection/_split.py:700: UserWarning: The least populated class in y has only 4 members, which is less than n_splits=5.\n",
      "  warnings.warn(\n",
      "/home/potatosalad/.local/share/virtualenvs/CSC_homework_3_group4-nat9sqnt/lib/python3.10/site-packages/sklearn/model_selection/_split.py:700: UserWarning: The least populated class in y has only 4 members, which is less than n_splits=5.\n",
      "  warnings.warn(\n",
      "/home/potatosalad/.local/share/virtualenvs/CSC_homework_3_group4-nat9sqnt/lib/python3.10/site-packages/sklearn/model_selection/_split.py:700: UserWarning: The least populated class in y has only 1 members, which is less than n_splits=5.\n",
      "  warnings.warn(\n",
      "/home/potatosalad/.local/share/virtualenvs/CSC_homework_3_group4-nat9sqnt/lib/python3.10/site-packages/sklearn/model_selection/_split.py:700: UserWarning: The least populated class in y has only 2 members, which is less than n_splits=5.\n",
      "  warnings.warn(\n",
      "/home/potatosalad/.local/share/virtualenvs/CSC_homework_3_group4-nat9sqnt/lib/python3.10/site-packages/sklearn/model_selection/_split.py:700: UserWarning: The least populated class in y has only 4 members, which is less than n_splits=5.\n",
      "  warnings.warn(\n",
      "/home/potatosalad/.local/share/virtualenvs/CSC_homework_3_group4-nat9sqnt/lib/python3.10/site-packages/sklearn/model_selection/_split.py:700: UserWarning: The least populated class in y has only 1 members, which is less than n_splits=5.\n",
      "  warnings.warn(\n",
      "/home/potatosalad/.local/share/virtualenvs/CSC_homework_3_group4-nat9sqnt/lib/python3.10/site-packages/sklearn/model_selection/_split.py:700: UserWarning: The least populated class in y has only 3 members, which is less than n_splits=5.\n",
      "  warnings.warn(\n",
      "/home/potatosalad/.local/share/virtualenvs/CSC_homework_3_group4-nat9sqnt/lib/python3.10/site-packages/sklearn/model_selection/_split.py:700: UserWarning: The least populated class in y has only 2 members, which is less than n_splits=5.\n",
      "  warnings.warn(\n",
      "/home/potatosalad/.local/share/virtualenvs/CSC_homework_3_group4-nat9sqnt/lib/python3.10/site-packages/sklearn/model_selection/_split.py:700: UserWarning: The least populated class in y has only 1 members, which is less than n_splits=5.\n",
      "  warnings.warn(\n",
      "/home/potatosalad/.local/share/virtualenvs/CSC_homework_3_group4-nat9sqnt/lib/python3.10/site-packages/sklearn/model_selection/_split.py:700: UserWarning: The least populated class in y has only 4 members, which is less than n_splits=5.\n",
      "  warnings.warn(\n",
      "/home/potatosalad/.local/share/virtualenvs/CSC_homework_3_group4-nat9sqnt/lib/python3.10/site-packages/sklearn/model_selection/_split.py:700: UserWarning: The least populated class in y has only 1 members, which is less than n_splits=5.\n",
      "  warnings.warn(\n",
      "/home/potatosalad/.local/share/virtualenvs/CSC_homework_3_group4-nat9sqnt/lib/python3.10/site-packages/sklearn/model_selection/_split.py:700: UserWarning: The least populated class in y has only 1 members, which is less than n_splits=5.\n",
      "  warnings.warn(\n",
      "/home/potatosalad/.local/share/virtualenvs/CSC_homework_3_group4-nat9sqnt/lib/python3.10/site-packages/sklearn/model_selection/_split.py:700: UserWarning: The least populated class in y has only 2 members, which is less than n_splits=5.\n",
      "  warnings.warn(\n",
      "/home/potatosalad/.local/share/virtualenvs/CSC_homework_3_group4-nat9sqnt/lib/python3.10/site-packages/sklearn/model_selection/_split.py:700: UserWarning: The least populated class in y has only 3 members, which is less than n_splits=5.\n",
      "  warnings.warn(\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Model performance for Training set Decision Tree_glass_Data_gridsearch\n",
      "- Accuracy: 1.0000\n",
      "- F1 score: 1.000000\n",
      "- Precision: 1.000000\n",
      "- Recall: 1.000000\n",
      "----------------------------------\n",
      "===================================\n",
      "Model performance for Test set Decision Tree_glass_Data_gridsearch\n",
      "- Accuracy: 0.9767\n",
      "- F1 score: 0.976591\n",
      "- Precision: 0.978295\n",
      "- Recall: 0.976744\n",
      "----------------------------------\n",
      "===================================\n",
      "Random Forest\n",
      "\n",
      "##################################################\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/potatosalad/.local/share/virtualenvs/CSC_homework_3_group4-nat9sqnt/lib/python3.10/site-packages/sklearn/model_selection/_split.py:700: UserWarning: The least populated class in y has only 2 members, which is less than n_splits=5.\n",
      "  warnings.warn(\n",
      "/home/potatosalad/.local/share/virtualenvs/CSC_homework_3_group4-nat9sqnt/lib/python3.10/site-packages/sklearn/model_selection/_split.py:700: UserWarning: The least populated class in y has only 1 members, which is less than n_splits=5.\n",
      "  warnings.warn(\n",
      "/home/potatosalad/.local/share/virtualenvs/CSC_homework_3_group4-nat9sqnt/lib/python3.10/site-packages/sklearn/model_selection/_split.py:700: UserWarning: The least populated class in y has only 3 members, which is less than n_splits=5.\n",
      "  warnings.warn(\n",
      "/home/potatosalad/.local/share/virtualenvs/CSC_homework_3_group4-nat9sqnt/lib/python3.10/site-packages/sklearn/model_selection/_split.py:700: UserWarning: The least populated class in y has only 1 members, which is less than n_splits=5.\n",
      "  warnings.warn(\n",
      "/home/potatosalad/.local/share/virtualenvs/CSC_homework_3_group4-nat9sqnt/lib/python3.10/site-packages/sklearn/model_selection/_split.py:700: UserWarning: The least populated class in y has only 2 members, which is less than n_splits=5.\n",
      "  warnings.warn(\n",
      "/home/potatosalad/.local/share/virtualenvs/CSC_homework_3_group4-nat9sqnt/lib/python3.10/site-packages/sklearn/model_selection/_split.py:700: UserWarning: The least populated class in y has only 4 members, which is less than n_splits=5.\n",
      "  warnings.warn(\n",
      "/home/potatosalad/.local/share/virtualenvs/CSC_homework_3_group4-nat9sqnt/lib/python3.10/site-packages/sklearn/model_selection/_split.py:700: UserWarning: The least populated class in y has only 1 members, which is less than n_splits=5.\n",
      "  warnings.warn(\n",
      "/home/potatosalad/.local/share/virtualenvs/CSC_homework_3_group4-nat9sqnt/lib/python3.10/site-packages/sklearn/model_selection/_split.py:700: UserWarning: The least populated class in y has only 4 members, which is less than n_splits=5.\n",
      "  warnings.warn(\n",
      "/home/potatosalad/.local/share/virtualenvs/CSC_homework_3_group4-nat9sqnt/lib/python3.10/site-packages/sklearn/model_selection/_split.py:700: UserWarning: The least populated class in y has only 4 members, which is less than n_splits=5.\n",
      "  warnings.warn(\n",
      "/home/potatosalad/.local/share/virtualenvs/CSC_homework_3_group4-nat9sqnt/lib/python3.10/site-packages/sklearn/model_selection/_split.py:700: UserWarning: The least populated class in y has only 1 members, which is less than n_splits=5.\n",
      "  warnings.warn(\n",
      "/home/potatosalad/.local/share/virtualenvs/CSC_homework_3_group4-nat9sqnt/lib/python3.10/site-packages/sklearn/model_selection/_split.py:700: UserWarning: The least populated class in y has only 1 members, which is less than n_splits=5.\n",
      "  warnings.warn(\n",
      "/home/potatosalad/.local/share/virtualenvs/CSC_homework_3_group4-nat9sqnt/lib/python3.10/site-packages/sklearn/model_selection/_split.py:700: UserWarning: The least populated class in y has only 2 members, which is less than n_splits=5.\n",
      "  warnings.warn(\n",
      "/home/potatosalad/.local/share/virtualenvs/CSC_homework_3_group4-nat9sqnt/lib/python3.10/site-packages/sklearn/model_selection/_split.py:700: UserWarning: The least populated class in y has only 1 members, which is less than n_splits=5.\n",
      "  warnings.warn(\n",
      "/home/potatosalad/.local/share/virtualenvs/CSC_homework_3_group4-nat9sqnt/lib/python3.10/site-packages/sklearn/model_selection/_split.py:700: UserWarning: The least populated class in y has only 2 members, which is less than n_splits=5.\n",
      "  warnings.warn(\n",
      "/home/potatosalad/.local/share/virtualenvs/CSC_homework_3_group4-nat9sqnt/lib/python3.10/site-packages/sklearn/model_selection/_split.py:700: UserWarning: The least populated class in y has only 1 members, which is less than n_splits=5.\n",
      "  warnings.warn(\n",
      "/home/potatosalad/.local/share/virtualenvs/CSC_homework_3_group4-nat9sqnt/lib/python3.10/site-packages/sklearn/model_selection/_split.py:700: UserWarning: The least populated class in y has only 4 members, which is less than n_splits=5.\n",
      "  warnings.warn(\n",
      "/home/potatosalad/.local/share/virtualenvs/CSC_homework_3_group4-nat9sqnt/lib/python3.10/site-packages/sklearn/model_selection/_split.py:700: UserWarning: The least populated class in y has only 2 members, which is less than n_splits=5.\n",
      "  warnings.warn(\n",
      "/home/potatosalad/.local/share/virtualenvs/CSC_homework_3_group4-nat9sqnt/lib/python3.10/site-packages/sklearn/model_selection/_split.py:700: UserWarning: The least populated class in y has only 1 members, which is less than n_splits=5.\n",
      "  warnings.warn(\n",
      "/home/potatosalad/.local/share/virtualenvs/CSC_homework_3_group4-nat9sqnt/lib/python3.10/site-packages/sklearn/model_selection/_split.py:700: UserWarning: The least populated class in y has only 1 members, which is less than n_splits=5.\n",
      "  warnings.warn(\n",
      "/home/potatosalad/.local/share/virtualenvs/CSC_homework_3_group4-nat9sqnt/lib/python3.10/site-packages/sklearn/model_selection/_split.py:700: UserWarning: The least populated class in y has only 3 members, which is less than n_splits=5.\n",
      "  warnings.warn(\n",
      "/home/potatosalad/.local/share/virtualenvs/CSC_homework_3_group4-nat9sqnt/lib/python3.10/site-packages/sklearn/model_selection/_split.py:700: UserWarning: The least populated class in y has only 3 members, which is less than n_splits=5.\n",
      "  warnings.warn(\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Model performance for Training set Random Forest_glass_Data_gridsearch\n",
      "- Accuracy: 1.0000\n",
      "- F1 score: 1.000000\n",
      "- Precision: 1.000000\n",
      "- Recall: 1.000000\n",
      "----------------------------------\n",
      "===================================\n",
      "Model performance for Test set Random Forest_glass_Data_gridsearch\n",
      "- Accuracy: 0.9535\n",
      "- F1 score: 0.953336\n",
      "- Precision: 0.960853\n",
      "- Recall: 0.953488\n",
      "----------------------------------\n",
      "===================================\n",
      "Gradient Boosting\n",
      "\n",
      "##################################################\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/potatosalad/.local/share/virtualenvs/CSC_homework_3_group4-nat9sqnt/lib/python3.10/site-packages/sklearn/model_selection/_split.py:700: UserWarning: The least populated class in y has only 2 members, which is less than n_splits=5.\n",
      "  warnings.warn(\n",
      "/home/potatosalad/.local/share/virtualenvs/CSC_homework_3_group4-nat9sqnt/lib/python3.10/site-packages/sklearn/model_selection/_split.py:700: UserWarning: The least populated class in y has only 2 members, which is less than n_splits=5.\n",
      "  warnings.warn(\n",
      "/home/potatosalad/.local/share/virtualenvs/CSC_homework_3_group4-nat9sqnt/lib/python3.10/site-packages/sklearn/model_selection/_split.py:700: UserWarning: The least populated class in y has only 1 members, which is less than n_splits=5.\n",
      "  warnings.warn(\n",
      "/home/potatosalad/.local/share/virtualenvs/CSC_homework_3_group4-nat9sqnt/lib/python3.10/site-packages/sklearn/model_selection/_split.py:700: UserWarning: The least populated class in y has only 4 members, which is less than n_splits=5.\n",
      "  warnings.warn(\n",
      "/home/potatosalad/.local/share/virtualenvs/CSC_homework_3_group4-nat9sqnt/lib/python3.10/site-packages/sklearn/model_selection/_split.py:700: UserWarning: The least populated class in y has only 3 members, which is less than n_splits=5.\n",
      "  warnings.warn(\n",
      "/home/potatosalad/.local/share/virtualenvs/CSC_homework_3_group4-nat9sqnt/lib/python3.10/site-packages/sklearn/model_selection/_split.py:700: UserWarning: The least populated class in y has only 1 members, which is less than n_splits=5.\n",
      "  warnings.warn(\n",
      "/home/potatosalad/.local/share/virtualenvs/CSC_homework_3_group4-nat9sqnt/lib/python3.10/site-packages/sklearn/model_selection/_split.py:700: UserWarning: The least populated class in y has only 1 members, which is less than n_splits=5.\n",
      "  warnings.warn(\n",
      "/home/potatosalad/.local/share/virtualenvs/CSC_homework_3_group4-nat9sqnt/lib/python3.10/site-packages/sklearn/model_selection/_split.py:700: UserWarning: The least populated class in y has only 2 members, which is less than n_splits=5.\n",
      "  warnings.warn(\n",
      "/home/potatosalad/.local/share/virtualenvs/CSC_homework_3_group4-nat9sqnt/lib/python3.10/site-packages/sklearn/model_selection/_split.py:700: UserWarning: The least populated class in y has only 4 members, which is less than n_splits=5.\n",
      "  warnings.warn(\n",
      "/home/potatosalad/.local/share/virtualenvs/CSC_homework_3_group4-nat9sqnt/lib/python3.10/site-packages/sklearn/model_selection/_split.py:700: UserWarning: The least populated class in y has only 1 members, which is less than n_splits=5.\n",
      "  warnings.warn(\n",
      "/home/potatosalad/.local/share/virtualenvs/CSC_homework_3_group4-nat9sqnt/lib/python3.10/site-packages/sklearn/model_selection/_split.py:700: UserWarning: The least populated class in y has only 1 members, which is less than n_splits=5.\n",
      "  warnings.warn(\n",
      "/home/potatosalad/.local/share/virtualenvs/CSC_homework_3_group4-nat9sqnt/lib/python3.10/site-packages/sklearn/model_selection/_split.py:700: UserWarning: The least populated class in y has only 3 members, which is less than n_splits=5.\n",
      "  warnings.warn(\n",
      "/home/potatosalad/.local/share/virtualenvs/CSC_homework_3_group4-nat9sqnt/lib/python3.10/site-packages/sklearn/model_selection/_split.py:700: UserWarning: The least populated class in y has only 4 members, which is less than n_splits=5.\n",
      "  warnings.warn(\n",
      "/home/potatosalad/.local/share/virtualenvs/CSC_homework_3_group4-nat9sqnt/lib/python3.10/site-packages/sklearn/model_selection/_split.py:700: UserWarning: The least populated class in y has only 1 members, which is less than n_splits=5.\n",
      "  warnings.warn(\n",
      "/home/potatosalad/.local/share/virtualenvs/CSC_homework_3_group4-nat9sqnt/lib/python3.10/site-packages/sklearn/model_selection/_split.py:700: UserWarning: The least populated class in y has only 1 members, which is less than n_splits=5.\n",
      "  warnings.warn(\n",
      "/home/potatosalad/.local/share/virtualenvs/CSC_homework_3_group4-nat9sqnt/lib/python3.10/site-packages/sklearn/model_selection/_split.py:700: UserWarning: The least populated class in y has only 2 members, which is less than n_splits=5.\n",
      "  warnings.warn(\n",
      "/home/potatosalad/.local/share/virtualenvs/CSC_homework_3_group4-nat9sqnt/lib/python3.10/site-packages/sklearn/model_selection/_split.py:700: UserWarning: The least populated class in y has only 4 members, which is less than n_splits=5.\n",
      "  warnings.warn(\n",
      "/home/potatosalad/.local/share/virtualenvs/CSC_homework_3_group4-nat9sqnt/lib/python3.10/site-packages/sklearn/model_selection/_split.py:700: UserWarning: The least populated class in y has only 1 members, which is less than n_splits=5.\n",
      "  warnings.warn(\n",
      "/home/potatosalad/.local/share/virtualenvs/CSC_homework_3_group4-nat9sqnt/lib/python3.10/site-packages/sklearn/model_selection/_split.py:700: UserWarning: The least populated class in y has only 1 members, which is less than n_splits=5.\n",
      "  warnings.warn(\n",
      "/home/potatosalad/.local/share/virtualenvs/CSC_homework_3_group4-nat9sqnt/lib/python3.10/site-packages/sklearn/model_selection/_split.py:700: UserWarning: The least populated class in y has only 2 members, which is less than n_splits=5.\n",
      "  warnings.warn(\n",
      "/home/potatosalad/.local/share/virtualenvs/CSC_homework_3_group4-nat9sqnt/lib/python3.10/site-packages/sklearn/model_selection/_split.py:700: UserWarning: The least populated class in y has only 3 members, which is less than n_splits=5.\n",
      "  warnings.warn(\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Model performance for Training set Gradient Boosting_glass_Data_gridsearch\n",
      "- Accuracy: 1.0000\n",
      "- F1 score: 1.000000\n",
      "- Precision: 1.000000\n",
      "- Recall: 1.000000\n",
      "----------------------------------\n",
      "===================================\n",
      "Model performance for Test set Gradient Boosting_glass_Data_gridsearch\n",
      "- Accuracy: 1.0000\n",
      "- F1 score: 1.000000\n",
      "- Precision: 1.000000\n",
      "- Recall: 1.000000\n",
      "----------------------------------\n",
      "===================================\n",
      "K-Nearest Neighbors\n",
      "\n",
      "##################################################\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/potatosalad/.local/share/virtualenvs/CSC_homework_3_group4-nat9sqnt/lib/python3.10/site-packages/sklearn/model_selection/_split.py:700: UserWarning: The least populated class in y has only 2 members, which is less than n_splits=5.\n",
      "  warnings.warn(\n",
      "/home/potatosalad/.local/share/virtualenvs/CSC_homework_3_group4-nat9sqnt/lib/python3.10/site-packages/sklearn/model_selection/_split.py:700: UserWarning: The least populated class in y has only 2 members, which is less than n_splits=5.\n",
      "  warnings.warn(\n",
      "/home/potatosalad/.local/share/virtualenvs/CSC_homework_3_group4-nat9sqnt/lib/python3.10/site-packages/sklearn/model_selection/_split.py:700: UserWarning: The least populated class in y has only 3 members, which is less than n_splits=5.\n",
      "  warnings.warn(\n",
      "/home/potatosalad/.local/share/virtualenvs/CSC_homework_3_group4-nat9sqnt/lib/python3.10/site-packages/sklearn/model_selection/_split.py:700: UserWarning: The least populated class in y has only 1 members, which is less than n_splits=5.\n",
      "  warnings.warn(\n",
      "/home/potatosalad/.local/share/virtualenvs/CSC_homework_3_group4-nat9sqnt/lib/python3.10/site-packages/sklearn/model_selection/_split.py:700: UserWarning: The least populated class in y has only 4 members, which is less than n_splits=5.\n",
      "  warnings.warn(\n",
      "/home/potatosalad/.local/share/virtualenvs/CSC_homework_3_group4-nat9sqnt/lib/python3.10/site-packages/sklearn/model_selection/_split.py:700: UserWarning: The least populated class in y has only 1 members, which is less than n_splits=5.\n",
      "  warnings.warn(\n",
      "/home/potatosalad/.local/share/virtualenvs/CSC_homework_3_group4-nat9sqnt/lib/python3.10/site-packages/sklearn/model_selection/_split.py:700: UserWarning: The least populated class in y has only 1 members, which is less than n_splits=5.\n",
      "  warnings.warn(\n",
      "/home/potatosalad/.local/share/virtualenvs/CSC_homework_3_group4-nat9sqnt/lib/python3.10/site-packages/sklearn/model_selection/_split.py:700: UserWarning: The least populated class in y has only 2 members, which is less than n_splits=5.\n",
      "  warnings.warn(\n",
      "/home/potatosalad/.local/share/virtualenvs/CSC_homework_3_group4-nat9sqnt/lib/python3.10/site-packages/sklearn/model_selection/_split.py:700: UserWarning: The least populated class in y has only 4 members, which is less than n_splits=5.\n",
      "  warnings.warn(\n",
      "/home/potatosalad/.local/share/virtualenvs/CSC_homework_3_group4-nat9sqnt/lib/python3.10/site-packages/sklearn/model_selection/_split.py:700: UserWarning: The least populated class in y has only 1 members, which is less than n_splits=5.\n",
      "  warnings.warn(\n",
      "/home/potatosalad/.local/share/virtualenvs/CSC_homework_3_group4-nat9sqnt/lib/python3.10/site-packages/sklearn/model_selection/_split.py:700: UserWarning: The least populated class in y has only 1 members, which is less than n_splits=5.\n",
      "  warnings.warn(\n",
      "/home/potatosalad/.local/share/virtualenvs/CSC_homework_3_group4-nat9sqnt/lib/python3.10/site-packages/sklearn/model_selection/_split.py:700: UserWarning: The least populated class in y has only 3 members, which is less than n_splits=5.\n",
      "  warnings.warn(\n",
      "/home/potatosalad/.local/share/virtualenvs/CSC_homework_3_group4-nat9sqnt/lib/python3.10/site-packages/sklearn/model_selection/_split.py:700: UserWarning: The least populated class in y has only 4 members, which is less than n_splits=5.\n",
      "  warnings.warn(\n",
      "/home/potatosalad/.local/share/virtualenvs/CSC_homework_3_group4-nat9sqnt/lib/python3.10/site-packages/sklearn/model_selection/_split.py:700: UserWarning: The least populated class in y has only 1 members, which is less than n_splits=5.\n",
      "  warnings.warn(\n",
      "/home/potatosalad/.local/share/virtualenvs/CSC_homework_3_group4-nat9sqnt/lib/python3.10/site-packages/sklearn/model_selection/_split.py:700: UserWarning: The least populated class in y has only 1 members, which is less than n_splits=5.\n",
      "  warnings.warn(\n",
      "/home/potatosalad/.local/share/virtualenvs/CSC_homework_3_group4-nat9sqnt/lib/python3.10/site-packages/sklearn/model_selection/_split.py:700: UserWarning: The least populated class in y has only 2 members, which is less than n_splits=5.\n",
      "  warnings.warn(\n",
      "/home/potatosalad/.local/share/virtualenvs/CSC_homework_3_group4-nat9sqnt/lib/python3.10/site-packages/sklearn/model_selection/_split.py:700: UserWarning: The least populated class in y has only 1 members, which is less than n_splits=5.\n",
      "  warnings.warn(\n",
      "/home/potatosalad/.local/share/virtualenvs/CSC_homework_3_group4-nat9sqnt/lib/python3.10/site-packages/sklearn/model_selection/_split.py:700: UserWarning: The least populated class in y has only 4 members, which is less than n_splits=5.\n",
      "  warnings.warn(\n",
      "/home/potatosalad/.local/share/virtualenvs/CSC_homework_3_group4-nat9sqnt/lib/python3.10/site-packages/sklearn/model_selection/_split.py:700: UserWarning: The least populated class in y has only 1 members, which is less than n_splits=5.\n",
      "  warnings.warn(\n",
      "/home/potatosalad/.local/share/virtualenvs/CSC_homework_3_group4-nat9sqnt/lib/python3.10/site-packages/sklearn/model_selection/_split.py:700: UserWarning: The least populated class in y has only 2 members, which is less than n_splits=5.\n",
      "  warnings.warn(\n",
      "/home/potatosalad/.local/share/virtualenvs/CSC_homework_3_group4-nat9sqnt/lib/python3.10/site-packages/sklearn/model_selection/_split.py:700: UserWarning: The least populated class in y has only 3 members, which is less than n_splits=5.\n",
      "  warnings.warn(\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Model performance for Training set K-Nearest Neighbors_glass_Data_gridsearch\n",
      "- Accuracy: 1.0000\n",
      "- F1 score: 1.000000\n",
      "- Precision: 1.000000\n",
      "- Recall: 1.000000\n",
      "----------------------------------\n",
      "===================================\n",
      "Model performance for Test set K-Nearest Neighbors_glass_Data_gridsearch\n",
      "- Accuracy: 0.9070\n",
      "- F1 score: 0.900506\n",
      "- Precision: 0.917636\n",
      "- Recall: 0.906977\n",
      "----------------------------------\n",
      "===================================\n",
      "SVM\n",
      "\n",
      "##################################################\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/potatosalad/.local/share/virtualenvs/CSC_homework_3_group4-nat9sqnt/lib/python3.10/site-packages/sklearn/model_selection/_split.py:700: UserWarning: The least populated class in y has only 2 members, which is less than n_splits=5.\n",
      "  warnings.warn(\n",
      "/home/potatosalad/.local/share/virtualenvs/CSC_homework_3_group4-nat9sqnt/lib/python3.10/site-packages/sklearn/model_selection/_split.py:700: UserWarning: The least populated class in y has only 2 members, which is less than n_splits=5.\n",
      "  warnings.warn(\n",
      "/home/potatosalad/.local/share/virtualenvs/CSC_homework_3_group4-nat9sqnt/lib/python3.10/site-packages/sklearn/model_selection/_split.py:700: UserWarning: The least populated class in y has only 1 members, which is less than n_splits=5.\n",
      "  warnings.warn(\n",
      "/home/potatosalad/.local/share/virtualenvs/CSC_homework_3_group4-nat9sqnt/lib/python3.10/site-packages/sklearn/model_selection/_split.py:700: UserWarning: The least populated class in y has only 3 members, which is less than n_splits=5.\n",
      "  warnings.warn(\n",
      "/home/potatosalad/.local/share/virtualenvs/CSC_homework_3_group4-nat9sqnt/lib/python3.10/site-packages/sklearn/model_selection/_split.py:700: UserWarning: The least populated class in y has only 4 members, which is less than n_splits=5.\n",
      "  warnings.warn(\n",
      "/home/potatosalad/.local/share/virtualenvs/CSC_homework_3_group4-nat9sqnt/lib/python3.10/site-packages/sklearn/model_selection/_split.py:700: UserWarning: The least populated class in y has only 1 members, which is less than n_splits=5.\n",
      "  warnings.warn(\n",
      "/home/potatosalad/.local/share/virtualenvs/CSC_homework_3_group4-nat9sqnt/lib/python3.10/site-packages/sklearn/model_selection/_split.py:700: UserWarning: The least populated class in y has only 1 members, which is less than n_splits=5.\n",
      "  warnings.warn(\n",
      "/home/potatosalad/.local/share/virtualenvs/CSC_homework_3_group4-nat9sqnt/lib/python3.10/site-packages/sklearn/model_selection/_split.py:700: UserWarning: The least populated class in y has only 2 members, which is less than n_splits=5.\n",
      "  warnings.warn(\n",
      "/home/potatosalad/.local/share/virtualenvs/CSC_homework_3_group4-nat9sqnt/lib/python3.10/site-packages/sklearn/model_selection/_split.py:700: UserWarning: The least populated class in y has only 4 members, which is less than n_splits=5.\n",
      "  warnings.warn(\n",
      "/home/potatosalad/.local/share/virtualenvs/CSC_homework_3_group4-nat9sqnt/lib/python3.10/site-packages/sklearn/model_selection/_split.py:700: UserWarning: The least populated class in y has only 3 members, which is less than n_splits=5.\n",
      "  warnings.warn(\n",
      "/home/potatosalad/.local/share/virtualenvs/CSC_homework_3_group4-nat9sqnt/lib/python3.10/site-packages/sklearn/model_selection/_split.py:700: UserWarning: The least populated class in y has only 4 members, which is less than n_splits=5.\n",
      "  warnings.warn(\n",
      "/home/potatosalad/.local/share/virtualenvs/CSC_homework_3_group4-nat9sqnt/lib/python3.10/site-packages/sklearn/model_selection/_split.py:700: UserWarning: The least populated class in y has only 1 members, which is less than n_splits=5.\n",
      "  warnings.warn(\n",
      "/home/potatosalad/.local/share/virtualenvs/CSC_homework_3_group4-nat9sqnt/lib/python3.10/site-packages/sklearn/model_selection/_split.py:700: UserWarning: The least populated class in y has only 1 members, which is less than n_splits=5.\n",
      "  warnings.warn(\n",
      "/home/potatosalad/.local/share/virtualenvs/CSC_homework_3_group4-nat9sqnt/lib/python3.10/site-packages/sklearn/model_selection/_split.py:700: UserWarning: The least populated class in y has only 1 members, which is less than n_splits=5.\n",
      "  warnings.warn(\n",
      "/home/potatosalad/.local/share/virtualenvs/CSC_homework_3_group4-nat9sqnt/lib/python3.10/site-packages/sklearn/model_selection/_split.py:700: UserWarning: The least populated class in y has only 1 members, which is less than n_splits=5.\n",
      "  warnings.warn(\n",
      "/home/potatosalad/.local/share/virtualenvs/CSC_homework_3_group4-nat9sqnt/lib/python3.10/site-packages/sklearn/model_selection/_split.py:700: UserWarning: The least populated class in y has only 4 members, which is less than n_splits=5.\n",
      "  warnings.warn(\n",
      "/home/potatosalad/.local/share/virtualenvs/CSC_homework_3_group4-nat9sqnt/lib/python3.10/site-packages/sklearn/model_selection/_split.py:700: UserWarning: The least populated class in y has only 2 members, which is less than n_splits=5.\n",
      "  warnings.warn(\n",
      "/home/potatosalad/.local/share/virtualenvs/CSC_homework_3_group4-nat9sqnt/lib/python3.10/site-packages/sklearn/model_selection/_split.py:700: UserWarning: The least populated class in y has only 1 members, which is less than n_splits=5.\n",
      "  warnings.warn(\n",
      "/home/potatosalad/.local/share/virtualenvs/CSC_homework_3_group4-nat9sqnt/lib/python3.10/site-packages/sklearn/model_selection/_split.py:700: UserWarning: The least populated class in y has only 2 members, which is less than n_splits=5.\n",
      "  warnings.warn(\n",
      "/home/potatosalad/.local/share/virtualenvs/CSC_homework_3_group4-nat9sqnt/lib/python3.10/site-packages/sklearn/model_selection/_split.py:700: UserWarning: The least populated class in y has only 1 members, which is less than n_splits=5.\n",
      "  warnings.warn(\n",
      "/home/potatosalad/.local/share/virtualenvs/CSC_homework_3_group4-nat9sqnt/lib/python3.10/site-packages/sklearn/model_selection/_split.py:700: UserWarning: The least populated class in y has only 3 members, which is less than n_splits=5.\n",
      "  warnings.warn(\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Model performance for Training set SVM_glass_Data_gridsearch\n",
      "- Accuracy: 0.9942\n",
      "- F1 score: 0.994067\n",
      "- Precision: 0.994245\n",
      "- Recall: 0.994152\n",
      "----------------------------------\n",
      "===================================\n",
      "Model performance for Test set SVM_glass_Data_gridsearch\n",
      "- Accuracy: 0.9767\n",
      "- F1 score: 0.975711\n",
      "- Precision: 0.981395\n",
      "- Recall: 0.976744\n",
      "----------------------------------\n",
      "===================================\n",
      "################################################\n",
      "################### ENSEMBLE ###################\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/potatosalad/.local/share/virtualenvs/CSC_homework_3_group4-nat9sqnt/lib/python3.10/site-packages/sklearn/linear_model/_logistic.py:458: ConvergenceWarning: lbfgs failed to converge (status=1):\n",
      "STOP: TOTAL NO. of ITERATIONS REACHED LIMIT.\n",
      "\n",
      "Increase the number of iterations (max_iter) or scale the data as shown in:\n",
      "    https://scikit-learn.org/stable/modules/preprocessing.html\n",
      "Please also refer to the documentation for alternative solver options:\n",
      "    https://scikit-learn.org/stable/modules/linear_model.html#logistic-regression\n",
      "  n_iter_i = _check_optimize_result(\n",
      "/home/potatosalad/.local/share/virtualenvs/CSC_homework_3_group4-nat9sqnt/lib/python3.10/site-packages/sklearn/linear_model/_logistic.py:458: ConvergenceWarning: lbfgs failed to converge (status=1):\n",
      "STOP: TOTAL NO. of ITERATIONS REACHED LIMIT.\n",
      "\n",
      "Increase the number of iterations (max_iter) or scale the data as shown in:\n",
      "    https://scikit-learn.org/stable/modules/preprocessing.html\n",
      "Please also refer to the documentation for alternative solver options:\n",
      "    https://scikit-learn.org/stable/modules/linear_model.html#logistic-regression\n",
      "  n_iter_i = _check_optimize_result(\n",
      "/home/potatosalad/.local/share/virtualenvs/CSC_homework_3_group4-nat9sqnt/lib/python3.10/site-packages/sklearn/linear_model/_logistic.py:458: ConvergenceWarning: lbfgs failed to converge (status=1):\n",
      "STOP: TOTAL NO. of ITERATIONS REACHED LIMIT.\n",
      "\n",
      "Increase the number of iterations (max_iter) or scale the data as shown in:\n",
      "    https://scikit-learn.org/stable/modules/preprocessing.html\n",
      "Please also refer to the documentation for alternative solver options:\n",
      "    https://scikit-learn.org/stable/modules/linear_model.html#logistic-regression\n",
      "  n_iter_i = _check_optimize_result(\n",
      "/home/potatosalad/.local/share/virtualenvs/CSC_homework_3_group4-nat9sqnt/lib/python3.10/site-packages/sklearn/linear_model/_logistic.py:458: ConvergenceWarning: lbfgs failed to converge (status=1):\n",
      "STOP: TOTAL NO. of ITERATIONS REACHED LIMIT.\n",
      "\n",
      "Increase the number of iterations (max_iter) or scale the data as shown in:\n",
      "    https://scikit-learn.org/stable/modules/preprocessing.html\n",
      "Please also refer to the documentation for alternative solver options:\n",
      "    https://scikit-learn.org/stable/modules/linear_model.html#logistic-regression\n",
      "  n_iter_i = _check_optimize_result(\n",
      "/home/potatosalad/.local/share/virtualenvs/CSC_homework_3_group4-nat9sqnt/lib/python3.10/site-packages/sklearn/linear_model/_logistic.py:458: ConvergenceWarning: lbfgs failed to converge (status=1):\n",
      "STOP: TOTAL NO. of ITERATIONS REACHED LIMIT.\n",
      "\n",
      "Increase the number of iterations (max_iter) or scale the data as shown in:\n",
      "    https://scikit-learn.org/stable/modules/preprocessing.html\n",
      "Please also refer to the documentation for alternative solver options:\n",
      "    https://scikit-learn.org/stable/modules/linear_model.html#logistic-regression\n",
      "  n_iter_i = _check_optimize_result(\n",
      "/home/potatosalad/.local/share/virtualenvs/CSC_homework_3_group4-nat9sqnt/lib/python3.10/site-packages/sklearn/linear_model/_logistic.py:458: ConvergenceWarning: lbfgs failed to converge (status=1):\n",
      "STOP: TOTAL NO. of ITERATIONS REACHED LIMIT.\n",
      "\n",
      "Increase the number of iterations (max_iter) or scale the data as shown in:\n",
      "    https://scikit-learn.org/stable/modules/preprocessing.html\n",
      "Please also refer to the documentation for alternative solver options:\n",
      "    https://scikit-learn.org/stable/modules/linear_model.html#logistic-regression\n",
      "  n_iter_i = _check_optimize_result(\n",
      "/home/potatosalad/.local/share/virtualenvs/CSC_homework_3_group4-nat9sqnt/lib/python3.10/site-packages/sklearn/linear_model/_logistic.py:458: ConvergenceWarning: lbfgs failed to converge (status=1):\n",
      "STOP: TOTAL NO. of ITERATIONS REACHED LIMIT.\n",
      "\n",
      "Increase the number of iterations (max_iter) or scale the data as shown in:\n",
      "    https://scikit-learn.org/stable/modules/preprocessing.html\n",
      "Please also refer to the documentation for alternative solver options:\n",
      "    https://scikit-learn.org/stable/modules/linear_model.html#logistic-regression\n",
      "  n_iter_i = _check_optimize_result(\n",
      "/home/potatosalad/.local/share/virtualenvs/CSC_homework_3_group4-nat9sqnt/lib/python3.10/site-packages/sklearn/linear_model/_logistic.py:458: ConvergenceWarning: lbfgs failed to converge (status=1):\n",
      "STOP: TOTAL NO. of ITERATIONS REACHED LIMIT.\n",
      "\n",
      "Increase the number of iterations (max_iter) or scale the data as shown in:\n",
      "    https://scikit-learn.org/stable/modules/preprocessing.html\n",
      "Please also refer to the documentation for alternative solver options:\n",
      "    https://scikit-learn.org/stable/modules/linear_model.html#logistic-regression\n",
      "  n_iter_i = _check_optimize_result(\n",
      "/home/potatosalad/.local/share/virtualenvs/CSC_homework_3_group4-nat9sqnt/lib/python3.10/site-packages/sklearn/linear_model/_logistic.py:458: ConvergenceWarning: lbfgs failed to converge (status=1):\n",
      "STOP: TOTAL NO. of ITERATIONS REACHED LIMIT.\n",
      "\n",
      "Increase the number of iterations (max_iter) or scale the data as shown in:\n",
      "    https://scikit-learn.org/stable/modules/preprocessing.html\n",
      "Please also refer to the documentation for alternative solver options:\n",
      "    https://scikit-learn.org/stable/modules/linear_model.html#logistic-regression\n",
      "  n_iter_i = _check_optimize_result(\n",
      "/home/potatosalad/.local/share/virtualenvs/CSC_homework_3_group4-nat9sqnt/lib/python3.10/site-packages/sklearn/linear_model/_logistic.py:458: ConvergenceWarning: lbfgs failed to converge (status=1):\n",
      "STOP: TOTAL NO. of ITERATIONS REACHED LIMIT.\n",
      "\n",
      "Increase the number of iterations (max_iter) or scale the data as shown in:\n",
      "    https://scikit-learn.org/stable/modules/preprocessing.html\n",
      "Please also refer to the documentation for alternative solver options:\n",
      "    https://scikit-learn.org/stable/modules/linear_model.html#logistic-regression\n",
      "  n_iter_i = _check_optimize_result(\n",
      "/home/potatosalad/.local/share/virtualenvs/CSC_homework_3_group4-nat9sqnt/lib/python3.10/site-packages/sklearn/linear_model/_logistic.py:458: ConvergenceWarning: lbfgs failed to converge (status=1):\n",
      "STOP: TOTAL NO. of ITERATIONS REACHED LIMIT.\n",
      "\n",
      "Increase the number of iterations (max_iter) or scale the data as shown in:\n",
      "    https://scikit-learn.org/stable/modules/preprocessing.html\n",
      "Please also refer to the documentation for alternative solver options:\n",
      "    https://scikit-learn.org/stable/modules/linear_model.html#logistic-regression\n",
      "  n_iter_i = _check_optimize_result(\n",
      "/home/potatosalad/.local/share/virtualenvs/CSC_homework_3_group4-nat9sqnt/lib/python3.10/site-packages/sklearn/linear_model/_logistic.py:458: ConvergenceWarning: lbfgs failed to converge (status=1):\n",
      "STOP: TOTAL NO. of ITERATIONS REACHED LIMIT.\n",
      "\n",
      "Increase the number of iterations (max_iter) or scale the data as shown in:\n",
      "    https://scikit-learn.org/stable/modules/preprocessing.html\n",
      "Please also refer to the documentation for alternative solver options:\n",
      "    https://scikit-learn.org/stable/modules/linear_model.html#logistic-regression\n",
      "  n_iter_i = _check_optimize_result(\n",
      "/home/potatosalad/.local/share/virtualenvs/CSC_homework_3_group4-nat9sqnt/lib/python3.10/site-packages/sklearn/linear_model/_logistic.py:458: ConvergenceWarning: lbfgs failed to converge (status=1):\n",
      "STOP: TOTAL NO. of ITERATIONS REACHED LIMIT.\n",
      "\n",
      "Increase the number of iterations (max_iter) or scale the data as shown in:\n",
      "    https://scikit-learn.org/stable/modules/preprocessing.html\n",
      "Please also refer to the documentation for alternative solver options:\n",
      "    https://scikit-learn.org/stable/modules/linear_model.html#logistic-regression\n",
      "  n_iter_i = _check_optimize_result(\n",
      "/home/potatosalad/.local/share/virtualenvs/CSC_homework_3_group4-nat9sqnt/lib/python3.10/site-packages/sklearn/linear_model/_logistic.py:458: ConvergenceWarning: lbfgs failed to converge (status=1):\n",
      "STOP: TOTAL NO. of ITERATIONS REACHED LIMIT.\n",
      "\n",
      "Increase the number of iterations (max_iter) or scale the data as shown in:\n",
      "    https://scikit-learn.org/stable/modules/preprocessing.html\n",
      "Please also refer to the documentation for alternative solver options:\n",
      "    https://scikit-learn.org/stable/modules/linear_model.html#logistic-regression\n",
      "  n_iter_i = _check_optimize_result(\n",
      "/home/potatosalad/.local/share/virtualenvs/CSC_homework_3_group4-nat9sqnt/lib/python3.10/site-packages/sklearn/linear_model/_logistic.py:458: ConvergenceWarning: lbfgs failed to converge (status=1):\n",
      "STOP: TOTAL NO. of ITERATIONS REACHED LIMIT.\n",
      "\n",
      "Increase the number of iterations (max_iter) or scale the data as shown in:\n",
      "    https://scikit-learn.org/stable/modules/preprocessing.html\n",
      "Please also refer to the documentation for alternative solver options:\n",
      "    https://scikit-learn.org/stable/modules/linear_model.html#logistic-regression\n",
      "  n_iter_i = _check_optimize_result(\n",
      "/home/potatosalad/.local/share/virtualenvs/CSC_homework_3_group4-nat9sqnt/lib/python3.10/site-packages/sklearn/linear_model/_logistic.py:458: ConvergenceWarning: lbfgs failed to converge (status=1):\n",
      "STOP: TOTAL NO. of ITERATIONS REACHED LIMIT.\n",
      "\n",
      "Increase the number of iterations (max_iter) or scale the data as shown in:\n",
      "    https://scikit-learn.org/stable/modules/preprocessing.html\n",
      "Please also refer to the documentation for alternative solver options:\n",
      "    https://scikit-learn.org/stable/modules/linear_model.html#logistic-regression\n",
      "  n_iter_i = _check_optimize_result(\n",
      "/home/potatosalad/.local/share/virtualenvs/CSC_homework_3_group4-nat9sqnt/lib/python3.10/site-packages/sklearn/linear_model/_logistic.py:458: ConvergenceWarning: lbfgs failed to converge (status=1):\n",
      "STOP: TOTAL NO. of ITERATIONS REACHED LIMIT.\n",
      "\n",
      "Increase the number of iterations (max_iter) or scale the data as shown in:\n",
      "    https://scikit-learn.org/stable/modules/preprocessing.html\n",
      "Please also refer to the documentation for alternative solver options:\n",
      "    https://scikit-learn.org/stable/modules/linear_model.html#logistic-regression\n",
      "  n_iter_i = _check_optimize_result(\n",
      "/home/potatosalad/.local/share/virtualenvs/CSC_homework_3_group4-nat9sqnt/lib/python3.10/site-packages/sklearn/linear_model/_logistic.py:458: ConvergenceWarning: lbfgs failed to converge (status=1):\n",
      "STOP: TOTAL NO. of ITERATIONS REACHED LIMIT.\n",
      "\n",
      "Increase the number of iterations (max_iter) or scale the data as shown in:\n",
      "    https://scikit-learn.org/stable/modules/preprocessing.html\n",
      "Please also refer to the documentation for alternative solver options:\n",
      "    https://scikit-learn.org/stable/modules/linear_model.html#logistic-regression\n",
      "  n_iter_i = _check_optimize_result(\n",
      "/home/potatosalad/.local/share/virtualenvs/CSC_homework_3_group4-nat9sqnt/lib/python3.10/site-packages/sklearn/linear_model/_logistic.py:458: ConvergenceWarning: lbfgs failed to converge (status=1):\n",
      "STOP: TOTAL NO. of ITERATIONS REACHED LIMIT.\n",
      "\n",
      "Increase the number of iterations (max_iter) or scale the data as shown in:\n",
      "    https://scikit-learn.org/stable/modules/preprocessing.html\n",
      "Please also refer to the documentation for alternative solver options:\n",
      "    https://scikit-learn.org/stable/modules/linear_model.html#logistic-regression\n",
      "  n_iter_i = _check_optimize_result(\n",
      "/home/potatosalad/.local/share/virtualenvs/CSC_homework_3_group4-nat9sqnt/lib/python3.10/site-packages/sklearn/linear_model/_logistic.py:458: ConvergenceWarning: lbfgs failed to converge (status=1):\n",
      "STOP: TOTAL NO. of ITERATIONS REACHED LIMIT.\n",
      "\n",
      "Increase the number of iterations (max_iter) or scale the data as shown in:\n",
      "    https://scikit-learn.org/stable/modules/preprocessing.html\n",
      "Please also refer to the documentation for alternative solver options:\n",
      "    https://scikit-learn.org/stable/modules/linear_model.html#logistic-regression\n",
      "  n_iter_i = _check_optimize_result(\n",
      "/home/potatosalad/.local/share/virtualenvs/CSC_homework_3_group4-nat9sqnt/lib/python3.10/site-packages/sklearn/linear_model/_logistic.py:458: ConvergenceWarning: lbfgs failed to converge (status=1):\n",
      "STOP: TOTAL NO. of ITERATIONS REACHED LIMIT.\n",
      "\n",
      "Increase the number of iterations (max_iter) or scale the data as shown in:\n",
      "    https://scikit-learn.org/stable/modules/preprocessing.html\n",
      "Please also refer to the documentation for alternative solver options:\n",
      "    https://scikit-learn.org/stable/modules/linear_model.html#logistic-regression\n",
      "  n_iter_i = _check_optimize_result(\n",
      "/home/potatosalad/.local/share/virtualenvs/CSC_homework_3_group4-nat9sqnt/lib/python3.10/site-packages/sklearn/linear_model/_logistic.py:458: ConvergenceWarning: lbfgs failed to converge (status=1):\n",
      "STOP: TOTAL NO. of ITERATIONS REACHED LIMIT.\n",
      "\n",
      "Increase the number of iterations (max_iter) or scale the data as shown in:\n",
      "    https://scikit-learn.org/stable/modules/preprocessing.html\n",
      "Please also refer to the documentation for alternative solver options:\n",
      "    https://scikit-learn.org/stable/modules/linear_model.html#logistic-regression\n",
      "  n_iter_i = _check_optimize_result(\n",
      "/home/potatosalad/.local/share/virtualenvs/CSC_homework_3_group4-nat9sqnt/lib/python3.10/site-packages/sklearn/linear_model/_logistic.py:458: ConvergenceWarning: lbfgs failed to converge (status=1):\n",
      "STOP: TOTAL NO. of ITERATIONS REACHED LIMIT.\n",
      "\n",
      "Increase the number of iterations (max_iter) or scale the data as shown in:\n",
      "    https://scikit-learn.org/stable/modules/preprocessing.html\n",
      "Please also refer to the documentation for alternative solver options:\n",
      "    https://scikit-learn.org/stable/modules/linear_model.html#logistic-regression\n",
      "  n_iter_i = _check_optimize_result(\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/potatosalad/.local/share/virtualenvs/CSC_homework_3_group4-nat9sqnt/lib/python3.10/site-packages/sklearn/linear_model/_logistic.py:458: ConvergenceWarning: lbfgs failed to converge (status=1):\n",
      "STOP: TOTAL NO. of ITERATIONS REACHED LIMIT.\n",
      "\n",
      "Increase the number of iterations (max_iter) or scale the data as shown in:\n",
      "    https://scikit-learn.org/stable/modules/preprocessing.html\n",
      "Please also refer to the documentation for alternative solver options:\n",
      "    https://scikit-learn.org/stable/modules/linear_model.html#logistic-regression\n",
      "  n_iter_i = _check_optimize_result(\n",
      "/home/potatosalad/.local/share/virtualenvs/CSC_homework_3_group4-nat9sqnt/lib/python3.10/site-packages/sklearn/linear_model/_logistic.py:458: ConvergenceWarning: lbfgs failed to converge (status=1):\n",
      "STOP: TOTAL NO. of ITERATIONS REACHED LIMIT.\n",
      "\n",
      "Increase the number of iterations (max_iter) or scale the data as shown in:\n",
      "    https://scikit-learn.org/stable/modules/preprocessing.html\n",
      "Please also refer to the documentation for alternative solver options:\n",
      "    https://scikit-learn.org/stable/modules/linear_model.html#logistic-regression\n",
      "  n_iter_i = _check_optimize_result(\n",
      "/home/potatosalad/.local/share/virtualenvs/CSC_homework_3_group4-nat9sqnt/lib/python3.10/site-packages/sklearn/linear_model/_logistic.py:458: ConvergenceWarning: lbfgs failed to converge (status=1):\n",
      "STOP: TOTAL NO. of ITERATIONS REACHED LIMIT.\n",
      "\n",
      "Increase the number of iterations (max_iter) or scale the data as shown in:\n",
      "    https://scikit-learn.org/stable/modules/preprocessing.html\n",
      "Please also refer to the documentation for alternative solver options:\n",
      "    https://scikit-learn.org/stable/modules/linear_model.html#logistic-regression\n",
      "  n_iter_i = _check_optimize_result(\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Model performance for train set ENSEMBLE_glass_Data_gridsearch\n",
      "- Accuracy: 1.0000\n",
      "- F1 score: 1.000000\n",
      "- Precision: 1.000000\n",
      "- Recall: 1.000000\n",
      "----------------------------------\n",
      "===================================\n",
      "Model performance for test set ENSEMBLE_glass_Data_gridsearch\n",
      "- Accuracy: 1.0000\n",
      "- F1 score: 1.000000\n",
      "- Precision: 1.000000\n",
      "- Recall: 1.000000\n",
      "----------------------------------\n",
      "===================================\n"
     ]
    }
   ],
   "source": [
    "# GRIDSEARCH\n",
    "fit_models(X_train, y_train, X_test, y_test,data_glass_target_names, models, params, \"glass_Data_gridsearch\", \"experiment_glass_gridsearch\",1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 80,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Fitting for parameters...\n",
      "X:      Id number: 1 to 214  RI: refractive index  \\\n",
      "79                    80               1.51590   \n",
      "161                  162               1.51934   \n",
      "109                  110               1.51818   \n",
      "127                  128               1.52081   \n",
      "95                    96               1.51860   \n",
      "..                   ...                   ...   \n",
      "106                  107               1.53125   \n",
      "14                    15               1.51763   \n",
      "92                    93               1.51588   \n",
      "179                  180               1.51852   \n",
      "102                  103               1.51820   \n",
      "\n",
      "     Na: Sodium (unit measurement: weight percent in corresponding oxide, as are attributes 4-10)  \\\n",
      "79                                               12.82                                              \n",
      "161                                              13.64                                              \n",
      "109                                              13.72                                              \n",
      "127                                              13.78                                              \n",
      "95                                               13.36                                              \n",
      "..                                                 ...                                              \n",
      "106                                              10.73                                              \n",
      "14                                               12.61                                              \n",
      "92                                               13.12                                              \n",
      "179                                              14.09                                              \n",
      "102                                              12.62                                              \n",
      "\n",
      "     Mg: Magnesium  Al: Aluminum  Si: Silicon  K: Potassium  Ca: Calcium  \\\n",
      "79            3.52          1.90        72.86          0.69         7.97   \n",
      "161           3.54          0.75        72.65          0.16         8.89   \n",
      "109           0.00          0.56        74.45          0.00        10.99   \n",
      "127           2.28          1.43        71.99          0.49         9.85   \n",
      "95            3.43          1.43        72.26          0.51         8.60   \n",
      "..             ...           ...          ...           ...          ...   \n",
      "106           0.00          2.10        69.81          0.58        13.30   \n",
      "14            3.59          1.31        73.29          0.58         8.50   \n",
      "92            3.41          1.58        73.26          0.07         8.39   \n",
      "179           2.19          1.66        72.67          0.00         9.32   \n",
      "102           2.76          0.83        73.81          0.35         9.42   \n",
      "\n",
      "     Ba: Barium  Fe: Iron  \n",
      "79         0.00      0.00  \n",
      "161        0.15      0.24  \n",
      "109        0.00      0.00  \n",
      "127        0.00      0.17  \n",
      "95         0.00      0.00  \n",
      "..          ...       ...  \n",
      "106        3.15      0.28  \n",
      "14         0.00      0.00  \n",
      "92         0.00      0.19  \n",
      "179        0.00      0.00  \n",
      "102        0.00      0.20  \n",
      "\n",
      "[171 rows x 10 columns]\n",
      "--------------------------------------------------\n",
      "Y: 79     2\n",
      "161    3\n",
      "109    2\n",
      "127    2\n",
      "95     2\n",
      "      ..\n",
      "106    2\n",
      "14     1\n",
      "92     2\n",
      "179    6\n",
      "102    2\n",
      "Name: Type of glass: (class attribute), Length: 171, dtype: int64\n",
      "--------------------------------------------------\n",
      "X_t:      Id number: 1 to 214  RI: refractive index  \\\n",
      "9                     10               1.51755   \n",
      "197                  198               1.51727   \n",
      "66                    67               1.52152   \n",
      "191                  192               1.51602   \n",
      "117                  118               1.51708   \n",
      "111                  112               1.52739   \n",
      "15                    16               1.51761   \n",
      "86                    87               1.51569   \n",
      "75                    76               1.51590   \n",
      "144                  145               1.51660   \n",
      "182                  183               1.51916   \n",
      "170                  171               1.52369   \n",
      "141                  142               1.51851   \n",
      "73                    74               1.51631   \n",
      "178                  179               1.51829   \n",
      "167                  168               1.51969   \n",
      "190                  191               1.51613   \n",
      "18                    19               1.51911   \n",
      "45                    46               1.51900   \n",
      "184                  185               1.51115   \n",
      "98                    99               1.51689   \n",
      "209                  210               1.51623   \n",
      "211                  212               1.52065   \n",
      "195                  196               1.51545   \n",
      "148                  149               1.51670   \n",
      "104                  105               1.52410   \n",
      "30                    31               1.51768   \n",
      "25                    26               1.51764   \n",
      "175                  176               1.52119   \n",
      "16                    17               1.51784   \n",
      "55                    56               1.51769   \n",
      "138                  139               1.51674   \n",
      "158                  159               1.51776   \n",
      "93                    94               1.51590   \n",
      "69                    70               1.52300   \n",
      "203                  204               1.51658   \n",
      "171                  172               1.51316   \n",
      "152                  153               1.51779   \n",
      "97                    98               1.51743   \n",
      "84                    85               1.51409   \n",
      "101                  102               1.51730   \n",
      "200                  201               1.51508   \n",
      "60                    61               1.51905   \n",
      "\n",
      "     Na: Sodium (unit measurement: weight percent in corresponding oxide, as are attributes 4-10)  \\\n",
      "9                                                13.00                                              \n",
      "197                                              14.70                                              \n",
      "66                                               13.05                                              \n",
      "191                                              14.85                                              \n",
      "117                                              13.72                                              \n",
      "111                                              11.02                                              \n",
      "15                                               12.81                                              \n",
      "86                                               13.24                                              \n",
      "75                                               13.02                                              \n",
      "144                                              12.99                                              \n",
      "182                                              14.15                                              \n",
      "170                                              13.44                                              \n",
      "141                                              13.20                                              \n",
      "73                                               13.34                                              \n",
      "178                                              14.46                                              \n",
      "167                                              12.64                                              \n",
      "190                                              13.88                                              \n",
      "18                                               13.90                                              \n",
      "45                                               13.49                                              \n",
      "184                                              17.38                                              \n",
      "98                                               12.67                                              \n",
      "209                                              14.14                                              \n",
      "211                                              14.36                                              \n",
      "195                                              14.14                                              \n",
      "148                                              13.24                                              \n",
      "104                                              13.83                                              \n",
      "30                                               12.65                                              \n",
      "25                                               12.98                                              \n",
      "175                                              12.97                                              \n",
      "16                                               12.68                                              \n",
      "55                                               12.45                                              \n",
      "138                                              12.79                                              \n",
      "158                                              13.53                                              \n",
      "93                                               13.24                                              \n",
      "69                                               13.31                                              \n",
      "203                                              14.80                                              \n",
      "171                                              13.02                                              \n",
      "152                                              13.64                                              \n",
      "97                                               12.20                                              \n",
      "84                                               14.25                                              \n",
      "101                                              12.35                                              \n",
      "200                                              15.15                                              \n",
      "60                                               13.60                                              \n",
      "\n",
      "     Mg: Magnesium  Al: Aluminum  Si: Silicon  K: Potassium  Ca: Calcium  \\\n",
      "9             3.60          1.36        72.99          0.57         8.40   \n",
      "197           0.00          2.34        73.28          0.00         8.95   \n",
      "66            3.65          0.87        72.22          0.19         9.85   \n",
      "191           0.00          2.38        73.28          0.00         8.76   \n",
      "117           3.68          1.81        72.06          0.64         7.88   \n",
      "111           0.00          0.75        73.08          0.00        14.96   \n",
      "15            3.54          1.23        73.24          0.58         8.39   \n",
      "86            3.49          1.47        73.25          0.38         8.03   \n",
      "75            3.58          1.51        73.12          0.69         7.96   \n",
      "144           3.18          1.23        72.97          0.58         8.81   \n",
      "182           0.00          2.09        72.74          0.00        10.88   \n",
      "170           0.00          1.58        72.22          0.32        12.24   \n",
      "141           3.63          1.07        72.83          0.57         8.41   \n",
      "73            3.57          1.57        72.87          0.61         7.89   \n",
      "178           2.24          1.62        72.38          0.00         9.26   \n",
      "167           0.00          1.65        73.75          0.38        11.53   \n",
      "190           1.78          1.79        73.10          0.00         8.67   \n",
      "18            3.73          1.18        72.12          0.06         8.89   \n",
      "45            3.48          1.35        71.95          0.55         9.00   \n",
      "184           0.00          0.34        75.41          0.00         6.65   \n",
      "98            2.88          1.71        73.21          0.73         8.54   \n",
      "209           0.00          2.88        72.61          0.08         9.18   \n",
      "211           0.00          2.02        73.42          0.00         8.44   \n",
      "195           0.00          2.68        73.39          0.08         9.07   \n",
      "148           3.57          1.38        72.70          0.56         8.44   \n",
      "104           2.90          1.17        71.15          0.08        10.79   \n",
      "30            3.56          1.30        73.08          0.61         8.69   \n",
      "25            3.54          1.21        73.00          0.65         8.53   \n",
      "175           0.33          1.51        73.39          0.13        11.27   \n",
      "16            3.67          1.16        73.11          0.61         8.70   \n",
      "55            2.71          1.29        73.70          0.56         9.06   \n",
      "138           3.52          1.54        73.36          0.66         7.90   \n",
      "158           3.41          1.52        72.04          0.58         8.79   \n",
      "93            3.34          1.47        73.10          0.39         8.22   \n",
      "69            3.58          0.82        71.99          0.12        10.17   \n",
      "203           0.00          1.99        73.11          0.00         8.28   \n",
      "171           0.00          3.04        70.48          6.21         6.96   \n",
      "152           3.65          0.65        73.00          0.06         8.93   \n",
      "97            3.25          1.16        73.55          0.62         8.90   \n",
      "84            3.09          2.08        72.28          1.10         7.08   \n",
      "101           2.72          1.63        72.87          0.70         9.23   \n",
      "200           0.00          2.25        73.50          0.00         8.34   \n",
      "60            3.62          1.11        72.64          0.14         8.76   \n",
      "\n",
      "     Ba: Barium  Fe: Iron  \n",
      "9          0.00      0.11  \n",
      "197        0.66      0.00  \n",
      "66         0.00      0.17  \n",
      "191        0.64      0.09  \n",
      "117        0.00      0.00  \n",
      "111        0.00      0.00  \n",
      "15         0.00      0.00  \n",
      "86         0.00      0.00  \n",
      "75         0.00      0.00  \n",
      "144        0.00      0.24  \n",
      "182        0.00      0.00  \n",
      "170        0.00      0.00  \n",
      "141        0.09      0.17  \n",
      "73         0.00      0.00  \n",
      "178        0.00      0.00  \n",
      "167        0.00      0.00  \n",
      "190        0.76      0.00  \n",
      "18         0.00      0.00  \n",
      "45         0.00      0.00  \n",
      "184        0.00      0.00  \n",
      "98         0.00      0.00  \n",
      "209        1.06      0.00  \n",
      "211        1.64      0.00  \n",
      "195        0.61      0.05  \n",
      "148        0.00      0.10  \n",
      "104        0.00      0.00  \n",
      "30         0.00      0.14  \n",
      "25         0.00      0.00  \n",
      "175        0.00      0.28  \n",
      "16         0.00      0.00  \n",
      "55         0.00      0.24  \n",
      "138        0.00      0.00  \n",
      "158        0.00      0.00  \n",
      "93         0.00      0.00  \n",
      "69         0.00      0.03  \n",
      "203        1.71      0.00  \n",
      "171        0.00      0.00  \n",
      "152        0.00      0.00  \n",
      "97         0.00      0.24  \n",
      "84         0.00      0.00  \n",
      "101        0.00      0.00  \n",
      "200        0.63      0.00  \n",
      "60         0.00      0.00  \n",
      "--------------------------------------------------\n",
      "Y_t: 9      1\n",
      "197    7\n",
      "66     1\n",
      "191    7\n",
      "117    2\n",
      "111    2\n",
      "15     1\n",
      "86     2\n",
      "75     2\n",
      "144    2\n",
      "182    6\n",
      "170    5\n",
      "141    2\n",
      "73     2\n",
      "178    6\n",
      "167    5\n",
      "190    7\n",
      "18     1\n",
      "45     1\n",
      "184    6\n",
      "98     2\n",
      "209    7\n",
      "211    7\n",
      "195    7\n",
      "148    3\n",
      "104    2\n",
      "30     1\n",
      "25     1\n",
      "175    5\n",
      "16     1\n",
      "55     1\n",
      "138    2\n",
      "158    3\n",
      "93     2\n",
      "69     1\n",
      "203    7\n",
      "171    5\n",
      "152    3\n",
      "97     2\n",
      "84     2\n",
      "101    2\n",
      "200    7\n",
      "60     1\n",
      "Name: Type of glass: (class attribute), dtype: int64\n",
      "--------------------------------------------------\n",
      "models: {'Logistic Regression': LogisticRegression(), 'Decision Tree': DecisionTreeClassifier(), 'Random Forest': RandomForestClassifier(random_state=42), 'Gradient Boosting': GradientBoostingClassifier(), 'K-Nearest Neighbors': KNeighborsClassifier(), 'SVM': SVC(probability=True)}\n",
      "--------------------------------------------------\n",
      "params: {'Logistic Regression': {'solver': ['newton-cholesky', 'liblinear', 'newton-cg', 'sag', 'saga'], 'penalty': ['l2'], 'C': [0.001, 0.01, 0.1, 1, 100], 'max_iter': [10000]}, 'Decision Tree': {'criterion': ['gini', 'entropy'], 'max_depth': [4, 6, 8, 10, 15, 50, 90]}, 'Random Forest': {'n_estimators': [200, 500], 'max_features': [None, 'sqrt', 'log2'], 'max_depth': [4, 5, 6, 7, 8], 'criterion': ['gini', 'entropy']}, 'Gradient Boosting': {'loss': ['log_loss'], 'learning_rate': [0.1, 0.01], 'max_depth': [3, 5, 8], 'max_features': ['log2', 'sqrt'], 'criterion': ['friedman_mse', 'squared_error'], 'subsample': [0.5, 0.8, 1.0], 'n_estimators': [10, 100, 1000]}, 'K-Nearest Neighbors': {'weights': ['uniform', 'distance'], 'leaf_size': [5, 15, 25]}, 'SVM': {'kernel': ['rbf'], 'C': [0.1, 1, 10, 100, 1000], 'degree': [1, 5], 'gamma': [1, 0.1, 0.01, 0.001]}}\n",
      "--------------------------------------------------\n",
      "run: glass_Data_no_gridsearch\n",
      "--------------------------------------------------\n",
      "folder: experiment_glass_no_gridsearch\n",
      "Logistic Regression\n",
      "\n",
      "##################################################\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Model performance for Training set Logistic Regression_glass_Data_no_gridsearch\n",
      "- Accuracy: 0.9766\n",
      "- F1 score: 0.976281\n",
      "- Precision: 0.978026\n",
      "- Recall: 0.976608\n",
      "----------------------------------\n",
      "===================================\n",
      "Model performance for Test set Logistic Regression_glass_Data_no_gridsearch\n",
      "- Accuracy: 0.9302\n",
      "- F1 score: 0.926155\n",
      "- Precision: 0.938630\n",
      "- Recall: 0.930233\n",
      "----------------------------------\n",
      "===================================\n",
      "Decision Tree\n",
      "\n",
      "##################################################\n",
      "Model performance for Training set Decision Tree_glass_Data_no_gridsearch\n",
      "- Accuracy: 1.0000\n",
      "- F1 score: 1.000000\n",
      "- Precision: 1.000000\n",
      "- Recall: 1.000000\n",
      "----------------------------------\n",
      "===================================\n",
      "Model performance for Test set Decision Tree_glass_Data_no_gridsearch\n",
      "- Accuracy: 0.9767\n",
      "- F1 score: 0.976591\n",
      "- Precision: 0.978295\n",
      "- Recall: 0.976744\n",
      "----------------------------------\n",
      "===================================\n",
      "Random Forest\n",
      "\n",
      "##################################################\n",
      "Model performance for Training set Random Forest_glass_Data_no_gridsearch\n",
      "- Accuracy: 1.0000\n",
      "- F1 score: 1.000000\n",
      "- Precision: 1.000000\n",
      "- Recall: 1.000000\n",
      "----------------------------------\n",
      "===================================\n",
      "Model performance for Test set Random Forest_glass_Data_no_gridsearch\n",
      "- Accuracy: 1.0000\n",
      "- F1 score: 1.000000\n",
      "- Precision: 1.000000\n",
      "- Recall: 1.000000\n",
      "----------------------------------\n",
      "===================================\n",
      "Gradient Boosting\n",
      "\n",
      "##################################################\n",
      "Model performance for Training set Gradient Boosting_glass_Data_no_gridsearch\n",
      "- Accuracy: 1.0000\n",
      "- F1 score: 1.000000\n",
      "- Precision: 1.000000\n",
      "- Recall: 1.000000\n",
      "----------------------------------\n",
      "===================================\n",
      "Model performance for Test set Gradient Boosting_glass_Data_no_gridsearch\n",
      "- Accuracy: 0.9767\n",
      "- F1 score: 0.976591\n",
      "- Precision: 0.978295\n",
      "- Recall: 0.976744\n",
      "----------------------------------\n",
      "===================================\n",
      "K-Nearest Neighbors\n",
      "\n",
      "##################################################\n",
      "Model performance for Training set K-Nearest Neighbors_glass_Data_no_gridsearch\n",
      "- Accuracy: 0.8889\n",
      "- F1 score: 0.884199\n",
      "- Precision: 0.888225\n",
      "- Recall: 0.888889\n",
      "----------------------------------\n",
      "===================================\n",
      "Model performance for Test set K-Nearest Neighbors_glass_Data_no_gridsearch\n",
      "- Accuracy: 0.8140\n",
      "- F1 score: 0.797535\n",
      "- Precision: 0.814599\n",
      "- Recall: 0.813953\n",
      "----------------------------------\n",
      "===================================\n",
      "SVM\n",
      "\n",
      "##################################################\n",
      "Model performance for Training set SVM_glass_Data_no_gridsearch\n",
      "- Accuracy: 0.9240\n",
      "- F1 score: 0.906435\n",
      "- Precision: 0.937154\n",
      "- Recall: 0.923977\n",
      "----------------------------------\n",
      "===================================\n",
      "Model performance for Test set SVM_glass_Data_no_gridsearch\n",
      "- Accuracy: 0.8605\n",
      "- F1 score: 0.846899\n",
      "- Precision: 0.869509\n",
      "- Recall: 0.860465\n",
      "----------------------------------\n",
      "===================================\n",
      "################################################\n",
      "################### ENSEMBLE ###################\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/potatosalad/.local/share/virtualenvs/CSC_homework_3_group4-nat9sqnt/lib/python3.10/site-packages/sklearn/linear_model/_logistic.py:458: ConvergenceWarning: lbfgs failed to converge (status=1):\n",
      "STOP: TOTAL NO. of ITERATIONS REACHED LIMIT.\n",
      "\n",
      "Increase the number of iterations (max_iter) or scale the data as shown in:\n",
      "    https://scikit-learn.org/stable/modules/preprocessing.html\n",
      "Please also refer to the documentation for alternative solver options:\n",
      "    https://scikit-learn.org/stable/modules/linear_model.html#logistic-regression\n",
      "  n_iter_i = _check_optimize_result(\n",
      "/home/potatosalad/.local/share/virtualenvs/CSC_homework_3_group4-nat9sqnt/lib/python3.10/site-packages/sklearn/linear_model/_logistic.py:458: ConvergenceWarning: lbfgs failed to converge (status=1):\n",
      "STOP: TOTAL NO. of ITERATIONS REACHED LIMIT.\n",
      "\n",
      "Increase the number of iterations (max_iter) or scale the data as shown in:\n",
      "    https://scikit-learn.org/stable/modules/preprocessing.html\n",
      "Please also refer to the documentation for alternative solver options:\n",
      "    https://scikit-learn.org/stable/modules/linear_model.html#logistic-regression\n",
      "  n_iter_i = _check_optimize_result(\n",
      "/home/potatosalad/.local/share/virtualenvs/CSC_homework_3_group4-nat9sqnt/lib/python3.10/site-packages/sklearn/linear_model/_logistic.py:458: ConvergenceWarning: lbfgs failed to converge (status=1):\n",
      "STOP: TOTAL NO. of ITERATIONS REACHED LIMIT.\n",
      "\n",
      "Increase the number of iterations (max_iter) or scale the data as shown in:\n",
      "    https://scikit-learn.org/stable/modules/preprocessing.html\n",
      "Please also refer to the documentation for alternative solver options:\n",
      "    https://scikit-learn.org/stable/modules/linear_model.html#logistic-regression\n",
      "  n_iter_i = _check_optimize_result(\n",
      "/home/potatosalad/.local/share/virtualenvs/CSC_homework_3_group4-nat9sqnt/lib/python3.10/site-packages/sklearn/linear_model/_logistic.py:458: ConvergenceWarning: lbfgs failed to converge (status=1):\n",
      "STOP: TOTAL NO. of ITERATIONS REACHED LIMIT.\n",
      "\n",
      "Increase the number of iterations (max_iter) or scale the data as shown in:\n",
      "    https://scikit-learn.org/stable/modules/preprocessing.html\n",
      "Please also refer to the documentation for alternative solver options:\n",
      "    https://scikit-learn.org/stable/modules/linear_model.html#logistic-regression\n",
      "  n_iter_i = _check_optimize_result(\n",
      "/home/potatosalad/.local/share/virtualenvs/CSC_homework_3_group4-nat9sqnt/lib/python3.10/site-packages/sklearn/linear_model/_logistic.py:458: ConvergenceWarning: lbfgs failed to converge (status=1):\n",
      "STOP: TOTAL NO. of ITERATIONS REACHED LIMIT.\n",
      "\n",
      "Increase the number of iterations (max_iter) or scale the data as shown in:\n",
      "    https://scikit-learn.org/stable/modules/preprocessing.html\n",
      "Please also refer to the documentation for alternative solver options:\n",
      "    https://scikit-learn.org/stable/modules/linear_model.html#logistic-regression\n",
      "  n_iter_i = _check_optimize_result(\n",
      "/home/potatosalad/.local/share/virtualenvs/CSC_homework_3_group4-nat9sqnt/lib/python3.10/site-packages/sklearn/linear_model/_logistic.py:458: ConvergenceWarning: lbfgs failed to converge (status=1):\n",
      "STOP: TOTAL NO. of ITERATIONS REACHED LIMIT.\n",
      "\n",
      "Increase the number of iterations (max_iter) or scale the data as shown in:\n",
      "    https://scikit-learn.org/stable/modules/preprocessing.html\n",
      "Please also refer to the documentation for alternative solver options:\n",
      "    https://scikit-learn.org/stable/modules/linear_model.html#logistic-regression\n",
      "  n_iter_i = _check_optimize_result(\n",
      "/home/potatosalad/.local/share/virtualenvs/CSC_homework_3_group4-nat9sqnt/lib/python3.10/site-packages/sklearn/linear_model/_logistic.py:458: ConvergenceWarning: lbfgs failed to converge (status=1):\n",
      "STOP: TOTAL NO. of ITERATIONS REACHED LIMIT.\n",
      "\n",
      "Increase the number of iterations (max_iter) or scale the data as shown in:\n",
      "    https://scikit-learn.org/stable/modules/preprocessing.html\n",
      "Please also refer to the documentation for alternative solver options:\n",
      "    https://scikit-learn.org/stable/modules/linear_model.html#logistic-regression\n",
      "  n_iter_i = _check_optimize_result(\n",
      "/home/potatosalad/.local/share/virtualenvs/CSC_homework_3_group4-nat9sqnt/lib/python3.10/site-packages/sklearn/linear_model/_logistic.py:458: ConvergenceWarning: lbfgs failed to converge (status=1):\n",
      "STOP: TOTAL NO. of ITERATIONS REACHED LIMIT.\n",
      "\n",
      "Increase the number of iterations (max_iter) or scale the data as shown in:\n",
      "    https://scikit-learn.org/stable/modules/preprocessing.html\n",
      "Please also refer to the documentation for alternative solver options:\n",
      "    https://scikit-learn.org/stable/modules/linear_model.html#logistic-regression\n",
      "  n_iter_i = _check_optimize_result(\n",
      "/home/potatosalad/.local/share/virtualenvs/CSC_homework_3_group4-nat9sqnt/lib/python3.10/site-packages/sklearn/linear_model/_logistic.py:458: ConvergenceWarning: lbfgs failed to converge (status=1):\n",
      "STOP: TOTAL NO. of ITERATIONS REACHED LIMIT.\n",
      "\n",
      "Increase the number of iterations (max_iter) or scale the data as shown in:\n",
      "    https://scikit-learn.org/stable/modules/preprocessing.html\n",
      "Please also refer to the documentation for alternative solver options:\n",
      "    https://scikit-learn.org/stable/modules/linear_model.html#logistic-regression\n",
      "  n_iter_i = _check_optimize_result(\n",
      "/home/potatosalad/.local/share/virtualenvs/CSC_homework_3_group4-nat9sqnt/lib/python3.10/site-packages/sklearn/linear_model/_logistic.py:458: ConvergenceWarning: lbfgs failed to converge (status=1):\n",
      "STOP: TOTAL NO. of ITERATIONS REACHED LIMIT.\n",
      "\n",
      "Increase the number of iterations (max_iter) or scale the data as shown in:\n",
      "    https://scikit-learn.org/stable/modules/preprocessing.html\n",
      "Please also refer to the documentation for alternative solver options:\n",
      "    https://scikit-learn.org/stable/modules/linear_model.html#logistic-regression\n",
      "  n_iter_i = _check_optimize_result(\n",
      "/home/potatosalad/.local/share/virtualenvs/CSC_homework_3_group4-nat9sqnt/lib/python3.10/site-packages/sklearn/linear_model/_logistic.py:458: ConvergenceWarning: lbfgs failed to converge (status=1):\n",
      "STOP: TOTAL NO. of ITERATIONS REACHED LIMIT.\n",
      "\n",
      "Increase the number of iterations (max_iter) or scale the data as shown in:\n",
      "    https://scikit-learn.org/stable/modules/preprocessing.html\n",
      "Please also refer to the documentation for alternative solver options:\n",
      "    https://scikit-learn.org/stable/modules/linear_model.html#logistic-regression\n",
      "  n_iter_i = _check_optimize_result(\n",
      "/home/potatosalad/.local/share/virtualenvs/CSC_homework_3_group4-nat9sqnt/lib/python3.10/site-packages/sklearn/linear_model/_logistic.py:458: ConvergenceWarning: lbfgs failed to converge (status=1):\n",
      "STOP: TOTAL NO. of ITERATIONS REACHED LIMIT.\n",
      "\n",
      "Increase the number of iterations (max_iter) or scale the data as shown in:\n",
      "    https://scikit-learn.org/stable/modules/preprocessing.html\n",
      "Please also refer to the documentation for alternative solver options:\n",
      "    https://scikit-learn.org/stable/modules/linear_model.html#logistic-regression\n",
      "  n_iter_i = _check_optimize_result(\n",
      "/home/potatosalad/.local/share/virtualenvs/CSC_homework_3_group4-nat9sqnt/lib/python3.10/site-packages/sklearn/linear_model/_logistic.py:458: ConvergenceWarning: lbfgs failed to converge (status=1):\n",
      "STOP: TOTAL NO. of ITERATIONS REACHED LIMIT.\n",
      "\n",
      "Increase the number of iterations (max_iter) or scale the data as shown in:\n",
      "    https://scikit-learn.org/stable/modules/preprocessing.html\n",
      "Please also refer to the documentation for alternative solver options:\n",
      "    https://scikit-learn.org/stable/modules/linear_model.html#logistic-regression\n",
      "  n_iter_i = _check_optimize_result(\n",
      "/home/potatosalad/.local/share/virtualenvs/CSC_homework_3_group4-nat9sqnt/lib/python3.10/site-packages/sklearn/linear_model/_logistic.py:458: ConvergenceWarning: lbfgs failed to converge (status=1):\n",
      "STOP: TOTAL NO. of ITERATIONS REACHED LIMIT.\n",
      "\n",
      "Increase the number of iterations (max_iter) or scale the data as shown in:\n",
      "    https://scikit-learn.org/stable/modules/preprocessing.html\n",
      "Please also refer to the documentation for alternative solver options:\n",
      "    https://scikit-learn.org/stable/modules/linear_model.html#logistic-regression\n",
      "  n_iter_i = _check_optimize_result(\n",
      "/home/potatosalad/.local/share/virtualenvs/CSC_homework_3_group4-nat9sqnt/lib/python3.10/site-packages/sklearn/linear_model/_logistic.py:458: ConvergenceWarning: lbfgs failed to converge (status=1):\n",
      "STOP: TOTAL NO. of ITERATIONS REACHED LIMIT.\n",
      "\n",
      "Increase the number of iterations (max_iter) or scale the data as shown in:\n",
      "    https://scikit-learn.org/stable/modules/preprocessing.html\n",
      "Please also refer to the documentation for alternative solver options:\n",
      "    https://scikit-learn.org/stable/modules/linear_model.html#logistic-regression\n",
      "  n_iter_i = _check_optimize_result(\n",
      "/home/potatosalad/.local/share/virtualenvs/CSC_homework_3_group4-nat9sqnt/lib/python3.10/site-packages/sklearn/linear_model/_logistic.py:458: ConvergenceWarning: lbfgs failed to converge (status=1):\n",
      "STOP: TOTAL NO. of ITERATIONS REACHED LIMIT.\n",
      "\n",
      "Increase the number of iterations (max_iter) or scale the data as shown in:\n",
      "    https://scikit-learn.org/stable/modules/preprocessing.html\n",
      "Please also refer to the documentation for alternative solver options:\n",
      "    https://scikit-learn.org/stable/modules/linear_model.html#logistic-regression\n",
      "  n_iter_i = _check_optimize_result(\n",
      "/home/potatosalad/.local/share/virtualenvs/CSC_homework_3_group4-nat9sqnt/lib/python3.10/site-packages/sklearn/linear_model/_logistic.py:458: ConvergenceWarning: lbfgs failed to converge (status=1):\n",
      "STOP: TOTAL NO. of ITERATIONS REACHED LIMIT.\n",
      "\n",
      "Increase the number of iterations (max_iter) or scale the data as shown in:\n",
      "    https://scikit-learn.org/stable/modules/preprocessing.html\n",
      "Please also refer to the documentation for alternative solver options:\n",
      "    https://scikit-learn.org/stable/modules/linear_model.html#logistic-regression\n",
      "  n_iter_i = _check_optimize_result(\n",
      "/home/potatosalad/.local/share/virtualenvs/CSC_homework_3_group4-nat9sqnt/lib/python3.10/site-packages/sklearn/linear_model/_logistic.py:458: ConvergenceWarning: lbfgs failed to converge (status=1):\n",
      "STOP: TOTAL NO. of ITERATIONS REACHED LIMIT.\n",
      "\n",
      "Increase the number of iterations (max_iter) or scale the data as shown in:\n",
      "    https://scikit-learn.org/stable/modules/preprocessing.html\n",
      "Please also refer to the documentation for alternative solver options:\n",
      "    https://scikit-learn.org/stable/modules/linear_model.html#logistic-regression\n",
      "  n_iter_i = _check_optimize_result(\n",
      "/home/potatosalad/.local/share/virtualenvs/CSC_homework_3_group4-nat9sqnt/lib/python3.10/site-packages/sklearn/linear_model/_logistic.py:458: ConvergenceWarning: lbfgs failed to converge (status=1):\n",
      "STOP: TOTAL NO. of ITERATIONS REACHED LIMIT.\n",
      "\n",
      "Increase the number of iterations (max_iter) or scale the data as shown in:\n",
      "    https://scikit-learn.org/stable/modules/preprocessing.html\n",
      "Please also refer to the documentation for alternative solver options:\n",
      "    https://scikit-learn.org/stable/modules/linear_model.html#logistic-regression\n",
      "  n_iter_i = _check_optimize_result(\n",
      "/home/potatosalad/.local/share/virtualenvs/CSC_homework_3_group4-nat9sqnt/lib/python3.10/site-packages/sklearn/linear_model/_logistic.py:458: ConvergenceWarning: lbfgs failed to converge (status=1):\n",
      "STOP: TOTAL NO. of ITERATIONS REACHED LIMIT.\n",
      "\n",
      "Increase the number of iterations (max_iter) or scale the data as shown in:\n",
      "    https://scikit-learn.org/stable/modules/preprocessing.html\n",
      "Please also refer to the documentation for alternative solver options:\n",
      "    https://scikit-learn.org/stable/modules/linear_model.html#logistic-regression\n",
      "  n_iter_i = _check_optimize_result(\n",
      "/home/potatosalad/.local/share/virtualenvs/CSC_homework_3_group4-nat9sqnt/lib/python3.10/site-packages/sklearn/linear_model/_logistic.py:458: ConvergenceWarning: lbfgs failed to converge (status=1):\n",
      "STOP: TOTAL NO. of ITERATIONS REACHED LIMIT.\n",
      "\n",
      "Increase the number of iterations (max_iter) or scale the data as shown in:\n",
      "    https://scikit-learn.org/stable/modules/preprocessing.html\n",
      "Please also refer to the documentation for alternative solver options:\n",
      "    https://scikit-learn.org/stable/modules/linear_model.html#logistic-regression\n",
      "  n_iter_i = _check_optimize_result(\n",
      "/home/potatosalad/.local/share/virtualenvs/CSC_homework_3_group4-nat9sqnt/lib/python3.10/site-packages/sklearn/linear_model/_logistic.py:458: ConvergenceWarning: lbfgs failed to converge (status=1):\n",
      "STOP: TOTAL NO. of ITERATIONS REACHED LIMIT.\n",
      "\n",
      "Increase the number of iterations (max_iter) or scale the data as shown in:\n",
      "    https://scikit-learn.org/stable/modules/preprocessing.html\n",
      "Please also refer to the documentation for alternative solver options:\n",
      "    https://scikit-learn.org/stable/modules/linear_model.html#logistic-regression\n",
      "  n_iter_i = _check_optimize_result(\n",
      "/home/potatosalad/.local/share/virtualenvs/CSC_homework_3_group4-nat9sqnt/lib/python3.10/site-packages/sklearn/linear_model/_logistic.py:458: ConvergenceWarning: lbfgs failed to converge (status=1):\n",
      "STOP: TOTAL NO. of ITERATIONS REACHED LIMIT.\n",
      "\n",
      "Increase the number of iterations (max_iter) or scale the data as shown in:\n",
      "    https://scikit-learn.org/stable/modules/preprocessing.html\n",
      "Please also refer to the documentation for alternative solver options:\n",
      "    https://scikit-learn.org/stable/modules/linear_model.html#logistic-regression\n",
      "  n_iter_i = _check_optimize_result(\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/potatosalad/.local/share/virtualenvs/CSC_homework_3_group4-nat9sqnt/lib/python3.10/site-packages/sklearn/linear_model/_logistic.py:458: ConvergenceWarning: lbfgs failed to converge (status=1):\n",
      "STOP: TOTAL NO. of ITERATIONS REACHED LIMIT.\n",
      "\n",
      "Increase the number of iterations (max_iter) or scale the data as shown in:\n",
      "    https://scikit-learn.org/stable/modules/preprocessing.html\n",
      "Please also refer to the documentation for alternative solver options:\n",
      "    https://scikit-learn.org/stable/modules/linear_model.html#logistic-regression\n",
      "  n_iter_i = _check_optimize_result(\n",
      "/home/potatosalad/.local/share/virtualenvs/CSC_homework_3_group4-nat9sqnt/lib/python3.10/site-packages/sklearn/linear_model/_logistic.py:458: ConvergenceWarning: lbfgs failed to converge (status=1):\n",
      "STOP: TOTAL NO. of ITERATIONS REACHED LIMIT.\n",
      "\n",
      "Increase the number of iterations (max_iter) or scale the data as shown in:\n",
      "    https://scikit-learn.org/stable/modules/preprocessing.html\n",
      "Please also refer to the documentation for alternative solver options:\n",
      "    https://scikit-learn.org/stable/modules/linear_model.html#logistic-regression\n",
      "  n_iter_i = _check_optimize_result(\n",
      "/home/potatosalad/.local/share/virtualenvs/CSC_homework_3_group4-nat9sqnt/lib/python3.10/site-packages/sklearn/linear_model/_logistic.py:458: ConvergenceWarning: lbfgs failed to converge (status=1):\n",
      "STOP: TOTAL NO. of ITERATIONS REACHED LIMIT.\n",
      "\n",
      "Increase the number of iterations (max_iter) or scale the data as shown in:\n",
      "    https://scikit-learn.org/stable/modules/preprocessing.html\n",
      "Please also refer to the documentation for alternative solver options:\n",
      "    https://scikit-learn.org/stable/modules/linear_model.html#logistic-regression\n",
      "  n_iter_i = _check_optimize_result(\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Model performance for train set ENSEMBLE_glass_Data_no_gridsearch\n",
      "- Accuracy: 1.0000\n",
      "- F1 score: 1.000000\n",
      "- Precision: 1.000000\n",
      "- Recall: 1.000000\n",
      "----------------------------------\n",
      "===================================\n",
      "Model performance for test set ENSEMBLE_glass_Data_no_gridsearch\n",
      "- Accuracy: 1.0000\n",
      "- F1 score: 1.000000\n",
      "- Precision: 1.000000\n",
      "- Recall: 1.000000\n",
      "----------------------------------\n",
      "===================================\n",
      "[Voting] ............ (2 of 6) Processing Decision Tree, total=   0.0s\n",
      "[Voting] ...... (5 of 6) Processing K-Nearest Neighbors, total=   0.0s\n",
      "[Voting] ...................... (6 of 6) Processing SVM, total=   0.0s\n",
      "[Voting] ...... (1 of 6) Processing Logistic Regression, total=   0.1s\n",
      "[Voting] ............ (3 of 6) Processing Random Forest, total=   0.2s\n",
      "[Voting] ........ (4 of 6) Processing Gradient Boosting, total=   0.5s\n",
      "[Voting] ............ (2 of 6) Processing Decision Tree, total=   0.0s\n",
      "[Voting] ...... (1 of 6) Processing Logistic Regression, total=   0.0s\n",
      "[Voting] ...... (5 of 6) Processing K-Nearest Neighbors, total=   0.0s\n",
      "[Voting] ...................... (6 of 6) Processing SVM, total=   0.0s\n",
      "[Voting] ............ (3 of 6) Processing Random Forest, total=   0.2s\n",
      "[Voting] ........ (4 of 6) Processing Gradient Boosting, total=   0.4s\n",
      "[Voting] ...................... (6 of 6) Processing SVM, total=   0.0s\n",
      "[Voting] ...... (5 of 6) Processing K-Nearest Neighbors, total=   0.0s\n",
      "[Voting] ............ (2 of 6) Processing Decision Tree, total=   0.0s\n",
      "[Voting] ...... (1 of 6) Processing Logistic Regression, total=   0.1s\n",
      "[Voting] ............ (3 of 6) Processing Random Forest, total=   0.2s\n",
      "[Voting] ........ (4 of 6) Processing Gradient Boosting, total=   0.5s\n",
      "[Voting] ............ (2 of 6) Processing Decision Tree, total=   0.0s\n",
      "[Voting] ...... (5 of 6) Processing K-Nearest Neighbors, total=   0.0s\n",
      "[Voting] ...................... (6 of 6) Processing SVM, total=   0.0s\n",
      "[Voting] ...... (1 of 6) Processing Logistic Regression, total=   0.1s\n",
      "[Voting] ............ (3 of 6) Processing Random Forest, total=   0.2s\n",
      "[Voting] ........ (4 of 6) Processing Gradient Boosting, total=   0.5s\n",
      "[Voting] ...... (5 of 6) Processing K-Nearest Neighbors, total=   0.0s\n",
      "[Voting] ............ (2 of 6) Processing Decision Tree, total=   0.0s\n",
      "[Voting] ...................... (6 of 6) Processing SVM, total=   0.0s\n",
      "[Voting] ...... (1 of 6) Processing Logistic Regression, total=   0.1s\n",
      "[Voting] ............ (3 of 6) Processing Random Forest, total=   0.2s\n",
      "[Voting] ........ (4 of 6) Processing Gradient Boosting, total=   0.6s\n",
      "[Voting] ............ (2 of 6) Processing Decision Tree, total=   0.0s\n",
      "[Voting] ...... (5 of 6) Processing K-Nearest Neighbors, total=   0.0s\n",
      "[Voting] ...................... (6 of 6) Processing SVM, total=   0.0s\n",
      "[Voting] ...... (1 of 6) Processing Logistic Regression, total=   0.1s\n",
      "[Voting] ............ (3 of 6) Processing Random Forest, total=   0.2s\n",
      "[Voting] ........ (4 of 6) Processing Gradient Boosting, total=   0.5s\n",
      "[Voting] ...... (5 of 6) Processing K-Nearest Neighbors, total=   0.0s\n",
      "[Voting] ...................... (6 of 6) Processing SVM, total=   0.0s\n",
      "[Voting] ............ (2 of 6) Processing Decision Tree, total=   0.0s\n",
      "[Voting] ...... (1 of 6) Processing Logistic Regression, total=   0.1s\n",
      "[Voting] ............ (3 of 6) Processing Random Forest, total=   0.2s\n",
      "[Voting] ........ (4 of 6) Processing Gradient Boosting, total=   0.3s\n",
      "[Voting] ........ (4 of 6) Processing Gradient Boosting, total=   0.3s\n",
      "[Voting] ...... (5 of 6) Processing K-Nearest Neighbors, total=   0.0s\n",
      "[Voting] ............ (2 of 6) Processing Decision Tree, total=   0.0s\n",
      "[Voting] ...................... (6 of 6) Processing SVM, total=   0.0s\n",
      "[Voting] ...... (1 of 6) Processing Logistic Regression, total=   0.1s\n",
      "[Voting] ............ (3 of 6) Processing Random Forest, total=   0.2s\n",
      "[Voting] ........ (4 of 6) Processing Gradient Boosting, total=   0.5s\n",
      "[Voting] ............ (2 of 6) Processing Decision Tree, total=   0.0s\n",
      "[Voting] ...... (5 of 6) Processing K-Nearest Neighbors, total=   0.0s\n",
      "[Voting] ...................... (6 of 6) Processing SVM, total=   0.0s\n",
      "[Voting] ...... (1 of 6) Processing Logistic Regression, total=   0.1s\n",
      "[Voting] ............ (3 of 6) Processing Random Forest, total=   0.3s\n",
      "[Voting] ........ (4 of 6) Processing Gradient Boosting, total=   0.5s\n",
      "[Voting] ............ (2 of 6) Processing Decision Tree, total=   0.0s\n",
      "[Voting] ...... (5 of 6) Processing K-Nearest Neighbors, total=   0.0s\n",
      "[Voting] ...................... (6 of 6) Processing SVM, total=   0.0s\n",
      "[Voting] ...... (1 of 6) Processing Logistic Regression, total=   0.1s\n",
      "[Voting] ............ (3 of 6) Processing Random Forest, total=   0.2s\n",
      "[Voting] ........ (4 of 6) Processing Gradient Boosting, total=   0.5s\n",
      "[Voting] ...... (5 of 6) Processing K-Nearest Neighbors, total=   0.0s\n",
      "[Voting] ...................... (6 of 6) Processing SVM, total=   0.0s\n",
      "[Voting] ............ (2 of 6) Processing Decision Tree, total=   0.0s\n",
      "[Voting] ...... (1 of 6) Processing Logistic Regression, total=   0.1s\n",
      "[Voting] ............ (3 of 6) Processing Random Forest, total=   0.2s\n",
      "[Voting] ........ (4 of 6) Processing Gradient Boosting, total=   0.5s\n",
      "[Voting] ...... (5 of 6) Processing K-Nearest Neighbors, total=   0.0s\n",
      "[Voting] ............ (2 of 6) Processing Decision Tree, total=   0.1s\n",
      "[Voting] ...................... (6 of 6) Processing SVM, total=   0.0s\n",
      "[Voting] ...... (1 of 6) Processing Logistic Regression, total=   0.1s\n",
      "[Voting] ............ (3 of 6) Processing Random Forest, total=   0.2s\n",
      "[Voting] ........ (4 of 6) Processing Gradient Boosting, total=   0.5s\n",
      "[Voting] ...... (5 of 6) Processing K-Nearest Neighbors, total=   0.0s\n",
      "[Voting] ............ (2 of 6) Processing Decision Tree, total=   0.0s\n",
      "[Voting] ...................... (6 of 6) Processing SVM, total=   0.0s\n",
      "[Voting] ...... (1 of 6) Processing Logistic Regression, total=   0.1s\n",
      "[Voting] ............ (3 of 6) Processing Random Forest, total=   0.2s\n",
      "[Voting] ........ (4 of 6) Processing Gradient Boosting, total=   0.5s\n",
      "[Voting] ...................... (6 of 6) Processing SVM, total=   0.0s\n",
      "[Voting] ............ (2 of 6) Processing Decision Tree, total=   0.0s\n",
      "[Voting] ...... (5 of 6) Processing K-Nearest Neighbors, total=   0.0s\n",
      "[Voting] ...................... (6 of 6) Processing SVM, total=   0.0s\n",
      "[Voting] ...... (1 of 6) Processing Logistic Regression, total=   0.1s\n",
      "[Voting] ............ (3 of 6) Processing Random Forest, total=   0.2s\n",
      "[Voting] ........ (4 of 6) Processing Gradient Boosting, total=   0.5s\n",
      "[Voting] ...... (5 of 6) Processing K-Nearest Neighbors, total=   0.0s\n",
      "[Voting] ...................... (6 of 6) Processing SVM, total=   0.0s\n",
      "[Voting] ............ (2 of 6) Processing Decision Tree, total=   0.0s\n",
      "[Voting] ...... (1 of 6) Processing Logistic Regression, total=   0.1s\n",
      "[Voting] ............ (3 of 6) Processing Random Forest, total=   0.3s\n",
      "[Voting] ........ (4 of 6) Processing Gradient Boosting, total=   0.5s\n",
      "[Voting] ...... (5 of 6) Processing K-Nearest Neighbors, total=   0.0s\n",
      "[Voting] ...................... (6 of 6) Processing SVM, total=   0.0s\n",
      "[Voting] ............ (2 of 6) Processing Decision Tree, total=   0.0s\n",
      "[Voting] ...... (1 of 6) Processing Logistic Regression, total=   0.1s\n",
      "[Voting] ............ (3 of 6) Processing Random Forest, total=   0.2s\n",
      "[Voting] ........ (4 of 6) Processing Gradient Boosting, total=   0.5s\n",
      "[Voting] ...... (5 of 6) Processing K-Nearest Neighbors, total=   0.0s\n",
      "[Voting] ...................... (6 of 6) Processing SVM, total=   0.0s\n",
      "[Voting] ............ (2 of 6) Processing Decision Tree, total=   0.0s\n",
      "[Voting] ...... (1 of 6) Processing Logistic Regression, total=   0.1s\n",
      "[Voting] ............ (3 of 6) Processing Random Forest, total=   0.2s\n",
      "[Voting] ........ (4 of 6) Processing Gradient Boosting, total=   0.3s\n",
      "[Voting] ...... (1 of 6) Processing Logistic Regression, total=   0.0s\n",
      "[Voting] ...... (5 of 6) Processing K-Nearest Neighbors, total=   0.0s\n",
      "[Voting] ............ (2 of 6) Processing Decision Tree, total=   0.0s\n",
      "[Voting] ...................... (6 of 6) Processing SVM, total=   0.0s\n",
      "[Voting] ...... (1 of 6) Processing Logistic Regression, total=   0.1s\n",
      "[Voting] ............ (3 of 6) Processing Random Forest, total=   0.2s\n",
      "[Voting] ........ (4 of 6) Processing Gradient Boosting, total=   0.5s\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[Voting] ............ (2 of 6) Processing Decision Tree, total=   0.0s\n",
      "[Voting] ...... (5 of 6) Processing K-Nearest Neighbors, total=   0.0s\n",
      "[Voting] ...................... (6 of 6) Processing SVM, total=   0.0s\n",
      "[Voting] ...... (1 of 6) Processing Logistic Regression, total=   0.1s\n",
      "[Voting] ............ (3 of 6) Processing Random Forest, total=   0.2s\n",
      "[Voting] ........ (4 of 6) Processing Gradient Boosting, total=   0.5s\n",
      "[Voting] ............ (2 of 6) Processing Decision Tree, total=   0.0s\n",
      "[Voting] ...... (5 of 6) Processing K-Nearest Neighbors, total=   0.0s\n",
      "[Voting] ...................... (6 of 6) Processing SVM, total=   0.0s\n",
      "[Voting] ...... (1 of 6) Processing Logistic Regression, total=   0.1s\n",
      "[Voting] ............ (3 of 6) Processing Random Forest, total=   0.2s\n",
      "[Voting] ........ (4 of 6) Processing Gradient Boosting, total=   0.5s\n",
      "[Voting] ............ (2 of 6) Processing Decision Tree, total=   0.0s\n",
      "[Voting] ...................... (6 of 6) Processing SVM, total=   0.0s\n",
      "[Voting] ...... (5 of 6) Processing K-Nearest Neighbors, total=   0.0s\n",
      "[Voting] ...... (1 of 6) Processing Logistic Regression, total=   0.1s\n",
      "[Voting] ............ (3 of 6) Processing Random Forest, total=   0.2s\n",
      "[Voting] ........ (4 of 6) Processing Gradient Boosting, total=   0.5s\n",
      "[Voting] ...... (5 of 6) Processing K-Nearest Neighbors, total=   0.0s\n",
      "[Voting] ............ (2 of 6) Processing Decision Tree, total=   0.0s\n",
      "[Voting] ...................... (6 of 6) Processing SVM, total=   0.0s\n",
      "[Voting] ...... (5 of 6) Processing K-Nearest Neighbors, total=   0.0s\n",
      "[Voting] ...... (1 of 6) Processing Logistic Regression, total=   0.1s\n",
      "[Voting] ............ (3 of 6) Processing Random Forest, total=   0.2s\n",
      "[Voting] ........ (4 of 6) Processing Gradient Boosting, total=   0.5s\n",
      "[Voting] ...... (5 of 6) Processing K-Nearest Neighbors, total=   0.0s\n",
      "[Voting] ............ (2 of 6) Processing Decision Tree, total=   0.0s\n",
      "[Voting] ...................... (6 of 6) Processing SVM, total=   0.0s\n",
      "[Voting] ...... (1 of 6) Processing Logistic Regression, total=   0.1s\n",
      "[Voting] ............ (3 of 6) Processing Random Forest, total=   0.2s\n",
      "[Voting] ........ (4 of 6) Processing Gradient Boosting, total=   0.5s\n",
      "[Voting] ...... (5 of 6) Processing K-Nearest Neighbors, total=   0.0s\n",
      "[Voting] ............ (2 of 6) Processing Decision Tree, total=   0.0s\n",
      "[Voting] ...................... (6 of 6) Processing SVM, total=   0.0s\n",
      "[Voting] ...... (1 of 6) Processing Logistic Regression, total=   0.1s\n",
      "[Voting] ............ (3 of 6) Processing Random Forest, total=   0.2s\n",
      "[Voting] ........ (4 of 6) Processing Gradient Boosting, total=   0.5s\n",
      "[Voting] ...... (5 of 6) Processing K-Nearest Neighbors, total=   0.0s\n",
      "[Voting] ............ (2 of 6) Processing Decision Tree, total=   0.0s\n",
      "[Voting] ...................... (6 of 6) Processing SVM, total=   0.0s\n",
      "[Voting] ...... (1 of 6) Processing Logistic Regression, total=   0.1s\n",
      "[Voting] ............ (3 of 6) Processing Random Forest, total=   0.3s\n",
      "[Voting] ........ (4 of 6) Processing Gradient Boosting, total=   0.3s\n",
      "[Voting] ............ (3 of 6) Processing Random Forest, total=   0.1s\n",
      "[Voting] ...... (5 of 6) Processing K-Nearest Neighbors, total=   0.0s\n",
      "[Voting] ............ (2 of 6) Processing Decision Tree, total=   0.0s\n",
      "[Voting] ...................... (6 of 6) Processing SVM, total=   0.0s\n",
      "[Voting] ...... (1 of 6) Processing Logistic Regression, total=   0.1s\n",
      "[Voting] ............ (3 of 6) Processing Random Forest, total=   0.2s\n",
      "[Voting] ........ (4 of 6) Processing Gradient Boosting, total=   0.6s\n",
      "[Voting] ............ (2 of 6) Processing Decision Tree, total=   0.0s\n",
      "[Voting] ...... (5 of 6) Processing K-Nearest Neighbors, total=   0.0s\n",
      "[Voting] ...................... (6 of 6) Processing SVM, total=   0.0s\n",
      "[Voting] ...... (1 of 6) Processing Logistic Regression, total=   0.1s\n",
      "[Voting] ............ (3 of 6) Processing Random Forest, total=   0.2s\n",
      "[Voting] ........ (4 of 6) Processing Gradient Boosting, total=   0.3s\n",
      "[Voting] ...... (5 of 6) Processing K-Nearest Neighbors, total=   0.0s\n",
      "[Voting] ............ (2 of 6) Processing Decision Tree, total=   0.0s\n",
      "[Voting] ...................... (6 of 6) Processing SVM, total=   0.0s\n",
      "[Voting] ...... (1 of 6) Processing Logistic Regression, total=   0.1s\n",
      "[Voting] ............ (3 of 6) Processing Random Forest, total=   0.2s\n",
      "[Voting] ........ (4 of 6) Processing Gradient Boosting, total=   0.4s\n",
      "[Voting] ...... (5 of 6) Processing K-Nearest Neighbors, total=   0.0s\n",
      "[Voting] ...................... (6 of 6) Processing SVM, total=   0.0s\n",
      "[Voting] ............ (2 of 6) Processing Decision Tree, total=   0.0s\n",
      "[Voting] ...... (1 of 6) Processing Logistic Regression, total=   0.1s\n",
      "[Voting] ............ (3 of 6) Processing Random Forest, total=   0.2s\n",
      "[Voting] ........ (4 of 6) Processing Gradient Boosting, total=   0.6s\n",
      "[Voting] ............ (2 of 6) Processing Decision Tree, total=   0.0s\n",
      "[Voting] ...................... (6 of 6) Processing SVM, total=   0.0s\n",
      "[Voting] ...... (5 of 6) Processing K-Nearest Neighbors, total=   0.1s\n",
      "[Voting] ...... (1 of 6) Processing Logistic Regression, total=   0.1s\n",
      "[Voting] ............ (3 of 6) Processing Random Forest, total=   0.2s\n",
      "[Voting] ........ (4 of 6) Processing Gradient Boosting, total=   0.3s\n",
      "[Voting] ............ (2 of 6) Processing Decision Tree, total=   0.0s\n",
      "[Voting] ...... (5 of 6) Processing K-Nearest Neighbors, total=   0.0s\n",
      "[Voting] ...................... (6 of 6) Processing SVM, total=   0.0s\n",
      "[Voting] ............ (2 of 6) Processing Decision Tree, total=   0.0s\n",
      "[Voting] ...... (1 of 6) Processing Logistic Regression, total=   0.1s\n",
      "[Voting] ............ (3 of 6) Processing Random Forest, total=   0.2s\n",
      "[Voting] ........ (4 of 6) Processing Gradient Boosting, total=   0.6s\n",
      "[Voting] ............ (2 of 6) Processing Decision Tree, total=   0.0s\n",
      "[Voting] ...... (5 of 6) Processing K-Nearest Neighbors, total=   0.0s\n",
      "[Voting] ...................... (6 of 6) Processing SVM, total=   0.0s\n",
      "[Voting] ...... (1 of 6) Processing Logistic Regression, total=   0.1s\n",
      "[Voting] ............ (3 of 6) Processing Random Forest, total=   0.2s\n",
      "[Voting] ........ (4 of 6) Processing Gradient Boosting, total=   0.6s\n",
      "[Voting] ...... (5 of 6) Processing K-Nearest Neighbors, total=   0.0s\n",
      "[Voting] ............ (2 of 6) Processing Decision Tree, total=   0.0s\n",
      "[Voting] ...................... (6 of 6) Processing SVM, total=   0.0s\n",
      "[Voting] ...... (1 of 6) Processing Logistic Regression, total=   0.1s\n",
      "[Voting] ............ (3 of 6) Processing Random Forest, total=   0.2s\n",
      "[Voting] ........ (4 of 6) Processing Gradient Boosting, total=   0.3s\n",
      "[Voting] ...... (5 of 6) Processing K-Nearest Neighbors, total=   0.0s\n",
      "[Voting] ...................... (6 of 6) Processing SVM, total=   0.0s\n",
      "[Voting] ............ (2 of 6) Processing Decision Tree, total=   0.0s\n",
      "[Voting] ...... (1 of 6) Processing Logistic Regression, total=   0.1s\n",
      "[Voting] ............ (3 of 6) Processing Random Forest, total=   0.2s\n",
      "[Voting] ........ (4 of 6) Processing Gradient Boosting, total=   0.4s\n",
      "[Voting] ...... (5 of 6) Processing K-Nearest Neighbors, total=   0.0s\n",
      "[Voting] ............ (2 of 6) Processing Decision Tree, total=   0.0s\n",
      "[Voting] ...................... (6 of 6) Processing SVM, total=   0.0s\n",
      "[Voting] ...... (1 of 6) Processing Logistic Regression, total=   0.1s\n",
      "[Voting] ............ (3 of 6) Processing Random Forest, total=   0.2s\n",
      "[Voting] ........ (4 of 6) Processing Gradient Boosting, total=   0.5s\n",
      "[Voting] ............ (2 of 6) Processing Decision Tree, total=   0.0s\n",
      "[Voting] ...... (5 of 6) Processing K-Nearest Neighbors, total=   0.0s\n",
      "[Voting] ...................... (6 of 6) Processing SVM, total=   0.0s\n",
      "[Voting] ...... (1 of 6) Processing Logistic Regression, total=   0.1s\n",
      "[Voting] ............ (3 of 6) Processing Random Forest, total=   0.2s\n",
      "[Voting] ........ (4 of 6) Processing Gradient Boosting, total=   0.3s\n",
      "[Voting] ............ (2 of 6) Processing Decision Tree, total=   0.0s\n",
      "[Voting] ...... (5 of 6) Processing K-Nearest Neighbors, total=   0.0s\n",
      "[Voting] ...................... (6 of 6) Processing SVM, total=   0.0s\n",
      "[Voting] ...... (1 of 6) Processing Logistic Regression, total=   0.1s\n",
      "[Voting] ............ (3 of 6) Processing Random Forest, total=   0.2s\n",
      "[Voting] ........ (4 of 6) Processing Gradient Boosting, total=   0.4s\n",
      "[Voting] ............ (2 of 6) Processing Decision Tree, total=   0.0s\n",
      "[Voting] ...... (5 of 6) Processing K-Nearest Neighbors, total=   0.0s\n",
      "[Voting] ...................... (6 of 6) Processing SVM, total=   0.0s\n",
      "[Voting] ...... (1 of 6) Processing Logistic Regression, total=   0.1s\n",
      "[Voting] ............ (3 of 6) Processing Random Forest, total=   0.2s\n",
      "[Voting] ........ (4 of 6) Processing Gradient Boosting, total=   0.5s\n",
      "[Voting] ...... (5 of 6) Processing K-Nearest Neighbors, total=   0.0s\n",
      "[Voting] ............ (2 of 6) Processing Decision Tree, total=   0.0s\n",
      "[Voting] ...................... (6 of 6) Processing SVM, total=   0.0s\n",
      "[Voting] ...... (1 of 6) Processing Logistic Regression, total=   0.1s\n",
      "[Voting] ............ (3 of 6) Processing Random Forest, total=   0.2s\n",
      "[Voting] ........ (4 of 6) Processing Gradient Boosting, total=   0.3s\n",
      "[Voting] ............ (2 of 6) Processing Decision Tree, total=   0.0s\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[Voting] ............ (2 of 6) Processing Decision Tree, total=   0.0s\n",
      "[Voting] ...... (5 of 6) Processing K-Nearest Neighbors, total=   0.0s\n",
      "[Voting] ...................... (6 of 6) Processing SVM, total=   0.0s\n",
      "[Voting] ...... (1 of 6) Processing Logistic Regression, total=   0.1s\n",
      "[Voting] ............ (3 of 6) Processing Random Forest, total=   0.3s\n",
      "[Voting] ........ (4 of 6) Processing Gradient Boosting, total=   0.3s\n",
      "[Voting] ...... (5 of 6) Processing K-Nearest Neighbors, total=   0.0s\n",
      "[Voting] ...................... (6 of 6) Processing SVM, total=   0.0s\n",
      "[Voting] ............ (2 of 6) Processing Decision Tree, total=   0.0s\n",
      "[Voting] ...... (1 of 6) Processing Logistic Regression, total=   0.1s\n",
      "[Voting] ............ (3 of 6) Processing Random Forest, total=   0.2s\n",
      "[Voting] ........ (4 of 6) Processing Gradient Boosting, total=   0.4s\n",
      "[Voting] ............ (2 of 6) Processing Decision Tree, total=   0.0s\n",
      "[Voting] ...... (5 of 6) Processing K-Nearest Neighbors, total=   0.0s\n",
      "[Voting] ...................... (6 of 6) Processing SVM, total=   0.0s\n",
      "[Voting] ...... (1 of 6) Processing Logistic Regression, total=   0.1s\n",
      "[Voting] ............ (3 of 6) Processing Random Forest, total=   0.2s\n",
      "[Voting] ........ (4 of 6) Processing Gradient Boosting, total=   0.4s\n",
      "[Voting] ...... (5 of 6) Processing K-Nearest Neighbors, total=   0.0s\n",
      "[Voting] ............ (2 of 6) Processing Decision Tree, total=   0.0s\n",
      "[Voting] ...... (5 of 6) Processing K-Nearest Neighbors, total=   0.0s\n",
      "[Voting] ...................... (6 of 6) Processing SVM, total=   0.0s\n",
      "[Voting] ...... (1 of 6) Processing Logistic Regression, total=   0.1s\n",
      "[Voting] ............ (3 of 6) Processing Random Forest, total=   0.2s\n",
      "[Voting] ........ (4 of 6) Processing Gradient Boosting, total=   0.6s\n",
      "[Voting] ...... (5 of 6) Processing K-Nearest Neighbors, total=   0.0s\n",
      "[Voting] ...................... (6 of 6) Processing SVM, total=   0.0s\n",
      "[Voting] ............ (2 of 6) Processing Decision Tree, total=   0.0s\n",
      "[Voting] ...... (1 of 6) Processing Logistic Regression, total=   0.0s\n",
      "[Voting] ............ (3 of 6) Processing Random Forest, total=   0.2s\n",
      "[Voting] ........ (4 of 6) Processing Gradient Boosting, total=   0.4s\n",
      "[Voting] ...................... (6 of 6) Processing SVM, total=   0.0s\n",
      "[Voting] ...................... (6 of 6) Processing SVM, total=   0.0s\n",
      "[Voting] ...... (5 of 6) Processing K-Nearest Neighbors, total=   0.0s\n",
      "[Voting] ............ (2 of 6) Processing Decision Tree, total=   0.0s\n",
      "[Voting] ...... (1 of 6) Processing Logistic Regression, total=   0.1s\n",
      "[Voting] ............ (3 of 6) Processing Random Forest, total=   0.2s\n",
      "[Voting] ........ (4 of 6) Processing Gradient Boosting, total=   0.5s\n",
      "[Voting] ...... (5 of 6) Processing K-Nearest Neighbors, total=   0.0s\n",
      "[Voting] ...................... (6 of 6) Processing SVM, total=   0.0s\n",
      "[Voting] ............ (2 of 6) Processing Decision Tree, total=   0.0s\n",
      "[Voting] ...... (1 of 6) Processing Logistic Regression, total=   0.1s\n",
      "[Voting] ............ (3 of 6) Processing Random Forest, total=   0.3s\n",
      "[Voting] ........ (4 of 6) Processing Gradient Boosting, total=   0.3s\n",
      "[Voting] ...... (1 of 6) Processing Logistic Regression, total=   0.0s\n",
      "[Voting] ...... (5 of 6) Processing K-Nearest Neighbors, total=   0.0s\n",
      "[Voting] ............ (2 of 6) Processing Decision Tree, total=   0.0s\n",
      "[Voting] ...................... (6 of 6) Processing SVM, total=   0.0s\n",
      "[Voting] ...... (1 of 6) Processing Logistic Regression, total=   0.1s\n",
      "[Voting] ............ (3 of 6) Processing Random Forest, total=   0.2s\n",
      "[Voting] ........ (4 of 6) Processing Gradient Boosting, total=   0.5s\n",
      "[Voting] ............ (2 of 6) Processing Decision Tree, total=   0.0s\n",
      "[Voting] ...... (5 of 6) Processing K-Nearest Neighbors, total=   0.0s\n",
      "[Voting] ...................... (6 of 6) Processing SVM, total=   0.0s\n",
      "[Voting] ...... (1 of 6) Processing Logistic Regression, total=   0.1s\n",
      "[Voting] ............ (3 of 6) Processing Random Forest, total=   0.2s\n",
      "[Voting] ........ (4 of 6) Processing Gradient Boosting, total=   0.3s\n",
      "[Voting] ............ (3 of 6) Processing Random Forest, total=   0.1s\n",
      "[Voting] ............ (2 of 6) Processing Decision Tree, total=   0.0s\n",
      "[Voting] ...... (5 of 6) Processing K-Nearest Neighbors, total=   0.0s\n",
      "[Voting] ...................... (6 of 6) Processing SVM, total=   0.0s\n",
      "[Voting] ...... (1 of 6) Processing Logistic Regression, total=   0.1s\n",
      "[Voting] ............ (3 of 6) Processing Random Forest, total=   0.2s\n",
      "[Voting] ........ (4 of 6) Processing Gradient Boosting, total=   0.5s\n",
      "[Voting] ...... (5 of 6) Processing K-Nearest Neighbors, total=   0.0s\n",
      "[Voting] ...................... (6 of 6) Processing SVM, total=   0.0s\n",
      "[Voting] ............ (2 of 6) Processing Decision Tree, total=   0.0s\n",
      "[Voting] ...... (1 of 6) Processing Logistic Regression, total=   0.1s\n",
      "[Voting] ............ (3 of 6) Processing Random Forest, total=   0.2s\n",
      "[Voting] ........ (4 of 6) Processing Gradient Boosting, total=   0.4s\n",
      "[Voting] ........ (4 of 6) Processing Gradient Boosting, total=   0.3s\n"
     ]
    }
   ],
   "source": [
    "# NO GRIDSEARCH\n",
    "fit_models(X_train, y_train, X_test, y_test,data_glass_target_names, models, params, \"glass_Data_no_gridsearch\", \"experiment_glass_no_gridsearch\",0)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## IRIS DATASET"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {
    "id": "sL6cDy2MHuaE"
   },
   "outputs": [],
   "source": [
    "iris = load_iris()\n",
    "target_names = iris.target_names\n",
    "X, y = iris.data, iris.target\n",
    "\n",
    "# X = df[0]\n",
    "# y = df[1]\n",
    "y = np.asarray(y)\n",
    "y = y.reshape(-1,)\n",
    "\n",
    "X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.20, random_state=42)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {
    "id": "wXFOys-dAcQQ",
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Fitting for parameters...\n",
      "X: [[4.6 3.6 1.  0.2]\n",
      " [5.7 4.4 1.5 0.4]\n",
      " [6.7 3.1 4.4 1.4]\n",
      " [4.8 3.4 1.6 0.2]\n",
      " [4.4 3.2 1.3 0.2]\n",
      " [6.3 2.5 5.  1.9]\n",
      " [6.4 3.2 4.5 1.5]\n",
      " [5.2 3.5 1.5 0.2]\n",
      " [5.  3.6 1.4 0.2]\n",
      " [5.2 4.1 1.5 0.1]\n",
      " [5.8 2.7 5.1 1.9]\n",
      " [6.  3.4 4.5 1.6]\n",
      " [6.7 3.1 4.7 1.5]\n",
      " [5.4 3.9 1.3 0.4]\n",
      " [5.4 3.7 1.5 0.2]\n",
      " [5.5 2.4 3.7 1. ]\n",
      " [6.3 2.8 5.1 1.5]\n",
      " [6.4 3.1 5.5 1.8]\n",
      " [6.6 3.  4.4 1.4]\n",
      " [7.2 3.6 6.1 2.5]\n",
      " [5.7 2.9 4.2 1.3]\n",
      " [7.6 3.  6.6 2.1]\n",
      " [5.6 3.  4.5 1.5]\n",
      " [5.1 3.5 1.4 0.2]\n",
      " [7.7 2.8 6.7 2. ]\n",
      " [5.8 2.7 4.1 1. ]\n",
      " [5.2 3.4 1.4 0.2]\n",
      " [5.  3.5 1.3 0.3]\n",
      " [5.1 3.8 1.9 0.4]\n",
      " [5.  2.  3.5 1. ]\n",
      " [6.3 2.7 4.9 1.8]\n",
      " [4.8 3.4 1.9 0.2]\n",
      " [5.  3.  1.6 0.2]\n",
      " [5.1 3.3 1.7 0.5]\n",
      " [5.6 2.7 4.2 1.3]\n",
      " [5.1 3.4 1.5 0.2]\n",
      " [5.7 3.  4.2 1.2]\n",
      " [7.7 3.8 6.7 2.2]\n",
      " [4.6 3.2 1.4 0.2]\n",
      " [6.2 2.9 4.3 1.3]\n",
      " [5.7 2.5 5.  2. ]\n",
      " [5.5 4.2 1.4 0.2]\n",
      " [6.  3.  4.8 1.8]\n",
      " [5.8 2.7 5.1 1.9]\n",
      " [6.  2.2 4.  1. ]\n",
      " [5.4 3.  4.5 1.5]\n",
      " [6.2 3.4 5.4 2.3]\n",
      " [5.5 2.3 4.  1.3]\n",
      " [5.4 3.9 1.7 0.4]\n",
      " [5.  2.3 3.3 1. ]\n",
      " [6.4 2.7 5.3 1.9]\n",
      " [5.  3.3 1.4 0.2]\n",
      " [5.  3.2 1.2 0.2]\n",
      " [5.5 2.4 3.8 1.1]\n",
      " [6.7 3.  5.  1.7]\n",
      " [4.9 3.1 1.5 0.2]\n",
      " [5.8 2.8 5.1 2.4]\n",
      " [5.  3.4 1.5 0.2]\n",
      " [5.  3.5 1.6 0.6]\n",
      " [5.9 3.2 4.8 1.8]\n",
      " [5.1 2.5 3.  1.1]\n",
      " [6.9 3.2 5.7 2.3]\n",
      " [6.  2.7 5.1 1.6]\n",
      " [6.1 2.6 5.6 1.4]\n",
      " [7.7 3.  6.1 2.3]\n",
      " [5.5 2.5 4.  1.3]\n",
      " [4.4 2.9 1.4 0.2]\n",
      " [4.3 3.  1.1 0.1]\n",
      " [6.  2.2 5.  1.5]\n",
      " [7.2 3.2 6.  1.8]\n",
      " [4.6 3.1 1.5 0.2]\n",
      " [5.1 3.5 1.4 0.3]\n",
      " [4.4 3.  1.3 0.2]\n",
      " [6.3 2.5 4.9 1.5]\n",
      " [6.3 3.4 5.6 2.4]\n",
      " [4.6 3.4 1.4 0.3]\n",
      " [6.8 3.  5.5 2.1]\n",
      " [6.3 3.3 6.  2.5]\n",
      " [4.7 3.2 1.3 0.2]\n",
      " [6.1 2.9 4.7 1.4]\n",
      " [6.5 2.8 4.6 1.5]\n",
      " [6.2 2.8 4.8 1.8]\n",
      " [7.  3.2 4.7 1.4]\n",
      " [6.4 3.2 5.3 2.3]\n",
      " [5.1 3.8 1.6 0.2]\n",
      " [6.9 3.1 5.4 2.1]\n",
      " [5.9 3.  4.2 1.5]\n",
      " [6.5 3.  5.2 2. ]\n",
      " [5.7 2.6 3.5 1. ]\n",
      " [5.2 2.7 3.9 1.4]\n",
      " [6.1 3.  4.6 1.4]\n",
      " [4.5 2.3 1.3 0.3]\n",
      " [6.6 2.9 4.6 1.3]\n",
      " [5.5 2.6 4.4 1.2]\n",
      " [5.3 3.7 1.5 0.2]\n",
      " [5.6 3.  4.1 1.3]\n",
      " [7.3 2.9 6.3 1.8]\n",
      " [6.7 3.3 5.7 2.1]\n",
      " [5.1 3.7 1.5 0.4]\n",
      " [4.9 2.4 3.3 1. ]\n",
      " [6.7 3.3 5.7 2.5]\n",
      " [7.2 3.  5.8 1.6]\n",
      " [4.9 3.6 1.4 0.1]\n",
      " [6.7 3.1 5.6 2.4]\n",
      " [4.9 3.  1.4 0.2]\n",
      " [6.9 3.1 4.9 1.5]\n",
      " [7.4 2.8 6.1 1.9]\n",
      " [6.3 2.9 5.6 1.8]\n",
      " [5.7 2.8 4.1 1.3]\n",
      " [6.5 3.  5.5 1.8]\n",
      " [6.3 2.3 4.4 1.3]\n",
      " [6.4 2.9 4.3 1.3]\n",
      " [5.6 2.8 4.9 2. ]\n",
      " [5.9 3.  5.1 1.8]\n",
      " [5.4 3.4 1.7 0.2]\n",
      " [6.1 2.8 4.  1.3]\n",
      " [4.9 2.5 4.5 1.7]\n",
      " [5.8 4.  1.2 0.2]\n",
      " [5.8 2.6 4.  1.2]\n",
      " [7.1 3.  5.9 2.1]]\n",
      "--------------------------------------------------\n",
      "Y: [0 0 1 0 0 2 1 0 0 0 2 1 1 0 0 1 2 2 1 2 1 2 1 0 2 1 0 0 0 1 2 0 0 0 1 0 1\n",
      " 2 0 1 2 0 2 2 1 1 2 1 0 1 2 0 0 1 1 0 2 0 0 1 1 2 1 2 2 1 0 0 2 2 0 0 0 1\n",
      " 2 0 2 2 0 1 1 2 1 2 0 2 1 2 1 1 1 0 1 1 0 1 2 2 0 1 2 2 0 2 0 1 2 2 1 2 1\n",
      " 1 2 2 0 1 2 0 1 2]\n",
      "--------------------------------------------------\n",
      "X_t: [[6.1 2.8 4.7 1.2]\n",
      " [5.7 3.8 1.7 0.3]\n",
      " [7.7 2.6 6.9 2.3]\n",
      " [6.  2.9 4.5 1.5]\n",
      " [6.8 2.8 4.8 1.4]\n",
      " [5.4 3.4 1.5 0.4]\n",
      " [5.6 2.9 3.6 1.3]\n",
      " [6.9 3.1 5.1 2.3]\n",
      " [6.2 2.2 4.5 1.5]\n",
      " [5.8 2.7 3.9 1.2]\n",
      " [6.5 3.2 5.1 2. ]\n",
      " [4.8 3.  1.4 0.1]\n",
      " [5.5 3.5 1.3 0.2]\n",
      " [4.9 3.1 1.5 0.1]\n",
      " [5.1 3.8 1.5 0.3]\n",
      " [6.3 3.3 4.7 1.6]\n",
      " [6.5 3.  5.8 2.2]\n",
      " [5.6 2.5 3.9 1.1]\n",
      " [5.7 2.8 4.5 1.3]\n",
      " [6.4 2.8 5.6 2.2]\n",
      " [4.7 3.2 1.6 0.2]\n",
      " [6.1 3.  4.9 1.8]\n",
      " [5.  3.4 1.6 0.4]\n",
      " [6.4 2.8 5.6 2.1]\n",
      " [7.9 3.8 6.4 2. ]\n",
      " [6.7 3.  5.2 2.3]\n",
      " [6.7 2.5 5.8 1.8]\n",
      " [6.8 3.2 5.9 2.3]\n",
      " [4.8 3.  1.4 0.3]\n",
      " [4.8 3.1 1.6 0.2]]\n",
      "--------------------------------------------------\n",
      "Y_t: [1 0 2 1 1 0 1 2 1 1 2 0 0 0 0 1 2 1 1 2 0 2 0 2 2 2 2 2 0 0]\n",
      "--------------------------------------------------\n",
      "models: {'Logistic Regression': LogisticRegression(), 'Decision Tree': DecisionTreeClassifier(), 'Random Forest': RandomForestClassifier(random_state=42), 'Gradient Boosting': GradientBoostingClassifier(), 'K-Nearest Neighbors': KNeighborsClassifier(), 'SVM': SVC(probability=True)}\n",
      "--------------------------------------------------\n",
      "params: {'Logistic Regression': {'solver': ['newton-cholesky', 'liblinear', 'newton-cg', 'sag', 'saga'], 'penalty': ['l2'], 'C': [0.001, 0.01, 0.1, 1, 100], 'max_iter': [10000]}, 'Decision Tree': {'criterion': ['gini', 'entropy'], 'max_depth': [4, 6, 8, 10, 15, 50, 90]}, 'Random Forest': {'n_estimators': [200, 500], 'max_features': [None, 'sqrt', 'log2'], 'max_depth': [4, 5, 6, 7, 8], 'criterion': ['gini', 'entropy']}, 'Gradient Boosting': {'loss': ['log_loss'], 'learning_rate': [0.1, 0.01], 'max_depth': [3, 5, 8], 'max_features': ['log2', 'sqrt'], 'criterion': ['friedman_mse', 'squared_error'], 'subsample': [0.5, 0.8, 1.0], 'n_estimators': [10, 100, 1000]}, 'K-Nearest Neighbors': {'weights': ['uniform', 'distance'], 'leaf_size': [5, 15, 25]}, 'SVM': {'kernel': ['rbf'], 'C': [0.1, 1, 10, 100, 1000], 'degree': [1, 5], 'gamma': [1, 0.1, 0.01, 0.001]}}\n",
      "--------------------------------------------------\n",
      "run: iris_Data\n",
      "--------------------------------------------------\n",
      "folder: experiment_iris_gridsearch\n",
      "Logistic Regression\n",
      "\n",
      "##################################################\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/potatosalad/.local/share/virtualenvs/CSC_homework_3_group4-nat9sqnt/lib/python3.10/site-packages/sklearn/model_selection/_split.py:700: UserWarning: The least populated class in y has only 1 members, which is less than n_splits=5.\n",
      "  warnings.warn(\n",
      "/home/potatosalad/.local/share/virtualenvs/CSC_homework_3_group4-nat9sqnt/lib/python3.10/site-packages/sklearn/model_selection/_split.py:700: UserWarning: The least populated class in y has only 1 members, which is less than n_splits=5.\n",
      "  warnings.warn(\n",
      "/home/potatosalad/.local/share/virtualenvs/CSC_homework_3_group4-nat9sqnt/lib/python3.10/site-packages/sklearn/model_selection/_split.py:700: UserWarning: The least populated class in y has only 1 members, which is less than n_splits=5.\n",
      "  warnings.warn(\n",
      "/home/potatosalad/.local/share/virtualenvs/CSC_homework_3_group4-nat9sqnt/lib/python3.10/site-packages/sklearn/model_selection/_split.py:700: UserWarning: The least populated class in y has only 1 members, which is less than n_splits=5.\n",
      "  warnings.warn(\n",
      "/home/potatosalad/.local/share/virtualenvs/CSC_homework_3_group4-nat9sqnt/lib/python3.10/site-packages/sklearn/model_selection/_split.py:700: UserWarning: The least populated class in y has only 1 members, which is less than n_splits=5.\n",
      "  warnings.warn(\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Model performance for Training set Logistic Regression_iris_Data\n",
      "- Accuracy: 0.9667\n",
      "- F1 score: 0.966667\n",
      "- Precision: 0.967459\n",
      "- Recall: 0.966667\n",
      "----------------------------------\n",
      "===================================\n",
      "Model performance for Test set Logistic Regression_iris_Data\n",
      "- Accuracy: 1.0000\n",
      "- F1 score: 1.000000\n",
      "- Precision: 1.000000\n",
      "- Recall: 1.000000\n",
      "----------------------------------\n",
      "===================================\n",
      "Decision Tree\n",
      "\n",
      "##################################################\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/potatosalad/.local/share/virtualenvs/CSC_homework_3_group4-nat9sqnt/lib/python3.10/site-packages/sklearn/model_selection/_split.py:700: UserWarning: The least populated class in y has only 1 members, which is less than n_splits=5.\n",
      "  warnings.warn(\n",
      "/home/potatosalad/.local/share/virtualenvs/CSC_homework_3_group4-nat9sqnt/lib/python3.10/site-packages/sklearn/model_selection/_split.py:700: UserWarning: The least populated class in y has only 1 members, which is less than n_splits=5.\n",
      "  warnings.warn(\n",
      "/home/potatosalad/.local/share/virtualenvs/CSC_homework_3_group4-nat9sqnt/lib/python3.10/site-packages/sklearn/model_selection/_split.py:700: UserWarning: The least populated class in y has only 1 members, which is less than n_splits=5.\n",
      "  warnings.warn(\n",
      "/home/potatosalad/.local/share/virtualenvs/CSC_homework_3_group4-nat9sqnt/lib/python3.10/site-packages/sklearn/model_selection/_split.py:700: UserWarning: The least populated class in y has only 1 members, which is less than n_splits=5.\n",
      "  warnings.warn(\n",
      "/home/potatosalad/.local/share/virtualenvs/CSC_homework_3_group4-nat9sqnt/lib/python3.10/site-packages/sklearn/model_selection/_split.py:700: UserWarning: The least populated class in y has only 1 members, which is less than n_splits=5.\n",
      "  warnings.warn(\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Model performance for Training set Decision Tree_iris_Data\n",
      "- Accuracy: 1.0000\n",
      "- F1 score: 1.000000\n",
      "- Precision: 1.000000\n",
      "- Recall: 1.000000\n",
      "----------------------------------\n",
      "===================================\n",
      "Model performance for Test set Decision Tree_iris_Data\n",
      "- Accuracy: 1.0000\n",
      "- F1 score: 1.000000\n",
      "- Precision: 1.000000\n",
      "- Recall: 1.000000\n",
      "----------------------------------\n",
      "===================================\n",
      "Random Forest\n",
      "\n",
      "##################################################\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/potatosalad/.local/share/virtualenvs/CSC_homework_3_group4-nat9sqnt/lib/python3.10/site-packages/sklearn/model_selection/_split.py:700: UserWarning: The least populated class in y has only 1 members, which is less than n_splits=5.\n",
      "  warnings.warn(\n",
      "/home/potatosalad/.local/share/virtualenvs/CSC_homework_3_group4-nat9sqnt/lib/python3.10/site-packages/sklearn/model_selection/_split.py:700: UserWarning: The least populated class in y has only 1 members, which is less than n_splits=5.\n",
      "  warnings.warn(\n",
      "/home/potatosalad/.local/share/virtualenvs/CSC_homework_3_group4-nat9sqnt/lib/python3.10/site-packages/sklearn/model_selection/_split.py:700: UserWarning: The least populated class in y has only 1 members, which is less than n_splits=5.\n",
      "  warnings.warn(\n",
      "/home/potatosalad/.local/share/virtualenvs/CSC_homework_3_group4-nat9sqnt/lib/python3.10/site-packages/sklearn/model_selection/_split.py:700: UserWarning: The least populated class in y has only 1 members, which is less than n_splits=5.\n",
      "  warnings.warn(\n",
      "/home/potatosalad/.local/share/virtualenvs/CSC_homework_3_group4-nat9sqnt/lib/python3.10/site-packages/sklearn/model_selection/_split.py:700: UserWarning: The least populated class in y has only 1 members, which is less than n_splits=5.\n",
      "  warnings.warn(\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Model performance for Training set Random Forest_iris_Data\n",
      "- Accuracy: 1.0000\n",
      "- F1 score: 1.000000\n",
      "- Precision: 1.000000\n",
      "- Recall: 1.000000\n",
      "----------------------------------\n",
      "===================================\n",
      "Model performance for Test set Random Forest_iris_Data\n",
      "- Accuracy: 1.0000\n",
      "- F1 score: 1.000000\n",
      "- Precision: 1.000000\n",
      "- Recall: 1.000000\n",
      "----------------------------------\n",
      "===================================\n",
      "Gradient Boosting\n",
      "\n",
      "##################################################\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/potatosalad/.local/share/virtualenvs/CSC_homework_3_group4-nat9sqnt/lib/python3.10/site-packages/sklearn/model_selection/_split.py:700: UserWarning: The least populated class in y has only 1 members, which is less than n_splits=5.\n",
      "  warnings.warn(\n",
      "/home/potatosalad/.local/share/virtualenvs/CSC_homework_3_group4-nat9sqnt/lib/python3.10/site-packages/sklearn/model_selection/_split.py:700: UserWarning: The least populated class in y has only 1 members, which is less than n_splits=5.\n",
      "  warnings.warn(\n",
      "/home/potatosalad/.local/share/virtualenvs/CSC_homework_3_group4-nat9sqnt/lib/python3.10/site-packages/sklearn/model_selection/_split.py:700: UserWarning: The least populated class in y has only 1 members, which is less than n_splits=5.\n",
      "  warnings.warn(\n",
      "/home/potatosalad/.local/share/virtualenvs/CSC_homework_3_group4-nat9sqnt/lib/python3.10/site-packages/sklearn/model_selection/_split.py:700: UserWarning: The least populated class in y has only 1 members, which is less than n_splits=5.\n",
      "  warnings.warn(\n",
      "/home/potatosalad/.local/share/virtualenvs/CSC_homework_3_group4-nat9sqnt/lib/python3.10/site-packages/sklearn/model_selection/_split.py:700: UserWarning: The least populated class in y has only 1 members, which is less than n_splits=5.\n",
      "  warnings.warn(\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Model performance for Training set Gradient Boosting_iris_Data\n",
      "- Accuracy: 0.9750\n",
      "- F1 score: 0.974988\n",
      "- Precision: 0.975178\n",
      "- Recall: 0.975000\n",
      "----------------------------------\n",
      "===================================\n",
      "Model performance for Test set Gradient Boosting_iris_Data\n",
      "- Accuracy: 1.0000\n",
      "- F1 score: 1.000000\n",
      "- Precision: 1.000000\n",
      "- Recall: 1.000000\n",
      "----------------------------------\n",
      "===================================\n",
      "K-Nearest Neighbors\n",
      "\n",
      "##################################################\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/potatosalad/.local/share/virtualenvs/CSC_homework_3_group4-nat9sqnt/lib/python3.10/site-packages/sklearn/model_selection/_split.py:700: UserWarning: The least populated class in y has only 1 members, which is less than n_splits=5.\n",
      "  warnings.warn(\n",
      "/home/potatosalad/.local/share/virtualenvs/CSC_homework_3_group4-nat9sqnt/lib/python3.10/site-packages/sklearn/model_selection/_split.py:700: UserWarning: The least populated class in y has only 1 members, which is less than n_splits=5.\n",
      "  warnings.warn(\n",
      "/home/potatosalad/.local/share/virtualenvs/CSC_homework_3_group4-nat9sqnt/lib/python3.10/site-packages/sklearn/model_selection/_split.py:700: UserWarning: The least populated class in y has only 1 members, which is less than n_splits=5.\n",
      "  warnings.warn(\n",
      "/home/potatosalad/.local/share/virtualenvs/CSC_homework_3_group4-nat9sqnt/lib/python3.10/site-packages/sklearn/model_selection/_split.py:700: UserWarning: The least populated class in y has only 1 members, which is less than n_splits=5.\n",
      "  warnings.warn(\n",
      "/home/potatosalad/.local/share/virtualenvs/CSC_homework_3_group4-nat9sqnt/lib/python3.10/site-packages/sklearn/model_selection/_split.py:700: UserWarning: The least populated class in y has only 1 members, which is less than n_splits=5.\n",
      "  warnings.warn(\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Model performance for Training set K-Nearest Neighbors_iris_Data\n",
      "- Accuracy: 1.0000\n",
      "- F1 score: 1.000000\n",
      "- Precision: 1.000000\n",
      "- Recall: 1.000000\n",
      "----------------------------------\n",
      "===================================\n",
      "Model performance for Test set K-Nearest Neighbors_iris_Data\n",
      "- Accuracy: 1.0000\n",
      "- F1 score: 1.000000\n",
      "- Precision: 1.000000\n",
      "- Recall: 1.000000\n",
      "----------------------------------\n",
      "===================================\n",
      "SVM\n",
      "\n",
      "##################################################\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/potatosalad/.local/share/virtualenvs/CSC_homework_3_group4-nat9sqnt/lib/python3.10/site-packages/sklearn/model_selection/_split.py:700: UserWarning: The least populated class in y has only 1 members, which is less than n_splits=5.\n",
      "  warnings.warn(\n",
      "/home/potatosalad/.local/share/virtualenvs/CSC_homework_3_group4-nat9sqnt/lib/python3.10/site-packages/sklearn/model_selection/_split.py:700: UserWarning: The least populated class in y has only 1 members, which is less than n_splits=5.\n",
      "  warnings.warn(\n",
      "/home/potatosalad/.local/share/virtualenvs/CSC_homework_3_group4-nat9sqnt/lib/python3.10/site-packages/sklearn/model_selection/_split.py:700: UserWarning: The least populated class in y has only 1 members, which is less than n_splits=5.\n",
      "  warnings.warn(\n",
      "/home/potatosalad/.local/share/virtualenvs/CSC_homework_3_group4-nat9sqnt/lib/python3.10/site-packages/sklearn/model_selection/_split.py:700: UserWarning: The least populated class in y has only 1 members, which is less than n_splits=5.\n",
      "  warnings.warn(\n",
      "/home/potatosalad/.local/share/virtualenvs/CSC_homework_3_group4-nat9sqnt/lib/python3.10/site-packages/sklearn/model_selection/_split.py:700: UserWarning: The least populated class in y has only 1 members, which is less than n_splits=5.\n",
      "  warnings.warn(\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Model performance for Training set SVM_iris_Data\n",
      "- Accuracy: 0.9667\n",
      "- F1 score: 0.966625\n",
      "- Precision: 0.969767\n",
      "- Recall: 0.966667\n",
      "----------------------------------\n",
      "===================================\n",
      "Model performance for Test set SVM_iris_Data\n",
      "- Accuracy: 0.9667\n",
      "- F1 score: 0.966411\n",
      "- Precision: 0.969444\n",
      "- Recall: 0.966667\n",
      "----------------------------------\n",
      "===================================\n",
      "################################################\n",
      "################### ENSEMBLE ###################\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/potatosalad/.local/share/virtualenvs/CSC_homework_3_group4-nat9sqnt/lib/python3.10/site-packages/sklearn/linear_model/_logistic.py:458: ConvergenceWarning: lbfgs failed to converge (status=1):\n",
      "STOP: TOTAL NO. of ITERATIONS REACHED LIMIT.\n",
      "\n",
      "Increase the number of iterations (max_iter) or scale the data as shown in:\n",
      "    https://scikit-learn.org/stable/modules/preprocessing.html\n",
      "Please also refer to the documentation for alternative solver options:\n",
      "    https://scikit-learn.org/stable/modules/linear_model.html#logistic-regression\n",
      "  n_iter_i = _check_optimize_result(\n",
      "/home/potatosalad/.local/share/virtualenvs/CSC_homework_3_group4-nat9sqnt/lib/python3.10/site-packages/sklearn/linear_model/_logistic.py:458: ConvergenceWarning: lbfgs failed to converge (status=1):\n",
      "STOP: TOTAL NO. of ITERATIONS REACHED LIMIT.\n",
      "\n",
      "Increase the number of iterations (max_iter) or scale the data as shown in:\n",
      "    https://scikit-learn.org/stable/modules/preprocessing.html\n",
      "Please also refer to the documentation for alternative solver options:\n",
      "    https://scikit-learn.org/stable/modules/linear_model.html#logistic-regression\n",
      "  n_iter_i = _check_optimize_result(\n",
      "/home/potatosalad/.local/share/virtualenvs/CSC_homework_3_group4-nat9sqnt/lib/python3.10/site-packages/sklearn/linear_model/_logistic.py:458: ConvergenceWarning: lbfgs failed to converge (status=1):\n",
      "STOP: TOTAL NO. of ITERATIONS REACHED LIMIT.\n",
      "\n",
      "Increase the number of iterations (max_iter) or scale the data as shown in:\n",
      "    https://scikit-learn.org/stable/modules/preprocessing.html\n",
      "Please also refer to the documentation for alternative solver options:\n",
      "    https://scikit-learn.org/stable/modules/linear_model.html#logistic-regression\n",
      "  n_iter_i = _check_optimize_result(\n",
      "/home/potatosalad/.local/share/virtualenvs/CSC_homework_3_group4-nat9sqnt/lib/python3.10/site-packages/sklearn/linear_model/_logistic.py:458: ConvergenceWarning: lbfgs failed to converge (status=1):\n",
      "STOP: TOTAL NO. of ITERATIONS REACHED LIMIT.\n",
      "\n",
      "Increase the number of iterations (max_iter) or scale the data as shown in:\n",
      "    https://scikit-learn.org/stable/modules/preprocessing.html\n",
      "Please also refer to the documentation for alternative solver options:\n",
      "    https://scikit-learn.org/stable/modules/linear_model.html#logistic-regression\n",
      "  n_iter_i = _check_optimize_result(\n",
      "/home/potatosalad/.local/share/virtualenvs/CSC_homework_3_group4-nat9sqnt/lib/python3.10/site-packages/sklearn/linear_model/_logistic.py:458: ConvergenceWarning: lbfgs failed to converge (status=1):\n",
      "STOP: TOTAL NO. of ITERATIONS REACHED LIMIT.\n",
      "\n",
      "Increase the number of iterations (max_iter) or scale the data as shown in:\n",
      "    https://scikit-learn.org/stable/modules/preprocessing.html\n",
      "Please also refer to the documentation for alternative solver options:\n",
      "    https://scikit-learn.org/stable/modules/linear_model.html#logistic-regression\n",
      "  n_iter_i = _check_optimize_result(\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Model performance for train set ENSEMBLE_iris_Data\n",
      "- Accuracy: 1.0000\n",
      "- F1 score: 1.000000\n",
      "- Precision: 1.000000\n",
      "- Recall: 1.000000\n",
      "----------------------------------\n",
      "===================================\n",
      "Model performance for test set ENSEMBLE_iris_Data\n",
      "- Accuracy: 1.0000\n",
      "- F1 score: 1.000000\n",
      "- Precision: 1.000000\n",
      "- Recall: 1.000000\n",
      "----------------------------------\n",
      "===================================\n"
     ]
    }
   ],
   "source": [
    "# GRIDSEARCH\n",
    "fit_models(X_train, y_train, X_test, y_test, iris.target_names, models, params, \"iris_Data\", \"experiment_iris_gridsearch\", 1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Fitting for parameters...\n",
      "X: [[4.6 3.6 1.  0.2]\n",
      " [5.7 4.4 1.5 0.4]\n",
      " [6.7 3.1 4.4 1.4]\n",
      " [4.8 3.4 1.6 0.2]\n",
      " [4.4 3.2 1.3 0.2]\n",
      " [6.3 2.5 5.  1.9]\n",
      " [6.4 3.2 4.5 1.5]\n",
      " [5.2 3.5 1.5 0.2]\n",
      " [5.  3.6 1.4 0.2]\n",
      " [5.2 4.1 1.5 0.1]\n",
      " [5.8 2.7 5.1 1.9]\n",
      " [6.  3.4 4.5 1.6]\n",
      " [6.7 3.1 4.7 1.5]\n",
      " [5.4 3.9 1.3 0.4]\n",
      " [5.4 3.7 1.5 0.2]\n",
      " [5.5 2.4 3.7 1. ]\n",
      " [6.3 2.8 5.1 1.5]\n",
      " [6.4 3.1 5.5 1.8]\n",
      " [6.6 3.  4.4 1.4]\n",
      " [7.2 3.6 6.1 2.5]\n",
      " [5.7 2.9 4.2 1.3]\n",
      " [7.6 3.  6.6 2.1]\n",
      " [5.6 3.  4.5 1.5]\n",
      " [5.1 3.5 1.4 0.2]\n",
      " [7.7 2.8 6.7 2. ]\n",
      " [5.8 2.7 4.1 1. ]\n",
      " [5.2 3.4 1.4 0.2]\n",
      " [5.  3.5 1.3 0.3]\n",
      " [5.1 3.8 1.9 0.4]\n",
      " [5.  2.  3.5 1. ]\n",
      " [6.3 2.7 4.9 1.8]\n",
      " [4.8 3.4 1.9 0.2]\n",
      " [5.  3.  1.6 0.2]\n",
      " [5.1 3.3 1.7 0.5]\n",
      " [5.6 2.7 4.2 1.3]\n",
      " [5.1 3.4 1.5 0.2]\n",
      " [5.7 3.  4.2 1.2]\n",
      " [7.7 3.8 6.7 2.2]\n",
      " [4.6 3.2 1.4 0.2]\n",
      " [6.2 2.9 4.3 1.3]\n",
      " [5.7 2.5 5.  2. ]\n",
      " [5.5 4.2 1.4 0.2]\n",
      " [6.  3.  4.8 1.8]\n",
      " [5.8 2.7 5.1 1.9]\n",
      " [6.  2.2 4.  1. ]\n",
      " [5.4 3.  4.5 1.5]\n",
      " [6.2 3.4 5.4 2.3]\n",
      " [5.5 2.3 4.  1.3]\n",
      " [5.4 3.9 1.7 0.4]\n",
      " [5.  2.3 3.3 1. ]\n",
      " [6.4 2.7 5.3 1.9]\n",
      " [5.  3.3 1.4 0.2]\n",
      " [5.  3.2 1.2 0.2]\n",
      " [5.5 2.4 3.8 1.1]\n",
      " [6.7 3.  5.  1.7]\n",
      " [4.9 3.1 1.5 0.2]\n",
      " [5.8 2.8 5.1 2.4]\n",
      " [5.  3.4 1.5 0.2]\n",
      " [5.  3.5 1.6 0.6]\n",
      " [5.9 3.2 4.8 1.8]\n",
      " [5.1 2.5 3.  1.1]\n",
      " [6.9 3.2 5.7 2.3]\n",
      " [6.  2.7 5.1 1.6]\n",
      " [6.1 2.6 5.6 1.4]\n",
      " [7.7 3.  6.1 2.3]\n",
      " [5.5 2.5 4.  1.3]\n",
      " [4.4 2.9 1.4 0.2]\n",
      " [4.3 3.  1.1 0.1]\n",
      " [6.  2.2 5.  1.5]\n",
      " [7.2 3.2 6.  1.8]\n",
      " [4.6 3.1 1.5 0.2]\n",
      " [5.1 3.5 1.4 0.3]\n",
      " [4.4 3.  1.3 0.2]\n",
      " [6.3 2.5 4.9 1.5]\n",
      " [6.3 3.4 5.6 2.4]\n",
      " [4.6 3.4 1.4 0.3]\n",
      " [6.8 3.  5.5 2.1]\n",
      " [6.3 3.3 6.  2.5]\n",
      " [4.7 3.2 1.3 0.2]\n",
      " [6.1 2.9 4.7 1.4]\n",
      " [6.5 2.8 4.6 1.5]\n",
      " [6.2 2.8 4.8 1.8]\n",
      " [7.  3.2 4.7 1.4]\n",
      " [6.4 3.2 5.3 2.3]\n",
      " [5.1 3.8 1.6 0.2]\n",
      " [6.9 3.1 5.4 2.1]\n",
      " [5.9 3.  4.2 1.5]\n",
      " [6.5 3.  5.2 2. ]\n",
      " [5.7 2.6 3.5 1. ]\n",
      " [5.2 2.7 3.9 1.4]\n",
      " [6.1 3.  4.6 1.4]\n",
      " [4.5 2.3 1.3 0.3]\n",
      " [6.6 2.9 4.6 1.3]\n",
      " [5.5 2.6 4.4 1.2]\n",
      " [5.3 3.7 1.5 0.2]\n",
      " [5.6 3.  4.1 1.3]\n",
      " [7.3 2.9 6.3 1.8]\n",
      " [6.7 3.3 5.7 2.1]\n",
      " [5.1 3.7 1.5 0.4]\n",
      " [4.9 2.4 3.3 1. ]\n",
      " [6.7 3.3 5.7 2.5]\n",
      " [7.2 3.  5.8 1.6]\n",
      " [4.9 3.6 1.4 0.1]\n",
      " [6.7 3.1 5.6 2.4]\n",
      " [4.9 3.  1.4 0.2]\n",
      " [6.9 3.1 4.9 1.5]\n",
      " [7.4 2.8 6.1 1.9]\n",
      " [6.3 2.9 5.6 1.8]\n",
      " [5.7 2.8 4.1 1.3]\n",
      " [6.5 3.  5.5 1.8]\n",
      " [6.3 2.3 4.4 1.3]\n",
      " [6.4 2.9 4.3 1.3]\n",
      " [5.6 2.8 4.9 2. ]\n",
      " [5.9 3.  5.1 1.8]\n",
      " [5.4 3.4 1.7 0.2]\n",
      " [6.1 2.8 4.  1.3]\n",
      " [4.9 2.5 4.5 1.7]\n",
      " [5.8 4.  1.2 0.2]\n",
      " [5.8 2.6 4.  1.2]\n",
      " [7.1 3.  5.9 2.1]]\n",
      "--------------------------------------------------\n",
      "Y: [0 0 1 0 0 2 1 0 0 0 2 1 1 0 0 1 2 2 1 2 1 2 1 0 2 1 0 0 0 1 2 0 0 0 1 0 1\n",
      " 2 0 1 2 0 2 2 1 1 2 1 0 1 2 0 0 1 1 0 2 0 0 1 1 2 1 2 2 1 0 0 2 2 0 0 0 1\n",
      " 2 0 2 2 0 1 1 2 1 2 0 2 1 2 1 1 1 0 1 1 0 1 2 2 0 1 2 2 0 2 0 1 2 2 1 2 1\n",
      " 1 2 2 0 1 2 0 1 2]\n",
      "--------------------------------------------------\n",
      "X_t: [[6.1 2.8 4.7 1.2]\n",
      " [5.7 3.8 1.7 0.3]\n",
      " [7.7 2.6 6.9 2.3]\n",
      " [6.  2.9 4.5 1.5]\n",
      " [6.8 2.8 4.8 1.4]\n",
      " [5.4 3.4 1.5 0.4]\n",
      " [5.6 2.9 3.6 1.3]\n",
      " [6.9 3.1 5.1 2.3]\n",
      " [6.2 2.2 4.5 1.5]\n",
      " [5.8 2.7 3.9 1.2]\n",
      " [6.5 3.2 5.1 2. ]\n",
      " [4.8 3.  1.4 0.1]\n",
      " [5.5 3.5 1.3 0.2]\n",
      " [4.9 3.1 1.5 0.1]\n",
      " [5.1 3.8 1.5 0.3]\n",
      " [6.3 3.3 4.7 1.6]\n",
      " [6.5 3.  5.8 2.2]\n",
      " [5.6 2.5 3.9 1.1]\n",
      " [5.7 2.8 4.5 1.3]\n",
      " [6.4 2.8 5.6 2.2]\n",
      " [4.7 3.2 1.6 0.2]\n",
      " [6.1 3.  4.9 1.8]\n",
      " [5.  3.4 1.6 0.4]\n",
      " [6.4 2.8 5.6 2.1]\n",
      " [7.9 3.8 6.4 2. ]\n",
      " [6.7 3.  5.2 2.3]\n",
      " [6.7 2.5 5.8 1.8]\n",
      " [6.8 3.2 5.9 2.3]\n",
      " [4.8 3.  1.4 0.3]\n",
      " [4.8 3.1 1.6 0.2]]\n",
      "--------------------------------------------------\n",
      "Y_t: [1 0 2 1 1 0 1 2 1 1 2 0 0 0 0 1 2 1 1 2 0 2 0 2 2 2 2 2 0 0]\n",
      "--------------------------------------------------\n",
      "models: {'Logistic Regression': LogisticRegression(), 'Decision Tree': DecisionTreeClassifier(), 'Random Forest': RandomForestClassifier(random_state=42), 'Gradient Boosting': GradientBoostingClassifier(), 'K-Nearest Neighbors': KNeighborsClassifier(), 'SVM': SVC(probability=True)}\n",
      "--------------------------------------------------\n",
      "params: {'Logistic Regression': {'solver': ['newton-cholesky', 'liblinear', 'newton-cg', 'sag', 'saga'], 'penalty': ['l2'], 'C': [0.001, 0.01, 0.1, 1, 100], 'max_iter': [10000]}, 'Decision Tree': {'criterion': ['gini', 'entropy'], 'max_depth': [4, 6, 8, 10, 15, 50, 90]}, 'Random Forest': {'n_estimators': [200, 500], 'max_features': [None, 'sqrt', 'log2'], 'max_depth': [4, 5, 6, 7, 8], 'criterion': ['gini', 'entropy']}, 'Gradient Boosting': {'loss': ['log_loss'], 'learning_rate': [0.1, 0.01], 'max_depth': [3, 5, 8], 'max_features': ['log2', 'sqrt'], 'criterion': ['friedman_mse', 'squared_error'], 'subsample': [0.5, 0.8, 1.0], 'n_estimators': [10, 100, 1000]}, 'K-Nearest Neighbors': {'weights': ['uniform', 'distance'], 'leaf_size': [5, 15, 25]}, 'SVM': {'kernel': ['rbf'], 'C': [0.1, 1, 10, 100, 1000], 'degree': [1, 5], 'gamma': [1, 0.1, 0.01, 0.001]}}\n",
      "--------------------------------------------------\n",
      "run: iris_Data_no_gridsearch\n",
      "--------------------------------------------------\n",
      "folder: experiment_iris_no_gridsearch\n",
      "Logistic Regression\n",
      "\n",
      "##################################################\n",
      "Model performance for Training set Logistic Regression_iris_Data_no_gridsearch\n",
      "- Accuracy: 0.9667\n",
      "- F1 score: 0.966667\n",
      "- Precision: 0.967459\n",
      "- Recall: 0.966667\n",
      "----------------------------------\n",
      "===================================\n",
      "Model performance for Test set Logistic Regression_iris_Data_no_gridsearch\n",
      "- Accuracy: 1.0000\n",
      "- F1 score: 1.000000\n",
      "- Precision: 1.000000\n",
      "- Recall: 1.000000\n",
      "----------------------------------\n",
      "===================================\n",
      "Decision Tree\n",
      "\n",
      "##################################################\n",
      "Model performance for Training set Decision Tree_iris_Data_no_gridsearch\n",
      "- Accuracy: 1.0000\n",
      "- F1 score: 1.000000\n",
      "- Precision: 1.000000\n",
      "- Recall: 1.000000\n",
      "----------------------------------\n",
      "===================================\n",
      "Model performance for Test set Decision Tree_iris_Data_no_gridsearch\n",
      "- Accuracy: 1.0000\n",
      "- F1 score: 1.000000\n",
      "- Precision: 1.000000\n",
      "- Recall: 1.000000\n",
      "----------------------------------\n",
      "===================================\n",
      "Random Forest\n",
      "\n",
      "##################################################\n",
      "Model performance for Training set Random Forest_iris_Data_no_gridsearch\n",
      "- Accuracy: 1.0000\n",
      "- F1 score: 1.000000\n",
      "- Precision: 1.000000\n",
      "- Recall: 1.000000\n",
      "----------------------------------\n",
      "===================================\n",
      "Model performance for Test set Random Forest_iris_Data_no_gridsearch\n",
      "- Accuracy: 1.0000\n",
      "- F1 score: 1.000000\n",
      "- Precision: 1.000000\n",
      "- Recall: 1.000000\n",
      "----------------------------------\n",
      "===================================\n",
      "Gradient Boosting\n",
      "\n",
      "##################################################\n",
      "Model performance for Training set Gradient Boosting_iris_Data_no_gridsearch\n",
      "- Accuracy: 1.0000\n",
      "- F1 score: 1.000000\n",
      "- Precision: 1.000000\n",
      "- Recall: 1.000000\n",
      "----------------------------------\n",
      "===================================\n",
      "Model performance for Test set Gradient Boosting_iris_Data_no_gridsearch\n",
      "- Accuracy: 1.0000\n",
      "- F1 score: 1.000000\n",
      "- Precision: 1.000000\n",
      "- Recall: 1.000000\n",
      "----------------------------------\n",
      "===================================\n",
      "K-Nearest Neighbors\n",
      "\n",
      "##################################################\n",
      "Model performance for Training set K-Nearest Neighbors_iris_Data_no_gridsearch\n",
      "- Accuracy: 0.9583\n",
      "- F1 score: 0.958314\n",
      "- Precision: 0.958490\n",
      "- Recall: 0.958333\n",
      "----------------------------------\n",
      "===================================\n",
      "Model performance for Test set K-Nearest Neighbors_iris_Data_no_gridsearch\n",
      "- Accuracy: 1.0000\n",
      "- F1 score: 1.000000\n",
      "- Precision: 1.000000\n",
      "- Recall: 1.000000\n",
      "----------------------------------\n",
      "===================================\n",
      "SVM\n",
      "\n",
      "##################################################\n",
      "Model performance for Training set SVM_iris_Data_no_gridsearch\n",
      "- Accuracy: 0.9667\n",
      "- F1 score: 0.966667\n",
      "- Precision: 0.967459\n",
      "- Recall: 0.966667\n",
      "----------------------------------\n",
      "===================================\n",
      "Model performance for Test set SVM_iris_Data_no_gridsearch\n",
      "- Accuracy: 1.0000\n",
      "- F1 score: 1.000000\n",
      "- Precision: 1.000000\n",
      "- Recall: 1.000000\n",
      "----------------------------------\n",
      "===================================\n",
      "################################################\n",
      "################### ENSEMBLE ###################\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/potatosalad/.local/share/virtualenvs/CSC_homework_3_group4-nat9sqnt/lib/python3.10/site-packages/sklearn/linear_model/_logistic.py:458: ConvergenceWarning: lbfgs failed to converge (status=1):\n",
      "STOP: TOTAL NO. of ITERATIONS REACHED LIMIT.\n",
      "\n",
      "Increase the number of iterations (max_iter) or scale the data as shown in:\n",
      "    https://scikit-learn.org/stable/modules/preprocessing.html\n",
      "Please also refer to the documentation for alternative solver options:\n",
      "    https://scikit-learn.org/stable/modules/linear_model.html#logistic-regression\n",
      "  n_iter_i = _check_optimize_result(\n",
      "/home/potatosalad/.local/share/virtualenvs/CSC_homework_3_group4-nat9sqnt/lib/python3.10/site-packages/sklearn/linear_model/_logistic.py:458: ConvergenceWarning: lbfgs failed to converge (status=1):\n",
      "STOP: TOTAL NO. of ITERATIONS REACHED LIMIT.\n",
      "\n",
      "Increase the number of iterations (max_iter) or scale the data as shown in:\n",
      "    https://scikit-learn.org/stable/modules/preprocessing.html\n",
      "Please also refer to the documentation for alternative solver options:\n",
      "    https://scikit-learn.org/stable/modules/linear_model.html#logistic-regression\n",
      "  n_iter_i = _check_optimize_result(\n",
      "/home/potatosalad/.local/share/virtualenvs/CSC_homework_3_group4-nat9sqnt/lib/python3.10/site-packages/sklearn/linear_model/_logistic.py:458: ConvergenceWarning: lbfgs failed to converge (status=1):\n",
      "STOP: TOTAL NO. of ITERATIONS REACHED LIMIT.\n",
      "\n",
      "Increase the number of iterations (max_iter) or scale the data as shown in:\n",
      "    https://scikit-learn.org/stable/modules/preprocessing.html\n",
      "Please also refer to the documentation for alternative solver options:\n",
      "    https://scikit-learn.org/stable/modules/linear_model.html#logistic-regression\n",
      "  n_iter_i = _check_optimize_result(\n",
      "/home/potatosalad/.local/share/virtualenvs/CSC_homework_3_group4-nat9sqnt/lib/python3.10/site-packages/sklearn/linear_model/_logistic.py:458: ConvergenceWarning: lbfgs failed to converge (status=1):\n",
      "STOP: TOTAL NO. of ITERATIONS REACHED LIMIT.\n",
      "\n",
      "Increase the number of iterations (max_iter) or scale the data as shown in:\n",
      "    https://scikit-learn.org/stable/modules/preprocessing.html\n",
      "Please also refer to the documentation for alternative solver options:\n",
      "    https://scikit-learn.org/stable/modules/linear_model.html#logistic-regression\n",
      "  n_iter_i = _check_optimize_result(\n",
      "/home/potatosalad/.local/share/virtualenvs/CSC_homework_3_group4-nat9sqnt/lib/python3.10/site-packages/sklearn/linear_model/_logistic.py:458: ConvergenceWarning: lbfgs failed to converge (status=1):\n",
      "STOP: TOTAL NO. of ITERATIONS REACHED LIMIT.\n",
      "\n",
      "Increase the number of iterations (max_iter) or scale the data as shown in:\n",
      "    https://scikit-learn.org/stable/modules/preprocessing.html\n",
      "Please also refer to the documentation for alternative solver options:\n",
      "    https://scikit-learn.org/stable/modules/linear_model.html#logistic-regression\n",
      "  n_iter_i = _check_optimize_result(\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Model performance for train set ENSEMBLE_iris_Data_no_gridsearch\n",
      "- Accuracy: 1.0000\n",
      "- F1 score: 1.000000\n",
      "- Precision: 1.000000\n",
      "- Recall: 1.000000\n",
      "----------------------------------\n",
      "===================================\n",
      "Model performance for test set ENSEMBLE_iris_Data_no_gridsearch\n",
      "- Accuracy: 1.0000\n",
      "- F1 score: 1.000000\n",
      "- Precision: 1.000000\n",
      "- Recall: 1.000000\n",
      "----------------------------------\n",
      "===================================\n"
     ]
    }
   ],
   "source": [
    "# NO GRIDSEARCH\n",
    "fit_models(X_train, y_train, X_test, y_test, iris.target_names, models, params, \"iris_Data_no_gridsearch\", \"experiment_iris_no_gridsearch\", 0)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# DEPLOYING THE VOTING SYSTEM"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 237,
   "metadata": {},
   "outputs": [],
   "source": [
    "def get_best_model_index(l):\n",
    "    best_i = 0\n",
    "    best_i_val = 0\n",
    "    for i in range(len(l)):\n",
    "        if l[i] > best_i_val:\n",
    "            best_i_val = l[i]\n",
    "            best_i = i\n",
    "        if l[i] == best_i_val:\n",
    "            best_i = random.choice([best_i, i])\n",
    "            best_i_val = l[best_i]\n",
    "    return best_i\n",
    "        \n",
    "def sortingPref(preference):\n",
    "    return np.argsort(preference)[::-1][:]\n",
    "\n",
    "def copeland(preferences):\n",
    "    ncandidates=len(preferences[0][0])\n",
    "    scores=np.zeros((len(preferences),ncandidates), dtype=\"i2\")\n",
    "    n_model=len(preferences[0])\n",
    "\n",
    "    '''\n",
    "    For each preference in the list of preferences\n",
    "    '''\n",
    "    for l in range(len(preferences)):\n",
    "        comparison = np.zeros((ncandidates,ncandidates), dtype=\"i2\")\n",
    "        profile=np.zeros((n_model,ncandidates), dtype=\"i2\")\n",
    "\n",
    "        profile = np.flip(np.argsort(preferences[l]))\n",
    "\n",
    "        for i in range(0, ncandidates):\n",
    "            for j in range(0,ncandidates):\n",
    "                _,i1=np.where(profile==i)\n",
    "                _,i2=np.where(profile==j)\n",
    "                comparison[i,j]=np.count_nonzero(np.less(i1,i2))\n",
    "        '''\n",
    "        Count how many pairwise is won or lose by any candidate\n",
    "        '''\n",
    "        comparison = comparison - comparison.T\n",
    "        '''\n",
    "        Remove negative numbers, i.e. candidates who lose pairwise\n",
    "        '''\n",
    "        comparison[comparison <0 ]=0\n",
    "        '''\n",
    "        For each candidate, count how many pairwise comparisons are won\n",
    "        '''\n",
    "        scores[l] = np.count_nonzero(comparison, axis=1)\n",
    "\n",
    "    return scores\n",
    "\n",
    "def rankaggr_brute(preferences):\n",
    "    '''\n",
    "    For each sample compute the score given the scoring vector\n",
    "    '''\n",
    "    n_candidates=len(preferences[0][0])\n",
    "    n_model=len(preferences[0])\n",
    "    n_samples = len(preferences)\n",
    "    scores=np.zeros((n_samples,n_candidates), dtype=\"f4\")\n",
    "   \n",
    "    for l in range(len(preferences)):\n",
    "        profile=[]\n",
    "        temp_ordered = np.flip(np.unique(preferences[l], axis=1), axis=1)\n",
    "        \n",
    "        for i in range(n_model):\n",
    "            temp = temp_ordered[i] \n",
    "            temp=[np.where(preferences[l][i]==temp[j])[0] for j in range(len(temp))]\n",
    "            profile.append(temp)\n",
    "\n",
    "        ranks = Dataset(profile)\n",
    "        sc = ScoringScheme()\n",
    "        if len(profile[0])>5:\n",
    "            consensus = KemRankAgg.compute_consensus(ranks, sc, Algorithm.ParCons)\n",
    "        else:\n",
    "            consensus = KemRankAgg.compute_consensus(ranks, sc, Algorithm.Exact)\n",
    "\n",
    "        for c in range(len(consensus.consensus_rankings[0])):\n",
    "            candidate = consensus.consensus_rankings[0][c][0]\n",
    "            scores[l][candidate] = n_candidates - c\n",
    "\n",
    "    return scores\n",
    "\n",
    "def scoringVec(scoring, preferences, weights=None):\n",
    "    n_candidate=len(preferences[0][0])\n",
    "    n_model=len(preferences[0])\n",
    "    if weights==None:\n",
    "        weights = np.ones(n_model, dtype=\"f4\")\n",
    "    if scoring is 'Plurality':\n",
    "        scoring=np.zeros(n_candidate, dtype='i2')\n",
    "        scoring[0]=1\n",
    "    if scoring is 'HalfApproval':\n",
    "        scoring=np.zeros(n_candidate, dtype='i2')\n",
    "        n=int(n_candidate/2)\n",
    "        if n_candidate%2 == 1: \n",
    "            n += 1\n",
    "        for i in range(n):\n",
    "            scoring[i]=1\n",
    "    if scoring is 'Borda':\n",
    "        scoring=[i for i in range(n_candidate)]\n",
    "        scoring.sort(reverse=True)\n",
    "\n",
    "    '''\n",
    "    For each sample compute the score given the scoring vector\n",
    "    '''\n",
    "    n_samples = len(preferences)\n",
    "    scores=np.zeros((n_samples,n_candidate), dtype=\"f4\")\n",
    "    for j in range(n_samples):\n",
    "        profile=np.zeros((n_model,n_candidate), dtype=\"i2\")\n",
    "        '''\n",
    "        For each model sort its preferences\n",
    "        '''\n",
    "        profile = np.flip(np.argsort(preferences[j]))\n",
    "        for i in range(n_candidate):\n",
    "            for m in range(n_model):\n",
    "                pos = profile[m,i]\n",
    "                scores[j,pos] += weights[m] * scoring[i]\n",
    "    return scores\n",
    "\n",
    "def votingSystem(scoring, models, x, y=[], classes=10, best_index=0, weights=None, proba=False):\n",
    "    if y != []:\n",
    "        y_val = np.reshape(y, (len(y), 1))\n",
    "    scores = []\n",
    "    if y != []:\n",
    "        fscores = []\n",
    "        \n",
    "    preferences = np.zeros((x.shape[0], len(models), classes))\n",
    "    for i in range(len(models)):\n",
    "        if hasattr(models[i], 'predict_proba'):\n",
    "            y_pred = models[i].predict_proba(x)\n",
    "        elif hasattr(models[i], 'decision_function'):\n",
    "            y_pred = models[i].decision_function(x)\n",
    "            y_pred = np.exp(y_pred) / np.sum(np.exp(y_pred), axis=1, keepdims=True)\n",
    "        else:\n",
    "            raise ValueError(\"Model doesn't have predict_proba or decision_function method\")\n",
    "        \n",
    "        preferences[:, i, :] = y_pred\n",
    "\n",
    "    '''\n",
    "    Compute voting winner for each sample given rankings from each model\n",
    "    '''\n",
    "    if scoring == 'Copeland':\n",
    "        scores = copeland(preferences)\n",
    "    elif scoring == \"Sum\":\n",
    "        scores = np.sum(preferences, axis=1)\n",
    "    elif scoring == \"Mean\":\n",
    "        scores = np.mean(preferences, axis=1)\n",
    "    elif scoring == \"Kemeny\":\n",
    "        scores = rankaggr_brute(preferences)\n",
    "    else:\n",
    "        scores = scoringVec(scoring, preferences, weights=weights)\n",
    "\n",
    "    if y != []:\n",
    "        return scores, fscores\n",
    "    return scores, None\n",
    "\n",
    "def predict(x, models, best_index, bestClassifier=False, nClasses=10, voting=\"Sum\", argMax=False, weighted=False, tiebreak=\"best\", epsilon=False ,weights=[]):\n",
    "    temp_fscores=None\n",
    "    if bestClassifier:\n",
    "        y_pred = models[best_index].predict(x)\n",
    "    else:\n",
    "        if weighted==False:\n",
    "            y_pred, temp_fscores=votingSystem(voting, models, x, classes=nClasses, proba=True)\n",
    "        else:\n",
    "            y_pred, temp_fscores=votingSystem(voting, models, x, classes=nClasses, proba=True, weights=weights)\n",
    "\n",
    "    if tiebreak is \"best\":\n",
    "        best_pred = models[best_index].predict(x)\n",
    "        return_y_pred=[]\n",
    "        if voting is \"Plurality\":\n",
    "            epsilon = 0.0\n",
    "\n",
    "        for i in range(len(y_pred)):\n",
    "            pred=y_pred[i]\n",
    "            winner = np.argwhere(pred == np.amax(pred))\n",
    "\n",
    "            if len(winner)>1: \n",
    "                pred=best_pred[i]\n",
    "            return_y_pred.append(pred)\n",
    "        y_pred=return_y_pred\n",
    "\n",
    "\n",
    "        if voting is \"Plurality\":\n",
    "            epsilon = epsilon / len(y_pred)\n",
    "\n",
    "    if argMax:\n",
    "        return np.argmax(y_pred, axis=1),temp_fscores\n",
    "\n",
    "    return y_pred, temp_fscores"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 76,
   "metadata": {},
   "outputs": [],
   "source": [
    "def get_model_weights(X_test, y_test, folder_path):\n",
    "\n",
    "    weights_gridsearch=[]\n",
    "    model_weights_gridsearch = {\n",
    "        \"Logistic Regression\"   :   0,\n",
    "        \"Random Forest\"         :   0,\n",
    "        \"Decision Tree\"         :   0,\n",
    "        \"K-Nearest Neighbors\"   :   0,\n",
    "        \"Gradient Boosting\"     :   0,\n",
    "        \"SVM\"                   :   0,\n",
    "        \"ensemble\"              :   0\n",
    "    }\n",
    "\n",
    "    # create an empty list to store the loaded files\n",
    "    loaded_models = []\n",
    "\n",
    "    # loop over the joblib files in the folder\n",
    "    for filename in os.listdir(folder_path):\n",
    "        if filename.endswith(\".joblib\"):\n",
    "            # load the joblib file\n",
    "            filepath = os.path.join(folder_path, filename)\n",
    "            loaded_model = joblib.load(filepath)\n",
    "\n",
    "            # add the loaded file to the list\n",
    "            loaded_models.append(loaded_model)\n",
    "\n",
    "    '''\n",
    "    Compute accuracy to store weights\n",
    "    '''\n",
    "    for m, i in zip(range(len(loaded_models)), model_weights_gridsearch):\n",
    "        print(loaded_models[m])\n",
    "        y_pred = loaded_models[m].predict(X_test)\n",
    "        tmp_f1 = f1_score(y_test, y_pred, average='weighted') \n",
    "        weights_gridsearch.append(tmp_f1)\n",
    "        model_weights_gridsearch[i] = round(tmp_f1/len(loaded_models) * 100,3)\n",
    "\n",
    "    return weights_gridsearch, model_weights_gridsearch, loaded_models"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## IRIS"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 276,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Pipeline(steps=[('standardscaler', StandardScaler()),\n",
      "                ('kneighborsclassifier', KNeighborsClassifier())])\n",
      "Pipeline(steps=[('standardscaler', StandardScaler()),\n",
      "                ('randomforestclassifier',\n",
      "                 RandomForestClassifier(random_state=42))])\n",
      "Pipeline(steps=[('standardscaler', StandardScaler()),\n",
      "                ('svc', SVC(probability=True))])\n",
      "Pipeline(steps=[('standardscaler', StandardScaler()),\n",
      "                ('decisiontreeclassifier', DecisionTreeClassifier())])\n",
      "Pipeline(steps=[('standardscaler', StandardScaler()),\n",
      "                ('logisticregression', LogisticRegression())])\n",
      "VotingClassifier(estimators=[('Logistic Regression', LogisticRegression()),\n",
      "                             ('Decision Tree', DecisionTreeClassifier()),\n",
      "                             ('Random Forest',\n",
      "                              RandomForestClassifier(random_state=42)),\n",
      "                             ('Gradient Boosting',\n",
      "                              GradientBoostingClassifier()),\n",
      "                             ('K-Nearest Neighbors', KNeighborsClassifier()),\n",
      "                             ('SVM', SVC(probability=True))],\n",
      "                 n_jobs=-2, verbose=True, voting='soft')\n",
      "Pipeline(steps=[('standardscaler', StandardScaler()),\n",
      "                ('gradientboostingclassifier', GradientBoostingClassifier())])\n"
     ]
    }
   ],
   "source": [
    "weights = []\n",
    "fscores = []\n",
    "saved_models = []\n",
    "best_index = 0\n",
    "epsilon = 0.0\n",
    "\n",
    "iris = load_iris()\n",
    "target_names = iris.target_names\n",
    "X, y = iris.data, iris.target\n",
    "y = np.asarray(y)\n",
    "y = y.reshape(-1,)\n",
    "X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.20, random_state=42)\n",
    "\n",
    "# define the folder containing the joblib files\n",
    "folder_path_gridsearch = \"./models/experiment_iris_gridsearch/\"\n",
    "folder_path_no_gridsearch = \"./models/experiment_iris_no_gridsearch/\"\n",
    "\n",
    "weights_gridsearch_iris_no_gridsearch, model_weights_iris_no_gridsearch, iris_models = get_model_weights(X_test, y_test, folder_path_no_gridsearch)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 277,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "MODEL WEIGHTS FOR IRIS WITHOUT GRIDSEARCH - USING TEST DATA\n",
      "[1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0]\n",
      "{'Logistic Regression': 14.286, 'Random Forest': 14.286, 'Decision Tree': 14.286, 'K-Nearest Neighbors': 14.286, 'Gradient Boosting': 14.286, 'SVM': 14.286, 'ensemble': 14.286}\n",
      "--------------------------------------------------\n",
      "6\n",
      "1.0\n"
     ]
    }
   ],
   "source": [
    "print(\"MODEL WEIGHTS FOR IRIS WITHOUT GRIDSEARCH - USING TEST DATA\")\n",
    "print(weights_gridsearch_iris_no_gridsearch)\n",
    "print(model_weights_iris_no_gridsearch)\n",
    "print(\"-\"*50)\n",
    "\n",
    "best_index = get_best_model_index(weights_gridsearch_iris_no_gridsearch)\n",
    "print(best_index)\n",
    "print(weights_gridsearch_iris_no_gridsearch[best_index])\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 278,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "id": "DpmfLCiaNkHd",
    "outputId": "7dc554ad-bf33-4dba-e630-f8c77227a46a",
    "scrolled": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[1 0 2 1 1 0 1 2 1 1 2 0 0 0 0 1 2 1 1 2 0 2 0 2 2 2 2 2 0 0]\n",
      "[1 0 2 1 1 0 1 2 1 1 2 0 0 0 0 1 2 1 1 2 0 2 0 2 2 2 2 2 0 0]\n",
      "1.0\n",
      "[[10  0  0]\n",
      " [ 0  9  0]\n",
      " [ 0  0 11]]\n"
     ]
    }
   ],
   "source": [
    "y_pred_vorace,_ = predict(X_test, iris_models, best_index, False, len(np.unique(y_test)), \"Plurality\", True, False, \"best\", False, weights_gridsearch_iris_no_gridsearch)\n",
    "print(y_pred_vorace)\n",
    "print(y_test)\n",
    "accuracy = accuracy_score(y_test, y_pred_vorace)\n",
    "print(accuracy)\n",
    "conf_mat = confusion_matrix(y_test, y_pred_vorace) \n",
    "print(conf_mat)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 279,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "id": "dbrabZXMNt6i",
    "outputId": "69c9b6d2-b506-4632-e3ab-8c6c4feda7c2",
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[1 0 2 1 1 0 1 2 1 1 2 0 0 0 0 1 2 1 1 2 0 2 0 2 2 2 2 2 0 0]\n",
      "[1 0 2 1 1 0 1 2 1 1 2 0 0 0 0 1 2 1 1 2 0 2 0 2 2 2 2 2 0 0]\n",
      "1.0\n",
      "[[10  0  0]\n",
      " [ 0  9  0]\n",
      " [ 0  0 11]]\n"
     ]
    }
   ],
   "source": [
    "y_pred_vorace,_ = predict(X_test, iris_models, best_index, False, len(np.unique(y_test)), \"Sum\", True, False, \"best\", False, weights_gridsearch_iris_no_gridsearch)\n",
    "print(y_pred_vorace)\n",
    "print(y_test)\n",
    "accuracy = accuracy_score(y_test, y_pred_vorace)\n",
    "print(accuracy)\n",
    "conf_mat = confusion_matrix(y_test, y_pred_vorace) \n",
    "print(conf_mat)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 280,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[1 0 2 1 1 0 1 2 1 1 2 0 0 0 0 1 2 1 1 2 0 2 0 2 2 2 2 2 0 0]\n",
      "[1 0 2 1 1 0 1 2 1 1 2 0 0 0 0 1 2 1 1 2 0 2 0 2 2 2 2 2 0 0]\n",
      "1.0\n",
      "[[10  0  0]\n",
      " [ 0  9  0]\n",
      " [ 0  0 11]]\n"
     ]
    }
   ],
   "source": [
    "y_pred_vorace,_ = predict(X_test, iris_models, best_index, False, len(np.unique(y_test)), \"Copeland\", True, False, \"best\", False, weights_gridsearch_iris_no_gridsearch)\n",
    "print(y_pred_vorace)\n",
    "print(y_test)\n",
    "accuracy = accuracy_score(y_test, y_pred_vorace)\n",
    "print(accuracy)\n",
    "conf_mat = confusion_matrix(y_test, y_pred_vorace) \n",
    "print(conf_mat)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 281,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[1 0 2 1 1 0 1 2 1 1 2 0 0 0 0 1 2 1 1 2 0 2 0 2 2 2 2 2 0 0]\n",
      "[1 0 2 1 1 0 1 2 1 1 2 0 0 0 0 1 2 1 1 2 0 2 0 2 2 2 2 2 0 0]\n",
      "1.0\n",
      "[[10  0  0]\n",
      " [ 0  9  0]\n",
      " [ 0  0 11]]\n"
     ]
    }
   ],
   "source": [
    "y_pred_vorace,_ = predict(X_test, iris_models, best_index, False, len(np.unique(y_test)), \"Borda\", True, False, \"best\", False, weights_gridsearch_iris_no_gridsearch)\n",
    "print(y_pred_vorace)\n",
    "print(y_test)\n",
    "accuracy = accuracy_score(y_test, y_pred_vorace)\n",
    "print(accuracy)\n",
    "conf_mat = confusion_matrix(y_test, y_pred_vorace) \n",
    "print(conf_mat)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 285,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[1 0 2 1 1 0 1 2 1 1 2 0 0 0 0 1 2 1 1 2 0 2 0 2 2 2 2 2 0 0]\n",
      "[1 0 2 1 1 0 1 2 1 1 2 0 0 0 0 1 2 1 1 2 0 2 0 2 2 2 2 2 0 0]\n",
      "1.0\n",
      "[[10  0  0]\n",
      " [ 0  9  0]\n",
      " [ 0  0 11]]\n"
     ]
    }
   ],
   "source": [
    "y_pred_vorace,_ = predict(X_test, iris_models, best_index, False, len(np.unique(y_test)), \"Kemeny\", True, False, \"best\", False, weights_gridsearch_iris_no_gridsearch)\n",
    "print(y_pred_vorace)\n",
    "print(y_test)\n",
    "accuracy = accuracy_score(y_test, y_pred_vorace)\n",
    "print(accuracy)\n",
    "conf_mat = confusion_matrix(y_test, y_pred_vorace) \n",
    "print(conf_mat)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## WINE"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 286,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "     Cultivars  Alcohol  Malic acid   Ash  Alcalinity of ash  Magnesium  \\\n",
      "0            1    14.23        1.71  2.43               15.6        127   \n",
      "1            1    13.20        1.78  2.14               11.2        100   \n",
      "2            1    13.16        2.36  2.67               18.6        101   \n",
      "3            1    14.37        1.95  2.50               16.8        113   \n",
      "4            1    13.24        2.59  2.87               21.0        118   \n",
      "..         ...      ...         ...   ...                ...        ...   \n",
      "173          3    13.71        5.65  2.45               20.5         95   \n",
      "174          3    13.40        3.91  2.48               23.0        102   \n",
      "175          3    13.27        4.28  2.26               20.0        120   \n",
      "176          3    13.17        2.59  2.37               20.0        120   \n",
      "177          3    14.13        4.10  2.74               24.5         96   \n",
      "\n",
      "     Total Phenols  Flavanoids  Nonflavanoid phenols  Proanthocyanins  \\\n",
      "0             2.80        3.06                  0.28             2.29   \n",
      "1             2.65        2.76                  0.26             1.28   \n",
      "2             2.80        3.24                  0.30             2.81   \n",
      "3             3.85        3.49                  0.24             2.18   \n",
      "4             2.80        2.69                  0.39             1.82   \n",
      "..             ...         ...                   ...              ...   \n",
      "173           1.68        0.61                  0.52             1.06   \n",
      "174           1.80        0.75                  0.43             1.41   \n",
      "175           1.59        0.69                  0.43             1.35   \n",
      "176           1.65        0.68                  0.53             1.46   \n",
      "177           2.05        0.76                  0.56             1.35   \n",
      "\n",
      "     Color intensity   Hue  OD280/OD315 of diluted wines  Proline  \n",
      "0               5.64  1.04                          3.92     1065  \n",
      "1               4.38  1.05                          3.40     1050  \n",
      "2               5.68  1.03                          3.17     1185  \n",
      "3               7.80  0.86                          3.45     1480  \n",
      "4               4.32  1.04                          2.93      735  \n",
      "..               ...   ...                           ...      ...  \n",
      "173             7.70  0.64                          1.74      740  \n",
      "174             7.30  0.70                          1.56      750  \n",
      "175            10.20  0.59                          1.56      835  \n",
      "176             9.30  0.60                          1.62      840  \n",
      "177             9.20  0.61                          1.60      560  \n",
      "\n",
      "[178 rows x 14 columns]\n",
      "Pipeline(steps=[('standardscaler', StandardScaler()),\n",
      "                ('logisticregression', LogisticRegression())])\n",
      "Pipeline(steps=[('standardscaler', StandardScaler()),\n",
      "                ('gradientboostingclassifier', GradientBoostingClassifier())])\n",
      "Pipeline(steps=[('standardscaler', StandardScaler()),\n",
      "                ('decisiontreeclassifier', DecisionTreeClassifier())])\n",
      "Pipeline(steps=[('standardscaler', StandardScaler()),\n",
      "                ('kneighborsclassifier', KNeighborsClassifier())])\n",
      "Pipeline(steps=[('standardscaler', StandardScaler()),\n",
      "                ('randomforestclassifier',\n",
      "                 RandomForestClassifier(random_state=42))])\n",
      "VotingClassifier(estimators=[('Logistic Regression', LogisticRegression()),\n",
      "                             ('Decision Tree', DecisionTreeClassifier()),\n",
      "                             ('Random Forest',\n",
      "                              RandomForestClassifier(random_state=42)),\n",
      "                             ('Gradient Boosting',\n",
      "                              GradientBoostingClassifier()),\n",
      "                             ('K-Nearest Neighbors', KNeighborsClassifier()),\n",
      "                             ('SVM', SVC(probability=True))],\n",
      "                 n_jobs=-2, verbose=True, voting='soft')\n",
      "Pipeline(steps=[('standardscaler', StandardScaler()),\n",
      "                ('svc', SVC(probability=True))])\n"
     ]
    }
   ],
   "source": [
    "weights = []\n",
    "fscores = []\n",
    "saved_models = []\n",
    "best_index = 0\n",
    "epsilon = 0.0\n",
    "\n",
    "data_wine=pd.read_csv('./data/wine.data',names=['Cultivars','Alcohol','Malic acid','Ash','Alcalinity of ash',\n",
    "'Magnesium','Total Phenols','Flavanoids','Nonflavanoid phenols',\n",
    "'Proanthocyanins','Color intensity','Hue','OD280/OD315 of diluted wines','Proline'])\n",
    "data_wine_target_names = np.array(['1','2','3'])\n",
    "print(data_wine)\n",
    "x=data_wine.drop(['Cultivars'], axis=1)\n",
    "y=data_wine['Cultivars']\n",
    "X_train, X_test, y_train, y_test = train_test_split(x, y, test_size=0.20, random_state=42)\n",
    "\n",
    "# define the folder containing the joblib files\n",
    "folder_path_gridsearch = \"./models/experiment_wine_gridsearch/\"\n",
    "folder_path_no_gridsearch = \"./models/experiment_wine_no_gridsearch/\"\n",
    "\n",
    "weights_gridsearch_wine_no_gridsearch, model_weights_wine_no_gridsearch, wine_models = get_model_weights(X_test, y_test, folder_path_no_gridsearch)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 287,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "MODEL WEIGHTS FOR WINE WITHOUT GRIDSEARCH - USING TEST DATA\n",
      "[1.0, 0.9439974457215836, 0.9439974457215836, 0.9436036129748098, 1.0, 1.0, 1.0]\n",
      "{'Logistic Regression': 14.286, 'Random Forest': 13.486, 'Decision Tree': 13.486, 'K-Nearest Neighbors': 13.48, 'Gradient Boosting': 14.286, 'SVM': 14.286, 'ensemble': 14.286}\n",
      "--------------------------------------------------\n",
      "6\n",
      "1.0\n"
     ]
    }
   ],
   "source": [
    "print(\"MODEL WEIGHTS FOR WINE WITHOUT GRIDSEARCH - USING TEST DATA\")\n",
    "print(weights_gridsearch_wine_no_gridsearch)\n",
    "print(model_weights_wine_no_gridsearch)\n",
    "print(\"-\"*50)\n",
    "\n",
    "best_index = get_best_model_index(weights_gridsearch_wine_no_gridsearch)\n",
    "print(best_index)\n",
    "print(weights_gridsearch_wine_no_gridsearch[best_index])\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 288,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "id": "DpmfLCiaNkHd",
    "outputId": "7dc554ad-bf33-4dba-e630-f8c77227a46a"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[1 1 3 1 2 1 2 3 2 3 1 3 1 2 1 2 2 2 1 2 1 2 2 3 3 3 2 2 2 1 1 2 3 1 1 1]\n",
      "19     1\n",
      "45     1\n",
      "140    3\n",
      "30     1\n",
      "67     2\n",
      "16     1\n",
      "119    2\n",
      "174    3\n",
      "109    2\n",
      "141    3\n",
      "24     1\n",
      "150    3\n",
      "41     1\n",
      "118    2\n",
      "15     1\n",
      "111    2\n",
      "113    2\n",
      "82     2\n",
      "9      1\n",
      "114    2\n",
      "18     1\n",
      "66     2\n",
      "60     2\n",
      "169    3\n",
      "171    3\n",
      "164    3\n",
      "117    2\n",
      "65     2\n",
      "90     2\n",
      "55     1\n",
      "29     1\n",
      "128    2\n",
      "145    3\n",
      "31     1\n",
      "12     1\n",
      "42     1\n",
      "Name: Cultivars, dtype: int64\n",
      "1.0\n",
      "[[14  0  0]\n",
      " [ 0 14  0]\n",
      " [ 0  0  8]]\n"
     ]
    }
   ],
   "source": [
    "y_pred_vorace,_ = predict(X_test, wine_models, best_index, False, len(np.unique(y_test)), \"Plurality\", True, False, \"best\", False, weights_gridsearch_wine_no_gridsearch)\n",
    "y_pred_vorace += 1 \n",
    "print(y_pred_vorace)\n",
    "print(y_test)\n",
    "accuracy = accuracy_score(y_test, y_pred_vorace)\n",
    "print(accuracy)\n",
    "conf_mat = confusion_matrix(y_test, y_pred_vorace)\n",
    "print(conf_mat)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 289,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "id": "dbrabZXMNt6i",
    "outputId": "69c9b6d2-b506-4632-e3ab-8c6c4feda7c2"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[1 1 3 1 2 1 2 3 2 3 1 3 1 2 1 2 2 2 1 2 1 2 2 3 3 3 2 2 2 1 1 2 3 1 1 1]\n",
      "19     1\n",
      "45     1\n",
      "140    3\n",
      "30     1\n",
      "67     2\n",
      "16     1\n",
      "119    2\n",
      "174    3\n",
      "109    2\n",
      "141    3\n",
      "24     1\n",
      "150    3\n",
      "41     1\n",
      "118    2\n",
      "15     1\n",
      "111    2\n",
      "113    2\n",
      "82     2\n",
      "9      1\n",
      "114    2\n",
      "18     1\n",
      "66     2\n",
      "60     2\n",
      "169    3\n",
      "171    3\n",
      "164    3\n",
      "117    2\n",
      "65     2\n",
      "90     2\n",
      "55     1\n",
      "29     1\n",
      "128    2\n",
      "145    3\n",
      "31     1\n",
      "12     1\n",
      "42     1\n",
      "Name: Cultivars, dtype: int64\n",
      "1.0\n",
      "[[14  0  0]\n",
      " [ 0 14  0]\n",
      " [ 0  0  8]]\n"
     ]
    }
   ],
   "source": [
    "y_pred_vorace,_ = predict(X_test, wine_models, best_index, False, len(np.unique(y_test)), \"Sum\", True, False, \"best\", False, weights_gridsearch_wine_no_gridsearch)\n",
    "y_pred_vorace += 1 \n",
    "print(y_pred_vorace)\n",
    "print(y_test)\n",
    "accuracy = accuracy_score(y_test, y_pred_vorace)\n",
    "print(accuracy)\n",
    "conf_mat = confusion_matrix(y_test, y_pred_vorace)\n",
    "print(conf_mat)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 290,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[1 1 3 1 2 1 2 3 2 3 1 3 1 2 1 2 2 2 1 2 1 2 2 3 3 3 2 2 2 1 1 2 3 1 1 1]\n",
      "19     1\n",
      "45     1\n",
      "140    3\n",
      "30     1\n",
      "67     2\n",
      "16     1\n",
      "119    2\n",
      "174    3\n",
      "109    2\n",
      "141    3\n",
      "24     1\n",
      "150    3\n",
      "41     1\n",
      "118    2\n",
      "15     1\n",
      "111    2\n",
      "113    2\n",
      "82     2\n",
      "9      1\n",
      "114    2\n",
      "18     1\n",
      "66     2\n",
      "60     2\n",
      "169    3\n",
      "171    3\n",
      "164    3\n",
      "117    2\n",
      "65     2\n",
      "90     2\n",
      "55     1\n",
      "29     1\n",
      "128    2\n",
      "145    3\n",
      "31     1\n",
      "12     1\n",
      "42     1\n",
      "Name: Cultivars, dtype: int64\n",
      "1.0\n",
      "[[14  0  0]\n",
      " [ 0 14  0]\n",
      " [ 0  0  8]]\n"
     ]
    }
   ],
   "source": [
    "y_pred_vorace,_ = predict(X_test, wine_models, best_index, False, len(np.unique(y_test)), \"Kemeny\", True, False, \"best\", False, weights_gridsearch_wine_no_gridsearch)\n",
    "y_pred_vorace += 1 \n",
    "print(y_pred_vorace)\n",
    "print(y_test)\n",
    "accuracy = accuracy_score(y_test, y_pred_vorace)\n",
    "print(accuracy)\n",
    "conf_mat = confusion_matrix(y_test, y_pred_vorace)\n",
    "print(conf_mat)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 291,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[1 1 3 1 2 1 2 3 2 3 1 3 1 2 1 2 2 2 1 2 1 2 2 3 3 3 2 2 2 1 1 2 3 1 1 1]\n",
      "19     1\n",
      "45     1\n",
      "140    3\n",
      "30     1\n",
      "67     2\n",
      "16     1\n",
      "119    2\n",
      "174    3\n",
      "109    2\n",
      "141    3\n",
      "24     1\n",
      "150    3\n",
      "41     1\n",
      "118    2\n",
      "15     1\n",
      "111    2\n",
      "113    2\n",
      "82     2\n",
      "9      1\n",
      "114    2\n",
      "18     1\n",
      "66     2\n",
      "60     2\n",
      "169    3\n",
      "171    3\n",
      "164    3\n",
      "117    2\n",
      "65     2\n",
      "90     2\n",
      "55     1\n",
      "29     1\n",
      "128    2\n",
      "145    3\n",
      "31     1\n",
      "12     1\n",
      "42     1\n",
      "Name: Cultivars, dtype: int64\n",
      "1.0\n",
      "[[14  0  0]\n",
      " [ 0 14  0]\n",
      " [ 0  0  8]]\n"
     ]
    }
   ],
   "source": [
    "y_pred_vorace,_ = predict(X_test, wine_models, best_index, False, len(np.unique(y_test)), \"Borda\", True, False, \"best\", False, weights_gridsearch_wine_no_gridsearch)\n",
    "y_pred_vorace += 1 \n",
    "print(y_pred_vorace)\n",
    "print(y_test)\n",
    "accuracy = accuracy_score(y_test, y_pred_vorace)\n",
    "print(accuracy)\n",
    "conf_mat = confusion_matrix(y_test, y_pred_vorace)\n",
    "print(conf_mat)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 292,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[1 1 3 1 2 1 2 3 2 3 1 3 1 2 1 2 2 2 1 2 1 2 2 3 3 3 2 2 2 1 1 2 3 1 1 1]\n",
      "19     1\n",
      "45     1\n",
      "140    3\n",
      "30     1\n",
      "67     2\n",
      "16     1\n",
      "119    2\n",
      "174    3\n",
      "109    2\n",
      "141    3\n",
      "24     1\n",
      "150    3\n",
      "41     1\n",
      "118    2\n",
      "15     1\n",
      "111    2\n",
      "113    2\n",
      "82     2\n",
      "9      1\n",
      "114    2\n",
      "18     1\n",
      "66     2\n",
      "60     2\n",
      "169    3\n",
      "171    3\n",
      "164    3\n",
      "117    2\n",
      "65     2\n",
      "90     2\n",
      "55     1\n",
      "29     1\n",
      "128    2\n",
      "145    3\n",
      "31     1\n",
      "12     1\n",
      "42     1\n",
      "Name: Cultivars, dtype: int64\n",
      "1.0\n",
      "[[14  0  0]\n",
      " [ 0 14  0]\n",
      " [ 0  0  8]]\n"
     ]
    }
   ],
   "source": [
    "y_pred_vorace,_ = predict(X_test, wine_models, best_index, False, len(np.unique(y_test)), \"Copeland\", True, False, \"best\", False, weights_gridsearch_wine_no_gridsearch)\n",
    "y_pred_vorace += 1 \n",
    "print(y_pred_vorace)\n",
    "print(y_test)\n",
    "accuracy = accuracy_score(y_test, y_pred_vorace)\n",
    "print(accuracy)\n",
    "conf_mat = confusion_matrix(y_test, y_pred_vorace)\n",
    "print(conf_mat)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## LYMPHOGRAPHY"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 327,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "     class: normal find, metastases, malign lymph, fibrosis  \\\n",
      "0                                                    3        \n",
      "1                                                    2        \n",
      "2                                                    3        \n",
      "3                                                    3        \n",
      "4                                                    2        \n",
      "..                                                 ...        \n",
      "143                                                  3        \n",
      "144                                                  2        \n",
      "145                                                  3        \n",
      "146                                                  2        \n",
      "147                                                  2        \n",
      "\n",
      "     lymphatics: normal, arched, deformed, displaced  \\\n",
      "0                                                  4   \n",
      "1                                                  3   \n",
      "2                                                  3   \n",
      "3                                                  3   \n",
      "4                                                  3   \n",
      "..                                               ...   \n",
      "143                                                3   \n",
      "144                                                2   \n",
      "145                                                2   \n",
      "146                                                2   \n",
      "147                                                2   \n",
      "\n",
      "     block of affere: no, yes  bl. of lymph. c: no, yes  \\\n",
      "0                           2                         1   \n",
      "1                           2                         1   \n",
      "2                           2                         2   \n",
      "3                           1                         1   \n",
      "4                           1                         1   \n",
      "..                        ...                       ...   \n",
      "143                         2                         1   \n",
      "144                         1                         1   \n",
      "145                         2                         1   \n",
      "146                         1                         1   \n",
      "147                         2                         2   \n",
      "\n",
      "     bl. of lymph. s: no, yes  by pass: no, yes  extravasates: no, yes  \\\n",
      "0                           1                 1                      1   \n",
      "1                           1                 2                      2   \n",
      "2                           2                 2                      2   \n",
      "3                           1                 1                      2   \n",
      "4                           1                 1                      1   \n",
      "..                        ...               ...                    ...   \n",
      "143                         1                 2                      2   \n",
      "144                         1                 1                      1   \n",
      "145                         1                 1                      2   \n",
      "146                         1                 1                      1   \n",
      "147                         1                 2                      2   \n",
      "\n",
      "     regeneration of: no, yes  early uptake in: no, yes  lym.nodes dimin: 0-3  \\\n",
      "0                           1                         2                     1   \n",
      "1                           1                         2                     1   \n",
      "2                           2                         2                     1   \n",
      "3                           1                         2                     1   \n",
      "4                           1                         1                     1   \n",
      "..                        ...                       ...                   ...   \n",
      "143                         1                         2                     1   \n",
      "144                         1                         1                     1   \n",
      "145                         1                         2                     1   \n",
      "146                         1                         2                     1   \n",
      "147                         1                         2                     1   \n",
      "\n",
      "     lym.nodes enlar: 1-4  changes in lym.: bean, oval, round  \\\n",
      "0                       2                                   2   \n",
      "1                       3                                   3   \n",
      "2                       4                                   3   \n",
      "3                       3                                   3   \n",
      "4                       2                                   2   \n",
      "..                    ...                                 ...   \n",
      "143                     2                                   2   \n",
      "144                     1                                   1   \n",
      "145                     3                                   3   \n",
      "146                     2                                   2   \n",
      "147                     3                                   3   \n",
      "\n",
      "     defect in node: no, lacunar, lac. marginal, lac. central  \\\n",
      "0                                                    2          \n",
      "1                                                    2          \n",
      "2                                                    3          \n",
      "3                                                    4          \n",
      "4                                                    4          \n",
      "..                                                 ...          \n",
      "143                                                  4          \n",
      "144                                                  1          \n",
      "145                                                  3          \n",
      "146                                                  4          \n",
      "147                                                  4          \n",
      "\n",
      "     changes in node: no, lacunar, lac. margin, lac. central  \\\n",
      "0                                                    4         \n",
      "1                                                    3         \n",
      "2                                                    4         \n",
      "3                                                    4         \n",
      "4                                                    3         \n",
      "..                                                 ...         \n",
      "143                                                  3         \n",
      "144                                                  1         \n",
      "145                                                  3         \n",
      "146                                                  2         \n",
      "147                                                  3         \n",
      "\n",
      "     changes in stru: no, grainy, drop-like, coarse, diluted, reticular, stripped, faint  \\\n",
      "0                                                    8                                     \n",
      "1                                                    4                                     \n",
      "2                                                    8                                     \n",
      "3                                                    4                                     \n",
      "4                                                    5                                     \n",
      "..                                                 ...                                     \n",
      "143                                                  5                                     \n",
      "144                                                  3                                     \n",
      "145                                                  8                                     \n",
      "146                                                  2                                     \n",
      "147                                                  4                                     \n",
      "\n",
      "     special forms: no, chalices, vesicles  dislocation of: no, yes  \\\n",
      "0                                        1                        1   \n",
      "1                                        2                        2   \n",
      "2                                        3                        2   \n",
      "3                                        3                        1   \n",
      "4                                        1                        2   \n",
      "..                                     ...                      ...   \n",
      "143                                      2                        2   \n",
      "144                                      1                        2   \n",
      "145                                      3                        2   \n",
      "146                                      1                        2   \n",
      "147                                      3                        2   \n",
      "\n",
      "     exclusion of no: no, yes  \\\n",
      "0                           2   \n",
      "1                           2   \n",
      "2                           2   \n",
      "3                           2   \n",
      "4                           2   \n",
      "..                        ...   \n",
      "143                         2   \n",
      "144                         2   \n",
      "145                         2   \n",
      "146                         2   \n",
      "147                         2   \n",
      "\n",
      "     no. of nodes in: 0-9, 10-19, 20-29, 30-39, 40-49, 50-59, 60-69, >=70  \n",
      "0                                                    2                     \n",
      "1                                                    2                     \n",
      "2                                                    7                     \n",
      "3                                                    6                     \n",
      "4                                                    1                     \n",
      "..                                                 ...                     \n",
      "143                                                  4                     \n",
      "144                                                  1                     \n",
      "145                                                  4                     \n",
      "146                                                  1                     \n",
      "147                                                  6                     \n",
      "\n",
      "[148 rows x 19 columns]\n",
      "Pipeline(steps=[('standardscaler', StandardScaler()),\n",
      "                ('decisiontreeclassifier', DecisionTreeClassifier())])\n",
      "Pipeline(steps=[('standardscaler', StandardScaler()),\n",
      "                ('kneighborsclassifier', KNeighborsClassifier())])\n",
      "Pipeline(steps=[('standardscaler', StandardScaler()),\n",
      "                ('randomforestclassifier',\n",
      "                 RandomForestClassifier(random_state=42))])\n",
      "VotingClassifier(estimators=[('Logistic Regression', LogisticRegression()),\n",
      "                             ('Decision Tree', DecisionTreeClassifier()),\n",
      "                             ('Random Forest',\n",
      "                              RandomForestClassifier(random_state=42)),\n",
      "                             ('Gradient Boosting',\n",
      "                              GradientBoostingClassifier()),\n",
      "                             ('K-Nearest Neighbors', KNeighborsClassifier()),\n",
      "                             ('SVM', SVC(probability=True))],\n",
      "                 n_jobs=-2, verbose=True, voting='soft')\n",
      "Pipeline(steps=[('standardscaler', StandardScaler()),\n",
      "                ('logisticregression', LogisticRegression())])\n",
      "Pipeline(steps=[('standardscaler', StandardScaler()),\n",
      "                ('svc', SVC(probability=True))])\n",
      "Pipeline(steps=[('standardscaler', StandardScaler()),\n",
      "                ('gradientboostingclassifier', GradientBoostingClassifier())])\n"
     ]
    }
   ],
   "source": [
    "weights = []\n",
    "fscores = []\n",
    "saved_models = []\n",
    "best_index = 0\n",
    "epsilon = 0.0\n",
    "\n",
    "data_lymphography=pd.read_csv('./data/lymphography.data',names=['class: normal find, metastases, malign lymph, fibrosis','lymphatics: normal, arched, deformed, displaced','block of affere: no, yes','bl. of lymph. c: no, yes','bl. of lymph. s: no, yes','by pass: no, yes','extravasates: no, yes', 'regeneration of: no, yes', 'early uptake in: no, yes', 'lym.nodes dimin: 0-3', 'lym.nodes enlar: 1-4', 'changes in lym.: bean, oval, round', 'defect in node: no, lacunar, lac. marginal, lac. central', 'changes in node: no, lacunar, lac. margin, lac. central', 'changes in stru: no, grainy, drop-like, coarse, diluted, reticular, stripped, faint', 'special forms: no, chalices, vesicles', 'dislocation of: no, yes', 'exclusion of no: no, yes', 'no. of nodes in: 0-9, 10-19, 20-29, 30-39, 40-49, 50-59, 60-69, >=70'])\n",
    "print(data_lymphography)\n",
    "data_lymphography_target_names = np.array(['normal find', 'metastases', 'malign lymph', 'fibrosis'])\n",
    "x=data_lymphography.drop(['class: normal find, metastases, malign lymph, fibrosis'], axis=1)\n",
    "y=data_lymphography['class: normal find, metastases, malign lymph, fibrosis']\n",
    "X_train, X_test, y_train, y_test = train_test_split(x, y, test_size=0.20, random_state=42)\n",
    "\n",
    "# define the folder containing the joblib files\n",
    "folder_path_gridsearch = \"./models/experiment_lymphography_gridsearch/\"\n",
    "folder_path_no_gridsearch = \"./models/experiment_lymphography_no_gridsearch/\"\n",
    "\n",
    "weights_gridsearch_lymphography_no_gridsearch, model_weights_lymphography_no_gridsearch, lymphography_models = get_model_weights(X_test, y_test, folder_path_no_gridsearch)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 328,
   "metadata": {
    "scrolled": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "MODEL WEIGHTS FOR LYMPHOGRAPHY WITHOUT GRIDSEARCH - USING TEST DATA\n",
      "[0.6542597187758478, 0.7415372361608921, 0.8045977011494254, 0.8628352490421456, 0.8331204767986378, 0.8045977011494254, 0.8477453580901858]\n",
      "{'Logistic Regression': 9.347, 'Random Forest': 10.593, 'Decision Tree': 11.494, 'K-Nearest Neighbors': 12.326, 'Gradient Boosting': 11.902, 'SVM': 11.494, 'ensemble': 12.111}\n",
      "--------------------------------------------------\n",
      "3\n",
      "0.8628352490421456\n"
     ]
    }
   ],
   "source": [
    "print(\"MODEL WEIGHTS FOR LYMPHOGRAPHY WITHOUT GRIDSEARCH - USING TEST DATA\")\n",
    "print(weights_gridsearch_lymphography_no_gridsearch)\n",
    "print(model_weights_lymphography_no_gridsearch)\n",
    "print(\"-\"*50)\n",
    "\n",
    "best_index = get_best_model_index(weights_gridsearch_lymphography_no_gridsearch)\n",
    "print(best_index)\n",
    "print(weights_gridsearch_lymphography_no_gridsearch[best_index])\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 329,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "id": "DpmfLCiaNkHd",
    "outputId": "7dc554ad-bf33-4dba-e630-f8c77227a46a"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[3 3 4 2 3 2 2 3 2 2 3 2 2 2 2 3 2 3 2 3 2 3 3 3 3 2 2 3 3 2]\n",
      "125    3\n",
      "51     3\n",
      "139    4\n",
      "19     2\n",
      "104    2\n",
      "12     2\n",
      "76     2\n",
      "31     3\n",
      "81     2\n",
      "9      3\n",
      "26     3\n",
      "96     2\n",
      "144    2\n",
      "67     2\n",
      "135    2\n",
      "66     3\n",
      "18     2\n",
      "69     3\n",
      "124    2\n",
      "30     3\n",
      "29     3\n",
      "105    3\n",
      "36     4\n",
      "118    3\n",
      "55     3\n",
      "22     2\n",
      "64     2\n",
      "131    3\n",
      "82     3\n",
      "11     2\n",
      "Name: class: normal find, metastases, malign lymph, fibrosis, dtype: int64\n",
      "0.8666666666666667\n",
      "[[13  1  0]\n",
      " [ 2 12  0]\n",
      " [ 0  1  1]]\n"
     ]
    }
   ],
   "source": [
    "y_pred_vorace,_ = predict(X_test, lymphography_models, best_index, False, len(np.unique(y_test))+1, \"Plurality\", True, False, \"best\", False, weights_gridsearch_lymphography_no_gridsearch)\n",
    "y_pred_vorace += 1\n",
    "print(y_pred_vorace)\n",
    "print(y_test)\n",
    "accuracy = accuracy_score(y_test, y_pred_vorace)\n",
    "print(accuracy)\n",
    "conf_mat = confusion_matrix(y_test, y_pred_vorace)\n",
    "print(conf_mat)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 330,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "id": "dbrabZXMNt6i",
    "outputId": "69c9b6d2-b506-4632-e3ab-8c6c4feda7c2"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[3 3 4 2 3 2 2 3 2 2 3 2 2 2 2 3 2 3 2 3 2 3 3 3 3 2 2 3 3 2]\n",
      "125    3\n",
      "51     3\n",
      "139    4\n",
      "19     2\n",
      "104    2\n",
      "12     2\n",
      "76     2\n",
      "31     3\n",
      "81     2\n",
      "9      3\n",
      "26     3\n",
      "96     2\n",
      "144    2\n",
      "67     2\n",
      "135    2\n",
      "66     3\n",
      "18     2\n",
      "69     3\n",
      "124    2\n",
      "30     3\n",
      "29     3\n",
      "105    3\n",
      "36     4\n",
      "118    3\n",
      "55     3\n",
      "22     2\n",
      "64     2\n",
      "131    3\n",
      "82     3\n",
      "11     2\n",
      "Name: class: normal find, metastases, malign lymph, fibrosis, dtype: int64\n",
      "0.8666666666666667\n",
      "[[13  1  0]\n",
      " [ 2 12  0]\n",
      " [ 0  1  1]]\n"
     ]
    }
   ],
   "source": [
    "y_pred_vorace,_ = predict(X_test, lymphography_models, best_index, False, len(np.unique(y_test))+1, \"Sum\", True, False, \"best\", False, weights_gridsearch_lymphography_no_gridsearch)\n",
    "y_pred_vorace += 1\n",
    "print(y_pred_vorace)\n",
    "print(y_test)\n",
    "accuracy = accuracy_score(y_test, y_pred_vorace)\n",
    "print(accuracy)\n",
    "conf_mat = confusion_matrix(y_test, y_pred_vorace)\n",
    "print(conf_mat)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 331,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[3 3 4 2 3 2 2 3 2 2 3 2 2 2 2 3 2 3 2 3 2 3 3 3 3 2 2 3 3 2]\n",
      "125    3\n",
      "51     3\n",
      "139    4\n",
      "19     2\n",
      "104    2\n",
      "12     2\n",
      "76     2\n",
      "31     3\n",
      "81     2\n",
      "9      3\n",
      "26     3\n",
      "96     2\n",
      "144    2\n",
      "67     2\n",
      "135    2\n",
      "66     3\n",
      "18     2\n",
      "69     3\n",
      "124    2\n",
      "30     3\n",
      "29     3\n",
      "105    3\n",
      "36     4\n",
      "118    3\n",
      "55     3\n",
      "22     2\n",
      "64     2\n",
      "131    3\n",
      "82     3\n",
      "11     2\n",
      "Name: class: normal find, metastases, malign lymph, fibrosis, dtype: int64\n",
      "0.8666666666666667\n",
      "[[13  1  0]\n",
      " [ 2 12  0]\n",
      " [ 0  1  1]]\n"
     ]
    }
   ],
   "source": [
    "y_pred_vorace,_ = predict(X_test, lymphography_models, best_index, False, len(np.unique(y_test))+1, \"Copeland\", True, False, \"best\", False, weights_gridsearch_lymphography_no_gridsearch)\n",
    "y_pred_vorace += 1\n",
    "print(y_pred_vorace)\n",
    "print(y_test)\n",
    "accuracy = accuracy_score(y_test, y_pred_vorace)\n",
    "print(accuracy)\n",
    "conf_mat = confusion_matrix(y_test, y_pred_vorace)\n",
    "print(conf_mat)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 298,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[3 3 4 2 3 2 2 3 2 2 3 2 2 2 2 3 2 3 2 3 2 3 3 3 3 2 2 3 3 2]\n",
      "125    3\n",
      "51     3\n",
      "139    4\n",
      "19     2\n",
      "104    2\n",
      "12     2\n",
      "76     2\n",
      "31     3\n",
      "81     2\n",
      "9      3\n",
      "26     3\n",
      "96     2\n",
      "144    2\n",
      "67     2\n",
      "135    2\n",
      "66     3\n",
      "18     2\n",
      "69     3\n",
      "124    2\n",
      "30     3\n",
      "29     3\n",
      "105    3\n",
      "36     4\n",
      "118    3\n",
      "55     3\n",
      "22     2\n",
      "64     2\n",
      "131    3\n",
      "82     3\n",
      "11     2\n",
      "Name: class: normal find, metastases, malign lymph, fibrosis, dtype: int64\n",
      "0.8666666666666667\n",
      "[[13  1  0]\n",
      " [ 2 12  0]\n",
      " [ 0  1  1]]\n"
     ]
    }
   ],
   "source": [
    "y_pred_vorace,_ = predict(X_test, lymphography_models, best_index, False, len(np.unique(y_test))+1, \"Borda\", True, False, \"best\", False, weights_gridsearch_lymphography_no_gridsearch)\n",
    "y_pred_vorace += 1\n",
    "print(y_pred_vorace)\n",
    "print(y_test)\n",
    "accuracy = accuracy_score(y_test, y_pred_vorace)\n",
    "print(accuracy)\n",
    "conf_mat = confusion_matrix(y_test, y_pred_vorace)\n",
    "print(conf_mat)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 332,
   "metadata": {
    "scrolled": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[3 3 2 2 3 2 2 2 2 2 3 2 1 2 2 3 3 3 2 2 2 3 3 3 2 2 2 3 3 2]\n",
      "125    3\n",
      "51     3\n",
      "139    4\n",
      "19     2\n",
      "104    2\n",
      "12     2\n",
      "76     2\n",
      "31     3\n",
      "81     2\n",
      "9      3\n",
      "26     3\n",
      "96     2\n",
      "144    2\n",
      "67     2\n",
      "135    2\n",
      "66     3\n",
      "18     2\n",
      "69     3\n",
      "124    2\n",
      "30     3\n",
      "29     3\n",
      "105    3\n",
      "36     4\n",
      "118    3\n",
      "55     3\n",
      "22     2\n",
      "64     2\n",
      "131    3\n",
      "82     3\n",
      "11     2\n",
      "Name: class: normal find, metastases, malign lymph, fibrosis, dtype: int64\n",
      "0.6666666666666666\n",
      "[[ 0  0  0  0]\n",
      " [ 1 11  2  0]\n",
      " [ 0  5  9  0]\n",
      " [ 0  1  1  0]]\n"
     ]
    }
   ],
   "source": [
    "y_pred_vorace,_ = predict(X_test, lymphography_models, best_index, False, len(np.unique(y_test))+1, \"Kemeny\", True, False, \"best\", False, weights_gridsearch_lymphography_no_gridsearch)\n",
    "y_pred_vorace += 1\n",
    "print(y_pred_vorace)\n",
    "print(y_test)\n",
    "accuracy = accuracy_score(y_test, y_pred_vorace)\n",
    "print(accuracy)\n",
    "conf_mat = confusion_matrix(y_test, y_pred_vorace)\n",
    "print(conf_mat)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## HEART"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 300,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "VotingClassifier(estimators=[('Logistic Regression', LogisticRegression()),\n",
      "                             ('Decision Tree', DecisionTreeClassifier()),\n",
      "                             ('Random Forest',\n",
      "                              RandomForestClassifier(random_state=42)),\n",
      "                             ('Gradient Boosting',\n",
      "                              GradientBoostingClassifier()),\n",
      "                             ('K-Nearest Neighbors', KNeighborsClassifier()),\n",
      "                             ('SVM', SVC(probability=True))],\n",
      "                 n_jobs=-2, verbose=True, voting='soft')\n",
      "Pipeline(steps=[('standardscaler', StandardScaler()),\n",
      "                ('decisiontreeclassifier', DecisionTreeClassifier())])\n",
      "Pipeline(steps=[('standardscaler', StandardScaler()),\n",
      "                ('kneighborsclassifier', KNeighborsClassifier())])\n",
      "Pipeline(steps=[('standardscaler', StandardScaler()),\n",
      "                ('logisticregression', LogisticRegression())])\n",
      "Pipeline(steps=[('standardscaler', StandardScaler()),\n",
      "                ('svc', SVC(probability=True))])\n",
      "Pipeline(steps=[('standardscaler', StandardScaler()),\n",
      "                ('gradientboostingclassifier', GradientBoostingClassifier())])\n",
      "Pipeline(steps=[('standardscaler', StandardScaler()),\n",
      "                ('randomforestclassifier',\n",
      "                 RandomForestClassifier(random_state=42))])\n"
     ]
    }
   ],
   "source": [
    "weights = []\n",
    "fscores = []\n",
    "saved_models = []\n",
    "best_index = 0\n",
    "epsilon = 0.0\n",
    "\n",
    "data_heart=pd.read_csv('./data/heart.data',names=['age','sex','chest pain type (4 values)', 'resting blood pressure', 'serum cholesterol in mg/dl', 'fasting blood sugar > 120 mg/dl', 'resting electrocardiographic results (values 0,1,2)', 'maximum heart rate achieved', 'exercise induced angina','oldpeak = ST depression induced by exercise relative to rest', 'the slope of the peak exercise ST segment','number of major vessels (0-3) colored by flourosopy','thal: 3 = normal; 6 = fixed defect; 7 = reversable defect', 'absence = 1, presence = 2'], delim_whitespace=True)\n",
    "# print(data_heart)\n",
    "data_heart_target_names = np.array(['Absence','Presence'])\n",
    "x=data_heart.drop(['absence = 1, presence = 2'], axis=1)\n",
    "y=data_heart['absence = 1, presence = 2']\n",
    "X_train, X_test, y_train, y_test = train_test_split(x, y, test_size=0.20, random_state=42)\n",
    "\n",
    "# define the folder containing the joblib files\n",
    "folder_path_gridsearch = \"./models/experiment_heart_gridsearch/\"\n",
    "folder_path_no_gridsearch = \"./models/experiment_heart_no_gridsearch/\"\n",
    "\n",
    "weights_gridsearch_heart_no_gridsearch, model_weights_heart_no_gridsearch, heart_models = get_model_weights(X_test, y_test, folder_path_no_gridsearch)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 301,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "MODEL WEIGHTS FOR HEART WITHOUT GRIDSEARCH - USING TEST DATA\n",
      "[0.8325445941026575, 0.6890284075495097, 0.810359231411863, 0.9069692189459209, 0.8877450980392156, 0.7380718954248366, 0.7552334943639291]\n",
      "{'Logistic Regression': 11.893, 'Random Forest': 9.843, 'Decision Tree': 11.577, 'K-Nearest Neighbors': 12.957, 'Gradient Boosting': 12.682, 'SVM': 10.544, 'ensemble': 10.789}\n",
      "--------------------------------------------------\n",
      "3\n",
      "0.9069692189459209\n"
     ]
    }
   ],
   "source": [
    "print(\"MODEL WEIGHTS FOR HEART WITHOUT GRIDSEARCH - USING TEST DATA\")\n",
    "print(weights_gridsearch_heart_no_gridsearch)\n",
    "print(model_weights_heart_no_gridsearch)\n",
    "print(\"-\"*50)\n",
    "\n",
    "best_index = get_best_model_index(weights_gridsearch_heart_no_gridsearch)\n",
    "print(best_index)\n",
    "print(weights_gridsearch_heart_no_gridsearch[best_index])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 303,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "id": "DpmfLCiaNkHd",
    "outputId": "7dc554ad-bf33-4dba-e630-f8c77227a46a",
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[2 2 1 1 1 2 2 1 1 1 1 1 2 2 2 1 2 1 2 1 2 1 1 1 1 2 1 2 1 1 1 1 1 1 1 2 2\n",
      " 1 2 2 1 1 1 1 1 1 1 2 2 1 1 1 1 2]\n",
      "30     2\n",
      "116    2\n",
      "79     1\n",
      "127    1\n",
      "196    1\n",
      "137    2\n",
      "209    1\n",
      "45     1\n",
      "158    1\n",
      "247    1\n",
      "183    1\n",
      "268    1\n",
      "227    2\n",
      "82     2\n",
      "165    1\n",
      "194    1\n",
      "226    2\n",
      "146    2\n",
      "104    2\n",
      "60     1\n",
      "221    2\n",
      "266    1\n",
      "46     2\n",
      "42     1\n",
      "185    1\n",
      "9      2\n",
      "22     1\n",
      "199    2\n",
      "109    1\n",
      "24     1\n",
      "113    1\n",
      "68     1\n",
      "144    2\n",
      "224    1\n",
      "252    2\n",
      "6      2\n",
      "120    2\n",
      "67     2\n",
      "119    2\n",
      "118    1\n",
      "25     1\n",
      "125    1\n",
      "244    1\n",
      "19     1\n",
      "77     1\n",
      "216    1\n",
      "90     1\n",
      "208    2\n",
      "93     2\n",
      "180    1\n",
      "15     1\n",
      "152    1\n",
      "232    1\n",
      "250    2\n",
      "Name: absence = 1, presence = 2, dtype: int64\n",
      "0.8518518518518519\n",
      "[[30  3]\n",
      " [ 5 16]]\n"
     ]
    }
   ],
   "source": [
    "y_pred_vorace,_ = predict(X_test, heart_models, best_index, False, len(np.unique(y_test)), \"Plurality\", True, False, \"best\", False, weights_gridsearch_heart_no_gridsearch)\n",
    "y_pred_vorace += 1\n",
    "print(y_pred_vorace)\n",
    "print(y_test)\n",
    "accuracy = accuracy_score(y_test, y_pred_vorace)\n",
    "print(accuracy)\n",
    "conf_mat = confusion_matrix(y_test, y_pred_vorace)\n",
    "print(conf_mat)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 304,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "id": "dbrabZXMNt6i",
    "outputId": "69c9b6d2-b506-4632-e3ab-8c6c4feda7c2"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[2 2 1 1 1 2 2 1 1 1 1 1 2 2 2 1 2 1 2 1 2 1 1 1 1 2 1 2 1 1 1 1 1 1 1 2 2\n",
      " 1 2 2 1 1 1 1 1 1 1 2 2 2 1 1 1 2]\n",
      "30     2\n",
      "116    2\n",
      "79     1\n",
      "127    1\n",
      "196    1\n",
      "137    2\n",
      "209    1\n",
      "45     1\n",
      "158    1\n",
      "247    1\n",
      "183    1\n",
      "268    1\n",
      "227    2\n",
      "82     2\n",
      "165    1\n",
      "194    1\n",
      "226    2\n",
      "146    2\n",
      "104    2\n",
      "60     1\n",
      "221    2\n",
      "266    1\n",
      "46     2\n",
      "42     1\n",
      "185    1\n",
      "9      2\n",
      "22     1\n",
      "199    2\n",
      "109    1\n",
      "24     1\n",
      "113    1\n",
      "68     1\n",
      "144    2\n",
      "224    1\n",
      "252    2\n",
      "6      2\n",
      "120    2\n",
      "67     2\n",
      "119    2\n",
      "118    1\n",
      "25     1\n",
      "125    1\n",
      "244    1\n",
      "19     1\n",
      "77     1\n",
      "216    1\n",
      "90     1\n",
      "208    2\n",
      "93     2\n",
      "180    1\n",
      "15     1\n",
      "152    1\n",
      "232    1\n",
      "250    2\n",
      "Name: absence = 1, presence = 2, dtype: int64\n",
      "0.8333333333333334\n",
      "[[29  4]\n",
      " [ 5 16]]\n"
     ]
    }
   ],
   "source": [
    "y_pred_vorace,_ = predict(X_test, heart_models, best_index, False, len(np.unique(y_test)), \"Sum\", True, False, \"best\", False, weights_gridsearch_heart_no_gridsearch)\n",
    "y_pred_vorace += 1\n",
    "print(y_pred_vorace)\n",
    "print(y_test)\n",
    "accuracy = accuracy_score(y_test, y_pred_vorace)\n",
    "print(accuracy)\n",
    "conf_mat = confusion_matrix(y_test, y_pred_vorace)\n",
    "print(conf_mat)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 305,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[2 2 1 1 1 2 2 1 1 1 1 1 2 2 2 1 2 1 2 1 2 1 1 1 1 2 1 2 1 1 1 1 1 1 1 2 2\n",
      " 1 2 2 1 1 1 1 1 1 1 2 2 1 1 1 1 2]\n",
      "30     2\n",
      "116    2\n",
      "79     1\n",
      "127    1\n",
      "196    1\n",
      "137    2\n",
      "209    1\n",
      "45     1\n",
      "158    1\n",
      "247    1\n",
      "183    1\n",
      "268    1\n",
      "227    2\n",
      "82     2\n",
      "165    1\n",
      "194    1\n",
      "226    2\n",
      "146    2\n",
      "104    2\n",
      "60     1\n",
      "221    2\n",
      "266    1\n",
      "46     2\n",
      "42     1\n",
      "185    1\n",
      "9      2\n",
      "22     1\n",
      "199    2\n",
      "109    1\n",
      "24     1\n",
      "113    1\n",
      "68     1\n",
      "144    2\n",
      "224    1\n",
      "252    2\n",
      "6      2\n",
      "120    2\n",
      "67     2\n",
      "119    2\n",
      "118    1\n",
      "25     1\n",
      "125    1\n",
      "244    1\n",
      "19     1\n",
      "77     1\n",
      "216    1\n",
      "90     1\n",
      "208    2\n",
      "93     2\n",
      "180    1\n",
      "15     1\n",
      "152    1\n",
      "232    1\n",
      "250    2\n",
      "Name: absence = 1, presence = 2, dtype: int64\n",
      "0.8518518518518519\n",
      "[[30  3]\n",
      " [ 5 16]]\n"
     ]
    }
   ],
   "source": [
    "y_pred_vorace,_ = predict(X_test, heart_models, best_index, False, len(np.unique(y_test)), \"Borda\", True, False, \"best\", False, weights_gridsearch_heart_no_gridsearch)\n",
    "y_pred_vorace += 1\n",
    "print(y_pred_vorace)\n",
    "print(y_test)\n",
    "accuracy = accuracy_score(y_test, y_pred_vorace)\n",
    "print(accuracy)\n",
    "conf_mat = confusion_matrix(y_test, y_pred_vorace)\n",
    "print(conf_mat)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 306,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[2 2 1 1 2 2 1 1 1 1 2 1 2 2 2 1 2 1 2 1 2 1 1 1 1 2 1 2 1 1 1 1 1 1 2 2 2\n",
      " 1 2 2 1 1 1 1 1 1 1 2 1 1 1 1 1 2]\n",
      "30     2\n",
      "116    2\n",
      "79     1\n",
      "127    1\n",
      "196    1\n",
      "137    2\n",
      "209    1\n",
      "45     1\n",
      "158    1\n",
      "247    1\n",
      "183    1\n",
      "268    1\n",
      "227    2\n",
      "82     2\n",
      "165    1\n",
      "194    1\n",
      "226    2\n",
      "146    2\n",
      "104    2\n",
      "60     1\n",
      "221    2\n",
      "266    1\n",
      "46     2\n",
      "42     1\n",
      "185    1\n",
      "9      2\n",
      "22     1\n",
      "199    2\n",
      "109    1\n",
      "24     1\n",
      "113    1\n",
      "68     1\n",
      "144    2\n",
      "224    1\n",
      "252    2\n",
      "6      2\n",
      "120    2\n",
      "67     2\n",
      "119    2\n",
      "118    1\n",
      "25     1\n",
      "125    1\n",
      "244    1\n",
      "19     1\n",
      "77     1\n",
      "216    1\n",
      "90     1\n",
      "208    2\n",
      "93     2\n",
      "180    1\n",
      "15     1\n",
      "152    1\n",
      "232    1\n",
      "250    2\n",
      "Name: absence = 1, presence = 2, dtype: int64\n",
      "0.8333333333333334\n",
      "[[29  4]\n",
      " [ 5 16]]\n"
     ]
    }
   ],
   "source": [
    "y_pred_vorace,_ = predict(X_test, heart_models, best_index, False, len(np.unique(y_test)), \"Kemeny\", True, False, \"best\", False, weights_gridsearch_heart_no_gridsearch)\n",
    "y_pred_vorace += 1\n",
    "print(y_pred_vorace)\n",
    "print(y_test)\n",
    "accuracy = accuracy_score(y_test, y_pred_vorace)\n",
    "print(accuracy)\n",
    "conf_mat = confusion_matrix(y_test, y_pred_vorace)\n",
    "print(conf_mat)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 307,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[2 2 1 1 1 2 2 1 1 1 1 1 2 2 2 1 2 1 2 1 2 1 1 1 1 2 1 2 1 1 1 1 1 1 1 2 2\n",
      " 1 2 2 1 1 1 1 1 1 1 2 2 1 1 1 1 2]\n",
      "30     2\n",
      "116    2\n",
      "79     1\n",
      "127    1\n",
      "196    1\n",
      "137    2\n",
      "209    1\n",
      "45     1\n",
      "158    1\n",
      "247    1\n",
      "183    1\n",
      "268    1\n",
      "227    2\n",
      "82     2\n",
      "165    1\n",
      "194    1\n",
      "226    2\n",
      "146    2\n",
      "104    2\n",
      "60     1\n",
      "221    2\n",
      "266    1\n",
      "46     2\n",
      "42     1\n",
      "185    1\n",
      "9      2\n",
      "22     1\n",
      "199    2\n",
      "109    1\n",
      "24     1\n",
      "113    1\n",
      "68     1\n",
      "144    2\n",
      "224    1\n",
      "252    2\n",
      "6      2\n",
      "120    2\n",
      "67     2\n",
      "119    2\n",
      "118    1\n",
      "25     1\n",
      "125    1\n",
      "244    1\n",
      "19     1\n",
      "77     1\n",
      "216    1\n",
      "90     1\n",
      "208    2\n",
      "93     2\n",
      "180    1\n",
      "15     1\n",
      "152    1\n",
      "232    1\n",
      "250    2\n",
      "Name: absence = 1, presence = 2, dtype: int64\n",
      "0.8518518518518519\n",
      "[[30  3]\n",
      " [ 5 16]]\n"
     ]
    }
   ],
   "source": [
    "y_pred_vorace,_ = predict(X_test, heart_models, best_index, False, len(np.unique(y_test)), \"Copeland\", True, False, \"best\", False, weights_gridsearch_heart_no_gridsearch)\n",
    "y_pred_vorace += 1\n",
    "print(y_pred_vorace)\n",
    "print(y_test)\n",
    "accuracy = accuracy_score(y_test, y_pred_vorace)\n",
    "print(accuracy)\n",
    "conf_mat = confusion_matrix(y_test, y_pred_vorace)\n",
    "print(conf_mat)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# GLASS"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 311,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "     Id number: 1 to 214  RI: refractive index  \\\n",
      "0                      1               1.52101   \n",
      "1                      2               1.51761   \n",
      "2                      3               1.51618   \n",
      "3                      4               1.51766   \n",
      "4                      5               1.51742   \n",
      "..                   ...                   ...   \n",
      "209                  210               1.51623   \n",
      "210                  211               1.51685   \n",
      "211                  212               1.52065   \n",
      "212                  213               1.51651   \n",
      "213                  214               1.51711   \n",
      "\n",
      "     Na: Sodium (unit measurement: weight percent in corresponding oxide, as are attributes 4-10)  \\\n",
      "0                                                13.64                                              \n",
      "1                                                13.89                                              \n",
      "2                                                13.53                                              \n",
      "3                                                13.21                                              \n",
      "4                                                13.27                                              \n",
      "..                                                 ...                                              \n",
      "209                                              14.14                                              \n",
      "210                                              14.92                                              \n",
      "211                                              14.36                                              \n",
      "212                                              14.38                                              \n",
      "213                                              14.23                                              \n",
      "\n",
      "     Mg: Magnesium  Al: Aluminum  Si: Silicon  K: Potassium  Ca: Calcium  \\\n",
      "0             4.49          1.10        71.78          0.06         8.75   \n",
      "1             3.60          1.36        72.73          0.48         7.83   \n",
      "2             3.55          1.54        72.99          0.39         7.78   \n",
      "3             3.69          1.29        72.61          0.57         8.22   \n",
      "4             3.62          1.24        73.08          0.55         8.07   \n",
      "..             ...           ...          ...           ...          ...   \n",
      "209           0.00          2.88        72.61          0.08         9.18   \n",
      "210           0.00          1.99        73.06          0.00         8.40   \n",
      "211           0.00          2.02        73.42          0.00         8.44   \n",
      "212           0.00          1.94        73.61          0.00         8.48   \n",
      "213           0.00          2.08        73.36          0.00         8.62   \n",
      "\n",
      "     Ba: Barium  Fe: Iron  Type of glass: (class attribute)  \n",
      "0          0.00       0.0                                 1  \n",
      "1          0.00       0.0                                 1  \n",
      "2          0.00       0.0                                 1  \n",
      "3          0.00       0.0                                 1  \n",
      "4          0.00       0.0                                 1  \n",
      "..          ...       ...                               ...  \n",
      "209        1.06       0.0                                 7  \n",
      "210        1.59       0.0                                 7  \n",
      "211        1.64       0.0                                 7  \n",
      "212        1.57       0.0                                 7  \n",
      "213        1.67       0.0                                 7  \n",
      "\n",
      "[214 rows x 11 columns]\n",
      "Pipeline(steps=[('standardscaler', StandardScaler()),\n",
      "                ('kneighborsclassifier', KNeighborsClassifier())])\n",
      "Pipeline(steps=[('standardscaler', StandardScaler()),\n",
      "                ('randomforestclassifier',\n",
      "                 RandomForestClassifier(random_state=42))])\n",
      "Pipeline(steps=[('standardscaler', StandardScaler()),\n",
      "                ('logisticregression', LogisticRegression())])\n",
      "Pipeline(steps=[('standardscaler', StandardScaler()),\n",
      "                ('svc', SVC(probability=True))])\n",
      "Pipeline(steps=[('standardscaler', StandardScaler()),\n",
      "                ('gradientboostingclassifier', GradientBoostingClassifier())])\n",
      "Pipeline(steps=[('standardscaler', StandardScaler()),\n",
      "                ('decisiontreeclassifier', DecisionTreeClassifier())])\n",
      "VotingClassifier(estimators=[('Logistic Regression', LogisticRegression()),\n",
      "                             ('Decision Tree', DecisionTreeClassifier()),\n",
      "                             ('Random Forest',\n",
      "                              RandomForestClassifier(random_state=42)),\n",
      "                             ('Gradient Boosting',\n",
      "                              GradientBoostingClassifier()),\n",
      "                             ('K-Nearest Neighbors', KNeighborsClassifier()),\n",
      "                             ('SVM', SVC(probability=True))],\n",
      "                 n_jobs=-2, verbose=True, voting='soft')\n"
     ]
    }
   ],
   "source": [
    "weights = []\n",
    "fscores = []\n",
    "saved_models = []\n",
    "best_index = 0\n",
    "epsilon = 0.0\n",
    "\n",
    "data_glass=pd.read_csv('./data/glass.data',names=['Id number: 1 to 214', 'RI: refractive index', 'Na: Sodium (unit measurement: weight percent in corresponding oxide, as are attributes 4-10)', 'Mg: Magnesium', 'Al: Aluminum', 'Si: Silicon','K: Potassium', 'Ca: Calcium','Ba: Barium','Fe: Iron','Type of glass: (class attribute)'])\n",
    "print(data_glass)\n",
    "data_glass_target_names = np.array(['1 building_windows_float_processed', '2 building_windows_non_float_processed', '3 vehicle_windows_float_processed','4 vehicle_windows_non_float_processed', '5 containers', '6 tableware', '7 headlamps'])\n",
    "x=data_glass.drop(['Type of glass: (class attribute)'], axis=1)\n",
    "y=data_glass['Type of glass: (class attribute)']\n",
    "X_train, X_test, y_train, y_test = train_test_split(x, y, test_size=0.20, random_state=42)\n",
    "\n",
    "# define the folder containing the joblib files\n",
    "folder_path_gridsearch = \"./models/experiment_glass_gridsearch/\"\n",
    "folder_path_no_gridsearch = \"./models/experiment_glass_no_gridsearch/\"\n",
    "\n",
    "weights_gridsearch_glass_no_gridsearch, model_weights_glass_no_gridsearch, glass_models = get_model_weights(X_test, y_test, folder_path_no_gridsearch)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 312,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "MODEL WEIGHTS FOR GLASS WITHOUT GRIDSEARCH - USING TEST DATA\n",
      "[0.797535070651112, 1.0, 0.926154647905674, 0.8468992248062015, 0.9765914385000192, 0.9765914385000192, 1.0]\n",
      "{'Logistic Regression': 11.393, 'Random Forest': 14.286, 'Decision Tree': 13.231, 'K-Nearest Neighbors': 12.099, 'Gradient Boosting': 13.951, 'SVM': 13.951, 'ensemble': 14.286}\n",
      "--------------------------------------------------\n",
      "6\n",
      "1.0\n"
     ]
    }
   ],
   "source": [
    "print(\"MODEL WEIGHTS FOR GLASS WITHOUT GRIDSEARCH - USING TEST DATA\")\n",
    "print(weights_gridsearch_glass_no_gridsearch)\n",
    "print(model_weights_glass_no_gridsearch)\n",
    "print(\"-\"*50)\n",
    "\n",
    "best_index = get_best_model_index(weights_gridsearch_glass_no_gridsearch)\n",
    "print(best_index)\n",
    "print(weights_gridsearch_glass_no_gridsearch[best_index])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 313,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "id": "DpmfLCiaNkHd",
    "outputId": "7dc554ad-bf33-4dba-e630-f8c77227a46a",
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[1 6 1 6 2 2 1 2 2 2 5 4 2 2 5 4 6 1 1 5 2 6 6 6 3 2 1 1 4 1 1 2 3 2 1 6 4\n",
      " 3 2 2 2 6 1]\n",
      "9      1\n",
      "197    7\n",
      "66     1\n",
      "191    7\n",
      "117    2\n",
      "111    2\n",
      "15     1\n",
      "86     2\n",
      "75     2\n",
      "144    2\n",
      "182    6\n",
      "170    5\n",
      "141    2\n",
      "73     2\n",
      "178    6\n",
      "167    5\n",
      "190    7\n",
      "18     1\n",
      "45     1\n",
      "184    6\n",
      "98     2\n",
      "209    7\n",
      "211    7\n",
      "195    7\n",
      "148    3\n",
      "104    2\n",
      "30     1\n",
      "25     1\n",
      "175    5\n",
      "16     1\n",
      "55     1\n",
      "138    2\n",
      "158    3\n",
      "93     2\n",
      "69     1\n",
      "203    7\n",
      "171    5\n",
      "152    3\n",
      "97     2\n",
      "84     2\n",
      "101    2\n",
      "200    7\n",
      "60     1\n",
      "Name: Type of glass: (class attribute), dtype: int64\n",
      "0.6511627906976745\n",
      "[[11  0  0  0  0  0  0]\n",
      " [ 0 14  0  0  0  0  0]\n",
      " [ 0  0  3  0  0  0  0]\n",
      " [ 0  0  0  0  0  0  0]\n",
      " [ 0  0  0  4  0  0  0]\n",
      " [ 0  0  0  0  3  0  0]\n",
      " [ 0  0  0  0  0  8  0]]\n"
     ]
    }
   ],
   "source": [
    "y_pred_vorace,_ = predict(X_test, glass_models, best_index, False, len(np.unique(y_test)), \"Plurality\", True, False, \"best\", False, weights_gridsearch_glass_no_gridsearch)\n",
    "y_pred_vorace += 1\n",
    "print(y_pred_vorace)\n",
    "print(y_test)\n",
    "accuracy = accuracy_score(y_test, y_pred_vorace)\n",
    "print(accuracy)\n",
    "conf_mat = confusion_matrix(y_test, y_pred_vorace)\n",
    "print(conf_mat)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 314,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "id": "dbrabZXMNt6i",
    "outputId": "69c9b6d2-b506-4632-e3ab-8c6c4feda7c2",
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[1 6 1 6 2 2 1 2 2 2 5 4 2 2 5 4 6 1 1 5 2 6 6 6 3 2 1 1 4 1 1 2 3 2 1 6 4\n",
      " 3 2 2 2 6 1]\n",
      "9      1\n",
      "197    7\n",
      "66     1\n",
      "191    7\n",
      "117    2\n",
      "111    2\n",
      "15     1\n",
      "86     2\n",
      "75     2\n",
      "144    2\n",
      "182    6\n",
      "170    5\n",
      "141    2\n",
      "73     2\n",
      "178    6\n",
      "167    5\n",
      "190    7\n",
      "18     1\n",
      "45     1\n",
      "184    6\n",
      "98     2\n",
      "209    7\n",
      "211    7\n",
      "195    7\n",
      "148    3\n",
      "104    2\n",
      "30     1\n",
      "25     1\n",
      "175    5\n",
      "16     1\n",
      "55     1\n",
      "138    2\n",
      "158    3\n",
      "93     2\n",
      "69     1\n",
      "203    7\n",
      "171    5\n",
      "152    3\n",
      "97     2\n",
      "84     2\n",
      "101    2\n",
      "200    7\n",
      "60     1\n",
      "Name: Type of glass: (class attribute), dtype: int64\n",
      "0.6511627906976745\n",
      "[[11  0  0  0  0  0  0]\n",
      " [ 0 14  0  0  0  0  0]\n",
      " [ 0  0  3  0  0  0  0]\n",
      " [ 0  0  0  0  0  0  0]\n",
      " [ 0  0  0  4  0  0  0]\n",
      " [ 0  0  0  0  3  0  0]\n",
      " [ 0  0  0  0  0  8  0]]\n"
     ]
    }
   ],
   "source": [
    "y_pred_vorace,_ = predict(X_test, glass_models, best_index, False, len(np.unique(y_test)), \"Borda\", True, False, \"best\", False, weights_gridsearch_glass_no_gridsearch)\n",
    "y_pred_vorace += 1\n",
    "print(y_pred_vorace)\n",
    "print(y_test)\n",
    "accuracy = accuracy_score(y_test, y_pred_vorace)\n",
    "print(accuracy)\n",
    "conf_mat = confusion_matrix(y_test, y_pred_vorace)\n",
    "print(conf_mat)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 315,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[1 6 1 6 2 2 1 2 2 2 5 4 2 2 5 4 6 1 1 5 2 6 6 6 3 2 1 1 4 1 1 2 3 2 1 6 4\n",
      " 3 2 2 2 6 1]\n",
      "9      1\n",
      "197    7\n",
      "66     1\n",
      "191    7\n",
      "117    2\n",
      "111    2\n",
      "15     1\n",
      "86     2\n",
      "75     2\n",
      "144    2\n",
      "182    6\n",
      "170    5\n",
      "141    2\n",
      "73     2\n",
      "178    6\n",
      "167    5\n",
      "190    7\n",
      "18     1\n",
      "45     1\n",
      "184    6\n",
      "98     2\n",
      "209    7\n",
      "211    7\n",
      "195    7\n",
      "148    3\n",
      "104    2\n",
      "30     1\n",
      "25     1\n",
      "175    5\n",
      "16     1\n",
      "55     1\n",
      "138    2\n",
      "158    3\n",
      "93     2\n",
      "69     1\n",
      "203    7\n",
      "171    5\n",
      "152    3\n",
      "97     2\n",
      "84     2\n",
      "101    2\n",
      "200    7\n",
      "60     1\n",
      "Name: Type of glass: (class attribute), dtype: int64\n",
      "0.6511627906976745\n",
      "[[11  0  0  0  0  0  0]\n",
      " [ 0 14  0  0  0  0  0]\n",
      " [ 0  0  3  0  0  0  0]\n",
      " [ 0  0  0  0  0  0  0]\n",
      " [ 0  0  0  4  0  0  0]\n",
      " [ 0  0  0  0  3  0  0]\n",
      " [ 0  0  0  0  0  8  0]]\n"
     ]
    }
   ],
   "source": [
    "y_pred_vorace,_ = predict(X_test, glass_models, best_index, False, len(np.unique(y_test)), \"Sum\", True, False, \"best\", False, weights_gridsearch_glass_no_gridsearch)\n",
    "y_pred_vorace += 1\n",
    "print(y_pred_vorace)\n",
    "print(y_test)\n",
    "accuracy = accuracy_score(y_test, y_pred_vorace)\n",
    "print(accuracy)\n",
    "conf_mat = confusion_matrix(y_test, y_pred_vorace)\n",
    "print(conf_mat)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 317,
   "metadata": {},
   "outputs": [
    {
     "ename": "AttributeError",
     "evalue": "module 'collections' has no attribute 'Iterable'",
     "output_type": "error",
     "traceback": [
      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[0;31mAttributeError\u001b[0m                            Traceback (most recent call last)",
      "\u001b[0;32m/tmp/ipykernel_62795/1136692444.py\u001b[0m in \u001b[0;36m<cell line: 1>\u001b[0;34m()\u001b[0m\n\u001b[0;32m----> 1\u001b[0;31m \u001b[0my_pred_vorace\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0m_\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mpredict\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mX_test\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mglass_models\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mbest_index\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;32mFalse\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mlen\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mnp\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0munique\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0my_test\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m\"Kemeny\"\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;32mTrue\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;32mFalse\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m\"best\"\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;32mFalse\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mweights_gridsearch_glass_no_gridsearch\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m      2\u001b[0m \u001b[0my_pred_vorace\u001b[0m \u001b[0;34m+=\u001b[0m \u001b[0;36m1\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m      3\u001b[0m \u001b[0mprint\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0my_pred_vorace\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m      4\u001b[0m \u001b[0mprint\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0my_test\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m      5\u001b[0m \u001b[0maccuracy\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0maccuracy_score\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0my_test\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0my_pred_vorace\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/tmp/ipykernel_62795/3667822946.py\u001b[0m in \u001b[0;36mpredict\u001b[0;34m(x, models, best_index, bestClassifier, nClasses, voting, argMax, weighted, tiebreak, epsilon, weights)\u001b[0m\n\u001b[1;32m    158\u001b[0m     \u001b[0;32melse\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    159\u001b[0m         \u001b[0;32mif\u001b[0m \u001b[0mweighted\u001b[0m\u001b[0;34m==\u001b[0m\u001b[0;32mFalse\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 160\u001b[0;31m             \u001b[0my_pred\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mtemp_fscores\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mvotingSystem\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mvoting\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mmodels\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mx\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mclasses\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mnClasses\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mproba\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;32mTrue\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    161\u001b[0m         \u001b[0;32melse\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    162\u001b[0m             \u001b[0my_pred\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mtemp_fscores\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mvotingSystem\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mvoting\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mmodels\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mx\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mclasses\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mnClasses\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mproba\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;32mTrue\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mweights\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mweights\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/tmp/ipykernel_62795/3667822946.py\u001b[0m in \u001b[0;36mvotingSystem\u001b[0;34m(scoring, models, x, y, classes, best_index, weights, proba)\u001b[0m\n\u001b[1;32m    144\u001b[0m         \u001b[0mscores\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mnp\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mmean\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mpreferences\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0maxis\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;36m1\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    145\u001b[0m     \u001b[0;32melif\u001b[0m \u001b[0mscoring\u001b[0m \u001b[0;34m==\u001b[0m \u001b[0;34m\"Kemeny\"\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 146\u001b[0;31m         \u001b[0mscores\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mrankaggr_brute\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mpreferences\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    147\u001b[0m     \u001b[0;32melse\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    148\u001b[0m         \u001b[0mscores\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mscoringVec\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mscoring\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mpreferences\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mweights\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mweights\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/tmp/ipykernel_62795/3667822946.py\u001b[0m in \u001b[0;36mrankaggr_brute\u001b[0;34m(preferences)\u001b[0m\n\u001b[1;32m     69\u001b[0m         \u001b[0msc\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mScoringScheme\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     70\u001b[0m         \u001b[0;32mif\u001b[0m \u001b[0mlen\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mprofile\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;36m0\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m>\u001b[0m\u001b[0;36m5\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 71\u001b[0;31m             \u001b[0mconsensus\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mKemRankAgg\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mcompute_consensus\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mranks\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0msc\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mAlgorithm\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mParCons\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m     72\u001b[0m         \u001b[0;32melse\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     73\u001b[0m             \u001b[0mconsensus\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mKemRankAgg\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mcompute_consensus\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mranks\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0msc\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mAlgorithm\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mExact\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/.local/share/virtualenvs/CSC_homework_3_group4-nat9sqnt/lib/python3.10/site-packages/corankco/kemrankagg.py\u001b[0m in \u001b[0;36mcompute_consensus\u001b[0;34m(dataset, scoring_scheme, algorithm, single_consensus)\u001b[0m\n\u001b[1;32m     15\u001b[0m                           single_consensus=False) -> Consensus:\n\u001b[1;32m     16\u001b[0m         \u001b[0malg\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mAlgorithmEnumeration\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mmedian_ranking_algorithms\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0malgorithm\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mvalue\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 17\u001b[0;31m         \u001b[0;32mreturn\u001b[0m \u001b[0malg\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mcompute_consensus_rankings\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mdataset\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mscoring_scheme\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0msingle_consensus\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m",
      "\u001b[0;32m~/.local/share/virtualenvs/CSC_homework_3_group4-nat9sqnt/lib/python3.10/site-packages/corankco/algorithms/parcons/parcons.py\u001b[0m in \u001b[0;36m__init__\u001b[0;34m(self, algorithm_to_complete, bound_for_exact)\u001b[0m\n\u001b[1;32m     16\u001b[0m             \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0malg\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0malgorithm_to_complete\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     17\u001b[0m         \u001b[0;32melse\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 18\u001b[0;31m             \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0malg\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mBioConsert\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mstarting_algorithms\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;32mNone\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m     19\u001b[0m         \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mbound_for_exact\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mbound_for_exact\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     20\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/.local/share/virtualenvs/CSC_homework_3_group4-nat9sqnt/lib/python3.10/site-packages/corankco/algorithms/bioconsert/bioconsert.py\u001b[0m in \u001b[0;36m__init__\u001b[0;34m(self, starting_algorithms)\u001b[0m\n\u001b[1;32m     13\u001b[0m     \u001b[0;32mdef\u001b[0m \u001b[0m__init__\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mstarting_algorithms\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;32mNone\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     14\u001b[0m         \u001b[0mis_valid\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0;32mTrue\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 15\u001b[0;31m         \u001b[0;32mif\u001b[0m \u001b[0misinstance\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mstarting_algorithms\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mcollections\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mIterable\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m     16\u001b[0m             \u001b[0;32mfor\u001b[0m \u001b[0mobj\u001b[0m \u001b[0;32min\u001b[0m \u001b[0mstarting_algorithms\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     17\u001b[0m                 \u001b[0;32mif\u001b[0m \u001b[0;32mnot\u001b[0m \u001b[0misinstance\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mobj\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mMedianRanking\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;31mAttributeError\u001b[0m: module 'collections' has no attribute 'Iterable'"
     ]
    }
   ],
   "source": [
    "y_pred_vorace,_ = predict(X_test, glass_models, best_index, False, len(np.unique(y_test)), \"Kemeny\", True, False, \"best\", False, weights_gridsearch_glass_no_gridsearch)\n",
    "y_pred_vorace += 1\n",
    "print(y_pred_vorace)\n",
    "print(y_test)\n",
    "accuracy = accuracy_score(y_test, y_pred_vorace)\n",
    "print(accuracy)\n",
    "conf_mat = confusion_matrix(y_test, y_pred_vorace)\n",
    "print(conf_mat)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 318,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[1 6 1 6 2 2 1 2 2 2 5 4 2 2 5 4 6 1 1 5 2 6 6 6 3 2 1 1 4 1 1 2 3 2 1 6 4\n",
      " 3 2 2 2 6 1]\n",
      "9      1\n",
      "197    7\n",
      "66     1\n",
      "191    7\n",
      "117    2\n",
      "111    2\n",
      "15     1\n",
      "86     2\n",
      "75     2\n",
      "144    2\n",
      "182    6\n",
      "170    5\n",
      "141    2\n",
      "73     2\n",
      "178    6\n",
      "167    5\n",
      "190    7\n",
      "18     1\n",
      "45     1\n",
      "184    6\n",
      "98     2\n",
      "209    7\n",
      "211    7\n",
      "195    7\n",
      "148    3\n",
      "104    2\n",
      "30     1\n",
      "25     1\n",
      "175    5\n",
      "16     1\n",
      "55     1\n",
      "138    2\n",
      "158    3\n",
      "93     2\n",
      "69     1\n",
      "203    7\n",
      "171    5\n",
      "152    3\n",
      "97     2\n",
      "84     2\n",
      "101    2\n",
      "200    7\n",
      "60     1\n",
      "Name: Type of glass: (class attribute), dtype: int64\n",
      "0.6511627906976745\n",
      "[[11  0  0  0  0  0  0]\n",
      " [ 0 14  0  0  0  0  0]\n",
      " [ 0  0  3  0  0  0  0]\n",
      " [ 0  0  0  0  0  0  0]\n",
      " [ 0  0  0  4  0  0  0]\n",
      " [ 0  0  0  0  3  0  0]\n",
      " [ 0  0  0  0  0  8  0]]\n"
     ]
    }
   ],
   "source": [
    "y_pred_vorace,_ = predict(X_test, glass_models, best_index, False, len(np.unique(y_test)), \"Copeland\", True, False, \"best\", False, weights_gridsearch_glass_no_gridsearch)\n",
    "y_pred_vorace += 1\n",
    "print(y_pred_vorace)\n",
    "print(y_test)\n",
    "accuracy = accuracy_score(y_test, y_pred_vorace)\n",
    "print(accuracy)\n",
    "conf_mat = confusion_matrix(y_test, y_pred_vorace)\n",
    "print(conf_mat)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "colab": {
   "authorship_tag": "ABX9TyPKA5jHgkTFWT5Nwiw/4GBy",
   "collapsed_sections": [],
   "include_colab_link": true,
   "mount_file_id": "16N80-trqoI7uWRAY_lXE74wYwWrW3QM3",
   "name": "vorace_test.ipynb",
   "provenance": []
  },
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.10.6"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 1
}