IPL Model Training.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "import numpy as np\n",
    "import matplotlib.pyplot as plt\n",
    "import seaborn as sns\n",
    "from pylab import rcParams\n",
    "\n",
    "rcParams[\"figure.figsize\"] = (10, 6)\n",
    "sns.set()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "# importing filtered csv \n",
    "\n",
    "main_data = pd.read_csv(\"eda_feature_engineering.csv\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>id</th>\n",
       "      <th>inning</th>\n",
       "      <th>over</th>\n",
       "      <th>ball</th>\n",
       "      <th>total_runs</th>\n",
       "      <th>is_wicket</th>\n",
       "      <th>batting_team</th>\n",
       "      <th>bowling_team</th>\n",
       "      <th>final_total_runs</th>\n",
       "      <th>wickets</th>\n",
       "      <th>runs</th>\n",
       "      <th>last_5_over_wickets</th>\n",
       "      <th>last_5_over_runs</th>\n",
       "      <th>last_5_over_balls</th>\n",
       "      <th>venue</th>\n",
       "      <th>winner</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>136141</th>\n",
       "      <td>981019</td>\n",
       "      <td>2</td>\n",
       "      <td>8.5</td>\n",
       "      <td>57</td>\n",
       "      <td>6</td>\n",
       "      <td>0</td>\n",
       "      <td>Royal Challengers Bangalore</td>\n",
       "      <td>Sunrisers Hyderabad</td>\n",
       "      <td>200</td>\n",
       "      <td>0</td>\n",
       "      <td>96</td>\n",
       "      <td>0</td>\n",
       "      <td>96</td>\n",
       "      <td>56</td>\n",
       "      <td>M Chinnaswamy Stadium</td>\n",
       "      <td>Sunrisers Hyderabad</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>106712</th>\n",
       "      <td>734033</td>\n",
       "      <td>1</td>\n",
       "      <td>11.6</td>\n",
       "      <td>75</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>Kings XI Punjab</td>\n",
       "      <td>Rajasthan Royals</td>\n",
       "      <td>179</td>\n",
       "      <td>2</td>\n",
       "      <td>103</td>\n",
       "      <td>2</td>\n",
       "      <td>103</td>\n",
       "      <td>74</td>\n",
       "      <td>Punjab Cricket Association Stadium, Mohali</td>\n",
       "      <td>Kings XI Punjab</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>188294</th>\n",
       "      <td>1216532</td>\n",
       "      <td>1</td>\n",
       "      <td>4.2</td>\n",
       "      <td>26</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>Sunrisers Hyderabad</td>\n",
       "      <td>Delhi Capitals</td>\n",
       "      <td>162</td>\n",
       "      <td>0</td>\n",
       "      <td>21</td>\n",
       "      <td>0</td>\n",
       "      <td>21</td>\n",
       "      <td>26</td>\n",
       "      <td>Sheikh Zayed Stadium</td>\n",
       "      <td>Sunrisers Hyderabad</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>131535</th>\n",
       "      <td>980979</td>\n",
       "      <td>2</td>\n",
       "      <td>7.3</td>\n",
       "      <td>45</td>\n",
       "      <td>6</td>\n",
       "      <td>0</td>\n",
       "      <td>Rising Pune Supergiants</td>\n",
       "      <td>Sunrisers Hyderabad</td>\n",
       "      <td>133</td>\n",
       "      <td>2</td>\n",
       "      <td>38</td>\n",
       "      <td>2</td>\n",
       "      <td>38</td>\n",
       "      <td>44</td>\n",
       "      <td>Dr. Y.S. Rajasekhara Reddy ACA-VDCA Cricket St...</td>\n",
       "      <td>Sunrisers Hyderabad</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>129176</th>\n",
       "      <td>980959</td>\n",
       "      <td>2</td>\n",
       "      <td>14.1</td>\n",
       "      <td>92</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>Kolkata Knight Riders</td>\n",
       "      <td>Royal Challengers Bangalore</td>\n",
       "      <td>189</td>\n",
       "      <td>4</td>\n",
       "      <td>106</td>\n",
       "      <td>4</td>\n",
       "      <td>106</td>\n",
       "      <td>91</td>\n",
       "      <td>M Chinnaswamy Stadium</td>\n",
       "      <td>Kolkata Knight Riders</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "             id  inning  over  ball  total_runs  is_wicket  \\\n",
       "136141   981019       2   8.5    57           6          0   \n",
       "106712   734033       1  11.6    75           1          0   \n",
       "188294  1216532       1   4.2    26           1          0   \n",
       "131535   980979       2   7.3    45           6          0   \n",
       "129176   980959       2  14.1    92           1          0   \n",
       "\n",
       "                       batting_team                 bowling_team  \\\n",
       "136141  Royal Challengers Bangalore          Sunrisers Hyderabad   \n",
       "106712              Kings XI Punjab             Rajasthan Royals   \n",
       "188294          Sunrisers Hyderabad               Delhi Capitals   \n",
       "131535      Rising Pune Supergiants          Sunrisers Hyderabad   \n",
       "129176        Kolkata Knight Riders  Royal Challengers Bangalore   \n",
       "\n",
       "        final_total_runs  wickets  runs  last_5_over_wickets  \\\n",
       "136141               200        0    96                    0   \n",
       "106712               179        2   103                    2   \n",
       "188294               162        0    21                    0   \n",
       "131535               133        2    38                    2   \n",
       "129176               189        4   106                    4   \n",
       "\n",
       "        last_5_over_runs  last_5_over_balls  \\\n",
       "136141                96                 56   \n",
       "106712               103                 74   \n",
       "188294                21                 26   \n",
       "131535                38                 44   \n",
       "129176               106                 91   \n",
       "\n",
       "                                                    venue  \\\n",
       "136141                              M Chinnaswamy Stadium   \n",
       "106712         Punjab Cricket Association Stadium, Mohali   \n",
       "188294                               Sheikh Zayed Stadium   \n",
       "131535  Dr. Y.S. Rajasekhara Reddy ACA-VDCA Cricket St...   \n",
       "129176                              M Chinnaswamy Stadium   \n",
       "\n",
       "                       winner  \n",
       "136141    Sunrisers Hyderabad  \n",
       "106712        Kings XI Punjab  \n",
       "188294    Sunrisers Hyderabad  \n",
       "131535    Sunrisers Hyderabad  \n",
       "129176  Kolkata Knight Riders  "
      ]
     },
     "execution_count": 3,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "main_data.sample(5)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(193096, 16)"
      ]
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "main_data.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array(['Kolkata Knight Riders', 'Royal Challengers Bangalore',\n",
       "       'Chennai Super Kings', 'Kings XI Punjab', 'Rajasthan Royals',\n",
       "       'Delhi Daredevils', 'Mumbai Indians', 'Deccan Chargers',\n",
       "       'Kochi Tuskers Kerala', 'Pune Warriors', 'Sunrisers Hyderabad',\n",
       "       'Rising Pune Supergiants', 'Gujarat Lions',\n",
       "       'Rising Pune Supergiant', 'Delhi Capitals'], dtype=object)"
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# here in current year 2021 some of team name has been changed and some of team are not playing\n",
    "# let's drop and rename teams\n",
    "\n",
    "main_data.batting_team.unique()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "# only taking more than 5 over records of match\n",
    "\n",
    "main_data = main_data.loc[main_data.over >= 5]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "# here batting team and balling team values are same\n",
    "\n",
    "teams_2021 = [\"Royal Challengers Bangalore\", \"Kolkata Knight Riders\", \"Rajasthan Royals\", \"Punjab Kings\", \"Sunrisers Hyderabad\", \"Delhi Capitals\", \"Mumbai Indians\", \"Chennai Super Kings\"]\n",
    "\n",
    "# here some of the teams name changed in year 2021 is \n",
    "# Delhi Daredevils becames Delhi Capitals\n",
    "# Kings XI Punjab becames Punjab Kings\n",
    "\n",
    "# let's rename this teams with current name\n",
    "\n",
    "main_data.replace({\"Delhi Daredevils\": \"Delhi Capitals\", \"Kings XI Punjab\": \"Punjab Kings\"}, inplace=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "# let's get those teams values which are not in year 2021 IPL\n",
    "ignoring_team = list(filter(lambda x: x not in teams_2021, main_data.batting_team.unique()))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "# let's drop only those teams data which are not in year 2021\n",
    "\n",
    "for i in ignoring_team:\n",
    "    main_data.drop(main_data.loc[(main_data.batting_team == i) | (main_data.bowling_team == i)].index, axis=0, inplace=True)\n",
    "    \n",
    "main_data = main_data.reset_index(drop=True)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Let's verifying 2021 teams with batting_team, bowling_team and winner"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "8"
      ]
     },
     "execution_count": 10,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(teams_2021)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "8"
      ]
     },
     "execution_count": 11,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "main_data.batting_team.nunique()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "8"
      ]
     },
     "execution_count": 12,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "main_data.bowling_team.nunique()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "8"
      ]
     },
     "execution_count": 13,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "main_data.winner.nunique()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [],
   "source": [
    "teams_2021.sort()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['Chennai Super Kings',\n",
       " 'Delhi Capitals',\n",
       " 'Kolkata Knight Riders',\n",
       " 'Mumbai Indians',\n",
       " 'Punjab Kings',\n",
       " 'Rajasthan Royals',\n",
       " 'Royal Challengers Bangalore',\n",
       " 'Sunrisers Hyderabad']"
      ]
     },
     "execution_count": 15,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "teams_2021"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([ True,  True,  True,  True,  True,  True,  True,  True])"
      ]
     },
     "execution_count": 16,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "teams_2021 == np.sort(main_data.batting_team.unique())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([ True,  True,  True,  True,  True,  True,  True,  True])"
      ]
     },
     "execution_count": 17,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "teams_2021 == np.sort(main_data.bowling_team.unique())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([ True,  True,  True,  True,  True,  True,  True,  True])"
      ]
     },
     "execution_count": 18,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "teams_2021 == np.sort(main_data.winner.unique())"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Feature Encoding"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'Sunrisers Hyderabad': 0,\n",
       " 'Delhi Capitals': 1,\n",
       " 'Rajasthan Royals': 2,\n",
       " 'Royal Challengers Bangalore': 3,\n",
       " 'Punjab Kings': 4,\n",
       " 'Kolkata Knight Riders': 5,\n",
       " 'Chennai Super Kings': 6,\n",
       " 'Mumbai Indians': 7}"
      ]
     },
     "execution_count": 19,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# here batting_team and bowling_team are in string format and ML model can not accept\n",
    "# string values so let's encode it into numeric with the help of winning team\n",
    "\n",
    "\n",
    "# here i'm going to do label encoding base on counting of highest winning team\n",
    "# here i'm ordering team base on winning count and making dict using team name as key and encoded values\n",
    "\n",
    "encoded_teams = {k:v for v, k in enumerate(main_data.winner.value_counts().sort_values().index, 0)}\n",
    "encoded_teams\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [],
   "source": [
    "# let's create a new data frame and map this encoded values with batting and bowling teams\n",
    "\n",
    "df1 = main_data.copy()\n",
    "\n",
    "\n",
    "df1.batting_team = df1.batting_team.map(encoded_teams)\n",
    "df1.bowling_team = df1.bowling_team.map(encoded_teams)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>id</th>\n",
       "      <th>inning</th>\n",
       "      <th>over</th>\n",
       "      <th>ball</th>\n",
       "      <th>total_runs</th>\n",
       "      <th>is_wicket</th>\n",
       "      <th>batting_team</th>\n",
       "      <th>bowling_team</th>\n",
       "      <th>final_total_runs</th>\n",
       "      <th>wickets</th>\n",
       "      <th>runs</th>\n",
       "      <th>last_5_over_wickets</th>\n",
       "      <th>last_5_over_runs</th>\n",
       "      <th>last_5_over_balls</th>\n",
       "      <th>venue</th>\n",
       "      <th>winner</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>335982</td>\n",
       "      <td>1</td>\n",
       "      <td>5.1</td>\n",
       "      <td>33</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>5</td>\n",
       "      <td>3</td>\n",
       "      <td>222</td>\n",
       "      <td>0</td>\n",
       "      <td>61</td>\n",
       "      <td>0</td>\n",
       "      <td>60</td>\n",
       "      <td>32</td>\n",
       "      <td>M Chinnaswamy Stadium</td>\n",
       "      <td>Kolkata Knight Riders</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>335982</td>\n",
       "      <td>1</td>\n",
       "      <td>5.2</td>\n",
       "      <td>34</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>5</td>\n",
       "      <td>3</td>\n",
       "      <td>222</td>\n",
       "      <td>1</td>\n",
       "      <td>61</td>\n",
       "      <td>1</td>\n",
       "      <td>60</td>\n",
       "      <td>33</td>\n",
       "      <td>M Chinnaswamy Stadium</td>\n",
       "      <td>Kolkata Knight Riders</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>335982</td>\n",
       "      <td>1</td>\n",
       "      <td>5.3</td>\n",
       "      <td>35</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>5</td>\n",
       "      <td>3</td>\n",
       "      <td>222</td>\n",
       "      <td>1</td>\n",
       "      <td>61</td>\n",
       "      <td>1</td>\n",
       "      <td>60</td>\n",
       "      <td>34</td>\n",
       "      <td>M Chinnaswamy Stadium</td>\n",
       "      <td>Kolkata Knight Riders</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>335982</td>\n",
       "      <td>1</td>\n",
       "      <td>5.4</td>\n",
       "      <td>36</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>5</td>\n",
       "      <td>3</td>\n",
       "      <td>222</td>\n",
       "      <td>1</td>\n",
       "      <td>61</td>\n",
       "      <td>1</td>\n",
       "      <td>60</td>\n",
       "      <td>35</td>\n",
       "      <td>M Chinnaswamy Stadium</td>\n",
       "      <td>Kolkata Knight Riders</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>335982</td>\n",
       "      <td>1</td>\n",
       "      <td>5.5</td>\n",
       "      <td>37</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>5</td>\n",
       "      <td>3</td>\n",
       "      <td>222</td>\n",
       "      <td>1</td>\n",
       "      <td>61</td>\n",
       "      <td>1</td>\n",
       "      <td>60</td>\n",
       "      <td>36</td>\n",
       "      <td>M Chinnaswamy Stadium</td>\n",
       "      <td>Kolkata Knight Riders</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "       id  inning  over  ball  total_runs  is_wicket  batting_team  \\\n",
       "0  335982       1   5.1    33           1          0             5   \n",
       "1  335982       1   5.2    34           0          1             5   \n",
       "2  335982       1   5.3    35           0          0             5   \n",
       "3  335982       1   5.4    36           0          0             5   \n",
       "4  335982       1   5.5    37           0          0             5   \n",
       "\n",
       "   bowling_team  final_total_runs  wickets  runs  last_5_over_wickets  \\\n",
       "0             3               222        0    61                    0   \n",
       "1             3               222        1    61                    1   \n",
       "2             3               222        1    61                    1   \n",
       "3             3               222        1    61                    1   \n",
       "4             3               222        1    61                    1   \n",
       "\n",
       "   last_5_over_runs  last_5_over_balls                  venue  \\\n",
       "0                60                 32  M Chinnaswamy Stadium   \n",
       "1                60                 33  M Chinnaswamy Stadium   \n",
       "2                60                 34  M Chinnaswamy Stadium   \n",
       "3                60                 35  M Chinnaswamy Stadium   \n",
       "4                60                 36  M Chinnaswamy Stadium   \n",
       "\n",
       "                  winner  \n",
       "0  Kolkata Knight Riders  \n",
       "1  Kolkata Knight Riders  \n",
       "2  Kolkata Knight Riders  \n",
       "3  Kolkata Knight Riders  \n",
       "4  Kolkata Knight Riders  "
      ]
     },
     "execution_count": 21,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df1.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "id                       int64\n",
       "inning                   int64\n",
       "over                   float64\n",
       "ball                     int64\n",
       "total_runs               int64\n",
       "is_wicket                int64\n",
       "batting_team             int64\n",
       "bowling_team             int64\n",
       "final_total_runs         int64\n",
       "wickets                  int64\n",
       "runs                     int64\n",
       "last_5_over_wickets      int64\n",
       "last_5_over_runs         int64\n",
       "last_5_over_balls        int64\n",
       "venue                   object\n",
       "winner                  object\n",
       "dtype: object"
      ]
     },
     "execution_count": 22,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# let's check the type of the df1 \n",
    "\n",
    "df1.dtypes"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Brabourne Stadium</th>\n",
       "      <th>Buffalo Park</th>\n",
       "      <th>De Beers Diamond Oval</th>\n",
       "      <th>Dr DY Patil Sports Academy</th>\n",
       "      <th>Dr. Y.S. Rajasekhara Reddy ACA-VDCA Cricket Stadium</th>\n",
       "      <th>Dubai International Cricket Stadium</th>\n",
       "      <th>Eden Gardens</th>\n",
       "      <th>Feroz Shah Kotla</th>\n",
       "      <th>Himachal Pradesh Cricket Association Stadium</th>\n",
       "      <th>Holkar Cricket Stadium</th>\n",
       "      <th>...</th>\n",
       "      <th>Rajiv Gandhi International Stadium, Uppal</th>\n",
       "      <th>Sardar Patel Stadium, Motera</th>\n",
       "      <th>Sawai Mansingh Stadium</th>\n",
       "      <th>Shaheed Veer Narayan Singh International Stadium</th>\n",
       "      <th>Sharjah Cricket Stadium</th>\n",
       "      <th>Sheikh Zayed Stadium</th>\n",
       "      <th>St George's Park</th>\n",
       "      <th>Subrata Roy Sahara Stadium</th>\n",
       "      <th>SuperSport Park</th>\n",
       "      <th>Wankhede Stadium</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>72087</th>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>77354</th>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>69791</th>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>3 rows × 29 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "       Brabourne Stadium  Buffalo Park  De Beers Diamond Oval  \\\n",
       "72087                  0             0                      0   \n",
       "77354                  0             0                      0   \n",
       "69791                  0             0                      0   \n",
       "\n",
       "       Dr DY Patil Sports Academy  \\\n",
       "72087                           0   \n",
       "77354                           0   \n",
       "69791                           0   \n",
       "\n",
       "       Dr. Y.S. Rajasekhara Reddy ACA-VDCA Cricket Stadium  \\\n",
       "72087                                                  0     \n",
       "77354                                                  0     \n",
       "69791                                                  0     \n",
       "\n",
       "       Dubai International Cricket Stadium  Eden Gardens  Feroz Shah Kotla  \\\n",
       "72087                                    0             0                 0   \n",
       "77354                                    0             0                 0   \n",
       "69791                                    0             0                 0   \n",
       "\n",
       "       Himachal Pradesh Cricket Association Stadium  Holkar Cricket Stadium  \\\n",
       "72087                                             0                       0   \n",
       "77354                                             0                       0   \n",
       "69791                                             0                       0   \n",
       "\n",
       "       ...  Rajiv Gandhi International Stadium, Uppal  \\\n",
       "72087  ...                                          0   \n",
       "77354  ...                                          0   \n",
       "69791  ...                                          1   \n",
       "\n",
       "       Sardar Patel Stadium, Motera  Sawai Mansingh Stadium  \\\n",
       "72087                             0                       0   \n",
       "77354                             0                       0   \n",
       "69791                             0                       0   \n",
       "\n",
       "       Shaheed Veer Narayan Singh International Stadium  \\\n",
       "72087                                                 1   \n",
       "77354                                                 0   \n",
       "69791                                                 0   \n",
       "\n",
       "       Sharjah Cricket Stadium  Sheikh Zayed Stadium  St George's Park  \\\n",
       "72087                        0                     0                 0   \n",
       "77354                        0                     0                 0   \n",
       "69791                        0                     0                 0   \n",
       "\n",
       "       Subrata Roy Sahara Stadium  SuperSport Park  Wankhede Stadium  \n",
       "72087                           0                0                 0  \n",
       "77354                           0                0                 1  \n",
       "69791                           0                0                 0  \n",
       "\n",
       "[3 rows x 29 columns]"
      ]
     },
     "execution_count": 23,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# venue column have object data types \n",
    "# so let's encoded venue column\n",
    "\n",
    "# here i'm doing one hot encoding for veune\n",
    "encoded_venue = pd.get_dummies(df1.venue, drop_first=True)\n",
    "# here i drop first venue value for prevent dummy variable trap when all values are 0 it indiicates\n",
    "# first column which value was Barabati Stadium\n",
    "\n",
    "encoded_venue.sample(3)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Goal\n",
    "- Here my goal is to predict score of batting team in IPL match\n",
    "- Here target variable is final_total_runs"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Feature Selection"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>over</th>\n",
       "      <th>wickets</th>\n",
       "      <th>runs</th>\n",
       "      <th>last_5_over_wickets</th>\n",
       "      <th>last_5_over_runs</th>\n",
       "      <th>batting_team</th>\n",
       "      <th>bowling_team</th>\n",
       "      <th>final_total_runs</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>5.1</td>\n",
       "      <td>0</td>\n",
       "      <td>61</td>\n",
       "      <td>0</td>\n",
       "      <td>60</td>\n",
       "      <td>5</td>\n",
       "      <td>3</td>\n",
       "      <td>222</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>5.2</td>\n",
       "      <td>1</td>\n",
       "      <td>61</td>\n",
       "      <td>1</td>\n",
       "      <td>60</td>\n",
       "      <td>5</td>\n",
       "      <td>3</td>\n",
       "      <td>222</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>5.3</td>\n",
       "      <td>1</td>\n",
       "      <td>61</td>\n",
       "      <td>1</td>\n",
       "      <td>60</td>\n",
       "      <td>5</td>\n",
       "      <td>3</td>\n",
       "      <td>222</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>5.4</td>\n",
       "      <td>1</td>\n",
       "      <td>61</td>\n",
       "      <td>1</td>\n",
       "      <td>60</td>\n",
       "      <td>5</td>\n",
       "      <td>3</td>\n",
       "      <td>222</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>5.5</td>\n",
       "      <td>1</td>\n",
       "      <td>61</td>\n",
       "      <td>1</td>\n",
       "      <td>60</td>\n",
       "      <td>5</td>\n",
       "      <td>3</td>\n",
       "      <td>222</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   over  wickets  runs  last_5_over_wickets  last_5_over_runs  batting_team  \\\n",
       "0   5.1        0    61                    0                60             5   \n",
       "1   5.2        1    61                    1                60             5   \n",
       "2   5.3        1    61                    1                60             5   \n",
       "3   5.4        1    61                    1                60             5   \n",
       "4   5.5        1    61                    1                60             5   \n",
       "\n",
       "   bowling_team  final_total_runs  \n",
       "0             3               222  \n",
       "1             3               222  \n",
       "2             3               222  \n",
       "3             3               222  \n",
       "4             3               222  "
      ]
     },
     "execution_count": 24,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# here i'm taking over, batting_team, bowling_team, wickets, runs, balls, last_5_over_wickets, last_5_over_runs, las_5_over_bals, venue feaures\n",
    "# and final_total_runs target variable\n",
    "\n",
    "df2 = df1[[\"over\", \"wickets\", \"runs\", \"last_5_over_wickets\", \"last_5_over_runs\",\"batting_team\", \"bowling_team\", \"final_total_runs\"]].copy()\n",
    "\n",
    "\n",
    "df2.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>over</th>\n",
       "      <th>wickets</th>\n",
       "      <th>runs</th>\n",
       "      <th>last_5_over_wickets</th>\n",
       "      <th>last_5_over_runs</th>\n",
       "      <th>batting_team</th>\n",
       "      <th>bowling_team</th>\n",
       "      <th>final_total_runs</th>\n",
       "      <th>Brabourne Stadium</th>\n",
       "      <th>Buffalo Park</th>\n",
       "      <th>...</th>\n",
       "      <th>Rajiv Gandhi International Stadium, Uppal</th>\n",
       "      <th>Sardar Patel Stadium, Motera</th>\n",
       "      <th>Sawai Mansingh Stadium</th>\n",
       "      <th>Shaheed Veer Narayan Singh International Stadium</th>\n",
       "      <th>Sharjah Cricket Stadium</th>\n",
       "      <th>Sheikh Zayed Stadium</th>\n",
       "      <th>St George's Park</th>\n",
       "      <th>Subrata Roy Sahara Stadium</th>\n",
       "      <th>SuperSport Park</th>\n",
       "      <th>Wankhede Stadium</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>37271</th>\n",
       "      <td>10.1</td>\n",
       "      <td>3</td>\n",
       "      <td>63</td>\n",
       "      <td>3</td>\n",
       "      <td>63</td>\n",
       "      <td>2</td>\n",
       "      <td>5</td>\n",
       "      <td>144</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>99893</th>\n",
       "      <td>5.5</td>\n",
       "      <td>0</td>\n",
       "      <td>55</td>\n",
       "      <td>0</td>\n",
       "      <td>54</td>\n",
       "      <td>0</td>\n",
       "      <td>7</td>\n",
       "      <td>151</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>102937</th>\n",
       "      <td>15.6</td>\n",
       "      <td>4</td>\n",
       "      <td>111</td>\n",
       "      <td>4</td>\n",
       "      <td>111</td>\n",
       "      <td>5</td>\n",
       "      <td>0</td>\n",
       "      <td>163</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>3 rows × 37 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "        over  wickets  runs  last_5_over_wickets  last_5_over_runs  \\\n",
       "37271   10.1        3    63                    3                63   \n",
       "99893    5.5        0    55                    0                54   \n",
       "102937  15.6        4   111                    4               111   \n",
       "\n",
       "        batting_team  bowling_team  final_total_runs  Brabourne Stadium  \\\n",
       "37271              2             5               144                  0   \n",
       "99893              0             7               151                  0   \n",
       "102937             5             0               163                  0   \n",
       "\n",
       "        Buffalo Park  ...  Rajiv Gandhi International Stadium, Uppal  \\\n",
       "37271              0  ...                                          0   \n",
       "99893              0  ...                                          0   \n",
       "102937             0  ...                                          0   \n",
       "\n",
       "        Sardar Patel Stadium, Motera  Sawai Mansingh Stadium  \\\n",
       "37271                              0                       1   \n",
       "99893                              0                       0   \n",
       "102937                             0                       0   \n",
       "\n",
       "        Shaheed Veer Narayan Singh International Stadium  \\\n",
       "37271                                                  0   \n",
       "99893                                                  0   \n",
       "102937                                                 0   \n",
       "\n",
       "        Sharjah Cricket Stadium  Sheikh Zayed Stadium  St George's Park  \\\n",
       "37271                         0                     0                 0   \n",
       "99893                         1                     0                 0   \n",
       "102937                        0                     1                 0   \n",
       "\n",
       "        Subrata Roy Sahara Stadium  SuperSport Park  Wankhede Stadium  \n",
       "37271                            0                0                 0  \n",
       "99893                            0                0                 0  \n",
       "102937                           0                0                 0  \n",
       "\n",
       "[3 rows x 37 columns]"
      ]
     },
     "execution_count": 25,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# here in df2 i don't have venue column for that i'm concatenate encoded_venue dataframe with df2\n",
    "\n",
    "df2 = pd.concat([df2, encoded_venue], axis=1).copy()\n",
    "df2.sample(3)\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Model Building"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {},
   "outputs": [],
   "source": [
    "# here let's creat feature matrix X and target vector y\n",
    "\n",
    "X = df2.drop(columns=\"final_total_runs\") # 38 features\n",
    "y = df2.final_total_runs"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(((88046, 36), (88046,)), ((22012, 36), (22012,)))"
      ]
     },
     "execution_count": 27,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# let's split the data into train and test part\n",
    "\n",
    "from sklearn.model_selection import train_test_split\n",
    "\n",
    "X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.2)\n",
    "\n",
    "(X_train.shape, y_train.shape), (X_test.shape, y_test.shape)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {},
   "outputs": [],
   "source": [
    "# scaling\n",
    "\n",
    "from sklearn.preprocessing import MinMaxScaler\n",
    "\n",
    "scaler = MinMaxScaler()\n",
    "\n",
    "X_train = scaler.fit_transform(X_train)\n",
    "X_test = scaler.transform(X_test)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.linear_model import LinearRegression, Ridge, Lasso\n",
    "from sklearn.tree import DecisionTreeRegressor\n",
    "from sklearn.ensemble import RandomForestRegressor, AdaBoostRegressor, GradientBoostingRegressor\n",
    "\n",
    "\n",
    "algos = {\n",
    "    \"Linear Regression\": {\n",
    "        \"model\": LinearRegression(),\n",
    "        \"params\": {}\n",
    "    },\n",
    "    \"Ridge\": {\n",
    "        \"model\": Ridge(),\n",
    "        \"params\": {\n",
    "            \"alpha\":np.arange(0.1, 1, 0.01)\n",
    "        }\n",
    "    },\n",
    "    \"Lasso\": {\n",
    "        \"model\": Lasso(),\n",
    "        \"params\": {\n",
    "            \"alpha\": np.arange(0.1, 1, 0.01)\n",
    "        }\n",
    "    },\n",
    "    \"Decision Tree\": {\n",
    "        \"model\": DecisionTreeRegressor(),\n",
    "        \"params\": {\n",
    "            \"criterion\": [\"mse\", \"friedman_mse\"],\n",
    "            \"splitter\": [\"best\", \"random\"],\n",
    "            \"max_depth\": [1, 3, 5, 7, 9, 10, 11, 12, 14, 15, 18, 20, 25, 28, 30, 33, 38, 40],\n",
    "            \"min_samples_split\": [2, 4, 6, 8, 10, 15, 20],\n",
    "            \"min_samples_leaf\": [i for i in range(1, 11)],\n",
    "            \"max_leaf_nodes\": [None] + [i for i in range(10, 91, 10)],\n",
    "            \"max_features\": [\"auto\", \"log2\", \"sqrt\", None]            \n",
    "        }\n",
    "    },\n",
    "    \"Random Forest\": {\n",
    "        \"model\": RandomForestRegressor(),\n",
    "        \"params\": {\n",
    "            \"n_estimators\": [100, 200, 300],\n",
    "            \"criterion\": [\"mse\", \"friedman_mse\"],\n",
    "            \"max_depth\": [1, 3, 5, 7, 9, 10, 11, 12, 14, 15, 18, 20, 25, 28, 30, 33, 38, 40],\n",
    "            \"min_samples_split\": [2, 4, 6, 8, 10, 15, 20],\n",
    "            \"min_samples_leaf\": [i for i in range(1, 11)],\n",
    "            \"max_leaf_nodes\": [None] + [i for i in range(10, 91, 10)],\n",
    "            \"max_features\": [\"auto\", \"log2\", \"sqrt\", None]\n",
    "        }\n",
    "    },\n",
    "    \"Ada Boost\": {\n",
    "        \"model\": AdaBoostRegressor(),\n",
    "        \"params\": {\n",
    "            \"n_estimators\": [100, 200, 300],\n",
    "            \"learning_rate\": np.arange(0.1, 1, 0.01),\n",
    "            \"loss\": ['linear', 'square', 'exponential']\n",
    "        }\n",
    "    },\n",
    "    \"Gradient Boost\": {\n",
    "        \"model\": GradientBoostingRegressor(),\n",
    "        \"params\": {\n",
    "            \"learning_rate\": np.arange(0.1, 1, 0.01),\n",
    "            \"n_estimators\": [100, 200, 300],\n",
    "            \"criterion\": ['friedman_mse', 'mse'],\n",
    "            \"min_samples_split\": [2, 4, 6, 8, 10, 15, 20],\n",
    "            \"min_samples_leaf\": [i for i in range(1, 11)],\n",
    "            \"max_depth\": [1, 3, 5, 7, 9, 10, 11, 12, 14, 15, 18, 20, 25, 28, 30, 33, 38, 40],\n",
    "            \"max_features\": [\"auto\", \"log2\", \"sqrt\", None],\n",
    "            \"max_leaf_nodes\": [None] + [i for i in range(10, 91, 10)],\n",
    "            \"alpha\": np.arange(0.1, 1, 0.01)\n",
    "        }\n",
    "    }\n",
    "}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Fitting 5 folds for each of 1 candidates, totalling 5 fits\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/opt/conda/lib/python3.9/site-packages/sklearn/model_selection/_search.py:285: UserWarning: The total space of parameters 1 is smaller than n_iter=15. Running 1 iterations. For exhaustive searches, use GridSearchCV.\n",
      "  warnings.warn(\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "---fitted---\n",
      "Linear Regression\n",
      "Fitting 5 folds for each of 15 candidates, totalling 75 fits\n",
      "---fitted---\n",
      "Ridge\n",
      "Fitting 5 folds for each of 15 candidates, totalling 75 fits\n",
      "---fitted---\n",
      "Lasso\n",
      "Fitting 5 folds for each of 15 candidates, totalling 75 fits\n",
      "---fitted---\n",
      "Decision Tree\n",
      "Fitting 5 folds for each of 15 candidates, totalling 75 fits\n",
      "---fitted---\n",
      "Random Forest\n",
      "Fitting 5 folds for each of 15 candidates, totalling 75 fits\n",
      "---fitted---\n",
      "Ada Boost\n",
      "Fitting 5 folds for each of 15 candidates, totalling 75 fits\n",
      "---fitted---\n",
      "Gradient Boost\n",
      "--------------------------------------------------------\n",
      "it takes 10.815720025698344 minutes\n",
      "--------------------------------------------------------\n"
     ]
    }
   ],
   "source": [
    "from sklearn.model_selection import RandomizedSearchCV\n",
    "import time\n",
    "\n",
    "start_time = time.time()\n",
    "best_model = {}\n",
    "best_model_details = []\n",
    "\n",
    "for model_name, values in algos.items():\n",
    "    rscv = RandomizedSearchCV(values[\"model\"], values[\"params\"], cv=5, n_iter=15, n_jobs=-1, verbose=2, random_state=4)\n",
    "    rscv.fit(X_train, y_train)\n",
    "    print(\"---fitted---\")\n",
    "    best_model[model_name] = rscv\n",
    "    best_model_details.append({\"Model Name\": model_name, \"Best Score\": rscv.best_score_, \"Best Parameters\": rscv.best_params_})\n",
    "    print(model_name)\n",
    "\n",
    "\n",
    "print(\"--------------------------------------------------------\")\n",
    "print(f\"it takes {(time.time() - start_time) / 60} minutes\")\n",
    "print(\"--------------------------------------------------------\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Model Name</th>\n",
       "      <th>Best Score</th>\n",
       "      <th>Best Parameters</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>Linear Regression</td>\n",
       "      <td>0.526527</td>\n",
       "      <td>{}</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>Ridge</td>\n",
       "      <td>0.526528</td>\n",
       "      <td>{'alpha': 0.19999999999999996}</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>Lasso</td>\n",
       "      <td>0.499577</td>\n",
       "      <td>{'alpha': 0.19999999999999996}</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>Decision Tree</td>\n",
       "      <td>0.501321</td>\n",
       "      <td>{'splitter': 'best', 'min_samples_split': 2, 'min_samples_leaf': 3, 'max_leaf_nodes': 90, 'max_features': 'auto', 'max_depth': 20, 'criterion': 'friedman_mse'}</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>Random Forest</td>\n",
       "      <td>0.672894</td>\n",
       "      <td>{'n_estimators': 300, 'min_samples_split': 15, 'min_samples_leaf': 5, 'max_leaf_nodes': None, 'max_features': 'log2', 'max_depth': 20, 'criterion': 'mse'}</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>Ada Boost</td>\n",
       "      <td>0.406849</td>\n",
       "      <td>{'n_estimators': 300, 'loss': 'linear', 'learning_rate': 0.2799999999999999}</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6</th>\n",
       "      <td>Gradient Boost</td>\n",
       "      <td>0.850539</td>\n",
       "      <td>{'n_estimators': 200, 'min_samples_split': 10, 'min_samples_leaf': 5, 'max_leaf_nodes': None, 'max_features': 'log2', 'max_depth': 14, 'learning_rate': 0.6799999999999997, 'criterion': 'mse', 'alpha': 0.7099999999999996}</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "          Model Name  Best Score  \\\n",
       "0  Linear Regression    0.526527   \n",
       "1              Ridge    0.526528   \n",
       "2              Lasso    0.499577   \n",
       "3      Decision Tree    0.501321   \n",
       "4      Random Forest    0.672894   \n",
       "5          Ada Boost    0.406849   \n",
       "6     Gradient Boost    0.850539   \n",
       "\n",
       "                                                                                                                                                                                                                Best Parameters  \n",
       "0                                                                                                                                                                                                                            {}  \n",
       "1                                                                                                                                                                                                {'alpha': 0.19999999999999996}  \n",
       "2                                                                                                                                                                                                {'alpha': 0.19999999999999996}  \n",
       "3                                                               {'splitter': 'best', 'min_samples_split': 2, 'min_samples_leaf': 3, 'max_leaf_nodes': 90, 'max_features': 'auto', 'max_depth': 20, 'criterion': 'friedman_mse'}  \n",
       "4                                                                    {'n_estimators': 300, 'min_samples_split': 15, 'min_samples_leaf': 5, 'max_leaf_nodes': None, 'max_features': 'log2', 'max_depth': 20, 'criterion': 'mse'}  \n",
       "5                                                                                                                                                  {'n_estimators': 300, 'loss': 'linear', 'learning_rate': 0.2799999999999999}  \n",
       "6  {'n_estimators': 200, 'min_samples_split': 10, 'min_samples_leaf': 5, 'max_leaf_nodes': None, 'max_features': 'log2', 'max_depth': 14, 'learning_rate': 0.6799999999999997, 'criterion': 'mse', 'alpha': 0.7099999999999996}  "
      ]
     },
     "execution_count": 31,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "pd.set_option('display.max_colwidth', None)\n",
    "pd.DataFrame(best_model_details)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Model Name</th>\n",
       "      <th>Test Score</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>Linear Regression</td>\n",
       "      <td>0.525730</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>Ridge</td>\n",
       "      <td>0.525731</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>Lasso</td>\n",
       "      <td>0.497081</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>Decision Tree</td>\n",
       "      <td>0.497885</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>Random Forest</td>\n",
       "      <td>0.681484</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>Ada Boost</td>\n",
       "      <td>0.398871</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6</th>\n",
       "      <td>Gradient Boost</td>\n",
       "      <td>0.871335</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "          Model Name  Test Score\n",
       "0  Linear Regression    0.525730\n",
       "1              Ridge    0.525731\n",
       "2              Lasso    0.497081\n",
       "3      Decision Tree    0.497885\n",
       "4      Random Forest    0.681484\n",
       "5          Ada Boost    0.398871\n",
       "6     Gradient Boost    0.871335"
      ]
     },
     "execution_count": 32,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "test_model = []\n",
    "\n",
    "for model_name, model in best_model.items():\n",
    "    test_model.append({\"Model Name\": model_name, \"Test Score\": model.score(X_test, y_test)})\n",
    "\n",
    "pd.DataFrame(test_model)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "metadata": {},
   "outputs": [],
   "source": [
    "# let's calculate error using MSE, RMSE and MAE for Gradient Boost algorithm\n",
    "\n",
    "from sklearn.metrics import mean_absolute_error, mean_squared_error\n",
    "\n",
    "train_model_error = []\n",
    "test_model_error = []\n",
    "\n",
    "for model_name, model in best_model.items():\n",
    "    y_pred_train = model.predict(X_train)\n",
    "    y_pred_test = model.predict(X_test)\n",
    "    \n",
    "    train_mae = mean_absolute_error(y_train, y_pred_train)\n",
    "    train_mse = mean_squared_error(y_train, y_pred_train)\n",
    "    train_rmse = np.sqrt(train_mse)\n",
    "\n",
    "    test_mae = mean_absolute_error(y_test, y_pred_test)\n",
    "    test_mse = mean_squared_error(y_test, y_pred_test)\n",
    "    test_rmse = np.sqrt(test_mse)\n",
    "    \n",
    "    train_model_error.append({\"Model Name\": model_name, \"Mean Absolute Error\": train_mae, \"Mean Squared Error\": train_mse, \"Root Mean Squared Error\": train_rmse})\n",
    "    test_model_error.append({\"Model Name\": model_name, \"Mean Absolute Error\": test_mae, \"Mean Squared Error\": test_mse, \"Root Mean Squared Error\": test_rmse})\n",
    "    \n",
    "\n",
    "train_model_error = pd.DataFrame(train_model_error)\n",
    "test_model_error = pd.DataFrame(test_model_error)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 34,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "-------- Training Data Error --------\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Model Name</th>\n",
       "      <th>Mean Absolute Error</th>\n",
       "      <th>Mean Squared Error</th>\n",
       "      <th>Root Mean Squared Error</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>Linear Regression</td>\n",
       "      <td>14.893146</td>\n",
       "      <td>407.665162</td>\n",
       "      <td>20.190720</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>Ridge</td>\n",
       "      <td>14.893306</td>\n",
       "      <td>407.665877</td>\n",
       "      <td>20.190737</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>Lasso</td>\n",
       "      <td>15.483640</td>\n",
       "      <td>431.115602</td>\n",
       "      <td>20.763323</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>Decision Tree</td>\n",
       "      <td>15.446022</td>\n",
       "      <td>420.360457</td>\n",
       "      <td>20.502694</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>Random Forest</td>\n",
       "      <td>11.296707</td>\n",
       "      <td>239.046683</td>\n",
       "      <td>15.461135</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>Ada Boost</td>\n",
       "      <td>17.624336</td>\n",
       "      <td>514.592183</td>\n",
       "      <td>22.684624</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6</th>\n",
       "      <td>Gradient Boost</td>\n",
       "      <td>1.632532</td>\n",
       "      <td>7.702044</td>\n",
       "      <td>2.775256</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "          Model Name  Mean Absolute Error  Mean Squared Error  \\\n",
       "0  Linear Regression            14.893146          407.665162   \n",
       "1              Ridge            14.893306          407.665877   \n",
       "2              Lasso            15.483640          431.115602   \n",
       "3      Decision Tree            15.446022          420.360457   \n",
       "4      Random Forest            11.296707          239.046683   \n",
       "5          Ada Boost            17.624336          514.592183   \n",
       "6     Gradient Boost             1.632532            7.702044   \n",
       "\n",
       "   Root Mean Squared Error  \n",
       "0                20.190720  \n",
       "1                20.190737  \n",
       "2                20.763323  \n",
       "3                20.502694  \n",
       "4                15.461135  \n",
       "5                22.684624  \n",
       "6                 2.775256  "
      ]
     },
     "execution_count": 34,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "print(\"-------- Training Data Error --------\")\n",
    "train_model_error"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 35,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "-------- Test Data Error --------\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Model Name</th>\n",
       "      <th>Mean Absolute Error</th>\n",
       "      <th>Mean Squared Error</th>\n",
       "      <th>Root Mean Squared Error</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>Linear Regression</td>\n",
       "      <td>14.977500</td>\n",
       "      <td>406.774691</td>\n",
       "      <td>20.168656</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>Ridge</td>\n",
       "      <td>14.977604</td>\n",
       "      <td>406.774002</td>\n",
       "      <td>20.168639</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>Lasso</td>\n",
       "      <td>15.555353</td>\n",
       "      <td>431.346365</td>\n",
       "      <td>20.768880</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>Decision Tree</td>\n",
       "      <td>15.710737</td>\n",
       "      <td>430.656772</td>\n",
       "      <td>20.752272</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>Random Forest</td>\n",
       "      <td>12.173585</td>\n",
       "      <td>273.186774</td>\n",
       "      <td>16.528363</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>Ada Boost</td>\n",
       "      <td>17.687932</td>\n",
       "      <td>515.579447</td>\n",
       "      <td>22.706375</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6</th>\n",
       "      <td>Gradient Boost</td>\n",
       "      <td>6.438084</td>\n",
       "      <td>110.353874</td>\n",
       "      <td>10.504945</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "          Model Name  Mean Absolute Error  Mean Squared Error  \\\n",
       "0  Linear Regression            14.977500          406.774691   \n",
       "1              Ridge            14.977604          406.774002   \n",
       "2              Lasso            15.555353          431.346365   \n",
       "3      Decision Tree            15.710737          430.656772   \n",
       "4      Random Forest            12.173585          273.186774   \n",
       "5          Ada Boost            17.687932          515.579447   \n",
       "6     Gradient Boost             6.438084          110.353874   \n",
       "\n",
       "   Root Mean Squared Error  \n",
       "0                20.168656  \n",
       "1                20.168639  \n",
       "2                20.768880  \n",
       "3                20.752272  \n",
       "4                16.528363  \n",
       "5                22.706375  \n",
       "6                10.504945  "
      ]
     },
     "execution_count": 35,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "print(\"-------- Test Data Error --------\")\n",
    "test_model_error"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Model Selection\n",
    "- Gradient Booost algorithm gives 0.850539 r2_score & 1.632532 MAE on train dataset and 0.871335 r2_score & 6.438084 MAE on test dataset\n",
    "- Random Forest algorithm gives 0.672894 r2_score & 11.296707 on train dataset and 0.681484 r2_score & 12.173585 on test dataset\n",
    "\n",
    "### Conclusion\n",
    "- Difference of train dataset MAE and test dataset MAE is 4.805552 for Gradient Boost\n",
    "- Difference of train dataset MAE and test dataset MAE is 0.876878 for Random Forest\n",
    "- Here accuracy of gradient boost is higher than random forest but Random Forest's MAE difference between train and test dataset is lower than Gradient Boost\n",
    "- so if i select Gradient Boost algorithm which has high r2_score but still our model not perform stable as it perform on train dataset because MAE difference between train and test dataset is high that's why it might generate overfitting issue.\n",
    "- if i select Radom Forest algorithm which has low r2_score but it will perfrom stable as it performed on train dataset because MAE difference between train and test dataset is low\n",
    "\n",
    "- I'm selecting **Random Forest** as final model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 36,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[CV] END ..........................alpha=0.23999999999999994; total time=   0.1s\n",
      "[CV] END ...........................alpha=0.5099999999999998; total time=   0.1s\n",
      "[CV] END ...........................alpha=0.8399999999999996; total time=   0.1s\n",
      "[CV] END ...........................alpha=0.3699999999999999; total time=   0.1s\n",
      "[CV] END ..........................alpha=0.48999999999999977; total time=   0.1s\n",
      "[CV] END ..........................alpha=0.19999999999999996; total time=   1.2s\n",
      "[CV] END ...........................alpha=0.7499999999999997; total time=   0.5s\n",
      "[CV] END ...........................alpha=0.7099999999999996; total time=   0.5s\n",
      "[CV] END ...........................alpha=0.5299999999999998; total time=   0.7s\n",
      "[CV] END ..........................alpha=0.29999999999999993; total time=   1.2s\n",
      "[CV] END ...........................alpha=0.3699999999999999; total time=   1.0s\n",
      "[CV] END criterion=friedman_mse, max_depth=5, max_features=sqrt, max_leaf_nodes=40, min_samples_leaf=3, min_samples_split=10, splitter=random; total time=   0.1s\n",
      "[CV] END criterion=friedman_mse, max_depth=10, max_features=auto, max_leaf_nodes=40, min_samples_leaf=2, min_samples_split=15, splitter=best; total time=   0.4s\n",
      "[CV] END criterion=friedman_mse, max_depth=40, max_features=auto, max_leaf_nodes=20, min_samples_leaf=5, min_samples_split=20, splitter=best; total time=   0.3s\n",
      "[CV] END criterion=friedman_mse, max_depth=40, max_features=auto, max_leaf_nodes=20, min_samples_leaf=5, min_samples_split=20, splitter=best; total time=   0.3s\n",
      "[CV] END criterion=friedman_mse, max_depth=7, max_features=None, max_leaf_nodes=50, min_samples_leaf=3, min_samples_split=6, splitter=best; total time=   0.3s\n",
      "[CV] END criterion=mse, max_depth=33, max_features=log2, max_leaf_nodes=60, min_samples_leaf=2, min_samples_split=4, n_estimators=100; total time=   5.7s\n",
      "[CV] END criterion=mse, max_depth=20, max_features=log2, max_leaf_nodes=None, min_samples_leaf=5, min_samples_split=15, n_estimators=300; total time=  19.9s\n",
      "[CV] END criterion=mse, max_depth=15, max_features=sqrt, max_leaf_nodes=None, min_samples_leaf=9, min_samples_split=10, n_estimators=200; total time=  14.9s\n",
      "[CV] END criterion=friedman_mse, max_depth=18, max_features=sqrt, max_leaf_nodes=40, min_samples_leaf=10, min_samples_split=8, n_estimators=200; total time=  12.3s\n",
      "[CV] END criterion=mse, max_depth=30, max_features=sqrt, max_leaf_nodes=30, min_samples_leaf=5, min_samples_split=20, n_estimators=100; total time=   6.0s\n",
      "[CV] END criterion=friedman_mse, max_depth=15, max_features=auto, max_leaf_nodes=80, min_samples_leaf=8, min_samples_split=20, n_estimators=200; total time=  42.9s\n",
      "[CV] END criterion=mse, max_depth=7, max_features=auto, max_leaf_nodes=10, min_samples_leaf=10, min_samples_split=2, n_estimators=200; total time=  27.8s\n",
      "[CV] END learning_rate=0.46999999999999986, loss=exponential, n_estimators=200; total time=  47.5s\n",
      "[CV] END learning_rate=0.5099999999999998, loss=exponential, n_estimators=200; total time=  46.2s\n",
      "[CV] END learning_rate=0.6199999999999998, loss=exponential, n_estimators=100; total time=  22.4s\n",
      "[CV] END learning_rate=0.9099999999999996, loss=square, n_estimators=100; total time=  20.9s\n",
      "[CV] END learning_rate=0.4299999999999998, loss=square, n_estimators=100; total time=  22.0s\n",
      "[CV] END learning_rate=0.16999999999999998, loss=square, n_estimators=200; total time=  44.2s\n",
      "[CV] END alpha=0.3699999999999999, criterion=friedman_mse, learning_rate=0.5499999999999998, max_depth=18, max_features=auto, max_leaf_nodes=30, min_samples_leaf=7, min_samples_split=15, n_estimators=300; total time= 1.0min\n",
      "[CV] END alpha=0.44999999999999984, criterion=friedman_mse, learning_rate=0.8699999999999996, max_depth=10, max_features=auto, max_leaf_nodes=50, min_samples_leaf=8, min_samples_split=15, n_estimators=200; total time=  48.2s\n",
      "[CV] END alpha=0.4099999999999998, criterion=friedman_mse, learning_rate=0.44999999999999984, max_depth=12, max_features=log2, max_leaf_nodes=90, min_samples_leaf=7, min_samples_split=2, n_estimators=300; total time=  16.7s\n",
      "[CV] END alpha=0.6699999999999997, criterion=friedman_mse, learning_rate=0.17999999999999997, max_depth=20, max_features=auto, max_leaf_nodes=40, min_samples_leaf=6, min_samples_split=20, n_estimators=200; total time=  50.1s\n",
      "[CV] END alpha=0.11, criterion=friedman_mse, learning_rate=0.8499999999999996, max_depth=9, max_features=sqrt, max_leaf_nodes=50, min_samples_leaf=7, min_samples_split=6, n_estimators=100; total time=   6.4s\n",
      "[CV] END ..........................alpha=0.19999999999999996; total time=   0.1s\n",
      "[CV] END ...........................alpha=0.8399999999999996; total time=   0.1s\n",
      "[CV] END ..........................alpha=0.22999999999999995; total time=   0.1s\n",
      "[CV] END ...........................alpha=0.7799999999999997; total time=   0.1s\n",
      "[CV] END ..........................alpha=0.23999999999999994; total time=   1.3s\n",
      "[CV] END ...........................alpha=0.7499999999999997; total time=   0.5s\n",
      "[CV] END ...........................alpha=0.7099999999999996; total time=   0.5s\n",
      "[CV] END ...........................alpha=0.5299999999999998; total time=   0.8s\n",
      "[CV] END ...........................alpha=0.8399999999999996; total time=   0.4s\n",
      "[CV] END ...........................alpha=0.9599999999999995; total time=   0.4s\n",
      "[CV] END ...........................alpha=0.3699999999999999; total time=   1.0s\n",
      "[CV] END ...........................alpha=0.7799999999999997; total time=   0.4s\n",
      "[CV] END criterion=friedman_mse, max_depth=7, max_features=sqrt, max_leaf_nodes=20, min_samples_leaf=2, min_samples_split=15, splitter=random; total time=   0.1s\n",
      "[CV] END criterion=friedman_mse, max_depth=12, max_features=sqrt, max_leaf_nodes=None, min_samples_leaf=10, min_samples_split=10, splitter=random; total time=   0.1s\n",
      "[CV] END criterion=friedman_mse, max_depth=15, max_features=sqrt, max_leaf_nodes=20, min_samples_leaf=10, min_samples_split=20, splitter=best; total time=   0.1s\n",
      "[CV] END criterion=friedman_mse, max_depth=15, max_features=sqrt, max_leaf_nodes=20, min_samples_leaf=10, min_samples_split=20, splitter=best; total time=   0.1s\n",
      "[CV] END criterion=mse, max_depth=30, max_features=log2, max_leaf_nodes=10, min_samples_leaf=3, min_samples_split=20, splitter=random; total time=   0.1s\n",
      "[CV] END criterion=mse, max_depth=30, max_features=log2, max_leaf_nodes=10, min_samples_leaf=3, min_samples_split=20, splitter=random; total time=   0.1s\n",
      "[CV] END criterion=mse, max_depth=30, max_features=log2, max_leaf_nodes=10, min_samples_leaf=3, min_samples_split=20, splitter=random; total time=   0.1s\n",
      "[CV] END criterion=mse, max_depth=30, max_features=log2, max_leaf_nodes=10, min_samples_leaf=3, min_samples_split=20, splitter=random; total time=   0.1s\n",
      "[CV] END criterion=friedman_mse, max_depth=28, max_features=sqrt, max_leaf_nodes=70, min_samples_leaf=7, min_samples_split=2, splitter=random; total time=   0.1s\n",
      "[CV] END criterion=friedman_mse, max_depth=20, max_features=auto, max_leaf_nodes=90, min_samples_leaf=3, min_samples_split=2, splitter=best; total time=   0.4s\n",
      "[CV] END criterion=mse, max_depth=30, max_features=log2, max_leaf_nodes=40, min_samples_leaf=8, min_samples_split=20, n_estimators=100; total time=   5.1s\n",
      "[CV] END criterion=mse, max_depth=28, max_features=sqrt, max_leaf_nodes=80, min_samples_leaf=4, min_samples_split=2, n_estimators=200; total time=  11.9s\n",
      "[CV] END criterion=mse, max_depth=25, max_features=None, max_leaf_nodes=20, min_samples_leaf=3, min_samples_split=15, n_estimators=200; total time=  31.3s\n",
      "[CV] END criterion=friedman_mse, max_depth=14, max_features=None, max_leaf_nodes=70, min_samples_leaf=3, min_samples_split=20, n_estimators=200; total time=  37.5s\n",
      "[CV] END criterion=friedman_mse, max_depth=10, max_features=sqrt, max_leaf_nodes=30, min_samples_leaf=8, min_samples_split=20, n_estimators=200; total time=  11.0s\n",
      "[CV] END criterion=mse, max_depth=7, max_features=auto, max_leaf_nodes=10, min_samples_leaf=10, min_samples_split=2, n_estimators=200; total time=  26.6s\n",
      "[CV] END learning_rate=0.4099999999999998, loss=exponential, n_estimators=300; total time= 1.1min\n",
      "[CV] END learning_rate=0.5099999999999998, loss=exponential, n_estimators=200; total time=  43.7s\n",
      "[CV] END learning_rate=0.9899999999999995, loss=linear, n_estimators=100; total time=   5.6s\n",
      "[CV] END learning_rate=0.30999999999999994, loss=linear, n_estimators=300; total time=  13.9s\n",
      "[CV] END learning_rate=0.8299999999999996, loss=exponential, n_estimators=300; total time= 1.0min\n",
      "[CV] END learning_rate=0.2799999999999999, loss=linear, n_estimators=300; total time=  13.3s\n",
      "[CV] END alpha=0.3699999999999999, criterion=friedman_mse, learning_rate=0.5499999999999998, max_depth=18, max_features=auto, max_leaf_nodes=30, min_samples_leaf=7, min_samples_split=15, n_estimators=300; total time= 1.0min\n",
      "[CV] END alpha=0.44999999999999984, criterion=friedman_mse, learning_rate=0.8699999999999996, max_depth=10, max_features=auto, max_leaf_nodes=50, min_samples_leaf=8, min_samples_split=15, n_estimators=200; total time=  47.7s\n",
      "[CV] END alpha=0.4099999999999998, criterion=friedman_mse, learning_rate=0.44999999999999984, max_depth=12, max_features=log2, max_leaf_nodes=90, min_samples_leaf=7, min_samples_split=2, n_estimators=300; total time=  16.7s\n",
      "[CV] END alpha=0.6699999999999997, criterion=friedman_mse, learning_rate=0.17999999999999997, max_depth=20, max_features=auto, max_leaf_nodes=40, min_samples_leaf=6, min_samples_split=20, n_estimators=200; total time=  52.7s\n",
      "[CV] END alpha=0.11, criterion=friedman_mse, learning_rate=0.8499999999999996, max_depth=9, max_features=sqrt, max_leaf_nodes=50, min_samples_leaf=7, min_samples_split=6, n_estimators=100; total time=   6.8s\n",
      "[CV] END ..........................alpha=0.23999999999999994; total time=   0.1s\n",
      "[CV] END ..........................alpha=0.29999999999999993; total time=   0.1s\n",
      "[CV] END ...........................alpha=0.9599999999999995; total time=   0.1s\n",
      "[CV] END ...........................alpha=0.3699999999999999; total time=   0.1s\n",
      "[CV] END ..........................alpha=0.48999999999999977; total time=   0.1s\n",
      "[CV] END ...........................alpha=0.8599999999999995; total time=   0.5s\n",
      "[CV] END ...........................alpha=0.8599999999999995; total time=   0.4s\n",
      "[CV] END ...........................alpha=0.8599999999999995; total time=   0.4s\n",
      "[CV] END ...........................alpha=0.7499999999999997; total time=   0.4s\n",
      "[CV] END ...........................alpha=0.7099999999999996; total time=   0.5s\n",
      "[CV] END ...........................alpha=0.5099999999999998; total time=   0.8s\n",
      "[CV] END ...........................alpha=0.8399999999999996; total time=   0.4s\n",
      "[CV] END ...........................alpha=0.9599999999999995; total time=   0.3s\n",
      "[CV] END ...........................alpha=0.3699999999999999; total time=   1.0s\n",
      "[CV] END ..........................alpha=0.48999999999999977; total time=   0.7s\n",
      "[CV] END criterion=friedman_mse, max_depth=7, max_features=sqrt, max_leaf_nodes=20, min_samples_leaf=2, min_samples_split=15, splitter=random; total time=   0.1s\n",
      "[CV] END criterion=friedman_mse, max_depth=12, max_features=sqrt, max_leaf_nodes=None, min_samples_leaf=10, min_samples_split=10, splitter=random; total time=   0.2s\n",
      "[CV] END criterion=mse, max_depth=38, max_features=None, max_leaf_nodes=50, min_samples_leaf=8, min_samples_split=4, splitter=random; total time=   0.3s\n",
      "[CV] END criterion=mse, max_depth=38, max_features=None, max_leaf_nodes=50, min_samples_leaf=8, min_samples_split=4, splitter=random; total time=   0.3s\n",
      "[CV] END criterion=mse, max_depth=7, max_features=sqrt, max_leaf_nodes=30, min_samples_leaf=4, min_samples_split=2, splitter=best; total time=   0.1s\n",
      "[CV] END criterion=mse, max_depth=7, max_features=sqrt, max_leaf_nodes=30, min_samples_leaf=4, min_samples_split=2, splitter=best; total time=   0.1s\n",
      "[CV] END criterion=friedman_mse, max_depth=7, max_features=None, max_leaf_nodes=50, min_samples_leaf=3, min_samples_split=6, splitter=best; total time=   0.3s\n",
      "[CV] END criterion=mse, max_depth=33, max_features=log2, max_leaf_nodes=60, min_samples_leaf=2, min_samples_split=4, n_estimators=100; total time=   5.2s\n",
      "[CV] END criterion=mse, max_depth=20, max_features=log2, max_leaf_nodes=None, min_samples_leaf=5, min_samples_split=15, n_estimators=300; total time=  20.0s\n",
      "[CV] END criterion=mse, max_depth=15, max_features=sqrt, max_leaf_nodes=None, min_samples_leaf=9, min_samples_split=10, n_estimators=200; total time=  13.9s\n",
      "[CV] END criterion=friedman_mse, max_depth=18, max_features=sqrt, max_leaf_nodes=40, min_samples_leaf=10, min_samples_split=8, n_estimators=200; total time=  12.3s\n",
      "[CV] END criterion=mse, max_depth=30, max_features=sqrt, max_leaf_nodes=30, min_samples_leaf=5, min_samples_split=20, n_estimators=100; total time=   6.1s\n",
      "[CV] END criterion=mse, max_depth=38, max_features=log2, max_leaf_nodes=80, min_samples_leaf=5, min_samples_split=20, n_estimators=300; total time=  17.7s\n",
      "[CV] END criterion=friedman_mse, max_depth=15, max_features=auto, max_leaf_nodes=80, min_samples_leaf=8, min_samples_split=20, n_estimators=200; total time=  42.9s\n",
      "[CV] END learning_rate=0.4099999999999998, loss=exponential, n_estimators=300; total time= 1.2min\n",
      "[CV] END learning_rate=0.4999999999999998, loss=square, n_estimators=200; total time=  45.2s\n",
      "[CV] END learning_rate=0.30999999999999994, loss=linear, n_estimators=300; total time=  15.4s\n",
      "[CV] END learning_rate=0.4299999999999998, loss=square, n_estimators=100; total time=  24.1s\n",
      "[CV] END learning_rate=0.16999999999999998, loss=square, n_estimators=200; total time=  28.0s\n",
      "[CV] END learning_rate=0.6799999999999997, loss=linear, n_estimators=200; total time=   6.0s\n",
      "[CV] END learning_rate=0.2799999999999999, loss=linear, n_estimators=300; total time=  12.7s\n",
      "[CV] END alpha=0.9599999999999995, criterion=friedman_mse, learning_rate=0.7399999999999997, max_depth=18, max_features=None, max_leaf_nodes=70, min_samples_leaf=3, min_samples_split=20, n_estimators=200; total time=  44.4s\n",
      "[CV] END alpha=0.43999999999999984, criterion=friedman_mse, learning_rate=0.2799999999999999, max_depth=7, max_features=None, max_leaf_nodes=10, min_samples_leaf=2, min_samples_split=6, n_estimators=200; total time=  39.0s\n",
      "[CV] END alpha=0.1, criterion=mse, learning_rate=0.9599999999999995, max_depth=40, max_features=sqrt, max_leaf_nodes=20, min_samples_leaf=7, min_samples_split=4, n_estimators=200; total time=  10.4s\n",
      "[CV] END alpha=0.48999999999999977, criterion=mse, learning_rate=0.8599999999999995, max_depth=7, max_features=log2, max_leaf_nodes=20, min_samples_leaf=3, min_samples_split=4, n_estimators=300; total time=  13.9s\n",
      "[CV] END alpha=0.1, criterion=mse, learning_rate=0.6599999999999997, max_depth=15, max_features=None, max_leaf_nodes=60, min_samples_leaf=1, min_samples_split=20, n_estimators=300; total time= 1.0min\n",
      "[CV] END alpha=0.7099999999999996, criterion=mse, learning_rate=0.6799999999999997, max_depth=14, max_features=log2, max_leaf_nodes=None, min_samples_leaf=5, min_samples_split=10, n_estimators=200; total time=  22.3s\n",
      "[CV] END .................................................... total time=   0.2s\n",
      "[CV] END ...........................alpha=0.8599999999999995; total time=   0.1s\n",
      "[CV] END ...........................alpha=0.7499999999999997; total time=   0.1s\n",
      "[CV] END ...........................alpha=0.7499999999999997; total time=   0.1s\n",
      "[CV] END ...........................alpha=0.7099999999999996; total time=   0.1s\n",
      "[CV] END ...........................alpha=0.5299999999999998; total time=   0.1s\n",
      "[CV] END ...........................alpha=0.5099999999999998; total time=   0.1s\n",
      "[CV] END ..........................alpha=0.29999999999999993; total time=   0.1s\n",
      "[CV] END ..........................alpha=0.22999999999999995; total time=   0.1s\n",
      "[CV] END ...........................alpha=0.7799999999999997; total time=   0.1s\n",
      "[CV] END ..........................alpha=0.23999999999999994; total time=   1.2s\n",
      "[CV] END ...........................alpha=0.8599999999999995; total time=   0.4s\n",
      "[CV] END ..........................alpha=0.46999999999999986; total time=   0.9s\n",
      "[CV] END ..........................alpha=0.29999999999999993; total time=   1.2s\n",
      "[CV] END ..........................alpha=0.22999999999999995; total time=   1.2s\n",
      "[CV] END ...........................alpha=0.7799999999999997; total time=   0.5s\n",
      "[CV] END criterion=friedman_mse, max_depth=5, max_features=sqrt, max_leaf_nodes=40, min_samples_leaf=3, min_samples_split=10, splitter=random; total time=   0.1s\n",
      "[CV] END criterion=friedman_mse, max_depth=10, max_features=auto, max_leaf_nodes=40, min_samples_leaf=2, min_samples_split=15, splitter=best; total time=   0.4s\n",
      "[CV] END criterion=friedman_mse, max_depth=3, max_features=auto, max_leaf_nodes=80, min_samples_leaf=5, min_samples_split=4, splitter=best; total time=   0.2s\n",
      "[CV] END criterion=friedman_mse, max_depth=3, max_features=auto, max_leaf_nodes=80, min_samples_leaf=5, min_samples_split=4, splitter=best; total time=   0.2s\n",
      "[CV] END criterion=friedman_mse, max_depth=28, max_features=sqrt, max_leaf_nodes=70, min_samples_leaf=7, min_samples_split=2, splitter=random; total time=   0.1s\n",
      "[CV] END criterion=friedman_mse, max_depth=28, max_features=sqrt, max_leaf_nodes=70, min_samples_leaf=7, min_samples_split=2, splitter=random; total time=   0.1s\n",
      "[CV] END criterion=mse, max_depth=30, max_features=log2, max_leaf_nodes=40, min_samples_leaf=8, min_samples_split=20, n_estimators=100; total time=   5.2s\n",
      "[CV] END criterion=mse, max_depth=28, max_features=sqrt, max_leaf_nodes=80, min_samples_leaf=4, min_samples_split=2, n_estimators=200; total time=  12.4s\n",
      "[CV] END criterion=mse, max_depth=15, max_features=sqrt, max_leaf_nodes=None, min_samples_leaf=9, min_samples_split=10, n_estimators=200; total time=  13.5s\n",
      "[CV] END criterion=mse, max_depth=5, max_features=log2, max_leaf_nodes=50, min_samples_leaf=6, min_samples_split=6, n_estimators=200; total time=   7.5s\n",
      "[CV] END criterion=mse, max_depth=5, max_features=log2, max_leaf_nodes=50, min_samples_leaf=6, min_samples_split=6, n_estimators=200; total time=   7.3s\n",
      "[CV] END criterion=friedman_mse, max_depth=14, max_features=None, max_leaf_nodes=70, min_samples_leaf=3, min_samples_split=20, n_estimators=200; total time=  40.0s\n",
      "[CV] END criterion=friedman_mse, max_depth=10, max_features=sqrt, max_leaf_nodes=30, min_samples_leaf=8, min_samples_split=20, n_estimators=200; total time=  10.8s\n",
      "[CV] END criterion=mse, max_depth=3, max_features=auto, max_leaf_nodes=50, min_samples_leaf=8, min_samples_split=6, n_estimators=100; total time=  11.2s\n",
      "[CV] END learning_rate=0.17999999999999997, loss=linear, n_estimators=300; total time=  23.2s\n",
      "[CV] END learning_rate=0.46999999999999986, loss=exponential, n_estimators=200; total time=  43.8s\n",
      "[CV] END learning_rate=0.4999999999999998, loss=square, n_estimators=200; total time=  20.0s\n",
      "[CV] END learning_rate=0.6799999999999997, loss=linear, n_estimators=300; total time=   7.4s\n",
      "[CV] END learning_rate=0.6199999999999998, loss=exponential, n_estimators=100; total time=  21.7s\n",
      "[CV] END learning_rate=0.9099999999999996, loss=square, n_estimators=100; total time=  20.4s\n",
      "[CV] END learning_rate=0.4299999999999998, loss=square, n_estimators=100; total time=  22.3s\n",
      "[CV] END learning_rate=0.16999999999999998, loss=square, n_estimators=200; total time=  44.1s\n",
      "[CV] END alpha=0.9599999999999995, criterion=friedman_mse, learning_rate=0.7399999999999997, max_depth=18, max_features=None, max_leaf_nodes=70, min_samples_leaf=3, min_samples_split=20, n_estimators=200; total time=  44.3s\n",
      "[CV] END alpha=0.43999999999999984, criterion=friedman_mse, learning_rate=0.2799999999999999, max_depth=7, max_features=None, max_leaf_nodes=10, min_samples_leaf=2, min_samples_split=6, n_estimators=200; total time=  40.2s\n",
      "[CV] END alpha=0.1, criterion=mse, learning_rate=0.9599999999999995, max_depth=40, max_features=sqrt, max_leaf_nodes=20, min_samples_leaf=7, min_samples_split=4, n_estimators=200; total time=  10.4s\n",
      "[CV] END alpha=0.7799999999999997, criterion=mse, learning_rate=0.29999999999999993, max_depth=20, max_features=log2, max_leaf_nodes=40, min_samples_leaf=3, min_samples_split=20, n_estimators=100; total time=   6.1s\n",
      "[CV] END alpha=0.7799999999999997, criterion=mse, learning_rate=0.29999999999999993, max_depth=20, max_features=log2, max_leaf_nodes=40, min_samples_leaf=3, min_samples_split=20, n_estimators=100; total time=   6.2s\n",
      "[CV] END alpha=0.1, criterion=mse, learning_rate=0.6599999999999997, max_depth=15, max_features=None, max_leaf_nodes=60, min_samples_leaf=1, min_samples_split=20, n_estimators=300; total time= 1.1min\n",
      "[CV] END alpha=0.7099999999999996, criterion=mse, learning_rate=0.6799999999999997, max_depth=14, max_features=log2, max_leaf_nodes=None, min_samples_leaf=5, min_samples_split=10, n_estimators=200; total time=  22.7s\n",
      "[CV] END .................................................... total time=   0.2s\n",
      "[CV] END ..........................alpha=0.19999999999999996; total time=   0.1s\n",
      "[CV] END ...........................alpha=0.8599999999999995; total time=   0.1s\n",
      "[CV] END ...........................alpha=0.7499999999999997; total time=   0.1s\n",
      "[CV] END ..........................alpha=0.46999999999999986; total time=   0.1s\n",
      "[CV] END ...........................alpha=0.7099999999999996; total time=   0.1s\n",
      "[CV] END ...........................alpha=0.5299999999999998; total time=   0.1s\n",
      "[CV] END ...........................alpha=0.5099999999999998; total time=   0.1s\n",
      "[CV] END ...........................alpha=0.8399999999999996; total time=   0.1s\n",
      "[CV] END ..........................alpha=0.22999999999999995; total time=   0.1s\n",
      "[CV] END ...........................alpha=0.7799999999999997; total time=   0.1s\n",
      "[CV] END ..........................alpha=0.19999999999999996; total time=   1.3s\n",
      "[CV] END ...........................alpha=0.7499999999999997; total time=   0.5s\n",
      "[CV] END ...........................alpha=0.5299999999999998; total time=   0.8s\n",
      "[CV] END ..........................alpha=0.29999999999999993; total time=   1.2s\n",
      "[CV] END ..........................alpha=0.22999999999999995; total time=   1.3s\n",
      "[CV] END ..........................alpha=0.48999999999999977; total time=   0.7s\n",
      "[CV] END criterion=friedman_mse, max_depth=10, max_features=auto, max_leaf_nodes=40, min_samples_leaf=2, min_samples_split=15, splitter=best; total time=   0.3s\n",
      "[CV] END criterion=mse, max_depth=38, max_features=None, max_leaf_nodes=50, min_samples_leaf=8, min_samples_split=4, splitter=random; total time=   0.3s\n",
      "[CV] END criterion=friedman_mse, max_depth=3, max_features=auto, max_leaf_nodes=80, min_samples_leaf=5, min_samples_split=4, splitter=best; total time=   0.2s\n",
      "[CV] END criterion=mse, max_depth=7, max_features=sqrt, max_leaf_nodes=30, min_samples_leaf=4, min_samples_split=2, splitter=best; total time=   0.1s\n",
      "[CV] END criterion=mse, max_depth=7, max_features=sqrt, max_leaf_nodes=30, min_samples_leaf=4, min_samples_split=2, splitter=best; total time=   0.1s\n",
      "[CV] END criterion=friedman_mse, max_depth=20, max_features=auto, max_leaf_nodes=90, min_samples_leaf=3, min_samples_split=2, splitter=best; total time=   0.4s\n",
      "[CV] END criterion=mse, max_depth=33, max_features=log2, max_leaf_nodes=60, min_samples_leaf=2, min_samples_split=4, n_estimators=100; total time=   5.2s\n",
      "[CV] END criterion=mse, max_depth=28, max_features=sqrt, max_leaf_nodes=80, min_samples_leaf=4, min_samples_split=2, n_estimators=200; total time=  12.0s\n",
      "[CV] END criterion=mse, max_depth=25, max_features=None, max_leaf_nodes=20, min_samples_leaf=3, min_samples_split=15, n_estimators=200; total time=  32.3s\n",
      "[CV] END criterion=mse, max_depth=30, max_features=sqrt, max_leaf_nodes=30, min_samples_leaf=5, min_samples_split=20, n_estimators=100; total time=   5.5s\n",
      "[CV] END criterion=mse, max_depth=38, max_features=log2, max_leaf_nodes=80, min_samples_leaf=5, min_samples_split=20, n_estimators=300; total time=  16.3s\n",
      "[CV] END criterion=friedman_mse, max_depth=15, max_features=auto, max_leaf_nodes=80, min_samples_leaf=8, min_samples_split=20, n_estimators=200; total time=  37.7s\n",
      "[CV] END learning_rate=0.17999999999999997, loss=linear, n_estimators=300; total time=  24.1s\n",
      "[CV] END learning_rate=0.46999999999999986, loss=exponential, n_estimators=200; total time=  44.4s\n",
      "[CV] END learning_rate=0.4999999999999998, loss=square, n_estimators=200; total time=  42.8s\n",
      "[CV] END learning_rate=0.9899999999999995, loss=linear, n_estimators=100; total time=   4.8s\n",
      "[CV] END learning_rate=0.9099999999999996, loss=square, n_estimators=100; total time=  21.2s\n",
      "[CV] END learning_rate=0.4299999999999998, loss=square, n_estimators=100; total time=  22.3s\n",
      "[CV] END learning_rate=0.16999999999999998, loss=square, n_estimators=200; total time=  44.0s\n",
      "[CV] END alpha=0.3699999999999999, criterion=friedman_mse, learning_rate=0.5499999999999998, max_depth=18, max_features=auto, max_leaf_nodes=30, min_samples_leaf=7, min_samples_split=15, n_estimators=300; total time=  59.9s\n",
      "[CV] END alpha=0.3199999999999999, criterion=friedman_mse, learning_rate=0.6799999999999997, max_depth=12, max_features=log2, max_leaf_nodes=70, min_samples_leaf=2, min_samples_split=4, n_estimators=300; total time=  14.8s\n",
      "[CV] END alpha=0.1, criterion=mse, learning_rate=0.9599999999999995, max_depth=40, max_features=sqrt, max_leaf_nodes=20, min_samples_leaf=7, min_samples_split=4, n_estimators=200; total time=  10.0s\n",
      "[CV] END alpha=0.48999999999999977, criterion=mse, learning_rate=0.8599999999999995, max_depth=7, max_features=log2, max_leaf_nodes=20, min_samples_leaf=3, min_samples_split=4, n_estimators=300; total time=  13.9s\n",
      "[CV] END alpha=0.7799999999999997, criterion=mse, learning_rate=0.29999999999999993, max_depth=20, max_features=log2, max_leaf_nodes=40, min_samples_leaf=3, min_samples_split=20, n_estimators=100; total time=   6.4s\n",
      "[CV] END alpha=0.1, criterion=mse, learning_rate=0.6599999999999997, max_depth=15, max_features=None, max_leaf_nodes=60, min_samples_leaf=1, min_samples_split=20, n_estimators=300; total time= 1.1min\n",
      "[CV] END alpha=0.7099999999999996, criterion=mse, learning_rate=0.6799999999999997, max_depth=14, max_features=log2, max_leaf_nodes=None, min_samples_leaf=5, min_samples_split=10, n_estimators=200; total time=  23.1s\n",
      "[CV] END .................................................... total time=   0.2s\n",
      "[CV] END ..........................alpha=0.19999999999999996; total time=   0.1s\n",
      "[CV] END ...........................alpha=0.8599999999999995; total time=   0.1s\n",
      "[CV] END ...........................alpha=0.7499999999999997; total time=   0.1s\n",
      "[CV] END ..........................alpha=0.46999999999999986; total time=   0.1s\n",
      "[CV] END ...........................alpha=0.7099999999999996; total time=   0.1s\n",
      "[CV] END ...........................alpha=0.5299999999999998; total time=   0.1s\n",
      "[CV] END ...........................alpha=0.5099999999999998; total time=   0.1s\n",
      "[CV] END ...........................alpha=0.8399999999999996; total time=   0.1s\n",
      "[CV] END ..........................alpha=0.22999999999999995; total time=   0.1s\n",
      "[CV] END ..........................alpha=0.48999999999999977; total time=   0.1s\n",
      "[CV] END ..........................alpha=0.19999999999999996; total time=   1.4s\n",
      "[CV] END ..........................alpha=0.46999999999999986; total time=   0.9s\n",
      "[CV] END ...........................alpha=0.5099999999999998; total time=   0.8s\n",
      "[CV] END ...........................alpha=0.8399999999999996; total time=   0.4s\n",
      "[CV] END ...........................alpha=0.9599999999999995; total time=   0.3s\n",
      "[CV] END ..........................alpha=0.22999999999999995; total time=   1.3s\n",
      "[CV] END ..........................alpha=0.48999999999999977; total time=   0.7s\n",
      "[CV] END criterion=friedman_mse, max_depth=7, max_features=sqrt, max_leaf_nodes=20, min_samples_leaf=2, min_samples_split=15, splitter=random; total time=   0.1s\n",
      "[CV] END criterion=friedman_mse, max_depth=5, max_features=sqrt, max_leaf_nodes=20, min_samples_leaf=5, min_samples_split=10, splitter=best; total time=   0.1s\n",
      "[CV] END criterion=friedman_mse, max_depth=5, max_features=sqrt, max_leaf_nodes=20, min_samples_leaf=5, min_samples_split=10, splitter=best; total time=   0.1s\n",
      "[CV] END criterion=friedman_mse, max_depth=5, max_features=sqrt, max_leaf_nodes=20, min_samples_leaf=5, min_samples_split=10, splitter=best; total time=   0.1s\n",
      "[CV] END criterion=friedman_mse, max_depth=40, max_features=auto, max_leaf_nodes=20, min_samples_leaf=5, min_samples_split=20, splitter=best; total time=   0.3s\n",
      "[CV] END criterion=mse, max_depth=28, max_features=auto, max_leaf_nodes=50, min_samples_leaf=6, min_samples_split=6, splitter=random; total time=   0.2s\n",
      "[CV] END criterion=friedman_mse, max_depth=20, max_features=auto, max_leaf_nodes=90, min_samples_leaf=3, min_samples_split=2, splitter=best; total time=   0.4s\n",
      "[CV] END criterion=mse, max_depth=30, max_features=log2, max_leaf_nodes=40, min_samples_leaf=8, min_samples_split=20, n_estimators=100; total time=   5.0s\n",
      "[CV] END criterion=mse, max_depth=28, max_features=sqrt, max_leaf_nodes=80, min_samples_leaf=4, min_samples_split=2, n_estimators=200; total time=  11.7s\n",
      "[CV] END criterion=mse, max_depth=25, max_features=None, max_leaf_nodes=20, min_samples_leaf=3, min_samples_split=15, n_estimators=200; total time=  32.1s\n",
      "[CV] END criterion=friedman_mse, max_depth=14, max_features=None, max_leaf_nodes=70, min_samples_leaf=3, min_samples_split=20, n_estimators=200; total time=  37.8s\n",
      "[CV] END criterion=mse, max_depth=3, max_features=auto, max_leaf_nodes=50, min_samples_leaf=8, min_samples_split=6, n_estimators=100; total time=  11.1s\n",
      "[CV] END criterion=mse, max_depth=7, max_features=auto, max_leaf_nodes=10, min_samples_leaf=10, min_samples_split=2, n_estimators=200; total time=  26.3s\n",
      "[CV] END learning_rate=0.4099999999999998, loss=exponential, n_estimators=300; total time= 1.1min\n",
      "[CV] END learning_rate=0.5099999999999998, loss=exponential, n_estimators=200; total time=  44.4s\n",
      "[CV] END learning_rate=0.9899999999999995, loss=linear, n_estimators=100; total time=   4.9s\n",
      "[CV] END learning_rate=0.30999999999999994, loss=linear, n_estimators=300; total time=  13.1s\n",
      "[CV] END learning_rate=0.8299999999999996, loss=exponential, n_estimators=300; total time= 1.1min\n",
      "[CV] END learning_rate=0.2799999999999999, loss=linear, n_estimators=300; total time=  14.2s\n",
      "[CV] END alpha=0.43999999999999984, criterion=friedman_mse, learning_rate=0.2799999999999999, max_depth=7, max_features=None, max_leaf_nodes=10, min_samples_leaf=2, min_samples_split=6, n_estimators=200; total time=  40.6s\n",
      "[CV] END alpha=0.43999999999999984, criterion=friedman_mse, learning_rate=0.2799999999999999, max_depth=7, max_features=None, max_leaf_nodes=10, min_samples_leaf=2, min_samples_split=6, n_estimators=200; total time=  40.7s\n",
      "[CV] END alpha=0.1, criterion=mse, learning_rate=0.9599999999999995, max_depth=40, max_features=sqrt, max_leaf_nodes=20, min_samples_leaf=7, min_samples_split=4, n_estimators=200; total time=  10.1s\n",
      "[CV] END alpha=0.48999999999999977, criterion=mse, learning_rate=0.8599999999999995, max_depth=7, max_features=log2, max_leaf_nodes=20, min_samples_leaf=3, min_samples_split=4, n_estimators=300; total time=  13.8s\n",
      "[CV] END alpha=0.1, criterion=mse, learning_rate=0.6599999999999997, max_depth=15, max_features=None, max_leaf_nodes=60, min_samples_leaf=1, min_samples_split=20, n_estimators=300; total time= 1.1min\n",
      "[CV] END alpha=0.7099999999999996, criterion=mse, learning_rate=0.6799999999999997, max_depth=14, max_features=log2, max_leaf_nodes=None, min_samples_leaf=5, min_samples_split=10, n_estimators=200; total time=  22.8s\n",
      "[CV] END .................................................... total time=   0.2s\n",
      "[CV] END ..........................alpha=0.19999999999999996; total time=   0.1s\n",
      "[CV] END ...........................alpha=0.8599999999999995; total time=   0.1s\n",
      "[CV] END ...........................alpha=0.7499999999999997; total time=   0.1s\n",
      "[CV] END ..........................alpha=0.46999999999999986; total time=   0.1s\n",
      "[CV] END ...........................alpha=0.7099999999999996; total time=   0.1s\n",
      "[CV] END ...........................alpha=0.5299999999999998; total time=   0.1s\n",
      "[CV] END ..........................alpha=0.29999999999999993; total time=   0.1s\n",
      "[CV] END ...........................alpha=0.9599999999999995; total time=   0.1s\n",
      "[CV] END ...........................alpha=0.3699999999999999; total time=   0.1s\n",
      "[CV] END ..........................alpha=0.48999999999999977; total time=   0.1s\n",
      "[CV] END ..........................alpha=0.19999999999999996; total time=   1.4s\n",
      "[CV] END ..........................alpha=0.46999999999999986; total time=   0.9s\n",
      "[CV] END ...........................alpha=0.5099999999999998; total time=   0.8s\n",
      "[CV] END ...........................alpha=0.8399999999999996; total time=   0.4s\n",
      "[CV] END ..........................alpha=0.22999999999999995; total time=   1.3s\n",
      "[CV] END ..........................alpha=0.48999999999999977; total time=   0.7s\n",
      "[CV] END criterion=friedman_mse, max_depth=7, max_features=sqrt, max_leaf_nodes=20, min_samples_leaf=2, min_samples_split=15, splitter=random; total time=   0.1s\n",
      "[CV] END criterion=friedman_mse, max_depth=12, max_features=sqrt, max_leaf_nodes=None, min_samples_leaf=10, min_samples_split=10, splitter=random; total time=   0.1s\n",
      "[CV] END criterion=friedman_mse, max_depth=5, max_features=sqrt, max_leaf_nodes=20, min_samples_leaf=5, min_samples_split=10, splitter=best; total time=   0.1s\n",
      "[CV] END criterion=friedman_mse, max_depth=15, max_features=sqrt, max_leaf_nodes=20, min_samples_leaf=10, min_samples_split=20, splitter=best; total time=   0.1s\n",
      "[CV] END criterion=mse, max_depth=28, max_features=auto, max_leaf_nodes=50, min_samples_leaf=6, min_samples_split=6, splitter=random; total time=   0.3s\n",
      "[CV] END criterion=mse, max_depth=28, max_features=auto, max_leaf_nodes=50, min_samples_leaf=6, min_samples_split=6, splitter=random; total time=   0.3s\n",
      "[CV] END criterion=friedman_mse, max_depth=7, max_features=None, max_leaf_nodes=50, min_samples_leaf=3, min_samples_split=6, splitter=best; total time=   0.3s\n",
      "[CV] END criterion=mse, max_depth=33, max_features=log2, max_leaf_nodes=60, min_samples_leaf=2, min_samples_split=4, n_estimators=100; total time=   5.7s\n",
      "[CV] END criterion=mse, max_depth=25, max_features=None, max_leaf_nodes=20, min_samples_leaf=3, min_samples_split=15, n_estimators=200; total time=  32.5s\n",
      "[CV] END criterion=mse, max_depth=5, max_features=log2, max_leaf_nodes=50, min_samples_leaf=6, min_samples_split=6, n_estimators=200; total time=   7.3s\n",
      "[CV] END criterion=friedman_mse, max_depth=14, max_features=None, max_leaf_nodes=70, min_samples_leaf=3, min_samples_split=20, n_estimators=200; total time=  39.3s\n",
      "[CV] END criterion=friedman_mse, max_depth=10, max_features=sqrt, max_leaf_nodes=30, min_samples_leaf=8, min_samples_split=20, n_estimators=200; total time=  11.1s\n",
      "[CV] END criterion=mse, max_depth=3, max_features=auto, max_leaf_nodes=50, min_samples_leaf=8, min_samples_split=6, n_estimators=100; total time=  11.4s\n",
      "[CV] END learning_rate=0.17999999999999997, loss=linear, n_estimators=300; total time=  23.0s\n",
      "[CV] END learning_rate=0.46999999999999986, loss=exponential, n_estimators=200; total time=  43.4s\n",
      "[CV] END learning_rate=0.4999999999999998, loss=square, n_estimators=200; total time=  41.4s\n",
      "[CV] END learning_rate=0.6199999999999998, loss=exponential, n_estimators=100; total time=  22.8s\n",
      "[CV] END learning_rate=0.8299999999999996, loss=exponential, n_estimators=300; total time= 1.0min\n",
      "[CV] END alpha=0.9599999999999995, criterion=friedman_mse, learning_rate=0.7399999999999997, max_depth=18, max_features=None, max_leaf_nodes=70, min_samples_leaf=3, min_samples_split=20, n_estimators=200; total time=  47.3s\n",
      "[CV] END alpha=0.43999999999999984, criterion=friedman_mse, learning_rate=0.2799999999999999, max_depth=7, max_features=None, max_leaf_nodes=10, min_samples_leaf=2, min_samples_split=6, n_estimators=200; total time=  41.5s\n",
      "[CV] END alpha=0.48999999999999977, criterion=mse, learning_rate=0.8599999999999995, max_depth=7, max_features=log2, max_leaf_nodes=20, min_samples_leaf=3, min_samples_split=4, n_estimators=300; total time=  13.4s\n",
      "[CV] END alpha=0.7799999999999997, criterion=mse, learning_rate=0.29999999999999993, max_depth=20, max_features=log2, max_leaf_nodes=40, min_samples_leaf=3, min_samples_split=20, n_estimators=100; total time=   6.2s\n",
      "[CV] END alpha=0.4099999999999998, criterion=friedman_mse, learning_rate=0.44999999999999984, max_depth=12, max_features=log2, max_leaf_nodes=90, min_samples_leaf=7, min_samples_split=2, n_estimators=300; total time=  17.4s\n",
      "[CV] END alpha=0.6699999999999997, criterion=friedman_mse, learning_rate=0.17999999999999997, max_depth=20, max_features=auto, max_leaf_nodes=40, min_samples_leaf=6, min_samples_split=20, n_estimators=200; total time=  54.4s\n",
      "[CV] END alpha=0.7099999999999996, criterion=friedman_mse, learning_rate=0.1, max_depth=38, max_features=auto, max_leaf_nodes=20, min_samples_leaf=10, min_samples_split=4, n_estimators=300; total time= 1.1min\n",
      "[CV] END .................................................... total time=   0.2s\n",
      "[CV] END ..........................alpha=0.19999999999999996; total time=   0.1s\n",
      "[CV] END ...........................alpha=0.8599999999999995; total time=   0.1s\n",
      "[CV] END ..........................alpha=0.46999999999999986; total time=   0.1s\n",
      "[CV] END ..........................alpha=0.46999999999999986; total time=   0.1s\n",
      "[CV] END ...........................alpha=0.7099999999999996; total time=   0.1s\n",
      "[CV] END ...........................alpha=0.5299999999999998; total time=   0.1s\n",
      "[CV] END ..........................alpha=0.29999999999999993; total time=   0.1s\n",
      "[CV] END ...........................alpha=0.9599999999999995; total time=   0.1s\n",
      "[CV] END ...........................alpha=0.3699999999999999; total time=   0.1s\n",
      "[CV] END ..........................alpha=0.23999999999999994; total time=   1.3s\n",
      "[CV] END ...........................alpha=0.8599999999999995; total time=   0.4s\n",
      "[CV] END ...........................alpha=0.7099999999999996; total time=   0.5s\n",
      "[CV] END ...........................alpha=0.5299999999999998; total time=   0.8s\n",
      "[CV] END ..........................alpha=0.29999999999999993; total time=   1.2s\n",
      "[CV] END ...........................alpha=0.7799999999999997; total time=   0.5s\n",
      "[CV] END ...........................alpha=0.7799999999999997; total time=   0.5s\n",
      "[CV] END criterion=friedman_mse, max_depth=5, max_features=sqrt, max_leaf_nodes=40, min_samples_leaf=3, min_samples_split=10, splitter=random; total time=   0.1s\n",
      "[CV] END criterion=friedman_mse, max_depth=12, max_features=sqrt, max_leaf_nodes=None, min_samples_leaf=10, min_samples_split=10, splitter=random; total time=   0.2s\n",
      "[CV] END criterion=friedman_mse, max_depth=15, max_features=sqrt, max_leaf_nodes=20, min_samples_leaf=10, min_samples_split=20, splitter=best; total time=   0.1s\n",
      "[CV] END criterion=friedman_mse, max_depth=15, max_features=sqrt, max_leaf_nodes=20, min_samples_leaf=10, min_samples_split=20, splitter=best; total time=   0.1s\n",
      "[CV] END criterion=mse, max_depth=28, max_features=auto, max_leaf_nodes=50, min_samples_leaf=6, min_samples_split=6, splitter=random; total time=   0.3s\n",
      "[CV] END criterion=mse, max_depth=28, max_features=auto, max_leaf_nodes=50, min_samples_leaf=6, min_samples_split=6, splitter=random; total time=   0.3s\n",
      "[CV] END criterion=friedman_mse, max_depth=7, max_features=None, max_leaf_nodes=50, min_samples_leaf=3, min_samples_split=6, splitter=best; total time=   0.3s\n",
      "[CV] END criterion=mse, max_depth=28, max_features=sqrt, max_leaf_nodes=80, min_samples_leaf=4, min_samples_split=2, n_estimators=200; total time=  11.7s\n",
      "[CV] END criterion=mse, max_depth=25, max_features=None, max_leaf_nodes=20, min_samples_leaf=3, min_samples_split=15, n_estimators=200; total time=  31.3s\n",
      "[CV] END criterion=friedman_mse, max_depth=14, max_features=None, max_leaf_nodes=70, min_samples_leaf=3, min_samples_split=20, n_estimators=200; total time=  37.8s\n",
      "[CV] END criterion=friedman_mse, max_depth=10, max_features=sqrt, max_leaf_nodes=30, min_samples_leaf=8, min_samples_split=20, n_estimators=200; total time=  10.2s\n",
      "[CV] END criterion=mse, max_depth=3, max_features=auto, max_leaf_nodes=50, min_samples_leaf=8, min_samples_split=6, n_estimators=100; total time=  11.0s\n",
      "[CV] END criterion=mse, max_depth=7, max_features=auto, max_leaf_nodes=10, min_samples_leaf=10, min_samples_split=2, n_estimators=200; total time=  23.7s\n",
      "[CV] END learning_rate=0.4099999999999998, loss=exponential, n_estimators=300; total time= 1.1min\n",
      "[CV] END learning_rate=0.5099999999999998, loss=exponential, n_estimators=200; total time=  44.8s\n",
      "[CV] END learning_rate=0.9899999999999995, loss=linear, n_estimators=100; total time=   4.5s\n",
      "[CV] END learning_rate=0.30999999999999994, loss=linear, n_estimators=300; total time=  14.2s\n",
      "[CV] END learning_rate=0.8299999999999996, loss=exponential, n_estimators=300; total time= 1.1min\n",
      "[CV] END alpha=0.9599999999999995, criterion=friedman_mse, learning_rate=0.7399999999999997, max_depth=18, max_features=None, max_leaf_nodes=70, min_samples_leaf=3, min_samples_split=20, n_estimators=200; total time=  47.4s\n",
      "[CV] END alpha=0.3199999999999999, criterion=friedman_mse, learning_rate=0.6799999999999997, max_depth=12, max_features=log2, max_leaf_nodes=70, min_samples_leaf=2, min_samples_split=4, n_estimators=300; total time=  14.5s\n",
      "[CV] END alpha=0.44999999999999984, criterion=friedman_mse, learning_rate=0.8699999999999996, max_depth=10, max_features=auto, max_leaf_nodes=50, min_samples_leaf=8, min_samples_split=15, n_estimators=200; total time=  47.0s\n",
      "[CV] END alpha=0.4099999999999998, criterion=friedman_mse, learning_rate=0.44999999999999984, max_depth=12, max_features=log2, max_leaf_nodes=90, min_samples_leaf=7, min_samples_split=2, n_estimators=300; total time=  17.1s\n",
      "[CV] END alpha=0.6699999999999997, criterion=friedman_mse, learning_rate=0.17999999999999997, max_depth=20, max_features=auto, max_leaf_nodes=40, min_samples_leaf=6, min_samples_split=20, n_estimators=200; total time=  55.0s\n",
      "[CV] END alpha=0.7099999999999996, criterion=friedman_mse, learning_rate=0.1, max_depth=38, max_features=auto, max_leaf_nodes=20, min_samples_leaf=10, min_samples_split=4, n_estimators=300; total time= 1.1min\n",
      "[CV] END ..........................alpha=0.23999999999999994; total time=   0.1s\n",
      "[CV] END ...........................alpha=0.8399999999999996; total time=   0.1s\n",
      "[CV] END ..........................alpha=0.22999999999999995; total time=   0.1s\n",
      "[CV] END ...........................alpha=0.7799999999999997; total time=   0.1s\n",
      "[CV] END ..........................alpha=0.23999999999999994; total time=   1.4s\n",
      "[CV] END ..........................alpha=0.46999999999999986; total time=   0.9s\n",
      "[CV] END ...........................alpha=0.5099999999999998; total time=   0.8s\n",
      "[CV] END ...........................alpha=0.8399999999999996; total time=   0.4s\n",
      "[CV] END ..........................alpha=0.22999999999999995; total time=   1.3s\n",
      "[CV] END ..........................alpha=0.48999999999999977; total time=   0.8s\n",
      "[CV] END criterion=friedman_mse, max_depth=7, max_features=sqrt, max_leaf_nodes=20, min_samples_leaf=2, min_samples_split=15, splitter=random; total time=   0.1s\n",
      "[CV] END criterion=friedman_mse, max_depth=12, max_features=sqrt, max_leaf_nodes=None, min_samples_leaf=10, min_samples_split=10, splitter=random; total time=   0.1s\n",
      "[CV] END criterion=friedman_mse, max_depth=5, max_features=sqrt, max_leaf_nodes=20, min_samples_leaf=5, min_samples_split=10, splitter=best; total time=   0.1s\n",
      "[CV] END criterion=mse, max_depth=38, max_features=None, max_leaf_nodes=50, min_samples_leaf=8, min_samples_split=4, splitter=random; total time=   0.3s\n",
      "[CV] END criterion=mse, max_depth=38, max_features=None, max_leaf_nodes=50, min_samples_leaf=8, min_samples_split=4, splitter=random; total time=   0.2s\n",
      "[CV] END criterion=friedman_mse, max_depth=28, max_features=sqrt, max_leaf_nodes=70, min_samples_leaf=7, min_samples_split=2, splitter=random; total time=   0.1s\n",
      "[CV] END criterion=friedman_mse, max_depth=28, max_features=sqrt, max_leaf_nodes=70, min_samples_leaf=7, min_samples_split=2, splitter=random; total time=   0.1s\n",
      "[CV] END criterion=friedman_mse, max_depth=7, max_features=None, max_leaf_nodes=50, min_samples_leaf=3, min_samples_split=6, splitter=best; total time=   0.3s\n",
      "[CV] END criterion=mse, max_depth=33, max_features=log2, max_leaf_nodes=60, min_samples_leaf=2, min_samples_split=4, n_estimators=100; total time=   5.2s\n",
      "[CV] END criterion=mse, max_depth=20, max_features=log2, max_leaf_nodes=None, min_samples_leaf=5, min_samples_split=15, n_estimators=300; total time=  19.9s\n",
      "[CV] END criterion=mse, max_depth=15, max_features=sqrt, max_leaf_nodes=None, min_samples_leaf=9, min_samples_split=10, n_estimators=200; total time=  14.4s\n",
      "[CV] END criterion=friedman_mse, max_depth=18, max_features=sqrt, max_leaf_nodes=40, min_samples_leaf=10, min_samples_split=8, n_estimators=200; total time=  12.5s\n",
      "[CV] END criterion=mse, max_depth=30, max_features=sqrt, max_leaf_nodes=30, min_samples_leaf=5, min_samples_split=20, n_estimators=100; total time=   6.1s\n",
      "[CV] END criterion=mse, max_depth=38, max_features=log2, max_leaf_nodes=80, min_samples_leaf=5, min_samples_split=20, n_estimators=300; total time=  17.9s\n",
      "[CV] END criterion=friedman_mse, max_depth=10, max_features=sqrt, max_leaf_nodes=30, min_samples_leaf=8, min_samples_split=20, n_estimators=200; total time=  11.0s\n",
      "[CV] END criterion=mse, max_depth=3, max_features=auto, max_leaf_nodes=50, min_samples_leaf=8, min_samples_split=6, n_estimators=100; total time=  12.3s\n",
      "[CV] END criterion=mse, max_depth=7, max_features=auto, max_leaf_nodes=10, min_samples_leaf=10, min_samples_split=2, n_estimators=200; total time=  28.9s\n",
      "[CV] END learning_rate=0.4099999999999998, loss=exponential, n_estimators=300; total time= 1.3min\n",
      "[CV] END learning_rate=0.6799999999999997, loss=linear, n_estimators=300; total time=  10.7s\n",
      "[CV] END learning_rate=0.6799999999999997, loss=linear, n_estimators=300; total time=   7.4s\n",
      "[CV] END learning_rate=0.6199999999999998, loss=exponential, n_estimators=100; total time=  26.0s\n",
      "[CV] END learning_rate=0.9099999999999996, loss=square, n_estimators=100; total time=  24.4s\n",
      "[CV] END learning_rate=0.16999999999999998, loss=square, n_estimators=200; total time=  51.8s\n",
      "[CV] END alpha=0.9599999999999995, criterion=friedman_mse, learning_rate=0.7399999999999997, max_depth=18, max_features=None, max_leaf_nodes=70, min_samples_leaf=3, min_samples_split=20, n_estimators=200; total time=  48.9s\n",
      "[CV] END alpha=0.3199999999999999, criterion=friedman_mse, learning_rate=0.6799999999999997, max_depth=12, max_features=log2, max_leaf_nodes=70, min_samples_leaf=2, min_samples_split=4, n_estimators=300; total time=  15.5s\n",
      "[CV] END alpha=0.44999999999999984, criterion=friedman_mse, learning_rate=0.8699999999999996, max_depth=10, max_features=auto, max_leaf_nodes=50, min_samples_leaf=8, min_samples_split=15, n_estimators=200; total time=  48.0s\n",
      "[CV] END alpha=0.4099999999999998, criterion=friedman_mse, learning_rate=0.44999999999999984, max_depth=12, max_features=log2, max_leaf_nodes=90, min_samples_leaf=7, min_samples_split=2, n_estimators=300; total time=  16.2s\n",
      "[CV] END alpha=0.34999999999999987, criterion=mse, learning_rate=0.7399999999999997, max_depth=3, max_features=log2, max_leaf_nodes=80, min_samples_leaf=5, min_samples_split=15, n_estimators=100; total time=   3.8s\n",
      "[CV] END alpha=0.34999999999999987, criterion=mse, learning_rate=0.7399999999999997, max_depth=3, max_features=log2, max_leaf_nodes=80, min_samples_leaf=5, min_samples_split=15, n_estimators=100; total time=   3.9s\n",
      "[CV] END alpha=0.34999999999999987, criterion=mse, learning_rate=0.7399999999999997, max_depth=3, max_features=log2, max_leaf_nodes=80, min_samples_leaf=5, min_samples_split=15, n_estimators=100; total time=   3.9s\n",
      "[CV] END alpha=0.34999999999999987, criterion=mse, learning_rate=0.7399999999999997, max_depth=3, max_features=log2, max_leaf_nodes=80, min_samples_leaf=5, min_samples_split=15, n_estimators=100; total time=   3.8s\n",
      "[CV] END alpha=0.34999999999999987, criterion=mse, learning_rate=0.7399999999999997, max_depth=3, max_features=log2, max_leaf_nodes=80, min_samples_leaf=5, min_samples_split=15, n_estimators=100; total time=   3.7s\n",
      "[CV] END alpha=0.7099999999999996, criterion=mse, learning_rate=0.6799999999999997, max_depth=14, max_features=log2, max_leaf_nodes=None, min_samples_leaf=5, min_samples_split=10, n_estimators=200; total time=  25.0s\n",
      "[CV] END alpha=0.11, criterion=friedman_mse, learning_rate=0.8499999999999996, max_depth=9, max_features=sqrt, max_leaf_nodes=50, min_samples_leaf=7, min_samples_split=6, n_estimators=100; total time=   7.1s\n",
      "[CV] END alpha=0.7099999999999996, criterion=friedman_mse, learning_rate=0.1, max_depth=38, max_features=auto, max_leaf_nodes=20, min_samples_leaf=10, min_samples_split=4, n_estimators=300; total time= 1.1min\n",
      "[CV] END ..........................alpha=0.23999999999999994; total time=   0.1s\n",
      "[CV] END ...........................alpha=0.5099999999999998; total time=   0.1s\n",
      "[CV] END ...........................alpha=0.9599999999999995; total time=   0.1s\n",
      "[CV] END ...........................alpha=0.3699999999999999; total time=   0.1s\n",
      "[CV] END ..........................alpha=0.48999999999999977; total time=   0.1s\n",
      "[CV] END ..........................alpha=0.19999999999999996; total time=   1.4s\n",
      "[CV] END ..........................alpha=0.46999999999999986; total time=   0.9s\n",
      "[CV] END ...........................alpha=0.5099999999999998; total time=   0.8s\n",
      "[CV] END ...........................alpha=0.9599999999999995; total time=   0.3s\n",
      "[CV] END ...........................alpha=0.9599999999999995; total time=   0.3s\n",
      "[CV] END ...........................alpha=0.3699999999999999; total time=   1.0s\n",
      "[CV] END ...........................alpha=0.7799999999999997; total time=   0.4s\n",
      "[CV] END criterion=friedman_mse, max_depth=5, max_features=sqrt, max_leaf_nodes=40, min_samples_leaf=3, min_samples_split=10, splitter=random; total time=   0.1s\n",
      "[CV] END criterion=friedman_mse, max_depth=10, max_features=auto, max_leaf_nodes=40, min_samples_leaf=2, min_samples_split=15, splitter=best; total time=   0.3s\n",
      "[CV] END criterion=friedman_mse, max_depth=40, max_features=auto, max_leaf_nodes=20, min_samples_leaf=5, min_samples_split=20, splitter=best; total time=   0.3s\n",
      "[CV] END criterion=friedman_mse, max_depth=40, max_features=auto, max_leaf_nodes=20, min_samples_leaf=5, min_samples_split=20, splitter=best; total time=   0.3s\n",
      "[CV] END criterion=friedman_mse, max_depth=20, max_features=auto, max_leaf_nodes=90, min_samples_leaf=3, min_samples_split=2, splitter=best; total time=   0.3s\n",
      "[CV] END criterion=mse, max_depth=30, max_features=log2, max_leaf_nodes=40, min_samples_leaf=8, min_samples_split=20, n_estimators=100; total time=   5.3s\n",
      "[CV] END criterion=mse, max_depth=20, max_features=log2, max_leaf_nodes=None, min_samples_leaf=5, min_samples_split=15, n_estimators=300; total time=  20.3s\n",
      "[CV] END criterion=mse, max_depth=15, max_features=sqrt, max_leaf_nodes=None, min_samples_leaf=9, min_samples_split=10, n_estimators=200; total time=  14.7s\n",
      "[CV] END criterion=friedman_mse, max_depth=18, max_features=sqrt, max_leaf_nodes=40, min_samples_leaf=10, min_samples_split=8, n_estimators=200; total time=  12.8s\n",
      "[CV] END criterion=mse, max_depth=38, max_features=log2, max_leaf_nodes=80, min_samples_leaf=5, min_samples_split=20, n_estimators=300; total time=  18.8s\n",
      "[CV] END criterion=friedman_mse, max_depth=15, max_features=auto, max_leaf_nodes=80, min_samples_leaf=8, min_samples_split=20, n_estimators=200; total time=  43.8s\n",
      "[CV] END learning_rate=0.17999999999999997, loss=linear, n_estimators=300; total time=  25.3s\n",
      "[CV] END learning_rate=0.5099999999999998, loss=exponential, n_estimators=200; total time=  51.1s\n",
      "[CV] END learning_rate=0.6799999999999997, loss=linear, n_estimators=300; total time=   8.6s\n",
      "[CV] END learning_rate=0.6799999999999997, loss=linear, n_estimators=300; total time=   7.2s\n",
      "[CV] END learning_rate=0.6199999999999998, loss=exponential, n_estimators=100; total time=  26.1s\n",
      "[CV] END learning_rate=0.9099999999999996, loss=square, n_estimators=100; total time=  24.4s\n",
      "[CV] END learning_rate=0.4299999999999998, loss=square, n_estimators=100; total time=  24.3s\n",
      "[CV] END learning_rate=0.6799999999999997, loss=linear, n_estimators=200; total time=   6.5s\n",
      "[CV] END learning_rate=0.6799999999999997, loss=linear, n_estimators=200; total time=   8.1s\n",
      "[CV] END learning_rate=0.6799999999999997, loss=linear, n_estimators=200; total time=   6.5s\n",
      "[CV] END learning_rate=0.2799999999999999, loss=linear, n_estimators=300; total time=  16.3s\n",
      "[CV] END alpha=0.3699999999999999, criterion=friedman_mse, learning_rate=0.5499999999999998, max_depth=18, max_features=auto, max_leaf_nodes=30, min_samples_leaf=7, min_samples_split=15, n_estimators=300; total time= 1.0min\n",
      "[CV] END alpha=0.3199999999999999, criterion=friedman_mse, learning_rate=0.6799999999999997, max_depth=12, max_features=log2, max_leaf_nodes=70, min_samples_leaf=2, min_samples_split=4, n_estimators=300; total time=  14.9s\n",
      "[CV] END alpha=0.1, criterion=mse, learning_rate=0.9599999999999995, max_depth=40, max_features=sqrt, max_leaf_nodes=20, min_samples_leaf=7, min_samples_split=4, n_estimators=200; total time=   9.7s\n",
      "[CV] END alpha=0.48999999999999977, criterion=mse, learning_rate=0.8599999999999995, max_depth=7, max_features=log2, max_leaf_nodes=20, min_samples_leaf=3, min_samples_split=4, n_estimators=300; total time=  13.9s\n",
      "[CV] END alpha=0.7799999999999997, criterion=mse, learning_rate=0.29999999999999993, max_depth=20, max_features=log2, max_leaf_nodes=40, min_samples_leaf=3, min_samples_split=20, n_estimators=100; total time=   6.5s\n",
      "[CV] END alpha=0.1, criterion=mse, learning_rate=0.6599999999999997, max_depth=15, max_features=None, max_leaf_nodes=60, min_samples_leaf=1, min_samples_split=20, n_estimators=300; total time= 1.1min\n",
      "[CV] END alpha=0.11, criterion=friedman_mse, learning_rate=0.8499999999999996, max_depth=9, max_features=sqrt, max_leaf_nodes=50, min_samples_leaf=7, min_samples_split=6, n_estimators=100; total time=   7.4s\n",
      "[CV] END alpha=0.7099999999999996, criterion=friedman_mse, learning_rate=0.1, max_depth=38, max_features=auto, max_leaf_nodes=20, min_samples_leaf=10, min_samples_split=4, n_estimators=300; total time= 1.1min\n",
      "[CV] END ..........................alpha=0.23999999999999994; total time=   0.1s\n",
      "[CV] END ..........................alpha=0.29999999999999993; total time=   0.1s\n",
      "[CV] END ...........................alpha=0.9599999999999995; total time=   0.1s\n",
      "[CV] END ...........................alpha=0.7799999999999997; total time=   0.1s\n",
      "[CV] END ..........................alpha=0.23999999999999994; total time=   1.3s\n",
      "[CV] END ...........................alpha=0.7499999999999997; total time=   0.4s\n",
      "[CV] END ...........................alpha=0.7099999999999996; total time=   0.4s\n",
      "[CV] END ...........................alpha=0.5299999999999998; total time=   0.7s\n",
      "[CV] END ..........................alpha=0.29999999999999993; total time=   1.1s\n",
      "[CV] END ...........................alpha=0.3699999999999999; total time=   1.1s\n",
      "[CV] END criterion=friedman_mse, max_depth=5, max_features=sqrt, max_leaf_nodes=40, min_samples_leaf=3, min_samples_split=10, splitter=random; total time=   0.1s\n",
      "[CV] END criterion=friedman_mse, max_depth=10, max_features=auto, max_leaf_nodes=40, min_samples_leaf=2, min_samples_split=15, splitter=best; total time=   0.3s\n",
      "[CV] END criterion=friedman_mse, max_depth=3, max_features=auto, max_leaf_nodes=80, min_samples_leaf=5, min_samples_split=4, splitter=best; total time=   0.2s\n",
      "[CV] END criterion=friedman_mse, max_depth=3, max_features=auto, max_leaf_nodes=80, min_samples_leaf=5, min_samples_split=4, splitter=best; total time=   0.2s\n",
      "[CV] END criterion=mse, max_depth=30, max_features=log2, max_leaf_nodes=10, min_samples_leaf=3, min_samples_split=20, splitter=random; total time=   0.1s\n",
      "[CV] END criterion=mse, max_depth=7, max_features=sqrt, max_leaf_nodes=30, min_samples_leaf=4, min_samples_split=2, splitter=best; total time=   0.1s\n",
      "[CV] END criterion=friedman_mse, max_depth=20, max_features=auto, max_leaf_nodes=90, min_samples_leaf=3, min_samples_split=2, splitter=best; total time=   0.3s\n",
      "[CV] END criterion=mse, max_depth=30, max_features=log2, max_leaf_nodes=40, min_samples_leaf=8, min_samples_split=20, n_estimators=100; total time=   5.4s\n",
      "[CV] END criterion=mse, max_depth=20, max_features=log2, max_leaf_nodes=None, min_samples_leaf=5, min_samples_split=15, n_estimators=300; total time=  20.8s\n",
      "[CV] END criterion=mse, max_depth=5, max_features=log2, max_leaf_nodes=50, min_samples_leaf=6, min_samples_split=6, n_estimators=200; total time=   7.5s\n",
      "[CV] END criterion=mse, max_depth=5, max_features=log2, max_leaf_nodes=50, min_samples_leaf=6, min_samples_split=6, n_estimators=200; total time=   7.1s\n",
      "[CV] END criterion=friedman_mse, max_depth=18, max_features=sqrt, max_leaf_nodes=40, min_samples_leaf=10, min_samples_split=8, n_estimators=200; total time=  11.6s\n",
      "[CV] END criterion=mse, max_depth=30, max_features=sqrt, max_leaf_nodes=30, min_samples_leaf=5, min_samples_split=20, n_estimators=100; total time=   5.5s\n",
      "[CV] END criterion=mse, max_depth=38, max_features=log2, max_leaf_nodes=80, min_samples_leaf=5, min_samples_split=20, n_estimators=300; total time=  16.2s\n",
      "[CV] END criterion=friedman_mse, max_depth=15, max_features=auto, max_leaf_nodes=80, min_samples_leaf=8, min_samples_split=20, n_estimators=200; total time=  39.0s\n",
      "[CV] END learning_rate=0.17999999999999997, loss=linear, n_estimators=300; total time=  24.7s\n",
      "[CV] END learning_rate=0.46999999999999986, loss=exponential, n_estimators=200; total time=  42.6s\n",
      "[CV] END learning_rate=0.4999999999999998, loss=square, n_estimators=200; total time=  41.1s\n",
      "[CV] END learning_rate=0.9899999999999995, loss=linear, n_estimators=100; total time=   5.0s\n",
      "[CV] END learning_rate=0.30999999999999994, loss=linear, n_estimators=300; total time=  11.9s\n",
      "[CV] END learning_rate=0.8299999999999996, loss=exponential, n_estimators=300; total time=  59.9s\n",
      "[CV] END learning_rate=0.6799999999999997, loss=linear, n_estimators=200; total time=   7.4s\n",
      "[CV] END learning_rate=0.2799999999999999, loss=linear, n_estimators=300; total time=  12.8s\n",
      "[CV] END alpha=0.3699999999999999, criterion=friedman_mse, learning_rate=0.5499999999999998, max_depth=18, max_features=auto, max_leaf_nodes=30, min_samples_leaf=7, min_samples_split=15, n_estimators=300; total time= 1.0min\n",
      "[CV] END alpha=0.3199999999999999, criterion=friedman_mse, learning_rate=0.6799999999999997, max_depth=12, max_features=log2, max_leaf_nodes=70, min_samples_leaf=2, min_samples_split=4, n_estimators=300; total time=  13.7s\n",
      "[CV] END alpha=0.44999999999999984, criterion=friedman_mse, learning_rate=0.8699999999999996, max_depth=10, max_features=auto, max_leaf_nodes=50, min_samples_leaf=8, min_samples_split=15, n_estimators=200; total time=  47.6s\n",
      "[CV] END alpha=0.6699999999999997, criterion=friedman_mse, learning_rate=0.17999999999999997, max_depth=20, max_features=auto, max_leaf_nodes=40, min_samples_leaf=6, min_samples_split=20, n_estimators=200; total time=  52.8s\n",
      "[CV] END alpha=0.11, criterion=friedman_mse, learning_rate=0.8499999999999996, max_depth=9, max_features=sqrt, max_leaf_nodes=50, min_samples_leaf=7, min_samples_split=6, n_estimators=100; total time=   6.9s\n",
      "[CV] END alpha=0.7099999999999996, criterion=friedman_mse, learning_rate=0.1, max_depth=38, max_features=auto, max_leaf_nodes=20, min_samples_leaf=10, min_samples_split=4, n_estimators=300; total time= 1.1min\n"
     ]
    }
   ],
   "source": [
    "# let's save gradient boost model and scaler in the form of pickle file\n",
    "# encoded_team and feature columns as JSON file for prediction purpose which i use in web app \n",
    "\n",
    "import pickle\n",
    "import json\n",
    "\n",
    "with open(\"artifacts/model.pickle\", \"wb\") as f:\n",
    "    pickle.dump(best_model[\"Random Forest\"], f)  \n",
    "    \n",
    "with open(\"artifacts/scaler.pickle\", \"wb\") as f:\n",
    "    pickle.dump(scaler, f)\n",
    "    \n",
    "with open(\"artifacts/encodedteams.json\", \"w\") as f:\n",
    "    json.dump(encoded_teams, f)\n",
    "    \n",
    "with open(\"artifacts/columns.json\", \"w\") as f:\n",
    "    json.dump({\"columns\": list(X.columns)}, f)"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.5"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}