New_XGB.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import gc\n",
    "import time\n",
    "import numpy as np\n",
    "import pandas as pd\n",
    "from time import time\n",
    "\n",
    "from sklearn.model_selection import train_test_split\n",
    "from sklearn.model_selection import KFold\n",
    "from sklearn.preprocessing import LabelEncoder\n",
    "\n",
    "from bayes_opt import BayesianOptimization\n",
    "\n",
    "import xgboost as xgb\n",
    "from xgboost import plot_importance\n",
    "\n",
    "import matplotlib.pyplot as plt\n",
    "\n",
    "pd.set_option('display.max_rows', 500)\n",
    "pd.set_option('display.max_columns', 500)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "PATH = \"/home/vishy/Desktop/Myfiles/AV/AMEX/\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Change this for validation with 10% from train\n",
    "is_valid = False"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "def handle_missing_inplace(df):\n",
    "    df['age_level'].fillna(value=99.0, inplace=True)\n",
    "    df['city_development_index'].fillna(value=99.0, inplace=True)\n",
    "    df['gender'].fillna(value='Unknown', inplace=True)\n",
    "    df['product_category_2'].fillna(value=999999.0, inplace=True)\n",
    "    df['user_depth'].fillna(value=99.0, inplace=True)\n",
    "    df['user_group_id'].fillna(value=99.0, inplace=True)\n",
    "    return df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "def timeFeatures(df):\n",
    "    # Make some new features with click_time column\n",
    "    df['day'] = df['DateTime'].dt.day.astype('uint8')\n",
    "    df['hour'] = df['DateTime'].dt.hour.astype('uint8')\n",
    "    df['minute'] = df['DateTime'].dt.minute.astype('uint8')\n",
    "    df['dow'] = df['DateTime'].dt.dayofweek.astype('uint8')\n",
    "    df.drop(['DateTime'], axis=1, inplace=True)\n",
    "    return df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(463291, 15)\n"
     ]
    }
   ],
   "source": [
    "train = pd.read_csv(PATH+'train.csv', low_memory=False, parse_dates=['DateTime'])\n",
    "train.rename(columns={'product':'prod'}, inplace=True)\n",
    "print(train.shape)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(128858, 14)\n"
     ]
    }
   ],
   "source": [
    "test = pd.read_csv(PATH+'test.csv', low_memory=False, parse_dates=['DateTime'])\n",
    "test.rename(columns={'product':'prod'}, inplace=True)\n",
    "print(test.shape)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "train = handle_missing_inplace(train)\n",
    "train = timeFeatures(train)\n",
    "\n",
    "test = handle_missing_inplace(test)\n",
    "test = timeFeatures(test)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "CLICK_ATTR_CATS = [['prod', 'campaign_id'],['prod', 'webpage_id'], ['prod', 'product_category_1'],\n",
    "                   ['user_group_id','gender'],['user_group_id','age_level'],['user_group_id', 'user_depth'],\n",
    "                   ['prod','age_level'], ['prod','user_depth'],['product_category_1','age_level'],\n",
    "                   ['product_category_1','user_depth']]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      ">> Calculating confidence-weighted rate for: ['prod', 'campaign_id'].\n",
      "   Saving to: prod_campaign_id_confRate. Group Max /Mean / Median / Min: 64962 / 5939.63 / 2162.0 / 1\n",
      ">> Calculating confidence-weighted rate for: ['prod', 'webpage_id'].\n",
      "   Saving to: prod_webpage_id_confRate. Group Max /Mean / Median / Min: 81829 / 6346.45 / 1871.0 / 1\n",
      ">> Calculating confidence-weighted rate for: ['prod', 'product_category_1'].\n",
      "   Saving to: prod_product_category_1_confRate. Group Max /Mean / Median / Min: 70930 / 10774.21 / 4588.0 / 1\n",
      ">> Calculating confidence-weighted rate for: ['user_group_id', 'gender'].\n",
      "   Saving to: user_group_id_gender_confRate. Group Max /Mean / Median / Min: 140317 / 30886.07 / 13779.0 / 33\n",
      ">> Calculating confidence-weighted rate for: ['user_group_id', 'age_level'].\n",
      "   Saving to: user_group_id_age_level_confRate. Group Max /Mean / Median / Min: 140317 / 33092.21 / 16011.0 / 153\n",
      ">> Calculating confidence-weighted rate for: ['user_group_id', 'user_depth'].\n",
      "   Saving to: user_group_id_user_depth_confRate. Group Max /Mean / Median / Min: 129024 / 11582.28 / 1268.0 / 1\n",
      ">> Calculating confidence-weighted rate for: ['prod', 'age_level'].\n",
      "   Saving to: prod_age_level_confRate. Group Max /Mean / Median / Min: 58839 / 5791.14 / 1902.5 / 1\n",
      ">> Calculating confidence-weighted rate for: ['prod', 'user_depth'].\n",
      "   Saving to: prod_user_depth_confRate. Group Max /Mean / Median / Min: 142927 / 11582.28 / 1706.5 / 174\n",
      ">> Calculating confidence-weighted rate for: ['product_category_1', 'age_level'].\n",
      "   Saving to: product_category_1_age_level_confRate. Group Max /Mean / Median / Min: 54536 / 11582.28 / 5530.0 / 4\n",
      ">> Calculating confidence-weighted rate for: ['product_category_1', 'user_depth'].\n",
      "   Saving to: product_category_1_user_depth_confRate. Group Max /Mean / Median / Min: 117589 / 23164.55 / 4762.0 / 1763\n",
      "(463291, 28) (128858, 27)\n"
     ]
    }
   ],
   "source": [
    "freqs = {}\n",
    "for cols in CLICK_ATTR_CATS:\n",
    "    \n",
    "    # New feature name\n",
    "    new_feature = '_'.join(cols)+'_confRate'    \n",
    "    \n",
    "    # Perform the groupby\n",
    "    group_object = train.groupby(cols)\n",
    "    \n",
    "    # Group sizes    \n",
    "    group_sizes = group_object.size()\n",
    "    log_group = np.log(100000) # 1000 views -> 60% confidence, 100 views -> 40% confidence \n",
    "    print(\">> Calculating confidence-weighted rate for: {}.\\n   Saving to: {}. Group Max /Mean / Median / Min: {} / {} / {} / {}\".format(\n",
    "        cols, new_feature,group_sizes.max(), np.round(group_sizes.mean(), 2), np.round(group_sizes.median(), 2),\n",
    "        group_sizes.min()))\n",
    "    \n",
    "    # Aggregation function\n",
    "    def rate_calculation(x):\n",
    "        \"\"\"Calculate the click rate. Scale by confidence\"\"\"\n",
    "        rate = x.sum() / float(x.count())\n",
    "        conf = np.min([1, np.log(x.count()) / log_group])\n",
    "        return rate * conf\n",
    "    \n",
    "    # Perform the merge\n",
    "    train = train.merge(\n",
    "        group_object['is_click']. \\\n",
    "            apply(rate_calculation). \\\n",
    "            reset_index(). \\\n",
    "            rename( \n",
    "                index=str,\n",
    "                columns={'is_click': new_feature}\n",
    "            )[cols + [new_feature]],\n",
    "        on=cols, how='left'\n",
    "    )\n",
    "    test = test.merge(\n",
    "        group_object['is_click']. \\\n",
    "            apply(rate_calculation). \\\n",
    "            reset_index(). \\\n",
    "            rename( \n",
    "                index=str,\n",
    "                columns={'is_click': new_feature}\n",
    "            )[cols + [new_feature]],\n",
    "        on=cols, how='left'\n",
    "    )\n",
    "    \n",
    "print(train.shape, test.shape)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(592149, 26)\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "0"
      ]
     },
     "execution_count": 11,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Isolate target\n",
    "y = train['is_click']\n",
    "# Drop is_click and session ID from train\n",
    "train.drop(['is_click','session_id'], axis=1, inplace=True)\n",
    "\n",
    "# Create Submission dataframe\n",
    "sub = pd.DataFrame()\n",
    "sub['session_id'] = test['session_id'].astype('int')\n",
    "\n",
    "# Drop sessionID from test rows\n",
    "test.drop(['session_id'], axis=1, inplace=True)\n",
    "gc.collect()\n",
    "\n",
    "# Create a pointer for train \n",
    "nrow_train = train.shape[0]\n",
    "\n",
    "# Concatenate for counting\n",
    "merge = pd.concat([train, test])\n",
    "print(merge.shape)\n",
    "\n",
    "del train, test\n",
    "gc.collect()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(592149, 30)\n"
     ]
    }
   ],
   "source": [
    "HISTORY_ADS = {\n",
    "    'identical_': ['user_id', 'prod', 'product_category_1', 'webpage_id', 'campaign_id'],\n",
    "    'user_prods': ['user_id', 'prod']\n",
    "}\n",
    "\n",
    "for fname, fset in HISTORY_ADS.items():\n",
    "    \n",
    "    merge['prev_'+fname] = merge.groupby(fset).cumcount().rename('prev_'+fname)\n",
    "    merge['future_'+fname] = merge.iloc[::-1].groupby(fset).cumcount().rename('future_'+fname).iloc[::-1]\n",
    "print(merge.shape)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [],
   "source": [
    "\n",
    "GROUPBY_AGGREGATIONS = [\n",
    "    {'groupby': ['user_id','prod','campaign_id'], 'select': 'day', 'agg': 'var'},\n",
    "    {'groupby': ['user_id','prod','product_category_1'], 'select': 'hour', 'agg': 'var'},\n",
    "    {'groupby': ['user_id','day','campaign_id'], 'select': 'hour', 'agg': 'var'},\n",
    "    {'groupby': ['user_id','day','hour'], 'select': 'campaign_id', 'agg': 'count'},\n",
    "    {'groupby': ['user_id', 'prod'], 'select': 'campaign_id', 'agg': 'count'},        \n",
    "    {'groupby': ['user_id', 'prod', 'webpage_id'], 'select': 'campaign_id', 'agg': 'count'},\n",
    "    {'groupby': ['user_id','prod','day','hour'], 'select': 'campaign_id', 'agg': 'count'},\n",
    "    {'groupby': ['user_id','prod','campaign_id'], 'select': 'hour', 'agg': 'mean'}, \n",
    "    \n",
    "\n",
    "    {'groupby': ['prod'], \n",
    "     'select': 'user_id', \n",
    "     'agg': lambda x: float(len(x)) / len(x.unique()), \n",
    "     'agg_name': 'AvgprodPerDistinct'\n",
    "    },\n",
    "    {'groupby': ['prod'], 'select': 'campaign_id', 'agg': 'count'},\n",
    "    {'groupby': ['campaign_id'], 'select': 'prod', 'agg': 'count'},\n",
    "    \n",
    "    {'groupby': ['user_id'], 'select': 'campaign_id', 'agg': 'nunique'}, \n",
    "    {'groupby': ['user_id'], 'select': 'prod', 'agg': 'nunique'}, \n",
    "    {'groupby': ['user_id','day'], 'select': 'hour', 'agg': 'nunique'}, \n",
    "    {'groupby': ['user_id','prod'], 'select': 'webpage_id', 'agg': 'nunique'}, \n",
    "    {'groupby': ['user_id'], 'select': 'product_category_1', 'agg': 'nunique'}, \n",
    "    {'groupby': ['prod'], 'select': 'campaign_id', 'agg': 'nunique'}, \n",
    "    {'groupby': ['user_id', 'product_category_1', 'webpage_id'], 'select': 'prod', 'agg': 'nunique'}, \n",
    "    {'groupby': ['user_id','product_category_1','webpage_id'], 'select': 'prod', 'agg': 'cumcount'}, \n",
    "    {'groupby': ['user_id'], 'select': 'prod', 'agg': 'cumcount'}, \n",
    "    {'groupby': ['user_id'], 'select': 'webpage_id', 'agg': 'cumcount'}\n",
    "]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Grouping by ['user_id', 'prod', 'campaign_id'], and aggregating day with var\n",
      "Grouping by ['user_id', 'prod', 'product_category_1'], and aggregating hour with var\n",
      "Grouping by ['user_id', 'day', 'campaign_id'], and aggregating hour with var\n",
      "Grouping by ['user_id', 'day', 'hour'], and aggregating campaign_id with count\n",
      "Grouping by ['user_id', 'prod'], and aggregating campaign_id with count\n",
      "Grouping by ['user_id', 'prod', 'webpage_id'], and aggregating campaign_id with count\n",
      "Grouping by ['user_id', 'prod', 'day', 'hour'], and aggregating campaign_id with count\n",
      "Grouping by ['user_id', 'prod', 'campaign_id'], and aggregating hour with mean\n",
      "Grouping by ['prod'], and aggregating user_id with AvgprodPerDistinct\n",
      "Grouping by ['prod'], and aggregating campaign_id with count\n",
      "Grouping by ['campaign_id'], and aggregating prod with count\n",
      "Grouping by ['user_id'], and aggregating campaign_id with nunique\n",
      "Grouping by ['user_id'], and aggregating prod with nunique\n",
      "Grouping by ['user_id', 'day'], and aggregating hour with nunique\n",
      "Grouping by ['user_id', 'prod'], and aggregating webpage_id with nunique\n",
      "Grouping by ['user_id'], and aggregating product_category_1 with nunique\n",
      "Grouping by ['prod'], and aggregating campaign_id with nunique\n",
      "Grouping by ['user_id', 'product_category_1', 'webpage_id'], and aggregating prod with nunique\n",
      "Grouping by ['user_id', 'product_category_1', 'webpage_id'], and aggregating prod with cumcount\n",
      "Grouping by ['user_id'], and aggregating prod with cumcount\n",
      "Grouping by ['user_id'], and aggregating webpage_id with cumcount\n",
      "(592149, 51)\n"
     ]
    }
   ],
   "source": [
    "# Apply all the groupby transformations\n",
    "for spec in GROUPBY_AGGREGATIONS:\n",
    "    \n",
    "\n",
    "    agg_name = spec['agg_name'] if 'agg_name' in spec else spec['agg']\n",
    "    new_feature = '{}_{}_{}'.format('_'.join(spec['groupby']), agg_name, spec['select'])\n",
    "    \n",
    "    print(\"Grouping by {}, and aggregating {} with {}\".format(\n",
    "        spec['groupby'], spec['select'], agg_name\n",
    "    ))\n",
    "    \n",
    "    all_features = list(set(spec['groupby'] + [spec['select']]))\n",
    "    gp = merge[all_features]. \\\n",
    "        groupby(spec['groupby'])[spec['select']]. \\\n",
    "        agg(spec['agg']). \\\n",
    "        reset_index(). \\\n",
    "        rename(index=str, columns={spec['select']: new_feature})\n",
    "\n",
    "    if 'cumcount' == spec['agg']:\n",
    "        merge[new_feature] = gp[0].values\n",
    "    else:\n",
    "        merge = merge.merge(gp, on=spec['groupby'], how='left')\n",
    "        \n",
    "    del gp\n",
    "    gc.collect()\n",
    "\n",
    "print(merge.shape)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "Index(['user_id', 'prod', 'campaign_id', 'webpage_id', 'product_category_1',\n",
       "       'product_category_2', 'user_group_id', 'gender', 'age_level',\n",
       "       'user_depth', 'city_development_index', 'var_1', 'day', 'hour',\n",
       "       'minute', 'dow', 'prod_campaign_id_confRate',\n",
       "       'prod_webpage_id_confRate', 'prod_product_category_1_confRate',\n",
       "       'user_group_id_gender_confRate', 'user_group_id_age_level_confRate',\n",
       "       'user_group_id_user_depth_confRate', 'prod_age_level_confRate',\n",
       "       'prod_user_depth_confRate', 'product_category_1_age_level_confRate',\n",
       "       'product_category_1_user_depth_confRate', 'prev_identical_',\n",
       "       'future_identical_', 'prev_user_prods', 'future_user_prods',\n",
       "       'user_id_prod_campaign_id_var_day',\n",
       "       'user_id_prod_product_category_1_var_hour',\n",
       "       'user_id_day_campaign_id_var_hour',\n",
       "       'user_id_day_hour_count_campaign_id', 'user_id_prod_count_campaign_id',\n",
       "       'user_id_prod_webpage_id_count_campaign_id',\n",
       "       'user_id_prod_day_hour_count_campaign_id',\n",
       "       'user_id_prod_campaign_id_mean_hour', 'prod_AvgprodPerDistinct_user_id',\n",
       "       'prod_count_campaign_id', 'campaign_id_count_prod',\n",
       "       'user_id_nunique_campaign_id', 'user_id_nunique_prod',\n",
       "       'user_id_day_nunique_hour', 'user_id_prod_nunique_webpage_id',\n",
       "       'user_id_nunique_product_category_1', 'prod_nunique_campaign_id',\n",
       "       'user_id_product_category_1_webpage_id_nunique_prod',\n",
       "       'user_id_product_category_1_webpage_id_cumcount_prod',\n",
       "       'user_id_cumcount_prod', 'user_id_cumcount_webpage_id'],\n",
       "      dtype='object')"
      ]
     },
     "execution_count": 15,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "merge.columns"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [],
   "source": [
    "cat_vars = ['user_id', 'prod', 'campaign_id','webpage_id', 'product_category_1','product_category_2',\n",
    "            'user_group_id', 'gender', 'age_level', 'user_depth','city_development_index', 'var_1',\n",
    "            'day','hour','minute', 'dow']\n",
    "\n",
    "contin_vars = ['prod_campaign_id_confRate',\n",
    "       'prod_webpage_id_confRate', 'prod_product_category_1_confRate',\n",
    "       'user_group_id_gender_confRate', 'user_group_id_age_level_confRate',\n",
    "       'user_group_id_user_depth_confRate', 'prod_age_level_confRate',\n",
    "       'prod_user_depth_confRate', 'product_category_1_age_level_confRate',\n",
    "       'product_category_1_user_depth_confRate', 'prev_identical_',\n",
    "       'future_identical_', 'prev_user_prods', 'future_user_prods',\n",
    "       'user_id_prod_campaign_id_var_day',\n",
    "       'user_id_prod_product_category_1_var_hour',\n",
    "       'user_id_day_campaign_id_var_hour',\n",
    "       'user_id_day_hour_count_campaign_id', 'user_id_prod_count_campaign_id',\n",
    "       'user_id_prod_webpage_id_count_campaign_id',\n",
    "       'user_id_prod_day_hour_count_campaign_id',\n",
    "       'user_id_prod_campaign_id_mean_hour', 'prod_AvgprodPerDistinct_user_id',\n",
    "       'prod_count_campaign_id', 'campaign_id_count_prod',\n",
    "       'user_id_nunique_campaign_id', 'user_id_nunique_prod',\n",
    "       'user_id_day_nunique_hour', 'user_id_prod_nunique_webpage_id',\n",
    "       'user_id_nunique_product_category_1', 'prod_nunique_campaign_id',\n",
    "       'user_id_product_category_1_webpage_id_nunique_prod',\n",
    "       'user_id_product_category_1_webpage_id_cumcount_prod',\n",
    "       'user_id_cumcount_prod', 'user_id_cumcount_webpage_id']\n",
    "\n",
    "for v in cat_vars: \n",
    "    merge[v] = merge[v].astype('category')\n",
    "\n",
    "for v in contin_vars: \n",
    "    merge[v] = merge[v].astype('float32')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(592149, 51)\n"
     ]
    }
   ],
   "source": [
    "lb = LabelEncoder()\n",
    "for v in cat_vars:\n",
    "    merge[v] = lb.fit_transform(merge[v])\n",
    "print(merge.shape)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(463291, 51) (128858, 51)\n"
     ]
    }
   ],
   "source": [
    "train = merge[:nrow_train]\n",
    "test = merge[nrow_train:]\n",
    "print(train.shape, test.shape)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# FInding best"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 129,
   "metadata": {},
   "outputs": [],
   "source": [
    "#params = {'eta': 0.005,'tree_method': \"hist\",'grow_policy': \"lossguide\",'subsample': 0.8,\n",
    "#          'colsample_bytree': 0.8, 'colsample_bylevel':0.7,'objective': 'binary:logistic', \n",
    "#          'eval_metric': 'auc', 'nthread':10,'random_state': 42, 'silent': True, 'max_depth':6,\n",
    "#          'scale_pos_weight':13, 'gamma': 5, 'lambda': 50, 'alpha':70  }"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "\"\\nx1, x2, y1, y2 = train_test_split(train, y, test_size=0.1, random_state=42)\\ndtrain = xgb.DMatrix(x1, y1)\\ndvalid = xgb.DMatrix(x2, y2)\\n\\ndel x1, y1, x2, y2 \\ngc.collect()\\nwatchlist = [(dtrain, 'train'), (dvalid, 'valid')]\\nmodel = xgb.train(params, dtrain, 10000, watchlist, maximize=True, early_stopping_rounds = 50, verbose_eval=500)\\n\""
      ]
     },
     "execution_count": 19,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "\"\"\"\n",
    "x1, x2, y1, y2 = train_test_split(train, y, test_size=0.1, random_state=42)\n",
    "dtrain = xgb.DMatrix(x1, y1)\n",
    "dvalid = xgb.DMatrix(x2, y2)\n",
    "\n",
    "del x1, y1, x2, y2 \n",
    "gc.collect()\n",
    "watchlist = [(dtrain, 'train'), (dvalid, 'valid')]\n",
    "model = xgb.train(params, dtrain, 10000, watchlist, maximize=True, early_stopping_rounds = 50, verbose_eval=500)\n",
    "\"\"\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [],
   "source": [
    "#dtest = xgb.DMatrix(test)\n",
    "#sub['is_click'] = model.predict(dtest, ntree_limit=model.best_ntree_limit)\n",
    "#sub.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [],
   "source": [
    "#sub.to_csv('more_xgb_7.csv', float_format='%.8f', index=False) #subsample': 0.9 - LB - 0.626989521070428"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Apply K fold XGB"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {},
   "outputs": [],
   "source": [
    "target_train = y.values\n",
    "\n",
    "train = np.array(train)\n",
    "test = np.array(test)\n",
    "\n",
    "xgb_preds = []"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(463291, 51) (128858, 51)\n"
     ]
    }
   ],
   "source": [
    "print(train.shape, test.shape)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {},
   "outputs": [],
   "source": [
    "K = 5\n",
    "kf = KFold(n_splits = K, random_state = 2018, shuffle = True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {},
   "outputs": [],
   "source": [
    "params = {'eta': 0.005,'tree_method': \"hist\",'grow_policy': \"lossguide\",'subsample': 0.8,\n",
    "          'colsample_bytree': 0.8, 'colsample_bylevel':0.7,'objective': 'binary:logistic', \n",
    "          'eval_metric': 'auc', 'nthread':10,'random_state': 42, 'silent': True, 'max_depth':6,\n",
    "          'scale_pos_weight':13, 'gamma': 5, 'lambda': 50, 'alpha':20}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[22:02:40] Tree method is selected to be 'hist', which uses a single updater grow_fast_histmaker.\n",
      "[0]\ttrain-auc:0.623044\tvalid-auc:0.613185\n",
      "Multiple eval metrics have been passed: 'valid-auc' will be used for early stopping.\n",
      "\n",
      "Will train until valid-auc hasn't improved in 50 rounds.\n",
      "[500]\ttrain-auc:0.657269\tvalid-auc:0.639317\n",
      "[1000]\ttrain-auc:0.671525\tvalid-auc:0.643225\n",
      "[1500]\ttrain-auc:0.682919\tvalid-auc:0.644983\n",
      "[2000]\ttrain-auc:0.693489\tvalid-auc:0.64604\n",
      "[2500]\ttrain-auc:0.702986\tvalid-auc:0.646798\n",
      "Stopping. Best iteration:\n",
      "[2887]\ttrain-auc:0.71024\tvalid-auc:0.647171\n",
      "\n",
      "[22:05:23] Tree method is selected to be 'hist', which uses a single updater grow_fast_histmaker.\n",
      "[0]\ttrain-auc:0.62671\tvalid-auc:0.616717\n",
      "Multiple eval metrics have been passed: 'valid-auc' will be used for early stopping.\n",
      "\n",
      "Will train until valid-auc hasn't improved in 50 rounds.\n",
      "[500]\ttrain-auc:0.658562\tvalid-auc:0.635702\n",
      "[1000]\ttrain-auc:0.672966\tvalid-auc:0.639259\n",
      "[1500]\ttrain-auc:0.68419\tvalid-auc:0.640691\n",
      "[2000]\ttrain-auc:0.694624\tvalid-auc:0.641602\n",
      "[2500]\ttrain-auc:0.704352\tvalid-auc:0.642311\n",
      "[3000]\ttrain-auc:0.713668\tvalid-auc:0.64278\n",
      "Stopping. Best iteration:\n",
      "[3042]\ttrain-auc:0.714423\tvalid-auc:0.642806\n",
      "\n",
      "[22:08:03] Tree method is selected to be 'hist', which uses a single updater grow_fast_histmaker.\n",
      "[0]\ttrain-auc:0.625204\tvalid-auc:0.621057\n",
      "Multiple eval metrics have been passed: 'valid-auc' will be used for early stopping.\n",
      "\n",
      "Will train until valid-auc hasn't improved in 50 rounds.\n",
      "[500]\ttrain-auc:0.656998\tvalid-auc:0.640481\n",
      "[1000]\ttrain-auc:0.670901\tvalid-auc:0.643477\n",
      "[1500]\ttrain-auc:0.682065\tvalid-auc:0.644961\n",
      "[2000]\ttrain-auc:0.692411\tvalid-auc:0.645839\n",
      "[2500]\ttrain-auc:0.702232\tvalid-auc:0.646732\n",
      "[3000]\ttrain-auc:0.711627\tvalid-auc:0.647199\n",
      "Stopping. Best iteration:\n",
      "[3438]\ttrain-auc:0.719481\tvalid-auc:0.647634\n",
      "\n",
      "[22:10:50] Tree method is selected to be 'hist', which uses a single updater grow_fast_histmaker.\n",
      "[0]\ttrain-auc:0.629121\tvalid-auc:0.60795\n",
      "Multiple eval metrics have been passed: 'valid-auc' will be used for early stopping.\n",
      "\n",
      "Will train until valid-auc hasn't improved in 50 rounds.\n",
      "[500]\ttrain-auc:0.659708\tvalid-auc:0.626504\n",
      "[1000]\ttrain-auc:0.673677\tvalid-auc:0.630689\n",
      "[1500]\ttrain-auc:0.684783\tvalid-auc:0.632658\n",
      "[2000]\ttrain-auc:0.695078\tvalid-auc:0.633781\n",
      "[2500]\ttrain-auc:0.705108\tvalid-auc:0.634412\n",
      "[3000]\ttrain-auc:0.714329\tvalid-auc:0.634842\n",
      "Stopping. Best iteration:\n",
      "[2989]\ttrain-auc:0.714127\tvalid-auc:0.634859\n",
      "\n",
      "[22:13:29] Tree method is selected to be 'hist', which uses a single updater grow_fast_histmaker.\n",
      "[0]\ttrain-auc:0.622984\tvalid-auc:0.622168\n",
      "Multiple eval metrics have been passed: 'valid-auc' will be used for early stopping.\n",
      "\n",
      "Will train until valid-auc hasn't improved in 50 rounds.\n",
      "[500]\ttrain-auc:0.656024\tvalid-auc:0.646039\n",
      "[1000]\ttrain-auc:0.670829\tvalid-auc:0.649402\n",
      "[1500]\ttrain-auc:0.682272\tvalid-auc:0.650762\n",
      "[2000]\ttrain-auc:0.692869\tvalid-auc:0.651804\n",
      "Stopping. Best iteration:\n",
      "[2406]\ttrain-auc:0.701271\tvalid-auc:0.652269\n",
      "\n",
      "Time taken is: 777.535796880722\n"
     ]
    }
   ],
   "source": [
    "start = time()\n",
    "\n",
    "for train_index, test_index in kf.split(train):\n",
    "    train_X, valid_X = train[train_index], train[test_index]\n",
    "    train_y, valid_y = target_train[train_index], target_train[test_index]\n",
    "\n",
    "    d_train = xgb.DMatrix(train_X, train_y)\n",
    "    d_valid = xgb.DMatrix(valid_X, valid_y)\n",
    "    d_test = xgb.DMatrix(test)\n",
    "    \n",
    "    watchlist = [(d_train, 'train'), (d_valid, 'valid')]\n",
    "    model = xgb.train(params, d_train, 10000, watchlist, maximize=True, verbose_eval=500, early_stopping_rounds=50)\n",
    "                        \n",
    "    xgb_pred = model.predict(d_test)\n",
    "    xgb_preds.append(list(xgb_pred))\n",
    "\n",
    "end = time()\n",
    "print ('Time taken is:', end-start)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {},
   "outputs": [],
   "source": [
    "preds=[]\n",
    "for i in range(len(xgb_preds[0])):\n",
    "    sum=0\n",
    "    for j in range(K):\n",
    "        sum+=xgb_preds[j][i]\n",
    "    preds.append(sum / K)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {},
   "outputs": [],
   "source": [
    "sub['is_click']=preds"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>session_id</th>\n",
       "      <th>is_click</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>411705</td>\n",
       "      <td>0.639891</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>208263</td>\n",
       "      <td>0.229040</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>239450</td>\n",
       "      <td>0.174025</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>547761</td>\n",
       "      <td>0.364239</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>574275</td>\n",
       "      <td>0.560393</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   session_id  is_click\n",
       "0      411705  0.639891\n",
       "1      208263  0.229040\n",
       "2      239450  0.174025\n",
       "3      547761  0.364239\n",
       "4      574275  0.560393"
      ]
     },
     "execution_count": 29,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "sub.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "metadata": {},
   "outputs": [],
   "source": [
    "sub.to_csv('xgb_5fold_v2.csv', float_format='%.8f', index=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 42,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 46,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 47,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.6"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}