WaveNet+DNN.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# 由于在boost模型建模的时候发现序列特征是最强的，与其手动调试不如搭建一个网络自动学习"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\Users\\Ksama\\AppData\\Local\\conda\\conda\\envs\\tensorflow\\lib\\site-packages\\tensorflow\\python\\framework\\dtypes.py:526: FutureWarning: Passing (type, 1) or '1type' as a synonym of type is deprecated; in a future version of numpy, it will be understood as (type, (1,)) / '(1,)type'.\n",
      "  _np_qint8 = np.dtype([(\"qint8\", np.int8, 1)])\n",
      "C:\\Users\\Ksama\\AppData\\Local\\conda\\conda\\envs\\tensorflow\\lib\\site-packages\\tensorflow\\python\\framework\\dtypes.py:527: FutureWarning: Passing (type, 1) or '1type' as a synonym of type is deprecated; in a future version of numpy, it will be understood as (type, (1,)) / '(1,)type'.\n",
      "  _np_quint8 = np.dtype([(\"quint8\", np.uint8, 1)])\n",
      "C:\\Users\\Ksama\\AppData\\Local\\conda\\conda\\envs\\tensorflow\\lib\\site-packages\\tensorflow\\python\\framework\\dtypes.py:528: FutureWarning: Passing (type, 1) or '1type' as a synonym of type is deprecated; in a future version of numpy, it will be understood as (type, (1,)) / '(1,)type'.\n",
      "  _np_qint16 = np.dtype([(\"qint16\", np.int16, 1)])\n",
      "C:\\Users\\Ksama\\AppData\\Local\\conda\\conda\\envs\\tensorflow\\lib\\site-packages\\tensorflow\\python\\framework\\dtypes.py:529: FutureWarning: Passing (type, 1) or '1type' as a synonym of type is deprecated; in a future version of numpy, it will be understood as (type, (1,)) / '(1,)type'.\n",
      "  _np_quint16 = np.dtype([(\"quint16\", np.uint16, 1)])\n",
      "C:\\Users\\Ksama\\AppData\\Local\\conda\\conda\\envs\\tensorflow\\lib\\site-packages\\tensorflow\\python\\framework\\dtypes.py:530: FutureWarning: Passing (type, 1) or '1type' as a synonym of type is deprecated; in a future version of numpy, it will be understood as (type, (1,)) / '(1,)type'.\n",
      "  _np_qint32 = np.dtype([(\"qint32\", np.int32, 1)])\n",
      "C:\\Users\\Ksama\\AppData\\Local\\conda\\conda\\envs\\tensorflow\\lib\\site-packages\\tensorflow\\python\\framework\\dtypes.py:535: FutureWarning: Passing (type, 1) or '1type' as a synonym of type is deprecated; in a future version of numpy, it will be understood as (type, (1,)) / '(1,)type'.\n",
      "  np_resource = np.dtype([(\"resource\", np.ubyte, 1)])\n",
      "Using TensorFlow backend.\n"
     ]
    }
   ],
   "source": [
    "import os\n",
    "import matplotlib.pyplot as plt\n",
    "import glob\n",
    "import gc\n",
    "from math import *\n",
    "import warnings\n",
    "import numpy as np\n",
    "import pandas as pd\n",
    "import tensorflow as tf\n",
    "from tqdm import tqdm\n",
    "from keras.layers import Dense, Dropout, LSTM,Reshape, GRU,Conv1D, Conv2D,Flatten,Permute, multiply,BatchNormalization, Activation, AveragePooling1D, GlobalAveragePooling1D, Lambda, Input, Concatenate, Add, UpSampling1D, Multiply\n",
    "from keras.models import Model\n",
    "from keras.objectives import mean_squared_error\n",
    "from keras import backend as K\n",
    "from keras.utils.vis_utils import plot_model\n",
    "from keras.losses import binary_crossentropy, categorical_crossentropy\n",
    "from keras.callbacks import ModelCheckpoint, EarlyStopping, TensorBoard, ReduceLROnPlateau,LearningRateScheduler\n",
    "from keras.initializers import random_normal\n",
    "from keras.optimizers import Adam, RMSprop, SGD\n",
    "from keras import regularizers\n",
    "from keras.callbacks import Callback\n",
    "from keras_self_attention import SeqSelfAttention\n",
    "from sklearn.metrics import cohen_kappa_score, f1_score,roc_auc_score\n",
    "from sklearn.model_selection import KFold, train_test_split\n",
    "from sklearn.preprocessing import StandardScaler, LabelEncoder,normalize\n",
    "\n",
    "warnings.filterwarnings('ignore')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "def reduce_mem(df):\n",
    "    start_mem = df.memory_usage().sum() / 1024 ** 2\n",
    "    for col in df.columns:\n",
    "        col_type = df[col].dtypes\n",
    "        if col_type != object:\n",
    "            c_min = df[col].min()\n",
    "            c_max = df[col].max()\n",
    "            if str(col_type)[:3] == 'int':\n",
    "                if c_min > np.iinfo(np.int8).min and c_max < np.iinfo(np.int8).max:\n",
    "                    df[col] = df[col].astype(np.int8)\n",
    "                elif c_min > np.iinfo(np.int16).min and c_max < np.iinfo(np.int16).max:\n",
    "                    df[col] = df[col].astype(np.int16)\n",
    "                elif c_min > np.iinfo(np.int32).min and c_max < np.iinfo(np.int32).max:\n",
    "                    df[col] = df[col].astype(np.int32)\n",
    "                elif c_min > np.iinfo(np.int64).min and c_max < np.iinfo(np.int64).max:\n",
    "                    df[col] = df[col].astype(np.int64)\n",
    "            else:\n",
    "                if c_min > np.finfo(np.float16).min and c_max < np.finfo(np.float16).max:\n",
    "                    df[col] = df[col].astype(np.float16)\n",
    "                elif c_min > np.finfo(np.float32).min and c_max < np.finfo(np.float32).max:\n",
    "                    df[col] = df[col].astype(np.float32)\n",
    "                else:\n",
    "                    df[col] = df[col].astype(np.float64)\n",
    "    end_mem = df.memory_usage().sum() / 1024 ** 2\n",
    "    print('{:.2f} Mb, {:.2f} Mb ({:.2f} %)'.format(start_mem, end_mem, 100 * (start_mem - end_mem) / start_mem))\n",
    "    gc.collect()\n",
    "    return df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "WARNING:root:\n",
      "DeepCTR version 0.7.4 detected. Your version is 0.7.3.\n",
      "Use `pip install -U deepctr` to upgrade.Changelog: https://github.com/shenweichen/DeepCTR/releases/tag/v0.7.4\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "7"
      ]
     },
     "execution_count": 3,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "train_df = pd.read_csv('../data/train.csv',low_memory=False)\n",
    "test_df = pd.read_csv('../data/test.csv',low_memory=False)\n",
    "\n",
    "#根据“第二次打比赛”队伍的发现进行简单的数据清洗\n",
    "train_df= train_df[(train_df['q']>0)]\n",
    "train_df = train_df[train_df['t']>-900]\n",
    "train_df = train_df[train_df['t']<1850]\n",
    "\n",
    "train_num = len(train_df)\n",
    "data = pd.concat([train_df, test_df], ignore_index=True)\n",
    "event = pd.read_csv('../data/event.csv')\n",
    "data = pd.merge(data, event, on='event_id', how='left')\n",
    "del train_df,test_df,event\n",
    "gc.collect()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "#非序列特征：\n",
    "#由于开始做深度模型的时间已经很晚了，故把精力放在序列的构造上，并没有过多的进行非序列特征的构造\n",
    "data['x_cmc'] = (data['x']-data['xcmc'])\n",
    "data['y_cmc'] = (data['y']-data['ycmc'])\n",
    "data['dis'] = np.sqrt(data['x_cmc']**2+data['y_cmc']**2)\n",
    "\n",
    "# t ,dis统计特征\n",
    "data['event_id_t_min'] = data.groupby('event_id')['t'].transform('min')\n",
    "data['event_id_t_max'] = data.groupby('event_id')['t'].transform('max')\n",
    "data['event_id_t_median'] = data.groupby('event_id')['t'].transform('median')\n",
    "data['event_id_t_mean'] = data.groupby('event_id')['t'].transform('mean')\n",
    "\n",
    "data['event_id_dis_min'] = data.groupby('event_id')['dis'].transform('min')\n",
    "data['event_id_dis_max'] = data.groupby('event_id')['dis'].transform('max')\n",
    "data['event_id_dis_median'] = data.groupby('event_id')['dis'].transform('median')\n",
    "data['event_id_dis_mean'] = data.groupby('event_id')['dis'].transform('mean')\n",
    "\n",
    "# t,dis\"偏移\"\n",
    "data['t_min_diff'] = data['t'] - data['event_id_t_min']\n",
    "data['t_max_diff'] = data['event_id_t_max'] - data['t']\n",
    "data['t_median_diff'] = data['event_id_t_median'] - data['t']\n",
    "data['t_mean_diff'] = data['event_id_t_mean'] - data['t']\n",
    "\n",
    "data['dis_min_diff'] = data['dis'] - data['event_id_dis_min']\n",
    "data['dis_max_diff'] = data['event_id_dis_max'] - data['dis']\n",
    "data['dis_median_diff'] = data['event_id_dis_median'] - data['dis']\n",
    "data['dis_mean_diff'] = data['event_id_dis_mean'] - data['dis']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "#归一化\n",
    "sclaer = StandardScaler()\n",
    "scale_fea = [x for x in data.columns if x not in ['event_id','hit_id','flag','z','level_0','index']]\n",
    "data[scale_fea] = sclaer.fit_transform(data[scale_fea].values)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████████████████████████████████████████████████████████| 64/64 [01:03<00:00,  1.00it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(13382309, 99)\n",
      "10209.89 Mb, 2641.81 Mb (74.12 %)\n"
     ]
    }
   ],
   "source": [
    "#构造序列特征，用做wavenet的输入\n",
    "seq_len = 64\n",
    "deleta = 2\n",
    "gap = int(seq_len/2)\n",
    "temp = data.sort_values(['q'])\n",
    "\n",
    "timing_cols=[]\n",
    "\n",
    "for i in tqdm(range(seq_len)):\n",
    "    data['t_gap_{}'.format(i)] = temp['t'].shift(deleta*(i-gap)).fillna(0)\n",
    "    timing_cols += ['t_gap_{}'.format(i)]\n",
    "print(data.shape)\n",
    "data = reduce_mem(data)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████████████████████████████████████████████████████████| 64/64 [00:48<00:00,  1.33it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(13382309, 163)\n",
      "4275.39 Mb, 4275.39 Mb (0.00 %)\n"
     ]
    }
   ],
   "source": [
    "temp = data.sort_values(['event_id','q'])\n",
    "\n",
    "q_cols=[]\n",
    "\n",
    "for i in tqdm(range(seq_len)):\n",
    "    data['q_gap_{}'.format(i)] = temp['q'].shift(deleta*(i - gap)).fillna(0)\n",
    "    q_cols += ['q_gap_{}'.format(i)]\n",
    "print(data.shape)\n",
    "del temp\n",
    "data = reduce_mem(data)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████████████████████████████████████████████████████████| 64/64 [00:49<00:00,  1.30it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(13382309, 227)\n",
      "5908.97 Mb, 5908.97 Mb (0.00 %)\n"
     ]
    }
   ],
   "source": [
    "temp = data.sort_values(['terror','q'])\n",
    "\n",
    "teq_cols=[]\n",
    "\n",
    "for i in tqdm(range(seq_len)):\n",
    "    data['tq_gap_{}'.format(i)] = temp['q'].shift(deleta*(i - gap)).fillna(0)\n",
    "    teq_cols += ['tq_gap_{}'.format(i)]\n",
    "print(data.shape)\n",
    "del temp\n",
    "data = reduce_mem(data)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████████████████████████████████████████████████████████| 64/64 [00:47<00:00,  1.34it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(13382309, 291)\n",
      "7542.56 Mb, 7542.56 Mb (0.00 %)\n"
     ]
    }
   ],
   "source": [
    "temp = data.sort_values(['hit_id'])\n",
    "\n",
    "ht_cols=[]\n",
    "\n",
    "for i in tqdm(range(seq_len)):\n",
    "    data['ht_gap_{}'.format(i)] = temp['t'].shift(deleta*(i - gap)).fillna(0)\n",
    "    ht_cols += ['ht_gap_{}'.format(i)]\n",
    "print(data.shape)\n",
    "del temp\n",
    "data = reduce_mem(data)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████████████████████████████████████████████████████████| 64/64 [00:52<00:00,  1.23it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(13382309, 355)\n",
      "9176.14 Mb, 9176.14 Mb (0.00 %)\n"
     ]
    }
   ],
   "source": [
    "temp = data.sort_values(['terror','q'])\n",
    "\n",
    "tet_cols=[]\n",
    "\n",
    "for i in tqdm(range(seq_len)):\n",
    "    data['tet_gap_{}'.format(i)] = temp['t'].shift(deleta*(i - gap)).fillna(0)\n",
    "    tet_cols += ['tet_gap_{}'.format(i)]\n",
    "print(data.shape)\n",
    "del temp\n",
    "data = reduce_mem(data)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████████████████████████████████████████████████████████| 64/64 [00:49<00:00,  1.29it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(13382309, 419)\n",
      "10809.72 Mb, 10809.72 Mb (0.00 %)\n"
     ]
    }
   ],
   "source": [
    "temp = data.sort_values(['event_id','q'])\n",
    "\n",
    "qdis_cols=[]\n",
    "\n",
    "for i in tqdm(range(seq_len)):\n",
    "    data['qdis_gap_{}'.format(i)] = temp['dis'].shift(deleta*(i - gap)).fillna(0)\n",
    "    qdis_cols += ['qdis_gap_{}'.format(i)]\n",
    "print(data.shape)\n",
    "del temp\n",
    "data = reduce_mem(data)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████████████████████████████████████████████████████████| 64/64 [00:52<00:00,  1.21it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(13382309, 483)\n",
      "12443.31 Mb, 12443.31 Mb (0.00 %)\n"
     ]
    }
   ],
   "source": [
    "temp = data.sort_values(['terror','q'])\n",
    "\n",
    "tqdis_cols=[]\n",
    "\n",
    "for i in tqdm(range(seq_len)):\n",
    "    data['tqdis_gap_{}'.format(i)] = temp['dis'].shift(deleta*(i - gap)).fillna(0)\n",
    "    tqdis_cols += ['tqdis_gap_{}'.format(i)]\n",
    "print(data.shape)\n",
    "del temp\n",
    "data = reduce_mem(data)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "10"
      ]
     },
     "execution_count": 13,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "train_df = data[:train_num].reset_index()\n",
    "test_df = data[train_num:].reset_index()\n",
    "train_df=train_df.sample(frac=1.0)\n",
    "train_df=train_df.reset_index(drop=True)\n",
    "del data\n",
    "gc.collect()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(9295798, 64, 7) (9295798, 31)\n"
     ]
    }
   ],
   "source": [
    "train_data_1 = np.array(train_df[timing_cols]).reshape(len(train_df), seq_len, 1)  \n",
    "train_data_2 = np.array(train_df[q_cols]).reshape(len(train_df), seq_len, 1)  \n",
    "train_data_3 = np.array(train_df[teq_cols]).reshape(len(train_df), seq_len, 1)  \n",
    "train_data_4 = np.array(train_df[ht_cols]).reshape(len(train_df), seq_len, 1)  \n",
    "train_data_5 = np.array(train_df[tet_cols]).reshape(len(train_df), seq_len, 1)  \n",
    "train_data_6 = np.array(train_df[qdis_cols]).reshape(len(train_df), seq_len, 1)  \n",
    "train_data_7 = np.array(train_df[tqdis_cols]).reshape(len(train_df), seq_len, 1)  \n",
    "\n",
    "train_data_seq = np.concatenate([train_data_1,train_data_2,train_data_3,train_data_4,train_data_5,train_data_6,train_data_7],axis=2)\n",
    "\n",
    "train_label = train_df['flag'].values\n",
    "train_data_sideinfo = train_df[scale_fea].values  \n",
    "del train_data_1,train_data_2,train_data_3,train_data_4,train_data_5,train_data_6,train_data_7\n",
    "\n",
    "test_data_1 = np.array(test_df[timing_cols]).reshape(len(test_df), seq_len, 1)  \n",
    "test_data_2 = np.array(test_df[q_cols]).reshape(len(test_df), seq_len, 1)  \n",
    "test_data_3 = np.array(test_df[teq_cols]).reshape(len(test_df), seq_len, 1)  \n",
    "test_data_4 = np.array(test_df[ht_cols]).reshape(len(test_df), seq_len, 1)  \n",
    "test_data_5 = np.array(test_df[tet_cols]).reshape(len(test_df), seq_len, 1)  \n",
    "test_data_6 = np.array(test_df[qdis_cols]).reshape(len(test_df), seq_len, 1)  \n",
    "test_data_7 = np.array(test_df[tqdis_cols]).reshape(len(test_df), seq_len, 1)  \n",
    "\n",
    "test_data_seq = np.concatenate([test_data_1,test_data_2,test_data_3,test_data_4,test_data_5,test_data_6,test_data_7],axis=2)\n",
    "\n",
    "test_data_sideinfo = test_df[scale_fea].values \n",
    "\n",
    "del test_data_1,test_data_2,test_data_3,test_data_4,test_data_5,test_data_6,test_data_7\n",
    "gc.collect()\n",
    "print(train_data_seq.shape,train_data_sideinfo.shape)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [],
   "source": [
    "# 训练集验证集按照 7:3比例分割\n",
    "train_size = int(len(train_data_lstm) * 0.7)\n",
    "# 训练集、验证集 seq特征\n",
    "X_train_seq, X_validate_seq = train_data_seq[:train_size], train_data_seq[train_size:]\n",
    "label_train, label_validate = train_label[:train_size], train_label[train_size:]\n",
    "\n",
    "# 训练集、验证集 dnn的特征\n",
    "X_train_side, X_validate_side = train_data_sideinfo[:train_size], train_data_sideinfo[train_size:]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [],
   "source": [
    "#采用CyclicLR学习率衰减\n",
    "class CyclicLR(tf.keras.callbacks.Callback):\n",
    "\n",
    "    def __init__(self, base_lr=0.001, max_lr=0.006, step_size=2000., mode='triangular',\n",
    "                 gamma=1., scale_fn=None, scale_mode='cycle'):\n",
    "        super(CyclicLR, self).__init__()\n",
    "\n",
    "        self.base_lr = base_lr\n",
    "        self.max_lr = max_lr\n",
    "        self.step_size = step_size\n",
    "        self.mode = mode\n",
    "        self.gamma = gamma\n",
    "        if scale_fn == None:\n",
    "            if self.mode == 'triangular':\n",
    "                self.scale_fn = lambda x: 1.\n",
    "                self.scale_mode = 'cycle'\n",
    "            elif self.mode == 'triangular2':\n",
    "                self.scale_fn = lambda x: 1 / (2. ** (x - 1))\n",
    "                self.scale_mode = 'cycle'\n",
    "            elif self.mode == 'exp_range':\n",
    "                self.scale_fn = lambda x: gamma ** (x)\n",
    "                self.scale_mode = 'iterations'\n",
    "        else:\n",
    "            self.scale_fn = scale_fn\n",
    "            self.scale_mode = scale_mode\n",
    "        self.clr_iterations = 0.\n",
    "        self.trn_iterations = 0.\n",
    "        self.history = {}\n",
    "\n",
    "        self._reset()\n",
    "\n",
    "    def _reset(self, new_base_lr=None, new_max_lr=None,\n",
    "               new_step_size=None):\n",
    "        \"\"\"Resets cycle iterations.\n",
    "        Optional boundary/step size adjustment.\n",
    "        \"\"\"\n",
    "        if new_base_lr != None:\n",
    "            self.base_lr = new_base_lr\n",
    "        if new_max_lr != None:\n",
    "            self.max_lr = new_max_lr\n",
    "        if new_step_size != None:\n",
    "            self.step_size = new_step_size\n",
    "        self.clr_iterations = 0.\n",
    "\n",
    "    def clr(self):\n",
    "        cycle = np.floor(1 + self.clr_iterations / (2 * self.step_size))\n",
    "        x = np.abs(self.clr_iterations / self.step_size - 2 * cycle + 1)\n",
    "        if self.scale_mode == 'cycle':\n",
    "            return self.base_lr + (self.max_lr - self.base_lr) * np.maximum(0, (1 - x)) * self.scale_fn(cycle)\n",
    "        else:\n",
    "            return self.base_lr + (self.max_lr - self.base_lr) * np.maximum(0, (1 - x)) * self.scale_fn(\n",
    "                self.clr_iterations)\n",
    "\n",
    "    def on_train_begin(self, logs={}):\n",
    "        logs = logs or {}\n",
    "\n",
    "        if self.clr_iterations == 0:\n",
    "            K.set_value(self.model.optimizer.lr, self.base_lr)\n",
    "        else:\n",
    "            K.set_value(self.model.optimizer.lr, self.clr())\n",
    "\n",
    "    def on_batch_end(self, epoch, logs=None):\n",
    "\n",
    "        logs = logs or {}\n",
    "        self.trn_iterations += 1\n",
    "        self.clr_iterations += 1\n",
    "\n",
    "        K.set_value(self.model.optimizer.lr, self.clr())\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [],
   "source": [
    "#wavenet的组成部分\n",
    "def wave_block(x,filters,kernel_size,n):\n",
    "    dilation_rates = [2**i for i in range(n)]\n",
    "    x = Conv1D(filters=filters,\n",
    "                kernel_size=1, \n",
    "                padding='same')(x)\n",
    "    res_x = x\n",
    "    for dilation_rate in dilation_rates:\n",
    "        tanh_out = Conv1D(filters=filters,\n",
    "                kernel_size=kernel_size, \n",
    "                padding='same',\n",
    "                activation = 'tanh',\n",
    "                dilation_rate=dilation_rate)(x)\n",
    "        sigm_out = Conv1D(filters=filters,\n",
    "                kernel_size=kernel_size, \n",
    "                padding='same',\n",
    "                activation = 'sigmoid',\n",
    "                dilation_rate=dilation_rate)(x)\n",
    "        x = Multiply()([tanh_out,sigm_out])\n",
    "        x = Conv1D(filters = filters,\n",
    "                       kernel_size = 1,\n",
    "                      padding='same')(x)\n",
    "\n",
    "        res_x = Add()([res_x,x])\n",
    "    return res_x\n",
    "\n",
    "#attention机制\n",
    "def attention_3d_block(inputs, seq_len=21):\n",
    "    a = Permute((2, 1))(inputs)\n",
    "    a = Dense(seq_len, activation='softmax')(a)\n",
    "    a_probs = Permute((2, 1))(a)\n",
    "    output_attention_mul = multiply([inputs, a_probs])\n",
    "    return output_attention_mul"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [],
   "source": [
    "def nn_model(seq_len,fea_len,att=True):\n",
    "    \n",
    "    seq_input =  Input((seq_len,7), name='seq_input')\n",
    "    nn_input = Input((fea_len,), name='nn_input')\n",
    "     \n",
    "#wavenet   \n",
    "    x = wave_block(seq_input,8,3,3)\n",
    "    x = wave_block(x,  12,3,2)\n",
    "    x = wave_block(x,  16,3,2)\n",
    "    seq_out = wave_block(x, 20,3,1)\n",
    "\n",
    "#合并  \n",
    "    seq_out = Dense(64,kernel_regularizer=regularizers.l2(0.01))(seq_out)\n",
    "    attention_mul = attention_3d_block(seq_out, seq_len=seq_len)\n",
    "    seq_out = Lambda(lambda x: K.sum(x, axis=1))(attention_mul)\n",
    "    \n",
    "#dnn input\n",
    "    dnn_input = Concatenate()([seq_out,nn_input])\n",
    "\n",
    "    x = Dense(512,activation='relu',kernel_regularizer=regularizers.l2(0.01))(dnn_input)\n",
    "    x = BatchNormalization()(x)\n",
    "    \n",
    "    x = Dense(128,activation='relu')(x)\n",
    "    x = BatchNormalization()(x)\n",
    "    \n",
    "    out = Dense(1, activation=\"sigmoid\")(x)\n",
    "    \n",
    "    model = Model(inputs=[seq_input, nn_input], outputs=out)\n",
    "    \n",
    "    return model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "WARNING:tensorflow:From C:\\Users\\Ksama\\AppData\\Local\\conda\\conda\\envs\\tensorflow\\lib\\site-packages\\tensorflow\\python\\framework\\op_def_library.py:263: colocate_with (from tensorflow.python.framework.ops) is deprecated and will be removed in a future version.\n",
      "Instructions for updating:\n",
      "Colocations handled automatically by placer.\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "WARNING:tensorflow:From C:\\Users\\Ksama\\AppData\\Local\\conda\\conda\\envs\\tensorflow\\lib\\site-packages\\tensorflow\\python\\framework\\op_def_library.py:263: colocate_with (from tensorflow.python.framework.ops) is deprecated and will be removed in a future version.\n",
      "Instructions for updating:\n",
      "Colocations handled automatically by placer.\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "WARNING:tensorflow:From C:\\Users\\Ksama\\AppData\\Local\\conda\\conda\\envs\\tensorflow\\lib\\site-packages\\keras\\backend\\tensorflow_backend.py:1242: calling reduce_sum_v1 (from tensorflow.python.ops.math_ops) with keep_dims is deprecated and will be removed in a future version.\n",
      "Instructions for updating:\n",
      "keep_dims is deprecated, use keepdims instead\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "WARNING:tensorflow:From C:\\Users\\Ksama\\AppData\\Local\\conda\\conda\\envs\\tensorflow\\lib\\site-packages\\keras\\backend\\tensorflow_backend.py:1242: calling reduce_sum_v1 (from tensorflow.python.ops.math_ops) with keep_dims is deprecated and will be removed in a future version.\n",
      "Instructions for updating:\n",
      "keep_dims is deprecated, use keepdims instead\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "WARNING:tensorflow:From C:\\Users\\Ksama\\AppData\\Local\\conda\\conda\\envs\\tensorflow\\lib\\site-packages\\keras\\backend\\tensorflow_backend.py:1208: calling reduce_max_v1 (from tensorflow.python.ops.math_ops) with keep_dims is deprecated and will be removed in a future version.\n",
      "Instructions for updating:\n",
      "keep_dims is deprecated, use keepdims instead\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "WARNING:tensorflow:From C:\\Users\\Ksama\\AppData\\Local\\conda\\conda\\envs\\tensorflow\\lib\\site-packages\\keras\\backend\\tensorflow_backend.py:1208: calling reduce_max_v1 (from tensorflow.python.ops.math_ops) with keep_dims is deprecated and will be removed in a future version.\n",
      "Instructions for updating:\n",
      "keep_dims is deprecated, use keepdims instead\n"
     ]
    }
   ],
   "source": [
    "model = nn_model(seq_len=seq_len,fea_len=len(scale_fea))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "__________________________________________________________________________________________________\n",
      "Layer (type)                    Output Shape         Param #     Connected to                     \n",
      "==================================================================================================\n",
      "seq_input (InputLayer)          (None, 64, 7)        0                                            \n",
      "__________________________________________________________________________________________________\n",
      "conv1d_1 (Conv1D)               (None, 64, 8)        64          seq_input[0][0]                  \n",
      "__________________________________________________________________________________________________\n",
      "conv1d_2 (Conv1D)               (None, 64, 8)        200         conv1d_1[0][0]                   \n",
      "__________________________________________________________________________________________________\n",
      "conv1d_3 (Conv1D)               (None, 64, 8)        200         conv1d_1[0][0]                   \n",
      "__________________________________________________________________________________________________\n",
      "multiply_1 (Multiply)           (None, 64, 8)        0           conv1d_2[0][0]                   \n",
      "                                                                 conv1d_3[0][0]                   \n",
      "__________________________________________________________________________________________________\n",
      "conv1d_4 (Conv1D)               (None, 64, 8)        72          multiply_1[0][0]                 \n",
      "__________________________________________________________________________________________________\n",
      "conv1d_5 (Conv1D)               (None, 64, 8)        200         conv1d_4[0][0]                   \n",
      "__________________________________________________________________________________________________\n",
      "conv1d_6 (Conv1D)               (None, 64, 8)        200         conv1d_4[0][0]                   \n",
      "__________________________________________________________________________________________________\n",
      "multiply_2 (Multiply)           (None, 64, 8)        0           conv1d_5[0][0]                   \n",
      "                                                                 conv1d_6[0][0]                   \n",
      "__________________________________________________________________________________________________\n",
      "conv1d_7 (Conv1D)               (None, 64, 8)        72          multiply_2[0][0]                 \n",
      "__________________________________________________________________________________________________\n",
      "conv1d_8 (Conv1D)               (None, 64, 8)        200         conv1d_7[0][0]                   \n",
      "__________________________________________________________________________________________________\n",
      "conv1d_9 (Conv1D)               (None, 64, 8)        200         conv1d_7[0][0]                   \n",
      "__________________________________________________________________________________________________\n",
      "add_1 (Add)                     (None, 64, 8)        0           conv1d_1[0][0]                   \n",
      "                                                                 conv1d_4[0][0]                   \n",
      "__________________________________________________________________________________________________\n",
      "multiply_3 (Multiply)           (None, 64, 8)        0           conv1d_8[0][0]                   \n",
      "                                                                 conv1d_9[0][0]                   \n",
      "__________________________________________________________________________________________________\n",
      "add_2 (Add)                     (None, 64, 8)        0           add_1[0][0]                      \n",
      "                                                                 conv1d_7[0][0]                   \n",
      "__________________________________________________________________________________________________\n",
      "conv1d_10 (Conv1D)              (None, 64, 8)        72          multiply_3[0][0]                 \n",
      "__________________________________________________________________________________________________\n",
      "add_3 (Add)                     (None, 64, 8)        0           add_2[0][0]                      \n",
      "                                                                 conv1d_10[0][0]                  \n",
      "__________________________________________________________________________________________________\n",
      "conv1d_11 (Conv1D)              (None, 64, 12)       108         add_3[0][0]                      \n",
      "__________________________________________________________________________________________________\n",
      "conv1d_12 (Conv1D)              (None, 64, 12)       444         conv1d_11[0][0]                  \n",
      "__________________________________________________________________________________________________\n",
      "conv1d_13 (Conv1D)              (None, 64, 12)       444         conv1d_11[0][0]                  \n",
      "__________________________________________________________________________________________________\n",
      "multiply_4 (Multiply)           (None, 64, 12)       0           conv1d_12[0][0]                  \n",
      "                                                                 conv1d_13[0][0]                  \n",
      "__________________________________________________________________________________________________\n",
      "conv1d_14 (Conv1D)              (None, 64, 12)       156         multiply_4[0][0]                 \n",
      "__________________________________________________________________________________________________\n",
      "conv1d_15 (Conv1D)              (None, 64, 12)       444         conv1d_14[0][0]                  \n",
      "__________________________________________________________________________________________________\n",
      "conv1d_16 (Conv1D)              (None, 64, 12)       444         conv1d_14[0][0]                  \n",
      "__________________________________________________________________________________________________\n",
      "multiply_5 (Multiply)           (None, 64, 12)       0           conv1d_15[0][0]                  \n",
      "                                                                 conv1d_16[0][0]                  \n",
      "__________________________________________________________________________________________________\n",
      "add_4 (Add)                     (None, 64, 12)       0           conv1d_11[0][0]                  \n",
      "                                                                 conv1d_14[0][0]                  \n",
      "__________________________________________________________________________________________________\n",
      "conv1d_17 (Conv1D)              (None, 64, 12)       156         multiply_5[0][0]                 \n",
      "__________________________________________________________________________________________________\n",
      "add_5 (Add)                     (None, 64, 12)       0           add_4[0][0]                      \n",
      "                                                                 conv1d_17[0][0]                  \n",
      "__________________________________________________________________________________________________\n",
      "conv1d_18 (Conv1D)              (None, 64, 16)       208         add_5[0][0]                      \n",
      "__________________________________________________________________________________________________\n",
      "conv1d_19 (Conv1D)              (None, 64, 16)       784         conv1d_18[0][0]                  \n",
      "__________________________________________________________________________________________________\n",
      "conv1d_20 (Conv1D)              (None, 64, 16)       784         conv1d_18[0][0]                  \n",
      "__________________________________________________________________________________________________\n",
      "multiply_6 (Multiply)           (None, 64, 16)       0           conv1d_19[0][0]                  \n",
      "                                                                 conv1d_20[0][0]                  \n",
      "__________________________________________________________________________________________________\n",
      "conv1d_21 (Conv1D)              (None, 64, 16)       272         multiply_6[0][0]                 \n",
      "__________________________________________________________________________________________________\n",
      "conv1d_22 (Conv1D)              (None, 64, 16)       784         conv1d_21[0][0]                  \n",
      "__________________________________________________________________________________________________\n",
      "conv1d_23 (Conv1D)              (None, 64, 16)       784         conv1d_21[0][0]                  \n",
      "__________________________________________________________________________________________________\n",
      "multiply_7 (Multiply)           (None, 64, 16)       0           conv1d_22[0][0]                  \n",
      "                                                                 conv1d_23[0][0]                  \n",
      "__________________________________________________________________________________________________\n",
      "add_6 (Add)                     (None, 64, 16)       0           conv1d_18[0][0]                  \n",
      "                                                                 conv1d_21[0][0]                  \n",
      "__________________________________________________________________________________________________\n",
      "conv1d_24 (Conv1D)              (None, 64, 16)       272         multiply_7[0][0]                 \n",
      "__________________________________________________________________________________________________\n",
      "add_7 (Add)                     (None, 64, 16)       0           add_6[0][0]                      \n",
      "                                                                 conv1d_24[0][0]                  \n",
      "__________________________________________________________________________________________________\n",
      "conv1d_25 (Conv1D)              (None, 64, 20)       340         add_7[0][0]                      \n",
      "__________________________________________________________________________________________________\n",
      "conv1d_26 (Conv1D)              (None, 64, 20)       1220        conv1d_25[0][0]                  \n",
      "__________________________________________________________________________________________________\n",
      "conv1d_27 (Conv1D)              (None, 64, 20)       1220        conv1d_25[0][0]                  \n",
      "__________________________________________________________________________________________________\n",
      "multiply_8 (Multiply)           (None, 64, 20)       0           conv1d_26[0][0]                  \n",
      "                                                                 conv1d_27[0][0]                  \n",
      "__________________________________________________________________________________________________\n",
      "conv1d_28 (Conv1D)              (None, 64, 20)       420         multiply_8[0][0]                 \n",
      "__________________________________________________________________________________________________\n",
      "add_8 (Add)                     (None, 64, 20)       0           conv1d_25[0][0]                  \n",
      "                                                                 conv1d_28[0][0]                  \n",
      "__________________________________________________________________________________________________\n",
      "dense_1 (Dense)                 (None, 64, 64)       1344        add_8[0][0]                      \n",
      "__________________________________________________________________________________________________\n",
      "permute_1 (Permute)             (None, 64, 64)       0           dense_1[0][0]                    \n",
      "__________________________________________________________________________________________________\n",
      "dense_2 (Dense)                 (None, 64, 64)       4160        permute_1[0][0]                  \n",
      "__________________________________________________________________________________________________\n",
      "permute_2 (Permute)             (None, 64, 64)       0           dense_2[0][0]                    \n",
      "__________________________________________________________________________________________________\n",
      "multiply_9 (Multiply)           (None, 64, 64)       0           dense_1[0][0]                    \n",
      "                                                                 permute_2[0][0]                  \n",
      "__________________________________________________________________________________________________\n",
      "lambda_1 (Lambda)               (None, 64)           0           multiply_9[0][0]                 \n",
      "__________________________________________________________________________________________________\n",
      "nn_input (InputLayer)           (None, 31)           0                                            \n",
      "__________________________________________________________________________________________________\n",
      "concatenate_1 (Concatenate)     (None, 95)           0           lambda_1[0][0]                   \n",
      "                                                                 nn_input[0][0]                   \n",
      "__________________________________________________________________________________________________\n",
      "dense_3 (Dense)                 (None, 512)          49152       concatenate_1[0][0]              \n",
      "__________________________________________________________________________________________________\n",
      "batch_normalization_1 (BatchNor (None, 512)          2048        dense_3[0][0]                    \n",
      "__________________________________________________________________________________________________\n",
      "dense_4 (Dense)                 (None, 128)          65664       batch_normalization_1[0][0]      \n",
      "__________________________________________________________________________________________________\n",
      "batch_normalization_2 (BatchNor (None, 128)          512         dense_4[0][0]                    \n",
      "__________________________________________________________________________________________________\n",
      "dense_5 (Dense)                 (None, 1)            129         batch_normalization_2[0][0]      \n",
      "==================================================================================================\n",
      "Total params: 133,773\n",
      "Trainable params: 132,493\n",
      "Non-trainable params: 1,280\n",
      "__________________________________________________________________________________________________\n"
     ]
    }
   ],
   "source": [
    "model.summary()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [],
   "source": [
    "#输出train&valid的auc\n",
    "def auroc(y_true, y_pred):\n",
    "    return tf.py_func(roc_auc_score, (y_true, y_pred), tf.double)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {},
   "outputs": [],
   "source": [
    "def lrs(epoch):\n",
    "    if epoch<10:\n",
    "        lr = learning_rate\n",
    "    elif epoch<20:\n",
    "        lr = learning_rate/10\n",
    "    elif epoch<40:\n",
    "        lr = learning_rate/100\n",
    "    elif epoch<70:\n",
    "        lr = learning_rate/500\n",
    "    else:\n",
    "        lr = learning_rate/1000\n",
    "    return lr\n",
    "\n",
    "lr_schedule = LearningRateScheduler(lrs)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "WARNING:tensorflow:From <ipython-input-21-0345cd89dcf8>:2: py_func (from tensorflow.python.ops.script_ops) is deprecated and will be removed in a future version.\n",
      "Instructions for updating:\n",
      "tf.py_func is deprecated in TF V2. Instead, use\n",
      "    tf.py_function, which takes a python function which manipulates tf eager\n",
      "    tensors instead of numpy arrays. It's easy to convert a tf eager tensor to\n",
      "    an ndarray (just call tensor.numpy()) but having access to eager tensors\n",
      "    means `tf.py_function`s can use accelerators such as GPUs as well as\n",
      "    being differentiable using a gradient tape.\n",
      "    \n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "WARNING:tensorflow:From <ipython-input-21-0345cd89dcf8>:2: py_func (from tensorflow.python.ops.script_ops) is deprecated and will be removed in a future version.\n",
      "Instructions for updating:\n",
      "tf.py_func is deprecated in TF V2. Instead, use\n",
      "    tf.py_function, which takes a python function which manipulates tf eager\n",
      "    tensors instead of numpy arrays. It's easy to convert a tf eager tensor to\n",
      "    an ndarray (just call tensor.numpy()) but having access to eager tensors\n",
      "    means `tf.py_function`s can use accelerators such as GPUs as well as\n",
      "    being differentiable using a gradient tape.\n",
      "    \n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[INFO] training model...\n",
      "WARNING:tensorflow:From C:\\Users\\Ksama\\AppData\\Local\\conda\\conda\\envs\\tensorflow\\lib\\site-packages\\tensorflow\\python\\ops\\math_ops.py:3066: to_int32 (from tensorflow.python.ops.math_ops) is deprecated and will be removed in a future version.\n",
      "Instructions for updating:\n",
      "Use tf.cast instead.\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "WARNING:tensorflow:From C:\\Users\\Ksama\\AppData\\Local\\conda\\conda\\envs\\tensorflow\\lib\\site-packages\\tensorflow\\python\\ops\\math_ops.py:3066: to_int32 (from tensorflow.python.ops.math_ops) is deprecated and will be removed in a future version.\n",
      "Instructions for updating:\n",
      "Use tf.cast instead.\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "WARNING:tensorflow:From C:\\Users\\Ksama\\AppData\\Local\\conda\\conda\\envs\\tensorflow\\lib\\site-packages\\tensorflow\\python\\ops\\math_grad.py:102: div (from tensorflow.python.ops.math_ops) is deprecated and will be removed in a future version.\n",
      "Instructions for updating:\n",
      "Deprecated in favor of operator or tf.math.divide.\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "WARNING:tensorflow:From C:\\Users\\Ksama\\AppData\\Local\\conda\\conda\\envs\\tensorflow\\lib\\site-packages\\tensorflow\\python\\ops\\math_grad.py:102: div (from tensorflow.python.ops.math_ops) is deprecated and will be removed in a future version.\n",
      "Instructions for updating:\n",
      "Deprecated in favor of operator or tf.math.divide.\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Train on 6507058 samples, validate on 2788740 samples\n",
      "Epoch 1/200\n",
      "6507058/6507058 [==============================] - 226s 35us/step - loss: 0.7023 - acc: 0.9628 - auroc: 0.9874 - val_loss: 0.1550 - val_acc: 0.9830 - val_auroc: 0.9983\n",
      "Epoch 2/200\n",
      "6507058/6507058 [==============================] - 202s 31us/step - loss: 0.0797 - acc: 0.9865 - auroc: 0.9989 - val_loss: 0.0508 - val_acc: 0.9882 - val_auroc: 0.9991\n",
      "Epoch 3/200\n",
      "6507058/6507058 [==============================] - 203s 31us/step - loss: 0.0431 - acc: 0.9888 - auroc: 0.9993 - val_loss: 0.0401 - val_acc: 0.9886 - val_auroc: 0.9992\n",
      "Epoch 4/200\n",
      "6507058/6507058 [==============================] - 201s 31us/step - loss: 0.0363 - acc: 0.9894 - auroc: 0.9993 - val_loss: 0.0344 - val_acc: 0.9896 - val_auroc: 0.9994\n",
      "Epoch 5/200\n",
      "6507058/6507058 [==============================] - 198s 30us/step - loss: 0.0333 - acc: 0.9898 - auroc: 0.9994 - val_loss: 0.0310 - val_acc: 0.9903 - val_auroc: 0.9994\n",
      "Epoch 6/200\n",
      "6507058/6507058 [==============================] - 197s 30us/step - loss: 0.0317 - acc: 0.9899 - auroc: 0.9994 - val_loss: 0.0300 - val_acc: 0.9904 - val_auroc: 0.9995\n",
      "Epoch 7/200\n",
      "6507058/6507058 [==============================] - 197s 30us/step - loss: 0.0307 - acc: 0.9901 - auroc: 0.9994 - val_loss: 0.0305 - val_acc: 0.9901 - val_auroc: 0.9994\n",
      "Epoch 8/200\n",
      "6507058/6507058 [==============================] - 197s 30us/step - loss: 0.0301 - acc: 0.9902 - auroc: 0.9994 - val_loss: 0.0310 - val_acc: 0.9897 - val_auroc: 0.9994\n",
      "Epoch 9/200\n",
      "6507058/6507058 [==============================] - 196s 30us/step - loss: 0.0296 - acc: 0.9903 - auroc: 0.9995 - val_loss: 0.0299 - val_acc: 0.9901 - val_auroc: 0.9994\n",
      "Epoch 10/200\n",
      "6507058/6507058 [==============================] - 198s 30us/step - loss: 0.0291 - acc: 0.9904 - auroc: 0.9995 - val_loss: 0.0279 - val_acc: 0.9907 - val_auroc: 0.9995\n",
      "Epoch 11/200\n",
      "6507058/6507058 [==============================] - 206s 32us/step - loss: 0.0287 - acc: 0.9905 - auroc: 0.9995 - val_loss: 0.0274 - val_acc: 0.9909 - val_auroc: 0.9995\n",
      "Epoch 12/200\n",
      "6507058/6507058 [==============================] - 207s 32us/step - loss: 0.0285 - acc: 0.9905 - auroc: 0.9995 - val_loss: 0.0281 - val_acc: 0.9907 - val_auroc: 0.9995\n",
      "Epoch 13/200\n",
      "6507058/6507058 [==============================] - 203s 31us/step - loss: 0.0283 - acc: 0.9906 - auroc: 0.9995 - val_loss: 0.0289 - val_acc: 0.9904 - val_auroc: 0.9995\n",
      "Epoch 14/200\n",
      "6507058/6507058 [==============================] - 204s 31us/step - loss: 0.0282 - acc: 0.9906 - auroc: 0.9995 - val_loss: 0.0283 - val_acc: 0.9905 - val_auroc: 0.9995\n",
      "Epoch 15/200\n",
      "6507058/6507058 [==============================] - 204s 31us/step - loss: 0.0279 - acc: 0.9906 - auroc: 0.9995 - val_loss: 0.0273 - val_acc: 0.9907 - val_auroc: 0.9995\n",
      "Epoch 16/200\n",
      "6506496/6507058 [============================>.] - ETA: 0s - loss: 0.0277 - acc: 0.9907 - auroc: 0.9995\n",
      "Epoch 00016: reducing learning rate to 3.1593976927979384e-06.\n",
      "6507058/6507058 [==============================] - 204s 31us/step - loss: 0.0277 - acc: 0.9907 - auroc: 0.9995 - val_loss: 0.0265 - val_acc: 0.9911 - val_auroc: 0.9995\n",
      "Epoch 17/200\n",
      "6507058/6507058 [==============================] - 205s 31us/step - loss: 0.0275 - acc: 0.9908 - auroc: 0.9995 - val_loss: 0.0274 - val_acc: 0.9908 - val_auroc: 0.9995\n",
      "Epoch 18/200\n",
      "6507058/6507058 [==============================] - 205s 31us/step - loss: 0.0275 - acc: 0.9907 - auroc: 0.9995 - val_loss: 0.0293 - val_acc: 0.9902 - val_auroc: 0.9994\n",
      "Epoch 19/200\n",
      "6507058/6507058 [==============================] - 204s 31us/step - loss: 0.0274 - acc: 0.9908 - auroc: 0.9995 - val_loss: 0.0281 - val_acc: 0.9906 - val_auroc: 0.9995\n",
      "Epoch 20/200\n",
      "6507058/6507058 [==============================] - 204s 31us/step - loss: 0.0273 - acc: 0.9908 - auroc: 0.9995 - val_loss: 0.0269 - val_acc: 0.9909 - val_auroc: 0.9995\n",
      "Epoch 21/200\n",
      "6506496/6507058 [============================>.] - ETA: 0s - loss: 0.0271 - acc: 0.9908 - auroc: 0.9995\n",
      "Epoch 00021: reducing learning rate to 3.9108433611545476e-07.\n",
      "6507058/6507058 [==============================] - 204s 31us/step - loss: 0.0271 - acc: 0.9908 - auroc: 0.9995 - val_loss: 0.0262 - val_acc: 0.9912 - val_auroc: 0.9996\n",
      "Epoch 22/200\n",
      "6507058/6507058 [==============================] - 203s 31us/step - loss: 0.0270 - acc: 0.9909 - auroc: 0.9995 - val_loss: 0.0270 - val_acc: 0.9909 - val_auroc: 0.9995\n",
      "Epoch 23/200\n",
      "6507058/6507058 [==============================] - 207s 32us/step - loss: 0.0269 - acc: 0.9909 - auroc: 0.9995 - val_loss: 0.0271 - val_acc: 0.9908 - val_auroc: 0.9995\n",
      "Epoch 24/200\n",
      "6507058/6507058 [==============================] - 204s 31us/step - loss: 0.0269 - acc: 0.9909 - auroc: 0.9995 - val_loss: 0.0274 - val_acc: 0.9907 - val_auroc: 0.9995\n",
      "Epoch 25/200\n",
      "6507058/6507058 [==============================] - 205s 31us/step - loss: 0.0268 - acc: 0.9909 - auroc: 0.9995 - val_loss: 0.0269 - val_acc: 0.9909 - val_auroc: 0.9995\n",
      "Epoch 26/200\n",
      "6506496/6507058 [============================>.] - ETA: 0s - loss: 0.0266 - acc: 0.9910 - auroc: 0.9995\n",
      "Epoch 00026: reducing learning rate to 2.4372287953156046e-06.\n",
      "6507058/6507058 [==============================] - 204s 31us/step - loss: 0.0266 - acc: 0.9910 - auroc: 0.9995 - val_loss: 0.0259 - val_acc: 0.9912 - val_auroc: 0.9996\n",
      "Epoch 27/200\n",
      "6507058/6507058 [==============================] - 205s 31us/step - loss: 0.0265 - acc: 0.9910 - auroc: 0.9995 - val_loss: 0.0260 - val_acc: 0.9912 - val_auroc: 0.9996\n",
      "Epoch 28/200\n",
      "6507058/6507058 [==============================] - 206s 32us/step - loss: 0.0265 - acc: 0.9910 - auroc: 0.9995 - val_loss: 0.0271 - val_acc: 0.9908 - val_auroc: 0.9995\n",
      "Epoch 29/200\n",
      "6507058/6507058 [==============================] - 206s 32us/step - loss: 0.0265 - acc: 0.9910 - auroc: 0.9995 - val_loss: 0.0281 - val_acc: 0.9905 - val_auroc: 0.9995\n",
      "Epoch 30/200\n",
      "6507058/6507058 [==============================] - 206s 32us/step - loss: 0.0264 - acc: 0.9910 - auroc: 0.9995 - val_loss: 0.0268 - val_acc: 0.9909 - val_auroc: 0.9995\n",
      "Epoch 31/200\n",
      "6506496/6507058 [============================>.] - ETA: 0s - loss: 0.0263 - acc: 0.9911 - auroc: 0.9996\n",
      "Epoch 00031: reducing learning rate to 5.205542220210191e-06.\n",
      "6507058/6507058 [==============================] - 205s 32us/step - loss: 0.0263 - acc: 0.9911 - auroc: 0.9996 - val_loss: 0.0258 - val_acc: 0.9913 - val_auroc: 0.9996\n",
      "Epoch 32/200\n",
      "6507058/6507058 [==============================] - 206s 32us/step - loss: 0.0262 - acc: 0.9911 - auroc: 0.9996 - val_loss: 0.0254 - val_acc: 0.9914 - val_auroc: 0.9996\n",
      "Epoch 33/200\n",
      "6507058/6507058 [==============================] - 206s 32us/step - loss: 0.0261 - acc: 0.9911 - auroc: 0.9996 - val_loss: 0.0265 - val_acc: 0.9910 - val_auroc: 0.9995\n",
      "Epoch 34/200\n",
      "6507058/6507058 [==============================] - 206s 32us/step - loss: 0.0262 - acc: 0.9910 - auroc: 0.9996 - val_loss: 0.0271 - val_acc: 0.9908 - val_auroc: 0.9995\n",
      "Epoch 35/200\n",
      "6507058/6507058 [==============================] - 206s 32us/step - loss: 0.0261 - acc: 0.9911 - auroc: 0.9996 - val_loss: 0.0266 - val_acc: 0.9910 - val_auroc: 0.9995\n",
      "Epoch 36/200\n",
      "6506496/6507058 [============================>.] - ETA: 0s - loss: 0.0260 - acc: 0.9911 - auroc: 0.9996\n",
      "Epoch 00036: reducing learning rate to 7.973855645104777e-06.\n",
      "6507058/6507058 [==============================] - 206s 32us/step - loss: 0.0260 - acc: 0.9911 - auroc: 0.9996 - val_loss: 0.0258 - val_acc: 0.9913 - val_auroc: 0.9996\n",
      "Epoch 37/200\n",
      "6507058/6507058 [==============================] - 206s 32us/step - loss: 0.0259 - acc: 0.9912 - auroc: 0.9996 - val_loss: 0.0252 - val_acc: 0.9914 - val_auroc: 0.9996\n",
      "Epoch 38/200\n",
      "6507058/6507058 [==============================] - 206s 32us/step - loss: 0.0258 - acc: 0.9911 - auroc: 0.9996 - val_loss: 0.0257 - val_acc: 0.9912 - val_auroc: 0.9996\n",
      "Epoch 39/200\n",
      "6507058/6507058 [==============================] - 207s 32us/step - loss: 0.0259 - acc: 0.9912 - auroc: 0.9996 - val_loss: 0.0264 - val_acc: 0.9910 - val_auroc: 0.9995\n",
      "Epoch 40/200\n",
      "6507058/6507058 [==============================] - 205s 32us/step - loss: 0.0259 - acc: 0.9911 - auroc: 0.9996 - val_loss: 0.0268 - val_acc: 0.9908 - val_auroc: 0.9995\n",
      "Epoch 41/200\n",
      "6506496/6507058 [============================>.] - ETA: 0s - loss: 0.0258 - acc: 0.9912 - auroc: 0.9996\n",
      "Epoch 00041: reducing learning rate to 1.0742168524302542e-05.\n",
      "6507058/6507058 [==============================] - 203s 31us/step - loss: 0.0258 - acc: 0.9912 - auroc: 0.9996 - val_loss: 0.0259 - val_acc: 0.9912 - val_auroc: 0.9996\n",
      "Epoch 42/200\n",
      "6507058/6507058 [==============================] - 203s 31us/step - loss: 0.0257 - acc: 0.9912 - auroc: 0.9996 - val_loss: 0.0250 - val_acc: 0.9915 - val_auroc: 0.9996\n",
      "Epoch 43/200\n",
      "6507058/6507058 [==============================] - 205s 32us/step - loss: 0.0256 - acc: 0.9913 - auroc: 0.9996 - val_loss: 0.0254 - val_acc: 0.9913 - val_auroc: 0.9996\n",
      "Epoch 44/200\n",
      "6507058/6507058 [==============================] - 201s 31us/step - loss: 0.0256 - acc: 0.9912 - auroc: 0.9996 - val_loss: 0.0260 - val_acc: 0.9910 - val_auroc: 0.9996\n",
      "Epoch 45/200\n",
      "6507058/6507058 [==============================] - 200s 31us/step - loss: 0.0257 - acc: 0.9912 - auroc: 0.9996 - val_loss: 0.0268 - val_acc: 0.9909 - val_auroc: 0.9995\n",
      "Epoch 46/200\n",
      "6506496/6507058 [============================>.] - ETA: 0s - loss: 0.0255 - acc: 0.9912 - auroc: 0.9996\n",
      "Epoch 00046: reducing learning rate to 1.3510481949197128e-05.\n",
      "6507058/6507058 [==============================] - 201s 31us/step - loss: 0.0255 - acc: 0.9912 - auroc: 0.9996 - val_loss: 0.0256 - val_acc: 0.9912 - val_auroc: 0.9996\n",
      "Epoch 47/200\n",
      "6507058/6507058 [==============================] - 199s 31us/step - loss: 0.0255 - acc: 0.9913 - auroc: 0.9996 - val_loss: 0.0250 - val_acc: 0.9915 - val_auroc: 0.9996\n",
      "Epoch 48/200\n",
      "6507058/6507058 [==============================] - 205s 31us/step - loss: 0.0254 - acc: 0.9913 - auroc: 0.9996 - val_loss: 0.0250 - val_acc: 0.9915 - val_auroc: 0.9996\n",
      "Epoch 49/200\n",
      "6507058/6507058 [==============================] - 205s 31us/step - loss: 0.0254 - acc: 0.9913 - auroc: 0.9996 - val_loss: 0.0257 - val_acc: 0.9912 - val_auroc: 0.9996\n",
      "Epoch 50/200\n",
      "6507058/6507058 [==============================] - 205s 31us/step - loss: 0.0254 - acc: 0.9913 - auroc: 0.9996 - val_loss: 0.0270 - val_acc: 0.9907 - val_auroc: 0.9995\n",
      "Epoch 51/200\n",
      "6506496/6507058 [============================>.] - ETA: 0s - loss: 0.0254 - acc: 0.9913 - auroc: 0.9996\n",
      "Epoch 00051: reducing learning rate to 1.6278795374091715e-05.\n",
      "6507058/6507058 [==============================] - 205s 31us/step - loss: 0.0254 - acc: 0.9913 - auroc: 0.9996 - val_loss: 0.0258 - val_acc: 0.9911 - val_auroc: 0.9996\n",
      "Epoch 52/200\n",
      "6507058/6507058 [==============================] - 206s 32us/step - loss: 0.0253 - acc: 0.9913 - auroc: 0.9996 - val_loss: 0.0249 - val_acc: 0.9914 - val_auroc: 0.9996\n",
      "Epoch 53/200\n",
      "6507058/6507058 [==============================] - 205s 32us/step - loss: 0.0253 - acc: 0.9913 - auroc: 0.9996 - val_loss: 0.0248 - val_acc: 0.9915 - val_auroc: 0.9996\n",
      "Epoch 54/200\n",
      "6507058/6507058 [==============================] - 206s 32us/step - loss: 0.0252 - acc: 0.9913 - auroc: 0.9996 - val_loss: 0.0251 - val_acc: 0.9914 - val_auroc: 0.9996\n",
      "Epoch 55/200\n",
      "6507058/6507058 [==============================] - 205s 32us/step - loss: 0.0252 - acc: 0.9913 - auroc: 0.9996 - val_loss: 0.0262 - val_acc: 0.9910 - val_auroc: 0.9995\n",
      "Epoch 56/200\n",
      "6506496/6507058 [============================>.] - ETA: 0s - loss: 0.0252 - acc: 0.9913 - auroc: 0.9996\n",
      "Epoch 00056: reducing learning rate to 1.90471087989863e-05.\n",
      "6507058/6507058 [==============================] - 205s 32us/step - loss: 0.0252 - acc: 0.9913 - auroc: 0.9996 - val_loss: 0.0259 - val_acc: 0.9911 - val_auroc: 0.9996\n",
      "Epoch 57/200\n",
      "6507058/6507058 [==============================] - 207s 32us/step - loss: 0.0252 - acc: 0.9913 - auroc: 0.9996 - val_loss: 0.0251 - val_acc: 0.9914 - val_auroc: 0.9996\n",
      "Epoch 58/200\n",
      "6507058/6507058 [==============================] - 204s 31us/step - loss: 0.0251 - acc: 0.9913 - auroc: 0.9996 - val_loss: 0.0246 - val_acc: 0.9916 - val_auroc: 0.9996\n",
      "Epoch 59/200\n",
      "6507058/6507058 [==============================] - 208s 32us/step - loss: 0.0252 - acc: 0.9914 - auroc: 0.9996 - val_loss: 0.0252 - val_acc: 0.9914 - val_auroc: 0.9996\n",
      "Epoch 60/200\n",
      "6507058/6507058 [==============================] - 209s 32us/step - loss: 0.0251 - acc: 0.9913 - auroc: 0.9996 - val_loss: 0.0261 - val_acc: 0.9910 - val_auroc: 0.9996\n",
      "Epoch 61/200\n",
      "6506496/6507058 [============================>.] - ETA: 0s - loss: 0.0251 - acc: 0.9913 - auroc: 0.9996\n",
      "Epoch 00061: reducing learning rate to 2.1815422223880886e-05.\n",
      "6507058/6507058 [==============================] - 205s 32us/step - loss: 0.0251 - acc: 0.9913 - auroc: 0.9996 - val_loss: 0.0262 - val_acc: 0.9910 - val_auroc: 0.9996\n",
      "Epoch 62/200\n",
      "6507058/6507058 [==============================] - 199s 31us/step - loss: 0.0250 - acc: 0.9914 - auroc: 0.9996 - val_loss: 0.0250 - val_acc: 0.9914 - val_auroc: 0.9996\n",
      "Epoch 63/200\n",
      "6507058/6507058 [==============================] - 197s 30us/step - loss: 0.0249 - acc: 0.9914 - auroc: 0.9996 - val_loss: 0.0245 - val_acc: 0.9916 - val_auroc: 0.9996\n",
      "Epoch 64/200\n",
      "6507058/6507058 [==============================] - 197s 30us/step - loss: 0.0249 - acc: 0.9914 - auroc: 0.9996 - val_loss: 0.0249 - val_acc: 0.9914 - val_auroc: 0.9996\n",
      "Epoch 65/200\n",
      "6507058/6507058 [==============================] - 197s 30us/step - loss: 0.0250 - acc: 0.9914 - auroc: 0.9996 - val_loss: 0.0260 - val_acc: 0.9911 - val_auroc: 0.9995\n",
      "Epoch 66/200\n",
      "6506496/6507058 [============================>.] - ETA: 0s - loss: 0.0250 - acc: 0.9914 - auroc: 0.9996\n",
      "Epoch 00066: reducing learning rate to 2.4583735648775473e-05.\n",
      "6507058/6507058 [==============================] - 197s 30us/step - loss: 0.0250 - acc: 0.9914 - auroc: 0.9996 - val_loss: 0.0258 - val_acc: 0.9911 - val_auroc: 0.9996\n",
      "Epoch 67/200\n",
      "6507058/6507058 [==============================] - 197s 30us/step - loss: 0.0250 - acc: 0.9914 - auroc: 0.9996 - val_loss: 0.0252 - val_acc: 0.9913 - val_auroc: 0.9996\n",
      "Epoch 68/200\n",
      "6507058/6507058 [==============================] - 197s 30us/step - loss: 0.0249 - acc: 0.9914 - auroc: 0.9996 - val_loss: 0.0245 - val_acc: 0.9915 - val_auroc: 0.9996\n",
      "Epoch 69/200\n",
      "6507058/6507058 [==============================] - 198s 30us/step - loss: 0.0248 - acc: 0.9914 - auroc: 0.9996 - val_loss: 0.0246 - val_acc: 0.9916 - val_auroc: 0.9996\n",
      "Epoch 70/200\n",
      "6507058/6507058 [==============================] - 205s 31us/step - loss: 0.0248 - acc: 0.9914 - auroc: 0.9996 - val_loss: 0.0250 - val_acc: 0.9914 - val_auroc: 0.9996\n",
      "Epoch 71/200\n",
      "6507058/6507058 [==============================] - 205s 31us/step - loss: 0.0249 - acc: 0.9914 - auroc: 0.9996 - val_loss: 0.0257 - val_acc: 0.9912 - val_auroc: 0.9996\n",
      "Epoch 72/200\n",
      "6507058/6507058 [==============================] - 205s 31us/step - loss: 0.0249 - acc: 0.9914 - auroc: 0.9996 - val_loss: 0.0250 - val_acc: 0.9914 - val_auroc: 0.9996\n",
      "Epoch 73/200\n",
      "6507058/6507058 [==============================] - 205s 31us/step - loss: 0.0248 - acc: 0.9914 - auroc: 0.9996 - val_loss: 0.0247 - val_acc: 0.9915 - val_auroc: 0.9996\n",
      "Epoch 74/200\n",
      "6506496/6507058 [============================>.] - ETA: 0s - loss: 0.0247 - acc: 0.9915 - auroc: 0.9996\n",
      "Epoch 00074: reducing learning rate to 7.010963781794998e-06.\n",
      "6507058/6507058 [==============================] - 204s 31us/step - loss: 0.0247 - acc: 0.9915 - auroc: 0.9996 - val_loss: 0.0244 - val_acc: 0.9916 - val_auroc: 0.9996\n",
      "Epoch 75/200\n",
      "6507058/6507058 [==============================] - 205s 31us/step - loss: 0.0247 - acc: 0.9915 - auroc: 0.9996 - val_loss: 0.0249 - val_acc: 0.9914 - val_auroc: 0.9996\n",
      "Epoch 76/200\n",
      "6507058/6507058 [==============================] - 205s 31us/step - loss: 0.0248 - acc: 0.9914 - auroc: 0.9996 - val_loss: 0.0267 - val_acc: 0.9907 - val_auroc: 0.9995\n",
      "Epoch 77/200\n",
      "6507058/6507058 [==============================] - 205s 31us/step - loss: 0.0248 - acc: 0.9914 - auroc: 0.9996 - val_loss: 0.0260 - val_acc: 0.9911 - val_auroc: 0.9996\n",
      "Epoch 78/200\n",
      "6507058/6507058 [==============================] - 205s 31us/step - loss: 0.0247 - acc: 0.9915 - auroc: 0.9996 - val_loss: 0.0246 - val_acc: 0.9915 - val_auroc: 0.9996\n",
      "Epoch 79/200\n",
      "6506496/6507058 [============================>.] - ETA: 0s - loss: 0.0246 - acc: 0.9915 - auroc: 0.9996\n",
      "Epoch 00079: reducing learning rate to 4.242650629748823e-06.\n",
      "6507058/6507058 [==============================] - 205s 31us/step - loss: 0.0246 - acc: 0.9915 - auroc: 0.9996 - val_loss: 0.0243 - val_acc: 0.9916 - val_auroc: 0.9996\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 80/200\n",
      "6507058/6507058 [==============================] - 203s 31us/step - loss: 0.0246 - acc: 0.9915 - auroc: 0.9996 - val_loss: 0.0247 - val_acc: 0.9915 - val_auroc: 0.9996\n",
      "Epoch 81/200\n",
      "6507058/6507058 [==============================] - 203s 31us/step - loss: 0.0246 - acc: 0.9914 - auroc: 0.9996 - val_loss: 0.0254 - val_acc: 0.9913 - val_auroc: 0.9996\n",
      "Epoch 82/200\n",
      "6507058/6507058 [==============================] - 203s 31us/step - loss: 0.0246 - acc: 0.9915 - auroc: 0.9996 - val_loss: 0.0258 - val_acc: 0.9911 - val_auroc: 0.9996\n",
      "Epoch 83/200\n",
      "6507058/6507058 [==============================] - 203s 31us/step - loss: 0.0246 - acc: 0.9915 - auroc: 0.9996 - val_loss: 0.0248 - val_acc: 0.9914 - val_auroc: 0.9996\n",
      "Epoch 84/200\n",
      "6506496/6507058 [============================>.] - ETA: 0s - loss: 0.0246 - acc: 0.9915 - auroc: 0.9996\n",
      "Epoch 00084: reducing learning rate to 1.4743373412784422e-06.\n",
      "6507058/6507058 [==============================] - 202s 31us/step - loss: 0.0246 - acc: 0.9915 - auroc: 0.9996 - val_loss: 0.0243 - val_acc: 0.9916 - val_auroc: 0.9996\n",
      "Epoch 85/200\n",
      "6507058/6507058 [==============================] - 203s 31us/step - loss: 0.0246 - acc: 0.9915 - auroc: 0.9996 - val_loss: 0.0244 - val_acc: 0.9915 - val_auroc: 0.9996\n",
      "Epoch 86/200\n",
      "6507058/6507058 [==============================] - 203s 31us/step - loss: 0.0246 - acc: 0.9915 - auroc: 0.9996 - val_loss: 0.0253 - val_acc: 0.9912 - val_auroc: 0.9996\n",
      "Epoch 87/200\n",
      "6507058/6507058 [==============================] - 203s 31us/step - loss: 0.0246 - acc: 0.9915 - auroc: 0.9996 - val_loss: 0.0256 - val_acc: 0.9911 - val_auroc: 0.9996\n",
      "Epoch 88/200\n",
      "6507058/6507058 [==============================] - 203s 31us/step - loss: 0.0245 - acc: 0.9915 - auroc: 0.9996 - val_loss: 0.0248 - val_acc: 0.9915 - val_auroc: 0.9996\n",
      "Epoch 89/200\n",
      "6506496/6507058 [============================>.] - ETA: 0s - loss: 0.0245 - acc: 0.9915 - auroc: 0.9996\n",
      "Epoch 00089: reducing learning rate to 1.3539758583647199e-06.\n",
      "6507058/6507058 [==============================] - 203s 31us/step - loss: 0.0245 - acc: 0.9915 - auroc: 0.9996 - val_loss: 0.0243 - val_acc: 0.9916 - val_auroc: 0.9996\n",
      "Epoch 90/200\n",
      "6507058/6507058 [==============================] - 203s 31us/step - loss: 0.0244 - acc: 0.9915 - auroc: 0.9996 - val_loss: 0.0252 - val_acc: 0.9914 - val_auroc: 0.9996\n",
      "Epoch 91/200\n",
      "6507058/6507058 [==============================] - 203s 31us/step - loss: 0.0245 - acc: 0.9915 - auroc: 0.9996 - val_loss: 0.0250 - val_acc: 0.9913 - val_auroc: 0.9996\n",
      "Epoch 92/200\n",
      "6507058/6507058 [==============================] - 203s 31us/step - loss: 0.0245 - acc: 0.9915 - auroc: 0.9996 - val_loss: 0.0259 - val_acc: 0.9910 - val_auroc: 0.9996\n",
      "Epoch 93/200\n",
      "6507058/6507058 [==============================] - 203s 31us/step - loss: 0.0244 - acc: 0.9915 - auroc: 0.9996 - val_loss: 0.0254 - val_acc: 0.9912 - val_auroc: 0.9996\n",
      "Epoch 94/200\n",
      "6506496/6507058 [============================>.] - ETA: 0s - loss: 0.0244 - acc: 0.9915 - auroc: 0.9996\n",
      "Epoch 00094: reducing learning rate to 4.122289283259306e-06.\n",
      "6507058/6507058 [==============================] - 203s 31us/step - loss: 0.0244 - acc: 0.9915 - auroc: 0.9996 - val_loss: 0.0244 - val_acc: 0.9916 - val_auroc: 0.9996\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "<keras.callbacks.History at 0x16281badac8>"
      ]
     },
     "execution_count": 25,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "#由于机器配置原因导致训练时间过久，故没有做cv，加上cv应该还会有提升\n",
    "learning_rate = 0.0006\n",
    "n_epoch=200\n",
    "batch_size = 2048\n",
    "cb_clr = CyclicLR(base_lr=1e-7, max_lr = 1e-4, step_size= int(1.0*(test_data_sideinfo.shape[0])/(batch_size*4)) , mode='exp_range', gamma=1.0, scale_fn=None, scale_mode='cycle')\n",
    "plateau = ReduceLROnPlateau(monitor=\"val_auroc\", verbose=1, mode='max', factor=0.3, patience=5)\n",
    "early_stopping = EarlyStopping(monitor='val_auroc', patience=10, mode='max')\n",
    "opt = Adam(lr=learning_rate)\n",
    "model.compile(\n",
    "              loss='binary_crossentropy',\n",
    "              optimizer=opt,\n",
    "             metrics=['accuracy',auroc])\n",
    "\n",
    "# train the model\n",
    "print(\"[INFO] training model...\")\n",
    "model.fit(\n",
    "    [X_train_seq,X_train_side], label_train,\n",
    "    validation_data=([X_validate_seq,X_validate_side], label_validate),\n",
    "    callbacks=[early_stopping,lr_schedule,cb_clr,plateau],shuffle=True,\n",
    "    epochs=n_epoch, batch_size=batch_size)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "AUC: 0.9996128383079947\n"
     ]
    }
   ],
   "source": [
    "pred_valid = model.predict([X_validate_seq,X_validate_side],batch_size=batch_size)\n",
    "pred = model.predict([test_data_seq,test_data_sideinfo],batch_size=batch_size)\n",
    "print(\"AUC: {}\".format(roc_auc_score(label_validate,pred_valid)))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 70,
   "metadata": {},
   "outputs": [],
   "source": [
    "#保存预测概率用于融合\n",
    "tt = pd.read_csv('../data/test.csv')\n",
    "sub = pd.DataFrame()\n",
    "tt['flag_pre'] =pred\n",
    "tt.loc[tt['t']>1850,'flag_pre']=1\n",
    "tt.loc[tt['t']<-900,'flag_pre']=0\n",
    "tt.loc[tt['q']<0,'flag_pre']=1\n",
    "sub['hit_id']=tt['hit_id']\n",
    "sub['flag_pred'] = tt['flag_pre']\n",
    "sub['event_id'] = tt['event_id']\n",
    "sub.to_csv('../result/wavenet_prob.csv',index=False)"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.5.6"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}