single_tower_transformer_bilstm.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/root/anaconda3/envs/tianchi_env/lib/python3.6/site-packages/sklearn/externals/joblib/__init__.py:15: FutureWarning: sklearn.externals.joblib is deprecated in 0.21 and will be removed in 0.23. Please import this functionality directly from joblib, which can be installed with: pip install joblib. If this warning is raised when loading pickled models, you may need to re-serialize those models with scikit-learn 0.21+.\n",
      "  warnings.warn(msg, category=FutureWarning)\n",
      "/root/anaconda3/envs/tianchi_env/lib/python3.6/site-packages/swifter/swifter.py:14: NumbaDeprecationWarning: An import was requested from a module that has moved location.\n",
      "Import requested from: 'numba.errors', please update to use 'numba.core.errors' or pin to Numba version 0.48.0. This alias will not be present in Numba version 0.50.0.\n",
      "  from numba.errors import TypingError\n",
      "/root/anaconda3/envs/tianchi_env/lib/python3.6/site-packages/swifter/swifter.py:14: NumbaDeprecationWarning: An import was requested from a module that has moved location.\n",
      "Import of 'TypingError' requested from: 'numba.errors', please update to use 'numba.core.errors' or pin to Numba version 0.48.0. This alias will not be present in Numba version 0.50.0.\n",
      "  from numba.errors import TypingError\n"
     ]
    }
   ],
   "source": [
    "from sklearn.externals import joblib\n",
    "import gc\n",
    "import torch.nn as nn\n",
    "import torch.nn.functional as F\n",
    "from torchvision import models\n",
    "import torch\n",
    "import numpy as np\n",
    "import swifter\n",
    "import torch.optim as optim\n",
    "import pandas as pd\n",
    "import time"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# CONFIGURATION"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "config={\n",
    "    'lstm_dropout':0.0,\n",
    "    'char_embedding_size':600,\n",
    "    'hidden_dims':384,\n",
    "    'rnn_layers':1,\n",
    "    'conv_in_dim':600,\n",
    "    'conv_out_dim':128,\n",
    "    'conv_kernel_size':3,\n",
    "    'img_vec_dim':2053,\n",
    "    'transformer_dim':768,\n",
    "    'img_transformer_dim':2048+5,\n",
    "    'encoder_layers':6,\n",
    "    'img_encoder_layers':3,\n",
    "    'heads':8,\n",
    "    'ffn_dim':2048,\n",
    "    'transformer_dropout':0.0\n",
    "}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "MAX_BOX = 10\n",
    "MAX_QUERY_CHAR = 12"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# MODEL"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "class Model(nn.Module):\n",
    "    def __init__(self, config):\n",
    "        super(Model, self).__init__()\n",
    "        self.keep_dropout = config['lstm_dropout']\n",
    "        self.char_embedding_size = config['char_embedding_size']\n",
    "        self.hidden_dims = config['hidden_dims']\n",
    "        self.rnn_layers = config['rnn_layers']\n",
    "\n",
    "        self.conv_in_dim = config['conv_in_dim']\n",
    "        self.conv_out_dim = config['conv_out_dim']\n",
    "        self.conv_kernel_size = config['conv_kernel_size']\n",
    "\n",
    "        self.dense_input_dim = config['img_vec_dim']\n",
    "        self.dense_output_dim = config['model_dim']\n",
    "        \n",
    "        encoder_layer = nn.TransformerEncoderLayer(d_model=config['transformer_dim'], \n",
    "                                                   nhead=config['heads'],\n",
    "                                                   dim_feedforward=config['ffn_dim'],\n",
    "                                                   dropout =0.0\n",
    "                                                  )\n",
    "        \n",
    "        img_encoder_layer = nn.TransformerEncoderLayer(d_model=config['img_transformer_dim'], \n",
    "                                                   nhead=config['heads'],\n",
    "                                                   dim_feedforward=config['ffn_dim'],\n",
    "                                                   dropout =0.0\n",
    "                                                  )\n",
    "        \n",
    "        self.query_img_transformer_encoder = nn.TransformerEncoder(encoder_layer, num_layers=config['encoder_layers'])\n",
    "        self.img_transformer_encoder = nn.TransformerEncoder(img_encoder_layer, num_layers=config['img_encoder_layers'])\n",
    "\n",
    "        self.lstm_net_query = nn.LSTM(self.char_embedding_size, self.hidden_dims,\n",
    "                                      num_layers=self.rnn_layers, dropout=self.keep_dropout,\n",
    "                                      bidirectional=True, batch_first=True)\n",
    "        self.conv1d = nn.Conv1d(in_channels=self.conv_in_dim, out_channels=self.conv_out_dim,\n",
    "                                kernel_size=self.conv_kernel_size)\n",
    "#         self.convs = nn.Sequential(self.conv1d, F.avg_pool1d(kernel_size=3, stride=1),nn.ReLU())\n",
    "        self.linearLayer = nn.Linear(self.dense_input_dim, self.dense_output_dim)\n",
    "\n",
    "        self.mlpLayer = nn.Sequential(nn.BatchNorm1d(768),nn.ReLU(), nn.Linear(768, 256), \n",
    "                                      nn.BatchNorm1d(256),nn.ReLU(), nn.Linear(256, 64), \n",
    "                                      nn.BatchNorm1d(64), nn.ReLU(),\n",
    "                                      nn.Linear(64, 1))\n",
    "\n",
    "    def forward(self, query_vec, label_text_vec, query_bert_vec, img_vec, pos_vec, padding_mask,img_padding_mask,lstm_seq_length):\n",
    "        pack = nn.utils.rnn.pack_padded_sequence(query_vec, lstm_seq_length, batch_first=True,enforce_sorted=False)\n",
    "        lstm_output, (h, c) = self.lstm_net_query(pack)\n",
    "        h = h.permute(1, 0, 2)\n",
    "        h = torch.cat((h[:,0,:],h[:,1,:]),dim=1)\n",
    "        query_bert_vec = query_bert_vec + h.view(query_bert_vec.shape[0], query_bert_vec.shape[1], query_bert_vec.shape[2])\n",
    "#         print('query_bert_vec size:', query_bert_vec.shape)\n",
    "        label_text_vec = label_text_vec.permute(0, 2, 1)\n",
    "#         print('label_text_vec size:', label_text_vec.shape)\n",
    "#         conv_output = self.convs(label_text_vec)\n",
    "        conv_output = self.conv1d(label_text_vec)\n",
    "        conv_output = F.avg_pool1d(conv_output, kernel_size=3, stride=1)\n",
    "#         print(conv_output.shape)\n",
    "        conv_output = conv_output.view(conv_output.shape[0], 1, conv_output.shape[1]*conv_output.shape[2])\n",
    "#         print('conv_output size:', conv_output.shape)\n",
    "#         print(img_vec.shape, pos_vec.shape)\n",
    "        img_vec = torch.cat((img_vec, pos_vec), dim=2)\n",
    "#         print(img_vec.shape)\n",
    "        img_outout = self.img_transformer_encoder(img_vec, src_key_padding_mask=img_padding_mask)\n",
    "\n",
    "        img_output = nn.functional.relu(self.linearLayer(img_vec))\n",
    "#         print('img_output size:', img_output.shape)\n",
    "#         print('query_bert_vec size:', query_bert_vec.shape)\n",
    "#         print('conv_output size:', conv_output.shape)\n",
    "        output = torch.cat((query_bert_vec, img_output), dim=1)\n",
    "#         print('output size:', output.shape)\n",
    "        output = output.permute(1, 0 ,2)\n",
    "        output = self.transformer_encoder(output, src_key_padding_mask=padding_mask).permute(1, 0 ,2)\n",
    "        output = output[:,0,:]\n",
    "        output = self.mlpLayer(output)\n",
    "#         print('output size:', output.shape)\n",
    "        return output\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "# loss function\n",
    "# loss_f = nn.BCEWithLogitsLoss()\n",
    "sigmoid=nn.Sigmoid()\n",
    "def loss_f(predict, label):\n",
    "    predict=sigmoid(predict)\n",
    "    return (-label*torch.log(predict)-(1-label)*torch.log((1-predict)*0.6)).mean()\n",
    "\n",
    "def calc_loss(predict, label):\n",
    "    loss = loss_f(predict.squeeze(dim=-1), label)\n",
    "    return loss"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "# generate neg samples\n",
    "from sklearn.utils import shuffle\n",
    "\n",
    "# def generate_negative_sample(train):\n",
    "#     train.reset_index(drop=True, inplace=True)\n",
    "#     df1 = shuffle(train, random_state=2020)\n",
    "#     df1.reset_index(drop=True, inplace=True)\n",
    "#     df1['old_query_id'] = train['query_id']\n",
    "#     df1['old_product_id'] = train['product_id']\n",
    "#     df1['query_albert_embedding'] = train['query_albert_embedding']\n",
    "#     df1['query_embedding'] = train['query_embedding']\n",
    "#     df['num_query_char'] = train['num_query_char']\n",
    "\n",
    "#     df1['label'] = df1.apply(lambda x: False if x['query_id']!=x['old_query_id']\n",
    "#                                                         and x['product_id']!=x['old_product_id'] else True, axis=1)\n",
    "\n",
    "#     output = df1[df1['label']==False]\n",
    "#     output['label']=0\n",
    "#     output = output[train.columns]\n",
    "#     print('negative sample number:', output.shape[0])\n",
    "#     del df1\n",
    "#     gc.collect()\n",
    "#     return output\n",
    "\n",
    "def generate_negative_sample(train, num):\n",
    "    train.reset_index(drop=True, inplace=True)\n",
    "    output=pd.DataFrame()\n",
    "    for i in range(0, num):\n",
    "        df = shuffle(train, random_state=2020+i)\n",
    "        df.reset_index(drop=True, inplace=True)\n",
    "        df['old_query_id'] = train['query_id']\n",
    "        df['old_product_id'] = train['product_id']\n",
    "        df['query_albert_embedding'] = train['query_albert_embedding']\n",
    "        df['query_embedding'] = train['query_embedding']\n",
    "        df['num_query_char'] = train['num_query_char']\n",
    "\n",
    "        df['label'] = df.apply(lambda x: False if x['query_id']!=x['old_query_id']\n",
    "                                                            and x['product_id']!=x['old_product_id'] else True, axis=1)\n",
    "\n",
    "        df = df[df['label']==False]\n",
    "        df['label']=0\n",
    "        df = df[train.columns]\n",
    "        \n",
    "        output = pd.concat((output, df), axis=0)\n",
    "    \n",
    "    output.reset_index(drop=True, inplace=True)\n",
    "    print('negative sample number:', output.shape[0])\n",
    "    del df\n",
    "    gc.collect()\n",
    "    return output\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# TRAIN DATA"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "# load 100000 samples \n",
    "def load_train_data(train_albert_path, train_data_path):\n",
    "    train_albert_embedded=joblib.load(train_albert_path)\n",
    "    train=joblib.load(train_data_path)\n",
    "    \n",
    "    train.reset_index(drop=True, inplace=True)\n",
    "    train_albert_embedded.reset_index(drop=True, inplace=True)\n",
    "    \n",
    "    # truncate\n",
    "    train['label_embedding']=train.swifter.apply(lambda x: x['label_embedding'][:MAX_BOX],axis=1)\n",
    "    train['pos']=train.swifter.apply(lambda x: x['pos'][:MAX_BOX],axis=1)\n",
    "    train['query_embedding']=train.swifter.apply(lambda x: x['query_embedding'][:MAX_QUERY_CHAR],axis=1)\n",
    "    train['num_box']=train.swifter.apply(lambda x: len(x['pos']),axis=1)\n",
    "    train['num_query_char']=train.swifter.apply(lambda x: len(x['query_embedding']),axis=1)\n",
    "    train['features']=train.swifter.apply(lambda x: [i for i in x['features']],axis=1)\n",
    "    train['features']=train.swifter.apply(lambda x: x['features'][:MAX_BOX],axis=1)\n",
    "    train['query_albert_embedding']=train_albert_embedded[0]\n",
    "    del train_albert_embedded\n",
    "    gc.collect()\n",
    "    return train"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# TRAIN MODEL"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "starting\n"
     ]
    }
   ],
   "source": [
    "import warnings\n",
    "warnings.filterwarnings(\"ignore\")\n",
    "# betas = (0.5, 0.999)\n",
    "lr = 0.0001\n",
    "weight_decay = 0\n",
    "\n",
    "model_ft = Model(config).to(torch.device(\"cuda\"))\n",
    "model_ft.load_state_dict(torch.load('model/0_200000_attention_bilstm_parameter.pkl'))\n",
    "params_to_update = list(model_ft.parameters())\n",
    "# Observe that all parameters are being optimized\n",
    "optimizer = optim.Adam(params_to_update, lr=lr)\n",
    "model_ft.train()\n",
    "print(\"starting\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "epoch: 1\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "ffa38a0ec5ec442ea11e47b7de085849",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, description='Pandas Apply', max=100000.0, style=ProgressStyle(descript…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "171b6e59945c4772814f0f5d7ec5f52b",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, description='Pandas Apply', max=100000.0, style=ProgressStyle(descript…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "779ddc13fb4b4ab1a88d274b379a5bbe",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, description='Pandas Apply', max=100000.0, style=ProgressStyle(descript…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "01cc1462e66a48aeabb0725622db429d",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, description='Pandas Apply', max=100000.0, style=ProgressStyle(descript…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "925371b1fd9f43be80f9d5aac0852882",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, description='Pandas Apply', max=100000.0, style=ProgressStyle(descript…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "7ae43cf65155475a8d9776a55241b03c",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, description='Pandas Apply', max=100000.0, style=ProgressStyle(descript…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "99a7f287b7bb45af8d736974c94df820",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, description='Pandas Apply', max=100000.0, style=ProgressStyle(descript…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "load 100000 train samples----------------------\n",
      "----trained samples: 1000\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "89e052dd301b40bfba608d18d6a4b188",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, description='Pandas Apply', max=1000.0, style=ProgressStyle(descriptio…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "1a8c0ff9c4a24dbc9a2b478dc520ea57",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, description='Pandas Apply', max=1000.0, style=ProgressStyle(descriptio…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "799dc8e6ba2041de921bd12c70433477",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, description='Pandas Apply', max=1000.0, style=ProgressStyle(descriptio…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "73a66cc19abb4198951828e06b13dff8",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, description='Pandas Apply', max=1000.0, style=ProgressStyle(descriptio…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "negative sample number: 1000\n",
      "epoch: 1 trained samples: 2000 ----average loss: 0.19538112916052341\n",
      "epoch: 1 trained samples: 2000 ----average prec: 0.9235\n",
      "----trained samples: 2000\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "48bada6bd4b245ad81be7b12890611bf",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, description='Pandas Apply', max=1000.0, style=ProgressStyle(descriptio…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "cc8133ca289d4281b12584f699064072",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, description='Pandas Apply', max=1000.0, style=ProgressStyle(descriptio…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "fef036fc61904d95a075a1aab803f106",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, description='Pandas Apply', max=1000.0, style=ProgressStyle(descriptio…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "cae93f98b9d8494099e31e901e1bfc11",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, description='Pandas Apply', max=1000.0, style=ProgressStyle(descriptio…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "negative sample number: 1000\n",
      "epoch: 1 trained samples: 3000 ----average loss: 0.20531382225453854\n",
      "epoch: 1 trained samples: 3000 ----average prec: 0.9190000000000002\n",
      "----trained samples: 3000\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "c4cf8168953d4f2ebb2026b57b1fe178",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, description='Pandas Apply', max=1000.0, style=ProgressStyle(descriptio…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "db76522201864c69afbf8ec4606c601c",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, description='Pandas Apply', max=1000.0, style=ProgressStyle(descriptio…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "504e4ca6d23043ebbe8996775261115b",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, description='Pandas Apply', max=1000.0, style=ProgressStyle(descriptio…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "c895c1a9a86b43e391c152f44e5a370b",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, description='Pandas Apply', max=1000.0, style=ProgressStyle(descriptio…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "negative sample number: 1000\n",
      "epoch: 1 trained samples: 4000 ----average loss: 0.20520890317857265\n",
      "epoch: 1 trained samples: 4000 ----average prec: 0.9230000000000002\n",
      "----trained samples: 4000\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "7d749b523c6d45a1bf69335c48befe3f",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, description='Pandas Apply', max=1000.0, style=ProgressStyle(descriptio…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "d87d5219c5234fff93923b01511603a4",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, description='Pandas Apply', max=1000.0, style=ProgressStyle(descriptio…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "858776999a1242fc9c4304c732ee866d",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, description='Pandas Apply', max=1000.0, style=ProgressStyle(descriptio…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "4fceb0ef451e463386f63919109c8099",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, description='Pandas Apply', max=1000.0, style=ProgressStyle(descriptio…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "negative sample number: 999\n",
      "epoch: 1 trained samples: 5000 ----average loss: 0.19212827272713184\n",
      "epoch: 1 trained samples: 5000 ----average prec: 0.9245000000000001\n",
      "----trained samples: 5000\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "f8ecab62d0d5460087e6c662eba61bfd",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, description='Pandas Apply', max=1000.0, style=ProgressStyle(descriptio…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "85ae3a65c3bc43e5987cc93faf299995",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, description='Pandas Apply', max=1000.0, style=ProgressStyle(descriptio…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "a41a244c0ba84b9190742725efb84e8b",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, description='Pandas Apply', max=1000.0, style=ProgressStyle(descriptio…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "cf800f8d8ee345ce9bde2b560dce7561",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, description='Pandas Apply', max=1000.0, style=ProgressStyle(descriptio…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "negative sample number: 999\n",
      "epoch: 1 trained samples: 6000 ----average loss: 0.19888261891901493\n",
      "epoch: 1 trained samples: 6000 ----average prec: 0.9225\n",
      "----trained samples: 6000\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "169bd26832014f69a2498e1d9bf0ea63",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, description='Pandas Apply', max=1000.0, style=ProgressStyle(descriptio…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "aa495e528a61479ca3123bcb47e7f913",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, description='Pandas Apply', max=1000.0, style=ProgressStyle(descriptio…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "14227ae95b464715a616162d5ca88406",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, description='Pandas Apply', max=1000.0, style=ProgressStyle(descriptio…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "87be0f5f53e74adcb5b151185a68ecb1",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, description='Pandas Apply', max=1000.0, style=ProgressStyle(descriptio…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "negative sample number: 999\n",
      "epoch: 1 trained samples: 7000 ----average loss: 0.18504626117646694\n",
      "epoch: 1 trained samples: 7000 ----average prec: 0.9314999999999999\n",
      "----trained samples: 7000\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "4106bb55508b43d4892c097b59e6ed76",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, description='Pandas Apply', max=1000.0, style=ProgressStyle(descriptio…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "8c8b8d923b2040f888e73762cc394ce2",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, description='Pandas Apply', max=1000.0, style=ProgressStyle(descriptio…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "3c1de2d2356647628d1cd5ecf5556cef",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, description='Pandas Apply', max=1000.0, style=ProgressStyle(descriptio…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "afed3807dcda4e548e618846c0af7214",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, description='Pandas Apply', max=1000.0, style=ProgressStyle(descriptio…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "negative sample number: 999\n",
      "epoch: 1 trained samples: 8000 ----average loss: 0.18001113273203373\n",
      "epoch: 1 trained samples: 8000 ----average prec: 0.9315000000000001\n",
      "----trained samples: 8000\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "527c0a5cfb994e489d4408ab6df9d673",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, description='Pandas Apply', max=1000.0, style=ProgressStyle(descriptio…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "6b5dc9276bf540918f0c20e5b49407a1",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, description='Pandas Apply', max=1000.0, style=ProgressStyle(descriptio…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "c7576c98d1074c0aa5e2088b8f5b48f1",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, description='Pandas Apply', max=1000.0, style=ProgressStyle(descriptio…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "45915b64e1224b6eb95b1065a77c9d6b",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, description='Pandas Apply', max=1000.0, style=ProgressStyle(descriptio…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "negative sample number: 997\n",
      "epoch: 1 trained samples: 9000 ----average loss: 0.2091795727610588\n",
      "epoch: 1 trained samples: 9000 ----average prec: 0.919\n",
      "----trained samples: 9000\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "4fd3d7ac587d416a9b7fd28f38dd7a0f",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, description='Pandas Apply', max=1000.0, style=ProgressStyle(descriptio…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "cccce2812a784315973f104fcdfc477a",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, description='Pandas Apply', max=1000.0, style=ProgressStyle(descriptio…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "4950af64b0f64ada9c1f989d1d9f569f",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, description='Pandas Apply', max=1000.0, style=ProgressStyle(descriptio…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "88e886da2ea747d0ae0a7ce727ec7533",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, description='Pandas Apply', max=1000.0, style=ProgressStyle(descriptio…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "negative sample number: 999\n",
      "epoch: 1 trained samples: 10000 ----average loss: 0.17931722290813923\n",
      "epoch: 1 trained samples: 10000 ----average prec: 0.9339999999999999\n",
      "----trained samples: 10000\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "e8bf291480794f878d86b9b0349c04ca",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, description='Pandas Apply', max=1000.0, style=ProgressStyle(descriptio…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "566cca109259470696dc616303abacec",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, description='Pandas Apply', max=1000.0, style=ProgressStyle(descriptio…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "c1e929c193f344c3a3d8683799e09727",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, description='Pandas Apply', max=1000.0, style=ProgressStyle(descriptio…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "7d111d470c964a0dbdb2d7486ae4454d",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, description='Pandas Apply', max=1000.0, style=ProgressStyle(descriptio…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "negative sample number: 1000\n",
      "epoch: 1 trained samples: 11000 ----average loss: 0.19236562587320805\n",
      "epoch: 1 trained samples: 11000 ----average prec: 0.9235000000000001\n",
      "----trained samples: 11000\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "0e8a8b0716d94325bcff0de08722230b",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, description='Pandas Apply', max=1000.0, style=ProgressStyle(descriptio…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "064cff4fce85485ebbb606cf8606e453",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, description='Pandas Apply', max=1000.0, style=ProgressStyle(descriptio…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "afd28dcaf65847a1a42c7d0298e881c9",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, description='Pandas Apply', max=1000.0, style=ProgressStyle(descriptio…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "3de07f36f43542bf828830ae87b9569f",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, description='Pandas Apply', max=1000.0, style=ProgressStyle(descriptio…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "negative sample number: 999\n",
      "epoch: 1 trained samples: 12000 ----average loss: 0.19617928471416235\n",
      "epoch: 1 trained samples: 12000 ----average prec: 0.922\n",
      "----trained samples: 12000\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "87b471fb5cf24f909e6ff471245e9a58",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, description='Pandas Apply', max=1000.0, style=ProgressStyle(descriptio…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "493ccefaf37347118b4c85199ec8852f",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, description='Pandas Apply', max=1000.0, style=ProgressStyle(descriptio…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "6346e43ea5f547b193f1732d09cefb8d",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, description='Pandas Apply', max=1000.0, style=ProgressStyle(descriptio…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "61411f5348214b1eb13b1d621f5bfc2b",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, description='Pandas Apply', max=1000.0, style=ProgressStyle(descriptio…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "negative sample number: 1000\n",
      "epoch: 1 trained samples: 13000 ----average loss: 0.19068784452974796\n",
      "epoch: 1 trained samples: 13000 ----average prec: 0.9295000000000001\n",
      "----trained samples: 13000\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "01b95a2a0fa64ed0ae85723ec5e86266",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, description='Pandas Apply', max=1000.0, style=ProgressStyle(descriptio…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "713843b8208d4bbab15b91c27a14c760",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, description='Pandas Apply', max=1000.0, style=ProgressStyle(descriptio…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "0fa58a951c6742659a91bd06d28b7b46",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, description='Pandas Apply', max=1000.0, style=ProgressStyle(descriptio…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "e506b73011e74d92a901a57ca1d63858",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, description='Pandas Apply', max=1000.0, style=ProgressStyle(descriptio…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "negative sample number: 999\n",
      "epoch: 1 trained samples: 14000 ----average loss: 0.1798284314572811\n",
      "epoch: 1 trained samples: 14000 ----average prec: 0.9315000000000001\n",
      "----trained samples: 14000\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "6234887b5f2643ec9ecf2d4721c671bf",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, description='Pandas Apply', max=1000.0, style=ProgressStyle(descriptio…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "ec588e3f1d6f4da7920a66be1e55969c",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, description='Pandas Apply', max=1000.0, style=ProgressStyle(descriptio…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "d5e2fa0456264a90947c9f51cf67a747",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, description='Pandas Apply', max=1000.0, style=ProgressStyle(descriptio…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "3122a763792d4eb6a20ae39aa3cf5506",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, description='Pandas Apply', max=1000.0, style=ProgressStyle(descriptio…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "negative sample number: 1000\n",
      "epoch: 1 trained samples: 15000 ----average loss: 0.1911078728735447\n",
      "epoch: 1 trained samples: 15000 ----average prec: 0.923\n",
      "----trained samples: 15000\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "d835e6878b914a2cb0a67cf56f65cc38",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, description='Pandas Apply', max=1000.0, style=ProgressStyle(descriptio…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "723bd018bbb447418e33a338fd4b2329",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, description='Pandas Apply', max=1000.0, style=ProgressStyle(descriptio…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "3bf38cc6c17b46b5bf363640e4e6b724",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, description='Pandas Apply', max=1000.0, style=ProgressStyle(descriptio…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "f6ef4925fc9142d793b1ee82da2fef60",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, description='Pandas Apply', max=1000.0, style=ProgressStyle(descriptio…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "negative sample number: 1000\n",
      "epoch: 1 trained samples: 16000 ----average loss: 0.19620885886251926\n",
      "epoch: 1 trained samples: 16000 ----average prec: 0.9279999999999999\n",
      "----trained samples: 16000\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "1a54d4b6c89c48329260d0acca7b3b8e",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, description='Pandas Apply', max=1000.0, style=ProgressStyle(descriptio…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "a4c6f202c7c5434d88b5f03a432e53ac",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, description='Pandas Apply', max=1000.0, style=ProgressStyle(descriptio…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "55c7ab87efe847809ba79a68c7fa4db9",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, description='Pandas Apply', max=1000.0, style=ProgressStyle(descriptio…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "64099613bf9d4b83828bd98cc5796154",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, description='Pandas Apply', max=1000.0, style=ProgressStyle(descriptio…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "negative sample number: 1000\n",
      "epoch: 1 trained samples: 17000 ----average loss: 0.18314483761787415\n",
      "epoch: 1 trained samples: 17000 ----average prec: 0.9275000000000001\n",
      "----trained samples: 17000\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "0b023d6f9f9d4bdf9e358e8cf6b4fe76",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, description='Pandas Apply', max=1000.0, style=ProgressStyle(descriptio…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "dd5d8bf3810d4b869c0f9e2cd22d537a",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, description='Pandas Apply', max=1000.0, style=ProgressStyle(descriptio…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "1239bbc138d04de399a3497c663d3045",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, description='Pandas Apply', max=1000.0, style=ProgressStyle(descriptio…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "25b151a7ce444546ab924096a0781310",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, description='Pandas Apply', max=1000.0, style=ProgressStyle(descriptio…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "negative sample number: 1000\n",
      "epoch: 1 trained samples: 18000 ----average loss: 0.18489277549088\n",
      "epoch: 1 trained samples: 18000 ----average prec: 0.9345000000000001\n",
      "----trained samples: 18000\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "0c09dcba267944ce9cd7c63dfe97a7eb",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, description='Pandas Apply', max=1000.0, style=ProgressStyle(descriptio…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "d2561043650d4a87beb05cd3c0815050",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, description='Pandas Apply', max=1000.0, style=ProgressStyle(descriptio…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "e0b8b302b287414e9905af88f035a665",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, description='Pandas Apply', max=1000.0, style=ProgressStyle(descriptio…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "22cb7df95be64fd1b6ee6daca23745a7",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, description='Pandas Apply', max=1000.0, style=ProgressStyle(descriptio…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "negative sample number: 1000\n",
      "epoch: 1 trained samples: 19000 ----average loss: 0.18346992880105972\n",
      "epoch: 1 trained samples: 19000 ----average prec: 0.932\n",
      "----trained samples: 19000\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "a1a1a24927f94e6599b2f1031fdf2cbf",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, description='Pandas Apply', max=1000.0, style=ProgressStyle(descriptio…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "da02cd0dfabb4a11a08e83259870045e",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, description='Pandas Apply', max=1000.0, style=ProgressStyle(descriptio…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "d4fe7a3724614119954bf2c8e1ff8705",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, description='Pandas Apply', max=1000.0, style=ProgressStyle(descriptio…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "066cdb6b281f4e23881ba12e9a1a234d",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, description='Pandas Apply', max=1000.0, style=ProgressStyle(descriptio…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "negative sample number: 999\n",
      "epoch: 1 trained samples: 20000 ----average loss: 0.17306738253682852\n",
      "epoch: 1 trained samples: 20000 ----average prec: 0.935\n",
      "----trained samples: 20000\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "76375433bfa8448388edb4f48346709f",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, description='Pandas Apply', max=1000.0, style=ProgressStyle(descriptio…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "fb85a5de15504ffc81954b498739abb4",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, description='Pandas Apply', max=1000.0, style=ProgressStyle(descriptio…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "322aa7ea20044655be4289fa127b2f5d",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, description='Pandas Apply', max=1000.0, style=ProgressStyle(descriptio…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "efbca760f81f4247856c8b0f1cc95d56",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, description='Pandas Apply', max=1000.0, style=ProgressStyle(descriptio…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "negative sample number: 1000\n",
      "epoch: 1 trained samples: 21000 ----average loss: 0.19667276553809643\n",
      "epoch: 1 trained samples: 21000 ----average prec: 0.9245000000000001\n",
      "----trained samples: 21000\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "9e75b1e34f3346f498502c2f29b05e84",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, description='Pandas Apply', max=1000.0, style=ProgressStyle(descriptio…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "d96ed2f480c2412ab89d56c959739cba",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, description='Pandas Apply', max=1000.0, style=ProgressStyle(descriptio…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "3edb101bee704135ace787c667565d36",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, description='Pandas Apply', max=1000.0, style=ProgressStyle(descriptio…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "5c4a04ddd1f7458a9c4f05b8dba951da",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, description='Pandas Apply', max=1000.0, style=ProgressStyle(descriptio…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "negative sample number: 1000\n",
      "epoch: 1 trained samples: 22000 ----average loss: 0.20838324539363384\n",
      "epoch: 1 trained samples: 22000 ----average prec: 0.925\n",
      "----trained samples: 22000\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "4dfc48153154425295f746b7b12b01f4",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, description='Pandas Apply', max=1000.0, style=ProgressStyle(descriptio…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "56afbd779ae44e8e9c44b5309fb41990",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, description='Pandas Apply', max=1000.0, style=ProgressStyle(descriptio…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "d53f0a271ab34022a334b388b12e484f",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, description='Pandas Apply', max=1000.0, style=ProgressStyle(descriptio…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "614560ad6fc84b7a915863eb036b30f0",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, description='Pandas Apply', max=1000.0, style=ProgressStyle(descriptio…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "negative sample number: 999\n",
      "epoch: 1 trained samples: 23000 ----average loss: 0.19547283835709095\n",
      "epoch: 1 trained samples: 23000 ----average prec: 0.9325000000000001\n",
      "----trained samples: 23000\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "2902bcfd1edb44f580639c4b60dbadfb",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, description='Pandas Apply', max=1000.0, style=ProgressStyle(descriptio…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "f5977bea09b04b368ac7f463eaa689d2",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, description='Pandas Apply', max=1000.0, style=ProgressStyle(descriptio…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "013dce04e5d44a18af81a0c719bfba55",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, description='Pandas Apply', max=1000.0, style=ProgressStyle(descriptio…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "16becbc8fc5544cd94a77fe33cc9d618",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, description='Pandas Apply', max=1000.0, style=ProgressStyle(descriptio…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "negative sample number: 1000\n",
      "epoch: 1 trained samples: 24000 ----average loss: 0.2059034202247858\n",
      "epoch: 1 trained samples: 24000 ----average prec: 0.9155000000000002\n",
      "----trained samples: 24000\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "c65f540d57154bc1b18e377c73eb79aa",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, description='Pandas Apply', max=1000.0, style=ProgressStyle(descriptio…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "15dc038c753e4bb0bfa913a3e18cfafc",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, description='Pandas Apply', max=1000.0, style=ProgressStyle(descriptio…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "457407b463954a0b8bdd8892a377fe1b",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, description='Pandas Apply', max=1000.0, style=ProgressStyle(descriptio…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "4ab87124bee046ebbb1d8849dda3442c",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, description='Pandas Apply', max=1000.0, style=ProgressStyle(descriptio…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "negative sample number: 1000\n",
      "epoch: 1 trained samples: 25000 ----average loss: 0.20449214056134224\n",
      "epoch: 1 trained samples: 25000 ----average prec: 0.9219999999999999\n",
      "----trained samples: 25000\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "5067bfb536cf4c4e873fa8f85b1b10ad",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, description='Pandas Apply', max=1000.0, style=ProgressStyle(descriptio…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "295e0f6eb6ba4519afbd310be1422adc",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, description='Pandas Apply', max=1000.0, style=ProgressStyle(descriptio…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "876650a8f38f4dc69e7d989bd4440edf",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, description='Pandas Apply', max=1000.0, style=ProgressStyle(descriptio…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "ea2e809f2c1541d6b21c3dc0824f8888",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, description='Pandas Apply', max=1000.0, style=ProgressStyle(descriptio…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "negative sample number: 1000\n",
      "epoch: 1 trained samples: 26000 ----average loss: 0.18599899299442768\n",
      "epoch: 1 trained samples: 26000 ----average prec: 0.923\n",
      "----trained samples: 26000\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "cbd10f582e574b22b846d406df5fe1f4",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, description='Pandas Apply', max=1000.0, style=ProgressStyle(descriptio…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "25d00029d8a04caeac0aaccc063f86b5",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, description='Pandas Apply', max=1000.0, style=ProgressStyle(descriptio…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "4cf2993e7e26408199053ecfbb3e704e",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, description='Pandas Apply', max=1000.0, style=ProgressStyle(descriptio…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "a26cfb84a9b9405d92456ae6ba93d62b",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, description='Pandas Apply', max=1000.0, style=ProgressStyle(descriptio…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "negative sample number: 999\n",
      "epoch: 1 trained samples: 27000 ----average loss: 0.17911900207400322\n",
      "epoch: 1 trained samples: 27000 ----average prec: 0.93\n",
      "----trained samples: 27000\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "d9ceca4c0aaf49479de0875fea186acb",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, description='Pandas Apply', max=1000.0, style=ProgressStyle(descriptio…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "4cd729eb629c41cf83bc64fb4d125d20",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, description='Pandas Apply', max=1000.0, style=ProgressStyle(descriptio…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "7f2084c0236a4d0fac440f5c8efbb656",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, description='Pandas Apply', max=1000.0, style=ProgressStyle(descriptio…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "193caf5623e84dddbca00549db00571f",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, description='Pandas Apply', max=1000.0, style=ProgressStyle(descriptio…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "negative sample number: 1000\n",
      "epoch: 1 trained samples: 28000 ----average loss: 0.18322807364165783\n",
      "epoch: 1 trained samples: 28000 ----average prec: 0.9309999999999999\n",
      "----trained samples: 28000\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "4c2462b8e1fd40a39d91540faa285930",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, description='Pandas Apply', max=1000.0, style=ProgressStyle(descriptio…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "d2f149a9ef8f4f2d8fe1dc0af054635f",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, description='Pandas Apply', max=1000.0, style=ProgressStyle(descriptio…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "8206e546156d474d8751898eb2171fc6",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, description='Pandas Apply', max=1000.0, style=ProgressStyle(descriptio…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "f083ecec53c54ae2948c0675e2395ba0",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, description='Pandas Apply', max=1000.0, style=ProgressStyle(descriptio…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "negative sample number: 998\n",
      "epoch: 1 trained samples: 29000 ----average loss: 0.17972762696444988\n",
      "epoch: 1 trained samples: 29000 ----average prec: 0.9284999999999999\n",
      "----trained samples: 29000\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "bada27fd318d4851b2e415a461abcd97",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, description='Pandas Apply', max=1000.0, style=ProgressStyle(descriptio…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "a5012b3e27af4e3c9ace605844fd3467",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, description='Pandas Apply', max=1000.0, style=ProgressStyle(descriptio…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "aa607d4e63c7417f8f12d2f0ea67a643",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, description='Pandas Apply', max=1000.0, style=ProgressStyle(descriptio…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "1683b73eccb2422b9c45bf9a110991c9",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, description='Pandas Apply', max=1000.0, style=ProgressStyle(descriptio…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "negative sample number: 1000\n",
      "epoch: 1 trained samples: 30000 ----average loss: 0.18965525552630424\n",
      "epoch: 1 trained samples: 30000 ----average prec: 0.925\n",
      "----trained samples: 30000\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "04f9a941a2884ae6b8d0b8baafc73544",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, description='Pandas Apply', max=1000.0, style=ProgressStyle(descriptio…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "af2fcedb0b1e4669a06d6d6a51a03c0c",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, description='Pandas Apply', max=1000.0, style=ProgressStyle(descriptio…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "6bd71bacfa1e4423b0cbdc5c07aaa19c",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, description='Pandas Apply', max=1000.0, style=ProgressStyle(descriptio…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "5af9fcbd0f0b48ec8b9e1cbfccc8cd9b",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, description='Pandas Apply', max=1000.0, style=ProgressStyle(descriptio…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "negative sample number: 1000\n",
      "epoch: 1 trained samples: 31000 ----average loss: 0.18210608884692192\n",
      "epoch: 1 trained samples: 31000 ----average prec: 0.933\n",
      "----trained samples: 31000\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "31063fc8f41a436e96b04bba1b3a55aa",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, description='Pandas Apply', max=1000.0, style=ProgressStyle(descriptio…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "609b3cc3d152404da9e829a7fce6702c",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, description='Pandas Apply', max=1000.0, style=ProgressStyle(descriptio…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "c49d8bf5a2fb44699977ab4700b089a9",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, description='Pandas Apply', max=1000.0, style=ProgressStyle(descriptio…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "8139d805de234def92765ecccad45638",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, description='Pandas Apply', max=1000.0, style=ProgressStyle(descriptio…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "negative sample number: 999\n",
      "epoch: 1 trained samples: 32000 ----average loss: 0.18216603063046932\n",
      "epoch: 1 trained samples: 32000 ----average prec: 0.932\n",
      "----trained samples: 32000\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "ef01f8c702754e5980464049ac2c2b89",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, description='Pandas Apply', max=1000.0, style=ProgressStyle(descriptio…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "a909e70913e74ab8a50965baca4fa291",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, description='Pandas Apply', max=1000.0, style=ProgressStyle(descriptio…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "b6fd06114a434b99a95170919dc8d120",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, description='Pandas Apply', max=1000.0, style=ProgressStyle(descriptio…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "30cc41130de345d0a23b43fb42447edf",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, description='Pandas Apply', max=1000.0, style=ProgressStyle(descriptio…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "negative sample number: 1000\n",
      "epoch: 1 trained samples: 33000 ----average loss: 0.1657354012131691\n",
      "epoch: 1 trained samples: 33000 ----average prec: 0.9395\n",
      "----trained samples: 33000\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "ccded4461e194c199dad48a121e64afe",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, description='Pandas Apply', max=1000.0, style=ProgressStyle(descriptio…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "5496881739464270a0347c4e9779e288",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, description='Pandas Apply', max=1000.0, style=ProgressStyle(descriptio…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "31d2bbce808f4f88b0a90ca8f96883ad",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, description='Pandas Apply', max=1000.0, style=ProgressStyle(descriptio…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "f280a20e72684d0ca7c03e9cdfc6fe2a",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, description='Pandas Apply', max=1000.0, style=ProgressStyle(descriptio…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "negative sample number: 999\n",
      "epoch: 1 trained samples: 34000 ----average loss: 0.166078582406044\n",
      "epoch: 1 trained samples: 34000 ----average prec: 0.937\n",
      "----trained samples: 34000\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "50b583265e9a45538e7fa0492067c564",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, description='Pandas Apply', max=1000.0, style=ProgressStyle(descriptio…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "892c3bb0897f4ba2815ec5f4b5f2174f",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, description='Pandas Apply', max=1000.0, style=ProgressStyle(descriptio…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "df470e3547134504bcca198befa07fd2",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, description='Pandas Apply', max=1000.0, style=ProgressStyle(descriptio…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "6da98486d49a451baec37ce691b66784",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, description='Pandas Apply', max=1000.0, style=ProgressStyle(descriptio…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "negative sample number: 1000\n",
      "epoch: 1 trained samples: 35000 ----average loss: 0.1790450345724821\n",
      "epoch: 1 trained samples: 35000 ----average prec: 0.9339999999999999\n",
      "----trained samples: 35000\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "c5f1c9640d264c81baa80496aeafeef7",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, description='Pandas Apply', max=1000.0, style=ProgressStyle(descriptio…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "cfa8715d66c14bbc9145cf038a39792a",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, description='Pandas Apply', max=1000.0, style=ProgressStyle(descriptio…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "5679450c0eee45c0945a3802ee99fc3f",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, description='Pandas Apply', max=1000.0, style=ProgressStyle(descriptio…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "db3aeab2d6644c2aa76925cce2db0d9b",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, description='Pandas Apply', max=1000.0, style=ProgressStyle(descriptio…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "negative sample number: 1000\n",
      "epoch: 1 trained samples: 36000 ----average loss: 0.1903560794889927\n",
      "epoch: 1 trained samples: 36000 ----average prec: 0.9274999999999999\n",
      "----trained samples: 36000\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "7e22e68926784e569581776fb04a9c46",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, description='Pandas Apply', max=1000.0, style=ProgressStyle(descriptio…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "74a98f1771b24f04b9aab8c707319603",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, description='Pandas Apply', max=1000.0, style=ProgressStyle(descriptio…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "cfcf43fde63c49c5bfc323b1b6940ad5",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, description='Pandas Apply', max=1000.0, style=ProgressStyle(descriptio…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "599b6f7f5fb3409c91d314baed6cf955",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, description='Pandas Apply', max=1000.0, style=ProgressStyle(descriptio…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "negative sample number: 999\n",
      "epoch: 1 trained samples: 37000 ----average loss: 0.17425949685275555\n",
      "epoch: 1 trained samples: 37000 ----average prec: 0.9335\n",
      "----trained samples: 37000\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "5ab24e0299a2413bbbfdcf7fabee5a17",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, description='Pandas Apply', max=1000.0, style=ProgressStyle(descriptio…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "3bfb2dd61cd84e66b5eea818aa6b14cb",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, description='Pandas Apply', max=1000.0, style=ProgressStyle(descriptio…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "3a42cb9e158f4dedb2721c36466923ae",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, description='Pandas Apply', max=1000.0, style=ProgressStyle(descriptio…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "6314ec4fd69845be8a5475f0ecab8cca",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, description='Pandas Apply', max=1000.0, style=ProgressStyle(descriptio…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "negative sample number: 1000\n",
      "epoch: 1 trained samples: 38000 ----average loss: 0.19134384207427502\n",
      "epoch: 1 trained samples: 38000 ----average prec: 0.927\n",
      "----trained samples: 38000\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "e8281c4f99b042fa9cd7f94a9cdc2943",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, description='Pandas Apply', max=1000.0, style=ProgressStyle(descriptio…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "f6c31187aa6a4b3ab1cdd240925e822a",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, description='Pandas Apply', max=1000.0, style=ProgressStyle(descriptio…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "70f8886e7d044b5e9bffc4527c5699ff",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, description='Pandas Apply', max=1000.0, style=ProgressStyle(descriptio…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "f545a0c727134c56b49c6b5b2a6dce02",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, description='Pandas Apply', max=1000.0, style=ProgressStyle(descriptio…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "negative sample number: 999\n",
      "epoch: 1 trained samples: 39000 ----average loss: 0.15895903669297695\n",
      "epoch: 1 trained samples: 39000 ----average prec: 0.9394999999999999\n",
      "----trained samples: 39000\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "460604ebda304ec59ee665fb4d3f1adb",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, description='Pandas Apply', max=1000.0, style=ProgressStyle(descriptio…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "fc37b3d61f62482b8723ae304d7bf2f6",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, description='Pandas Apply', max=1000.0, style=ProgressStyle(descriptio…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "4eb3f6df10654666bd48ff3dc2e1f3c7",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, description='Pandas Apply', max=1000.0, style=ProgressStyle(descriptio…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "aee67f85ccf745428b09a8e747170763",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, description='Pandas Apply', max=1000.0, style=ProgressStyle(descriptio…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "negative sample number: 1000\n",
      "epoch: 1 trained samples: 40000 ----average loss: 0.1801391988992691\n",
      "epoch: 1 trained samples: 40000 ----average prec: 0.932\n",
      "----trained samples: 40000\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "36138176ab1f402786e8bc163411f38a",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, description='Pandas Apply', max=1000.0, style=ProgressStyle(descriptio…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "59ee1e97eb9341a7ac90e7e189582668",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, description='Pandas Apply', max=1000.0, style=ProgressStyle(descriptio…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "bae0a088000a40369b1e1e3e42f53268",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, description='Pandas Apply', max=1000.0, style=ProgressStyle(descriptio…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "596b2ab5f8204ccb80cefd71fa3b2fea",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, description='Pandas Apply', max=1000.0, style=ProgressStyle(descriptio…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "negative sample number: 1000\n",
      "epoch: 1 trained samples: 41000 ----average loss: 0.19684368930757046\n",
      "epoch: 1 trained samples: 41000 ----average prec: 0.9280000000000002\n",
      "----trained samples: 41000\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "b968f34837594bc78789511cbf5acdc5",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, description='Pandas Apply', max=1000.0, style=ProgressStyle(descriptio…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "8d4fabc342784a6a8cabd45332ed6bb0",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, description='Pandas Apply', max=1000.0, style=ProgressStyle(descriptio…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "379f0492762d47b0acb570f8c0d74f53",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, description='Pandas Apply', max=1000.0, style=ProgressStyle(descriptio…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "4862718852984e3ba9829a3eaff26459",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, description='Pandas Apply', max=1000.0, style=ProgressStyle(descriptio…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "negative sample number: 1000\n",
      "epoch: 1 trained samples: 42000 ----average loss: 0.17262257542461157\n",
      "epoch: 1 trained samples: 42000 ----average prec: 0.9335000000000001\n",
      "----trained samples: 42000\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "21fffa99ee194c47ad966751b1f269ed",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, description='Pandas Apply', max=1000.0, style=ProgressStyle(descriptio…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "a2f4142014fa4b1a81a53bbc111f429e",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, description='Pandas Apply', max=1000.0, style=ProgressStyle(descriptio…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "06f8de3578d64b03aeeb271406800c6f",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, description='Pandas Apply', max=1000.0, style=ProgressStyle(descriptio…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "69b3e1c7faef45b199fafb1faa1b9230",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, description='Pandas Apply', max=1000.0, style=ProgressStyle(descriptio…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "negative sample number: 999\n",
      "epoch: 1 trained samples: 43000 ----average loss: 0.17168565280735493\n",
      "epoch: 1 trained samples: 43000 ----average prec: 0.9339999999999999\n",
      "----trained samples: 43000\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "a771f03bdcee4e80a000c4dc8492f0fd",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, description='Pandas Apply', max=1000.0, style=ProgressStyle(descriptio…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "0418eab790ed40db8e829f11c04913c7",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, description='Pandas Apply', max=1000.0, style=ProgressStyle(descriptio…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "fcdf73f64bd147de8034c16523ab966a",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, description='Pandas Apply', max=1000.0, style=ProgressStyle(descriptio…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "6de28c8252234c5fb1d96b8b63a55495",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, description='Pandas Apply', max=1000.0, style=ProgressStyle(descriptio…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "negative sample number: 999\n",
      "epoch: 1 trained samples: 44000 ----average loss: 0.1664688792079687\n",
      "epoch: 1 trained samples: 44000 ----average prec: 0.9359999999999999\n",
      "----trained samples: 44000\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "c190aba663be4b228c0de8284d4874db",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, description='Pandas Apply', max=1000.0, style=ProgressStyle(descriptio…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "a13103e73f9c461b8735c492a2e3cac4",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, description='Pandas Apply', max=1000.0, style=ProgressStyle(descriptio…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "1adb539c3f914d0fae82cb082f66e4ab",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, description='Pandas Apply', max=1000.0, style=ProgressStyle(descriptio…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "b0c024fbb9304aa19e174464f15cd7b7",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, description='Pandas Apply', max=1000.0, style=ProgressStyle(descriptio…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "negative sample number: 1000\n",
      "epoch: 1 trained samples: 45000 ----average loss: 0.18871484883129597\n",
      "epoch: 1 trained samples: 45000 ----average prec: 0.9295\n",
      "----trained samples: 45000\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "d5600526aa434ea5ab385819b4b2efe7",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, description='Pandas Apply', max=1000.0, style=ProgressStyle(descriptio…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "70df860959564f52a54489b9e4455127",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, description='Pandas Apply', max=1000.0, style=ProgressStyle(descriptio…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "0558f170eb6f4becb8f0b2871a44727e",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, description='Pandas Apply', max=1000.0, style=ProgressStyle(descriptio…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "3d26212f727548d2aabeb373014c2ad9",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, description='Pandas Apply', max=1000.0, style=ProgressStyle(descriptio…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "negative sample number: 999\n",
      "epoch: 1 trained samples: 46000 ----average loss: 0.16764375567436218\n",
      "epoch: 1 trained samples: 46000 ----average prec: 0.9395000000000001\n",
      "----trained samples: 46000\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "142bdde648c34933b29802284406f617",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, description='Pandas Apply', max=1000.0, style=ProgressStyle(descriptio…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "f575a9c6c8ea4485a4886e2d37b1167b",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, description='Pandas Apply', max=1000.0, style=ProgressStyle(descriptio…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "0224cc9147404b17925aa82f8876f206",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, description='Pandas Apply', max=1000.0, style=ProgressStyle(descriptio…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "d7ddeb6c48db4aaf9dd4b1b14d339e15",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, description='Pandas Apply', max=1000.0, style=ProgressStyle(descriptio…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "negative sample number: 999\n",
      "epoch: 1 trained samples: 47000 ----average loss: 0.1677377177402377\n",
      "epoch: 1 trained samples: 47000 ----average prec: 0.9325\n",
      "----trained samples: 47000\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "c6be6129b71a4b1b855e7af6b60bb955",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, description='Pandas Apply', max=1000.0, style=ProgressStyle(descriptio…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "67ffeb946dcf4e358d51e21451077959",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, description='Pandas Apply', max=1000.0, style=ProgressStyle(descriptio…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "9ece7a2800fe45b0964ee33140050c44",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, description='Pandas Apply', max=1000.0, style=ProgressStyle(descriptio…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "64e1e7ae240b491da7fb8afbd49f0dc6",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, description='Pandas Apply', max=1000.0, style=ProgressStyle(descriptio…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "negative sample number: 1000\n",
      "epoch: 1 trained samples: 48000 ----average loss: 0.1663004644215107\n",
      "epoch: 1 trained samples: 48000 ----average prec: 0.933\n",
      "----trained samples: 48000\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "a38ab31e6a0741b98703571f1e37336c",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, description='Pandas Apply', max=1000.0, style=ProgressStyle(descriptio…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "26c964e98bad48cfbcc9f0100c5b1a5d",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, description='Pandas Apply', max=1000.0, style=ProgressStyle(descriptio…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "4eaccf2992154a9fa8c71a090a683177",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, description='Pandas Apply', max=1000.0, style=ProgressStyle(descriptio…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "d90d214c5408489f9b01bab8d9fa466f",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, description='Pandas Apply', max=1000.0, style=ProgressStyle(descriptio…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "negative sample number: 1000\n",
      "epoch: 1 trained samples: 49000 ----average loss: 0.16846242360770702\n",
      "epoch: 1 trained samples: 49000 ----average prec: 0.9365000000000001\n",
      "----trained samples: 49000\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "92512638e4c64946bdbddd8307fc5eb2",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, description='Pandas Apply', max=1000.0, style=ProgressStyle(descriptio…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "b90fc914034f4a8d896214091c8b06ac",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, description='Pandas Apply', max=1000.0, style=ProgressStyle(descriptio…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "2faa5017d0a7415b81b2ff1df1bd94b1",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, description='Pandas Apply', max=1000.0, style=ProgressStyle(descriptio…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "57ae4c28ad0143e29157ed69f3c7f717",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, description='Pandas Apply', max=1000.0, style=ProgressStyle(descriptio…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "negative sample number: 1000\n",
      "epoch: 1 trained samples: 50000 ----average loss: 0.19170199520885944\n",
      "epoch: 1 trained samples: 50000 ----average prec: 0.9259999999999999\n",
      "----trained samples: 50000\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "2efcbd6d373c4a7ebd7566df415fee2b",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, description='Pandas Apply', max=1000.0, style=ProgressStyle(descriptio…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "12e867b783624dff91d5326d9302bf7a",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, description='Pandas Apply', max=1000.0, style=ProgressStyle(descriptio…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "fe981b55b7eb46e4b8357d3eaeb93cd0",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, description='Pandas Apply', max=1000.0, style=ProgressStyle(descriptio…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "fe8e434372a5489b82a59fc7f1595928",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, description='Pandas Apply', max=1000.0, style=ProgressStyle(descriptio…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "negative sample number: 999\n",
      "epoch: 1 trained samples: 51000 ----average loss: 0.19624954275786877\n",
      "epoch: 1 trained samples: 51000 ----average prec: 0.9235\n",
      "----trained samples: 51000\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "012e436f2ac04b72b3805e1fc97c6264",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, description='Pandas Apply', max=1000.0, style=ProgressStyle(descriptio…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "50db455ad66c41a0912f53a02ca40bec",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, description='Pandas Apply', max=1000.0, style=ProgressStyle(descriptio…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "e4f7749b163d48baa664c3970933e096",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, description='Pandas Apply', max=1000.0, style=ProgressStyle(descriptio…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "2f92163caa264960bcd0f70f8ea5eead",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, description='Pandas Apply', max=1000.0, style=ProgressStyle(descriptio…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "negative sample number: 999\n",
      "epoch: 1 trained samples: 52000 ----average loss: 0.18985562585294247\n",
      "epoch: 1 trained samples: 52000 ----average prec: 0.9225000000000001\n",
      "----trained samples: 52000\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "f67f0aa04a994be99689e29176079095",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, description='Pandas Apply', max=1000.0, style=ProgressStyle(descriptio…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "017bce25b4fb40be824be6b0b32b6f73",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, description='Pandas Apply', max=1000.0, style=ProgressStyle(descriptio…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "3424de1400c04ed78b8d68d571b65a53",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, description='Pandas Apply', max=1000.0, style=ProgressStyle(descriptio…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "371b817cbf9c48c0b88cfb5e964f5ab7",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, description='Pandas Apply', max=1000.0, style=ProgressStyle(descriptio…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "negative sample number: 1000\n",
      "epoch: 1 trained samples: 53000 ----average loss: 0.16733840573579073\n",
      "epoch: 1 trained samples: 53000 ----average prec: 0.9384999999999999\n",
      "----trained samples: 53000\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "03b67bb49eaa4db784f1006f3b745335",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, description='Pandas Apply', max=1000.0, style=ProgressStyle(descriptio…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "6152b1e16f7043baab61b0cddf64e931",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, description='Pandas Apply', max=1000.0, style=ProgressStyle(descriptio…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "55a455368d7248a58655233d781bc883",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, description='Pandas Apply', max=1000.0, style=ProgressStyle(descriptio…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "5026f2b576404fb292c4d85f355f07d0",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, description='Pandas Apply', max=1000.0, style=ProgressStyle(descriptio…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "negative sample number: 1000\n",
      "epoch: 1 trained samples: 54000 ----average loss: 0.18272214010357857\n",
      "epoch: 1 trained samples: 54000 ----average prec: 0.926\n",
      "----trained samples: 54000\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "77ddd8d0fd204334b587f0c2db9b80c4",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, description='Pandas Apply', max=1000.0, style=ProgressStyle(descriptio…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "b1ac5406e9f44d84947962cb567566cd",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, description='Pandas Apply', max=1000.0, style=ProgressStyle(descriptio…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "66425b5e45e34e90a928f30998e59f2c",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, description='Pandas Apply', max=1000.0, style=ProgressStyle(descriptio…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "9333b649b53248289323f0b50853b647",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, description='Pandas Apply', max=1000.0, style=ProgressStyle(descriptio…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "negative sample number: 1000\n",
      "epoch: 1 trained samples: 55000 ----average loss: 0.16868013516068459\n",
      "epoch: 1 trained samples: 55000 ----average prec: 0.9260000000000002\n",
      "----trained samples: 55000\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "e9f33fff6cbb42ed8e77bcfb8571a900",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, description='Pandas Apply', max=1000.0, style=ProgressStyle(descriptio…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "721332ee96f64f0fb34240bd6a030b10",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, description='Pandas Apply', max=1000.0, style=ProgressStyle(descriptio…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "2fa1728a4bd1445095ee5780f8bea574",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, description='Pandas Apply', max=1000.0, style=ProgressStyle(descriptio…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "f1cbc1b233014007b7fadeeed12ea671",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, description='Pandas Apply', max=1000.0, style=ProgressStyle(descriptio…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "negative sample number: 1000\n",
      "epoch: 1 trained samples: 56000 ----average loss: 0.17376871593296528\n",
      "epoch: 1 trained samples: 56000 ----average prec: 0.938\n",
      "----trained samples: 56000\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "366e84c1e58a4576a5942f4927674e61",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, description='Pandas Apply', max=1000.0, style=ProgressStyle(descriptio…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "2772ca91dda2494fa4cb46412c5312ad",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, description='Pandas Apply', max=1000.0, style=ProgressStyle(descriptio…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "c858121a78834068bf44f62445f5be90",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, description='Pandas Apply', max=1000.0, style=ProgressStyle(descriptio…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "00c7ece4c4ea42dcb3cc916464a8fbc0",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, description='Pandas Apply', max=1000.0, style=ProgressStyle(descriptio…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "negative sample number: 1000\n",
      "epoch: 1 trained samples: 57000 ----average loss: 0.174593148753047\n",
      "epoch: 1 trained samples: 57000 ----average prec: 0.9355\n",
      "----trained samples: 57000\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "f7ef0e2514524802a23f076538dc6a72",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, description='Pandas Apply', max=1000.0, style=ProgressStyle(descriptio…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "002f2e34b242473b88bda2db6c4f757e",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, description='Pandas Apply', max=1000.0, style=ProgressStyle(descriptio…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "34fe512865fd4d669ad3ffeff145e88d",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, description='Pandas Apply', max=1000.0, style=ProgressStyle(descriptio…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "d0ae098e2b3048078f4a7e5a2f05d390",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, description='Pandas Apply', max=1000.0, style=ProgressStyle(descriptio…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "negative sample number: 999\n",
      "epoch: 1 trained samples: 58000 ----average loss: 0.1700175404548645\n",
      "epoch: 1 trained samples: 58000 ----average prec: 0.9390000000000001\n",
      "----trained samples: 58000\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "1e22bd1d9d414cd7ae6b583623b19431",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, description='Pandas Apply', max=1000.0, style=ProgressStyle(descriptio…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "e7ed456f0a0244dcae1c0d6403382932",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, description='Pandas Apply', max=1000.0, style=ProgressStyle(descriptio…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "21c483c60db14ea7a6b3fcefaad5b69c",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, description='Pandas Apply', max=1000.0, style=ProgressStyle(descriptio…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "d627c3d74d034569be70f59503c70ca4",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, description='Pandas Apply', max=1000.0, style=ProgressStyle(descriptio…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "negative sample number: 999\n",
      "epoch: 1 trained samples: 59000 ----average loss: 0.1700602639466524\n",
      "epoch: 1 trained samples: 59000 ----average prec: 0.935\n",
      "----trained samples: 59000\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "b7c0718082eb446b8262e3fefda948b5",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, description='Pandas Apply', max=1000.0, style=ProgressStyle(descriptio…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "69a62345b4d14a7db6aa6f697ef04a58",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, description='Pandas Apply', max=1000.0, style=ProgressStyle(descriptio…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "503aa7bf41424fcc94a57045c46c410c",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, description='Pandas Apply', max=1000.0, style=ProgressStyle(descriptio…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "309be8538d524e16b4db58ca9e78cac7",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, description='Pandas Apply', max=1000.0, style=ProgressStyle(descriptio…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "negative sample number: 999\n",
      "epoch: 1 trained samples: 60000 ----average loss: 0.1755796568468213\n",
      "epoch: 1 trained samples: 60000 ----average prec: 0.9365\n",
      "----trained samples: 60000\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "2620cf84ddbd4b15a0391165136300bb",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, description='Pandas Apply', max=1000.0, style=ProgressStyle(descriptio…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "031b196d3ecb4167ad5d603592876693",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, description='Pandas Apply', max=1000.0, style=ProgressStyle(descriptio…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "ad26583775874d8a9dff877df0c2442c",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, description='Pandas Apply', max=1000.0, style=ProgressStyle(descriptio…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "d9d30430a8c84854a5e4f992de2d74de",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, description='Pandas Apply', max=1000.0, style=ProgressStyle(descriptio…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "negative sample number: 1000\n",
      "epoch: 1 trained samples: 61000 ----average loss: 0.16775978915393353\n",
      "epoch: 1 trained samples: 61000 ----average prec: 0.942\n",
      "----trained samples: 61000\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "81f87fbec22149ea8a801ef9b42b48fc",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, description='Pandas Apply', max=1000.0, style=ProgressStyle(descriptio…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "68027eb14b80431eb8f7b58c29b328b5",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, description='Pandas Apply', max=1000.0, style=ProgressStyle(descriptio…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "708821da16664cd6aed1ba4fa2184d7a",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, description='Pandas Apply', max=1000.0, style=ProgressStyle(descriptio…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "0b566673c9404e179e8a61b83beb3c4d",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, description='Pandas Apply', max=1000.0, style=ProgressStyle(descriptio…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "negative sample number: 999\n",
      "epoch: 1 trained samples: 62000 ----average loss: 0.17535782046616077\n",
      "epoch: 1 trained samples: 62000 ----average prec: 0.931\n",
      "----trained samples: 62000\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "2ea9e08c8ef44f77909ea55cc6b047cd",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, description='Pandas Apply', max=1000.0, style=ProgressStyle(descriptio…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "c4311a1319454bc3979ea88540989dc0",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, description='Pandas Apply', max=1000.0, style=ProgressStyle(descriptio…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "874d7cf788644bfa81c9a65fbf2684e9",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, description='Pandas Apply', max=1000.0, style=ProgressStyle(descriptio…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "4e8fcb272aca44058da6fe050d5db402",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, description='Pandas Apply', max=1000.0, style=ProgressStyle(descriptio…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "negative sample number: 998\n",
      "epoch: 1 trained samples: 63000 ----average loss: 0.1728160548955202\n",
      "epoch: 1 trained samples: 63000 ----average prec: 0.9365\n",
      "----trained samples: 63000\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "91512615809945be9e080e8166e2f504",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, description='Pandas Apply', max=1000.0, style=ProgressStyle(descriptio…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "6ff9168c67d840168606d1c839e821e0",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, description='Pandas Apply', max=1000.0, style=ProgressStyle(descriptio…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "c4043bf79b6442dba13c906752a7f559",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, description='Pandas Apply', max=1000.0, style=ProgressStyle(descriptio…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "fb53098e29e84714b58a9120c8c4aac8",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, description='Pandas Apply', max=1000.0, style=ProgressStyle(descriptio…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "negative sample number: 999\n",
      "epoch: 1 trained samples: 64000 ----average loss: 0.1766745988279581\n",
      "epoch: 1 trained samples: 64000 ----average prec: 0.938\n",
      "----trained samples: 64000\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "1a5168dbf3b745988569233b9b245676",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, description='Pandas Apply', max=1000.0, style=ProgressStyle(descriptio…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "6143ccdedc514c0c8bd0389f438be5a7",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, description='Pandas Apply', max=1000.0, style=ProgressStyle(descriptio…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "fcc08c3851b84c188b03bd0022e83499",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, description='Pandas Apply', max=1000.0, style=ProgressStyle(descriptio…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "ebedb98e60f84bd59353ae4158e7455e",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, description='Pandas Apply', max=1000.0, style=ProgressStyle(descriptio…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "negative sample number: 1000\n",
      "epoch: 1 trained samples: 65000 ----average loss: 0.16218827106058598\n",
      "epoch: 1 trained samples: 65000 ----average prec: 0.9390000000000001\n",
      "----trained samples: 65000\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "f6dbdefbf60b4aa6a396e1a0e4a7f4c8",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, description='Pandas Apply', max=1000.0, style=ProgressStyle(descriptio…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "0cc24033ba55406296683e06ce2a6ac8",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, description='Pandas Apply', max=1000.0, style=ProgressStyle(descriptio…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "30cc54b5230f4faf9c37ab00c97a0f40",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, description='Pandas Apply', max=1000.0, style=ProgressStyle(descriptio…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "e0240a0df3d74e51bdbe156024a7fee4",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, description='Pandas Apply', max=1000.0, style=ProgressStyle(descriptio…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "negative sample number: 999\n",
      "epoch: 1 trained samples: 66000 ----average loss: 0.16258556954562664\n",
      "epoch: 1 trained samples: 66000 ----average prec: 0.946\n",
      "----trained samples: 66000\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "dbf69b27a25043d0a90880547cca86df",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, description='Pandas Apply', max=1000.0, style=ProgressStyle(descriptio…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "0231442a01d5404db38d40193d6848fb",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, description='Pandas Apply', max=1000.0, style=ProgressStyle(descriptio…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "37cc97a59a9a4313ab86eca9740cc4dc",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, description='Pandas Apply', max=1000.0, style=ProgressStyle(descriptio…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "29f6079842824071b0fa681bd83c3959",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, description='Pandas Apply', max=1000.0, style=ProgressStyle(descriptio…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "negative sample number: 1000\n",
      "epoch: 1 trained samples: 67000 ----average loss: 0.15632114093750715\n",
      "epoch: 1 trained samples: 67000 ----average prec: 0.942\n",
      "----trained samples: 67000\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "ef8cfddcaba540e78fd9530ab09cee34",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, description='Pandas Apply', max=1000.0, style=ProgressStyle(descriptio…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "49c6ba63bc74466e889a8e05cd70fd3a",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, description='Pandas Apply', max=1000.0, style=ProgressStyle(descriptio…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "ea6230ef2b6b47dc9691953b1c14f73d",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, description='Pandas Apply', max=1000.0, style=ProgressStyle(descriptio…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "c73c4fe947e1448a81ed8daeadec27d4",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, description='Pandas Apply', max=1000.0, style=ProgressStyle(descriptio…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "negative sample number: 1000\n",
      "epoch: 1 trained samples: 68000 ----average loss: 0.1663501476868987\n",
      "epoch: 1 trained samples: 68000 ----average prec: 0.939\n",
      "----trained samples: 68000\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "5ec26cc91bdc4c3aa46dd88d781e4f4a",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, description='Pandas Apply', max=1000.0, style=ProgressStyle(descriptio…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "7f11b178b9e74dd4a62f30a7e9e3ecc7",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, description='Pandas Apply', max=1000.0, style=ProgressStyle(descriptio…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "c3053851a59a4e029ae63b198ae14988",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, description='Pandas Apply', max=1000.0, style=ProgressStyle(descriptio…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "4ff73e00f023423b983464103b91cc9c",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, description='Pandas Apply', max=1000.0, style=ProgressStyle(descriptio…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "def generate_train(train, allsizes):\n",
    "    length = train.shape[0]\n",
    "    for i in range(0, length, allsizes):\n",
    "        yield train.iloc[i:i+allsizes, :]\n",
    "        \n",
    "def generate_batch_train(query_vec, \n",
    "                         label_text_vec, \n",
    "                         query_bert_vec, \n",
    "                         img_vec, pos_vec, \n",
    "                         padding_mask, \n",
    "                         lstm_seq_length, \n",
    "                         label,\n",
    "                         allsizes, \n",
    "                         batch):\n",
    "    length = allsizes*2\n",
    "    for i in range(0, length, batch):\n",
    "        yield query_vec[i:i+batch], label_text_vec[i:i+batch], query_bert_vec[i:i+batch], img_vec[i:i+batch], pos_vec[i:i+batch], padding_mask[i:i+batch], lstm_seq_length[i:i+batch], label[i:i+batch]\n",
    "        \n",
    "def cal_prec(predict,labels):\n",
    "    prec=0\n",
    "    for p, l in zip(predict,labels):\n",
    "        if p>0.5 and l==1:\n",
    "            prec+=1\n",
    "        if p<=0.5 and l==0:\n",
    "            prec+=1\n",
    "            \n",
    "    return prec/len(predict)\n",
    "\n",
    "allsizes=1000\n",
    "batch=250\n",
    "epoch=50\n",
    "\n",
    "for e in range(1, epoch):\n",
    "    print('epoch:', e)\n",
    "    for s in range(1, 3):\n",
    "        train_albert_path='../albert/avgpool_albert_embedded_'+str(s)+'00000'\n",
    "        train_data_path='data/temp_data_'+str(s)+'00000.pkl'\n",
    "        train = load_train_data(train_albert_path, train_data_path)\n",
    "        \n",
    "        train=shuffle(train, random_state=2020)\n",
    "        train.reset_index(drop=True, inplace=True)\n",
    "        gc.collect()\n",
    "        \n",
    "        print('load '+str(s*100000)+' train samples----------------------')\n",
    "    \n",
    "        trained_samples=allsizes+(s-1)*100000\n",
    "        for big_batch_train in generate_train(train, allsizes):\n",
    "            print('----trained samples:', trained_samples)\n",
    "            \n",
    "            trained_samples += allsizes\n",
    "\n",
    "            big_batch_train['label']=1\n",
    "\n",
    "            # padding\n",
    "            def padding_seq(sequence, length, max_length):\n",
    "                if length >= max_length:\n",
    "                    return sequence\n",
    "                else:\n",
    "                    return sequence + [np.zeros(len(sequence[0])) for _ in range(max_length-length)]\n",
    "            big_batch_train['label_embedding']=big_batch_train.swifter.apply(lambda x: padding_seq(x['label_embedding'], x['num_box'], MAX_BOX),axis=1)\n",
    "            big_batch_train['pos']=big_batch_train.swifter.apply(lambda x: padding_seq(x['pos'], x['num_box'], MAX_BOX),axis=1)\n",
    "            big_batch_train['query_embedding']=big_batch_train.swifter.apply(lambda x: padding_seq(x['query_embedding'], x['num_query_char'], MAX_QUERY_CHAR),axis=1)\n",
    "            big_batch_train['features']=big_batch_train.swifter.apply(lambda x: padding_seq(x['features'], x['num_box'], MAX_BOX),axis=1)\n",
    "            gc.collect()\n",
    "\n",
    "            # generate negative samples\n",
    "            negative_train = generate_negative_sample(big_batch_train)\n",
    "            if len(negative_train)<len(big_batch_train):\n",
    "                    negative_train=pd.concat((negative_train,\n",
    "                                              negative_train.iloc[0:(len(big_batch_train)-len(negative_train)),:]),axis=0)\n",
    "\n",
    "            big_batch_train = pd.concat((big_batch_train, negative_train), axis=0)\n",
    "            big_batch_train.reset_index(drop=True, inplace=True)\n",
    "            big_batch_train=shuffle(big_batch_train, random_state=2020)\n",
    "            big_batch_train.reset_index(drop=True, inplace=True)\n",
    "\n",
    "            del negative_train\n",
    "            gc.collect()\n",
    "\n",
    "            query_vec = torch.Tensor(big_batch_train['query_embedding'])\n",
    "            lstm_seq_length = list(big_batch_train['num_query_char'])\n",
    "\n",
    "            label_text_vec = torch.Tensor(big_batch_train['label_embedding'])\n",
    "            query_bert_vec = torch.Tensor(big_batch_train['query_albert_embedding']).view(allsizes*2, 1, 768)\n",
    "\n",
    "            img_vec = torch.Tensor(big_batch_train['features'])\n",
    "            pos_vec = torch.Tensor(big_batch_train['pos'])\n",
    "\n",
    "            label =torch.Tensor(big_batch_train['label'])\n",
    "\n",
    "            padding_mask =[]\n",
    "            for i in list(big_batch_train['num_box']):\n",
    "                mask=[0]*(i+2) + [1]*(MAX_BOX-i)\n",
    "                padding_mask.append(mask)\n",
    "            padding_mask = torch.ByteTensor(padding_mask)\n",
    "            gc.collect()\n",
    "\n",
    "            # train model in per batch samples\n",
    "            train_count = 0\n",
    "            avg_loss = 0\n",
    "            avg_prec = 0\n",
    "            for i in generate_batch_train(query_vec,label_text_vec,query_bert_vec,img_vec,pos_vec,padding_mask,lstm_seq_length,label,allsizes, batch):\n",
    "                b_query_vec, b_label_text_vec, b_query_bert_vec, b_img_vec, b_pos_vec, b_padding_mask, b_lstm_seq_length, b_label=i\n",
    "                optimizer.zero_grad()\n",
    "                prediction=model_ft(b_query_vec.cuda(), b_label_text_vec.cuda(), b_query_bert_vec.cuda(), b_img_vec.cuda(), \n",
    "                                    b_pos_vec.cuda(), b_padding_mask.cuda(), b_lstm_seq_length)\n",
    "\n",
    "                loss = calc_loss(prediction, b_label.cuda())\n",
    "                prec = cal_prec(prediction, b_label.cuda())\n",
    "\n",
    "                avg_loss += loss.item()\n",
    "                avg_prec += prec\n",
    "                loss.backward()\n",
    "                optimizer.step()\n",
    "                train_count += 1\n",
    "\n",
    "            print('epoch:', e, 'trained samples:', trained_samples, '----average loss:', avg_loss / train_count)\n",
    "            print('epoch:', e, 'trained samples:', trained_samples, '----average prec:', avg_prec / train_count)\n",
    "\n",
    "\n",
    "        del train\n",
    "        gc.collect()\n",
    "        \n",
    "        torch.save(model_ft.state_dict(), 'model/' + str(e) + '_'+ str(s*100000) + '_attention_bilstm_parameter.pkl')\n",
    "        print('model/' + str(e) + '_' + str(s*100000) + '_attention_bilstm_parameter.pkl has been saved!')\n",
    "        "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 55,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "torch.Size([10, 12, 600]) torch.Size([10, 10, 600]) torch.Size([10, 1, 768]) torch.Size([10, 10, 2048]) torch.Size([10, 10, 5]) (10, 12) (10,)\n",
      "0:loss------- tensor(0.6204, device='cuda:0', grad_fn=<BinaryCrossEntropyWithLogitsBackward>)\n",
      "1:loss------- tensor(0.4574, device='cuda:0', grad_fn=<BinaryCrossEntropyWithLogitsBackward>)\n",
      "2:loss------- tensor(0.3725, device='cuda:0', grad_fn=<BinaryCrossEntropyWithLogitsBackward>)\n",
      "3:loss------- tensor(0.3211, device='cuda:0', grad_fn=<BinaryCrossEntropyWithLogitsBackward>)\n",
      "4:loss------- tensor(0.2872, device='cuda:0', grad_fn=<BinaryCrossEntropyWithLogitsBackward>)\n",
      "5:loss------- tensor(0.2631, device='cuda:0', grad_fn=<BinaryCrossEntropyWithLogitsBackward>)\n",
      "6:loss------- tensor(0.2458, device='cuda:0', grad_fn=<BinaryCrossEntropyWithLogitsBackward>)\n",
      "7:loss------- tensor(0.2322, device='cuda:0', grad_fn=<BinaryCrossEntropyWithLogitsBackward>)\n",
      "8:loss------- tensor(0.2214, device='cuda:0', grad_fn=<BinaryCrossEntropyWithLogitsBackward>)\n",
      "9:loss------- tensor(0.2132, device='cuda:0', grad_fn=<BinaryCrossEntropyWithLogitsBackward>)\n",
      "10:loss------- tensor(0.2067, device='cuda:0', grad_fn=<BinaryCrossEntropyWithLogitsBackward>)\n",
      "11:loss------- tensor(0.2012, device='cuda:0', grad_fn=<BinaryCrossEntropyWithLogitsBackward>)\n",
      "12:loss------- tensor(0.1963, device='cuda:0', grad_fn=<BinaryCrossEntropyWithLogitsBackward>)\n",
      "13:loss------- tensor(0.1924, device='cuda:0', grad_fn=<BinaryCrossEntropyWithLogitsBackward>)\n",
      "14:loss------- tensor(0.1896, device='cuda:0', grad_fn=<BinaryCrossEntropyWithLogitsBackward>)\n",
      "15:loss------- tensor(0.1872, device='cuda:0', grad_fn=<BinaryCrossEntropyWithLogitsBackward>)\n",
      "16:loss------- tensor(0.1848, device='cuda:0', grad_fn=<BinaryCrossEntropyWithLogitsBackward>)\n",
      "17:loss------- tensor(0.1825, device='cuda:0', grad_fn=<BinaryCrossEntropyWithLogitsBackward>)\n",
      "18:loss------- tensor(0.1802, device='cuda:0', grad_fn=<BinaryCrossEntropyWithLogitsBackward>)\n",
      "19:loss------- tensor(0.1782, device='cuda:0', grad_fn=<BinaryCrossEntropyWithLogitsBackward>)\n",
      "20:loss------- tensor(0.1764, device='cuda:0', grad_fn=<BinaryCrossEntropyWithLogitsBackward>)\n",
      "21:loss------- tensor(0.1746, device='cuda:0', grad_fn=<BinaryCrossEntropyWithLogitsBackward>)\n",
      "22:loss------- tensor(0.1730, device='cuda:0', grad_fn=<BinaryCrossEntropyWithLogitsBackward>)\n",
      "23:loss------- tensor(0.1715, device='cuda:0', grad_fn=<BinaryCrossEntropyWithLogitsBackward>)\n",
      "24:loss------- tensor(0.1701, device='cuda:0', grad_fn=<BinaryCrossEntropyWithLogitsBackward>)\n",
      "25:loss------- tensor(0.1688, device='cuda:0', grad_fn=<BinaryCrossEntropyWithLogitsBackward>)\n",
      "26:loss------- tensor(0.1675, device='cuda:0', grad_fn=<BinaryCrossEntropyWithLogitsBackward>)\n",
      "27:loss------- tensor(0.1664, device='cuda:0', grad_fn=<BinaryCrossEntropyWithLogitsBackward>)\n",
      "28:loss------- tensor(0.1652, device='cuda:0', grad_fn=<BinaryCrossEntropyWithLogitsBackward>)\n",
      "29:loss------- tensor(0.1641, device='cuda:0', grad_fn=<BinaryCrossEntropyWithLogitsBackward>)\n",
      "30:loss------- tensor(0.1630, device='cuda:0', grad_fn=<BinaryCrossEntropyWithLogitsBackward>)\n",
      "31:loss------- tensor(0.1620, device='cuda:0', grad_fn=<BinaryCrossEntropyWithLogitsBackward>)\n",
      "32:loss------- tensor(0.1611, device='cuda:0', grad_fn=<BinaryCrossEntropyWithLogitsBackward>)\n",
      "33:loss------- tensor(0.1602, device='cuda:0', grad_fn=<BinaryCrossEntropyWithLogitsBackward>)\n",
      "34:loss------- tensor(0.1594, device='cuda:0', grad_fn=<BinaryCrossEntropyWithLogitsBackward>)\n",
      "35:loss------- tensor(0.1587, device='cuda:0', grad_fn=<BinaryCrossEntropyWithLogitsBackward>)\n",
      "36:loss------- tensor(0.1581, device='cuda:0', grad_fn=<BinaryCrossEntropyWithLogitsBackward>)\n",
      "37:loss------- tensor(0.1574, device='cuda:0', grad_fn=<BinaryCrossEntropyWithLogitsBackward>)\n",
      "38:loss------- tensor(0.1569, device='cuda:0', grad_fn=<BinaryCrossEntropyWithLogitsBackward>)\n",
      "39:loss------- tensor(0.1563, device='cuda:0', grad_fn=<BinaryCrossEntropyWithLogitsBackward>)\n",
      "40:loss------- tensor(0.1558, device='cuda:0', grad_fn=<BinaryCrossEntropyWithLogitsBackward>)\n",
      "41:loss------- tensor(0.1553, device='cuda:0', grad_fn=<BinaryCrossEntropyWithLogitsBackward>)\n",
      "42:loss------- tensor(0.1549, device='cuda:0', grad_fn=<BinaryCrossEntropyWithLogitsBackward>)\n",
      "43:loss------- tensor(0.1544, device='cuda:0', grad_fn=<BinaryCrossEntropyWithLogitsBackward>)\n",
      "44:loss------- tensor(0.1541, device='cuda:0', grad_fn=<BinaryCrossEntropyWithLogitsBackward>)\n",
      "45:loss------- tensor(0.1537, device='cuda:0', grad_fn=<BinaryCrossEntropyWithLogitsBackward>)\n",
      "46:loss------- tensor(0.1533, device='cuda:0', grad_fn=<BinaryCrossEntropyWithLogitsBackward>)\n",
      "47:loss------- tensor(0.1530, device='cuda:0', grad_fn=<BinaryCrossEntropyWithLogitsBackward>)\n",
      "48:loss------- tensor(0.1527, device='cuda:0', grad_fn=<BinaryCrossEntropyWithLogitsBackward>)\n",
      "49:loss------- tensor(0.1524, device='cuda:0', grad_fn=<BinaryCrossEntropyWithLogitsBackward>)\n",
      "50:loss------- tensor(0.1521, device='cuda:0', grad_fn=<BinaryCrossEntropyWithLogitsBackward>)\n",
      "51:loss------- tensor(0.1518, device='cuda:0', grad_fn=<BinaryCrossEntropyWithLogitsBackward>)\n",
      "52:loss------- tensor(0.1515, device='cuda:0', grad_fn=<BinaryCrossEntropyWithLogitsBackward>)\n",
      "53:loss------- tensor(0.1512, device='cuda:0', grad_fn=<BinaryCrossEntropyWithLogitsBackward>)\n",
      "54:loss------- tensor(0.1510, device='cuda:0', grad_fn=<BinaryCrossEntropyWithLogitsBackward>)\n",
      "55:loss------- tensor(0.1507, device='cuda:0', grad_fn=<BinaryCrossEntropyWithLogitsBackward>)\n",
      "56:loss------- tensor(0.1505, device='cuda:0', grad_fn=<BinaryCrossEntropyWithLogitsBackward>)\n",
      "57:loss------- tensor(0.1502, device='cuda:0', grad_fn=<BinaryCrossEntropyWithLogitsBackward>)\n",
      "58:loss------- tensor(0.1500, device='cuda:0', grad_fn=<BinaryCrossEntropyWithLogitsBackward>)\n",
      "59:loss------- tensor(0.1498, device='cuda:0', grad_fn=<BinaryCrossEntropyWithLogitsBackward>)\n",
      "60:loss------- tensor(0.1496, device='cuda:0', grad_fn=<BinaryCrossEntropyWithLogitsBackward>)\n",
      "61:loss------- tensor(0.1494, device='cuda:0', grad_fn=<BinaryCrossEntropyWithLogitsBackward>)\n",
      "62:loss------- tensor(0.1492, device='cuda:0', grad_fn=<BinaryCrossEntropyWithLogitsBackward>)\n",
      "63:loss------- tensor(0.1490, device='cuda:0', grad_fn=<BinaryCrossEntropyWithLogitsBackward>)\n",
      "64:loss------- tensor(0.1488, device='cuda:0', grad_fn=<BinaryCrossEntropyWithLogitsBackward>)\n",
      "65:loss------- tensor(0.1486, device='cuda:0', grad_fn=<BinaryCrossEntropyWithLogitsBackward>)\n",
      "66:loss------- tensor(0.1484, device='cuda:0', grad_fn=<BinaryCrossEntropyWithLogitsBackward>)\n",
      "67:loss------- tensor(0.1483, device='cuda:0', grad_fn=<BinaryCrossEntropyWithLogitsBackward>)\n",
      "68:loss------- tensor(0.1481, device='cuda:0', grad_fn=<BinaryCrossEntropyWithLogitsBackward>)\n",
      "69:loss------- tensor(0.1479, device='cuda:0', grad_fn=<BinaryCrossEntropyWithLogitsBackward>)\n",
      "70:loss------- tensor(0.1478, device='cuda:0', grad_fn=<BinaryCrossEntropyWithLogitsBackward>)\n",
      "71:loss------- tensor(0.1476, device='cuda:0', grad_fn=<BinaryCrossEntropyWithLogitsBackward>)\n",
      "72:loss------- tensor(0.1475, device='cuda:0', grad_fn=<BinaryCrossEntropyWithLogitsBackward>)\n",
      "73:loss------- tensor(0.1473, device='cuda:0', grad_fn=<BinaryCrossEntropyWithLogitsBackward>)\n",
      "74:loss------- tensor(0.1472, device='cuda:0', grad_fn=<BinaryCrossEntropyWithLogitsBackward>)\n",
      "75:loss------- tensor(0.1470, device='cuda:0', grad_fn=<BinaryCrossEntropyWithLogitsBackward>)\n",
      "76:loss------- tensor(0.1469, device='cuda:0', grad_fn=<BinaryCrossEntropyWithLogitsBackward>)\n",
      "77:loss------- tensor(0.1468, device='cuda:0', grad_fn=<BinaryCrossEntropyWithLogitsBackward>)\n",
      "78:loss------- tensor(0.1466, device='cuda:0', grad_fn=<BinaryCrossEntropyWithLogitsBackward>)\n",
      "79:loss------- tensor(0.1465, device='cuda:0', grad_fn=<BinaryCrossEntropyWithLogitsBackward>)\n",
      "80:loss------- tensor(0.1464, device='cuda:0', grad_fn=<BinaryCrossEntropyWithLogitsBackward>)\n",
      "81:loss------- tensor(0.1463, device='cuda:0', grad_fn=<BinaryCrossEntropyWithLogitsBackward>)\n",
      "82:loss------- tensor(0.1461, device='cuda:0', grad_fn=<BinaryCrossEntropyWithLogitsBackward>)\n",
      "83:loss------- tensor(0.1460, device='cuda:0', grad_fn=<BinaryCrossEntropyWithLogitsBackward>)\n",
      "84:loss------- tensor(0.1459, device='cuda:0', grad_fn=<BinaryCrossEntropyWithLogitsBackward>)\n",
      "85:loss------- tensor(0.1458, device='cuda:0', grad_fn=<BinaryCrossEntropyWithLogitsBackward>)\n",
      "86:loss------- tensor(0.1457, device='cuda:0', grad_fn=<BinaryCrossEntropyWithLogitsBackward>)\n",
      "87:loss------- tensor(0.1456, device='cuda:0', grad_fn=<BinaryCrossEntropyWithLogitsBackward>)\n",
      "88:loss------- tensor(0.1454, device='cuda:0', grad_fn=<BinaryCrossEntropyWithLogitsBackward>)\n",
      "89:loss------- tensor(0.1453, device='cuda:0', grad_fn=<BinaryCrossEntropyWithLogitsBackward>)\n",
      "90:loss------- tensor(0.1452, device='cuda:0', grad_fn=<BinaryCrossEntropyWithLogitsBackward>)\n",
      "91:loss------- tensor(0.1451, device='cuda:0', grad_fn=<BinaryCrossEntropyWithLogitsBackward>)\n",
      "92:loss------- tensor(0.1450, device='cuda:0', grad_fn=<BinaryCrossEntropyWithLogitsBackward>)\n",
      "93:loss------- tensor(0.1449, device='cuda:0', grad_fn=<BinaryCrossEntropyWithLogitsBackward>)\n",
      "94:loss------- tensor(0.1448, device='cuda:0', grad_fn=<BinaryCrossEntropyWithLogitsBackward>)\n",
      "95:loss------- tensor(0.1447, device='cuda:0', grad_fn=<BinaryCrossEntropyWithLogitsBackward>)\n",
      "96:loss------- tensor(0.1446, device='cuda:0', grad_fn=<BinaryCrossEntropyWithLogitsBackward>)\n",
      "97:loss------- tensor(0.1445, device='cuda:0', grad_fn=<BinaryCrossEntropyWithLogitsBackward>)\n",
      "98:loss------- tensor(0.1444, device='cuda:0', grad_fn=<BinaryCrossEntropyWithLogitsBackward>)\n",
      "99:loss------- tensor(0.1443, device='cuda:0', grad_fn=<BinaryCrossEntropyWithLogitsBackward>)\n"
     ]
    }
   ],
   "source": [
    "print(query_vec[0:10].shape, \n",
    "      label_text_vec[0:10].shape, \n",
    "      query_bert_vec[0:10].shape, \n",
    "      img_vec[0:10].shape, \n",
    "      pos_vec[0:10].shape, \n",
    "      np.array(padding_mask[0:10]).shape, \n",
    "      np.array(lstm_seq_length[0:10]).shape)\n",
    "\n",
    "query_vec1=torch.Tensor(query_vec[0:10].numpy().astype('float16'))\n",
    "label_text_vec1=torch.Tensor(label_text_vec[0:10].numpy().astype('float16'))\n",
    "query_bert_vec1=torch.Tensor(query_bert_vec[0:10].numpy().astype('float16'))\n",
    "img_vec1=torch.Tensor(img_vec[0:10].numpy().astype('float16'))\n",
    "pos_vec1=torch.Tensor(pos_vec[0:10].numpy().astype('float16'))\n",
    "\n",
    "\n",
    "labels=torch.Tensor(big_batch_train['label'])[:10]\n",
    "for i in range(100):\n",
    "    \n",
    "    optimizer.zero_grad()\n",
    "    prediction=model_ft(query_vec1.cuda(), label_text_vec1.cuda(), query_bert_vec1.cuda(), \n",
    "                 img_vec1.cuda(), pos_vec1.cuda(), torch.ByteTensor(padding_mask[0:10]).cuda(), lstm_seq_length[0:10])\n",
    "\n",
    "    loss = calc_loss(prediction, labels.cuda())\n",
    "    print(str(i)+':loss-------',loss)\n",
    "\n",
    "#             print('train prec:', cal_prec(predict,labels))\n",
    "\n",
    "    loss.backward()\n",
    "    optimizer.step()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# VALID DATA"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "10ad63fea5ec4ff7ab310b4ee6ec93b5",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, description='Pandas Apply', max=14720.0, style=ProgressStyle(descripti…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "806a94951ae74cc2ad38de6e046d67b0",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, description='Pandas Apply', max=14720.0, style=ProgressStyle(descripti…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "fe13ecbd14ab40329d7b9698b41d4a70",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, description='Pandas Apply', max=14720.0, style=ProgressStyle(descripti…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "3f41c5c71a0a48deb2cfed9de3c3a396",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, description='Pandas Apply', max=14720.0, style=ProgressStyle(descripti…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "83be3d2c937d4990a64e09edcf85c037",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, description='Pandas Apply', max=14720.0, style=ProgressStyle(descripti…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "e1345ab66c8842919bb26e95e7a03396",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, description='Pandas Apply', max=14720.0, style=ProgressStyle(descripti…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "643f873d893e4406ab7156e0442a007a",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, description='Pandas Apply', max=14720.0, style=ProgressStyle(descripti…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "29bca62087a747518f7e1a9bf39e1e6e",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, description='Pandas Apply', max=14720.0, style=ProgressStyle(descripti…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "7c7a17313dd947cea76f5e3bbfd2cec9",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, description='Pandas Apply', max=14720.0, style=ProgressStyle(descripti…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "49d1b943aa9d456ca5a9107a41449ce1",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, description='Pandas Apply', max=14720.0, style=ProgressStyle(descripti…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "6674f47ee4db4431a82cf61e36cc1260",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, description='Pandas Apply', max=14720.0, style=ProgressStyle(descripti…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>product_id</th>\n",
       "      <th>query_id</th>\n",
       "      <th>features</th>\n",
       "      <th>label_embedding</th>\n",
       "      <th>query_embedding</th>\n",
       "      <th>pos</th>\n",
       "      <th>num_box</th>\n",
       "      <th>num_query_char</th>\n",
       "      <th>query_albert_embedding</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>103047314</td>\n",
       "      <td>0</td>\n",
       "      <td>[[0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0,...</td>\n",
       "      <td>[[0.04083252, -0.043029785, 0.17004395, -0.861...</td>\n",
       "      <td>[[-0.18566895, 0.065979004, -0.25219727, -0.11...</td>\n",
       "      <td>[[0.105, 0.4175, 0.1425, 0.59625, 0.1417966534...</td>\n",
       "      <td>2</td>\n",
       "      <td>3</td>\n",
       "      <td>[0.73320353, -0.7702514, 0.52037656, -0.972297...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>103030678</td>\n",
       "      <td>0</td>\n",
       "      <td>[[0.0, 0.0, 0.0022290119, 0.0, 0.0, 0.18552563...</td>\n",
       "      <td>[[0.04083252, -0.043029785, 0.17004395, -0.861...</td>\n",
       "      <td>[[-0.18566895, 0.065979004, -0.25219727, -0.11...</td>\n",
       "      <td>[[0.52375, 0.97375, 0.00625, 0.29125, 0.128249...</td>\n",
       "      <td>9</td>\n",
       "      <td>3</td>\n",
       "      <td>[0.73320353, -0.7702514, 0.52037656, -0.972297...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>103031217</td>\n",
       "      <td>0</td>\n",
       "      <td>[[0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0,...</td>\n",
       "      <td>[[0.04083252, -0.043029785, 0.17004395, -0.861...</td>\n",
       "      <td>[[-0.18566895, 0.065979004, -0.25219727, -0.11...</td>\n",
       "      <td>[[0.264, 0.564, 0.09221311475409837, 0.9672131...</td>\n",
       "      <td>2</td>\n",
       "      <td>3</td>\n",
       "      <td>[0.73320353, -0.7702514, 0.52037656, -0.972297...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>103026621</td>\n",
       "      <td>0</td>\n",
       "      <td>[[0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0,...</td>\n",
       "      <td>[[-0.27514648, 0.08642578, -0.28344727, -0.252...</td>\n",
       "      <td>[[-0.18566895, 0.065979004, -0.25219727, -0.11...</td>\n",
       "      <td>[[0.4325, 0.63125, 0.566200215285253, 0.725511...</td>\n",
       "      <td>10</td>\n",
       "      <td>3</td>\n",
       "      <td>[0.73320353, -0.7702514, 0.52037656, -0.972297...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>103017048</td>\n",
       "      <td>0</td>\n",
       "      <td>[[0.0, 0.0, 0.05747965, 0.0, 0.0, 0.0, 0.0, 0....</td>\n",
       "      <td>[[0.04083252, -0.043029785, 0.17004395, -0.861...</td>\n",
       "      <td>[[-0.18566895, 0.065979004, -0.25219727, -0.11...</td>\n",
       "      <td>[[0.2375, 0.6975, 0.34875, 0.90375, 0.25529960...</td>\n",
       "      <td>2</td>\n",
       "      <td>3</td>\n",
       "      <td>[0.73320353, -0.7702514, 0.52037656, -0.972297...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>14715</th>\n",
       "      <td>103044330</td>\n",
       "      <td>500</td>\n",
       "      <td>[[0.0, 0.0, 0.027551055, 0.0, 0.32401124, 0.0,...</td>\n",
       "      <td>[[-0.19830322, 0.09790802, 0.42492676, -0.3308...</td>\n",
       "      <td>[[-0.59521484, 0.828125, -0.07165527, -0.15124...</td>\n",
       "      <td>[[0.24375, 0.41625, 0.04875, 0.23875, 0.032774...</td>\n",
       "      <td>5</td>\n",
       "      <td>3</td>\n",
       "      <td>[0.59812593, -0.666215, 0.56656677, -0.9123924...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>14716</th>\n",
       "      <td>103016765</td>\n",
       "      <td>500</td>\n",
       "      <td>[[0.0, 0.0, 4.595919, 0.0, 0.0, 0.03517687, 0....</td>\n",
       "      <td>[[0.107177734, -0.19818115, -0.11024475, 0.392...</td>\n",
       "      <td>[[-0.59521484, 0.828125, -0.07165527, -0.15124...</td>\n",
       "      <td>[[0.16533333333333333, 1.3266666666666667, 0.1...</td>\n",
       "      <td>3</td>\n",
       "      <td>3</td>\n",
       "      <td>[0.59812593, -0.666215, 0.56656677, -0.9123924...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>14717</th>\n",
       "      <td>103013406</td>\n",
       "      <td>500</td>\n",
       "      <td>[[0.017800761, 0.0, 0.050141126, 0.0, 0.0, 0.0...</td>\n",
       "      <td>[[0.10523733, -0.16297404, -0.12767537, -0.051...</td>\n",
       "      <td>[[-0.59521484, 0.828125, -0.07165527, -0.15124...</td>\n",
       "      <td>[[0.28, 1.4893333333333334, 0.0, 0.66666666666...</td>\n",
       "      <td>3</td>\n",
       "      <td>3</td>\n",
       "      <td>[0.59812593, -0.666215, 0.56656677, -0.9123924...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>14718</th>\n",
       "      <td>103012098</td>\n",
       "      <td>500</td>\n",
       "      <td>[[0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0,...</td>\n",
       "      <td>[[-0.27514648, 0.08642578, -0.28344727, -0.252...</td>\n",
       "      <td>[[-0.59521484, 0.828125, -0.07165527, -0.15124...</td>\n",
       "      <td>[[0.135, 0.211, 0.888, 0.996, 0.00820799179200...</td>\n",
       "      <td>6</td>\n",
       "      <td>3</td>\n",
       "      <td>[0.59812593, -0.666215, 0.56656677, -0.9123924...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>14719</th>\n",
       "      <td>103017846</td>\n",
       "      <td>500</td>\n",
       "      <td>[[0.0, 0.0, 2.8108034, 0.0, 0.0, 0.029381203, ...</td>\n",
       "      <td>[[-0.45617676, -0.29260254, -0.21557617, 0.159...</td>\n",
       "      <td>[[-0.59521484, 0.828125, -0.07165527, -0.15124...</td>\n",
       "      <td>[[0.00625, 0.16875, 0.43875, 0.5675, 0.0209218...</td>\n",
       "      <td>3</td>\n",
       "      <td>3</td>\n",
       "      <td>[0.59812593, -0.666215, 0.56656677, -0.9123924...</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>14720 rows × 9 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "       product_id  query_id  \\\n",
       "0       103047314         0   \n",
       "1       103030678         0   \n",
       "2       103031217         0   \n",
       "3       103026621         0   \n",
       "4       103017048         0   \n",
       "...           ...       ...   \n",
       "14715   103044330       500   \n",
       "14716   103016765       500   \n",
       "14717   103013406       500   \n",
       "14718   103012098       500   \n",
       "14719   103017846       500   \n",
       "\n",
       "                                                features  \\\n",
       "0      [[0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0,...   \n",
       "1      [[0.0, 0.0, 0.0022290119, 0.0, 0.0, 0.18552563...   \n",
       "2      [[0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0,...   \n",
       "3      [[0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0,...   \n",
       "4      [[0.0, 0.0, 0.05747965, 0.0, 0.0, 0.0, 0.0, 0....   \n",
       "...                                                  ...   \n",
       "14715  [[0.0, 0.0, 0.027551055, 0.0, 0.32401124, 0.0,...   \n",
       "14716  [[0.0, 0.0, 4.595919, 0.0, 0.0, 0.03517687, 0....   \n",
       "14717  [[0.017800761, 0.0, 0.050141126, 0.0, 0.0, 0.0...   \n",
       "14718  [[0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0,...   \n",
       "14719  [[0.0, 0.0, 2.8108034, 0.0, 0.0, 0.029381203, ...   \n",
       "\n",
       "                                         label_embedding  \\\n",
       "0      [[0.04083252, -0.043029785, 0.17004395, -0.861...   \n",
       "1      [[0.04083252, -0.043029785, 0.17004395, -0.861...   \n",
       "2      [[0.04083252, -0.043029785, 0.17004395, -0.861...   \n",
       "3      [[-0.27514648, 0.08642578, -0.28344727, -0.252...   \n",
       "4      [[0.04083252, -0.043029785, 0.17004395, -0.861...   \n",
       "...                                                  ...   \n",
       "14715  [[-0.19830322, 0.09790802, 0.42492676, -0.3308...   \n",
       "14716  [[0.107177734, -0.19818115, -0.11024475, 0.392...   \n",
       "14717  [[0.10523733, -0.16297404, -0.12767537, -0.051...   \n",
       "14718  [[-0.27514648, 0.08642578, -0.28344727, -0.252...   \n",
       "14719  [[-0.45617676, -0.29260254, -0.21557617, 0.159...   \n",
       "\n",
       "                                         query_embedding  \\\n",
       "0      [[-0.18566895, 0.065979004, -0.25219727, -0.11...   \n",
       "1      [[-0.18566895, 0.065979004, -0.25219727, -0.11...   \n",
       "2      [[-0.18566895, 0.065979004, -0.25219727, -0.11...   \n",
       "3      [[-0.18566895, 0.065979004, -0.25219727, -0.11...   \n",
       "4      [[-0.18566895, 0.065979004, -0.25219727, -0.11...   \n",
       "...                                                  ...   \n",
       "14715  [[-0.59521484, 0.828125, -0.07165527, -0.15124...   \n",
       "14716  [[-0.59521484, 0.828125, -0.07165527, -0.15124...   \n",
       "14717  [[-0.59521484, 0.828125, -0.07165527, -0.15124...   \n",
       "14718  [[-0.59521484, 0.828125, -0.07165527, -0.15124...   \n",
       "14719  [[-0.59521484, 0.828125, -0.07165527, -0.15124...   \n",
       "\n",
       "                                                     pos  num_box  \\\n",
       "0      [[0.105, 0.4175, 0.1425, 0.59625, 0.1417966534...        2   \n",
       "1      [[0.52375, 0.97375, 0.00625, 0.29125, 0.128249...        9   \n",
       "2      [[0.264, 0.564, 0.09221311475409837, 0.9672131...        2   \n",
       "3      [[0.4325, 0.63125, 0.566200215285253, 0.725511...       10   \n",
       "4      [[0.2375, 0.6975, 0.34875, 0.90375, 0.25529960...        2   \n",
       "...                                                  ...      ...   \n",
       "14715  [[0.24375, 0.41625, 0.04875, 0.23875, 0.032774...        5   \n",
       "14716  [[0.16533333333333333, 1.3266666666666667, 0.1...        3   \n",
       "14717  [[0.28, 1.4893333333333334, 0.0, 0.66666666666...        3   \n",
       "14718  [[0.135, 0.211, 0.888, 0.996, 0.00820799179200...        6   \n",
       "14719  [[0.00625, 0.16875, 0.43875, 0.5675, 0.0209218...        3   \n",
       "\n",
       "       num_query_char                             query_albert_embedding  \n",
       "0                   3  [0.73320353, -0.7702514, 0.52037656, -0.972297...  \n",
       "1                   3  [0.73320353, -0.7702514, 0.52037656, -0.972297...  \n",
       "2                   3  [0.73320353, -0.7702514, 0.52037656, -0.972297...  \n",
       "3                   3  [0.73320353, -0.7702514, 0.52037656, -0.972297...  \n",
       "4                   3  [0.73320353, -0.7702514, 0.52037656, -0.972297...  \n",
       "...               ...                                                ...  \n",
       "14715               3  [0.59812593, -0.666215, 0.56656677, -0.9123924...  \n",
       "14716               3  [0.59812593, -0.666215, 0.56656677, -0.9123924...  \n",
       "14717               3  [0.59812593, -0.666215, 0.56656677, -0.9123924...  \n",
       "14718               3  [0.59812593, -0.666215, 0.56656677, -0.9123924...  \n",
       "14719               3  [0.59812593, -0.666215, 0.56656677, -0.9123924...  \n",
       "\n",
       "[14720 rows x 9 columns]"
      ]
     },
     "execution_count": 10,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# load data\n",
    "valid_albert_embedding=joblib.load('data/valid_query_albert.pkl')\n",
    "valid=joblib.load('data/valid_data.pkl')\n",
    "# truncate\n",
    "valid['label_embedding']=valid.swifter.apply(lambda x: x['label_embedding'][:MAX_BOX],axis=1)\n",
    "valid['pos']=valid.swifter.apply(lambda x: x['pos'][:MAX_BOX],axis=1)\n",
    "valid['query_embedding']=valid.swifter.apply(lambda x: x['query_embedding'][:MAX_QUERY_CHAR],axis=1)\n",
    "valid['num_box']=valid.swifter.apply(lambda x: len(x['pos']),axis=1)\n",
    "valid['num_query_char']=valid.swifter.apply(lambda x: len(x['query_embedding']),axis=1)\n",
    "valid['features']=valid.swifter.apply(lambda x: [i for i in x['features']],axis=1)\n",
    "valid['features']=valid.swifter.apply(lambda x: x['features'][:MAX_BOX],axis=1)\n",
    "# padding\n",
    "def padding_seq(sequence, length, max_length):\n",
    "    if length >= max_length:\n",
    "        return sequence\n",
    "    else:\n",
    "        return sequence + [np.zeros(len(sequence[0])) for _ in range(max_length-length)]\n",
    "    \n",
    "valid['label_embedding']=valid.swifter.apply(lambda x: padding_seq(x['label_embedding'], x['num_box'], MAX_BOX),axis=1)\n",
    "valid['pos']=valid.swifter.apply(lambda x: padding_seq(x['pos'], x['num_box'], MAX_BOX),axis=1)\n",
    "valid['query_embedding']=valid.swifter.apply(lambda x: padding_seq(x['query_embedding'], x['num_query_char'], MAX_QUERY_CHAR),axis=1)\n",
    "valid['features']=valid.swifter.apply(lambda x: padding_seq(x['features'], x['num_box'], MAX_BOX),axis=1)\n",
    "valid['query_albert_embedding']=valid_albert_embedding['text_vec']\n",
    "gc.collect()\n",
    "valid"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "start\n",
      "0\n",
      "1000\n",
      "2000\n",
      "3000\n",
      "4000\n",
      "5000\n",
      "6000\n",
      "7000\n",
      "8000\n",
      "9000\n",
      "10000\n",
      "11000\n",
      "12000\n",
      "13000\n",
      "14000\n",
      "[[0, 103059361, 103017048, 103028493, 103020398, 103031217], [1, 103009387, 103065134, 103022099, 103049171, 103032961], [2, 103059018, 103059469, 103057259, 103039546, 103039137], [3, 103062592, 103040728, 103034297, 103046951, 103064331], [4, 103041391, 103042861, 103048149, 103028489, 103048840], [5, 103054044, 103033019, 103019152, 103007847, 103007999], [6, 103017342, 103038649, 103029444, 103000072, 103041148], [7, 103013757, 103041020, 103029770, 103023432, 103047484], [8, 103065668, 103040021, 103007787, 103061603, 103049972], [9, 103033259, 103040021, 103063825, 103061603, 103010304], [10, 103039501, 103028931, 103045719, 103049707, 103018933], [11, 103008253, 103049300, 103036002, 103008494, 103042885], [12, 103008191, 103018646, 103004892, 103042584, 103005351], [13, 103056700, 103004351, 103029379, 103056822, 103039899], [14, 103050414, 103038336, 103029023, 103012854, 103014269], [15, 103049190, 103063535, 103022584, 103002639, 103054344], [16, 103032192, 103040243, 103040492, 103005021, 103027014], [17, 103017785, 103006499, 103042956, 103005923, 103046185], [18, 103014889, 103035193, 103047396, 103039725, 103062572], [19, 103017923, 103007847, 103044472, 103015032, 103012103], [20, 103009922, 103026549, 103013648, 103060274, 103004289], [21, 103019534, 103036133, 103058358, 103046620, 103036746], [22, 103012397, 103010599, 103061698, 103037835, 103062284], [23, 103032359, 103037470, 103066758, 103031425, 103033407], [24, 103062396, 103056822, 103039899, 103064438, 103034701], [25, 103016662, 103067003, 103004374, 100705143, 103052090], [26, 103042161, 103062913, 103039664, 103020228, 103065065], [27, 103049065, 103058734, 103043401, 103050353, 103031895], [28, 103058333, 103017208, 103024762, 103064771, 103009874], [29, 103031074, 103043031, 103028424, 103006765, 103057233], [30, 103033633, 103009553, 103052467, 103050789, 103037984], [31, 103026392, 103026596, 103062739, 103034512, 103065152], [32, 103042988, 103007218, 103044054, 103048840, 103050899], [33, 103014889, 103057890, 103039725, 103044881, 103062361], [34, 103008291, 103058464, 103027402, 103034108, 103021319], [35, 103054938, 103002085, 103048539, 103053062, 103037566], [36, 103034167, 103002508, 103032732, 103047359, 103047700], [37, 103019817, 101003809, 103007308, 103043602, 103060483], [38, 103021726, 103046705, 103061079, 103001398, 103030271], [39, 103026288, 103018242, 103051989, 103009786, 103065750], [40, 103040583, 103009631, 103015105, 103037010, 103025469], [41, 103063825, 103012215, 103027101, 103061603, 103063648], [42, 103061702, 103044893, 103054440, 103021241, 103035465], [43, 103005913, 103032359, 103044806, 103053155, 103003163], [44, 103047268, 103006984, 103030171, 103065568, 103001854], [45, 103013021, 103060296, 103043408, 103039191, 103019072], [46, 103007155, 103049248, 103003558, 103018808, 103019197], [47, 103027506, 103046125, 103045946, 103047590, 103014548], [48, 103035193, 103057890, 103039725, 103062572, 103031533], [49, 103010683, 103016108, 103057074, 103009263, 103047909], [50, 103006635, 103009944, 103062592, 103020371, 103016681], [51, 103011108, 103065939, 103042693, 103024786, 103053721], [52, 103039986, 103047896, 103002498, 103009013, 103021772], [53, 103018608, 103053768, 103049272, 103035314, 103009841], [54, 103024252, 103011719, 103021726, 103048370, 103026560], [55, 103055856, 103006984, 103007812, 103030171, 103025469], [56, 103035193, 103062572, 103044881, 103008301, 103050471], [57, 103005206, 103059970, 103030731, 103064176, 103016093], [59, 103064677, 103054414, 103037398, 103029387, 103037349], [60, 103055856, 103011449, 103062822, 103002700, 103033767], [61, 103026785, 103048863, 103033419, 103006969, 103066912], [62, 103057890, 103051038, 103035422, 103052613, 103001216], [63, 103029361, 103004289, 103057449, 103015032, 103008480], [64, 103013138, 103015833, 103026330, 103011990, 103054714], [65, 103012651, 103049140, 103012582, 103053162, 103007125], [66, 103011035, 103057890, 103052613, 103001216, 103014889], [67, 103054847, 103050388, 103004670, 103041335, 103049653], [68, 103006930, 103032359, 103036020, 103034838, 103048256], [69, 103018244, 103054944, 103038166, 103028032, 103050055], [70, 103058908, 103064868, 103022545, 103010263, 102406628], [71, 103015366, 103007549, 103033317, 103058035, 103060186], [72, 103029776, 103006892, 103032408, 103021880, 103004839], [73, 103057493, 103023442, 103022679, 103024270, 103055200], [74, 103061799, 103033283, 103043118, 103007637, 103064520], [75, 103001806, 103002508, 103051468, 103011601, 103009976], [76, 103000469, 103066091, 103021823, 103062799, 103045948], [77, 103051636, 103048841, 103030140, 103006261, 103028476], [78, 103031987, 103028950, 103003292, 103003009, 103006320], [79, 103058633, 103003972, 103000427, 103006249, 103013966], [80, 103044120, 103011458, 103054704, 103018625, 103001287], [81, 103005069, 103062093, 103003859, 103053705, 103023304], [82, 103024053, 103005546, 103029776, 103056822, 103021880], [83, 103017824, 103042954, 103052326, 103007895, 103034339], [84, 103005190, 103028222, 103064329, 103060981, 103030591], [85, 103058016, 103060376, 103056903, 103004949, 103020300], [86, 103045683, 103048734, 103028662, 103042338, 103029482], [87, 103049371, 103023903, 103002912, 103032953, 103065768], [88, 103039397, 103066917, 103012776, 103060583, 103010882], [89, 103020204, 103037241, 103046529, 103003729, 103006404], [90, 103047010, 103048149, 103059469, 103060830, 103035830], [91, 103006819, 103032207, 103041098, 103054935, 103004730], [92, 103055656, 103062396, 103056411, 103015007, 103057949], [93, 103001114, 103002772, 103056023, 103066227, 103044650], [94, 103056023, 103055718, 103016017, 103039172, 103004263], [95, 103057728, 103048804, 103027579, 103043195, 103057890], [96, 103035193, 103014889, 103057890, 103011035, 103050471], [97, 103018675, 103029675, 103037519, 103060590, 103022025], [98, 103027731, 103061218, 103057562, 103013017, 103062417], [99, 103062572, 103035193, 103007340, 103015422, 103060811], [100, 103044806, 103056025, 103038249, 103012284, 103040723], [101, 103036180, 103036723, 103020103, 103046224, 103064044], [102, 103023089, 103009922, 103055512, 103037194, 103012103], [103, 103027579, 103047291, 103063996, 103055189, 103036156], [104, 103055644, 103035984, 103047809, 103057804, 103053551], [105, 103002772, 103014612, 103006733, 103066227, 103044650], [106, 103016017, 103056023, 103001114, 103014612, 103039172], [107, 103001220, 103039899, 103056822, 103010786, 103013823], [108, 103066370, 103050137, 103009810, 103034675, 103016095], [109, 103062541, 103005621, 103006328, 103030129, 103029995], [110, 103007874, 103015605, 103061403, 103056252, 103020101], [111, 103031154, 103037264, 103011719, 103018347, 103029068], [112, 103016662, 103004374, 103005249, 103058920, 100705143], [113, 103007569, 103025355, 103064146, 103029553, 103037010], [114, 103046485, 103059883, 103034287, 103055434, 103013951], [115, 103024163, 103061191, 103015366, 103060186, 103033192], [116, 103017471, 103065870, 103037932, 103047494, 103008417], [117, 103040124, 103047467, 103016559, 103051417, 103006397], [118, 103001100, 103060998, 103061627, 103002292, 103004026], [119, 103037938, 103063972, 103035360, 103010474, 103031395], [120, 103035179, 103013111, 103054599, 103008661, 103015513], [121, 103030049, 103046927, 103042985, 103048021, 103049530], [122, 103028751, 103057367, 103040020, 103054401, 103011273], [123, 103018143, 103025248, 103033196, 103054554, 103017136], [124, 103047720, 103012668, 103041207, 103018405, 103015867], [125, 103027393, 103031012, 103060293, 103047042, 103008121], [126, 103034766, 102137929, 103022099, 103065134, 103008355], [127, 103018719, 103044298, 103050209, 103023699, 103045954], [128, 103017892, 103019283, 103030130, 103016353, 103053344], [129, 103044657, 103015451, 103051758, 103056063, 103017963], [130, 103048946, 103022431, 103053979, 103030120, 103061131], [131, 103060531, 103023283, 101905883, 103031068, 103017129], [132, 103025280, 103060074, 103020504, 103032289, 103053039], [133, 103024252, 103034994, 103040271, 103026560, 103055395], [134, 103060419, 103006885, 103002921, 103000825, 103034606], [135, 103020909, 103045451, 103002693, 103014581, 103047386], [136, 103039297, 103012050, 103064567, 103032763, 103065605], [137, 103063045, 103060192, 103025477, 103003126, 103005924], [138, 103008732, 103031819, 103032545, 103028541, 103012733], [139, 103009324, 103001285, 103049636, 103045345, 103030864], [140, 103035014, 103031140, 103006758, 103042220, 103009427], [141, 103053156, 103054102, 103001220, 103052394, 103002557], [142, 103021438, 103009066, 103059443, 103043661, 103050594], [143, 103023328, 103038398, 103038377, 103048641, 103046356], [144, 103026544, 103039277, 103062701, 103061322, 103004804], [145, 103044982, 103009737, 103066487, 103063373, 103014599], [146, 103055856, 103007812, 103017005, 103050766, 103045682], [147, 103051099, 103043704, 103063209, 103020177, 103034512], [148, 103059052, 103066322, 103004315, 103058139, 103000734], [149, 103035193, 103027611, 103011147, 103038206, 103027352], [150, 103008173, 103020070, 103033071, 103052561, 103043978], [151, 103019370, 103018140, 103066407, 103028152, 103025020], [152, 103009140, 103027446, 103037581, 103064520, 103033318], [153, 103061974, 103042785, 103041171, 103022424, 103053222], [154, 103008000, 103052741, 103054510, 103019972, 103028777], [155, 103014917, 103016017, 103002772, 103037919, 103001608], [156, 103057074, 103012716, 103010683, 103009156, 103011604], [157, 103042430, 103031371, 103011099, 103050414, 103046249], [158, 103035396, 103018286, 103043607, 103006807, 103059695], [159, 103014917, 103037919, 103014005, 103016088, 103042876], [160, 103063488, 103041824, 103027879, 103035547, 103026314], [161, 103061698, 103054989, 103012397, 103010599, 103011169], [162, 103010608, 103018899, 103033758, 103013962, 103027428], [163, 103017632, 103044804, 103049952, 103039949, 103061599], [164, 103022228, 103001248, 103034339, 103038160, 103036928], [165, 103023745, 103008810, 103046863, 103052891, 103001954], [166, 103037645, 103010786, 103053162, 103007125, 103024901], [167, 103005300, 103052394, 103023516, 103016361, 103006897], [168, 103009553, 103029636, 103051449, 103024338, 103025402], [169, 103040419, 103062448, 103066613, 103055708, 103035420], [170, 103051099, 103063209, 103008156, 103005927, 103057008], [171, 103055673, 103066551, 103059840, 103002722, 103000969], [172, 103038280, 103008463, 103024396, 103039110, 103062898], [173, 103026340, 103026911, 103061075, 103011514, 103011159], [174, 103052305, 103042572, 103011826, 103018040, 103017022], [175, 103047844, 103031567, 103065954, 103025508, 103017400], [176, 103006872, 103028222, 103005190, 103057880, 103049639], [177, 103040829, 103032181, 103024006, 103059300, 103041010], [178, 103063915, 103009057, 103043689, 103062568, 103028563], [179, 103060981, 103008463, 103028101, 103059372, 103065919], [180, 103021350, 103002851, 103005459, 103010786, 103032454], [181, 103041553, 103062658, 103021746, 103049297, 103061648], [182, 103011868, 103065653, 103007700, 103009685, 103060700], [183, 103001220, 103052394, 103010786, 103013823, 103053162], [184, 103015917, 100829477, 103035133, 103065548, 103013985], [185, 103057363, 103026850, 103021727, 103011171, 103003910], [186, 103027549, 103005234, 103018585, 103055571, 103006491], [187, 103013021, 103015593, 103056257, 103033050, 103041263], [188, 103050810, 103056822, 103029379, 103064438, 103039899], [189, 103034750, 103027284, 103012121, 103040459, 103044671], [190, 103020607, 103024435, 103006321, 103032545, 103054364], [191, 103052116, 103052452, 103015390, 103000731, 103044846], [192, 103010359, 101162349, 100113596, 103056866, 103002400], [193, 103018717, 103015382, 103008896, 103054632, 103010888], [194, 103005190, 103008463, 103018150, 103028941, 103000731], [195, 103066767, 103024218, 103045911, 103040395, 103011411], [196, 103028222, 103005190, 103038280, 103057880, 103025072], [197, 103014612, 103033962, 103039172, 103002772, 103006733], [198, 103054948, 103011502, 103051717, 103024780, 103028456], [199, 103025522, 103063956, 103018200, 103007665, 103039852], [200, 103001114, 103002772, 103014612, 103066227, 103006733], [201, 103056023, 103004263, 103016017, 103002772, 103006733], [202, 103032036, 103040717, 103047677, 103052128, 103050796], [203, 103020485, 103059228, 103060186, 103039211, 103009922], [204, 103048003, 103039137, 103004275, 103039032, 103002921], [205, 103002127, 103066762, 103064112, 103064987, 103032899], [206, 103032587, 103057949, 103015268, 103029776, 103006892], [207, 103062036, 103019772, 103006465, 103001782, 103000087], [208, 103003061, 103043723, 103004643, 103018942, 103001038], [209, 103045346, 103050758, 103049993, 103017158, 103064520], [210, 103008836, 103018912, 103006466, 103024515, 103007091], [211, 103036020, 103036156, 103006930, 103014030, 103009813], [212, 103064911, 103038249, 103010675, 103029804, 103023403], [213, 103053162, 103013823, 103057949, 103058909, 103035604], [214, 103019300, 103002675, 103010042, 103029615, 103045507], [215, 103029776, 103006892, 103002701, 103005546, 103064438], [216, 103055856, 103022275, 103051426, 103050512, 103025206], [217, 103001639, 103016971, 103063719, 103040124, 103016194], [218, 103058032, 103063744, 103026784, 103052936, 103032605], [219, 103021911, 103008291, 103061530, 103058464, 103009333], [220, 103013662, 103019745, 103052719, 103063609, 103007634], [221, 103031699, 103052394, 103023516, 103046558, 103037311], [222, 103007823, 103064520, 103046451, 103032386, 103033337], [223, 103009813, 103032359, 103021798, 103008809, 103000103], [224, 103002772, 103014612, 103066227, 103044650, 103006733], [225, 103035193, 103055561, 103062572, 103057090, 103011035], [226, 103011891, 103014837, 103008930, 103007042, 103056023], [227, 103003973, 103004574, 103042197, 103011889, 103007968], [228, 103033684, 103014363, 103063013, 103017052, 103049924], [229, 103046885, 103054347, 103043910, 103040887, 103033318], [230, 103031568, 103021530, 103042967, 103027616, 103011681], [231, 103001114, 103056023, 103039172, 103016017, 103012458], [232, 103006930, 103015288, 103027579, 103030627, 103034838], [233, 103031342, 103027030, 103001254, 103012914, 103037957], [234, 103056426, 103029496, 103065304, 103029804, 103033333], [235, 103057952, 103030171, 103040689, 103055856, 103002181], [236, 103035657, 103052394, 103001220, 103053162, 103012582], [237, 103059891, 103028189, 103024532, 103063913, 103038725], [238, 103007101, 103033063, 103013924, 103040777, 103040699], [239, 103009056, 103055310, 103045040, 103018456, 103055759], [240, 103036075, 103060601, 103051057, 103040740, 103047734], [241, 103066317, 103045713, 103030637, 103015206, 103000415], [242, 103017396, 103022635, 103000522, 103034516, 103015469], [243, 103056080, 103045300, 103003989, 103054678, 103017390], [244, 103034864, 103032359, 103036020, 103046258, 103034838], [245, 103012597, 103015366, 103028718, 103053996, 103057143], [246, 103052083, 103001220, 103036158, 103013823, 103000830], [247, 103053340, 103037542, 103059015, 103012475, 103061703], [248, 103014860, 103033949, 103049832, 103015658, 103050924], [249, 103045316, 103029565, 103046564, 103051687, 103065619], [250, 103064646, 103046234, 103032359, 103056978, 103030817], [251, 103029676, 103034780, 103031090, 103040306, 103007789], [252, 103060252, 103022643, 103036076, 103005904, 103031621], [253, 103029045, 103056274, 103028173, 103007904, 103007468], [254, 103000810, 103029379, 103039899, 103064438, 103007921], [255, 103046857, 103055856, 100544434, 103021726, 103051426], [256, 103062396, 103043119, 103015881, 103039899, 103064438], [257, 103014612, 103012458, 103039172, 103016017, 103002772], [258, 103036606, 103001220, 103013823, 103053162, 103007125], [259, 103062462, 103023304, 103056196, 103028653, 103060020], [260, 103055048, 103059355, 103046041, 103034535, 103000104], [261, 103003595, 103041756, 103030111, 103047764, 103030955], [262, 103023109, 103025766, 103031874, 103065177, 103044510], [263, 103048025, 103031980, 103000708, 103009727, 103055153], [264, 103001114, 103039172, 103012458, 103056023, 103002772], [265, 103040811, 103046519, 103040344, 103033850, 103013040], [266, 103005300, 103063825, 103040021, 103013823, 103012582], [267, 103044707, 103048445, 103006671, 103002516, 103058545], [268, 103008722, 103044707, 103036625, 103039280, 103001302], [269, 103055322, 103029604, 103035783, 103038584, 103036600], [270, 103054849, 103031025, 103050213, 103041838, 103000196], [271, 103021460, 103015076, 103020307, 103020259, 103036464], [272, 103016361, 103053134, 103052394, 103064341, 103005300], [273, 103061630, 103048852, 103062368, 103000660, 103049158], [274, 103019631, 103026228, 103040861, 103007025, 103016189], [275, 103060913, 103010227, 103030278, 103043687, 103040856], [276, 103054342, 103008909, 103024878, 103065455, 103014543], [277, 103008393, 103014892, 103003911, 103053696, 103033039], [278, 103011913, 103014368, 103022319, 103053712, 103013615], [279, 103006930, 103009813, 103001305, 103032359, 103036020], [280, 103001856, 103065954, 103056606, 103033647, 100640027], [281, 103015614, 103024324, 103062372, 103032650, 103045920], [282, 103056640, 103006930, 103036020, 103009813, 103034838], [283, 103065129, 103056903, 103020300, 103022523, 103010132], [284, 103030563, 103056335, 103059443, 103014182, 103030955], [285, 103052171, 103063154, 103041986, 103041048, 103060759], [286, 103041640, 103020683, 103031161, 103037221, 103037346], [287, 103066243, 103025866, 103046969, 103004191, 103059268], [288, 103054949, 103034046, 103039206, 103042027, 103018077], [289, 103051210, 103056901, 103059355, 103050533, 103021535], [290, 103040811, 103016085, 103040344, 103046519, 103056126], [291, 103037638, 103011089, 103033962, 103004661, 103026375], [292, 103044024, 103047256, 103051924, 103029025, 103036932], [293, 103015789, 103048494, 103015408, 103044726, 103021036], [294, 103050106, 103045752, 103031140, 103047392, 103002027], [295, 103010786, 103025866, 103013823, 103053162, 103059268], [296, 103004898, 103030092, 103002069, 103021827, 103000704], [297, 103059012, 103026242, 103012568, 103055048, 103002415], [298, 103031819, 103026800, 103033545, 103022816, 103024435], [299, 103048840, 103028489, 103006885, 103029698, 103042115], [300, 103042546, 103060489, 103037453, 103016366, 103035380], [301, 103034580, 103038341, 103051906, 103058656, 103061530], [302, 103024524, 103066065, 103034672, 103063552, 103018276], [303, 103027579, 103044806, 103014213, 103040723, 103016165], [304, 103006930, 103036020, 103029119, 103038249, 103014174], [305, 103048670, 103008927, 103004015, 103037663, 103001480], [306, 103043321, 103058380, 103046746, 103048901, 103004888], [307, 103025858, 103016715, 103016736, 103009033, 103045604], [308, 103011061, 103032468, 103029721, 103006612, 103016656], [309, 103022495, 103055764, 103008510, 103045560, 103006819], [310, 103052736, 103032919, 103032816, 103046759, 103028407], [311, 103059469, 103008190, 103028489, 103029698, 103040121], [312, 103057074, 103023470, 103009156, 103002498, 103011604], [314, 103063340, 103008369, 103000005, 103006617, 103026743], [315, 103019688, 103018611, 103042293, 103030594, 103021811], [316, 103057508, 103053714, 103011035, 103050471, 103024250], [317, 103034182, 103001220, 103024454, 103043686, 103053162], [318, 103056990, 103043602, 103000331, 103004894, 103060483], [319, 103020745, 103016266, 103051176, 103041440, 103044728], [320, 103062396, 103029379, 103039899, 103064438, 103053177], [321, 103017253, 103001507, 103065018, 103006086, 103036007], [322, 103065649, 103014834, 103027633, 103007320, 103000805], [323, 103054675, 103064602, 103021811, 103048125, 103011935], [324, 103006930, 103032359, 103034838, 103039725, 103011147], [325, 103007847, 103002417, 103056196, 103051833, 103017724], [326, 103054401, 103036634, 103015213, 103055677, 103005801], [327, 103066748, 103015105, 103055012, 103030955, 103030171], [328, 103062396, 103056822, 103039899, 103029776, 103024053], [329, 103015792, 103002571, 103020292, 103049221, 103027743], [330, 103065930, 103014238, 103060812, 103060723, 103031820], [331, 103014612, 103024111, 103056023, 103012458, 103066524], [332, 103012140, 103056023, 103016017, 103014612, 103002772], [333, 103037010, 103021726, 103057452, 103054209, 103031427], [334, 103039172, 103002772, 103014612, 103012458, 103006733], [335, 103026647, 103046397, 103019982, 103053980, 103039872], [336, 103001114, 103039774, 103002772, 103056023, 103016017], [337, 103040706, 103051641, 103043272, 103064509, 103063969], [338, 103018538, 103052812, 103059457, 103020198, 103060258], [339, 103051098, 103033757, 103038938, 103060290, 103010357], [340, 103008217, 103053199, 103000069, 103022458, 103058532], [341, 103019439, 103002976, 103037631, 103062141, 103053205], [342, 103045142, 103025706, 103011908, 103045215, 103054352], [343, 103016642, 103011473, 103018765, 103008021, 103047558], [344, 103064510, 103027171, 103044158, 103033906, 101205239], [345, 103039725, 103056532, 103035193, 103011035, 103047967], [346, 103014889, 103007340, 103030635, 103044881, 103050471], [347, 103030730, 103037010, 103012242, 103051426, 103020731], [348, 103005300, 103023516, 103005805, 103027908, 103048717], [349, 103062572, 103035193, 103044881, 103011147, 103007340], [350, 103031976, 103054881, 103052394, 103040642, 103012387], [351, 103059215, 103047309, 103050821, 103011052, 103006012], [352, 103056240, 103052394, 103022039, 103065640, 103024376], [353, 103016931, 103051649, 103029789, 103066937, 103036152], [354, 103029804, 103052001, 103063479, 103010236, 103042089], [355, 103003060, 103011990, 103048149, 103029142, 103028489], [356, 103046326, 103042693, 103005194, 103059694, 103019075], [357, 103052394, 103012582, 103053162, 103007125, 103024376], [358, 103062572, 103014889, 103057890, 103050471, 103043127], [359, 103003721, 103007989, 103007785, 103013217, 103046548], [360, 103036142, 103025936, 103036156, 103034838, 103064568], [361, 103053433, 103051546, 103004963, 103065198, 103066530], [362, 103013233, 103030866, 103021205, 103028283, 103007756], [363, 103066524, 103012954, 103001897, 103000960, 103036281], [364, 103031705, 103021798, 103000103, 103037458, 103036020], [365, 103066502, 103006404, 103049158, 103003397, 103023036], [366, 103059469, 103060207, 103028489, 103029698, 103059092], [367, 103001727, 103002610, 103010384, 103001143, 103030767], [368, 103039172, 103056023, 100454655, 103014612, 103006733], [369, 103040820, 103012938, 103021495, 103055451, 103020438], [370, 103030786, 103028483, 103041353, 103060485, 103057091], [371, 103010887, 103045292, 103055841, 103055451, 103007216], [372, 103029427, 103029544, 103013694, 103048454, 103000541], [373, 103019030, 103033282, 103066606, 103002831, 103030725], [374, 103051480, 103037016, 103030570, 103024624, 103048886], [375, 103033251, 103004053, 103016331, 103036041, 103002534], [376, 103024978, 103042679, 102301030, 103016656, 103008915], [377, 103029708, 103059299, 103007812, 103003595, 103041756], [378, 103014612, 103056023, 103002772, 103001114, 103066227], [379, 103002772, 103064582, 103006733, 103066227, 103044650], [380, 103046228, 103015230, 103024173, 103011554, 103031025], [381, 103057074, 103014093, 103010683, 103004988, 103009256], [382, 103047044, 103059934, 103023075, 103058765, 103054979], [383, 103007643, 103009832, 103024891, 103006264, 103065095], [384, 103003973, 103064096, 103019072, 100417141, 103013021], [385, 103062396, 103029776, 103006975, 103024053, 103005546], [386, 103061588, 103048025, 103059718, 103057899, 103049700], [387, 103027692, 103001138, 103009324, 103038369, 103031140], [388, 103046848, 103002138, 103058400, 103042241, 103029698], [389, 103023247, 103052509, 103004915, 103004779, 103062173], [390, 103020941, 103066324, 103066185, 103013757, 103050466], [391, 102184985, 103063487, 103057038, 103026868, 103061322], [392, 103046933, 103065842, 103026617, 103008750, 103058572], [394, 103028376, 103042428, 103023959, 103058871, 103052478], [395, 103011634, 103017910, 103026787, 103014742, 103054955], [396, 103001581, 103058464, 103048388, 103040552, 103040259], [397, 103007927, 103051493, 103010657, 103016404, 103043428], [398, 103046668, 103051510, 103037588, 103007021, 103028926], [399, 103050858, 103050215, 103063534, 103064771, 103058250], [400, 103001220, 103010786, 103013823, 103053162, 103025866], [402, 103017441, 103042667, 103059980, 103001402, 103060926], [403, 103024846, 103038033, 103009004, 103060812, 103060723], [405, 103054755, 103011554, 103037400, 103021815, 103021221], [406, 103057074, 103050580, 103020029, 103009256, 103011604], [407, 103014612, 103002772, 103056023, 103016017, 103006733], [408, 103026330, 103051515, 103045281, 103016636, 103007445], [409, 103009304, 103059052, 103016201, 103006386, 103026339], [410, 103063509, 103026847, 103042267, 103047808, 103015321], [411, 103002848, 103005906, 103029817, 103037503, 103000331], [412, 103019805, 103048868, 103050858, 103047138, 103003764], [413, 103037387, 103050858, 103050215, 103062203, 103003764], [414, 103036158, 103029747, 103006886, 103013823, 103053162], [415, 103006481, 103015287, 103041998, 103063955, 103009893], [416, 103025149, 103028687, 103028141, 103033384, 103007741], [417, 103063825, 103027101, 103065668, 103032515, 103042959], [418, 103010381, 103002772, 103006733, 103066227, 103044650], [419, 103001114, 103056023, 103014612, 103012458, 103039172], [420, 103062796, 103056822, 103050865, 103064928, 103006161], [421, 103029804, 103021131, 103066509, 103036156, 103009012], [422, 103006892, 103029776, 103008836, 103047989, 103033878], [423, 103018611, 103042293, 103033664, 103057905, 103051130], [424, 103047833, 103056822, 103019507, 103029379, 103004886], [425, 103039206, 103064824, 103013772, 103016635, 103006062], [426, 103001114, 103043305, 103056023, 103016017, 103014612], [427, 103043856, 103063670, 103030571, 103000852, 103062335], [428, 103022848, 103012663, 103030895, 103025172, 103033923], [429, 103053980, 103000069, 103056871, 103023948, 103055694], [430, 103003987, 103009739, 103032596, 103008570, 103029814], [431, 103026835, 103032155, 103035298, 103025163, 103006851], [432, 103043789, 103007217, 103009205, 103001124, 103038799], [433, 103057074, 103004869, 103010683, 103047909, 103003417], [434, 103005069, 103007974, 103062093, 103052471, 103043961], [435, 103005757, 103039614, 103066459, 103002181, 103012580], [436, 103019700, 103044754, 103065841, 103047268, 103021726], [437, 103007170, 103043961, 103009922, 103007847, 103012103], [438, 103001114, 103002772, 103044650, 103066227, 103048154], [439, 103065328, 103066324, 103064267, 103064199, 103005815], [440, 103019822, 103019604, 103047198, 103006574, 103018611], [441, 103062396, 103001220, 103063197, 103012446, 103048795], [442, 103015357, 103059469, 103039137, 103016660, 103021989], [443, 103061879, 103018358, 103021726, 103047542, 103032642], [444, 103038953, 103002520, 103041998, 103018611, 103025546], [445, 103014238, 103031820, 103018898, 103046248, 103059536], [446, 103027934, 103015881, 103018904, 103054665, 103034704], [447, 103051125, 103035334, 103009451, 103016190, 103018902], [448, 103021811, 103006694, 103011935, 103004633, 103004404], [449, 103041256, 103050274, 103011053, 103050850, 103064520], [450, 103007523, 103051594, 103048300, 103060621, 103033318], [451, 103053350, 101916078, 103048149, 103031000, 103011211], [452, 103064211, 103011097, 103040493, 103041225, 103026108], [453, 103029776, 103050021, 103046602, 103026181, 103006388], [454, 103045830, 103049635, 103024989, 103033218, 103051016], [455, 103029090, 103047125, 103028437, 103006090, 103052415], [456, 103042400, 103044439, 103015661, 103019172, 103040404], [457, 103016017, 103014612, 103039172, 103012458, 103052872], [458, 103048071, 103066758, 103004309, 103051148, 103040810], [459, 103015378, 103050371, 103004867, 103035666, 103056146], [460, 103014889, 103062572, 103050471, 103047967, 103057890], [461, 103055677, 103054401, 103015213, 103065357, 101293512], [462, 103032934, 103028762, 103056517, 103031154, 103047535], [463, 103049079, 103065007, 103016875, 103042764, 103051614], [464, 103011035, 103047967, 103036702, 103035422, 103009629], [465, 103062572, 103014889, 103039725, 103050471, 103044881], [466, 103043393, 103017181, 103059538, 103013743, 103041724], [467, 103056023, 103039172, 103002772, 103012458, 103006733], [468, 103013243, 103017414, 103005351, 101512688, 103029796], [469, 103014530, 103000384, 103040881, 103055153, 103002732], [470, 103050215, 103058333, 103033358, 103024762, 103003764], [471, 103017735, 103063450, 103039575, 103014987, 103002497], [472, 103011990, 103056385, 103045566, 103038334, 103031000], [473, 103060918, 103031140, 103066291, 103048190, 103017165], [474, 103001114, 103039172, 103002772, 103006733, 103066227], [475, 103012031, 103047493, 103035497, 103030720, 103006361], [476, 103007789, 103009490, 103048473, 103052394, 103013850], [477, 103058862, 103046563, 103033412, 103009553, 103060879], [478, 103014612, 103056023, 103066524, 103000960, 103012458], [479, 103058134, 103019272, 103046000, 103023992, 103041784], [480, 103042620, 103017483, 103052257, 103004951, 103057670], [481, 103043222, 103029444, 103019373, 103049683, 103002972], [482, 103011035, 103039725, 103001145, 103035193, 103001216], [483, 103015833, 103031000, 103036969, 103054714, 103058182], [484, 103012582, 103053162, 103034780, 103065874, 103001220], [485, 103028149, 103029129, 103043377, 103027743, 103000736], [486, 103018488, 103051019, 103034780, 103009490, 103016304], [487, 103033936, 103062396, 103064438, 103005300, 103056822], [488, 103001114, 103016017, 103002772, 103014612, 103035809], [489, 103026860, 103064232, 103061110, 103048692, 103052127], [490, 103066684, 103010317, 103003950, 103008419, 103056311], [491, 103063720, 103030955, 103021726, 103008396, 103008272], [492, 103003181, 103030501, 103048830, 103050172, 103020408], [493, 103015746, 103066910, 103020600, 103039521, 103017708], [494, 103042798, 103028494, 103013369, 103045133, 103038537], [495, 103056980, 103032934, 103062416, 103056517, 103041506], [496, 103061127, 103017742, 103030368, 103050771, 103019377], [497, 103035788, 103060860, 103060783, 103011726, 103016524], [498, 103006923, 103013021, 103035595, 103035303, 103007452], [499, 103040130, 103004211, 103056018, 103004228, 103055945], [500, 103044806, 103057030, 103006399, 103040723, 103014213]]\n"
     ]
    }
   ],
   "source": [
    "# predict\n",
    "\n",
    "import warnings\n",
    "warnings.filterwarnings(\"ignore\")\n",
    "\n",
    "model_ft = Model(config).to(torch.device(\"cpu\"))\n",
    "model_ft.load_state_dict(torch.load('model/0_200000_attention_bilstm_parameter.pkl'))\n",
    "model_ft.eval()\n",
    "\n",
    "query_vec = torch.Tensor(valid['query_embedding'])\n",
    "lstm_seq_length = list(valid['num_query_char'])\n",
    "label_text_vec = torch.Tensor(valid['label_embedding'])\n",
    "query_bert_vec = torch.Tensor(valid['query_albert_embedding']).view(14720, 1, 768)\n",
    "img_vec = torch.Tensor(valid['features'])\n",
    "pos_vec = torch.Tensor(valid['pos'])\n",
    "\n",
    "padding_mask =[]\n",
    "for i in list(valid['num_box']):\n",
    "    mask=[0]*(i+2) + [1]*(MAX_BOX-i)\n",
    "    padding_mask.append(mask)\n",
    "padding_mask = torch.ByteTensor(padding_mask)\n",
    "gc.collect()\n",
    "        \n",
    "# prediction=model_ft(query_vec.cuda(), label_text_vec.cuda(), query_bert_vec.cuda(), img_vec.cuda(), \n",
    "#                     pos_vec.cuda(), padding_mask.cuda(), lstm_seq_length)\n",
    "print('start')\n",
    "res = []\n",
    "for i in range(0, 14720, 1000):\n",
    "    print(i)\n",
    "    prediction=model_ft(query_vec[i:i+1000], label_text_vec[i:i+1000], query_bert_vec[i:i+1000], img_vec[i:i+1000], \n",
    "                        pos_vec[i:i+1000], padding_mask[i:i+1000], lstm_seq_length[i:i+1000])\n",
    "    res += [i[0].item() for i in prediction.detach().numpy()]\n",
    "    gc.collect()\n",
    "\n",
    "valid['prediction']=res\n",
    "sorted_valid=valid.sort_values(by=['query_id','prediction'],ascending=[True, False])\n",
    "submission=[]\n",
    "for i in valid['query_id'].unique():\n",
    "    submission.append([i]+list(sorted_valid[sorted_valid['query_id']==i]['product_id'].iloc[:5]))\n",
    "print(submission)\n",
    "submission=pd.DataFrame(submission, columns=['query-id','product1','product2',\n",
    "                                             'product3','product4','product5'],dtype=np.int)\n",
    "\n",
    "submission\n",
    "submission.to_csv('valid_pred.csv', index=0)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# TEST DATA"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "fba6154afd584107ac58c00995cf50b7",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, description='Pandas Apply', max=28830.0, style=ProgressStyle(descripti…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "2a41d5f4ca544311ad7a071fbd230698",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, description='Pandas Apply', max=28830.0, style=ProgressStyle(descripti…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "b7ba2ff01ef64a35b207d3a87745c442",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, description='Pandas Apply', max=28830.0, style=ProgressStyle(descripti…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "d0051e10ee5d4d159e40628dcbad8b13",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, description='Pandas Apply', max=28830.0, style=ProgressStyle(descripti…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "0b9b127bdf71438a9fd17a1b4477395b",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, description='Pandas Apply', max=28830.0, style=ProgressStyle(descripti…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "02a075b2447144848eac1c688af8d2b7",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, description='Pandas Apply', max=28830.0, style=ProgressStyle(descripti…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "5d3b63eea3294800aaf9b97e6becc90b",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, description='Pandas Apply', max=28830.0, style=ProgressStyle(descripti…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "f68d4d531d0745c1accb4d65bc596735",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, description='Pandas Apply', max=28830.0, style=ProgressStyle(descripti…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "3506c52484984c23940c4f775060c5b8",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, description='Pandas Apply', max=28830.0, style=ProgressStyle(descripti…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "0fda10dc6d2146f096866069b541ac4b",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, description='Pandas Apply', max=28830.0, style=ProgressStyle(descripti…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "fc295a39468943aeb7bea3a4ed110346",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, description='Pandas Apply', max=28830.0, style=ProgressStyle(descripti…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "12"
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "test_albert_embedding=joblib.load('data/test_query_albert.pkl')\n",
    "test=joblib.load('data/test_data.pkl')\n",
    "# truncate\n",
    "test['label_embedding']=test.swifter.apply(lambda x: x['label_embedding'][:MAX_BOX],axis=1)\n",
    "test['pos']=test.swifter.apply(lambda x: x['pos'][:MAX_BOX],axis=1)\n",
    "test['query_embedding']=test.swifter.apply(lambda x: x['query_embedding'][:MAX_QUERY_CHAR],axis=1)\n",
    "test['num_box']=test.swifter.apply(lambda x: len(x['pos']),axis=1)\n",
    "test['num_query_char']=test.swifter.apply(lambda x: len(x['query_embedding']),axis=1)\n",
    "test['features']=test.swifter.apply(lambda x: [i for i in x['features']],axis=1)\n",
    "test['features']=test.swifter.apply(lambda x: x['features'][:MAX_BOX],axis=1)\n",
    "# padding\n",
    "def padding_seq(sequence, length, max_length):\n",
    "    if length >= max_length:\n",
    "        return sequence\n",
    "    else:\n",
    "        return sequence + [np.zeros(len(sequence[0])) for _ in range(max_length-length)]\n",
    "    \n",
    "test['label_embedding']=test.swifter.apply(lambda x: padding_seq(x['label_embedding'], x['num_box'], MAX_BOX),axis=1)\n",
    "test['pos']=test.swifter.apply(lambda x: padding_seq(x['pos'], x['num_box'], MAX_BOX),axis=1)\n",
    "test['query_embedding']=test.swifter.apply(lambda x: padding_seq(x['query_embedding'], x['num_query_char'], MAX_QUERY_CHAR),axis=1)\n",
    "test['features']=test.swifter.apply(lambda x: padding_seq(x['features'], x['num_box'], MAX_BOX),axis=1)\n",
    "test['query_albert_embedding']=test_albert_embedding['text_vec']\n",
    "gc.collect()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "start\n",
      "0\n",
      "1000\n",
      "2000\n",
      "3000\n",
      "4000\n",
      "5000\n",
      "6000\n",
      "7000\n",
      "8000\n",
      "9000\n",
      "10000\n",
      "11000\n",
      "12000\n",
      "13000\n",
      "14000\n",
      "15000\n",
      "16000\n",
      "17000\n",
      "18000\n",
      "19000\n",
      "20000\n",
      "21000\n",
      "22000\n",
      "23000\n",
      "24000\n",
      "25000\n",
      "26000\n",
      "27000\n",
      "28000\n"
     ]
    }
   ],
   "source": [
    "import warnings\n",
    "warnings.filterwarnings(\"ignore\")\n",
    "\n",
    "model_ft = Model(config).to(torch.device(\"cpu\"))\n",
    "model_ft.load_state_dict(torch.load('model/9_1000000_attention_bilstm_parameter.pkl'))\n",
    "model_ft.eval()\n",
    "\n",
    "query_vec = torch.Tensor(test['query_embedding'])\n",
    "lstm_seq_length = list(test['num_query_char'])\n",
    "label_text_vec = torch.Tensor(test['label_embedding'])\n",
    "query_bert_vec = torch.Tensor(test['query_albert_embedding']).view(28830, 1, 768)\n",
    "img_vec = torch.Tensor(test['features'])\n",
    "pos_vec = torch.Tensor(test['pos'])\n",
    "\n",
    "padding_mask =[]\n",
    "for i in list(test['num_box']):\n",
    "    mask=[0]*(i+2) + [1]*(MAX_BOX-i)\n",
    "    padding_mask.append(mask)\n",
    "padding_mask = torch.ByteTensor(padding_mask)\n",
    "gc.collect()\n",
    "\n",
    "print('start')\n",
    "res = []\n",
    "for i in range(0, 28830, 1000):\n",
    "    print(i)\n",
    "    prediction=model_ft(query_vec[i:i+1000], label_text_vec[i:i+1000], query_bert_vec[i:i+1000], img_vec[i:i+1000], \n",
    "                        pos_vec[i:i+1000], padding_mask[i:i+1000], lstm_seq_length[i:i+1000])\n",
    "    res += [i[0].item() for i in prediction.detach().numpy()]\n",
    "    gc.collect()\n",
    "\n",
    "test['prediction']=res\n",
    "        \n",
    "# prediction=model_ft(query_vec, label_text_vec, query_bert_vec, img_vec, \n",
    "#                     pos_vec, padding_mask, lstm_seq_length)\n",
    "\n",
    "# test['prediction']=[i[0].item() for i in prediction.detach().numpy()]\n",
    "sorted_test=test.sort_values(by=['query_id','prediction'],ascending=[True, False])\n",
    "submission=[]\n",
    "for i in test['query_id'].unique():\n",
    "    submission.append([i]+list(sorted_test[sorted_test['query_id']==i]['product_id'].iloc[:5]))\n",
    "submission=pd.DataFrame(submission, columns=['query-id','product1','product2',\n",
    "                                             'product3','product4','product5'],dtype=np.int)\n",
    "\n",
    "submission\n",
    "submission.to_csv('submission/single_tower_attention_bilstm_8_1000000_05220926_sub.csv', index=0)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0"
      ]
     },
     "execution_count": 14,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "del sorted_valid\n",
    "gc.collect()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.10"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}