run-bert.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "I0211 20:28:30.347209 139670379685696 file_utils.py:38] PyTorch version 1.4.0 available.\n"
     ]
    }
   ],
   "source": [
    "import pandas as pd\n",
    "import numpy as np\n",
    "import datetime\n",
    "import importlib\n",
    "import pickle\n",
    "\n",
    "import data_utils\n",
    "import model_utils\n",
    "import train_utils\n",
    "import evaluation\n",
    "import submission\n",
    "importlib.reload(data_utils)\n",
    "importlib.reload(model_utils)\n",
    "importlib.reload(train_utils)\n",
    "importlib.reload(evaluation)\n",
    "importlib.reload(submission)\n",
    "\n",
    "import torch\n",
    "import torch.nn as nn\n",
    "from torchtext.data import TabularDataset, Field, RawField, BucketIterator, Iterator\n",
    "from transformers import BertForSequenceClassification, BertTokenizer, BertModel"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Training "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "collapsed": true,
    "scrolled": false
   },
   "outputs": [],
   "source": [
    "train_data, val_data, test_data = data_utils.prep_all_data(\n",
    "    train_file=\"train_real.csv\", val_file=\"val.csv\", test_file=\"test.csv\", batch_size=32)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "I0211 20:28:43.068177 139670379685696 tokenization_utils.py:418] loading file https://s3.amazonaws.com/models.huggingface.co/bert/bert-large-uncased-vocab.txt from cache at /home/molly/.cache/torch/transformers/9b3c03a36e83b13d5ba95ac965c9f9074a99e14340c523ab405703179e79fc46.542ce4285a40d23a559526243235df47c5f75c197f04f37d1a0c124c32c9a084\n",
      "I0211 20:28:43.542003 139670379685696 configuration_utils.py:254] loading configuration file https://s3.amazonaws.com/models.huggingface.co/bert/bert-large-uncased-config.json from cache at /home/molly/.cache/torch/transformers/6dfaed860471b03ab5b9acb6153bea82b6632fb9bbe514d3fff050fe1319ee6d.fc076a4d5f1edf25ea3a2bd66e9f6f295dcd64c81dfef5b3f5a3eb2a82751ad1\n",
      "I0211 20:28:43.543458 139670379685696 configuration_utils.py:290] Model config BertConfig {\n",
      "  \"architectures\": [\n",
      "    \"BertForMaskedLM\"\n",
      "  ],\n",
      "  \"attention_probs_dropout_prob\": 0.1,\n",
      "  \"bos_token_id\": 0,\n",
      "  \"do_sample\": false,\n",
      "  \"eos_token_ids\": 0,\n",
      "  \"finetuning_task\": null,\n",
      "  \"hidden_act\": \"gelu\",\n",
      "  \"hidden_dropout_prob\": 0.1,\n",
      "  \"hidden_size\": 1024,\n",
      "  \"id2label\": {\n",
      "    \"0\": \"LABEL_0\",\n",
      "    \"1\": \"LABEL_1\"\n",
      "  },\n",
      "  \"initializer_range\": 0.02,\n",
      "  \"intermediate_size\": 4096,\n",
      "  \"is_decoder\": false,\n",
      "  \"label2id\": {\n",
      "    \"LABEL_0\": 0,\n",
      "    \"LABEL_1\": 1\n",
      "  },\n",
      "  \"layer_norm_eps\": 1e-12,\n",
      "  \"length_penalty\": 1.0,\n",
      "  \"max_length\": 20,\n",
      "  \"max_position_embeddings\": 512,\n",
      "  \"model_type\": \"bert\",\n",
      "  \"num_attention_heads\": 16,\n",
      "  \"num_beams\": 1,\n",
      "  \"num_hidden_layers\": 24,\n",
      "  \"num_labels\": 46,\n",
      "  \"num_return_sequences\": 1,\n",
      "  \"output_attentions\": false,\n",
      "  \"output_hidden_states\": false,\n",
      "  \"output_past\": true,\n",
      "  \"pad_token_id\": 0,\n",
      "  \"pruned_heads\": {},\n",
      "  \"repetition_penalty\": 1.0,\n",
      "  \"temperature\": 1.0,\n",
      "  \"top_k\": 50,\n",
      "  \"top_p\": 1.0,\n",
      "  \"torchscript\": false,\n",
      "  \"type_vocab_size\": 2,\n",
      "  \"use_bfloat16\": false,\n",
      "  \"vocab_size\": 30522\n",
      "}\n",
      "\n",
      "I0211 20:28:43.869621 139670379685696 modeling_utils.py:458] loading weights file https://s3.amazonaws.com/models.huggingface.co/bert/bert-large-uncased-pytorch_model.bin from cache at /home/molly/.cache/torch/transformers/54da47087cc86ce75324e4dc9bbb5f66c6e83a7c6bd23baea8b489acc8d09aa4.4d5343a4b979c4beeaadef17a0453d1bb183dd9b084f58b84c7cc781df343ae6\n",
      "I0211 20:28:51.737640 139670379685696 modeling_utils.py:543] Weights of BertForSequenceClassification not initialized from pretrained model: ['classifier.weight', 'classifier.bias']\n",
      "I0211 20:28:51.738670 139670379685696 modeling_utils.py:549] Weights from pretrained model not used in BertForSequenceClassification: ['cls.predictions.bias', 'cls.predictions.transform.dense.weight', 'cls.predictions.transform.dense.bias', 'cls.predictions.decoder.weight', 'cls.seq_relationship.weight', 'cls.seq_relationship.bias', 'cls.predictions.transform.LayerNorm.weight', 'cls.predictions.transform.LayerNorm.bias']\n"
     ]
    }
   ],
   "source": [
    "m = model_utils.BertMovie()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch: 0, LR: 1e-05, Train Loss: 575.9486, Val Loss: 294.2758, Val f1 0.126\n",
      "Epoch: 1, LR: 1e-05, Train Loss: 254.9178, Val Loss: 200.7394, Val f1 0.126\n",
      "Epoch: 2, LR: 1e-05, Train Loss: 194.9430, Val Loss: 169.1680, Val f1 0.126\n",
      "Epoch: 3, LR: 1e-05, Train Loss: 171.5428, Val Loss: 154.6524, Val f1 0.126\n",
      "Epoch: 4, LR: 1e-05, Train Loss: 159.7396, Val Loss: 146.6598, Val f1 0.126\n",
      "Epoch: 5, LR: 1e-05, Train Loss: 152.4854, Val Loss: 140.9484, Val f1 0.126\n",
      "Epoch: 6, LR: 1e-05, Train Loss: 145.5534, Val Loss: 133.2889, Val f1 0.138\n",
      "Epoch: 7, LR: 1e-05, Train Loss: 135.8025, Val Loss: 121.8225, Val f1 0.194\n",
      "Epoch: 8, LR: 1e-05, Train Loss: 122.2849, Val Loss: 107.0806, Val f1 0.311\n",
      "Epoch: 9, LR: 1e-05, Train Loss: 107.3720, Val Loss: 93.5628, Val f1 0.511\n",
      "Epoch: 10, LR: 1e-05, Train Loss: 93.7622, Val Loss: 83.2128, Val f1 0.592\n",
      "Epoch: 11, LR: 1e-05, Train Loss: 82.6304, Val Loss: 74.1128, Val f1 0.663\n",
      "Epoch: 12, LR: 1e-05, Train Loss: 73.2444, Val Loss: 67.7242, Val f1 0.714\n",
      "Epoch: 13, LR: 1e-05, Train Loss: 65.1703, Val Loss: 61.5826, Val f1 0.771\n",
      "Epoch: 14, LR: 1e-05, Train Loss: 58.5020, Val Loss: 57.2118, Val f1 0.788\n",
      "Epoch: 15, LR: 1e-05, Train Loss: 53.6812, Val Loss: 54.0881, Val f1 0.806\n",
      "Epoch: 16, LR: 1e-05, Train Loss: 50.6586, Val Loss: 54.7853, Val f1 0.762\n",
      "Epoch: 17, LR: 1e-05, Train Loss: 47.3859, Val Loss: 48.3467, Val f1 0.815\n",
      "Epoch: 18, LR: 1e-05, Train Loss: 41.9762, Val Loss: 45.9706, Val f1 0.822\n",
      "Epoch: 19, LR: 1e-05, Train Loss: 37.8886, Val Loss: 43.8428, Val f1 0.842\n",
      "Epoch: 20, LR: 1e-05, Train Loss: 34.9756, Val Loss: 41.7727, Val f1 0.835\n",
      "Epoch: 21, LR: 1e-05, Train Loss: 32.5284, Val Loss: 41.9260, Val f1 0.845\n",
      "Epoch: 22, LR: 1e-05, Train Loss: 29.6503, Val Loss: 40.9931, Val f1 0.833\n",
      "Epoch: 23, LR: 1e-05, Train Loss: 27.6405, Val Loss: 39.8228, Val f1 0.835\n",
      "Epoch: 24, LR: 1e-05, Train Loss: 25.6984, Val Loss: 39.8015, Val f1 0.832\n",
      "Epoch: 25, LR: 1e-05, Train Loss: 24.1560, Val Loss: 37.9759, Val f1 0.839\n",
      "Epoch: 26, LR: 1e-05, Train Loss: 22.4563, Val Loss: 38.7498, Val f1 0.840\n",
      "Epoch: 27, LR: 1e-05, Train Loss: 21.0099, Val Loss: 35.7461, Val f1 0.860\n",
      "Epoch: 28, LR: 1e-05, Train Loss: 19.8577, Val Loss: 36.3108, Val f1 0.855\n",
      "Epoch: 29, LR: 1e-05, Train Loss: 18.5522, Val Loss: 36.4158, Val f1 0.838\n",
      "Epoch: 30, LR: 1e-05, Train Loss: 17.4381, Val Loss: 36.2165, Val f1 0.844\n",
      "Epoch: 31, LR: 1e-05, Train Loss: 16.7051, Val Loss: 35.2999, Val f1 0.849\n",
      "Epoch: 32, LR: 1e-05, Train Loss: 15.8515, Val Loss: 33.3490, Val f1 0.845\n",
      "Epoch: 33, LR: 1e-05, Train Loss: 14.7824, Val Loss: 33.5636, Val f1 0.857\n",
      "Epoch: 34, LR: 1e-05, Train Loss: 14.3342, Val Loss: 32.3687, Val f1 0.864\n",
      "Epoch: 35, LR: 1e-05, Train Loss: 13.6027, Val Loss: 33.7949, Val f1 0.845\n",
      "Epoch: 36, LR: 1e-05, Train Loss: 12.8502, Val Loss: 33.0860, Val f1 0.861\n",
      "Epoch: 37, LR: 1e-05, Train Loss: 11.7904, Val Loss: 32.2312, Val f1 0.863\n",
      "Epoch: 38, LR: 1e-05, Train Loss: 11.4762, Val Loss: 31.7555, Val f1 0.868\n",
      "Epoch: 39, LR: 1e-05, Train Loss: 10.8131, Val Loss: 32.3396, Val f1 0.870\n",
      "Epoch: 40, LR: 1e-05, Train Loss: 10.2882, Val Loss: 31.8992, Val f1 0.868\n",
      "Epoch: 41, LR: 1e-05, Train Loss: 10.0386, Val Loss: 34.1724, Val f1 0.859\n",
      "Epoch: 42, LR: 1e-05, Train Loss: 9.8574, Val Loss: 33.1547, Val f1 0.854\n",
      "Epoch: 43, LR: 1e-05, Train Loss: 8.9776, Val Loss: 32.1491, Val f1 0.873\n",
      "Epoch: 44, LR: 1e-05, Train Loss: 8.6643, Val Loss: 30.7707, Val f1 0.876\n",
      "Epoch: 45, LR: 1e-05, Train Loss: 8.2422, Val Loss: 30.5755, Val f1 0.874\n",
      "Epoch: 46, LR: 1e-05, Train Loss: 7.8900, Val Loss: 31.8520, Val f1 0.867\n",
      "Epoch: 47, LR: 1e-05, Train Loss: 7.5843, Val Loss: 31.1046, Val f1 0.879\n",
      "Epoch: 48, LR: 1e-05, Train Loss: 7.3640, Val Loss: 32.7880, Val f1 0.861\n",
      "Epoch: 49, LR: 1e-05, Train Loss: 6.8976, Val Loss: 31.1305, Val f1 0.869\n",
      "Epoch: 50, LR: 1e-05, Train Loss: 7.2577, Val Loss: 35.1165, Val f1 0.860\n",
      "Epoch: 51, LR: 1e-05, Train Loss: 6.9778, Val Loss: 33.0328, Val f1 0.853\n",
      "Epoch: 52, LR: 1e-05, Train Loss: 6.2672, Val Loss: 33.1725, Val f1 0.867\n",
      "Epoch: 53, LR: 1e-05, Train Loss: 6.1084, Val Loss: 33.2630, Val f1 0.864\n",
      "Epoch: 54, LR: 1e-05, Train Loss: 6.1773, Val Loss: 32.5268, Val f1 0.866\n",
      "Epoch: 55, LR: 1e-05, Train Loss: 6.3351, Val Loss: 32.8058, Val f1 0.864\n",
      "Epoch: 56, LR: 1.0000000000000002e-06, Train Loss: 5.7071, Val Loss: 33.2700, Val f1 0.877\n",
      "Epoch: 57, LR: 1.0000000000000002e-06, Train Loss: 5.2626, Val Loss: 33.1389, Val f1 0.879\n",
      "Epoch: 58, LR: 1.0000000000000002e-06, Train Loss: 5.1346, Val Loss: 33.1049, Val f1 0.871\n",
      "Epoch: 59, LR: 1.0000000000000002e-06, Train Loss: 4.9401, Val Loss: 32.5822, Val f1 0.877\n",
      "Epoch: 60, LR: 1.0000000000000002e-06, Train Loss: 4.9255, Val Loss: 32.1955, Val f1 0.882\n",
      "Epoch: 61, LR: 1.0000000000000002e-06, Train Loss: 4.8411, Val Loss: 31.6494, Val f1 0.883\n",
      "Epoch: 62, LR: 1.0000000000000002e-06, Train Loss: 4.8380, Val Loss: 33.1755, Val f1 0.878\n",
      "Epoch: 63, LR: 1.0000000000000002e-06, Train Loss: 4.8096, Val Loss: 32.6636, Val f1 0.876\n",
      "Epoch: 64, LR: 1.0000000000000002e-06, Train Loss: 4.7685, Val Loss: 32.4881, Val f1 0.880\n",
      "Epoch: 65, LR: 1.0000000000000002e-06, Train Loss: 4.6767, Val Loss: 32.6026, Val f1 0.877\n",
      "Epoch: 66, LR: 1.0000000000000002e-06, Train Loss: 4.6837, Val Loss: 31.0713, Val f1 0.886\n",
      "Epoch: 67, LR: 1.0000000000000002e-07, Train Loss: 4.6278, Val Loss: 30.9401, Val f1 0.884\n",
      "Epoch: 68, LR: 1.0000000000000002e-07, Train Loss: 4.6874, Val Loss: 30.9898, Val f1 0.884\n",
      "Epoch: 69, LR: 1.0000000000000002e-07, Train Loss: 4.6056, Val Loss: 31.0017, Val f1 0.884\n",
      "Epoch: 70, LR: 1.0000000000000002e-07, Train Loss: 4.6700, Val Loss: 31.0336, Val f1 0.884\n",
      "Epoch: 71, LR: 1.0000000000000002e-07, Train Loss: 4.6437, Val Loss: 31.0463, Val f1 0.884\n",
      "Epoch: 72, LR: 1.0000000000000002e-07, Train Loss: 4.6639, Val Loss: 31.0681, Val f1 0.886\n",
      "Epoch: 73, LR: 1.0000000000000002e-07, Train Loss: 4.6812, Val Loss: 31.1064, Val f1 0.886\n",
      "Epoch: 74, LR: 1.0000000000000002e-07, Train Loss: 4.5884, Val Loss: 31.1310, Val f1 0.885\n",
      "Epoch: 75, LR: 1.0000000000000002e-07, Train Loss: 4.5675, Val Loss: 31.1406, Val f1 0.885\n",
      "Epoch: 76, LR: 1.0000000000000002e-07, Train Loss: 4.5740, Val Loss: 31.1599, Val f1 0.886\n",
      "Epoch: 77, LR: 1.0000000000000002e-07, Train Loss: 4.5827, Val Loss: 31.1748, Val f1 0.887\n",
      "Epoch: 78, LR: 1.0000000000000004e-08, Train Loss: 4.5898, Val Loss: 31.1762, Val f1 0.885\n",
      "Epoch: 79, LR: 1.0000000000000004e-08, Train Loss: 4.5759, Val Loss: 31.1772, Val f1 0.885\n",
      "Epoch: 80, LR: 1.0000000000000004e-08, Train Loss: 4.5617, Val Loss: 31.1785, Val f1 0.885\n",
      "Epoch: 81, LR: 1.0000000000000004e-08, Train Loss: 4.5758, Val Loss: 31.1776, Val f1 0.885\n",
      "Epoch: 82, LR: 1.0000000000000004e-08, Train Loss: 4.6973, Val Loss: 31.1806, Val f1 0.887\n",
      "Epoch: 83, LR: 1.0000000000000004e-08, Train Loss: 4.5535, Val Loss: 31.1810, Val f1 0.887\n",
      "Epoch: 84, LR: 1.0000000000000004e-08, Train Loss: 4.5851, Val Loss: 31.1808, Val f1 0.887\n",
      "Epoch: 85, LR: 1.0000000000000004e-08, Train Loss: 4.6109, Val Loss: 31.1859, Val f1 0.887\n",
      "Epoch: 86, LR: 1.0000000000000004e-08, Train Loss: 4.6063, Val Loss: 31.1912, Val f1 0.887\n",
      "Epoch: 87, LR: 1.0000000000000004e-08, Train Loss: 4.5650, Val Loss: 31.1925, Val f1 0.887\n",
      "Epoch: 88, LR: 1.0000000000000004e-08, Train Loss: 4.5462, Val Loss: 31.1922, Val f1 0.887\n"
     ]
    }
   ],
   "source": [
    "result = train_utils.train(train_data, val_data, m,\n",
    "                          lr=1e-5, print_freq=1, max_epoch=100)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/molly/anaconda3/lib/python3.6/site-packages/torch/serialization.py:360: UserWarning: Couldn't retrieve source code for container of type BertMovie. It won't be checked for correctness upon loading.\n",
      "  \"type \" + obj.__name__ + \". It won't be checked \"\n"
     ]
    }
   ],
   "source": [
    "torch.save(result[\"trained_model\"], \"./data/model_checkpoints/bert_large_feb05.mdl\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Ensemble"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/molly/anaconda3/lib/python3.6/site-packages/torch/serialization.py:593: SourceChangeWarning: source code of class 'model_utils.GRU' has changed. you can retrieve the original source code by accessing the object's source attribute or set `torch.nn.Module.dump_patches = True` and use the patch tool to revert the changes.\n",
      "  warnings.warn(msg, SourceChangeWarning)\n"
     ]
    }
   ],
   "source": [
    "train_data, val_data, test_data = data_utils.prep_all_data(\n",
    "    train_file=\"train_real.csv\", val_file=\"val.csv\", test_file=\"test.csv\")\n",
    "train_data_phrase, val_data_phrase, test_data_phrase = data_utils.prep_all_data(\n",
    "    path=\"./data/movie_name_condensed_data/\", \n",
    "    train_file=\"train_real.csv\", val_file=\"val.csv\", test_file=\"test.csv\")\n",
    "\n",
    "m_gram = torch.load(\"./data/model_checkpoints/ngram_MLP_Jan31.mdl\")\n",
    "m_gram_phrase = torch.load(\"./data/model_checkpoints/ngram_phrase_MLP_Jan31.mdl\")\n",
    "m_GRU = torch.load(\"./data/model_checkpoints/GRU_Jan31.mdl\")\n",
    "m_GRU_phrase = torch.load(\"./data/model_checkpoints/GRU_phrase_Jan31.mdl\")\n",
    "m_bert = result[\"trained_model\"]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 187,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "ensemble f1 val score: 0.818118118118118\n"
     ]
    }
   ],
   "source": [
    "importlib.reload(submission)\n",
    "\n",
    "ensemble = submission.Ensemble(models=[m_gram, m_gram_phrase], \n",
    "                               val_data=[val_data, val_data_phrase],\n",
    "                               test_data=[test_data, test_data_phrase])\n",
    "df_val = ensemble.get_ensemble_result(submission=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "ensemble f1 val score: 0.8632632632632633\n"
     ]
    }
   ],
   "source": [
    "importlib.reload(submission)\n",
    "\n",
    "ensemble = submission.Ensemble(models=[m_bert, m_gram, m_gram_phrase], \n",
    "                               val_data=[val_data, val_data, val_data_phrase],\n",
    "                               test_data=[test_data, test_data, test_data_phrase])\n",
    "df_val = ensemble.get_ensemble_result(submission=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "ensemble f1 val score: 0.8922922922922921\n"
     ]
    }
   ],
   "source": [
    "importlib.reload(submission)\n",
    "\n",
    "ensemble = submission.Ensemble(models=[m_bert, m_gram_phrase], \n",
    "                               val_data=[val_data, val_data_phrase],\n",
    "                               test_data=[test_data, test_data_phrase])\n",
    "df_val = ensemble.get_ensemble_result(submission=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "ensemble f1 val score: 0.8896896896896896\n"
     ]
    }
   ],
   "source": [
    "importlib.reload(submission)\n",
    "\n",
    "ensemble = submission.Ensemble(models=[m_bert], \n",
    "                               val_data=[val_data],\n",
    "                               test_data=[test_data])\n",
    "df_val = ensemble.get_ensemble_result(submission=False)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Generate submission file for kaggle "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "df = ensemble.get_ensemble_result(submission=True)\n",
    "ensemble.save_submission_file(\"bert-large+ngram-phrase\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "collapsed": true
   },
   "source": [
    "### Error analysis for best model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(53, 3)"
      ]
     },
     "execution_count": 20,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "errors = df_val[df_val[\"CORE RELATIONS\"]!=df_val.raw_label]; errors.shape\n",
    "correct = df_val[df_val[\"CORE RELATIONS\"]==df_val.raw_label]; errors.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.8408408408408409"
      ]
     },
     "execution_count": 21,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "accuracy = 1-len(errors)/len(df_val); accuracy"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "movie_other                                                8\n",
       "other                                                      8\n",
       "NO_REL                                                     4\n",
       "movie.rating movie.initial_release_date                    3\n",
       "movie.directed_by                                          3\n",
       "movie.starring.actor actor.gender                          3\n",
       "movie.starring.actor movie.starring.character              2\n",
       "movie.star_rating                                          2\n",
       "movie.rating                                               2\n",
       "movie.genre movie.initial_release_date                     1\n",
       "movie.country movie.genre                                  1\n",
       "movie.locations movie.country                              1\n",
       "movie.country                                              1\n",
       "movie.genre movie.description                              1\n",
       "movie.country movie.subjects                               1\n",
       "movie.rating movie.subjects                                1\n",
       "movie.locations movie.initial_release_date                 1\n",
       "person.nationality                                         1\n",
       "movie.showing                                              1\n",
       "movie.country movie.initial_release_date                   1\n",
       "movie.starring.actor movie.produced_by                     1\n",
       "movie.locations movie.directed_by                          1\n",
       "rt.runtime movie.runtime                                   1\n",
       "movie.language                                             1\n",
       "movie.country movie.language movie.initial_release_date    1\n",
       "movie.production_companies                                 1\n",
       "movie.starring.actor movie.award actor.nationality         1\n",
       "Name: raw_label, dtype: int64"
      ]
     },
     "execution_count": 22,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "errors.raw_label.value_counts()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {
    "scrolled": false
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style>\n",
       "    .dataframe thead tr:only-child th {\n",
       "        text-align: right;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: left;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>CORE RELATIONS</th>\n",
       "      <th>text</th>\n",
       "      <th>raw_label</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>ID</th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>2861</th>\n",
       "      <td>movie.showing</td>\n",
       "      <td>where can i rent this movie</td>\n",
       "      <td>movie_other</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2896</th>\n",
       "      <td>movie.genre</td>\n",
       "      <td>search for dramas like downton abbey</td>\n",
       "      <td>movie_other</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2904</th>\n",
       "      <td>movie.subjects</td>\n",
       "      <td>what other movies are about the holocaust simi...</td>\n",
       "      <td>movie_other</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2914</th>\n",
       "      <td>movie.subjects</td>\n",
       "      <td>after watching this movie will i have nightmar...</td>\n",
       "      <td>movie_other</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2922</th>\n",
       "      <td>movie.language</td>\n",
       "      <td>what do the actors speak</td>\n",
       "      <td>other</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2939</th>\n",
       "      <td>movie.genre</td>\n",
       "      <td>scary</td>\n",
       "      <td>other</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2943</th>\n",
       "      <td>movie.subjects</td>\n",
       "      <td>zombies</td>\n",
       "      <td>other</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2951</th>\n",
       "      <td>movie.country movie.language</td>\n",
       "      <td>arabic life of a movie</td>\n",
       "      <td>other</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2964</th>\n",
       "      <td>movie.directed_by</td>\n",
       "      <td>did you have fun directing gilda mister vidor</td>\n",
       "      <td>movie_other</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2971</th>\n",
       "      <td>movie.subjects</td>\n",
       "      <td>england will you make movie about the life of ...</td>\n",
       "      <td>movie_other</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2977</th>\n",
       "      <td>NO_REL</td>\n",
       "      <td>goto greyscale</td>\n",
       "      <td>other</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3004</th>\n",
       "      <td>NO_REL</td>\n",
       "      <td>mister lester did you get to meet the beatles</td>\n",
       "      <td>other</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3009</th>\n",
       "      <td>movie.estimated_budget</td>\n",
       "      <td>paramount- do you pay all the expenses for the...</td>\n",
       "      <td>other</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3013</th>\n",
       "      <td>other</td>\n",
       "      <td>r info</td>\n",
       "      <td>movie_other</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3065</th>\n",
       "      <td>movie.rating</td>\n",
       "      <td>what is the g rating</td>\n",
       "      <td>movie_other</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3120</th>\n",
       "      <td>movie_other</td>\n",
       "      <td>was the poularity good or bad</td>\n",
       "      <td>other</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                    CORE RELATIONS  \\\n",
       "ID                                   \n",
       "2861                 movie.showing   \n",
       "2896                   movie.genre   \n",
       "2904                movie.subjects   \n",
       "2914                movie.subjects   \n",
       "2922                movie.language   \n",
       "2939                   movie.genre   \n",
       "2943                movie.subjects   \n",
       "2951  movie.country movie.language   \n",
       "2964             movie.directed_by   \n",
       "2971                movie.subjects   \n",
       "2977                        NO_REL   \n",
       "3004                        NO_REL   \n",
       "3009        movie.estimated_budget   \n",
       "3013                         other   \n",
       "3065                  movie.rating   \n",
       "3120                   movie_other   \n",
       "\n",
       "                                                   text    raw_label  \n",
       "ID                                                                    \n",
       "2861                        where can i rent this movie  movie_other  \n",
       "2896               search for dramas like downton abbey  movie_other  \n",
       "2904  what other movies are about the holocaust simi...  movie_other  \n",
       "2914  after watching this movie will i have nightmar...  movie_other  \n",
       "2922                           what do the actors speak        other  \n",
       "2939                                              scary        other  \n",
       "2943                                            zombies        other  \n",
       "2951                             arabic life of a movie        other  \n",
       "2964      did you have fun directing gilda mister vidor  movie_other  \n",
       "2971  england will you make movie about the life of ...  movie_other  \n",
       "2977                                     goto greyscale        other  \n",
       "3004      mister lester did you get to meet the beatles        other  \n",
       "3009  paramount- do you pay all the expenses for the...        other  \n",
       "3013                                             r info  movie_other  \n",
       "3065                               what is the g rating  movie_other  \n",
       "3120                      was the poularity good or bad        other  "
      ]
     },
     "execution_count": 23,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "errors[errors.raw_label.isin([\"other\", \"movie_other\"])]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 205,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style>\n",
       "    .dataframe thead tr:only-child th {\n",
       "        text-align: right;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: left;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>CORE RELATIONS</th>\n",
       "      <th>text</th>\n",
       "      <th>raw_label</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>ID</th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>2844</th>\n",
       "      <td>movie_other</td>\n",
       "      <td>find movie based on criteria</td>\n",
       "      <td>movie_other</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2855</th>\n",
       "      <td>movie_other</td>\n",
       "      <td>search by stars</td>\n",
       "      <td>movie_other</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2857</th>\n",
       "      <td>movie_other</td>\n",
       "      <td>searching for cheap movies</td>\n",
       "      <td>movie_other</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2861</th>\n",
       "      <td>movie_other</td>\n",
       "      <td>where can i rent this movie</td>\n",
       "      <td>movie_other</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2889</th>\n",
       "      <td>movie_other</td>\n",
       "      <td>middle aged white actors</td>\n",
       "      <td>movie_other</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2894</th>\n",
       "      <td>movie_other</td>\n",
       "      <td>movies like in july</td>\n",
       "      <td>movie_other</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2922</th>\n",
       "      <td>other</td>\n",
       "      <td>what do the actors speak</td>\n",
       "      <td>other</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2939</th>\n",
       "      <td>other</td>\n",
       "      <td>scary</td>\n",
       "      <td>other</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2954</th>\n",
       "      <td>other</td>\n",
       "      <td>can i have a map of britain please</td>\n",
       "      <td>other</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2964</th>\n",
       "      <td>movie_other</td>\n",
       "      <td>did you have fun directing gilda mister vidor</td>\n",
       "      <td>movie_other</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2972</th>\n",
       "      <td>other</td>\n",
       "      <td>find map of canada</td>\n",
       "      <td>other</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2973</th>\n",
       "      <td>other</td>\n",
       "      <td>find select info</td>\n",
       "      <td>other</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2976</th>\n",
       "      <td>other</td>\n",
       "      <td>give me more information about the text highli...</td>\n",
       "      <td>other</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2977</th>\n",
       "      <td>other</td>\n",
       "      <td>goto greyscale</td>\n",
       "      <td>other</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2986</th>\n",
       "      <td>movie_other</td>\n",
       "      <td>how popular do you think the movie will be</td>\n",
       "      <td>movie_other</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3006</th>\n",
       "      <td>movie_other</td>\n",
       "      <td>movie parental guidance rating scale</td>\n",
       "      <td>movie_other</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3014</th>\n",
       "      <td>movie_other</td>\n",
       "      <td>r rating means sex or nudity</td>\n",
       "      <td>movie_other</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3022</th>\n",
       "      <td>other</td>\n",
       "      <td>show information in english</td>\n",
       "      <td>other</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3030</th>\n",
       "      <td>other</td>\n",
       "      <td>show me the credits in orange</td>\n",
       "      <td>other</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3057</th>\n",
       "      <td>other</td>\n",
       "      <td>what does the flag of greece look like</td>\n",
       "      <td>other</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3079</th>\n",
       "      <td>other</td>\n",
       "      <td>searching for maps</td>\n",
       "      <td>other</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3096</th>\n",
       "      <td>other</td>\n",
       "      <td>july</td>\n",
       "      <td>other</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3097</th>\n",
       "      <td>movie_other</td>\n",
       "      <td>pg thirteen information</td>\n",
       "      <td>movie_other</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3115</th>\n",
       "      <td>other</td>\n",
       "      <td>want page</td>\n",
       "      <td>other</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3120</th>\n",
       "      <td>other</td>\n",
       "      <td>was the poularity good or bad</td>\n",
       "      <td>other</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "     CORE RELATIONS                                               text  \\\n",
       "ID                                                                       \n",
       "2844    movie_other                       find movie based on criteria   \n",
       "2855    movie_other                                    search by stars   \n",
       "2857    movie_other                         searching for cheap movies   \n",
       "2861    movie_other                        where can i rent this movie   \n",
       "2889    movie_other                           middle aged white actors   \n",
       "2894    movie_other                                movies like in july   \n",
       "2922          other                           what do the actors speak   \n",
       "2939          other                                              scary   \n",
       "2954          other                 can i have a map of britain please   \n",
       "2964    movie_other      did you have fun directing gilda mister vidor   \n",
       "2972          other                                 find map of canada   \n",
       "2973          other                                   find select info   \n",
       "2976          other  give me more information about the text highli...   \n",
       "2977          other                                     goto greyscale   \n",
       "2986    movie_other         how popular do you think the movie will be   \n",
       "3006    movie_other               movie parental guidance rating scale   \n",
       "3014    movie_other                       r rating means sex or nudity   \n",
       "3022          other                        show information in english   \n",
       "3030          other                      show me the credits in orange   \n",
       "3057          other             what does the flag of greece look like   \n",
       "3079          other                                 searching for maps   \n",
       "3096          other                                               july   \n",
       "3097    movie_other                            pg thirteen information   \n",
       "3115          other                                          want page   \n",
       "3120          other                      was the poularity good or bad   \n",
       "\n",
       "        raw_label  \n",
       "ID                 \n",
       "2844  movie_other  \n",
       "2855  movie_other  \n",
       "2857  movie_other  \n",
       "2861  movie_other  \n",
       "2889  movie_other  \n",
       "2894  movie_other  \n",
       "2922        other  \n",
       "2939        other  \n",
       "2954        other  \n",
       "2964  movie_other  \n",
       "2972        other  \n",
       "2973        other  \n",
       "2976        other  \n",
       "2977        other  \n",
       "2986  movie_other  \n",
       "3006  movie_other  \n",
       "3014  movie_other  \n",
       "3022        other  \n",
       "3030        other  \n",
       "3057        other  \n",
       "3079        other  \n",
       "3096        other  \n",
       "3097  movie_other  \n",
       "3115        other  \n",
       "3120        other  "
      ]
     },
     "execution_count": 205,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "correct[correct.raw_label.isin([\"other\", \"movie_other\"])]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 350,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "df_train = pd.read_csv(\"./data/train_real.csv\", index_col=\"ID\")[[\"text\", \"raw_label\"]]\n",
    "df_test = pd.read_csv(\"./data/holdout_test.csv\", index_col=\"ID\")[[\"text\", \"raw_label\"]]\n",
    "df_val = pd.read_csv(\"./data/val.csv\", index_col=\"ID\")[[\"text\", \"raw_label\"]]\n",
    "df_all_train = pd.read_csv(\"./data/train.csv\", index_col=\"ID\")[[\"text\", \"raw_label\"]]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 351,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style>\n",
       "    .dataframe thead tr:only-child th {\n",
       "        text-align: right;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: left;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>text</th>\n",
       "      <th>raw_label</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>ID</th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>735</th>\n",
       "      <td>i need movie information for life is beautiful</td>\n",
       "      <td>NO_REL</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>737</th>\n",
       "      <td>i want to know more about the movie life is be...</td>\n",
       "      <td>NO_REL</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>738</th>\n",
       "      <td>i would like info on the movie life is beautiful</td>\n",
       "      <td>NO_REL</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>739</th>\n",
       "      <td>please show me info on the movie life is beaut...</td>\n",
       "      <td>NO_REL</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>740</th>\n",
       "      <td>search for the movie life is beautiful</td>\n",
       "      <td>NO_REL</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>742</th>\n",
       "      <td>can i see more info the movie life is beautiful</td>\n",
       "      <td>NO_REL</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>743</th>\n",
       "      <td>can you show me info on the movie life is beau...</td>\n",
       "      <td>NO_REL</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>744</th>\n",
       "      <td>display information for the movie life is beau...</td>\n",
       "      <td>NO_REL</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>773</th>\n",
       "      <td>i am looking for the information on the movie ...</td>\n",
       "      <td>NO_REL</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>782</th>\n",
       "      <td>what are the plots of hotel rwanda</td>\n",
       "      <td>NO_REL</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>784</th>\n",
       "      <td>bings movie page finding nemo</td>\n",
       "      <td>NO_REL</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>785</th>\n",
       "      <td>display information for film finding nemo</td>\n",
       "      <td>NO_REL</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>786</th>\n",
       "      <td>finding nemo movie shortcut</td>\n",
       "      <td>NO_REL</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>787</th>\n",
       "      <td>whats some details for the movie finding nemo</td>\n",
       "      <td>NO_REL</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>790</th>\n",
       "      <td>finding nemo movie page</td>\n",
       "      <td>NO_REL</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>791</th>\n",
       "      <td>find movie finding nemo</td>\n",
       "      <td>NO_REL</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>792</th>\n",
       "      <td>display all details of original finding nemo m...</td>\n",
       "      <td>NO_REL</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>793</th>\n",
       "      <td>find all details of original finding nemo movie</td>\n",
       "      <td>NO_REL</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>812</th>\n",
       "      <td>find ms. congeniality</td>\n",
       "      <td>NO_REL</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>818</th>\n",
       "      <td>find films</td>\n",
       "      <td>NO_REL</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>819</th>\n",
       "      <td>find movies</td>\n",
       "      <td>NO_REL</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>820</th>\n",
       "      <td>in search for movies</td>\n",
       "      <td>NO_REL</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>821</th>\n",
       "      <td>looking for movies</td>\n",
       "      <td>NO_REL</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>949</th>\n",
       "      <td>information about movie</td>\n",
       "      <td>NO_REL</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>950</th>\n",
       "      <td>show me information about movie</td>\n",
       "      <td>NO_REL</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>952</th>\n",
       "      <td>tell me about movie</td>\n",
       "      <td>NO_REL</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1024</th>\n",
       "      <td>dirty dancing movie</td>\n",
       "      <td>NO_REL</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1025</th>\n",
       "      <td>dirty dancing movie short cuts</td>\n",
       "      <td>NO_REL</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1027</th>\n",
       "      <td>will you please show me info on the movie dirt...</td>\n",
       "      <td>NO_REL</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1028</th>\n",
       "      <td>find the movie dirty dancing</td>\n",
       "      <td>NO_REL</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3085</th>\n",
       "      <td>funnie</td>\n",
       "      <td>other</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3086</th>\n",
       "      <td>genuine</td>\n",
       "      <td>other</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3089</th>\n",
       "      <td>list page</td>\n",
       "      <td>other</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3090</th>\n",
       "      <td>noisy</td>\n",
       "      <td>other</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3091</th>\n",
       "      <td>search databases</td>\n",
       "      <td>other</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3092</th>\n",
       "      <td>search popularity</td>\n",
       "      <td>other</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3093</th>\n",
       "      <td>search selected info</td>\n",
       "      <td>other</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3094</th>\n",
       "      <td>september twenty eight</td>\n",
       "      <td>other</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3095</th>\n",
       "      <td>dirty dan</td>\n",
       "      <td>other</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3098</th>\n",
       "      <td>adventeurous</td>\n",
       "      <td>other</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3099</th>\n",
       "      <td>find selected info</td>\n",
       "      <td>other</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3100</th>\n",
       "      <td>has he written anything new</td>\n",
       "      <td>other</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3101</th>\n",
       "      <td>love</td>\n",
       "      <td>other</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3102</th>\n",
       "      <td>wilford</td>\n",
       "      <td>other</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3103</th>\n",
       "      <td>william</td>\n",
       "      <td>other</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3104</th>\n",
       "      <td>pull up photos of sexy women</td>\n",
       "      <td>other</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3105</th>\n",
       "      <td>show me pictures of beautiful men</td>\n",
       "      <td>other</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3106</th>\n",
       "      <td>show me pictures of beautiful women</td>\n",
       "      <td>other</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3108</th>\n",
       "      <td>what does person look like</td>\n",
       "      <td>other</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3109</th>\n",
       "      <td>on julie</td>\n",
       "      <td>other</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3110</th>\n",
       "      <td>on july</td>\n",
       "      <td>other</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3111</th>\n",
       "      <td>send dredd movie</td>\n",
       "      <td>other</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3113</th>\n",
       "      <td>would you make a sequel</td>\n",
       "      <td>other</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3114</th>\n",
       "      <td>show me the highlighted topics</td>\n",
       "      <td>other</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3116</th>\n",
       "      <td>are the animated charactors protrayed as small...</td>\n",
       "      <td>other</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3117</th>\n",
       "      <td>what was the country where nobody knows takes ...</td>\n",
       "      <td>other</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3119</th>\n",
       "      <td>show me famous people</td>\n",
       "      <td>other</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3122</th>\n",
       "      <td>tyler p new projects</td>\n",
       "      <td>other</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3124</th>\n",
       "      <td>how many shaft movies are there</td>\n",
       "      <td>NO_REL</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3166</th>\n",
       "      <td>who are these directors</td>\n",
       "      <td>NO_REL</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>365 rows × 2 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "                                                   text raw_label\n",
       "ID                                                               \n",
       "735      i need movie information for life is beautiful    NO_REL\n",
       "737   i want to know more about the movie life is be...    NO_REL\n",
       "738    i would like info on the movie life is beautiful    NO_REL\n",
       "739   please show me info on the movie life is beaut...    NO_REL\n",
       "740              search for the movie life is beautiful    NO_REL\n",
       "742     can i see more info the movie life is beautiful    NO_REL\n",
       "743   can you show me info on the movie life is beau...    NO_REL\n",
       "744   display information for the movie life is beau...    NO_REL\n",
       "773   i am looking for the information on the movie ...    NO_REL\n",
       "782                  what are the plots of hotel rwanda    NO_REL\n",
       "784                       bings movie page finding nemo    NO_REL\n",
       "785           display information for film finding nemo    NO_REL\n",
       "786                         finding nemo movie shortcut    NO_REL\n",
       "787       whats some details for the movie finding nemo    NO_REL\n",
       "790                             finding nemo movie page    NO_REL\n",
       "791                             find movie finding nemo    NO_REL\n",
       "792   display all details of original finding nemo m...    NO_REL\n",
       "793     find all details of original finding nemo movie    NO_REL\n",
       "812                               find ms. congeniality    NO_REL\n",
       "818                                          find films    NO_REL\n",
       "819                                         find movies    NO_REL\n",
       "820                                in search for movies    NO_REL\n",
       "821                                  looking for movies    NO_REL\n",
       "949                             information about movie    NO_REL\n",
       "950                     show me information about movie    NO_REL\n",
       "952                                 tell me about movie    NO_REL\n",
       "1024                                dirty dancing movie    NO_REL\n",
       "1025                     dirty dancing movie short cuts    NO_REL\n",
       "1027  will you please show me info on the movie dirt...    NO_REL\n",
       "1028                       find the movie dirty dancing    NO_REL\n",
       "...                                                 ...       ...\n",
       "3085                                             funnie     other\n",
       "3086                                            genuine     other\n",
       "3089                                          list page     other\n",
       "3090                                              noisy     other\n",
       "3091                                   search databases     other\n",
       "3092                                  search popularity     other\n",
       "3093                               search selected info     other\n",
       "3094                             september twenty eight     other\n",
       "3095                                          dirty dan     other\n",
       "3098                                       adventeurous     other\n",
       "3099                                 find selected info     other\n",
       "3100                        has he written anything new     other\n",
       "3101                                               love     other\n",
       "3102                                            wilford     other\n",
       "3103                                            william     other\n",
       "3104                       pull up photos of sexy women     other\n",
       "3105                  show me pictures of beautiful men     other\n",
       "3106                show me pictures of beautiful women     other\n",
       "3108                         what does person look like     other\n",
       "3109                                           on julie     other\n",
       "3110                                            on july     other\n",
       "3111                                   send dredd movie     other\n",
       "3113                            would you make a sequel     other\n",
       "3114                     show me the highlighted topics     other\n",
       "3116  are the animated charactors protrayed as small...     other\n",
       "3117  what was the country where nobody knows takes ...     other\n",
       "3119                              show me famous people     other\n",
       "3122                               tyler p new projects     other\n",
       "3124                    how many shaft movies are there    NO_REL\n",
       "3166                            who are these directors    NO_REL\n",
       "\n",
       "[365 rows x 2 columns]"
      ]
     },
     "execution_count": 351,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_train[df_train.raw_label.isin([\"NO_REL\", \"other\"])]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 265,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(315,)"
      ]
     },
     "execution_count": 265,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_val.index.value_counts().shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 266,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(2702,)"
      ]
     },
     "execution_count": 266,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_train.index.value_counts().shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 268,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(3338,)"
      ]
     },
     "execution_count": 268,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_all_train.index.value_counts().shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 269,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "3338"
      ]
     },
     "execution_count": 269,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "321 + 315 + 2702"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 271,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(333, 2)"
      ]
     },
     "execution_count": 271,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_val.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.2"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}