language_model_raw.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Tokenization"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import numpy as np\n",
    "import pandas as pd\n",
    "import matplotlib.pyplot as plt\n",
    "import os\n",
    "from pathlib import Path\n",
    "from collections import defaultdict\n",
    "from fastai.text import *\n",
    "np.random.seed(42)\n",
    "\n",
    "import warnings\n",
    "warnings.filterwarnings(action='ignore')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "path = Path('/home/paperspace/suicides')\n",
    "lm_path = Path('data/lm/')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "df = pd.read_csv(path/'data/std_format_raw_data.csv',index_col=0)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "f = Path('data/lm/tmp/tokens.npy')\n",
    "if not f.is_file():\n",
    "    texts = df.text.apply(lambda x: 'xbof '+x)\n",
    "    tok = Tokenizer()\n",
    "    tokens = []\n",
    "    for i,text in enumerate(texts):\n",
    "        tokens.append(tok.proc_text(text))\n",
    "        if i%(len(texts)//10) == 0:\n",
    "            print(i/len(texts)*100,' percent done')\n",
    "    lm_path.mkdir(exist_ok=True)\n",
    "    (lm_path/'tmp').mkdir(exist_ok=True)\n",
    "    np.save(lm_path/'tmp'/'tokens.npy', tokens)\n",
    "else:\n",
    "    tokens = np.load(lm_path/'tmp'/'tokens.npy')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Full vocab size:  377618\n",
      "Top 20 occuring words:\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "[('*', 11203884),\n",
       " ('\\n', 9508265),\n",
       " ('.', 6671207),\n",
       " ('t_up', 5542099),\n",
       " ('-', 3509054),\n",
       " (',', 3497876),\n",
       " (':', 3433106),\n",
       " (']', 2658757),\n",
       " ('[', 2523815),\n",
       " ('the', 2064814),\n",
       " (')', 2061053),\n",
       " ('(', 1908151),\n",
       " ('and', 1813555),\n",
       " ('to', 1522031),\n",
       " ('of', 1521027),\n",
       " ('was', 1424638),\n",
       " ('with', 1085054),\n",
       " ('\\n\\n', 1079766),\n",
       " ('/', 1045644),\n",
       " ('a', 1009812)]"
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# get most commonly occuring tokens\n",
    "freq = Counter(p for o in tokens for p in o)\n",
    "print('Full vocab size: ',len(freq))\n",
    "print('Top 20 occuring words:')\n",
    "freq.most_common(20)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "max_vocab = 150000\n",
    "min_freq = 2"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "truncated vocab size:  118646\n"
     ]
    }
   ],
   "source": [
    "int_to_str = [o for o,c in freq.most_common(max_vocab) if c>min_freq]\n",
    "int_to_str.insert(0, '_pad_')\n",
    "int_to_str.insert(0, '_unk_')\n",
    "str_to_int = defaultdict(lambda:0, {v:k for k,v in enumerate(int_to_str)})\n",
    "\n",
    "vocab_size = len(int_to_str)\n",
    "print('truncated vocab size: ',vocab_size)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Forward Language Model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "f = Path('data/lm/tmp/language_model.npy')\n",
    "g = Path('data/lm/tmp/int_to_str.npy')\n",
    "if not (f.is_file() and g.is_file()):\n",
    "    language_model = np.array([[str_to_int[o] for o in p] for p in tokens])\n",
    "    np.save(lm_path/'tmp'/'language_model.npy', language_model)\n",
    "    pickle.dump(int_to_str, open(lm_path/'tmp'/'int_to_str.pkl', 'wb'))\n",
    "else:\n",
    "    language_model = np.load(lm_path/'tmp'/'language_model.npy')\n",
    "    int_to_str = pickle.load(open(lm_path/'tmp'/'int_to_str.pkl', 'rb'))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "#! wget -nH -r -np -P {lm_path} http://files.fast.ai/models/wt103/\n",
    "weights = torch.load(lm_path/'models/wt103/fwd_wt103.h5', map_location=lambda storage, loc: storage)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [],
   "source": [
    "encoder_weights = to_np(weights['0.encoder.weight'])\n",
    "row_means = encoder_weights.mean(0)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [],
   "source": [
    "embedding_size = 400\n",
    "num_hidden = 1150\n",
    "num_layers = 3"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [],
   "source": [
    "# key used for wiki103\n",
    "int_to_str_wiki = pickle.load((lm_path/'models'/'wt103'/'itos_wt103.pkl').open('rb'))\n",
    "str_to_int_wiki = defaultdict(lambda:-1, {v:k for k,v in enumerate(int_to_str_wiki)})"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [],
   "source": [
    "# map suicide embedding weights to wiki103 embedding weights\n",
    "new_weights = np.zeros((vocab_size, embedding_size), dtype=np.float32)\n",
    "for i,w in enumerate(int_to_str):\n",
    "    row = str_to_int_wiki[w]\n",
    "    new_weights[i] = encoder_weights[row] if row>=0 else row_means"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [],
   "source": [
    "weights['0.encoder.weight'] = T(new_weights)\n",
    "weights['0.encoder_with_dropout.embed.weight'] = T(np.copy(new_weights))\n",
    "weights['1.decoder.weight'] = T(np.copy(new_weights))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [],
   "source": [
    "# hyperparameters for language model\n",
    "wd =1e-7\n",
    "bptt = 70\n",
    "bs = 32\n",
    "lr = 1e-3\n",
    "opt = partial(optim.Adam, betas=(0.8, 0.99))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [],
   "source": [
    "size = len(language_model)\n",
    "lm_train,lm_val = sklearn.model_selection.train_test_split(language_model[:size], test_size=0.1)\n",
    "\n",
    "dl_train = LanguageModelLoader(np.concatenate(lm_train), bs, bptt)\n",
    "dl_val = LanguageModelLoader(np.concatenate(lm_val), bs, bptt)\n",
    "model = LanguageModelData(path, 1, vocab_size, dl_train, dl_val, bs=bs, bptt=bptt)\n",
    "dropouts = np.array([0.25, 0.1, 0.2, 0.02, 0.15])*0.7"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [],
   "source": [
    "learner = model.get_model(opt, embedding_size, num_hidden, num_layers, \n",
    "    dropouti=dropouts[0], dropout=dropouts[1], wdrop=dropouts[2], dropoute=dropouts[3], dropouth=dropouts[4])\n",
    "\n",
    "learner.metrics = [accuracy]\n",
    "learner.freeze_to(-1)\n",
    "learner.model.load_state_dict(weights)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [],
   "source": [
    "#learner.lr_find(start_lr=lr/10, end_lr=lr*10, linear=True)\n",
    "#learner.sched.plot()\n",
    "#learner.load('lm_last_fit_full')\n",
    "learner.save_encoder('lm_last_fit_full_encoder')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "4011d568fa764f049e04b252275a384d",
       "version_major": 2,
       "version_minor": 0
      },
      "text/html": [
       "<p>Failed to display Jupyter Widget of type <code>HBox</code>.</p>\n",
       "<p>\n",
       "  If you're reading this message in the Jupyter Notebook or JupyterLab Notebook, it may mean\n",
       "  that the widgets JavaScript is still loading. If this message persists, it\n",
       "  likely means that the widgets JavaScript library is either not installed or\n",
       "  not enabled. See the <a href=\"https://ipywidgets.readthedocs.io/en/stable/user_install.html\">Jupyter\n",
       "  Widgets Documentation</a> for setup instructions.\n",
       "</p>\n",
       "<p>\n",
       "  If you're reading this message in another frontend (for example, a static\n",
       "  rendering on GitHub or <a href=\"https://nbviewer.jupyter.org/\">NBViewer</a>),\n",
       "  it may mean that your frontend doesn't currently support widgets.\n",
       "</p>\n"
      ],
      "text/plain": [
       "HBox(children=(IntProgress(value=0, description='Epoch', max=1), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "  0%|          | 50/59020 [00:08<2:42:47,  6.04it/s, loss=8.03] \n",
      "  0%|          | 51/59020 [00:08<2:42:42,  6.04it/s, loss=8.03]"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Exception in thread Thread-4:\n",
      "Traceback (most recent call last):\n",
      "  File \"/home/paperspace/anaconda3/envs/fastai/lib/python3.6/threading.py\", line 916, in _bootstrap_inner\n",
      "    self.run()\n",
      "  File \"/home/paperspace/anaconda3/envs/fastai/lib/python3.6/site-packages/tqdm/_tqdm.py\", line 144, in run\n",
      "    for instance in self.tqdm_cls._instances:\n",
      "  File \"/home/paperspace/anaconda3/envs/fastai/lib/python3.6/_weakrefset.py\", line 60, in __iter__\n",
      "    for itemref in self.data:\n",
      "RuntimeError: Set changed size during iteration\n",
      "\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "epoch      trn_loss   val_loss   accuracy                           \n",
      "    0      2.871062   2.591112   0.512873  \n",
      "\n"
     ]
    }
   ],
   "source": [
    "learner.fit(lr, 1, wds=wd, use_clr=(32,2), cycle_len=1)\n",
    "learner.save('lm_last_fit_full')\n",
    "learner.save_encoder('lm_last_fit_full_encoder')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAXAAAAD8CAYAAABuHP8oAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDIuMS4xLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvAOZPmwAAIABJREFUeJzt3Xl8FfW9//HXJwlJCHsgIJsEEBGRPaKgghWrbNVra71qrdpq7bW93i7684JSd4SqbbW1dalXpdflVi1WC24IKCoIhk0QkDVsIgkJawIhy/f3xzk5ZCUnmJOZSd7PxyOPzPmemXM+33B4Z/Kd78yYcw4REQmeOK8LEBGRE6MAFxEJKAW4iEhAKcBFRAJKAS4iElAKcBGRgFKAi4gElAJcRCSgFOAiIgGVEIsX7dChg0tPT4/FS4uINEpLly7d45xLq8s2MQnw9PR0MjMzY/HSIiKNkpltres2GkIREQkoBbiISEApwEVEAkoBLiISUApwEZGAUoCLiARUVAFuZr8ysy/MbLWZvWxmybEuTEREjq/WADezrsB/ARnOuTOAeODKWBTz8pJtLN26NxYvLSLS6ER7Ik8C0NzMioAU4KtYFDN55ioAsqZPiMXLi4g0KrXugTvndgKPANuAXcB+59x7sS5MRESOL5ohlHbApUBPoAvQwsyuqWa9m8ws08wyc3Jy6r9SERGpIJqDmBcCW5xzOc65ImAmMLLySs65p51zGc65jLS0Ol2PJeK8Ph0AOFpcekLbi4g0JdEE+DbgbDNLMTMDxgBrY1HMzn2HAfh6/5FYvLyISKMSzRj4YuA1YBmwKrzN07Eo5tZv9wWgoKg4Fi8vItKoRDULxTl3N3B3jGuhXUozAPYVFMX6rUREAs9XZ2K2iQT4UY8rERHxP18FeNuURAD2H9YeuIhIbXwV4GVDKDv3Hva4EhER//NVgCcnxAPwx3kbPa5ERMT/fBXgcXHmdQkiIoHhqwAvr7hEJ/OIiByPbwN8zyHNRBEROR7fBfj1I9MBOFSomSgiIsfjuwAf068jALnaAxcROS7fBXib5qGphAeP6HR6EZHj8V2At0wKnd2/+6AuaCUicjy+C/Bm8aGS7nx9tceViIj4m+8CvHtqitcliIgEQrT3xGxQLRLjOamNbnwvInI8vtsDBzizZyotknz5u0VExDd8GeCtk5tpFoqISC18GeA5BwvZsiff6zJERHzNlwG+aHMuAIcKtRcuIlITXwb494Z2A+CLnfs9rkRExL98GeApiaHrgj/x4SaPKxER8S9fBvjN5/cG4OL+J3lciYiIf/kywNuGb602eeYqjysREfEvXwZ482bxXpcgIuJ7vjxbxswY1L1t5MqEIiJSlS/3wAFaJyewY2+B12WIiPiWL/fAAT7asAcA5xxmutmxiEhlvt0D79gqCdDJPCIiNfFtgN8+9jQA8vJ1azURker4NsDbt0wEdHd6EZGa1BrgZtbXzFaU+zpgZr+MdWEdWoSGUHIPFcb6rUREAqnWg5jOuS+BwQBmFg/sBF6PcV2RPXANoYiIVK+uQyhjgE3Oua2xKKa81BahAM9VgIuIVKuuAX4l8HJ1T5jZTWaWaWaZOTk537iw5GbxtExKYI+GUEREqhV1gJtZInAJ8Gp1zzvnnnbOZTjnMtLS0uqluNQWieTqIKaISLXqsgc+DljmnNsdq2Iq25ZXwJsrv2qotxMRCZS6BPhV1DB8IiIiDS+qADezFODbwMzYllPRL8b0AaC4pLQh31ZEJBCiCnDnXIFzrr1zrkHvcdYufF3wA7pDvYhIFb49ExPghcXbAHh6wWaPKxER8R9fB/igbm0BeFL3xhQRqcLXAX5R/05elyAi4lu+DvCB3dp4XYKIiG/5OsBPap3sdQkiIr7l6wDXnXhERGrm6wAXEZGa+T7Ae6W1ICnB92WKiDQ43yfjead0IFEBLiJShe+TsWPrZA4eKeZIUYnXpYiI+IrvA7xs+GThpj0eVyIi4i++D/C1uw4C8OPnMz2uRETEX3wf4Hd953QAfjq6l8eViIj4i+8DvE3z0BUJX1+20+NKRET8xfcBXib7oO6NKSJSXoLXBURj9Klp7C3QvTFFRMoLxB54e93cWESkikAEeLsWieTlK8BFRMoLRICXOsfhohLyC3VrNRGRMoEI8Oc+yQLg1czt3hYiIuIjgQjwrm2bA7ByR4PeU1lExNcCEeAdWiYC8PpyzQUXESkTiAB/9MohXpcgIuI7gQjw9PYpXpcgIuI7gQhwM+OyIV3p1q6516WIiPhGIAIcoGOrJLIPFOKc87oUERFfCE6At07maEkp+wqKvC5FRMQXAhPgnVonAbqolYhImagC3MzamtlrZrbOzNaa2YhYF1ZZx1bJAOw+cKSh31pExJeivRrhY8A7zrnLzSwRaPBpIdoDFxGpqNYAN7PWwCjgegDn3FGgwa8sVbYHvmXPoYZ+axERX4pmCKUXkAM8Z2bLzewZM2sR47qqaJ4YD8Cf529q6LcWEfGlaAI8ARgKPOGcGwLkA5Mqr2RmN5lZppll5uTk1HOZIiJSWTQBvgPY4ZxbHH78GqFAr8A597RzLsM5l5GWllafNYqISDVqDXDn3NfAdjPrG24aA6yJaVU1GHfGSQAUlZR68fYiIr4S7TzwW4AXzexzYDDwYOxKqllOeAbKgvUaohERiWoaoXNuBZAR41pq1Sw+9Ptm6uy1jOnXyeNqRES8FZgzMQFG9m4PwOY9+R5XIiLivUAF+PeGdQNg1Kk6SCoiEqgAbx++M4/GwEVEAhbgifGBKldEJKYClYhmFlnemqtxcBFp2gIV4OWt2qk71ItI0xa4AE+IC+2FK8BFpKkLXIAP6NYGgNUKcBFp4gIX4D07hC6E+MnGXI8rERHxVuAC/JdjTvW6BBERXwhcgJ/cvsFvBiQi4kuBC/DySkud1yWIiHgm0AG+cJPGwUWk6QpkgD/0vYEAZOlkHhFpwgIZ4BeeHrqU7JR/rva4EhER7wQywFNbJHpdgoiI5wIZ4OXpQKaINFWBDfCze6UCsP9wkceViIh4I7ABPmFgFwCWbdvrcSUiIt4IbIBnhW+rdsOMTI8rERHxRmAD/Gfn9/a6BBERTwU2wNu3TIosO6cDmSLS9AQ2wMtbsGGP1yWIiDS4QAf4yN7tAbj7DZ3QIyJNT6AD/KejQ+PgWbkFHlciItLwAh3gZXvgAIePlnhYiYhIwwt0gDeLP1b+3oKjHlYiItLwAh3g5ZXNCxcRaSqiCnAzyzKzVWa2wsx8debMs9dnAPDEh5s8rkREpGEl1GHdbznnfDdfb/SpHQHIPlDocSUiIg0r8EMo8XEGwJe7D3pciYhIw4o2wB3wnpktNbObYlnQN1FwtNjrEkREGky0AX6Oc24oMA74uZmNqryCmd1kZplmlpmTk1OvRUbr9Lve9eR9RUS8EFWAO+e+Cn/PBl4HhlezztPOuQznXEZaWlr9VlmLKRP6Nej7iYj4Qa0BbmYtzKxV2TJwEeCrc9d/fE7PyHJhsU7oEZGmIZo98E7Ax2a2ElgCzHbOvRPbsuomLnwgE6DvFF+VJiISM7VOI3TObQYGNUAtIiJSB4GfRlhmzGkdI8u6LoqINAWNJsB/d8WxPxJ+9PwSDysREWkYjSbA26YkMv27AwD4dHOex9WIiMReowlwgImDukSWdx844mElIiKx16gCvGXSsWOyb6zY6WElIiKx16gCHGDIyW0BePCtdR5XIiISW40uwJ+//thJonn5usmDiDRejS7A26Q0iywPvX8O50yf52E1IiKx0+gCHOC/x54WWd6577CHlYiIxE6jDPBhPdpVeOyc86gSEZHYaZQBPrxnKi/deBbn9ekAwIbsQx5XJCJS/xplgAOMPKUDaS2TALjoDws4VKibPYhI49JoAxygdfNjBzRHPzTfw0pEROpfow7w/3dx38hyrqYUikgj06gDvEVSArdccIrXZYiIxESjDnCAn3/rWICnT5rNxmzdvV5EGodGH+DJzeIrPL7w9ws8qkREpH41+gAHePzqIV6XICJS75pEgE8c2KXC47vfWE1JqU7uEZFgaxIBDvDMtRmR5RmLttL7jrc8rEZE5JtrMgF+4emd6J7a3OsyRETqTZMJcIBHLh9U4fE7q3d5VImIyDfXpAL8rF7tmfOrUZHH//HCMhasz/GwIhGRE9ekAhygT6dWfHbnhZHH1z6rO9iLSDA1uQAHSGuVVOHxz15cykuLt3lUjYjIiWmSAQ5w+bBukeW3Vn3NHa+v8rAaEZG6a7IBfuf4flXaps5e40ElIiInpskGeLsWiWx+cHyFtr9+tIUlW/Io1Uk+IhIATTbAAeLijDm/GkVqi8RI2xVPLeJP8zaya7/upSki/hZ1gJtZvJktN7NZsSyoofXp1IoZPxpeoe0P769nxLR5HDxS5FFVIiK1q8se+C+AtbEqxEsDurWpvv2e9xq4EhGR6EUV4GbWDZgAPBPbcvznD3PWe12CiEi1ot0DfxS4HSitaQUzu8nMMs0sMycneGc3zrt1NO1SmlVpf2zuBp76cBNFJTV2XUTEE7UGuJlNBLKdc0uPt55z7mnnXIZzLiMtLa3eCmwovdJasvyui2gWb1Wem/b2Ovrc+TZ7DhV6UJmISPUSoljnHOASMxsPJAOtzewF59w1sS3NGxumjmd7XgH7Cor4zuMfV3guMyuPsWd09qgyEZGKat0Dd85Nds51c86lA1cC8xpreJfpnprCgG5tWHLHmArt//HCMo2Ji4hvNOl54LVpV25+eJnH5m4gfdJsCotLPKhIROSYOgW4c+4D59zEWBXjN83ia/7xPPdJVsMVIiJSDe2B12Ld/WN58pphVdqnv73Og2pERI5RgNciuVk8Y884qdrn0ifNZu7a3Q1ckYhIiAI8Sh/d/i0e+t7AKu03zMhk/rpsFm3K9aAqEWnKzLn6v/JeRkaGy8zMrPfX9YvhU98n+2DVOeHXj0znnkv6sz2vgJTEeNq3TKpmaxGRqsxsqXMuoy7baA/8BEwad1q17c8vzGLRplzOe2g+wx54v4GrEpGmRgF+Ai4d3LXG567666eR5W///kPOuPvdhihJRJogBfgJiI8zvjukK62Tj38i64bsQxwqLG6gqkSkqVGAn6Df//tgPr/nYrKmT6h13R/+z2IAvt5/JNZliUgTogCvB1nTJ/D41UNqfP6jDXu4+43VnD1tLpNn6ubJIlI/FOD1ZOLALpzRtXWNz89YtBWAl5dsI33SbA7obj8i8g0pwOvRrFvOi3rdgbrbj4h8Q9FcTlbq4L5L+5O1p4ARvduz/3ARt726ssZ1f/l/y/l853425+Rz47k9mTLx9AasVESCTifyxFhpqeP3c9bz+PyNta775n+ew0mtk+nYOrkBKhMRPzmRE3kU4A1kU84hxvzuw6jWvSKjGzeffwo9O7SIcVUi4hcnEuAaQmkgvdNaEmdQGsXvy1cyd7B82z5Gn5rGbRf3JblZfOwLFJHA0R54Aystdby2dAe3/+PzOm13SseWvP/r0TGqSkS8pmuhBEBcnHHFmd15+PJjVzace2vtwbwx+xAPzFpD+qTZ7Cs4GssSRSQgFOAe+X5Gd1LDt2zrndYyqm2e+XgLAIPvm8P0t9fx+vId5ISvivjsx1tInzSbWPxFJSL+pDFwDy2afAFlefvs9Rn8+Pnoh52e/HATAIkJcUy7bAD3zVoDQP7RElom6Z9VpCnQHriHkhLiIwcoLzitE1nTJ9CqjuF7tLiUW8vNNX9r1S425RwCIL+wmL35oeGWxZtz2Z5XUE+Vi4gf6CCmz5SUOrbm5nNBlFMOa9KpdRK7D1S96cS8W0eTX1jCjr0FDO3Rjk6acy7iC5oH3gjtLygic2seN8yIzc+za9vmvP7zkXRslcyXXx8k91AhI0/pEPX2WXvy6dE+BTOLSX0iTYVmoTRCbVKaMaZfp8jjTQ+OJ2v6BJbcMaZeXn/nvsMMnzqXBetzuPjRBVz9zOKot129cz/nP/IBz32SVS+1iEjdKMAD4ssHxvLp5DHEx4X2dDu2Tuat/4r+4lm1ufbZJZFl5xz/++lWZi7bUeP6uw8cYfn2fUBo3B1g2ltreeHTrfVWk4gcn6YrBERSQjwntal4RubpXY5dvjZr+gTSJ82ul/fqOfmtyPJjczewNTd08HPlXRfRunkCZsZZD86NrJO5dS+HCot5asFmAK45u0fkuUOFxTz2/nr+e+xpJMRrf0GkPul/VMDN+dUo5t92foW2LdPGR7Vtr7Tar7VSFt4Ag+57jwt//2G11zIf/9hH1W4/7P45/PWjLUx7e12V57IPHmHX/sMV2tbvPshdb6ymJHzNgeKSUu771xqyD+huRiKV6SBmI7IttwAz6J6awshpc7lkcFfaNG9GRno7zkxPBeDTzblc+fSnPP+jMzm/b0e25RYw6uH59V5L82bxPPBvZ1SY4rj63osrzFEv+4th3q2j6RU+men7Ty7ks6y9NIs3vrh3LDfM+IyPNuzhwn4deea6M3HOsS2vgB7tdaEvaVw0C0XqrLC4hL5T3mmw97v6rJMZ0as9f5y7gQ3Zhyo899drM7h/1hq2VTNf/ZxT2vPijWfzy/9bzj9XfMWnk8dwUpvqp0D++pUVzFy2k/+5LoN567LZnJPPyzedzcrt+3h16Xbuv/SMRj9rZsH6HIb3TK3ThdAWbtxDn06tSGuVFMPKpCYxmYViZslmtsTMVprZF2Z274mXKH6TlHDsP/gd40+jcw2hWF9eWryNW15eXiW8AX7yt8xqwxtgyZY8AP654isAdu4LrZeZlcfucsMrpaWOmct2AnDDjExeXLyNRZtz2bD7IFf/9VNe+HQbBwuLj1tjYXEJy7btJftgMIdtlm7dy7XPLuF3731Zp+2ufmYxZ059P0ZVSSxEMwZeCFzgnBsEDAbGmtnZsS1LvHDTqN7M+PFwr8uoVlGJI79c8L6/NhuAy59cVOGAak3h/O0/LCD/aAkA/1i6g6PFpZSWOlbt2M/3nlgIQM7BQq54ahF9p7zDd/+ykOFT53LjjMzI9WWOFJVw4EjRN76Y2Ccb93A4XEssbAr/clwc/qUXjaKS0liVU29KSh3FUdb5xAeb+CwrjzVfHWjUZyDXaQjFzFKAj4GbnXM1ThjWEEqw5OUfpbikNHInoMqzWa4d0YM7xvcjzoy/Z27nN/9cDYRmvlS3flAkJsRxtDgUCK2TE2iRlMCu/dXvdS/7zbcZev+cyONV91xEq+RmQGjI5uMNe8gvLGbB7d+ifcsklm/by2V/WchPR/Vi6da9vPofI9iUc4ibX1gW+evjB2edzNTLBtRYX1FJKTOX7eDyYd0j00drc/7D88kqd+B57X1jaZ547K+sHXsLOFpcGjnmUGb/4SIG3Ru6T2vZv6vf9L/rHfKPllSpr7iklL8t2so1Z/cgMSG0T1r5M7ll2vhah812HzjCPW9+wcPfH0TLpAReW7qD215dyYap4+hz59tAbH82MTuRx8zizWwFkA3MqS68zewmM8s0s8ycnJy61CAeS22RWOE2blnTJ7Bx6jievGYYI3u3575LzyC5WTyJCXH88OwerLt/LBunjous37FVEokJcWRNn8APy00hBHj86iH07dSqxve+anj3+u9QlMrCG+DAkeIawxuoMif++08u4mhxKemTZjNz2U6yDxaSf7SEYQ+8j3OOy/4S2qt/asFmMrfu5eYXlnHh7xdUGDp6cfG2Ku+zPa+A9EmzeWPFTp79eAv//Y9V/Hn+Rr78+iDOuQpXm7zk8Y9JnzSb0tJQe/qk2RXCG+DGv31GnzvfYvLMVQCc+9v5XPC7D3ln9S7eX7ObN1d+xdpdB7j4Dwuq7bdzx/Z6nXNMnb2GXfsPU1xSytbcfJZuzYt6r/hErdi+jwffWhv5C6qsv0u37sU5x0tLtnHfrDU88t6XOOcoreauKfsPV505Vdm1/7OEt1d/zYvhcxnK7mf7dHh6LIR+MZRd9TNrTz5Ze/Lro4snrK574G2B14FbnHOra1pPe+BN2/6CIt5d8zWDurWl70nHwru6PfWyPZpDhcWccfe7VZ6/+fzePPHBptgVe4JaJMZzuKgkqjssHY8Z/Os/z2Xinz6u8lyfji2rPVYw/bsDuHL4yXX+yychziiOouCnfjiMLm2a8+bKnfxj2U7y8o/y+T0XMfCe96pd/7w+HUhKiOei/p14a9UuHr96KM9/soWbzz+F+Dhjc84hDheVsPvAEd5a9TVvrviKubeOpntqCgCHj5Ywcvpc9hYUcc93Tuf6c3pGXts5V+G8hMq6tEnmq0q/eJ//0Zlc/9xnVdad+bORZGblcdOo3lzwyAec26cD9116Bs99soVeaS25rtzJbOXPq3jwsgHc8fqqCq917yX9ufvNLwCYMqEfN5zb8xsfGG+QWShmdjeQ75x7pKZ1FOBSnRHT5rJr/xHm3TqaOWt2s373IX53xaDI8z9+/jPmrcvmocsHctmQrsSZER9nJzREkzV9Al/vP8LZ0+bWvrLUavSpaXy4vn7/sp51y7n079L6uAEdC0/8YCg3v7isXl/zioxuPHT5oNpXPI6YBLiZpQFFzrl9ZtYceA/4rXNuVk3bKMClOgVHi9lXUESXts3rtN3W3HxGP/xB5PGWaeNr/U9ftmdfNo5ZV6ktEsnL152PYu2uiadHrmUfdN90fDxWNzXuDMwws3hCY+avHC+8RWqSkphASmLdr97Qo32LKgehsqZPiIwH3/rKSob0aMc1Z51c5c/YDi0TKzyedUvF4YqubZvztxuGs3L7Pn79ykrMwDlYOOkCbpyRyZSJ/Rj7aPVnmZb589VDWb/7II/N3VDnvjV1jSW8AT7fsY/c/KN8q2/HBntPncgjjd7SrXsjUwU3Th3HnDW7+c0bXzBlQj/+bUjXWrcvP4STNX0Cve94K3Kqf1lb5fWq85cfDGXJljyeX5h1Ar2ou/+64BT+OG9jg7yX1346qhdrdh3gow17vC7lhPfEdTlZkWoM69GOF244i0sHdyEhPo5xAzqTOeXCqMIb4KWfnAXA0ikXAvDuL0dFnnviB0Mjy/+4eWSF7S46PXQZ4GnfHUDW9AmMH9CZey7pz5xfhbbv1aFFlevRvPSTs5h/2/m0TWlWpY61941lyoR+FdrGDziJ+y/tD8B3BnXhk0kXAPDovw/mFxeeWuU1OrdJZvK406rt59CT21bbXllZ/fWtpuDLDP/cIXQD8F9V06/J4/vxzHV1yr46W3XPRfzkvJ5cOrhLpK3yrKuGpj1wkRMweebnjOjdgUsGdalxHecc+w8X0TYlscZ1AB56Zx1/+WATD18+kO9nHJtWuXDjHjLSU3lvzdf06tCywtUny/b2N0wdR7NarvIYmn4IceG55M99soV7/7WGH57dg+tG9uC+WWtZsD6H+y/tz8SBXdi+t4BLHv8ksv2sW87l1czt3HBuL05unxJ5zcrHIYac3JYfndOTLm2Sue3VlTjgquEnM7xnKt8NT6uM9Pnygdz+2ucV2rKmT+Cd1btYtXM/f56/KdJWvr/rHxhHYkJc5K+qLm2SmXfb+ZFLBlz19Kcs2pwb2qZ9Ck/9MIOs3Hx++r9La/z5dG6TTG7+0ci00iV3jGHyzFVMHn8a3dqlcNpv3uHxq4cwceCxf+ucg4WktkjkwOEihpQ7P6B8zXWla6GIBNCRohJeWryN60amR33Czva8AuLjrM4HhCF0wsrEP33MSzeeRZ/wHP0jRSUkJcRFjiEcKSrhtN+ErpFTWyC9tHgbJ6emcG6fmu/kdOsrK8nLL2TCwC6kt09h5vKdvFRpHnz595mxMIthPdpxRtc2AHzrkQ/Ysie/wrGQ/MJiEuKtwuUgjhSV8PX+IyzZkscVZ4Z+GR4+WkK/u96hX+fWvPyTs8g+WMhj729g9qpdPHNtBuec0oHmifGRXxJ1DeDyQ2dTJvTjxvN61Wn7MgpwEak3Jxpo0dieV8B5D83n3FM68PHGPbW+z/7DRWzZk8/g7tEN81S2Mfsg3dqlVLi4V0mpq/ALM33SbIb3TOWVn46o02tPf3sdT35Y8S+GE6EAF5F6U1hcQmkpFU7Fr0/vr9nNmT1TWb5tL4ePljBuQOeYvE+0DhUWk5QQV+uQVGXOOV7J3E7H1snfaAaKAlxEJKA0C0VEpAlRgIuIBJQCXEQkoBTgIiIBpQAXEQkoBbiISEApwEVEAkoBLiISUDE5kcfMcoCtJ7h5B8D7a0LWj8bUF2hc/WlMfQH1x8+i7UsP51xaXV44JgH+TZhZZl3PRvKrxtQXaFz9aUx9AfXHz2LZFw2hiIgElAJcRCSg/BjgT3tdQD1qTH2BxtWfxtQXUH/8LGZ98d0YuIiIRMePe+AiIhIF3wS4mY01sy/NbKOZTfK6nvLM7Fkzyzaz1eXaUs1sjpltCH9vF243M/tjuB+fm9nQcttcF15/g5ldV659mJmtCm/zRyu7Z1Rs+tLdzOab2Voz+8LMfhHw/iSb2RIzWxnuz73h9p5mtjhc29/NLDHcnhR+vDH8fHq515ocbv/SzC4u196gn00zizez5WY2qxH0JSv8WVhhZpnhtqB+1tqa2Wtmti78/2eE530J3fDU2y8gHtgE9AISgZXA6V7XVa6+UcBQYHW5toeASeHlScBvw8vjgbcBA84GFofbU4HN4e/twsvtws8tAUaEt3kbGBfDvnQGhoaXWwHrgdMD3B8DWoaXmwGLw3W+AlwZbn8SuDm8/DPgyfDylcDfw8unhz93SUDP8Ocx3ovPJvBr4CVgVvhxkPuSBXSo1BbUz9oM4MbwciLQ1uu+xOwfro4/mBHAu+UeTwYme11XpRrTqRjgXwKdw8udgS/Dy08BV1VeD7gKeKpc+1Phts7AunLtFdZrgH69AXy7MfQHSAGWAWcROnEiofLnC3gXGBFeTgivZ5U/c2XrNfRnE+gGzAUuAGaFawtkX8LvkUXVAA/cZw1oDWwhfNzQL33xyxBKV2B7ucc7wm1+1sk5twsg/L3sZng19eV47TuqaY+58J/cQwjttQa2P+EhhxVANjCH0F7mPudccTU1ROoOP78faE/d+xkrjwK3A6Xhx+0Jbl8AHPCemS01s5vLPWQoAAACWElEQVTCbUH8rPUCcoDnwsNbz5hZCzzui18CvLqxnqBOj6mpL3Vtjykzawn8A/ilc+7A8Vatps1X/XHOlTjnBhPaex0O9DtODb7tj5lNBLKdc0vLNx/n/X3bl3LOcc4NBcYBPzezUcdZ18/9SSA0jPqEc24IkE9oyKQmDdIXvwT4DqB7ucfdgK88qiVau82sM0D4e3a4vaa+HK+9WzXtMWNmzQiF94vOuZnh5sD2p4xzbh/wAaExx7ZmllBNDZG6w8+3AfKoez9j4RzgEjPLAv6P0DDKowSzLwA4574Kf88GXif0CzaIn7UdwA7n3OLw49cIBbq3fYnl+FcdxpcSCA3m9+TYwZX+XtdVqcZ0Ko6BP0zFgxcPhZcnUPHgxZJweyqhMbR24a8tQGr4uc/C65YdvBgfw34Y8Dfg0UrtQe1PGtA2vNwc+AiYCLxKxQN/Pwsv/5yKB/5eCS/3p+KBv82EDvp58tkEzufYQcxA9gVoAbQqt7wQGBvgz9pHQN/w8j3hfnjal5h+COv4wxlPaEbEJuBOr+upVNvLwC6giNBvyhsIjTXOBTaEv5f9Ixjw53A/VgEZ5V7nx8DG8NePyrVnAKvD2zxOpQMl9dyXcwn9afY5sCL8NT7A/RkILA/3ZzVwV7i9F6Gj+hsJBWBSuD05/Hhj+Ple5V7rznDNX1JuBoAXn00qBngg+xKue2X464uy9wvwZ20wkBn+rP2TUAB72hediSkiElB+GQMXEZE6UoCLiASUAlxEJKAU4CIiAaUAFxEJKAW4iEhAKcBFRAJKAS4iElD/H+0NrAKaG9QMAAAAAElFTkSuQmCC\n",
      "text/plain": [
       "<matplotlib.figure.Figure at 0x7fae46692278>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "learner.sched.plot_loss()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Backward Language Model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {},
   "outputs": [],
   "source": [
    "f = Path('data/lm/tmp/language_model.npy')\n",
    "g = Path('data/lm/tmp/int_to_str.npy')\n",
    "if not (f.is_file() and g.is_file()):\n",
    "    language_model = np.array([[str_to_int[o] for o in p] for p in tokens])\n",
    "    np.save(lm_path/'tmp'/'language_model.npy', language_model)\n",
    "    pickle.dump(int_to_str, open(lm_path/'tmp'/'int_to_str.pkl', 'wb'))\n",
    "else:\n",
    "    language_model = np.load(lm_path/'tmp'/'language_model.npy')\n",
    "    int_to_str = pickle.load(open(lm_path/'tmp'/'int_to_str.pkl', 'rb'))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {},
   "outputs": [],
   "source": [
    "reversed_language_model = []\n",
    "for doc_vector in language_model:\n",
    "    reversed_doc_vector = doc_vector[::-1]#[:-1]\n",
    "    #reversed_doc_vector.insert(0,str_to_int('xbof'))\n",
    "    reversed_language_model.append(reversed_doc_vector)\n",
    "reversed_language_model = np.array(reversed_language_model)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {},
   "outputs": [],
   "source": [
    "weights = torch.load(lm_path/'models/wt103/bwd_wt103.h5', map_location=lambda storage, loc: storage)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {},
   "outputs": [],
   "source": [
    "encoder_weights = to_np(weights['0.encoder.weight'])\n",
    "row_means = encoder_weights.mean(0)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {},
   "outputs": [],
   "source": [
    "embedding_size = 400\n",
    "num_hidden = 1150\n",
    "num_layers = 3"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {},
   "outputs": [],
   "source": [
    "# key used for wiki103\n",
    "int_to_str_wiki = pickle.load((lm_path/'models'/'wt103'/'itos_wt103.pkl').open('rb'))\n",
    "str_to_int_wiki = defaultdict(lambda:-1, {v:k for k,v in enumerate(int_to_str_wiki)})"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {},
   "outputs": [],
   "source": [
    "# map suicide embedding weights to wiki103 embedding weights\n",
    "new_weights = np.zeros((vocab_size, embedding_size), dtype=np.float32)\n",
    "for i,w in enumerate(int_to_str):\n",
    "    row = str_to_int_wiki[w]\n",
    "    new_weights[i] = encoder_weights[row] if row>=0 else row_means"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "metadata": {},
   "outputs": [],
   "source": [
    "weights['0.encoder.weight'] = T(new_weights)\n",
    "weights['0.encoder_with_dropout.embed.weight'] = T(np.copy(new_weights))\n",
    "weights['1.decoder.weight'] = T(np.copy(new_weights))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "metadata": {},
   "outputs": [],
   "source": [
    "# hyperparameters for language model\n",
    "wd =1e-7\n",
    "bptt = 70\n",
    "bs = 32\n",
    "lr = 1e-3\n",
    "opt = partial(optim.Adam, betas=(0.8, 0.99))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "metadata": {},
   "outputs": [],
   "source": [
    "size = len(reversed_language_model)\n",
    "lm_train,lm_val = sklearn.model_selection.train_test_split(reversed_language_model[:size], test_size=0.1)\n",
    "\n",
    "dl_train = LanguageModelLoader(np.concatenate(lm_train), bs, bptt)\n",
    "dl_val = LanguageModelLoader(np.concatenate(lm_val), bs, bptt)\n",
    "model = LanguageModelData(path, 1, vocab_size, dl_train, dl_val, bs=bs, bptt=bptt)\n",
    "dropouts = np.array([0.25, 0.1, 0.2, 0.02, 0.15])*0.7"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "metadata": {},
   "outputs": [],
   "source": [
    "learner = model.get_model(opt, embedding_size, num_hidden, num_layers, \n",
    "    dropouti=dropouts[0], dropout=dropouts[1], wdrop=dropouts[2], dropoute=dropouts[3], dropouth=dropouts[4])\n",
    "\n",
    "learner.metrics = [accuracy]\n",
    "learner.freeze_to(-1)\n",
    "learner.model.load_state_dict(weights)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "6beb2d7c7ad94a29ad6eae084a74abde",
       "version_major": 2,
       "version_minor": 0
      },
      "text/html": [
       "<p>Failed to display Jupyter Widget of type <code>HBox</code>.</p>\n",
       "<p>\n",
       "  If you're reading this message in the Jupyter Notebook or JupyterLab Notebook, it may mean\n",
       "  that the widgets JavaScript is still loading. If this message persists, it\n",
       "  likely means that the widgets JavaScript library is either not installed or\n",
       "  not enabled. See the <a href=\"https://ipywidgets.readthedocs.io/en/stable/user_install.html\">Jupyter\n",
       "  Widgets Documentation</a> for setup instructions.\n",
       "</p>\n",
       "<p>\n",
       "  If you're reading this message in another frontend (for example, a static\n",
       "  rendering on GitHub or <a href=\"https://nbviewer.jupyter.org/\">NBViewer</a>),\n",
       "  it may mean that your frontend doesn't currently support widgets.\n",
       "</p>\n"
      ],
      "text/plain": [
       "HBox(children=(IntProgress(value=0, description='Epoch', max=1), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "epoch      trn_loss   val_loss   accuracy                           \n",
      "    0      3.095271   2.806746   0.486801  \n",
      "\n"
     ]
    }
   ],
   "source": [
    "learner.fit(lr, 1, wds=wd, use_clr=(32,2), cycle_len=1)\n",
    "learner.save('lm_last_fit_full_rev')\n",
    "learner.save_encoder('lm_last_fit_full_rev_encoder')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.4"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}