Human_similarity_judgment.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import os\n",
    "import pickle\n",
    "import argparse\n",
    "import matplotlib.pyplot as plt\n",
    "import numpy as np\n",
    "import pandas as pd\n",
    "\n",
    "from operator import itemgetter\n",
    "from sklearn.metrics.pairwise import cosine_similarity\n",
    "from scipy.stats import spearmanr"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "data_dir = 'data'\n",
    "wsd_dir = 'WS353'\n",
    "lan_to_file = {'de': 'German', 'en': 'English', 'it': 'Italian'}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "def CommonPairsOfWords(sim_ds, word2idx):\n",
    "    # --- Check for common words between the similarity dataset and the embeddings ---\n",
    "    # Words in the similarity dataset\n",
    "    sim_words = set(sim_ds.Word1.values).union(set(sim_ds.Word2.values))\n",
    "    # Embedded words\n",
    "    train_words = set(word2idx.keys())\n",
    "    # Words in common\n",
    "    common_words = [w for w in sim_words if w in train_words]\n",
    "    print(f\"There are {len(common_words)} words in common out of {len(sim_words)} in the word similarity dataset.\")\n",
    "    \n",
    "    # --- Check for common pairs of words ---\n",
    "    # Pairs of words in the word similarity dataset\n",
    "    pair_of_words = sim_ds[['Word1', 'Word2']].values\n",
    "\n",
    "    # Construction of necessary data structures\n",
    "    ## Common pairs of words, where words are converted into indices wrt our embedding model.\n",
    "    common_pairs_idx = []\n",
    "    ## List of indice of the common pairs of words wrt the word similarity dataset\n",
    "    common_is = []\n",
    "\n",
    "    for i, pair in enumerate(pair_of_words):\n",
    "        if pair[0] in common_words and pair[1] in common_words:\n",
    "            common_pairs_idx.append([word2idx[pair[0]], word2idx[pair[1]]])\n",
    "            common_is.append(i)\n",
    "    print(f'There are {len(common_pairs_idx)} common pairs of words, out of {len(pair_of_words)} in the word similarity dataset.')\n",
    "    \n",
    "    return common_pairs_idx, common_is\n",
    "\n",
    "def ComputeEmbeddingSimilarity(common_pairs_idx, idx2vec):\n",
    "    # List of cosine similarity scores of common pairs of words\n",
    "    train_score = []\n",
    "    for pair in common_pairs_idx:\n",
    "        val = cosine_similarity(idx2vec[pair[0]].reshape(1,-1), idx2vec[pair[1]].reshape(1,-1)).item()\n",
    "        train_score.append(max(0,val))\n",
    "    train_score = np.array(train_score).reshape(-1,1)\n",
    "    return train_score\n",
    "\n",
    "\n",
    "def ComputeSimilarityStats(common_sim, train_score, train_score_ngrams):\n",
    "    \n",
    "    # Let rescale cosine similarity to get values comparable with the average score in sim_ds\n",
    "    common_sim.loc[:,'base'] = np.round(train_score * 10, 2)\n",
    "    common_sim.loc[:,'ngram'] = np.round(train_score_ngrams * 10, 2)\n",
    "    \n",
    "    means = common_sim.mean(axis=0)[['Average Score', 'base', 'ngram']]\n",
    "    print('Means of similarity scores:')\n",
    "    print(means)\n",
    "    print('')\n",
    "\n",
    "    # Let compute how much each rescaled cosine similarity differs from the human similarity score, for both embedding approach\n",
    "    common_sim.loc[:, 'bdiff'] = np.abs(common_sim['Average Score']  - common_sim['base'])\n",
    "    common_sim.loc[:, 'bdiff_v2'] = np.abs(common_sim['Average Score']  - (common_sim['base']+means['ngram']-means['base']))\n",
    "    \n",
    "    common_sim.loc[:, 'ngdiff'] = np.abs(common_sim['Average Score'] - common_sim['ngram'])\n",
    "    \n",
    "    # Let compute the difference between the performance of the two models\n",
    "    common_sim.loc[:, 'bngdiff'] = common_sim['bdiff'] - common_sim['ngdiff']\n",
    "    common_sim.loc[:, 'bngdiff_v2'] = common_sim['bdiff_v2'] - common_sim['ngdiff']\n",
    "    \n",
    "    ngram_better = (common_sim['bngdiff'] > 0).sum()\n",
    "    print(f'''Similarity scores of ngram based approach are closer to the human similarity scores in {ngram_better} case(s) out out of {len(common_sim)} cases. ({100*round(ngram_better/len(common_sim), 3)}%)''')\n",
    "    \n",
    "    ngram_equal = (common_sim['bngdiff'] == 0).sum()\n",
    "    print(f'''Similarity scores of both approaches work equivalently in {ngram_equal} case(s).({100*round(ngram_equal/len(common_sim), 3)}%)''')\n",
    "    ngram_worse = (common_sim['bngdiff'] < 0).sum()\n",
    "    print(f'''Similarity scores of ngram based approach underperfor in {ngram_worse} case(s). ({100*round(ngram_worse/len(common_sim), 2)}%)''')\n",
    "    \n",
    "    print('')\n",
    "    print('After adjusting the scores of the base model by adding the difference between the mean of the ngram and the base model scores')\n",
    "    ngram_better_v2 = (common_sim['bngdiff_v2'] > 0).sum()\n",
    "    print(f'''Similarity scores of ngram based approach are closer to the human similarity scores in {ngram_better_v2} case(s) out out of {len(common_sim)} cases. ({100*round(ngram_better_v2/len(common_sim), 3)}%)''')\n",
    "    ngram_equal_v2 = (common_sim['bngdiff_v2'] == 0).sum()\n",
    "    print(f'''Similarity scores of both approaches work equivalently in {ngram_equal_v2} case(s).({100*round(ngram_equal_v2/len(common_sim), 3)}%)''')\n",
    "    ngram_worse_v2 = (common_sim['bngdiff_v2'] < 0).sum()\n",
    "    print(f'''Similarity scores of ngram based approach underperfor in {ngram_worse_v2} case(s). ({100*round(ngram_worse_v2/len(common_sim), 2)}%)''')\n",
    "    \n",
    "    pd.set_option('display.max_rows', None)\n",
    "    \n",
    "    return (common_sim[['Word1', 'Word2', 'Average Score', 'base', 'ngram', 'bdiff', 'bdiff_v2', 'ngdiff', 'bngdiff', 'bngdiff_v2']],\n",
    "            {'outperforming': ngram_better, 'downperforming': ngram_worse, 'equivalent': ngram_equal},\n",
    "            {'outperforming': ngram_better_v2, 'downperforming': ngram_worse_v2, 'equivalent': ngram_equal_v2})\n",
    "\n",
    "def RelevantPairs(common_sim, worse_threshold, better_threshold):\n",
    "    much_worse = common_sim[common_sim.bngdiff < -worse_threshold]\n",
    "    much_better = common_sim[common_sim.bngdiff > better_threshold]\n",
    "    much_worse_v2 = common_sim[common_sim.bngdiff_v2 < -worse_threshold]\n",
    "    much_better_v2 = common_sim[common_sim.bngdiff_v2 > better_threshold]\n",
    "    \n",
    "    return (much_worse.sort_values('bngdiff'),\n",
    "            much_better.sort_values('bngdiff', ascending = False),\n",
    "            much_worse.sort_values('bngdiff_v2'),\n",
    "            much_better.sort_values('bngdiff_v2', ascending = False))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "common_sim = {}\n",
    "ngram_stats = {}\n",
    "spearman_scores = {}\n",
    "ngram_stats_v2 = {}"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## English"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "lan = 'en'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Word1</th>\n",
       "      <th>Word2</th>\n",
       "      <th>1</th>\n",
       "      <th>2</th>\n",
       "      <th>3</th>\n",
       "      <th>4</th>\n",
       "      <th>5</th>\n",
       "      <th>6</th>\n",
       "      <th>7</th>\n",
       "      <th>8</th>\n",
       "      <th>9</th>\n",
       "      <th>10</th>\n",
       "      <th>11</th>\n",
       "      <th>12</th>\n",
       "      <th>13</th>\n",
       "      <th>Average Score</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>love</td>\n",
       "      <td>sex</td>\n",
       "      <td>9.0</td>\n",
       "      <td>9.0</td>\n",
       "      <td>7.0</td>\n",
       "      <td>7.0</td>\n",
       "      <td>8</td>\n",
       "      <td>10.0</td>\n",
       "      <td>9.0</td>\n",
       "      <td>7.0</td>\n",
       "      <td>10.0</td>\n",
       "      <td>8.0</td>\n",
       "      <td>9.0</td>\n",
       "      <td>8.0</td>\n",
       "      <td>7</td>\n",
       "      <td>8.31</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>tiger</td>\n",
       "      <td>cat</td>\n",
       "      <td>7.0</td>\n",
       "      <td>9.0</td>\n",
       "      <td>9.0</td>\n",
       "      <td>7.0</td>\n",
       "      <td>8</td>\n",
       "      <td>7.0</td>\n",
       "      <td>10.0</td>\n",
       "      <td>7.0</td>\n",
       "      <td>10.0</td>\n",
       "      <td>8.0</td>\n",
       "      <td>9.0</td>\n",
       "      <td>8.0</td>\n",
       "      <td>9</td>\n",
       "      <td>8.31</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>tiger</td>\n",
       "      <td>tiger</td>\n",
       "      <td>10.0</td>\n",
       "      <td>10.0</td>\n",
       "      <td>10.0</td>\n",
       "      <td>10.0</td>\n",
       "      <td>10</td>\n",
       "      <td>10.0</td>\n",
       "      <td>10.0</td>\n",
       "      <td>10.0</td>\n",
       "      <td>10.0</td>\n",
       "      <td>10.0</td>\n",
       "      <td>10.0</td>\n",
       "      <td>10.0</td>\n",
       "      <td>10</td>\n",
       "      <td>10.00</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>book</td>\n",
       "      <td>paper</td>\n",
       "      <td>9.0</td>\n",
       "      <td>8.0</td>\n",
       "      <td>7.0</td>\n",
       "      <td>6.0</td>\n",
       "      <td>6</td>\n",
       "      <td>9.0</td>\n",
       "      <td>8.0</td>\n",
       "      <td>7.0</td>\n",
       "      <td>10.0</td>\n",
       "      <td>5.0</td>\n",
       "      <td>9.0</td>\n",
       "      <td>7.0</td>\n",
       "      <td>9</td>\n",
       "      <td>7.69</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>computer</td>\n",
       "      <td>keyboard</td>\n",
       "      <td>9.0</td>\n",
       "      <td>10.0</td>\n",
       "      <td>5.0</td>\n",
       "      <td>7.0</td>\n",
       "      <td>8</td>\n",
       "      <td>8.0</td>\n",
       "      <td>9.0</td>\n",
       "      <td>10.0</td>\n",
       "      <td>10.0</td>\n",
       "      <td>7.0</td>\n",
       "      <td>9.0</td>\n",
       "      <td>9.0</td>\n",
       "      <td>8</td>\n",
       "      <td>8.38</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "      Word1     Word2     1     2     3     4   5     6     7     8     9  \\\n",
       "0      love       sex   9.0   9.0   7.0   7.0   8  10.0   9.0   7.0  10.0   \n",
       "1     tiger       cat   7.0   9.0   9.0   7.0   8   7.0  10.0   7.0  10.0   \n",
       "2     tiger     tiger  10.0  10.0  10.0  10.0  10  10.0  10.0  10.0  10.0   \n",
       "3      book     paper   9.0   8.0   7.0   6.0   6   9.0   8.0   7.0  10.0   \n",
       "4  computer  keyboard   9.0  10.0   5.0   7.0   8   8.0   9.0  10.0  10.0   \n",
       "\n",
       "     10    11    12  13  Average Score  \n",
       "0   8.0   9.0   8.0   7           8.31  \n",
       "1   8.0   9.0   8.0   9           8.31  \n",
       "2  10.0  10.0  10.0  10          10.00  \n",
       "3   5.0   9.0   7.0   9           7.69  \n",
       "4   7.0   9.0   9.0   8           8.38  "
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# --- Import word similarity dataset WS353 for the selected language ---\n",
    "sim_ds = pd.read_csv(os.path.join(wsd_dir, f'MWS353_{lan_to_file[lan]}.txt'), sep=\",\")\n",
    "sim_ds.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "# --- Import data of our embedding model ---\n",
    "word2idx = pickle.load(open(os.path.join(data_dir, lan, 'word2idx.dat'), 'rb'))\n",
    "# Embeddings without subwords' information\n",
    "idx2vec = pickle.load(open(os.path.join(data_dir, lan, 'idx2vec.dat'), 'rb'))\n",
    "# Embeddings with subwords' information\n",
    "idx2vec_ngrams = pickle.load(open(os.path.join(data_dir, lan, 'idx2vec_ngrams.dat'), 'rb'))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "There are 335 words in common out of 435 in the word similarity dataset.\n",
      "There are 229 common pairs of words, out of 350 in the word similarity dataset.\n"
     ]
    }
   ],
   "source": [
    "# --- Identify common pairs of words ---\n",
    "common_pairs_idx, common_is = CommonPairsOfWords(sim_ds, word2idx)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "# --- Compute cosine similarity of embeddings without subwords' information for common pairs of words ---\n",
    "train_score = ComputeEmbeddingSimilarity(common_pairs_idx, idx2vec)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [],
   "source": [
    "# --- Compute cosine similarity of embeddings with subwords' information for common pairs of words ---\n",
    "train_score_ngrams = ComputeEmbeddingSimilarity(common_pairs_idx, idx2vec_ngrams)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Means of similarity scores:\n",
      "Average Score    5.396114\n",
      "base             1.565983\n",
      "ngram            5.155066\n",
      "dtype: float64\n",
      "\n",
      "Similarity scores of ngram based approach are closer to the human similarity scores in 169 case(s) out out of 229 cases. (73.8%)\n",
      "Similarity scores of both approaches work equivalently in 0 case(s).(0.0%)\n",
      "Similarity scores of ngram based approach underperfor in 60 case(s). (26.0%)\n",
      "\n",
      "After adjusting the scores of the base model by adding the difference between the mean of the ngram and the base model scores\n",
      "Similarity scores of ngram based approach are closer to the human similarity scores in 82 case(s) out out of 229 cases. (35.8%)\n",
      "Similarity scores of both approaches work equivalently in 0 case(s).(0.0%)\n",
      "Similarity scores of ngram based approach underperfor in 147 case(s). (64.0%)\n"
     ]
    }
   ],
   "source": [
    "(cs, ns, ns_v2 ) = ComputeSimilarityStats(sim_ds.iloc[common_is], train_score, train_score_ngrams)\n",
    "common_sim[lan] = cs\n",
    "ngram_stats[lan] = ns\n",
    "ngram_stats_v2[lan] = ns_v2"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [],
   "source": [
    "# --- Compute Spearman's rank correlation coefficients\n",
    "sim_score = common_sim[lan]['Average Score'].values.reshape(-1,1)\n",
    "ss = {'base': spearmanr(sim_score, train_score),\n",
    "      'ngram': spearmanr(sim_score, train_score_ngrams)}\n",
    "spearman_scores[lan] = ss"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [],
   "source": [
    "(much_worse, much_better, much_worse_v2, much_better_v2) = RelevantPairs(common_sim[lan], worse_threshold = 0, better_threshold = 0)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Word1</th>\n",
       "      <th>Word2</th>\n",
       "      <th>Average Score</th>\n",
       "      <th>base</th>\n",
       "      <th>ngram</th>\n",
       "      <th>bdiff</th>\n",
       "      <th>bdiff_v2</th>\n",
       "      <th>ngdiff</th>\n",
       "      <th>bngdiff</th>\n",
       "      <th>bngdiff_v2</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>260</th>\n",
       "      <td>profit</td>\n",
       "      <td>warning</td>\n",
       "      <td>0.77</td>\n",
       "      <td>0.87</td>\n",
       "      <td>8.95</td>\n",
       "      <td>0.10</td>\n",
       "      <td>3.689083</td>\n",
       "      <td>8.18</td>\n",
       "      <td>-8.08</td>\n",
       "      <td>-4.490917</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>323</th>\n",
       "      <td>sugar</td>\n",
       "      <td>approach</td>\n",
       "      <td>1.00</td>\n",
       "      <td>1.50</td>\n",
       "      <td>7.99</td>\n",
       "      <td>0.50</td>\n",
       "      <td>4.089083</td>\n",
       "      <td>6.99</td>\n",
       "      <td>-6.49</td>\n",
       "      <td>-2.900917</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>215</th>\n",
       "      <td>governor</td>\n",
       "      <td>interview</td>\n",
       "      <td>1.31</td>\n",
       "      <td>1.58</td>\n",
       "      <td>7.41</td>\n",
       "      <td>0.27</td>\n",
       "      <td>3.859083</td>\n",
       "      <td>6.10</td>\n",
       "      <td>-5.83</td>\n",
       "      <td>-2.240917</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>33</th>\n",
       "      <td>king</td>\n",
       "      <td>cabbage</td>\n",
       "      <td>1.15</td>\n",
       "      <td>1.24</td>\n",
       "      <td>6.34</td>\n",
       "      <td>0.09</td>\n",
       "      <td>3.679083</td>\n",
       "      <td>5.19</td>\n",
       "      <td>-5.10</td>\n",
       "      <td>-1.510917</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>181</th>\n",
       "      <td>century</td>\n",
       "      <td>nation</td>\n",
       "      <td>1.46</td>\n",
       "      <td>1.39</td>\n",
       "      <td>6.37</td>\n",
       "      <td>0.07</td>\n",
       "      <td>3.519083</td>\n",
       "      <td>4.91</td>\n",
       "      <td>-4.84</td>\n",
       "      <td>-1.390917</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "        Word1      Word2  Average Score  base  ngram  bdiff  bdiff_v2  ngdiff  \\\n",
       "260    profit    warning           0.77  0.87   8.95   0.10  3.689083    8.18   \n",
       "323     sugar   approach           1.00  1.50   7.99   0.50  4.089083    6.99   \n",
       "215  governor  interview           1.31  1.58   7.41   0.27  3.859083    6.10   \n",
       "33       king    cabbage           1.15  1.24   6.34   0.09  3.679083    5.19   \n",
       "181   century     nation           1.46  1.39   6.37   0.07  3.519083    4.91   \n",
       "\n",
       "     bngdiff  bngdiff_v2  \n",
       "260    -8.08   -4.490917  \n",
       "323    -6.49   -2.900917  \n",
       "215    -5.83   -2.240917  \n",
       "33     -5.10   -1.510917  \n",
       "181    -4.84   -1.390917  "
      ]
     },
     "execution_count": 15,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Pairs of words of worst underperforming cases of ngram based model\n",
    "much_worse.head(5)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Word1</th>\n",
       "      <th>Word2</th>\n",
       "      <th>Average Score</th>\n",
       "      <th>base</th>\n",
       "      <th>ngram</th>\n",
       "      <th>bdiff</th>\n",
       "      <th>bdiff_v2</th>\n",
       "      <th>ngdiff</th>\n",
       "      <th>bngdiff</th>\n",
       "      <th>bngdiff_v2</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>263</th>\n",
       "      <td>dollar</td>\n",
       "      <td>buck</td>\n",
       "      <td>9.54</td>\n",
       "      <td>0.68</td>\n",
       "      <td>8.47</td>\n",
       "      <td>8.86</td>\n",
       "      <td>5.270917</td>\n",
       "      <td>1.07</td>\n",
       "      <td>7.79</td>\n",
       "      <td>4.200917</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>197</th>\n",
       "      <td>treatment</td>\n",
       "      <td>recovery</td>\n",
       "      <td>7.54</td>\n",
       "      <td>0.54</td>\n",
       "      <td>7.81</td>\n",
       "      <td>7.00</td>\n",
       "      <td>3.410917</td>\n",
       "      <td>0.27</td>\n",
       "      <td>6.73</td>\n",
       "      <td>3.140917</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>34</th>\n",
       "      <td>king</td>\n",
       "      <td>queen</td>\n",
       "      <td>8.46</td>\n",
       "      <td>0.09</td>\n",
       "      <td>6.77</td>\n",
       "      <td>8.37</td>\n",
       "      <td>4.780917</td>\n",
       "      <td>1.69</td>\n",
       "      <td>6.68</td>\n",
       "      <td>3.090917</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>282</th>\n",
       "      <td>seafood</td>\n",
       "      <td>sea</td>\n",
       "      <td>7.94</td>\n",
       "      <td>0.13</td>\n",
       "      <td>6.49</td>\n",
       "      <td>7.81</td>\n",
       "      <td>4.220917</td>\n",
       "      <td>1.45</td>\n",
       "      <td>6.36</td>\n",
       "      <td>2.770917</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>49</th>\n",
       "      <td>movie</td>\n",
       "      <td>star</td>\n",
       "      <td>7.46</td>\n",
       "      <td>0.00</td>\n",
       "      <td>6.14</td>\n",
       "      <td>7.46</td>\n",
       "      <td>3.870917</td>\n",
       "      <td>1.32</td>\n",
       "      <td>6.14</td>\n",
       "      <td>2.550917</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "         Word1     Word2  Average Score  base  ngram  bdiff  bdiff_v2  ngdiff  \\\n",
       "263     dollar      buck           9.54  0.68   8.47   8.86  5.270917    1.07   \n",
       "197  treatment  recovery           7.54  0.54   7.81   7.00  3.410917    0.27   \n",
       "34        king     queen           8.46  0.09   6.77   8.37  4.780917    1.69   \n",
       "282    seafood       sea           7.94  0.13   6.49   7.81  4.220917    1.45   \n",
       "49       movie      star           7.46  0.00   6.14   7.46  3.870917    1.32   \n",
       "\n",
       "     bngdiff  bngdiff_v2  \n",
       "263     7.79    4.200917  \n",
       "197     6.73    3.140917  \n",
       "34      6.68    3.090917  \n",
       "282     6.36    2.770917  \n",
       "49      6.14    2.550917  "
      ]
     },
     "execution_count": 16,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Pairs of words of best outerperforming cases of ngram based model\n",
    "much_better.head(5)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Word1</th>\n",
       "      <th>Word2</th>\n",
       "      <th>Average Score</th>\n",
       "      <th>base</th>\n",
       "      <th>ngram</th>\n",
       "      <th>bdiff</th>\n",
       "      <th>bdiff_v2</th>\n",
       "      <th>ngdiff</th>\n",
       "      <th>bngdiff</th>\n",
       "      <th>bngdiff_v2</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>178</th>\n",
       "      <td>prejudice</td>\n",
       "      <td>recognition</td>\n",
       "      <td>2.73</td>\n",
       "      <td>0.06</td>\n",
       "      <td>8.51</td>\n",
       "      <td>2.67</td>\n",
       "      <td>0.919083</td>\n",
       "      <td>5.78</td>\n",
       "      <td>-3.11</td>\n",
       "      <td>-4.860917</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>260</th>\n",
       "      <td>profit</td>\n",
       "      <td>warning</td>\n",
       "      <td>0.77</td>\n",
       "      <td>0.87</td>\n",
       "      <td>8.95</td>\n",
       "      <td>0.10</td>\n",
       "      <td>3.689083</td>\n",
       "      <td>8.18</td>\n",
       "      <td>-8.08</td>\n",
       "      <td>-4.490917</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>270</th>\n",
       "      <td>luxury</td>\n",
       "      <td>car</td>\n",
       "      <td>5.12</td>\n",
       "      <td>0.47</td>\n",
       "      <td>0.00</td>\n",
       "      <td>4.65</td>\n",
       "      <td>1.060917</td>\n",
       "      <td>5.12</td>\n",
       "      <td>-0.47</td>\n",
       "      <td>-4.059083</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>212</th>\n",
       "      <td>life</td>\n",
       "      <td>term</td>\n",
       "      <td>5.35</td>\n",
       "      <td>1.22</td>\n",
       "      <td>0.83</td>\n",
       "      <td>4.13</td>\n",
       "      <td>0.540917</td>\n",
       "      <td>4.52</td>\n",
       "      <td>-0.39</td>\n",
       "      <td>-3.979083</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>315</th>\n",
       "      <td>population</td>\n",
       "      <td>development</td>\n",
       "      <td>4.42</td>\n",
       "      <td>1.06</td>\n",
       "      <td>8.50</td>\n",
       "      <td>3.36</td>\n",
       "      <td>0.229083</td>\n",
       "      <td>4.08</td>\n",
       "      <td>-0.72</td>\n",
       "      <td>-3.850917</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "          Word1        Word2  Average Score  base  ngram  bdiff  bdiff_v2  \\\n",
       "178   prejudice  recognition           2.73  0.06   8.51   2.67  0.919083   \n",
       "260      profit      warning           0.77  0.87   8.95   0.10  3.689083   \n",
       "270      luxury          car           5.12  0.47   0.00   4.65  1.060917   \n",
       "212        life         term           5.35  1.22   0.83   4.13  0.540917   \n",
       "315  population  development           4.42  1.06   8.50   3.36  0.229083   \n",
       "\n",
       "     ngdiff  bngdiff  bngdiff_v2  \n",
       "178    5.78    -3.11   -4.860917  \n",
       "260    8.18    -8.08   -4.490917  \n",
       "270    5.12    -0.47   -4.059083  \n",
       "212    4.52    -0.39   -3.979083  \n",
       "315    4.08    -0.72   -3.850917  "
      ]
     },
     "execution_count": 17,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Pairs of words of worst underperforming cases of ngram based model after rescaling base scores\n",
    "much_worse_v2.head(5)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Word1</th>\n",
       "      <th>Word2</th>\n",
       "      <th>Average Score</th>\n",
       "      <th>base</th>\n",
       "      <th>ngram</th>\n",
       "      <th>bdiff</th>\n",
       "      <th>bdiff_v2</th>\n",
       "      <th>ngdiff</th>\n",
       "      <th>bngdiff</th>\n",
       "      <th>bngdiff_v2</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>263</th>\n",
       "      <td>dollar</td>\n",
       "      <td>buck</td>\n",
       "      <td>9.54</td>\n",
       "      <td>0.68</td>\n",
       "      <td>8.47</td>\n",
       "      <td>8.86</td>\n",
       "      <td>5.270917</td>\n",
       "      <td>1.07</td>\n",
       "      <td>7.79</td>\n",
       "      <td>4.200917</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>197</th>\n",
       "      <td>treatment</td>\n",
       "      <td>recovery</td>\n",
       "      <td>7.54</td>\n",
       "      <td>0.54</td>\n",
       "      <td>7.81</td>\n",
       "      <td>7.00</td>\n",
       "      <td>3.410917</td>\n",
       "      <td>0.27</td>\n",
       "      <td>6.73</td>\n",
       "      <td>3.140917</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>34</th>\n",
       "      <td>king</td>\n",
       "      <td>queen</td>\n",
       "      <td>8.46</td>\n",
       "      <td>0.09</td>\n",
       "      <td>6.77</td>\n",
       "      <td>8.37</td>\n",
       "      <td>4.780917</td>\n",
       "      <td>1.69</td>\n",
       "      <td>6.68</td>\n",
       "      <td>3.090917</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>282</th>\n",
       "      <td>seafood</td>\n",
       "      <td>sea</td>\n",
       "      <td>7.94</td>\n",
       "      <td>0.13</td>\n",
       "      <td>6.49</td>\n",
       "      <td>7.81</td>\n",
       "      <td>4.220917</td>\n",
       "      <td>1.45</td>\n",
       "      <td>6.36</td>\n",
       "      <td>2.770917</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>49</th>\n",
       "      <td>movie</td>\n",
       "      <td>star</td>\n",
       "      <td>7.46</td>\n",
       "      <td>0.00</td>\n",
       "      <td>6.14</td>\n",
       "      <td>7.46</td>\n",
       "      <td>3.870917</td>\n",
       "      <td>1.32</td>\n",
       "      <td>6.14</td>\n",
       "      <td>2.550917</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "         Word1     Word2  Average Score  base  ngram  bdiff  bdiff_v2  ngdiff  \\\n",
       "263     dollar      buck           9.54  0.68   8.47   8.86  5.270917    1.07   \n",
       "197  treatment  recovery           7.54  0.54   7.81   7.00  3.410917    0.27   \n",
       "34        king     queen           8.46  0.09   6.77   8.37  4.780917    1.69   \n",
       "282    seafood       sea           7.94  0.13   6.49   7.81  4.220917    1.45   \n",
       "49       movie      star           7.46  0.00   6.14   7.46  3.870917    1.32   \n",
       "\n",
       "     bngdiff  bngdiff_v2  \n",
       "263     7.79    4.200917  \n",
       "197     6.73    3.140917  \n",
       "34      6.68    3.090917  \n",
       "282     6.36    2.770917  \n",
       "49      6.14    2.550917  "
      ]
     },
     "execution_count": 18,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Pairs of words of best outerperforming cases of ngram based model after rescaling base scores\n",
    "much_better_v2.head(5)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## German"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [],
   "source": [
    "lan = 'de'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Word1</th>\n",
       "      <th>Word2</th>\n",
       "      <th>1</th>\n",
       "      <th>2</th>\n",
       "      <th>3</th>\n",
       "      <th>4</th>\n",
       "      <th>5</th>\n",
       "      <th>6</th>\n",
       "      <th>7</th>\n",
       "      <th>8</th>\n",
       "      <th>9</th>\n",
       "      <th>10</th>\n",
       "      <th>11</th>\n",
       "      <th>12</th>\n",
       "      <th>13</th>\n",
       "      <th>Average Score</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>Liebe</td>\n",
       "      <td>Sex</td>\n",
       "      <td>8</td>\n",
       "      <td>9</td>\n",
       "      <td>8</td>\n",
       "      <td>9</td>\n",
       "      <td>9</td>\n",
       "      <td>8.0</td>\n",
       "      <td>9.0</td>\n",
       "      <td>9.0</td>\n",
       "      <td>7.0</td>\n",
       "      <td>10.0</td>\n",
       "      <td>7.0</td>\n",
       "      <td>10</td>\n",
       "      <td>7.0</td>\n",
       "      <td>8.46</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>Tiger</td>\n",
       "      <td>Katze</td>\n",
       "      <td>8</td>\n",
       "      <td>7</td>\n",
       "      <td>7</td>\n",
       "      <td>8</td>\n",
       "      <td>7</td>\n",
       "      <td>8.0</td>\n",
       "      <td>9.0</td>\n",
       "      <td>10.0</td>\n",
       "      <td>9.0</td>\n",
       "      <td>8.0</td>\n",
       "      <td>7.0</td>\n",
       "      <td>8</td>\n",
       "      <td>7.0</td>\n",
       "      <td>7.92</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>Tiger</td>\n",
       "      <td>Tiger</td>\n",
       "      <td>10</td>\n",
       "      <td>10</td>\n",
       "      <td>10</td>\n",
       "      <td>10</td>\n",
       "      <td>10</td>\n",
       "      <td>10.0</td>\n",
       "      <td>10.0</td>\n",
       "      <td>10.0</td>\n",
       "      <td>10.0</td>\n",
       "      <td>10.0</td>\n",
       "      <td>10.0</td>\n",
       "      <td>10</td>\n",
       "      <td>10.0</td>\n",
       "      <td>10.00</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>Buch</td>\n",
       "      <td>Papier</td>\n",
       "      <td>9</td>\n",
       "      <td>8</td>\n",
       "      <td>8</td>\n",
       "      <td>4</td>\n",
       "      <td>8</td>\n",
       "      <td>8.0</td>\n",
       "      <td>9.0</td>\n",
       "      <td>10.0</td>\n",
       "      <td>9.0</td>\n",
       "      <td>8.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>5</td>\n",
       "      <td>6.0</td>\n",
       "      <td>7.08</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>Computer</td>\n",
       "      <td>Tastatur</td>\n",
       "      <td>8</td>\n",
       "      <td>7</td>\n",
       "      <td>8</td>\n",
       "      <td>7</td>\n",
       "      <td>9</td>\n",
       "      <td>8.0</td>\n",
       "      <td>9.0</td>\n",
       "      <td>10.0</td>\n",
       "      <td>8.0</td>\n",
       "      <td>10.0</td>\n",
       "      <td>6.0</td>\n",
       "      <td>6</td>\n",
       "      <td>8.0</td>\n",
       "      <td>8.00</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "      Word1     Word2   1   2   3   4   5     6     7     8     9    10    11  \\\n",
       "0     Liebe       Sex   8   9   8   9   9   8.0   9.0   9.0   7.0  10.0   7.0   \n",
       "1     Tiger     Katze   8   7   7   8   7   8.0   9.0  10.0   9.0   8.0   7.0   \n",
       "2     Tiger     Tiger  10  10  10  10  10  10.0  10.0  10.0  10.0  10.0  10.0   \n",
       "3      Buch    Papier   9   8   8   4   8   8.0   9.0  10.0   9.0   8.0   0.0   \n",
       "4  Computer  Tastatur   8   7   8   7   9   8.0   9.0  10.0   8.0  10.0   6.0   \n",
       "\n",
       "   12    13  Average Score  \n",
       "0  10   7.0           8.46  \n",
       "1   8   7.0           7.92  \n",
       "2  10  10.0          10.00  \n",
       "3   5   6.0           7.08  \n",
       "4   6   8.0           8.00  "
      ]
     },
     "execution_count": 20,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# --- Import word similarity dataset WS353 for the selected language ---\n",
    "sim_ds = pd.read_csv(os.path.join(wsd_dir, f'MWS353_{lan_to_file[lan]}.txt'), sep=\",\")\n",
    "sim_ds.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [],
   "source": [
    "# --- Import data of our embedding model ---\n",
    "word2idx = pickle.load(open(os.path.join(data_dir, lan, 'word2idx.dat'), 'rb'))\n",
    "# Embeddings without subwords' information\n",
    "idx2vec = pickle.load(open(os.path.join(data_dir, lan, 'idx2vec.dat'), 'rb'))\n",
    "# Embeddings with subwords' information\n",
    "idx2vec_ngrams = pickle.load(open(os.path.join(data_dir, lan, 'idx2vec_ngrams.dat'), 'rb'))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "There are 359 words in common out of 455 in the word similarity dataset.\n",
      "There are 237 common pairs of words, out of 350 in the word similarity dataset.\n"
     ]
    }
   ],
   "source": [
    "# --- Identify common pairs of words ---\n",
    "common_pairs_idx, common_is = CommonPairsOfWords(sim_ds, word2idx)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {},
   "outputs": [],
   "source": [
    "# --- Compute cosine similarity of embeddings without subwords' information for common pairs of words ---\n",
    "train_score = ComputeEmbeddingSimilarity(common_pairs_idx, idx2vec)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {},
   "outputs": [],
   "source": [
    "# --- Compute cosine similarity of embeddings with subwords' information for common pairs of words ---\n",
    "train_score_ngrams = ComputeEmbeddingSimilarity(common_pairs_idx, idx2vec_ngrams)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Means of similarity scores:\n",
      "Average Score    5.086540\n",
      "base             1.669662\n",
      "ngram            3.834135\n",
      "dtype: float64\n",
      "\n",
      "Similarity scores of ngram based approach are closer to the human similarity scores in 157 case(s) out out of 237 cases. (66.2%)\n",
      "Similarity scores of both approaches work equivalently in 3 case(s).(1.3%)\n",
      "Similarity scores of ngram based approach underperfor in 77 case(s). (32.0%)\n",
      "\n",
      "After adjusting the scores of the base model by adding the difference between the mean of the ngram and the base model scores\n",
      "Similarity scores of ngram based approach are closer to the human similarity scores in 94 case(s) out out of 237 cases. (39.7%)\n",
      "Similarity scores of both approaches work equivalently in 0 case(s).(0.0%)\n",
      "Similarity scores of ngram based approach underperfor in 143 case(s). (60.0%)\n"
     ]
    }
   ],
   "source": [
    "(cs, ns, ns_v2 ) = ComputeSimilarityStats(sim_ds.iloc[common_is], train_score, train_score_ngrams)\n",
    "common_sim[lan] = cs\n",
    "ngram_stats[lan] = ns\n",
    "ngram_stats_v2[lan] = ns_v2"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {},
   "outputs": [],
   "source": [
    "# --- Compute Spearman's rank correlation coefficients\n",
    "sim_score = common_sim[lan]['Average Score'].values.reshape(-1,1)\n",
    "ss = {'base': spearmanr(sim_score, train_score),\n",
    "      'ngram': spearmanr(sim_score, train_score_ngrams)}\n",
    "spearman_scores[lan] = ss"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {},
   "outputs": [],
   "source": [
    "(much_worse, much_better, much_worse_v2, much_better_v2) = RelevantPairs(common_sim[lan], worse_threshold = 0, better_threshold = 0)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Word1</th>\n",
       "      <th>Word2</th>\n",
       "      <th>Average Score</th>\n",
       "      <th>base</th>\n",
       "      <th>ngram</th>\n",
       "      <th>bdiff</th>\n",
       "      <th>bdiff_v2</th>\n",
       "      <th>ngdiff</th>\n",
       "      <th>bngdiff</th>\n",
       "      <th>bngdiff_v2</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>91</th>\n",
       "      <td>Mittag</td>\n",
       "      <td>Faden</td>\n",
       "      <td>0.15</td>\n",
       "      <td>3.18</td>\n",
       "      <td>9.76</td>\n",
       "      <td>3.03</td>\n",
       "      <td>5.194473</td>\n",
       "      <td>9.61</td>\n",
       "      <td>-6.58</td>\n",
       "      <td>-4.415527</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>45</th>\n",
       "      <td>Arafat</td>\n",
       "      <td>Frieden</td>\n",
       "      <td>2.46</td>\n",
       "      <td>1.95</td>\n",
       "      <td>9.14</td>\n",
       "      <td>0.51</td>\n",
       "      <td>1.654473</td>\n",
       "      <td>6.68</td>\n",
       "      <td>-6.17</td>\n",
       "      <td>-5.025527</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>89</th>\n",
       "      <td>Akkord</td>\n",
       "      <td>Lächeln</td>\n",
       "      <td>0.31</td>\n",
       "      <td>2.29</td>\n",
       "      <td>8.36</td>\n",
       "      <td>1.98</td>\n",
       "      <td>4.144473</td>\n",
       "      <td>8.05</td>\n",
       "      <td>-6.07</td>\n",
       "      <td>-3.905527</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>317</th>\n",
       "      <td>Moral</td>\n",
       "      <td>Heirat</td>\n",
       "      <td>2.81</td>\n",
       "      <td>2.41</td>\n",
       "      <td>9.21</td>\n",
       "      <td>0.40</td>\n",
       "      <td>1.764473</td>\n",
       "      <td>6.40</td>\n",
       "      <td>-6.00</td>\n",
       "      <td>-4.635527</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>39</th>\n",
       "      <td>Heilig</td>\n",
       "      <td>Sex</td>\n",
       "      <td>0.69</td>\n",
       "      <td>1.71</td>\n",
       "      <td>7.41</td>\n",
       "      <td>1.02</td>\n",
       "      <td>3.184473</td>\n",
       "      <td>6.72</td>\n",
       "      <td>-5.70</td>\n",
       "      <td>-3.535527</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "      Word1    Word2  Average Score  base  ngram  bdiff  bdiff_v2  ngdiff  \\\n",
       "91   Mittag    Faden           0.15  3.18   9.76   3.03  5.194473    9.61   \n",
       "45   Arafat  Frieden           2.46  1.95   9.14   0.51  1.654473    6.68   \n",
       "89   Akkord  Lächeln           0.31  2.29   8.36   1.98  4.144473    8.05   \n",
       "317   Moral   Heirat           2.81  2.41   9.21   0.40  1.764473    6.40   \n",
       "39   Heilig      Sex           0.69  1.71   7.41   1.02  3.184473    6.72   \n",
       "\n",
       "     bngdiff  bngdiff_v2  \n",
       "91     -6.58   -4.415527  \n",
       "45     -6.17   -5.025527  \n",
       "89     -6.07   -3.905527  \n",
       "317    -6.00   -4.635527  \n",
       "39     -5.70   -3.535527  "
      ]
     },
     "execution_count": 29,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Pairs of words of worst underperforming cases of ngram based model\n",
    "much_worse.head(5)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Word1</th>\n",
       "      <th>Word2</th>\n",
       "      <th>Average Score</th>\n",
       "      <th>base</th>\n",
       "      <th>ngram</th>\n",
       "      <th>bdiff</th>\n",
       "      <th>bdiff_v2</th>\n",
       "      <th>ngdiff</th>\n",
       "      <th>bngdiff</th>\n",
       "      <th>bngdiff_v2</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>254</th>\n",
       "      <td>Kalkulation</td>\n",
       "      <td>Berechnung</td>\n",
       "      <td>9.54</td>\n",
       "      <td>1.73</td>\n",
       "      <td>9.36</td>\n",
       "      <td>7.81</td>\n",
       "      <td>5.645527</td>\n",
       "      <td>0.18</td>\n",
       "      <td>7.63</td>\n",
       "      <td>5.465527</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>48</th>\n",
       "      <td>Gesetz</td>\n",
       "      <td>Anwalt</td>\n",
       "      <td>8.38</td>\n",
       "      <td>0.71</td>\n",
       "      <td>8.01</td>\n",
       "      <td>7.67</td>\n",
       "      <td>5.505527</td>\n",
       "      <td>0.37</td>\n",
       "      <td>7.30</td>\n",
       "      <td>5.135527</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>Tiger</td>\n",
       "      <td>Katze</td>\n",
       "      <td>7.92</td>\n",
       "      <td>0.09</td>\n",
       "      <td>8.71</td>\n",
       "      <td>7.83</td>\n",
       "      <td>5.665527</td>\n",
       "      <td>0.79</td>\n",
       "      <td>7.04</td>\n",
       "      <td>4.875527</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>Computer</td>\n",
       "      <td>Tastatur</td>\n",
       "      <td>8.00</td>\n",
       "      <td>1.23</td>\n",
       "      <td>8.47</td>\n",
       "      <td>6.77</td>\n",
       "      <td>4.605527</td>\n",
       "      <td>0.47</td>\n",
       "      <td>6.30</td>\n",
       "      <td>4.135527</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>105</th>\n",
       "      <td>Tiger</td>\n",
       "      <td>Katze</td>\n",
       "      <td>6.92</td>\n",
       "      <td>0.09</td>\n",
       "      <td>8.71</td>\n",
       "      <td>6.83</td>\n",
       "      <td>4.665527</td>\n",
       "      <td>1.79</td>\n",
       "      <td>5.04</td>\n",
       "      <td>2.875527</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "           Word1       Word2  Average Score  base  ngram  bdiff  bdiff_v2  \\\n",
       "254  Kalkulation  Berechnung           9.54  1.73   9.36   7.81  5.645527   \n",
       "48        Gesetz      Anwalt           8.38  0.71   8.01   7.67  5.505527   \n",
       "1          Tiger       Katze           7.92  0.09   8.71   7.83  5.665527   \n",
       "4       Computer    Tastatur           8.00  1.23   8.47   6.77  4.605527   \n",
       "105        Tiger       Katze           6.92  0.09   8.71   6.83  4.665527   \n",
       "\n",
       "     ngdiff  bngdiff  bngdiff_v2  \n",
       "254    0.18     7.63    5.465527  \n",
       "48     0.37     7.30    5.135527  \n",
       "1      0.79     7.04    4.875527  \n",
       "4      0.47     6.30    4.135527  \n",
       "105    1.79     5.04    2.875527  "
      ]
     },
     "execution_count": 30,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Pairs of words of best outerperforming cases of ngram based model\n",
    "much_better.head(5)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Word1</th>\n",
       "      <th>Word2</th>\n",
       "      <th>Average Score</th>\n",
       "      <th>base</th>\n",
       "      <th>ngram</th>\n",
       "      <th>bdiff</th>\n",
       "      <th>bdiff_v2</th>\n",
       "      <th>ngdiff</th>\n",
       "      <th>bngdiff</th>\n",
       "      <th>bngdiff_v2</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>206</th>\n",
       "      <td>Harvard</td>\n",
       "      <td>Yale</td>\n",
       "      <td>7.85</td>\n",
       "      <td>5.63</td>\n",
       "      <td>2.47</td>\n",
       "      <td>2.22</td>\n",
       "      <td>0.055527</td>\n",
       "      <td>5.38</td>\n",
       "      <td>-3.16</td>\n",
       "      <td>-5.324473</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>45</th>\n",
       "      <td>Arafat</td>\n",
       "      <td>Frieden</td>\n",
       "      <td>2.46</td>\n",
       "      <td>1.95</td>\n",
       "      <td>9.14</td>\n",
       "      <td>0.51</td>\n",
       "      <td>1.654473</td>\n",
       "      <td>6.68</td>\n",
       "      <td>-6.17</td>\n",
       "      <td>-5.025527</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>317</th>\n",
       "      <td>Moral</td>\n",
       "      <td>Heirat</td>\n",
       "      <td>2.81</td>\n",
       "      <td>2.41</td>\n",
       "      <td>9.21</td>\n",
       "      <td>0.40</td>\n",
       "      <td>1.764473</td>\n",
       "      <td>6.40</td>\n",
       "      <td>-6.00</td>\n",
       "      <td>-4.635527</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>293</th>\n",
       "      <td>Meisterschaft</td>\n",
       "      <td>Turnier</td>\n",
       "      <td>8.05</td>\n",
       "      <td>3.64</td>\n",
       "      <td>1.30</td>\n",
       "      <td>4.41</td>\n",
       "      <td>2.245527</td>\n",
       "      <td>6.75</td>\n",
       "      <td>-2.34</td>\n",
       "      <td>-4.504473</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>91</th>\n",
       "      <td>Mittag</td>\n",
       "      <td>Faden</td>\n",
       "      <td>0.15</td>\n",
       "      <td>3.18</td>\n",
       "      <td>9.76</td>\n",
       "      <td>3.03</td>\n",
       "      <td>5.194473</td>\n",
       "      <td>9.61</td>\n",
       "      <td>-6.58</td>\n",
       "      <td>-4.415527</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "             Word1    Word2  Average Score  base  ngram  bdiff  bdiff_v2  \\\n",
       "206        Harvard     Yale           7.85  5.63   2.47   2.22  0.055527   \n",
       "45          Arafat  Frieden           2.46  1.95   9.14   0.51  1.654473   \n",
       "317          Moral   Heirat           2.81  2.41   9.21   0.40  1.764473   \n",
       "293  Meisterschaft  Turnier           8.05  3.64   1.30   4.41  2.245527   \n",
       "91          Mittag    Faden           0.15  3.18   9.76   3.03  5.194473   \n",
       "\n",
       "     ngdiff  bngdiff  bngdiff_v2  \n",
       "206    5.38    -3.16   -5.324473  \n",
       "45     6.68    -6.17   -5.025527  \n",
       "317    6.40    -6.00   -4.635527  \n",
       "293    6.75    -2.34   -4.504473  \n",
       "91     9.61    -6.58   -4.415527  "
      ]
     },
     "execution_count": 31,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Pairs of words of worst underperforming cases of ngram based model after rescaling base scores\n",
    "much_worse_v2.head(5)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Word1</th>\n",
       "      <th>Word2</th>\n",
       "      <th>Average Score</th>\n",
       "      <th>base</th>\n",
       "      <th>ngram</th>\n",
       "      <th>bdiff</th>\n",
       "      <th>bdiff_v2</th>\n",
       "      <th>ngdiff</th>\n",
       "      <th>bngdiff</th>\n",
       "      <th>bngdiff_v2</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>254</th>\n",
       "      <td>Kalkulation</td>\n",
       "      <td>Berechnung</td>\n",
       "      <td>9.54</td>\n",
       "      <td>1.73</td>\n",
       "      <td>9.36</td>\n",
       "      <td>7.81</td>\n",
       "      <td>5.645527</td>\n",
       "      <td>0.18</td>\n",
       "      <td>7.63</td>\n",
       "      <td>5.465527</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>48</th>\n",
       "      <td>Gesetz</td>\n",
       "      <td>Anwalt</td>\n",
       "      <td>8.38</td>\n",
       "      <td>0.71</td>\n",
       "      <td>8.01</td>\n",
       "      <td>7.67</td>\n",
       "      <td>5.505527</td>\n",
       "      <td>0.37</td>\n",
       "      <td>7.30</td>\n",
       "      <td>5.135527</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>Tiger</td>\n",
       "      <td>Katze</td>\n",
       "      <td>7.92</td>\n",
       "      <td>0.09</td>\n",
       "      <td>8.71</td>\n",
       "      <td>7.83</td>\n",
       "      <td>5.665527</td>\n",
       "      <td>0.79</td>\n",
       "      <td>7.04</td>\n",
       "      <td>4.875527</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>Computer</td>\n",
       "      <td>Tastatur</td>\n",
       "      <td>8.00</td>\n",
       "      <td>1.23</td>\n",
       "      <td>8.47</td>\n",
       "      <td>6.77</td>\n",
       "      <td>4.605527</td>\n",
       "      <td>0.47</td>\n",
       "      <td>6.30</td>\n",
       "      <td>4.135527</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>105</th>\n",
       "      <td>Tiger</td>\n",
       "      <td>Katze</td>\n",
       "      <td>6.92</td>\n",
       "      <td>0.09</td>\n",
       "      <td>8.71</td>\n",
       "      <td>6.83</td>\n",
       "      <td>4.665527</td>\n",
       "      <td>1.79</td>\n",
       "      <td>5.04</td>\n",
       "      <td>2.875527</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "           Word1       Word2  Average Score  base  ngram  bdiff  bdiff_v2  \\\n",
       "254  Kalkulation  Berechnung           9.54  1.73   9.36   7.81  5.645527   \n",
       "48        Gesetz      Anwalt           8.38  0.71   8.01   7.67  5.505527   \n",
       "1          Tiger       Katze           7.92  0.09   8.71   7.83  5.665527   \n",
       "4       Computer    Tastatur           8.00  1.23   8.47   6.77  4.605527   \n",
       "105        Tiger       Katze           6.92  0.09   8.71   6.83  4.665527   \n",
       "\n",
       "     ngdiff  bngdiff  bngdiff_v2  \n",
       "254    0.18     7.63    5.465527  \n",
       "48     0.37     7.30    5.135527  \n",
       "1      0.79     7.04    4.875527  \n",
       "4      0.47     6.30    4.135527  \n",
       "105    1.79     5.04    2.875527  "
      ]
     },
     "execution_count": 32,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Pairs of words of best outerperforming cases of ngram based model after rescaling base scores\n",
    "much_better_v2.head(5)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Italian"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "metadata": {},
   "outputs": [],
   "source": [
    "lan = 'it'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 34,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Word1</th>\n",
       "      <th>Word2</th>\n",
       "      <th>1</th>\n",
       "      <th>2</th>\n",
       "      <th>3</th>\n",
       "      <th>4</th>\n",
       "      <th>5</th>\n",
       "      <th>6</th>\n",
       "      <th>7</th>\n",
       "      <th>8</th>\n",
       "      <th>9</th>\n",
       "      <th>10</th>\n",
       "      <th>11</th>\n",
       "      <th>12</th>\n",
       "      <th>13</th>\n",
       "      <th>Average Score</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>amore</td>\n",
       "      <td>sesso</td>\n",
       "      <td>8.0</td>\n",
       "      <td>8.0</td>\n",
       "      <td>7.0</td>\n",
       "      <td>8.0</td>\n",
       "      <td>6.0</td>\n",
       "      <td>8</td>\n",
       "      <td>7.0</td>\n",
       "      <td>8.0</td>\n",
       "      <td>9.0</td>\n",
       "      <td>7</td>\n",
       "      <td>10.0</td>\n",
       "      <td>9.0</td>\n",
       "      <td>10.0</td>\n",
       "      <td>8.08</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>tigre</td>\n",
       "      <td>gatto</td>\n",
       "      <td>8.0</td>\n",
       "      <td>10.0</td>\n",
       "      <td>6.0</td>\n",
       "      <td>7.0</td>\n",
       "      <td>9.0</td>\n",
       "      <td>7</td>\n",
       "      <td>8.0</td>\n",
       "      <td>7.0</td>\n",
       "      <td>10.0</td>\n",
       "      <td>7</td>\n",
       "      <td>9.0</td>\n",
       "      <td>8.0</td>\n",
       "      <td>10.0</td>\n",
       "      <td>8.15</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>tigre</td>\n",
       "      <td>tigre</td>\n",
       "      <td>10.0</td>\n",
       "      <td>10.0</td>\n",
       "      <td>10.0</td>\n",
       "      <td>10.0</td>\n",
       "      <td>10.0</td>\n",
       "      <td>10</td>\n",
       "      <td>10.0</td>\n",
       "      <td>10.0</td>\n",
       "      <td>10.0</td>\n",
       "      <td>10</td>\n",
       "      <td>10.0</td>\n",
       "      <td>10.0</td>\n",
       "      <td>10.0</td>\n",
       "      <td>10.00</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>libro</td>\n",
       "      <td>carta</td>\n",
       "      <td>6.0</td>\n",
       "      <td>10.0</td>\n",
       "      <td>8.0</td>\n",
       "      <td>6.0</td>\n",
       "      <td>8.0</td>\n",
       "      <td>8</td>\n",
       "      <td>7.0</td>\n",
       "      <td>8.0</td>\n",
       "      <td>10.0</td>\n",
       "      <td>9</td>\n",
       "      <td>10.0</td>\n",
       "      <td>9.0</td>\n",
       "      <td>10.0</td>\n",
       "      <td>8.38</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>computer</td>\n",
       "      <td>tastiera</td>\n",
       "      <td>6.0</td>\n",
       "      <td>10.0</td>\n",
       "      <td>10.0</td>\n",
       "      <td>7.0</td>\n",
       "      <td>8.0</td>\n",
       "      <td>6</td>\n",
       "      <td>6.0</td>\n",
       "      <td>8.0</td>\n",
       "      <td>10.0</td>\n",
       "      <td>9</td>\n",
       "      <td>10.0</td>\n",
       "      <td>9.0</td>\n",
       "      <td>10.0</td>\n",
       "      <td>8.38</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "      Word1     Word2     1     2     3     4     5   6     7     8     9  10  \\\n",
       "0     amore     sesso   8.0   8.0   7.0   8.0   6.0   8   7.0   8.0   9.0   7   \n",
       "1     tigre     gatto   8.0  10.0   6.0   7.0   9.0   7   8.0   7.0  10.0   7   \n",
       "2     tigre     tigre  10.0  10.0  10.0  10.0  10.0  10  10.0  10.0  10.0  10   \n",
       "3     libro     carta   6.0  10.0   8.0   6.0   8.0   8   7.0   8.0  10.0   9   \n",
       "4  computer  tastiera   6.0  10.0  10.0   7.0   8.0   6   6.0   8.0  10.0   9   \n",
       "\n",
       "     11    12    13  Average Score  \n",
       "0  10.0   9.0  10.0           8.08  \n",
       "1   9.0   8.0  10.0           8.15  \n",
       "2  10.0  10.0  10.0          10.00  \n",
       "3  10.0   9.0  10.0           8.38  \n",
       "4  10.0   9.0  10.0           8.38  "
      ]
     },
     "execution_count": 34,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# --- Import word similarity dataset WS353 for the selected language ---\n",
    "sim_ds = pd.read_csv(os.path.join(wsd_dir, f'MWS353_{lan_to_file[lan]}.txt'), sep=\",\")\n",
    "sim_ds.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 35,
   "metadata": {},
   "outputs": [],
   "source": [
    "# --- Import data of our embedding model ---\n",
    "word2idx = pickle.load(open(os.path.join(data_dir, lan, 'word2idx.dat'), 'rb'))\n",
    "# Embeddings without subwords' information\n",
    "idx2vec = pickle.load(open(os.path.join(data_dir, lan, 'idx2vec.dat'), 'rb'))\n",
    "# Embeddings with subwords' information\n",
    "idx2vec_ngrams = pickle.load(open(os.path.join(data_dir, lan, 'idx2vec_ngrams.dat'), 'rb'))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 36,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "There are 353 words in common out of 432 in the word similarity dataset.\n",
      "There are 251 common pairs of words, out of 350 in the word similarity dataset.\n"
     ]
    }
   ],
   "source": [
    "# --- Identify common pairs of words ---\n",
    "common_pairs_idx, common_is = CommonPairsOfWords(sim_ds, word2idx)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 37,
   "metadata": {},
   "outputs": [],
   "source": [
    "# --- Compute cosine similarity of embeddings without subwords' information for common pairs of words ---\n",
    "train_score = ComputeEmbeddingSimilarity(common_pairs_idx, idx2vec)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 38,
   "metadata": {},
   "outputs": [],
   "source": [
    "# --- Compute cosine similarity of embeddings with subwords' information for common pairs of words ---\n",
    "train_score_ngrams = ComputeEmbeddingSimilarity(common_pairs_idx, idx2vec_ngrams)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 40,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Means of similarity scores:\n",
      "Average Score    6.224303\n",
      "base             1.733347\n",
      "ngram            4.434382\n",
      "dtype: float64\n",
      "\n",
      "Similarity scores of ngram based approach are closer to the human similarity scores in 222 case(s) out out of 251 cases. (88.4%)\n",
      "Similarity scores of both approaches work equivalently in 2 case(s).(0.8%)\n",
      "Similarity scores of ngram based approach underperfor in 27 case(s). (11.0%)\n",
      "\n",
      "After adjusting the scores of the base model by adding the difference between the mean of the ngram and the base model scores\n",
      "Similarity scores of ngram based approach are closer to the human similarity scores in 116 case(s) out out of 251 cases. (46.2%)\n",
      "Similarity scores of both approaches work equivalently in 0 case(s).(0.0%)\n",
      "Similarity scores of ngram based approach underperfor in 135 case(s). (54.0%)\n"
     ]
    }
   ],
   "source": [
    "(cs, ns, ns_v2 ) = ComputeSimilarityStats(sim_ds.iloc[common_is], train_score, train_score_ngrams)\n",
    "common_sim[lan] = cs\n",
    "ngram_stats[lan] = ns\n",
    "ngram_stats_v2[lan] = ns_v2"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 41,
   "metadata": {},
   "outputs": [],
   "source": [
    "# --- Compute Spearman's rank correlation coefficients\n",
    "sim_score = common_sim[lan]['Average Score'].values.reshape(-1,1)\n",
    "ss = {'base': spearmanr(sim_score, train_score),\n",
    "      'ngram': spearmanr(sim_score, train_score_ngrams)}\n",
    "spearman_scores[lan] = ss"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 42,
   "metadata": {},
   "outputs": [],
   "source": [
    "(much_worse, much_better, much_worse_v2, much_better_v2) = RelevantPairs(common_sim[lan], worse_threshold = 0, better_threshold = 0)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 43,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Word1</th>\n",
       "      <th>Word2</th>\n",
       "      <th>Average Score</th>\n",
       "      <th>base</th>\n",
       "      <th>ngram</th>\n",
       "      <th>bdiff</th>\n",
       "      <th>bdiff_v2</th>\n",
       "      <th>ngdiff</th>\n",
       "      <th>bngdiff</th>\n",
       "      <th>bngdiff_v2</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>91</th>\n",
       "      <td>mezzogiorno</td>\n",
       "      <td>corda</td>\n",
       "      <td>0.42</td>\n",
       "      <td>0.32</td>\n",
       "      <td>8.45</td>\n",
       "      <td>0.10</td>\n",
       "      <td>2.601036</td>\n",
       "      <td>8.03</td>\n",
       "      <td>-7.93</td>\n",
       "      <td>-5.428964</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>81</th>\n",
       "      <td>cimitero</td>\n",
       "      <td>foresta</td>\n",
       "      <td>1.54</td>\n",
       "      <td>1.65</td>\n",
       "      <td>8.29</td>\n",
       "      <td>0.11</td>\n",
       "      <td>2.811036</td>\n",
       "      <td>6.75</td>\n",
       "      <td>-6.64</td>\n",
       "      <td>-3.938964</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>323</th>\n",
       "      <td>zucchero</td>\n",
       "      <td>approccio</td>\n",
       "      <td>1.46</td>\n",
       "      <td>1.28</td>\n",
       "      <td>5.90</td>\n",
       "      <td>0.18</td>\n",
       "      <td>2.521036</td>\n",
       "      <td>4.44</td>\n",
       "      <td>-4.26</td>\n",
       "      <td>-1.918964</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>177</th>\n",
       "      <td>volontario</td>\n",
       "      <td>motto</td>\n",
       "      <td>3.38</td>\n",
       "      <td>3.61</td>\n",
       "      <td>7.43</td>\n",
       "      <td>0.23</td>\n",
       "      <td>2.931036</td>\n",
       "      <td>4.05</td>\n",
       "      <td>-3.82</td>\n",
       "      <td>-1.118964</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>248</th>\n",
       "      <td>produzione</td>\n",
       "      <td>escursione</td>\n",
       "      <td>1.31</td>\n",
       "      <td>0.06</td>\n",
       "      <td>5.31</td>\n",
       "      <td>1.25</td>\n",
       "      <td>1.451036</td>\n",
       "      <td>4.00</td>\n",
       "      <td>-2.75</td>\n",
       "      <td>-2.548964</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "           Word1       Word2  Average Score  base  ngram  bdiff  bdiff_v2  \\\n",
       "91   mezzogiorno       corda           0.42  0.32   8.45   0.10  2.601036   \n",
       "81      cimitero     foresta           1.54  1.65   8.29   0.11  2.811036   \n",
       "323     zucchero   approccio           1.46  1.28   5.90   0.18  2.521036   \n",
       "177   volontario       motto           3.38  3.61   7.43   0.23  2.931036   \n",
       "248   produzione  escursione           1.31  0.06   5.31   1.25  1.451036   \n",
       "\n",
       "     ngdiff  bngdiff  bngdiff_v2  \n",
       "91     8.03    -7.93   -5.428964  \n",
       "81     6.75    -6.64   -3.938964  \n",
       "323    4.44    -4.26   -1.918964  \n",
       "177    4.05    -3.82   -1.118964  \n",
       "248    4.00    -2.75   -2.548964  "
      ]
     },
     "execution_count": 43,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Pairs of words of worst underperforming cases of ngram based model\n",
    "much_worse.head(5)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 44,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Word1</th>\n",
       "      <th>Word2</th>\n",
       "      <th>Average Score</th>\n",
       "      <th>base</th>\n",
       "      <th>ngram</th>\n",
       "      <th>bdiff</th>\n",
       "      <th>bdiff_v2</th>\n",
       "      <th>ngdiff</th>\n",
       "      <th>bngdiff</th>\n",
       "      <th>bngdiff_v2</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>228</th>\n",
       "      <td>hotel</td>\n",
       "      <td>prenotazione</td>\n",
       "      <td>8.85</td>\n",
       "      <td>1.72</td>\n",
       "      <td>8.72</td>\n",
       "      <td>7.13</td>\n",
       "      <td>4.428964</td>\n",
       "      <td>0.13</td>\n",
       "      <td>7.00</td>\n",
       "      <td>4.298964</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>179</th>\n",
       "      <td>decorazione</td>\n",
       "      <td>valore</td>\n",
       "      <td>6.35</td>\n",
       "      <td>0.00</td>\n",
       "      <td>5.90</td>\n",
       "      <td>6.35</td>\n",
       "      <td>3.648964</td>\n",
       "      <td>0.45</td>\n",
       "      <td>5.90</td>\n",
       "      <td>3.198964</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>72</th>\n",
       "      <td>cibo</td>\n",
       "      <td>frutta</td>\n",
       "      <td>7.77</td>\n",
       "      <td>0.59</td>\n",
       "      <td>9.12</td>\n",
       "      <td>7.18</td>\n",
       "      <td>4.478964</td>\n",
       "      <td>1.35</td>\n",
       "      <td>5.83</td>\n",
       "      <td>3.128964</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>264</th>\n",
       "      <td>dollaro</td>\n",
       "      <td>profitto</td>\n",
       "      <td>6.31</td>\n",
       "      <td>0.65</td>\n",
       "      <td>6.47</td>\n",
       "      <td>5.66</td>\n",
       "      <td>2.958964</td>\n",
       "      <td>0.16</td>\n",
       "      <td>5.50</td>\n",
       "      <td>2.798964</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>246</th>\n",
       "      <td>lista</td>\n",
       "      <td>categoria</td>\n",
       "      <td>6.62</td>\n",
       "      <td>1.17</td>\n",
       "      <td>6.75</td>\n",
       "      <td>5.45</td>\n",
       "      <td>2.748964</td>\n",
       "      <td>0.13</td>\n",
       "      <td>5.32</td>\n",
       "      <td>2.618964</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "           Word1         Word2  Average Score  base  ngram  bdiff  bdiff_v2  \\\n",
       "228        hotel  prenotazione           8.85  1.72   8.72   7.13  4.428964   \n",
       "179  decorazione        valore           6.35  0.00   5.90   6.35  3.648964   \n",
       "72          cibo        frutta           7.77  0.59   9.12   7.18  4.478964   \n",
       "264      dollaro      profitto           6.31  0.65   6.47   5.66  2.958964   \n",
       "246        lista     categoria           6.62  1.17   6.75   5.45  2.748964   \n",
       "\n",
       "     ngdiff  bngdiff  bngdiff_v2  \n",
       "228    0.13     7.00    4.298964  \n",
       "179    0.45     5.90    3.198964  \n",
       "72     1.35     5.83    3.128964  \n",
       "264    0.16     5.50    2.798964  \n",
       "246    0.13     5.32    2.618964  "
      ]
     },
     "execution_count": 44,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Pairs of words of best outerperforming cases of ngram based model\n",
    "much_better.head(5)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 45,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Word1</th>\n",
       "      <th>Word2</th>\n",
       "      <th>Average Score</th>\n",
       "      <th>base</th>\n",
       "      <th>ngram</th>\n",
       "      <th>bdiff</th>\n",
       "      <th>bdiff_v2</th>\n",
       "      <th>ngdiff</th>\n",
       "      <th>bngdiff</th>\n",
       "      <th>bngdiff_v2</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>91</th>\n",
       "      <td>mezzogiorno</td>\n",
       "      <td>corda</td>\n",
       "      <td>0.42</td>\n",
       "      <td>0.32</td>\n",
       "      <td>8.45</td>\n",
       "      <td>0.10</td>\n",
       "      <td>2.601036</td>\n",
       "      <td>8.03</td>\n",
       "      <td>-7.93</td>\n",
       "      <td>-5.428964</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>293</th>\n",
       "      <td>campionato</td>\n",
       "      <td>torneo</td>\n",
       "      <td>8.54</td>\n",
       "      <td>3.06</td>\n",
       "      <td>1.48</td>\n",
       "      <td>5.48</td>\n",
       "      <td>2.778964</td>\n",
       "      <td>7.06</td>\n",
       "      <td>-1.58</td>\n",
       "      <td>-4.281036</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>68</th>\n",
       "      <td>ragazzo</td>\n",
       "      <td>giovanotto</td>\n",
       "      <td>9.12</td>\n",
       "      <td>3.56</td>\n",
       "      <td>2.14</td>\n",
       "      <td>5.56</td>\n",
       "      <td>2.858964</td>\n",
       "      <td>6.98</td>\n",
       "      <td>-1.42</td>\n",
       "      <td>-4.121036</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>205</th>\n",
       "      <td>cento</td>\n",
       "      <td>percento</td>\n",
       "      <td>8.77</td>\n",
       "      <td>1.89</td>\n",
       "      <td>0.53</td>\n",
       "      <td>6.88</td>\n",
       "      <td>4.178964</td>\n",
       "      <td>8.24</td>\n",
       "      <td>-1.36</td>\n",
       "      <td>-4.061036</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>260</th>\n",
       "      <td>profitto</td>\n",
       "      <td>avvertimento</td>\n",
       "      <td>2.81</td>\n",
       "      <td>0.38</td>\n",
       "      <td>7.02</td>\n",
       "      <td>2.43</td>\n",
       "      <td>0.271036</td>\n",
       "      <td>4.21</td>\n",
       "      <td>-1.78</td>\n",
       "      <td>-3.938964</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "           Word1         Word2  Average Score  base  ngram  bdiff  bdiff_v2  \\\n",
       "91   mezzogiorno         corda           0.42  0.32   8.45   0.10  2.601036   \n",
       "293   campionato        torneo           8.54  3.06   1.48   5.48  2.778964   \n",
       "68       ragazzo    giovanotto           9.12  3.56   2.14   5.56  2.858964   \n",
       "205        cento      percento           8.77  1.89   0.53   6.88  4.178964   \n",
       "260     profitto  avvertimento           2.81  0.38   7.02   2.43  0.271036   \n",
       "\n",
       "     ngdiff  bngdiff  bngdiff_v2  \n",
       "91     8.03    -7.93   -5.428964  \n",
       "293    7.06    -1.58   -4.281036  \n",
       "68     6.98    -1.42   -4.121036  \n",
       "205    8.24    -1.36   -4.061036  \n",
       "260    4.21    -1.78   -3.938964  "
      ]
     },
     "execution_count": 45,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Pairs of words of worst underperforming cases of ngram based model after rescaling base scores\n",
    "much_worse_v2.head(5)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 46,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Word1</th>\n",
       "      <th>Word2</th>\n",
       "      <th>Average Score</th>\n",
       "      <th>base</th>\n",
       "      <th>ngram</th>\n",
       "      <th>bdiff</th>\n",
       "      <th>bdiff_v2</th>\n",
       "      <th>ngdiff</th>\n",
       "      <th>bngdiff</th>\n",
       "      <th>bngdiff_v2</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>228</th>\n",
       "      <td>hotel</td>\n",
       "      <td>prenotazione</td>\n",
       "      <td>8.85</td>\n",
       "      <td>1.72</td>\n",
       "      <td>8.72</td>\n",
       "      <td>7.13</td>\n",
       "      <td>4.428964</td>\n",
       "      <td>0.13</td>\n",
       "      <td>7.00</td>\n",
       "      <td>4.298964</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>179</th>\n",
       "      <td>decorazione</td>\n",
       "      <td>valore</td>\n",
       "      <td>6.35</td>\n",
       "      <td>0.00</td>\n",
       "      <td>5.90</td>\n",
       "      <td>6.35</td>\n",
       "      <td>3.648964</td>\n",
       "      <td>0.45</td>\n",
       "      <td>5.90</td>\n",
       "      <td>3.198964</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>72</th>\n",
       "      <td>cibo</td>\n",
       "      <td>frutta</td>\n",
       "      <td>7.77</td>\n",
       "      <td>0.59</td>\n",
       "      <td>9.12</td>\n",
       "      <td>7.18</td>\n",
       "      <td>4.478964</td>\n",
       "      <td>1.35</td>\n",
       "      <td>5.83</td>\n",
       "      <td>3.128964</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>264</th>\n",
       "      <td>dollaro</td>\n",
       "      <td>profitto</td>\n",
       "      <td>6.31</td>\n",
       "      <td>0.65</td>\n",
       "      <td>6.47</td>\n",
       "      <td>5.66</td>\n",
       "      <td>2.958964</td>\n",
       "      <td>0.16</td>\n",
       "      <td>5.50</td>\n",
       "      <td>2.798964</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>60</th>\n",
       "      <td>bere</td>\n",
       "      <td>orecchio</td>\n",
       "      <td>0.53</td>\n",
       "      <td>1.44</td>\n",
       "      <td>1.43</td>\n",
       "      <td>0.91</td>\n",
       "      <td>3.611036</td>\n",
       "      <td>0.90</td>\n",
       "      <td>0.01</td>\n",
       "      <td>2.711036</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "           Word1         Word2  Average Score  base  ngram  bdiff  bdiff_v2  \\\n",
       "228        hotel  prenotazione           8.85  1.72   8.72   7.13  4.428964   \n",
       "179  decorazione        valore           6.35  0.00   5.90   6.35  3.648964   \n",
       "72          cibo        frutta           7.77  0.59   9.12   7.18  4.478964   \n",
       "264      dollaro      profitto           6.31  0.65   6.47   5.66  2.958964   \n",
       "60          bere      orecchio           0.53  1.44   1.43   0.91  3.611036   \n",
       "\n",
       "     ngdiff  bngdiff  bngdiff_v2  \n",
       "228    0.13     7.00    4.298964  \n",
       "179    0.45     5.90    3.198964  \n",
       "72     1.35     5.83    3.128964  \n",
       "264    0.16     5.50    2.798964  \n",
       "60     0.90     0.01    2.711036  "
      ]
     },
     "execution_count": 46,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Pairs of words of best outerperforming cases of ngram based model after rescaling base scores\n",
    "much_better_v2.head(5)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Comparison"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 47,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>outperforming</th>\n",
       "      <th>downperforming</th>\n",
       "      <th>equivalent</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>English</th>\n",
       "      <td>169</td>\n",
       "      <td>60</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>German</th>\n",
       "      <td>157</td>\n",
       "      <td>77</td>\n",
       "      <td>3</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Italian</th>\n",
       "      <td>222</td>\n",
       "      <td>27</td>\n",
       "      <td>2</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "         outperforming  downperforming  equivalent\n",
       "English            169              60           0\n",
       "German             157              77           3\n",
       "Italian            222              27           2"
      ]
     },
     "execution_count": 47,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "stats = pd.DataFrame.from_dict(ngram_stats, orient = 'index')\n",
    "stats.index = ['English', 'German', 'Italian']\n",
    "stats"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 48,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<matplotlib.axes._subplots.AxesSubplot at 0x7fc7281a8c90>"
      ]
     },
     "execution_count": 48,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAA20AAAJOCAYAAAAkve/mAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4xLjMsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+AADFEAAAgAElEQVR4nOzdebweZWE3/N8ViCQlKSCLleUhgFhBQiGEVYQoiqCyvlJAn5Kggrzio/K2tZRWDRYV+1Jq1bZWqwQXFAqyKLYVLchuSCSyI1gDRCibbCEECczzx0wON4dzcrKc5FyE7/fzOZ9zz3bNNXPNPff9O3PNnNI0TQAAAKjTqJGuAAAAAIMT2gAAAComtAEAAFRMaAMAAKiY0AYAAFAxoQ0AAKBiQhusIqWUU0opD5VS/mek61KDUsobSil3lFLml1IOHun6rIhSyvRSyrdGuA7TSilXjtC655ZS3rIU800opTSllDVXRb36rXvE9s+qsCzbV0qZUUo5ZQnTNyyl3F5KGbOMdbi5lDJlCdMvK6W8f1nKHGlD7ava69GdX7dcynmrPieXUk4qpfzrMJSzfSnl6uGoE6xKQhsMovsi+lT3AXZ/KeWMUsq45SxrsyR/mmTbpmn+YHhr+pL1qSRfappmXNM0F4x0ZXhp68Lga0a6HquJE5Oc0TTNwmVZqGma1zdNc1myav+QUUqZUkqZtyrW9VLTnV//eylnr+acPFCbNk3zmaZpVjj0N01zQ5JHSykHrGhZsCoJbbBkBzRNMy7JpCQ7J/nrZS2gu6qweZKHm6Z5YDmXXx1tnuTmVbGiUsoaq2I9rHqr8ftjRJRS1koyNcmIXjl+uRrh43m5z8kvwffht5N8YKQrActCaIOl0DTNb5L8e5LtkqSUsk4p5WullPtKKb/puj6u0U2bVkq5qpTy96WU3ya5LMklSTburtrN6OY7sOtO9GjXbWibxevrrvL9RSnlhiRPllLW7Mb9eSnlhlLKk936X1VK+fdSyhOllB+XUtbrKePfSin/U0p5rJRyeSnl9T3TZpRS/rGUcnG37M9KKVv1TH99KeWSUspvu6uMJ3XjR5VSTiyl/KqU8nAp5ZxSyisH22+llGNKKXd25VxUStm4G/+rJFsm+X63T9YaYNm5pZQ/67b3sVLK2b3dtUopH+v2/72llPf3Xmnptu+fSyk/LKU8meRNpZR3lFKuL6U8Xkq5p5Qyvaesxd32ju6mPVJKOa6UsnO3/kdLKV8a4jAZ09XxiVLKz0spf9RT/uJ99kQp5ZZSyiE9015TSvlpt40PlVLO7pn2up52uL2U8sc909bv9unjpZSZSfrab4B9uUzb17XzX5dS7iqlPFBK+UYpZZ2e6X/STXu4lPJX/da1TMdIz3JHl1K+3zN8ZynlnJ7he0opO3Svm1LK8aWUO5LcUUq5vJvtF93xdPjgqylf7Pb1baWUffqt/9aujf67lPKBnmkblFJ+0O2n35ZSriiljOqmbVxKOa+U8mAp5dellA8vYRtnlFL+qbTv2fmlPU/8QSnl812b3FZK2bFn/m1Ke254tLTnigN7pi2x/Zd07Axh1ySPNk0zryvnTaWUG3vK/XG3vsXDV5auK13pusmWUvZLclKSw7vt/EVP+Zt32/1EKeVHpZQNespa0jnxBVdSu315Sill7bTn5sXn1/mlO8/02x9vL+1774nSnrP/rBv/om6l/deVZINuXz5R2vfq5t18J5dSvti9Hl3a8/LfdsNjSykLS3dOHmLbBjrf71ja88gTpT0n9J77Bj0eB9ju/ufFAc/7ZYBzcndsX9St485SyjE95U4vpZxbSvlWKeXxJNO6cf/WjXuilHJjKeW1pZS/LO155J5Syr49ZRxdBnjPDdampd/V26XYp4N+fqT9XN6nDPDZA9VqmsaPHz8D/CSZm+Qt3evN0v4F8m+64QuS/EuStZNslGRmkg9006YlWZTk/yRZM8nYJFOSzOsp+7VJnkzy1iSjk3wsyZ1JXtGz7jndesf2jLs2yauSbJLkgSQ/T7JjkrWS/FeST/as471JxnfTPp9kTs+0GUl+m2SXro7fTvLdbtr4JPel7c45phvetZv20a4Om3bl/kuS7wyy/96c5KG0VynXSvLFJJcPtH+XsP9nJtk4ySuT3JrkuG7afkn+J8nrk/xekm8maZK8pmf7HkvyhrR/nBrTtcHEbnj7JPcnObibf0K3/Je7efdNsrBr54169vfeg9R1epJnkryra88/S/LrJKO76Yd12zEqyeFd27+6m/adJH/VU889u/FrJ7knydFdG03q9ufru+nfTXJON992SX6T5MpB6rdM25f22Lkz7Ze4cUm+l+Sb3bRtk8xPslfXrqenPd4Xv1cGPUZ66rHmAHXcMsmj3X54dZK7kvymZ9ojSUZ1w03aP4S8Ms+/P/raf5B9MK2r5wldGx3eHSOv7Ka/I23wKUn2TrIgyaRu2me7fTe6+3ljN9+oJLOTfCLJK7p6/neStw1ShxldG+7UtcN/dcfJUUnWSHJKkku7eUd3bXBSV/abkzyR5A+Hav8MfezMSHLKIHU8PsnFPcNjkjyVZIOurP9Jcm/a88LYbtr6A5wzpyf5Vr+yL0vyq7Tnv7Hd8KlLeU58Qfv2bkP6nV8H2a77kryxe71eT9tOS7/3TV58Lnkizx/v/9Czn9+c5Mbu9R7dtv2sZ9ovlud837X3XXn+WH1X2vPL4u0d8HgcZLv7b8uA5/2BzslJfprkn7pjYIckDybZp9857+C074Ox3biFSd7Wlf+NtMf3X3X1PCbJr3vKX9J77kVtmp5jain36YCfHz3lPZ5k+yUdN3781PQz4hXw46fWn+6kPz/tF8m7ug+vsWlD09Ppvix28x6Z579sTUtyd7+yXvABlOTjSc7pGR6V9kvXlJ51v3eA+rynZ/i8JP/cM/x/klwwyLas2314r9MNz0jyrz3T357ktp5tuX6Qcm5d/KHdDb+6++Ae6Ev415L8bc/wuG7eCT3bM1Ro+989w3+b5Mvd668n+WzPtNfkxV9OvjFE+34+yd93ryd0y2/SM/3hJIf3298fHaSs6Umu7deefV8SB5h/TpKDutffSPKVJJv2m+fwJFf0G/cvST6Z9gv+M0le1zPtMxk6tC3V9iX5SZIP9kz7w8XtnDag9H7RWzvJ7/L8l/VBj5EsIbR1896TNmAc0e2TmUlelzZ8XNQzX5Pkzf2WXZrQdm96vtx25f/JIPNfkOQj3etPJbmwf/lpr0r1f6//Zdr7wQYqc0aSr/Z7z97aMzwx7VWupP0i/j/pgmo37jvdsbbE9l/SsdNTj8FC21/1tm837ookhybZLcmP0obF/ZK8KckN/d6zQ4W2v+4Z/mCS/+heD3VOXNHQdnfa7nC/P8BxMVRo6z3exyV5Ns8HrIVJ1k97H+BJSeZ185yc5AtLuW1z03O+TxsQ+x+rV/ds74DH4yDb3X9bBjzvD9B+m3XbOb5n+meTzOhp38v7rWt6kkt6hg9I+xm6Rjc8vqvPukvxnntRm+aFoW1p9umAnx89436TZK+h9qEfP7X86B4JS3Zw0zTrNk2zedM0H2ya5qm0/f5HJ7mv65bxaNovRBv1LHfPEOVunDYIJkmapnmuW2aTIcq4v+f1UwMMj0vae7hKKaeWtova42k/wJL2r+WL9T7FcsHiZdN+WP9qkHpvnuT8nu2+Ne0H+6sGmLf/Ns5PGxQ2GWDewQxWx43zwv0z0L56wbhSyq6llEtL243tsSTH5YX7I1nK/TuIvvV17Tmvq2dKKUeVUub07Lftetb9sbR/aZ7ZdfV5bzd+8yS7Ll6mW+49Sf4gyYZpQ1DvNt6VoS3t9r2g7brXa6Zt5xfs+6ZpnkzbrostyzHS30/Tflnbq3t9Wdq/wO/dDfca6j02kN80TdP0DN+V59to/1LKtV1XsEfTfqFd3Eb/f9q/4v+o68Z1Yjd+87RduHrb6KQseVuXpQ3u6Y6l3vpukqHbf0nHzlAeSfvluteytMtQlvSeHuqcuCL+n7RtelfXxXH3ZVi293ifn/Zq1cbd58GstPth8b65Ou0V/t59s6zn+40z8LG62GDH49IYbP/3t3GS3zZN80S/OizrZ9RDTdM82zOcPP85taT33FCWZp8Ota3j0/5RFl4ShDZYdvekvdK2QRfo1m2a5vebpnl9zzzNIMsudm/aL1ZJ2htt0oal3yxDGUvy7iQHJXlLknXSXuFI2nAwlHsy+P1R9yTZv2e7122aZkzT3vPXX/9tXDvtX6QHmndZ3Ze2+91imw0wT//9d1aSi5Js1jTNOmm7Fy3N/lhafXXo7i/ZNMm9pb3/5atJPpS2G9m6SW5avO6maf6naZpjmqbZOO2VgH/q7kG5J8lP++3rcU3T/L9puyktygu3+38N47a8oO26shel/UJ2X79t/b207brYshwj/S0OB2/sXv80g4eD5Xl/bNK91xb7X2nbaK20VxpPS/Kqro1+mOfb6Immaf60aZot0149+P9Kez/cPWm7e/Vu6/imad6+HHXr794km/W7V+l/pX3/DNX+Szp2hnJD2q5nvfqHtiW1y2LL2j5DnRMXpO0KvVhvAB1yXU3TXNc0zUFp/7h2QdqrhUnbxa6v3FLKQMG293gfl7a73b3dqJ+m7Qq5Y5LruuG3pe2CuPhey2U939+XgY/Vxdsy2PE4nO5N8spSSm+AX3z8DVTnZTLUe24pyl6afbqk9W+cthvq7ctWcxg5Qhsso6Zp7kvbRejvSim/X9oHL2xVStl7GYo5J8k7Sin7lFJGp71/7Om0f6UdDuO78h5O+4XkM8uw7A+S/EEp5aPdzejjSym7dtO+nOTT5fkb8TcspRw0SDlnJTm6lLJD9wH9mbT3e8xdju3p75yu7G260PCJpVhmfNq/HC8speySNtgOp51KKYeW9ilqH027/69N232wSftFO6WUo9M90KYbPqyUsjiAPtLN+2zadnhtaR/6Mbr72bmUsk33l+vvJZleSvm9Usq2aZ/4N1y+k+SEUsoW3ZfUzyQ5u2maRUnOTfLOUsqepZRXpO2q1ftZsizHSH8/TdvlbmzTPgjjirTd8NZPcv0Qy96f9p6yJdkoyYe7fXlYkm3SflF8Rdr7lR5MsqiUsn/a+/7SbcM7S/vAmJL2Pphnu5+ZSR4v7UMkxnZXuLcrpey8lNu7JD9LGyg+1tV3Stov6N9divYf9NhZivXOTLJuKaX3isXVabvI7pJkZtM0N6e7mpfng0l/9yeZUAZ5QMYAhjonzkny7m4f75c2MPaua/3S87CcXqWUV5RS3lNKWadpmmfyfBsmyS+SvL47T41J2wWvv7f3HO9/k/Y8tvgq00/T3pN4S9M0v0t7FfL9acP8g0u5bf1dkzaUf7i0DyU5NO2+X7w9gx2Pw6bbvquTfLaUMqaUsn2S96W9D244LPE9lyHaNCv+GTolyX81TfP08lQeRoLQBsvnqLQfOrek/aJ9btp7d5ZK0zS3J/nfaR/O8VDaL2MHdB/6w+Eb6R7k0NXx2mWo2xNpb+4+IG33kjvSfpFO2pvwL0rbLeeJrtxdBynnJ2nvOzgv7V+Ot0p7r9IKa5rm35N8IcmlabsJXdNNWtIH8AeTfKqr9yfy/F/ah8uFae8leiTJnyQ5tGmaZ5qmuSXJ33V1vD/tfUtX9Sy3c5KflVLmp923H2ma5tddO+ybdp/dm7YtPpf2i07SXrkb142fkeSMYdyWr6d9uMvlaR8ksDDt/VfpvrAfnzaU39dtb+//U1rqY6S/pml+mfYemCu64cfTPtjjqp4uVoOZnuTM0nYHHOxJiT9LsnXa99ynk7yraZqHu3394bTHxCNpA/1FPcttneTHXd2uSfJPTdNc1tXpgLQPafh1V+6/pr26vUK6c8GBSfbvyv2nJEc1TXNbN8ug7b8Ux85Q652R9vy0eNyTaR96dHPPOeqaJHc1g/8bk3/rfj9cSvn5Uqx3qHPiR7pxi7t6XtCz7G1p/9Dw3137v+jpkWnfk3NL2138uMXb1x1zn0rbvnckGegflJ+V9l7S36Z9iMx7eqZdnfbetsXh9Za075e+MLus5/tu/KFp77d7JO155Xs9swx4PA5U1go6Mm0vjXuTnJ/2nshLhqPgod5zQ7XpMHyGviftH5jgJaO8sMs0wEtPdwXhpiRrdVeDgOVUStkwbXDesbtvC1YbpZSJSb7SNM2y3NcII05oA16SSvu/zi5O2/3wzCTPNU1z8MjWCgBg+OkeCbxUfSDt/RC/Sns/x9I8ZAEA4CXHlTYAAICKudIGAABQsTVHugJJssEGGzQTJkwY6WoAAACMiNmzZz/UNM2GA02rIrRNmDAhs2bNGulqAAAAjIhSyl2DTdM9EgAAoGJCGwAAQMWENgAAgIpVcU8bAADwYs8880zmzZuXhQsXjnRVGCZjxozJpptumtGjRy/1MkIbAABUat68eRk/fnwmTJiQUspIV4cV1DRNHn744cybNy9bbLHFUi+neyQAAFRq4cKFWX/99QW21UQpJeuvv/4yXzkV2gAAoGIC2+pledpTaAMAAKiYe9oAAOAlYsKJFw9reXNPfcewljdjxozsu+++2XjjjYetzCOPPDI333xzjj766JxwwgnDVm6SvP3tb89ZZ52Vddddd1jLHW5CGwAAMCxmzJiR7bbbblhC26JFi/LQQw/l6quvzl133bVMy6255tLFnB/+8IfLW71VSmgDAAAGdfrpp+frX/96kuT9739/Dj744Lzzne/MTTfdlCQ57bTTMn/+/Gy33XaZNWtW3vOe92Ts2LG55pprss022+Twww/PpZdemiQ566yz8prXvCYPPvhgjjvuuNx9991Jks9//vN5wxvekOnTp+fee+/N3Llzs8EGG+Smm27KAw88kB122CFf/OIXM378+Bx33HFZsGBBttpqq3z961/PeuutlylTpmSPPfbIVVddlQMPPDA33nhjxo4dm9tuuy133XVXzjjjjJx55pm55pprsuuuu2bGjBlJkgkTJmTWrFmZP39+9t9//+y55565+uqrs8kmm+TCCy/M2LFjc9111+V973tf1l577ey5557593//975tX1Xc0wYAAAxo9uzZOeOMM/Kzn/0s1157bb761a/mkUceGXDed73rXZk8eXK+/e1vZ86cORk7dmyS5Pd///czc+bMfOhDH8pHP/rRJMlHPvKRnHDCCbnuuuty3nnn5f3vf/8L1nnhhRfmrLPOykUXXZStttoqc+bMyRvf+MYcddRR+dznPpcbbrghEydOzMknn9y33KOPPpqf/vSn+dM//dMkySOPPJL/+q//yt///d/ngAMOyAknnJCbb745N954Y+bMmfOi+t9xxx05/vjjc/PNN2fdddfNeeedlyQ5+uij8+UvfznXXHNN1lhjjeHZscvIlTYAAGBAV155ZQ455JCsvfbaSZJDDz00V1xxxTKVceSRR/b9XnxP2o9//OPccsstffM8/vjjeeKJJ5IkBx54YF/g6/XYY4/l0Ucfzd57750kmTp1ag477LC+6YcffvgL5j/ggANSSsnEiRPzqle9KhMnTkySvP71r8/cuXOzww47vGD+LbbYom/cTjvtlLlz5+bRRx/NE088kT322CNJ8u53vzs/+MEPlmn7h4PQBgAADKhpmheNe/TRR/Pcc8/1DQ/1P8d6H3G/+PVzzz2Xa665ZsBwtjggLqv+y6211lpJklGjRvW9Xjy8aNGiFy3fO88aa6yRp556asDtHwm6RwIAAAPaa6+9csEFF2TBggV58sknc/7552f//ffPAw88kIcffjhPP/30C648jR8/vu+K2WJnn3123+/dd989SbLvvvvmS1/6Ut88A3VX7G+dddbJeuut13el75vf/GbfVbeVZb311sv48eNz7bXXJkm++93vrtT1DcaVNgAAeIkY7kf0D2XSpEmZNm1adtlllyTtg0h23nnnfOITn8iuu+6aLbbYIq973ev65p82bVqOO+64vgeRJMnTTz+dXXfdNc8991y+853vJEm+8IUv5Pjjj8/222+fRYsWZa+99sqXv/zlIetz5pln9j2IZMstt8wZZ5yxErb6hb72ta/lmGOOydprr50pU6ZknXXWWenr7K/UcMlv8uTJzaxZs0a6GgAAUJVbb70122yzzUhXY7ktfjrjBhtsMNJVWW7z58/PuHHjkiSnnnpq7rvvvvzDP/zDCpU5ULuWUmY3TTN5oPldaQMAABjExRdfnM9+9rNZtGhRNt98875/F7AqCW0AAMBKMXfu3JGuwgo7/PDDX/RkylXNg0gAAAAqJrQBAABUTGgDAAComNAGAABQMQ8iAQCAl4rpw/w/wqY/tmyzT5+ecePG5c/+7M+Gtx7D4LbbbssRRxyRUkrOPffcbLXVVsNW9qxZs/KNb3wjX/jCF4atzGUhtAEAsMImnHjxSFdhuazqf1bNyvHss8/mggsuyEEHHZSTTz55qZZpmiZN02TUqKE7H06ePDmTJw/4L9RWCd0jAQCAQX3605/OH/7hH+Ytb3lLbr/99iTJnDlzsttuu2X77bfPIYcckkceeSQPPPBAdtpppyTJL37xi5RScvfddydJttpqqyxYsCDTpk3Lhz/84eyxxx7Zcsstc+655yZJLrvssuy111455JBDsu222+a4447Lc889lyT50Y9+lN133z2TJk3KYYcdlvnz5ydp/3H3pz71qey55545++yz8/nPfz7/+q//mje96U1JktNPPz3bbbddtttuu3z+859P0v4Lgm222SYf/OAHM2nSpNxzzz0ZN25c/uIv/iI77bRT3vKWt2TmzJmZMmVKttxyy1x00UV99XvnO9+ZpL3a+N73vrdvnt6rb3/zN3+T173udXnrW9+aI488MqeddtqwtIHQBgAADGj27Nn57ne/m+uvvz7f+973ct111yVJjjrqqHzuc5/LDTfckIkTJ+bkk0/ORhttlIULF+bxxx/PFVdckcmTJ+eKK67IXXfdlY022ii/93u/lyS57777cuWVV+YHP/hBTjzxxL51zZw5M3/3d3+XG2+8Mb/61a/yve99Lw899FBOOeWU/PjHP87Pf/7zTJ48OaeffnrfMmPGjMmVV16Zd7/73TnuuONywgkn5NJLL83s2bNzxhln5Gc/+1muvfbafPWrX83111+fJLn99ttz1FFH5frrr8/mm2+eJ598MlOmTMns2bMzfvz4/PVf/3UuueSSnH/++fnEJz4x4H657bbb8p//+Z+ZOXNmTj755DzzzDOZNWtWzjvvvL59NWvWrGFrB90jAQCAAV1xxRU55JBD+gLXgQcemCeffDKPPvpo9t577yTJ1KlTc9hhhyVJ9thjj1x11VW5/PLLc9JJJ+U//uM/0jRN3vjGN/aVefDBB2fUqFHZdtttc//99/eN32WXXbLlllsmSY488shceeWVGTNmTG655Za84Q1vSJL87ne/y+677963zGD/9PrKK6/MIYcckrXXXjtJcuihh+aKK67IgQcemM033zy77bZb37yveMUrst9++yVJJk6cmLXWWiujR4/OxIkTB/3n4O94xzuy1lprZa211spGG22U+++/P1deeWUOOuigjB07NklywAEHLOVeHprQBgAADKqUstTzvvGNb+y7unbQQQflc5/7XEopfV0Lk2Sttdbqe900zaDrKaWkaZq89a1vzXe+850B17c4lPXXW+5Qy4wePbpv3aNGjeqr36hRo7Jo0aIBy+jdhjXWWCOLFi1a4jpXlO6RAADAgPbaa6+cf/75eeqpp/LEE0/k+9//ftZee+2st956ueKKK5Ik3/zmN/uuuu2111751re+la233jqjRo3KK1/5yvzwhz/su1K2JDNnzsyvf/3rPPfcczn77LOz5557ZrfddstVV12VO++8M0myYMGC/PKXv1yqel9wwQVZsGBBnnzyyZx//vkvuNq3Muy55575/ve/n4ULF2b+/Pm5+OLheziPK20AAPBSsYyP6F9RkyZNyuGHH54ddtghm2++eV/wOfPMM3PcccdlwYIF2XLLLXPGGWckaR8OkrShKWmDzLx587LeeusNua7dd989J554Ym688ca+h5KMGjUqM2bMyJFHHpmnn346SXLKKafkta997ZD1njZtWnbZZZckyfvf//7suOOOg3Z3HA4777xzDjzwwPzRH/1RNt9880yePDnrrDM8/6KhrMzLeEtr8uTJzXDeqAcAwKrlkf8rx6233pptttlmpKux0l122WU57bTT8oMf/GCkq7JC5s+fn3HjxmXBggXZa6+98pWvfCWTJk160XwDtWspZXbTNAP+XwFX2gAAAIbBsccem1tuuSULFy7M1KlTBwxsy0NoAwAARtSUKVMyZcqUka7GCjvrrLNWSrkeRAIAAFAxoQ0AAKBiQhsAAEDFhDYAAICKeRAJAAC8REw8c+Kwlnfj1BuHtbzB3Hvvvfnwhz+cc889d5mXXdF/B/CZz3wmJ5100nItWwtX2gAAgJVq4403Xq7ANhw+85nPjMh6h5PQBgAADOpb3/pWdtlll+ywww75wAc+kGeffTZnnHFGXvva12bvvffOMccckw996ENJkmnTpr0gnI0bNy5JMnfu3Gy33XZJkl133TU333xz3zxTpkzJ7NmzM3PmzOyxxx7Zcccds8cee+T2229/UV2efPLJvPe9783OO++cHXfcMRdeeGGSZMaMGTn00EOz3377Zeutt87HPvaxJMmJJ56Yp556KjvssEPe8573rJwdtAoIbQAAwIBuvfXWnH322bnqqqsyZ86crLHGGvnWt76VT37yk7nqqqtyySWX5JZbblmmMo844oicc845SZL77rsv9957b3baaae87nWvy+WXX57rr78+n/rUpwbs0vjpT386b37zm3Pdddfl0ksvzZ//+Z/nySefTJLMmTMnZ599dm688cacffbZueeee3Lqqadm7NixmTNnTr797W+v+A4ZIe5pAwAABvSTn/wks2fPzs4775wkeeqpp3L11VdnypQp2XDDDZMkhx9+eH75y18udZl//Md/nLe+9a05+eSTc8455+Swww5Lkjz22GOZOnVq7rjjjpRS8swzz7xo2R/96Ee56F1buj4AABjiSURBVKKLctpppyVJFi5cmLvvvjtJss8++2SdddZJkmy77ba56667stlmmy3/xlfElTYAAGBATdNk6tSpmTNnTubMmZPbb78906dPTyllwPnXXHPNPPfcc33L/u53v3vRPJtssknWX3/93HDDDTn77LNzxBFHJEk+/vGP501velNuuummfP/738/ChQsHrM95553XV5+7774722yzTZJkrbXW6ptvjTXWyKJFi1Z4+2shtAEAAAPaZ599cu655+aBBx5Ikvz2t7/NjjvumMsuuywPP/xwnnnmmfzbv/1b3/wTJkzI7NmzkyQXXnjhgFfLkraL5N/+7d/msccey8SJ7RMxH3vssWyyySZJ2nvUBvK2t70tX/ziF9M0TZLk+uuvH3IbRo8ePWg9Xip0jwQAgJeIVfWI/sW23XbbnHLKKdl3333z3HPPZfTo0fnHf/zHTJ8+Pbvvvnte/epXZ9KkSXn22WeTJMccc0wOOuig7LLLLtlnn32y9tprD1juu971rnzkIx/Jxz/+8b5xH/vYxzJ16tScfvrpefOb3zzgch//+Mfz0Y9+NNtvv32apsmECROG/FcAxx57bLbffvtMmjTpJXtfW1mcUkfS5MmTm1mzZo10NQAAWE4TTrx4pKuwXOae+o6RrsIS3XrrrX3d/2o1Y8aMzJo1K1/60pdGuiovGQO1aylldtM0kweaX/dIAACAiukeCQAALLdp06Zl2rRpI12N1ZorbQAAULEabmdi+CxPewptAABQqTFjxuThhx8W3FYTTdPk4YcfzpgxY5ZpOd0jAQCgUptuumnmzZuXBx98cKSrwjAZM2ZMNt1002VaRmgDAIBKjR49OltsscVIV4MRpnskAABAxYQ2AACAigltAAAAFRPaAAAAKia0AQAAVExoAwAAqJjQBgAAUDGhDQAAoGJCGwAAQMWENgAAgIoJbQAAABUT2gAAAComtAEAAFRMaAMAAKiY0AYAAFAxoQ0AAKBiQhsAAEDFhDYAAICKCW0AAAAVGzK0lVI2K6VcWkq5tZRycynlI934V5ZSLiml3NH9Xq8bX0opXyil3FlKuaGUMmllbwQAAMDqammutC1K8qdN02yTZLckx5dStk1yYpKfNE2zdZKfdMNJsn+SrbufY5P887DXGgAA4GViyNDWNM19TdP8vHv9RJJbk2yS5KAkZ3aznZnk4O71QUm+0bSuTbJuKeXVw15zAACAl4FluqetlDIhyY5JfpbkVU3T3Je0wS7JRt1smyS5p2exed24/mUdW0qZVUqZ9eCDDy57zQEAAF4Gljq0lVLGJTkvyUebpnl8SbMOMK550Yim+UrTNJObppm84YYbLm01AAAAXlaWKrSVUkanDWzfbprme93o+xd3e+x+P9CNn5dks57FN01y7/BUFwAA4OVlaZ4eWZJ8LcmtTdOc3jPpoiRTu9dTk1zYM/6o7imSuyV5bHE3SgAAAJbNmksxzxuS/EmSG0spc7pxJyU5Nck5pZT3Jbk7yWHdtB8meXuSO5MsSHL0sNYYAADgZWTI0NY0zZUZ+D61JNlngPmbJMevYL0AAADIMj49EgAAgFVLaAMAAKiY0AYAAFAxoQ0AAKBiQhsAAEDFhDYAAICKCW0AAAAVE9oAAAAqJrQBAABUTGgDAAComNAGAABQMaENAACgYkIbAABAxYQ2AACAigltAAAAFRPaAAAAKia0AQAAVExoAwAAqJjQBgAAUDGhDQAAoGJCGwAAQMWENgAAgIoJbQAAABUT2gAAAComtAEAAFRMaAMAAKiY0AYAAFAxoQ0AAKBiQhsAAEDFhDYAAICKCW0AAAAVE9oAAAAqJrQBAABUTGgDAAComNAGAABQMaENAACgYkIbAABAxYQ2AACAigltAAAAFRPaAAAAKia0AQAAVExoAwAAqJjQBgAAUDGhDQAAoGJCGwAAQMWENgAAgIoJbQAAABUT2gAAAComtAEAAFRMaAMAAKiY0AYAAFAxoQ0AAKBiQhsAAEDFhDYAAICKCW0AAAAVE9oAAAAqJrQBAABUTGgDAAComNAGAABQMaENAACgYkIbAABAxYQ2AACAigltAAAAFRPaAAAAKia0AQAAVExoAwAAqJjQBgAAULE1R7oCDG7CiRePdBWWy9xT3zHSVQAAgNWGK20AAAAVE9oAAAAqJrQBAABUTGgDAAComNAGAABQMaENAACgYkIbAABAxYQ2AACAigltAAAAFRPaAAAAKia0AQAAVExoAwAAqJjQBgAAUDGhDQAAoGJCGwAAQMWENgAAgIoJbQAAABUT2gAAAComtAEAAFRMaAMAAKiY0AYAAFAxoQ0AAKBiQhsAAEDFhDYAAICKCW0AAAAVE9oAAAAqJrQBAABUbM2RrgAAq9aEEy8e6Sosl7mnvmOkqwAAI8KVNgAAgIoJbQAAABUT2gAAAComtAEAAFRMaAMAAKiY0AYAAFAxoQ0AAKBiQhsAAEDFhDYAAICKCW0AAAAVE9oAAAAqJrQBAABUbMjQVkr5einlgVLKTT3jppdSflNKmdP9vL1n2l+WUu4spdxeSnnbyqo4AADAy8HSXGmbkWS/Acb/fdM0O3Q/P0ySUsq2SY5I8vpumX8qpawxXJUFAAB4uRkytDVNc3mS3y5leQcl+W7TNE83TfPrJHcm2WUF6gcAAPCytiL3tH2olHJD131yvW7cJknu6ZlnXjfuRUopx5ZSZpVSZj344IMrUA0AAIDV1/KGtn9OslWSHZLcl+TvuvFlgHmbgQpomuYrTdNMbppm8oYbbric1QAAAFi9LVdoa5rm/qZpnm2a5rkkX83zXSDnJdmsZ9ZNk9y7YlUEAAB4+Vqu0FZKeXXP4CFJFj9Z8qIkR5RS1iqlbJFk6yQzV6yKAAAAL19rDjVDKeU7SaYk2aCUMi/JJ5NMKaXskLbr49wkH0iSpmluLqWck+SWJIuSHN80zbMrp+oAAACrvyFDW9M0Rw4w+mtLmP/TST69IpUCAACgtSJPjwQAAGAlE9oAAAAqJrQBAABUTGgDAAComNAGAABQMaENAACgYkIbAABAxYQ2AACAigltAAAAFRPaAAAAKia0AQAAVExoAwAAqJjQBgAAUDGhDQAAoGJCGwAAQMWENgAAgIoJbQAAABUT2gAAAComtAEAAFRMaAMAAKiY0AYAAFAxoQ0AAKBiQhsAAEDFhDYAAICKCW0AAAAVE9oAAAAqJrQBAABUTGgDAAComNAGAABQMaENAACgYkIbAABAxYQ2AACAigltAAAAFRPaAAAAKia0AQAAVExoAwAAqJjQBgAAUDGhDQAAoGJCGwAAQMWENgAAgIoJbQAAABUT2gAAAComtAEAAFRMaAMAAKiY0AYAAFAxoQ0AAKBiQhsAAEDFhDYAAICKCW0AAAAVE9oAAAAqJrQBAABUTGgDAAComNAGAABQMaENAACgYkIbAABAxYQ2AACAigltAAAAFRPaAAAAKia0AQAAVExoAwAAqJjQBgAAUDGhDQAAoGJCGwAAQMWENgAAgIoJbQAAABUT2gAAAComtAEAAFRMaAMAAKiY0AYAAFAxoQ0AAKBiQhsAAEDFhDYAAICKCW0AAAAVE9oAAAAqJrQBAABUTGgDAAComNAGAABQMaENAACgYkIbAABAxYQ2AACAigltAAAAFRPaAAAAKia0AQAAVExoAwAAqJjQBgAAUDGhDQAAoGJCGwAAQMWENgAAgIoJbQAAABUT2gAAAComtAEAAFRMaAMAAKiY0AYAAFAxoQ0AAKBiQhsAAEDFhDYAAICKCW0AAAAVE9oAAAAqJrQBAABUTGgDAAComNAGAABQMaENAACgYkIbAABAxYQ2AACAigltAAAAFRPaAAAAKia0AQAAVExoAwAAqNiQoa2U8vVSygOllJt6xr2ylHJJKeWO7vd63fhSSvlCKeXOUsoNpZRJK7PyAAAAq7uludI2I8l+/cadmOQnTdNsneQn3XCS7J9k6+7n2CT/PDzVBAAAeHkaMrQ1TXN5kt/2G31QkjO712cmObhn/Dea1rVJ1i2lvHq4KgsAAPBys7z3tL2qaZr7kqT7vVE3fpMk9/TMN68b9yKllGNLKbNKKbMefPDB5awGAADA6m24H0RSBhjXDDRj0zRfaZpmctM0kzfccMNhrgYAAMDqYXlD2/2Luz12vx/oxs9LslnPfJsmuXf5qwcAAPDytryh7aIkU7vXU5Nc2DP+qO4pkrsleWxxN0oAAACW3ZpDzVBK+U6SKUk2KKXMS/LJJKcmOaeU8r4kdyc5rJv9h0nenuTOJAuSHL0S6gwAAPCyMWRoa5rmyEEm7TPAvE2S41e0UgAAALSG+0EkAAAADCOhDQAAoGJCGwAAQMWENgAAgIoJbQAAABUT2gAAAComtAEAAFRMaAMAAKiY0AYAAFAxoQ0AAKBiQhsAAEDFhDYAAICKCW0AAAAVE9oAAAAqJrQBAABUTGgDAAComNAGAABQMaENAACgYkIbAABAxYQ2AACAigltAAAAFRPaAAAAKia0AQAAVExoAwAAqJjQBgAAUDGhDQAAoGJCGwAAQMWENgAAgIoJbQAAABUT2gAAAComtAEAAFRMaAMAAKiY0AYAAFAxoQ0AAKBiQhsAAEDFhDYAAICKCW0AAAAVE9oAAAAqJrQBAABUTGgDAAComNAGAABQMaENAACgYkIbAABAxYQ2AACAigltAAAAFRPaAAAAKia0AQAAVGzNka4AACyV6euMdA2W3/THRroGALyEudIGAABQMaENAACgYkIbAABAxYQ2AACAigltAAAAFRPaAAAAKia0AQAAVExoAwAAqJjQBgAAUDGhDQAAoGJCGwAAQMWENgAAgIoJbQAAABUT2gAAAComtAEAAFRMaAMAAKiY0AYAAFAxoQ0AAKBiQhsAAEDFhDYAAICKCW0AAAAVE9oAAAAqJrQBAABUTGgDAAComNAGAABQMaENAACgYkIbAABAxYQ2AACAigltAAAAFRPaAAAAKia0AQAAVExoAwAAqJjQBgAAUDGhDQAAoGJCGwAAQMWENgAAgIoJbQAAABUT2gAAAComtAEAAFRszZGuAKuh6euMdA2W3/THRroGAADwAq60AQAAVExoAwAAqJjQBgAAUDGhDQAAoGJCGwAAQMWENgAAgIoJbQAAABUT2gAAAComtAEAAFRMaAMAAKiY0AYAAFAxoQ0AAKBiQhsAAEDFhDYAAICKCW0AAAAVE9oAAAAqJrQBAABUTGgDAAComNAGAABQsTVXZOFSytwkTyR5Nsmipmkml1JemeTsJBOSzE3yx03TPLJi1QQAAHh5Go4rbW9qmmaHpmkmd8MnJvlJ0zRbJ/lJNwwAAMByWBndIw9Kcmb3+swkB6+EdQAAALwsrGhoa5L8qJQyu5RybDfuVU3T3Jck3e+NBlqwlHJsKWVWKWXWgw8+uILVAAAAWD2t0D1tSd7QNM29pZSNklxSSrltaRdsmuYrSb6SJJMnT25WsB4AAACrpRW60tY0zb3d7weSnJ9klyT3l1JenSTd7wdWtJIAAAAvV8sd2kopa5dSxi9+nWTfJDcluSjJ1G62qUkuXNFKAgAAvFytSPfIVyU5v5SyuJyzmqb5j1LKdUnOKaW8L8ndSQ5b8WoCAAC8PC13aGua5r+T/NEA4x9Oss+KVAoAAIDWynjkPwAAAMNEaAMAAKiY0AYAAFAxoQ0AAKBiQhsAAEDFhDYAAICKCW0AAAAVE9oAAAAqJrQBAABUTGgDAAComNAGAABQMaENAACgYkIbAABAxYQ2AACAigltAAAAFRPaAAAAKia0AQAAVExoAwAAqJjQBgAAUDGhDQAAoGJCGwAAQMWENgAAgIoJbQAAABUT2gAAAComtAEAAFRMaAMAAKiY0AYAAFAxoQ0AAKBiQhsAAEDFhDYAAICKCW0AAAAVE9oAAAAqJrQBAABUTGgDAAComNAGAABQMaENAACgYkIbAABAxYQ2AACAigltAAAAFRPaAAAAKia0AQAAVExoAwAAqJjQBgAAUDGhDQAAoGJCGwAAQMWENgAAgIoJbQAAABUT2gAAACq25khXAAAARsz0dUa6Bstv+mMjXQNWEVfaAAAAKia0AQAAVExoAwAAqJjQBgAAUDGhDQAAoGJCGwAAQMWENgAAgIoJbQAAABUT2gAAAComtAEAAFRMaAMAAKiY0AYAAFAxoQ0AAKBiQhsAAEDFhDYAAICKCW0AAAAVE9oAAAAqJrQBAABUTGgDAAComNAGAABQMaENAACgYkIbAABAxYQ2AACAigltAAAAFRPaAAAAKia0AQAAVExoAwAAqJjQBgAAUDGhDQAAoGJCGwAAQMWENgAAgIoJbQAAABUT2gAAAComtAEAAFRMaAMAAKiY0AYAAFAxoQ0AAKBiQhsAAEDFhDYAAICKCW0AAAAVE9oAAAAqJrQBAABUTGgDAAComNAGAABQMaENAACgYkIbAABAxYQ2AACAigltAAAAFRPaAAAAKia0AQAAVExoAwAAqJjQBgAAUDGhDQAAoGJCGwAAQMWENgAAgIqtOdIVAIDV3cQzJ450FZbLjVNvHOkqABBX2gAAAKrmShsAALwEuYr/8uFKGwAAQMVWWmgrpexXSrm9lHJnKeXElbUeAACA1dlKCW2llDWS/GOS/ZNsm+TIUsq2K2NdAAAAq7OVdaVtlyR3Nk3z303T/C7Jd5MctJLWBQAAsNpaWQ8i2STJPT3D85Ls2jtDKeXYJMd2g/NLKbevpLqwipWVv4oNkjy0Uko+eRXUHlguL+lzS25aOcWuZGWacyKrP+eWVc+5ZVCbDzZhZYW2gVqiecFA03wlyVdW0vpZjZVSZjVNM3mk6wGsXpxbgJXBuYXhsLK6R85LslnP8KZJ7l1J6wIAAFhtrazQdl2SrUspW5RSXpHkiCQXraR1AQAArLZWSvfIpmkWlVI+lOQ/k6yR5OtN09y8MtbFy5JutcDK4NwCrAzOLayw0jTN0HMBAAAwIv5ve/cb2lUVx3H8/WmstIgRJEGUzkiTVrZEB4X9wUroQZEVDQaREFISPRgESUIhQYk9CC0zyScL2oOIFdEDNdQHZsbStZwTQXCNKCmCkgppuH17cM/a5Qfbb/3222938/OCce8995zzO3cPvr9z7jn3d6ft5dpmZmZmZmY2dR60mZmZmZmZFZgHbVZTkoYl9eb+Nk2hrr/S9npJn0yQr1HS7HyRiZlNSNJ1kjolnZV0XNJRSetmul1mNrfk+hyNktomkf+/voeklZJ2THcbbW6brve0mY3nQkQ0V7PCiPgZeLKadZpZ8UkS8BnQERFtKW0R8Ogky9dFxPA0NtHM5p5GoA3onGyBiDgGHJuuBtmlwTNtVgiSfpC0RVKPpD5Jy1L6AklfpvTdkgYlXVtSNn83q0lSd5rFOyFpScpWJ+kDSf2S9kuaX+NLNLPqWwMMRcT7owkRMRgR70iqk/SWpG9TLHgOQNL9kg5J6gT6Uvw4LWmPpJOSPpL0oKQjks5IaknlWiR9Lem7tL0lpa+X1CVpb8q/bSb+EWZWM1uBe1I/oz3FkMOpn9Ij6e7SAinufJH2HUusIh60Wa3NL1ke2Zo791tErAB2AS+ltNeAgyn9U2BhmfqfB7an2byVZC96B1gC7IyIJuAP4IkqXY+ZzZwmoGecc88C5yNiFbAK2CBpcTrXAmyOiFvT8c3AdmA5sIzsLvpqsjj0SspzGrg3Iu4EXgXeyH1WM9AK3A60SrqxCtdmZsW0CTgcEc0R8TbwK/BQ6qe0AuWWQTqWWEW8PNJqbaLlkV1pexx4PO2vBtYBRMReSb+Xqf8osFnSDUBXRJzJVlAxEBG9ufobK2y/mRWUpJ1kMWMIGASWSxpdOt1AdvNmCOiOiIFc0YGI6Et19AMHIiIk9TEWKxqAjjR7H0B9rvyBiDifyp8CFgE/TsMlmlnx1APvSmoGhoGlZfI7llhFPNNmRfJP2g4zdkNB/6eCiOgke57lArBP0pqSukvrN7PZqx9YMXoQES8ADwALyGLHi+lueHNELI6I/Snr3yX15OPDSO54hLFY8TpwKCJuAx4B5o1T3vHF7NLSDvwC3EG2wufyMvkdS6wiHrRZ0X0FPAUgaS1wzUSZJd0EnI2IHcDnZMudzGxuOgjMk7Qxl3Zl2u4DNkqqB5C0VNJVU/isBuCntL9+CvWY2ez2J3B17rgBOBcRI8DTQF2Z8o4lVhEP2qzWSp9p21om/xZgraQe4GHgHFnAHE8rcFJSL9mzKR9WpdVmVjgREcBjwH2SBiR1Ax3Ay8Ae4BTQk36oaDdTu2u9DXhT0hHKd8rMbO46AVyU9L2kduA94BlJ35AtjSydyS/lWGIVUfadZ1ZMkq4AhiPioqS7gF3VfmWAmZmZmVmRea2sFd1C4GNJl5H9gMCGGW6PmZmZmVlNeabNzMzMzMyswPxMm5mZmZmZWYF50GZmZmZmZlZgHrSZmZmZmZkVmAdtZmZmZmZmBeZBm5mZmZmZWYH9CyxgMOYsESEvAAAAAElFTkSuQmCC\n",
      "text/plain": [
       "<Figure size 1080x720 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "stats.plot(kind='bar', rot = 0, figsize=(15,10),\n",
    "           title = 'Performance of ngram based model wrt base model (without subwords information)')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 49,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>outperforming</th>\n",
       "      <th>downperforming</th>\n",
       "      <th>equivalent</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>English</th>\n",
       "      <td>82</td>\n",
       "      <td>147</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>German</th>\n",
       "      <td>94</td>\n",
       "      <td>143</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Italian</th>\n",
       "      <td>116</td>\n",
       "      <td>135</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "         outperforming  downperforming  equivalent\n",
       "English             82             147           0\n",
       "German              94             143           0\n",
       "Italian            116             135           0"
      ]
     },
     "execution_count": 49,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "stats_v2 = pd.DataFrame.from_dict(ngram_stats_v2, orient = 'index')\n",
    "stats_v2.index = ['English', 'German', 'Italian']\n",
    "stats_v2"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 50,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<matplotlib.axes._subplots.AxesSubplot at 0x7fc72cd29850>"
      ]
     },
     "execution_count": 50,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAA20AAAJOCAYAAAAkve/mAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4xLjMsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+AADFEAAAgAElEQVR4nOzdeZglZX0v8O9vABkEBGQxAoYBghEEwjKsIo6iBhc2rwTQRFARuepVuSaGmKhg1GgucU9i3BhcUAjI4paIyr7PyMiOoLKMEDYZtgFloO4fVT0cmu7pnumGKZjP53n66VP7W9t76nvqPXWqaZoAAADQT1OWdgEAAAAYndAGAADQY0IbAABAjwltAAAAPSa0AQAA9JjQBgAA0GNCG+NWVR+tqjuq6n+Wdln6oKpeVFXXVtV9VbX30i7PRFTVEVX1zaVchoOq6pyltOzrq+rl4xhvWlU1VbX8k1GuYcteatvnybA461dVM6vqoxNc3geq6iuTUZ4+mMixOda04z0/aFXV2lV1TVVNXczprqiqGYsYfkZVHTzhAj6JJuNcXZrl6N7fNxrnuL2+JhirzluM+WxZVedNRplYPELb01j3RvtAV4HcWlVHV9UqSziv5yV5X5LNmqb5o8kt6VPWR5J8oWmaVZqmOXlpF4antu6i+U+WdjmWFU3TfLxpmoOTpRvGeVo6PMnRTdM8uDgTNU3zwqZpzkie3A/SqmpGVc19Mpb1VNO9v/96nKP35ppgpH06WOdNRNM0lyaZV1V7THReLB6h7elvj6ZpVkmyTZLtkvzD4s6gu5DZIMmdTdPctoTTPx1tkOSKJ2NBVbXck7EcnnxP4/NjmWWfPrVM5v6qqhWTHJhkqbZcWFYt5XNvia8JnoJ1xreSvH1pF2JZI7QtI5qm+W2SHyXZPEmqarWq+mpV3VJVv+2aPi7XDTuoqs6tqk9X1e+SnJHktCTrdnftZnbj7dk155jXNdvYdGh53V2+v62qS5PcX1XLd/3+pqourar7u+U/p6p+VFX3VtVPqmqNgXn8Z1X9T1XdXVVnVdULB4bNrKp/raofdNNeWFUbDwx/YVWdVlW/6+4yfqDrP6WqDq+qX1XVnVV1fFU9e7TtVlVvq6rruvmcWlXrdv1/lWSjJN/rtsmKI0x7fVX9dbe+d1fVcYPNZarq/d32v7mqDh6809Kt379X1Q+r6v4kL62q11TVJVV1T1XdVFVHDMxr6E7Bm7thd1XVoVW1Xbf8eVX1hTEOk6ldGe+tqp9X1Z8NzH9om91bVVdW1T4Dw/6kqs7s1vGOqjpuYNgLBvbDNVX1FwPD1uy26T1VdVGShftvhG25WOvX7ed/qKobquq2qvp6Va02MPyvumF3VtXfD1vWYh0jA9O9uaq+N9B9XVUdP9B9U1Vt1b1uquqdVXVtkmur6qxutF90x9N+oy+mPt9t66urardhy7+q20e/rqq3Dwxbq6q+322n31XV2VU1pRu2blWdWFW3V9Vvqurdi1jHmVX1b9Wes/dVW0/8UVV9ptsnV1fV1gPjb1pt3TCv2rpiz4Fhi9z/izp2FqXbr9t2r/+y29abdd0HV9XJ3evBOxlD239et147DczvqG7dflNVr1rEckeq80bdtlW1fVXN6tb/1qr61MCwXarqvG673VRVB3X9R60DRijPour45br1uqOqfp3kNePYtNtVe+7fVW2rjandvNbojq3bu2Hfr6r1B8pxUHc83tttgzcODHtLd8zeVVX/XVUbjLIuU6vqm9Wej/Oq6uKqek437NldeW7u5nPywHQj1t/dsMecg12/RdVXr+7W/95ue/71KNtphyTzmqaZ20330qq6bGA+P+mO96Huc6prStcdQy+vqt2TfCDJft3x+IuB+W9Q7Xl3b1X9uKrWGpjXot6TH3Mnv7rmglW1ctprg6H39/sGt9NY618jNCMevqwka3Xb9d5q3ys26MY7sqo+371eodrrgn/uuleqqgeruyYYY91GOve2rvZ97N5q35MG33tHrQ9HWO/h78sjXnfUCNcE1Z7/p3bLuK6q3jYw3yOq6oTuuL4nyUFdv//s+t1bVZdV1fOr6u+qfR+7qapeOTCPN9cIdf5o+7SG3b0dxzYd9fol7XXhbjXCtQ9PoKZp/D1N/5Jcn+Tl3evnpf0E6B+77pOT/EeSlZOsk+SiJG/vhh2UZEGS/5Nk+SQrJZmRZO7AvJ+f5P4kr0iyQpL3J7kuyTMGlj2nW+5KA/0uSPKcJOsluS3Jz5NsnWTFJD9L8uGBZbwlyardsM8kmTMwbGaS3yXZvivjt5J8pxu2apJb0jbnnNp179ANe29XhvW7+f5Hkm+Psv1eluSOtHcpV0zy+SRnjbR9F7H9L0qybpJnJ7kqyaHdsN2T/E+SFyZ5ZpJvJGmS/MnA+t2d5EVpP1yZ2u2DLbruLZPcmmTvbvxp3fRf7MZ9ZZIHu/28zsD2fskoZT0iyUNJXt/tz79O8pskK3TD9+3WY0qS/bp9/9xu2LeT/P1AOXfp+q+c5KYkb+720Tbd9nxhN/w7SY7vxts8yW+TnDNK+RZr/dIeO9elfRNdJcl3k3yjG7ZZkvuS7Nrt10+lPd6HzpVRj5GBciw/Qhk3SjKv2w7PTXJDkt8ODLsryZSuu0n7Qciz8+j5sXD/j7INDurKeVi3j/brjpFnd8Nfkzb4VJKXJJmfZJtu2D91226F7u/F3XhTksxO8qEkz+jK+eskfz5KGWZ2+3Dbbj/8rDtO3pRkuSQfTXJ6N+4K3T74QDfvlyW5N8mfjrX/M/axMzPJR0cp49eTvK97/aUkv0ryvweGHTZwzH9ztP3abe+HkrytW7f/neTmJLWI831hnTfWtk1yfpK/6l6vkmTH7vUfd9vpgG4brplkq27YjIxdByw/jjr+0CRXd2V9dpLTh6//COt2+cD45w5t/658/yttPbZqkv9McvLAfrxnYJ8/d2Af7p32+Ni028f/kOS8UZb/9iTf65axXNrj71ndsB8kOS7JGt32GqoDxqq/H3MOZuxj7pYkL+5er5Hu3BqhrO9M8oOB7qlJHkiyVjff/0l7HK3aLfeBJGuO8J59RLrjc2BeZ6Q9np/fTXtGkk+M8z35MfVLBs6hDHt/H2W9Rlz/tOfJOcPGHf5edm8erW8/m0fP85cluax7vXO3bhcODPvFONft+jz23HtG2vp3qK58fdpzeWh9R6wPR1nv4esy4nXH8P3XdZ+Z5N+6Y2CrJLcn2W1g/z6U9jyY0pX7iLTva3/ezf/raevXv+/K+bYkvxmY/6Lq/Mft0zy2zhvPNh3x+mVgfvck2XJRx42/yf1b6gXw9wTu3Pakuy/theQNXeWxUtrQ9Pt0F4vduAfk0Yutg5LcOGxej6kAknwwyfED3VPSXnTNGFj2W0YozxsHuk9M8u8D3f8n3Zv9COuyeld5rtZ1z0zylYHhr05y9cC6XDLKfK4aqjS77ud2FedIF+FfTfLPA92rdONOG1ifsULbXw50/3OSL3avv5bknwaG/Uke/+bw9TH272eSfLp7Pa2bfr2B4Xcm2W/Y9n7vKPM6IskFw/bnwjfpEcafk2Sv7vXX014crz9snP2SnD2s338k+XDaC6+HkrxgYNjHM3ZoG9f6JflpkncMDPvTof2c9iJ68I125SR/yKMXS6MeI1lEaOvGvSntxd7+3Ta5KMkL0l4InjowXpPkZcOmfcxF1QjzPijDQkM3/78aZfyTk7yne/2RJKcMn3/auwLDz/W/S/t9nJHmOTPJl4eds1cNdG+R9i5D0l4I/U+6oNr1+3Z3rC1y/y/q2Bkox2ih7a1D27rblwfn0Q90bsijFzVHZOzQdt1A9zO7cf5oEef7Wwa6F7lt097dOzLJWiOMc9Jox8GwcUeqA5bP2HX8zzJwAZb2Q5BFHdfXDxv/1Ul+Ncq4WyW5a+Dcmpc21K00bLwfJXnrQPeUtBedG4wwz7ckOS/DLhDTnpuPJFljhGnGqr8fcw6O45i7MW14fNYY++TvM1C/dP3OTvK6JDsm+XHaDyt2T/LSJJcO285jhbZ/GOh+R5L/6l6P9Z78mPolix/aRlz/jC+0Dda3qyR5OI8GrAfTBv/D037AM7cb58gknxvnul2fx557u+bxdeV5A+s7Yn04ynoPX5cRrztG2H/P69Zz1YHh/5Rk5sD+PWvYso5IctpA9x5pr+GW67pX7cqz+ihlHazzH7dP89g6bzzbdMTrl4F+v02y61jb0N/k/Wke+fS3d9M0qzdNs0HTNO9omuaBtO2uV0hyS3dbfF7aN6d1Bqa7aYz5rpv2AihJ0jTNI900640xj1sHXj8wQvcqycLmO5+otonaPWkrkKT9tHLI4FMs5w9Nm7ay/NUo5d4gyUkD631V2or1OSOMO3wd70sbFNYbYdzRjFbGdfPY7TPStnpMv6raoapOr7YZ0t1pPy1fa9g049q+o1i4vG5/zu3Kmap6U1XNGdhumw8s+/1pP+m7qGtq8Zau/wZJdhiappvujUn+KMnaaS8uB9fxhoxtvOv3mH3XvR66mH3Mtm+a5v60+3XI4hwjw52Z9s1y1+71GWk/AX1J1z1orHNsJL9tunfLzg15dB+9qqou6JrizEt7QTG0j/5f2k9Rf9w1ozm8679B2iY0g/voA1n0ui7OPripO5YGy7text7/izp2xnJmkhdX1R+lDYfHJXlRVU1LslraDxzGa+H52zTN/O7luM6hjL1t35r20+6rq23u99qu/6j11zjrgKFlL6qOH17/jOfcGz7+0HH3zKr6j2qbpd6TNoyuXlXLdefWfl05b+malb1goIyfHSjf79LWIyPVr99I8t9JvlNtM8h/rqoV0m6r3zVNc9cI04yn/h6+vxZ1zP2vtOfUDdU28dspI7sr7cX1oMWpF8ayqPeUsd6TJ2K86z+Swfr2vrT7et3uemRW2u0wtG3OS9vCZHDbLO71xroZua4cMlp9OB6jbf/h1k17bN47rAyLe410R9M0Dw90J49eJy2qzh/LeLbpWOu6atoPZXiSCG3LppvSfgq7VhfoVm+a5llN07xwYJxmlGmH3Jz2TS5J+0WbtG+gv12MeSzKG5LsleTlaS+0pg0tahzT3pTRvx91U5JXDaz36k3TTG3a7/wNN3wdV077ieBI4y6uW9I2vxvyvBHGGb79jk1yapLnNU2zWtrmHePZHuO1sAzVtu9fP8nN1X7/4MtJ3pW2Gc/qaZtKVZI0TfM/TdO8rWmaddN+Evtv1X4H4KYkZw7b1qs0TfO/0zYTWZDHrvcfT+K6PGbfdfNekPYN8ZZh6/rMtPt1yOIcI8MNXZy9uHt9Zka/OFuS82O97lwb8sdp99GKae80HpXkOd0++mEe3Uf3Nk3zvqZpNkr76e3/rfb7cDelbW4zuK6rNk3z6iUo23A3J3lePfa7In+c9vwZa/8v6thZpKZprkt7gfHutJ9k35v24uOQtHcEHhlpssVZsUUtfuD1Irdt0zTXNk1zQNog9ckkJ3R1zKLqr/HWAWPV8Y85BzK+c2/4+Dd3r9+X9k72Dk3TPCvtxXfy6LH3303TvCLtXbGr09YlQ2V8+7Dts1LTNI97lHjTNA81TXNk0zSbpW1G99q0TXJvSvLsqlp9hPKOp/4evr9GPeaaprm4aZq90u6vk9PeLRvJpWnD+KDhoW1R9cJIZRuPsd6T56e9Wzxk8AOQMZe1iPW/f3C+3Yclww3Wt6ukbW43dPycmbYp5NZJLu66/zxtE8Sh75ou7vXGLRm5rhxal9Hqw8l0c9pjczDAD9V/I5V5sYxV549j3uPZpota/rppm6Fes3glZyKEtmVQ0zS3pG2i8S9V9axqH7ywcVW9ZDFmc3yS11TVbt0nnu9Le5EwWb/dsWo3vzvTviF8fDGm/X6SP6qq91b7ZeBVq2qHbtgXk3ysHv0i9NpVtdco8zk2yZuraquugvx42vb21y/B+gx3fDfvTbvQ8KFxTLNq2k/uHqyq7dMG28m0bVW9rtqnWL037fa/IG0TpybthXaq6s3pHmjTde9bjz544K5u3IfT7ofnV/vQjxW6v+2qatPuk8PvJjmi+6R+s7RPXJss305yWFVt2F0kfDzJcU3TLEhyQpLXVvuwh2ekbSozWBcuzjEy3Jlpmzyt1LQPIjg7bTOoNZNcMsa0t6b93tOirJPk3d223Dft94F+mPbNc8V0YajaB2YMfmH9tdU+MKbSfg/h4e7voiT3VPsl/pWqvcO9eVVtN871XZQL017Qvb8r74y0F0jfGcf+H/XYGeeyz0z7IcPQBfEZw7qHuz1tM7tx/R7TOC1y21b7kJS1uxA59Gn1w2m/J/PyqvqLah+osGZ1D7DJOOuAcdTxx6c9jtav9kEP47nT8M5u/GenvWM49MChVdPeAZjXDfvw0ATVPmhqzy4w/T5tU6+huwZfTPJ31T1gqtoHp+w70oKrfZjHFtU+SOWetM0cH+7W80dpPyhaoztOhkLj4tbfox5zVfWMqnpjVa3WNM1DefQcGslFae80Dt6xOC9tsN0+yUVN01yR7s5eHg0mw92aZFqN8oCMEYz1njwnyRu643D3tIFxcFlr1sDDmgaNsf6/SPLCbjtPTdsEb7hXD9S3/5h2PwzdZTozbQC/smmaP6Q9Vw9O+4HH7eNct+HOT/uh0Lu7c+h1abf90PqMVh9Omm79zkvyT9U+SGfLtHfXvzVJi1hknZ8x9mkmfg03I8nPmqb5/ZIUniUjtC273pT2pL8y7YX2CWk/CR2XpmmuSfKXab/cfUfai7E9ukp3Mnw93YMcujJesBhluzftl2v3SPsJ+7VpL6ST9kvQp6ZtFnFvN98dRpnPT9O2+z4x7Sd3G6f9rtKENU3zoySfS/sAgOvSvskkbaU5mnck+UhX7g9l9E96l9QpaZsy3ZXkr5K8rvuE+8ok/9KV8da031s6d2C67ZJcWFX3pd2272ma5jfdfnhl2m12c9p98cm0bzRJexG9Std/ZpKjJ3Fdvpa2SdVZab/I/WDa71+lu2B6Z9qLulu69R38PZtxHyPDNU3zy7QXpmd33fekffjEuQNNXEZzRJJjqm2aNdqTEi9Msknac+5jSV7fNM2d3bZ+d9pj4q60F/OnDky3SZKfdGU7P8m/NU1zRlemPdJ+D+k33Xy/kvbu9oR0dcGeSV7VzfffkrypaZqru1FG3f/jOHbGcmbaMHHWKN3Dyzo/7fY8t9v+O45zOaMax7bdPckV3Xnz2ST7N03zYNM0N6Zt5vS+tM3I5iQZepLr4tQBi6rjv5y2ueEv0j4M6rvjWKVj0wbBX3d/Qz9U/Jm03026I+258l8D00zp1uPmbl1e0q1DmqY5Ke0+/U61zSovT3usjOSPuvLfk7a58pl59JH6f5U2xF2d9mFE7+3mv1j19ziOub9Kcn1X1kPTvv+NNJ8/pD2e/3Kg3/1pt/MVA++R5ye5oRn9Z3T+s/t/Z1X9fLRyDyxjrPfk93T9hpp9njww7dVpP+j6dXf8P+7pkRll/bs67yNp65drk4z0g/THpg3zv0v7EJk3Dgw7L+3xM3RuXpm2vl54ri7u9UbX/3Vpv293V9r3tcFjfMT6cKR5TdABaVsJ3ZzkpLTfjzxtMmY8Vp0/1j6dhGu4N6b94IUnUTXNEt+dBSZJdwfh8iQrdneDAHgKqqq1035ws3XTfm8LnjaqaoskX2qaZnG+18gkENpgKan2t85+kLb54TFJHmmaZu+lWyoAAPpG80hYet6etj36r9K2px/zIQsAACx73GkDAADoMXfaAAAAemz5pV2AJFlrrbWaadOmLe1iAAAALBWzZ8++o2matUca1ovQNm3atMyaNWtpFwMAAGCpqKobRhumeSQAAECPCW0AAAA9JrQBAAD0WC++0wYAADzeQw89lLlz5+bBBx9c2kVhkkydOjXrr79+VlhhhXFPI7QBAEBPzZ07N6uuumqmTZuWqlraxWGCmqbJnXfemblz52bDDTcc93SaRwIAQE89+OCDWXPNNQW2p4mqypprrrnYd06FNgAA6DGB7ellSfan0AYAANBjvtMGAABPEdMO/8Gkzu/6T7xmUuc3c+bMvPKVr8y66647afM84IADcsUVV+TNb35zDjvssEmbb5K8+tWvzrHHHpvVV199Uuc72YQ2AABgUsycOTObb775pIS2BQsW5I477sh5552XG264YbGmW3758cWcH/7wh0tavCeV0AYAAIzqU5/6VL72ta8lSQ4++ODsvffeee1rX5vLL788SXLUUUflvvvuy+abb55Zs2bljW98Y1ZaaaWcf/752XTTTbPffvvl9NNPT5Ice+yx+ZM/+ZPcfvvtOfTQQ3PjjTcmST7zmc/kRS96UY444ojcfPPNuf7667PWWmvl8ssvz2233Zatttoqn//857Pqqqvm0EMPzfz587Pxxhvna1/7WtZYY43MmDEjO++8c84999zsueeeueyyy7LSSivl6quvzg033JCjjz46xxxzTM4///zssMMOmTlzZpJk2rRpmTVrVu6777686lWvyi677JLzzjsv6623Xk455ZSstNJKufjii/PWt741K6+8cnbZZZf86Ec/WrjuTxbfaQMAAEY0e/bsHH300bnwwgtzwQUX5Mtf/nLuuuuuEcd9/etfn+nTp+db3/pW5syZk5VWWilJ8qxnPSsXXXRR3vWud+W9731vkuQ973lPDjvssFx88cU58cQTc/DBBz9mmaecckqOPfbYnHrqqdl4440zZ86cvPjFL86b3vSmfPKTn8yll16aLbbYIkceeeTC6ebNm5czzzwz73vf+5Ikd911V372s5/l05/+dPbYY48cdthhueKKK3LZZZdlzpw5jyv/tddem3e+85254oorsvrqq+fEE09Mkrz5zW/OF7/4xZx//vlZbrnlJmfDLiZ32gAAgBGdc8452WeffbLyyisnSV73utfl7LPPXqx5HHDAAQv/D30n7Sc/+UmuvPLKhePcc889uffee5Mke+6558LAN+juu+/OvHnz8pKXvCRJcuCBB2bfffddOHy//fZ7zPh77LFHqipbbLFFnvOc52SLLbZIkrzwhS/M9ddfn6222uox42+44YYL+2277ba5/vrrM2/evNx7773ZeeedkyRveMMb8v3vf3+x1n8yCG0AAMCImqZ5XL958+blkUceWdg91m+ODT7ifuj1I488kvPPP3/EcDYUEBfX8OlWXHHFJMmUKVMWvh7qXrBgweOmHxxnueWWywMPPDDi+i8NmkcCAAAj2nXXXXPyySdn/vz5uf/++3PSSSflVa96VW677bbceeed+f3vf/+YO0+rrrrqwjtmQ4477riF/3faaackyStf+cp84QtfWDjOSM0Vh1tttdWyxhprLLzT941vfGPhXbcnyhprrJFVV101F1xwQZLkO9/5zhO6vNG40wYAAE8Rk/2I/rFss802Oeigg7L99tsnaR9Est122+VDH/pQdthhh2y44YZ5wQtesHD8gw46KIceeujCB5Ekye9///vssMMOeeSRR/Ltb387SfK5z30u73znO7PllltmwYIF2XXXXfPFL35xzPIcc8wxCx9EstFGG+Xoo49+Atb6sb761a/mbW97W1ZeeeXMmDEjq6222hO+zOGqD7f8pk+f3syaNWtpFwMAAHrlqquuyqabbrq0i7HEhp7OuNZaay3toiyx++67L6usskqS5BOf+ERuueWWfPazn53QPEfar1U1u2ma6SON704bAADAKH7wgx/kn/7pn7JgwYJssMEGC38u4MkktAEAAE+I66+/fmkXYcL222+/xz2Z8snmQSQAAAA9JrQBAAD0mNAGAADQY0IbAABAj3kQCQAAPFUcMcm/EXbE3Ys3+hFHZJVVVslf//VfT245JsHVV1+d/fffP1WVE044IRtvvPGkzXvWrFn5+te/ns997nOTNs/FIbQx+Sa7MnkyLWbFBQDA0vfwww/n5JNPzl577ZUjjzxyXNM0TZOmaTJlytiND6dPn57p00f8CbUnheaRAADAqD72sY/lT//0T/Pyl78811xzTZJkzpw52XHHHbPllltmn332yV133ZXbbrst2267bZLkF7/4RaoqN954Y5Jk4403zvz583PQQQfl3e9+d3beeedstNFGOeGEE5IkZ5xxRnbdddfss88+2WyzzXLooYfmkUceSZL8+Mc/zk477ZRtttkm++67b+67774k7Q93f+QjH8kuu+yS4447Lp/5zGfyla98JS996UuTJJ/61Key+eabZ/PNN89nPvOZJO1PEGy66aZ5xzvekW222SY33XRTVllllfzt3/5ttt1227z85S/PRRddlBkzZmSjjTbKqaeeurB8r33ta5O0dxvf8pa3LBxn8O7bP/7jP+YFL3hBXvGKV+SAAw7IUUcdNSn7QGgDAABGNHv27HznO9/JJZdcku9+97u5+OKLkyRvetOb8slPfjKXXnpptthiixx55JFZZ5118uCDD+aee+7J2WefnenTp+fss8/ODTfckHXWWSfPfOYzkyS33HJLzjnnnHz/+9/P4YcfvnBZF110Uf7lX/4ll112WX71q1/lu9/9bu6444589KMfzU9+8pP8/Oc/z/Tp0/OpT31q4TRTp07NOeeckze84Q059NBDc9hhh+X000/P7Nmzc/TRR+fCCy/MBRdckC9/+cu55JJLkiTXXHNN3vSmN+WSSy7JBhtskPvvvz8zZszI7Nmzs+qqq+Yf/uEfctppp+Wkk07Khz70oRG3y9VXX53//u//zkUXXZQjjzwyDz30UGbNmpUTTzxx4baaNWvWpO0HzSMBAIARnX322dlnn30WBq4999wz999/f+bNm5eXvOQlSZIDDzww++67b5Jk5513zrnnnpuzzjorH/jAB/Jf//VfaZomL37xixfOc++9986UKVOy2Wab5dZbb13Yf/vtt89GG22UJDnggANyzjnnZOrUqbnyyivzohe9KEnyhz/8ITvttNPCaUb70etzzjkn++yzT1ZeeeUkyete97qcffbZ2XPPPbPBBhtkxx13XDjuM57xjOy+++5Jki222CIrrrhiVlhhhWyxxRaj/jj4a17zmqy44opZccUVs8466+TWW2/NOeeckw0HRD8AACAASURBVL322isrrbRSkmSPPfYY51Yem9AGAACMqqrGPe6LX/zihXfX9tprr3zyk59MVS1sWpgkK6644sLXTdOMupyqStM0ecUrXpFvf/vbIy5vKJQNNzjfsaZZYYUVFi57ypQpC8s3ZcqULFiwYMR5DK7DcsstlwULFixymROleSQAADCiXXfdNSeddFIeeOCB3Hvvvfne976XlVdeOWussUbOPvvsJMk3vvGNhXfddt1113zzm9/MJptskilTpuTZz352fvjDHy68U7YoF110UX7zm9/kkUceyXHHHZdddtklO+64Y84999xcd911SZL58+fnl7/85bjKffLJJ2f+/Pm5//77c9JJJz3mbt8TYZdddsn3vve9PPjgg7nvvvvygx/8YNLm7U4bAAA8VTzJT7reZpttst9++2WrrbbKBhtssDD4HHPMMTn00EMzf/78bLTRRjn66KOTtA8HSdrQlLRBZu7cuVljjTXGXNZOO+2Uww8/PJdddtnCh5JMmTIlM2fOzAEHHJDf//73SZKPfvSjef7znz9muQ866KBsv/32SZKDDz44W2+99ajNHSfDdtttlz333DN/9md/lg022CDTp0/PaqtNzlPV64m8jTde06dPbybzi3osZR75DwAwKa666qpsuummS7sYT7gzzjgjRx11VL7//e8v7aJMyH333ZdVVlkl8+fPz6677povfelL2WabbR433kj7tapmN00z4u8KuNMGAAAwCQ455JBceeWVefDBB3PggQeOGNiWhNAGAAAsVTNmzMiMGTOWdjEm7Nhjj31C5utBJAAAAD0mtAEAAPSY0AYAANBjQhsAAECPeRAJAE8Nfk4EIFscs8Wkzu+yAy+b1PmN5uabb8673/3unHDCCYs97UR/DuDjH/94PvCBDyzRtH3hThsAAPCEWnfddZcosE2Gj3/840tluZNJaAMAAEb1zW9+M9tvv3222mqrvP3tb8/DDz+co48+Os9//vPzkpe8JG9729vyrne9K0ly0EEHPSacrbLKKkmS66+/PptvvnmSZIcddsgVV1yxcJwZM2Zk9uzZueiii7Lzzjtn6623zs4775xrrrnmcWW5//7785a3vCXbbbddtt5665xyyilJkpkzZ+Z1r3tddt9992yyySZ5//vfnyQ5/PDD88ADD2SrrbbKG9/4xidmAz0JhDYAAGBEV111VY477rice+65mTNnTpZbbrl885vfzIc//OGce+65Oe2003LllVcu1jz333//HH/88UmSW265JTfffHO23XbbvOAFL8hZZ52VSy65JB/5yEdGbNL4sY99LC972cty8cUX5/TTT8/f/M3f5P7770+SzJkzJ8cdd1wuu+yyHHfccbnpppvyiU98IiuttFLmzJmTb33rWxPfIEuJ77QBAAAj+ulPf5rZs2dnu+22S5I88MADOe+88zJjxoysvfbaSZL99tsvv/zlL8c9z7/4i7/IK17xihx55JE5/vjjs++++yZJ7r777hx44IG59tprU1V56KGHHjftj3/845x66qk56qijkiQPPvhgbrzxxiTJbrvtltVWa7//vNlmm+WGG27I8573vCVf+R5xpw0AABhR0zQ58MADM2fOnMyZMyfXXHNNjjjiiFTViOMvv/zyeeSRRxZO+4c//OFx46y33npZc801c+mll+a4447L/vvvnyT54Ac/mJe+9KW5/PLL873vfS8PPvjgiOU58cQTF5bnxhtvzKabbpokWXHFFReOt9xyy2XBggUTXv++ENoAAIAR7bbbbjnhhBNy2223JUl+97vfZeutt84ZZ5yRO++8Mw899FD+8z//c+H406ZNy+zZs5Mkp5xyyoh3y5K2ieQ///M/5+67784WW7RPxLz77ruz3nrrJWm/ozaSP//zP8/nP//5NE2TJLnkkkvGXIcVVlhh1HI8VWgeCQAATxFP1iP6h2y22Wb56Ec/mle+8pV55JFHssIKK+Rf//Vfc8QRR2SnnXbKc5/73GyzzTZ5+OGHkyRve9vbstdee2X77bfPbrvtlpVXXnnE+b7+9a/Pe97znnzwgx9c2O/9739/DjzwwHzqU5/Ky172shGn++AHP5j3vve92XLLLdM0TaZNmzbmTwEccsgh2XLLLbPNNts8Zb/XVkMpddQRqr6W5LVJbmuaZvNhw/46yf9LsnbTNHdUe5/0s0lenWR+koOapvn5WIWYPn16M2vWrCVcBXrHbykBTwR1C7AMuuqqqxY2/+urmTNnZtasWfnCF76wtIvylDHSfq2q2U3TTB9p/PE0j5yZZPfhPavqeUlekeTGgd6vSrJJ93dIkn8fV6kBAAAY0ZihrWmas5L8boRBn07y/iSDt+r2SvL1pnVBktWr6rmTUlIAAKB3DjroIHfZnmBL9CCSqtozyW+bpvnFsEHrJblpoHtu12+keRxSVbOqatbtt9++JMUAAICnvbG+zsRTy5Lsz8UObVX1zCR/n+RDIw0eod+IpWqa5ktN00xvmmb60G88AAAAj5o6dWruvPNOwe1pomma3HnnnZk6depiTbckT4/cOMmGSX7R/T7D+kl+XlXbp72zNvgLdusnuXkJlgEAAMu89ddfP3Pnzo2WaU8fU6dOzfrrr79Y0yx2aGua5rIk6wx1V9X1SaZ3T488Ncm7quo7SXZIcnfTNLcs7jIAAID2N8Y23HDDpV0MlrIxm0dW1beTnJ/kT6tqblW9dRGj/zDJr5Ncl+TLSd4xKaUEAABYRo15p61pmgPGGD5t4HWT5J0TLxYAAADJEj49EgAAgCeH0AYAANBjQhsAAECPCW0AAAA9JrQBAAD0mNAGAADQY0IbAABAjwltAAAAPSa0AQAA9JjQBgAA0GNCGwAAQI8tv7QLAAAAS80Rqy3tEiy5I+5e2iXgSeJOGwAAQI8JbQAAAD0mtAEAAPSY0AYAANBjQhsAAECPCW0AAAA9JrQBAAD0mNAGAADQY0IbAABAjwltAAAAPSa0AQAA9JjQBgAA0GNCGwAAQI8JbQAAAD0mtAEAAPSY0AYAANBjQhsAAECPCW0AAAA9JrQBAAD0mNAGAADQY0IbAABAjwltAAAAPSa0AQAA9JjQBgAA0GNCGwAAQI8JbQAAAD0mtAEAAPSY0AYAANBjQhsAAECPCW0AAAA9JrQBAAD0mNAGAADQY0IbAABAjwltAAAAPSa0AQAA9JjQBgAA0GNCGwAAQI8JbQAAAD0mtAEAAPSY0AYAANBjQhsAAECPCW0AAAA9JrQBAAD0mNAGAADQY0IbAABAjwltAAAAPSa0AQAA9JjQBgAA0GNCGwAAQI+NGdqq6mtVdVtVXT7Q7/9V1dVVdWlVnVRVqw8M+7uquq6qrqmqP3+iCg4AALAsGM+dtplJdh/W77QkmzdNs2WSXyb5uySpqs2S7J/khd00/1ZVy01aaQEAAJYxY4a2pmnOSvK7Yf1+3DTNgq7zgiTrd6/3SvKdpml+3zTNb5Jcl2T7SSwvAADAMmUyvtP2liQ/6l6vl+SmgWFzu36PU1WHVNWsqpp1++23T0IxAAAAnn4mFNqq6u+TLEjyraFeI4zWjDRt0zRfappmetM009dee+2JFAMAAOBpa/klnbCqDkzy2iS7NU0zFMzmJnnewGjrJ7l5yYsHAACwbFuiO21VtXuSv02yZ9M08wcGnZpk/6pasao2TLJJkosmXkwAAIBl05h32qrq20lmJFmrquYm+XDap0WumOS0qkqSC5qmObRpmiuq6vgkV6ZtNvnOpmkefqIKDwAA8HQ3ZmhrmuaAEXp/dRHjfyzJxyZSKAAAAFqT8fRIAAAAniBCGwAAQI8JbQAAAD0mtAEAAPSY0AYAANBjQhsAAECPCW0AAAA9JrQBAAD02Jg/rg0AAGOZdvgPlnYRlsj1U5d2CWBs7rQBAAD0mNAGAADQY0IbAABAjwltAAAAPSa0AQAA9JjQBgAA0GNCGwAAQI8JbQAAAD0mtAEAAPSY0AYAANBjQhsAAECPCW0AAAA9JrQBAAD0mNAGAADQY0IbAABAjwltAAAAPSa0AQAA9JjQBgAA0GNCGwAAQI8JbQAAAD0mtAEAAPSY0AYAANBjQhsAAECPCW0AAAA9JrQBAAD0mNAGAADQY0IbAABAjwltAAAAPSa0AQAA9JjQBgAA0GNCGwAAQI8JbQAAAD0mtAEAAPSY0AYAANBjQhsAAECPCW0AAAA9JrQBAAD0mNAGAADQY0IbAABAjwltAAAAPSa0AQAA9JjQBgAA0GNCGwAAQI8JbQAAAD0mtAEAAPSY0AYAANBjQhsAAECPCW0AAAA9JrQBAAD0mNAGAADQY0IbAABAjwltAAAAPSa0AQAA9NiYoa2qvlZVt1XV5QP9nl1Vp1XVtd3/Nbr+VVWfq6rrqurSqtrmiSw8AADA09147rTNTLL7sH6HJ/lp0zSbJPlp150kr0qySfd3SJJ/n5xiAgAALJvGDG1N05yV5HfDeu+V5Jju9TFJ9h7o//WmdUGS1avquZNVWAAAgGXNkn6n7TlN09ySJN3/dbr+6yW5aWC8uV2/x6mqQ6pqVlXNuv3225ewGAAAAE9vk/0gkhqhXzPSiE3TfKlpmulN00xfe+21J7kYAAAATw9LGtpuHWr22P2/res/N8nzBsZbP8nNS148AACAZduShrZTkxzYvT4wySkD/d/UPUVyxyR3DzWjBAAAYPEtP9YIVfXtJDOSrFVVc5N8OMknkhxfVW9NcmOSfbvRf5jk1UmuSzI/yZufgDIDAAAsM8YMbU3THDDKoN1GGLdJ8s6JFgoAAIDWZD+IBAAAgEkktAEAAPSY0AYAANBjQhsAAECPCW0AAAA9JrQBAAD0mNAGAADQY0IbAABAjwltAAAAPSa0AQAA9JjQBgAA0GNCGwAAQI8JbQAAAD0mtAEAAPSY0AYAANBjyy/tAgDw5Jp2+A+WdhGWyPVTl3YJAGDpcKcNAACgx4Q2AACAHhPaAAAAekxoAwAA6DGhDQAAoMeENgAAgB4T2gAAAHpMaAMAAOgxoQ0AAKDHhDYAAIAeE9oAAAB6TGgDAADoMaENAACgx4Q2AACAHhPaAAAAekxoAwAA6DGhDQAAoMeENgAAgB4T2gAAAHpMaAMAAOgxoQ0AAKDHhDYAAIAeE9oAAAB6TGgDAADoMaENAACgx4Q2AACAHhPaAAAAekxoAwAA6DGhDQAAoMeENgAAgB4T2gAAAHpMaAMAAOgxoQ0AAKDHhDYAAIAeE9oAAAB6TGgDAADoMaENAACgx4Q2AACAHhPaAAAAekxoAwAA6DGhDQAAoMeWX9oFYHTTDv/B0i7CErl+6tIuAQAAPH240wYAANBjQhsAAECPCW0AAAA9NqHQVlWHVdUVVXV5VX27qqZW1YZVdWFVXVtVx1XVMyarsAAAAMuaJQ5tVbVekncnmd40zeZJlkuyf5JPJvl00zSbJLkryVsno6AAAADLook2j1w+yUpVtXySZya5JcnLkpzQDT8myd4TXAYAAMAya4lDW9M0v01yVJIb04a1u5PMTjKvaZoF3Whzk6w30vRVdUhVzaqqWbfffvuSFgMAAOBpbSLNI9dIsleSDZOsm2TlJK8aYdRmpOmbpvlS0zTTm6aZvvbaay9pMQAAAJ7WJtI88uVJftM0ze1N0zyU5LtJdk6yetdcMknWT3LzBMsIAACwzJpIaLsxyY5V9cyqqiS7JbkyyelJXt+Nc2CSUyZWRAAAgGXXRL7TdmHaB478PMll3by+lORvk/zfqrouyZpJvjoJ5QQAAFgmLT/2KKNrmubDST48rPevk2w/kfkCAADQmugj/wEAAHgCCW0AAAA9JrQBAAD0mNAGAADQY0IbAABAjwltAAAAPSa0AQAA9JjQBgAA0GNCGwAAQI8JbQAAAD0mtAEAAPSY0AYAANBjQhsAAECPCW0AAAA9JrQBAAD0mNAGAADQY0IbAABAjwltAAAAPSa0AQAA9JjQBgAA0GNCGwAAQI8JbQAAAD0mtAEAAPSY0AYAANBjQhsAAECPCW0AAAA9JrQBAAD0mNAGAADQY0IbAABAjwltAAAAPSa0AQAA9JjQBgAA0GNCGwAAQI8JbQAAAD0mtAEAAPSY0AYAANBjQhsAAECPCW0AAAA9JrQBAAD0mNAGAADQY0IbAABAjwltAAAAPSa0AQAA9JjQBgAA0GNCGwAAQI8JbQAAAD0mtAEAAPSY0AYAANBjQhsAAECPCW0AAAA9JrQBAAD0mNAGAADQY0IbAABAjwltAAAAPSa0AQAA9JjQBgAA0GNCGwAAQI8JbQAAAD0mtAEAAPSY0AYAANBjQhsAAECPTSi0VdXqVXVCVV1dVVdV1U5V9eyqOq2qru3+rzFZhQUAAFjWTPRO22eT/FfTNC9I8mdJrkpyeJKfNk2zSZKfdt0AAAAsgSUObVX1rCS7JvlqkjRN84emaeYl2SvJMd1oxyTZe6KFBAAAWFZN5E7bRkluT3J0VV1SVV+pqpWTPKdpmluSpPu/zkgTV9UhVTWrqmbdfvvtEygGAADA09dEQtvySbZJ8u9N02yd5P4sRlPIpmm+1DTN9KZppq+99toTKAYAAMDT10RC29wkc5umubDrPiFtiLu1qp6bJN3/2yZWRAAAgGXXEoe2pmn+J8lNVfWnXa/dklyZ5NQkB3b9DkxyyoRKCAAAsAxbfoLT/58k36qqZyT5dZI3pw2Cx1fVW5PcmGTfCS4DAABgmTWh0NY0zZwk00cYtNtE5gsAAEBror/TBgAAwBNIaAMAAOgxoQ0AAKDHhDYAAIAeE9oAAAB6TGgDAADoMaENAACgx4Q2AACAHhPaAAAAekxoAwAA6DGhDQAAoMeENgAAgB4T2gAAAHpMaAMAAOgxoQ0AAKDHhDYAAIAeE9oAAAB6TGgDAADoMaENAACgx4Q2AACAHhPaAAAAekxoAwAA6DGhDQAAoMeENgAAgB4T2gAAAHpMaAMAAOgxoQ0AAKDHhDYAAIAeE9oAAAB6TGgDAADoMaENAACgx4Q2AACAHhPaAAAAekxoAwAA6DGhDQAAoMeENgAAgB4T2gAAAHpMaAMAAOgxoQ0AAKDHhDYAAIAeE9oAAAB6TGgDAADoMaENAACgx4Q2AACAHhPaAAAAekxoAwAA6DGhDQAAoMeENgAAgB4T2gAAAHpMaAMAAOgxoQ0AAKDHhDYAAIAeE9oAAAB6TGgDAADoMaENAACgx4Q2AACAHhPaAAAAekxoAwAA6DGhDQAAoMeENgAAgB4T2gAAAHpMaAMAAOixCYe2qlquqi6pqu933RtW1YVVdW1VHVdVz5h4MQEAAJZNk3Gn7T1Jrhro/mSSTzdNs0mSu5K8dRKWAQAAsEyaUGirqvWTvCbJV7ruSvKyJCd0oxyTZO+JLAMAAGBZNtE7bZ9J8v4kj3TdayaZ1zTNgq57bpL1Rpqwqg6pqllVNev222+fYDEAAACenpY4tFXVa5Pc1jTN7MHeI4zajDR90zRfappmetM009dee+0lLQYAAMDT2vITmPZFSfasqlcnmZrkWWnvvK1eVct3d9vWT3LzxIsJAACwbFriO21N0/xd0zTrN00zLcn+SX7WNM0bk5ye5PXdaAcmOWXCpQQAAFhGPRG/0/a3Sf5vVV2X9jtuX30ClgEAALBMmEjzyIWapjkjyRnd618n2X4y5gsAALCseyLutAEAADBJhDYAAIAeE9oAAAB6TGgDAADoMaENAACgx4Q2AACAHhPaAAAAekxoAwAA6DGhDQAAoMeENgAAgB4T2gAAAHpMaAMAAOgxoQ0AAKDHhDYAAIAeE9oAAAB6TGgDAADoMaENAACgx4Q2AACAHhPaAAAAekxoAwAA6DGhDQAAoMeENgAAgB4T2gAAAHpMaAMAAOgxoQ0AAKDHhDYAAIAeE9oAAAB6TGgDAADoMaENAACgx4Q2AACAHhPaAAAAekxoAwAA6DGhDQAAoMeENgAAgB4T2gAAAHpMaAMAAOgxoQ0AAKDHhDYAAIAeE9oAAAB6TGgDAADoMaENAACgx4Q2AACAHhPaAAAAekxoAwAA6DGhDQAAoMeENgAAgB4T2gAAAHpMaAMAAOgxoQ0AAKDHhDYAAIAeE9oAAAB6TGgDAADoMaENAACgx4Q2AACAHhPaAAAAekxoAwAA6DGhDQAAoMeENgAAgB4T2gAAAHpMaAMAAOgxoQ0AAKDHlji0VdXzqur0qrqqqq6oqvd0/Z9dVadV1bXd/zUmr7gAAADLloncaVuQ5H1N02yaZMck76yqzZIcnuSnTdNskuSnXTcAAABLYIlDW9M0tzT/v727DbXsqs8A/jwd40ttCQVHsb5kUhqVpNpRxpRWbUVbqR/qS7VODUgEMa2oYKDQoNASCjVYqGibqqktTMFQxEYN/WAs6ge12jiOqZP4gpI0vgWN0Fot0pDJ6oez09xeOrnx3nvuLO/8fjCcs/dea+11huE/+zl7nXPGOLE8/36SLyZ5TJIXJjm2NDuW5EU7nSQAAMDZalc+09b2UJKnJvmXJI8aY9yRrIJdkkeeps9lbY+3PX7nnXfuxjQAAAD2nR2HtrY/leQfkrxhjPGfD7TfGOOaMcaRMcaRgwcP7nQaAAAA+9KOQlvbc7IKbO8ZY1y37P5220cvxx+d5Ds7myIAAMDZayffHtkkf5Pki2OMP99w6Pokly7PL03ywe1PDwAA4Oz2oB30fUaSVyQ52famZd8bk1yV5L1tX5Xka0l+Z2dTBAAAOHttO7SNMT6RpKc5/NztjgsAAMB9duXbIwEAAFgPoQ0AAGBiQhsAAMDEhDYAAICJCW0AAAATE9oAAAAmJrQBAABMTGgDAACYmNAGAAAwMaENAABgYkIbAADAxIQ2AACAiQltAAAAExPaAAAAJia0AQAATExoAwAAmJjQBgAAMDGhDQAAYGJCGwAAwMSENgAAgIkJbQAAABMT2gAAACYmtAEAAExMaAMAAJiY0AYAADAxoQ0AAGBiQhsAAMDEhDYAAICJCW0AAAATE9oAAAAmJrQBAABMTGgDAACYmNAGAAAwMaENAABgYkIbAADAxIQ2AACAiQltAAAAExPaAAAAJia0AQAATExoAwAAmJjQBgAAMDGhDQAAYGJCGwAAwMSENgAAgIkJbQAAABMT2gAAACYmtAEAAExMaAMAAJiY0AYAADAxoQ0AAGBiQhsAAMDEhDYAAICJCW0AAAATE9oAAAAmJrQBAABMTGgDAACYmNAGAAAwMaENAABgYkIbAADAxIQ2AACAiQltAAAAExPaAAAAJra20Nb2N9t+ue1X216xrvMAAADsZ2sJbW0PJLk6yfOTXJjk5W0vXMe5AAAA9rN13Wm7OMlXxxi3jjHuSvL3SV64pnMBAADsWw9a07iPSfL1DdvfSPJLGxu0vSzJZcvmD9p+eU1zYY91/ad4RJLvrmXkK/dg9sC2qC3AOqgtTOS80x1YV2j7//4Fjf+zMcY1Sa5Z0/nZx9oeH2McOdPzAPYXtQVYB7WF3bCu5ZHfSPK4DduPTfKtNZ0LAABg31pXaPtMkgvant/2wUl+N8n1azoXAADAvrWW5ZFjjLvbvi7JDUkOJPnbMcYt6zgXZyXLaoF1UFuAdVBb2LGOMbZuBQAAwBmxth/XBgAAYOeENgAAgIkJbeyptqfa3rThzxU7GOsHy+PPtn3f/bQ71Pbm7Z4HmFfbR7W9tu2tbT/b9lNtX3ym5wXsLxuuOQ61veQBtP/fa4+2R9q+fd1zZH9b1++0wen8cIxxeDcHHGN8K8lLd3NMYH5tm+QDSY6NMS5Z9p2X5AUPsP+BMcapNU4R2H8OJbkkybUPtMMY43iS4+uaEGcHd9qYQtt/a3tl2xNtT7Z90rL/YNt/Wva/q+3tbR+xqe/Gd7Muanvjchfv820vWJodaPvXbW9p++G2D9vjlwjsvuckuWuM8c57d4wxbh9j/EXbA23/rO1nllrwe0nS9tltP9b22iQnl/rxpbbvbntz2/e0/fW2n2z7lbYXL/0ubvvPbT+3PD5x2f/Ktte1/dDS/i1n4i8C2DNXJXnWcp1x+VJDPr5cp5xo+yubOyx15x+X52oJ2yK0sdcetml55NENx747xnhaknck+YNl3x8n+eiy//1JHr/F+L+f5G3L3bwjWf3Qe5JckOTqMcZFSf4jyUt26fUAZ85FSU6c5tirknxvjPH0JE9P8uq25y/HLk7ypjHGhcv2zyd5W5KnJHlSVu+iPzOrOvTGpc2XkvzqGOOpSf4oyZ9uONfhJEeTPDnJ0baP24XXBszpiiQfH2McHmO8Ncl3kvzGcp1yNMlWyyDVErbF8kj22v0tj7xuefxskt9enj8zyYuTZIzxobb/vsX4n0rypraPTXLdGOMrqxVUuW2McdOG8Q9tc/7ApNpenVXNuCvJ7Ume0vbepdPnZvXmzV1Jbhxj3Lah621jjJPLGLck+cgYY7Q9mftqxblJji1370eSczb0/8gY43tL/y8kOS/J19fwEoH5nJPkL9seTnIqyRO2aK+WsC3utDGT/14eT+W+NxT6owwwxrg2q8+z/DDJDW2fs2nszeMDP75uSfK0ezfGGK9N8twkB7OqHa9f3g0/PMY4f4zx4aXpf20aZ2N9uGfD9j25r1b8SZKPjTF+IclvJXnoafqrL3B2uTzJt5P8YlYrfB68RXu1hG0R2pjdJ5K8LEnaPi/Jz9xf47Y/l+TWMcbbk1yf1XInYH/6aJKHtn3Nhn0/uTzekOQ1bc9JkrZPaPvwHZzr3CTfXJ6/cgfjAD/evp/kpzdsn5vkjjHGPUlekeTAFv3VErZFaGOvbf5M21VbtL8yyfPankjy/CR3ZFUwT+dokpvb3pTVZ1P+bldmDUxnjDGSvCjJr7W9re2NSY4l+cMk707yhSQnli8qeld29q71W5K8ue0ns/VFGbB/fT7J3W3/te3lSf4qyaVtP53V0sjNzScA3QAAAGRJREFUd/I3U0vYlq7+z4M5tX1IklNjjLvb/nKSd+z2TwYAAMDMrJVldo9P8t62P5HVFwi8+gzPBwAA9pQ7bQAAABPzmTYAAICJCW0AAAATE9oAAAAmJrQBAABMTGgDAACY2P8A6kS1Aw+RooAAAAAASUVORK5CYII=\n",
      "text/plain": [
       "<Figure size 1080x720 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "stats_v2.plot(kind='bar', rot = 0, figsize=(15,10),\n",
    "           title = 'Performance of ngram based model wrt base model with rescaled base scores (without subwords information)')"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.6"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}