word2vec_baseline.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import numpy as np\n",
    "import pandas as pd\n",
    "import seaborn as sns\n",
    "import matplotlib.pyplot as plt"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "df_selected = pd.read_csv('rusentiment/Dataset/rusentiment_preselected_posts.csv')\n",
    "df_random = pd.read_csv('rusentiment/Dataset/rusentiment_random_posts.csv')\n",
    "df_test = pd.read_csv('rusentiment/Dataset/rusentiment_test.csv')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Preselected length:6950\n",
      "Random length:21268\n",
      "Test length:2967\n"
     ]
    }
   ],
   "source": [
    "print(f'Preselected length:{len(df_selected)}')\n",
    "print(f'Random length:{len(df_random)}')\n",
    "print(f'Test length:{len(df_test)}')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "df_train = pd.concat((df_selected, df_random))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAtUAAAHjCAYAAADseONGAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDIuMi4zLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvIxREBQAAIABJREFUeJzt3X+0XWV95/HPVxIJESQQIguJbahlLIoWIQtBbEeLRcAuwSkq/kTHTmyhVp2xNUyn4o/SiaOjDp0BSjUVq2gp6oIWWkCEalXEBBGBoInKSMSRFAgFERX6zB9nR69w8+s+Sc5NeL3Wuuvs8+y9z3ku93DOm80+51RrLQAAwNQ9atwTAACA7Z2oBgCATqIaAAA6iWoAAOgkqgEAoJOoBgCATqIaAAA6iWoAAOgkqgEAoNOMcU9gqvbaa6+2YMGCcU8DAIAd2PLly/+ltTZvY9ttt1G9YMGCLFu2bNzTAABgB1ZV/3dTtnP6BwAAdBLVAADQSVQDAECn7facagAAtr6f/OQnWb16de6///5xT2WrmjVrVubPn5+ZM2dOaX9RDQDAeq1evTq77bZbFixYkKoa93S2itZa7rjjjqxevTr77bfflG7D6R8AAKzX/fffn7lz5+6wQZ0kVZW5c+d2HY0X1QAAbNCOHNTr9P6OohoAADo5pxoAgE22YPHFW/T2blny/A2uX7t2bc4777ycfPLJm3W7xx57bM4777zMmTOnZ3qbzJFqAACmrbVr1+bMM8982PiDDz64wf0uueSSbRbUiSPVAABMY4sXL843v/nNHHTQQZk5c2Z23XXX7LPPPrnuuuty00035fjjj8+tt96a+++/P294wxuyaNGiJMmCBQuybNmy3HvvvTnmmGPyrGc9K1/4whey77775sILL8wuu+yyRefpSDUAANPWkiVL8sQnPjHXXXdd3v3ud+eaa67J6aefnptuuilJsnTp0ixfvjzLli3LGWeckTvuuONht7Fy5cqccsopufHGGzNnzpx84hOf2OLzdKQaAIDtxqGHHvpznyV9xhln5FOf+lSS5NZbb83KlSszd+7cn9tnv/32y0EHHZQkOeSQQ3LLLbds8XmJagAAthuPecxjfrp81VVX5dOf/nS++MUvZvbs2Xn2s5896WdN77zzzj9d3mmnnfLDH/5wi8/L6R8AAExbu+22W+65555J1919993ZY489Mnv27Nx88825+uqrt/HsfsaRagAANtnGPgJvS5s7d26OOOKIHHjggdlll12y9957/3Td0UcfnbPPPjtPe9rT8qQnPSmHHXbYNp3bRNVaG9ud91i4cGFbtmzZuKcBALBDW7FiRQ444IBxT2ObmOx3rarlrbWFG9vX6R8AANBJVAMAQCfnVHfY0l/TOVXb+twmAAB+niPVAADQSVQDAEAnUQ0AAJ2cUw0AwKZ72+5b+Pbu3uDqtWvX5rzzzsvJJ5+82Tf9/ve/P4sWLcrs2bOnOrtN5kg1AADT1tq1a3PmmWdOad/3v//9ue+++7bwjCbnSDUAANPW4sWL881vfjMHHXRQfvM3fzOPe9zjcv755+dHP/pRXvjCF+btb397fvCDH+TFL35xVq9enQcffDB/8id/ku9///u57bbb8pznPCd77bVXrrzyyq06T1ENAMC0tWTJktxwww257rrrctlll+WCCy7INddck9ZaXvCCF+Szn/1s1qxZk8c//vG5+OLRxx3ffffd2X333fPe9743V155Zfbaa6+tPk+nfwAAsF247LLLctlll+XpT396Dj744Nx8881ZuXJlnvrUp+bTn/503vKWt+Rzn/tcdt99C5/3vQkcqQYAYLvQWsupp56a173udQ9bt3z58lxyySU59dRTc9RRR+Wtb33rNp2bI9UAAExbu+22W+65554kyfOe97wsXbo09957b5Lku9/9bm6//fbcdtttmT17dl7xilfkzW9+c6699tqH7bu1OVINAMCm28hH4G1pc+fOzRFHHJEDDzwwxxxzTF72spfl8MMPT5Lsuuuu+chHPpJVq1blD//wD/OoRz0qM2fOzFlnnZUkWbRoUY455pjss88+W/2NitVa26p3sLUsXLiwLVu2bKxzWLD44rHe/zq3LHn+uKcAAOygVqxYkQMOOGDc09gmJvtdq2p5a23hxvZ1+gcAAHQS1QAA0ElUAwCwQdvr6cKbo/d3FNUAAKzXrFmzcscdd+zQYd1ayx133JFZs2ZN+TZ8+gcAAOs1f/78rF69OmvWrBn3VLaqWbNmZf78+VPeX1QDALBeM2fOzH777TfuaUx7Tv8AAIBOohoAADqJagAA6CSqAQCgk6gGAIBOohoAADqJagAA6CSqAQCgk6gGAIBOohoAADqJagAA6CSqAQCgk6gGAIBOohoAADqJagAA6CSqAQCgk6gGAIBOohoAADqJagAA6CSqAQCgk6gGAIBOohoAADqJagAA6CSqAQCgk6gGAIBOG43qqlpaVbdX1Q0TxvasqsurauVwuccwXlV1RlWtqqrrq+rgCfucNGy/sqpOmjB+SFV9bdjnjKqqLf1LAgDA1rQpR6o/lOToh4wtTnJFa23/JFcM15PkmCT7Dz+LkpyVjCI8yWlJnpHk0CSnrQvxYZtFE/Z76H0BAMC0ttGobq19NsmdDxk+Lsm5w/K5SY6fMP7hNnJ1kjlVtU+S5yW5vLV2Z2vtriSXJzl6WPfY1toXW2styYcn3BYAAGwXpnpO9d6tte8lyXD5uGF83yS3Tthu9TC2ofHVk4xPqqoWVdWyqlq2Zs2aKU4dAAC2rC39RsXJzoduUxifVGvtnNbawtbawnnz5k1xigAAsGVNNaq/P5y6keHy9mF8dZInTNhufpLbNjI+f5JxAADYbkw1qi9Ksu4TPE5KcuGE8VcNnwJyWJK7h9NDLk1yVFXtMbxB8agklw7r7qmqw4ZP/XjVhNsCAIDtwoyNbVBVH0vy7CR7VdXqjD7FY0mS86vqtUm+k+RFw+aXJDk2yaok9yV5TZK01u6sqncm+fKw3Ttaa+ve/Ph7GX3CyC5J/mH4AQCA7cZGo7q19tL1rDpykm1bklPWcztLkyydZHxZkgM3Ng8AAJiufKMiAAB0EtUAANBJVAMAQCdRDQAAnUQ1AAB0EtUAANBJVAMAQCdRDQAAnUQ1AAB0EtUAANBJVAMAQCdRDQAAnUQ1AAB0EtUAANBJVAMAQCdRDQAAnUQ1AAB0EtUAANBJVAMAQCdRDQAAnUQ1AAB0EtUAANBJVAMAQCdRDQAAnUQ1AAB0EtUAANBJVAMAQCdRDQAAnUQ1AAB0EtUAANBJVAMAQCdRDQAAnUQ1AAB0EtUAANBJVAMAQCdRDQAAnUQ1AAB0EtUAANBJVAMAQCdRDQAAnUQ1AAB0EtUAANBJVAMAQCdRDQAAnUQ1AAB0EtUAANBJVAMAQCdRDQAAnUQ1AAB0EtUAANBJVAMAQCdRDQAAnUQ1AAB0EtUAANBJVAMAQCdRDQAAnUQ1AAB0EtUAANBJVAMAQCdRDQAAnUQ1AAB0EtUAANBJVAMAQCdRDQAAnbqiuqreVFU3VtUNVfWxqppVVftV1ZeqamVV/U1VPXrYdufh+qph/YIJt3PqMP71qnpe368EAADb1pSjuqr2TfIHSRa21g5MslOSE5O8K8n7Wmv7J7kryWuHXV6b5K7W2i8ned+wXarqycN+T0lydJIzq2qnqc4LAAC2td7TP2Yk2aWqZiSZneR7SX4jyQXD+nOTHD8sHzdcz7D+yKqqYfzjrbUftda+nWRVkkM75wUAANvMlKO6tfbdJO9J8p2MYvruJMuTrG2tPTBstjrJvsPyvkluHfZ9YNh+7sTxSfYBAIBpr+f0jz0yOsq8X5LHJ3lMkmMm2bSt22U969Y3Ptl9LqqqZVW1bM2aNZs/aQAA2Ap6Tv94bpJvt9bWtNZ+kuSTSZ6ZZM5wOkiSzE9y27C8OskTkmRYv3uSOyeOT7LPz2mtndNaW9haWzhv3ryOqQMAwJbTE9XfSXJYVc0ezo0+MslNSa5McsKwzUlJLhyWLxquZ1j/mdZaG8ZPHD4dZL8k+ye5pmNeAACwTc3Y+CaTa619qaouSHJtkgeSfCXJOUkuTvLxqvrTYeyDwy4fTPLXVbUqoyPUJw63c2NVnZ9RkD+Q5JTW2oNTnRcAAGxrU47qJGmtnZbktIcMfyuTfHpHa+3+JC9az+2cnuT0nrkAAMC4+EZFAADoJKoBAKCTqAYAgE6iGgAAOolqAADoJKoBAKCTqAYAgE6iGgAAOolqAADoJKoBAKCTqAYAgE6iGgAAOolqAADoJKoBAKCTqAYAgE6iGgAAOolqAADoJKoBAKCTqAYAgE6iGgAAOolqAADoJKoBAKCTqAYAgE6iGgAAOolqAADoJKoBAKCTqAYAgE6iGgAAOolqAADoJKoBAKCTqAYAgE6iGgAAOolqAADoJKoBAKCTqAYAgE6iGgAAOolqAADoJKoBAKCTqAYAgE6iGgAAOolqAADoJKoBAKCTqAYAgE6iGgAAOolqAADoJKoBAKCTqAYAgE6iGgAAOolqAADoJKoBAKCTqAYAgE6iGgAAOolqAADoJKoBAKCTqAYAgE6iGgAAOolqAADoJKoBAKCTqAYAgE6iGgAAOolqAADoJKoBAKCTqAYAgE6iGgAAOs0Y9wRgR7Ng8cXjnkJuWfL8cU8BAB5Ruo5UV9Wcqrqgqm6uqhVVdXhV7VlVl1fVyuFyj2HbqqozqmpVVV1fVQdPuJ2Thu1XVtVJvb8UAABsS72nf/yvJP/YWvuVJL+aZEWSxUmuaK3tn+SK4XqSHJNk/+FnUZKzkqSq9kxyWpJnJDk0yWnrQhwAALYHU47qqnpskl9P8sEkaa39uLW2NslxSc4dNjs3yfHD8nFJPtxGrk4yp6r2SfK8JJe31u5srd2V5PIkR091XgAAsK31HKn+pSRrkvxVVX2lqj5QVY9Jsndr7XtJMlw+bth+3yS3Tth/9TC2vvGHqapFVbWsqpatWbOmY+oAALDl9ET1jCQHJzmrtfb0JD/Iz071mExNMtY2MP7wwdbOaa0tbK0tnDdv3ubOFwAAtoqeqF6dZHVr7UvD9QsyiuzvD6d1ZLi8fcL2T5iw//wkt21gHAAAtgtTjurW2v9LcmtVPWkYOjLJTUkuSrLuEzxOSnLhsHxRklcNnwJyWJK7h9NDLk1yVFXtMbxB8ahhDAAAtgu9n1P9+iQfrapHJ/lWktdkFOrnV9Vrk3wnyYuGbS9JcmySVUnuG7ZNa+3Oqnpnki8P272jtXZn57wAAGCb6Yrq1tp1SRZOsurISbZtSU5Zz+0sTbK0Zy4AADAuvqYcAAA6iWoAAOgkqgEAoJOoBgCATqIaAAA6iWoAAOgkqgEAoJOoBgCATqIaAAA6iWoAAOgkqgEAoJOoBgCATqIaAAA6iWoAAOgkqgEAoJOoBgCATqIaAAA6iWoAAOgkqgEAoJOoBgCATqIaAAA6iWoAAOgkqgEAoJOoBgCATqIaAAA6iWoAAOgkqgEAoJOoBgCATqIaAAA6iWoAAOgkqgEAoJOoBgCATqIaAAA6iWoAAOgkqgEAoJOoBgCATqIaAAA6iWoAAOgkqgEAoJOoBgCATqIaAAA6iWoAAOgkqgEAoJOoBgCATqIaAAA6iWoAAOgkqgEAoJOoBgCATqIaAAA6iWoAAOgkqgEAoJOoBgCATqIaAAA6iWoAAOgkqgEAoJOoBgCATqIaAAA6iWoAAOgkqgEAoJOoBgCATqIaAAA6iWoAAOgkqgEAoJOoBgCATjN6b6CqdkqyLMl3W2u/VVX7Jfl4kj2TXJvkla21H1fVzkk+nOSQJHckeUlr7ZbhNk5N8tokDyb5g9bapb3zAphOFiy+eNxTyC1Lnj/uKQDssLbEkeo3JFkx4fq7kryvtbZ/krsyiuUMl3e11n45yfuG7VJVT05yYpKnJDk6yZlDqAMAwHahK6qran6S5yf5wHC9kvxGkguGTc5NcvywfNxwPcP6I4ftj0vy8dbaj1pr306yKsmhPfMCAIBtqfdI9fuT/FGSfxuuz02ytrX2wHB9dZJ9h+V9k9yaJMP6u4ftfzo+yT4/p6oWVdWyqlq2Zs2azqkDAMCWMeWorqrfSnJ7a235xOFJNm0bWbehfX5+sLVzWmsLW2sL582bt1nzBQCAraXnjYpHJHlBVR2bZFaSx2Z05HpOVc0YjkbPT3LbsP3qJE9IsrqqZiTZPcmdE8bXmbgPAABMe1M+Ut1aO7W1Nr+1tiCjNxp+prX28iRXJjlh2OykJBcOyxcN1zOs/0xrrQ3jJ1bVzsMnh+yf5JqpzgsAALa17o/Um8Rbkny8qv40yVeSfHAY/2CSv66qVRkdoT4xSVprN1bV+UluSvJAklNaaw9uhXkBAMBWsUWiurV2VZKrhuVvZZJP72it3Z/kRevZ//Qkp2+JuQAAwLbmGxUBAKCTqAYAgE6iGgAAOolqAADoJKoBAKCTqAYAgE6iGgAAOm2NL38BADbBgsUXj3sKuWXJ88c9BdghOFINAACdRDUAAHQS1QAA0ElUAwBAJ1ENAACdRDUAAHQS1QAA0ElUAwBAJ1ENAACdRDUAAHQS1QAA0ElUAwBAJ1ENAACdRDUAAHQS1QAA0ElUAwBAJ1ENAACdRDUAAHQS1QAA0ElUAwBAJ1ENAACdZox7AgAA/MyCxRePewq5Zcnzxz2F7Y4j1QAA0ElUAwBAJ1ENAACdRDUAAHQS1QAA0ElUAwBAJ1ENAACdRDUAAHQS1QAA0ElUAwBAJ1ENAACdRDUAAHQS1QAA0ElUAwBAJ1ENAACdRDUAAHQS1QAA0ElUAwBAJ1ENAACdRDUAAHQS1QAA0ElUAwBAJ1ENAACdRDUAAHQS1QAA0ElUAwBAJ1ENAACdRDUAAHQS1QAA0ElUAwBAJ1ENAACdRDUAAHQS1QAA0ElUAwBApylHdVU9oaqurKoVVXVjVb1hGN+zqi6vqpXD5R7DeFXVGVW1qqqur6qDJ9zWScP2K6vqpP5fCwAAtp2eI9UPJPkvrbUDkhyW5JSqenKSxUmuaK3tn+SK4XqSHJNk/+FnUZKzklGEJzktyTOSHJrktHUhDgAA24MpR3Vr7XuttWuH5XuSrEiyb5Ljkpw7bHZukuOH5eOSfLiNXJ1kTlXtk+R5SS5vrd3ZWrsryeVJjp7qvAAAYFvbIudUV9WCJE9P8qUke7fWvpeMwjvJ44bN9k1y64TdVg9j6xuf7H4WVdWyqlq2Zs2aLTF1AADo1h3VVbVrkk8keWNr7V83tOkkY20D4w8fbO2c1trC1trCefPmbf5kAQBgK+iK6qqamVFQf7S19slh+PvDaR0ZLm8fxlcnecKE3ecnuW0D4wAAsF3o+fSPSvLBJCtaa++dsOqiJOs+weOkJBdOGH/V8CkghyW5ezg95NIkR1XVHsMbFI8axgAAYLswo2PfI5K8MsnXquq6Yey/JlmS5Pyqem2S7yR50bDukiTHJlmV5L4kr0mS1tqdVfXOJF8etntHa+3OjnkBAMA2NeWobq39cyY/HzpJjpxk+5bklPXc1tIkS6c6FwAAGCffqAgAAJ1ENQAAdBLVAADQSVQDAEAnUQ0AAJ1ENQAAdBLVAADQSVQDAEAnUQ0AAJ1ENQAAdBLVAADQSVQDAEAnUQ0AAJ1ENQAAdBLVAADQSVQDAECnGeOeAFvA23Yf9wySt9097hkAAIyNI9UAANBJVAMAQCdRDQAAnUQ1AAB0EtUAANBJVAMAQCdRDQAAnUQ1AAB0EtUAANBJVAMAQCdRDQAAnUQ1AAB0EtUAANBJVAMAQCdRDQAAnUQ1AAB0EtUAANBJVAMAQCdRDQAAnUQ1AAB0EtUAANBJVAMAQCdRDQAAnUQ1AAB0EtUAANBJVAMAQCdRDQAAnUQ1AAB0EtUAANBJVAMAQCdRDQAAnWaMewLAVvC23cc9g5G33T3uGQDANuFINQAAdBLVAADQSVQDAEAnUQ0AAJ28UREAgJ/nDe+bzZFqAADo5Eg1wCOFI08AW40j1QAA0ElUAwBAJ6d/AMAjmdOCYItwpBoAADqJagAA6CSqAQCgk6gGAIBOohoAADqJagAA6DRtorqqjq6qr1fVqqpaPO75AADAppoWUV1VOyX5P0mOSfLkJC+tqiePd1YAALBppkVUJzk0yarW2rdaaz9O8vEkx415TgAAsEmqtTbuOaSqTkhydGvtd4brr0zyjNba7z9ku0VJFg1Xn5Tk69t0otPXXkn+ZdyTYFrxmGAyHhdMxuOCyXhc/MwvttbmbWyj6fI15TXJ2MNqv7V2TpJztv50ti9Vtay1tnDc82D68JhgMh4XTMbjgsl4XGy+6XL6x+okT5hwfX6S28Y0FwAA2CzTJaq/nGT/qtqvqh6d5MQkF415TgAAsEmmxekfrbUHqur3k1yaZKckS1trN455WtsTp8TwUB4TTMbjgsl4XDAZj4vNNC3eqAgAANuz6XL6BwAAbLdENQAAdBLVO4CqWlBVL5vivvdu6fkwXlX1u1X1qmH51VX1+AnrPuDbSh85quqWqtprkvEvjGM+TH9VNaeqTp5w/fFVdcE458T2YX3PN48konrHsCDJpFFdVdPizahsO621s1trHx6uvjrJ4yes+53W2k1jmRjTRmvtmeOeA9PWnCQ/jerW2m2ttRPGOB/YbojqMRqOMK+oqr+sqhur6rKq2qWqnlhV/1hVy6vqc1X1K8P2Hxq+fXLd/uuOMi9J8mtVdV1VvWk4Ovm3VfV3SS6rql2r6oqquraqvlZVvgJ+mhoeEzdX1blVdX1VXVBVs6vqyKr6yvD3W1pVOw/bL6mqm4Zt3zOMva2q3jw8VhYm+ejw2Nilqq6qqoVV9XtV9T8m3O+rq+rPh+VXVNU1wz5/UVU7jeOfBZunqh5TVRdX1Ver6oaqesmEdbsMzyn/abh+73D57Kr6bFV9angcnV1VXhemsSm8bjyxqq6uqi9X1Tsm/O3X97qwJMkTh3//3z3c3w3DPl+qqqdMmMtVVXXI8NhbOtzHV7zGjN9kzwfDkeR3Dc/v11TVLw/bzquqTwx/vy9X1RETbuNhf9eq2qmq3jM8bq6vqtdPuOvXT3hM/coYfvXxaq35GdNPRkeYH0hy0HD9/CSvSHJFkv2HsWck+cyw/KEkJ0zY/97h8tlJ/n7C+Ksz+kKdPYfrM5I8dljeK8mq/OyTX+4d9z8HPw97TLQkRwzXlyb5b0luTfLvhrEPJ3ljkj2TfH3C33LOcPm2JG8elq9KsnDC7V+VUWjPS7Jqwvg/JHlWkgOS/F2SmcP4mUleNe5/Ln426bHz20n+csL13ZPcMjymPj3x7/iQ5477k/xSRh9nevnE5xg/0+9nCq8bf5/kpcPy707420/6ujDc/g0Pub8bhuU3JXn7sLxPkm8My3+W5BXD8pwk30jymHH/s3ok/2zg+eCPh+uvytANSc5L8qxh+ReSrNjQ3zXJ7yX5RJIZw7p1rXFLktcPyycn+cC4/zls6x9HJMbv262164bl5Rk9gT0zyd9W1XVJ/iKjJ6/NdXlr7c5huZL8WVVdn9GL675J9u6aNVvTra21zw/LH0lyZEaPk28MY+cm+fUk/5pREH2gqv5Dkvs29Q5aa2uSfKuqDququUmelOTzw30dkuTLw+PvyIyCi+nva0meOxyJ+rXW2t3D+IVJ/qr97JSgh7qmtfat1tqDST6W0X9cMb1tzuvG4Un+dlg+b8JtTOV14fwkLxqWXzzhdo9Ksni476uSzMoozhif9T0ffGzC5eHD8nOT/O/h73dRksdW1W5Z/9/1uUnObq09kCQTWiNJPjlcrntcPqI433b8fjRh+cGMntTWttYOmmTbBzKcslNVleTRG7jdH0xYfnlGRyYPaa39pKpuyehfDqanTfrw+Db60qRDMwrfE5P8fpLf2Iz7+ZuMXhhvTvKp1lobHlfnttZO3cw5M2attW9U1SFJjk3y36vqsmHV55McU1XnteEQ0kN33ch1pp/Ned1Yn81+XWitfbeq7qiqpyV5SZLXDasqyW+31r6+GffPVrSB54OJ/36vW35UksNbaz+ceBvD68HD/q7D+PqeJ9Y9Nh/MI7AxHamefv41yber6kXJ6MFbVb86rLslo6OISXJckpnD8j1JdtvAbe6e5PbhifM5SX5xi8+aLekXqmrdEYSXZnQUacG689+SvDLJP1XVrkl2b61dktHpIJO9oG7osfHJJMcP9/E3w9gVSU6oqsclSVXtWVUeL9uBGn3Ky32ttY8keU+Sg4dVb01yR0an8kzm0KrabziX+iVJ/nmrT5YtbUOvG1dndCpAMvqP73XW97qwsdeTjyf5o4yee742jF2a0bm0Ndz/03t/Ifps4PngJRMuvzgsX5bRQZl1+657LVnf3/WyJL9bwwchVNWeW+v32N6I6unp5UleW1VfTXJjRgGdJH+Z5N9X1TUZnTO37mj09UkeGN6Q8KZJbu+jSRZW1bLhtm/eqrOn14okJw3/W3bPJO9L8pqM/tfu15L8W5KzM3rh+/thu3/K6HzHh/pQkrOHNx3tMnFFa+2uJDcl+cXW2jXD2E0ZncN92XC7l2dqpx+x7T01yTXD/6r94yR/OmHdG5PMqglvTp3gixm9Oe2GJN9O8qmtPVG2ivW9brwxyX8eXjf2SbLuNIBJXxdaa3ck+fzw5rZ3T3I/F2QU5+dPGHtnRgd5rh/e1PjOLfqbMRXrez7Yuaq+lOQN+dlrxh9k9Fi4vqpuyujc+2T9f9cPJPnOMP7VrOfTxx6JfE05TCNVtSCjN48cOOap8AhQVc/O6E2tvzXuubB1VNXsJD8cTu86MaM3Lfp0jkeg4RSfha21fxn3XHZUj7jzXQDgEeSQjN6EVknWJvmPY54P7LAcqQYAgE7OqQYAgE6iGgAAOolqAADoJKoBtmNVde9G1i8YPg5rc27zQ1V1Qt/MAB5ZRDUAAHQS1QA7gKrataquqKprq+prVTXxs4hnVNW5w5c7XDB8dnGq6pCq+qeqWl5Vl1aVL/oBmCJRDbBjuD/JC1trByd5TpL/ue7rhZM8Kck5rbWnZfSV1idX1cwkf57khNbaIUmWJjl9DPMG2CH48heAHUO7IOpWAAAA50lEQVQl+bOq+vWMvsp+3yR7D+tuba19flj+SEZfS/yPSQ5McvnQ3jsl+d42nTHADkRUA+wYXp5kXpJDWms/Gb6SeNaw7qHf8tUyivAbW2uHb7spAuy4nP4BsGPYPcntQ1A/J8kvTlj3C1W1Lp5fmuSfk3w9ybx141U1s6qesk1nDLADEdUAO4aPJllYVcsyOmp984R1K5KcVFXXJ9kzyVmttR8nOSHJu6rqq0muS/LMbTxngB1GtfbQ/ysIAABsDkeqAQCgk6gGAIBOohoAADqJagAA6CSqAQCgk6gGAIBOohoAADr9f+T5RodH9lOcAAAAAElFTkSuQmCC\n",
      "text/plain": [
       "<Figure size 864x576 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "fig, ax = plt.subplots(figsize=(12, 8))\n",
    "freq = pd.DataFrame({'train': df_train.groupby('label').text.count(), 'test':df_test.groupby('label').text.count()})\n",
    "freq.sort_values(by='train', ascending=False).plot.bar(rot=0, ax=ax);"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Original code https://github.com/akutuzov/webvectors/blob/master/preprocessing/rus_preprocessing_udpipe.py\n",
    "from ufal.udpipe import Model, Pipeline\n",
    "\n",
    "#! wget http://rusvectores.org/static/models/udpipe_syntagrus.model\n",
    "udpipe_filename = 'models/russian-syntagrus-ud-2.2-conll18-180430.udpipe'\n",
    "udpipe_model = Model.load(udpipe_filename)\n",
    "process_pipeline = Pipeline(udpipe_model, 'tokenize', Pipeline.DEFAULT, Pipeline.DEFAULT, 'conllu')\n",
    "\n",
    "def tag_ud(pipeline, text='Текст нужно передать функции в виде строки!', pos=True):\n",
    "    # если частеречные тэги не нужны (например, их нет в модели), выставьте pos=False\n",
    "    # в этом случае на выход будут поданы только леммы\n",
    "\n",
    "    # обрабатываем текст, получаем результат в формате conllu:\n",
    "    processed = pipeline.process(text)\n",
    "\n",
    "    # пропускаем строки со служебной информацией:\n",
    "    content = [l for l in processed.split('\\n') if not l.startswith('#')]\n",
    "\n",
    "    # извлекаем из обработанного текста лемму и тэг\n",
    "    tagged = [w.split('\\t')[2].lower() + '_' + w.split('\\t')[3] for w in content if w]\n",
    "\n",
    "    tagged_propn = []\n",
    "    propn = []\n",
    "    for t in tagged:\n",
    "        if t.endswith('PROPN'):\n",
    "            if propn:\n",
    "                propn.append(t)\n",
    "            else:\n",
    "                propn = [t]\n",
    "        elif t.endswith('PUNCT'):\n",
    "            propn = []\n",
    "            continue  # я здесь пропускаю знаки препинания, но вы можете поступить по-другому\n",
    "        else:\n",
    "            if len(propn) > 1:\n",
    "                name = '::'.join([x.split('_')[0] for x in propn]) + '_PROPN'\n",
    "                tagged_propn.append(name)\n",
    "            elif len(propn) == 1:\n",
    "                tagged_propn.append(propn[0])\n",
    "            tagged_propn.append(t)\n",
    "            propn = []\n",
    "    if not pos:\n",
    "        tagged_propn = [t.split('_')[0] for t in tagged_propn]\n",
    "    return tagged_propn\n",
    "\n",
    "df_train['lemmatized_tokens'] = df_train.apply(lambda x: tag_ud(process_pipeline, x['text'], pos=True), axis=1)\n",
    "df_test['lemmatized_tokens'] = df_test.apply(lambda x: tag_ud(process_pipeline, x['text'], pos=True), axis=1)\n",
    "\n",
    "df_train = df_train[~df_train.lemmatized_tokens.isnull()]\n",
    "df_test = df_test[~df_test.lemmatized_tokens.isnull()]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "import gensim\n",
    "\n",
    "w2v_filename = 'models/taiga_upos_skipgram_300_2_2018.vec.gz'\n",
    "\n",
    "w2v_model = gensim.models.KeyedVectors.load_word2vec_format(w2v_filename, binary=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "def create_matrix_with_mean_embeddings(docs, w2v_model, verbose=False):\n",
    "    embeddings_dim = w2v_model.vector_size\n",
    "    docs_len = len(docs)\n",
    "    X = np.zeros((docs_len, embeddings_dim), dtype=np.float32)\n",
    "\n",
    "    empty_vocab = []\n",
    "    for i, tokens in enumerate(docs):\n",
    "        tokens_embeddings = [w2v_model.get_vector(t) for t in tokens if t in w2v_model.vocab]\n",
    "        if len(tokens_embeddings) > 0:\n",
    "            mean_embeddings = np.mean(tokens_embeddings, axis=0)\n",
    "            X[i] = mean_embeddings\n",
    "        else:\n",
    "            empty_vocab.append(tokens)\n",
    "            \n",
    "    if verbose:\n",
    "        print(f'Docs with empty vocabulary: {len(empty_vocab)}')\n",
    "        print(empty_vocab)\n",
    "    return X"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "X_train dim:(28218, 300)\n",
      "X_test dim:(2967, 300)\n",
      "y_train dim:(28218,)\n",
      "y_test dim:(2967,)\n"
     ]
    }
   ],
   "source": [
    "from sklearn import preprocessing\n",
    "le = preprocessing.LabelEncoder()\n",
    "\n",
    "X_train = create_matrix_with_mean_embeddings(df_train.lemmatized_tokens, w2v_model)\n",
    "X_test = create_matrix_with_mean_embeddings(df_test.lemmatized_tokens, w2v_model)\n",
    "\n",
    "y_train  = le.fit_transform(df_train['label'].values)\n",
    "y_test  = le.transform(df_test['label'].values) \n",
    "\n",
    "print(f'X_train dim:{X_train.shape}')\n",
    "print(f'X_test dim:{X_test.shape}')\n",
    "print(f'y_train dim:{y_train.shape}')\n",
    "print(f'y_test dim:{y_test.shape}')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Pipeline(memory=None,\n",
      "     steps=[('std_scaler', StandardScaler(copy=True, with_mean=True, with_std=True)), ('svc', LinearSVC(C=1.0, class_weight=None, dual=False, fit_intercept=True,\n",
      "     intercept_scaling=1, loss='squared_hinge', max_iter=1000,\n",
      "     multi_class='ovr', penalty='l1', random_state=None, tol=0.0001,\n",
      "     verbose=0))])\n",
      "Recall 5-folds mean:0.5501\n",
      "Precision 5-folds mean:0.5634\n",
      "f1-score 5-folds mean:0.5105\n"
     ]
    }
   ],
   "source": [
    "from sklearn.svm import LinearSVC\n",
    "from sklearn.preprocessing import StandardScaler\n",
    "from sklearn.pipeline import Pipeline\n",
    "from sklearn.model_selection import RandomizedSearchCV\n",
    "from sklearn.model_selection import StratifiedKFold\n",
    "from sklearn.model_selection import cross_val_score\n",
    "from scipy.stats import expon\n",
    "\n",
    "def make_cross_val(model, X_train, y_train, cv):\n",
    "    precision = cross_val_score(model, X_train, y_train, cv=cv, scoring='precision_weighted', n_jobs=-1)\n",
    "    recall = cross_val_score(model, X_train, y_train, cv=cv, scoring='recall_weighted', n_jobs=-1)\n",
    "    f1_score = cross_val_score(model, X_train, y_train, cv=cv, scoring='f1_weighted', n_jobs=-1)\n",
    "    \n",
    "    print(model)\n",
    "    print(f'Recall {k}-folds mean:{np.mean(recall):.4f}')\n",
    "    print(f'Precision {k}-folds mean:{np.mean(precision):.4f}')\n",
    "    print(f'f1-score {k}-folds mean:{np.mean(f1_score):.4f}')\n",
    "    \n",
    "svc_pipeline = Pipeline([('std_scaler',StandardScaler()), ('svc', LinearSVC(penalty='l1', dual=False))])\n",
    "\n",
    "# k = 5\n",
    "# skf = StratifiedKFold(n_splits=k, random_state=42)\n",
    "\n",
    "# C_distr = expon(scale=2)\n",
    "# param_grid = {'svc__C': C_distr}\n",
    "# svc_grid = RandomizedSearchCV(svc_pipeline, param_grid, n_iter=25, scoring='f1_macro', cv=skf, n_jobs=-1)\n",
    "\n",
    "# svc_tuned = svc_grid.fit(X_train, y_train)\n",
    "make_cross_val(svc_pipeline, X_train, y_train, skf)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Precision: 0.6066\n",
      "Recall: 0.6181\n",
      "f1-score: 0.5890\n"
     ]
    }
   ],
   "source": [
    "from sklearn.metrics import f1_score, precision_score, recall_score, confusion_matrix\n",
    "\n",
    "def score_model(y_pred, y_true):\n",
    "    average = 'weighted'\n",
    "    f1 = f1_score(y_true, y_pred, average=average)\n",
    "    precision = precision_score(y_true, y_pred, average=average)\n",
    "    recall = recall_score(y_true, y_pred, average=average)\n",
    "    return f1, precision, recall\n",
    "\n",
    "svc_pipeline.fit(X_train, y_train)\n",
    "y_pred = svc_pipeline.predict(X_test)\n",
    "\n",
    "f1, precision, recall = score_model(y_pred,y_test)\n",
    "print(f'Precision: {precision:.4f}')\n",
    "print(f'Recall: {recall:.4f}')\n",
    "print(f'f1-score: {f1:.4f}')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/nerdikon/anaconda3/lib/python3.7/site-packages/sklearn/preprocessing/label.py:151: DeprecationWarning: The truth value of an empty array is ambiguous. Returning False, but in future this will result in an error. Use `array.size > 0` to check that an array is not empty.\n",
      "  if diff:\n"
     ]
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAjoAAAHjCAYAAAAqkbW6AAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDIuMi4zLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvIxREBQAAIABJREFUeJzs3Xd4FdW6x/Hvu3cSCALSk1AEVBSlWsB6QKV3K4rHgh7FBiIc8WBDwXKv/do9qCgcC1Wq1ISmgkfAAggovaZgIAlFINlZ94+EkJAEomZSdn6f55nHzMyaWcVh8matNTPmnENEREQkGPmKuwAiIiIiXlGgIyIiIkFLgY6IiIgELQU6IiIiErQU6IiIiEjQUqAjIiIiQUuBjoiIiAQtBToiIiIStBToiIiISNAKKe4C5Gf99a30ymaP9Vmxt7iLEPT2pacVdxHKhAC6XXgtUNwFKCM2b91qRZrhz5MK/x9Pk+uKtg4noR4dERERCVoKdERERCRoKdARERGRoKVAR0RERIKWAh0REREJWiX2qSsRERHxlgsU/vN0JeqRK9SjIyIiIkFMgY6IiIgELQ1diYiIlFWB4H+pqXp0REREJGgp0BEREZEiZWadzewXM9tgZkPz2P+amf2YufxqZknZ9gWy7Zt2srw0dCUiIlJGOQ++x3eyp67MzA+8DXQAdgDLzGyac25NVrmcG5Qt/QDgvGyn+N0517Kg5VGPjoiIiBSl1sAG59wm59wRYCzQ6wTp+wCf/9nMFOiIiIhIoTGzfma2PNvS77gkdYDt2dZ3ZG7L61z1gYbA/Gyby2ee91szu/pk5dHQlYiIiBQa59xIYOQJkuQ1uuXySXsTMNE5l/3Nhqc553aZ2enAfDNb5ZzbmF9mCnRERETKKg/ejFwAO4B62dbrArvySXsT8ED2Dc65XZn/3WRmC8mYv5NvoKOhKxERESlKy4BGZtbQzMLICGZyPT1lZmcDVYGl2bZVNbNymT/XAC4D1hx/bHbq0REREZEi45xLM7P+wBzAD4xyzv1sZiOA5c65o0FPH2Cscy77sNY5wL/NLJ2Mzpr/zf60Vl4U6IiIiJRRrpjejOycmwnMPG7bsOPWn87juCVAsz+Sl4auREREJGgp0BEREZGgpUBHREREgpbm6IiIiJRV+nq5iIiISOmlQEdERESCloauREREyigvvl5e0qhHR0RERIKWAh0REREJWhq6EhERKauK56OeRUo9OiIiIhK0FOiIiIhI0FKgIyIiIkFLc3RERETKqOL6enlRUo+OiIiIBC0FOiIiIhK0NHQlIiJSVmnoSkRERKT0UqAjIiIiQUuBTgFUaHkJ9V+fSP03v6Dq1bfnm67ixVfRaOIyyp1xTsaGkBAi7h/Gaa98zmkvf0p4k/OLqMSlxyVt2jApZi6TF8Rw+7335NofGhbG82++zuQFMXw8eSJRdeoA4A8J4emXX2TsrC+ZMG82fe+7N+uYipUq8cI7bzExeg4T5s2m2XnnFVl9SqK/tW3L7PkxzF20kLvvuy/X/tCwMF576y3mLlrI+ClTqFO3bo79UbVr8/2an7mz390AREZFMWbs58yMiWbGvLncdscdRVKPkqxN27bMnT+fmEWLuCePNg4LC+P1t94iZtEiJmZr48suv5wpM2bw5Zw5TJkxg4svvTTrmK7duzNj9mxmzZvHI48+WmR1KanatG1LzPz5LFi0iHvzaeM333qLBYsWMTlbG1epUoXPxo5l9Zo1DB8xIscxDw8ZwjdLl7J6zZoiqUNJ5NIDhb6UNAp0Tsbno+Zdj7DzuYFsHdSbSpd3JKxuw1zJrHwFqnS9kd9/XZW17dT21wCw7Z992DmiPzVuewjMiqzoJZ3P5+NfI57mwb7/4IaOnenUszsNzzwzR5pevW9gX3Iy11zZjs8+/IgBQx8BoH3XLoSFhXFTl27c0uNqrr35pqwg6OGnnmTJosVc374Tfbr2YPOGDUVet5LC5/Mx7JkR3HV7X7q170D3nj05o1HONr7hxt6kJCfTse0VfPzhhzw8dGiO/Y8Oe5KvFi7MWg8E0vjfZ5+la7v23Hj1Ndx82625zlmW+Hw+nn7mGf5x++10bt+e7j17cmajRjnS3HDjjSQnJ9OubVs++vBDHsls471799Lvzjvp1qkTQwYP5uXXXgMyfjkPfewxbrv5Zrp06ECNGjW45LLLirxuJYXP52PEM8/Q9/bb6di+PT3zaOPemW18Zdu2fPjhhwzNbOPDhw/z6ssv8/xzz+U6b3R0NFf36lUkdZDio0DnJMqf2YTUuO2kJeyEtDT2fTOPU1q1zZWu+k33snfqf3CpR7K2hdVtyMFVywAIpOwl/eD+Y709QpMWLdi+dSs7t28nLTWVudO/pG2H9jnStO3QnhmTJgMQM2s2rS+9JGOHc5SvUAG/30/58uVJTU3lwP79nFKxIue1bsXUceMBSEtNZf++fUVar5KkecuWbN2ylR3bt5OamsqX06fTrkPHHGmu6tCRyZMmATBn5kwuuexYr0K7jh3ZsW0b639dn7Vtd8Ju1qz+GYADBw6wacNGIiIii6A2JVOLli3ZumUL27O1cfsOHXKkad+hQ1Ybz545MytoWfPzzyQkJACw/tdfKVeuHGFhYdQ77TQ2b97Mnj17AFjy9dd07tKlCGtVshzfxtOnT6fDcW3coUMHJmW28ayZM7k0s41///13li9fzuHDh3Od98cffmB3ZvtL8PI80DGzcDM72+t8vBJSrSZpv8VnraclxhNSrWaONOUankVojQgOrPg6x/bDW9ZzSus24PMTUqs25U5vTGj1iCIpd2lQKzKC+NjYrPWEuDhqReZsn1oRx9IEAgH279vPqVWrEj1rNocOHmT2f5cy45vFfPL+B6QkJ1OnXj2S9uzhqZde4NMZ03jif5+nfHh4kdarJImIjCAudlfWenxsLBHHtXFEZASxuzLSBAIB9u3bR9WqVQkPD+fu++7lrf97Pd/z16lbl3OanMtPP/7oTQVKgYjISGKzXcdxsbFEREbmTpOtjfdntnF2nbt2Zc3PP3PkyBG2btnCGWecQZ26dfH7/bTv1ImoqCjvK1NCRebRxpEnaeN9ebSxlE2eBjpm1gP4EZidud7SzKadIH0/M1tuZsvHbtrtZdEKLq+hJudy7K/RdzC7R/9frmQp86eRlpjAaS+MoeYdgzn0y8oSOX5ZbPJoW5e9bfNJg3M0bdGcQCBA54svpWebK7jlrn9Qp149/CF+zm7ShImffsbfu/fk94MH6Xtf7rk/ZYVx8ja2fP4/DBg8iNEffMjBgwfzPHeFChV44713eX7ECA7s3184BS6F8hqMLmgbH9WoUSMeGTqUJzPn4qSkpDDs8cd5/a23GDtxIjt37CAtLfgfA85PYbSx5M0F0gp9KWm8fo/O00BrYCGAc+5HM2uQX2Ln3EhgJMD661uViCs0LTGBkBrH/gIOqR5B2t7fstZ94RUoV+8M6g5/DwB/lerU/tcr7HrhnxzeuJbfPn4tK23d5z4kNXZ70RW+hEuIjSMi21+ptSIj2R2fsxs5IS4jTUJcHH6/n4qVKpKclESnXj1ZuvgrAmlp7E3cw0/LV3BO82b88N/vSIiL4+cffwIyhrv65jHJuayIi4sjMqp21npEVBQJx7VxXGwcUbVrE5/ZxpUqVSIpKYkWLVvSqUtXHn70USpXrky6S+fw4cN8OnoMISEhvPHee0yfMoV5s+cUdbVKlLi4uBy9LZFRUSTEx+dMExtLVO3axGVdxxltDBm9Fe+MHMnDgwezbdu2rGPmx8QwPyYGgBv79CEQKLt/JMXm0cbxJ2njStnaWMo2r4eu0pxzyR7n4alDG9YQFnUaIbVqQ0gIlS7rwIFli7P2px88wKY7O7Dl/l5sub8Xh9avzgpyLKwcVq48ABWat4ZAGkd2bC6uqpQ4a1aupF6D+tSuW5eQ0FA69ujG4uiYHGkWR8fQ/bqMSd3tunRm2dJvAYjfuYsLL7kYgPLh4TQ97zy2bNxI4m+/ER8bS/3TMyaMt770UjaV4cnIq376iQYNG1C3Xl1CQ0Pp1qMH8+fNy5FmfvQ8rrnuOgA6de3Kt0uWAPD3G3rT7vLLaXf55YweNYp/v/02n44eA8BzL77Apg0b+PiDD4u0PiXRyp9+on7DhtStVy+rjWOOa+OY6OisNu6crY0rVa7M+x99xMsvvsj3y5fnOKZa9eoAVK5cmb/feivjx44tgtqUTCt/+okG2dq4R48eRB/XxtHR0VyX2cZdunZlaWYbi3jdo7PazG4G/GbWCHgQKF1XX3qAhA9epM4Tb4DPT8r8aRzZsYlqN97D4Y1rObB8cb6H+k+tRp0n3gSXTtqe3cS98VQRFrzkCwQCvPTUcN4c8xF+n59pEyawaf167hk0kLWrVrM4Ooap48Yz4rVXmLwghpTkJB4b8BAA4//zCU+99ALj5szCzJg+cSIb1v0CwEtPjeCZ114lNCyUndu2M3zIv4qzmsUqEAgwYtgwPhgzBr/fz6Tx49mwfj0PDh7E6pWrmB8dzcRx43nptVeZu2ghyUlJDOo/4ITnvODCC7n6uuv4Ze1apsycCcCrL73I4gULi6BGJU8gEGD4sGF8lNnGE8aPZ/369QwcPJjVK1cSEx3N+HHjeOW114hZtIikpCQe6t8fgFtvv536DRrwwIABPDAgo9373norexITefKppzjn3HMBePP119myuez+kRQIBHhq2DDGjBmDL1sbDxo8mFUrVxIdHc24ceN47bXXWLBoEclJSQzIbGOAr77+moqVKhEaGkqHjh257dZb2bB+PUMffZSevXoRHh7Okm+/ZdzYsbz+f7mnIQS1EjjUVNjMyzFMM6sAPA4cfcxjDvCsc+7QyY4tKUNXwazPir3FXYSgty89+G8iJUEA3S68VnYHzorW5q1bi/QdJIenPlno/3jK9XqmRL1HxesenbOdc4+TEeyIiIiIFCmv5+i8ambrzOwZM2vicV4iIiIiOXjao+Ocu9LMIoHewEgzqwyMc84962W+IiIicnJl4ZUnnr8w0DkX55x7A7iXjHfqDPM6TxERERHw/oWB55jZ02a2GniLjCeu6p7kMBEREZFC4fVk5I+Az4GOzrldJ0ssIiIiRagMPF7u9Rydi708v4iIiMiJeBLomNl451xvM1sFOV5wYYBzzjX3Il8RERGR7Lzq0RmY+d/uHp1fRERE/qKS+BHOwubJZGTnXGzmj/c757ZmX4D7vchTRERE5HheP17eIY9tXTzOU0RERATwbo7OfWT03JxuZiuz7aoEfONFniIiIiLH82qOzmfALOB/gKHZtu9zzu3xKE8RERH5I8rAHB1PAh3nXDKQDPQBMLNaQHmgoplVdM5t8yJfERERkey8fjNyDzNbD2wGFgFbyOjpEREREfGc129Gfha4GIh2zp1nZleS2csjIiIixUsf9fzrUp1ziYDPzHzOuQVAS4/zFBEREQG879FJMrOKwGLgUzNLAIJ/5pOIiIiUCF736PQCfgcGAbOBjUAPj/MUERERAbz/qOeBbKujvcxLRERE/iA9Xv7XmNk+cn7UEzIeO18O/NM5t8nL/EVERKRs83qOzqvALjJeIGjATUAk8AswCrjC4/xFRESkDPM60OnsnLso2/pIM/vWOTfCzB7zOG8RERE5ARfQ4+V/VbqZ9TYzX+bSO9u+44e0RERERAqV14HO34FbgQQgPvPnW8wsHOjvcd4iIiJSxnn91NUm8n+c/Gsv8xYREZETc2XgqSuvv3V1lpnFmNnqzPXmZvaEl3mKiIiIHOX10NX7wKNAKoBzbiUZT16JiIiIeM7rQKeCc+6747YFfz+ZiIiIlAheP17+m5mdQeYTVmZ2PRDrcZ4iIiJSEOnB3/fgdaDzADASaGxmO4HNZDyJJSIiIuI5rwOdncBHwAKgGpAC3A6M8DhfEREREc8DnalAEvA9GZ+CEBERkRKiLLwZ2etAp65zrrPHeYiIiIjkyeunrpaYWTOP8xARERHJk9c9OpcDfc1sM3CYjC+YO+dcc4/zFREREfE80Oni8flFRETkz9Icnb/GObfVy/OLiIiInIjXPTp/2vhfKxd3EYLegsENi7sIQe/l93cXdxHKhOD/m7T4zd23o7iLIPKnlNhAR0RERLylr5eLiIiIlGIKdERERCRoaehKRESkrCoDT12pR0dERESClgIdERERCVoKdERERCRoaY6OiIhIGVUWvl6uHh0REREJWgp0REREpEiZWWcz+8XMNpjZ0HzS9DazNWb2s5l9lm377Wa2PnO5/WR5aehKRESkjHLpRT90ZWZ+4G2gA7ADWGZm05xza7KlaQQ8ClzmnNtrZrUyt1cDngIuBBywIvPYvfnlpx4dERERKUqtgQ3OuU3OuSPAWKDXcWnuBt4+GsA45xIyt3cC5jnn9mTumwd0PlFmCnRERESkKNUBtmdb35G5LbuzgLPM7Bsz+9bMOv+BY3PQ0JWIiIgUGjPrB/TLtmmkc25k9iR5HOaOWw8BGgFXAHWBr8ysaQGPzXUiERERKYs8eLw8M6gZeYIkO4B62dbrArvySPOtcy4V2Gxmv5AR+OwgI/jJfuzCE5VHQ1ciIiJSlJYBjcysoZmFATcB045LMwW4EsDMapAxlLUJmAN0NLOqZlYV6Ji5LV/q0REREZEi45xLM7P+ZAQofmCUc+5nMxsBLHfOTeNYQLMGCABDnHOJAGb2DBnBEsAI59yeE+WnQEdERKSMKq43IzvnZgIzj9s2LNvPDhicuRx/7ChgVEHz0tCViIiIBC0FOiIiIhK0NHQlIiJSRrlAenEXwXPq0REREZGgpUBHREREgpYCHREREQlamqMjIiJSVmmOjoiIiEjppUBHREREgpaGrkRERMqo4nozclFSj46IiIgELQU6IiIiErQU6IiIiEjQ0hwdERGRMsoFXHEXwXPq0REREZGgpUBHREREgpaGrkRERMoofb1cREREpBRToCMiIiJBS0NXIiIiZZSGrkRERERKMfXoFMDpl7Wi478ewHw+fvxiJktHjc2xv/Wt19Py2q6kBwIc3JvEjGEvkRKbAMBVg/px5t8uwnzG5qUrmPvC28VRhVLBf1oLyre5A8xH6poYjqyYmitNyJmXEHbRDeAc6b9t5dDcN7BKNQjv+jCYD3x+UlfOJnX1vGKoQcl35mWt6PKv/pjPz/dffMnXoz7Psf+SW2/g/KxrOZkpw14kOTaeBq1a0nnIA1npajQ8jYmPjGDdgm+KugolXqPLWtH1X/3x+fys+OJLFh/XxpfeegMXZrbxgb3JTB72Ikmx8QCcGlmLa55+mMqRtcA5xjwwlKRd8cVRjRLn4jZ/459PPYHP52fquPGMeW9kjv2hYWE8/cqLNG7alOSkJB7vP5DYnTsJCQ3l0eee4ZxmTXEunVeGP8v3//0OgJDQUIYMH8YFF19Eeno67778GgtmzymO6omHFOichPl8dH7sQT7r9wgp8bu58/N3WL9wKb9t2pqVJn7dBkb1uY+0Q4c5v3cP2g3qx+RHnqVOi3Op27IJ719/NwC3jX6d0y5swbblPxVXdUouM8pf8Q8OTnkWtz+RCjf+D2mblpO+d+exJKdGEnbh1Ryc+CQcPoCFVwbAHdjLwQlPQHoahJbjlJtfIW3zctyBvcVVmxLJfD66PTaQMf2GkBK/m36fv8cvC5ewO9u1HLtuPSP73EvqocO06t2TjoPuYcIjI9iy7Efe651xHYdXrsSDX37CxqXLi6sqJZb5fPR4bCAfZbbxvZ+/x9o82vjdzDZu3bsnnQbdw7hHRgBw/XOPsvD9T9j47QrCwsvjXPC/zK0gfD4fj4x4mv639iUhLo7RUyfxVfR8Nm/YkJWmZ+/r2ZecwnVXtqdD9270HzqExwc8xNU39Qbg5i7dqVq9Gv/30Yf07XUtzjnueOA+9ibu4fqrOmJmVK5SpbiqKB7S0NVJ1G7amD3bdpK0M5b0tDTWzF7AWVdemiPN1mU/knboMAA7V66lUkTNjB0OQsqF4Q8NwR8Wij/Ez4FE/fLNiy/iTNKT4nApCZAeIO3XJYSc3ipHmrAm7UhdOQcOHwDA/Z6SsSM9kBHkAPhDM3p2JJc6TRuzZ9su9u6MJZCWxurZ82l85WU50mxZ9iOpmdfy9pVrqHz0Ws7m3A5t2fD1d1np5Ji6TRuTmK2NV82ezznHtfHmfNq45un18fn9bPx2BQBHfj+kNs7UpEVzdmzdyq7t20lLTWXu9C9p06FdjjRtO7Tny0lfADB/1mxaXXoJAA0bncmyJUsA2Ju4h/0pKZzTvBkAPW+4no/feQ8A5xzJe8ve/dmlu0JfShr9RjiJShE12Be/O2s9JX43lWrVyDd9y2u6sPHrjG7RnSvXsHXZjwyMmcDAmPFsWrKcxM3bPC9zaeQ7pRrp+xOz1tP3J2IVq+VIY1Vr46sSRYXrRlDhhmfxn9bi2L6K1anQ5yUq9n2XIyumqjcnD5UjapAcn5C1nnySa/n8a7qy/uv/5tretMuVrJoV40kZS7vj2zglfjeVT9DGF2Rr4xr16/L7vv30eXU4948bSafB92A+3aIBakZGEh8bm7WeEBdHzciInGkiIoiPjQMgEAiwf99+Tq1alfVr19G2Q3v8fj+169alcbOmRERFUbFSJQDuHfwQY6ZP4X/efoNqNaoXXaWkyHjyr8jM9plZSh7LPjNLOcFx/cxsuZktX7ZnZ37Jil1+3clNu7UnqslZfPvxeACq1qtNjYan8UaHG3mj/Y3Ub30e9S5oVpRFLT3Mcm87rp3NfFiVKA5OHs7vc16nfLt7IaxCRtL9iRz8fAgH/vMgoee0xcJPLYpSlzInb+OjmndrT+0mZ/PNx+NybK9YoxoRZ57OhiXLvChgEMjdxvndL1p0a0+dJmfzVWYb+0L8NDi/GbNfeY/3br6XanVrc36vzp6WtrTI6/aQ+/6Q9/U9ffxEEmLjGD1tMoOGPc7KFd8TCKThDwkhonYUP634ntt6XM2q73/gwceGelMBKVaeBDrOuUrOucp5LJWcc5VPcNxI59yFzrkLW1Wr40XR/rB98b8dG4oCKkfUZP/uxFzpGlx0PpfdfTPjH3ySQGoqAGe3u5ydK9eS+vshUn8/xMavv6NO83OLrOylSfr+RHwVj/015atYPVevTPr+PaRtWgbpAVzKbtL37sJXJSpHGndgL+mJ2/HXblwk5S5NUuJ3c2pEraz1UyNqsi+Pa/n0i86nzd238PmDj2ddy0c17XQla+d/TXpawPPylkbHt3HlfNr4jIvOp+3dt/BJtjZOjt9N7LoN7N0ZS3ognbXzvybqnEZFVvaSLCE2joioY//Wa0VGsjtbzxlk9PJEREUC4Pf7qVipIslJSQQCAV579nlu6daTIf3uo1LlymzfvJXkvXv5/eBBFs6ZC0D0zFk0btKk6CpVQriAK/SlpCmSflEzq2Vmpx1diiLPwrLr53VUq1+HU+tE4gsJ4dzOV/LrwiU50kQ0PpOuwwYx/sEnObgnKWt7cmwCp13YHPP78IX4Oe3C5iRu0tBVXtLjN+KrEoVVrgk+PyFnXUra5pyTXdM2fUdI3aYAWPlK+KpEkZ4Sj51SLWNuDkC5U/BHnU160q6irkKJd/RarlInEn9ICE07X8W6467lyMZn0mPYYD578HEOZLuWj2ra5SoNW53Azp/XUb1+HapmtnGzPNo4qvGZ9Bo2mE+Pa+Odq3+hfOVKVKia0Rt5euvz2L1xKwJrVq6iXoMG1K5bl5DQUDr26MZX0Tmvw8XRMXS77loArurSmeVLvwWgXPnylA8PB6D15ZcRCASyJjF/FTOfCy6+CIBWl16aY3KzBA9Pn7oys57AK0BtIAGoD6wFSk3Y7ALpzHn+Tfq8+wI+v4+fpszit41baXN/X2LX/ML6hUtpN7gfoRXCue7lYQAkxyUw4cEnWTdvMQ1an0e/SR/gHGz6ZhnrFy0t5hqVUC6dQ4tGUaHn4+DzkbpmAel7dhB2UW8CCRsJbF5BYNtPhJzWggp/fxXS0zn8zSdwaD++es0od/ltgAOMIz9MJz1xe3HXqMRJD6Qz8/k3uPXdF/H5ffwwZRa7N27hyvvvYNeaX/hl4RI6Dr6XsArh9H75aQCS4+L5/MEnAKhSO4JTI2qyVU8N5is9kM6M59/g9sw2XjFlFgkbt9Du/jvYueYX1i1cQufMNr4ps42T4uL59MEncOnpzH7lXe58/xUwY9eaX1k+aUbxVqiECAQCvPTUcN4YMwqfz8/0CRPZtH4D/QYNZO2qVXwVPZ9p4yYw/LWXmbQgmpTkJB4fMAiAatWr88aYUaSnO3bHxfHU4IezzvvWCy8x/NWXGTTscZIS9zDiEQ1dBSPz8vFFM/sJuAqIds6dZ2ZXAn2cc/1OduxzzduVvP6vIPPg3Zp457WX39998kTyl2kgzXtz9+0o7iKUCd9tXp/XjCTPxD3QqtB/10a+vaxI63AyXr9HJ9U5l2hmPjPzOecWmNkLHucpIiIiBeDKwF8JXgc6SWZWEVgMfGpmCUCax3mKiIiIAN5PRu4FHAQGAbOBjUAPj/MUERERATzs0TEzPzDVOdceSAdGe5WXiIiI/HEl8XHwwuZZj45zLgAcNDO9uU1ERESKhddzdA4Bq8xsHnDg6Ebn3IMe5ysiIiLieaDzZeaSXfD3k4mIiJQC6enFXQLveR3oVHHOvZ59g5kN9DhPEREREcD7p65uz2NbX4/zFBEREQE86tExsz7AzUBDM5uWbVclIPcX7kREREQ84NXQ1RIgFqhBxreujtoHrPQoTxEREfkD9GbkP8k5txXYClzixflFRERECsLrr5fv49hTVmFAKHDAOVfZy3xFREREwONAxzlXKfu6mV0NtPYyTxERESmYsjB05fVTVzk456YAVxVlniIiIlJ2eT10dW22VR9wIXphoIiIiBQRr18YmP1L5WnAFjK+aC4iIiLiOa/n6Nzh5flFRETkzysLn4DwdI6OmZ1lZjFmtjpzvbmZPeFlniIiIiJHeT0Z+X3gUSAVwDm3ErjJ4zxFREREAO/n6FRwzn1nZtm3pXmcp4iIiBSAHi//634zszPIfNLKzK4n49MQIiIiIp7zukfJhbHOAAAgAElEQVTnAWAk0NjMdgKbgb97nKeIiIgI4H2gsxP4CFgAVANSgNuBER7nKyIiIieRnm4nT1TKeR3oTAWSgO+BXR7nJSIiIpKD14FOXedcZ4/zEBEREcmT15ORl5hZM4/zEBEREcmT1z06lwN9zWwzcBgwwDnnmnucr4iIiJxEWXgzsteBThePzy8iIiKSL6+/dbXVy/OLiIiInIjXc3REREREio0CHREREQlaXs/RERERkRJK37oSERERKcUU6IiIiEjQ0tCViIhIGVUWvnWlHh0REREJWgp0REREJGiV2KGrSSl616DXYl6PLe4iBL2PL6lQ3EUoE67/Orm4ixD0DpWFx3MkKJXYQEdERES8lV4G4lcNXYmIiEjQUqAjIiIiQUtDVyIiImWUHi8XERERKcUU6IiIiEjQ0tCViIhIGeU0dCUiIiJSeinQERERkSJlZp3N7Bcz22BmQ0+Q7nozc2Z2YeZ6AzP73cx+zFzeO1leGroSERGRImNmfuBtoAOwA1hmZtOcc2uOS1cJeBD473Gn2Oica1nQ/NSjIyIiUkalpxf+UgCtgQ3OuU3OuSPAWKBXHumeAV4EDv2VOirQERERkUJjZv3MbHm2pd9xSeoA27Ot78jclv0c5wH1nHMz8siioZn9YGaLzOxvJyuPhq5ERESk0DjnRgIjT5Akr0e9XNZOMx/wGtA3j3SxwGnOuUQzuwCYYmZNnHMp+WWmQEdERKSMKqY3I+8A6mVbrwvsyrZeCWgKLDQzgEhgmpn1dM4tBw4DOOdWmNlG4CxgeX6ZaehKREREitIyoJGZNTSzMOAmYNrRnc65ZOdcDedcA+dcA+BboKdzbrmZ1cyczIyZnQ40AjadKDP16IiIiEiRcc6lmVl/YA7gB0Y55342sxHAcufctBMc3gYYYWZpQAC41zm350T5KdARERGRIuWcmwnMPG7bsHzSXpHt50nApD+SlwIdERGRMkpfLxcREREpxRToiIiISNDS0JWIiEgZFdDQlYiIiEjppUBHREREgpaGrkRERMooPXUlIiIiUoop0BEREZGgpUBHREREgpbm6IiIiJRR6U5zdERERERKLQU6IiIiErQ0dCUiIlJGpacXdwm8l2+gY2bTAZfffudcT09KJCIiIlJITtSj83KRlUJERETEA/kGOs65RUVZEBEREZHCdtI5OmbWCPgf4Fyg/NHtzrnTPSyXiIiIeCygx8sB+Ah4F0gDrgTGAP/xslAiIiIihaEggU64cy4GMOfcVufc08BV3hZLRERE5K8ryOPlh8zMB6w3s/7ATqCWt8USERERr+nr5RkeAioADwIXALcCt3tZKBEREZHCcNIeHefcsswf9wN3eFuckunStm14eNgT+P1+Jo8bz8fv/jvH/tCwMJ559SXOadqUpKS9DO0/kNgdOwkJCeHJF56ncZMmhIT4mfHFFD56571iqkXJ16rN5fR/4lF8fj8zx0/k839/kGN/81YX8MATj3L62WfxzEMPs3j2XAAiatdm+Duv4/P5CQkNYfKYT5n++bjiqEKJV77ZJVS99WHw+TiwcAopM0bn2H/K37pT5aaBBPYmALBv3ngOLJpK6GlnUa3vUCz8FEhPJ2XaKA7+d15xVKFEurjN3/jnU0/g8/mZOm48Y94bmWN/aFgYT7/yIo2bNiU5KYnH+w8kdudOQkJDefS5ZzinWVOcS+eV4c/y/X+/o8IppzBy/OdZx9eKjGDWlGm89sxzRV21EuOytm3417Bh+Pw+vhg3nlHv5ryXhoaF8dyrL3NuZhsP6T+AXTt20rVXL/rec3dWurMaN+bG7j34Zc1a3h39ETVq1cLv9/P9suU8/+Qw0svCG/TKmII8dbWAPF4c6JwrE/N0fD4f/xrxNPffcjvxcXF8Mu0LFs2LYfOGDVlpru59AynJyfS6oh0de3Rj4NBHGNp/IO27diEsLIwbO3ejfPnyTIyezexp04ndsbMYa1Qy+Xw+Bj79BENuv4vdcfG8+8U4lsQsYOuGjVlp4nfF8sIjj9H7rpzxduLu3QzofTOpR1IpX6ECo2ZOZUnMfBITdhd1NUo281H19n+R8MIDBPbEEzliDAe/X0zars05kh387zz2jnkxxzZ35BCJ/36KtPjt+KvUIPKZT/h91VLcwf1FWYMSyefz8ciIp+l/a18S4uIYPXUSX0XPz3GP6Nn7evYlp3Ddle3p0L0b/YcO4fEBD3H1Tb0BuLlLd6pWr8b/ffQhfXtdy8EDB7il27F3so6eNpmFc+YWed1KCp/Px2MjhtPvltuIj4vj82lTWDgvmk3Z2vja3r1JSU6h+xVX0blHdx4a+i8e6f8gM6dOZebUqQA0OvtsXn//3/yyZi0ADz8wgAP7M67hV999h47dujJ7+oyir2Ax0lNXGR4GhmQuTwI/Asu9LFRJ0rRlC3Zs3crO7dtJS01lzvQvuaJj+xxprujYnhmTJgMQM3M2rS69BACHIzy8An6/n3Lly5N6JJUD+/SLIS+NWzRj59ZtxG7fQVpqKvO/nMWl7XPG0vE7d7Hpl19z/cWVlppK6pFUAMLCQjGfPuGWl7AzmpAWv53A7p0QSOPgt3OpcEHbAh2bFreNtPjtAASSfiOQsgd/papeFrfUaNKiOTu2bmVX5j1i7vQvadOhXY40bTu058tJXwAwf9axe0TDRmeybMkSAPYm7mF/SgrnNG+W49h6DepTrXp1fvhuGWVV05Yt2JbtPjx7+gyu7NghR5orOrZn2qRJAMybOYuLLr0013m69OzBrGnTs9aPBjkhISGEhobiXL4fA5BS7KS/EZxzK7It3zjnBgMXFeTkluEWMxuWuX6ambX+i2UuUjUjIojbFZu1nhAbR62IiHzTBAIB9u/bT5WqVYmZOZvffz/I3O+WMnPJYv7z/gekJCcXaflLixoRESTExmWt/xYXR82Igs95rxkVyfszJjP2q/mMHfmBenPy4K9ai8Ce+Kz1tD0J+KvmbuMKra4i8rnPqTHgBfzVInLtDzu9CeYPJS1hh6flLS1qRkYSH5vtHhEXR83I3PeI+Mzr++g94tSqVVm/dh1tO7TH7/dTu25dGjdrSkRUVI5jO/bowbwvv/S+IiVYREQk8dnuw/GxsbnuwxEREVlpMtp4H1Wq5gzGO3XvliPQAXh3zMcsXLGMAwcOMG/mLI9qIMXppIGOmVXLttQws05AZAHP/w5wCdAnc30f8PYJ8upnZsvNbPlv+1IKmIW3zHJ36x0f9eeXpkmL5gQCATpddCnd/3YFt9z1D+rUq+dZWUuzvNuw4Mfvjo3j7u7XcGu7znS6phdVq1cvxNIFibx6qI9r5N9/+Iqdg3oQ93gfDv38HdXveTrHft+p1al+7wgS3x/+x/4HBbE8Lt1cbZPX9Y1zTB8/kYTYOEZPm8ygYY+zcsX3BAJpOZJ16NGNudPK1nBKLnk233HX30nu1c1atuDQ74fY8OuvOdLcd1tfrmp9EWFhYbTOoxdISr+C9PGvIGOoagWwFPgn8I8Cnv8i59wDwCEA59xeICy/xM65kc65C51zF9aoVLmAWXgrIS6OyNrH/sKqFRXJ7oSEfNP4/X4qVqpIclISXXr1ZOmir0hLS2Nv4h5+WrGCc4/rlpYMu+PiqBV1LH6uERnJb8e1c0EkJuxmy/qNNGt1QWEWLygE9iTk6KEJqVaLQFLOnq/0/cmQljEMuH/BZMIanJO1z8qfQq2HXydp4jsc2bi6aApdCiTExuXohakVGcnu+Nz3iIjM6zv7PSIQCPDas89zS7eeDOl3H5UqV2b75q1ZxzU6pzEhIX7Wrf65aCpTQsXHxRGR7T4cERWV6z6cPU1GG1ciOSkpa3/nHj1y9eYcdeTwERZGR3Nlh/Z57g9m6c4KfSlpChLonOOcO90519A518g51xEo6GBxqpn5yZzMbGY1gVI1pf3nn1ZSr0F9atetS0hoKJ16dGPRvJgcaRbNi6H7ddcA0K5rZ5Yt+RaA2F27aHXpxQCUDw+n2XnnsWXjRiS3dStXU6d+fSLr1iEkNJSrunVhacyCAh1bIzKCsHLlAKhYuTJNLjiP7Zs2n+SosufIpjWERtbDX7M2+EOocHFHfv9+cY40vlOP9YSFn9+G1KMTlf0h1HzoJQ58/SW/f5fz+i/r1qxcRb0GDbLuER17dOOr6JxttDg6hm7XXQvAVV06s3xpxj2iXPnylA8PB6D15ZcRCARyTGLu2KM7c8p6bw4Z9+H6DRpQJ7ONO/fozsJ50TnSLJwXQ8/rrgOgQ9cufLdkadY+M6Nj1y7Mmn4s0AmvUIEaNWsCGYHR5VdewWbdn4NSQV4YuAQ4/7htS/PYlpc3gMlALTN7DrgeeOIPlbCYBQIBXhg2nLfHfITP72fa+AlsWr+eewcNZM2q1SyOjmHK+PE88+orTF0YQ3JSEo8OeAiA8WM+4emXXmDC3FmYGdMmTGT9ul+KuUYlU3ogwJvDn+OFj97H7/cxa8JktqzfQN+B/fl19c8siVnA2c2aMuLdN6hYuTKXXHUlfQf2584uPal/xunc++gjGcMFZoz/4CM2/7q+uKtU8qQH2DPmJWoNeRN8fg4snkbqzk2ceu09HNm8lt9/WEylTjcRfl4bSA+Qvj+FxJFPA1Dhog6UO/t8fBVP5ZS/dQcgceRwUrf9eoIMy4ZAIMBLTw3njTGj8Pn8TJ8wkU3rN9Bv0EDWrlrFV9HzmTZuAsNfe5lJC6JJSU7i8QGDAKhWvTpvjBlFerpjd1wcTw1+OMe523frykN33FUc1SpRAoEAzw97mnfHjMbv9zFl/AQ2rl/P/YMeYs2qVSyMjmHy+HE8/+qrzFg4n+SkZB4Z8GDW8Rdc1Jr4uDh2bt+etS28QgXe+OB9wsLC8Pl9fLdkKRM+/aw4qices/xmmZtZJFAH+AS4mWOjpJWB95xzjQuUgVljoF3m8THOubUFOe78BmdqAoDHqoTkO4ooheTjSyoUdxHKhOu/1iR/rx1ygeIuQpmwcsumIh37mdXqqkL/Xdtl2fwSNX51oh6dTkBfoC7wCscCnRTgsYKc3MxeB8Y55/KdgCwiIiLilXwDHefcaGC0mV3nnJv0J8//PfCEmZ1FxhDWOOdcmXkHj4iIiBSvgkxGvsDMqhxdMbOqZvZsQU7unBvtnOsKtAZ+BV4wM02eEBERkSJRkECni3Mu6xm9zEfEu/7BfM4EGgMNgHV/8FgRERHxQMAV/lLSFCTQ8ZtZuaMrZhYOlDtB+ixmdrQHZwTwM3CBc67HnyqpiIiIyB9UkMfLPwFizOyjzPU7gNEnSJ/dZuAS59xvf6ZwIiIiIn/FSQMd59yLZrYSaE/Gk1ezgfonOsbMGjvn1gHfAaeZ2WnHnfP7P19kERERKQwl8U3Gha0gPToAcWS80bg3Gb00J3sKazDQj4zH0o/ngKvy2C4iIiJSqPINdDIfCb+JjA9yJgLjyHjB4JUnO6lzrl/mj12cc4eOO2/5P19cERERkYI7UY/OOuAroIdzbgOAmQ36g+fP6/MReW0TERGRIhYo40NX15HRo7PAzGYDYzn2duQTyvb5iHAzO4+cn4/QO/FFRESkSJzozciTgclmdgpwNTAIiDCzd4HJzrm5Jzhv9s9HvJpt+z4K+PkIERERkb+qIE9dHQA+BT41s2rADcBQIN9Ap5A+HyEiIiLylxT0qSsAnHN7gH9nLvkys1ucc58ADcxscB7neTWPw0RERKQIlcQ3GRe2PxTo/AGnZP63okfnFxERETkpTwId59y/M/873Ivzi4iIiBREQb519aeZ2YtmVtnMQs0sxsx+M7NbvMxTRERECiaAFfpS0nga6AAdnXMpQHdgB3AWMMTjPEVEREQA7wOd0Mz/dgU+z5zMLCIiIlIkvJqMfNR0M1sH/A7cb2Y1gUMnOUZERESkUHga6DjnhprZC0CKcy5gZgeAXl7mKSIiIgWjx8v/IjMLBW4F2pgZwCLgPS/zFBERETnK66Grd8mYp/NO5vqtmdvu8jhfEREREc8DnVbOuRbZ1ueb2U8e5ykiIiIFECjuAhQBr5+6CpjZGUdXzOx0yka7ioiISAngdY/OEGCBmW3KXG8A3OFxniIiIiKA9z0635DxAdD0zOXfwFKP8xQREZECCHiwlDRe9+iMAVKAZzLX+wD/AW7wOF8RERERzwOds4+bjLxAk5FFRESkqHg9dPWDmV18dMXMLiJjOEtERETEc1736FwE3GZm2zLXTwPWmtkqwDnnmnucv4iIiOSjJH5tvLB5Heh09vj8IiIiIvny+ltXW708v4iIiMiJeN2jIyIiIiVUwAX/Vz29nowsIiIiUmwU6IiIiEjQUqAjIiIiQUtzdERERMqokvjJhsJWYgOdKiFhxV2EoHckPb24ixD0rv06qbiLUCbM7FW9uIsQ9NpMji/uIoj8KRq6EhERkaBVYnt0RERExFtlYehKPToiIiIStBToiIiISNDS0JWIiEgZpaErERERkVJMgY6IiIgUKTPrbGa/mNkGMxuax/57zWyVmf1oZl+b2bnZ9j2aedwvZtbpZHkp0BEREZEiY2Z+4G2gC3Au0Cd7IJPpM+dcM+dcS+BF4NXMY88FbgKaAJ2BdzLPly8FOiIiImVUAFfoSwG0BjY45zY5544AY4Fe2RM451KyrZ4CWSfuBYx1zh12zm0GNmSeL18KdERERKTQmFk/M1uebel3XJI6wPZs6zsytx1/ngfMbCMZPToP/pFjs9NTVyIiIlJonHMjgZEnSGJ5HZbHed4G3jazm4EngNsLemx2CnRERETKqGJ6vHwHUC/bel1g1wnSjwXe/ZPHauhKREREitQyoJGZNTSzMDImF0/LnsDMGmVb7Qasz/x5GnCTmZUzs4ZAI+C7E2WmHh0REREpMs65NDPrD8wB/MAo59zPZjYCWO6cmwb0N7P2QCqwl4xhKzLTjQfWAGnAA865E3ZMKdARERGRIuWcmwnMPG7bsGw/DzzBsc8BzxU0LwU6IiIiZVTAFehx8FJNc3REREQkaCnQERERkaCloSsREZEySl8vFxERESnFFOiIiIhI0NLQlYiISBlVwI9wlmrq0REREZGgpUBHREREgpYCHREREQlamqMjIiJSRmmOjoiIiEgppkBHREREgpaGrkRERMoovRlZREREpBRToCMiIiJBS4GOiIiIBC3N0RERESmjAk6Pl4uIiIiUWgp0REREJGhp6EpERKSM0puRRUREREoxBToiIiIStDR0JSIiUkZp6EoAaNXmckbP/ZL/xMymzz135drfvNUF/HvqROatW0mbzh2ztkfUrs17UyYwctoXjJo1jR59bizKYpc6F7W5nM+iZzF2/hxuuffuXPtbtLqQD6dNYuGvq7miS6cc+1756H1m/fgdL3zwXlEVt9S4pE0bJsXMZfKCGG6/955c+0PDwnj+zdeZvCCGjydPJKpOHQBCQkMZ9uL/MnbWl3w2czoXXHRR1jH3PzyYGd98xeLVPxVZPUqLsMYXU+2x8VR7fCIV2t2Wa3/51t2o8exsqg75D1WH/IfyF/fM2lfz1SVZ20+966WiLHaJd3nbtsyaH8OcRQu5+777cu0PDQvj1bfeYs6ihYybMoU6devm2B9VuzYr1vzMnf2O3Vtivv6aaXNmM3nmTCZOn+Z5HaR4qEfnJHw+HwOffoIht9/F7rh43v1iHEtiFrB1w8asNPG7YnnhkcfofdcdOY5N3L2bAb1vJvVIKuUrVGDUzKksiZlPYsLuoq5Giefz+Rg8fBiDbruThLh4Ppgyga+j57PluHZ+/pFH6XPXnbmO/+z9DylfPpyeNyuYzM7n8/GvEU/zwK23Ex8Xx5ipX7A4OobNGzZkpenV+wb2JSdzzZXt6Ni9GwOGPsJjAwZyzU0ZbXlTl25UrV6NNz4axW29rsE5x+Lo+Ywb/R8mL4gurqqVTOaj0vVD2PvuANKTEqg6+GMOr/6KQPzmHMkO/RDN/kkv5zrcpR5m70u3FlVpSw2fz8ewZ0Zw599vIT4ujgnTpjE/eh4b1x+7jq+/sTcpycl0ansFXXv04J9DhzK4f/+s/Y8Oe5KvFi7Mde7bbupD0t69RVENKSbq0TmJxi2asXPrNmK37yAtNZX5X87i0vZX5UgTv3MXm375lfT09Bzb01JTST2SCkBYWCjmU3Pn55wWzdmxdRu7Mts5esZMLu/QLkeauJ072bjuV9LTc3e1rljyLQcPHCiq4pYaTVq0YPvWrezcvp201FTmTv+Sth3a50jTtkN7ZkyaDEDMrNm0vvQSABo2OpNlS5YCsDdxD/tSUji3eTMAVv/4I4m7FbAfL6T+uaT9toP0xF0QSOPwD/Mo16xNcRer1GvesiXbtmxlx/btpKamMnP6dNp16JgjTbsOHZkyaRIAc2bO5JLLLj22r2NHtm/bxoZf1xdpuaVk8Pw3r5lda2avmtkrZnaN1/kVthoRESTExmWt/xYXR82IWgU+vmZUJO/PmMzYr+YzduQH6s3JR83ICBJiY7PWd8fGUTMiohhLFBxqRUYQn61dE+LiqBWZs11rRRxLEwgE2L9vP6dWrcr6tWtp26E9fr+f2nXrck6zpkRERRVp+Usb/6m1SN8bn7WenpSA79SaudKVa34l1R75hMp9/wdflWP3EwsJo+rgj6n60IeEKUDKEhEZQWzsrqz1uNhYIo6/jiMjiN2VkSYQCLBv3z6qVK1KeHg4d993L2//3+u5zutwfPjJf5g0Yzq9+/TxthIlVLpzhb6UNJ4OXZnZO8CZwOeZm+4xs/bOuQfySd8P6Adwds1Ialeu6mXxCsTMcm37I/8fd8fGcXf3a6heqybPvPsmi2fNZW9iYiGWMDjkbmVwJfAfTKmT5/XrTpoG55g2fiINzziTMdMmE7dzFytXfE8gEPCooEHsuPY+vPorDq2YC4FUyl96DZVvfoqkdzJuiYnDe5Ge8hu+6rWp+sDbJO3aSCBxZ3GUuoQ5+XWc170a5xgweBAff/AhBw8ezLX75muvIyEhgWrVqzPqk0/YtHEjy7/7rtBKLSWD13N02gJNXeYVaWajgVX5JXbOjQRGAlx15rkl4rfc7rg4akVFZq3XiIzkt4SEP3yexITdbFm/kWatLmDx7LmFWcSgkBAXT61svQU1o/5cO0tOCbFxOXphakVGsjs+Z7smxGWkSYiLw+/3U7FSRZKTkgB49dnnstJ9OHE82zZvKZJyl1aB5AR8VY/1NPiq1CI95bccadzBlKyfDy2dSsUex+aRHE2bnriLIxu+J6Tu2Qp0gPi4OKKiametR0ZFkXDcdRwfG0dU7drEZ17HlSpVIikpieYtW9KpS1eGPPoolSpXJt2lc/jwYT4dPYaEzHvMnsREoufMoXnLFgp0gpDXQ1e/AKdlW68HrPQ4z0K1buVq6tSvT2TdOoSEhnJVty4sjVlQoGNrREYQVq4cABUrV6bJBeexfdPmkxxVNq1buYp6DeoTldnO7bt35Zvo+cVdrFJvzcqV1GtQn9p16xISGkrHHt1YHB2TI83i6Bi6X5cxqtyuS2eWLf0WgHLly1M+PByAiy6/jEAgLcckZsktbdtaQmrUw1ctCvwhlDuvA4dXL86Rxle5etbPYU3/RiB+CwAWXgn8oRk/n3IqoQ1bkBan+wXAqp9+on7DBtSpV5fQ0FC69ujB/HnzcqSZHz2Pq6+7DoBOXbvy7ZIlANxyQ2/aXX457S6/nDGjRjHy7bf5dPQYwsPDOeWUUwAIDw/nsjZ/49dffi3SepUEAVyhLyWN1z061YG1ZnY0RG4FLDWzaQDOuZ75HllCpAcCvDn8OV746H38fh+zJkxmy/oN9B3Yn19X/8ySmAWc3awpI959g4qVK3PJVVfSd2B/7uzSk/pnnM69jz6S0XVtxvgPPmKzJsPlKRAI8OrTz/Dq6A/x+Xx8OWESm9dv4B8PDWDdqtV8E7OAxs2b8vy7b1Hp1Mpc1u5K/jGwP7d27gHA2+M+4bTTT6fCKRX44puF/O/QJ/juq6+LuVbFLxAI8NJTw3lzzEf4fX6mTZjApvXruWfQQNauWs3i6BimjhvPiNdeYfKCGFKSk3js/9u78/ioqvv/469PBpCIKAQri6IgbhUE3CuiWFlUFG2FSq3rz1ZrFVSoVqwWFX7fVqvF1q8LoqKiVQggsioQFBQ3UCurCyA7SbBIggoqmXy+f9ybMISEzdxJMvN+Ph7zyF3O3HPvmTt3PrnnnHv63gJAVqNGPDLiGYqLi1mfl8/A/reWbvemAX/inAsvpG5mJpPfmc34UdkM+9fDVXWY1UdxnK/HPkiD6x/GMjLY8v5E4nnLqXfedWxd9Qk/LHqLzDN7s0/rM/DiOL55E5teHARArHEL9r9kAO6OmbE557kdemulq3g8zuCBA3l6xAgyYjHGZmezdMkS+vbvx8L5C3gjJ4cxo7L5+0NDmDprJoUFBfTv03en22x04IE8MmwYALFaMSaNH8/sWbOScTiSZBZlOwgz67Sz9e5e4VlVXaquUtkPZXqJSeX7zouqehfSwpSLGu06kfwoZ47L33Ui+dE+XbmivCaLkbmpTcdK/619eOHspB7DrkR6R2dngYyIiIhI1CIJdMxstrt3NLOvYbsKOwPc3fePIl8RERHZfdWxTU1liyTQcfeO4d/6UWxfREREZHdE2uvKzH5bzrL7osxTREREpETUva56mdl37v5vKH2AYN2I8xQREZHdEE+DB7NGHehcDEwws2LgPOArd78h4jxFREREgOgaI2clzP4OeAV4GxhkZlnu/lUU+YqIiIgkiuqOzofs2NvqfKB7OH94RPmKiIjIbkqHXleRNEZ295bufjgwAGjv7i2B4cA8oFcUeYqIiIiUFfVYV3e5+yYz6wh0BZ4FHo84TxEREREg+kAnHv49Hxjq7uOBOhHnKSIiIgJE3+tqrZk9AXQB7jezfYg+uBIREZHdUJwG3cujDhtAEAIAABptSURBVDouAaYC57p7AZAF3BZxniIiIiJA9IN6bgZeTpjPBXKjzFNERESkRNRVVyIiIlJNqXu5iIiISA2mQEdERERSlgIdERERSVlqoyMiIpKm1EZHREREpAZToCMiIiIpS4GOiIiIpCwFOiIiIpKy1BhZREQkTWmsKxEREZEaTIGOiIiIpCxVXYmIiKQpPUdHREREpAZToCMiIiIpS1VXIiIiaSquXlciIiIiNZcCHREREUlZCnREREQkZamNjoiISJoqVvdyERERkZpLgY6IiIikLFVdiYiIpCl1LxcRERGpwartHZ0VW7+t6l1IeXWwqt6FlLevVduvWEr5+bj1Vb0LKe/TKf+o6l0Q2Su6oyMiIiIpS/9uioiIpKlitdERERERqbkU6IiIiEjKUqAjIiKSpuJ4pb92h5mda2afmdlSMxtQzvozzewjMysys15l1sXN7OPwNWFXeamNjoiIiCSNmcWAR4GuwBpgrplNcPfFCclWAVcDt5aziS3u3n5381OgIyIiIsl0CrDU3b8AMLORwEVAaaDj7ivCdcU/NjMFOiIiImmq2H90HLE3DgZWJ8yvAU7dg/fXNbMPgCLgPnd/ZWeJFeiIiIhIpTGz64DrEhYNc/dhiUnKedue9HM/1N3XmdnhwOtmtsDdl1WUWIGOiIiIVJowqBm2kyRrgOYJ84cA6/Zg++vCv1+Y2UzgeKDCQEe9rkRERCSZ5gJHmllLM6sD/BrYZe8pADNraGb7hNMHAqeT0LanPLqjIyIikqaK96jGqHK4e5GZ9QGmAjFguLsvMrNBwAfuPsHMTgbGAQ2BHmZ2r7u3Bn4KPBE2Us4gaKOjQEdERESqD3efAkwps2xgwvRcgiqtsu97BzhuT/JS1ZWIiIikLN3RERERSVNxDeopIiIiUnMp0BEREZGUpUBHREREUpba6IiIiKSpquhenmy6oyMiIiIpS4GOiIiIpCxVXYmIiKSpYnUvFxEREam5FOiIiIhIylLVlYiISJoqruodSALd0REREZGUpUBHREREUpYCHREREUlZaqMjIiKSptS9XERERKQGU6AjIiIiKUtVVyIiImlKg3qKiIiI1GAKdERERCRlKdARERGRlKU2OiIiImlK3ctFREREajAFOiIiIpKyVHUlIiKSptKhe7kCnQqc2akTA+++m4xYjOyRIxn6+OPbra9Tpw4PDhlCm+OOo2DjRvr26cPaNWto0KABjw4dStu2bRk7Zgz3DBxY+p4XR47koIMO4rvvvgPgqiuuYMOGDUk9ruqmY6dO3Hn3QDJiMcaMHMWTZcq5dp063D9kCK2Pa0PBxgL6h+VcommzZkzKmc6j//wnw4c9CcCM2bP59ttviMeLiceL6NXjwqQeU3XWodOZ3DrwLmKxGONGZfPs409st752nToMHvIAP23ThoKCjQzoczO5a9ZSq1Yt/nL/XzmmdWtq1Yox6eVXeOaxoVV0FNVPx05nMuDuu4nFMhg7chRPPb592dSuU4e/DflH6Xn8xz59WLdmLc0OOZiJM3JYsewLAOb95z8MuvMuALpf2INrb7wBd/gyP5/bb+lHwcaNST+26urNjz7nf4ZPori4mF91OZnrLu603fq/Dp/M+wuDcv3u+x/YUPgtH7wQXI//PuJVZn34GcXFzuntjuDO316AmSX9GCQ5FOiUIyMjg3sHD+bKyy4jLy+PVyZMICcnh6VLlpSmuaR3bzYVFnJ2p05c0KMHtw8YwE19+vD999/z0IMPctTRR3PU0UfvsO1+N9/MggULknk41VZGRgYDBw/imssuJz8vj9ETJvB6znSWLVlamqZX70vYVFjIOZ3OonuPHvxxwAD69+lTuv6OgX/hrZkzd9j2lb++VD8KZWRkZHD7oHu44fKryM/L44UJLzNr+gyWL91W3r+45FdsKizkorM6063H+dw84E8M6HMzXbqfR506deh97vnUrVuXMTmv8dqEieSuWVuFR1Q9ZGRkcOfgQVx72RXk5+UxasJ43sjJ2e487hmex+d1+jnn9biA/gMGcGufvgCsXrmSnt3P326bsViMAXcP5MIu3SjYuJE/3jGA31x1JY/9819JPbbqKh4vZtCTE3jm7mto3Gh/ev3pMc4++RiOaN64NM2fr9lWps9PfofFy3MB+OjTlXz0yUomDLkJgN/c+QRzFi3n1DaHJ/cgJGnURqcc7dq3Z+WKFaxevZqtW7cyaeJEunbtul2aLl27MnbsWABenTKFDqefDsCWLVv44IMP+P7775O+3zVN2/btWbViJWvCcp4ycSKdu3bbLk3nrt14JSznqVOmcNrpHbat69aN1atWsfTzJciutWnfjjUrV7J29WqKtm5l6sTJnNWty3ZpzurWhUljxwEwY8prnNzhNAAcJzNzX2KxGPvUrcvWH7by7dffJP0YqqPj2rdjdZnz+Odlrhdnd+3K+PA8njblVX6WcB6Xx8wwMzL33ReAevvtx5f566M5gBpo/tI1HNa0Ec2bZFGndi3O79iWGXM+qTD95NnzuaBjWyAo2x+2FrG1KM4PRUVsjRdzYIP9krXr1U4xXumv6kaBTjmaNGlCbm5u6Xxubi6NmzTZLk3jJk3IXbcOgHg8ztdff03Dhg13ue2/P/ggk6ZMoc9NN1XuTtdAjZs0Jjd3Xel8Xm4ujZs03i7NQU0a71DODRo2JDMzk2v/cD2PlvMfruM8/cLzjJ00kUsuvTTag6hBftK4MXnrtp3X63PzOKhx4wrTxONxvvn6Gxo0bMiMKa+xZctmps15lynvvMnzTz7FpsLCpO5/ddW4zPUiPzdvh+vFQU22L9eS8xjg4ObNGTNlEs+OGskJJ58MQFFREYPv+guvTH2VmXPfp9WRRzJ21KgkHVH1l7+hkCaNDiidb9zoAPK/2lRu2rXrN7ImfyM/O64VAMcffSintjmcjr/9Gx1/+zfOaH8krQ45KCn7LVUj0qorMzsKuA04LDEvdz+7gvTXAdcBNMrKYv/9qk+U7WWeNVBefW7ZNGX1u/lm8vPzqVevHo8NHcovL76YcS+/XKn7WbPsugzLrTd3p2//fjz71NNs3rx5h9W/ubgn69evJ6tRI4a/8AJfLFvGB3PmVNpe11S7c85WlKZ1u7bE43HOObUD9Q/Yn6ezR/L+7HdYu3p1ZPtbc+x9uX65/ku6nHY6hQUFHNumDQ8/+QQXdT2H77/7jt6XX0av7hewetUq7hx0L9feeANP/O8jkR1FTVLelbaiFjaTZ8/nnNNaE4sF/9evzN3AsjVfMuvJ2wG45t7hzF20nJNbt4xmZ6XKRX1HZzTwEXAXQcBT8iqXuw9z95Pc/aSqDHLy8vJo2rRp6XzTpk1Zn5+/fZrcXJo2awYE9en169enoKBgp9vND7fx7bffMmH8eNq1b1/Je16z5Ofl0bRps9L5Jk2bsr7M7fn83Lxyy7lt+/bcdscdzJg9myuvuYbrbryRy666EoD164NtfLVhAzlTp9K2fbskHVH1tj4vjybNtp3XBzVtwpfr11eYJhaLsV/9/SgsKOC8iy7k3VlvUVRUxMYNXzHvww85tu1xSd3/6io/L3e760Xjpk12uF7k525frvXr16ewoICtP/xAYXjdWLxwIatXrqJFy5Ycc+yxAKxetQqA1yZNpv2JJyTjcGqEJo0OIG/DtjuK+RsKOShr/3LTTnl7Puefse0aMP39RbQ7qjn1MvehXuY+nHHCUXz8uQL2VBZ1oFPk7o+7+xx3/7DkFXGeP9r8efNo0bIlhzRvTu3atbmgRw9ypk/fLs2MnBx69uwJwHndu/PuO+/sdJuxWKy0aqtWrVqc3bkzn3/2WTQHUEMsmDePw1q24ODmh1C7dm269+jB62XK+fWc6fwiLOdzunfnvbCcL//VJXTu2JHOHTsyYvhwhj36KP9+bgSZmZnUq1cPgMzMTE4/8ww+/+zzpB5XdbVo3nyatziMZoccQq3atTmnx/nMmj5juzSzps/ggp6/BKBz93OZ+857AOSuW8fJHX4GQN3MTI47/nhWLFuW3AOophbOm8+hLbc/j9+YnrNdmjdycrgoPI+7dT+P9995F4CGWVlkZASX4UOaN+ewli1Ys2oV+Xl5tDrySBpmZQHQ4YyOfLFU5V3iuCMOZkXuf1md/xU/bC1i8uz5nH3yT3dI98XaL9n0zRaOP/rQ0mXNDmzA3MXLKYrH2VoUZ+6i5bQ65CfJ3P1qpdgr/1XdRFJ1ZWZZ4eREM7sBGAeUts5196+iyLeyxONx7hk4kOdGjCAjFmN0djZLlizhlv79WTB/PjNychg1ahRDHnqI12fNorCggJsSegK9OXs2+9WvT+3atenarRtXXXEFa9es4dnnn6d2rVpkxGK8PXs2I196qQqPsurF43EGDxzI02E5j83OZumSJfTt34+F8xfwRk4OY0Zl8/eHhjB11kwKCwroH/ZUqUijAw/kkWHDAIjVijFp/Hhmz5qVjMOp9uLxOPcPvJdHRzxDRizGhOzRfLFkCdf3u5nFCxbyZs4MXsnOZvCQfzB+5gwKCwq4o+8tAGSPeIF7Hrif0dNexcyYMHoMSz5N70C9RDwe538G3s2wESPIiGUwLns0y5YsoU//fiwKz+Oxo0Zx30MP8eqsNygsKCztcXXSqafQp38/4kVx4sVxBv35LgrDtk+P/fNfPDd6FEVbi8hdu5Y///HWqjzMaqVWLMbA313I7wY9Q7zY6dn5RI48tDH/emk6bVodQudTgqBn8lvz6N6x7XZVh+ec1ob3Fiyjxy0PYwZnHH9UuUGSpA7bVbuSvdqo2XKCatTyqk3d3XfZj+/www6rhnFhaqlTYa22VJZ9TU9wSIbvPV7Vu5DyFk15sKp3IT207pnUC/MJLY6o9N/aj1YsrVY/LpFchd1drbpERESquerYHbyyRdpGx8xuNLMGCfMNw6osERERkchF3Rj5Wncv7Yrk7huBayPOU0RERASIPtDJsIRWYGYWA+pEnKeIiIgIEP1YV1OBbDMbStA4+XrgtYjzFBERkd2QDm10og50bgd+D/yBoAfWNOCpiPMUERERASIOdNy92MyeBV53dz10Q0RERJIq6l5XFwIfE1ZXmVl7M5sQZZ4iIiKye9wr/1XdRN0Y+W7gFKAAwN0/BlpEnKeIiIgIkJyxrgp3nUxERESk8kXdGHmhmf0GiJnZkcBNwM5HvxQREZGkSIdeV1Hf0ekLtCYY0PNFoBC4JeI8RURERIDoe11tBu40s7+6+7dR5iUiIiJSVtS9rjqY2WLgk3C+nZk9FmWeIiIiIiWirrp6CDgH2ADg7vOAMyPOU0RERHaDR/CqbqIOdHD31WUWxaPOU0RERASi73W12sw6AG5mdQh6XX0ScZ4iIiIiQPSBzvXAv4CDgbUEg3zeGHGeIiIishvSoXt51L2u/gtcFmUeIiIiIhWJutfV4WY20cy+NLP1ZjbezA6PMk8RERGRElE3Rn4RyAaaAs2A0cBLEecpIiIiAkQf6Ji7P+/uReHrBapn7zMREZG0kw7dy6NujPyGmQ0ARhIcf29gspllAbj7VxHnLyIiImks6kCnd/j392wL9Ay4JpxXex0RERGJTNRVV7cD7dy9JfAMMA/o6e4t3V1BjoiISBVKh6qrqAOdu9x9k5l1BLoCzwKPR5yniIiICBB9oFMy3MP5wFB3Hw/UiThPERERESD6NjprzewJoAtwv5ntQxLG1xIREZFdS4cnI0cddFxCMOzDue5eAGQBt0Wcp4iIiAgQ/RAQm4GXE+Zzgdwo8xQREREpoWokERERSVlRt9ERERGRair1W+jojo6IiIikMAU6IiIikrJUdSUiIpKmVHUlIiIiUoMp0BEREZGUpUBHREREUpba6IiIiKSpdGijY+7pcJjJYWbXufuwqt6PVKYyjp7KODlUztFTGQuo6qqyXVfVO5AGVMbRUxknh8o5eipjUaAjIiIiqUuBjoiIiKQsBTqVS3XB0VMZR09lnBwq5+ipjEWNkUVERCR16Y6OiIiIpCwFOiIiIpKyFOhExMwamNkNCfPNzGxMVe5TqjCzFmb2m7187zeVvT+pxMyuN7Mrw+mrzaxZwrqnzOzYqtu7msvMVpjZgeUsf6cq9kfKV9HnJDWbAp3oNABKAx13X+fuvapwf1JJC6DcQMfM9LTvH8Hdh7r7iHD2aqBZwrrfufviKtmxFOXuHap6H0RSXdoGOuFdgU/M7EkzW2Rm08ws08xamdlrZvahmb1lZseE6VuZ2XtmNtfMBpXcGTCz/cxshpl9ZGYLzOyiMIv7gFZm9rGZPRDmtzB8z/tm1jphX2aa2YlmVs/Mhod5/CdhWylhL8r8WTPrlfD+krsx9wFnhGXbL7zzMNrMJgLTdvKZpLSwfD81s+fMbL6ZjTGzfc2sc3g+LQjPr33C9PeZ2eIw7YPhsnvM7Naw3E8C/h2Wc2Z4np5kZn8ws78n5Hu1mf1vOH25mc0J3/OEmcWqoiyqUvg9nmxm88xsoZn1TliXGZ7r14bzJdeRs8zsTTMbF34mQ80sba/PUH45hndc7g/PsTlmdkSY9idmNja8ds41s9MTtrHDNdXMYmb2YPidmG9mfROy7ptw7TimCg5dKpu7p+WL4K5AEdA+nM8GLgdmAEeGy04FXg+nJwGXhtPXA9+E07WA/cPpA4GlgIXbX1gmv4XhdD/g3nC6KfB5OP1X4PJwugHwOVCvqsuqCsv8WaBXwvtLyvwsYFLC8quBNUDWzj6TxG2k4issXwdOD+eHA3cBq4GjwmUjgFuALOCzhHJpEP69B7g1nJ4JnJSw/ZkEwc9PgKUJy18FOgI/BSYCtcPljwFXVnW5VMHn0BN4MmH+AGBF+PnkJJZJmXP6O+BwIAZMTzz30/G1k3K8M5y/suQ6ALwIdAynDwU+CafLvaYCfwDGArXCdSXXjhVA33D6BuCpqi4HvX78K63/YwCWu/vH4fSHBBeiDsBoM/sYeIIgEAE4DRgdTr+YsA0D/mpm8wkuYgcDjXeRbzbwq3D6koTtdgMGhHnPBOoSfGlTyZ6U+Z6Y7u5fhdN785mkitXu/nY4/QLQmaDMPw+XPQecCWwi+GF9yswuBjbvbgbu/iXwhZn9zMwaAUcDb4d5nQjMDT/LzgQ/3OlmAdAlvPNwhrsXhsvHA8/4tqrBsua4+xfuHgdeIgge01lF5fhSwt/TwukuwCPheTcB2N/M6lPxNbULMNTdiwASrh0AL4d/S65PUsOle3uG7xOm4wQ/hgXu3n4PtnEZwX+4J7r7VjNbQfBlqpC7rzWzDWbWFugN/D5cZUBPd/9sD/KvafakzIsIq1fNzIA6O9nutwnTe/yZpJDdejCWuxeZ2SkEwcivgT7A2XuQzyiCIP1TYJy7e/gZPefud+zhPqcUd//czE4EugN/M7Np4aq3gfPM7EV3L+9zKrssrR9ytpNyTCyXkukM4DR335K4jfCc3OGaGi6vqHxLrlFx9BuZEtL9jk5Zm4DlZvYrCL4MZtYuXPcewa1UCH4YShwArA9/UH8OHBYu/xqov5O8RgJ/Ag5w9wXhsqkE9cMW5n/8jz2gGmBnZb6C4A4BwEVA7XB6V2Vb0WeSDg41s5L/ci8luKPVoqQtA3AFMMvM9iM496YQVGWVF2jurJxfBn4R5jEqXDYD6GVmBwGYWZaZpVPZA0EPS2Czu78APAicEK4aCGwgqNIrzylm1jJsm9MbmB35zlZjOynH3gl/3w2npxEE6yXvLTmfK7qmTgOut7DzgpllRXUcUvUU6OzoMuC3ZjYPWETwAwvBj0F/M5tDULVSchv138BJZvZB+N5PAdx9A/B22IjugXLyGUMQMGUnLBtM8GM+34KGy4Mr9ciqr4rK/EmgU1jmp7Ltrs18oChspNivnO2V+5mkiU+Aq8JquyzgIeD/EVQNLgCKgaEEAcykMN0sgnZjZT0LDLWwMXLiCnffCCwGDnP3OeGyxQRtgqaF253O3lVD1nTHAXPC6pI7gf+fsO4WoK4lNOZO8C5BQ/uFwHJgXNQ7Ws1VVI77mNn7wM1sO29vIvjOzzezxQTtKKHia+pTwKpw+Twq6MUpqUFDQOwmM9sX2BLeov81QcPktOjNIzWDmbUgaJzZpop3RfaQmZ1F0Aj8gqrel+osrIY+yd3/W9X7IjWH6h9334kEjd0MKACuqeL9ERERkV3QHR0RERFJWWqjIyIiIilLgY6IiIikLAU6IiIikrIU6IjUUGYWD7t+L7RgrK99f8S2zjKzSeH0hWY2YCdpG5jZDRWt38n77jGzW/d2H0VE9oYCHZGaa4u7tw+7k//AtmeHAKUPX9zj77i7T3D3+3aSpAHBOEAiItWeAh2R1PAWcIRtGyH+MeAjoLmZdTOzd8MRmUeHT0XGzM61YLTz2cDFJRuyYDTyR8LpxhaMqD0vfHUgeKhdq/Bu0gNhutssGCF6vpndm7CtO83sMzPLIRgTS0QkqRToiNRw4WPszyMYBBGCgGKEux9P8DTpu4Au7n4C8AHBE77rEjx5ugdwBtCkgs0/DMxy93YEj+BfBAwAloV3k24zs27AkcApBENJnGhmZ4bjFP0aOJ4gkDq5kg9dRGSX9MBAkZorM3w8PgR3dJ4GmgEr3f29cPnPgGMJhiOBYGDUd4FjCEY1XwJgZi8A15WTx9nAlQDhqNqFZtawTJpu4es/4fx+BIFPfYIBPzeHeUz4UUcrIrIXFOiI1Fxbyo76HgYziSO5GzDd3S8tk649lTc6tgF/c/cnyuRxSyXmISKyV1R1JZLa3gNOLxm93Mz2NbOjCAY6bWlmrcJ0l1bw/hnAH8L3xsxsf3Yc1XwqcE1C25+DwxHM3wR+aWaZZlafoJpMRCSpFOiIpDB3/xK4GngpHFH8PeAYd/+OoKpqctgYeWUFm7gZ+Hk48vmHQGt330BQFbbQzB5w92nAi8C7YboxQH13/wgYBXwMjCWoXhMRSSqNdSUiIiIpS3d0REREJGUp0BEREZGUpUBHREREUpYCHREREUlZCnREREQkZSnQERERkZSlQEdERERS1v8BWnYjVrqa0OUAAAAASUVORK5CYII=\n",
      "text/plain": [
       "<Figure size 720x576 with 2 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "from sklearn.metrics import confusion_matrix\n",
    "conf_mat = confusion_matrix(y_test, y_pred)\n",
    "conf_mat_ratios = np.round(conf_mat / np.sum(conf_mat, axis=0),3)\n",
    "fig, ax = plt.subplots(figsize=(10,8))\n",
    "\n",
    "labels = sorted(list(set(le.inverse_transform(y_pred))))\n",
    "sns.heatmap(conf_mat_ratios, annot=True, center=0, xticklabels=labels, yticklabels=labels)\n",
    "plt.ylabel('Actual')\n",
    "plt.xlabel('Predicted')\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/nerdikon/anaconda3/lib/python3.7/site-packages/sklearn/preprocessing/label.py:151: DeprecationWarning: The truth value of an empty array is ambiguous. Returning False, but in future this will result in an error. Use `array.size > 0` to check that an array is not empty.\n",
      "  if diff:\n",
      "/home/nerdikon/anaconda3/lib/python3.7/site-packages/sklearn/preprocessing/label.py:151: DeprecationWarning: The truth value of an empty array is ambiguous. Returning False, but in future this will result in an error. Use `array.size > 0` to check that an array is not empty.\n",
      "  if diff:\n"
     ]
    }
   ],
   "source": [
    "y_pred_labels = le.inverse_transform(y_pred)\n",
    "y_test_labels = le.inverse_transform(y_test)\n",
    "\n",
    "results = pd.DataFrame({'text':df_test.text, 'lemmatized_tokens':df_test.lemmatized_tokens,'predicted':y_pred_labels, 'actual':y_test_labels})\n",
    "errors = results[results.predicted != results.actual]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAskAAAHVCAYAAAADyWaQAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDIuMi4zLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvIxREBQAAIABJREFUeJzt3X+8XWV9J/rPdwg0/CoYiL6UIGEsCm0pAU8VBK2KtYLzAq1yxaslcplGR21re603dm5HOmO9tPDS1tspDhVKGCuUIiiitUEoY4cKmEBEJCJRo6RYiCARBlCBZ/7YK3h4OCEn4ZycE3y/X6/92ms969lrf/c5K2t/8pxnr12ttQAAAD/xb2a6AAAAmG2EZAAA6AjJAADQEZIBAKAjJAMAQEdIBgCAjpAMAAAdIRkAADpCMgAAdObMdAFJsvfee7eFCxfOdBkAADzFrVy58nuttfmb6zcrQvLChQuzYsWKmS4DAICnuKr69mT6mW4BAAAdIRkAADpCMgAAdGbFnGQAAJIf//jHWbduXR588MGZLmW7N3fu3CxYsCA77rjjVj1eSAYAmCXWrVuX3XffPQsXLkxVzXQ5263WWu66666sW7cu+++//1btw3QLAIBZ4sEHH8xee+0lID9JVZW99trrSY3IC8kAALOIgDw1nuzPUUgGAICOOckAALPUwqWfmdL9rT3t1Zvts8MOO+Tggw/OQw89lIMOOijLli3LLrvsslXPd9VVV+WMM87IZZddlksvvTQ333xzli5dOmHfe+65Jx//+Mfz9re/fYue49RTT81uu+2Wd7/73VtV46YYSQYA4FE777xzVq1alZtuuik77bRTPvKRjzxme2stjzzyyBbv97jjjttkQE5GIfkv//Ivt3i/00VIBgBgQi9+8YuzZs2arF27NgcddFDe/va357DDDsttt92W5cuX54gjjshhhx2WE044Iffdd1+S5HOf+1wOPPDAHHXUUbn44osf3de5556bd77znUmSO+64I6997WtzyCGH5JBDDsk///M/Z+nSpfnGN76RRYsW5fd///eTJKeffnp++Zd/Ob/0S7+U973vfY/u64//+I/zvOc9L694xStyyy23TMtrF5IBAHichx56KH//93+fgw8+OElyyy235KSTTsoNN9yQXXfdNe9///vz+c9/Ptdff33GxsbywQ9+MA8++GB+8zd/M5/+9KfzT//0T/nXf/3XCff927/92/mVX/mVfPnLX87111+fX/iFX8hpp52W5zznOVm1alVOP/30LF++PLfeemuuu+66rFq1KitXrswXvvCFrFy5MhdccEFuuOGGXHzxxfnSl740La/fnGQAAB71wAMPZNGiRUlGI8mnnHJKbr/99uy33345/PDDkyTXXHNNbr755hx55JFJkh/96Ec54ogj8rWvfS37779/DjjggCTJm9/85px11lmPe44rr7wy5513XpLRHOg99tgj3//+9x/TZ/ny5Vm+fHkOPfTQJMl9992XW2+9Nffee29e+9rXPjpP+rjjjpuGn4KQDADAOBvnJPd23XXXR5dba/nVX/3VnH/++Y/ps2rVqim7hF1rLe9973vz1re+9THtf/Znf7ZNLpNnugUAAFvk8MMPz9VXX501a9YkSe6///58/etfz4EHHphvfetb+cY3vpEkjwvRGx199NE588wzkyQPP/xwfvCDH2T33XfPvffe+2ifX/u1X8s555zz6Fznf/mXf8mdd96Zl7zkJbnkkkvywAMP5N57782nP/3paXmNRpIBAGapyVyybSbMnz8/5557bt74xjfmhz/8YZLk/e9/f5773OfmrLPOyqtf/ersvffeOeqoo3LTTTc97vF//ud/niVLluTss8/ODjvskDPPPDNHHHFEjjzyyPziL/5ijjnmmJx++ulZvXp1jjjiiCTJbrvtlo997GM57LDD8oY3vCGLFi3Kfvvtlxe/+MXT8hqrtTYtO94SY2NjbcWKFTNdBgDAjFq9enUOOuigmS7jKWOin2dVrWytjW3usaZbAABAR0gGAICOOcnjTPVXP26N2Tr3CADgp8lmR5Kr6nlVtWrc7QdV9a6qmldVl1fVrcP904b+VVUfrqo1VXVjVR02/S8DAACmzmZDcmvtltbaotbaoiTPT3J/kkuSLE1yRWvtgCRXDOtJckySA4bbkiRnTkfhAAAwXbZ0TvLRSb7RWvt2kuOTLBvalyV5zbB8fJLz2sg1SfasqmdOSbUAALANbOmc5BOTbLwq9DNaa99Nktbad6vq6UP7PkluG/eYdUPbd8fvqKqWZDTSnGc/+9lbWAYAwE+BU/eY4v1tmFS3Sy65JL/+67+e1atX58ADD9xkv3PPPTevfOUr86xnPWuryrnqqqtyxhln5LLLLtuqx0+nSY8kV9VOSY5L8neb6zpB2+MuxtxaO6u1NtZaG5s/f/5kywAAYJqdf/75Oeqoo3LBBRc8Yb9zzz03t99++zaqatvakukWxyS5vrV2x7B+x8ZpFMP9nUP7uiT7jnvcgiRPzZ8eAMBTzH333Zerr746Z5999mNC8p/+6Z/m4IMPziGHHJKlS5fmoosuyooVK/KmN70pixYtygMPPJCFCxfme9/7XpJkxYoVeelLX5okue666/KiF70ohx56aF70ohfllltumYmXtkW2ZLrFG/OTqRZJcmmSxUlOG+4/Na79nVV1QZIXJtmwcVoGAACz2yc/+cm86lWvynOf+9zMmzcv119/fe6444588pOfzLXXXptddtkld999d+bNm5e/+Iu/yBlnnJGxsSf+ArsDDzwwX/jCFzJnzpx8/vOfzx/8wR/kE5/4xDZ6RVtnUiG5qnZJ8qtJ3jqu+bQkF1bVKUm+k+SEof2zSY5NsiajK2GcPGXVAgAwrc4///y8613vSpKceOKJOf/88/PII4/k5JNPzi677JIkmTdv3hbtc8OGDVm8eHFuvfXWVFV+/OMfT3ndU21SIbm1dn+Svbq2uzK62kXftyV5x5RUBwDANnPXXXflyiuvzE033ZSqysMPP5yqyute97pUTfSxs8eaM2dOHnnkkSTJgw8++Gj7H/7hH+ZlL3tZLrnkkqxdu/bRaRizma+lBgAgSXLRRRflpJNOyre//e2sXbs2t912W/bff//Mmzcv55xzTu6///4kyd13350k2X333XPvvfc++viFCxdm5cqVSfKY6RQbNmzIPvvsk2T0Yb/tga+lBgCYrSZ5ybapcv7552fp0qWPaXvd616X1atX57jjjsvY2Fh22mmnHHvssfnABz6Qt7zlLXnb296WnXfeOV/84hfzvve9L6eccko+8IEP5IUvfOGj+3jPe96TxYsX54Mf/GBe/vKXb9PXtLVqNDtiZo2NjbUVK1bMdBlZuPQzM11C1p726pkuAQCYIatXr85BBx0002U8ZUz086yqla21J/6kYUy3AACAxxGSAQCgIyQDAMwis2Eq7FPBk/05CskAALPE3Llzc9dddwnKT1JrLXfddVfmzp271ftwdQsAgFliwYIFWbduXdavXz/TpWz35s6dmwULFmz144VkAIBZYscdd8z+++8/02UQ0y0AAOBxhGQAAOgIyQAA0BGSAQCgIyQDAEBHSAYAgI6QDAAAHSEZAAA6QjIAAHSEZAAA6AjJAADQEZIBAKAjJAMAQEdIBgCAjpAMAAAdIRkAADpCMgAAdIRkAADoCMkAANARkgEAoCMkAwBAR0gGAICOkAwAAB0hGQAAOkIyAAB0hGQAAOgIyQAA0BGSAQCgIyQDAEBHSAYAgI6QDAAAHSEZAAA6QjIAAHSEZAAA6AjJAADQEZIBAKAjJAMAQEdIBgCAjpAMAAAdIRkAADqTCslVtWdVXVRVX6uq1VV1RFXNq6rLq+rW4f5pQ9+qqg9X1ZqqurGqDpvelwAAAFNrsiPJf57kc621A5MckmR1kqVJrmitHZDkimE9SY5JcsBwW5LkzCmtGAAAptlmQ3JV/WySlyQ5O0laaz9qrd2T5Pgky4Zuy5K8Zlg+Psl5beSaJHtW1TOnvHIAAJgmkxlJ/rdJ1if566q6oao+WlW7JnlGa+27STLcP33ov0+S28Y9ft3Q9hhVtaSqVlTVivXr1z+pFwEAAFNpMiF5TpLDkpzZWjs0yf/KT6ZWTKQmaGuPa2jtrNbaWGttbP78+ZMqFgAAtoXJhOR1Sda11q4d1i/KKDTfsXEaxXB/57j++457/IIkt09NuQAAMP02G5Jba/+a5Laqet7QdHSSm5NcmmTx0LY4yaeG5UuTnDRc5eLwJBs2TssAAIDtwZxJ9vutJH9TVTsl+WaSkzMK2BdW1SlJvpPkhKHvZ5Mcm2RNkvuHvgAAsN2YVEhura1KMjbBpqMn6NuSvONJ1gUAADPGN+4BAEBHSAYAgI6QDAAAHSEZAAA6QjIAAHSEZAAA6AjJAADQEZIBAKAjJAMAQEdIBgCAjpAMAAAdIRkAADpCMgAAdIRkAADoCMkAANARkgEAoCMkAwBAR0gGAICOkAwAAB0hGQAAOkIyAAB0hGQAAOgIyQAA0BGSAQCgIyQDAEBHSAYAgI6QDAAAHSEZAAA6QjIAAHSEZAAA6AjJAADQEZIBAKAjJAMAQEdIBgCAjpAMAAAdIRkAADpCMgAAdIRkAADoCMkAANARkgEAoCMkAwBAR0gGAICOkAwAAB0hGQAAOkIyAAB0hGQAAOgIyQAA0BGSAQCgIyQDAEBHSAYAgM6kQnJVra2qr1TVqqpaMbTNq6rLq+rW4f5pQ3tV1Yerak1V3VhVh03nCwAAgKm2JSPJL2utLWqtjQ3rS5Nc0Vo7IMkVw3qSHJPkgOG2JMmZU1UsAABsC09musXxSZYNy8uSvGZc+3lt5Joke1bVM5/E8wAAwDY12ZDckiyvqpVVtWRoe0Zr7btJMtw/fWjfJ8lt4x67bmh7jKpaUlUrqmrF+vXrt656AACYBnMm2e/I1trtVfX0JJdX1deeoG9N0NYe19DaWUnOSpKxsbHHbQcAgJkyqZHk1trtw/2dSS5J8oIkd2ycRjHc3zl0X5dk33EPX5Dk9qkqGAAApttmQ3JV7VpVu29cTvLKJDcluTTJ4qHb4iSfGpYvTXLScJWLw5Ns2DgtAwAAtgeTmW7xjCSXVNXG/h9vrX2uqr6U5MKqOiXJd5KcMPT/bJJjk6xJcn+Sk6e8agAAmEabDcmttW8mOWSC9ruSHD1Be0vyjimpDgAAZoBv3AMAgI6QDAAAHSEZAAA6QjIAAHSEZAAA6AjJAADQEZIBAKAjJAMAQEdIBgCAjpAMAAAdIRkAADpCMgAAdIRkAADoCMkAANARkgEAoCMkAwBAR0gGAICOkAwAAB0hGQAAOkIyAAB0hGQAAOgIyQAA0BGSAQCgIyQDAEBHSAYAgI6QDAAAHSEZAAA6QjIAAHSEZAAA6AjJAADQEZIBAKAjJAMAQEdIBgCAjpAMAAAdIRkAADpCMgAAdIRkAADoCMkAANARkgEAoCMkAwBAR0gGAICOkAwAAB0hGQAAOkIyAAB0hGQAAOgIyQAA0BGSAQCgIyQDAEBHSAYAgM6kQ3JV7VBVN1TVZcP6/lV1bVXdWlV/W1U7De0/M6yvGbYvnJ7SAQBgemzJSPLvJFk9bv1PknyotXZAku8nOWVoPyXJ91trP5fkQ0M/AADYbkwqJFfVgiSvTvLRYb2SvDzJRUOXZUleMywfP6xn2H700B8AALYLkx1J/rMk70nyyLC+V5J7WmsPDevrkuwzLO+T5LYkGbZvGPo/RlUtqaoVVbVi/fr1W1k+AABMvc2G5Kr6d0nubK2tHN88Qdc2iW0/aWjtrNbaWGttbP78+ZMqFgAAtoU5k+hzZJLjqurYJHOT/GxGI8t7VtWcYbR4QZLbh/7rkuybZF1VzUmyR5K7p7xyAACYJpsdSW6tvbe1tqC1tjDJiUmubK29Kck/Jnn90G1xkk8Ny5cO6xm2X9lae9xIMgAAzFZP5jrJ/0+S36uqNRnNOT57aD87yV5D++8lWfrkSgQAgG1rMtMtHtVauyrJVcPyN5O8YII+DyY5YQpqAwCAGeEb9wAAoCMkAwBAR0gGAICOkAwAAB0hGQAAOkIyAAB0hGQAAOgIyQAA0BGSAQCgIyQDAEBHSAYAgI6QDAAAHSEZAAA6QjIAAHSEZAAA6AjJAADQEZIBAKAjJAMAQEdIBgCAjpAMAAAdIRkAADpCMgAAdIRkAADoCMkAANARkgEAoCMkAwBAR0gGAICOkAwAAB0hGQAAOkIyAAB0hGQAAOgIyQAA0BGSAQCgIyQDAEBHSAYAgI6QDAAAHSEZAAA6QjIAAHSEZAAA6AjJAADQEZIBAKAjJAMAQEdIBgCAjpAMAAAdIRkAADpCMgAAdIRkAADoCMkAANARkgEAoCMkAwBAZ7MhuarmVtV1VfXlqvpqVf3R0L5/VV1bVbdW1d9W1U5D+88M62uG7Qun9yUAAMDUmsxI8g+TvLy1dkiSRUleVVWHJ/mTJB9qrR2Q5PtJThn6n5Lk+621n0vyoaEfAABsNzYbktvIfcPqjsOtJXl5kouG9mVJXjMsHz+sZ9h+dFXVlFUMAADTbFJzkqtqh6paleTOJJcn+UaSe1prDw1d1iXZZ1jeJ8ltSTJs35Bkrwn2uaSqVlTVivXr1z+5VwEAAFNoUiG5tfZwa21RkgVJXpDkoIm6DfcTjRq3xzW0dlZrbay1NjZ//vzJ1gsAANNui65u0Vq7J8lVSQ5PsmdVzRk2LUhy+7C8Lsm+STJs3yPJ3VNRLAAAbAuTubrF/Krac1jeOckrkqxO8o9JXj90W5zkU8PypcN6hu1XttYeN5IMAACz1ZzNd8kzkyyrqh0yCtUXttYuq6qbk1xQVe9PckOSs4f+Zyf571W1JqMR5BOnoW4AAJg2mw3JrbUbkxw6Qfs3M5qf3Lc/mOSEKakOGDl1j5muIDl1w0xXAADbzGRGkgEA2J4ZbNlivpYaAAA6QjIAAHSEZAAA6AjJAADQEZIBAKAjJAMAQEdIBgCAjpAMAAAdIRkAADpCMgAAdIRkAADoCMkAANARkgEAoCMkAwBAR0gGAICOkAwAAB0hGQAAOkIyAAB0hGQAAOgIyQAA0BGSAQCgIyQDAEBHSAYAgI6QDAAAHSEZAAA6QjIAAHSEZAAA6AjJAADQEZIBAKAjJAMAQEdIBgCAjpAMAAAdIRkAADpCMgAAdObMdAEwmy1c+pmZLiFJsnbuTFcAAD9dhOTZ5tQ9ZrqCkVM3zHQFAAAzxnQLAADoCMkAANARkgEAoCMkAwBAR0gGAICOkAwAAB0hGQAAOkIyAAB0hGQAAOgIyQAA0BGSAQCgIyQDAEBnsyG5qvatqn+sqtVV9dWq+p2hfV5VXV5Vtw73Txvaq6o+XFVrqurGqjpsul8EAABMpcmMJD+U5P9urR2U5PAk76iqn0+yNMkVrbUDklwxrCfJMUkOGG5Lkpw55VUDAMA02mxIbq19t7V2/bB8b5LVSfZJcnySZUO3ZUleMywfn+S8NnJNkj2r6plTXjkAAEyTLZqTXFULkxya5Nokz2itfTcZBekkTx+67ZPktnEPWze0AQDAdmHSIbmqdkvyiSTvaq394Im6TtDWJtjfkqpaUVUr1q9fP9kyAABg2k0qJFfVjhkF5L9prV08NN+xcRrFcH/n0L4uyb7jHr4gye39PltrZ7XWxlprY/Pnz9/a+gEAYMpN5uoWleTsJKtbax8ct+nSJIuH5cVJPjWu/aThKheHJ9mwcVoGAABsD+ZMos+RSX4jyVeqatXQ9gdJTktyYVWdkuQ7SU4Ytn02ybFJ1iS5P8nJU1oxAABMs82G5Nba/8zE84yT5OgJ+rck73iSdQEAwIyZzEgyALC9OHWPma4gOXXDTFcAT5qvpQYAgI6QDAAAHSEZAAA6QjIAAHSEZAAA6AjJAADQEZIBAKAjJAMAQEdIBgCAjpAMAAAdIRkAADpCMgAAdIRkAADoCMkAANARkgEAoCMkAwBAR0gGAICOkAwAAB0hGQAAOkIyAAB0hGQAAOgIyQAA0BGSAQCgIyQDAEBHSAYAgI6QDAAAHSEZAAA6QjIAAHSEZAAA6AjJAADQEZIBAKAjJAMAQEdIBgCAzpyZLgCArXTqHjNdQXLqhpmuAGBaGEkGAICOkAwAAB0hGQAAOkIyAAB0hGQAAOgIyQAA0BGSAQCgIyQDAEBHSAYAgI6QDAAAHSEZAAA6QjIAAHSEZAAA6AjJAADQEZIBAKCz2ZBcVedU1Z1VddO4tnlVdXlV3TrcP21or6r6cFWtqaobq+qw6SweAACmw2RGks9N8qqubWmSK1prByS5YlhPkmOSHDDcliQ5c2rKBACAbWezIbm19oUkd3fNxydZNiwvS/Kace3ntZFrkuxZVc+cqmIBAGBb2No5yc9orX03SYb7pw/t+yS5bVy/dUPb41TVkqpaUVUr1q9fv5VlAADA1JvqD+7VBG1too6ttbNaa2OttbH58+dPcRkAALD1tjYk37FxGsVwf+fQvi7JvuP6LUhy+9aXBwAA297WhuRLkywelhcn+dS49pOGq1wcnmTDxmkZAACwvZizuQ5VdX6SlybZu6rWJXlfktOSXFhVpyT5TpIThu6fTXJskjVJ7k9y8jTUDAAA02qzIbm19sZNbDp6gr4tyTuebFEAADCTfOMeAAB0hGQAAOgIyQAA0BGSAQCgIyQDAEBHSAYAgM5mLwEHwGMtXPqZmS4hSbJ27kxXAPDUZSQZAAA6QjIAAHSEZAAA6AjJAADQEZIBAKDj6hYAMAVc9QSeWowkAwBAR0gGAICOkAwAAB0hGQAAOkIyAAB0hGQAAOgIyQAA0BGSAQCgIyQDAEBHSAYAgI6QDAAAHSEZAAA6QjIAAHTmzHQBAABPVQuXfmamS0iSrJ070xVsf4wkAwBAR0gGAICOkAwAAB0hGQAAOkIyAAB0hGQAAOgIyQAA0BGSAQCgIyQDAEBHSAYAgI6QDAAAHSEZAAA6QjIAAHSEZAAA6AjJAADQEZIBAKAjJAMAQEdIBgCAjpAMAAAdIRkAADpCMgAAdIRkAADoCMkAANCZlpBcVa+qqluqak1VLZ2O5wAAgOky5SG5qnZI8l+THJPk55O8sap+fqqfBwAApst0jCS/IMma1to3W2s/SnJBkuOn4XkAAGBaVGttandY9fokr2qt/fth/TeSvLC19s6u35IkS4bV5yW5ZUoL2X7tneR7M10Es47jgok4LpiI44KJOC5+Yr/W2vzNdZozDU9cE7Q9Lom31s5KctY0PP92rapWtNbGZroOZhfHBRNxXDARxwUTcVxsuemYbrEuyb7j1hckuX0angcAAKbFdITkLyU5oKr2r6qdkpyY5NJpeB4AAJgWUz7dorX2UFW9M8k/JNkhyTmtta9O9fM8hZmCwkQcF0zEccFEHBdMxHGxhab8g3sAALC98417AADQEZIBAKAjJM9CVbWwqv7PrXzsfVNdDzOnqt5WVScNy2+pqmeN2/ZR32bJlqiqtVW190zXwbZTVXtW1dvHrT+rqi6ayZqYfpv6t15V/zwT9WyvhOTZaWGSCUNyVU3Hta2ZpVprH2mtnTesviXJs8Zt+/ettZtnpDBge7FnkkdDcmvt9tba62ewHmZQa+1FM13D9kRInkLDCPDqqvqrqvpqVS2vqp2r6jlV9bmqWllV/1RVBw79zx2+oXDj4zeOAp+W5MVVtaqqfncYQfy7qvp0kuVVtVtVXVFV11fVV6rK137PQsPx8LWqWlZVN1bVRVW1S1UdXVU3DL+7c6rqZ4b+p1XVzUPfM4a2U6vq3cNxMpbkb4bjYuequqqqxqrqP1TVn4573rdU1f8/LL+5qq4bHvPfqmqHmfhZ8HhVtWtVfaaqvlxVN1XVG4bRnz8ZfmfXVdXPDX3nV9UnqupLw+3Icfs4Z2i7YeO5oKp2qKozhmPsxqr6rXFP/Vvjzh0HzsBLZ5yteN94TlVdM/zO//PG940neF84LclzhnPA6cPz3TQ85tqq+oVxtVxVVc/f1HHF7DTRuWTctp2H4+g3h/WNx8tLq+oLVXXJ8L7zkaqSCXutNbcpumU0AvxQkkXD+oVJ3pzkiiQHDG0vTHLlsHxuktePe/x9w/1Lk1w2rv0tGX1Jy7xhfU6Snx2W906yJj+5Usl9M/1zcHvM8dCSHDmsn5Pk/01yW5LnDm3nJXlXknkZfTX7xt/jnsP9qUnePSxflWRs3P6vyig4z0+yZlz73yc5KslBST6dZMeh/S+TnDTTPxe3R39Pr0vyV+PW90iyNsl/HNZP2ngeSPLxJEcNy89OsnpY/kCSN288ZpJ8PcmuSf5Dkk8kmTNs23juWJvkt4bltyf56Ez/HH7ab1vxvnFZkjcOy28b974x4fvCsP+buue7aVj+3SR/NCw/M8nXn+i4mumfldsmj6FNnUsWJvn8+PN+HpszHkzybzO6XO/lGZdH3EY3/2uYet9qra0alldmdJC+KMnfVdWqJP8to5PRlrq8tXb3sFxJPlBVN2b0D2CfJM94UlUzXW5rrV09LH8sydEZHSNfH9qWJXlJkh9kdML6aFX9epL7J/sErbX1Sb5ZVYdX1V5Jnpfk6uG5np/kS8Oxd3RGJ0Rmh68kecUwcvzi1tqGof38cfdHDMuvSPIXw+/x0iQ/W1W7J3llkqVD+1VJ5mYUol+R5COttYeSZNy5I0kuHu43np+YeVvyvnFEkr8blj8+bh9b875wYZIThuX/Y9x+N3VcMTtt6lzyqSR/3X4yZa93XWvtm621hzM63xy1LYrdnpjfOvV+OG754YxOUve01hZN0PehDFNeqqqS7PQE+/1f45bflNHo4fNbaz+uqrUZncSYfSZ1IfI2+hKeF2QUZE9M8s4kL9+C5/nbjN7kvpbkktZaG46pZa21925hzWwDrbWvV9Xzkxyb5P+rquUbN43vNtz/myRHtNYeGL+P4Xf8utbaLRO0b+rY23iOejjeA2aLLXnf2JQtfl9orf1LVd1VVb+U5A1J3jpsmvC4YnZ6gnPJ1UmOqaqPt2H4uH/oZtZ/6hlJnn4/SPKtqjohGb15VdUhw7a1GY30JcnxSXYclu9NsvsT7HOPJHfuoCICAAACQUlEQVQOJ8KXJdlvyqtmqjy7qjaOBr4xoxGehRvnmib5jST/o6p2S7JHa+2zGU2/mOjN8YmOi4uTvGZ4jr8d2q5I8vqqenqSVNW8qnKszBI1ulLJ/a21jyU5I8lhw6Y3jLv/4rC8PKP/OG187Mbj4x8ymmNcQ/uh4/q/rYYP+lbVvOl6HUyLJ3rfuCajP68no/9Qb7Sp94XNvZ9ckOQ9GZ1/vjK0beq4YhZ6gnPJf0pyV0ZT7Sbygqraf5iL/IYk/3Pai93OCMnbxpuSnFJVX07y1YwCcZL8VZJfqarrMppztnG0+MYkDw2T8H93gv39TZKxqlox7Ptr01o9T8bqJIuHP4HOS/KhJCdn9GfUryR5JMlHMnoTu2zo9z8ymivYOzfJR4YP4Ow8fkNr7ftJbk6yX2vtuqHt5ozmQC8f9nt5tm6qD9Pj4CTXDX/S/o9J3j+0/0xVXZvkd/KT4+C3M/o3f2NV3ZzRXNQk+S8Z/ef6xuHDWP9laP9oku8M7V/OJq6Ww6y2qfeNdyX5veF945lJNv5pfcL3hdbaXUmuHj7QdfoEz3NRRmH7wnFtmzqumJ02dS5JRsfL3Br34e5xvpjRBztvSvKtJJdMd6HbG19LDdOkqhZm9MGrX5zhUthODH8iH2utfW+ma2F2qqpdkjwwTKk6MaMP8bn6BFukql6a0YfC/91M1zKbmY8GANuP52f0Ic5Kck+S/2uG64GnLCPJAADQMScZAAA6QjIAAHSEZAAA6AjJAADQEZIBAKDzvwE1K7nj0EJVdgAAAABJRU5ErkJggg==\n",
      "text/plain": [
       "<Figure size 864x576 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "fig, ax = plt.subplots(figsize=(12, 8))\n",
    "freq = pd.DataFrame({'Predicted': errors.predicted.value_counts(), 'Actual':errors.actual.value_counts()})\n",
    "freq.sort_values(by='Predicted', ascending=False).plot.bar(rot=0, ax=ax);"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [],
   "source": [
    "import torch\n",
    "import torch.nn.functional as F\n",
    "\n",
    "class OneHiddenLayerNet(torch.nn.Module):\n",
    "    def __init__(self, embedding_dim, hidden_dim, out_dim):\n",
    "        super().__init__()\n",
    "        \n",
    "        self.embedding_dim = embedding_dim\n",
    "        self.hidden_size = hidden_dim\n",
    "        self.out_dim = out_dim\n",
    "        \n",
    "#         self.criterion = torch.nn.CrossEntropyLoss()\n",
    "#         self.optimizer = torch.optim.Adam(net.parameters(), lr = 0.001)\n",
    "\n",
    "        self.classifier = torch.nn.Sequential(\n",
    "            torch.nn.Linear(self.embedding_dim, self.hidden_size),\n",
    "            torch.nn.ReLU(),\n",
    "            torch.nn.Linear(self.hidden_size, self.out_dim),\n",
    "        )\n",
    "\n",
    "    def forward(self, inputs):\n",
    "        logits = self.classifier(inputs)\n",
    "        outputs = F.softmax(logits, dim=-1)\n",
    "\n",
    "        return outputs\n",
    "\n",
    "#     def backward(self, y_pred, target):\n",
    "#         loss = self.criterion(y_pred, target)\n",
    "\n",
    "#         # Zero the gradients\n",
    "#         self.optimizer.zero_grad()\n",
    "\n",
    "#         # perform a backward pass (backpropagation)\n",
    "#         loss.backward()\n",
    "\n",
    "#         # Update the parameters\n",
    "#         self.optimizer.step()\n",
    "#         return loss"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [],
   "source": [
    "from torch.autograd import Variable\n",
    "\n",
    "def get_batch(X, y, i, batch_size):\n",
    "    X_batch = X[i*batch_size:i*batch_size+batch_size]\n",
    "    y_batch = y[i*batch_size:i*batch_size+batch_size]\n",
    "    return Variable(torch.FloatTensor(X_batch)), Variable(torch.LongTensor(y_batch))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 62,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 0/50, Loss: 1.4031\n",
      "Epoch 10/50, Loss: 1.2678\n",
      "Epoch 20/50, Loss: 1.2453\n",
      "Epoch 30/50, Loss: 1.2251\n",
      "Epoch 40/50, Loss: 1.2161\n",
      "Epoch 50/50, Loss: 1.1921\n"
     ]
    }
   ],
   "source": [
    "net = OneHiddenLayerNet(embedding_dim=300, hidden_dim=256, out_dim=5)\n",
    "batch_size = 64\n",
    "num_epochs = 50\n",
    "\n",
    "criterion = torch.nn.CrossEntropyLoss()\n",
    "optimizer = torch.optim.Adam(net.parameters(), lr = 0.001)\n",
    "# optimizer = torch.optim.SGD(net.parameters(), lr=0.01, momentum=0.9)\n",
    "\n",
    "for epoch in range(num_epochs+1):\n",
    "    total_batch = int(len(X_train)/batch_size)\n",
    "    for i in range(total_batch):\n",
    "        X_batch, y_batch = get_batch(X_train, y_train, i, batch_size)\n",
    "        optimizer.zero_grad()\n",
    "        outputs = net(X_batch)\n",
    "        loss = criterion(outputs, y_batch)\n",
    "        loss.backward()\n",
    "        optimizer.step()\n",
    "    if epoch % 10 == 0:\n",
    "        print(f'Epoch {epoch}/{num_epochs}, Loss: {loss.item():.4f}')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 63,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Precision: 0.6205\n",
      "Recall: 0.6316\n",
      "f1-score: 0.6207\n"
     ]
    }
   ],
   "source": [
    "# Test the Model\n",
    "X_batch_test = Variable(torch.FloatTensor(X_test))\n",
    "y_batch_test = Variable(torch.LongTensor(y_test))\n",
    "\n",
    "outputs = net(X_batch_test)\n",
    "_, y_pred = torch.max(outputs.data, 1)\n",
    "\n",
    "f1, precision, recall = score_model(y_pred,y_test)\n",
    "print(f'Precision: {precision:.4f}')\n",
    "print(f'Recall: {recall:.4f}')\n",
    "print(f'f1-score: {f1:.4f}')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 64,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/nerdikon/anaconda3/lib/python3.7/site-packages/sklearn/preprocessing/label.py:151: DeprecationWarning: The truth value of an empty array is ambiguous. Returning False, but in future this will result in an error. Use `array.size > 0` to check that an array is not empty.\n",
      "  if diff:\n"
     ]
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAjoAAAHjCAYAAAAqkbW6AAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDIuMi4zLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvIxREBQAAIABJREFUeJzs3Xl4FdX9x/H394YgICB7AgQEFQVlsyJuCIKyE7BuuC9t5ecCKq64oeLS1gWr1aq0akVtBUUgQAAJAaqCVbTKvkMgkJuwJaBsyc35/ZEQbjaImrlJ7v28nmeeZGbOzDkzjodvzjJjzjlEREREwpGvogsgIiIi4hUFOiIiIhK2FOiIiIhI2FKgIyIiImFLgY6IiIiELQU6IiIiErYU6IiIiEjYUqAjIiIiYUuBjoiIiIStahVdgNL80OdcvbLZY9esTq/oIoS9XPQYh0JdX3RFFyHsZeQequgiRIRNKSkW0gyXTyr/SuqMy0N7DcegFh0REREJWwp0REREJGwp0BEREZGwpUBHREREwpYCHREREQlblXbWlYiIiHjLBQLlfs5KNeUKteiIiIhIGFOgIyIiImFLXVciIiKRKpBT0SXwnFp0REREJGwp0BEREZGwpa4rERGRCOVyy7/rSrOuREREREJEgY6IiIiELQU6IiIiErY0RkdERCRSefBm5MpGLToiIiISthToiIiISNhS15WIiEiEcnozsoiIiEjVpUBHREREwpYCHREREQlbGqMjIiISqTRGR0RERKTqUqAjIiIiYUtdVyIiIhHKi6+XVzZq0REREZGQMrN+ZrbazNaZ2agS9r9sZt/nL2vMLDNoXyBoX8Kx8lKLjoiIiISMmUUBrwO9gVTgGzNLcM6tOJzGOTcyKP0I4MygU+x3znUua34KdERERCJVxXzUsyuwzjm3AcDMPgKGACtKSX8N8MQvzUxdVyIiIlJuzGyYmS0OWoYVSdIc2BK0npq/raRznQi0BpKDNtfIP+9XZnbpscqjFh0REREpN865ccC4oySxkg4rJe3VwCfOueCmp5bOuW1mdhKQbGZLnXPrS8tMLToiIiISSqlAi6D1OGBbKWmvBv4dvME5ty3/5wZgPoXH7xSjFh0REZEIVUFfL/8GaGNmrYGt5AUz1xZNZGanAfWBRUHb6gP7nHMHzawRcAHw/NEyU6AjIiIiIeOcyzGz4cBsIAp4xzm33MzGAIudc4enjF8DfOScC+7Wage8ZWa55PVK/Sl4tlZJFOiIiIhISDnnEoHEIttGF1l/soTjFgIdfk5eCnREREQilT7qKSIiIlJ1KdARERGRsKVApwzqdDmX096eQNt3P6bJ0BtKTXfChT3p9NlX1GzTFoB6vfpy6hvjC5aOsxZS46Q2oSp2pdetR3dmJM9l1oJ5/OH224rtj65enZde+yuzFszjoymTaRaX9z6pZnHN+W71Sj5NnMGniTN44tlnAKh1/PEF2z5NnMGX//uWUaMfD+k1VXbdevRgZvJcZi+Yz623315sf3T16ox97TVmL5jPhClTaB4XB0DzuDi+X72KyYmJTE5M5Mlnnw110Su1c7tfyMdzZzNpXhI33lb03Wh59/XZv/6FSfOSeGfyJzRtnvcsV4uO5vHn/8S/Zk7nw8QEfnNOVyDvWf5gRkLB8tm3/2Xk44+G9Joqmx49ejA3OZn5CxZwewnPbvXq1XnttdeYv2ABU6ZMIS7/2QW44447mL9gAXOTk+nevXvB9i+++IJZs2eTmJhIwrRpIbmOysblBsp9qWw0RudYfD6aD7+fDaPuIntHBm3++i5Ziz7n4OZNhZPVrEWjS6/ip5XLCrZlJs8mM3k2ADVanUyrp57nwIa1oSx9peXz+Xjs6TH84bobSPf7mZAwlXlJSaxfu64gzeVDr2JPVhb9evSkf/wg7hs1ivuGjwBgS0oKlw0YWOic+376qdC2j6cnMGfW7NBcUBXg8/kY/fQYfnfd9aT7/XyckEBy0pxC9/yK/Hvet8dFDIiP575Ro7h3+HAANqek8NsBAyqq+JWWz+fjwTFPMvyGm8nw+3lv6iQ+T0pm47oj93XwVVewN2sPl/e8hN6DBjJ81AM8OuIeLr36KgCu7T+I+g0b8Jd33+bmIZex76efuH7g4ILj30uYzPzZn4X82ioLn8/HmKef5vrrrsPv95OQkMCcpCTWrT1Sn141dChZWVlc1KMH8fHxjBo1iuHDh3NKmzbEx8fTp3dvmsTE8OGHH9LzoovIzc0F4Jqrr2b37t0VdWkSAmrROYZap53OoW2pHPJvw+XkkLlgDiec371YutibhpEx8QPcoUMlnqdez95kzpvjdXGrjA6dO7F5UwqpW7aQnZ3NzGnT6NW7d6E0vXr3ZsqkSQB8ljiTcy84v8znP7FVKxo0bMi3X39druWuyjp27lzonidOm8bFvfsUSnNx7z4F93x2YiLn/Yx7HqnO6NSR1JQUtm3ZQk52Np9Nm0H33hcXStOj9yXMmPQpAMkzZ3H2+ecB0LrNKXyzcCEAu3fu4sc9e2jXsfCEkhatTqRBw4b87+tvQnA1lVPnzp1J2bSJLfnP7rRp0+hTpL7o07s3k/Kf3cTERM6/4IKC7dOmTePQoUOkbtlCyqZNdO5c5u9BShjwPNAxs5r5L/2pkqIbNebQ9oyC9eztGUQ3bFwoTc2TTyW6cQx7//tlqeep1+MSMudH7l9kRcXExuJPSytY96f5aRIbWyRNDP5teWkCgQB79+6lXv36ADRv0YJJidN5b8JHnHX22cXOP2BwPLOmz/DwCqqemNgY0tKOvHzUn5ZGTGxMoTRNYmNI25aXpug9j2vRgk8TZ/D+hAkl3vNI1Tg2lvSgZznD76dxkfvaOCaG9DQ/kHdff9z7IyfUr8/alavo0fsSoqKiaBYXR9sO7Ylp2rTQsX3i45kzI7Kf5ZjYWLYF3eO0tDRiitUXsWwr8uzWr1+/+LF+f8GxDnj/gw+YNn0611xzjfcXIhXC064rM4sHXgSqA63NrDMwxjk3uJT0w4BhAI+1a80VcU28LF4ZlfBJjuBXF5nR7LZ72Pzi06WeoVbbM8g9eIADmzaUf/GqKCvxvhb+1IlZ8TTOObZnbOfi8y4gKzOT09u3569/f4vBvfvy048/FqQbMDieh+65t9zLXbWVfD8LpSjhnuMcGRkZ9DrvfDIzMzmjfXte+/s4BvXuU+ieR6qSbllZnmWcY9rET2h98sm8lzCZtK1bWfLtdwSKTPftHT+QJ++9vxxLXPWUfIvLVl+Uth3g8ssuIyMjg4YNG/LBBx+wfv16vo6wVuAKejNySHndovMkeZ9jzwRwzn0PtCotsXNunHOui3OuS+UIciB7RwbVGx8pS3TjJmTv2l6w7qtZixqtTuKUF/5Gu/GTqdXuDFqPeaFgQDJAvYsuUbdVEX5/GrFBf7nGNo0lIz29cJo0P7HN8tJERUVRp04dsjIzyT50iKzMTABWLFvGlpTNtGrduuC409q1IyqqGiuWLUOOSPf7adq0WcF6bNOmZKRnFE6T5qdps7w0h+95Zv49z8y/58vz73nroHseyTLS/IVaYZrExrK9yH3N8PuJaZrXihAVFUXtOrXJyswkEAjw8jPPcf3AwTww7Hbq1K3Llo0pBce1adeWatWiWLVseWguppLy+/00C7rHTZs2LaG+SKNZCc+uPy2t8LGxR+qajIy8/047d+5k9uzZdFKXVljyOtDJcc5leZyHp/atXkn15i2oHtsUq1aNej16k7Xo84L9uft+YvmV/Vh5429ZeeNv2bdyORtHP8D+tavyEphxwoUXkzlfgU6wZT8s4cTWrWjeIo7o6Gj6x8czb05SoTTzkpK49PLLAegzoD//XZj3uZP6DRrg8+U9unEtWnBi61akbt5ccNyAwfEkJiQghS394YdC93xAfDzJcwo/l8lJcwrued8BA/gqf/xISfd8S9A9j2QrliylRatWNIuLo1p0NH3iB/J50txCaf6TNJeBl18GQK/+/Vi86CsAjqtRgxo1awLQtdsFBAKBQoOY+8QPYnbC9BBdSeX1ww8/0Kp1a+JatCA6Opr4+HjmFHl25yQlcXn+sztgwAAW5j+7c+bMIT4+nurVqxPXogWtWrfm+++/p2bNmhx//PEA1KxZkwu7d2fN6tWhvTAJCa9nXS0zs2uBKDNrA9wFLPQ4z/KVG2Dray9y0nOvgM/HrtnTOZiykZgbb2X/mlXs+erzox5+fIczyd6RwSF/aR9mjUyBQIBnRz/B38ePxxflY/LEj1m3di3D7x3J8iVLmZeUxKQJE/jzyy8za8E8MjOzuD9/xlWXc7oy4t6R5OQEyM0N8NQjj5GVdSSe7jdoILfdfEtFXVqlFQgEeHr0aN4ePx5fVBSTJk5k3dq1jLh3JMvy7/knEyby/Mtjmb1gPlmZmdybf8/PPqcrI+69l0BOgEBugCcfebTQPY9kgUCAF554ilfHv4PPF8W0jz9hw9p1DBt5NyuXLuXzpGQSJnzMUy+/yKR5SezJyuTRESMBaNCwIa+Of4fcXMd2v58ninRRXTJwAPfc8oeKuKxKJRAIMHr0aMaPH09UVBQTJ05k7dq1jLz3XpYuWUJSUhITJ0xg7MsvM3/BAjIzMxmRP1tw7dq1TJ8xgzlJSeTk5DD68cfJzc2lUaNGjBs3DoCoatWYOnUqCxYsqMjLrBgR0HVlRfs5y/XkZrWAR4HDUztmA8845w4c69gf+pzrXcEEgGtWpx87kfwquegxDoW6vuiKLkLYy8gteUaplK9NKSklDUnyzMGpj5d7JXXckKdDeg3H4nWLzmnOuUfJC3ZEREREQsrrMTpjzWyVmT1tZmd4nJeIiIhIIZ626DjneppZLHAVMM7M6gITnHPPeJmviIiIHFtl/GRDefP8hYHOOb9z7lXgNuB7YLTXeYqIiIiAx4GOmbUzsyfNbBnwGnkzruKOcZiIiIhIufB6MPK7wL+BPs45za8WERGpTCJgernXY3TO9fL8IiIiIkfjSaBjZhOdc1eZ2VKKfBkKcM65jl7kKyIiIhLMqxadu/N/DvLo/CIiIvIr6aOev5BzLi3/1zuccynBC3CHF3mKiIiIFOX19PLeJWzr73GeIiIiIoB3Y3RuJ6/l5iQzWxK0qw7wpRd5ioiIiBTl1RidfwEzgT8Co4K273XO7fIoTxEREfk5ImCMjieBjnMuC8gCrgEwsyZADaC2mdV2zm32Il8RERGRYF6/GTnezNYCG4EFwCbyWnpEREREPOf1m5GfAc4FkpxzZ5pZT/JbeURERKRi6aOev162c24n4DMzn3NuHtDZ4zxFREREAO9bdDLNrDbwH+BDM8sAwn/kk4iIiFQKXrfoDAH2AyOBWcB6IN7jPEVEREQA7z/q+VPQ6nte5iUiIiI/k6aX/zpmtpfCH/WEvGnni4H7nHMbvMxfREREIpvXY3TGAtvIe4GgAVcDscBq4B3gIo/zFxERkQjmdaDTzzl3TtD6ODP7yjk3xswe8ThvEREROQoX0PTyXyvXzK4yM1/+clXQvqJdWiIiIiLlyutA5zrgBiADSM///XozqwkM9zhvERERiXBez7raQOnTyb/wMm8RERE5OhcBs668/tbVqWY218yW5a93NLPHvMxTRERE5DCvu67+DjwMZAM455aQN/NKRERExHNeBzq1nHNfF9kW/u1kIiIiUil4Pb18h5mdTP4MKzO7AkjzOE8REREpi9zwb3vwOtC5ExgHtDWzrcBG8mZiiYiIiHjO60BnK/AuMA9oAOwBbgLGeJyviIiIiOeBzlQgE/iOvE9BiIiISCURCW9G9jrQiXPO9fM4DxEREZESeT3raqGZdfA4DxEREZESed2i0w242cw2AgfJ+4K5c8519DhfEREREc8Dnf4en19ERER+KY3R+XWccylenl9ERETkaLxu0fnFPk+vU9FFCHtfP9a2oosQ9h55ZV1FFyEinF2tZkUXIey9lLmhoosg8otU2kBHREREvKWvl4uIiIhUYQp0REREJGyp60pERCRSRcCsK7XoiIiISNhSoCMiIiJhS4GOiIiIhC2N0REREYlQkfD1crXoiIiISNhSoCMiIiJhS11XIiIiEcrlqutKREREpMpSoCMiIiJhS4GOiIiIhC2N0REREYlUml4uIiIiUnUp0BEREZGwpa4rERGRCKU3I4uIiIhUYQp0REREJGyp60pERCRCuUBuRRfBc2rRERERkZAys35mttrM1pnZqFLSXGVmK8xsuZn9K2j7TWa2Nn+56Vh5qUVHREREQsbMooDXgd5AKvCNmSU451YEpWkDPAxc4JzbbWZN8rc3AJ4AugAO+Db/2N2l5acWHREREQmlrsA659wG59wh4CNgSJE0twKvHw5gnHMZ+dv7AnOcc7vy980B+h0tMwU6IiIikSqQW+6LmQ0zs8VBy7AiuTYHtgStp+ZvC3YqcKqZfWlmX5lZv59xbCHquhIREZFy45wbB4w7ShIr6bAi69WANsBFQBzwuZm1L+OxhahFR0REREIpFWgRtB4HbCshzVTnXLZzbiOwmrzApyzHFqJAR0REJEK5QKDclzL4BmhjZq3NrDpwNZBQJM0UoCeAmTUirytrAzAb6GNm9c2sPtAnf1up1HUlIiIiIeOcyzGz4eQFKFHAO8655WY2BljsnEvgSECzAggADzjndgKY2dPkBUsAY5xzu46WnwIdERERCSnnXCKQWGTb6KDfHXBv/lL02HeAd8qal7quREREJGypRUdERCRCucBRJyyFBbXoiIiISNhSoCMiIiJhS11XIiIiEUpfLxcRERGpwhToiIiISNhS15WIiEiEUteViIiISBWmFp0yaHl+Fy586A7M52PF5Jl8986EQvvPuHIQHYcOJjeQS/b+/cwb8zK7N2ymSfvT6Pn4SADM4Os332dD8pcVcQlVwsKNu3kxeRMB57i0Qwy3nNO8WJrPVu1g3MJUzKBN41o8N+hUvtmcxdh5mwrSbNq1n+cGnUrPNg1CWPqqod0F53DZqLvxRflYNGk6SW9/UGh/zxuHct7lgwgEAvy4K5N/Pf5HdqelF+yvcXwtHkn4kCVz/8Mnz70c6uJXCc3O70KXB/Pqi3WTZ7L83cL1RZsrBnHa0MG43Fxy9u3nq6dfJmvDZo5vFsPgT99mT0oqADuWrOS/z75SEZdQKZ3fozsPjn4cX1QUkydM4N033iq0P7p6dZ4Z+yLt2rcnK3M3Dw2/i22pWxkwZDA3/d+tBenatG3LNYMGsyVlM+9+/FHB9iaxsSROmcoLY54J2TVJaCjQOQbz+ejxyAim/t9D/Ji+g6v+9Rob5y9i94bNBWnWJCaz/OPpALTqcR7d7r+NaXc8wq51m5h47R24QC61GjXg6o/fZOOCRRHRVPhzBXIdf0rayN+uPJ2YOtW54YOl9Di5Pic1qlWQZvPu/fzz6628c2176taoxq6fsgE4u+UJ/PumTgBk7c/m0rf/x7mtTqiQ66jMzOfjysfu5fVbR5Lpz+D+Cf9g2bwv8G/YVJAmdeUaXhj6B7IPHKTb0EsZct8d/PP+Jwr2DxhxK+sWf18Bpa8azOej68MjSLrtIfal76D/h6+RumARWUH1xaaZyaz9JK++iOtxHmfddxvJdz4CwI+p25gx9LYKKXtl5vP5eHjMk9x2/U2k+/18mDCZBXPmsmHduoI0v73qSvZkZTH4ol70jR/E3aMe4qHhd5E4NYHEqXnfizzltFP5y9/fYvWKlQAMHRBfcPy/pk1l7qyjfhtSqih1XR1DTPvTyNqyjT1b/eTm5LB21nxOuuj8Qmmyf9pX8Ht0zRrg8t40mXPgYEFQE3VcdQj/F1D+Ysv9P9Kifg3i6tUgOspHn7aNmL9+d6E0k5dkcGXnWOrWyIvPGxwfXew8c9fs4vzW9akZHRWSclclJ3Zox/bNqexM3UYgJ4fvZibRoVe3QmnWfvM/sg8cBGDTD8upF9O4YF+L00+jTsP6rFr4dUjLXZU0bH8ae7ds48f8+iJl9nxaHKW+qBZUX0jp2nfuxJaUFLZu2UJOdjazp03noj6XFEpzUZ9LmDbpUwCSEmfS9fzzip2n/+B4ZiVML7a9ZatWNGjYkO++/qbYvnDncl25L5WNWnSO4fgmjdjr316w/mPGDmI6tC2WrsPQwXS+4XJ80dWYcuuDBdtjOrSl11P3UadpDEmP/lmtOaXI2HuImDrHFazH1K7OsrS9hdKk7N4PwO/+tYyAc/zf+XGc37p+oTSzV+3gui7NvC9wFVSvSWMy/RkF65np2zmxw+mlpj/3skGs+Py/AJgZlz4wnPcffppTzznL87JWVbWaNOKnoPrip/QdNCqhvjh16GBOvz6vvpgz7Eh9Ubt5LAM/eoPsH/fx/evvkvG/ZSEpd2XXJCYG/7a0gvX0ND8dOncqkia2IE0gEODHvXupV78+mbuP/MHUZ9BA7rm1eItZv8GDmD19hkell4rmSYuOme01sz0lLHvNbM9RjhtmZovNbPGXO1O9KNrPZ1Z8Wwl/gS2dkMD7g25i0V/+wdm3XluwPX3pKv592a18fO1wzvr91URVL94KISU3dlmRex/IdWzevZ+3hp7OcwPb8PTsDew9kFOwf/uPh1i3Yx/nqduqZCU8y66U1oQug/rQ8oy2JL/7LwC6Xf1bVvxnUaFASUpQxvpizYQEpsTfxHev/IMO+fXF/u27mNTvOmZcfTuLX3qTbn98mOjjaxU7NhIVrQug+G0t+dYfSdS+cycO7D/A+jVriqXrGz+IWQnTfnU5pXLyJNBxztVxztUtYanjnKt7lOPGOee6OOe6XNAwzoui/Ww/pW+nTuyR5vvaTRrxU8bOUtOvmTWf1j0vKLZ998bNZO8/QMNTWntSzqoupk510vceLFhP//EQjWpXL5LmOHqc0oDoKB/N69XgxAY12Lz7QMH+Oat30rNN3n4pLjM9g3qxTQrW68U0Zs/2HcXSnXpuF/oMu5FxIx4iJztvHFTrTu258NrLeWL2x1x6/510HdyP+Hs0lqSofenbOT6ovjg+phH7t5deX2yaNZ8WF+XVF7nZ2RzKymvF3LVyLXtT06hzYuWoBytaut9PbLOmBesxTWPZnpFeapqoqChq16lDVmZmwf5+pQQzp7ZrS7WoaqxcFpmtZy7gyn2pbELyL4KZNTGzloeXUORZXtKXr+aEls2p0zwWX7VqtOl3ERsXLCqU5oSWR2YHtep+DlmbtwJQp3kslv+Pbp2mTah/Ygv2bPOHrvBVyOmxtdmy+wBbMw+QHcjls1U76HFy4W6pi06pz+LNeQ2Cu/dls3n3AZrXO9LdNXvVDvq2bRTSclclm5etonHLFjRo3pSoatX4Tf9LWDqv8CzAuLZtuPqJB/j78FH8uOvIPxLjR43hyd6X81TfK5ny4ut8nTCLaX95M9SXUOntXL6aOi2bU7tZXn1xYt+L2FKkvqgTVF/EXXgOe/Pri+Pqn4D58uqL2s1jqduyOT+mpiGw/IcltGzVimZxcVSLjqZv/CAWzJlbKM2COXOJv/wyAC4Z0J9vFh6572ZG7wH9mTWt+PicfoPjmTVNrTnhzNMxOmY2GHgJaAZkACcCK4EzvMy3PLlALv/542sMeeOPedPLp8xm1/oUut5xExnL17BpwSI6Xj2EuHPPJDc7wMG9e0l6/HkAmp3Znt/8bii52QGcy2X+c69yILPUnruIVs1nPHhxa4ZPWkkg1zGkQxNOblSLN77YzOmxtelxSgPOa1WPrzZlccU73+Pzwd09TqRezbyuwG1ZB0jfe5CzWpTaYBjxcgMBPnluLHe8NRZflI+vJs/Av34jA+78PZuXr2LZ/C8Zct+dVK9Vk1vGPg3A7rR0/j5iVAWXvOpwgVy+/tNrXJxfX6ybOpus9Sl0uv0mdq5YQ+qCRZx29RCannMmuTkBDu3Zy5ej8+qLmN90oNMdN5GbE8Dl5vLfZ17h0J69x8gxMgQCAf40+ineGP9PfFE+pk78hPVr13L7yHtYsXQpC5LmMnniRJ4d+xIJ85PZk5nJQyPuLjj+rHO6ku73s3XLlmLn7jNwAMNv+X0oL0dCzErroy+Xk5v9APQCkpxzZ5pZT+Aa59ywYx37Wqfela/9K8zcPLz4e2qkfD3yyrpjJ5Jf7exqNSu6CGHvpcwNFV2EiPD9pvUljDbyjv/Os8v939rY178J6TUci9ezrrKdczvNzGdmPufcPDP7s8d5ioiISBm4QEWXwHteBzqZZlYb+A/woZllADnHOEZERESkXHg9GHkIsA8YCcwC1gPxRz1CREREpJx41qJjZlHAVOfcJUAu8J5XeYmIiMjPVxmng5c3z1p0nHMBYJ+Z6e1tIiIiUiG8HqNzAFhqZnOAnw5vdM7d5XG+IiIiIp4HOjPyl2Dh304mIiJSBeRGwOcXvQ506jnnXgneYGZ3l5ZYREREpDx5PevqphK23exxniIiIiKARy06ZnYNcC3Q2swSgnbVAUr/wp2IiIhIOfKq62ohkAY0Iu9bV4ftBZZ4lKeIiIj8DHoz8i/knEsBUoDzvDi/iIiISFl4/fXyvRyZZVUdiAZ+cs7pE9MiIiLiOU8DHedcneB1M7sU6OplniIiIlI2kdB15fWsq0Kcc1OAXqHMU0RERCKX111XlwWt+oAu6IWBIiIiEiJevzAw+EvlOcAm8r5oLiIiIuI5r8fo3OLl+UVEROSXi4RPQHg6RsfMTjWzuWa2LH+9o5k95mWeIiIiIod5PRj578DDQDaAc24JcLXHeYqIiIgA3o/RqeWc+9rMgrfleJyniIiIlIGml/96O8zsZPJnWpnZFeR9GkJERETEc1636NwJjAPamtlWYCNwncd5ioiIiADeBzpbgXeBeUADYA9wEzDG43xFRETkGHJz7diJqjivA52pQCbwHbDN47xERERECvE60IlzzvXzOA8RERGREnk9GHmhmXXwOA8RERGREnndotMNuNnMNgIHAQOcc66jx/mKiIjIMUTCm5G9DnT6e3x+ERERkVJ5/a2rFC/PLyIiInI0XrfoiIiISCWlNyOLiIiIVGEKdERERCRsKdARERGRsKUxOiIiIhEqEj4BoRYdERERCVsKdERERCRsqetKREQkQuXJIBDYAAAgAElEQVRGwPTyShvovJW5vqKLEPY++VNqRRch7E2Mr1vRRYgIg6dmVHQRwl4urqKLIPKLqOtKREREwlalbdERERERb2nWlYiIiEgVpkBHREREwpYCHREREQlbGqMjIiISoZzG6IiIiIhUXQp0REREJGyp60pERCRC5eZWdAm8pxYdERERCVsKdERERCRsKdARERGRsKUxOiIiIhFKn4AQERERqcIU6IiIiEjYUteViIhIhFLXlYiIiEg5M7N+ZrbazNaZ2aijpLvCzJyZdclfb2Vm+83s+/zlzWPlpRYdERERCRkziwJeB3oDqcA3ZpbgnFtRJF0d4C7gv0VOsd4517ms+alFR0REJEIFcq3clzLoCqxzzm1wzh0CPgKGlJDuaeB54MCvuUYFOiIiIlJuzGyYmS0OWoYVSdIc2BK0npq/LfgcZwItnHPTS8iitZn9z8wWmNmFxyqPuq5ERESk3DjnxgHjjpKkpGYfV7DTzAe8DNxcQro0oKVzbqeZnQVMMbMznHN7SstMLToiIiISSqlAi6D1OGBb0HodoD0w38w2AecCCWbWxTl30Dm3E8A59y2wHjj1aJmpRUdERCRCVdD08m+ANmbWGtgKXA1ce3incy4LaHR43czmA/c75xabWWNgl3MuYGYnAW2ADUfLTIGOiIiIhIxzLsfMhgOzgSjgHefccjMbAyx2ziUc5fDuwBgzywECwG3OuV1Hy0+BjoiIiISUcy4RSCyybXQpaS8K+n0SMOnn5KVAR0REJELlOr0ZWURERKTKUqAjIiIiYUuBjoiIiIQtjdERERGJULm5FV0C76lFR0RERMJWqS06ZjaNoFcyF+WcG+xJiURERETKydG6rl4MWSlEREQk5AIRML281EDHObcglAURERERKW/HHIxsZm2APwKnAzUOb3fOneRhuURERER+tbLMunoXeIK8T6b3BG6h5E+si4iISBVSQR/1DKmyzLqq6ZybC5hzLsU59yTQy9tiiYiIiPx6ZWnROWBmPmBt/tdGtwJNvC2WiIiIyK9Xlhade4BawF3AWcANwE1eFkpERESkPBwz0HHOfeOc+9E5l+qcu8U5d5lz7qtQFK6yuKBHdxKSk5ixIJnf335bsf3R1avzwmuvMmNBMh9O+ZRmcc0BGHjpED5OnF6w/LBxHaed3i7Uxa8yunbvxvg5iXyYPItr/+8PxfZ3PLsL46ZOYu7qpfTo16dge0yzZrw19RP+Me1T3p05jcHXDA1lsauU6m3PpcEjE2nw6CfUuvjGYvtrdB1Io2dmUf+B96n/wPvUODfvdVnRp5xVsK3+A+/T+IX/UL1D91AXv8o5t/uFTJg7i4/nzeGG24YV29+5axfemzaZL9auoGf/vhVQwqrjgh7dSZibxPT5yfyulHr4+ddeZfr8wvXwgCFDmJg4vWD5fsORenjE/ffx2cIv+Gr50pBeS2UScFbuS2VTlllX8yjhxYHOuYgYp+Pz+Xj06acYdt2N+P1+PkqYwrykJDasXVeQ5rKhV7Enaw8De/SiX/wgRo56iAeG38WMKVOZMWUqAG1OO41X//EWq1esrKhLqdR8Ph93P/k499/0e7b703lz8kS+nDuPlHXrC9JkbNvGnx58mKG3/q7QsTu3b2f4ldeQfSibmrVq8e7MBL6cm8zOjO2hvozKzXzUueIBdr8xgtzMDOrf+08OLvucQPrGQskO/C+JHycVfo1W9rpv2f3CDXmnqVWXho9+wqFV/w1Z0asin8/H/WOe4K4bbiHD7+fdqZP4PGkum4Ke6fStaTz9wCiuvfX3FVjSys/n8/HImKcYdv2NpPv9/DthCvPnJLFhXVA9fFVePTzoorx6+J5RD/Hg8LtInDqVxKlH6uFX/n6kHl4wdy7/fm880+cnV8h1SWiUpevqfuCB/OVx4HtgsZeFqkw6dO7E5k0ppG7ZQk52NjOnTadn796F0vTsfQkJkyYBMCdxJudccH6x8/QfHE9iwrSQlLkqatupI1tTNpO2JZWc7GySpydywSWFY2n/1m1sWL0GV+TjLDnZ2WQfygby/qozX+X7i6IyqHbi6eTsSCV35zYI5HDwf3M47he0yhzXqReHVi6C7IMelDJ8nN6pI6kpKWzLrzvmTJtB996XFEqTtnUr61atLvZMS2HtO3dic0oKW/Pv5axp0+nZp3A9fFGfIvXw+SXXwzOD6uEl//ueHdv1B1G4K0vX1bdBy5fOuXuBc8pycstzvZmNzl9vaWZdf2WZQ6pJbCz+tLSC9fS0NGJiY4qkicG/LS9NIBDgx717qVe/fqE0/eIHMnOqAp3SNI5pwvY0f8H6dn86jWNijnJEkeObxvL2jClM/CKZf7/1tlpzShB1QhNyd6cXrOdmZuA7oXGxdMd17EmDBz+g7s1/xFev+LyDGmf25sB3n3la1nDQODaGjKBnOsPvp3Fs2Z9pOSImJpb0bYXr4SZF6oeYmJiCNKXVw30HDSwU6AjkOiv3pbI5ZqBjZg2ClkZm1heILeP5/wacB1yTv74XeP0oeQ0zs8VmtnjXj3vKmIW3SvpP5lzhnjyz4qmC03To3IkD+w+wbs2a8i5e+CjpHpb+qbVitqf5+f3AS7muV1/6XjaE+g0blmfpwleRZ/ngss/ZOeZSdj1/PYfWfE3da58otN9XtyHVmp3MoVURNUzvFympXih6v6WMSryVRe6l6mEpRVm6rr4lr6vqW2ARcB9Q1g7lc5xzdwIHAJxzu4HqpSV2zo1zznVxznVpULtuGbPwVrrfT2zTpgXrMU2bkpGeUThNmp/YZnlpoqKiqF2nDlmZmQX7+8er2+pYtvvTadz0SPzcODaGHUXuc1nszNjOprXr6Hj2WeVZvLAQyMrAV//IX8G+ek3I3bOjUBq3bw8E8roBDyyaSrUWbQvtP67zJRxcsgByA94XuIrLSPPTJOiZbhIby/Zf8ExLXj0c06xwPbw9I6PUNCXVw/3i49WaE6HKEui0c86d5Jxr7Zxr45zrA3xTxvNnm1kU+YOZzawxUKU6o5f9sIQTW7eieYs4qkVH0z9+EPPnJBVKMz9pLoMvvxyA3gP68/XCRQX7zIw+A/szS/+DHdXqJUuJa3UisXHNqRYdTa9BA1g4d16Zjm0cG0P1444DoHbdurQ/6zds3rDxGEdFnpzNK6nWqAW+Bk0hqhrHndmbg8v+UyiNr+6RlrDq7S8kkL6p0P4av+mjbqsyWrlkKS1ataJpXF7d0Tt+IJ8nza3oYlVJy39YwomtWtE8/172K6kennOMenhAf2ZOUz0cicrywsCFwG+KbFtUwraSvApMBpqY2bPAFcBjP6uEFSwQCPDc6Cd5c/x7REX5mDzxY9avXcud997D8iVLmZ80l08nTOCPL49lxoJksjKzeHD4XQXHn3VOV/xpflK3bKnAq6j8AoEArzz1DC/88x/4fD5mfvIpm9au45Z7RrB66TIWzp3HaR3a88wbf6X2CXU5r1dPbr57BLf0j6flySdzxyMP4pzDzJjwj3fYuGZtRV9S5ZMbYO+kF6l326uYz8f+/04j4N/I8f2Hkb15JYeWf07N7kM57owLcbkB3L497PnXmILDfQ2a4qvXhOz131XgRVQdgUCAF58Ywyvj38bni2L6x5+wce06bh15F6uWLuPzpGTadezAn998nTon1KXbxT259Z67uLbvwIoueqVzuB5+I78enpJfD98x8h5WLM2rhydPnMBzY8cyfX5+PTyicD2c7veztUg9PHLUQwwYMpgaNWsyZ9GXfDphIm/85ZVQX16FqozTwcubFevnPLzDLBZoDnwAXMuRXtK6wJvOubYlHlj8PG2Bi/OPn+ucK9P86g4nnqTObI81rHZcRRch7E2MrxxdsOFu8NTdFV2EsLfP5VR0ESLCkk0bQhp5zDy7V7n/W9v/m+RKFT0drUWnL3AzEAe8xJFAZw/wSFlObmavABOcc6UOQBYRERHxSqmBjnPuPeA9M7vcOTfpF57/O+AxMzuVvC6sCc65iHkHj4iISGUWiIC+k7IMRj7LzOodXjGz+mb2TFlO7px7zzk3AOgKrAH+bGYaPCEiIiIhUZZAp79zrmCOXv4U8QE/M59TgLZAK2DVzzxWRERE5Bcpy6yrKDM7zjl3EMDMagJlGsVqZn8GLgPWAxOBp4ODJhEREak4lfFNxuWtLIHOB8BcM3s3f/0W4L0ynn8jcJ5zbscxU4qIiIiUs2MGOs65581sCXAJeTOvZgEnHu0YM2vrnFsFfA20NLOWRc6pF3GIiIiI58rSogPgJ++NxleR10pzrFlY9wLDyJuWXpQDepWwXURERKRclRro5E8Jv5q8D3LuBCaQ94LBnsc6qXNuWP6v/Z1zB4qct8YvL66IiIiUl0h4M/LRZl2tIu+NxvHOuW7Oub8CP/dLfgvLuE1ERESk3B2t6+py8lp05pnZLOAjjrwd+aiCPh9R08zOpPDnI2r98uKKiIiIlN3R3ow8GZhsZscDlwIjgRgzewOY7Jw72ieMgz8fMTZo+17K+PkIERER8VYkvBm5LLOufgI+BD40swbAlcAooNRAp5w+HyEiIiLyq5R11hUAzrldwFv5S6nM7Hrn3AdAKzO7t4TzjC3hMBEREZFy9bMCnZ/h+PyftT06v4iIiMgxeRLoOOfeyv/5lBfnFxERkV8vULY5RlVaWT7q+YuZ2fNmVtfMos1srpntMLPrvcxTRERE5DBPAx2gj3NuDzAISAVOBR7wOE8RERERwLsxOodF5/8cAPzbObfLLPybyURERKoCTS//9aaZ2SpgP3CHmTUGDhzjGBEREZFy4WnXlXNuFHAe0MU5lw38BAzxMk8RERGRwzxt0TGzaOAGoHt+l9UC4E0v8xQREZGy+bkfsKyKvO66eoO8cTp/y1+/IX/bHzzOV0RERMTzQOds51ynoPVkM/vB4zxFREREAO+nlwfM7OTDK2Z2EpHRUiYiIiKVgNctOg8A88xsQ/56K+AWj/MUERGRMoiElgevW3S+JO8DoLn5y1vAIo/zFBEREQG8b9EZD+wBns5fvwZ4H7jS43xFREREPA90TisyGHmeBiOLiIhUDvqo56/3PzM79/CKmZ1DXneWiIiIiOe8btE5B7jRzDbnr7cEVprZUsA55zp6nL+IiIhEMK8DnX4en19ERESkVJ4GOs65FC/PLyIiIr9cwIX/58u9HqMjIiIiUmEU6IiIiEjY8nqMjoiIiFRSejOyiIiISBWmQEdERETCVqXtujohqnpFFyHs7cvNqegihL0+U3ZUdBEiwvNNWlZ0EcLeiPSNFV0E8YC6rkRERESqMAU6IiIiErYU6IiIiEjYqrRjdERERMRbGqMjIiIiUoUp0BEREZGwpa4rERGRCBVAH/UUERERqbIU6IiIiEjYUqAjIiIiYUuBjoiISIQKeLCUhZn1M7PVZrbOzEaVsP82M1tqZt+b2RdmdnrQvofzj1ttZn2PlZcCHREREQkZM4sCXgf6A6cD1wQHMvn+5Zzr4JzrDDwPjM0/9nTgauAMoB/wt/zzlUqBjoiIiIRSV2Cdc26Dc+4Q8BEwJDiBc25P0OrxUDA9bAjwkXPuoHNuI7Au/3yl0vRyERGRCBVw5T+93MyGAcOCNo1zzo0LWm8ObAlaTwXOKeE8dwL3AtWBXkHHflXk2OZHK48CHRERESk3+UHNuKMksZIOK+E8rwOvm9m1wGPATWU9Npi6rkRERCSUUoEWQetxwLajpP8IuPQXHqtAR0REJFJV0Kyrb4A2ZtbazKqTN7g4ITiBmbUJWh0IrM3/PQG42syOM7PWQBvg66Nlpq4rERERCRnnXI6ZDQdmA1HAO8655WY2BljsnEsAhpvZJUA2sJu8bivy000EVgA5wJ3OuaPGVwp0REREJKScc4lAYpFto4N+v/soxz4LPFvWvNR1JSIiImFLLToiIiIRSl8vFxEREanCFOiIiIhI2FLXlYiISIRS15WIiIhIFaZAR0RERMKWAh0REREJWxqjIyIiEqHK+MmGKk0tOiIiIhK2FOiIiIhI2FLXlYiISIQKOE0vFxEREamyFOiIiIhI2FLXlYiISITSm5FFREREqjAFOiIiIhK2FOiIiIhI2NIYHRERkQilMToiIiIiVZgCnTI4p3s3/pU0k4+SZ3P9bbcW29/p7C68nTCJ+WuWcVH/vgXbT2nXljc/+Yj3Z03jn4lT6TWwfyiLXaWd2/1CPp47m0nzkrjxtmHF9p/Z9WzGT5vCwrUr6dW/XwWUsOo4v0d3psydQ8L8ZG65/f+K7Y+uXp0/v/YqCfOTeX/KJJrFNQdgwJDBTEicVrB8t2Etp53eDoB+g+P5eFYiE2fO4PX33qVe/fohvabKrOG5Z3PBx+/RbdIHtLrxmlLTxfTqTp+v51G33akARJ9Qly5/G0uv+Ym0vf+uUBW3yriwRw9mJc/lswXzufX224vtj65enZdfe43PFsxn4pQpNI+LK7S/abNmfLdiOb8bdqQOf+6F51n47WKmfTbb8/JLxVGgcww+n497nxrN/bfcyvV9B3FJ/EBanXJyoTTp29J47sGHSUqYXmj7wQMHeOb+h7ihXzz33Xwrdz3+MLXr1All8askn8/Hg2Oe5O6b/8DQPv3pO3gQrU85pVAa/9ZtjHngIT5LmFZBpawafD4fD495kjtv/h2X9e5Lv8HxnFTkXv72qivZk5XF4It68cHb73L3qIcASJyawNAB8QwdEM+jI+9jW2oqq1esJCoqigdHP86t11zHVf0HsnblKq6+6YaKuLzKx+ej3YN3893do/hy6M007Xsxx7c+sViyqFo1aTn0MjKXrijYlnvwEOveeoc1r74RyhJXCT6fj9FPj+EPN93MwEt6M2jwYE5uU/g5vnLoVezJyqJPj4v459tvc/+oUYX2Pzz6cT6fP7/Qtk8//oQ/3HST18Wv1HKdK/elslGgcwztOnUkNWUz27akkpOdTdL0RLr1vrhQGv/WraxftYbc3ML/gbds3ETqphQAdmZkkLlzF/UaNghZ2auqMzp1JDUlhW1btpCTnc1n02bQvcg9T9u6lXWrVhe751JY+86d2JKSwtb8ezl72nQu6nNJoTQX9bmEaZM+BSApcSZdzz+v2Hn6D45nVn4gb2ZgULNWTQCOr1Ob7ekZHl9J1XDCGW3Zl7qN/dvScDk5+D9Lpkn3C4qlO+X/fsfG9z8i99Chgm2BAwfI/GEZuQcPFUsf6Tp27kzKphRSt2whOzubGdOmcXHvPoXS9Ordh8mTJgEwOzGR8y44v2DfxX36kLp5M2vXrC10zOKvvyYrM8v7C5AK5XmgY2aXmdlYM3vJzH7rdX7lrXFsDBlpaQXr29P8NI6J+dnnadexA9Wio9masrk8ixeWGsfGkh50zzP8fhrH/vx7LtAkJgb/tiP3Mj3NT5Miz2+TmNiCNIFAgB/37i3WFdVn0EBm5ree5eTk8Nxjo/l4ViJzvl7ESaecwuQJEz2+kqqhRuNGHAgK+g5kbOe4xo0Kpalz6inUiGnCji++CnXxqqyY2Bj8adsK1tPT0ogpUifExMaQti0vTSAQYO/evdSvX5+aNWty6+238dpfXglpmaXy8DTQMbO/AbcBS4FlwP+Z2etHST/MzBab2WL/nkwvi1ZmVsI29zOb5ho2bszjY5/njw8+8rOPjURW8k0PeTnCgZVwM4veypLud/Bz2r5zJw7sP8D6NWsAqFatGldefx1XDxxM767nsXbVKn53R/ExExGp5Ie30P7TRt7J6lf+FrIihQMroSYuWpeW/Kw7Rtw7kvf+8Tb79u3zrHxVWQBX7ktl4/X08h5Ae5f/RJrZe+QFPSVyzo0DxgF0O6ltpbhbGf50mjRtWrDeuGksOzLK3kxfq/bxPP/2m/z9pb+w/PsfvChi2MlI8xMTdM+bxMaqa+QXSvf7iW125F7GNI1le0Z6iWky/H6ioqKoXacOWZlH/tDoFz+IWUFjoQ4PSE7dnNc6+dmMRH53+21eXkaVcSBjOzVimhSs12jSmIPbdxasV6tVi9ont+bsN/4CQPWGDej84rN8f/+j7Fm5JuTlrSr8fj+xTZsVrMc0bUpGkTrBn+anabNmpOc/x3Xq1CEzM5NOnTvTt/8A7n/4YerWrUuuy+XgwYN8+N74UF+GVBCvu65WAy2D1lsASzzOs1ytWrKUFq1OpGlcc6pFR3PJoAF8mZRcpmOrRUfz3JuvMWvyVObN1Kj+slqxZCktWrWiWVwc1aKj6RM/kM+T5lZ0saqk5T8soWXQvewbP4gFcwrfywVz5hJ/+WUAXDKgP98sXFSwz8zoPaA/s6YdGWif4U/npDanUL9B3nizc7t1Y8O6dSG4mspvz4pV1GrRnJrNYrFq1Yjt04uMzxcW7M/56Sfm97mUzy+9hs8vvYasZSsU5JTB0h9+oFXrVsS1iCM6OpqB8fEkz5lTKE1y0hx+e/nlAPQdMICvFubd9+uuvIqLu3Xj4m7deO+dd3jr9dcV5EQYr1t0GgIrzezr/PWzgUVmlgDgnBvscf6/WiAQYOyTTzP2vbfx+XzM+HgSG9eu4/f3jGDV0mV8OXcebTu257k3XqPOCXW54OKe/P7u4dzQL55eA/rR+ewunFCvHgMuzxue9OwDD7Nu5aoKvqrKLRAI8MITT/Hq+Hfw+aKY9vEnbFi7jmEj72bl0qV8npRMu44deP7Nv1H3hLpceHFPht1zF1f3HVDRRa90AoEAfxr9FG+M/ye+KB9TJ37C+rVruX3kPaxYupQFSXOZPHEiz459iYT5yezJzOShEXcXHH/WOV1J9/vZumVLwbbtGRm89cqrvD3x3+Rk55C2dSuj73+wIi6v0nGBXFa98Cq/efV5zOdj67SZ/LRhEycPu4U9K1ezPSjoKcmFU/5NteNrYdHRNOnRjW/veoCfNqaEqPSVVyAQYMzo0fxj/HiioqKYNHEi69au5a57R7JsyVKSk5L4ZMJEXnh5LJ8tmE9WZiYjh4845nlfevVVup53LvXr12fBV4v468sv84nGm4Ud83LMiJn1ONp+59yC0vZVlq6rcHbIBSq6CGHvkMut6CJEhOebtDx2IvlVRqRvrOgiRITVKZtKGujlmTvbX1Du/9a+vuzLkF7DsXjaonO0QEZERETEa54EOmb2hXOum5ntpdCUAwxwzrm6XuQrIiIiEsyTQMc51y3/p14DLCIiUkkFIuDVHV6/R+f3JWz7k5d5ioiIiBzm9ayrK8zsgHPuQyh4gWANj/MUERERAbwPdC4DEswsF+gP7HLO3eFxniIiIiKAd4ORg79c+QdgCvAlMMbMGjjndnmRr4iIiJRdZfxkQ3nzqkXnW4rPthoIHH6j20ke5SsiIiJSwJPByM651s65k4BRQGfnXGvgHeAH4Aov8hQREREpyutvXT3mnNtjZt2A3sA/gTc8zlNERETKINe5cl8qG68DncPfGBgIvOmcmwpU9zhPEREREcD7QGermb0FXAUkmtlxIchTREREBPB+evlVQD/gRedcppk1BR7wOE8REREpA826+pWcc/uAT4PW04A0L/MUEREROUzdSCIiIhK2FOiIiIhI2PJ6jI6IiIhUUpEwRkctOiIiIhK2FOiIiIhI2FLXlYiISISqjG8yLm9q0REREZGwpUBHREREwpYCHREREQlbGqMjIiISoTS9XERERKQKU6AjIiIiYUtdVyIiIhEqoOnlIiIiIlWXAh0REREJW+q6EhERiVC5mnUlIiIiUnUp0BEREZGwpUBHREREwpbG6IiIiEQoTS8XERERqcIU6IiIiEjYUteViIhIhMpV15WIiIhI1VVpW3S2BfZXdBHCXjRW0UUIe1G6xyExIn1jRRch7K1OfKmiiyDyi6hFR0RERMJWpW3REREREW8F9AkIERERkapLgY6IiIiELXVdiYiIRKhcl1vRRfCcWnREREQkpMysn5mtNrN1ZjaqhP3dzew7M8sxsyuK7AuY2ff5S8Kx8lKLjoiIiISMmUUBrwO9gVTgGzNLcM6tCEq2GbgZuL+EU+x3znUua34KdERERCJUbsXMuuoKrHPObQAws4+AIUBBoOOc25S/71f3ranrSkRERMqNmQ0zs8VBy7AiSZoDW4LWU/O3lVWN/PN+ZWaXHiuxWnRERESk3DjnxgHjjpKkpFfG/5ympZbOuW1mdhKQbGZLnXPrS0usFh0REREJpVSgRdB6HLCtrAc757bl/9wAzAfOPFp6BToiIiIRKuBcuS9l8A3Qxsxam1l14GrgmLOnAMysvpkdl/97I+ACgsb2lESBjoiIiISMcy4HGA7MBlYCE51zy81sjJkNBjCzs80sFbgSeMvMlucf3g5YbGY/APOAPxWZrVWMxuiIiIhISDnnEoHEIttGB/3+DXldWv/f3p3HV1Gdfxz/PAlBQcEItCQBFNytC1iRKqAoCAqKbQVBxIVaVLBgBanFnxQVbcVqpVVUFKpA3dhE2WRJgCiKUqQQEIvBymJNwLIEbShKOL8/5iTebKyZ3OTe7/v1uq/MdufMPHfu3Cdzzswp+b73gXMOpSwlOiIiInEqSreXVypVXYmIiEjMUqIjIiIiMUuJjoiIiMQstdERERGJU/sO7nbwak1XdERERCRmKdERERGRmKWqKxERkTh1xF2DVwO6oiMiIiIxS4mOiIiIxCxVXYmIiMQp3XUlIiIiUo0p0REREZGYpURHREREYpba6IiIiMQp9V4uIiIiUo0p0REREZGYpaorERGROKXby0VERESqMSU6IiIiErOU6IiIiEjMUhsdERGROKXby0VERESqMSU6IiIiErNUdSUiIhKnVHUVxy5p1470hQtZmJlJv/79S82vWbMmT40ezcLMTN54800aNW4MQHJyMq+8/jqr167lwREjir0nKSmJ3z/6KBmLFrEgI4MrO3eulH2pyi5u1465CzOYn7mY28qIc1LNmowaPZr5mYuZHCE5sQIAABRUSURBVBHnQqlpaaxY+zG33n5b0bSMJUuYMW8ub86Zw7SZM0Lfh6qubbtLmL0wg7mZi+jbv1+p+Uk1a/Kn0U8zN3MRr785nbTGjQBIa9yIFes+4Y05s3ljzmwe+P0jpd47etxY3po/N/R9qE7COKaltHdWfMoVA56k451P8MIbmaXmf/nVTm4aPpaf3fM0XQc9ReZH6wDY8XU+Nw0fy3k3PMiIsTo/xANd0SlDQkICDz38MDf37k1ubi5vzphBeno667Ozi5bp0bMnu/LyaN+uHVd37cpvhw7lrgED2LNnD6OeeILTTj+d004/vdh6fzVgANu2baPDZZdhZiQnJ1f2rlUpCQkJDH94BL/ofSNbcnOZOmMGC9MX8Fn2+qJlruvZg115eXRqdyldunZlyNChDBowoGj+fcN/x7uLF5da9y3X92LHjh2VsRtVWkJCAsMeHkHf3jexJTeXSTPeYlF6erEYd/MxvrLdZXTuejX3DB3KPQMGArB540au7XJVmeu+/MoryM//b6XsR3UR5jEt3yso2MeIsTN46YFbaVi/Lt3vfZb2F5zBKU0aFi3z3NRFdG59DjdceSHrN2/h9kcmsPD5ezkqqQa/7tWR7E1byN60JYp7IZVFV3TK0LxFCzZu2MDmzZv57rvvmDVzJh07diy2zOUdOzJt2jQA3p4zh9Zt2gCwe/duli9fzp49e0qtt3uPHjz3zDMAOOfi/of43BYt2LhhI1/4OM+eOZMOHTsVW6Z9x05M93GeN2cOF7VpXTSvQ6dOfLFpE9mfZiNlO6dFczZFxPjtmTNpX+JYbt+xI2/6GM+f8zYXRsS4PLVr1+aWvr/k+adHh7Ld1ZWO6cqRtf4LTkytT5OUetRMqsFVbc8lY9knxZYx4Jv84Dz8df4eflivLgC1j65JyzObclSS/s8H2Ocq/lXVKNEpQ0pKCjk5OUXjOTk5NExJKbZMw5QUcr78EoCCggK+/vprjj/++HLXWadu8CUbPGQIM2bPZvSzz9KgQYMQtr76aJjSkNycL4vGt+Tk0DClYallyopzrVq1uK1/P0b/+S9lrNnx15f/xrRZM+nRq1eYu1DlNUxJITfiWM7NyeWHpY7lhuR+GSxTGONkfyw3atKEaXNmMWHS65x/wQVF7xl4z2DGjx3H7t27K2Evqo/wjmmJtGVbHin1jysab1j/OLZs31VsmQE9OzDznZVc0ncktz8ynmF9u1b2ZkoVEWqiY2anmdlYM5tvZgsLX/tZ/nYzW25my3d9802Ym3bIXIn+QMzsgMtEqpGYSFpaGh8tX841V13FP1as4L7776/w7axOjAPHsLw4Dxw8iAnj/kp+fn6p+b2u7ca1V13Nbbf0offNN9OyVauK2+hqpqwYc5Ax/mrrV3S4qA3dulzNYw8/wh+f+jPHHHssZ/zoTE5o2pSMefPD2uxqK6xjWoor60xbMqqzl2Tx88t+zDvjhvLCsD7c+5fJ7Nu3rzI2T6qYsK/dTQHGAGOBggMt7Jx7AXgB4KQTT4zaBbDc3FxSU1OLxlNTU9m6pXhdbm5ODqlpaeTm5pKYmEidOnXYuXNnuevcsWMH+fn5zJsbNNycM3s21/XsGc4OVBO5ubmkpKYVjTdMTWXrlq3Fl8nJJTUtjS0l4ty8RQuu6NyFIffdR926ddnn9rFnzx5emTCRrVuDdWzfto0F8+ZxbovmLF+2rFL3rarIzc0hJeJYTklNKeNYziUlLbVYjPP8sZz37bcArF2zhs0bN9G0WTPObn4uZ51zNguWvEtijUTq16/P+Ndfo8/18X31DMI7pqW4lPrHkbstr2h8y7a8oqqpQlMzljPud30AOO/0E9jz3V527MqnfvKxlbmpUgWEXXW11zn3nHNumXPuo8JXyGUesaxVq2jarBmNmzQhKSmJq7t2JX3BgmLLZKSn061bNwA6d+nC0vffP+B6M9LTufCiiwBo3aZNscbN8Wj1qlU0bdaUxk0ak5SUxFVdu7KwRJwXpi/g5z7OV3Tpwgc+zr2v60GHtm3p0LYtE158keefeYZXJkykVq1aHHPMMQDUqlWLNpdcTPa6Tyt1v6qSNauyOLFZUxr5GHfu2pVFC9KLLbMoPZ2f+Rh36tKZD99fCsDx9eqRkBCcIho3acKJzZryxaZNTHr5FS5tdSEd217Mjd2vY8PnnyvJ8cI4pqW0c05pxIac/7B5y3a+/W4vs5dk0f6CM4stk9ogmaVZnwHw2Rdb2fPtXuodd0w0NrdK24er8FdVE8oVHTOr5wdnmtmdwHSgqHWuc257GOVWlIKCAh4cPpwJEyeSkJjIlMmTyc7O5u7Bg1mdlUVGejqTJk3iyVGjWJiZSd7OndwVcdfEO0uWcGydOiQlJdGxUyduuekm1mdn89jIkTw5ahS/Gz6c7du3c++QIVHcy+grKChgxPDhjJs4kcTERKZNnsz67GzuGjyINVmrWZieztRJk3l81JPMz1xM3s6dDPJ3A5WnfoMGPPPCCwAk1khk1ltv8W5m6VtP40VBQQG/H/4AYydOJCExgemTp7A+O5sBgwfxcdZqFqWnM23SJB4bNYq5mYvYuTOPIT7GLX/SioGDB7F3bwH79hXw0P8NIy8v7wAlxrcwjmkprUZiIsP7XkPfES9RsM/RrcP5nHpCQ/7y2gLOPrkxHVqdydA+nRn27HTGz3wPM2PkwO5F1Ybt7/gj3+zew3d7C0j/cC0vPvCLYndsSWyx/bUrOeyVmn1OUI1aRgMBnHPupAOtI5pVV/EiqcyPRypSomJcKQqq4H+RsWbdnD9FexPiw1ndKvWk0aLpyRX+5Vm54bMqdeIL5YqOc65ZGOsVERGRilMVq5oqWth3Xf3KzJIjxo/3VVkiIiIioQu7MfJtzrmiW5GcczsAPddcREREKkXYiU6CRTw0wswSgZohlykiIiIChP8cnXnAZDMbQ9A4uR+gHgBFRESqgBDuR6pywk50fgvcAfQnuANrPjAu5DJFREREgJATHefcPjMbDyx0zq0LsywRERGRksK+6+oaYCW+usrMWpjZjDDLFBERkYMTD09GDrsx8gNAK2AngHNuJdA05DJFREREgMrp60rPjBcREZGoCLsx8hozuwFINLNTgbuAA/d+KSIiIqGrehVNFS/sKzoDgbMIOvR8FcgD7g65TBEREREg/Luu8oH7zewPzrn/hlmWiIiISElh33XV2szWAp/48eZm9myYZYqIiIgUCrvqahRwBbANwDm3Crgk5DJFRETkIOj28grgnNtcYlJB2GWKiIiIQPh3XW02s9aAM7OaBHddfRJymSIiIiJA+Fd0+gG/AhoB/wZa+HERERGJMhfCq6oJ+66r/wC9wyxDREREpDxh33V1kpnNNLOvzGyrmb1lZieFWaaIiIhIobCrrl4FJgOpQBowBXgt5DJFREREgPATHXPO/c05t9e/XqZqVuGJiIjEHbXROXKLzGwo8DrB/vcEZptZPQDn3PaQyxcREZE4Fnai09P/vYPvEz0DbvXjaq8jIiIioQm76uq3QHPnXDPgJWAV0M0518w5pyRHREQkivRk5CM3zDm3y8zaAh2B8cBzIZcpIiIiAoSf6BR293AVMMY59xZQM+QyRURERIDwE51/m9nzQA9gjpkdVQllioiIyEGIh7uuwk46egDzgCudczuBesBvQi5TREREBAi/C4h84I2I8RwgJ8wyRURERAqpGklERERiVtjP0REREZEqqiq2qalouqIjIiIiMUuJjoiIiMQsVV2JiIjEKVVdiYiIiFRjSnREREQkZinRERERkZilNjoiIiJxKh7a6Jhz8bCblcPMbnfOvRDt7YhlinH4FOPKoTiHTzEWUNVVRbs92hsQBxTj8CnGlUNxDp9iLEp0REREJHYp0REREZGYpUSnYqkuOHyKcfgU48qhOIdPMRY1RhYREZHYpSs6IiIiErOU6IiIiEjMUqITEjNLNrM7I8bTzGxqNLcpVphZUzO74TDf+01Fb08sMbN+ZnazH+5jZmkR88aZ2Y+it3XVl5ltMLMGZUx/PxrbI2Ur73OS6k2JTniSgaJExzn3pXOuexS3J5Y0BcpMdMxMT/s+As65Mc65iX60D5AWMa+vc25tVDYsRjnnWkd7G0RiXdwmOv6qwCdmNtbMPjaz+WZWy8xONrO5ZvaRmb1rZmf45U82sw/M7O9mNqLwyoCZHWtmGWa2wsxWm9lPfREjgZPNbKWZPe7LW+Pf86GZnRWxLYvN7HwzO8bMXvRl/CNiXTHhMGI+3sy6R7y/8GrMSOBiH9tB/srDFDObCczfz2cS03x8/2lmE8wsy8ymmlltM+vgj6fV/vg6yi8/0szW+mWf8NMeNLMhPu4tgVd8nGv547SlmfU3sz9GlNvHzJ72wzea2TL/nufNLDEasYgm/z2ebWarzGyNmfWMmFfLH+u3+fHC88ilZvaOmU33n8kYM4vb8zOUHUd/xeUxf4wtM7NT/LI/MLNp/tz5dzNrE7GOUudUM0s0syf8dyLLzAZGFD0w4txxRhR2XSqacy4uXwRXBfYCLfz4ZOBGIAM41U/7CbDQD88CevnhfsA3frgGUNcPNwDWA+bXv6ZEeWv88CDgIT+cCnzqh/8A3OiHk4FPgWOiHasoxnw80D3i/YUxvxSYFTG9D/AFUG9/n0nkOmLx5ePrgDZ+/EVgGLAZOM1PmwjcDdQD1kXEJdn/fRAY4ocXAy0j1r+YIPn5AbA+YvrbQFvgTGAmkOSnPwvcHO24ROFz6AaMjRg/DtjgP5/0yJiUOKb/B5wEJAILIo/9eHztJ473+/GbC88DwKtAWz98AvCJHy7znAr0B6YBNfy8wnPHBmCgH74TGBftOOh15K+4/o8B+Nw5t9IPf0RwImoNTDGzlcDzBIkIwEXAFD/8asQ6DPiDmWURnMQaAQ0PUO5k4Do/3CNivZ2Aob7sxcDRBF/aWHIoMT8UC5xz2/3w4XwmsWKzc+49P/wy0IEg5p/6aROAS4BdBD+s48zsWiD/YAtwzn0F/MvMLjSz+sDpwHu+rPOBv/vPsgPBD3e8WQ1c7q88XOycy/PT3wJect9XDZa0zDn3L+dcAfAaQfIYz8qL42sRfy/yw5cDo/1xNwOoa2Z1KP+cejkwxjm3FyDi3AHwhv9beH6Sai7e2zPsiRguIPgx3Omca3EI6+hN8B/u+c6578xsA8GXqVzOuX+b2TYzOxfoCdzhZxnQzTm37hDKr24OJeZ78dWrZmZAzf2s978Rw4f8mcSQg3owlnNur5m1IkhGrgcGAO0PoZxJBEn6P4HpzjnnP6MJzrn7DnGbY4pz7lMzOx/oAjxqZvP9rPeAzmb2qnOurM+p5LS4fsjZfuIYGZfC4QTgIufc7sh1+GOy1DnVTy8vvoXnqAL0GxkT4v2KTkm7gM/N7DoIvgxm1tzP+4DgUioEPwyFjgO2+h/Uy4AT/fSvgTr7Ket14F7gOOfcaj9tHkH9sPnyzzvSHaoG9hfzDQRXCAB+CiT54QPFtrzPJB6cYGaF/+X2Irii1bSwLQNwE5BpZscSHHtzCKqyyko09xfnN4Cf+TIm+WkZQHcz+yGAmdUzs3iKPRDcYQnkO+deBp4AfuxnDQe2EVTplaWVmTXzbXN6AktC39gqbD9x7Bnxd6kfnk+QrBe+t/B4Lu+cOh/oZ/7mBTOrF9Z+SPQp0SmtN/BLM1sFfEzwAwvBj8FgM1tGULVSeBn1FaClmS337/0ngHNuG/Ceb0T3eBnlTCVImCZHTHuY4Mc8y4KGyw9X6J5VXeXFfCzQzsf8J3x/1SYL2OsbKQ4qY31lfiZx4hPgFl9tVw8YBfyCoGpwNbAPGEOQwMzyy2UStBsraTwwxnxj5MgZzrkdwFrgROfcMj9tLUGboPl+vQs4vGrI6u4cYJmvLrkfeCRi3t3A0RbRmDvCUoKG9muAz4HpYW9oFVdeHI8ysw+BX/P9cXsXwXc+y8zWErSjhPLPqeOATX76Ksq5i1Nig7qAOEhmVhvY7S/RX0/QMDku7uaR6sHMmhI0zjw7ypsih8jMLiVoBH51tLelKvPV0C2dc/+J9rZI9aH6x4N3PkFjNwN2ArdGeXtERETkAHRFR0RERGKW2uiIiIhIzFKiIyIiIjFLiY6IiIjELCU6ItWUmRX4W7/XWNDXV+0jWNelZjbLD19jZkP3s2yymd1Z3vz9vO9BMxtyuNsoInI4lOiIVF+7nXMt/O3k3/L9s0OAoocvHvJ33Dk3wzk3cj+LJBP0AyQiUuUp0RGJDe8Cp9j3PcQ/C6wAmphZJzNb6ntknuKfioyZXWlBb+dLgGsLV2RBb+Sj/XBDC3rUXuVfrQkeaneyv5r0uF/uNxb0EJ1lZg9FrOt+M1tnZukEfWKJiFQqJToi1Zx/jH1ngk4QIUgoJjrnziN4mvQw4HLn3I+B5QRP+D6a4MnTXYGLgZRyVv8UkOmca07wCP6PgaHAZ/5q0m/MrBNwKtCKoCuJ883sEt9P0fXAeQSJ1AUVvOsiIgekBwaKVF+1/OPxIbii81cgDdjonPvAT78Q+BFBdyQQdIy6FDiDoFfzbAAzexm4vYwy2gM3A/hetfPM7PgSy3Tyr3/48WMJEp86BB1+5vsyZhzR3oqIHAYlOiLV1+6Svb77ZCayJ3cDFjjnepVYrgUV1zu2AY86554vUcbdFViGiMhhUdWVSGz7AGhT2Hu5mdU2s9MIOjptZmYn++V6lfP+DKC/f2+imdWldK/m84BbI9r+NPI9mL8D/NzMaplZHYJqMhGRSqVERySGOee+AvoAr/kexT8AznDO/Y+gqmq2b4y8sZxV/Bq4zPd8/hFwlnNuG0FV2Boze9w5Nx94FVjql5sK1HHOrQAmASuBaQTVayIilUp9XYmIiEjM0hUdERERiVlKdERERCRmKdERERGRmKVER0RERGKWEh0RERGJWUp0REREJGYp0REREZGY9f+C9bAtgiMM7AAAAABJRU5ErkJggg==\n",
      "text/plain": [
       "<Figure size 720x576 with 2 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "from sklearn.metrics import confusion_matrix\n",
    "conf_mat = confusion_matrix(y_test, y_pred)\n",
    "conf_mat_ratios = np.round(conf_mat / np.sum(conf_mat, axis=0),3)\n",
    "fig, ax = plt.subplots(figsize=(10,8))\n",
    "\n",
    "labels = sorted(list(set(le.inverse_transform(y_pred))))\n",
    "sns.heatmap(conf_mat_ratios, annot=True, center=0, xticklabels=labels, yticklabels=labels)\n",
    "plt.ylabel('Actual')\n",
    "plt.xlabel('Predicted')\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 65,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/nerdikon/anaconda3/lib/python3.7/site-packages/sklearn/preprocessing/label.py:151: DeprecationWarning: The truth value of an empty array is ambiguous. Returning False, but in future this will result in an error. Use `array.size > 0` to check that an array is not empty.\n",
      "  if diff:\n",
      "/home/nerdikon/anaconda3/lib/python3.7/site-packages/sklearn/preprocessing/label.py:151: DeprecationWarning: The truth value of an empty array is ambiguous. Returning False, but in future this will result in an error. Use `array.size > 0` to check that an array is not empty.\n",
      "  if diff:\n"
     ]
    }
   ],
   "source": [
    "y_pred_labels = le.inverse_transform(y_pred)\n",
    "y_test_labels = le.inverse_transform(y_test)\n",
    "\n",
    "results = pd.DataFrame({'text':df_test.text, 'lemmatized_tokens':df_test.lemmatized_tokens,'predicted':y_pred_labels, 'actual':y_test_labels})\n",
    "errors = results[results.predicted != results.actual]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 66,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAskAAAHVCAYAAAADyWaQAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDIuMi4zLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvIxREBQAAH6dJREFUeJzt3X+8XHV95/H3pwQMvwoNRB9KLGEtClUk4K2CoFWxVrEPUJEVVwu6VLRqre1aN9rtSrfUpcJDbdcVSoUF1hq0CIr4o1Eo1eIPTCAiEpGoUSIWImiEBVTgu3/ck3j5ciE34V7mBp7Px+M+5sx3zsx8597DzIuTMzPVWgsAAPBLvzLqCQAAwGwjkgEAoCOSAQCgI5IBAKAjkgEAoCOSAQCgI5IBAKAjkgEAoCOSAQCgM2fUE0iSXXfdtS1cuHDU0wAA4CFu+fLlP2qtzd/YerMikhcuXJhly5aNehoAADzEVdX3prKewy0AAKAjkgEAoCOSAQCgMyuOSQYAIPnFL36RNWvW5I477hj1VLZ4c+fOzYIFC7L11ltv1vVFMgDALLFmzZrsuOOOWbhwYapq1NPZYrXWctNNN2XNmjXZY489Nus2HG4BADBL3HHHHdlll10E8gNUVdlll10e0B55kQwAMIsI5OnxQH+PIhkAADqOSQYAmKUWLv7ktN7e6hNfuNF1ttpqq+yzzz658847s/fee+ess87Kdtttt1n3d8kll+Tkk0/OhRdemAsuuCBXX311Fi9ePOm6P/nJT/KhD30or3/96zfpPo4//vjssMMOectb3rJZc7wv9iQDALDBtttumxUrVuSqq67KNttsk1NPPfUel7fWcvfdd2/y7R522GH3GcjJeCS///3v3+TbnSkiGQCAST3jGc/IqlWrsnr16uy99955/etfn/333z/XXXddli5dmgMPPDD7779/jjzyyNx6661Jks985jPZa6+9cvDBB+e8887bcFtnnnlm3vjGNyZJbrjhhrz4xS/Ovvvum3333Tdf/OIXs3jx4nz729/OokWL8md/9mdJkpNOOim/9Vu/lSc/+cl5xzveseG2/vqv/zpPeMIT8tznPjfXXHPNjDx2kQwAwL3ceeed+fSnP5199tknSXLNNdfk6KOPzhVXXJHtt98+J5xwQj73uc/l8ssvz9jYWN797nfnjjvuyGte85p84hOfyBe+8IX8+7//+6S3/aY3vSm//du/na997Wu5/PLL88QnPjEnnnhiHve4x2XFihU56aSTsnTp0lx77bW57LLLsmLFiixfvjyf//zns3z58pxzzjm54oorct555+WrX/3qjDx+xyQDALDB7bffnkWLFiUZ35N87LHH5vrrr8/uu++eAw44IEny5S9/OVdffXUOOuigJMnPf/7zHHjggfnmN7+ZPfbYI3vuuWeS5JWvfGVOO+20e93HxRdfnLPPPjvJ+DHQO+20U3784x/fY52lS5dm6dKl2W+//ZIkt956a6699trccsstefGLX7zhOOnDDjtsBn4LIhkAgAnWH5Pc23777Tcst9byO7/zO1myZMk91lmxYsW0fYRday1ve9vb8trXvvYe4+9973sflI/Jc7gFAACb5IADDsill16aVatWJUluu+22fOtb38pee+2V7373u/n2t7+dJPeK6PUOOeSQnHLKKUmSu+66Kz/96U+z44475pZbbtmwzu/+7u/mjDPO2HCs8w9+8IPceOONeeYzn5nzzz8/t99+e2655ZZ84hOfmJHHaE8yAMAsNZWPbBuF+fPn58wzz8zLX/7y/OxnP0uSnHDCCXn84x+f0047LS984Quz66675uCDD85VV111r+v/7d/+bY477ricfvrp2WqrrXLKKafkwAMPzEEHHZQnPelJecELXpCTTjopK1euzIEHHpgk2WGHHfLBD34w+++/f172spdl0aJF2X333fOMZzxjRh5jtdZm5IY3xdjYWFu2bNmopwEAMFIrV67M3nvvPeppPGRM9vusquWttbGNXdfhFgAA0BHJAADQcUzyBNP91Y+bY7YeewQA8HBiTzIAAHREMgAAdEQyAAB0HJMMADBbHb/TNN/euimtdv755+clL3lJVq5cmb322us+1zvzzDPzvOc9L495zGM2azqXXHJJTj755Fx44YWbdf2ZZE8yAAD3sGTJkhx88ME555xz7ne9M888M9dff/2DNKsHl0gGAGCDW2+9NZdeemlOP/30e0Tyu971ruyzzz7Zd999s3jx4px77rlZtmxZXvGKV2TRokW5/fbbs3DhwvzoRz9KkixbtizPetazkiSXXXZZnv70p2e//fbL05/+9FxzzTWjeGibxOEWAABs8LGPfSzPf/7z8/jHPz7z5s3L5ZdfnhtuuCEf+9jH8pWvfCXbbbddbr755sybNy/ve9/7cvLJJ2ds7P6/wG6vvfbK5z//+cyZMyef+9zn8va3vz0f/ehHH6RHtHlEMgAAGyxZsiRvfvObkyRHHXVUlixZkrvvvjuvfvWrs9122yVJ5s2bt0m3uW7duhxzzDG59tprU1X5xS9+Me3znm4iGQCAJMlNN92Uiy++OFdddVWqKnfddVeqKkcccUSqaqPXnzNnTu6+++4kyR133LFh/C/+4i/y7Gc/O+eff35Wr1694TCM2cwxyQAAJEnOPffcHH300fne976X1atX57rrrssee+yRefPm5Ywzzshtt92WJLn55puTJDvuuGNuueWWDddfuHBhli9fniT3OJxi3bp12W233ZKMv9lvS2BPMgDAbDXFj2ybLkuWLMnixYvvMXbEEUdk5cqVOeywwzI2NpZtttkmhx56aN75znfmVa96VV73utdl2223zZe+9KW84x3vyLHHHpt3vvOdedrTnrbhNt761rfmmGOOybvf/e485znPeVAf0+aq1tqo55CxsbG2bNmyUU8jCxd/ctRTyOoTXzjqKQAAI7Jy5crsvffeo57GQ8Zkv8+qWt5au/93GsbhFgAAcC8iGQAAOiIZAGAWmQ2Hwj4UPNDfo0gGAJgl5s6dm5tuukkoP0Cttdx0002ZO3fuZt+GT7cAAJglFixYkDVr1mTt2rWjnsoWb+7cuVmwYMFmX18kAwDMEltvvXX22GOPUU+DONwCAADuRSQDAEBHJAMAQEckAwBARyQDAEBHJAMAQEckAwBARyQDAEBHJAMAQEckAwBARyQDAEBHJAMAQEckAwBARyQDAEBHJAMAQEckAwBARyQDAEBHJAMAQEckAwBARyQDAEBHJAMAQEckAwBARyQDAEBHJAMAQEckAwBARyQDAEBnSpFcVaur6utVtaKqlg1j86rqs1V17XD6a8N4VdXfVdWqqrqyqvafyQcAAADTbVP2JD+7tbaotTY2nF+c5KLW2p5JLhrOJ8kLkuw5/ByX5JTpmiwAADwYHsjhFocnOWtYPivJiyaMn93GfTnJzlX16AdwPwAA8KCaaiS3JEuranlVHTeMPaq19sMkGU4fOYzvluS6CdddM4zdQ1UdV1XLqmrZ2rVrN2/2AAAwA+ZMcb2DWmvXV9Ujk3y2qr55P+vWJGPtXgOtnZbktCQZGxu71+UAADAqU9qT3Fq7fji9Mcn5SZ6a5Ib1h1EMpzcOq69J8tgJV1+Q5PrpmjAAAMy0jUZyVW1fVTuuX07yvCRXJbkgyTHDasck+fiwfEGSo4dPuTggybr1h2UAAMCWYCqHWzwqyflVtX79D7XWPlNVX03ykao6Nsn3kxw5rP+pJIcmWZXktiSvnvZZAwDADNpoJLfWvpNk30nGb0pyyCTjLckbpmV2AAAwAr5xDwAAOiIZAAA6IhkAADoiGQAAOiIZAAA6IhkAADoiGQAAOiIZAAA6IhkAADoiGQAAOiIZAAA6IhkAADoiGQAAOiIZAAA6IhkAADoiGQAAOiIZAAA6IhkAADoiGQAAOiIZAAA6IhkAADoiGQAAOiIZAAA6IhkAADoiGQAAOiIZAAA6IhkAADoiGQAAOiIZAAA6IhkAADoiGQAAOiIZAAA6IhkAADoiGQAAOiIZAAA6IhkAADoiGQAAOiIZAAA6IhkAADoiGQAAOiIZAAA6IhkAADoiGQAAOiIZAAA6IhkAADoiGQAAOiIZAAA6IhkAADoiGQAAOiIZAAA6IhkAADoiGQAAOiIZAAA6IhkAADoiGQAAOiIZAAA6IhkAADoiGQAAOiIZAAA6IhkAADoiGQAAOiIZAAA6IhkAADoiGQAAOlOO5KraqqquqKoLh/N7VNVXquraqvpwVW0zjD9iOL9quHzhzEwdAABmxqbsSf7jJCsnnP+bJO9pre2Z5MdJjh3Gj03y49babyR5z7AeAABsMaYUyVW1IMkLk3xgOF9JnpPk3GGVs5K8aFg+fDif4fJDhvUBAGCLMNU9ye9N8tYkdw/nd0nyk9bancP5NUl2G5Z3S3JdkgyXrxvWv4eqOq6qllXVsrVr127m9AEAYPptNJKr6veS3NhaWz5xeJJV2xQu++VAa6e11sZaa2Pz58+f0mQBAODBMGcK6xyU5LCqOjTJ3CS/mvE9yztX1Zxhb/GCJNcP669J8tgka6pqTpKdktw87TMHAIAZstE9ya21t7XWFrTWFiY5KsnFrbVXJPmXJC8dVjsmyceH5QuG8xkuv7i1dq89yQAAMFs9kM9J/q9J/rSqVmX8mOPTh/HTk+wyjP9pksUPbIoAAPDgmsrhFhu01i5Jcsmw/J0kT51knTuSHDkNcwMAgJHwjXsAANARyQAA0BHJAADQEckAANARyQAA0BHJAADQEckAANARyQAA0BHJAADQEckAANARyQAA0BHJAADQEckAANARyQAA0BHJAADQEckAANARyQAA0BHJAADQEckAANARyQAA0BHJAADQEckAANARyQAA0BHJAADQEckAANARyQAA0BHJAADQEckAANARyQAA0BHJAADQEckAANARyQAA0BHJAADQEckAANARyQAA0BHJAADQEckAANARyQAA0BHJAADQEckAANARyQAA0BHJAADQEckAANARyQAA0BHJAADQEckAANARyQAA0BHJAADQEckAANARyQAA0BHJAADQEckAANARyQAA0BHJAADQEckAANARyQAA0BHJAADQEckAANARyQAA0BHJAADQEckAANARyQAA0BHJAADQEckAANDZaCRX1dyquqyqvlZV36iqvxzG96iqr1TVtVX14araZhh/xHB+1XD5wpl9CAAAML2msif5Z0me01rbN8miJM+vqgOS/E2S97TW9kzy4yTHDusfm+THrbXfSPKeYT0AANhibDSS27hbh7NbDz8tyXOSnDuMn5XkRcPy4cP5DJcfUlU1bTMGAIAZNqVjkqtqq6pakeTGJJ9N8u0kP2mt3TmssibJbsPybkmuS5Lh8nVJdpnkNo+rqmVVtWzt2rUP7FEAAMA0mlIkt9buaq0tSrIgyVOT7D3ZasPpZHuN270GWjuttTbWWhubP3/+VOcLAAAzbs6mrNxa+0lVXZLkgCQ7V9WcYW/xgiTXD6utSfLYJGuqak6SnZLcPH1Tfog7fqdRz2Dc8etGPQMAgJGZyqdbzK+qnYflbZM8N8nKJP+S5KXDasck+fiwfMFwPsPlF7fW7rUnGQAAZqup7El+dJKzqmqrjEf1R1prF1bV1UnOqaoTklyR5PRh/dOT/N+qWpXxPchHzcC8AQBgxmw0kltrVybZb5Lx72T8+OR+/I4kR07L7IBxs+EwHIfgAPAw4hv3AACgI5IBAKAjkgEAoCOSAQCgI5IBAKAjkgEAoCOSAQCgI5IBAKAjkgEAoCOSAQCgI5IBAKAjkgEAoCOSAQCgI5IBAKAjkgEAoCOSAQCgI5IBAKAjkgEAoCOSAQCgI5IBAKAjkgEAoCOSAQCgI5IBAKAjkgEAoCOSAQCgI5IBAKAjkgEAoCOSAQCgI5IBAKAjkgEAoCOSAQCgI5IBAKAjkgEAoCOSAQCgM2fUEwBgMx2/06hnkBy/btQzAJgR9iQDAEBHJAMAQEckAwBARyQDAEBHJAMAQEckAwBARyQDAEBHJAMAQEckAwBARyQDAEBHJAMAQEckAwBARyQDAEBHJAMAQEckAwBARyQDAEBHJAMAQEckAwBARyQDAEBHJAMAQEckAwBARyQDAEBHJAMAQEckAwBARyQDAEBHJAMAQEckAwBARyQDAEBHJAMAQGejkVxVj62qf6mqlVX1jar642F8XlV9tqquHU5/bRivqvq7qlpVVVdW1f4z/SAAAGA6TWVP8p1J/ktrbe8kByR5Q1X9ZpLFSS5qre2Z5KLhfJK8IMmew89xSU6Z9lkDAMAM2mgkt9Z+2Fq7fFi+JcnKJLslOTzJWcNqZyV50bB8eJKz27gvJ9m5qh497TMHAIAZsknHJFfVwiT7JflKkke11n6YjId0kkcOq+2W5LoJV1szjPW3dVxVLauqZWvXrt30mQMAwAyZciRX1Q5JPprkza21n97fqpOMtXsNtHZaa22stTY2f/78qU4DAABm3JQiuaq2zngg/2Nr7bxh+Ib1h1EMpzcO42uSPHbC1RckuX56pgsAADNvKp9uUUlOT7KytfbuCRddkOSYYfmYJB+fMH708CkXByRZt/6wDAAA2BLMmcI6ByX5/SRfr6oVw9jbk5yY5CNVdWyS7yc5crjsU0kOTbIqyW1JXj2tMwYAgBm20Uhurf1bJj/OOEkOmWT9luQND3BeAAAwMr5xDwAAOiIZAAA6IhkAADoiGQAAOiIZAAA6IhkAADoiGQAAOiIZAAA6IhkAADpT+VpqeNhauPiTo55CkmT13FHPAAAeXuxJBgCAjkgGAICOSAYAgI5IBgCAjkgGAICOSAYAgI5IBgCAjkgGAICOSAYAgI5IBgCAjkgGAICOSAYAgI5IBgCAjkgGAICOSAYAgI5IBgCAjkgGAICOSAYAgI5IBgCAjkgGAICOSAYAgI5IBgCAjkgGAIDOnFFPAGBLs3DxJ0c9hSTJ6rmjngHAQ5c9yQAA0LEnGQAeSo7fadQzSI5fN+oZwANmTzIAAHREMgAAdEQyAAB0RDIAAHREMgAAdEQyAAB0RDIAAHREMgAAdEQyAAB0RDIAAHR8LTUATIOFiz856ikkSVbPHfUM4KHBnmQAAOiIZAAA6IhkAADoiGQAAOiIZAAA6IhkAADoiGQAAOiIZAAA6IhkAADoiGQAAOiIZAAA6MwZ9QQAAJhhx+806hkkx68b9Qw2iT3JAADQEckAANARyQAA0BHJAADQEckAANDZaCRX1RlVdWNVXTVhbF5Vfbaqrh1Of20Yr6r6u6paVVVXVtX+Mzl5AACYCVPZk3xmkud3Y4uTXNRa2zPJRcP5JHlBkj2Hn+OSnDI90wQAgAfPRiO5tfb5JDd3w4cnOWtYPivJiyaMn93GfTnJzlX16OmaLAAAPBg295jkR7XWfpgkw+kjh/Hdklw3Yb01wxgAAGwxpvuNezXJWJt0xarjqmpZVS1bu3btNE8DAAA23+ZG8g3rD6MYTm8cxtckeeyE9RYkuX6yG2itndZaG2utjc2fP38zpwEAANNvcyP5giTHDMvHJPn4hPGjh0+5OCDJuvWHZQAAwJZizsZWqKolSZ6VZNeqWpPkHUlOTPKRqjo2yfeTHDms/qkkhyZZleS2JK+egTkDAMCM2mgkt9Zefh8XHTLJui3JGx7opAAAYJR84x4AAHREMgAAdEQyAAB0RDIAAHREMgAAdEQyAAB0RDIAAHREMgAAdEQyAAB0RDIAAHREMgAAdEQyAAB0RDIAAHREMgAAdEQyAAB0RDIAAHTmjHoCAAAPVQsXf3LUU0iSrJ476hlseexJBgCAjkgGAICOSAYAgI5IBgCAjkgGAICOSAYAgI5IBgCAjkgGAICOSAYAgI5IBgCAjkgGAICOSAYAgI5IBgCAjkgGAICOSAYAgI5IBgCAjkgGAICOSAYAgI5IBgCAjkgGAICOSAYAgI5IBgCAjkgGAICOSAYAgI5IBgCAjkgGAICOSAYAgI5IBgCAjkgGAICOSAYAgI5IBgCAjkgGAICOSAYAgI5IBgCAjkgGAICOSAYAgI5IBgCAjkgGAICOSAYAgI5IBgCAjkgGAICOSAYAgI5IBgCAjkgGAICOSAYAgI5IBgCAjkgGAICOSAYAgM6MRHJVPb+qrqmqVVW1eCbuAwAAZsq0R3JVbZXkfyd5QZLfTPLyqvrN6b4fAACYKTOxJ/mpSVa11r7TWvt5knOSHD4D9wMAADOiWmvTe4NVL03y/NbaHwznfz/J01prb+zWOy7JccPZJyS5ZlonsuXaNcmPRj0JZh3bBZOxXTAZ2wWTsV380u6ttfkbW2nODNxxTTJ2rxJvrZ2W5LQZuP8tWlUta62NjXoezC62CyZju2AytgsmY7vYdDNxuMWaJI+dcH5Bkutn4H4AAGBGzEQkfzXJnlW1R1Vtk+SoJBfMwP0AAMCMmPbDLVprd1bVG5P8c5KtkpzRWvvGdN/PQ5hDUJiM7YLJ2C6YjO2CydguNtG0v3EPAAC2dL5xDwAAOiIZAAA6InkWqqqFVfWfNvO6t073fBidqnpdVR09LL+qqh4z4bIP+DbLh4+qWl1Vu04y/sVRzIctQ1XtXFWvn3D+MVV17ijnxOx3X883DzcieXZamGTSSK6qmfhsa2ap1tqprbWzh7OvSvKYCZf9QWvt6pFMjFmjtfb0Uc+BWW3nJBsiubV2fWvtpSOcD2wxRPI0GvYAr6yqf6iqb1TV0qratqoeV1WfqarlVfWFqtprWP/M4RsK119//V7gE5M8o6pWVNWfDHsQ/6mqPpFkaVXtUFUXVdXlVfX1qvK137PQsD18s6rOqqorq+rcqtquqg6pqiuGv90ZVfWIYf0Tq+rqYd2Th7Hjq+otw3YyluQfh+1i26q6pKrGquoPq+pdE+73VVX1v4blV1bVZcN1/r6qthrF74JNU1XbV9Unq+prVXVVVb1swmXbDs8nrxnO3zqcPquqPl9V5w/b0alV5Tl+ltuM143HVdWXq+qrVfU/Jvz97+t14cQkjxueA04a7u+q4TpfqaonTpjLJVX1lGH7O2O4jyu8xozWZM8Hw57evxme3y+rqt8Y1p1fVR8d/nZfraqDJtzGvf6mVbVVVZ08bDNXVtUfTbjrP5qwPe01goc+eq01P9P0k/E9wHcmWTSc/0iSVya5KMmew9jTklw8LJ+Z5KUTrn/rcPqsJBdOGH9Vxr+kZd5wfk6SXx2Wd02yKr/8pJJbR/178HOP7aElOWg4f0aS/5bkuiSPH8bOTvLmJPMy/tXs6/+OOw+nxyd5y7B8SZKxCbd/ScbDeX6SVRPGP53k4CR7J/lEkq2H8fcnOXrUvxc/U9p2jkjyDxPO75Rk9bBNfW7i37F73rgjyX/I+Mdvfnbi84uf2fmzGa8bFyZ5+bD8ugl//0lfF4bbv6q7v6uG5T9J8pfD8qOTfGtYfmeSVw7LOyf5VpLtR/27erj+3M/zwZ8P54/O0AxJPpTk4GH515OsvL+/aZI/TPLRJHOGy9Z3xuokfzQsvz7JB0b9exjFj70M0++7rbUVw/LyjD8hPT3JP1XViiR/n/Eno0312dbazcNyJXlnVV2Z8RfM3ZI86gHNmplyXWvt0mH5g0kOyfg28q1h7Kwkz0zy04wHzgeq6iVJbpvqHbTW1ib5TlUdUFW7JHlCkkuH+3pKkq8O294hGQ8oZr+vJ3nusKfoGa21dcP4x5P8n/bLQ3B6l7XWvtNauyvJkoz/zxKz36a8bhyY5J+G5Q9NuI3NeV34SJIjh+X/OOF2n5dk8XDflySZm/HgYjTu6/lgyYTTA4fl5yZ53/C3uyDJr1bVjrnvv+lzk5zaWrszSSZ0RpKcN5yu3yYfdhzfOv1+NmH5row/Sf2ktbZoknXvzHDIS1VVkm3u53b/34TlV2R87+FTWmu/qKrVGd/gmX2m9EHkbfxLeJ6a8ZA9KskbkzxnE+7nwxl/kftmkvNba23Yps5qrb1tE+fMiLXWvlVVT0lyaJL/WVVLh4suTfKCqvpQG3bx9FfdyHlmp0153bgvm/y60Fr7QVXdVFVPTvKyJK8dLqokR7TWrtmE+2eG3M/zwcT/vtcv/0qSA1trt0+8jeH14F5/02H8vp4n1m+Xd+Vh2ov2JM+8nyb5blUdmYxvkFW173DZ6ozv6UuSw5NsPSzfkmTH+7nNnZLcODwRPjvJ7tM+a6bLr1fV+v/Df3nG9/AsXH/8WJLfT/KvVbVDkp1aa5/K+OEXk7043t92cV6SFw338eFh7KIkL62qRyZJVc2rKtvKFqDGP8XkttbaB5OcnGT/4aL/nuSmjB86M5mnVtUew7HIL0vybzM+WWbC/b1ufDnj//yejP8P9Xr39bqwsdeTc5K8NePPP18fxv4548ej1nD/+z3QB8Tmu5/ng5dNOP3SsLw04ztZ1l93/WvJff1NlyZ5XQ0fClBV82bqcWyJRPKD4xVJjq2qryX5RsaDOEn+IclvV9VlGT/mbP3e4iuT3DkcpP8nk9zePyYZq6plw21/c0ZnzwOxMskxwz+BzkvyniSvzvg/o349yd1JTs34i9iFw3r/mvFjBXtnJjl1eAPOthMvaK39OMnVSXZvrV02jF2d8WOglw63+9ls3qE+PPj2SXLZ8E+jf57khAmXvTnJ3JrwZs0JvpTxN2pdleS7Sc6f6YkyY+7rdePNSf50eN14dJL1//Q+6etCa+2mJJcOb/g6aZL7OTfjsf2RCWN/lfGdNlcOb/L7q2l9ZGyq+3o+eERVfSXJH+eXrxlvyvh2cGVVXZ3x49aT+/6bfiDJ94fxr+U+Plnr4crXUsMMqaqFGX8zxZNGPBUeBqrqWRl/k+fvjXouzJyq2i7J7cMhVUdl/E18Pn3iYWY4nGastfajUc/loexheYwJAGyhnpLxN2ZVkp8k+c8jng88ZNmTDAAAHcckAwBARyQDAEBHJAMAQEckAwBARyQDAEDn/wMO0N78WVsRRQAAAABJRU5ErkJggg==\n",
      "text/plain": [
       "<Figure size 864x576 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "fig, ax = plt.subplots(figsize=(12, 8))\n",
    "freq = pd.DataFrame({'Predicted': errors.predicted.value_counts(), 'Actual':errors.actual.value_counts()})\n",
    "freq.sort_values(by='Predicted', ascending=False).plot.bar(rot=0, ax=ax);"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 67,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>text</th>\n",
       "      <th>lemmatized_tokens</th>\n",
       "      <th>predicted</th>\n",
       "      <th>actual</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>Скоро ты вернешься домой, грязный, не бритый н...</td>\n",
       "      <td>[скоро_ADV, ты_PRON, вернуться_VERB, домой_ADV...</td>\n",
       "      <td>negative</td>\n",
       "      <td>positive</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>Наши красавцы. 1:3 в первом периоде и 7:3 в ит...</td>\n",
       "      <td>[наш_DET, красавец_NOUN, 1_NUM, 3_NUM, в_ADP, ...</td>\n",
       "      <td>neutral</td>\n",
       "      <td>positive</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8</th>\n",
       "      <td>нельзя общаться с людьми, которые нуждаются в ...</td>\n",
       "      <td>[нельзя_ADV, общаться_VERB, с_ADP, человек_NOU...</td>\n",
       "      <td>neutral</td>\n",
       "      <td>negative</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>10</th>\n",
       "      <td>Друзья, скиньте клёвой музычки, в тачке не чег...</td>\n",
       "      <td>[друг_NOUN, скинить_NOUN, клёвый_ADJ, музычка_...</td>\n",
       "      <td>positive</td>\n",
       "      <td>neutral</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>11</th>\n",
       "      <td>Селфи дня ))</td>\n",
       "      <td>[селфь_NOUN, день_NOUN]</td>\n",
       "      <td>neutral</td>\n",
       "      <td>positive</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>14</th>\n",
       "      <td>Хочу такую зиму. ❄</td>\n",
       "      <td>[хотеть_VERB, такой_DET, зима_NOUN, ❄_SYM]</td>\n",
       "      <td>neutral</td>\n",
       "      <td>positive</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>17</th>\n",
       "      <td>Я не писал такое оскобление всем!!!!\\nМеня, ви...</td>\n",
       "      <td>[я_PRON, не_PART, писать_VERB, такой_DET, оско...</td>\n",
       "      <td>neutral</td>\n",
       "      <td>skip</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>18</th>\n",
       "      <td>...и так начало мини отпуска))И новых приключе...</td>\n",
       "      <td>[и_CCONJ, так_ADV, начать_NOUN, минить_NOUN, о...</td>\n",
       "      <td>neutral</td>\n",
       "      <td>positive</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>22</th>\n",
       "      <td>Аю,слова правильные,но такие унылые(((Сьешь ко...</td>\n",
       "      <td>[ай_VERB, ,слово_NOUN, правильный_ADJ, но_CCON...</td>\n",
       "      <td>skip</td>\n",
       "      <td>negative</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>25</th>\n",
       "      <td>по кругу, твою подругу КСЮХУ....))))))ахахахаахах</td>\n",
       "      <td>[по_ADP, круг_NOUN, твой_DET, подружать_NOUN, ...</td>\n",
       "      <td>neutral</td>\n",
       "      <td>skip</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>26</th>\n",
       "      <td>без тебя_(</td>\n",
       "      <td>[без_ADP, тебя__NOUN]</td>\n",
       "      <td>neutral</td>\n",
       "      <td>negative</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>27</th>\n",
       "      <td>я прыгнул в бездну из уюта пора готовиться к з...</td>\n",
       "      <td>[я_PRON, прыгать_VERB, в_ADP, бездна_NOUN, из_...</td>\n",
       "      <td>neutral</td>\n",
       "      <td>negative</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>29</th>\n",
       "      <td>амммм, Каринка)))</td>\n",
       "      <td>[амммма_NOUN]</td>\n",
       "      <td>neutral</td>\n",
       "      <td>skip</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>30</th>\n",
       "      <td>З 19 вересня тебе!</td>\n",
       "      <td>[з_ADV, 19_NUM, вересня_NOUN, ты_PRON]</td>\n",
       "      <td>neutral</td>\n",
       "      <td>skip</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>31</th>\n",
       "      <td>Мы растем!</td>\n",
       "      <td>[мы_PRON, расти_VERB]</td>\n",
       "      <td>speech</td>\n",
       "      <td>positive</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>33</th>\n",
       "      <td>на аву прямо сейчааас!</td>\n",
       "      <td>[на_ADP, ав_NOUN, прямо_ADV, сейчааас_ADV]</td>\n",
       "      <td>positive</td>\n",
       "      <td>neutral</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>37</th>\n",
       "      <td>Продам котеночка...Очень  ласковый и умный реб...</td>\n",
       "      <td>[продать_VERB, котеночка_NOUN, очень_ADV, ласк...</td>\n",
       "      <td>skip</td>\n",
       "      <td>neutral</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>38</th>\n",
       "      <td>хоть и качество плохое, но обещал!!!!) ребятки...</td>\n",
       "      <td>[хоть_ADV, и_PART, качество_NOUN, плохой_ADJ, ...</td>\n",
       "      <td>positive</td>\n",
       "      <td>neutral</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>40</th>\n",
       "      <td>нахуй Лешего на аву поставила? О_о</td>\n",
       "      <td>[наховать_ADV, лешего_ADJ, на_ADP, ав_NOUN, ст...</td>\n",
       "      <td>neutral</td>\n",
       "      <td>negative</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>43</th>\n",
       "      <td>Я доверяю немногим. Но эти немногие, мне реаль...</td>\n",
       "      <td>[я_PRON, доверять_VERB, немногие_NOUN, но_CCON...</td>\n",
       "      <td>neutral</td>\n",
       "      <td>positive</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>46</th>\n",
       "      <td>Димон, в Дубаях еть крутой шейх, который на те...</td>\n",
       "      <td>[димон_PART, в_ADP, дубаи_PROPN, еть_VERB, кру...</td>\n",
       "      <td>neutral</td>\n",
       "      <td>positive</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>47</th>\n",
       "      <td>ПИДОР го скайп</td>\n",
       "      <td>[пидор_NOUN, й_ADJ, скайп_NOUN]</td>\n",
       "      <td>negative</td>\n",
       "      <td>skip</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>48</th>\n",
       "      <td>чувак тупо лол___)) ахаххахаах</td>\n",
       "      <td>[чувак_NOUN, тупый_NOUN, лол____NOUN, ахаххаха...</td>\n",
       "      <td>neutral</td>\n",
       "      <td>positive</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>51</th>\n",
       "      <td>Вот и моя первая выставка!</td>\n",
       "      <td>[вот_PART, и_PART, мой_DET, первый_ADJ, выстав...</td>\n",
       "      <td>neutral</td>\n",
       "      <td>positive</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>54</th>\n",
       "      <td>Кто-то заходит на мою страничку, с умилением с...</td>\n",
       "      <td>[кто-то_PRON, заходить_VERB, на_ADP, мой_DET, ...</td>\n",
       "      <td>negative</td>\n",
       "      <td>positive</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>58</th>\n",
       "      <td>З Днем Народження!!!😊</td>\n",
       "      <td>[за_NOUN, дний_NOUN, 😊_SYM]</td>\n",
       "      <td>neutral</td>\n",
       "      <td>skip</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>61</th>\n",
       "      <td>Даже моя бабуля так не матерится, как он)))))</td>\n",
       "      <td>[даже_PART, мой_DET, бабуль_NOUN, так_ADV, не_...</td>\n",
       "      <td>skip</td>\n",
       "      <td>neutral</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>63</th>\n",
       "      <td>После 6-ти часовой задержки в Бангкоке и второ...</td>\n",
       "      <td>[после_ADP, 6_NUM, часовой_ADJ, задержка_NOUN,...</td>\n",
       "      <td>neutral</td>\n",
       "      <td>positive</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>67</th>\n",
       "      <td>Друзья - это редкие драгоценности, они принося...</td>\n",
       "      <td>[друг_NOUN, это_PRON, редкий_ADJ, драгоценност...</td>\n",
       "      <td>neutral</td>\n",
       "      <td>positive</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>68</th>\n",
       "      <td>Заболiв!(((</td>\n",
       "      <td>[заболiть_NOUN]</td>\n",
       "      <td>neutral</td>\n",
       "      <td>skip</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2886</th>\n",
       "      <td>и тебя с праздником :) \\nВсех душевно больных)...</td>\n",
       "      <td>[и_CCONJ, ты_PRON, с_ADP, праздник_NOUN, весь_...</td>\n",
       "      <td>neutral</td>\n",
       "      <td>speech</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2887</th>\n",
       "      <td>В Общество похуистов приходит журналист брать ...</td>\n",
       "      <td>[в_ADP, общество_PROPN, похуист_NOUN, приходит...</td>\n",
       "      <td>negative</td>\n",
       "      <td>skip</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2888</th>\n",
       "      <td>Нужен мне как воздух😔\\nВ понедельник,или вторн...</td>\n",
       "      <td>[нужный_ADJ, я_PRON, как_SCONJ, воздух😔_NOUN, ...</td>\n",
       "      <td>neutral</td>\n",
       "      <td>skip</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2891</th>\n",
       "      <td>А я люблю его… он мне нравится… и хрен кто дог...</td>\n",
       "      <td>[а_CCONJ, я_PRON, любить_VERB, он_PRON, он_PRO...</td>\n",
       "      <td>neutral</td>\n",
       "      <td>positive</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2892</th>\n",
       "      <td>Подарите, а?</td>\n",
       "      <td>[дарить_VERB, а_CCONJ]</td>\n",
       "      <td>positive</td>\n",
       "      <td>neutral</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2893</th>\n",
       "      <td>любимому две ложечки  сахара!!!</td>\n",
       "      <td>[любимый_NOUN, два_NUM, ложечка_NOUN, сахара_N...</td>\n",
       "      <td>skip</td>\n",
       "      <td>positive</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2899</th>\n",
       "      <td>#ахахах ...</td>\n",
       "      <td>[#ахаха_NOUN]</td>\n",
       "      <td>neutral</td>\n",
       "      <td>positive</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2903</th>\n",
       "      <td>приспичило белке купить под конец осени кепку,...</td>\n",
       "      <td>[приспичить_VERB, белок_NOUN, купить_VERB, под...</td>\n",
       "      <td>neutral</td>\n",
       "      <td>skip</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2905</th>\n",
       "      <td>Вятка! Спасибо за великолепные выходные! Это б...</td>\n",
       "      <td>[вяток_NOUN, спасибо_NOUN, за_ADP, великолепны...</td>\n",
       "      <td>positive</td>\n",
       "      <td>speech</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2911</th>\n",
       "      <td>Вот оно - искусство!!! ))</td>\n",
       "      <td>[вот_PART, оно_PRON, искусство_NOUN]</td>\n",
       "      <td>neutral</td>\n",
       "      <td>positive</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2917</th>\n",
       "      <td>помню как было тяжело , но очень хочу это все ...</td>\n",
       "      <td>[помнить_VERB, как_SCONJ, быть_AUX, тяжелый_AD...</td>\n",
       "      <td>positive</td>\n",
       "      <td>negative</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2920</th>\n",
       "      <td>Когда-нибудь и мы поженимся.</td>\n",
       "      <td>[когда-нибудь_PRON, и_CCONJ, мы_PRON, жениться...</td>\n",
       "      <td>neutral</td>\n",
       "      <td>skip</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2921</th>\n",
       "      <td>Я прощаю одиночество и грусть, ты сказал, что ...</td>\n",
       "      <td>[я_PRON, прощать_VERB, одиночество_NOUN, и_CCO...</td>\n",
       "      <td>positive</td>\n",
       "      <td>neutral</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2925</th>\n",
       "      <td>Иногда увольняться полезно! ;)))</td>\n",
       "      <td>[иногда_ADV, увольняться_VERB, полезный_ADJ]</td>\n",
       "      <td>neutral</td>\n",
       "      <td>positive</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2931</th>\n",
       "      <td>НУФВСё!!!\\n\\nГАМОВЕР УЧЕБЕ \\nя теперь маджистер</td>\n",
       "      <td>[нуфвсе_PRON, гамовереть_NOUN, учы_PRON, я_PRO...</td>\n",
       "      <td>neutral</td>\n",
       "      <td>positive</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2934</th>\n",
       "      <td>ты чоооо-о не поздравил меня с этим великим пр...</td>\n",
       "      <td>[ты_PRON, чоооо-о_ADV, не_PART, поздравить_VER...</td>\n",
       "      <td>speech</td>\n",
       "      <td>neutral</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2936</th>\n",
       "      <td>С Днём Рождения!!!:*</td>\n",
       "      <td>[с_ADP, одн_DET]</td>\n",
       "      <td>neutral</td>\n",
       "      <td>speech</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2937</th>\n",
       "      <td>многие пиздят в инете за этот фильм) от создат...</td>\n",
       "      <td>[много_NUM, пиздить_VERB, в_ADP, инет_NOUN, за...</td>\n",
       "      <td>neutral</td>\n",
       "      <td>skip</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2940</th>\n",
       "      <td>Танюшечка❤</td>\n",
       "      <td>[танюшечка❤_NOUN]</td>\n",
       "      <td>neutral</td>\n",
       "      <td>positive</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2942</th>\n",
       "      <td>кшод4 аркгаркгаи</td>\n",
       "      <td>[кшод4_NUM, аркгаркгай_NOUN]</td>\n",
       "      <td>neutral</td>\n",
       "      <td>skip</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2945</th>\n",
       "      <td>с регой!)кк учёба?</td>\n",
       "      <td>[с_ADP, рега_NOUN, )кк_NOUN, учёба_NOUN]</td>\n",
       "      <td>neutral</td>\n",
       "      <td>speech</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2949</th>\n",
       "      <td>Тащуся 8-)</td>\n",
       "      <td>[тащусь_VERB, 8_NUM]</td>\n",
       "      <td>neutral</td>\n",
       "      <td>positive</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2951</th>\n",
       "      <td>ПАЗДРАВЛЯЮ!!!</td>\n",
       "      <td>[паздравлять_VERB]</td>\n",
       "      <td>neutral</td>\n",
       "      <td>speech</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2954</th>\n",
       "      <td>ВАЩЕЕЕ КРАСАВАА!!!</td>\n",
       "      <td>[ващео_ADV]</td>\n",
       "      <td>neutral</td>\n",
       "      <td>positive</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2955</th>\n",
       "      <td>В эту песню я просто влюблен еще с 90-х. Мы го...</td>\n",
       "      <td>[в_ADP, этот_DET, песня_NOUN, я_PRON, просто_P...</td>\n",
       "      <td>negative</td>\n",
       "      <td>positive</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2957</th>\n",
       "      <td>אימון קרב מגע מתקדם ביחידה מובחרת - Krav Maga</td>\n",
       "      <td>[אימון_NOUN, קרב_NOUN, מגע_NOUN, מתקדם_NOUN, ב...</td>\n",
       "      <td>neutral</td>\n",
       "      <td>skip</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2959</th>\n",
       "      <td>Бля я и забыл, какой он ахуенный!</td>\n",
       "      <td>[бль_VERB, я_PRON, и_CCONJ, забыть_VERB, какой...</td>\n",
       "      <td>negative</td>\n",
       "      <td>positive</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2961</th>\n",
       "      <td>Ярких эмоций, незабываемых встреч и полета душ...</td>\n",
       "      <td>[яркий_ADJ, эмоция_NOUN, незабываемый_ADJ, вст...</td>\n",
       "      <td>positive</td>\n",
       "      <td>speech</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2964</th>\n",
       "      <td>Ты одна половинка - я вторая половинка...... А...</td>\n",
       "      <td>[ты_PRON, один_NUM, половинка_NOUN, я_PRON, вт...</td>\n",
       "      <td>neutral</td>\n",
       "      <td>skip</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2966</th>\n",
       "      <td>Ахахахахах.Прикольно попереписувались.\\nаааааа...</td>\n",
       "      <td>[ахахахаха_NOUN, прикольно_ADV, попереписувать...</td>\n",
       "      <td>positive</td>\n",
       "      <td>skip</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>1093 rows × 4 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "                                                   text  \\\n",
       "1     Скоро ты вернешься домой, грязный, не бритый н...   \n",
       "5     Наши красавцы. 1:3 в первом периоде и 7:3 в ит...   \n",
       "8     нельзя общаться с людьми, которые нуждаются в ...   \n",
       "10    Друзья, скиньте клёвой музычки, в тачке не чег...   \n",
       "11                                         Селфи дня ))   \n",
       "14                                   Хочу такую зиму. ❄   \n",
       "17    Я не писал такое оскобление всем!!!!\\nМеня, ви...   \n",
       "18    ...и так начало мини отпуска))И новых приключе...   \n",
       "22    Аю,слова правильные,но такие унылые(((Сьешь ко...   \n",
       "25    по кругу, твою подругу КСЮХУ....))))))ахахахаахах   \n",
       "26                                           без тебя_(   \n",
       "27    я прыгнул в бездну из уюта пора готовиться к з...   \n",
       "29                                    амммм, Каринка)))   \n",
       "30                                   З 19 вересня тебе!   \n",
       "31                                           Мы растем!   \n",
       "33                               на аву прямо сейчааас!   \n",
       "37    Продам котеночка...Очень  ласковый и умный реб...   \n",
       "38    хоть и качество плохое, но обещал!!!!) ребятки...   \n",
       "40                   нахуй Лешего на аву поставила? О_о   \n",
       "43    Я доверяю немногим. Но эти немногие, мне реаль...   \n",
       "46    Димон, в Дубаях еть крутой шейх, который на те...   \n",
       "47                                       ПИДОР го скайп   \n",
       "48                       чувак тупо лол___)) ахаххахаах   \n",
       "51                           Вот и моя первая выставка!   \n",
       "54    Кто-то заходит на мою страничку, с умилением с...   \n",
       "58                                З Днем Народження!!!😊   \n",
       "61        Даже моя бабуля так не матерится, как он)))))   \n",
       "63    После 6-ти часовой задержки в Бангкоке и второ...   \n",
       "67    Друзья - это редкие драгоценности, они принося...   \n",
       "68                                          Заболiв!(((   \n",
       "...                                                 ...   \n",
       "2886  и тебя с праздником :) \\nВсех душевно больных)...   \n",
       "2887  В Общество похуистов приходит журналист брать ...   \n",
       "2888  Нужен мне как воздух😔\\nВ понедельник,или вторн...   \n",
       "2891  А я люблю его… он мне нравится… и хрен кто дог...   \n",
       "2892                                       Подарите, а?   \n",
       "2893                    любимому две ложечки  сахара!!!   \n",
       "2899                                        #ахахах ...   \n",
       "2903  приспичило белке купить под конец осени кепку,...   \n",
       "2905  Вятка! Спасибо за великолепные выходные! Это б...   \n",
       "2911                          Вот оно - искусство!!! ))   \n",
       "2917  помню как было тяжело , но очень хочу это все ...   \n",
       "2920                       Когда-нибудь и мы поженимся.   \n",
       "2921  Я прощаю одиночество и грусть, ты сказал, что ...   \n",
       "2925                   Иногда увольняться полезно! ;)))   \n",
       "2931    НУФВСё!!!\\n\\nГАМОВЕР УЧЕБЕ \\nя теперь маджистер   \n",
       "2934  ты чоооо-о не поздравил меня с этим великим пр...   \n",
       "2936                               С Днём Рождения!!!:*   \n",
       "2937  многие пиздят в инете за этот фильм) от создат...   \n",
       "2940                                         Танюшечка❤   \n",
       "2942                                   кшод4 аркгаркгаи   \n",
       "2945                                 с регой!)кк учёба?   \n",
       "2949                                         Тащуся 8-)   \n",
       "2951                                      ПАЗДРАВЛЯЮ!!!   \n",
       "2954                                 ВАЩЕЕЕ КРАСАВАА!!!   \n",
       "2955  В эту песню я просто влюблен еще с 90-х. Мы го...   \n",
       "2957      אימון קרב מגע מתקדם ביחידה מובחרת - Krav Maga   \n",
       "2959                  Бля я и забыл, какой он ахуенный!   \n",
       "2961  Ярких эмоций, незабываемых встреч и полета душ...   \n",
       "2964  Ты одна половинка - я вторая половинка...... А...   \n",
       "2966  Ахахахахах.Прикольно попереписувались.\\nаааааа...   \n",
       "\n",
       "                                      lemmatized_tokens predicted    actual  \n",
       "1     [скоро_ADV, ты_PRON, вернуться_VERB, домой_ADV...  negative  positive  \n",
       "5     [наш_DET, красавец_NOUN, 1_NUM, 3_NUM, в_ADP, ...   neutral  positive  \n",
       "8     [нельзя_ADV, общаться_VERB, с_ADP, человек_NOU...   neutral  negative  \n",
       "10    [друг_NOUN, скинить_NOUN, клёвый_ADJ, музычка_...  positive   neutral  \n",
       "11                              [селфь_NOUN, день_NOUN]   neutral  positive  \n",
       "14           [хотеть_VERB, такой_DET, зима_NOUN, ❄_SYM]   neutral  positive  \n",
       "17    [я_PRON, не_PART, писать_VERB, такой_DET, оско...   neutral      skip  \n",
       "18    [и_CCONJ, так_ADV, начать_NOUN, минить_NOUN, о...   neutral  positive  \n",
       "22    [ай_VERB, ,слово_NOUN, правильный_ADJ, но_CCON...      skip  negative  \n",
       "25    [по_ADP, круг_NOUN, твой_DET, подружать_NOUN, ...   neutral      skip  \n",
       "26                                [без_ADP, тебя__NOUN]   neutral  negative  \n",
       "27    [я_PRON, прыгать_VERB, в_ADP, бездна_NOUN, из_...   neutral  negative  \n",
       "29                                        [амммма_NOUN]   neutral      skip  \n",
       "30               [з_ADV, 19_NUM, вересня_NOUN, ты_PRON]   neutral      skip  \n",
       "31                                [мы_PRON, расти_VERB]    speech  positive  \n",
       "33           [на_ADP, ав_NOUN, прямо_ADV, сейчааас_ADV]  positive   neutral  \n",
       "37    [продать_VERB, котеночка_NOUN, очень_ADV, ласк...      skip   neutral  \n",
       "38    [хоть_ADV, и_PART, качество_NOUN, плохой_ADJ, ...  positive   neutral  \n",
       "40    [наховать_ADV, лешего_ADJ, на_ADP, ав_NOUN, ст...   neutral  negative  \n",
       "43    [я_PRON, доверять_VERB, немногие_NOUN, но_CCON...   neutral  positive  \n",
       "46    [димон_PART, в_ADP, дубаи_PROPN, еть_VERB, кру...   neutral  positive  \n",
       "47                      [пидор_NOUN, й_ADJ, скайп_NOUN]  negative      skip  \n",
       "48    [чувак_NOUN, тупый_NOUN, лол____NOUN, ахаххаха...   neutral  positive  \n",
       "51    [вот_PART, и_PART, мой_DET, первый_ADJ, выстав...   neutral  positive  \n",
       "54    [кто-то_PRON, заходить_VERB, на_ADP, мой_DET, ...  negative  positive  \n",
       "58                          [за_NOUN, дний_NOUN, 😊_SYM]   neutral      skip  \n",
       "61    [даже_PART, мой_DET, бабуль_NOUN, так_ADV, не_...      skip   neutral  \n",
       "63    [после_ADP, 6_NUM, часовой_ADJ, задержка_NOUN,...   neutral  positive  \n",
       "67    [друг_NOUN, это_PRON, редкий_ADJ, драгоценност...   neutral  positive  \n",
       "68                                      [заболiть_NOUN]   neutral      skip  \n",
       "...                                                 ...       ...       ...  \n",
       "2886  [и_CCONJ, ты_PRON, с_ADP, праздник_NOUN, весь_...   neutral    speech  \n",
       "2887  [в_ADP, общество_PROPN, похуист_NOUN, приходит...  negative      skip  \n",
       "2888  [нужный_ADJ, я_PRON, как_SCONJ, воздух😔_NOUN, ...   neutral      skip  \n",
       "2891  [а_CCONJ, я_PRON, любить_VERB, он_PRON, он_PRO...   neutral  positive  \n",
       "2892                             [дарить_VERB, а_CCONJ]  positive   neutral  \n",
       "2893  [любимый_NOUN, два_NUM, ложечка_NOUN, сахара_N...      skip  positive  \n",
       "2899                                      [#ахаха_NOUN]   neutral  positive  \n",
       "2903  [приспичить_VERB, белок_NOUN, купить_VERB, под...   neutral      skip  \n",
       "2905  [вяток_NOUN, спасибо_NOUN, за_ADP, великолепны...  positive    speech  \n",
       "2911               [вот_PART, оно_PRON, искусство_NOUN]   neutral  positive  \n",
       "2917  [помнить_VERB, как_SCONJ, быть_AUX, тяжелый_AD...  positive  negative  \n",
       "2920  [когда-нибудь_PRON, и_CCONJ, мы_PRON, жениться...   neutral      skip  \n",
       "2921  [я_PRON, прощать_VERB, одиночество_NOUN, и_CCO...  positive   neutral  \n",
       "2925       [иногда_ADV, увольняться_VERB, полезный_ADJ]   neutral  positive  \n",
       "2931  [нуфвсе_PRON, гамовереть_NOUN, учы_PRON, я_PRO...   neutral  positive  \n",
       "2934  [ты_PRON, чоооо-о_ADV, не_PART, поздравить_VER...    speech   neutral  \n",
       "2936                                   [с_ADP, одн_DET]   neutral    speech  \n",
       "2937  [много_NUM, пиздить_VERB, в_ADP, инет_NOUN, за...   neutral      skip  \n",
       "2940                                  [танюшечка❤_NOUN]   neutral  positive  \n",
       "2942                       [кшод4_NUM, аркгаркгай_NOUN]   neutral      skip  \n",
       "2945           [с_ADP, рега_NOUN, )кк_NOUN, учёба_NOUN]   neutral    speech  \n",
       "2949                               [тащусь_VERB, 8_NUM]   neutral  positive  \n",
       "2951                                 [паздравлять_VERB]   neutral    speech  \n",
       "2954                                        [ващео_ADV]   neutral  positive  \n",
       "2955  [в_ADP, этот_DET, песня_NOUN, я_PRON, просто_P...  negative  positive  \n",
       "2957  [אימון_NOUN, קרב_NOUN, מגע_NOUN, מתקדם_NOUN, ב...   neutral      skip  \n",
       "2959  [бль_VERB, я_PRON, и_CCONJ, забыть_VERB, какой...  negative  positive  \n",
       "2961  [яркий_ADJ, эмоция_NOUN, незабываемый_ADJ, вст...  positive    speech  \n",
       "2964  [ты_PRON, один_NUM, половинка_NOUN, я_PRON, вт...   neutral      skip  \n",
       "2966  [ахахахаха_NOUN, прикольно_ADV, попереписувать...  positive      skip  \n",
       "\n",
       "[1093 rows x 4 columns]"
      ]
     },
     "execution_count": 67,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "errors"
   ]
  },
  {
   "cell_type": "raw",
   "metadata": {},
   "source": [
    "Without POS\n",
    "\n",
    "best_svc_pipeline = Pipeline([('tfidf',TfidfVectorizer(ngram_range=(1,1), sublinear_tf=False)), ('svc', LinearSVC(C=0.6235, penalty='l1', dual=False))])\n",
    "Precision: 0.6748\n",
    "Recall: 0.6896\n",
    "f1-score: 0.6702\n",
    "\n",
    "Witg POS\n",
    "best_svc_pipeline = Pipeline([('tfidf',TfidfVectorizer(ngram_range=(1,1), sublinear_tf=False)), ('svc', LinearSVC(C=0.5, penalty='l1', dual=False))])\n",
    "Precision: 0.6678\n",
    "Recall: 0.6781\n",
    "f1-score: 0.6606"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.0"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}