final_version_for_tira.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[nltk_data] Downloading package stopwords to /home/katya/nltk_data...\n",
      "[nltk_data]   Package stopwords is already up-to-date!\n",
      "[nltk_data] Downloading package punkt to /home/katya/nltk_data...\n",
      "[nltk_data]   Package punkt is already up-to-date!\n"
     ]
    }
   ],
   "source": [
    "import xml.etree.ElementTree as ET\n",
    "import requests\n",
    "from string import punctuation \n",
    "import json\n",
    "import pickle\n",
    "from nltk.tokenize import word_tokenize\n",
    "from nltk.stem import WordNetLemmatizer \n",
    "from nltk.corpus import stopwords\n",
    "import nltk\n",
    "from nltk import pos_tag\n",
    "from nltk.corpus import wordnet\n",
    "import re\n",
    "import pandas as pd\n",
    "#nltk.download('stopwords')\n",
    "#nltk.download('punkt')\n",
    "#nltk.download('averaged_perceptron_tagger')\n",
    "import sys\n",
    "import os\n",
    "from zipfile import ZipFile\n",
    "import pytrec_eval\n",
    "from boilerpy3 import extractors\n",
    "from urllib.error import HTTPError\n",
    "from elasticsearch import Elasticsearch\n",
    "from elasticsearch import NotFoundError\n",
    "import time\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "input_dir = 'data_final'\n",
    "output_dir = 'results_final'\n",
    "\n",
    "#input_dir = sys.argv[1]\n",
    "#output_dir = sys.argv[2]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "#parse the XML-file with queries\n",
    "\n",
    "mytree = ET.parse(input_dir+'/topics-task-2-only-titles.xml')\n",
    "myroot = mytree.getroot()\n",
    "\n",
    "#preprocess the queries\n",
    "q = []\n",
    "topics = []\n",
    "for item in myroot:\n",
    "    d = {}\n",
    "    for x in item:\n",
    "        d[x.tag] = x.text.strip('\\n')\n",
    "        #print(d)\n",
    "        \n",
    "        if x.tag == \"title\":\n",
    "            #append the query to an array\n",
    "            q.append(x.text)\n",
    "    topics.append(d)\n",
    "    \n",
    "#save topics as json\n",
    "#with open(input_dir+'/topics_final.json', 'w') as file:\n",
    "#    json.dump(topics, file)\n",
    "    \n",
    "results = []\n",
    "chatnoir = \"https://www.chatnoir.eu/api/v1/_search\"\n",
    "attr = {\"apikey\": \"7dd15626-53aa-46c6-bd34-b2feaa2d9d81\",\n",
    "        \"query\": \"hello world\",\n",
    "        \"index\": \"cw12\",\n",
    "        \"pretty\": True\n",
    "}\n",
    "\n",
    "for x in q:\n",
    "    attr[\"query\"] = x\n",
    "    #somehow index doesn't work correctly when passed as an array (it only searches the 1st index of the array), so search\n",
    "    #in each index separately and sum up the results\n",
    "    response = requests.post(chatnoir, data = attr)\n",
    "    res = response.json()[\"meta\"][\"total_results\"]\n",
    "    \n",
    "    results.append(res)\n",
    "    \n",
    "#save results as txt\n",
    "\n",
    "#with open(input_dir+\"/results_final.txt\", \"wb\") as fp:   #Pickling\n",
    "#    pickle.dump(results, fp)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "#RETRIEVAL\n",
    "#open the file with topics\n",
    "#f = open(input_dir+\"/topics_final.json\", encoding='utf8')\n",
    "#topics = json.load(f)\n",
    "\n",
    "#open the file with amount of results for each topic\n",
    "#with open(input_dir+\"/results_final.txt\", \"rb\") as fp:   # Unpickling\n",
    "#    results = pickle.load(fp)\n",
    " \n",
    "#preprocess the queries\n",
    "for i in range(len(topics)):\n",
    "    topics[i]['title'] = topics[i]['title'].replace(' ', ' AND ')\n",
    "    topics[i]['title'] = re.sub('[?:,]', '', topics[i]['title'])\n",
    "    topics[i]['results'] = results[i]\n",
    "    \n",
    "attr = {\"apikey\": \"7dd15626-53aa-46c6-bd34-b2feaa2d9d81\",\n",
    "        \"query\": \"\",\n",
    "        \"index\": \"cw12\",\n",
    "        \"size\": 10\n",
    "       }\n",
    "\n",
    "#save first 110 docs for each topic\n",
    "extractor = extractors.ArticleExtractor()\n",
    "def topics_iter(q):\n",
    "    docs = []\n",
    "    attr[\"query\"] = q['title']\n",
    "    #attr['size'] = q['results']\n",
    "    if q['results']<=110:\n",
    "        num_of_res = q['results']\n",
    "    else:\n",
    "        num_of_res = 110\n",
    "    count = 0\n",
    "    print(attr)\n",
    "    print(num_of_res)\n",
    "    url = \"https://www.chatnoir.eu/api/v1/_search?\"\n",
    "    while count < num_of_res:\n",
    "        attr[\"from\"] = count\n",
    "        while True:\n",
    "            try:\n",
    "                r = requests.post(url, json = attr)\n",
    "                res = r.json()\n",
    "                print(count)\n",
    "                #print(res)\n",
    "                res_len = len(res['results'])\n",
    "                print(res_len)\n",
    "            except KeyError:\n",
    "                continue\n",
    "            break\n",
    "        \n",
    "    \n",
    "        for i in range(res_len):\n",
    "            \n",
    "            doc_url = \"https://www.chatnoir.eu/cache?uuid=\"+res['results'][i]['uuid']+\"&index=cw12&raw\"\n",
    "            #print(doc_url)\n",
    "            try:\n",
    "                doc = extractor.get_doc_from_url(doc_url)\n",
    "                content = doc.content\n",
    "                title = doc.title\n",
    "                res['results'][i]['document'] = content\n",
    "            except HTTPError:\n",
    "                print(\"HTTPError\")\n",
    "                continue\n",
    "        docs.append(res['results'])\n",
    "        count+=10\n",
    "        \n",
    "    return docs"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'apikey': '7dd15626-53aa-46c6-bd34-b2feaa2d9d81', 'query': 'What AND is AND better AND at AND reducing AND fever AND in AND children AND Ibuprofen AND or AND Aspirin', 'index': 'cw12', 'size': 10}\n",
      "110\n",
      "0\n",
      "10\n",
      "10\n",
      "10\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Warning: SAX input contains nested A elements -- You have probably hit a bug in your HTML parser (e.g., NekoHTML bug #2909310). Please clean the HTML externally and feed it to BoilerPy3 again. Trying to recover somehow...\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "20\n",
      "10\n",
      "30\n",
      "10\n",
      "40\n",
      "10\n",
      "50\n",
      "10\n",
      "60\n",
      "10\n",
      "70\n",
      "10\n",
      "80\n",
      "10\n",
      "90\n",
      "10\n",
      "100\n",
      "10\n",
      "{'apikey': '7dd15626-53aa-46c6-bd34-b2feaa2d9d81', 'query': 'What AND are AND the AND best AND rice AND cookers', 'index': 'cw12', 'size': 10, 'from': 100}\n",
      "110\n",
      "0\n",
      "10\n",
      "10\n",
      "10\n",
      "20\n",
      "10\n",
      "30\n",
      "10\n",
      "40\n",
      "10\n",
      "50\n",
      "10\n",
      "60\n",
      "10\n",
      "70\n",
      "10\n",
      "80\n",
      "10\n",
      "90\n",
      "10\n",
      "100\n",
      "10\n",
      "{'apikey': '7dd15626-53aa-46c6-bd34-b2feaa2d9d81', 'query': 'Should AND I AND buy AND steel AND or AND ceramic AND knives', 'index': 'cw12', 'size': 10, 'from': 100}\n",
      "110\n",
      "0\n",
      "10\n",
      "10\n",
      "10\n",
      "20\n",
      "10\n",
      "30\n",
      "10\n",
      "40\n",
      "10\n",
      "HTTPError\n",
      "50\n",
      "10\n",
      "60\n",
      "10\n",
      "70\n",
      "10\n",
      "80\n",
      "10\n",
      "90\n",
      "10\n",
      "100\n",
      "10\n",
      "{'apikey': '7dd15626-53aa-46c6-bd34-b2feaa2d9d81', 'query': 'Is AND morning AND or AND afternoon AND sun AND the AND best AND for AND fruit AND trees', 'index': 'cw12', 'size': 10, 'from': 100}\n",
      "110\n",
      "0\n",
      "10\n",
      "10\n",
      "10\n",
      "20\n",
      "10\n",
      "30\n",
      "10\n",
      "40\n",
      "10\n",
      "50\n",
      "10\n",
      "60\n",
      "10\n",
      "70\n",
      "10\n",
      "80\n",
      "10\n",
      "90\n",
      "10\n",
      "100\n",
      "10\n",
      "{'apikey': '7dd15626-53aa-46c6-bd34-b2feaa2d9d81', 'query': 'What AND is AND better AND for AND back AND pain AND chiropractic AND therapy AND or AND physical AND therapy', 'index': 'cw12', 'size': 10, 'from': 100}\n",
      "110\n",
      "0\n",
      "10\n",
      "10\n",
      "10\n",
      "20\n",
      "10\n",
      "30\n",
      "10\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Warning: SAX input contains nested A elements -- You have probably hit a bug in your HTML parser (e.g., NekoHTML bug #2909310). Please clean the HTML externally and feed it to BoilerPy3 again. Trying to recover somehow...\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "40\n",
      "10\n",
      "50\n",
      "10\n",
      "60\n",
      "10\n",
      "70\n",
      "10\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Warning: SAX input contains nested A elements -- You have probably hit a bug in your HTML parser (e.g., NekoHTML bug #2909310). Please clean the HTML externally and feed it to BoilerPy3 again. Trying to recover somehow...\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "80\n",
      "10\n",
      "90\n",
      "10\n",
      "100\n",
      "10\n",
      "{'apikey': '7dd15626-53aa-46c6-bd34-b2feaa2d9d81', 'query': 'Is AND Kenya AND or AND Tanzania AND better AND for AND a AND safari', 'index': 'cw12', 'size': 10, 'from': 100}\n",
      "110\n",
      "0\n",
      "10\n",
      "10\n",
      "10\n",
      "20\n",
      "10\n",
      "30\n",
      "10\n",
      "40\n",
      "10\n",
      "50\n",
      "10\n",
      "60\n",
      "10\n",
      "70\n",
      "10\n",
      "80\n",
      "10\n",
      "90\n",
      "10\n",
      "100\n",
      "10\n",
      "{'apikey': '7dd15626-53aa-46c6-bd34-b2feaa2d9d81', 'query': \"How AND is AND a AND Master's AND degree AND different AND from AND a AND Bachelor's AND degree\", 'index': 'cw12', 'size': 10, 'from': 100}\n",
      "110\n",
      "0\n",
      "10\n",
      "10\n",
      "10\n",
      "20\n",
      "10\n",
      "30\n",
      "10\n",
      "40\n",
      "10\n",
      "50\n",
      "10\n",
      "60\n",
      "10\n",
      "70\n",
      "10\n",
      "80\n",
      "10\n",
      "90\n",
      "10\n",
      "100\n",
      "10\n",
      "{'apikey': '7dd15626-53aa-46c6-bd34-b2feaa2d9d81', 'query': 'Which AND is AND better AND Family AND Guy AND or AND The AND Simpsons', 'index': 'cw12', 'size': 10, 'from': 100}\n",
      "110\n",
      "0\n",
      "10\n",
      "10\n",
      "10\n",
      "20\n",
      "10\n",
      "30\n",
      "10\n",
      "40\n",
      "10\n",
      "50\n",
      "10\n",
      "60\n",
      "10\n",
      "70\n",
      "10\n",
      "80\n",
      "10\n",
      "90\n",
      "10\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Warning: SAX input contains nested A elements -- You have probably hit a bug in your HTML parser (e.g., NekoHTML bug #2909310). Please clean the HTML externally and feed it to BoilerPy3 again. Trying to recover somehow...\n",
      "Warning: SAX input contains nested A elements -- You have probably hit a bug in your HTML parser (e.g., NekoHTML bug #2909310). Please clean the HTML externally and feed it to BoilerPy3 again. Trying to recover somehow...\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "100\n",
      "10\n",
      "{'apikey': '7dd15626-53aa-46c6-bd34-b2feaa2d9d81', 'query': 'Which AND is AND more AND difficult AND skiing AND or AND snowboarding', 'index': 'cw12', 'size': 10, 'from': 100}\n",
      "110\n",
      "0\n",
      "10\n",
      "10\n",
      "10\n",
      "20\n",
      "10\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Warning: SAX input contains nested A elements -- You have probably hit a bug in your HTML parser (e.g., NekoHTML bug #2909310). Please clean the HTML externally and feed it to BoilerPy3 again. Trying to recover somehow...\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "30\n",
      "10\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Warning: SAX input contains nested A elements -- You have probably hit a bug in your HTML parser (e.g., NekoHTML bug #2909310). Please clean the HTML externally and feed it to BoilerPy3 again. Trying to recover somehow...\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "40\n",
      "10\n",
      "50\n",
      "10\n",
      "60\n",
      "10\n"
     ]
    },
    {
     "ename": "KeyboardInterrupt",
     "evalue": "",
     "output_type": "error",
     "traceback": [
      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[0;31mKeyboardInterrupt\u001b[0m                         Traceback (most recent call last)",
      "\u001b[0;32m<ipython-input-5-f01f160d9d82>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m\u001b[0m\n\u001b[1;32m      1\u001b[0m \u001b[0;32mfor\u001b[0m \u001b[0mq\u001b[0m \u001b[0;32min\u001b[0m \u001b[0mtopics\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m----> 2\u001b[0;31m     \u001b[0mq\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;34m'documents'\u001b[0m\u001b[0;34m]\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mtopics_iter\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mq\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m      3\u001b[0m     \u001b[0;32mwith\u001b[0m \u001b[0mopen\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0minput_dir\u001b[0m\u001b[0;34m+\u001b[0m\u001b[0;34m\"/docs_final/docs_for_topic_{}.txt\"\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mformat\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mq\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;34m'number'\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m\"w\"\u001b[0m\u001b[0;34m)\u001b[0m \u001b[0;32mas\u001b[0m \u001b[0mf\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m      4\u001b[0m         \u001b[0mjson\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mdump\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mq\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mf\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m      5\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m<ipython-input-4-13c6817e99c3>\u001b[0m in \u001b[0;36mtopics_iter\u001b[0;34m(q)\u001b[0m\n\u001b[1;32m     54\u001b[0m             \u001b[0;31m#print(doc_url)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     55\u001b[0m             \u001b[0;32mtry\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 56\u001b[0;31m                 \u001b[0mdoc\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mextractor\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mget_doc_from_url\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mdoc_url\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m     57\u001b[0m                 \u001b[0mcontent\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mdoc\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mcontent\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     58\u001b[0m                 \u001b[0mtitle\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mdoc\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mtitle\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/anaconda3/lib/python3.7/site-packages/boilerpy3/extractors.py\u001b[0m in \u001b[0;36mget_doc_from_url\u001b[0;34m(self, url)\u001b[0m\n\u001b[1;32m     43\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     44\u001b[0m     \u001b[0;32mdef\u001b[0m \u001b[0mget_doc_from_url\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0murl\u001b[0m\u001b[0;34m)\u001b[0m \u001b[0;34m->\u001b[0m \u001b[0mTextDocument\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 45\u001b[0;31m         \u001b[0;32mreturn\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mget_doc\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mread_from_url\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0murl\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m     46\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     47\u001b[0m     \u001b[0;32mdef\u001b[0m \u001b[0mget_doc\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mtext\u001b[0m\u001b[0;34m)\u001b[0m \u001b[0;34m->\u001b[0m \u001b[0mTextDocument\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/anaconda3/lib/python3.7/site-packages/boilerpy3/extractors.py\u001b[0m in \u001b[0;36mread_from_url\u001b[0;34m(self, url)\u001b[0m\n\u001b[1;32m     60\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     61\u001b[0m     \u001b[0;32mdef\u001b[0m \u001b[0mread_from_url\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0murl\u001b[0m\u001b[0;34m:\u001b[0m \u001b[0mstr\u001b[0m\u001b[0;34m)\u001b[0m \u001b[0;34m->\u001b[0m \u001b[0mstr\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 62\u001b[0;31m         \u001b[0;32mwith\u001b[0m \u001b[0murllib\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mrequest\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0murlopen\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0murl\u001b[0m\u001b[0;34m)\u001b[0m \u001b[0;32mas\u001b[0m \u001b[0murl_obj\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m     63\u001b[0m             \u001b[0mtext\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0murl_obj\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mread\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     64\u001b[0m             \u001b[0mencoding\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mget_url_encoding\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0murl_obj\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/anaconda3/lib/python3.7/urllib/request.py\u001b[0m in \u001b[0;36murlopen\u001b[0;34m(url, data, timeout, cafile, capath, cadefault, context)\u001b[0m\n\u001b[1;32m    220\u001b[0m     \u001b[0;32melse\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    221\u001b[0m         \u001b[0mopener\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0m_opener\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 222\u001b[0;31m     \u001b[0;32mreturn\u001b[0m \u001b[0mopener\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mopen\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0murl\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mdata\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mtimeout\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    223\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    224\u001b[0m \u001b[0;32mdef\u001b[0m \u001b[0minstall_opener\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mopener\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/anaconda3/lib/python3.7/urllib/request.py\u001b[0m in \u001b[0;36mopen\u001b[0;34m(self, fullurl, data, timeout)\u001b[0m\n\u001b[1;32m    523\u001b[0m             \u001b[0mreq\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mmeth\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mreq\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    524\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 525\u001b[0;31m         \u001b[0mresponse\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_open\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mreq\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mdata\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    526\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    527\u001b[0m         \u001b[0;31m# post-process response\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/anaconda3/lib/python3.7/urllib/request.py\u001b[0m in \u001b[0;36m_open\u001b[0;34m(self, req, data)\u001b[0m\n\u001b[1;32m    541\u001b[0m         \u001b[0mprotocol\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mreq\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mtype\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    542\u001b[0m         result = self._call_chain(self.handle_open, protocol, protocol +\n\u001b[0;32m--> 543\u001b[0;31m                                   '_open', req)\n\u001b[0m\u001b[1;32m    544\u001b[0m         \u001b[0;32mif\u001b[0m \u001b[0mresult\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    545\u001b[0m             \u001b[0;32mreturn\u001b[0m \u001b[0mresult\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/anaconda3/lib/python3.7/urllib/request.py\u001b[0m in \u001b[0;36m_call_chain\u001b[0;34m(self, chain, kind, meth_name, *args)\u001b[0m\n\u001b[1;32m    501\u001b[0m         \u001b[0;32mfor\u001b[0m \u001b[0mhandler\u001b[0m \u001b[0;32min\u001b[0m \u001b[0mhandlers\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    502\u001b[0m             \u001b[0mfunc\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mgetattr\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mhandler\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mmeth_name\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 503\u001b[0;31m             \u001b[0mresult\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mfunc\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m*\u001b[0m\u001b[0margs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    504\u001b[0m             \u001b[0;32mif\u001b[0m \u001b[0mresult\u001b[0m \u001b[0;32mis\u001b[0m \u001b[0;32mnot\u001b[0m \u001b[0;32mNone\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    505\u001b[0m                 \u001b[0;32mreturn\u001b[0m \u001b[0mresult\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/anaconda3/lib/python3.7/urllib/request.py\u001b[0m in \u001b[0;36mhttps_open\u001b[0;34m(self, req)\u001b[0m\n\u001b[1;32m   1360\u001b[0m         \u001b[0;32mdef\u001b[0m \u001b[0mhttps_open\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mreq\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1361\u001b[0m             return self.do_open(http.client.HTTPSConnection, req,\n\u001b[0;32m-> 1362\u001b[0;31m                 context=self._context, check_hostname=self._check_hostname)\n\u001b[0m\u001b[1;32m   1363\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1364\u001b[0m         \u001b[0mhttps_request\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mAbstractHTTPHandler\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mdo_request_\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/anaconda3/lib/python3.7/urllib/request.py\u001b[0m in \u001b[0;36mdo_open\u001b[0;34m(self, http_class, req, **http_conn_args)\u001b[0m\n\u001b[1;32m   1317\u001b[0m             \u001b[0;32mtry\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1318\u001b[0m                 h.request(req.get_method(), req.selector, req.data, headers,\n\u001b[0;32m-> 1319\u001b[0;31m                           encode_chunked=req.has_header('Transfer-encoding'))\n\u001b[0m\u001b[1;32m   1320\u001b[0m             \u001b[0;32mexcept\u001b[0m \u001b[0mOSError\u001b[0m \u001b[0;32mas\u001b[0m \u001b[0merr\u001b[0m\u001b[0;34m:\u001b[0m \u001b[0;31m# timeout error\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1321\u001b[0m                 \u001b[0;32mraise\u001b[0m \u001b[0mURLError\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0merr\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/anaconda3/lib/python3.7/http/client.py\u001b[0m in \u001b[0;36mrequest\u001b[0;34m(self, method, url, body, headers, encode_chunked)\u001b[0m\n\u001b[1;32m   1250\u001b[0m                 encode_chunked=False):\n\u001b[1;32m   1251\u001b[0m         \u001b[0;34m\"\"\"Send a complete request to the server.\"\"\"\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m-> 1252\u001b[0;31m         \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_send_request\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mmethod\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0murl\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mbody\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mheaders\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mencode_chunked\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m   1253\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1254\u001b[0m     \u001b[0;32mdef\u001b[0m \u001b[0m_send_request\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mmethod\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0murl\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mbody\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mheaders\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mencode_chunked\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/anaconda3/lib/python3.7/http/client.py\u001b[0m in \u001b[0;36m_send_request\u001b[0;34m(self, method, url, body, headers, encode_chunked)\u001b[0m\n\u001b[1;32m   1296\u001b[0m             \u001b[0;31m# default charset of iso-8859-1.\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1297\u001b[0m             \u001b[0mbody\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0m_encode\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mbody\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m'body'\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m-> 1298\u001b[0;31m         \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mendheaders\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mbody\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mencode_chunked\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mencode_chunked\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m   1299\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1300\u001b[0m     \u001b[0;32mdef\u001b[0m \u001b[0mgetresponse\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/anaconda3/lib/python3.7/http/client.py\u001b[0m in \u001b[0;36mendheaders\u001b[0;34m(self, message_body, encode_chunked)\u001b[0m\n\u001b[1;32m   1245\u001b[0m         \u001b[0;32melse\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1246\u001b[0m             \u001b[0;32mraise\u001b[0m \u001b[0mCannotSendHeader\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m-> 1247\u001b[0;31m         \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_send_output\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mmessage_body\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mencode_chunked\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mencode_chunked\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m   1248\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1249\u001b[0m     def request(self, method, url, body=None, headers={}, *,\n",
      "\u001b[0;32m~/anaconda3/lib/python3.7/http/client.py\u001b[0m in \u001b[0;36m_send_output\u001b[0;34m(self, message_body, encode_chunked)\u001b[0m\n\u001b[1;32m   1024\u001b[0m         \u001b[0mmsg\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0;34mb\"\\r\\n\"\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mjoin\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_buffer\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1025\u001b[0m         \u001b[0;32mdel\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_buffer\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m-> 1026\u001b[0;31m         \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0msend\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mmsg\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m   1027\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1028\u001b[0m         \u001b[0;32mif\u001b[0m \u001b[0mmessage_body\u001b[0m \u001b[0;32mis\u001b[0m \u001b[0;32mnot\u001b[0m \u001b[0;32mNone\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/anaconda3/lib/python3.7/http/client.py\u001b[0m in \u001b[0;36msend\u001b[0;34m(self, data)\u001b[0m\n\u001b[1;32m    964\u001b[0m         \u001b[0;32mif\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0msock\u001b[0m \u001b[0;32mis\u001b[0m \u001b[0;32mNone\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    965\u001b[0m             \u001b[0;32mif\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mauto_open\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 966\u001b[0;31m                 \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mconnect\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    967\u001b[0m             \u001b[0;32melse\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    968\u001b[0m                 \u001b[0;32mraise\u001b[0m \u001b[0mNotConnected\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/anaconda3/lib/python3.7/http/client.py\u001b[0m in \u001b[0;36mconnect\u001b[0;34m(self)\u001b[0m\n\u001b[1;32m   1412\u001b[0m             \u001b[0;34m\"Connect to a host on a given (SSL) port.\"\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1413\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m-> 1414\u001b[0;31m             \u001b[0msuper\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mconnect\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m   1415\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1416\u001b[0m             \u001b[0;32mif\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_tunnel_host\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/anaconda3/lib/python3.7/http/client.py\u001b[0m in \u001b[0;36mconnect\u001b[0;34m(self)\u001b[0m\n\u001b[1;32m    936\u001b[0m         \u001b[0;34m\"\"\"Connect to the host and port specified in __init__.\"\"\"\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    937\u001b[0m         self.sock = self._create_connection(\n\u001b[0;32m--> 938\u001b[0;31m             (self.host,self.port), self.timeout, self.source_address)\n\u001b[0m\u001b[1;32m    939\u001b[0m         \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0msock\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0msetsockopt\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0msocket\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mIPPROTO_TCP\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0msocket\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mTCP_NODELAY\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;36m1\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    940\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/anaconda3/lib/python3.7/socket.py\u001b[0m in \u001b[0;36mcreate_connection\u001b[0;34m(address, timeout, source_address)\u001b[0m\n\u001b[1;32m    705\u001b[0m     \u001b[0mhost\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mport\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0maddress\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    706\u001b[0m     \u001b[0merr\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0;32mNone\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 707\u001b[0;31m     \u001b[0;32mfor\u001b[0m \u001b[0mres\u001b[0m \u001b[0;32min\u001b[0m \u001b[0mgetaddrinfo\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mhost\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mport\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;36m0\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mSOCK_STREAM\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    708\u001b[0m         \u001b[0maf\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0msocktype\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mproto\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mcanonname\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0msa\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mres\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    709\u001b[0m         \u001b[0msock\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0;32mNone\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/anaconda3/lib/python3.7/socket.py\u001b[0m in \u001b[0;36mgetaddrinfo\u001b[0;34m(host, port, family, type, proto, flags)\u001b[0m\n\u001b[1;32m    750\u001b[0m     \u001b[0;31m# and socket type values to enum constants.\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    751\u001b[0m     \u001b[0maddrlist\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0;34m[\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 752\u001b[0;31m     \u001b[0;32mfor\u001b[0m \u001b[0mres\u001b[0m \u001b[0;32min\u001b[0m \u001b[0m_socket\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mgetaddrinfo\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mhost\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mport\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mfamily\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mtype\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mproto\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mflags\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    753\u001b[0m         \u001b[0maf\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0msocktype\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mproto\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mcanonname\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0msa\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mres\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    754\u001b[0m         addrlist.append((_intenum_converter(af, AddressFamily),\n",
      "\u001b[0;31mKeyboardInterrupt\u001b[0m: "
     ]
    }
   ],
   "source": [
    "for q in topics:\n",
    "    q['documents'] = topics_iter(q)\n",
    "    with open(input_dir+\"/docs_final/docs_for_topic_{}.txt\".format(q['number']), \"w\") as f:\n",
    "        json.dump(q, f)\n",
    "        \n",
    "    f.close()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#lemmatize topics\n",
    "#f = open(input_dir+'/topics_final.json')\n",
    "#topics = json.load(f)\n",
    "#f.close()\n",
    "lis =[]\n",
    "for i in range(len(topics)):\n",
    "    x=(topics[i]['title'])\n",
    "    lis.append(re.sub('[?:,]', '', x))\n",
    "converted_list = [x.lower() for x in lis]\n",
    "#print (\"Topics: \", converted_list)\n",
    "#print(\"\\n\")\n",
    "\n",
    "tokenized_sents = [word_tokenize(i) for i in converted_list]\n",
    "#for i in tokenized_sents:\n",
    "    #print (i)\n",
    "lis3 =[]\n",
    "\n",
    "for i in tokenized_sents:\n",
    "    tokens_without_sw = [word for word in i if not word in stopwords.words()]\n",
    "    lemmatizer = WordNetLemmatizer()\n",
    "    lemmatized_output_0 = ([lemmatizer.lemmatize(w,pos=\"n\") for w in tokens_without_sw])\n",
    "    lemmatized_output_1 = ' '.join(([lemmatizer.lemmatize(w,pos=\"v\") for w in lemmatized_output_0]))\n",
    "    lis3.append(lemmatized_output_1)   \n",
    "#print(\"Lemmatized verbs and nouns: \\n\", lis3)\n",
    "#with open(input_dir+'/topics_final_lemmatized.txt', 'wb') as fp:\n",
    "#    pickle.dump(lis3, fp)\n",
    "#fp.close()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#create and save bulk data for index\n",
    "#input_dir = 'data/docs_final'\n",
    "#output_dir = 'res'\n",
    "def strip_punct(s):\n",
    "    s = re.sub('[^A-Za-z0-9]', ' ', s)\n",
    "    s = s.lower()\n",
    "    return \" \".join(s.split())\n",
    "\n",
    "count = 0\n",
    "c=0\n",
    "t = 1\n",
    "url = 'https://demo.webis.de/targer-api/classifyCombo'\n",
    "headers = {'accept': 'application/json', 'Content-Type': 'text/plain'}\n",
    "bulk_data = []\n",
    "#extract docs from zip-files\n",
    "for filename in os.listdir(input_dir+'/docs_final'):\n",
    "    if filename.endswith(\".zip\"):\n",
    "        with ZipFile((input_dir+\"/{}\").format(filename), 'r') as zip:\n",
    "            zip.extractall(input_dir)\n",
    "            \n",
    "for filename in os.listdir(input_dir+'/docs_final'):\n",
    "    if filename.endswith(\".txt\"):\n",
    "        name = re.split('_|\\.', filename)\n",
    "        num = next(obj for obj in name if obj.isdigit())\n",
    "        with open((input_dir+\"/{}\").format(filename), \"r\") as f:\n",
    "            topic = json.load(f)\n",
    "            topic['title'] = strip_punct(topic['title']).lower()\n",
    "            print(filename)\n",
    "            print(topic['title'])\n",
    "            print(t)\n",
    "            t+=1\n",
    "            for n in topic[\"documents\"]:\n",
    "                print(len(n))\n",
    "                if count<10:\n",
    "                    count+=1\n",
    "                    for doc in n:\n",
    "                        try:\n",
    "                            doc_raw = doc['document'].rstrip('\\n')\n",
    "                            doc_raw = doc_raw.rstrip('\\\\n')\n",
    "                            \n",
    "                            doc['lem'] = word_tokenize(strip_punct(doc_raw).lower())\n",
    "                            tokens_without_sw = [word for word in doc['lem'] if not word in stopwords.words()]\n",
    "                            lemmatizer = WordNetLemmatizer()\n",
    "                            lemmatized_output_0 = ([lemmatizer.lemmatize(w,pos=\"n\") for w in tokens_without_sw])\n",
    "                            doc['lem'] = ' '.join(([lemmatizer.lemmatize(w,pos=\"v\") for w in lemmatized_output_0]))\n",
    "                            doc['lem'] = doc['lem'].rstrip()\n",
    "                            \n",
    "                            title_lem = strip_punct(doc['title'].lower().rstrip())\n",
    "                            title_lem = word_tokenize(title_lem)\n",
    "                            tokens_without_sw = [word for word in title_lem if not word in stopwords.words()]\n",
    "                            lemmatizer = WordNetLemmatizer()\n",
    "                            lemmatized_output_0 = ([lemmatizer.lemmatize(w,pos=\"n\") for w in tokens_without_sw])\n",
    "                            title_lem = ' '.join(([lemmatizer.lemmatize(w,pos=\"v\") for w in lemmatized_output_0]))\n",
    "                            \n",
    "                            \n",
    "                            \n",
    "                            b = {\n",
    "                                    'query': topic['title'],\n",
    "                                    'title': doc['title'],\n",
    "                                    'title_lem': title_lem,\n",
    "                                    'num': num,\n",
    "                                    'uuid': doc['uuid'],\n",
    "                                    'score': doc['score'],\n",
    "                                    'document': doc['document'],\n",
    "                                    'lem': doc['lem']\n",
    "                                }\n",
    "\n",
    "                            templ = {'index': {'_index': 'test_index', \n",
    "                                           '_type': 'doc', \n",
    "                                           '_id': doc['trec_id']}}\n",
    "                            bulk_data.append(templ)\n",
    "                            bulk_data.append(b)\n",
    "\n",
    "                            c+=1\n",
    "                        except KeyError:\n",
    "                            pass\n",
    "                else:\n",
    "                    break\n",
    "            count=0\n",
    "            \n",
    "#with open(input_dir+\"/bulk_data_final.json\", \"w\") as f:\n",
    "#    json.dump(bulk_data,f)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#create synonyms\n",
    "#with open(input_dir+'/topics_final_lemmatized.txt', 'rb') as f:\n",
    "#    queries_lem = pickle.load(f)\n",
    "#f.close()\n",
    "df = pd.DataFrame(columns = ['query', 'syn'])\n",
    "df['query'] = queries_lem\n",
    "#print(df)\n",
    "#df.to_csv(input_dir+'/q_for_syn_final.tsv', sep = '\\t')\n",
    "\n",
    "#define words for synonyms in the .tsv file manually!"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#df = pd.read_csv(input_dir+'/q_for_syn_final.tsv', sep = '\\t')\n",
    "#df = df.fillna(0)\n",
    "\n",
    "for idx, row in df.iterrows():\n",
    "    if row['query']!=0:\n",
    "        tokenized = word_tokenize(row['query'])\n",
    "        pos_tagged = nltk.pos_tag(tokenized)\n",
    "        satz_synonyms = []\n",
    "        for wort in pos_tagged:\n",
    "            if  wort[1] != 'RB' and wort[1] != 'JJ' and wort[1] != 'JJS' and wort[1]!='RBR'and wort[1]!='RBS':\n",
    "                #print(wort[0])\n",
    "                for syn in wordnet.synsets(wort[0]):\n",
    "                    for l in syn.lemmas()[:1]:\n",
    "                        for n in l.name().split():\n",
    "                            if n not in satz_synonyms:\n",
    "                                satz_synonyms.append(n)\n",
    "        else:\n",
    "            satz_synonyms.append(wort[0])\n",
    "        s = ' '.join(str(v) for v in satz_synonyms)\n",
    "        s = ''.join(s)\n",
    "        s = s.split('_')\n",
    "        s = ' '.join(s)\n",
    "        #print(s)\n",
    "        df.at[idx,['syn']] = s\n",
    "#df.to_csv(input_dir+'/q_for_syn_final.tsv', sep = '\\t', index = False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#create index\n",
    "\n",
    "es = Elasticsearch()\n",
    "\n",
    "'''\n",
    "for filename in os.listdir(input_dir):\n",
    "    if filename.endswith(\".zip\"):\n",
    "        with ZipFile(filename, 'r') as zip:\n",
    "            zip.extractall()\n",
    "'''            \n",
    "def chunks(lst, n):\n",
    "    \"\"\"Yield successive n-sized chunks from lst.\"\"\"\n",
    "    for i in range(0, len(lst), n):\n",
    "        yield lst[i:i + n]\n",
    "\n",
    "def search_extended(search_param, filename, q, ind):\n",
    "    qid = []\n",
    "    Q0 = []\n",
    "    doc = []\n",
    "    rank = []\n",
    "    score = []\n",
    "    tag = []\n",
    "\n",
    "    num=1\n",
    "#['bool']['should']\n",
    "    for idx, row in q.iterrows():\n",
    "            \n",
    "            #for key in x['match']:\n",
    "                #x['match'][key] = row['query']\n",
    "        for x in search_param['query']['bool']['should']:\n",
    "            for key in x[\"match\"]:\n",
    "                #print(key)\n",
    "                if \"boost\" in x['match'][key]:\n",
    "                    x['match'][key][\"query\"] = row['query']\n",
    "                else:\n",
    "                    x['match'][key] = row['syn']\n",
    "        #print(search_param)\n",
    "        response = es.search(index=ind, body=search_param)\n",
    "        r = 1\n",
    "        for x in response['hits']['hits']:\n",
    "            qid.append(num)\n",
    "            Q0.append('Q0')\n",
    "            doc.append(x['_id'])\n",
    "            rank.append(r)\n",
    "            score.append(x['_score'])\n",
    "            tag.append('uh-t2-thor')\n",
    "            r+=1\n",
    "        num+=1\n",
    "    \n",
    "   \n",
    "    qrels = {'qid': qid, 'Q0': Q0, 'doc': doc, 'rank':rank, 'score':score, 'tag':tag}\n",
    "\n",
    "    df = pd.DataFrame(qrels)\n",
    "    df.to_csv((output_dir+'/'+'run.txt'), sep = ' ', index = False, header = False)\n",
    "    \n",
    "    \n",
    "def create_index(b, k1, index):\n",
    "    \n",
    "    #create template for index\n",
    "    request_body = {\n",
    "    \"settings\" : {\n",
    "\t        \"number_of_shards\": 1,\n",
    "\t        \"number_of_replicas\": 0,\n",
    "        \"similarity\": {\n",
    "      \"default\": { \n",
    "        \"type\": \"BM25\",\n",
    "        \"b\":b,\n",
    "        \"k1\":k1\n",
    "      }\n",
    "    }\n",
    "\t    }\n",
    "\t}\n",
    "\n",
    "    es.indices.create(index = index, body = request_body)\n",
    "    \n",
    "    \n",
    "    #load data to index\n",
    "    #with open(input_dir+'/bulk_data_final.json') as f:\n",
    "    #    bulk_data = json.load(f)\n",
    "        \n",
    "    for x in bulk_data[::2]:\n",
    "        x['index']['_index']=index\n",
    "        \n",
    "    bulks = chunks(bulk_data, 100)\n",
    "    for x in bulks:\n",
    "        res = es.bulk(index = index, body = x)\n",
    "    \n",
    "def create_run(d, run_name, index, mode):\n",
    "    #d, run_name, index\n",
    "    \n",
    "    if mode=='simple':\n",
    "        #f = open(input_dir+\"/topics_final_lemmatized.txt\", 'rb')\n",
    "        #queries_lem = pickle.load(f)\n",
    "        #f.close()\n",
    "        queries_lem = lis3\n",
    "        search_param = {\n",
    "    \n",
    "        'size': 50,\n",
    "        \"query\": {\n",
    "        \"bool\": {\n",
    "        \"should\": d\n",
    "        }\n",
    "        }\n",
    "        }\n",
    "        #print(search_param)\n",
    "\n",
    "        search(search_param, run_name, queries_lem, index)\n",
    "    elif mode=='syn':\n",
    "        #df_syn = pd.read_csv(input_dir+'/q_for_syn.tsv', sep = '\\t')\n",
    "        search_param = {\n",
    "    \n",
    "        'size': 50,\n",
    "        \"query\": {\n",
    "        \"bool\": {\n",
    "        \"should\": d\n",
    "        }\n",
    "        }\n",
    "        }\n",
    "        #print(search_param)\n",
    "\n",
    "        search_extended(search_param, run_name, df, index)\n",
    "    else:\n",
    "        print('Wrong arguments!')\n",
    "        \n",
    "index = 'final_0.68'\n",
    "create_index(0.68, 1.2, index)\n",
    "\n",
    "#create run for doc+title with synonyms\n",
    "d=[]\n",
    "d.append({\"match\": {\"title_lem\": {\n",
    "                        \"query\":\"\",\n",
    "                    \"boost\":5}}})\n",
    "d.append({\"match\": {\"lem\": {\n",
    "                        \"query\":\"\",\n",
    "                    \"boost\":5}}})\n",
    "d.append({\"match\": {\"title_lem\": {\"\"}}})\n",
    "d.append({\"match\": {\"lem\": {\"\"}}})\n",
    "create_run(d, 'run_title_doc_syn', index, 'syn')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "es.indices.delete(index=index)"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.6"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}