notebooks/questions_binary_classification.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# The training and testing data should be POS tagged (e.g., mystem for Russian) and saved as word POS word POS ...\n",
    "# as implemented in notebook rule_based_classification.ipynb\n",
    "\n",
    "# This code uses predictions from individual models saved in files\n",
    "# input files:\n",
    "# CNN predictions in form of probabilities as .txt file: probability\\tquestion, where probability - a prob. that a question is comp.\n",
    "# BERT predictions in form of probabilities as .txt file: probability1\\t\\tprobaility2\\tquestion, where probability1 is a prob. that a question is not comp., probability2 - otherwise\n",
    "# ML (log. regr) predictions as .txt file: lr_probabilities\\tquestion"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Define helper functions"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import json\n",
    "import subprocess\n",
    "import string\n",
    "import re\n",
    "from tqdm import tqdm_notebook as tqdm\n",
    "import pandas as pd\n",
    "import numpy as np\n",
    "from os.path import join\n",
    "\n",
    "### Helper functions to read probabilities\n",
    "import glob\n",
    "import os\n",
    "\n",
    "def read_cnn_questions_probabilities():\n",
    "    cnn_questions=[]\n",
    "    cnn_probabilities=[]\n",
    "    for n in range(10): # adjusted to 10-Fold Crossvalidation producing a separ. file for each split\n",
    "        path_cnn = 'path_to_CNN_predictions'\n",
    "        filename = glob.glob(os.path.join(path_cnn, 'file' + str(n) + '.txt')) # change to your file names  \n",
    "        with open(filename[0]) as f:\n",
    "            content = f.readlines()\n",
    "        cnn_probabilities += [float(x.strip('\\n').split('\\t')[0]) for x in content[1:]] # ignore heading in the input file\n",
    "        cnn_questions += [x.strip('\\n').split('\\t')[1] for x in content[1:]] # ignore heading in the input file\n",
    "    return  cnn_probabilities, cnn_questions \n",
    "\n",
    "def read_bert_questions_probabilities():\n",
    "    bert_questions=[]\n",
    "    bert_probabilities=[]\n",
    "    for n in range(10):\n",
    "        path_bert = 'path_to_BERT_predictions'  \n",
    "        filename = glob.glob(os.path.join(path_bert, 'file' + str(n) + '.txt'))\n",
    "        with open(filename[0]) as f:\n",
    "            content = f.readlines()\n",
    "        bert_probabilities += [float(x.strip('\\n').split('\\t')[1]) for x in content] # no heading in the input file\n",
    "        bert_questions += [x.strip('\\n').split('\\t')[2] for x in content] # no heading in the input file\n",
    "    return bert_probabilities, bert_questions\n",
    "\n",
    "def read_ml_questions_probabilities():\n",
    "    ml_questions=[]\n",
    "    lr_probabilities=[]\n",
    "    for n in range(10):\n",
    "        path_ml = 'path_to_ML_predictions'\n",
    "        filename = glob.glob(os.path.join(path_ml, 'file' + str(n) + '.txt'))\n",
    "        with open(filename[0]) as f:\n",
    "            content = f.readlines()\n",
    "        lr_probabilities += [float(x.strip('\\n').split('\\t')[0]) for x in content[1:]] # ignore heading in the input file\n",
    "        ml_questions += [x.strip('\\n').split('\\t')[1]for x in content[1:]]\n",
    "    return  lr_probabilities, ml_questions\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Building Ensemble classifier on a training set"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Reading probabilities in a dataframe\n",
    "\n",
    "cnn_probabilities, cnn_questions = read_cnn_questions_probabilities()\n",
    "bert_probabilities, bert_questions = read_bert_questions_probabilities()\n",
    "\n",
    "neural_predictions_df = pd.DataFrame({'cnn_probabilities':cnn_probabilities, 'bert_probabilities':bert_probabilities, \n",
    "                                      'lr_probabilities':lr_probabilities, 'bert_questions':bert_questions, 'cnn_questions':cnn_questions, 'ml_questions':ml_questions})\n",
    "\n",
    "# all_predictions_df contains probabilities of all classifiers along with questions\n",
    "# Columns: 'cnn_probabilities', 'bert_probabilities', 'lr_probabilities', 'bert_questions', 'cnn_questions', 'ml_questions'\n",
    "\n",
    "global all_predictions_df\n",
    "all_predictions_df = neural_predictions_df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Read data from train split\n",
    "\n",
    "dataset = 'train-binary.txt'\n",
    "data = pd.read_csv(join('path_to_train_data', dataset), sep='\\t')\n",
    "\n",
    "questions_original = np.array([question for question in data['question'].tolist()]) # questions before POS tagging\n",
    "questions = np.array([pos(mst.process(strip_punct(question))) for question in data['question'].tolist()]) # produce POS tagging on a fly\n",
    "# see notebook rule_based_classification.ipynb for example with mystem in Russian\n",
    "# or ignore if training data was already POS tagged\n",
    "labels = np.array(data.comp.tolist())\n",
    "\n",
    "dataset = 'train-binary-afterpb.txt' # dataset produced after removing questions classified with rules (see notebook rule_based_classification.ipynb)\n",
    "# which produce predicitons with precision 1. In our case, the first 7 rules from the notebook\n",
    "\n",
    "data_afterpb_df = pd.read_csv(join('path_to_train_data', dataset), sep='\\t')\n",
    "\n",
    "questions_afterpb = np.array([pos(mst.process(strip_punct(question))) for question in data_afterpb_df['question'].tolist()]) # produce POS tagging on a fly\n",
    "# or ignore if training data was already POS tagged\n",
    "\n",
    "labels_afterpb = np.array(data_afterpb_df.comp.tolist())\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Predictions based on the ML probabilities combined with rules"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Define helper functions\n",
    "\n",
    "from collections import Counter\n",
    "from sklearn.metrics import classification_report\n",
    "\n",
    "# We add the 8th rule (in Russian) which recalls extra 20% but makes precision not 1 any more.\n",
    "# We will then fix FP using ML\n",
    "\n",
    "def predict_pattern(question):\n",
    "    if (re.search('луч.{0,1}ше ', question) and not re.search('как ', question)): prediction = 1\n",
    "    elif (re.search('comp', question) and re.search(' или | vs | vs.', question) and question.find('comp') < question.find(' или ') and not re.search('более comp или conj менее comp', question)): prediction = 1\n",
    "    elif (re.search('как', question) and re.search('правильно', question) and re.search(' или ', question)) or (re.search('как', question) and re.search('пишется | писать | написать', question) and re.search(' или ', question)): prediction = 1\n",
    "    elif (re.search('что ', question) and re.search(r'общего | сходст| схож', question) and re.search(' и | от | или | между | vs | vs. | versus ', question)): prediction = 1\n",
    "    elif (re.search('выб+рать|купить|взять', question) and re.search(r' или | между | vs | vs. | versus ', question)): prediction = 1\n",
    "    elif ((re.search(' в ', question)) and re.search(' сравнении ', question)) or ((re.search(' по ', question)) and re.search(' сравнению ', question)): prediction = 1\n",
    "    elif (re.search('преимуществ|недостаток', question) and re.search('перед | над | сравнен | vs | vs. | versus', question)): prediction = 1\n",
    "    elif (re.search(' отлич| разница | различие | различ', question) and re.search(' и | от | или | между | vs | vs. | versus', question) and not re.search('что ', question)): prediction = 1\n",
    "    else: prediction = 0\n",
    "    return prediction\n",
    "\n",
    "def binary_pred(prob, thre=0.5):\n",
    "    if prob >= thre: pred = 1\n",
    "    else: pred = 0\n",
    "    return pred\n",
    "\n",
    "def precision_recall(probabilities):\n",
    "    comb_predictions = []\n",
    "    thresholds = []\n",
    "\n",
    "    for pr_pos in np.arange(0.0, 0.99, 0.001): # manipulate the step of the probability threshold in order to draw precision-recall curve\n",
    "        comb_predictions_ = []\n",
    "        pr_pos = round(pr_pos,5)\n",
    "        for item in probabilities:\n",
    "            pred = binary_pred(item, thre=pr_pos)\n",
    "            comb_predictions_.append(pred)\n",
    "        comb_predictions_ = np.pad(comb_predictions_, (0,476), 'constant', constant_values=1) # predictions would be padded with ones for those predicted by the rules as 1\n",
    "        comb_predictions.append(comb_predictions_)\n",
    "        thresholds.append(pr_pos)\n",
    "    predictions_thresholds = list(zip(comb_predictions, thresholds))\n",
    "    dicts_low = {}\n",
    "    keys = [str(item[1]) for item in predictions_thresholds]\n",
    "    for i in range(len(keys)):\n",
    "            dicts_low[keys[i]] = predictions_thresholds[i][0]\n",
    "            \n",
    "    comb_predictions = []\n",
    "    thresholds = []\n",
    "\n",
    "    for pr_pos in np.arange(0.99, 1.00001, 0.00001): # the high probabilities are very senstitive to the step in making decision so that we reduces the step \n",
    "        comb_predictions_ = []\n",
    "        pr_pos = round(pr_pos,5)\n",
    "        for item in probabilities:\n",
    "            pred = binary_pred(item, thre=pr_pos)\n",
    "            comb_predictions_.append(pred)\n",
    "        comb_predictions_ = np.pad(comb_predictions_, (0,476), 'constant', constant_values=1)\n",
    "        comb_predictions.append(comb_predictions_)\n",
    "        thresholds.append(pr_pos)\n",
    "    predictions_thresholds = list(zip(comb_predictions, thresholds))\n",
    "    dicts_hi = {}\n",
    "    keys = [str(item[1]) for item in predictions_thresholds]\n",
    "    for i in range(len(keys)):\n",
    "            dicts_hi[keys[i]] = predictions_thresholds[i][0]\n",
    "    \n",
    "    dicts={}\n",
    "    dicts = {**dicts_low, **dicts_hi}\n",
    "\n",
    "    labels_afterpb_padded = np.pad(labels_afterpb, (0,476), 'constant', constant_values=1)\n",
    "    recall = []\n",
    "    precision = []\n",
    "    thresholds = []\n",
    "    for keys, values in dicts.items():\n",
    "        precision_ = classification_report(y_true=labels_afterpb_padded, y_pred=values, output_dict=True)['1']['precision']\n",
    "        recall_ = classification_report(y_true=labels_afterpb_padded, y_pred=values, output_dict=True)['1']['recall']\n",
    "        precision.append(round(precision_, 3))\n",
    "        recall.append(round(recall_,3))\n",
    "        thresholds.append(keys)\n",
    "    return precision, recall, thresholds"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "%%time\n",
    "# Calculate precision, recall and respective decision probability thresholds for predicitions\n",
    "# from log. regr., CNN, BERT\n",
    "\n",
    "precision_lr, recall_lr, thresholds_lr = precision_recall(lr_probabilities)\n",
    "precision_bert, recall_bert, thresholds_bert = precision_recall(bert_probabilities)\n",
    "precision_cnn, recall_cnn, thresholds_cnn = precision_recall(cnn_probabilities)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "%%time\n",
    "# Build ensemble classifiers, where comb_no_bert would mean Ensemble-CNN, comb_no_cnn - Ensemble-BERT, and comb_predictions - Ensemble-combi \n",
    "# The calculations would be done in two parts for the lower classifier confedence probabilities with a enlarged step\n",
    "# and for higher probabilites with a reduced step.\n",
    "\n",
    "comb_no_bert = []\n",
    "comb_no_cnn = []\n",
    "comb_predictions = []\n",
    "thresholds = []\n",
    "\n",
    "for pr_pos in np.arange(0.0, 0.99, 0.001):\n",
    "    comb_predictions_ = []\n",
    "    comb_no_bert_ = []\n",
    "    comb_no_cnn_ = []\n",
    "    pr_pos = round(pr_pos,5)\n",
    "    for item in list(zip(lr_probabilities, cnn_probabilities, bert_probabilities, ml_questions)):\n",
    "        pred_lr = binary_pred(item[0], thre=0.45)                 # these thresholds correspond to values where the model\n",
    "        pred_cnn = binary_pred(item[1], thre=0.995)               # achieves a maximal recall at a precision of 1\n",
    "        pred_bert = binary_pred(item[2], thre=0.998)              # for a comparative questions class\n",
    "        if pred_bert == 1: comb_predictions_.append(pred_bert)\n",
    "        elif pred_cnn == 1: comb_predictions_.append(pred_cnn)\n",
    "        elif pred_lr == 1: comb_predictions_.append(pred_lr)\n",
    "        else:\n",
    "            if binary_pred(item[1], thre=pr_pos) == binary_pred(item[2], thre=pr_pos): pred = binary_pred(item[2], thre=pr_pos)\n",
    "            else: pred = predict_pattern(item[3])\n",
    "            comb_predictions_.append(pred)\n",
    "\n",
    "        if pred_cnn == 1: comb_no_bert_.append(pred_cnn)\n",
    "        else:\n",
    "            if binary_pred(item[0], thre=pr_pos) == binary_pred(item[1], thre=pr_pos): pred = binary_pred(item[1], thre=pr_pos)\n",
    "            else: pred = predict_pattern(item[4])\n",
    "            comb_no_bert_.append(pred)\n",
    "            \n",
    "        if pred_bert == 1: comb_no_cnn_.append(pred_bert)\n",
    "        elif pred_lr == 1: comb_no_cnn_.append(pred_lr)\n",
    "        else:\n",
    "            if binary_pred(item[0], thre=pr_pos) == binary_pred(item[2], thre=pr_pos): pred = binary_pred(item[2], thre=pr_pos)\n",
    "            else: pred = predict_pattern(item[4])\n",
    "            comb_no_cnn_.append(pred)\n",
    "            \n",
    "    comb_no_bert_ = np.pad(comb_no_bert_, (0,476), 'constant', constant_values=1)         # padding by ones to reflect the number of questions \n",
    "    comb_no_cnn_ = np.pad(comb_no_cnn_, (0,476), 'constant', constant_values=1)           # classified as comparative by rules with precision 1\n",
    "    comb_predictions_ = np.pad(comb_predictions_, (0,476), 'constant', constant_values=1)\n",
    "    comb_predictions.append(comb_predictions_)\n",
    "    comb_no_bert.append(comb_no_bert_)\n",
    "    comb_no_cnn.append(comb_no_cnn_)\n",
    "    thresholds.append(pr_pos)\n",
    "\n",
    "def create_dict_low(predictions, thresholds):\n",
    "    predictions_thresholds = list(zip(predictions, thresholds))\n",
    "    dicts_low = {}\n",
    "    keys = [str(item[1]) for item in predictions_thresholds]\n",
    "    for i in range(len(keys)):\n",
    "            dicts_low[keys[i]] = predictions_thresholds[i][0]\n",
    "    return dicts_low\n",
    "\n",
    "dicts_low_comb = create_dict_low(comb_predictions, thresholds)\n",
    "dicts_low_no_bert = create_dict_low(comb_no_bert, thresholds)\n",
    "dicts_low_no_cnn = create_dict_low(comb_no_cnn, thresholds)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "%%time\n",
    "\n",
    "# Follows the instructions above but for higher classifiers' confidence probabilities\n",
    "\n",
    "comb_no_bert = []\n",
    "comb_no_cnn = []\n",
    "comb_predictions = []\n",
    "thresholds = []\n",
    "\n",
    "for pr_pos in np.arange(0.99, 1.00001, 0.00001):\n",
    "    comb_predictions_ = []\n",
    "    comb_no_bert_ = []\n",
    "    comb_no_cnn_ = []\n",
    "    pr_pos = round(pr_pos,5)\n",
    "    for item in list(zip(lr_probabilities, cnn_probabilities, bert_probabilities, ml_questions)):\n",
    "        pred_lr = binary_pred(item[0], thre=0.45)\n",
    "        pred_cnn = binary_pred(item[1], thre=0.995)\n",
    "        pred_bert = binary_pred(item[2], thre=0.998)\n",
    "        if pred_bert == 1: comb_predictions_.append(pred_bert)\n",
    "        elif pred_cnn == 1: comb_predictions_.append(pred_cnn)\n",
    "        elif pred_lr == 1: comb_predictions_.append(pred_lr)\n",
    "        else:\n",
    "            if binary_pred(item[1], thre=pr_pos) == binary_pred(item[2], thre=pr_pos): pred = binary_pred(item[2], thre=pr_pos)\n",
    "            else: pred = predict_pattern(item[4])\n",
    "            comb_predictions_.append(pred)\n",
    "\n",
    "        if pred_cnn == 1: comb_no_bert_.append(pred_cnn)\n",
    "        else:\n",
    "            if binary_pred(item[0], thre=pr_pos) == binary_pred(item[1], thre=pr_pos): pred = binary_pred(item[1], thre=pr_pos)\n",
    "            else: pred = predict_pattern(item[3])\n",
    "            comb_no_bert_.append(pred)\n",
    "            \n",
    "        if pred_bert == 1: comb_no_cnn_.append(pred_bert)\n",
    "        elif pred_lr == 1: comb_no_cnn_.append(pred_lr)\n",
    "        else:\n",
    "            if binary_pred(item[0], thre=pr_pos) == binary_pred(item[2], thre=pr_pos): pred = binary_pred(item[2], thre=pr_pos)\n",
    "            else: pred = predict_pattern(item[4])\n",
    "            comb_no_cnn_.append(pred)\n",
    "            \n",
    "    comb_no_bert_ = np.pad(comb_no_bert_, (0,476), 'constant', constant_values=1)\n",
    "    comb_no_cnn_ = np.pad(comb_no_cnn_, (0,476), 'constant', constant_values=1)\n",
    "    majority_vote_pred_ = np.pad(majority_vote_pred_, (0,476), 'constant', constant_values=1)\n",
    "    comb_predictions_ = np.pad(comb_predictions_, (0,476), 'constant', constant_values=1)\n",
    "    comb_predictions.append(comb_predictions_)\n",
    "    majority_vote_pred.append(majority_vote_pred_)\n",
    "    comb_no_bert.append(comb_no_bert_)\n",
    "    comb_no_cnn.append(comb_no_cnn_)\n",
    "    thresholds.append(pr_pos)\n",
    "\n",
    "def create_dict_hi(predictions, thresholds):\n",
    "    predictions_thresholds = list(zip(predictions, thresholds))\n",
    "    dicts_hi = {}\n",
    "    keys = [str(item[1]) for item in predictions_thresholds]\n",
    "    for i in range(len(keys)):\n",
    "            dicts_hi[keys[i]] = predictions_thresholds[i][0]\n",
    "    return dicts_hi\n",
    "\n",
    "dicts_hi_comb = create_dict_hi(comb_predictions, thresholds)\n",
    "dicts_hi_no_bert = create_dict_hi(comb_no_bert, thresholds)\n",
    "dicts_hi_no_cnn = create_dict_hi(comb_no_cnn, thresholds)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "%%time\n",
    "\n",
    "# Merge dictionaries calculated above and extract precisions, recalls, f1's, and probability thresholds\n",
    "\n",
    "def prec_rec_f1(dicts_low, dicts_hi):\n",
    "    thresholds_comb = []\n",
    "    dicts = {**dicts_low, **dicts_hi}\n",
    "    labels_afterpb_padded = np.pad(labels_afterpb, (0,476), 'constant', constant_values=1)\n",
    "    recall = []\n",
    "    precision = []\n",
    "    f1 = []\n",
    "    for keys, values in dicts.items():\n",
    "        precision_ = classification_report(y_true=labels_afterpb_padded, y_pred=values, output_dict=True)['1']['precision']\n",
    "        recall_ = classification_report(y_true=labels_afterpb_padded, y_pred=values, output_dict=True)['1']['recall']\n",
    "        f1_ = classification_report(y_true=labels_afterpb_padded, y_pred=values, output_dict=True)['1']['f1-score']\n",
    "        precision.append(round(precision_, 3))\n",
    "        recall.append(round(recall_,3))\n",
    "        f1.append(round(f1_,3))\n",
    "        thresholds_comb.append(float(keys))\n",
    "    return precision, recall, f1, thresholds_comb\n",
    "\n",
    "precision, recall, f1, thresholds = prec_rec_f1(dicts_low_comb, dicts_hi_comb)\n",
    "precision_no_bert, recall_no_bert, f1_no_bert, thresholds_no_bert = prec_rec_f1(dicts_low_no_bert, dicts_hi_no_bert)\n",
    "precision_no_cnn, recall_no_cnn, f1_no_cnn, thresholds_no_cnn = prec_rec_f1(dicts_low_no_cnn, dicts_hi_no_cnn)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Read rule-based predictions calculated and saved in rule_based_calssification.ipynb\n",
    "\n",
    "pb_df = pd.read_csv('../prrecall-pb.txt', sep='\\t')\n",
    "precision_pb = pb_df.precision_pb\n",
    "recall_pb = pb_df.recall_pb"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Plot precision-recall curves for classification of the comparative question class\n",
    "\n",
    "%matplotlib inline\n",
    "import matplotlib.pyplot as plt\n",
    "import numpy as np\n",
    "\n",
    "font = {'size' : 22}\n",
    "plt.rc('font', **font)\n",
    "\n",
    "fig = plt.figure(figsize=(25, 6))\n",
    "ax1 = fig.add_subplot(1, 2, 1)\n",
    "ax2 = fig.add_subplot(1, 2, 2)\n",
    "\n",
    "major_ticks = np.arange(0, 1.01, 0.05)\n",
    "minor_ticks = np.arange(0, 1.01, 0.01)\n",
    "\n",
    "ax1.set_xticks(major_ticks)\n",
    "ax1.set_xticks(minor_ticks, minor=True)\n",
    "ax1.set_yticks(major_ticks)\n",
    "ax1.set_yticks(minor_ticks, minor=True)\n",
    "ax2.set_xticks(major_ticks)\n",
    "ax2.set_xticks(minor_ticks, minor=True)\n",
    "ax2.set_yticks(major_ticks)\n",
    "ax2.set_yticks(minor_ticks, minor=True)\n",
    "ax2.label_outer()\n",
    "\n",
    "axes = plt.gca()\n",
    "ax1.set_xlim([0.39,0.71])\n",
    "ax1.set_ylim([0.89,1.01])\n",
    "ax2.set_xlim([0.49,0.81])\n",
    "ax2.set_ylim([0.89,1.01])\n",
    "\n",
    "ax1.grid(which='minor', alpha=0.2)\n",
    "ax1.grid(which='major', alpha=0.5)\n",
    "ax2.grid(which='minor', alpha=0.2)\n",
    "ax2.grid(which='major', alpha=0.5)\n",
    "\n",
    "ax1.plot(recall_pb, precision_pb, marker='o', label='Rule-based', linestyle='dashed', \n",
    "        linewidth=2, markersize=10)\n",
    "ax1.plot(recall_cnn+[0.55], precision_cnn+[1.0], marker='d', label='CNN', linestyle='dashed', \n",
    "        linewidth=2, markersize=10, markevery = 0.025)\n",
    "ax1.plot(recall_bert+[0.49], precision_bert+[1.0], marker='s', label='BERT', linestyle='dashed', \n",
    "        linewidth=2, markersize=10, markevery = 0.03)\n",
    "ax1.plot(recall_lr+[0.55], precision_lr+[1.0], marker='v', label='Log. regression', linestyle='dashed', \n",
    "        linewidth=2, markersize=10, markevery = 0.025)\n",
    "\n",
    "\n",
    "ax2.plot(recall+[0.60], precision+[1.0], marker='o', label='Ensemble-combi', linestyle='dashed', \n",
    "        linewidth=2, markersize=10, markevery = 0.03)\n",
    "ax2.plot(recall_no_bert+[0.629], precision_no_bert+[1.0], marker='d', label='Ensemble-CNN', linestyle='dashed', \n",
    "        linewidth=2, markersize=10, markevery = 0.025)\n",
    "ax2.plot(recall_no_cnn+[0.628], precision_no_cnn+[1.0], marker='s', label='Ensemble-BERT', linestyle='dashed', \n",
    "        linewidth=2, markersize=10, markevery = 0.025)\n",
    "\n",
    "ax1.set(xlabel='Recall', ylabel='Precision')\n",
    "ax2.set(xlabel='Recall')\n",
    "\n",
    "ax1.legend(loc=3, ncol=1)\n",
    "ax2.legend(loc=3, ncol=1)\n",
    "plt.tight_layout()\n",
    "\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## The same can be then applied for the test set to evaluate classification performance"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.8"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}