notebooks/multiclass_classification.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "# This code uses predictions from individual models saved in files"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Define helper functions"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "import glob\n",
    "import os\n",
    "import pandas as pd\n",
    "\n",
    "def binary_pred(prob, thre=0.5):\n",
    "    if prob >= thre: pred = 1\n",
    "    else: pred = 0\n",
    "    return pred"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "dataset = 'train-multiclass-ya.txt' # annotated multicalss dataset\n",
    "data = pd.read_csv('path_to_multiclass_dataset' + dataset, sep='\\t')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Read BERT probibility predicitions for each subclass\n",
    "\n",
    "classes = [ 'opinion/argument', 'reason/factoid', 'context/aspect', 'preference', 'direct', 'super', 'method']\n",
    "\n",
    "i=0\n",
    "for cl in classes:\n",
    "    for n in range(10):     # considered 10 splits for 10-Fold Crossvalidation\n",
    "        path_bert = 'path_to_BERT_multiclass_predictions'\n",
    "        filename = glob.glob(os.path.join(path_bert, 'file' + classes[i] + str(n)))\n",
    "        data_frame = pd.read_csv(filename[0], delimiter='\\t', header=None)\n",
    "        data_frame_ = pd.DataFrame({cl: data_frame[1]})\n",
    "        if n == 0: data_frame_bert_ = data_frame_\n",
    "        else: data_frame_bert_ = pd.concat([data_frame_bert_, data_frame_])\n",
    "    if i == 0: data_frame_bert = data_frame_bert_  \n",
    "    else: data_frame_bert[cl] = pd.Series(data_frame_bert_[cl].tolist(), index=data_frame_bert.index) \n",
    "    i+=1\n",
    "    \n",
    "# Read CNN probibility predicitions for into dataframe each crossvalidation split contains all subclasses in one file\n",
    "\n",
    "for n in range(10):\n",
    "    path_cnn = 'path_to_CNN_multiclass_predictions'\n",
    "    filename = glob.glob(os.path.join(path_cnn, 'file' + str(n) + '.txt'))\n",
    "    data_frame_ = pd.read_csv(filename[0], delimiter='\\t')\n",
    "    if n == 0: data_frame = data_frame_\n",
    "    else: data_frame = pd.concat([data_frame, data_frame_])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "%%time\n",
    "\n",
    "from sklearn.metrics import precision_recall_fscore_support, f1_score, recall_score, precision_score\n",
    "\n",
    "# Calculate predictions for each subclass based on the probabilitites\n",
    "\n",
    "classes = [ 'opinion/argument', 'reason/factoid', 'context/aspect', 'preference', 'direct', 'super', 'method']\n",
    "\n",
    "precision_cnn = []\n",
    "recall_cnn = []\n",
    "f1_cnn = []\n",
    "\n",
    "precision_bert = []\n",
    "recall_bert = []\n",
    "f1_bert = []\n",
    "\n",
    "for cl in classes:\n",
    "    if cl == 'preference': labels = [1.0 if x==2.0 else x for x in data[cl].tolist()] # convert labels 1 and 2 for preference to 1\n",
    "    else: labels = data[cl].tolist()\n",
    "        \n",
    "    precision_cnn_cl = []\n",
    "    recall_cnn_cl = []\n",
    "    f1_cnn_cl = []\n",
    "    precision_bert_cl = []\n",
    "    recall_bert_cl = []\n",
    "    f1_bert_cl = []\n",
    "    \n",
    "    for pr_pos in np.arange(0., 1.001, 0.0005):\n",
    "        thre = round(pr_pos, 5)\n",
    "        pred_cnn_ = []\n",
    "        precision_cnn_ = []\n",
    "        recall_cnn_ = []\n",
    "        f1_cnn_ = []\n",
    "        pred_bert_ = []\n",
    "        precision_bert_ = []\n",
    "        recall_bert_ = []\n",
    "        f1_bert_ = []\n",
    "\n",
    "        pred_cnn_ = [binary_pred(prob, thre) for prob in data_frame[cl].tolist()]\n",
    "        pred_bert_ = [binary_pred(prob, thre) for prob in data_frame_bert[cl].tolist()]\n",
    "        \n",
    "        precision_cnn_ = round(precision_score(y_true=labels, y_pred=pred_cnn_, average='binary'),2)\n",
    "        recall_cnn_ = round(recall_score(y_true=labels, y_pred=pred_cnn_, average='binary'),2)\n",
    "        f1_cnn_ =  round(f1_score(y_true=labels, y_pred=pred_cnn_, average='binary'),2)\n",
    "        \n",
    "        precision_bert_ = round(precision_score(y_true=labels, y_pred=pred_bert_, average='binary'),2)\n",
    "        recall_bert_ = round(recall_score(y_true=labels, y_pred=pred_bert_, average='binary'),2)\n",
    "        f1_bert_ = round(f1_score(y_true=labels, y_pred=pred_bert_, average='binary'),2)\n",
    "        \n",
    "        precision_cnn_cl.append(precision_cnn_)\n",
    "        recall_cnn_cl.append(recall_cnn_)\n",
    "        f1_cnn_cl.append(f1_cnn_)\n",
    "        \n",
    "        precision_bert_cl.append(precision_bert_)\n",
    "        recall_bert_cl.append(recall_bert_)\n",
    "        f1_bert_cl.append(f1_bert_)\n",
    "\n",
    "    precision_cnn.append(precision_cnn_cl)\n",
    "    recall_cnn.append(recall_cnn_cl)\n",
    "    f1_cnn.append(f1_cnn_cl)\n",
    "    \n",
    "    precision_bert.append(precision_bert_cl)\n",
    "    recall_bert.append(recall_bert_cl)\n",
    "    f1_bert.append(f1_bert_cl)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "%matplotlib inline\n",
    "import matplotlib.pyplot as plt\n",
    "\n",
    "classes = [ 'opinion/argument', 'reason/factoid', 'context/aspect', 'preference', 'direct', 'super', 'method']\n",
    "titles = [ 'Opinion/argument', 'Reason/factoid', 'Context/aspect & Method', 'Preference', 'Direct', 'Super']\n",
    "\n",
    "fig, axs = plt.subplots(2,3, figsize=(23, 9))\n",
    "fig.subplots_adjust(hspace = .4, wspace=.001)\n",
    "font = {'size' : 17}\n",
    "plt.rc('font', **font)\n",
    "\n",
    "major_ticks = np.arange(0, 1.01, 0.1)\n",
    "minor_ticks = np.arange(0, 1.01, 0.05)\n",
    "\n",
    "axs = axs.ravel()\n",
    "\n",
    "for i in range(6):\n",
    "    if i == 2:\n",
    "        axs[2].plot(recall_bert[2], \n",
    "                    precision_bert[2], \n",
    "                    marker='o', label='BERT (context/aspect)', linestyle='dashed', \n",
    "                    linewidth=2, markersize=10, markevery = 0.05)\n",
    "\n",
    "        axs[2].plot(recall_cnn[2],\n",
    "                    precision_cnn[2], \n",
    "                    marker='d', label='CNN (context/aspect)', linestyle='dashed', \n",
    "                    linewidth=2, markersize=10, markevery = 0.05)\n",
    "        \n",
    "        axs[2].plot(recall_bert[6], \n",
    "                    precision_bert[6], \n",
    "                    marker='o', label='BERT (method)', linestyle='dashed', \n",
    "                    linewidth=2, markersize=10, markevery = 0.05)\n",
    "\n",
    "        axs[2].plot(recall_cnn[6],\n",
    "                    precision_cnn[6], \n",
    "                    marker='d', label='CNN (method)', linestyle='dashed', \n",
    "                    linewidth=2, markersize=10, markevery = 0.05)\n",
    "        \n",
    "    else:\n",
    "        axs[i].plot(recall_bert[i], \n",
    "                    precision_bert[i], \n",
    "                    marker='o', label='BERT', linestyle='dashed', \n",
    "                    linewidth=2, markersize=10, markevery = 0.05)\n",
    "\n",
    "        axs[i].plot(recall_cnn[i],\n",
    "                    precision_cnn[i], \n",
    "                    marker='d', label='CNN', linestyle='dashed', \n",
    "                    linewidth=2, markersize=10, markevery = 0.05)\n",
    "    \n",
    "    \n",
    "    axs[i].set_xticks(major_ticks)\n",
    "    axs[i].set_xticks(minor_ticks, minor=True)\n",
    "    axs[i].set_yticks(major_ticks)\n",
    "    axs[i].set_yticks(minor_ticks, minor=True)\n",
    "    \n",
    "    axs[i].grid(which='minor', alpha=0.2)\n",
    "    axs[i].grid(which='major', alpha=0.5)\n",
    "    axs[i].grid(which='minor', alpha=0.2)\n",
    "    axs[i].grid(which='major', alpha=0.5)\n",
    "    \n",
    "    axs[i].set_xlim([0.41,1.05])\n",
    "    axs[i].set_ylim([0.66,1.02])\n",
    "    axs[i].tick_params(axis='both', which='major', labelsize=16)\n",
    "    \n",
    "    axs[i].set_title(titles[i])\n",
    "    axs[3].set_xlabel('Recall', fontsize = 17)\n",
    "    axs[4].set_xlabel('Recall', fontsize = 17)\n",
    "    axs[5].set_xlabel('Recall', fontsize = 17)\n",
    "    axs[0].set_ylabel('Precision', fontsize = 17)\n",
    "    axs[3].set_ylabel('Precision', fontsize = 17)\n",
    "\n",
    "    axs[i].legend(loc=3, ncol=1)\n",
    "    axs[i].legend(loc=3, ncol=1)\n",
    "    axs[2].legend(loc=1, ncol=1)\n",
    "    axs[1].legend(loc=3, ncol=1)\n",
    "    \n",
    "plt.show()"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.8"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}