notebooks/rule_based_classification.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Define helper functions"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import json\n",
    "import subprocess\n",
    "import string\n",
    "import re\n",
    "from tqdm import tqdm_notebook as tqdm\n",
    "import pandas as pd\n",
    "from os.path import join\n",
    "from sklearn.metrics import classification_report\n",
    "import matplotlib.pyplot as plt\n",
    "\n",
    "\n",
    "class Mystem(object):\n",
    "    \"\"\"\n",
    "    Integrates mystem results to do POS-tagging. The executable mystem should be placed in the same directory or\n",
    "    change the input path in the line specified\n",
    "    \"\"\"\n",
    "    \n",
    "    def __init__(self):\n",
    "        self._proc = None\n",
    "\n",
    "    def _start(self):\n",
    "        self._proc = subprocess.Popen(\n",
    "           \"./mystem --format json -cgi --eng-gr\".split(), # here you can specify location of mystem. Leave unchange if mystem is in the same directory as this code\n",
    "           stdin=subprocess.PIPE, stdout=subprocess.PIPE)    \n",
    "\n",
    "    def _getProc(self):\n",
    "        if self._proc is None:\n",
    "            self._start()\n",
    "        return self._proc\n",
    "    \n",
    "    def process(self, text):\n",
    "        p = self._getProc()\n",
    "        p.stdin.write(text.strip().encode('utf8'))\n",
    "        p.stdin.write('\\n'.encode('utf8'))\n",
    "        p.stdin.flush()\n",
    "        return json.loads(p.stdout.readline().decode('utf8'))\n",
    "    \n",
    "mst = Mystem()\n",
    "\n",
    "def strip_punct(s):\n",
    "    \"\"\"\n",
    "    Removes punctuation\n",
    "    \"\"\"\n",
    "    s = re.sub('[^А-Яа-яЁёA-Za-z0-9]', ' ', s.lower())\n",
    "    return \" \".join(s.split())\n",
    "\n",
    "def pos(l):\n",
    "    \"\"\"\n",
    "    Processes output json results from mystem and returns POS tags.\n",
    "    In case a token isn't in Russian, mystem returns an empty field, which is replaced by the tag 'latin'.\n",
    "    Returns tokens (words) followed by the POS tag. Some POS tags are a single-charachter. When deciding to remove a single charachter tokens,\n",
    "    consider doing so before POS-tagging.\n",
    "    \"\"\"\n",
    "    dd=[]\n",
    "    for d1 in l:\n",
    "        l2 = d1.get('analysis', [])\n",
    "        l3 = d1.get('text', [])\n",
    "        if l2 != []:\n",
    "            dd.append(l3)\n",
    "            grammems = []\n",
    "            for d2 in l2:\n",
    "                if 'gr' in d2:\n",
    "                    grammems.append(d2['gr'])\n",
    "                    grammems_str = ' '.join(grammems).lower()\n",
    "            pos = ''\n",
    "            if re.search('comp', grammems_str): pos = 'comp'\n",
    "            elif re.search('supr', grammems_str): pos = 'supr'\n",
    "            else: pos = grammems[0].split('=')[0].split(',')[0]\n",
    "            dd.append(pos)\n",
    "        elif 'analysis' in d1:  \n",
    "            dd.append(l3)\n",
    "            dd.append('latin')\n",
    "    return ' '.join(dd).lower()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "!chmod +x mystem # could need changing the mode"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Read data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "dataset = 'train.txt' # specify the name of the input file, which is in a format: question\\tlabel\\n\n",
    "data = pd.read_csv(join('path', dataset), sep='\\t') # specify the path to the input file\n",
    "\n",
    "questions_original = np.array([question for question in data['question'].tolist()])\n",
    "questions = np.array([pos(mst.process(strip_punct(question))) for question in data['question'].tolist()]) # apply mystem after punctuation removal\n",
    "labels = np.array(data.comp.tolist())"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Rule-based classification"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# calculates and prints classification report for each rule independently\n",
    "# the 15 rules are ordered as they occur in the publication\n",
    "\n",
    "precision_pb = []\n",
    "recall_pb = []\n",
    "\n",
    "patterns = ['\\t\\tif (re.search(\\'луч.{0,1}ше \\', question) and not re.search(\\'как \\', question)): predictions.append(1)\\n',\n",
    "           '\\t\\tif (re.search(\\'comp\\', question) and re.search(\\' или | vs | vs. | versus \\', question) and question.find(\\'comp\\') < question.find(\\' или \\') and not re.search(\\'более comp или conj менее comp\\', question)): predictions.append(1)\\n',\n",
    "           '\\t\\tif (re.search(\\'как\\', question) and re.search(\\'правильно\\', question) and re.search(\\' или \\', question)) or (re.search(\\'как\\', question) and re.search(\\'пишется | писать | написать\\', question) and re.search(\\' или \\', question)): predictions.append(1)\\n',\n",
    "           '\\t\\tif (re.search(\\'что \\', question) and re.search(\\'общего | сходст| схож\\', question) and re.search(\\' и | от | или | между | vs | vs. | versus \\', question)): predictions.append(1)\\n',\n",
    "           '\\t\\tif (re.search(\\'выб+рать|купить|взять\\', question) and re.search(\\' или | между | vs | vs. | versus \\', question)): predictions.append(1)\\n',\n",
    "           '\\t\\tif ((re.search(\\' в \\', question)) and re.search(\\' сравнении \\', question)) or ((re.search(\\' по \\', question)) and re.search(\\' сравнению \\', question)): predictions.append(1)\\n',\n",
    "           '\\t\\tif (re.search(\\'преимуществ|недостаток\\', question) and re.search(\\'перед | над | сравнен | vs | vs. | versus\\', question)): predictions.append(1)\\n',\n",
    "           '\\t\\tif (re.search(\\' отлич| разница | различ\\', question) and re.search(\\' и | от | или | между | vs | vs. | versus\\', question) and not re.search(\\'что \\', question)): predictions.append(1)\\n',\n",
    "           '\\t\\tif (re.search(\\'луч.{0,1}ше \\', question)): predictions.append(1)\\n',\n",
    "           '\\t\\tif (re.search(\\'comp\\', question) and re.search(\\'как.{2,2}\\', question) and not re.search(\\' или | vs | vs. | versus \\', question) and not re.search(\\'как \\', question)): predictions.append(1)\\n',\n",
    "           '\\t\\tif (re.search(\\' или \\', question)): predictions.append(1)\\n',\n",
    "           '\\t\\tif (re.search(\\'comp\\', question)): predictions.append(1)\\n',\n",
    "           '\\t\\tif (re.search(\\'comp\\', question) and re.search(\\'как.{2,2}\\', question) and not re.search(\\' или | vs | vs. | versus \\', question)): predictions.append(1)\\n',\n",
    "           '\\t\\tif (re.search(\\'supr\\', question)): predictions.append(1)\\n',\n",
    "           '\\t\\tif (re.search(\\' плюс\\', question) and re.search(\\' минус\\', question)): predictions.append(1)\\n']\n",
    "\n",
    "func1 = 'def predict_str(corpus):\\n\\tpredictions = []\\n\\tfor question in corpus:\\n'\n",
    "func2 = '\\t\\telse: predictions.append(0)\\n\\r\\treturn predictions'\n",
    "\n",
    "for pattern in patterns:\n",
    "    func = func1 + pattern + func2\n",
    "    exec(func)\n",
    "    predictions = predict_str(questions)\n",
    "    print(pattern)\n",
    "    print(classification_report(y_true=labels, y_pred=predictions, digits=4))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# plots a precision-recall curve for a comparative question class (labeled with 1 in a dataset)\n",
    "\n",
    "precision_pb = []\n",
    "recall_pb = []\n",
    "patterns = ['\\t\\tif (re.search(\\'луч.{0,1}ше \\', question) and not re.search(\\'как \\', question)): predictions.append(1)\\n',\n",
    "           '\\t\\telif (re.search(\\'comp\\', question) and re.search(\\' или | vs | vs. \\', question) and question.find(\\'comp\\') < question.find(\\' или \\') and not re.search(\\'более comp или conj менее comp\\', question)): predictions.append(1)\\n',\n",
    "           '\\t\\telif (re.search(\\'как\\', question) and re.search(\\'правильно\\', question) and re.search(\\' или \\', question)) or (re.search(\\'как\\', question) and re.search(\\'пишется | писать | написать\\', question) and re.search(\\' или \\', question)): predictions.append(1)\\n',\n",
    "           '\\t\\telif (re.search(\\'что \\', question) and re.search(\\'общего | сходст| схож\\', question) and re.search(\\' и | от | или | между | vs | vs. | versus \\', question)): predictions.append(1)\\n',\n",
    "           '\\t\\telif (re.search(\\'выб+рать|купить|взять\\', question) and re.search(\\' или | между | vs | vs. | versus \\', question)): predictions.append(1)\\n',\n",
    "           '\\t\\telif ((re.search(\\' в \\', question)) and re.search(\\' сравнении \\', question)) or ((re.search(\\' по \\', question)) and re.search(\\' сравнению \\', question)): predictions.append(1)\\n',\n",
    "           '\\t\\telif (re.search(\\'преимуществ|недостаток\\', question) and re.search(\\'перед | над | сравнен | vs | vs. | versus\\', question)): predictions.append(1)\\n',\n",
    "           '\\t\\telif (re.search(\\' отлич| разница | различ\\', question) and re.search(\\' и | от | или | между | vs | vs. | versus\\', question) and not re.search(\\'что \\', question)): predictions.append(1)\\n',\n",
    "           '\\t\\telif (re.search(\\'луч.{0,1}ше \\', question)): predictions.append(1)\\n',\n",
    "           '\\t\\telif (re.search(\\'comp\\', question) and re.search(\\'как.{2,2}\\', question) and not re.search(\\' или | vs | vs. | versus \\', question) and not re.search(\\'как \\', question)): predictions.append(1)\\n',\n",
    "           '\\t\\telif (re.search(\\' или \\', question)): predictions.append(1)\\n',\n",
    "           '\\t\\telif (re.search(\\'comp\\', question)): predictions.append(1)\\n',\n",
    "           '\\t\\telif (re.search(\\'comp\\', question) and re.search(r\\'как.{2,2}\\', question) and not re.search(\\' или | vs | vs. | versus \\', question)): predictions.append(1)\\n',\n",
    "           '\\t\\telif (re.search(\\'supr\\', question)): predictions.append(1)\\n',\n",
    "           '\\t\\telif (re.search(\\' плюс\\', question) and re.search(\\' минус\\', question)): predictions.append(1)\\n']\n",
    "\n",
    "func1 = 'def predict_str(corpus):\\n\\tpredictions = []\\n\\tfor question in corpus:\\n'\n",
    "func2 = '\\t\\telse: predictions.append(0)\\n\\r\\treturn predictions'\n",
    "for pattern in patterns:\n",
    "    func = func1 + pattern + func2\n",
    "    func1 += pattern\n",
    "    exec(func)\n",
    "    predictions = predict_str(questions)\n",
    "    precision = classification_report(y_true=labels, y_pred=predictions, output_dict=True)['1']['precision'] # collects precision for a comparative question class '1'\n",
    "    recall = classification_report(y_true=labels, y_pred=predictions, output_dict=True)['1']['recall'] # collects recall for a comparative question class '1'\n",
    "    precision_pb.append(round(precision,4))\n",
    "    recall_pb.append(round(recall,4))\n",
    "    \n",
    "\n",
    "# Save precision-recall results for rules\n",
    "\n",
    "pb_df = pd.DataFrame({'precision_pb': precision_pb, 'recall_pb': recall_pb})\n",
    "# pb_df.to_csv('../prrecall-pb.txt', sep='\\t', index=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "%matplotlib inline\n",
    "\n",
    "font = {'size' : 18}\n",
    "plt.rc('font', **font)\n",
    "\n",
    "fig = plt.figure(figsize=(20,10))\n",
    "ax = fig.add_subplot(1, 1, 1)\n",
    "\n",
    "major_ticks = np.arange(0, 1.01, 0.05)\n",
    "minor_ticks = np.arange(0, 1.01, 0.01)\n",
    "\n",
    "ax.set_xticks(major_ticks)\n",
    "ax.set_xticks(minor_ticks, minor=True)\n",
    "ax.set_yticks(major_ticks)\n",
    "ax.set_yticks(minor_ticks, minor=True)\n",
    "axes = plt.gca()\n",
    "axes.set_xlim([0.0,1.02])\n",
    "axes.set_ylim([0.55,1.02])\n",
    "\n",
    "ax.grid(which='both')\n",
    "\n",
    "\n",
    "ax.grid(which='minor', alpha=0.2)\n",
    "ax.grid(which='major', alpha=0.5)\n",
    "\n",
    "ax.plot(recall_pb, precision_pb, marker='^', label='Pattern-based', linestyle='dashed', \n",
    "        linewidth=2, markersize=12)\n",
    "\n",
    "plt.xlabel('Recall', fontsize=22)\n",
    "plt.ylabel('Precision', fontsize=22)\n",
    "plt.legend(bbox_to_anchor=(0., 1.02, 1., .102), loc=3,\n",
    "           ncol=3, mode=\"expand\", borderaxespad=0.)\n",
    "\n",
    "plt.show()"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.8"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}