6. correlation_of_traditional_metrics_combined.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "import numpy as np\n",
    "from scipy.stats import pearsonr \n",
    "\n",
    "LANG = 'FR'\n",
    "MAIN = f'resulting_data/{LANG}_latin/'\n",
    "MAIN = f'resulting_data/{LANG}/'\n",
    "\n",
    "path_trad_BERT = MAIN+'BERT_traditional.csv'\n",
    "path_trad_PG = MAIN+'PG_traditional.csv'\n",
    "\n",
    "# BERT dan PG\n",
    "path_human_focus = f'mturk/annotation_result/{LANG}/human_focus_final.csv'\n",
    "path_human_coverage = f'mturk/annotation_result/{LANG}/human_coverage_final.csv'\n",
    "\n",
    "def eval (h, c):\n",
    "    for col in c.columns[1:]:\n",
    "        score = pearsonr(h, c[col])[0]\n",
    "        print(col+','+str(score))\n",
    "    print()\n",
    "\n",
    "trad_PG = pd.read_csv(path_trad_PG)\n",
    "trad_BERT = pd.read_csv(path_trad_BERT)\n",
    "\n",
    "human_foc = pd.read_csv(path_human_focus)\n",
    "human_cov = pd.read_csv(path_human_coverage)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Focus resulting_data/FR/ 270\n",
      "ROUGE-1-F,0.6874740321113832\n",
      "ROUGE-2-F,0.6799190552046659\n",
      "ROUGE-3-F,0.6081305118686484\n",
      "ROUGE-L-F,0.6882174617801713\n",
      "ROUGE-S-F,0.6447637391989007\n",
      "ROUGE-SU-F,0.6577396219950404\n",
      "ROUGE-W-F,0.6709771145427735\n",
      "ROUGE-1-P,0.6804011075928594\n",
      "ROUGE-2-P,0.671098802366808\n",
      "ROUGE-3-P,0.5927664054203179\n",
      "ROUGE-L-P,0.6821431674619156\n",
      "ROUGE-S-P,0.5976985048016697\n",
      "ROUGE-SU-P,0.6101079486795062\n",
      "ROUGE-W-P,0.6739934069178942\n",
      "ROUGE-1-R,0.5752843977354121\n",
      "ROUGE-2-R,0.6364710433544669\n",
      "ROUGE-3-R,0.5759852674908228\n",
      "ROUGE-L-R,0.5904697139625815\n",
      "ROUGE-S-R,0.5293587230757817\n",
      "ROUGE-SU-R,0.5355784414843315\n",
      "ROUGE-W-R,0.5896057127024138\n",
      "METEOR,0.6326720975942426\n",
      "BLEU-1,0.6444508742156065\n",
      "BLEU-2,0.6995377008145883\n",
      "BLEU-3,0.6762749749673049\n",
      "BLEU-4,0.6442807614861038\n",
      "\n",
      "Coverage resulting_data/FR/ 270\n",
      "ROUGE-1-F,0.6616479227194398\n",
      "ROUGE-2-F,0.6728076118403017\n",
      "ROUGE-3-F,0.6003201679317891\n",
      "ROUGE-L-F,0.6633003415034971\n",
      "ROUGE-S-F,0.6279715138126616\n",
      "ROUGE-SU-F,0.6391804341744104\n",
      "ROUGE-W-F,0.6697219246504104\n",
      "ROUGE-1-P,0.5815632131244921\n",
      "ROUGE-2-P,0.611022956069986\n",
      "ROUGE-3-P,0.5422204526218455\n",
      "ROUGE-L-P,0.5872279336732806\n",
      "ROUGE-S-P,0.5099351438487592\n",
      "ROUGE-SU-P,0.5181828969801308\n",
      "ROUGE-W-P,0.575746293633218\n",
      "ROUGE-1-R,0.6650469666495125\n",
      "ROUGE-2-R,0.7051712488954263\n",
      "ROUGE-3-R,0.6331502513546159\n",
      "ROUGE-L-R,0.6723553417671411\n",
      "ROUGE-S-R,0.6659365443546006\n",
      "ROUGE-SU-R,0.6746943235516996\n",
      "ROUGE-W-R,0.6684361416399791\n",
      "METEOR,0.6732377622519391\n",
      "BLEU-1,0.5698825260295257\n",
      "BLEU-2,0.6362907427156115\n",
      "BLEU-3,0.6152849325613522\n",
      "BLEU-4,0.5858284162479273\n",
      "\n"
     ]
    }
   ],
   "source": [
    "#focus combined\n",
    "filtered_human = []\n",
    "filtered_trad = pd.DataFrame(columns=trad_PG.columns)\n",
    "for idx, row in human_foc.iterrows():\n",
    "    filtered_human.append(row['score'])\n",
    "    if row['model'] == 'BERT':\n",
    "        filtered_trad = filtered_trad.append(trad_BERT[trad_BERT['id']==row['id']])\n",
    "    elif row['model'] == 'PG':\n",
    "        filtered_trad = filtered_trad.append(trad_PG[trad_PG['id']==row['id']])\n",
    "print('Focus', MAIN, len(filtered_human))\n",
    "eval(filtered_human, filtered_trad)\n",
    "\n",
    "#coveraged combined\n",
    "filtered_human = []\n",
    "filtered_trad = pd.DataFrame(columns=trad_PG.columns)\n",
    "for idx, row in human_cov.iterrows():\n",
    "    filtered_human.append(row['score'])\n",
    "    if row['model'] == 'BERT':\n",
    "        filtered_trad = filtered_trad.append(trad_BERT[trad_BERT['id']==row['id']])\n",
    "    elif row['model'] == 'PG':\n",
    "        filtered_trad = filtered_trad.append(trad_PG[trad_PG['id']==row['id']])\n",
    "print('Coverage', MAIN, len(filtered_human))\n",
    "eval(filtered_human, filtered_trad)"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.6"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}