04-evaluating-models.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "We compare the outputs of the models using the metrics in notebook 2. "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 35,
   "metadata": {},
   "outputs": [],
   "source": [
    "## load in the results and the truth: \n",
    "\n",
    "\n",
    "import pandas as pd\n",
    "import numpy as np\n",
    "test = pd.read_csv(\"data/test.txt\", header=None, sep=\" \")\n",
    "y_test = test[0]\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 36,
   "metadata": {},
   "outputs": [],
   "source": [
    "pred_lm6 = pd.read_csv(\"data/pred_lm6.txt\", header=None, sep=\" \")[1]\n",
    "pred_lm2 = pd.read_csv(\"data/pred_lm2.txt\", header=None, sep=\" \")[1]\n",
    "pred_lr6 = pd.read_csv(\"data/pred_lr6.txt\", header=None, sep=\" \")[1]\n",
    "pred_lr2 = pd.read_csv(\"data/pred_lr2.txt\", header=None, sep=\" \")[1]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 37,
   "metadata": {},
   "outputs": [],
   "source": [
    "def bubble_sort(vals_list):\n",
    "    swaps = 0\n",
    "    n=len(vals_list)\n",
    "    sorted = 0\n",
    "    while sorted == 0:\n",
    "        swaps_this_pass = 0\n",
    "        for i in range(0, n-1):\n",
    "\n",
    "            if vals_list[i]>vals_list[i+1]:\n",
    "                vals_list[i], vals_list[i+1] = vals_list[i+1], vals_list[i]\n",
    "                swaps_this_pass = swaps_this_pass + 1\n",
    "\n",
    "        if swaps_this_pass==0:\n",
    "            sorted=1\n",
    "        swaps = swaps + swaps_this_pass        \n",
    "        n = n-1 #ith pass of bubble sort puts the nth value in order.\n",
    "            #so there is no need to consider this. \n",
    "    return(swaps)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 38,
   "metadata": {},
   "outputs": [],
   "source": [
    "def bubble_swaps(group):\n",
    "    truth_by_pred = group.sort_values(by = \"predicted\").truth.tolist()\n",
    "    swaps = bubble_sort(truth_by_pred)\n",
    "    return swaps"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 39,
   "metadata": {},
   "outputs": [],
   "source": [
    "import math\n",
    "\"\"\" this returns 0 if all of the ordered data is undesirable\"\"\"\n",
    "def ndcg_p(ordered_data, p):\n",
    "    \"\"\"normalised discounted cumulative gain\"\"\"\n",
    "    if sum(ordered_data)==0:\n",
    "        return 0\n",
    "    else:\n",
    "        indexloop = range(0, p)\n",
    "        DCG_p = 0\n",
    "        for index in indexloop:\n",
    "            current_ratio=(2**(ordered_data[index])-1)*(math.log((float(index)+2), 2)**(-1))\n",
    "            DCG_p = DCG_p + current_ratio\n",
    "        sorted_data= sorted(ordered_data,reverse=True)\n",
    "        n = len(ordered_data)\n",
    "        indexloop = range(0, n)\n",
    "        iDCG_p = 0\n",
    "        for index in indexloop:\n",
    "            current_ratio=(2**(sorted_data[index])-1)*((math.log((index+2), 2))**(-1))\n",
    "            iDCG_p = iDCG_p + current_ratio\n",
    "        return(DCG_p/iDCG_p)\n",
    "    \n",
    "    \n",
    "def compute_at_p(ordered_data, p): \n",
    "    \"\"\"\n",
    "    returns -1 if number of items in list is less than p\n",
    "    else returns ndcg@p\n",
    "    \"\"\"\n",
    "    n = len(ordered_data)\n",
    "    if n<p:\n",
    "        result = np.nan\n",
    "    else:\n",
    "        result = ndcg_p(ordered_data, p)\n",
    "    \n",
    "    return result\n",
    "\n",
    "\n",
    "def sort_ndcg(unordered_group, p):\n",
    "    ordered_group = unordered_group.sort_values(by = \"predicted\").truth.tolist()\n",
    "    return compute_at_p(ordered_group, p)\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 40,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0.7349503461764648\n",
      "0.6996125253331467\n",
      "0.6611331099722424\n",
      "0.6856596521689255\n"
     ]
    }
   ],
   "source": [
    "from sklearn.metrics import mean_squared_error\n",
    "\n",
    "print(mean_squared_error(pred_lm6, y_test))\n",
    "print(mean_squared_error(pred_lm2, y_test))\n",
    "print(mean_squared_error(pred_lr6, y_test))\n",
    "print(mean_squared_error(pred_lr2, y_test))\n",
    "\n",
    "\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 41,
   "metadata": {},
   "outputs": [],
   "source": [
    "preds_df_lm6 = pd.DataFrame({'qid': test[1], 'truth' :y_test, 'predicted' : pred_lm6 })"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 42,
   "metadata": {},
   "outputs": [],
   "source": [
    "groups_lm6 = preds_df_lm6.groupby('qid')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 43,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<pandas.core.groupby.generic.DataFrameGroupBy object at 0x129624f98>"
      ]
     },
     "execution_count": 43,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "groups_lm6"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 44,
   "metadata": {},
   "outputs": [],
   "source": [
    "pairwise_lm6 = groups_lm6.apply(bubble_swaps)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 45,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "2190.582"
      ]
     },
     "execution_count": 45,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "np.mean(pairwise_lm6)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 46,
   "metadata": {},
   "outputs": [],
   "source": [
    "ndcg_lm6_100 = groups_lm6.apply(sort_ndcg, p=100)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 47,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.3536186971513519"
      ]
     },
     "execution_count": 47,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "np.nanmean(ndcg_lm6_100)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 48,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "2350.1675\n",
      "0.38323430363485084\n"
     ]
    }
   ],
   "source": [
    "preds_df_lm2 = pd.DataFrame({'qid': test[1], 'truth' :y_test, 'predicted' : pred_lm2 })\n",
    "groups_lm2 = preds_df_lm2.groupby('qid')\n",
    "pairwise_lm2 = groups_lm2.apply(bubble_swaps)\n",
    "print(np.mean(pairwise_lm2))\n",
    "ndcg_lm2_100 = groups_lm2.apply(sort_ndcg, p=100)\n",
    "print(np.nanmean(ndcg_lm2_100))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 53,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0.7116672567519972\n"
     ]
    }
   ],
   "source": [
    "ndcg_lm2_500 = groups_lm2.apply(sort_ndcg, p=500)\n",
    "print(np.nanmean(ndcg_lm2_500))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 52,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0.707635785366559\n"
     ]
    }
   ],
   "source": [
    "ndcg_lm6_500 = groups_lm6.apply(sort_ndcg, p=500)\n",
    "print(np.nanmean(ndcg_lm6_500))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.3"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}