w3 - task_Metrics.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 204,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "import sklearn.metrics as metrics"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 217,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# Загрузите файл classification.csv. \n",
    "# В нем записаны истинные классы объектов выборки (колонка true) и ответы некоторого классификатора (колонка pred)."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 313,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "data = pd.read_csv('classification.csv')\n",
    "y_true = data.iloc[:,0]\n",
    "y_pred = data.iloc[:,1]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 206,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>true</th>\n",
       "      <th>pred</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6</th>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7</th>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8</th>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9</th>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   true  pred\n",
       "0     1     0\n",
       "1     1     1\n",
       "2     1     1\n",
       "3     0     0\n",
       "4     1     1\n",
       "5     1     1\n",
       "6     1     1\n",
       "7     0     1\n",
       "8     0     0\n",
       "9     0     1"
      ]
     },
     "execution_count": 206,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "data.head(10)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 208,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# 2. Заполните таблицу ошибок классификации. Для этого подсчитайте величины TP, FP, FN и TN согласно их определениям.\n",
    "# Например, FP — это количество объектов, имеющих класс 0, но отнесенных алгоритмом к классу 1.\n",
    "# Ответ в данном вопросе — четыре числа через пробел."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 238,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def write_answer_1(a1):\n",
    "    with open(\"ans1.txt\", \"w\") as fout:\n",
    "        fout.write(str(a1))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 318,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "clf_table = {'tp': (1, 1), 'fp': (0, 1), 'fn': (1, 0), 'tn': (0, 0)}\n",
    "for name, res in clf_table.items():\n",
    "    clf_table[name] = len(data[(y_true == res[0]) & (y_pred == res[1])])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 319,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "dict_items([('fn', 59), ('fp', 34), ('tn', 64), ('tp', 43)])"
      ]
     },
     "execution_count": 319,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "clf_table.items()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 239,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "write_answer_1('{tp} {fp} {fn} {tn}'.format(**clf_table))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 240,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "#3. Посчитайте основные метрики качества классификатора:\n",
    "# Accuracy (доля верно угаданных) — sklearn.metrics.accuracy_score\n",
    "# Precision (точность) — sklearn.metrics.precision_score\n",
    "# Recall (полнота) — sklearn.metrics.recall_score\n",
    "# F-мера — sklearn.metrics.f1_score\n",
    "# В качестве ответа укажите эти четыре числа через пробел."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 242,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "def write_answer_2(a1,a2,a3,a4):\n",
    "    with open(\"ans2.txt\", \"w\") as fout:\n",
    "        fout.write(str(a1))\n",
    "        fout.write(\" \")\n",
    "        fout.write(str(a2))\n",
    "        fout.write(\" \")\n",
    "        fout.write(str(a3))\n",
    "        fout.write(\" \")\n",
    "        fout.write(str(a4))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 243,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# Accuracy (доля верно угаданных)\n",
    "acc = metrics.accuracy_score(data['true'], data['pred'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 244,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# Precision (точность) \n",
    "pr = metrics.precision_score(data['true'], data['pred'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 245,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# Recall (полнота) \n",
    "rec = metrics.recall_score(data['true'], data['pred'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 246,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# F-мера \n",
    "f1 = metrics.f1_score(data['true'], data['pred'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 247,
   "metadata": {
    "collapsed": false,
    "scrolled": true
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(0.53500000000000003,\n",
       " 0.55844155844155841,\n",
       " 0.42156862745098039,\n",
       " 0.48044692737430167)"
      ]
     },
     "execution_count": 247,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "acc,pr,rec,f1"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 331,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "write_answer_2('{0:.2f}'.format(acc),'{0:.2f}'.format(pr),'{0:.2f}'.format(rec),'{0:.2f}'.format(f1))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 250,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# 4. Имеется четыре обученных классификатора. В файле scores.csv записаны истинные классы \n",
    "# и значения степени принадлежности положительному классу для каждого классификатора на некоторой выборке:\n",
    "# для логистической регрессии — вероятность положительного класса (колонка score_logreg),\n",
    "# для SVM — отступ от разделяющей поверхности (колонка score_svm),\n",
    "# для метрического алгоритма — взвешенная сумма классов соседей (колонка score_knn),\n",
    "# для решающего дерева — доля положительных объектов в листе (колонка score_tree).\n",
    "# Загрузите этот файл."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 307,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "data2 = pandas.read_csv('scores.csv')\n",
    "y2_true = data2.iloc[:,0]\n",
    "y2_pred = data2.iloc[:,1:]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 308,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(200,)"
      ]
     },
     "execution_count": 308,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "y2_true.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 309,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(200, 4)"
      ]
     },
     "execution_count": 309,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "y2_pred.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 252,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>true</th>\n",
       "      <th>score_logreg</th>\n",
       "      <th>score_svm</th>\n",
       "      <th>score_knn</th>\n",
       "      <th>score_tree</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>0</td>\n",
       "      <td>0.683832</td>\n",
       "      <td>0.145976</td>\n",
       "      <td>0.787063</td>\n",
       "      <td>0.500000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>1</td>\n",
       "      <td>0.801966</td>\n",
       "      <td>0.239511</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>0.833333</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>0</td>\n",
       "      <td>0.382315</td>\n",
       "      <td>-0.245701</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>1</td>\n",
       "      <td>0.506797</td>\n",
       "      <td>-0.137058</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.105263</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>1</td>\n",
       "      <td>0.488781</td>\n",
       "      <td>-0.154148</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.105263</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>0</td>\n",
       "      <td>0.108506</td>\n",
       "      <td>-0.790864</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6</th>\n",
       "      <td>0</td>\n",
       "      <td>0.413749</td>\n",
       "      <td>-0.185143</td>\n",
       "      <td>0.283053</td>\n",
       "      <td>0.363636</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7</th>\n",
       "      <td>0</td>\n",
       "      <td>0.961081</td>\n",
       "      <td>0.830973</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>1.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8</th>\n",
       "      <td>0</td>\n",
       "      <td>0.055733</td>\n",
       "      <td>-0.872805</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.100000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9</th>\n",
       "      <td>0</td>\n",
       "      <td>0.168708</td>\n",
       "      <td>-0.518390</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.100000</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   true  score_logreg  score_svm  score_knn  score_tree\n",
       "0     0      0.683832   0.145976   0.787063    0.500000\n",
       "1     1      0.801966   0.239511   1.000000    0.833333\n",
       "2     0      0.382315  -0.245701   0.000000    0.000000\n",
       "3     1      0.506797  -0.137058   0.000000    0.105263\n",
       "4     1      0.488781  -0.154148   0.000000    0.105263\n",
       "5     0      0.108506  -0.790864   0.000000    0.000000\n",
       "6     0      0.413749  -0.185143   0.283053    0.363636\n",
       "7     0      0.961081   0.830973   1.000000    1.000000\n",
       "8     0      0.055733  -0.872805   0.000000    0.100000\n",
       "9     0      0.168708  -0.518390   0.000000    0.100000"
      ]
     },
     "execution_count": 252,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "data2.head(10)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 253,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# 5. Посчитайте площадь под ROC-кривой для каждого классификатора. \n",
    "# Какой классификатор имеет наибольшее значение метрики AUC-ROC (укажите название столбца)? \n",
    "# Воспользуйтесь функцией sklearn.metrics.roc_auc_score."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 257,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def write_answer_3(a1):\n",
    "    with open(\"ans3.txt\", \"w\") as fout:\n",
    "        fout.write(str(a1))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 310,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "scores = {}\n",
    "for clf in data2.columns[1:]:\n",
    "    scores[clf] = metrics.roc_auc_score(y2_true, data2[clf])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 311,
   "metadata": {
    "collapsed": false,
    "scrolled": true
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'score_knn': 0.63515406162464982,\n",
       " 'score_logreg': 0.71918767507002801,\n",
       " 'score_svm': 0.70868347338935567,\n",
       " 'score_tree': 0.69192677070828335}"
      ]
     },
     "execution_count": 311,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "scores"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 325,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "write_answer_3('score_logreg')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 365,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# 6. Какой классификатор достигает наибольшей точности (Precision) при полноте (Recall) не менее 70% ?\n",
    "# Чтобы получить ответ на этот вопрос, найдите все точки precision-recall-кривой \n",
    "# с помощью функции sklearn.metrics.precision_recall_curve. Она возвращает три массива: precision, recall, thresholds. \n",
    "# В них записаны точность и полнота при определенных порогах, указанных в массиве thresholds. \n",
    "# Найдите максимальное значение точности среди тех записей, для которых полнота не меньше, чем 0.7."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 337,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def write_answer_4(a1):\n",
    "    with open(\"ans4.txt\", \"w\") as fout:\n",
    "        fout.write(str(a1))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 363,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "pr_scores = {}\n",
    "for clf in data2.columns[1:]:\n",
    "    pr_curve = metrics.precision_recall_curve(y2_true, data2[clf])\n",
    "    pr_curve_2 = pandas.DataFrame({'precision': pr_curve[0], 'recall': pr_curve[1]})\n",
    "    pr_scores[clf] = pr_curve_2[pr_curve_2['recall'] >= 0.7]['precision'].max()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 367,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(array([ 0.49      ,  0.50273224,  0.50847458,  0.53289474,  0.59848485,\n",
       "         0.65178571,  0.67708333,  0.66292135,  0.67073171,  0.7       ,\n",
       "         0.71794872,  0.75      ,  1.        ]),\n",
       " array([ 1.        ,  0.93877551,  0.91836735,  0.82653061,  0.80612245,\n",
       "         0.74489796,  0.66326531,  0.60204082,  0.56122449,  0.42857143,\n",
       "         0.28571429,  0.18367347,  0.        ]),\n",
       " array([ 0.        ,  0.1       ,  0.10526316,  0.31578947,  0.36363636,\n",
       "         0.4       ,  0.5       ,  0.64285714,  0.7       ,  0.83333333,\n",
       "         0.92307692,  1.        ]))"
      ]
     },
     "execution_count": 367,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# массив precision\n",
    "# массив recall\n",
    "# массив thresholds\n",
    "pr_curve[0:3]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 364,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>precision</th>\n",
       "      <th>recall</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>0.490000</td>\n",
       "      <td>1.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>0.502732</td>\n",
       "      <td>0.938776</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>0.508475</td>\n",
       "      <td>0.918367</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>0.532895</td>\n",
       "      <td>0.826531</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>0.598485</td>\n",
       "      <td>0.806122</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>0.651786</td>\n",
       "      <td>0.744898</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6</th>\n",
       "      <td>0.677083</td>\n",
       "      <td>0.663265</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7</th>\n",
       "      <td>0.662921</td>\n",
       "      <td>0.602041</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8</th>\n",
       "      <td>0.670732</td>\n",
       "      <td>0.561224</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9</th>\n",
       "      <td>0.700000</td>\n",
       "      <td>0.428571</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>10</th>\n",
       "      <td>0.717949</td>\n",
       "      <td>0.285714</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>11</th>\n",
       "      <td>0.750000</td>\n",
       "      <td>0.183673</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>12</th>\n",
       "      <td>1.000000</td>\n",
       "      <td>0.000000</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "    precision    recall\n",
       "0    0.490000  1.000000\n",
       "1    0.502732  0.938776\n",
       "2    0.508475  0.918367\n",
       "3    0.532895  0.826531\n",
       "4    0.598485  0.806122\n",
       "5    0.651786  0.744898\n",
       "6    0.677083  0.663265\n",
       "7    0.662921  0.602041\n",
       "8    0.670732  0.561224\n",
       "9    0.700000  0.428571\n",
       "10   0.717949  0.285714\n",
       "11   0.750000  0.183673\n",
       "12   1.000000  0.000000"
      ]
     },
     "execution_count": 364,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "pr_curve_2"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 323,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'score_knn': 0.60655737704918034,\n",
       " 'score_logreg': 0.63025210084033612,\n",
       " 'score_svm': 0.6228070175438597,\n",
       " 'score_tree': 0.6517857142857143}"
      ]
     },
     "execution_count": 323,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    " pr_scores"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 338,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "write_answer_4('score_tree')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "anaconda-cloud": {},
  "kernelspec": {
   "display_name": "Python [Root]",
   "language": "python",
   "name": "Python [Root]"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.5.2"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 0
}