w4 - task_LinReg_Salary.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "import numpy as np\n",
    "from sklearn.feature_extraction.text import TfidfVectorizer\n",
    "from sklearn.feature_extraction import DictVectorizer\n",
    "from sklearn.linear_model import Ridge\n",
    "from scipy.sparse import hstack"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# 1. Загрузите данные об описаниях вакансий и соответствующих годовых зарплатах из файла salary-train.csv."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 42,
   "metadata": {},
   "outputs": [],
   "source": [
    "data = pd.read_csv('salary-train.csv')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "data_train = pd.read_csv('salary-train.csv')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 40,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "data_test = pd.read_csv('salary-test-mini.csv')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>FullDescription</th>\n",
       "      <th>LocationNormalized</th>\n",
       "      <th>ContractTime</th>\n",
       "      <th>SalaryNormalized</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>International Sales Manager London ****k  ****...</td>\n",
       "      <td>London</td>\n",
       "      <td>permanent</td>\n",
       "      <td>33000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>An ideal opportunity for an individual that ha...</td>\n",
       "      <td>London</td>\n",
       "      <td>permanent</td>\n",
       "      <td>50000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>Online Content and Brand Manager// Luxury Reta...</td>\n",
       "      <td>South East London</td>\n",
       "      <td>permanent</td>\n",
       "      <td>40000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>A great local marketleader is seeking a perman...</td>\n",
       "      <td>Dereham</td>\n",
       "      <td>permanent</td>\n",
       "      <td>22500</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>Registered Nurse / RGN  Nursing Home for Young...</td>\n",
       "      <td>Sutton Coldfield</td>\n",
       "      <td>NaN</td>\n",
       "      <td>20355</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                                     FullDescription LocationNormalized  \\\n",
       "0  International Sales Manager London ****k  ****...             London   \n",
       "1  An ideal opportunity for an individual that ha...             London   \n",
       "2  Online Content and Brand Manager// Luxury Reta...  South East London   \n",
       "3  A great local marketleader is seeking a perman...            Dereham   \n",
       "4  Registered Nurse / RGN  Nursing Home for Young...   Sutton Coldfield   \n",
       "\n",
       "  ContractTime  SalaryNormalized  \n",
       "0    permanent             33000  \n",
       "1    permanent             50000  \n",
       "2    permanent             40000  \n",
       "3    permanent             22500  \n",
       "4          NaN             20355  "
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "data.head(5)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# 2. Проведите предобработку:\n",
    "# - Приведите тексты к нижнему регистру (text.lower())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 48,
   "metadata": {},
   "outputs": [],
   "source": [
    "data_train.apply(lambda x: x.astype(str).str.lower())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 49,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0    international sales manager london ****k  ****...\n",
       "1    an ideal opportunity for an individual that ha...\n",
       "2    online content and brand manager// luxury reta...\n",
       "3    a great local marketleader is seeking a perman...\n",
       "4    registered nurse / rgn  nursing home for young...\n",
       "5    sales and marketing assistant will provide adm...\n",
       "6    vacancy ladieswear fashion area manager / regi...\n",
       "7    reference: lr/jan/**** our client is one of th...\n",
       "8    sponsorship manager london the company a marke...\n",
       "9    about barclays barclays moves, lends, invests ...\n",
       "Name: FullDescription, dtype: object"
      ]
     },
     "execution_count": 49,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "data_train['FullDescription'][:10]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# Замените все, кроме букв и цифр, на пробелы — это облегчит дальнейшее разделение текста на слова. \n",
    "# Для такой замены в строке text подходит следующий вызов: re.sub('[^a-zA-Z0-9]', ' ', text). \n",
    "# Также можно воспользоваться методом replace у DataFrame, чтобы сразу преобразовать все тексты:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 50,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "data_train['FullDescription'].replace('[^a-zA-Z0-9]', ' ', regex=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 51,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0    international sales manager london     k      ...\n",
       "1    an ideal opportunity for an individual that ha...\n",
       "2    online content and brand manager   luxury reta...\n",
       "3    a great local marketleader is seeking a perman...\n",
       "4    registered nurse   rgn  nursing home for young...\n",
       "5    sales and marketing assistant will provide adm...\n",
       "6    vacancy ladieswear fashion area manager   regi...\n",
       "7    reference  lr jan      our client is one of th...\n",
       "8    sponsorship manager london the company a marke...\n",
       "9    about barclays barclays moves  lends  invests ...\n",
       "Name: FullDescription, dtype: object"
      ]
     },
     "execution_count": 51,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "data_train['FullDescription'][:10]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 39,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# Примените TfidfVectorizer для преобразования текстов в векторы признаков. \n",
    "# Оставьте только те слова, которые встречаются хотя бы в 5 объектах (параметр min_df у TfidfVectorizer)."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 68,
   "metadata": {},
   "outputs": [],
   "source": [
    "vec = TfidfVectorizer(min_df=5)\n",
    "x_train_transform = vec.fit_transform(data_train['FullDescription'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 52,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# Замените пропуски в столбцах LocationNormalized и ContractTime на специальную строку 'nan'. "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 64,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "data_train['LocationNormalized'].fillna('nan', inplace=True)\n",
    "data_train['ContractTime'].fillna('nan', inplace=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 66,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>FullDescription</th>\n",
       "      <th>LocationNormalized</th>\n",
       "      <th>ContractTime</th>\n",
       "      <th>SalaryNormalized</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>international sales manager london     k      ...</td>\n",
       "      <td>london</td>\n",
       "      <td>permanent</td>\n",
       "      <td>33000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>an ideal opportunity for an individual that ha...</td>\n",
       "      <td>london</td>\n",
       "      <td>permanent</td>\n",
       "      <td>50000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>online content and brand manager   luxury reta...</td>\n",
       "      <td>south east london</td>\n",
       "      <td>permanent</td>\n",
       "      <td>40000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>a great local marketleader is seeking a perman...</td>\n",
       "      <td>dereham</td>\n",
       "      <td>permanent</td>\n",
       "      <td>22500</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>registered nurse   rgn  nursing home for young...</td>\n",
       "      <td>sutton coldfield</td>\n",
       "      <td>nan</td>\n",
       "      <td>20355</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>sales and marketing assistant will provide adm...</td>\n",
       "      <td>crawley</td>\n",
       "      <td>nan</td>\n",
       "      <td>22500</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6</th>\n",
       "      <td>vacancy ladieswear fashion area manager   regi...</td>\n",
       "      <td>uk</td>\n",
       "      <td>permanent</td>\n",
       "      <td>32000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7</th>\n",
       "      <td>reference  lr jan      our client is one of th...</td>\n",
       "      <td>bristol</td>\n",
       "      <td>permanent</td>\n",
       "      <td>30000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8</th>\n",
       "      <td>sponsorship manager london the company a marke...</td>\n",
       "      <td>central london</td>\n",
       "      <td>permanent</td>\n",
       "      <td>31500</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9</th>\n",
       "      <td>about barclays barclays moves  lends  invests ...</td>\n",
       "      <td>south east london</td>\n",
       "      <td>permanent</td>\n",
       "      <td>42499</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                                     FullDescription LocationNormalized  \\\n",
       "0  international sales manager london     k      ...             london   \n",
       "1  an ideal opportunity for an individual that ha...             london   \n",
       "2  online content and brand manager   luxury reta...  south east london   \n",
       "3  a great local marketleader is seeking a perman...            dereham   \n",
       "4  registered nurse   rgn  nursing home for young...   sutton coldfield   \n",
       "5  sales and marketing assistant will provide adm...            crawley   \n",
       "6  vacancy ladieswear fashion area manager   regi...                 uk   \n",
       "7  reference  lr jan      our client is one of th...            bristol   \n",
       "8  sponsorship manager london the company a marke...     central london   \n",
       "9  about barclays barclays moves  lends  invests ...  south east london   \n",
       "\n",
       "  ContractTime SalaryNormalized  \n",
       "0    permanent            33000  \n",
       "1    permanent            50000  \n",
       "2    permanent            40000  \n",
       "3    permanent            22500  \n",
       "4          nan            20355  \n",
       "5          nan            22500  \n",
       "6    permanent            32000  \n",
       "7    permanent            30000  \n",
       "8    permanent            31500  \n",
       "9    permanent            42499  "
      ]
     },
     "execution_count": 66,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "data_train.head(10)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 67,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# Примените DictVectorizer для получения one-hot-кодирования признаков LocationNormalized и ContractTime."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 71,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "enc = DictVectorizer()\n",
    "x_train_categ = enc.fit_transform(data_train[['LocationNormalized', 'ContractTime']].to_dict('records'))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 73,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# Объедините все полученные признаки в одну матрицу \"объекты-признаки\". \n",
    "# Обратите внимание, что матрицы для текстов и категориальных признаков являются разреженными. \n",
    "# Для объединения их столбцов нужно воспользоваться функцией scipy.sparse.hstack."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 75,
   "metadata": {},
   "outputs": [],
   "source": [
    "x_train = hstack([x_train_transform, x_train_categ])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 81,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "y_train = data_train['SalaryNormalized']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 78,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# 3. Обучите гребневую регрессию с параметрами alpha=1 и random_state=241. \n",
    "# Целевая переменная записана в столбце SalaryNormalized."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 82,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "Ridge(alpha=1, copy_X=True, fit_intercept=True, max_iter=None,\n",
       "   normalize=False, random_state=241, solver='auto', tol=0.001)"
      ]
     },
     "execution_count": 82,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "model = Ridge(alpha=1,random_state=241)\n",
    "model.fit(x_train, y_train)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 83,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# 4. Постройте прогнозы для двух примеров из файла salary-test-mini.csv. \n",
    "# Значения полученных прогнозов являются ответом на задание. Укажите их через пробел."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 84,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "x_test_transform = vec.transform(data_test['FullDescription'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 85,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "x_test_categ = enc.transform(data_test[['LocationNormalized', 'ContractTime']].to_dict('records'))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 86,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "x_test = hstack([x_test_transform, x_test_categ])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 95,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "55850.59 37182.51\n"
     ]
    }
   ],
   "source": [
    "y_test = model.predict(x_test)\n",
    "print('{:0.2f} {:0.2f}'.format(y_test[0],y_test[1]))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "anaconda-cloud": {},
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.4"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 1
}