pandas_tutorial.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# ~ ~ ~ ~ DFW PYTHONEERS~ ~ ~ ~ \n",
    "# Pandas Tutorial\n",
    "### https://pandas.pydata.org/index.html\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<style>body {\n",
       "    margin: 0;\n",
       "    font-family: Helvetica;\n",
       "}\n",
       "table.dataframe {\n",
       "    border-collapse: collapse;\n",
       "    border: none;\n",
       "}\n",
       "table.dataframe tr {\n",
       "    border: none;\n",
       "}\n",
       "table.dataframe td, table.dataframe th {\n",
       "    margin: 0;\n",
       "    border: 1px solid white;\n",
       "    padding-left: 0.25em;\n",
       "    padding-right: 0.25em;\n",
       "}\n",
       "table.dataframe th:not(:empty) {\n",
       "    background-color: #fec;\n",
       "    text-align: left;\n",
       "    font-weight: normal;\n",
       "}\n",
       "table.dataframe tr:nth-child(2) th:empty {\n",
       "    border-left: none;\n",
       "    border-right: 1px dashed #888;\n",
       "}\n",
       "table.dataframe td {\n",
       "    border: 2px solid #ccf;\n",
       "    background-color: #f4f4ff;\n",
       "}\n",
       "h3 {\n",
       "    color: white;\n",
       "    background-color: black;\n",
       "    padding: 0.5em;\n",
       "}\n",
       "</style>"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "execution_count": 1,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "from IPython.core.display import HTML\n",
    "css = open('style-table.css').read() + open('style-notebook.css').read()\n",
    "HTML('<style>{}</style>'.format(css))\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'0.23.4'"
      ]
     },
     "execution_count": 2,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "import pandas as pd\n",
    "pd.__version__"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "import matplotlib.pyplot as plt\n",
    "import seaborn as sns\n",
    "%matplotlib inline"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Dataset: Stanford Open Policing Project\n",
    "\n",
    "# https://openpolicing.stanford.edu/\n",
    "\n",
    "## Metadata: https://github.com/5harad/openpolicing/blob/master/DATA-README.md\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#                                                  TUTORIALS"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Wall time: 449 ms\n"
     ]
    }
   ],
   "source": [
    "%%time\n",
    "# Reading the CSV file in the dataframe df\n",
    "df= pd.read_csv('police.csv')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {
    "collapsed": true
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>stop_date</th>\n",
       "      <th>stop_time</th>\n",
       "      <th>county_name</th>\n",
       "      <th>driver_gender</th>\n",
       "      <th>driver_age_raw</th>\n",
       "      <th>driver_age</th>\n",
       "      <th>driver_race</th>\n",
       "      <th>violation_raw</th>\n",
       "      <th>violation</th>\n",
       "      <th>search_conducted</th>\n",
       "      <th>search_type</th>\n",
       "      <th>stop_outcome</th>\n",
       "      <th>is_arrested</th>\n",
       "      <th>stop_duration</th>\n",
       "      <th>drugs_related_stop</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>1/2/2005</td>\n",
       "      <td>1:55</td>\n",
       "      <td>NaN</td>\n",
       "      <td>M</td>\n",
       "      <td>1985.0</td>\n",
       "      <td>20.0</td>\n",
       "      <td>White</td>\n",
       "      <td>Speeding</td>\n",
       "      <td>Speeding</td>\n",
       "      <td>False</td>\n",
       "      <td>NaN</td>\n",
       "      <td>Citation</td>\n",
       "      <td>False</td>\n",
       "      <td>0-15 Min</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>1/18/2005</td>\n",
       "      <td>8:15</td>\n",
       "      <td>NaN</td>\n",
       "      <td>M</td>\n",
       "      <td>1965.0</td>\n",
       "      <td>40.0</td>\n",
       "      <td>White</td>\n",
       "      <td>Speeding</td>\n",
       "      <td>Speeding</td>\n",
       "      <td>False</td>\n",
       "      <td>NaN</td>\n",
       "      <td>Citation</td>\n",
       "      <td>False</td>\n",
       "      <td>0-15 Min</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>1/23/2005</td>\n",
       "      <td>23:15</td>\n",
       "      <td>NaN</td>\n",
       "      <td>M</td>\n",
       "      <td>1972.0</td>\n",
       "      <td>33.0</td>\n",
       "      <td>White</td>\n",
       "      <td>Speeding</td>\n",
       "      <td>Speeding</td>\n",
       "      <td>False</td>\n",
       "      <td>NaN</td>\n",
       "      <td>Citation</td>\n",
       "      <td>False</td>\n",
       "      <td>0-15 Min</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>2/20/2005</td>\n",
       "      <td>17:15</td>\n",
       "      <td>NaN</td>\n",
       "      <td>M</td>\n",
       "      <td>1986.0</td>\n",
       "      <td>19.0</td>\n",
       "      <td>White</td>\n",
       "      <td>Call for Service</td>\n",
       "      <td>Other</td>\n",
       "      <td>False</td>\n",
       "      <td>NaN</td>\n",
       "      <td>Arrest Driver</td>\n",
       "      <td>True</td>\n",
       "      <td>16-30 Min</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>3/14/2005</td>\n",
       "      <td>10:00</td>\n",
       "      <td>NaN</td>\n",
       "      <td>F</td>\n",
       "      <td>1984.0</td>\n",
       "      <td>21.0</td>\n",
       "      <td>White</td>\n",
       "      <td>Speeding</td>\n",
       "      <td>Speeding</td>\n",
       "      <td>False</td>\n",
       "      <td>NaN</td>\n",
       "      <td>Citation</td>\n",
       "      <td>False</td>\n",
       "      <td>0-15 Min</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   stop_date stop_time  county_name driver_gender  driver_age_raw  driver_age  \\\n",
       "0   1/2/2005      1:55          NaN             M          1985.0        20.0   \n",
       "1  1/18/2005      8:15          NaN             M          1965.0        40.0   \n",
       "2  1/23/2005     23:15          NaN             M          1972.0        33.0   \n",
       "3  2/20/2005     17:15          NaN             M          1986.0        19.0   \n",
       "4  3/14/2005     10:00          NaN             F          1984.0        21.0   \n",
       "\n",
       "  driver_race     violation_raw violation  search_conducted search_type  \\\n",
       "0       White          Speeding  Speeding             False         NaN   \n",
       "1       White          Speeding  Speeding             False         NaN   \n",
       "2       White          Speeding  Speeding             False         NaN   \n",
       "3       White  Call for Service     Other             False         NaN   \n",
       "4       White          Speeding  Speeding             False         NaN   \n",
       "\n",
       "    stop_outcome is_arrested stop_duration  drugs_related_stop  \n",
       "0       Citation       False      0-15 Min               False  \n",
       "1       Citation       False      0-15 Min               False  \n",
       "2       Citation       False      0-15 Min               False  \n",
       "3  Arrest Driver        True     16-30 Min               False  \n",
       "4       Citation       False      0-15 Min               False  "
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df.head()\n",
    "#df.tail()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(91741, 15)"
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df.shape\n",
    "#df.shape[0]\n",
    "#df.shape[1]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "91741"
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Alternatively you can find out the # of records from len function\n",
    "len(df)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {
    "collapsed": true
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "stop_date              object\n",
       "stop_time              object\n",
       "county_name           float64\n",
       "driver_gender          object\n",
       "driver_age_raw        float64\n",
       "driver_age            float64\n",
       "driver_race            object\n",
       "violation_raw          object\n",
       "violation              object\n",
       "search_conducted         bool\n",
       "search_type            object\n",
       "stop_outcome           object\n",
       "is_arrested            object\n",
       "stop_duration          object\n",
       "drugs_related_stop       bool\n",
       "dtype: object"
      ]
     },
     "execution_count": 8,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df.dtypes"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "Index(['stop_date', 'stop_time', 'county_name', 'driver_gender',\n",
       "       'driver_age_raw', 'driver_age', 'driver_race', 'violation_raw',\n",
       "       'violation', 'search_conducted', 'search_type', 'stop_outcome',\n",
       "       'is_arrested', 'stop_duration', 'drugs_related_stop'],\n",
       "      dtype='object')"
      ]
     },
     "execution_count": 9,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df.columns"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Series, Chain & Math operation"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0    20.0\n",
       "1    40.0\n",
       "2    33.0\n",
       "3    19.0\n",
       "4    21.0\n",
       "Name: driver_age, dtype: float64"
      ]
     },
     "execution_count": 10,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Series\n",
    "df['driver_age'].head(5)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {
    "collapsed": true
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>stop_date</th>\n",
       "      <th>stop_time</th>\n",
       "      <th>county_name</th>\n",
       "      <th>driver_gender</th>\n",
       "      <th>driver_age_raw</th>\n",
       "      <th>driver_age</th>\n",
       "      <th>driver_race</th>\n",
       "      <th>violation_raw</th>\n",
       "      <th>violation</th>\n",
       "      <th>search_conducted</th>\n",
       "      <th>search_type</th>\n",
       "      <th>stop_outcome</th>\n",
       "      <th>is_arrested</th>\n",
       "      <th>stop_duration</th>\n",
       "      <th>drugs_related_stop</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>1/2/2005</td>\n",
       "      <td>1:55</td>\n",
       "      <td>NaN</td>\n",
       "      <td>M</td>\n",
       "      <td>1985.0</td>\n",
       "      <td>20.0</td>\n",
       "      <td>White</td>\n",
       "      <td>Speeding</td>\n",
       "      <td>Speeding</td>\n",
       "      <td>False</td>\n",
       "      <td>NaN</td>\n",
       "      <td>Citation</td>\n",
       "      <td>False</td>\n",
       "      <td>0-15 Min</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>1/18/2005</td>\n",
       "      <td>8:15</td>\n",
       "      <td>NaN</td>\n",
       "      <td>M</td>\n",
       "      <td>1965.0</td>\n",
       "      <td>40.0</td>\n",
       "      <td>White</td>\n",
       "      <td>Speeding</td>\n",
       "      <td>Speeding</td>\n",
       "      <td>False</td>\n",
       "      <td>NaN</td>\n",
       "      <td>Citation</td>\n",
       "      <td>False</td>\n",
       "      <td>0-15 Min</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>1/23/2005</td>\n",
       "      <td>23:15</td>\n",
       "      <td>NaN</td>\n",
       "      <td>M</td>\n",
       "      <td>1972.0</td>\n",
       "      <td>33.0</td>\n",
       "      <td>White</td>\n",
       "      <td>Speeding</td>\n",
       "      <td>Speeding</td>\n",
       "      <td>False</td>\n",
       "      <td>NaN</td>\n",
       "      <td>Citation</td>\n",
       "      <td>False</td>\n",
       "      <td>0-15 Min</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>2/20/2005</td>\n",
       "      <td>17:15</td>\n",
       "      <td>NaN</td>\n",
       "      <td>M</td>\n",
       "      <td>1986.0</td>\n",
       "      <td>19.0</td>\n",
       "      <td>White</td>\n",
       "      <td>Call for Service</td>\n",
       "      <td>Other</td>\n",
       "      <td>False</td>\n",
       "      <td>NaN</td>\n",
       "      <td>Arrest Driver</td>\n",
       "      <td>True</td>\n",
       "      <td>16-30 Min</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>3/14/2005</td>\n",
       "      <td>10:00</td>\n",
       "      <td>NaN</td>\n",
       "      <td>F</td>\n",
       "      <td>1984.0</td>\n",
       "      <td>21.0</td>\n",
       "      <td>White</td>\n",
       "      <td>Speeding</td>\n",
       "      <td>Speeding</td>\n",
       "      <td>False</td>\n",
       "      <td>NaN</td>\n",
       "      <td>Citation</td>\n",
       "      <td>False</td>\n",
       "      <td>0-15 Min</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   stop_date stop_time  county_name driver_gender  driver_age_raw  driver_age  \\\n",
       "0   1/2/2005      1:55          NaN             M          1985.0        20.0   \n",
       "1  1/18/2005      8:15          NaN             M          1965.0        40.0   \n",
       "2  1/23/2005     23:15          NaN             M          1972.0        33.0   \n",
       "3  2/20/2005     17:15          NaN             M          1986.0        19.0   \n",
       "4  3/14/2005     10:00          NaN             F          1984.0        21.0   \n",
       "\n",
       "  driver_race     violation_raw violation  search_conducted search_type  \\\n",
       "0       White          Speeding  Speeding             False         NaN   \n",
       "1       White          Speeding  Speeding             False         NaN   \n",
       "2       White          Speeding  Speeding             False         NaN   \n",
       "3       White  Call for Service     Other             False         NaN   \n",
       "4       White          Speeding  Speeding             False         NaN   \n",
       "\n",
       "    stop_outcome is_arrested stop_duration  drugs_related_stop  \n",
       "0       Citation       False      0-15 Min               False  \n",
       "1       Citation       False      0-15 Min               False  \n",
       "2       Citation       False      0-15 Min               False  \n",
       "3  Arrest Driver        True     16-30 Min               False  \n",
       "4       Citation       False      0-15 Min               False  "
      ]
     },
     "execution_count": 11,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "h = df.head()\n",
    "h"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0    20000.0\n",
       "1    40000.0\n",
       "2    33000.0\n",
       "3    19000.0\n",
       "4    21000.0\n",
       "Name: driver_age, dtype: float64"
      ]
     },
     "execution_count": 12,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Performing Mathematical operations\n",
    "h['driver_age'] * 1000\n",
    "#h['driver_age'] //10 "
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Filtering"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {
    "collapsed": true
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0    False\n",
       "1     True\n",
       "2     True\n",
       "3    False\n",
       "4     True\n",
       "Name: driver_age, dtype: bool"
      ]
     },
     "execution_count": 13,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "h.driver_age > 20"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {
    "collapsed": true
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>stop_date</th>\n",
       "      <th>stop_time</th>\n",
       "      <th>county_name</th>\n",
       "      <th>driver_gender</th>\n",
       "      <th>driver_age_raw</th>\n",
       "      <th>driver_age</th>\n",
       "      <th>driver_race</th>\n",
       "      <th>violation_raw</th>\n",
       "      <th>violation</th>\n",
       "      <th>search_conducted</th>\n",
       "      <th>search_type</th>\n",
       "      <th>stop_outcome</th>\n",
       "      <th>is_arrested</th>\n",
       "      <th>stop_duration</th>\n",
       "      <th>drugs_related_stop</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>1/23/2005</td>\n",
       "      <td>23:15</td>\n",
       "      <td>NaN</td>\n",
       "      <td>M</td>\n",
       "      <td>1972.0</td>\n",
       "      <td>33.0</td>\n",
       "      <td>White</td>\n",
       "      <td>Speeding</td>\n",
       "      <td>Speeding</td>\n",
       "      <td>False</td>\n",
       "      <td>NaN</td>\n",
       "      <td>Citation</td>\n",
       "      <td>False</td>\n",
       "      <td>0-15 Min</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>3/14/2005</td>\n",
       "      <td>10:00</td>\n",
       "      <td>NaN</td>\n",
       "      <td>F</td>\n",
       "      <td>1984.0</td>\n",
       "      <td>21.0</td>\n",
       "      <td>White</td>\n",
       "      <td>Speeding</td>\n",
       "      <td>Speeding</td>\n",
       "      <td>False</td>\n",
       "      <td>NaN</td>\n",
       "      <td>Citation</td>\n",
       "      <td>False</td>\n",
       "      <td>0-15 Min</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   stop_date stop_time  county_name driver_gender  driver_age_raw  driver_age  \\\n",
       "2  1/23/2005     23:15          NaN             M          1972.0        33.0   \n",
       "4  3/14/2005     10:00          NaN             F          1984.0        21.0   \n",
       "\n",
       "  driver_race violation_raw violation  search_conducted search_type  \\\n",
       "2       White      Speeding  Speeding             False         NaN   \n",
       "4       White      Speeding  Speeding             False         NaN   \n",
       "\n",
       "  stop_outcome is_arrested stop_duration  drugs_related_stop  \n",
       "2     Citation       False      0-15 Min               False  \n",
       "4     Citation       False      0-15 Min               False  "
      ]
     },
     "execution_count": 14,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "#h[h.driver_age > 20]\n",
    "h[(h.driver_age > 20) & (h.driver_age < 40)]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {
    "collapsed": true
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>stop_date</th>\n",
       "      <th>stop_time</th>\n",
       "      <th>county_name</th>\n",
       "      <th>driver_gender</th>\n",
       "      <th>driver_age_raw</th>\n",
       "      <th>driver_age</th>\n",
       "      <th>driver_race</th>\n",
       "      <th>violation_raw</th>\n",
       "      <th>violation</th>\n",
       "      <th>search_conducted</th>\n",
       "      <th>search_type</th>\n",
       "      <th>stop_outcome</th>\n",
       "      <th>is_arrested</th>\n",
       "      <th>stop_duration</th>\n",
       "      <th>drugs_related_stop</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>3/14/2005</td>\n",
       "      <td>10:00</td>\n",
       "      <td>NaN</td>\n",
       "      <td>F</td>\n",
       "      <td>1984.0</td>\n",
       "      <td>21.0</td>\n",
       "      <td>White</td>\n",
       "      <td>Speeding</td>\n",
       "      <td>Speeding</td>\n",
       "      <td>False</td>\n",
       "      <td>NaN</td>\n",
       "      <td>Citation</td>\n",
       "      <td>False</td>\n",
       "      <td>0-15 Min</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>1/23/2005</td>\n",
       "      <td>23:15</td>\n",
       "      <td>NaN</td>\n",
       "      <td>M</td>\n",
       "      <td>1972.0</td>\n",
       "      <td>33.0</td>\n",
       "      <td>White</td>\n",
       "      <td>Speeding</td>\n",
       "      <td>Speeding</td>\n",
       "      <td>False</td>\n",
       "      <td>NaN</td>\n",
       "      <td>Citation</td>\n",
       "      <td>False</td>\n",
       "      <td>0-15 Min</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   stop_date stop_time  county_name driver_gender  driver_age_raw  driver_age  \\\n",
       "4  3/14/2005     10:00          NaN             F          1984.0        21.0   \n",
       "2  1/23/2005     23:15          NaN             M          1972.0        33.0   \n",
       "\n",
       "  driver_race violation_raw violation  search_conducted search_type  \\\n",
       "4       White      Speeding  Speeding             False         NaN   \n",
       "2       White      Speeding  Speeding             False         NaN   \n",
       "\n",
       "  stop_outcome is_arrested stop_duration  drugs_related_stop  \n",
       "4     Citation       False      0-15 Min               False  \n",
       "2     Citation       False      0-15 Min               False  "
      ]
     },
     "execution_count": 15,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "h[(h.driver_age > 20) & (h.driver_age < 40)].sort_values('driver_age')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Introducing Set_index(['column']) function"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {
    "collapsed": true
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>stop_time</th>\n",
       "      <th>county_name</th>\n",
       "      <th>driver_gender</th>\n",
       "      <th>driver_age_raw</th>\n",
       "      <th>driver_age</th>\n",
       "      <th>driver_race</th>\n",
       "      <th>violation_raw</th>\n",
       "      <th>violation</th>\n",
       "      <th>search_conducted</th>\n",
       "      <th>search_type</th>\n",
       "      <th>stop_outcome</th>\n",
       "      <th>is_arrested</th>\n",
       "      <th>stop_duration</th>\n",
       "      <th>drugs_related_stop</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>stop_date</th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>1/2/2005</th>\n",
       "      <td>1:55</td>\n",
       "      <td>NaN</td>\n",
       "      <td>M</td>\n",
       "      <td>1985.0</td>\n",
       "      <td>20.0</td>\n",
       "      <td>White</td>\n",
       "      <td>Speeding</td>\n",
       "      <td>Speeding</td>\n",
       "      <td>False</td>\n",
       "      <td>NaN</td>\n",
       "      <td>Citation</td>\n",
       "      <td>False</td>\n",
       "      <td>0-15 Min</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1/18/2005</th>\n",
       "      <td>8:15</td>\n",
       "      <td>NaN</td>\n",
       "      <td>M</td>\n",
       "      <td>1965.0</td>\n",
       "      <td>40.0</td>\n",
       "      <td>White</td>\n",
       "      <td>Speeding</td>\n",
       "      <td>Speeding</td>\n",
       "      <td>False</td>\n",
       "      <td>NaN</td>\n",
       "      <td>Citation</td>\n",
       "      <td>False</td>\n",
       "      <td>0-15 Min</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1/23/2005</th>\n",
       "      <td>23:15</td>\n",
       "      <td>NaN</td>\n",
       "      <td>M</td>\n",
       "      <td>1972.0</td>\n",
       "      <td>33.0</td>\n",
       "      <td>White</td>\n",
       "      <td>Speeding</td>\n",
       "      <td>Speeding</td>\n",
       "      <td>False</td>\n",
       "      <td>NaN</td>\n",
       "      <td>Citation</td>\n",
       "      <td>False</td>\n",
       "      <td>0-15 Min</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2/20/2005</th>\n",
       "      <td>17:15</td>\n",
       "      <td>NaN</td>\n",
       "      <td>M</td>\n",
       "      <td>1986.0</td>\n",
       "      <td>19.0</td>\n",
       "      <td>White</td>\n",
       "      <td>Call for Service</td>\n",
       "      <td>Other</td>\n",
       "      <td>False</td>\n",
       "      <td>NaN</td>\n",
       "      <td>Arrest Driver</td>\n",
       "      <td>True</td>\n",
       "      <td>16-30 Min</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3/14/2005</th>\n",
       "      <td>10:00</td>\n",
       "      <td>NaN</td>\n",
       "      <td>F</td>\n",
       "      <td>1984.0</td>\n",
       "      <td>21.0</td>\n",
       "      <td>White</td>\n",
       "      <td>Speeding</td>\n",
       "      <td>Speeding</td>\n",
       "      <td>False</td>\n",
       "      <td>NaN</td>\n",
       "      <td>Citation</td>\n",
       "      <td>False</td>\n",
       "      <td>0-15 Min</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "          stop_time  county_name driver_gender  driver_age_raw  driver_age  \\\n",
       "stop_date                                                                    \n",
       "1/2/2005       1:55          NaN             M          1985.0        20.0   \n",
       "1/18/2005      8:15          NaN             M          1965.0        40.0   \n",
       "1/23/2005     23:15          NaN             M          1972.0        33.0   \n",
       "2/20/2005     17:15          NaN             M          1986.0        19.0   \n",
       "3/14/2005     10:00          NaN             F          1984.0        21.0   \n",
       "\n",
       "          driver_race     violation_raw violation  search_conducted  \\\n",
       "stop_date                                                             \n",
       "1/2/2005        White          Speeding  Speeding             False   \n",
       "1/18/2005       White          Speeding  Speeding             False   \n",
       "1/23/2005       White          Speeding  Speeding             False   \n",
       "2/20/2005       White  Call for Service     Other             False   \n",
       "3/14/2005       White          Speeding  Speeding             False   \n",
       "\n",
       "          search_type   stop_outcome is_arrested stop_duration  \\\n",
       "stop_date                                                        \n",
       "1/2/2005          NaN       Citation       False      0-15 Min   \n",
       "1/18/2005         NaN       Citation       False      0-15 Min   \n",
       "1/23/2005         NaN       Citation       False      0-15 Min   \n",
       "2/20/2005         NaN  Arrest Driver        True     16-30 Min   \n",
       "3/14/2005         NaN       Citation       False      0-15 Min   \n",
       "\n",
       "           drugs_related_stop  \n",
       "stop_date                      \n",
       "1/2/2005                False  \n",
       "1/18/2005               False  \n",
       "1/23/2005               False  \n",
       "2/20/2005               False  \n",
       "3/14/2005               False  "
      ]
     },
     "execution_count": 16,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df.set_index(['stop_date']).head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {
    "collapsed": true
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th>county_name</th>\n",
       "      <th>driver_gender</th>\n",
       "      <th>driver_age_raw</th>\n",
       "      <th>driver_age</th>\n",
       "      <th>driver_race</th>\n",
       "      <th>violation_raw</th>\n",
       "      <th>violation</th>\n",
       "      <th>search_conducted</th>\n",
       "      <th>search_type</th>\n",
       "      <th>stop_outcome</th>\n",
       "      <th>is_arrested</th>\n",
       "      <th>stop_duration</th>\n",
       "      <th>drugs_related_stop</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>stop_date</th>\n",
       "      <th>stop_time</th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th rowspan=\"5\" valign=\"top\">1/1/2006</th>\n",
       "      <th>11:10</th>\n",
       "      <td>NaN</td>\n",
       "      <td>M</td>\n",
       "      <td>1958.0</td>\n",
       "      <td>48.0</td>\n",
       "      <td>White</td>\n",
       "      <td>Speeding</td>\n",
       "      <td>Speeding</td>\n",
       "      <td>False</td>\n",
       "      <td>NaN</td>\n",
       "      <td>Citation</td>\n",
       "      <td>False</td>\n",
       "      <td>0-15 Min</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>13:05</th>\n",
       "      <td>NaN</td>\n",
       "      <td>M</td>\n",
       "      <td>1972.0</td>\n",
       "      <td>34.0</td>\n",
       "      <td>White</td>\n",
       "      <td>Speeding</td>\n",
       "      <td>Speeding</td>\n",
       "      <td>False</td>\n",
       "      <td>NaN</td>\n",
       "      <td>Citation</td>\n",
       "      <td>False</td>\n",
       "      <td>0-15 Min</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>14:10</th>\n",
       "      <td>NaN</td>\n",
       "      <td>M</td>\n",
       "      <td>1983.0</td>\n",
       "      <td>23.0</td>\n",
       "      <td>White</td>\n",
       "      <td>Registration Violation</td>\n",
       "      <td>Registration/plates</td>\n",
       "      <td>True</td>\n",
       "      <td>Incident to Arrest</td>\n",
       "      <td>Arrest Driver</td>\n",
       "      <td>True</td>\n",
       "      <td>16-30 Min</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>17:18</th>\n",
       "      <td>NaN</td>\n",
       "      <td>M</td>\n",
       "      <td>1974.0</td>\n",
       "      <td>32.0</td>\n",
       "      <td>White</td>\n",
       "      <td>Other Traffic Violation</td>\n",
       "      <td>Moving violation</td>\n",
       "      <td>False</td>\n",
       "      <td>NaN</td>\n",
       "      <td>Warning</td>\n",
       "      <td>False</td>\n",
       "      <td>0-15 Min</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1:20</th>\n",
       "      <td>NaN</td>\n",
       "      <td>M</td>\n",
       "      <td>1981.0</td>\n",
       "      <td>25.0</td>\n",
       "      <td>White</td>\n",
       "      <td>Other Traffic Violation</td>\n",
       "      <td>Moving violation</td>\n",
       "      <td>False</td>\n",
       "      <td>NaN</td>\n",
       "      <td>Citation</td>\n",
       "      <td>False</td>\n",
       "      <td>0-15 Min</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                     county_name driver_gender  driver_age_raw  driver_age  \\\n",
       "stop_date stop_time                                                          \n",
       "1/1/2006  11:10              NaN             M          1958.0        48.0   \n",
       "          13:05              NaN             M          1972.0        34.0   \n",
       "          14:10              NaN             M          1983.0        23.0   \n",
       "          17:18              NaN             M          1974.0        32.0   \n",
       "          1:20               NaN             M          1981.0        25.0   \n",
       "\n",
       "                    driver_race            violation_raw            violation  \\\n",
       "stop_date stop_time                                                             \n",
       "1/1/2006  11:10           White                 Speeding             Speeding   \n",
       "          13:05           White                 Speeding             Speeding   \n",
       "          14:10           White   Registration Violation  Registration/plates   \n",
       "          17:18           White  Other Traffic Violation     Moving violation   \n",
       "          1:20            White  Other Traffic Violation     Moving violation   \n",
       "\n",
       "                     search_conducted         search_type   stop_outcome  \\\n",
       "stop_date stop_time                                                        \n",
       "1/1/2006  11:10                 False                 NaN       Citation   \n",
       "          13:05                 False                 NaN       Citation   \n",
       "          14:10                  True  Incident to Arrest  Arrest Driver   \n",
       "          17:18                 False                 NaN        Warning   \n",
       "          1:20                  False                 NaN       Citation   \n",
       "\n",
       "                    is_arrested stop_duration  drugs_related_stop  \n",
       "stop_date stop_time                                                \n",
       "1/1/2006  11:10           False      0-15 Min               False  \n",
       "          13:05           False      0-15 Min               False  \n",
       "          14:10            True     16-30 Min               False  \n",
       "          17:18           False      0-15 Min               False  \n",
       "          1:20            False      0-15 Min               False  "
      ]
     },
     "execution_count": 17,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df.set_index(['stop_date', 'stop_time']).sort_index().head()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Reset Index()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "df.reset_index('stop_time')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Introducing loc function"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "df.loc([''])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#df.loc['Sleuth', 1972]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Group by"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "c = df\n",
    "c.head(1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "grp_obj = c.groupby(['driver_age'])\n",
    "grp_obj.is_arrested.sum().sort_index()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "grp_obj2 = c.groupby(['driver_gender','driver_age', ])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "grp_obj2.search_conducted.agg(['sum', 'size', 'min', 'max', 'mean'])"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "collapsed": true
   },
   "source": [
    "### Finding out the total null values in the various columns"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "df.isnull().sum()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "True == 1"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "df1.isnull()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# TASKS/EXERCISES"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### TASK: Remove the columns that only contains the missing values"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "df.drop('county_name', axis = 'columns', inplace = True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#Alternatively: \n",
    "df = df.drop('county_name', axis = 'columns')\n",
    "#Previous approach is more memory efficient and pythonic"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "df.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "df.columns"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "# Equivalent method to do that is\n",
    "df.dropna(axis='columns', how='all').shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#Key things that we learnt:\n",
    "    #Multiple ways to do the same in python"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "\n",
    " ### TASK: Do men or women speed more often?\n",
    " \n",
    " ### Columns we will use: driver gender, violation"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Filter for speeding --> Chaining to select --> driver gender --> value counts\n",
    "\n",
    "df[df.violation == 'Speeding'].driver_gender.value_counts(normalize = True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# For Males\n",
    "df[df.driver_gender == 'M'].violation.value_counts(normalize = True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# For Females\n",
    "df[df.driver_gender == 'F'].violation.value_counts(normalize = True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Doing it in one line of code\n",
    "df.groupby('driver_gender').violation.value_counts(normalize = True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#another way to do it is\n",
    "df.loc[df['violation'] == 'Speeding', 'driver_gender'].value_counts(normalize = True)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Task: Does gender affect who gets searched during a stop"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "df.head()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Columns we will use: driver_gender, search_conducted"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "df.search_conducted.value_counts(normalize = True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "df.groupby('driver_gender').search_conducted.mean()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Value counts drop the Nan by default"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Task: During the search, how often is the driver frisked"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "df.search_type.value_counts(dropna = False)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Pandas String Method to apply across the Pandas data series"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Provide the link of Str functions and talk about it"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "df['frisk'] = df.search_type.str.contains('Protective Frisk')\n",
    "df.frisk.value_counts(dropna = False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "df.frisk.sum()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "df.frisk.mean() # ignores the Nan (missing) values and Pandas mostly ignores the missing values all over the place."
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### TASK: Which year had the least number of stops"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Columns we will use: stop_date"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "df.head(1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "combined_date_time = df.stop_date.str.cat(df.stop_time, sep= ' ')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "combined_date_time.head() # "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Introducing datetime functions\n",
    "df['stop_date_time'] = pd.to_datetime(combined_date_time)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "df.dtypes"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Introducing dt functions"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "df.stop_date_time.dt.year.value_counts()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### TASK: How does drug activity change by time of the day?\n",
    "\n",
    "### Columns we will use: stop_date_time, drugs_related_stop"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "df.groupby(df.stop_date_time.dt.hour).drugs_related_stop.mean()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "df.groupby(df.stop_date_time.dt.hour).drugs_related_stop.mean().plot()"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.1"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}