Mission210Solution.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Show Number</th>\n",
       "      <th>Air Date</th>\n",
       "      <th>Round</th>\n",
       "      <th>Category</th>\n",
       "      <th>Value</th>\n",
       "      <th>Question</th>\n",
       "      <th>Answer</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>4680</td>\n",
       "      <td>2004-12-31</td>\n",
       "      <td>Jeopardy!</td>\n",
       "      <td>HISTORY</td>\n",
       "      <td>$200</td>\n",
       "      <td>For the last 8 years of his life, Galileo was ...</td>\n",
       "      <td>Copernicus</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>4680</td>\n",
       "      <td>2004-12-31</td>\n",
       "      <td>Jeopardy!</td>\n",
       "      <td>ESPN's TOP 10 ALL-TIME ATHLETES</td>\n",
       "      <td>$200</td>\n",
       "      <td>No. 2: 1912 Olympian; football star at Carlisl...</td>\n",
       "      <td>Jim Thorpe</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>4680</td>\n",
       "      <td>2004-12-31</td>\n",
       "      <td>Jeopardy!</td>\n",
       "      <td>EVERYBODY TALKS ABOUT IT...</td>\n",
       "      <td>$200</td>\n",
       "      <td>The city of Yuma in this state has a record av...</td>\n",
       "      <td>Arizona</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>4680</td>\n",
       "      <td>2004-12-31</td>\n",
       "      <td>Jeopardy!</td>\n",
       "      <td>THE COMPANY LINE</td>\n",
       "      <td>$200</td>\n",
       "      <td>In 1963, live on \"The Art Linkletter Show\", th...</td>\n",
       "      <td>McDonald's</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>4680</td>\n",
       "      <td>2004-12-31</td>\n",
       "      <td>Jeopardy!</td>\n",
       "      <td>EPITAPHS &amp; TRIBUTES</td>\n",
       "      <td>$200</td>\n",
       "      <td>Signer of the Dec. of Indep., framer of the Co...</td>\n",
       "      <td>John Adams</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>19994</th>\n",
       "      <td>3582</td>\n",
       "      <td>2000-03-14</td>\n",
       "      <td>Jeopardy!</td>\n",
       "      <td>U.S. GEOGRAPHY</td>\n",
       "      <td>$200</td>\n",
       "      <td>Of 8, 12 or 18, the number of U.S. states that...</td>\n",
       "      <td>18</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>19995</th>\n",
       "      <td>3582</td>\n",
       "      <td>2000-03-14</td>\n",
       "      <td>Jeopardy!</td>\n",
       "      <td>POP MUSIC PAIRINGS</td>\n",
       "      <td>$200</td>\n",
       "      <td>...&amp; the New Power Generation</td>\n",
       "      <td>Prince</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>19996</th>\n",
       "      <td>3582</td>\n",
       "      <td>2000-03-14</td>\n",
       "      <td>Jeopardy!</td>\n",
       "      <td>HISTORIC PEOPLE</td>\n",
       "      <td>$200</td>\n",
       "      <td>In 1589 he was appointed professor of mathemat...</td>\n",
       "      <td>Galileo</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>19997</th>\n",
       "      <td>3582</td>\n",
       "      <td>2000-03-14</td>\n",
       "      <td>Jeopardy!</td>\n",
       "      <td>1998 QUOTATIONS</td>\n",
       "      <td>$200</td>\n",
       "      <td>Before the grand jury she said, \"I'm really so...</td>\n",
       "      <td>Monica Lewinsky</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>19998</th>\n",
       "      <td>3582</td>\n",
       "      <td>2000-03-14</td>\n",
       "      <td>Jeopardy!</td>\n",
       "      <td>LLAMA-RAMA</td>\n",
       "      <td>$200</td>\n",
       "      <td>Llamas are the heftiest South American members...</td>\n",
       "      <td>Camels</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>19999 rows × 7 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "       Show Number    Air Date      Round                         Category  \\\n",
       "0             4680  2004-12-31  Jeopardy!                          HISTORY   \n",
       "1             4680  2004-12-31  Jeopardy!  ESPN's TOP 10 ALL-TIME ATHLETES   \n",
       "2             4680  2004-12-31  Jeopardy!      EVERYBODY TALKS ABOUT IT...   \n",
       "3             4680  2004-12-31  Jeopardy!                 THE COMPANY LINE   \n",
       "4             4680  2004-12-31  Jeopardy!              EPITAPHS & TRIBUTES   \n",
       "...            ...         ...        ...                              ...   \n",
       "19994         3582  2000-03-14  Jeopardy!                   U.S. GEOGRAPHY   \n",
       "19995         3582  2000-03-14  Jeopardy!               POP MUSIC PAIRINGS   \n",
       "19996         3582  2000-03-14  Jeopardy!                  HISTORIC PEOPLE   \n",
       "19997         3582  2000-03-14  Jeopardy!                  1998 QUOTATIONS   \n",
       "19998         3582  2000-03-14  Jeopardy!                       LLAMA-RAMA   \n",
       "\n",
       "       Value                                           Question  \\\n",
       "0       $200  For the last 8 years of his life, Galileo was ...   \n",
       "1       $200  No. 2: 1912 Olympian; football star at Carlisl...   \n",
       "2       $200  The city of Yuma in this state has a record av...   \n",
       "3       $200  In 1963, live on \"The Art Linkletter Show\", th...   \n",
       "4       $200  Signer of the Dec. of Indep., framer of the Co...   \n",
       "...      ...                                                ...   \n",
       "19994   $200  Of 8, 12 or 18, the number of U.S. states that...   \n",
       "19995   $200                      ...& the New Power Generation   \n",
       "19996   $200  In 1589 he was appointed professor of mathemat...   \n",
       "19997   $200  Before the grand jury she said, \"I'm really so...   \n",
       "19998   $200  Llamas are the heftiest South American members...   \n",
       "\n",
       "                Answer  \n",
       "0           Copernicus  \n",
       "1           Jim Thorpe  \n",
       "2              Arizona  \n",
       "3           McDonald's  \n",
       "4           John Adams  \n",
       "...                ...  \n",
       "19994               18  \n",
       "19995           Prince  \n",
       "19996          Galileo  \n",
       "19997  Monica Lewinsky  \n",
       "19998           Camels  \n",
       "\n",
       "[19999 rows x 7 columns]"
      ]
     },
     "execution_count": 1,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "import pandas\n",
    "import csv\n",
    "\n",
    "jeopardy = pandas.read_csv(\"jeopardy.csv\")\n",
    "\n",
    "jeopardy"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "Index(['Show Number', ' Air Date', ' Round', ' Category', ' Value',\n",
       "       ' Question', ' Answer'],\n",
       "      dtype='object')"
      ]
     },
     "execution_count": 2,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "jeopardy.columns"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "jeopardy.columns = ['Show Number', 'Air Date', 'Round', 'Category', 'Value', 'Question', 'Answer']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "import re\n",
    "\n",
    "def normalize_text(text):\n",
    "    text = text.lower()\n",
    "    text = re.sub(\"[^A-Za-z0-9\\s]\", \"\", text)\n",
    "    text = re.sub(\"\\s+\", \" \", text)\n",
    "    return text\n",
    "\n",
    "def normalize_values(text):\n",
    "    text = re.sub(\"[^A-Za-z0-9\\s]\", \"\", text)\n",
    "    try:\n",
    "        text = int(text)\n",
    "    except Exception:\n",
    "        text = 0\n",
    "    return text"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "jeopardy[\"clean_question\"] = jeopardy[\"Question\"].apply(normalize_text)\n",
    "jeopardy[\"clean_answer\"] = jeopardy[\"Answer\"].apply(normalize_text)\n",
    "jeopardy[\"clean_value\"] = jeopardy[\"Value\"].apply(normalize_values)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Show Number</th>\n",
       "      <th>Air Date</th>\n",
       "      <th>Round</th>\n",
       "      <th>Category</th>\n",
       "      <th>Value</th>\n",
       "      <th>Question</th>\n",
       "      <th>Answer</th>\n",
       "      <th>clean_question</th>\n",
       "      <th>clean_answer</th>\n",
       "      <th>clean_value</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>4680</td>\n",
       "      <td>2004-12-31</td>\n",
       "      <td>Jeopardy!</td>\n",
       "      <td>HISTORY</td>\n",
       "      <td>$200</td>\n",
       "      <td>For the last 8 years of his life, Galileo was ...</td>\n",
       "      <td>Copernicus</td>\n",
       "      <td>for the last 8 years of his life galileo was u...</td>\n",
       "      <td>copernicus</td>\n",
       "      <td>200</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>4680</td>\n",
       "      <td>2004-12-31</td>\n",
       "      <td>Jeopardy!</td>\n",
       "      <td>ESPN's TOP 10 ALL-TIME ATHLETES</td>\n",
       "      <td>$200</td>\n",
       "      <td>No. 2: 1912 Olympian; football star at Carlisl...</td>\n",
       "      <td>Jim Thorpe</td>\n",
       "      <td>no 2 1912 olympian football star at carlisle i...</td>\n",
       "      <td>jim thorpe</td>\n",
       "      <td>200</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>4680</td>\n",
       "      <td>2004-12-31</td>\n",
       "      <td>Jeopardy!</td>\n",
       "      <td>EVERYBODY TALKS ABOUT IT...</td>\n",
       "      <td>$200</td>\n",
       "      <td>The city of Yuma in this state has a record av...</td>\n",
       "      <td>Arizona</td>\n",
       "      <td>the city of yuma in this state has a record av...</td>\n",
       "      <td>arizona</td>\n",
       "      <td>200</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>4680</td>\n",
       "      <td>2004-12-31</td>\n",
       "      <td>Jeopardy!</td>\n",
       "      <td>THE COMPANY LINE</td>\n",
       "      <td>$200</td>\n",
       "      <td>In 1963, live on \"The Art Linkletter Show\", th...</td>\n",
       "      <td>McDonald's</td>\n",
       "      <td>in 1963 live on the art linkletter show this c...</td>\n",
       "      <td>mcdonalds</td>\n",
       "      <td>200</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>4680</td>\n",
       "      <td>2004-12-31</td>\n",
       "      <td>Jeopardy!</td>\n",
       "      <td>EPITAPHS &amp; TRIBUTES</td>\n",
       "      <td>$200</td>\n",
       "      <td>Signer of the Dec. of Indep., framer of the Co...</td>\n",
       "      <td>John Adams</td>\n",
       "      <td>signer of the dec of indep framer of the const...</td>\n",
       "      <td>john adams</td>\n",
       "      <td>200</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>19994</th>\n",
       "      <td>3582</td>\n",
       "      <td>2000-03-14</td>\n",
       "      <td>Jeopardy!</td>\n",
       "      <td>U.S. GEOGRAPHY</td>\n",
       "      <td>$200</td>\n",
       "      <td>Of 8, 12 or 18, the number of U.S. states that...</td>\n",
       "      <td>18</td>\n",
       "      <td>of 8 12 or 18 the number of us states that tou...</td>\n",
       "      <td>18</td>\n",
       "      <td>200</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>19995</th>\n",
       "      <td>3582</td>\n",
       "      <td>2000-03-14</td>\n",
       "      <td>Jeopardy!</td>\n",
       "      <td>POP MUSIC PAIRINGS</td>\n",
       "      <td>$200</td>\n",
       "      <td>...&amp; the New Power Generation</td>\n",
       "      <td>Prince</td>\n",
       "      <td>the new power generation</td>\n",
       "      <td>prince</td>\n",
       "      <td>200</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>19996</th>\n",
       "      <td>3582</td>\n",
       "      <td>2000-03-14</td>\n",
       "      <td>Jeopardy!</td>\n",
       "      <td>HISTORIC PEOPLE</td>\n",
       "      <td>$200</td>\n",
       "      <td>In 1589 he was appointed professor of mathemat...</td>\n",
       "      <td>Galileo</td>\n",
       "      <td>in 1589 he was appointed professor of mathemat...</td>\n",
       "      <td>galileo</td>\n",
       "      <td>200</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>19997</th>\n",
       "      <td>3582</td>\n",
       "      <td>2000-03-14</td>\n",
       "      <td>Jeopardy!</td>\n",
       "      <td>1998 QUOTATIONS</td>\n",
       "      <td>$200</td>\n",
       "      <td>Before the grand jury she said, \"I'm really so...</td>\n",
       "      <td>Monica Lewinsky</td>\n",
       "      <td>before the grand jury she said im really sorry...</td>\n",
       "      <td>monica lewinsky</td>\n",
       "      <td>200</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>19998</th>\n",
       "      <td>3582</td>\n",
       "      <td>2000-03-14</td>\n",
       "      <td>Jeopardy!</td>\n",
       "      <td>LLAMA-RAMA</td>\n",
       "      <td>$200</td>\n",
       "      <td>Llamas are the heftiest South American members...</td>\n",
       "      <td>Camels</td>\n",
       "      <td>llamas are the heftiest south american members...</td>\n",
       "      <td>camels</td>\n",
       "      <td>200</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>19999 rows × 10 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "       Show Number    Air Date      Round                         Category  \\\n",
       "0             4680  2004-12-31  Jeopardy!                          HISTORY   \n",
       "1             4680  2004-12-31  Jeopardy!  ESPN's TOP 10 ALL-TIME ATHLETES   \n",
       "2             4680  2004-12-31  Jeopardy!      EVERYBODY TALKS ABOUT IT...   \n",
       "3             4680  2004-12-31  Jeopardy!                 THE COMPANY LINE   \n",
       "4             4680  2004-12-31  Jeopardy!              EPITAPHS & TRIBUTES   \n",
       "...            ...         ...        ...                              ...   \n",
       "19994         3582  2000-03-14  Jeopardy!                   U.S. GEOGRAPHY   \n",
       "19995         3582  2000-03-14  Jeopardy!               POP MUSIC PAIRINGS   \n",
       "19996         3582  2000-03-14  Jeopardy!                  HISTORIC PEOPLE   \n",
       "19997         3582  2000-03-14  Jeopardy!                  1998 QUOTATIONS   \n",
       "19998         3582  2000-03-14  Jeopardy!                       LLAMA-RAMA   \n",
       "\n",
       "      Value                                           Question  \\\n",
       "0      $200  For the last 8 years of his life, Galileo was ...   \n",
       "1      $200  No. 2: 1912 Olympian; football star at Carlisl...   \n",
       "2      $200  The city of Yuma in this state has a record av...   \n",
       "3      $200  In 1963, live on \"The Art Linkletter Show\", th...   \n",
       "4      $200  Signer of the Dec. of Indep., framer of the Co...   \n",
       "...     ...                                                ...   \n",
       "19994  $200  Of 8, 12 or 18, the number of U.S. states that...   \n",
       "19995  $200                      ...& the New Power Generation   \n",
       "19996  $200  In 1589 he was appointed professor of mathemat...   \n",
       "19997  $200  Before the grand jury she said, \"I'm really so...   \n",
       "19998  $200  Llamas are the heftiest South American members...   \n",
       "\n",
       "                Answer                                     clean_question  \\\n",
       "0           Copernicus  for the last 8 years of his life galileo was u...   \n",
       "1           Jim Thorpe  no 2 1912 olympian football star at carlisle i...   \n",
       "2              Arizona  the city of yuma in this state has a record av...   \n",
       "3           McDonald's  in 1963 live on the art linkletter show this c...   \n",
       "4           John Adams  signer of the dec of indep framer of the const...   \n",
       "...                ...                                                ...   \n",
       "19994               18  of 8 12 or 18 the number of us states that tou...   \n",
       "19995           Prince                           the new power generation   \n",
       "19996          Galileo  in 1589 he was appointed professor of mathemat...   \n",
       "19997  Monica Lewinsky  before the grand jury she said im really sorry...   \n",
       "19998           Camels  llamas are the heftiest south american members...   \n",
       "\n",
       "          clean_answer  clean_value  \n",
       "0           copernicus          200  \n",
       "1           jim thorpe          200  \n",
       "2              arizona          200  \n",
       "3            mcdonalds          200  \n",
       "4           john adams          200  \n",
       "...                ...          ...  \n",
       "19994               18          200  \n",
       "19995           prince          200  \n",
       "19996          galileo          200  \n",
       "19997  monica lewinsky          200  \n",
       "19998           camels          200  \n",
       "\n",
       "[19999 rows x 10 columns]"
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "jeopardy"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "jeopardy[\"Air Date\"] = pandas.to_datetime(jeopardy[\"Air Date\"])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "Show Number                int64\n",
       "Air Date          datetime64[ns]\n",
       "Round                     object\n",
       "Category                  object\n",
       "Value                     object\n",
       "Question                  object\n",
       "Answer                    object\n",
       "clean_question            object\n",
       "clean_answer              object\n",
       "clean_value                int64\n",
       "dtype: object"
      ]
     },
     "execution_count": 8,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "jeopardy.dtypes"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "def count_matches(row):\n",
    "    split_answer = row[\"clean_answer\"].split()\n",
    "    split_question = row[\"clean_question\"].split()\n",
    "    if \"the\" in split_answer:\n",
    "        split_answer.remove(\"the\")\n",
    "    if len(split_answer) == 0:\n",
    "        return 0\n",
    "    match_count = 0\n",
    "    for item in split_answer:\n",
    "        if item in split_question:\n",
    "            match_count += 1\n",
    "    return match_count / len(split_answer)\n",
    "\n",
    "jeopardy[\"answer_in_question\"] = jeopardy.apply(count_matches, axis=1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.059001965249777744"
      ]
     },
     "execution_count": 10,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "jeopardy[\"answer_in_question\"].mean()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Recycled questions\n",
    "\n",
    "On average, the answer only makes up for about `6%` of the question.  This isn't a huge number, and means that we probably can't just hope that hearing a question will enable us to figure out the answer.  We'll probably have to study."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.6876260592169776"
      ]
     },
     "execution_count": 11,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "question_overlap = []\n",
    "terms_used = set()\n",
    "\n",
    "jeopardy = jeopardy.sort_values(\"Air Date\")\n",
    "\n",
    "for i, row in jeopardy.iterrows():\n",
    "        split_question = row[\"clean_question\"].split(\" \")\n",
    "        split_question = [q for q in split_question if len(q) > 5]\n",
    "        match_count = 0\n",
    "        for word in split_question:\n",
    "            if word in terms_used:\n",
    "                match_count += 1\n",
    "        for word in split_question:\n",
    "            terms_used.add(word)\n",
    "        if len(split_question) > 0:\n",
    "            match_count /= len(split_question)\n",
    "        question_overlap.append(match_count)\n",
    "jeopardy[\"question_overlap\"] = question_overlap\n",
    "\n",
    "jeopardy[\"question_overlap\"].mean()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Low value vs high value questions\n",
    "\n",
    "There is about `70%` overlap between terms in new questions and terms in old questions.  This only looks at a small set of questions, and it doesn't look at phrases, it looks at single terms.  This makes it relatively insignificant, but it does mean that it's worth looking more into the recycling of questions."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [],
   "source": [
    "def determine_value(row):\n",
    "    value = 0\n",
    "    if row[\"clean_value\"] > 800:\n",
    "        value = 1\n",
    "    return value\n",
    "\n",
    "jeopardy[\"high_value\"] = jeopardy.apply(determine_value, axis=1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [],
   "source": [
    "def count_usage(term):\n",
    "    low_count = 0\n",
    "    high_count = 0\n",
    "    for i, row in jeopardy.iterrows():\n",
    "        if term in row[\"clean_question\"].split(\" \"):\n",
    "            if row[\"high_value\"] == 1:\n",
    "                high_count += 1\n",
    "            else:\n",
    "                low_count += 1\n",
    "    return high_count, low_count"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[(0, 1),\n",
       " (1, 0),\n",
       " (0, 1),\n",
       " (1, 0),\n",
       " (1, 0),\n",
       " (0, 2),\n",
       " (3, 8),\n",
       " (1, 0),\n",
       " (0, 1),\n",
       " (0, 1)]"
      ]
     },
     "execution_count": 14,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "from random import choice\n",
    "\n",
    "terms_used_list = list(terms_used)\n",
    "comparison_terms = [choice(terms_used_list) for _ in range(10)]\n",
    "\n",
    "observed_expected = []\n",
    "\n",
    "for term in comparison_terms:\n",
    "    observed_expected.append(count_usage(term))\n",
    "\n",
    "observed_expected"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[Power_divergenceResult(statistic=0.401962846126884, pvalue=0.5260772985705469),\n",
       " Power_divergenceResult(statistic=2.487792117195675, pvalue=0.11473257634454047),\n",
       " Power_divergenceResult(statistic=0.401962846126884, pvalue=0.5260772985705469),\n",
       " Power_divergenceResult(statistic=2.487792117195675, pvalue=0.11473257634454047),\n",
       " Power_divergenceResult(statistic=2.487792117195675, pvalue=0.11473257634454047),\n",
       " Power_divergenceResult(statistic=0.803925692253768, pvalue=0.3699222378079571),\n",
       " Power_divergenceResult(statistic=0.01052283698924083, pvalue=0.9182956181393399),\n",
       " Power_divergenceResult(statistic=2.487792117195675, pvalue=0.11473257634454047),\n",
       " Power_divergenceResult(statistic=0.401962846126884, pvalue=0.5260772985705469),\n",
       " Power_divergenceResult(statistic=0.401962846126884, pvalue=0.5260772985705469)]"
      ]
     },
     "execution_count": 15,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "from scipy.stats import chisquare\n",
    "import numpy as np\n",
    "\n",
    "high_value_count = jeopardy[jeopardy[\"high_value\"] == 1].shape[0]\n",
    "low_value_count = jeopardy[jeopardy[\"high_value\"] == 0].shape[0]\n",
    "\n",
    "chi_squared = []\n",
    "for obs in observed_expected:\n",
    "    total = sum(obs)\n",
    "    total_prop = total / jeopardy.shape[0]\n",
    "    high_value_exp = total_prop * high_value_count\n",
    "    low_value_exp = total_prop * low_value_count\n",
    "    \n",
    "    observed = np.array([obs[0], obs[1]])\n",
    "    expected = np.array([high_value_exp, low_value_exp])\n",
    "    chi_squared.append(chisquare(observed, expected))\n",
    "\n",
    "chi_squared"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Chi-squared results\n",
    "\n",
    "None of the terms had a significant difference in usage between high value and low value rows.  Additionally, the frequencies were all lower than `5`, so the chi-squared test isn't as valid.  It would be better to run this test with only terms that have higher frequencies."
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.6"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}