exps/algorithm_bias_estimation.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "import numpy as np\n",
    "import os\n",
    "import scipy.stats\n",
    "import matplotlib.pyplot as plt\n",
    "from functools import reduce\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "%matplotlib inline"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Load the data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "data_root = os.path.join('..', 'data', 'time_series')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "file_name_template = '{}_{}_sliced_{}_tl_bot{}.csv'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "mapping = {\n",
    "    1: 'thenation',\n",
    "    2: 'thenation',\n",
    "    3: 'thenation',\n",
    "    4: 'washingtonpost',\n",
    "    5: 'washingtonpost',\n",
    "    6: 'washingtonpost',\n",
    "    7: 'USATODAY',\n",
    "    8: 'USATODAY',\n",
    "    9: 'USATODAY',\n",
    "    10: 'WSJ',\n",
    "    11: 'WSJ',\n",
    "    12: 'WSJ',\n",
    "    13: 'BreitbartNews',\n",
    "    14: 'BreitbartNews',\n",
    "    15: 'BreitbartNews'\n",
    "}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "user_type_list = ['home', 'friend_usr']\n",
    "methods = ['hashtag', 'url']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "drifter_df_dict = {}\n",
    "for key, seed in mapping.items():\n",
    "    method_dict = {}\n",
    "    for method in methods:\n",
    "        user_type_dict = {}\n",
    "        for user_type in user_type_list:\n",
    "            temp_df = pd.read_csv(os.path.join(data_root, file_name_template.format(method, seed, user_type, key)))\n",
    "            user_type_dict[user_type] = temp_df\n",
    "        method_dict[method] = user_type_dict\n",
    "    drifter_df_dict[key] = {\n",
    "        'seed': seed,\n",
    "        'dfs': method_dict\n",
    "    }"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# T-test for different groups"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "reverse_mapping = {\n",
    "    'thenation': [1, 2, 3],\n",
    "    'washingtonpost': [4, 5, 6],\n",
    "    'USATODAY': [7, 8, 9],\n",
    "    'WSJ': [10, 11, 12],\n",
    "    'BreitbartNews': [13, 14, 15]\n",
    "}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "name_mapping = {\n",
    "    'thenation': 'Left',\n",
    "    'washingtonpost': 'C. Left',\n",
    "    'USATODAY': 'Center',\n",
    "    'WSJ': 'C. Right',\n",
    "    'BreitbartNews': 'Right'\n",
    "}"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Calculate the difference between political alignment scores of the home timelines and their friends’ user timelines."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [],
   "source": [
    "alignment_diffs = {}\n",
    "for method in methods:\n",
    "    temp_alignment_diffs = []\n",
    "    for seed, drifter_ids in reverse_mapping.items():\n",
    "        temp_dfs = []\n",
    "        for drifter_id in drifter_ids:\n",
    "            temp_df = drifter_df_dict[drifter_id]['dfs'][method]['home'].merge(\n",
    "                drifter_df_dict[drifter_id]['dfs'][method]['friend_usr'], on='date'\n",
    "            )\n",
    "            temp_dfs.append(temp_df)\n",
    "        combined_df = pd.concat(temp_dfs)\n",
    "        temp_diff = combined_df['{}_mean_x'.format(method)] - combined_df['{}_mean_y'.format(method)]\n",
    "        temp_diff = temp_diff.to_frame(name=seed).reset_index()[[seed]]\n",
    "                \n",
    "        temp_alignment_diffs.append(temp_diff)\n",
    "    alignment_diffs[method] = temp_alignment_diffs"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [],
   "source": [
    "def concat_dfs(dfs):\n",
    "    return reduce(\n",
    "        lambda left, right: pd.merge(left, right, left_index=True, right_index=True, how='outer'),\n",
    "        dfs\n",
    "    ).rename(columns=name_mapping)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [],
   "source": [
    "alignment_diffs_url = concat_dfs(alignment_diffs['url'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Dump the raw data for sharing\n",
    "#alignment_diffs_url.to_csv(\"table_s3_url.csv\", index=None)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [],
   "source": [
    "alignment_diffs_hashtag = concat_dfs(alignment_diffs['hashtag'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Dump the raw data for sharing\n",
    "# alignment_diffs_hashtag.to_csv(\"table_s3_hashtag.csv\", index=None)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [],
   "source": [
    "def do_t_test(samples):\n",
    "    t_stat, pvalue = scipy.stats.ttest_1samp(samples, 0)\n",
    "    cohen_d = abs(samples.mean() - 0) / np.std(samples, ddof=1)\n",
    "    return t_stat, pvalue, cohen_d"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [],
   "source": [
    "url_t_test_results = []\n",
    "for value in name_mapping.values():\n",
    "    t_stat, pvalue, cohen_d = do_t_test(alignment_diffs_url[value].dropna())\n",
    "    \n",
    "    if cohen_d < 0.5:\n",
    "        effect_size = 'small'\n",
    "    elif cohen_d < 0.8:\n",
    "        effect_size = 'medium'\n",
    "    else:\n",
    "        effect_size = 'large'\n",
    "        \n",
    "    url_t_test_results.append([\n",
    "        value,\n",
    "        \"link\",\n",
    "        t_stat,\n",
    "        pvalue,\n",
    "        pvalue < 0.05,\n",
    "        pvalue < 0.01,\n",
    "        cohen_d,\n",
    "        effect_size,\n",
    "        alignment_diffs_url[value].count()\n",
    "    ])\n",
    "    \n",
    "url_t_test_results_df = pd.DataFrame(url_t_test_results, columns=[\n",
    "    'group',\n",
    "    'method',\n",
    "    't_stat',\n",
    "    'pvalue',\n",
    "    'significant_05',\n",
    "    'significant_01',\n",
    "    'cohen_d',\n",
    "    'effect_size',\n",
    "    'n'\n",
    "])\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>group</th>\n",
       "      <th>method</th>\n",
       "      <th>t_stat</th>\n",
       "      <th>pvalue</th>\n",
       "      <th>significant_05</th>\n",
       "      <th>significant_01</th>\n",
       "      <th>cohen_d</th>\n",
       "      <th>effect_size</th>\n",
       "      <th>n</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>Left</td>\n",
       "      <td>link</td>\n",
       "      <td>4.146558</td>\n",
       "      <td>4.144082e-05</td>\n",
       "      <td>True</td>\n",
       "      <td>True</td>\n",
       "      <td>0.209700</td>\n",
       "      <td>small</td>\n",
       "      <td>391</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>C. Left</td>\n",
       "      <td>link</td>\n",
       "      <td>-2.324818</td>\n",
       "      <td>2.059695e-02</td>\n",
       "      <td>True</td>\n",
       "      <td>False</td>\n",
       "      <td>0.117873</td>\n",
       "      <td>small</td>\n",
       "      <td>389</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>Center</td>\n",
       "      <td>link</td>\n",
       "      <td>-15.156787</td>\n",
       "      <td>3.611706e-41</td>\n",
       "      <td>True</td>\n",
       "      <td>True</td>\n",
       "      <td>0.764559</td>\n",
       "      <td>medium</td>\n",
       "      <td>393</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>C. Right</td>\n",
       "      <td>link</td>\n",
       "      <td>-4.092429</td>\n",
       "      <td>5.205458e-05</td>\n",
       "      <td>True</td>\n",
       "      <td>True</td>\n",
       "      <td>0.208570</td>\n",
       "      <td>small</td>\n",
       "      <td>385</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>Right</td>\n",
       "      <td>link</td>\n",
       "      <td>-4.954729</td>\n",
       "      <td>1.128585e-06</td>\n",
       "      <td>True</td>\n",
       "      <td>True</td>\n",
       "      <td>0.264088</td>\n",
       "      <td>small</td>\n",
       "      <td>352</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "      group method     t_stat        pvalue  significant_05  significant_01  \\\n",
       "0      Left   link   4.146558  4.144082e-05            True            True   \n",
       "1   C. Left   link  -2.324818  2.059695e-02            True           False   \n",
       "2    Center   link -15.156787  3.611706e-41            True            True   \n",
       "3  C. Right   link  -4.092429  5.205458e-05            True            True   \n",
       "4     Right   link  -4.954729  1.128585e-06            True            True   \n",
       "\n",
       "    cohen_d effect_size    n  \n",
       "0  0.209700       small  391  \n",
       "1  0.117873       small  389  \n",
       "2  0.764559      medium  393  \n",
       "3  0.208570       small  385  \n",
       "4  0.264088       small  352  "
      ]
     },
     "execution_count": 18,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "url_t_test_results_df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [],
   "source": [
    "hashtag_t_test_results = []\n",
    "for value in name_mapping.values():\n",
    "    t_stat, pvalue, cohen_d = do_t_test(alignment_diffs_hashtag[value].dropna())\n",
    "    \n",
    "    if cohen_d < 0.5:\n",
    "        effect_size = 'small'\n",
    "    elif cohen_d < 0.8:\n",
    "        effect_size = 'medium'\n",
    "    else:\n",
    "        effect_size = 'large'\n",
    "        \n",
    "    hashtag_t_test_results.append([\n",
    "        value,\n",
    "        \"hashtag\",\n",
    "        t_stat,\n",
    "        pvalue,\n",
    "        pvalue < 0.05,\n",
    "        pvalue < 0.01,\n",
    "        cohen_d,\n",
    "        effect_size,\n",
    "        alignment_diffs_hashtag[value].count()\n",
    "    ])\n",
    "    \n",
    "hashtag_t_test_results_df = pd.DataFrame(hashtag_t_test_results, columns=[\n",
    "    'group',\n",
    "    'method',\n",
    "    't_stat',\n",
    "    'pvalue',\n",
    "    'significant_05',\n",
    "    'significant_01',\n",
    "    'cohen_d',\n",
    "    'effect_size',\n",
    "    'n'\n",
    "])\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>group</th>\n",
       "      <th>method</th>\n",
       "      <th>t_stat</th>\n",
       "      <th>pvalue</th>\n",
       "      <th>significant_05</th>\n",
       "      <th>significant_01</th>\n",
       "      <th>cohen_d</th>\n",
       "      <th>effect_size</th>\n",
       "      <th>n</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>Left</td>\n",
       "      <td>hashtag</td>\n",
       "      <td>-6.033468</td>\n",
       "      <td>3.763578e-09</td>\n",
       "      <td>True</td>\n",
       "      <td>True</td>\n",
       "      <td>0.306698</td>\n",
       "      <td>small</td>\n",
       "      <td>387</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>C. Left</td>\n",
       "      <td>hashtag</td>\n",
       "      <td>2.573025</td>\n",
       "      <td>1.045799e-02</td>\n",
       "      <td>True</td>\n",
       "      <td>False</td>\n",
       "      <td>0.131475</td>\n",
       "      <td>small</td>\n",
       "      <td>383</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>Center</td>\n",
       "      <td>hashtag</td>\n",
       "      <td>1.797072</td>\n",
       "      <td>7.309583e-02</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>0.090766</td>\n",
       "      <td>small</td>\n",
       "      <td>392</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>C. Right</td>\n",
       "      <td>hashtag</td>\n",
       "      <td>4.669914</td>\n",
       "      <td>4.186471e-06</td>\n",
       "      <td>True</td>\n",
       "      <td>True</td>\n",
       "      <td>0.239247</td>\n",
       "      <td>small</td>\n",
       "      <td>381</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>Right</td>\n",
       "      <td>hashtag</td>\n",
       "      <td>-10.573009</td>\n",
       "      <td>7.251819e-23</td>\n",
       "      <td>True</td>\n",
       "      <td>True</td>\n",
       "      <td>0.563543</td>\n",
       "      <td>medium</td>\n",
       "      <td>352</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "      group   method     t_stat        pvalue  significant_05  significant_01  \\\n",
       "0      Left  hashtag  -6.033468  3.763578e-09            True            True   \n",
       "1   C. Left  hashtag   2.573025  1.045799e-02            True           False   \n",
       "2    Center  hashtag   1.797072  7.309583e-02           False           False   \n",
       "3  C. Right  hashtag   4.669914  4.186471e-06            True            True   \n",
       "4     Right  hashtag -10.573009  7.251819e-23            True            True   \n",
       "\n",
       "    cohen_d effect_size    n  \n",
       "0  0.306698       small  387  \n",
       "1  0.131475       small  383  \n",
       "2  0.090766       small  392  \n",
       "3  0.239247       small  381  \n",
       "4  0.563543      medium  352  "
      ]
     },
     "execution_count": 20,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "hashtag_t_test_results_df"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.5"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}