notebooks/Standard errors.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "id": "a34f30c6",
   "metadata": {
    "pycharm": {
     "name": "#%% md\n"
    }
   },
   "source": [
    "# Standard errors\n",
    "\n",
    "This is a notebook demonstrating how to obtain standard errors for your generated impact estimates."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "id": "43b770ca",
   "metadata": {},
   "outputs": [],
   "source": [
    "%load_ext autoreload\n",
    "%autoreload 2\n",
    "import os, sys\n",
    "import warnings\n",
    "warnings.filterwarnings('ignore') # suppress sklearn deprecation warnings for now..\n",
    "\n",
    "import pandas as pd\n",
    "import numpy as np\n",
    "from sklearn.model_selection import train_test_split\n",
    "\n",
    "# the below checks for whether we run dowhy, causaltune, and FLAML from source\n",
    "root_path = root_path = os.path.realpath('../..')\n",
    "try:\n",
    "    import causaltune\n",
    "except ModuleNotFoundError:\n",
    "    sys.path.append(os.path.join(root_path, \"causaltune\"))\n",
    "\n",
    "try:\n",
    "    import dowhy\n",
    "except ModuleNotFoundError:\n",
    "    sys.path.append(os.path.join(root_path, \"dowhy\"))\n",
    "\n",
    "try:\n",
    "    import flaml\n",
    "except ModuleNotFoundError:\n",
    "    sys.path.append(os.path.join(root_path, \"FLAML\"))\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "53241021",
   "metadata": {
    "pycharm": {
     "name": "#%%\n"
    }
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<style>.container { width:100% !important; }</style>"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "# this makes the notebook expand to full width of the browser window\n",
    "from IPython.core.display import display, HTML\n",
    "display(HTML(\"<style>.container { width:100% !important; }</style>\"))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "id": "5ed9b5f7",
   "metadata": {
    "pycharm": {
     "name": "#%%\n"
    }
   },
   "outputs": [],
   "source": [
    "%%javascript\n",
    "\n",
    "// turn off scrollable windows for large output\n",
    "IPython.OutputArea.prototype._should_scroll = function(lines) {\n",
    "    return false;\n",
    "}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "da208ce6",
   "metadata": {
    "pycharm": {
     "name": "#%%\n"
    }
   },
   "outputs": [],
   "source": [
    "from causaltune import CausalTune\n",
    "from causaltune.datasets import synth_ihdp"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "ab536d1b",
   "metadata": {},
   "source": [
    "## Loading data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "id": "96719b4d",
   "metadata": {},
   "outputs": [],
   "source": [
    "# load toy dataset and apply standard pre-processing\n",
    "cd = synth_ihdp()\n",
    "cd.preprocess_dataset()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "id": "49e4721b",
   "metadata": {
    "pycharm": {
     "name": "#%%\n"
    }
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>treatment</th>\n",
       "      <th>y_factual</th>\n",
       "      <th>random</th>\n",
       "      <th>x1</th>\n",
       "      <th>x2</th>\n",
       "      <th>x3</th>\n",
       "      <th>x4</th>\n",
       "      <th>x5</th>\n",
       "      <th>x6</th>\n",
       "      <th>x7</th>\n",
       "      <th>...</th>\n",
       "      <th>x16</th>\n",
       "      <th>x17</th>\n",
       "      <th>x18</th>\n",
       "      <th>x19</th>\n",
       "      <th>x20</th>\n",
       "      <th>x21</th>\n",
       "      <th>x22</th>\n",
       "      <th>x23</th>\n",
       "      <th>x24</th>\n",
       "      <th>x25</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>1</td>\n",
       "      <td>5.599916</td>\n",
       "      <td>1.0</td>\n",
       "      <td>-0.528603</td>\n",
       "      <td>-0.343455</td>\n",
       "      <td>1.128554</td>\n",
       "      <td>0.161703</td>\n",
       "      <td>-0.316603</td>\n",
       "      <td>1.295216</td>\n",
       "      <td>1.0</td>\n",
       "      <td>...</td>\n",
       "      <td>1.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>0</td>\n",
       "      <td>6.875856</td>\n",
       "      <td>1.0</td>\n",
       "      <td>-1.736945</td>\n",
       "      <td>-1.802002</td>\n",
       "      <td>0.383828</td>\n",
       "      <td>2.244319</td>\n",
       "      <td>-0.629189</td>\n",
       "      <td>1.295216</td>\n",
       "      <td>0.0</td>\n",
       "      <td>...</td>\n",
       "      <td>1.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>0</td>\n",
       "      <td>2.996273</td>\n",
       "      <td>0.0</td>\n",
       "      <td>-0.807451</td>\n",
       "      <td>-0.202946</td>\n",
       "      <td>-0.360898</td>\n",
       "      <td>-0.879606</td>\n",
       "      <td>0.808706</td>\n",
       "      <td>-0.526556</td>\n",
       "      <td>0.0</td>\n",
       "      <td>...</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>0</td>\n",
       "      <td>1.366206</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.390083</td>\n",
       "      <td>0.596582</td>\n",
       "      <td>-1.850350</td>\n",
       "      <td>-0.879606</td>\n",
       "      <td>-0.004017</td>\n",
       "      <td>-0.857787</td>\n",
       "      <td>0.0</td>\n",
       "      <td>...</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>0</td>\n",
       "      <td>1.963538</td>\n",
       "      <td>0.0</td>\n",
       "      <td>-1.045228</td>\n",
       "      <td>-0.602710</td>\n",
       "      <td>0.011465</td>\n",
       "      <td>0.161703</td>\n",
       "      <td>0.683672</td>\n",
       "      <td>-0.360940</td>\n",
       "      <td>1.0</td>\n",
       "      <td>...</td>\n",
       "      <td>1.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>5 rows × 28 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "   treatment  y_factual  random        x1        x2        x3        x4  \\\n",
       "0          1   5.599916     1.0 -0.528603 -0.343455  1.128554  0.161703   \n",
       "1          0   6.875856     1.0 -1.736945 -1.802002  0.383828  2.244319   \n",
       "2          0   2.996273     0.0 -0.807451 -0.202946 -0.360898 -0.879606   \n",
       "3          0   1.366206     1.0  0.390083  0.596582 -1.850350 -0.879606   \n",
       "4          0   1.963538     0.0 -1.045228 -0.602710  0.011465  0.161703   \n",
       "\n",
       "         x5        x6   x7  ...  x16  x17  x18  x19  x20  x21  x22  x23  x24  \\\n",
       "0 -0.316603  1.295216  1.0  ...  1.0  1.0  1.0  1.0  0.0  0.0  0.0  0.0  0.0   \n",
       "1 -0.629189  1.295216  0.0  ...  1.0  1.0  1.0  1.0  0.0  0.0  0.0  0.0  0.0   \n",
       "2  0.808706 -0.526556  0.0  ...  1.0  0.0  1.0  1.0  0.0  0.0  0.0  0.0  0.0   \n",
       "3 -0.004017 -0.857787  0.0  ...  1.0  0.0  1.0  1.0  0.0  0.0  0.0  0.0  0.0   \n",
       "4  0.683672 -0.360940  1.0  ...  1.0  1.0  1.0  1.0  0.0  0.0  0.0  0.0  0.0   \n",
       "\n",
       "   x25  \n",
       "0  0.0  \n",
       "1  0.0  \n",
       "2  0.0  \n",
       "3  0.0  \n",
       "4  0.0  \n",
       "\n",
       "[5 rows x 28 columns]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "# inspect the preprocessed dataset\n",
    "display(cd.data.head())"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "d4d1871f",
   "metadata": {},
   "source": [
    "## Model training and standard errors"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "id": "fd4b291e",
   "metadata": {},
   "outputs": [],
   "source": [
    "# training configs\n",
    "\n",
    "# set evaluation metric\n",
    "metric = \"energy_distance\"\n",
    "\n",
    "# it's best to specify either time_budget or components_time_budget, \n",
    "# and let the other one be inferred; time in seconds\n",
    "time_budget = None\n",
    "components_time_budget = 10\n",
    "\n",
    "# specify training set size\n",
    "train_size = 0.7\n"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "e0f63d12",
   "metadata": {},
   "source": [
    "Note that in the example below, we are passing `'cheap_inference'` to `estimator_list`. This configuration will restrict the selection of estimators to the ones that have analytical standard errors.\n",
    "\n",
    "Now if `outcome_model=\"auto\"` in the CausalTune constructor, we search over a simultaneous search space for the EconML estimators and for FLAML wrappers for common regressors. The old behavior is now achieved by `outcome_model=\"nested\"` (Refitting AutoML for each estimator).\n",
    "\n",
    "You can also preprocess the data in the CausalityDataset using one of the popular category encoders: OneHot, WoE, Label, Target."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "id": "097c923e",
   "metadata": {
    "pycharm": {
     "name": "#%%\n"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Fitting a Propensity-Weighted scoring estimator to be used in scoring tasks\n",
      "Propensity Model Fitted Successfully\n",
      "---------------------\n",
      "Best estimator: backdoor.econml.dr.ForestDRLearner\n",
      "Best config: {'estimator': {'estimator_name': 'backdoor.econml.dr.ForestDRLearner', 'min_propensity': 4.1309041114224745e-06, 'n_estimators': 51, 'min_samples_split': 2, 'min_samples_leaf': 5, 'min_weight_fraction_leaf': 0.0, 'max_features': 'log2', 'min_impurity_decrease': 0, 'max_samples': 0.4714678358460523, 'min_balancedness_tol': 0.48107268073765275, 'honest': 1, 'subforest_size': 5}, 'outcome_estimator': {'alpha': 0.0680343251051132, 'fit_intercept': True, 'eps': 3.581001561497127e-16, 'estimator_name': 'lasso_lars'}}\n",
      "Best score: 0.19782534210362535\n"
     ]
    }
   ],
   "source": [
    "ct = CausalTune(\n",
    "    estimator_list='cheap_inference',\n",
    "    metric=metric,\n",
    "    verbose=0,\n",
    "    components_verbose=0,\n",
    "    time_budget=time_budget,\n",
    "    components_time_budget=components_time_budget,\n",
    "    train_size=train_size,\n",
    "    outcome_model=\"auto\"\n",
    ")\n",
    "\n",
    "\n",
    "# run causaltune\n",
    "ct.fit(data=cd, outcome=cd.outcomes[0])\n",
    "\n",
    "print('---------------------')\n",
    "# return best estimator\n",
    "print(f\"Best estimator: {ct.best_estimator}\")\n",
    "# config of best estimator:\n",
    "print(f\"Best config: {ct.best_config}\")\n",
    "# best score:\n",
    "print(f\"Best score: {ct.best_score}\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "id": "dd8b4d04",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([[3.06847504],\n",
       "       [5.10172326],\n",
       "       [2.3049086 ],\n",
       "       [4.39115942],\n",
       "       [4.38397264]])"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "# obtaining effect estimates\n",
    "\n",
    "test_df = ct.test_df\n",
    "\n",
    "cates = ct.effect(test_df)\n",
    "display(cates[:5,])"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "8c819410",
   "metadata": {},
   "source": [
    "Below we show how to generate standard errors using `CausalTune.effect_stderr()`. By default, this will use the `best_estimator` identified during training.\n",
    "\n",
    "If this estimator does not have analytical standard errors, it will be refitted `n_bootstrap_samples`-times on the training data."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "id": "0ee744d2",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([[0.74527346],\n",
       "       [0.76067972],\n",
       "       [0.48614067],\n",
       "       [0.42494167],\n",
       "       [0.52123297]])"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "# generating standard errors by refitting train_df \n",
    "se = ct.effect_stderr(ct.test_df)\n",
    "display(se[:5,])"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "9a474ab5",
   "metadata": {},
   "source": [
    "In addition to merely generating standard errors, we have the option to generate various other statistical inferences for the effect, such as the standard error, z-test score, and p-value for each sample `X{i}`."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "id": "277adbcc",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>point_estimate</th>\n",
       "      <th>stderr</th>\n",
       "      <th>zstat</th>\n",
       "      <th>pvalue</th>\n",
       "      <th>ci_lower</th>\n",
       "      <th>ci_upper</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>X</th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>3.068</td>\n",
       "      <td>0.745</td>\n",
       "      <td>4.117</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.843</td>\n",
       "      <td>4.294</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>5.102</td>\n",
       "      <td>0.761</td>\n",
       "      <td>6.707</td>\n",
       "      <td>0.0</td>\n",
       "      <td>3.851</td>\n",
       "      <td>6.353</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>2.305</td>\n",
       "      <td>0.486</td>\n",
       "      <td>4.741</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.505</td>\n",
       "      <td>3.105</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>4.391</td>\n",
       "      <td>0.425</td>\n",
       "      <td>10.334</td>\n",
       "      <td>0.0</td>\n",
       "      <td>3.692</td>\n",
       "      <td>5.090</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>4.384</td>\n",
       "      <td>0.521</td>\n",
       "      <td>8.411</td>\n",
       "      <td>0.0</td>\n",
       "      <td>3.527</td>\n",
       "      <td>5.241</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   point_estimate  stderr   zstat  pvalue  ci_lower  ci_upper\n",
       "X                                                            \n",
       "0           3.068   0.745   4.117     0.0     1.843     4.294\n",
       "1           5.102   0.761   6.707     0.0     3.851     6.353\n",
       "2           2.305   0.486   4.741     0.0     1.505     3.105\n",
       "3           4.391   0.425  10.334     0.0     3.692     5.090\n",
       "4           4.384   0.521   8.411     0.0     3.527     5.241"
      ]
     },
     "execution_count": 11,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "ct.effect_inference(test_df)[0].summary_frame(alpha=0.1, value=0, decimals=3).head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "2b200c45-d652-42a8-b8f1-611a119143c3",
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.10.14"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}