AB_testing.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 265,
   "metadata": {},
   "outputs": [],
   "source": [
    "import numpy as np\n",
    "import seaborn as sn\n",
    "import matplotlib.pyplot as plt\n",
    "import pandas as pd\n",
    "from sklearn.pipeline import Pipeline\n",
    "import scipy.stats as st\n",
    "import statsmodels.stats.api as sm\n",
    "from statsmodels.stats.proportion import proportions_ztest, proportion_confint\n",
    "from scipy.stats import ttest_1samp\n",
    "\n",
    "pd.set_option('display.max_columns', 500)\n",
    "plt.rc('text', usetex=True)\n",
    "plt.rc('font', family='serif')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "After clustering the customers in ``clustering_analysis.ipynb``, we explore this segmentation with the following null hypothesis:\n",
    "\n",
    "$H_0$: in the 2 clustering scenario, the cluster with customers with higher average income will deal a higher number of times with the company in long term\n",
    "\n",
    "An alternative hypothesis $H_1$ is a negative of $H_0$, in other words this customer group is not dealing with the company more than any group under some statistical significance applied in our test."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Unnamed: 0</th>\n",
       "      <th>ID</th>\n",
       "      <th>Year_Birth</th>\n",
       "      <th>Education</th>\n",
       "      <th>Marital_Status</th>\n",
       "      <th>Income</th>\n",
       "      <th>Kidhome</th>\n",
       "      <th>Teenhome</th>\n",
       "      <th>Dt_Customer</th>\n",
       "      <th>Recency</th>\n",
       "      <th>...</th>\n",
       "      <th>NumStorePurchases</th>\n",
       "      <th>NumWebVisitsMonth</th>\n",
       "      <th>AcceptedCmp3</th>\n",
       "      <th>AcceptedCmp4</th>\n",
       "      <th>AcceptedCmp5</th>\n",
       "      <th>AcceptedCmp1</th>\n",
       "      <th>AcceptedCmp2</th>\n",
       "      <th>Complain</th>\n",
       "      <th>Response</th>\n",
       "      <th>Accepted</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>0</td>\n",
       "      <td>5524</td>\n",
       "      <td>1957</td>\n",
       "      <td>Graduation</td>\n",
       "      <td>Single</td>\n",
       "      <td>58138.0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>2012-04-09</td>\n",
       "      <td>58</td>\n",
       "      <td>...</td>\n",
       "      <td>4</td>\n",
       "      <td>7</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>1</td>\n",
       "      <td>2174</td>\n",
       "      <td>1954</td>\n",
       "      <td>Graduation</td>\n",
       "      <td>Single</td>\n",
       "      <td>46344.0</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>2014-08-03</td>\n",
       "      <td>38</td>\n",
       "      <td>...</td>\n",
       "      <td>2</td>\n",
       "      <td>5</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>2</td>\n",
       "      <td>4141</td>\n",
       "      <td>1965</td>\n",
       "      <td>Graduation</td>\n",
       "      <td>Together</td>\n",
       "      <td>71613.0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>2013-08-21</td>\n",
       "      <td>26</td>\n",
       "      <td>...</td>\n",
       "      <td>10</td>\n",
       "      <td>4</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>3</td>\n",
       "      <td>6182</td>\n",
       "      <td>1984</td>\n",
       "      <td>Graduation</td>\n",
       "      <td>Together</td>\n",
       "      <td>26646.0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>2014-10-02</td>\n",
       "      <td>26</td>\n",
       "      <td>...</td>\n",
       "      <td>4</td>\n",
       "      <td>6</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>4</td>\n",
       "      <td>5324</td>\n",
       "      <td>1981</td>\n",
       "      <td>PhD</td>\n",
       "      <td>Married</td>\n",
       "      <td>58293.0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>2014-01-19</td>\n",
       "      <td>94</td>\n",
       "      <td>...</td>\n",
       "      <td>6</td>\n",
       "      <td>5</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>5 rows × 29 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "   Unnamed: 0    ID  Year_Birth   Education Marital_Status   Income  Kidhome  \\\n",
       "0           0  5524        1957  Graduation         Single  58138.0        0   \n",
       "1           1  2174        1954  Graduation         Single  46344.0        1   \n",
       "2           2  4141        1965  Graduation       Together  71613.0        0   \n",
       "3           3  6182        1984  Graduation       Together  26646.0        1   \n",
       "4           4  5324        1981         PhD        Married  58293.0        1   \n",
       "\n",
       "   Teenhome Dt_Customer  Recency  ...  NumStorePurchases  NumWebVisitsMonth  \\\n",
       "0         0  2012-04-09       58  ...                  4                  7   \n",
       "1         1  2014-08-03       38  ...                  2                  5   \n",
       "2         0  2013-08-21       26  ...                 10                  4   \n",
       "3         0  2014-10-02       26  ...                  4                  6   \n",
       "4         0  2014-01-19       94  ...                  6                  5   \n",
       "\n",
       "   AcceptedCmp3  AcceptedCmp4  AcceptedCmp5  AcceptedCmp1  AcceptedCmp2  \\\n",
       "0             0             0             0             0             0   \n",
       "1             0             0             0             0             0   \n",
       "2             0             0             0             0             0   \n",
       "3             0             0             0             0             0   \n",
       "4             0             0             0             0             0   \n",
       "\n",
       "   Complain  Response  Accepted  \n",
       "0         0         1         0  \n",
       "1         0         0         0  \n",
       "2         0         0         0  \n",
       "3         0         0         0  \n",
       "4         0         0         0  \n",
       "\n",
       "[5 rows x 29 columns]"
      ]
     },
     "execution_count": 21,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "data_cleaned = pd.read_csv('dataset/cleaned_marketing_campaign.csv', sep=',')\n",
    "data_cleaned.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0    1747\n",
       "1     322\n",
       "2      81\n",
       "3      44\n",
       "4      11\n",
       "Name: Accepted, dtype: int64"
      ]
     },
     "execution_count": 26,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "data_cleaned['Accepted'].value_counts()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "metadata": {},
   "outputs": [],
   "source": [
    "data_cleaned['Accepted'] = (data_cleaned['Accepted'] != 0).astype(int)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 34,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0    1747\n",
       "1     458\n",
       "Name: Accepted, dtype: int64"
      ]
     },
     "execution_count": 34,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "data_cleaned['Accepted'].value_counts()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 69,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Confidence Interval for the whole dataset: (0.18759767337763758, 0.22782182775614926)\n"
     ]
    }
   ],
   "source": [
    "m,n = data_cleaned.shape\n",
    "SE = st.sem(data_cleaned['Accepted'])\n",
    "SE = np.std(data_cleaned['Accepted'])/np.sqrt(m)\n",
    "mean = np.mean(data_cleaned['Accepted'])\n",
    "\n",
    "t_value = st.t.ppf(q=0.99, df=m-1)\n",
    "\n",
    "CI = mean - t_value * SE, mean + t_value * SE\n",
    "print('Confidence Interval for the whole dataset:', CI)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 68,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([0, 1])"
      ]
     },
     "execution_count": 68,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "y_pred = np.load('dataset/prediction_2_clusters.npy')\n",
    "np.unique(y_pred)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 107,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAjsAAAGwCAYAAABPSaTdAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjYuMywgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/P9b71AAAACXBIWXMAAA9hAAAPYQGoP6dpAAA4F0lEQVR4nO3deXxU9b3/8XdCdiEJIWYrCatCQIIgkuIKJYLRS2vltkqJglqg/kCUtC5pRRHbG6otRZFKe28FeoGiPoq7RSEgiAaEYGQbEGgwLAkwxJCVJCTf3x+WuU7ZMyeZmZPX8/GYxyNnmc/5zDdq3p75nnMCjDFGAAAANhXo7QYAAABaEmEHAADYGmEHAADYGmEHAADYGmEHAADYGmEHAADYGmEHAADYWpC3G/AFTU1NOnz4sDp06KCAgABvtwMAAC6CMUaVlZVKSkpSYOC5z98QdiQdPnxYycnJ3m4DAAA0w4EDB9S5c+dzbifsSOrQoYOkbwYrMjLSy90AAICLUVFRoeTkZNff8XMh7Eiur64iIyMJOwAA+JkLTUFhgjIAALA1wg4AALA1wg4AALA15uwAAODHGhsb1dDQ4O02WkRwcLDatWvncR3CDgAAfsgYo9LSUpWXl3u7lRYVHR2thIQEj+6DR9gBAMAPnQ46cXFxioiIsN1NcY0xqqmp0dGjRyVJiYmJza5F2AEAwM80Nja6gk6nTp283U6LCQ8PlyQdPXpUcXFxzf5KiwnKAAD4mdNzdCIiIrzcScs7/Rk9mZdE2AEAwE/Z7aurs7HiMxJ2AACArTFnBwAAmyguLpbT6Wy148XGxiolJaXVjtdcXg0769at0/PPP6+CggKVlJTojTfe0B133OHafq5TV88995weffRRSVLXrl311VdfuW3Pzc3VE0880WJ9AwDga4qLi5Wa2ls1NbWtdsyIiHA5HLt8PvB4NexUV1erf//+uv/++3XnnXeesb2kpMRt+R//+IceeOABjR492m39zJkzNWHCBNfyhZ5+CgCA3TidTtXU1Gpx9gilJse0+PEcB8qUNftDOZ3OSwo7FzrR0RK8GnYyMzOVmZl5zu0JCQluy2+99ZaGDRum7t27u63v0KHDGfsCANAWpSbHaGCPOG+3cU4XOtHREvxmzs6RI0f03nvvadGiRWdsmzVrlp599lmlpKToJz/5iaZNm6agoHN/tLq6OtXV1bmWKyoqWqRn4GJY/R27v3yHDqBtutCJjpbgN2Fn0aJF6tChwxkpcOrUqRo4cKBiYmL06aefKicnRyUlJZo9e/Y5a+Xm5uqZZ55p6ZaBC2qJ79j95Tt0AGgtfhN2XnnlFY0dO1ZhYWFu67Ozs10/p6WlKSQkRJMmTVJubq5CQ0PPWisnJ8ftfRUVFUpOTm6ZxoHzsPo79uZ+hw4AduYXYefjjz/W7t279eqrr15w3/T0dJ06dUr79+9Xr169zrpPaGjoOYMQ4A2+/h07APgzv7ip4F/+8hddc8016t+//wX3LSwsVGBgoOLi+MMBAAC8fGanqqpKe/fudS0XFRWpsLBQMTExrlPwFRUVev311/X73//+jPfn5+dr48aNGjZsmDp06KD8/HxNmzZNWVlZ6tixY6t9DgAA4Lu8GnY2b96sYcOGuZZPz6MZN26cFi5cKElatmyZjDEaM2bMGe8PDQ3VsmXLNGPGDNXV1albt26aNm2a23wcAADaEseBMp8+zsWc6LCaV8PO0KFDZYw57z4TJ07UxIkTz7pt4MCB2rBhQ0u0BgCAX4mNjVVERLiyZn/YaseMiAhXbGzsJb3nYk50WM0vJigDAIDzS0lJkcOxy+efjXUxJzqsRtgBAMAmUlJSuO3EWfjF1VgAAADNRdgBAAC2RtgBAAC2RtgBAAC2RtgBAAC2RtgBAAC2RtgBAAC2xn12AACwieLiYp+/qaA3EHYAALCB4uJi9U5NVW1NTasdMzwiQrscjksOPPPmzdPzzz+v0tJS9e/fX3PnztXgwYNbqEvCDgAAtuB0OlVbU6NRD+eqU+duLX684weL9M4LOXI6nZcUdl599VVlZ2dr/vz5Sk9P15w5czRy5Ejt3r1bcXFxLdIrYQcAABvp1LmbErr38XYb5zR79mxNmDBB9913nyRp/vz5eu+99/TKK6/oiSeeaJFjMkEZAAC0ivr6ehUUFCgjI8O1LjAwUBkZGcrPz2+x4xJ2AABAq3A6nWpsbFR8fLzb+vj4eJWWlrbYcQk7AADA1gg7AACgVcTGxqpdu3Y6cuSI2/ojR44oISGhxY5L2AEAAK0iJCRE11xzjfLy8lzrmpqalJeXpyFDhrTYcbkaCwAAtJrs7GyNGzdOgwYN0uDBgzVnzhxVV1e7rs5qCYQdAABs5PjBIp8+zl133aVjx47pqaeeUmlpqa6++mqtWLHijEnLViLsAM1g1S3ZHQ6HBd0AwDfzYcIjIvTOCzmtdszwiAjFxsZe8vumTJmiKVOmtEBHZ0fYAS5RcXGxUlN7q6am1rKa1a14e3cA9pSSkqJdDgfPxjoLwg5wiZxOp2pqarU4e4RSk2M8qvX+5v2avmSD6uvrLeoOQFuWkpLiF+GjtRF2gGZKTY7RwB6ePcfFcaDMom4AAOfCpecAAMDWCDsAAPgpY4y3W2hxVnxGwg4AAH4mODhYklTTBi5uOP0ZT3/m5mDODgAAfqZdu3aKjo7W0aNHJUkREREKCAjwclfWMsaopqZGR48eVXR0tNq1a9fsWoQdAAD80OlnSZ0OPHYVHR3t8XOzCDsAAPihgIAAJSYmKi4uTg0NDd5up0UEBwd7dEbnNMIOAAB+rF27dpYEAjtjgjIAALA1wg4AALA1wg4AALA1wg4AALA1wg4AALA1wg4AALA1wg4AALA17rMD2JDD4bCsVmxsrFJSUiyrBwCtzathZ926dXr++edVUFCgkpISvfHGG7rjjjtc28ePH69Fixa5vWfkyJFasWKFa7msrEwPPfSQ3nnnHQUGBmr06NF64YUX1L59+9b6GIDPKPm6WgGSsrKyLKsZEREuh2MXgQeA3/Jq2Kmurlb//v11//3368477zzrPrfeeqsWLFjgWg4NDXXbPnbsWJWUlGjlypVqaGjQfffdp4kTJ2rp0qUt2jvgi8qr6mQk/XHCEKX36eJxPceBMmXN/lBOp5OwA8BveTXsZGZmKjMz87z7hIaGnvMBYA6HQytWrNCmTZs0aNAgSdLcuXN122236Xe/+52SkpIs7xnwB1cmRWlgjzhvtwEAPsHnJyh/9NFHiouLU69evfTggw/q+PHjrm35+fmKjo52BR1JysjIUGBgoDZu3HjOmnV1daqoqHB7AQAAe/LpsHPrrbfqr3/9q/Ly8vTb3/5Wa9euVWZmphobGyVJpaWliotz/7/XoKAgxcTEqLS09Jx1c3NzFRUV5XolJye36OcAAADe49NXY919992un/v166e0tDT16NFDH330kYYPH97sujk5OcrOznYtV1RUEHgAALApnz6z8++6d++u2NhY7d27V5KUkJCgo0ePuu1z6tQplZWVnXOej/TNPKDIyEi3FwAAsCe/CjsHDx7U8ePHlZiYKEkaMmSIysvLVVBQ4Npn9erVampqUnp6urfaBAAAPsSrX2NVVVW5ztJIUlFRkQoLCxUTE6OYmBg988wzGj16tBISErRv3z499thj6tmzp0aOHClJSk1N1a233qoJEyZo/vz5amho0JQpU3T33XdzJRYAAJDk5bCzefNmDRs2zLV8eh7NuHHj9PLLL2vr1q1atGiRysvLlZSUpBEjRujZZ591u9fOkiVLNGXKFA0fPtx1U8EXX3yx1T8LAHxbcXGxnE6nZfW4kzXQfF4NO0OHDpUx5pzbP/jggwvWiImJ4QaCAHxKcXGxeqemqramxrKa4RER2uVwEHiAZvDpq7EAwB85nU7V1tRo1MO56tS5m8f1jh8s0jsv5HAna6CZCDsA0EI6de6mhO59vN0G0Ob51dVYAAAAl4qwAwAAbI2wAwAAbI2wAwAAbI0JymgTrLznicPhsKQOAKB1EHZge8XFxUpN7a2amlpL61ZbeA8VAEDLIezA9pxOp2pqarU4e4RSk2M8rvf+5v2avmSD6uvrLegOANDSCDtoM1KTYzSwR5zHdRwHyizoBgDQWgg7AOAnrJovxnO20NYQdgDAx1V97ZQUoKysLEvq8ZwttDWEHQDwcXXVlZKMht7/S3VNTfOoFs/ZQltE2AEAPxGdmMKztoBm4KaCAADA1gg7AADA1gg7AADA1gg7AADA1gg7AADA1gg7AADA1rj0HMAFcedeAP6MsAPgnEq+rlaAZNmdeyMiwuVw7CLwAGhVhB0A51ReVScj6Y8Thii9TxePajkOlClr9ofcuddHWHW2TuKMHXwfYQfABV2ZFGXJE+PhfVY/Z0viWVvwfYQdAJBUXFwsp9NpSS0rz5pYzcrnbEk8awv+gbADoM0rLi5W79RU1dbUWFq3pqbW0npW4jlbaEsIOwDaPKfTqdqaGo16OFedOnfzuN6+Lev18d9eUn19nQXdAfAUYQcA/qVT526WnO04frDIgm4AWIWbCgIAAFsj7AAAAFsj7AAAAFsj7AAAAFsj7AAAAFsj7AAAAFsj7AAAAFsj7AAAAFsj7AAAAFsj7AAAAFsj7AAAAFsj7AAAAFvzathZt26dRo0apaSkJAUEBOjNN990bWtoaNDjjz+ufv366bLLLlNSUpLuvfdeHT582K1G165dFRAQ4PaaNWtWK38SAADgq7wadqqrq9W/f3/NmzfvjG01NTXasmWLpk+fri1btmj58uXavXu3vv/975+x78yZM1VSUuJ6PfTQQ63RPgAA8ANB3jx4ZmamMjMzz7otKipKK1eudFv30ksvafDgwSouLlZKSoprfYcOHZSQkHDRx62rq1NdXZ1ruaKi4hI7BwAA/sKv5uycOHFCAQEBio6Odls/a9YsderUSQMGDNDzzz+vU6dOnbdObm6uoqKiXK/k5OQW7BoAAHiTV8/sXIqTJ0/q8ccf15gxYxQZGelaP3XqVA0cOFAxMTH69NNPlZOTo5KSEs2ePfuctXJycpSdne1arqioIPAAAGBTfhF2Ghoa9OMf/1jGGL388stu274dWtLS0hQSEqJJkyYpNzdXoaGhZ60XGhp6zm0AAMBefP5rrNNB56uvvtLKlSvdzuqcTXp6uk6dOqX9+/e3ToMAAMCn+fSZndNBZ8+ePVqzZo06dep0wfcUFhYqMDBQcXFxrdAhAADwdV4NO1VVVdq7d69ruaioSIWFhYqJiVFiYqL+8z//U1u2bNG7776rxsZGlZaWSpJiYmIUEhKi/Px8bdy4UcOGDVOHDh2Un5+vadOmKSsrSx07dvTWxwKANsfhcFhSJzY21u1qW8AKXg07mzdv1rBhw1zLp+ffjBs3TjNmzNDbb78tSbr66qvd3rdmzRoNHTpUoaGhWrZsmWbMmKG6ujp169ZN06ZNc5vHAwBoOVVfOyUFKCsry5J64RER2uVwEHhgKa+GnaFDh8oYc87t59smSQMHDtSGDRusbgsAcJHqqislGQ29/5fqmprmUa3jB4v0zgs5cjqdhB1Yyqfn7AAA/EN0YooSuvfxdhvAWfn81VgAAACeIOwAAABb42ss+Kzi4mI5nU6P61h1lQgAwD8RduCTiouLlZraWzU1tZbVrK6psawWAMB/EHbgk5xOp2pqarU4e4RSk2M8qvX+5v2avmSD6uvrLeoOAOBPCDvwaanJMRrYw7O7YTsOlFnUDQDAHzFBGQAA2BphBwAA2BphBwAA2BphBwAA2BphBwAA2BphBwAA2BphBwAA2BphBwAA2BphBwAA2BphBwAA2BphBwAA2BphBwAA2BphBwAA2BphBwAA2BphBwAA2BphBwAA2BphBwAA2BphBwAA2FqQtxsA0LY4HA7LasXGxiolJcWyegDsibADoFWUfF2tAElZWVmW1YyICJfDsYvAA+C8CDsAWkV5VZ2MpD9OGKL0Pl08ruc4UKas2R/K6XQSdgCcF2EHQKu6MilKA3vEebsNAG0IE5QBAICtEXYAAICtEXYAAICtEXYAAICtEXYAAICtEXYAAICtEXYAAICtEXYAAICtEXYAAICtEXYAAICteTXsrFu3TqNGjVJSUpICAgL05ptvum03xuipp55SYmKiwsPDlZGRoT179rjtU1ZWprFjxyoyMlLR0dF64IEHVFVV1YqfAgAA+DKvhp3q6mr1799f8+bNO+v25557Ti+++KLmz5+vjRs36rLLLtPIkSN18uRJ1z5jx47Vjh07tHLlSr377rtat26dJk6c2FofAQAA+LhmhZ3u3bvr+PHjZ6wvLy9X9+7dL7pOZmamfv3rX+uHP/zhGduMMZozZ46efPJJ/eAHP1BaWpr++te/6vDhw64zQA6HQytWrND//M//KD09XTfccIPmzp2rZcuW6fDhw835aAAAwGaaFXb279+vxsbGM9bX1dXp0KFDHjclSUVFRSotLVVGRoZrXVRUlNLT05Wfny9Jys/PV3R0tAYNGuTaJyMjQ4GBgdq4ceM5a9fV1amiosLtBQAA7CnoUnZ+++23XT9/8MEHioqKci03NjYqLy9PXbt2taSx0tJSSVJ8fLzb+vj4eNe20tJSxcXFuW0PCgpSTEyMa5+zyc3N1TPPPGNJnwAAwLddUti54447JEkBAQEaN26c27bg4GB17dpVv//97y1rrqXk5OQoOzvbtVxRUaHk5GQvdgQAAFrKJYWdpqYmSVK3bt20adMmxcbGtkhTkpSQkCBJOnLkiBITE13rjxw5oquvvtq1z9GjR93ed+rUKZWVlbnefzahoaEKDQ21vmkAAOBzmjVnp6ioqEWDjvRNoEpISFBeXp5rXUVFhTZu3KghQ4ZIkoYMGaLy8nIVFBS49lm9erWampqUnp7eov0BAAD/cElndr4tLy9PeXl5Onr0qOuMz2mvvPLKRdWoqqrS3r17XctFRUUqLCxUTEyMUlJS9Mgjj+jXv/61rrjiCnXr1k3Tp09XUlKS6+u01NRU3XrrrZowYYLmz5+vhoYGTZkyRXfffbeSkpKa+9EAAICNNCvsPPPMM5o5c6YGDRqkxMREBQQENOvgmzdv1rBhw1zLp+fRjBs3TgsXLtRjjz2m6upqTZw4UeXl5brhhhu0YsUKhYWFud6zZMkSTZkyRcOHD1dgYKBGjx6tF198sVn9AAAA+2lW2Jk/f74WLlyoe+65x6ODDx06VMaYc24PCAjQzJkzNXPmzHPuExMTo6VLl3rUBwAAsK9mzdmpr6/XddddZ3UvAAAAlmtW2PnpT3/K2RQAAOAXmvU11smTJ/XnP/9Zq1atUlpamoKDg922z54925LmAAAAPNWssLN161bXvW62b9/utq25k5UBAABaQrPCzpo1a6zuAwAAoEU0a84OAACAv2jWmZ1hw4ad9+uq1atXN7shAAAAKzUr7Jyer3NaQ0ODCgsLtX379jMeEAoAAOBNzQo7f/jDH866fsaMGaqqqvKoIQAAACtZOmcnKyvrop+LBQAA0BosDTv5+fluz60CAADwtmZ9jXXnnXe6LRtjVFJSos2bN2v69OmWNAYAAGCFZoWdqKgot+XAwED16tVLM2fO1IgRIyxpDAAAwArNCjsLFiywug8AAIAW0aywc1pBQYEcDockqW/fvhowYIAlTQEAAFilWWHn6NGjuvvuu/XRRx8pOjpaklReXq5hw4Zp2bJluvzyy63sEQAAoNmadTXWQw89pMrKSu3YsUNlZWUqKyvT9u3bVVFRoalTp1rdIwAAQLM168zOihUrtGrVKqWmprrW9enTR/PmzWOCMgAA8CnNOrPT1NSk4ODgM9YHBwerqanJ46YAAACs0qyw873vfU8PP/ywDh8+7Fp36NAhTZs2TcOHD7esOQAAAE81K+y89NJLqqioUNeuXdWjRw/16NFD3bp1U0VFhebOnWt1jwAAAM3WrDk7ycnJ2rJli1atWqVdu3ZJklJTU5WRkWFpcwAAAJ66pLCzevVqTZkyRRs2bFBkZKRuueUW3XLLLZKkEydOqG/fvpo/f75uvPHGFmkWAP7d6Xt9ebsGAN91SWFnzpw5mjBhgiIjI8/YFhUVpUmTJmn27NmEHQAtruTrakkBysrKsqxmTU2tZbUA+I5LCjtffPGFfvvb355z+4gRI/S73/3O46YA4ELKq+okGQ29/5fqmprmUa19W9br47+9pPr6OmuaA+BTLinsHDly5KyXnLuKBQXp2LFjHjcFABcrOjFFCd37eFTj+MEii7oB4Isu6Wqs73znO9q+ffs5t2/dulWJiYkeNwUAAGCVSwo7t912m6ZPn66TJ0+esa22tlZPP/20/uM//sOy5gAAADx1SV9jPfnkk1q+fLmuvPJKTZkyRb169ZIk7dq1S/PmzVNjY6N+9atftUijAAAAzXFJYSc+Pl6ffvqpHnzwQeXk5MgYI0kKCAjQyJEjNW/ePMXHx7dIowAAAM1xyTcV7NKli95//319/fXX2rt3r4wxuuKKK9SxY8eW6A8AAMAjzbqDsiR17NhR1157rZW9AAAAWK5Zz8YCAADwF4QdAABga4QdAABga4QdAABga4QdAABga4QdAABga4QdAABgaz4fdrp27aqAgIAzXpMnT5YkDR069IxtP/vZz7zcNQAA8BXNvqlga9m0aZMaGxtdy9u3b9ctt9yiH/3oR651EyZM0MyZM13LERERrdojvlFcXCyn02lJLYfDYUkdAAB8PuxcfvnlbsuzZs1Sjx49dPPNN7vWRUREKCEhobVbw7cUFxcrNbW3ampqLa1bXVNjaT0AQNvj82Hn2+rr67V48WJlZ2crICDAtX7JkiVavHixEhISNGrUKE2fPv28Z3fq6upUV1fnWq6oqGjRvtsCp9OpmppaLc4eodTkGI/rvb95v6Yv2aD6+noLugMAtGV+FXbefPNNlZeXa/z48a51P/nJT9SlSxclJSVp69atevzxx7V7924tX778nHVyc3P1zDPPtELHbU9qcowG9ojzuI7jQJkF3QAA4Gdh5y9/+YsyMzOVlJTkWjdx4kTXz/369VNiYqKGDx+uffv2qUePHmetk5OTo+zsbNdyRUWFkpOTW65xAADgNX4Tdr766iutWrXqvGdsJCk9PV2StHfv3nOGndDQUIWGhlreIwAA8D0+f+n5aQsWLFBcXJxuv/328+5XWFgoSUpMTGyFrgAAgK/zizM7TU1NWrBggcaNG6egoP9red++fVq6dKluu+02derUSVu3btW0adN00003KS0tzYsdAwAAX+EXYWfVqlUqLi7W/fff77Y+JCREq1at0pw5c1RdXa3k5GSNHj1aTz75pJc6BQAAvsYvws6IESNkjDljfXJystauXeuFjgAAgL/wmzk7AAAAzUHYAQAAtkbYAQAAtkbYAQAAtkbYAQAAtkbYAQAAtkbYAQAAtkbYAQAAtkbYAQAAtkbYAQAAtkbYAQAAtkbYAQAAtkbYAQAAtkbYAQAAthbk7QYAwBOVlZVyOp0e1wBgX4QdAH7JWVkrSdq0aZM+373fo1p1h7+UJJ2sPelpWwB8EGEHgF+qrGmQJF3dNUZX9urpUa0vPjumTdukhlMNVrQGwMcQdgD4tQ7hwYqNDPOoxmVhwRZ1A8AXMUEZAADYGmEHAADYGmEHAADYGmEHAADYGmEHAADYGmEHAADYGmEHAADYGmEHAADYGmEHAADYGmEHAADYGmEHAADYGmEHAADYGmEHAADYGmEHAADYGmEHAADYGmEHAADYGmEHAADYGmEHAADYGmEHAADYGmEHAADYGmEHAADYGmEHAADYmk+HnRkzZiggIMDt1bt3b9f2kydPavLkyerUqZPat2+v0aNH68iRI17sGAAA+BqfDjuS1LdvX5WUlLhe69evd22bNm2a3nnnHb3++utau3atDh8+rDvvvNOL3QIAAF8T5O0GLiQoKEgJCQlnrD9x4oT+8pe/aOnSpfre974nSVqwYIFSU1O1YcMGffe73z1nzbq6OtXV1bmWKyoqrG8cAAD4BJ8/s7Nnzx4lJSWpe/fuGjt2rIqLiyVJBQUFamhoUEZGhmvf3r17KyUlRfn5+eetmZubq6ioKNcrOTm5RT8DAADwHp8OO+np6Vq4cKFWrFihl19+WUVFRbrxxhtVWVmp0tJShYSEKDo62u098fHxKi0tPW/dnJwcnThxwvU6cOBAC34KAADgTT79NVZmZqbr57S0NKWnp6tLly567bXXFB4e3uy6oaGhCg0NtaJFAADg43w67Py76OhoXXnlldq7d69uueUW1dfXq7y83O3szpEjR846xweAb9hTckId9x31uM7hsmoLugHQFvhV2KmqqtK+fft0zz336JprrlFwcLDy8vI0evRoSdLu3btVXFysIUOGeLlTAP/OWVErKUAP/jlf0vnn1V2Kkw2NltUCYE8+HXZ+8YtfaNSoUerSpYsOHz6sp59+Wu3atdOYMWMUFRWlBx54QNnZ2YqJiVFkZKQeeughDRky5LxXYgHwjsraBklGN3z/LvXs2d3jeps2btaOT1bqVGOT583BpzgcDstqxcbGKiUlxbJ68E8+HXYOHjyoMWPG6Pjx47r88st1ww03aMOGDbr88sslSX/4wx8UGBio0aNHq66uTiNHjtQf//hHL3cN4HyiYuOU0NnzPz4Ru/ZZ0A18SdXXTkkBysrKsqxmeESEdjkcBJ42zqfDzrJly867PSwsTPPmzdO8efNaqSMAQEupq66UZDT0/l+qa2qax/WOHyzSOy/kyOl0EnbaOJ8OOwCAtic6MUUJ3ft4uw3YiE/fZwcAAMBThB0AAGBrhB0AAGBrhB0AAGBrhB0AAGBrhB0AAGBrhB0AAGBrhB0AAGBrhB0AAGBrhB0AAGBrhB0AAGBrhB0AAGBrhB0AAGBrhB0AAGBrhB0AAGBrhB0AAGBrhB0AAGBrhB0AAGBrhB0AAGBrhB0AAGBrQd5uAN5TXFwsp9NpSS2Hw2FJHQAArEbYaaOKi4uVmtpbNTW1ltatrqmxtB4AAJ4i7LRRTqdTNTW1Wpw9QqnJMR7Xe3/zfk1fskH19fUWdAcAgHUIO21canKMBvaI87iO40CZBd0AAGA9wg4AwNasnFMYGxurlJQUy+qhdRB2AAC2VPW1U1KAsrKyLKsZHhGhXQ4HgcfPEHYAALZUV10pyWjo/b9U19Q0j+sdP1ikd17IkdPpJOz4GcIOAMDWohNTlNC9j7fbgBdxU0EAAGBrhB0AAGBrhB0AAGBrhB0AAGBrhB0AAGBrhB0AAGBrhB0AAGBrhB0AAGBr3FQQAP6lpqpaTqfT4zqVlZUWdAPAKoQdAG1eXUOjJGmHY6d2Hzrmeb3DX0qSTtae9LgWAM/5dNjJzc3V8uXLtWvXLoWHh+u6667Tb3/7W/Xq1cu1z9ChQ7V27Vq3902aNEnz589v7XYB+KlTjU2SpF6Jkeqf1tPjel98dkybtkkNpxo8rgXAcz4ddtauXavJkyfr2muv1alTp/TLX/5SI0aM0M6dO3XZZZe59pswYYJmzpzpWo6IiPBGuwD8XERYkGIjwzyuc1lYsAXdALCKT4edFStWuC0vXLhQcXFxKigo0E033eRaHxERoYSEhIuuW1dXp7q6OtdyRUWF5822kuLiYkvmFDgcDgu6QVuxp+SEOu476lGNw2XVFnUDAJfGp8POvztx4oQkKSYmxm39kiVLtHjxYiUkJGjUqFGaPn36ec/u5Obm6plnnmnRXltCcXGxUlN7q6am1rKa1TU1ltWC/TgraiUF6ME/50vKt6TmyX/NjwGA1uI3YaepqUmPPPKIrr/+el111VWu9T/5yU/UpUsXJSUlaevWrXr88ce1e/duLV++/Jy1cnJylJ2d7VquqKhQcnJyi/ZvBafTqZqaWi3OHqHU5JgLv+E83t+8X9OXbFB9fb1F3cGOKmsbJBnd8P271LNnd49qbdq4WTs+WemaHwMArcVvws7kyZO1fft2rV+/3m39xIkTXT/369dPiYmJGj58uPbt26cePXqctVZoaKhCQ0NbtN+WlJoco4E94jyq4ThQZlE3aAuiYuOU0DnFoxoRu/ZZ1A0AXBq/uKnglClT9O6772rNmjXq3LnzefdNT0+XJO3du7c1WgMAAD7Op8/sGGP00EMP6Y033tBHH32kbt26XfA9hYWFkqTExMQW7g4AAPgDnw47kydP1tKlS/XWW2+pQ4cOKi0tlSRFRUUpPDxc+/bt09KlS3XbbbepU6dO2rp1q6ZNm6abbrpJaWlpXu4eAAD4Ap8OOy+//LKkb24c+G0LFizQ+PHjFRISolWrVmnOnDmqrq5WcnKyRo8erSeffNIL3QIAAF/k02HHGHPe7cnJyWfcPRkAAODb/GKCMgAAQHMRdgAAgK0RdgAAgK0RdgAAgK0RdgAAgK0RdgAAgK0RdgAAgK0RdgAAgK359E0FAcCf1VRVy+l0elynsrLSgm6AtouwAwAWq2tolCTtcOzU7kPHPK93+EtJ0snakx7XAtoiwg4AWOxUY5MkqVdipPqn9fS43hefHdOmbVLDqQaPawFtEWEHAFpIRFiQYiPDPK5zWViwBd0AbRcTlAEAgK0RdgAAgK0RdgAAgK0RdgAAgK0RdgAAgK0RdgAAgK1x6TkAAJfA4XBYUic2NlYpKSmW1ML5EXYAALgIVV87JQUoKyvLknrhERHa5XAQeFoBYQcAgItQV10pyWjo/b9U19Q0j2odP1ikd17IkdPpJOy0AsIOAACXIDoxRQnd+3i7DVwCJigDAABbI+wAAABbI+wAAABbI+wAAABbY4IyYEN7Sk6o476jHtc5XFZtQTcA4F2EHcBGnBW1kgL04J/zJeVbVvdkQ6NltQCgtRF2WlhxcbGcTqcltay6ayfsq7K2QZLRDd+/Sz17dve43qaNm7Xjk5U61djkeXMA4CWEnRZUXFys1NTeqqmptbRudU2NpfVgP1GxcUro7PmNyiJ27bOgGwDwLsJOC3I6naqpqdXi7BFKTY7xuN77m/dr+pINqq+vt6A7AADaBsJOK0hNjtHAHnEe13EcKLOgGwAA2hYuPQcAALZG2AEAALbG11hAM1lxL5uiIxUWdQNcmsrKSkuuFK2srLSgG6BlEXaAS2T9vWwCdOSEtVfsAedy8uRJSdKmTZv0+e79HterO/zlN3VrT3pcC2gphB3gEll5L5udu/bps/df0/biMm3hjse4gJqqao/PxpSfKJck9U2OUd8+PT3u6YvPjmnTNqnhVIPHtdoiK++fFhsbq5QUz285YUeEHaCZrLiXzZ79hyUFKHf5NuUu32ZNY+KOx3ZT96/f5w7HTu0+dMyzWv86ExMaFKjYyDCPe7ssLNjjGm1R1ddOSQHKysqyrGZ4RIR2ORwEnrMg7LQCqy4ZZ36H/dSfrJVkNPDW0Urrc6XH9bjjsT2d/n32SoxU/zTPzsZs/KRU27ZJRvwz4k111ZWSjIbe/0t1TU3zuN7xg0V654UcOZ1Ows5Z2CbszJs3T88//7xKS0vVv39/zZ07V4MHD/ZqTyUlJZIClDX7QwurMr/DjjrEXM4dj3FBEWFBHp+NCQtuZ1E3sEJ0YooSuvfxdhu2Z4uw8+qrryo7O1vz589Xenq65syZo5EjR2r37t2Ki/P8Zn7NVV5eLslo6J1Z6to12eN6Vs7v8JezRFzxBAAXz8o5QHV1dQoNDbWklrfnE9ki7MyePVsTJkzQfffdJ0maP3++3nvvPb3yyit64oknvNydFG3Rc4qsn9/hu2eJuOIJAC5eS8wBUkCAZIwlpbw9n8jvw059fb0KCgqUk5PjWhcYGKiMjAzl55/9j2RdXZ3q6upcyydOnJAkVVRYewag5l8P7DxQVKSG+roL7H1hJV8VSTJK6f9dJSUmeVSr9MgR7f/8E71XcEDVDZ6f1t64u1SS9F5Bsf7p9PzZXWu2HpJk1OOa63X55fEe1bL6s27e882ZpoNffSU1eTbv4fiRb8at9OABbQvxvDdfrkdvvlHP6t5KDx2WJJXs3aFtIZ79STn05Q5J0oFdhWqo8/x/Tny5nuW97SiQZHTFTT9Qp0TPv0k4tn+X9m1cZUm96rKj2rbyNe3fv1/R0dEe9/Ztp/9umwuFMuPnDh06ZCSZTz/91G39o48+agYPHnzW9zz99NNGEi9evHjx4sXLBq8DBw6cNyv4/Zmd5sjJyVF2drZruampSWVlZerUqZMCAgIuqVZFRYWSk5N14MABRUZGWt1qm8AYeobx8xxj6BnGz3OMYfMYY1RZWamkpPN/2+H3YSc2Nlbt2rXTkSNH3NYfOXJECQkJZ31PaGjoGZOuPD21FhkZyT+gHmIMPcP4eY4x9Azj5znG8NJFRUVdcB+/fxBoSEiIrrnmGuXl5bnWNTU1KS8vT0OGDPFiZwAAwBf4/ZkdScrOzta4ceM0aNAgDR48WHPmzFF1dbXr6iwAANB22SLs3HXXXTp27JieeuoplZaW6uqrr9aKFSsUH+/ZVTwXIzQ0VE8//bRl9yJoixhDzzB+nmMMPcP4eY4xbFkBxlh0ET0AAIAP8vs5OwAAAOdD2AEAALZG2AEAALZG2AEAALZG2PHQvHnz1LVrV4WFhSk9PV2fffaZt1tqcbm5ubr22mvVoUMHxcXF6Y477tDu3bvd9jl58qQmT56sTp06qX379ho9evQZN34sLi7W7bffroiICMXFxenRRx/VqVOn3Pb56KOPNHDgQIWGhqpnz55auHDhGf34++9g1qxZCggI0COPPOJax/hd2KFDh5SVlaVOnTopPDxc/fr10+bNm13bjTF66qmnlJiYqPDwcGVkZGjPnj1uNcrKyjR27FhFRkYqOjpaDzzwgKqqqtz22bp1q2688UaFhYUpOTlZzz333Bm9vP766+rdu7fCwsLUr18/vf/++y3zoS3S2Nio6dOnq1u3bgoPD1ePHj307LPPuj1fiPFzt27dOo0aNUpJSUkKCAjQm2++6bbdl8brYnppczx+OFUbtmzZMhMSEmJeeeUVs2PHDjNhwgQTHR1tjhw54u3WWtTIkSPNggULzPbt201hYaG57bbbTEpKiqmqqnLt87Of/cwkJyebvLw8s3nzZvPd737XXHfdda7tp06dMldddZXJyMgwn3/+uXn//fdNbGysycnJce3zz3/+00RERJjs7Gyzc+dOM3fuXNOuXTuzYsUK1z7+/jv47LPPTNeuXU1aWpp5+OGHXesZv/MrKyszXbp0MePHjzcbN240//znP80HH3xg9u7d69pn1qxZJioqyrz55pvmiy++MN///vdNt27dTG1trWufW2+91fTv399s2LDBfPzxx6Znz55mzJgxru0nTpww8fHxZuzYsWb79u3mb3/7mwkPDzd/+tOfXPt88sknpl27dua5554zO3fuNE8++aQJDg4227Zta53BaIbf/OY3plOnTubdd981RUVF5vXXXzft27c3L7zwgmsfxs/d+++/b371q1+Z5cuXG0nmjTfecNvuS+N1Mb20NYQdDwwePNhMnjzZtdzY2GiSkpJMbm6uF7tqfUePHjWSzNq1a40xxpSXl5vg4GDz+uuvu/ZxOBxGksnPzzfGfPMfjsDAQFNaWura5+WXXzaRkZGmrq7OGGPMY489Zvr27et2rLvuusuMHDnStezPv4PKykpzxRVXmJUrV5qbb77ZFXYYvwt7/PHHzQ033HDO7U1NTSYhIcE8//zzrnXl5eUmNDTU/O1vfzPGGLNz504jyWzatMm1zz/+8Q8TEBBgDh06ZIwx5o9//KPp2LGja0xPH7tXr16u5R//+Mfm9ttvdzt+enq6mTRpkmcfsgXdfvvt5v7773dbd+edd5qxY8caYxi/C/n3sONL43UxvbRFfI3VTPX19SooKFBGRoZrXWBgoDIyMpSfn+/FzlrfiRMnJEkxMTGSpIKCAjU0NLiNTe/evZWSkuIam/z8fPXr18/txo8jR45URUWFduzY4drn2zVO73O6hr//DiZPnqzbb7/9jM/I+F3Y22+/rUGDBulHP/qR4uLiNGDAAP33f/+3a3tRUZFKS0vdPltUVJTS09PdxjA6OlqDBg1y7ZORkaHAwEBt3LjRtc9NN92kkJAQ1z4jR47U7t279fXXX7v2Od84+6LrrrtOeXl5+vLLLyVJX3zxhdavX6/MzExJjN+l8qXxuphe2iLCTjM5nU41NjaecZfm+Ph4lZaWeqmr1tfU1KRHHnlE119/va666ipJUmlpqUJCQs54uOq3x6a0tPSsY3d62/n2qaioUG1trV//DpYtW6YtW7YoNzf3jG2M34X985//1Msvv6wrrrhCH3zwgR588EFNnTpVixYtkvR/Y3C+z1ZaWqq4uDi37UFBQYqJibFknH15DJ944gndfffd6t27t4KDgzVgwAA98sgjGjt2rCTG71L50nhdTC9tkS0eFwHvmTx5srZv367169d7uxW/ceDAAT388MNauXKlwsLCvN2OX2pqatKgQYP0X//1X5KkAQMGaPv27Zo/f77GjRvn5e5832uvvaYlS5Zo6dKl6tu3rwoLC/XII48oKSmJ8YMtcWanmWJjY9WuXbszrpA5cuSIEhISvNRV65oyZYreffddrVmzRp07d3atT0hIUH19vcrLy932//bYJCQknHXsTm873z6RkZEKDw/3299BQUGBjh49qoEDByooKEhBQUFau3atXnzxRQUFBSk+Pp7xu4DExET16dPHbV1qaqqKi4sl/d8YnO+zJSQk6OjRo27bT506pbKyMkvG2ZfH8NFHH3Wd3enXr5/uueceTZs2zXWmkfG7NL40XhfTS1tE2GmmkJAQXXPNNcrLy3Ota2pqUl5enoYMGeLFzlqeMUZTpkzRG2+8odWrV6tbt25u26+55hoFBwe7jc3u3btVXFzsGpshQ4Zo27Ztbv/yr1y5UpGRka4/YkOGDHGrcXqf0zX89XcwfPhwbdu2TYWFha7XoEGDNHbsWNfPjN/5XX/99Wfc7uDLL79Uly5dJEndunVTQkKC22erqKjQxo0b3cawvLxcBQUFrn1Wr16tpqYmpaenu/ZZt26dGhoaXPusXLlSvXr1UseOHV37nG+cfVFNTY0CA93/89+uXTs1NTVJYvwulS+N18X00iZ5e4a0P1u2bJkJDQ01CxcuNDt37jQTJ0400dHRblfI2NGDDz5ooqKizEcffWRKSkpcr5qaGtc+P/vZz0xKSopZvXq12bx5sxkyZIgZMmSIa/vpS6dHjBhhCgsLzYoVK8zll19+1kunH330UeNwOMy8efPOeum0HX4H374ayxjG70I+++wzExQUZH7zm9+YPXv2mCVLlpiIiAizePFi1z6zZs0y0dHR5q233jJbt241P/jBD856KfCAAQPMxo0bzfr1680VV1zhdilweXm5iY+PN/fcc4/Zvn27WbZsmYmIiDjjUuCgoCDzu9/9zjgcDvP000/75KXT3zZu3Djzne98x3Xp+fLly01sbKx57LHHXPswfu4qKyvN559/bj7//HMjycyePdt8/vnn5quvvjLG+NZ4XUwvbQ1hx0Nz5841KSkpJiQkxAwePNhs2LDB2y21OElnfS1YsMC1T21trfl//+//mY4dO5qIiAjzwx/+0JSUlLjV2b9/v8nMzDTh4eEmNjbW/PznPzcNDQ1u+6xZs8ZcffXVJiQkxHTv3t3tGKfZ4Xfw72GH8buwd955x1x11VUmNDTU9O7d2/z5z392297U1GSmT59u4uPjTWhoqBk+fLjZvXu32z7Hjx83Y8aMMe3btzeRkZHmvvvuM5WVlW77fPHFF+aGG24woaGh5jvf+Y6ZNWvWGb289tpr5sorrzQhISGmb9++5r333rP+A1uooqLCPPzwwyYlJcWEhYWZ7t27m1/96ldulzwzfu7WrFlz1v/ujRs3zhjjW+N1Mb20NQHGfOuWmQAAADbDnB0AAGBrhB0AAGBrhB0AAGBrhB0AAGBrhB0AAGBrhB0AAGBrhB0AAGBrhB0AAGBrhB0AAGBrhB0AXjd+/Hjdcccd3m4DgE0RdgAAgK0RdgD4lKFDh2rq1Kl67LHHFBMTo4SEBM2YMcNtn/Lyck2aNEnx8fEKCwvTVVddpXfffde1/e9//7v69u2r0NBQde3aVb///e/d3t+1a1f9+te/1r333qv27durS5cuevvtt3Xs2DH94Ac/UPv27ZWWlqbNmze7vW/9+vW68cYbFR4eruTkZE2dOlXV1dUtNhYArEHYAeBzFi1apMsuu0wbN27Uc889p5kzZ2rlypWSpKamJmVmZuqTTz7R4sWLtXPnTs2aNUvt2rWTJBUUFOjHP/6x7r77bm3btk0zZszQ9OnTtXDhQrdj/OEPf9D111+vzz//XLfffrvuuece3XvvvcrKytKWLVvUo0cP3XvvvTr9rOR9+/bp1ltv1ejRo7V161a9+uqrWr9+vaZMmdKqYwPg0vHUcwBeN378eJWXl+vNN9/U0KFD1djYqI8//ti1ffDgwfre976nWbNm6cMPP1RmZqYcDoeuvPLKM2qNHTtWx44d04cffuha99hjj+m9997Tjh07JH1zZufGG2/U//7v/0qSSktLlZiYqOnTp2vmzJmSpA0bNmjIkCEqKSlRQkKCfvrTn6pdu3b605/+5Kq7fv163XzzzaqurlZYWFiLjA0Az3FmB4DPSUtLc1tOTEzU0aNHJUmFhYXq3LnzWYOOJDkcDl1//fVu666//nrt2bNHjY2NZz1GfHy8JKlfv35nrDt93C+++EILFy5U+/btXa+RI0eqqalJRUVFzf2oAFpBkLcbAIB/Fxwc7LYcEBCgpqYmSVJ4eLjlxwgICDjnutPHraqq0qRJkzR16tQzaqWkpFjSE4CWQdgB4FfS0tJ08OBBffnll2c9u5OamqpPPvnEbd0nn3yiK6+80jWvpzkGDhyonTt3qmfPns2uAcA7+BoLgF+5+eabddNNN2n06NFauXKlioqK9I9//EMrVqyQJP385z9XXl6enn32WX355ZdatGiRXnrpJf3iF7/w6LiPP/64Pv30U02ZMkWFhYXas2eP3nrrLSYoA36AsAPA7/z973/XtddeqzFjxqhPnz567LHHXPNxBg4cqNdee03Lli3TVVddpaeeekozZ87U+PHjPTpmWlqa1q5dqy+//FI33nijBgwYoKeeekpJSUkWfCIALYmrsQAAgK1xZgcAANgaYQcAANgaYQcAANgaYQcAANgaYQcAANgaYQcAANgaYQcAANgaYQcAANgaYQcAANgaYQcAANgaYQcAANja/we4hh6Ojh0HnwAAAABJRU5ErkJggg==",
      "text/plain": [
       "<Figure size 640x480 with 1 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "sn.histplot(data=data_cleaned, hue=y_pred, x='Income', alpha=0.6)\n",
    "plt.legend(labels=[1,0])\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 116,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0    629\n",
      "1    324\n",
      "Name: Accepted, dtype: int64\n",
      "0    1118\n",
      "1     134\n",
      "Name: Accepted, dtype: int64\n",
      "Acceptance rate for cluster 0: 51.51%\n",
      "Acceptance rate for cluster 1: 11.99%\n"
     ]
    }
   ],
   "source": [
    "accep_0 = data_cleaned.iloc[y_pred == 0]['Accepted'] #higher income\n",
    "accep_1 = data_cleaned.iloc[y_pred == 1]['Accepted'] #lower income\n",
    "\n",
    "print(accep_0.value_counts())\n",
    "print(accep_1.value_counts())\n",
    "\n",
    "print('Acceptance rate for cluster 0: %.2f%%'%(100 * (accep_0 == 1).astype(int).sum()/(accep_0 == 0).astype(int).sum()))\n",
    "print('Acceptance rate for cluster 1: %.2f%%'%(100 * (accep_1 == 1).astype(int).sum()/(accep_1 == 0).astype(int).sum()))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 121,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Confidence Interval for the whole dataset: (0.30423715940279245, 0.37572086787947406)\n",
      "Mean value of acceptances of cluster 1: 0.10702875399361023\n"
     ]
    }
   ],
   "source": [
    "m,n = data_cleaned.shape\n",
    "\n",
    "mean_0 = np.mean(accep_0)\n",
    "mean_1 = np.mean(accep_1)\n",
    "\n",
    "SE_0 = st.sem(accep_0)\n",
    "\n",
    "t_value = st.t.ppf(q=0.99, df=m-1)\n",
    "\n",
    "CI_0 = mean_0 - t_value * SE_0, mean_0 + t_value * SE_0\n",
    "print('Confidence Interval for the whole dataset:', CI_0)\n",
    "print('Mean value of acceptances of cluster 1:', mean_1)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "We then reject the null hypothesis $H_0$ with 99% C.L., since the mean of the cluster 1 does not fall into the CI from cluster 0."
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Let us check with other approach using the p-value. We need to establish some parameters to accomplish the A/B test procedure:\n",
    "\n",
    "- **Power of the test** ($1 - \\beta$): This is the probability of successfully predict that both customer cluster differs in Accepted Rate defined in the dataset, when they actually are different. We set it as 99%.\n",
    "- **Significance** ($\\alpha$): The probability of performing a hypothesis test with result out of the defined confidence level. We set it as 1%.\n",
    "- **Effect size**: Can be calculated using the above metrics, and provides how both clusters would differ given some expected (or desired) change in the rate."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 257,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "-0.893825491622076"
      ]
     },
     "execution_count": 257,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "effect_size = sm.proportion_effectsize(0.1199, 0.5151)\n",
    "power_ = 0.99\n",
    "alpha_ = 0.01\n",
    "\n",
    "effect_size"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 249,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "61"
      ]
     },
     "execution_count": 249,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "required_n = sm.NormalIndPower().solve_power(\n",
    "    effect_size,\n",
    "    power=power_,\n",
    "    alpha=alpha_,\n",
    "    ratio=1\n",
    "    )\n",
    "required_n = int(np.ceil(required_n))\n",
    "required_n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 250,
   "metadata": {},
   "outputs": [],
   "source": [
    "control_sample = data_cleaned.iloc[y_pred == 0].sample(required_n)\n",
    "treatment_sample = data_cleaned.iloc[y_pred == 1].sample(required_n)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 251,
   "metadata": {},
   "outputs": [],
   "source": [
    "i_con = control_sample.index\n",
    "i_trea = treatment_sample.index"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 252,
   "metadata": {},
   "outputs": [],
   "source": [
    "ab_test = pd.concat([control_sample, treatment_sample], axis=0)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 253,
   "metadata": {},
   "outputs": [],
   "source": [
    "control_results = ab_test.loc[i_con]['Accepted']\n",
    "treatment_results = ab_test.loc[i_trea]['Accepted']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 254,
   "metadata": {},
   "outputs": [],
   "source": [
    "successes = [control_results.sum(), treatment_results.sum()]\n",
    "nobs = [control_results.count(), treatment_results.count()]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 261,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0.05041955573335427\n",
      "(0.09082134999499744, 0.0001453219375038145) (0.36819504344762555, 0.196575989537906)\n"
     ]
    }
   ],
   "source": [
    "z_stat, pvalue = proportions_ztest(successes, nobs=nobs)\n",
    "(lower_control, lower_treat), (upper_control, upper_treat) = proportion_confint(successes, nobs=nobs, alpha=alpha_)\n",
    "\n",
    "print(pvalue)\n",
    "print((lower_control, lower_treat), (upper_control, upper_treat))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Now, we will perform the above test 10,000 times but for samples with 100 size, in order to check if p-values generally lies before the threshold impossed by $\\alpha$:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 297,
   "metadata": {},
   "outputs": [],
   "source": [
    "p_value_list = []\n",
    "required_n = 100\n",
    "\n",
    "for i in range(10000):\n",
    "    control_sample = data_cleaned.iloc[y_pred == 0].sample(required_n)\n",
    "    treatment_sample = data_cleaned.iloc[y_pred == 1].sample(required_n)\n",
    "\n",
    "    i_con = control_sample.index\n",
    "    i_trea = treatment_sample.index\n",
    "\n",
    "    ab_test = pd.concat([control_sample, treatment_sample], axis=0)\n",
    "\n",
    "    control_results = ab_test.loc[i_con]['Accepted']\n",
    "    treatment_results = ab_test.loc[i_trea]['Accepted']\n",
    "\n",
    "    successes = [control_results.sum(), treatment_results.sum()]\n",
    "    nobs = [control_results.count(), treatment_results.count()]\n",
    "\n",
    "    z_stat, p_value = proportions_ztest(successes, nobs=nobs)\n",
    "    (lower_control, lower_treat), (upper_control, upper_treat) = proportion_confint(successes, nobs=nobs, alpha=alpha_)\n",
    "\n",
    "    p_value_list.append(p_value)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 298,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAkQAAAGyCAYAAAAFw9vDAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjYuMywgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/P9b71AAAACXBIWXMAAA9hAAAPYQGoP6dpAAApG0lEQVR4nO3db2wj52Hn8R+1f7irXUuztJ1N7crxji8o2hRoQlE9oECvaUQlQBoc0JpcH+ADkl4rqogDFOfWIpQrLl1cLqqY9IAWcRBqjSZF4UNX5KZv0he2xv1zBfqiXLEG+g/oHSduWG9sx6aG9K52uWuL94LVSJSolUTNajic7wcY5JnnGVKPOJH52+d5ZibSarVaAgAACLEhvzsAAADgNwIRAAAIPQIRAAAIPQIRAAAIPQIRAAAIPQIRAAAIPQIRAAAIPQIRAAAIPQIRAAAIveN+/NByuSzLsiRJpVJJly9flmEYkiTbtlUsFmWapmzbViaTOXQbAADAvfgSiCzL0uzsrCQpl8tpcnJSKysrkqR0Ou2WbdvW9PS0CoXCodoAAADu5cinzMrlsubn5939VCqlcrks27Zl23bHsaZpuiNJvbYBAADs5cgDUTwe1+XLl919x3EkSbFYTJZlKRaLdRwfi8XcKbZe2gAAAPbiy5RZKpVyy1euXFEymZRhGG442q5Wq/Xctl2z2VSz2XT319fXVavV9OCDDyoSiez7dwAAAP5ptVp699139cgjj2ho6PDjO74Eog2O46hYLLprf+51nFdt8/PzunTp0j57CAAA+lm1WtWP/uiPHvp9fA1E2WxWy8vL7tVghmHsGNWp1WoyDKPntu3m5ub07LPPuvv1el2PPfaYqtWqRkZGvPnF0JN//dd/1W8svarhcx9w69ZW39LvXvzo5v/Zb96UHnmkXb5+XTpzxoeeAgD81mg0NDY2pgceeMCT9/MtEOVyOWWzWZmm6Y7kJJNJ5fP5HccmEgmZptlT23bRaFTRaHRH/cjICIHIZw888IBOnBrWidObIefErWE98MADm+fm2LHNF4yMEIgAIOS8Wu7iy40Zi8Wi4vG4G4aWlpZkGIZM0+w4zrZtJRKJQ7UBAADs5chHiGzbVjqd7qgzDEOZTEaSVCgUlM1mNTExoVKp1HEvoV7bMECOH5c++9nNMgAAHoi0Wq2W353wU6PR0OjoqOr1OlNmPqtWq3rmxRUNx867dWu1N/X80+MaGxvzsWcAgH7j9fc3zzIDAAChx5wDgqXVktbW2uXhYYl7RwEAPMAIEYJlbU06e7a9bQQjAAAOiUAEAABCj0AEAABCj0AEAABCj0AEAABCj0AEAABCj0AEAABCj/sQIViOHZNSqc0yAAAeIBAhWE6dknhOHQDAY0yZAQCA0CMQAQCA0CMQIVhu3mw/vywSaZcBAPAAgQgAAIQegQgAAIQegQgAAIQegQgAAIQegQgAAIQegQgAAIQed6pGsBw7Jn3605tlAAA8QCBCsJw6Jf3pn/rdCwDAgGHKDAAAhB6BCAAAhB6BCMFy86Z05kx749EdAACPsIYIwbO25ncPAAADhhEiAAAQegQiAAAQegQiAAAQegQiAAAQegQiAAAQelxlhmAZGpJ+7uc2ywAAeIBAhGA5fVr6i7/wuxcAgAHDP7EBAEDoEYgAAEDo+RKIyuWyxsfHd9QXi0U5jiPHcbq+plwuS5Js23bLG/u5XE7FYlG5XK7r6zEgbt6UHn64vfHoDgCAR448EBWLRUnqCDQb0um0zp07p3PnzikSiSgSiSiXy0mS8vm8xsfHFYlENDMzI9M0O143OzurVCqlVCql6enpo/ll4I+3325vAAB45MgXVadSqa71juOoUCh0tOdyOc3OzkqSxsfHtbq6KkkyDMM9xrbtjvcxTVOWZXncawAAMMj6ag3R1jBULBZ3hCfDMDrCkCRZlqVYLNZRF4vFuo5AAQAAdNM3l91vDTqO46hWq3VMizmO4063lUold9pst/VCtVqta32z2VSz2XT3G43G4TsPAAACrW8C0VbZbFYLCwsddZlMxg1NpmlqampKlUpl1/fYLSjNz8/r0qVLXnUVAAAMgL6aMpPaQcayrB1TY1vXCpmmKdu2Zdu2DMPYMRpUq9V2vH7D3Nyc6vW6u1WrVa9/BQAAEDB9F4iuXbu2I8yUy2VNTk7uODYWiymZTHZ9n0Qi0bU+Go1qZGSkY0OADA1JiUR749EdAACP+Dpl5jhO1/CzfZG0aZodU2iWZSmVSnVdZG3bthKJxK4jRAi406elUsnvXgAABsyRByLLsrS8vCypvZ5nYmJix9VkWxdTS+0F14lEQrlcToZhqFKpqFAouO2FQkHZbFYTExMqlUodbQAAAHuJtFqtlt+d8FOj0dDo6Kjq9TrTZz6rVqt65sUVDcfOu3VrtTf1/NPjGhsb87FnAIB+4/X3N4swECxra9Ljj7e3tTW/ewMAGBB9edk9sKtWS/qXf9ksAwDgAUaIAABA6BGIAABA6BGIAABA6BGIAABA6BGIAABA6HGVGYIlEpF+4ic2ywAAeIBAhGAZHpb+4R/87gUAYMAwZQYAAEKPQAQAAEKPQIRgWVuTPvKR9sajOwAAHmENEYKl1ZL+8R83ywAAeIARIgAAEHoEIgAAEHoEIgAAEHoEIgAAEHoEIgAAEHpcZYZgiUSkD31oswwAgAcIRAiW4WHptdf87gUAYMAwZQYAAEKPQAQAAEKPQIRguXVLmphob7du+d0bAMCAYA0RgmV9Xbp2bbMMAIAHGCECAAChRyACAAChRyACAAChRyACAAChRyACAAChx1VmCJ6HHvK7BwCAAUMgQrCcOSP98Id+9wIAMGCYMgMAAKFHIAIAAKFHIEKw3Lolffzj7Y1HdwAAPMIaIgTL+rr0l3+5WQYAwAO+BKJyuazp6WmtrKzsqJekeDwu27blOI7i8bgkybZtFYtFmaYp27aVyWRkGMaebQAAAHs58kC0EVw2ws9W+Xxei4uLkqRkMqlCoeC2pdNpN0DZtq3p6Wm3/V5tAAAAeznyQJRKpXZtGx8f1+rqqiR1jPDYtt1xnGmasixrzzYAAID96LtF1YZh7JjusixLsVisoy4Wi6lcLt+zDQAAYD/6alG14zgqFouSpFKppJmZGZmmKcdxuh5fq9Xu2dZNs9lUs9l09xuNxqH6DAAAgq+vAtHWxdCmaWpqakqVSmXX43cLQ/dqm5+f16VLlw7RS/hueNjvHgAABkxfTZltXQ+0ccWYbdsyDGPHiE+tVnOn13Zr62Zubk71et3dqtWq578H7qMzZ6SbN9vbmTN+9wYAMCD6JhCVy2VNTk7uqI/FYkomk11fk0gk7tnWTTQa1cjISMcGAADCzdcpM8dxOqbIFhYW3DbLspRKpbousrZtW4lEYs82AACA/TjyQGRZlpaXlyW11/NMTEy4wSeRSCiXy8kwDFUqlY57CRUKBWWzWU1MTKhUKu27DQPm9m3pySfb5atXpVOn/O0PAGAgRFqtVsvvTvip0WhodHRU9Xqd6TOfVatVPfPiioZj5926tdqbev7pcY2NjbUrbt6Uzp5tl2/cYB0RAISU19/ffbOGCAAAwC8EIgAAEHoEIgAAEHoEIgAAEHoEIgAAEHoEIgAAEHp99SwzYE9nzkjhvlMEAOA+YIQIAACEHoEIAACEHoEIwXL7tpROt7fbt/3uDQBgQBCIECzvvy8Vi+3t/ff97g0AYEAQiAAAQOgRiAAAQOgRiAAAQOgRiAAAQOgRiAAAQOgRiAAAQOjx6A4Ey/CwdOPGZhkAAA8QiBAskUj7eWYAAHiIKTMAABB6BCIES7Mpfe5z7a3Z9Ls3AIABQSBCsLz3nvSHf9je3nvP794AAAYEgQgAAIQegQgAAIQegQgAAIQegQgAAIQegQgAAIQegQgAAIQed6pGsAwPS2+9tVkGAMADBCIESyQiPfyw370AAAwYpswAAEDoEYgQLM2m9Mwz7Y1HdwAAPEIgQrC89570jW+0Nx7dAQDwCIEIAACEHoEIAACEni9XmZXLZU1PT2tlZWVHvWVZkqRSqaTLly/LMAy3TZLi8bhs25bjOIrH45Ik27ZVLBZlmqZs21Ymk3FfBwAAsJcjD0QbwWUj4GxlWZZmZ2clSblcTpOTk25oyufzWlxclCQlk0kVCgX3del02j3Otm1NT093tAMAANzLkU+ZpVIpd2Rnq3K5rPn5+Y7jyuWybNuWJI2Pj2t1dVWrq6taXl52R4A22jeYpumOMgEAAOxH36whisfjunz5srvvOI4kKRaLuXWGYeyYCrMsq+OYjdd0G4ECAADopq/uVJ1KpdzylStXlEwm3QDkOI6KxaKk9vqimZkZmabpBqftarVa1/pms6nmlvvXNBoNbzqPo3H6tPS9722WAQDwQF8Fog0b4WfrouutC6VN09TU1JQqlco936Ob+fl5Xbp0ycvu4igNDUmPP+53LwAAA6Zvpsy2ymazHeuEpM61QhtXk9m2LcMwdowG1Wq1Xa8ym5ubU71ed7dqtXo/fgUAABAgfReIcrmcstmsOx3mOI7K5bImJyd3HBuLxZRMJru+TyKR6FofjUY1MjLSsSFA7tyRnnuuvd2543dvAAADwtdAtH1aq1gsKh6Pu2FoaWlJhmHINE0tLCy4x1mWpVQq5bZtZdu2EokE9yEaVHfvSl/7Wnu7e9fv3gAABsSRryGyLEvLy8uS2ut5JiYmlEqlZNu20ul0x7GGYbhrhxKJhHK5nAzDUKVS6bjPUKFQUDab1cTEhEqlEvcgAgAABxJptVotvzvhp0ajodHRUdXrdabPfFatVvXMiysajp1369Zqb+r5p8c1NjbWrrh5Uzp7tl2+cUM6c8aHngIA/Ob193ffrSECAAA4agQiAAAQegQiAAAQegQiAAAQen15p2pgV6dPS3//95tlAAA8QCBCsAwNSR/5iN+9AAAMGKbMAABA6DFChGC5c0f6ylfa5S9+UTp50t/+AAAGAoEIwXL3rnTpUrv83HMEIgCAJ5gyAwAAoUcgAgAAoUcgAgAAoUcgAgAAoUcgAgAAoUcgAgAAocdl9wiWU6ekv/mbzTIAAB4gECFYjh2TJib87gUAYMAwZQYAAEKPESIEy5070u/9Xrv867/OnaoBAJ4gECFY7t6VZmfb5c9/nkAEAPAEU2YAACD0CEQAACD0eg5Er732mluu1+u6evVqRx0AAEBQ9ByILMtyy6Ojo3ryySc76gAAAILiQIuq6/W6lpaWFIlEtLy8vKN9ZWVFv/qrv+pZ5wAAAI7CgQLR6OioksmkFhYWVKlUdOHChY722Y2rfwAAAALkwJfdX7hwQd/85jf1yiuvaHJysqONNUS4706dkv78zzfLAAB4oOf7EE1OTurVV19VrVZz6/L5vK5cueJJx4Cujh2TPv5xv3sBABgwPQeiixcvynEcGYbh1v3t3/6tF30CAAA4Uj0HoqmpKU1PT3fUXb169dAdAu7p7l1pcbFdzmSkEyf87Q8AYCD0HIieeOKJfdUBnrpzR/rCF9rlz32OQAQA8ETPgahSqSifz2tiYkKS1Gq1tLS0pFKp5FnnAAAAjkLPN2bM5/O6cOGCWq2WWq2WJLn/CwAAECQ9jxAtLCzsuOw+mUweukMAAABHrecRou1hSJLOnTt3qM4AAAD4oecRoj/7sz/r2HccR/l8Xi+99NKery2Xy5qentbKykpHvW3bKhaLMk1Ttm0rk8m4l/X32gYAALCXngNRJpPR+Pi4u27IsixNTU3t+bqN4FIul3e0pdNpNyTZtq3p6WkVCoVDtQEAAOzlUGuInnzyyY66V155Zc/XpVKprvW2bXfsm6Ypy7IO1YYBFI1K3/3uZhkAAA/0HIi2hyFJikQiPXfEsizFYrGOulgspnK5rGvXrvXUFo/He+4P+tTx49Iv/ILfvQAADJieA9HXvva1jv133nlHjuPoE5/4RE/v5zhO1/pardZzWzfNZlPNZtPdbzQaB+kmAAAYQD1fZfbHf/zH7j2IWq2WTNPU7/zO73jZN0m7B6Ve2+bn5zU6OupuY2Njh+sgjtbdu9K3v93e7t71uzcAgAHh6X2IDsMwjB2jOrVaTYZh9NzWzdzcnJ599ll3v9FoEIqC5M4d6Zd/uV1Op3l0BwDAE4e6D1Gj0dALL7ygF1544dBTT7vd1DGRSPTc1k00GtXIyEjHBgAAwq3nQPS9731Pn/jEJ/Tyyy/r5Zdf1vj4uF599dUDvcfWaS3TNDvabNtWIpGQYRg9twEAAOxHz1NmV69e1bVr1zrq5ubm9NGPfvSer7MsS8vLy5La63kmJibcS/ELhYKy2awmJiZUKpU67iXUaxsAAMBeIq0en8h69erVHZfed6vrd41GQ6Ojo6rX60yf+axareqZF1c0HDvv1q3V3tTzT49vrvO6eVM6e7ZdvnFDOnPGh54CAPzm9fd3z1Nm22+IKLWn0QAAAIKm5ymzZDKpT37ykxofH5fUngpbWFjwrGMAAABHpecRoo997GPK5/PufYgWFxd7vikjsG/RqLS01N54dAcAwCP7HiF67bXXOvYff/xxXbhwQXNzcxodHfW6X0B3x4+37z8EAICH9j1CtLy8LNM0tbCw0LF+qFar6erVq/rOd75zXzoIAABwv+17hMg0Ta2srOhjH/tYR/2FCxd04cIF1et1fec739Ev/dIved5JwPXee9Kf/Em7/Iu/2B4xAgDgkPb9bVKv1+/5qI7R0VH1eAU/sH/NpnTxYrt84waBCADgiX1Pme329PitVldXD9UZAAAAP+w7EFUqFU+OAQAA6Df7DkSmaeqFF17Ytf2FF15QLBbzpFMAAABHad8LMKanp/XJT35ShUJBv/Zrv6YLFy5Iat+xOp/PS5Jeeuml+9NLAACA++hAN2Z8+eWXNTk5qV/5lV/R+Pi44vG4UqmU4vE4YQgAAATWgS/RmZ2d1ezsrPvcso2RIgAAgKDq+ZplghB8cfKk9K1vbZYBAPAAN3FBsJw4IX3uc373AgAwYHp+uCsAAMCgYIQIwfLee9LGAv5PfYo7VQMAPMG3CYKl2ZQ+85l2mUd3AAA8wpQZAAAIPQIRAAAIPQIRAAAIPQIRAAAIPQIRAAAIPQIRAAAIPa5ZRrCcPCl9/eubZQAAPEAgQrCcOCE984zfvQAADBimzAAAQOgxQoRgef996a/+ql3+2Z+Vjh3ztz8AgIFAIEKw3L4t/fzPt8s3bkhnzvjbHwDAQGDKDAAAhB6BCAAAhB6BCAAAhB6BCAAAhB6BCAAAhB6BCAAAhB6X3SNYTpyQcrnNMgAAHui7QFQsFpVMJiVJhmF0tJXLZUlSPB6XbdtyHEfxeFySZNu2isWiTNOUbdvKZDI7Xo8BcPKk9NxzfvcCADBg+m7KLJ1O69y5czp37pwikYgikYhy/zYikM/nNT4+rkgkopmZGZmm2fG62dlZpVIppVIpTU9P+/UrAACAgOmrESLHcVQoFJRKpdy6XC6n2dlZSdL4+LhWV1cldY4e2bbd8T6macqyrPvfYRy999+X/m2kUPE4j+4AAHii70aItoahYrHYsS+1g9D2qTDLshSLxTrqYrGYO8WGAXL7tvTTP93ebt/2uzcAgAHRVyNEW4OO4ziq1Wod02KO46hYLEqSSqWSO23mOE7X96vVajvqms2mms2mu99oNLzpPAAACKy+CkRbZbNZLSwsdNRtXShtmqampqZUqVR2fY9uQWl+fl6XLl3ysqsAACDg+m7KTGoHGcuydkyNbV0rtHE1mW3bMgxjx2hQrVbrepXZ3Nyc6vW6u1Wr1fvxKwAAgADpy0B07dq1rpfcT05O7jg2Fou5l+lvl0gkdtRFo1GNjIx0bAAAINz6csqsXC7vWCRtmmbHFJplWUqlUl0XWdu2rUQiwX2IAADAvvRlIJLUsZhaai+4TiQSyuVyMgxDlUpFhULBbS8UCspms5qYmFCpVOpoAwAAuJdIq9Vq+d0JPzUaDY2OjqperzN95rNqtapnXlzRcOy8W7dWe1PPPz2usbGxdsWdO9JXvtIuf/GL7TtXAwBCx+vv774dIQK6OnlS+u3f9rsXAIAB05eLqgEAAI4SI0QIlvV16Z/+qV3+8R+Xhsj0AIDDIxAhWG7dkn7yJ9vlGzekM2f87Q8AYCDwz2sAABB6BCIAABB6BCIAABB6BCIAABB6BCIAABB6BCIAABB6XHaPYDlxQvrN39wsAwDgAQIRguXkSemrX/W7FwCAAcOUGQAACD1GiBAs6+vS97/fLj/2GI/uAAB4gkCEYLl1S7pwoV3m0R0AAI/wz2sAABB6BCIAABB6BCIAABB6BCIAABB6BCIAABB6BCIAABB6XHaPYDl+XPr85zfLAAB4gG8UBEs0Kj3/vN+9AAAMGKbMAABA6DFChGBptaS3326XH3pIikT87Q8AYCAQiBAsa2vSBz7QLvPoDgCAR5gyAwAAoUcgAgAAoUcgAgAAoUcgAgAAoUcgAgAAoUcgAgAAocdl9wiW48elz352swwAgAf4RkGwRKPSt7/tdy8AAAOGKTMAABB6fTdCVC6XJUnxeFy2bctxHMXjcUmSbdsqFosyTVO2bSuTycgwjD3bMEBarfbdqiVpeJhHdwAAPNF3gSifz2txcVGSlEwmVSgU3LZ0Oq2VlRVJ7QA0PT3ttt+rDQNkbU06e7Zd5tEdAACP9F0gGh8f1+rqqiR1jPDYtt1xnGmasixrzzYAAIC99OUaIsMwdkx3WZalWCzWUReLxVQul+/ZBgAAsJe+GyFyHEfFYlGSVCqVNDMzI9M05ThO1+Nrtdo927ZrNptqNpvufqPROHSfAQBAsPVdINq6GNo0TU1NTalSqex6/G5haLe2+fl5Xbp06ZC9BAAAg6Tvpsy2rgfauGLMtm0ZhrFjxKdWq7nTa7u1bTc3N6d6ve5u1Wr1vvweAAAgOPoqEJXLZU1OTu6oj8ViSiaTXV+TSCTu2bZdNBrVyMhIxwYAAMKtr6bMTNPUwsKCu29ZllKpVNdF1rZtK5FI7NmGAXPsmJRKbZYBAPBAXwUiwzCUSCSUy+VkGIYqlUrHvYQKhYKy2awmJiZUKpX23YYBcuqUxLkFAHgs0mq1Wn53wk+NRkOjo6Oq1+tMn/msWq3qmRdXNBw779at1d7U80+Pa2xszMeeAQD6jdff3321hggAAMAPBCIEy82b7eeXRSLtMgAAHiAQAQCA0CMQAQCA0CMQAQCA0CMQAQCA0CMQAQCA0CMQAQCA0OurO1UDezp2TPr0pzfLAAB4gECEvtZaX9f169c7K7/5TT366KMaGmKAEwDgDQIR+trtRk2zS29o9OEfuHW36m/rDz7/KR7nAQDwDIEIfe/U6EMdzzcDAMBrzDkgUKLNW1r67/9Jj/7Yj/HoDgCAZxghQuCcutuU7vrdCwDAIGGECAAAhB6BCAAAhB6BCAAAhB6BCAAAhB6BCAAAhB5XmSFQ1iND+rsLH9GHzz+gU9ypGgDgEb5RECh3T0b13zJf1g+XlqTTp/3uDgBgQBCIAABA6BGIAABA6BGIECjR5i390Zc/q0c++lEe3QEA8AyLqhE4ozcbElkIAOAhRogAAEDoEYgAAEDoEYgAAEDoEYgAAEDoEYgAAEDocZUZAmU9MqT/++i/04ceHNZJHt0BAPAI3ygIlLsno/qNL3xVb373uzy6AwDgGQIRAAAIPQIRAAAIPQIRAuXkndu6vJDRj/zMz0hra353BwAwIPpuUXW5XJZlWZKkUqmky5cvyzAMt02S4vG4bNuW4ziKx+OSJNu2VSwWZZqmbNtWJpNxX4fBEWm1dN75oeRIarX87g4AYED0XSCyLEuzs7OSpFwup8nJSa2srEiS8vm8FhcXJUnJZFKFQsF9XTqddo+zbVvT09Md7QAAALvpqymzcrms+fl5dz+VSqlcLsu2bUnS+Pi4VldXtbq6quXlZXcEaKN9g2ma7igTAADAXvoqEMXjcV2+fNnddxxHkhSLxdw6wzB2TIVZltVxzMZrNqbYAAAA7qXvpsxSqZRbvnLlipLJpBuAHMdRsViU1F5fNDMzI9M03eC0Xa1W21HXbDbVbDbd/Uaj4V3nAQBAIPVdINqwEX421gVJ6lgobZqmpqamVKlU7vke283Pz+vSpUtedxcAAARYX02ZbZXNZjvWCUmda4U2riazbVuGYewYDarVal2vMpubm1O9Xne3arV6v34F3AetSETf/8CY7n74w1Ik4nd3AAADoi8DUS6XUzabdafDHMdRuVzW5OTkjmNjsZiSyWTX90kkEjvqotGoRkZGOjYEx52Tp/SF//r7euOVV6ThYb+7AwAYEH0XiIrFouLxuBuGlpaWZBiGTNPUwsKCe5xlWUqlUm7bVrZtK5FIcB8iAACwL321hsi2baXT6Y46wzDctUOJREK5XE6GYahSqXTcZ6hQKCibzWpiYkKlUol7EAEAgH2LtFrhvt1vo9HQ6Oio6vU602c+q1areubFFQ3Hzrt179j/oKHhEZ374Jik9qM7vvw/PqsfGT2lE6++yrQZAISU19/ffTVCBOwl0mrpsbeq0lvi0R0AAM/03RoiAACAo0YgAgAAoUcgAgAAoUcgAgAAoUcgAgAAocdVZgiUViSiN42H9eDZqI7z6A4AgEcYIUKg3Dl5StPZRf3gr/+aexABADxDIAIAAKFHIAIAAKFHIEKgnLjT1O9+/Tmd/8xnpFu3/O4OAGBAsKgagTLUWteHX/9/0uuS1tf97g4AYEAwQgQAAEKPQAQAAEKPQAQAAEKPQAQAAEKPQAQAAEKPq8wQOPUzIzobPa5jfncEADAwCEQIlGb0tP7zF7+l35p8VI/UalKt5rY9+uijGhpi0BMAcHAEIgTO7UZNs0tvaPThH7h1t+pv6w8+/ymNjY352DMAQFARiBBIp0Yf0nDsvN/dAAAMCOYXECgn7jT1tf/9VeW+dUkn7jT97g4AYEAwQoRAGWqt66eq/+yWN7TW13X9+vUdx7OuCACwHwQiDATWFQEADoNAhIHBuiIAQK+YSwAAAKHHCBEGFuuKAAD7RSDCwGJdEQBgvwhECJxbJ04qosi+jmVdEQBgPwhECJRm9LT+47PPa2h4ROeip/3uDgBgQBCIECrd1hWtr7fvZ7R9XRFrjQAgPAhECJVu64pWq/+soVNnNfrwI24da40AIFwIRAiUE3eb+nLh96Vjx/X1X/9funsieuD32L6u6JbztoaGR1hrBAAhRiBCoAytr+vf238nSfrG+voeRwMAsD8EIuAQ1tfX9frrr++oZ/0RAATLwAQi27ZVLBZlmqZs21Ymk5FhGH53CwOkW/i5fv26futP/k6njYfcOtYfAUDwDEwgSqfTWllZkdQOR9PT0yoUCj73CoPk9ddf13/5xks6PboZflar/6zh84+z/ggAAm4gApFt2x37pmnKsiyfeoNB0O3y/OvXr+v0yM4F2ft5LZf2A0B/G4hAZFmWYrFYR10sFlO5XFY8HvepVwiy3S7PHz7/uIZ7fO32S/vXnLf0P3/xp/TII5t13YKT13XS/oJYtylCr3+Gl/a7nusw6778WjPGWjXg/huIQOQ4Ttf6Wq22o67ZbKrZbLr79XpdktRoNO5L37B/7777rt59q6q7t9fcuhvvXNfQjbqG1t+TJEXv3NbGmar/4DU1T57qepwnddEzHX15785t3Xir2vNrhyLHOupuvvOGnnn+X3Q29gG3rvGD1zQUHb6vdbffXdVXnv4P+uAHP6h7eeONN/TFF/+PTj1w7r79DC9162+3fuz3uMP8DK/59XOBfnbjxg1JUqvV8uYNWwNgYWGhlUwmO+pM02wVCoUdx37pS19qSWJjY2NjY2MbgK1SqXiSJQZihMgwjB2jQbVaretVZnNzc3r22Wfdfcdx9KEPfUjf//73NTo6er+7intoNBoaGxtTtVrVyMiI390JPc5H/+Bc9A/ORf+o1+t67LHHdiyZ6dVABKJkMql8Pr+jPpFI7KiLRqOKRnfe3Xh0dJT/c/eJkZERzkUf4Xz0D85F/+Bc9A+v1tENxGo80zQ79m3bViKR4D5EAABgXwZihEiSCoWCstmsJiYmVCqVuAcRAADYt4EJRKZpamFhQZKUSqX2/bpoNKovfelLXafRcLQ4F/2F89E/OBf9g3PRP7w+F5FWy6vr1QAAAIJpINYQAQAAHAaBCAAAhB6BCAAAhN7ALKo+KNu2VSwWZZqmbNtWJpO552X65XJZ09PTWllZObpODqiDfPYHPU84GP4O+sdBzkW5XHYfYF0qlXT58mX+Ljx0kHOxcR4cx1GpVNJTTz3FMzQ91Ot3QDab1dzc3MH+Ljy533UAxeNxt1ypVFqpVGrXYwuFQmtlZaUV4o/LUwf57A9yLA6Ov4P+cZBzsbCw0FHe+loc3kHOhWEYrZWVlVar1Wrl8/mWaZr3vX9h0st3wMZ/p1ZXVw/0s0I5ZWbbdse+aZpuyu8mlUqR+D1ykM/+oOcJB8PfQf84yLkol8uan59391OplMrl8o73QG8O+ndRKBQ6/i4YqfNOr98Btm3vuGHzfoQyEFmWtePZJ7FYTOVy2acehcdBPnvO0/3F59s/DnIu4vG4Ll++7O47juMej8M76N9FMpl0y4VCQTMzM/e1f2HSy3+jisXige5FuFUo1xBt/Adku+0PiIX3DvLZc57uLz7f/nHQc7H1P/hXrlxRMplkZMIjvfxdlMtlXblyRVNTU8pkMvepZ+Fz0HPhOM6h/g5COUK0m90+fNx/B/nsOU/3F59v/9jrXDiOo2KxyKOKjsC9zkU8Htfc3JwqlYqKxeLRdSqkdjsXS0tLHSN2BzVQI0SLi4uqVCq7tk9NTbn/ktqeMGu1Gv/COgIH+ew5T/cXn2//6PVcZLNZLS8vc8481Ou5MAxD6XRaU1NTWl1d5Zx44CDnwrIsXbx48XA/8KArvgdBpVLZcVWGYRh7rkgP6cflqYN89r2eJ+wPfwf9o5dzsbCw0KpUKq1Wq9VaXV3l78IjBzkXy8vLLcMwOl4ryb3qDIdz0HORz+fdTVJrdnb2QOdioEaI9mv76nPbtpVIJNzUWS6XZRhG11Xqh52jDLuDfPZ7HYvD4e+gfxz0XBSLRcXjcZmmKcdxtLS0xNoVjxzkXMRisY4pmo02rsb0xkHOxfapspmZGc3MzBzsarPDJrigqlQqrdnZ2VahUGjNzs52JM5UKtVxn4/l5eXW7OysmzgLhYIPPR4cB/ns73UsDo+/g/6x33OxMQqxdds6SoHDO8jfRaFQcEclUqmUO2oHbxzkXLRa7dHShYWFlqRWJpM50AgRT7sHAAChx1VmAAAg9AhEAAAg9AhEAAAg9AhEAAAg9AhEAAAg9AhEAAAg9AhEAELHsiyNj49rcXHR764A6BMEIgChk0wm9dRTT/ndDQB9hEAEAABCj0AEAABCj0AEoG9YlqUnnnhCMzMzKhaLWlxc1MzMjBzH2fU1xWJRTzzxhNLptKT2g2e3vodlWVpcXFQ2m73nzx0fH1cul+t4T8uyOo7J5XIqFov3fC8AwRTKp90D6E/JZFKpVEoPPvigUqmUpHY4SafTWl5e7vqaVCqlWq2mlZUVSZJhGMpms8pkMopEIqpUKkomk25A2njf7T9365qiVCqlK1euuPu2bSubzbo/o1arKZfLaXZ21rPfHYC/CEQA+o5hGG45lUopnU7LcZyO+q0ymYzOnTunfD4vx3FkmqYkaXV1VYZhyLZt1Wo12bbdU3/y+bxisVjHiFGpVOrpvQD0JwIRgMCYmppSrVaTJJmmqUKh4LZdvHjRvYw+k8lIkubn593Rpo2Q1Kt4PK5kMunub/wMAIOBNUQA+s7WNUPFYlHJZFKGYWh5eVkrKytaWVnpCEOSlM1mtbCwoFgsJqm95qdcLmt2dlamabrvuXWUZ+vPMQxD77zzjrtvWZbb/tRTT3W8bvv7AAg+RogA9J1KpeIGklKptCP8dGOaZscoTiKRkGEYbnBJp9PK5/MyTVPlctldI5RMJhWPx3Xx4kVls1n3+GQyqXw+r3g8rng8roWFBWWzWU1MTLjtAAZHpNVqtfzuBABsyGazeuKJJ5iSAnCkmDIDAAChRyAC0Dcsy5JlWSoUCiqXy353B0CIMGUGAABCjxEiAAAQegQiAAAQegQiAAAQegQiAAAQegQiAAAQegQiAAAQegQiAAAQegQiAAAQegQiAAAQev8fPSsjghJkl/sAAAAASUVORK5CYII=",
      "text/plain": [
       "<Figure size 640x480 with 1 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "ax = sn.histplot(x=p_value_list, bins=200)\n",
    "ax.set_xlabel(r'p-value')\n",
    "ax.set_xlim(-0.1,0.4)\n",
    "ax.axvline(alpha_, linestyle='--', color='r')\n",
    "ax.set_ylim(0,2000)\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 300,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Percentage of p-values below the alpha threshold: 94.25%\n"
     ]
    }
   ],
   "source": [
    "print(f'Percentage of p-values below the alpha threshold: {100 * (np.array(p_value_list) < alpha_).sum()/len(p_value_list)}%')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "It shows that we can exclude the null hypothesis with more confidence, then customers from the cluster 0 tend to be converted with a good level of confidence."
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.5"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}