documentation/demos/benchmark-small-data.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "import numpy as np"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Generate random sequences of length 10"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "np.random.seed(101)\n",
    "seqs = np.random.choice(10, size=(20000, 10))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Generate target based on the sum of fields 1-3 and 4-6, multiplied by fields 8-10. Field 7 is ignored"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "v = np.sum((seqs[:, :3] + seqs[:, :3:6]) * seqs[:, -3:], axis=1)\n",
    "target = (v / (np.max(v) / 15)).astype(int)\n",
    "target[target > 9] = 9"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Check target label distribution"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0    1590\n",
       "1    2860\n",
       "2    3303\n",
       "3    3120\n",
       "4    2596\n",
       "5    2129\n",
       "6    1577\n",
       "7    1200\n",
       "8     710\n",
       "9     915\n",
       "Name: count, dtype: int64"
      ]
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "pd.Series(target).value_counts().sort_index()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Create synthetic dataset"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>sequenceId</th>\n",
       "      <th>subsequenceId</th>\n",
       "      <th>inputCol</th>\n",
       "      <th>10</th>\n",
       "      <th>9</th>\n",
       "      <th>8</th>\n",
       "      <th>7</th>\n",
       "      <th>6</th>\n",
       "      <th>5</th>\n",
       "      <th>4</th>\n",
       "      <th>3</th>\n",
       "      <th>2</th>\n",
       "      <th>1</th>\n",
       "      <th>target</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>itemId</td>\n",
       "      <td>1</td>\n",
       "      <td>6</td>\n",
       "      <td>7</td>\n",
       "      <td>9</td>\n",
       "      <td>8</td>\n",
       "      <td>4</td>\n",
       "      <td>8</td>\n",
       "      <td>5</td>\n",
       "      <td>0</td>\n",
       "      <td>5</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>itemId</td>\n",
       "      <td>8</td>\n",
       "      <td>1</td>\n",
       "      <td>3</td>\n",
       "      <td>8</td>\n",
       "      <td>3</td>\n",
       "      <td>3</td>\n",
       "      <td>2</td>\n",
       "      <td>8</td>\n",
       "      <td>9</td>\n",
       "      <td>3</td>\n",
       "      <td>8</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>2</td>\n",
       "      <td>2</td>\n",
       "      <td>itemId</td>\n",
       "      <td>7</td>\n",
       "      <td>0</td>\n",
       "      <td>9</td>\n",
       "      <td>7</td>\n",
       "      <td>9</td>\n",
       "      <td>8</td>\n",
       "      <td>4</td>\n",
       "      <td>3</td>\n",
       "      <td>3</td>\n",
       "      <td>7</td>\n",
       "      <td>5</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>3</td>\n",
       "      <td>3</td>\n",
       "      <td>itemId</td>\n",
       "      <td>4</td>\n",
       "      <td>8</td>\n",
       "      <td>7</td>\n",
       "      <td>6</td>\n",
       "      <td>9</td>\n",
       "      <td>4</td>\n",
       "      <td>2</td>\n",
       "      <td>7</td>\n",
       "      <td>7</td>\n",
       "      <td>7</td>\n",
       "      <td>7</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>4</td>\n",
       "      <td>4</td>\n",
       "      <td>itemId</td>\n",
       "      <td>0</td>\n",
       "      <td>4</td>\n",
       "      <td>1</td>\n",
       "      <td>8</td>\n",
       "      <td>3</td>\n",
       "      <td>1</td>\n",
       "      <td>8</td>\n",
       "      <td>4</td>\n",
       "      <td>3</td>\n",
       "      <td>2</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   sequenceId  subsequenceId inputCol  10  9  8  7  6  5  4  3  2  1  target\n",
       "0           0              0   itemId   1  6  7  9  8  4  8  5  0  5       1\n",
       "1           1              1   itemId   8  1  3  8  3  3  2  8  9  3       8\n",
       "2           2              2   itemId   7  0  9  7  9  8  4  3  3  7       5\n",
       "3           3              3   itemId   4  8  7  6  9  4  2  7  7  7       7\n",
       "4           4              4   itemId   0  4  1  8  3  1  8  4  3  2       0"
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "data_dict = {\n",
    "    \"sequenceId\": np.arange(20000),\n",
    "    \"subsequenceId\": np.arange(20000),\n",
    "    \"inputCol\": np.repeat(\"itemId\", 20000),\n",
    "}\n",
    "for i in range(10):\n",
    "    data_dict[str(10 - i)] = seqs[:, i]\n",
    "\n",
    "data_dict[\"target\"] = target\n",
    "data = pd.DataFrame(data_dict)\n",
    "data.head(5)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(20000, 14)"
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "data.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "import os\n",
    "import numpy as np"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "10"
      ]
     },
     "execution_count": 8,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(np.unique(data[\"target\"]))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Write train and test data to separate files"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "project_path = \"/Users/leon.luithlen/psprojects/test-sequifier\"\n",
    "data.iloc[:10000, :].to_csv(\n",
    "    f\"{project_path}/train_data.csv\", sep=\",\", decimal=\".\", index=None\n",
    ")\n",
    "data.iloc[10000:, :].to_csv(\n",
    "    f\"{project_path}/test_data.csv\", sep=\",\", decimal=\".\", index=None\n",
    ")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Train a model on train data using the sequifier cli and infer on test data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "sequifier --train --on-unprocessed --config-path=/Users/leon.luithlen/psprojects/test-sequifier/configs/train.yaml\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "sh: sequifier: command not found\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "32512"
      ]
     },
     "execution_count": 10,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "command = f\"sequifier --train --on-unprocessed --config-path={project_path}/configs/train.yaml\"\n",
    "print(command)\n",
    "os.system(command)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 42,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "sh: sequifier: command not found\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "32512"
      ]
     },
     "execution_count": 42,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "command = f\"sequifier --infer --config-path={project_path}/configs/infer.yaml\"\n",
    "print(command)\n",
    "os.system(command)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Load predictions from inference"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 35,
   "metadata": {},
   "outputs": [],
   "source": [
    "preds = pd.read_csv(\n",
    "    f\"{project_path}/outputs/predictions/sequifier-default-best-300-predictions.csv\"\n",
    ")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Evaluate transformer test set accuracy"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 36,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.8895"
      ]
     },
     "execution_count": 36,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "np.mean(target[10000:] == preds[\"model_output\"].values)"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Train and evaluate random forest classifier"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.ensemble import RandomForestClassifier\n",
    "\n",
    "rf = RandomForestClassifier(n_estimators=1000)\n",
    "rf.fit(data.iloc[:10000, 3:-1], data.iloc[:10000, -1])\n",
    "rf_preds = rf.predict(data.iloc[10000:, 3:-1])\n",
    "np.mean(rf_preds == target[10000:])"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Train and evaluate XGBoost classifier"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.6967"
      ]
     },
     "execution_count": 42,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "import xgboost as xgb\n",
    "\n",
    "D_train = xgb.DMatrix(\n",
    "    data.iloc[:10000, 3:-1].values, label=data.iloc[:10000, -1].values\n",
    ")\n",
    "D_test = xgb.DMatrix(data.iloc[10000:, 3:-1].values, label=data.iloc[10000:, -1].values)\n",
    "param = {\"eta\": 0.3, \"max_depth\": 5, \"objective\": \"multi:softprob\", \"num_class\": 10}\n",
    "\n",
    "steps = 1000  # The number of training iterations\n",
    "model = xgb.train(param, D_train, steps)\n",
    "\n",
    "xg_preds = model.predict(D_test)\n",
    "xg_best_preds = np.asarray([np.argmax(line) for line in xg_preds])\n",
    "np.mean(xg_best_preds == target[10000:])"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Train and evaluate logistic regression"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.5663"
      ]
     },
     "execution_count": 47,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "from sklearn.preprocessing import OneHotEncoder\n",
    "from sklearn.linear_model import LogisticRegression\n",
    "\n",
    "encoder = OneHotEncoder()\n",
    "X_encoded = encoder.fit_transform(data.iloc[:, 3:-1])\n",
    "lr = LogisticRegression(penalty=\"l2\", max_iter=10000)\n",
    "lr.fit(X_encoded[:10000, :], target[:10000])\n",
    "lr_preds = lr.predict(X_encoded[10000:, :])\n",
    "np.mean(lr_preds == target[10000:])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "dev",
   "language": "python",
   "name": "dev"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.11.7"
  },
  "vscode": {
   "interpreter": {
    "hash": "ad64dbc0878696d9b422cb07e63deba6d531b70d397a0e11e1b180ef0a0a6bf4"
   }
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}