multi_label_trainer.ipynb

{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "accelerator": "GPU",
    "colab": {
      "name": "multi-label-trainer.ipynb",
      "provenance": [],
      "collapsed_sections": [],
      "include_colab_link": true
    },
    "kernelspec": {
      "display_name": "Python 3 (ipykernel)",
      "language": "python",
      "name": "python3"
    },
    "language_info": {
      "codemirror_mode": {
        "name": "ipython",
        "version": 3
      },
      "file_extension": ".py",
      "mimetype": "text/x-python",
      "name": "python",
      "nbconvert_exporter": "python",
      "pygments_lexer": "ipython3",
      "version": "3.8.2"
    }
  },
  "cells": [
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "view-in-github",
        "colab_type": "text"
      },
      "source": [
        "<a href=\"https://colab.research.google.com/github/sayakpaul/A-B-testing-with-Machine-Learning/blob/master/multi_label_trainer.ipynb\" target=\"_parent\"><img src=\"https://colab.research.google.com/assets/colab-badge.svg\" alt=\"Open In Colab\"/></a>"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "5GICQpY-zws7"
      },
      "source": [
        "## Imports"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "ho5uPff1fLoH"
      },
      "source": [
        "from tensorflow.keras import layers\n",
        "from tensorflow import keras\n",
        "import tensorflow as tf\n",
        "\n",
        "from sklearn.preprocessing import MultiLabelBinarizer\n",
        "from sklearn.model_selection import train_test_split\n",
        "from ast import literal_eval\n",
        "import pandas as pd"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "qyeQtKSezymP"
      },
      "source": [
        "## Read data and perform basic EDA"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 206
        },
        "id": "yFo2pNYbf2Du",
        "outputId": "d487ddac-8552-4543-f7f7-ecc3a1c2f9a8"
      },
      "source": [
        "arxiv_data = pd.read_csv(\n",
        "    \"https://github.com/soumik12345/multi-label-text-classification/releases/download/v0.2/arxiv_data.csv\"\n",
        ")\n",
        "arxiv_data.head()"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/html": [
              "<div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>titles</th>\n",
              "      <th>summaries</th>\n",
              "      <th>terms</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>0</th>\n",
              "      <td>Survey on Semantic Stereo Matching / Semantic ...</td>\n",
              "      <td>Stereo matching is one of the widely used tech...</td>\n",
              "      <td>['cs.CV', 'cs.LG']</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1</th>\n",
              "      <td>FUTURE-AI: Guiding Principles and Consensus Re...</td>\n",
              "      <td>The recent advancements in artificial intellig...</td>\n",
              "      <td>['cs.CV', 'cs.AI', 'cs.LG']</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>2</th>\n",
              "      <td>Enforcing Mutual Consistency of Hard Regions f...</td>\n",
              "      <td>In this paper, we proposed a novel mutual cons...</td>\n",
              "      <td>['cs.CV', 'cs.AI']</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>3</th>\n",
              "      <td>Parameter Decoupling Strategy for Semi-supervi...</td>\n",
              "      <td>Consistency training has proven to be an advan...</td>\n",
              "      <td>['cs.CV']</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>4</th>\n",
              "      <td>Background-Foreground Segmentation for Interio...</td>\n",
              "      <td>To ensure safety in automated driving, the cor...</td>\n",
              "      <td>['cs.CV', 'cs.LG']</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "</div>"
            ],
            "text/plain": [
              "                                              titles  ...                        terms\n",
              "0  Survey on Semantic Stereo Matching / Semantic ...  ...           ['cs.CV', 'cs.LG']\n",
              "1  FUTURE-AI: Guiding Principles and Consensus Re...  ...  ['cs.CV', 'cs.AI', 'cs.LG']\n",
              "2  Enforcing Mutual Consistency of Hard Regions f...  ...           ['cs.CV', 'cs.AI']\n",
              "3  Parameter Decoupling Strategy for Semi-supervi...  ...                    ['cs.CV']\n",
              "4  Background-Foreground Segmentation for Interio...  ...           ['cs.CV', 'cs.LG']\n",
              "\n",
              "[5 rows x 3 columns]"
            ]
          },
          "metadata": {},
          "execution_count": 2
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "Em_8mJvUgKY-",
        "outputId": "96ecef3c-6ff5-4e15-9cb3-be8835bee5e8"
      },
      "source": [
        "print(f\"There are {len(arxiv_data)} rows in the dataset.\")"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "There are 51774 rows in the dataset.\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "k9Vb9jtK5zjg",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "38aa5188-62a0-480b-d829-0cab6cd141d2"
      },
      "source": [
        "total_duplicate_titles = sum(arxiv_data[\"titles\"].duplicated())\n",
        "print(f\"There are {total_duplicate_titles} duplicate titles.\")"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "There are 12802 duplicate titles.\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "2259X-rf6OLY",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "309cfa18-56ff-4c4d-b7b8-1a66bec152d5"
      },
      "source": [
        "arxiv_data = arxiv_data[~arxiv_data[\"titles\"].duplicated()]\n",
        "print(f\"There are {len(arxiv_data)} rows in the deduplicated dataset.\")"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "There are 38972 rows in the deduplicated dataset.\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "TmgkCCr2g0w5",
        "outputId": "3b7268e4-86eb-43e9-852f-8d4da544cdc3"
      },
      "source": [
        "# There are some terms with occurence as low as 1.\n",
        "sum(arxiv_data[\"terms\"].value_counts() == 1)"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "2321"
            ]
          },
          "metadata": {},
          "execution_count": 10
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "hyBJNHMdifJ-",
        "outputId": "b220cf2b-341f-46da-ad09-0c36ca02f4f8"
      },
      "source": [
        "# How many unique terms?\n",
        "arxiv_data[\"terms\"].nunique()"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "3157"
            ]
          },
          "metadata": {},
          "execution_count": 11
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "77ZoCzrMhLxc",
        "outputId": "733bee4c-11f8-44ce-9b0e-53cd1bafa637"
      },
      "source": [
        "# Filtering the rare terms.\n",
        "arxiv_data_filtered = arxiv_data.groupby(\"terms\").filter(lambda x: len(x) > 1)\n",
        "arxiv_data_filtered.shape"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "(36651, 3)"
            ]
          },
          "metadata": {},
          "execution_count": 12
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "MxrG9tim0QNr"
      },
      "source": [
        "## Convert the string labels to list of strings. \n",
        "\n",
        "The initial labels are represented as raw strings. Here we make them `List[str]` for a more compact representation. "
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "LIEGLc61iwbQ",
        "outputId": "ebddd3c1-8f4b-42cf-e236-356a00c88f51"
      },
      "source": [
        "arxiv_data_filtered[\"terms\"] = arxiv_data_filtered[\"terms\"].apply(\n",
        "    lambda x: literal_eval(x)\n",
        ")\n",
        "arxiv_data_filtered[\"terms\"].values[:5]"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "array([list(['cs.CV', 'cs.LG']), list(['cs.CV', 'cs.AI', 'cs.LG']),\n",
              "       list(['cs.CV', 'cs.AI']), list(['cs.CV']),\n",
              "       list(['cs.CV', 'cs.LG'])], dtype=object)"
            ]
          },
          "metadata": {},
          "execution_count": 13
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "zjFB8Uoo0cXM"
      },
      "source": [
        "## Stratified splits because of class imbalance"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "EbKDVTKPgOKe",
        "outputId": "38a3b8b4-0dd1-4181-ee36-e7e3dd8ed953"
      },
      "source": [
        "test_split = 0.1\n",
        "\n",
        "# Initial train and test split.\n",
        "train_df, test_df = train_test_split(\n",
        "    arxiv_data_filtered,\n",
        "    test_size=test_split,\n",
        "    stratify=arxiv_data_filtered[\"terms\"].values,\n",
        ")\n",
        "\n",
        "# Splitting the test set further into validation\n",
        "# and new test sets.\n",
        "val_df = test_df.sample(frac=0.5)\n",
        "test_df.drop(val_df.index, inplace=True)\n",
        "\n",
        "print(f\"Number of rows in training set: {len(train_df)}\")\n",
        "print(f\"Number of rows in validation set: {len(val_df)}\")\n",
        "print(f\"Number of rows in test set: {len(test_df)}\")"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Number of rows in training set: 32985\n",
            "Number of rows in validation set: 1833\n",
            "Number of rows in test set: 1833\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "/usr/local/lib/python3.7/dist-packages/pandas/core/frame.py:4174: SettingWithCopyWarning: \n",
            "A value is trying to be set on a copy of a slice from a DataFrame\n",
            "\n",
            "See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy\n",
            "  errors=errors,\n"
          ]
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "96Ew2PPI0lVc"
      },
      "source": [
        "## Multi-label binarization"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "1vgxbdwGf07E",
        "outputId": "6f2d9d79-cf1d-433d-d1f8-aabaffb103ea"
      },
      "source": [
        "mlb = MultiLabelBinarizer()\n",
        "mlb.fit_transform(train_df[\"terms\"])\n",
        "mlb.classes_"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "array(['14J60 (Primary) 14F05, 14J26 (Secondary)', '62H30', '62H35',\n",
              "       '62H99', '65D19', '68', '68Q32', '68T01', '68T05', '68T07',\n",
              "       '68T10', '68T30', '68T45', '68T99', '68Txx', '68U01', '68U10',\n",
              "       'E.5; E.4; E.2; H.1.1; F.1.1; F.1.3', 'F.2.2; I.2.7', 'G.3',\n",
              "       'H.3.1; H.3.3; I.2.6; I.2.7', 'H.3.1; I.2.6; I.2.7', 'I.2',\n",
              "       'I.2.0; I.2.6', 'I.2.1', 'I.2.10', 'I.2.10; I.2.6',\n",
              "       'I.2.10; I.4.8', 'I.2.10; I.4.8; I.5.4', 'I.2.10; I.4; I.5',\n",
              "       'I.2.10; I.5.1; I.4.8', 'I.2.1; J.3', 'I.2.6', 'I.2.6, I.5.4',\n",
              "       'I.2.6; I.2.10', 'I.2.6; I.2.7', 'I.2.6; I.2.7; H.3.1; H.3.3',\n",
              "       'I.2.6; I.2.8', 'I.2.6; I.2.9', 'I.2.6; I.5.1', 'I.2.6; I.5.4',\n",
              "       'I.2.7', 'I.2.8', 'I.2; I.2.6; I.2.7', 'I.2; I.4; I.5', 'I.2; I.5',\n",
              "       'I.2; J.2', 'I.4', 'I.4.0', 'I.4.3', 'I.4.4', 'I.4.5', 'I.4.6',\n",
              "       'I.4.6; I.4.8', 'I.4.8', 'I.4.9', 'I.4.9; I.5.4', 'I.4; I.5',\n",
              "       'I.5.4', 'K.3.2', 'astro-ph.IM', 'cond-mat.dis-nn',\n",
              "       'cond-mat.mtrl-sci', 'cond-mat.soft', 'cond-mat.stat-mech',\n",
              "       'cs.AI', 'cs.AR', 'cs.CC', 'cs.CE', 'cs.CG', 'cs.CL', 'cs.CR',\n",
              "       'cs.CV', 'cs.CY', 'cs.DB', 'cs.DC', 'cs.DM', 'cs.DS', 'cs.ET',\n",
              "       'cs.FL', 'cs.GR', 'cs.GT', 'cs.HC', 'cs.IR', 'cs.IT', 'cs.LG',\n",
              "       'cs.LO', 'cs.MA', 'cs.MM', 'cs.MS', 'cs.NA', 'cs.NE', 'cs.NI',\n",
              "       'cs.PF', 'cs.PL', 'cs.RO', 'cs.SC', 'cs.SD', 'cs.SE', 'cs.SI',\n",
              "       'cs.SY', 'econ.EM', 'econ.GN', 'eess.AS', 'eess.IV', 'eess.SP',\n",
              "       'eess.SY', 'hep-ex', 'hep-ph', 'math.AP', 'math.AT', 'math.CO',\n",
              "       'math.DS', 'math.FA', 'math.IT', 'math.LO', 'math.NA', 'math.OC',\n",
              "       'math.PR', 'math.ST', 'nlin.AO', 'nlin.CD', 'physics.ao-ph',\n",
              "       'physics.bio-ph', 'physics.chem-ph', 'physics.comp-ph',\n",
              "       'physics.data-an', 'physics.flu-dyn', 'physics.geo-ph',\n",
              "       'physics.med-ph', 'physics.optics', 'physics.soc-ph', 'q-bio.BM',\n",
              "       'q-bio.GN', 'q-bio.MN', 'q-bio.NC', 'q-bio.OT', 'q-bio.QM',\n",
              "       'q-bio.TO', 'q-fin.CP', 'q-fin.EC', 'q-fin.GN', 'q-fin.PM',\n",
              "       'q-fin.RM', 'q-fin.ST', 'q-fin.TR', 'quant-ph', 'stat.AP',\n",
              "       'stat.CO', 'stat.ME', 'stat.ML', 'stat.TH'], dtype=object)"
            ]
          },
          "metadata": {},
          "execution_count": 16
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "a2kFVBCG0oXV"
      },
      "source": [
        "## Data preprocessing and `tf.data.Dataset` objects\n",
        "\n",
        "Get percentile estimates of the sequence lengths. "
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "kCR-_Iw3gyT-",
        "outputId": "f2939ed0-79dc-44c5-931d-af117ce0fc8d"
      },
      "source": [
        "train_df[\"summaries\"].apply(lambda x: len(x.split(\" \"))).describe()"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "count    32985.000000\n",
              "mean       156.502471\n",
              "std         41.538054\n",
              "min          5.000000\n",
              "25%        128.000000\n",
              "50%        154.000000\n",
              "75%        183.000000\n",
              "max        462.000000\n",
              "Name: summaries, dtype: float64"
            ]
          },
          "metadata": {},
          "execution_count": 17
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "2rdivJop02xG"
      },
      "source": [
        "Notice that 50% of the abstracts have a length of 158. So, any number near that is a good enough approximate for the maximum sequence length. "
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "QoMgEZrtVBDS"
      },
      "source": [
        "max_seqlen = 150\n",
        "batch_size = 128\n",
        "\n",
        "\n",
        "def unify_text_length(text, label):\n",
        "    unified_text = tf.strings.substr(text, 0, max_seqlen)\n",
        "    return tf.expand_dims(unified_text, -1), label\n",
        "\n",
        "\n",
        "def make_dataset(dataframe, train=True):\n",
        "    label_binarized = mlb.transform(dataframe[\"terms\"].values)\n",
        "    dataset = tf.data.Dataset.from_tensor_slices(\n",
        "        (dataframe[\"summaries\"].values, label_binarized)\n",
        "    )\n",
        "    if train:\n",
        "        dataset = dataset.shuffle(batch_size * 10)\n",
        "    dataset = dataset.map(unify_text_length).cache()\n",
        "    return dataset.batch(batch_size).prefetch(tf.data.AUTOTUNE)"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "F_vrgkCXrWOS"
      },
      "source": [
        "train_dataset = make_dataset(train_df)\n",
        "validation_dataset = make_dataset(val_df, False)\n",
        "test_dataset = make_dataset(test_df, False)"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "-1Bb4Xnm1EwK"
      },
      "source": [
        "## Dataset preview"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "w-8k7gScVoz6",
        "outputId": "78ffcba1-795e-4a7e-c945-4b54523d87e8"
      },
      "source": [
        "text_batch, label_batch = next(iter(train_dataset))\n",
        "\n",
        "for i, text in enumerate(text_batch[:5]):\n",
        "    label = label_batch[i].numpy()[None, ...]\n",
        "    print(f\"Abstract: {text[0]}\")\n",
        "    print(f\"Label(s): {mlb.inverse_transform(label)[0]}\")\n",
        "    print(\" \")"
      ],
      "execution_count": null,
      "outputs": [
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Abstract: b'We study the effect of the stochastic gradient noise on the training of\\ngenerative adversarial networks (GANs) and show that it can prevent the\\nconver'\n",
            "Label(s): ('cs.LG', 'math.OC', 'stat.ML')\n",
            " \n",
            "Abstract: b'Sensitive medical data is often subject to strict usage constraints. In this\\npaper, we trained a generative adversarial network (GAN) on real-world\\nel'\n",
            "Label(s): ('cs.LG',)\n",
            " \n",
            "Abstract: b'Popular rotated detection methods usually use five parameters (coordinates of\\nthe central point, width, height, and rotation angle) to describe the ro'\n",
            "Label(s): ('cs.CV',)\n",
            " \n",
            "Abstract: b'FPN is a common component used in object detectors, it supplements\\nmulti-scale information by adjacent level features interpolation and summation.\\nHow'\n",
            "Label(s): ('cs.CV',)\n",
            " \n",
            "Abstract: b'Data for Image segmentation models can be costly to obtain due to the\\nprecision required by human annotators. We run a series of experiments showing\\nt'\n",
            "Label(s): ('cs.CV',)\n",
            " \n"
          ]
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "Yfbnoi-y1IoB"
      },
      "source": [
        "## Vocabulary size for vectorization"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "IIW42KOgyfE7",
        "outputId": "b6508f2b-623a-45ee-bfb7-479416ec5c52"
      },
      "source": [
        "train_df[\"total_words\"] = train_df[\"summaries\"].str.split().str.len()\n",
        "vocabulary_size = train_df[\"total_words\"].max()\n",
        "print(f\"Vocabulary size: {vocabulary_size}\")"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Vocabulary size: 498\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "/usr/local/lib/python3.7/dist-packages/ipykernel_launcher.py:1: SettingWithCopyWarning: \n",
            "A value is trying to be set on a copy of a slice from a DataFrame.\n",
            "Try using .loc[row_indexer,col_indexer] = value instead\n",
            "\n",
            "See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy\n",
            "  \"\"\"Entry point for launching an IPython kernel.\n"
          ]
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "3TBUFHm11M3G"
      },
      "source": [
        "## Create model with `TextVectorization`"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "4XX7ovyPokNs"
      },
      "source": [
        "text_vectorizer = layers.TextVectorization(\n",
        "    max_tokens=vocabulary_size, ngrams=2, output_mode=\"tf_idf\"\n",
        ")\n",
        "\n",
        "with tf.device(\"/CPU:0\"):\n",
        "    text_vectorizer.adapt(train_dataset.map(lambda text, label: text))\n",
        "\n",
        "\n",
        "def make_model():\n",
        "    shallow_mlp_model = keras.Sequential(\n",
        "        [\n",
        "            keras.Input(shape=(), dtype=tf.string),\n",
        "            text_vectorizer,\n",
        "            layers.Dense(512, activation=\"relu\"),\n",
        "            layers.Dense(256, activation=\"relu\"),\n",
        "            layers.Dense(len(mlb.classes_), activation=\"softmax\"),\n",
        "        ]\n",
        "    )\n",
        "    return shallow_mlp_model"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "i7i71RLtugUs"
      },
      "source": [
        "With the CPU placement, we run into: \n",
        "\n",
        "```\n",
        "(1) Invalid argument: During Variant Host->Device Copy: non-DMA-copy attempted of tensor type: string\n",
        "```"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "58tUdCsQuI1P",
        "outputId": "2c58e032-0da5-434a-b735-44b2d35fa920"
      },
      "source": [
        "shallow_mlp_model = make_model()\n",
        "shallow_mlp_model.summary()"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Model: \"sequential\"\n",
            "_________________________________________________________________\n",
            "Layer (type)                 Output Shape              Param #   \n",
            "=================================================================\n",
            "text_vectorization (TextVect (None, 498)               1         \n",
            "_________________________________________________________________\n",
            "dense (Dense)                (None, 512)               255488    \n",
            "_________________________________________________________________\n",
            "dense_1 (Dense)              (None, 256)               131328    \n",
            "_________________________________________________________________\n",
            "dense_2 (Dense)              (None, 152)               39064     \n",
            "=================================================================\n",
            "Total params: 425,881\n",
            "Trainable params: 425,880\n",
            "Non-trainable params: 1\n",
            "_________________________________________________________________\n"
          ]
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "y1Hr9D0O1Tw0"
      },
      "source": [
        "## Train the model"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "WCoaRkA-wsC3",
        "outputId": "198a94a5-04f0-4584-8b3e-27328f894e5e"
      },
      "source": [
        "epochs = 20\n",
        "\n",
        "shallow_mlp_model.compile(\n",
        "    loss=\"binary_crossentropy\", optimizer=\"adam\", metrics=[\"categorical_accuracy\"]\n",
        ")\n",
        "\n",
        "shallow_mlp_model.fit(train_dataset, validation_data=validation_dataset, epochs=epochs)"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Epoch 1/20\n",
            "258/258 [==============================] - 4s 8ms/step - loss: 0.0382 - categorical_accuracy: 0.5880 - val_loss: 0.0218 - val_categorical_accuracy: 0.6416\n",
            "Epoch 2/20\n",
            "258/258 [==============================] - 2s 7ms/step - loss: 0.0224 - categorical_accuracy: 0.6366 - val_loss: 0.0214 - val_categorical_accuracy: 0.6454\n",
            "Epoch 3/20\n",
            "258/258 [==============================] - 2s 8ms/step - loss: 0.0216 - categorical_accuracy: 0.6410 - val_loss: 0.0213 - val_categorical_accuracy: 0.6465\n",
            "Epoch 4/20\n",
            "258/258 [==============================] - 2s 7ms/step - loss: 0.0211 - categorical_accuracy: 0.6441 - val_loss: 0.0213 - val_categorical_accuracy: 0.6476\n",
            "Epoch 5/20\n",
            "258/258 [==============================] - 2s 7ms/step - loss: 0.0205 - categorical_accuracy: 0.6484 - val_loss: 0.0213 - val_categorical_accuracy: 0.6487\n",
            "Epoch 6/20\n",
            "258/258 [==============================] - 2s 7ms/step - loss: 0.0200 - categorical_accuracy: 0.6535 - val_loss: 0.0214 - val_categorical_accuracy: 0.6503\n",
            "Epoch 7/20\n",
            "258/258 [==============================] - 2s 7ms/step - loss: 0.0194 - categorical_accuracy: 0.6582 - val_loss: 0.0217 - val_categorical_accuracy: 0.6508\n",
            "Epoch 8/20\n",
            "258/258 [==============================] - 2s 7ms/step - loss: 0.0187 - categorical_accuracy: 0.6662 - val_loss: 0.0222 - val_categorical_accuracy: 0.6519\n",
            "Epoch 9/20\n",
            "258/258 [==============================] - 2s 7ms/step - loss: 0.0181 - categorical_accuracy: 0.6742 - val_loss: 0.0229 - val_categorical_accuracy: 0.6481\n",
            "Epoch 10/20\n",
            "258/258 [==============================] - 2s 7ms/step - loss: 0.0174 - categorical_accuracy: 0.6813 - val_loss: 0.0237 - val_categorical_accuracy: 0.6394\n",
            "Epoch 11/20\n",
            "258/258 [==============================] - 2s 7ms/step - loss: 0.0168 - categorical_accuracy: 0.6866 - val_loss: 0.0244 - val_categorical_accuracy: 0.6307\n",
            "Epoch 12/20\n",
            "258/258 [==============================] - 2s 7ms/step - loss: 0.0165 - categorical_accuracy: 0.6861 - val_loss: 0.0244 - val_categorical_accuracy: 0.6290\n",
            "Epoch 13/20\n",
            "258/258 [==============================] - 2s 7ms/step - loss: 0.0159 - categorical_accuracy: 0.6899 - val_loss: 0.0253 - val_categorical_accuracy: 0.6432\n",
            "Epoch 14/20\n",
            "258/258 [==============================] - 2s 7ms/step - loss: 0.0150 - categorical_accuracy: 0.6983 - val_loss: 0.0261 - val_categorical_accuracy: 0.6416\n",
            "Epoch 15/20\n",
            "258/258 [==============================] - 2s 7ms/step - loss: 0.0142 - categorical_accuracy: 0.7049 - val_loss: 0.0271 - val_categorical_accuracy: 0.6350\n",
            "Epoch 16/20\n",
            "258/258 [==============================] - 2s 7ms/step - loss: 0.0135 - categorical_accuracy: 0.7095 - val_loss: 0.0285 - val_categorical_accuracy: 0.6323\n",
            "Epoch 17/20\n",
            "258/258 [==============================] - 2s 7ms/step - loss: 0.0128 - categorical_accuracy: 0.7126 - val_loss: 0.0297 - val_categorical_accuracy: 0.6296\n",
            "Epoch 18/20\n",
            "258/258 [==============================] - 2s 7ms/step - loss: 0.0121 - categorical_accuracy: 0.7162 - val_loss: 0.0312 - val_categorical_accuracy: 0.6247\n",
            "Epoch 19/20\n",
            "258/258 [==============================] - 2s 7ms/step - loss: 0.0115 - categorical_accuracy: 0.7166 - val_loss: 0.0324 - val_categorical_accuracy: 0.6263\n",
            "Epoch 20/20\n",
            "258/258 [==============================] - 2s 7ms/step - loss: 0.0109 - categorical_accuracy: 0.7184 - val_loss: 0.0339 - val_categorical_accuracy: 0.6230\n"
          ]
        },
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "<keras.callbacks.History at 0x7f70b6e88810>"
            ]
          },
          "metadata": {},
          "execution_count": 23
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "wBvqeuk88G9r"
      },
      "source": [
        "## Evaluate the model"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "sxz8yDaT8MdL",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "d83176bd-34ea-4bac-ff75-7af429e0d475"
      },
      "source": [
        "_, categorical_acc = shallow_mlp_model.evaluate(test_dataset)\n",
        "print(f\"Categorical accuracy on the test set: {round(categorical_acc * 100, 2)}%.\")"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "15/15 [==============================] - 0s 11ms/step - loss: 0.0339 - categorical_accuracy: 0.6230\n",
            "Categorical accuracy on the test set: 62.3%.\n"
          ]
        }
      ]
    }
  ]
}