10_2_Convolution_for_MNIST_1D.ipynb

{
  "cells": [
    {
      "cell_type": "markdown",
      "metadata": {
        "colab_type": "text",
        "id": "view-in-github"
      },
      "source": [
        "<a href=\"https://colab.research.google.com/github/udlbook/udlbook/blob/main/Notebooks/Chap10/10_2_Convolution_for_MNIST_1D.ipynb\" target=\"_parent\"><img src=\"https://colab.research.google.com/assets/colab-badge.svg\" alt=\"Open In Colab\"/></a>"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "t9vk9Elugvmi"
      },
      "source": [
        "# **Notebook 10.2: Convolution for MNIST-1D**\n",
        "\n",
        "This notebook investigates a 1D convolutional network for MNIST-1D as in figure 10.7 and 10.8a.\n",
        "\n",
        "Work through the cells below, running each cell in turn. In various places you will see the words \"TO DO\". Follow the instructions at these places and make predictions about what is going to happen or write code to complete the functions.\n",
        "\n",
        "Contact me at udlbookmail@gmail.com if you find any mistakes or have any suggestions.\n",
        "\n"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 1,
      "metadata": {
        "id": "D5yLObtZCi9J"
      },
      "outputs": [],
      "source": [
        "# Run this if you're in a Colab to install MNIST 1D repository\n",
        "# !pip install git+https://github.com/greydanus/mnist1d\n",
        "\n",
        "# Run this as well if you do not have this package installed:\n",
        "# !pip install pytorch"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 2,
      "metadata": {
        "id": "YrXWAH7sUWvU"
      },
      "outputs": [],
      "source": [
        "import numpy as np\n",
        "import os\n",
        "import torch, torch.nn as nn\n",
        "from torch.utils.data import TensorDataset, DataLoader\n",
        "from torch.optim.lr_scheduler import StepLR\n",
        "import matplotlib.pyplot as plt\n",
        "import mnist1d\n",
        "import random"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 3,
      "metadata": {
        "id": "twI72ZCrCt5z"
      },
      "outputs": [
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Successfully loaded data from ./mnist1d_data.pkl\n",
            "Examples in training set: 4000\n",
            "Examples in test set: 1000\n",
            "Length of each example: 40\n"
          ]
        }
      ],
      "source": [
        "args = mnist1d.data.get_dataset_args()\n",
        "data = mnist1d.data.get_dataset(args, path='./mnist1d_data.pkl', download=False, regenerate=False)\n",
        "\n",
        "# The training and test input and outputs are in\n",
        "# data['x'], data['y'], data['x_test'], and data['y_test']\n",
        "print(\"Examples in training set: {}\".format(len(data['y'])))\n",
        "print(\"Examples in test set: {}\".format(len(data['y_test'])))\n",
        "print(\"Length of each example: {}\".format(data['x'].shape[-1]))"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 4,
      "metadata": {
        "id": "8bKADvLHbiV5"
      },
      "outputs": [
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Train data: 4000 examples (columns), each of which has 40 dimensions (rows)\n",
            "Validation data: 1000 examples (columns), each of which has 40 dimensions (rows)\n"
          ]
        }
      ],
      "source": [
        "# Load in the data\n",
        "train_data_x = data['x'].transpose()\n",
        "train_data_y = data['y']\n",
        "val_data_x = data['x_test'].transpose()\n",
        "val_data_y = data['y_test']\n",
        "# Print out sizes\n",
        "print(\"Train data: %d examples (columns), each of which has %d dimensions (rows)\"%((train_data_x.shape[1],train_data_x.shape[0])))\n",
        "print(\"Validation data: %d examples (columns), each of which has %d dimensions (rows)\"%((val_data_x.shape[1],val_data_x.shape[0])))"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "_sFvRDGrl4qe"
      },
      "source": [
        "Define the network"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 5,
      "metadata": {
        "id": "FslroPJJffrh"
      },
      "outputs": [],
      "source": [
        "# There are 40 input dimensions and 10 output dimensions for this data\n",
        "# The inputs correspond to the 40 offsets in the MNIST1D template.\n",
        "D_i = 40\n",
        "# The outputs correspond to the 10 digits\n",
        "D_o = 10\n",
        "\n",
        "\n",
        "# TODO✅ Create a model with the following layers\n",
        "# 1. Convolutional layer, (input=length 40 and 1 channel, kernel size 3, stride 2, padding=\"valid\", 15 output channels )\n",
        "# 2. ReLU\n",
        "# 3. Convolutional layer, (input=length 19 and 15 channels, kernel size 3, stride 2, padding=\"valid\", 15 output channels )\n",
        "# 4. ReLU\n",
        "# 5. Convolutional layer, (input=length 9 and 15 channels, kernel size 3, stride 2, padding=\"valid\", 15 output channels)\n",
        "# 6. ReLU\n",
        "# 7. Flatten (converts 4x15) to length 60\n",
        "# 8. Linear layer (input size = 60, output size = 10)\n",
        "# References:\n",
        "# https://pytorch.org/docs/1.13/generated/torch.nn.Conv1d.html?highlight=conv1d#torch.nn.Conv1d\n",
        "# https://pytorch.org/docs/stable/generated/torch.nn.Flatten.html\n",
        "# https://pytorch.org/docs/1.13/generated/torch.nn.Linear.html?highlight=linear#torch.nn.Linear\n",
        "\n",
        "# NOTE THAT THE CONVOLUTIONAL LAYERS NEED TO TAKE THE NUMBER OF INPUT CHANNELS AS A PARAMETER\n",
        "# AND NOT THE INPUT SIZE.\n",
        "\n",
        "# Replace the following function:\n",
        "# model = nn.Sequential(\n",
        "# nn.Flatten(),\n",
        "# nn.Linear(40, 100),\n",
        "# nn.ReLU(),\n",
        "# nn.Linear(100, 100),\n",
        "# nn.ReLU(),\n",
        "# nn.Linear(100, 10))\n",
        "\n",
        "model = nn.Sequential(\n",
        "    nn.Conv1d(in_channels=1, out_channels=15, kernel_size=3, stride=2, padding=\"valid\"),\n",
        "    nn.ReLU(),\n",
        "    nn.Conv1d(in_channels=15, out_channels=15, kernel_size=3, stride=2, padding=\"valid\"),\n",
        "    nn.ReLU(),\n",
        "    nn.Conv1d(in_channels=15, out_channels=15, kernel_size=3, stride=2, padding=\"valid\"),\n",
        "    nn.ReLU(),\n",
        "    nn.Flatten(),\n",
        "    nn.Linear(in_features=60, out_features=10)\n",
        ")\n",
        "\n"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 6,
      "metadata": {
        "id": "YgLaex1pfhqz"
      },
      "outputs": [],
      "source": [
        "# He initialization of weights\n",
        "def weights_init(layer_in):\n",
        "  if isinstance(layer_in, nn.Linear):\n",
        "    nn.init.kaiming_uniform_(layer_in.weight)\n",
        "    layer_in.bias.data.fill_(0.0)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 7,
      "metadata": {
        "id": "NYw8I_3mmX5c"
      },
      "outputs": [
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Epoch     0, train loss 1.819255, train error 71.53,  val loss 1.826534, percent error 74.50\n",
            "Epoch     1, train loss 1.540612, train error 57.28,  val loss 1.551777, percent error 58.70\n",
            "Epoch     2, train loss 1.374423, train error 56.65,  val loss 1.379486, percent error 58.20\n",
            "Epoch     3, train loss 1.214649, train error 49.58,  val loss 1.240618, percent error 52.70\n",
            "Epoch     4, train loss 1.144867, train error 47.30,  val loss 1.154199, percent error 49.00\n",
            "Epoch     5, train loss 1.149517, train error 46.60,  val loss 1.174016, percent error 49.50\n",
            "Epoch     6, train loss 1.087694, train error 42.88,  val loss 1.101884, percent error 44.70\n",
            "Epoch     7, train loss 1.087531, train error 41.15,  val loss 1.121379, percent error 44.90\n",
            "Epoch     8, train loss 0.845398, train error 33.68,  val loss 0.886344, percent error 35.70\n",
            "Epoch     9, train loss 0.749691, train error 29.25,  val loss 0.806012, percent error 33.10\n",
            "Epoch    10, train loss 0.678785, train error 26.18,  val loss 0.774437, percent error 29.90\n",
            "Epoch    11, train loss 0.617844, train error 23.97,  val loss 0.700192, percent error 26.40\n",
            "Epoch    12, train loss 0.596091, train error 22.65,  val loss 0.705177, percent error 28.40\n",
            "Epoch    13, train loss 0.477130, train error 17.95,  val loss 0.585542, percent error 21.50\n",
            "Epoch    14, train loss 0.400067, train error 14.53,  val loss 0.520807, percent error 19.10\n",
            "Epoch    15, train loss 0.434445, train error 16.12,  val loss 0.555931, percent error 20.40\n",
            "Epoch    16, train loss 0.304585, train error 11.20,  val loss 0.434924, percent error 15.40\n",
            "Epoch    17, train loss 0.269201, train error 9.72,  val loss 0.441706, percent error 13.90\n",
            "Epoch    18, train loss 0.277064, train error 10.00,  val loss 0.436229, percent error 13.90\n",
            "Epoch    19, train loss 0.315977, train error 12.00,  val loss 0.521532, percent error 16.80\n",
            "Epoch    20, train loss 0.163267, train error 5.47,  val loss 0.357855, percent error 11.30\n",
            "Epoch    21, train loss 0.176150, train error 6.88,  val loss 0.376560, percent error 11.80\n",
            "Epoch    22, train loss 0.163613, train error 5.82,  val loss 0.377238, percent error 10.60\n",
            "Epoch    23, train loss 0.141227, train error 4.72,  val loss 0.359900, percent error 10.40\n",
            "Epoch    24, train loss 0.158443, train error 6.03,  val loss 0.379480, percent error 12.00\n",
            "Epoch    25, train loss 0.145435, train error 5.15,  val loss 0.421231, percent error 12.70\n",
            "Epoch    26, train loss 0.118435, train error 3.57,  val loss 0.369915, percent error 9.80\n",
            "Epoch    27, train loss 0.113728, train error 3.68,  val loss 0.379096, percent error 10.30\n",
            "Epoch    28, train loss 0.128505, train error 4.53,  val loss 0.413432, percent error 11.40\n",
            "Epoch    29, train loss 0.122811, train error 4.25,  val loss 0.402811, percent error 11.20\n",
            "Epoch    30, train loss 0.108459, train error 3.55,  val loss 0.399652, percent error 11.30\n",
            "Epoch    31, train loss 0.101419, train error 3.18,  val loss 0.363614, percent error 10.90\n",
            "Epoch    32, train loss 0.132031, train error 4.55,  val loss 0.376170, percent error 10.90\n",
            "Epoch    33, train loss 0.109052, train error 3.80,  val loss 0.381664, percent error 11.20\n",
            "Epoch    34, train loss 0.088507, train error 2.82,  val loss 0.382703, percent error 10.10\n",
            "Epoch    35, train loss 0.087899, train error 2.85,  val loss 0.396745, percent error 11.00\n",
            "Epoch    36, train loss 0.116272, train error 4.25,  val loss 0.436972, percent error 11.90\n",
            "Epoch    37, train loss 0.104614, train error 3.88,  val loss 0.467987, percent error 10.80\n",
            "Epoch    38, train loss 0.077558, train error 2.47,  val loss 0.396682, percent error 10.50\n",
            "Epoch    39, train loss 0.068329, train error 2.03,  val loss 0.405541, percent error 10.10\n",
            "Epoch    40, train loss 0.056319, train error 1.53,  val loss 0.419005, percent error 10.30\n",
            "Epoch    41, train loss 0.056620, train error 1.50,  val loss 0.393807, percent error 9.70\n",
            "Epoch    42, train loss 0.056464, train error 1.53,  val loss 0.416975, percent error 9.70\n",
            "Epoch    43, train loss 0.053462, train error 1.45,  val loss 0.416939, percent error 10.80\n",
            "Epoch    44, train loss 0.054943, train error 1.50,  val loss 0.420231, percent error 9.60\n",
            "Epoch    45, train loss 0.048920, train error 1.18,  val loss 0.421658, percent error 9.70\n",
            "Epoch    46, train loss 0.052367, train error 1.35,  val loss 0.427477, percent error 10.10\n",
            "Epoch    47, train loss 0.045768, train error 1.10,  val loss 0.434439, percent error 10.20\n",
            "Epoch    48, train loss 0.044444, train error 0.95,  val loss 0.437225, percent error 9.70\n",
            "Epoch    49, train loss 0.045134, train error 1.15,  val loss 0.425683, percent error 10.40\n",
            "Epoch    50, train loss 0.044140, train error 1.03,  val loss 0.438483, percent error 10.30\n",
            "Epoch    51, train loss 0.041470, train error 0.90,  val loss 0.438727, percent error 10.10\n",
            "Epoch    52, train loss 0.041207, train error 0.97,  val loss 0.457767, percent error 10.00\n",
            "Epoch    53, train loss 0.041666, train error 0.95,  val loss 0.456766, percent error 9.90\n",
            "Epoch    54, train loss 0.041275, train error 0.88,  val loss 0.453863, percent error 10.10\n",
            "Epoch    55, train loss 0.036217, train error 0.75,  val loss 0.452738, percent error 10.30\n",
            "Epoch    56, train loss 0.035952, train error 0.65,  val loss 0.459155, percent error 9.90\n",
            "Epoch    57, train loss 0.035756, train error 0.68,  val loss 0.463037, percent error 10.00\n",
            "Epoch    58, train loss 0.037323, train error 0.88,  val loss 0.461388, percent error 10.20\n",
            "Epoch    59, train loss 0.035636, train error 0.68,  val loss 0.460830, percent error 10.50\n",
            "Epoch    60, train loss 0.031879, train error 0.50,  val loss 0.470960, percent error 10.00\n",
            "Epoch    61, train loss 0.031166, train error 0.45,  val loss 0.471043, percent error 10.20\n",
            "Epoch    62, train loss 0.031918, train error 0.53,  val loss 0.469893, percent error 10.40\n",
            "Epoch    63, train loss 0.031105, train error 0.43,  val loss 0.473254, percent error 10.50\n",
            "Epoch    64, train loss 0.029578, train error 0.43,  val loss 0.479804, percent error 10.40\n",
            "Epoch    65, train loss 0.029840, train error 0.43,  val loss 0.485581, percent error 10.60\n",
            "Epoch    66, train loss 0.030224, train error 0.43,  val loss 0.484385, percent error 10.30\n",
            "Epoch    67, train loss 0.029518, train error 0.35,  val loss 0.488774, percent error 10.50\n",
            "Epoch    68, train loss 0.028461, train error 0.35,  val loss 0.490979, percent error 10.50\n",
            "Epoch    69, train loss 0.028266, train error 0.32,  val loss 0.491343, percent error 10.60\n",
            "Epoch    70, train loss 0.029026, train error 0.40,  val loss 0.494545, percent error 10.00\n",
            "Epoch    71, train loss 0.028892, train error 0.38,  val loss 0.503956, percent error 10.50\n",
            "Epoch    72, train loss 0.029863, train error 0.47,  val loss 0.503412, percent error 10.40\n",
            "Epoch    73, train loss 0.026824, train error 0.30,  val loss 0.495079, percent error 10.30\n",
            "Epoch    74, train loss 0.028013, train error 0.45,  val loss 0.490354, percent error 10.70\n",
            "Epoch    75, train loss 0.026329, train error 0.32,  val loss 0.498616, percent error 10.80\n",
            "Epoch    76, train loss 0.026223, train error 0.30,  val loss 0.510332, percent error 10.50\n",
            "Epoch    77, train loss 0.026364, train error 0.25,  val loss 0.501925, percent error 10.30\n",
            "Epoch    78, train loss 0.025483, train error 0.30,  val loss 0.509124, percent error 10.70\n",
            "Epoch    79, train loss 0.025481, train error 0.28,  val loss 0.507241, percent error 10.70\n",
            "Epoch    80, train loss 0.025215, train error 0.28,  val loss 0.512343, percent error 10.20\n",
            "Epoch    81, train loss 0.024547, train error 0.25,  val loss 0.511892, percent error 10.30\n",
            "Epoch    82, train loss 0.024375, train error 0.28,  val loss 0.511262, percent error 10.60\n",
            "Epoch    83, train loss 0.024371, train error 0.22,  val loss 0.512935, percent error 10.40\n",
            "Epoch    84, train loss 0.024064, train error 0.28,  val loss 0.513039, percent error 10.70\n",
            "Epoch    85, train loss 0.024267, train error 0.28,  val loss 0.517678, percent error 10.30\n",
            "Epoch    86, train loss 0.023926, train error 0.28,  val loss 0.511538, percent error 10.60\n",
            "Epoch    87, train loss 0.023877, train error 0.22,  val loss 0.519036, percent error 10.40\n",
            "Epoch    88, train loss 0.023719, train error 0.25,  val loss 0.517295, percent error 10.20\n",
            "Epoch    89, train loss 0.023541, train error 0.20,  val loss 0.516541, percent error 10.60\n",
            "Epoch    90, train loss 0.023344, train error 0.25,  val loss 0.516277, percent error 10.70\n",
            "Epoch    91, train loss 0.023544, train error 0.20,  val loss 0.519844, percent error 10.60\n",
            "Epoch    92, train loss 0.023165, train error 0.22,  val loss 0.519360, percent error 10.70\n",
            "Epoch    93, train loss 0.023011, train error 0.22,  val loss 0.524835, percent error 10.60\n",
            "Epoch    94, train loss 0.023154, train error 0.20,  val loss 0.527369, percent error 10.70\n",
            "Epoch    95, train loss 0.022742, train error 0.20,  val loss 0.520445, percent error 10.80\n",
            "Epoch    96, train loss 0.023884, train error 0.25,  val loss 0.526235, percent error 10.80\n",
            "Epoch    97, train loss 0.022471, train error 0.15,  val loss 0.523767, percent error 10.80\n",
            "Epoch    98, train loss 0.022415, train error 0.20,  val loss 0.530886, percent error 10.40\n",
            "Epoch    99, train loss 0.022189, train error 0.22,  val loss 0.526752, percent error 10.80\n"
          ]
        },
        {
          "data": {
            "image/png": "iVBORw0KGgoAAAANSUhEUgAAAkgAAAHHCAYAAABEEKc/AAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjkuMiwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy8hTgPZAAAACXBIWXMAAA9hAAAPYQGoP6dpAABl10lEQVR4nO3dd3hTVQMG8Ddd6UwXndJChbL3tGxkL9kKVAVBUChL5FNQWSpTRGSLo6CAKCgylA1WwLL3KltmBy1tKKUz5/vjkLRJd2mbtH1/z5Mnyb03957c2yZvzjn3XIUQQoCIiIiIdMyMXQAiIiIiU8OARERERGSAAYmIiIjIAAMSERERkQEGJCIiIiIDDEhEREREBhiQiIiIiAwwIBEREREZYEAiIiIiMsCARES5unXrFhQKBVatWqWbNn36dCgUijy9XqFQYPr06YVapjZt2qBNmzaFus7SomLFihgyZIixi0FUojEgUZm2atUqKBQK3c3a2hpVqlTB6NGjERERUWjbSUhIwPTp0/H333/nafm///4bCoUCGzduzPe2XnnlFdja2uLx48fZLhMYGAgrKytER0fne/3F6eLFi5g+fTpu3bpl7KLoaI+N9mZubg53d3f069cPly5dMnbxspTf/fjgwQNMmjQJbdu2hYODAxQKRY5/u//++y9atGgBW1tbeHp6YuzYsYiPj8/TtuLi4vDBBx/A398fNjY2qFChAoYNG4bbt29nWvbevXt49dVX4eTkBJVKhZ49e+LGjRt52g5RfjEgEQH49NNP8dNPP2HJkiVo1qwZli9fjoCAACQkJBTK+hMSEjBjxow8B6TnERgYiKdPn2LTpk3ZlmXz5s3o3LkzXF1dC7ydTz75BE+fPi3w6/Pi4sWLmDFjRpZf7Lt27cKuXbuKdPs5GTt2LH766Sd89913CAwMxJ9//omWLVsiPDzcaGXKTk77MSthYWGYO3cu7t27h9q1a+e47OnTp9GuXTskJCRgwYIFePvtt7Fy5Ur0798/1+1oNBp06NABy5YtQ+/evbF48WIMHDgQGzZsQLNmzfRCfnx8PNq2bYuQkBB89NFHmDFjBk6dOoXWrVubfNCnksnC2AUgMgVdunRBo0aNAABvv/02XF1dsWDBAmzevBkDBw4s8Ho1Gg2Sk5MLq5h58sorr8DBwQHr1q3Dm2++mWn+5s2b8eTJEwQGBj7XdiwsLGBhYbyPECsrK6NtGwBatmyJfv366Z5XrVoVI0eOxI8//ogPPvjAiCV7fg0bNkR0dDRcXFywcePGHMPORx99BGdnZ/z9999QqVQAZBPf8OHDsWvXLnTs2DHb1x4+fBjHjh3DkiVLEBQUpJtetWpVDB06FHv27EHv3r0BAMuWLcPVq1dx9OhRNG7cGID8v61Vqxa+/PJLzJo1qzDeOpEOa5CIsvDyyy8DAG7evAkAmD9/Ppo1awZXV1fY2NigYcOGWTZ/KRQKjB49GmvXrkXNmjWhVCqxYsUKuLm5AQBmzJiha5opSJ+cy5cvZ9n0kJGNjQ369OmDvXv3IjIyMtP8devWwcHBAa+88gpiYmIwceJE1K5dG/b29lCpVOjSpQvOnDmTa1my6oOUlJSE9957D25ubrpt3L17N9Nr//vvP4waNQpVq1aFjY0NXF1d0b9/f70ajlWrVum+mNu2bavbb9pauKz6IEVGRmLYsGHw8PCAtbU16tati9WrV+sto+1PNX/+fKxcuRKVKlWCUqlE48aNcezYsVzfd3ZatmwJALh+/bre9Hv37mHo0KHw8PCAUqlEzZo18cMPP2R6/eLFi1GzZk3Y2trC2dkZjRo1wrp163TzhwwZgooVK2Z6XW59wXLbj1lxcHCAi4tLTm8XAKBWq7F79268/vrrunAEAG+++Sbs7e3x66+/5vp6APDw8NCb7uXlBUD+LWtt3LgRjRs31oUjAKhWrRratWuX63aICoI1SERZ0H7JaZugvv76a7zyyisIDAxEcnIy1q9fj/79+2Pbtm3o1q2b3mv37duHX3/9FaNHj0a5cuVQt25dLF++HCNHjkTv3r3Rp08fAECdOnXyXa7q1aujdevWuTbVBQYGYvXq1bpyaMXExGDnzp0YOHAgbGxscOHCBfzxxx/o378//Pz8EBERgW+++QatW7fGxYsX4e3tna/yvf3221izZg0GDRqEZs2aYd++fZn2DwAcO3YM//77LwYMGIDy5cvj1q1bWL58Odq0aYOLFy/C1tYWrVq1wtixY7Fo0SJ89NFHqF69um4fZOXp06do06YNrl27htGjR8PPzw8bNmzAkCFDEBsbi3Hjxuktv27dOjx+/BjvvPMOFAoF5s2bhz59+uDGjRuwtLTM1/sGoAt3zs7OumkRERF46aWXdMHZzc0N27dvx7Bhw6BWqzF+/HgAwLfffouxY8eiX79+GDduHBITE3H27FkcOXIEgwYNyndZMsrvfsyPc+fOITU1VVf7qmVlZYV69erh1KlTOb6+UaNGsLOzw5QpU+Di4oKqVavi2rVr+OCDD9C4cWO0b98egKyJPXv2LIYOHZppHU2aNMGuXbvw+PFjODg4PPd7ItIRRGVYcHCwACD27NkjoqKixJ07d8T69euFq6ursLGxEXfv3hVCCJGQkKD3uuTkZFGrVi3x8ssv600HIMzMzMSFCxf0pkdFRQkAYtq0aXkq1/79+wUAsWHDhkzrb926da6vT01NFV5eXiIgIEBv+ooVKwQAsXPnTiGEEImJiSItLU1vmZs3bwqlUik+/fRTvWkARHBwsG7atGnTRMaPkNOnTwsAYtSoUXrrGzRoUKb3brg/hRAiNDRUABA//vijbtqGDRsEALF///5My7du3VpvXyxcuFAAEGvWrNFNS05OFgEBAcLe3l6o1Wq99+Lq6ipiYmJ0y27evFkAEFu3bs20rYy0x+aHH34QUVFR4v79+2LHjh2icuXKQqFQiKNHj+qWHTZsmPDy8hIPHz7UW8eAAQOEo6Ojbj/07NlT1KxZM8ftDh48WFSoUCHTdMPjIIQQFSpUEIMHD9Y9z2k/5ian12rn/fPPP5nm9e/fX3h6eua6/m3btgkvLy8BQHfr1KmTePz4sW4Z7f9Pxr9JraVLlwoA4vLly/l7Y0S5YBMbEYD27dvDzc0NPj4+GDBgAOzt7bFp0ya88MILAPSr+h89eoS4uDi0bNkSJ0+ezLSu1q1bo0aNGkVSTiFEnjp6m5ubY8CAAQgNDdVrtlq3bh08PDzQrl07AIBSqYSZmfwYSEtLQ3R0NOzt7VG1atUs31tO/vrrLwCy83JG2lqSjDLuz5SUFERHR6Ny5cpwcnLK93Yzbt/T01Ovz5ilpaXujKqQkBC95V977TW92h5tE1lez4oaOnQo3Nzc4O3tjc6dOyMuLg4//fSTrglICIHffvsNPXr0gBACDx8+1N06deqEuLg43Xt1cnLC3bt3n6uJzxi0nfSVSmWmedbW1nnqxO/m5ob69etj5syZ+OOPPzB9+nQcOHAAb731Vp63k3EZosLCJjYiAEuXLkWVKlVgYWEBDw8PVK1aVRccAGDbtm34/PPPcfr0aSQlJemmZ9X3w8/Pr1jKnJvAwEB89dVXWLduHT766CPcvXsXBw4cwNixY2Fubg5ANl18/fXXWLZsGW7evIm0tDTd6/N7htt///0HMzMzVKpUSW961apVMy379OlTzJ49G8HBwbh37x6EELp5cXFx+dpuxu37+/vrHTcgvSnpv//+05vu6+ur91wblh49epSn7U2dOhUtW7ZEfHw8Nm3ahPXr1+ttOyoqCrGxsVi5ciVWrlyZ5Tq0fcQ+/PBD7NmzB02aNEHlypXRsWNHDBo0CM2bN89TWYxFG3Qz/k9oJSYm6gXhrNy4cQNt27bFjz/+iL59+wIAevbsqRvHafv27ejSpUuu28lYFqLCwoBEBNmPwbAfhdaBAwfwyiuvoFWrVli2bBm8vLxgaWmJ4OBgvU60WqbyQd2wYUNUq1YNP//8Mz766CP8/PPPEELonb02a9YsTJkyBUOHDsVnn30GFxcXmJmZYfz48dBoNEVWtjFjxiA4OBjjx49HQEAAHB0doVAoMGDAgCLdbkbakGgoY1jLSe3atXV9ZHr16oWEhAQMHz4cLVq0gI+Pj+59vP766xg8eHCW69D2Q6tevTrCwsKwbds27NixA7/99huWLVuGqVOnYsaMGQCyDuMA9EJtcdN2pn7w4EGmeQ8ePMi1D9uqVauQmJiI7t27601/5ZVXAACHDh1Cly5d4OLiAqVSme12AOS7vxxRbhiQiHLx22+/wdraGjt37tSr4g8ODs7zOvI64nRhCwwMxJQpU3D27FmsW7cO/v7+emcBbdy4EW3btsX333+v97rY2FiUK1cuX9uqUKECNBoNrl+/rldrFBYWlmnZjRs3YvDgwfjyyy910xITExEbG6u3XH72W4UKFXD27FloNBq9mpzLly/r5helOXPmYNOmTZg5c6buzEUHBwekpaXpglRO7Ozs8Nprr+G1115DcnIy+vTpg5kzZ2Ly5MmwtraGs7Nzpv0DZK4Zy0pR/f3VqlULFhYWOH78OF599VXd9OTkZJw+fVpvWlYiIiIghMgU8lJSUgAAqampAAAzMzPUrl0bx48fz7SOI0eO4MUXX2QHbSp07INElAtzc3MoFAq9D/Fbt27hjz/+yPM6bG1tASDLL7j8yMtp/hlpa4umTp2K06dPZxr7yNzcPFONyYYNG3Dv3r18l61Lly4AgEWLFulNX7hwYaZls9ru4sWLM31R2tnZAcjbfuvatSvCw8Pxyy+/6KalpqZi8eLFsLe3R+vWrfPyNgqsUqVK6Nu3L1atWoXw8HCYm5ujb9+++O2333D+/PlMy0dFRekeGw50aGVlhRo1akAIoQsLlSpVQlxcHM6ePatb7sGDB9kOCJpRfvZjfjg6OqJ9+/ZYs2aN3qCOP/30E+Lj4/XGT0pISMDly5fx8OFD3bQqVapACJHpNP2ff/4ZAFC/fn3dtH79+uHYsWN6ISksLAz79u3L06CURPnFGiSiXHTr1g0LFixA586dMWjQIERGRmLp0qWoXLmy3pdVTmxsbFCjRg388ssvqFKlClxcXFCrVi3UqlUrX2XJ62n+Wn5+fmjWrBk2b94MAJkCUvfu3fHpp5/irbfeQrNmzXDu3DmsXbsWL774Yr7KBQD16tXDwIEDsWzZMsTFxaFZs2bYu3cvrl27lmnZ7t2746effoKjoyNq1KiB0NBQ7NmzJ1O/p3r16sHc3Bxz585FXFwclEolXn75Zbi7u2da54gRI/DNN99gyJAhOHHiBCpWrIiNGzfi0KFDWLhwYbHUMPzvf//Dr7/+ioULF2LOnDmYM2cO9u/fj6ZNm2L48OGoUaMGYmJicPLkSezZswcxMTEAgI4dO8LT0xPNmzeHh4cHLl26hCVLlqBbt266cg8YMAAffvghevfujbFjxyIhIQHLly9HlSpVcu3Ynp/9qPX5558DAC5cuABAhp6DBw8CkKOoa82cORPNmjVD69atMWLECNy9exdffvklOnbsiM6dO+uWO3r0KNq2bYtp06bpxgAbMmQI5s+fj3feeQenTp1CzZo1cfLkSXz33XeoWbOmbpBIABg1ahS+/fZbdOvWDRMnToSlpSUWLFgADw8PvP/++3k9RER5Z6Sz54hMgvY0/2PHjuW43Pfffy/8/f2FUqkU1apVE8HBwVmeXg1ABAUFZbmOf//9VzRs2FBYWVnlesr/857mn5H2NOgmTZpkmpeYmCjef/994eXlJWxsbETz5s1FaGhoplPo83KavxBCPH36VIwdO1a4uroKOzs70aNHD3Hnzp1M7/fRo0firbfeEuXKlRP29vaiU6dO4vLly5lOTxdCiG+//Va8+OKLwtzcXO90c8MyCiFERESEbr1WVlaidu3aemXO+F6++OKLTPsjt+MiRPbHRqtNmzZCpVKJ2NhYXZmCgoKEj4+PsLS0FJ6enqJdu3Zi5cqVutd88803olWrVsLV1VUolUpRqVIl8b///U/ExcXprXvXrl2iVq1awsrKSlStWlWsWbMmT6f5C5H9fswOMpx2b3gzdODAAdGsWTNhbW0t3NzcRFBQkG5YBcP9Zrh/7969K4YOHSr8/PyElZWV8PLyEsOHDxdRUVGZtnPnzh3Rr18/oVKphL29vejevbu4evVqju+DqKAUQuSxRyIRERFRGcE+SEREREQGGJCIiIiIDDAgERERERkwakD6559/0KNHD3h7e0OhUGQ6bVoIgalTp8LLyws2NjZo3749rl69qrdMTEwMAgMDoVKp4OTkhGHDhiE+Pr4Y3wURERGVNkYNSE+ePEHdunWxdOnSLOfPmzcPixYtwooVK3DkyBHY2dmhU6dOuqHlAXna8oULF7B7925s27YN//zzD0aMGFFcb4GIiIhKIZM5i02hUGDTpk3o1asXAFl75O3tjffffx8TJ04EIK/R5OHhgVWrVmHAgAG4dOkSatSogWPHjukuE7Fjxw507doVd+/e5dDzREREVCAmO1DkzZs3ER4erjdEv6OjI5o2bYrQ0FDdlcqdnJz0rqHVvn17mJmZ4ciRI3qDjGWUlJSkd9FDjUaDmJgYuLq6Gu2SEERERJQ/Qgg8fvwY3t7emS5U/bxMNiCFh4cDADw8PPSme3h46OaFh4dnGgnWwsICLi4uumWyMnv2bN0FIImIiKhku3PnDsqXL1+o6zTZgFSUJk+ejAkTJuiex8XFwdfXF3fu3IFKpTJiyYiIiCiv1Go1fHx8iuRSQiYbkDw9PQHIqz17eXnppkdERKBevXq6ZSIjI/Vel5qaipiYGN3rs6JUKvWuyq6lUqkYkIiIiEqYougeY7LjIPn5+cHT0xN79+7VTVOr1Thy5AgCAgIAAAEBAYiNjcWJEyd0y+zbtw8ajQZNmzYt9jITERFR6WDUGqT4+Hi9K33fvHkTp0+fhouLC3x9fTF+/Hh8/vnn8Pf3h5+fH6ZMmQJvb2/dmW7Vq1dH586dMXz4cKxYsQIpKSkYPXo0BgwYwDPYiIiIqMCMGpCOHz+Otm3b6p5r+wUNHjwYq1atwgcffIAnT55gxIgRiI2NRYsWLbBjxw5YW1vrXrN27VqMHj0a7dq1g5mZGfr27YtFixYV+3shIiKi0sNkxkEyJrVaDUdHR8TFxbEPEhFRCZeWloaUlBRjF4MKgaWlJczNzbOdX5Tf3ybbSZuIiCg/hBAIDw9HbGyssYtChcjJyQmenp7FPk4hAxIREZUK2nDk7u4OW1tbDvxbwgkhkJCQoDtbPeMZ7cWBAYmIiEq8tLQ0XThydXU1dnGokNjY2AAAIiMj4e7unmNzW2Ez2dP8iYiI8krb58jW1tbIJaHCpj2mxd2vjAGJiIhKDTarlT7GOqYMSEREREQGGJCIiIhKiYoVK2LhwoXGLkapwE7aRERERtSmTRvUq1evUILNsWPHYGdn9/yFIgYkIiIiUyaEQFpaGiwscv/KdnNzK4YSlQ1sYiMiIjKSIUOGICQkBF9//TUUCgUUCgVWrVoFhUKB7du3o2HDhlAqlTh48CCuX7+Onj17wsPDA/b29mjcuDH27Nmjtz7DJjaFQoHvvvsOvXv3hq2tLfz9/bFly5ZifpclEwMSERGVTkIAT54U/y0fV/D6+uuvERAQgOHDh+PBgwd48OABfHx8AACTJk3CnDlzcOnSJdSpUwfx8fHo2rUr9u7di1OnTqFz587o0aMHbt++neM2ZsyYgVdffRVnz55F165dERgYiJiYmOfatWUBm9iIiKh0SkgA7O2Lf7vx8UAe+wE5OjrCysoKtra28PT0BABcvnwZAPDpp5+iQ4cOumVdXFxQt25d3fPPPvsMmzZtwpYtWzB69OhstzFkyBAMHDgQADBr1iwsWrQIR48eRefOnfP91soS1iARERGZoEaNGuk9j4+Px8SJE1G9enU4OTnB3t4ely5dyrUGqU6dOrrHdnZ2UKlUust3UPZYg0RERKWTra2szTHGdguB4dloEydOxO7duzF//nxUrlwZNjY26NevH5KTk3Ncj6Wlpd5zhUIBjUZTKGUszRiQiIiodFIo8tzUZUxWVlZIS0vLdblDhw5hyJAh6N27NwBZo3Tr1q0iLl3ZxSY2IiIiI6pYsSKOHDmCW7du4eHDh9nW7vj7++P333/H6dOncebMGQwaNIg1QUWIAYmIiMiIJk6cCHNzc9SoUQNubm7Z9ilasGABnJ2d0axZM/To0QOdOnVCgwYNirm0ZYdCiHycj1hKqdVqODo6Ii4uDiqVytjFISKifEpMTMTNmzfh5+cHa2trYxeHClFOx7Yov79Zg0RERERkgAGJiIiIyAADEhEREZEBBiQiIiIiAwxIRERERAYYkIiIiIgMMCARERERGWBAIiIiIjLAgERERERkgAGJiIioBKtYsSIWLlyoe65QKPDHH39ku/ytW7egUChw+vTp59puYa3HVFkYuwBERERUeB48eABnZ+dCXeeQIUMQGxurF7x8fHzw4MEDlCtXrlC3ZSoYkIiIiEoRT0/PYtmOubl5sW3LGNjERkREZCQrV66Et7c3NBqN3vSePXti6NChuH79Onr27AkPDw/Y29ujcePG2LNnT47rNGxiO3r0KOrXrw9ra2s0atQIp06d0ls+LS0Nw4YNg5+fH2xsbFC1alV8/fXXuvnTp0/H6tWrsXnzZigUCigUCvz9999ZNrGFhISgSZMmUCqV8PLywqRJk5Camqqb36ZNG4wdOxYffPABXFxc4OnpienTp+d/xxUD1iAREVGpJASQkFD827W1BRSKvC3bv39/jBkzBvv370e7du0AADExMdixYwf++usvxMfHo2vXrpg5cyaUSiV+/PFH9OjRA2FhYfD19c11/fHx8ejevTs6dOiANWvW4ObNmxg3bpzeMhqNBuXLl8eGDRvg6uqKf//9FyNGjICXlxdeffVVTJw4EZcuXYJarUZwcDAAwMXFBffv39dbz71799C1a1cMGTIEP/74Iy5fvozhw4fD2tpaLwStXr0aEyZMwJEjRxAaGoohQ4agefPm6NChQ952WjFhQCIiolIpIQGwty/+7cbHA3Z2eVvW2dkZXbp0wbp163QBaePGjShXrhzatm0LMzMz1K1bV7f8Z599hk2bNmHLli0YPXp0rutft24dNBoNvv/+e1hbW6NmzZq4e/cuRo4cqVvG0tISM2bM0D338/NDaGgofv31V7z66quwt7eHjY0NkpKScmxSW7ZsGXx8fLBkyRIoFApUq1YN9+/fx4cffoipU6fCzEw2WtWpUwfTpk0DAPj7+2PJkiXYu3evyQUkNrEREREZUWBgIH777TckJSUBANauXYsBAwbAzMwM8fHxmDhxIqpXrw4nJyfY29vj0qVLuH37dp7WfenSJdSpUwfW1ta6aQEBAZmWW7p0KRo2bAg3NzfY29tj5cqVed5Gxm0FBARAkaH6rHnz5oiPj8fdu3d10+rUqaP3Oi8vL0RGRuZrW8WBNUhERFQq2drK2hxjbDc/evToASEE/vzzTzRu3BgHDhzAV199BQCYOHEidu/ejfnz56Ny5cqwsbFBv379kJycXGjlXb9+PSZOnIgvv/wSAQEBcHBwwBdffIEjR44U2jYysrS01HuuUCgy9cEyBQxIRERUKikUeW/qMiZra2v06dMHa9euxbVr11C1alU0aNAAAHDo0CEMGTIEvXv3BiD7FN26dSvP665evTp++uknJCYm6mqRDh8+rLfMoUOH0KxZM4waNUo37fr163rLWFlZIS0tLddt/fbbbxBC6GqRDh06BAcHB5QvXz7PZTYVbGIjIiIyssDAQPz555/44YcfEBgYqJvu7++P33//HadPn8aZM2cwaNCgfNW2DBo0CAqFAsOHD8fFixfx119/Yf78+XrL+Pv74/jx49i5cyeuXLmCKVOm4NixY3rLVKxYEWfPnkVYWBgePnyIlJSUTNsaNWoU7ty5gzFjxuDy5cvYvHkzpk2bhgkTJuj6H5UkJa/EREREpczLL78MFxcXhIWFYdCgQbrpCxYsgLOzM5o1a4YePXqgU6dOutqlvLC3t8fWrVtx7tw51K9fHx9//DHmzp2rt8w777yDPn364LXXXkPTpk0RHR2tV5sEAMOHD0fVqlXRqFEjuLm54dChQ5m29cILL+Cvv/7C0aNHUbduXbz77rsYNmwYPvnkk3zuDdOgEEIIYxfC2NRqNRwdHREXFweVSmXs4hARUT4lJibi5s2b8PPz0+uQTCVfTse2KL+/WYNEREREZIABiYiIiMgAAxIRERGRAQYkIiIiIgMMSEREVGrwvKPSx1jHlAGJiIhKPO3ozAnGuDotFSntMTUcgbuocSRtIiIq8czNzeHk5KS7ppetra3eNcGo5BFCICEhAZGRkXBycoK5uXmxbp8BiYiISgXtleZN8cKnVHBOTk66Y1ucGJCIiKhUUCgU8PLygru7e5aXwqCSx9LSsthrjrQYkIiIqFQxNzc32pcqlR7spE1ERERkgAGJiIiIyAADEhEREZEBBiQiIiIiAwxIRERERAYYkIiIiIgMMCARERERGWBAIiIiIjLAgERERERkgAGJiIiIyAADEhEREZEBBiQiIiIiAwxIRERERAYYkIiIiIgMmHRASktLw5QpU+Dn5wcbGxtUqlQJn332GYQQumWEEJg6dSq8vLxgY2OD9u3b4+rVq0YsNREREZV0Jh2Q5s6di+XLl2PJkiW4dOkS5s6di3nz5mHx4sW6ZebNm4dFixZhxYoVOHLkCOzs7NCpUyckJiYaseRERERUkilExuoYE9O9e3d4eHjg+++/103r27cvbGxssGbNGggh4O3tjffffx8TJ04EAMTFxcHDwwOrVq3CgAED8rQdtVoNR0dHxMXFQaVSFcl7ISIiosJVlN/fJl2D1KxZM+zduxdXrlwBAJw5cwYHDx5Ely5dAAA3b95EeHg42rdvr3uNo6MjmjZtitDQ0GzXm5SUBLVarXcjIiIi0rIwdgFyMmnSJKjValSrVg3m5uZIS0vDzJkzERgYCAAIDw8HAHh4eOi9zsPDQzcvK7Nnz8aMGTOKruBERERUopl0DdKvv/6KtWvXYt26dTh58iRWr16N+fPnY/Xq1c+13smTJyMuLk53u3PnTiGVmIiIiEoDk65B+t///odJkybp+hLVrl0b//33H2bPno3BgwfD09MTABAREQEvLy/d6yIiIlCvXr1s16tUKqFUKou07ERERFRymXQNUkJCAszM9Itobm4OjUYDAPDz84Onpyf27t2rm69Wq3HkyBEEBAQUa1mJiIio9DDpGqQePXpg5syZ8PX1Rc2aNXHq1CksWLAAQ4cOBQAoFAqMHz8en3/+Ofz9/eHn54cpU6bA29sbvXr1Mm7hiYiIqMQy6YC0ePFiTJkyBaNGjUJkZCS8vb3xzjvvYOrUqbplPvjgAzx58gQjRoxAbGwsWrRogR07dsDa2tqIJSciIqKSzKTHQSouHAeJiIio5Cmz4yARERERGQMDEhEREZEBBiQiIiIiAwxIRERERAYYkIiIiIgMMCARERERGWBAIiIiIjLAgERERERkgAGJiIiIyAADEhEREZEBBiQiIiIiAwxIRERERAYYkIiIiIgMMCARERERGWBAIiIiIjLAgERERERkgAEpAyGMXQIiIiIyBQxIGTx5YuwSEBERkSlgQMogKsrYJSAiIiJTwICUQXS0sUtAREREpoABKQPWIBERERHAgKSHAYmIiIgABiQ9DEhEREQEMCDpefjQ2CUgIiIiU8CAlAEDEhEREQEMSHrYxEZEREQAA5IeBiQiIiICGJD0PHzIa40QERERA5KeqChej42IiIgYkPSkpSkQG2vsUhAREZGxMSAZiIw0dgmIiIjI2BiQDLCjNhERETEgGWANEhERETEgGWBAIiIiIgYkAwxIRERExIBkgH2QiIiIiAHJAGuQiIiIiAHJAAMSERERMSAZiIzQGLsIREREZGQMSAaiInmtESIiorKOAcnAwxgzpKUZuxRERERkTAxIBoRQIDra2KUgIiIiY2JAysAZMhmxozYREVHZxoCUgRvkIEgcC4mIiKhsY0DKQBuQWINERERUtjEgZVAODwEwIBEREZV1DEgZsImNiIiIAAYkPaxBIiIiIoABSU96HyQOFklERFSWMSBloKtBus+RIomIiMoyBqQM3OyeAgCiIhiQiIiIyjIGpAzcysmmtciH3C1ERERlGZNABuU8LQAAsfGWSE42cmGIiIjIaBiQMnB6wR7mSAXAU/2JiIjKMgakDMw83DgWEhERETEg6fHwgDvkIEgcC4mIiKjsYkDKiAGJiIiIwICkz92dF6wlIiIiBiQ9GWqQ2AeJiIio7GJAysjTM72JLVxj5MIQERGRsTAgZeTiAnfFs8uN3OVASERERGUVA1JG5uZwc0oBAEQ+4OVGiIiIyioGJAPubvJyI1EPFUYuCRERERmLyQeke/fu4fXXX4erqytsbGxQu3ZtHD9+XDdfCIGpU6fCy8sLNjY2aN++Pa5evVrg7bl7yl0SGWv53GUnIiKiksmkA9KjR4/QvHlzWFpaYvv27bh48SK+/PJLODs765aZN28eFi1ahBUrVuDIkSOws7NDp06dkJiYWKBtuvlYAwCeJFniyZNCeRtERERUwlgYuwA5mTt3Lnx8fBAcHKyb5ufnp3sshMDChQvxySefoGfPngCAH3/8ER4eHvjjjz8wYMCAfG/TwccJSiQiCdaIigLs7J7/fRAREVHJYtI1SFu2bEGjRo3Qv39/uLu7o379+vj2229182/evInw8HC0b99eN83R0RFNmzZFaGhotutNSkqCWq3Wu2kpvDw5FhIREVEZZ9IB6caNG1i+fDn8/f2xc+dOjBw5EmPHjsXq1asBAOHh4QAADw8Pvdd5eHjo5mVl9uzZcHR01N18fHzSZ2YcC4mjaRMREZVJJh2QNBoNGjRogFmzZqF+/foYMWIEhg8fjhUrVjzXeidPnoy4uDjd7c6dO+kzPT15uREiIqIyzqQDkpeXF2rUqKE3rXr16rh9+zYAwNPTEwAQERGht0xERIRuXlaUSiVUKpXeLcNGWYNERERUxpl0QGrevDnCwsL0pl25cgUVKlQAIDtse3p6Yu/evbr5arUaR44cQUBAQME2mqGJLeoeR9MmIiIqi0w6IL333ns4fPgwZs2ahWvXrmHdunVYuXIlgoKCAAAKhQLjx4/H559/ji1btuDcuXN488034e3tjV69ehVso/b2cLeMBQAsXWmBwEBg924gjQNrExERlRkKIYQwdiFysm3bNkyePBlXr16Fn58fJkyYgOHDh+vmCyEwbdo0rFy5ErGxsWjRogWWLVuGKlWq5HkbarUajo6OiIuLg0qlwnXftuh1ZxHOo7ZumRdeAN58E3j/fcDVtVDfIhERERWA4fd3YTL5gFQcMu3g5s0h/v0Xx+fsxarbL+Pnn4FHj+Syw4cDK1cat7xERERUtAHJpJvYjMbTEwoAje0vYelS4MED4Isv5Kx9+4xaMiIiIioGDEhZ0Z4B92wsJaUSGDECMDMDrl8H7t0zYtmIiIioyDEgZUUbkDZvBg4cAISASgXUry8n//OP8YpGRERERY8BKSstWwLm5sC5c0CrVkDz5sDmzWjVUnbXCgkxcvmIiIioSDEgZaVNG+DSJdmuplQCoaFAr15otXEsANYgERERlXYMSNnx9we++Qa4dQuYNAlQqdDy7joAMjtxlG0iIqLSiwEpN56ewOzZwIYNcEUMalnJkb0PHDByuYiIiKjIMCDl1bMe2q2TdwNgPyQiIqLSjAEpr9zcgBdeQCvIDkjsh0RERFR6MSDlR716uoB09mz66NpERERUujAg5Uf9+vBEBKqowiEEcPCgsQtERERERYEBKT/q1QMAtLb8FwCb2YiIiEorBqT8eNZRu1XsFgDsqE1ERFRaMSDlR8WKgEqFVmnyirUnTwKPHxu3SERERFT4GJDyw8wMqFcPvriDiuUeIy0N+PdfYxeKiIiIChsDUn4964fUyu0yAPZDIiIiKo3yHZBSUlJgYWGB8+fPF0V5TJ92wMhnzWzsh0RERFT65DsgWVpawtfXF2lpaUVRHtOnrUG6vx4AcPQo8PSpEctDREREha5ATWwff/wxPvroI8TExBR2eUxfjRqApSUqxZ+Gt0cqUlKAI0eMXSgiIiIqTBYFedGSJUtw7do1eHt7o0KFCrCzs9Obf/LkyUIpnEmysgJq1YLi1Cm0qvwA6yN8sH8/0KaNsQtGREREhaVAAalXr16FXIwSpl494NQpdHY9hvXwwS+/ANOnAwqFsQtGREREhaFAAWnatGmFXY6SpX59IDgYfZLXY6RNH4SFAcePA40bG7tgREREVBgKFJC0Tpw4gUuXLgEAatasifrPzvAq9Z511HY4H4pevYCffwbWrGFAIiIiKi0UQgiR3xdFRkZiwIAB+Pvvv+Hk5AQAiI2NRdu2bbF+/Xq4ubkVdjmLlFqthqOjI+Li4qBSqfLyAsDREQDw17pYdBvkCDc34N49wNKyiAtLREREAArw/Z0PBTqLbcyYMXj8+DEuXLiAmJgYxMTE4Pz581Cr1Rg7dmyhFtAkqVRApUoAgI4ux+HuDkRFAbt2GblcREREVCgKFJB27NiBZcuWoXr16rppNWrUwNKlS7F9+/ZCK5xJe9acaHH2JAYOlJN++smI5SEiIqJCU6CApNFoYJlFW5KlpSU0Gs1zF6pEeNYPCadP44035MPNm4G4OKOViIiIiApJgQLSyy+/jHHjxuH+/fu6affu3cN7772Hdu3aFVrhTJq2Q/qpU2jQAKhWDUhMBH77zbjFIiIioudXoIC0ZMkSqNVqVKxYEZUqVUKlSpXg5+cHtVqNxYsXF3YZTZO2BiksDIqnCbpaJDazERERlXwFOosNAIQQ2LNnDy5flle1r169Otq3b1+ohSsuBeoFLwTg6QlERgKHD+M/z6aoWFHO+u8/wNe3yIpLREREMLGz2FJSUmBhYYELFy6gQ4cOGDNmDMaMGVNiw1GBKRR6zWwVKgCtW8un69bpL/rkCXD2rMxUREREZPryHZAsLS3h6+uLtLS0oihPydKwobxfvx4QQq+ZTaMBQkKAoUNlRVPdusB33xmvqERERJR3BeqD9PHHH+Ojjz5CTExMYZenZBkxArC2lklowwb07QsolcDFi0CFCvICtsHBQHy8XHz9eqOWloiIiPKoQH2Q6tevj2vXriElJQUVKlSAnZ2d3vyTJ08WWgGLw3O1Yc6YIa9UW748cPkyXn3LDhs2yFkODsBrr8mg9PrrcpTtmBjA3r6w3wEREVHZU5R9kAp0LbZevXoVaiFKtA8+AFatAm7dAmbPxvz5n8PDAwgIAHr1AmxtZd+jKVOAmzeBv/8Gunc3bpGJiIgoZ/kOSKmpqVAoFBg6dCjKly9fFGUqWWxsgAULgD59gC++gO9bb2Hx4kp6iygUQKdOwIoVwI4dDEhERESmLt99kCwsLPDFF18gNTW1KMpTMvXqBXToACQnA++9l+UinTvL+507i69YREREVDAFHkk7JCSksMtScikUwKJFgIUFsHUrkMX16Nq2lbOvXQOuXzdCGYmIiCjPCtQHqUuXLpg0aRLOnTuHhg0bZuqk/corrxRK4UqUatWAceOAL7+U9y+/LE9pe0alApo3lye87dwJjBplxLISERFRjgp0FpuZWfYVTwqFosSNkVRoveDVaqBqVSA8HGjZEpg9W6aiZ+bMASZPBnr0ALZsKYSCExERlWEmNZI2AGg0mmxvJS0cFSqVSvbEViqBAweAFi2Abt2AU6cAyI7aALBvn+yuRERERKYpXwGpa9euiIuL0z2fM2cOYmNjdc+jo6NRo0aNQitcidSzJ3D1KjB8OGBuDvz1F9CgAfDaa6hbMQ4eHvLSI4cOGbugRERElJ18BaSdO3ciKSlJ93zWrFl6o2mnpqYiLCys8EpXUvn4ACtXApcuAYMGyU7cv/4Ks0+no2NHuciOHcYsIBEREeUkXwHJsLtSAbovlS3+/sDatcAvv8jn69ahcwfZBMnT/YmIiExXgfogUT716gWUKwdERqKD5d9QKIAzZ4AHD4xdMCIiIspKvgKSQqGAQqHINI1yYWkJDBgAAHDb+gMaNpSTd+0yYpmIiIgoW/kaB0kIgSFDhkD5bHyfxMREvPvuu7pxkDL2TyIDb7wBLFkCbNqEzqOTcPy4Ejt2AIMHG7tgREREZChfAWmwwbf566+/nmmZN9988/lKVFo1biz7JF29ik5W+/E5OmP3biAtTZ7slpwsL2ablATUri37dRMREZFx5CsgBQcHF1U5Sj+FAnj9dWDaNLx05Gs4OnZGdDTQrh1w754MR9ohpLZu5QVtiYiIjImdtItTYCAAwGLfLnRs9RSAvPTItWvp4QgADh40RuGIiIhIiwGpOFWqBDRrBmg0mF93DaZPlwNv79sna5G++koudvWqUUtJRERU5hXoYrX0HF5/Hfj3X/j+uRzTTg7Xm1W1qrxnQCIiIjIu1iAVt1dflaf9nzoFXLigN8vfX95fvQpoNEYoGxEREQFgQCp+rq5A167y8dq1erMqVgQsLIDERNnkRkRERMbBgGQM2uER1q7VqyqysABefFE+ZjMbERGR8TAgGUP37oBKBdy+DezfrzdL28x25YoRykVEREQAGJCMw9oaGDhQPn7nHSA2VjcrYz8kIiIiMg4GJGOZNUt2Orp+HXjzTV1TW5UqcjZrkIiIiIyHAclYXFyA334DlEo5dPacOQBYg0RERGQKGJCMqUEDYOlS+fiTT4Ddu3UB6cYNIDXVeEUjIiIqyxiQjG3YMHkTAhg4ED7iNpRKICUF+O8/YxeOiIiobGJAMgVLlsjapOhomL3aD5Uryf5IbGYjIiIyDgYkU2BtDWzcCDg7A8eOoYr1bQAMSERERMZSogLSnDlzoFAoMH78eN20xMREBAUFwdXVFfb29ujbty8iIiKMV8iC8vMDhstrs/lr5ClsPJONiIjIOEpMQDp27Bi++eYb1KlTR2/6e++9h61bt2LDhg0ICQnB/fv30adPHyOV8jnVrQsA8H98EgBrkIiIiIylRASk+Ph4BAYG4ttvv4Wzs7NuelxcHL7//nssWLAAL7/8Mho2bIjg4GD8+++/OHz4sBFLXEDPwl+V+38DYEAiIiIylhIRkIKCgtCtWze0b99eb/qJEyeQkpKiN71atWrw9fVFaGhotutLSkqCWq3Wu5mEqlUBS0v4Pz0DALh1C0hONm6RiIiIyiKTD0jr16/HyZMnMXv27EzzwsPDYWVlBScnJ73pHh4eCA8Pz3ads2fPhqOjo+7m4+NT2MUuGEtLoEYNeCIc9tap0GjkeEhERERUvEw6IN25cwfjxo3D2rVrYW1tXWjrnTx5MuLi4nS3O3fuFNq6n1udOlAA8HeOAsCO2kRERMZg0gHpxIkTiIyMRIMGDWBhYQELCwuEhIRg0aJFsLCwgIeHB5KTkxGb4WKvABAREQFPT89s16tUKqFSqfRuJuNZPyR/85sA2A+JiIjIGEw6ILVr1w7nzp3D6dOndbdGjRohMDBQ99jS0hJ79+7VvSYsLAy3b99GQECAEUv+HLQBKUH2Q2JAIiIiKn4Wxi5AThwcHFCrVi29aXZ2dnB1ddVNHzZsGCZMmAAXFxeoVCqMGTMGAQEBeOmll4xR5OenPZPt0REAI9nERkREZAQmHZDy4quvvoKZmRn69u2LpKQkdOrUCcuWLTN2sQrOwwNwc4N/VBgA1iAREREZg0IIIYxdCGNTq9VwdHREXFycafRHat8eD/eehhseAgCePAFsbY1cJiIiIhNTlN/fJt0HqcyqUweuiIaz8gkA4No1I5eHiIiojGFAMkXaU/2VvGgtERGRMTAgmSLtmWyJ5wEwIBERERU3BiRTVKMGYGaGKsnnAHCwSCIiouLGgGSKrK2BqlXhD1l1xBokIiKi4sWAZKpq12ZAIiIiMhIGJFNVp44uIEVEAGq1kctDRERUhjAgmao6deAINdwtogGwFomIiKg4MSCZKu0lR9IuAQCOHzdmYYiIiMoWBiRT5esLqFToLrYCAObNA1JSjFwmIiKiMoIByVQpFECdOhiNJXBXPcWNG8CqVcYuFBERUdnAgGTK6tSBHRIwudEeAMCnnwKJiUYuExERURnAgGTKnvVDetf8W7zwAnD3LvDtt0YuExERURnAgGTKngUk6wsnMGWKnDRzJpCQYMQyERERlQEMSKasVi15f/8+3urxEH5+ckykpUuNWywiIqLSjgHJlDk4AC++CACwCjuHadPk5DlzOHAkERFRUWJAMnUNGsj7f/5BYCBQtSoQEwMsXGjUUhEREZVqDEimrksXeb91KywsgBkz5NMvv5RBiYiIiAofA5Kp69ZNjol04gRw7x7695d9t9VqYOVKYxeOiIiodGJAMnUeHkDTpvLxtm0wMwMmTJBPly8H0tKMVzQiIqLSigGpJHjlFXm/ZQsA4LXXAFdX4PZtYNs2I5aLiIiolGJAKgm0AWnvXuDJE1hbA8OGyUk85Z+IiKjwMSCVBDVqAH5+QFISsHs3AGDkSNk1afduICzMyOUjIiIqZRiQSgKFIlMzW8WKQPfuctKyZcYpFhERUWnFgFRSaAPStm26ntlBQXLSqlVAfLxxikVERFQaMSCVFC1bAo6OQFQUcPQoAKBDB8DfX57yv2aNkctHRERUijAglRSWlumDRj5rZjMzA0aNkpOWLgWEMFLZiIiIShkGpJLEoB8SAAwZAtjaAufPAwcOGKdYREREpQ0DUknSuTNgbg5cvAhcvw4AcHICXn9dzl6yxHhFIyIiKk0YkEoSZ2egVSv5eOtW3WRtZ+1Nm4D7941QLiIiolKGAamk0TazZQhIdeoALVoAqanyjDYiIiJ6PgxIJU2PHvI+JAR49Eg3+e235f0PP7CzNhER0fNiQCppKlWSI2unpQG//66b3K8fYG8vuyaxszYREdHzYUAqiYYMkfeLFumqi+zsgAED5OQffjBOsYiIiEoLBqSS6O235bn9Z88Cf/+tmzx0qLzfsEEOHpmV2Fjg6tUiLyEREVGJxoBUEjk7A4MHy8dff62b/NJLQLVqQEIC8MsvmV8WHw80bixb6J6NEkBERERZYEAqqcaOlfdbtujSjkKRXouUVTPbhx8C167Js93YT4mIiCh7DEglVbVqcuBIIYDFi3WT33xTjiV5+LAcT1Jr715g2bL05ydPFmNZiYiIShgGpJJs/Hh5/8MPuk5HHh5A9+5ycnCwvFer02uWXnxR3jMgERERZY8BqSTr2BGoXh14/Dg9DSE9DP34I5CSAvzvf8Dt20DFisD69XLe6dNypAAiIiLKjAGpJFMo0vsiLVqkSzxdusiapMhI4L33gJUr5SLBwUCDBvIEuCdPeDYbERFRdhiQSro33pBntd24AWzbBgCwtJR9kQBg6VJ5P2YM0KaN7J9Ut66cxmY2IiKirDEglXR2dsCIEfJxhlP+tc1sgBx8e/bs9OcNGsj7U6eKoXxEREQlEANSaRAUJKuG9u8Hjh0DIE9y69IFUCpl05qdXfri2oDEGiQiIqKsMSCVBj4+wMCB8vHo0YBGAwDYtAm4exdo2VJ/8fr15f3Jk7ywLRERUVYYkEqLuXMBBwfg6FHgu+8AyNqjcuUyL1qzpuynFBsL3LpVrKUkIiIqERiQSgtvb+Czz+TjSZOAqKhsF7WyAmrXlo/ZzEZERJQZA1JpEhQE1KsHPHokryuSA3bUJiIiyh4DUmliYQEsXy4fBwcDBw9muyg7ahMREWWPAam0eeklYPhw+XjkSDmUdha0HbVPnGBHbSIiIkMMSKXR7NmAqytw/rwcYTsLdeoAZmZytO0HD4q5fERERCaOAak0cnUF5s2Tj6dNAy5cyLSIra28jBvAZjYiIiJDDEil1ZAhQKtW8qJrbdsC585lWoQdtYmIiLLGgFRamZnJkSIbNpSn/LdtC5w5o7dIxgEjiYiIKB0DUmnm4gLs3g00bgxERwMvv6xXXcQz2YiIiLLGgFTaOTsDu3YBTZsCMTEyJB0/DkAOmQQAt28DDx8ar4hERESmhgGpLHBykiEpIEBeX6RDByAiAo6OQOXKchH2QyIiIkrHgFRWqFTAzp3y1LXYWGDzZgDsh0RERJQVBqSyxMEBGDhQPt6+HQDPZCMiIsoKA1JZ06WLvN+zB0hOZkdtIiKiLDAglTUNGgDu7kB8PHDokK6J7epVIC7OuEUjIiIyFQxIZY2ZGdCpk3y8fTvc3NI7aq9YYbxiERERmRIGpLJI28z2rB/SJ5/Ip7NmyTEliYiIyjqTDkizZ89G48aN4eDgAHd3d/Tq1QthYWF6yyQmJiIoKAiurq6wt7dH3759ERERYaQSlxAdO8qapPPngTt38MYb8mw2tRqYPt3YhSMiIjI+kw5IISEhCAoKwuHDh7F7926kpKSgY8eOePLkiW6Z9957D1u3bsWGDRsQEhKC+/fvo0+fPkYsdQng6ioHjgSAHTtgZgZ8+aV8+s03wKVLmV+i0cgapmnTACGKr6hERETGoBCi5HzdRUVFwd3dHSEhIWjVqhXi4uLg5uaGdevWoV+/fgCAy5cvo3r16ggNDcVLL72Up/Wq1Wo4OjoiLi4OKpWqKN+C6fjsM2DqVKB3b+D33wEAPXsCW7YA3bsDW7emL5qWBrz9NrBqlXx+9ixQu3bxF5mIiCijovz+NukaJENxz06zcnFxAQCcOHECKSkpaN++vW6ZatWqwdfXF6GhodmuJykpCWq1Wu9W5hic7g8A8+YBFhbAtm3Avn1ydkoK8MYb6eEIkJd3IyIiKs1KTEDSaDQYP348mjdvjlq1agEAwsPDYWVlBScnJ71lPTw8EB4enu26Zs+eDUdHR93Nx8enKItumrSn+z9+DBw6BACoWhV49105+/33gcREYMAA4OefZXDSZioGJCIiKu1KTEAKCgrC+fPnsX79+ude1+TJkxEXF6e73blzpxBKWMIYnO6vNW0a4OgInD4N1KkjW9+srOT9nDlymZAQICmp+ItMRERUXEpEQBo9ejS2bduG/fv3o3z58rrpnp6eSE5ORmxsrN7yERER8PT0zHZ9SqUSKpVK71YmGZzuDwDlygEffywfX70KWFvL/kg9esh+Rx4ewNOnQA4tmERERCWeSQckIQRGjx6NTZs2Yd++ffDz89Ob37BhQ1haWmLv3r26aWFhYbh9+zYCAgKKu7glj8Hp/lpjxwL16smapO3b5WIAoFAA2u5ebGYjIqLSzKQDUlBQENasWYN169bBwcEB4eHhCA8Px9OnTwEAjo6OGDZsGCZMmID9+/fjxIkTeOuttxAQEJDnM9jKNIPT/bWUSllDFB4OtGmj/5IOHeQ9AxIREZVmJh2Qli9fjri4OLRp0wZeXl662y+//KJb5quvvkL37t3Rt29ftGrVCp6envj92WnrlAdZNLMBsmnN2jrz4toapOPHgUePirhsRERERlKixkEqKmVyHCSt48eBxo0BBwfg4UPZIzsXNWrIwSQ3bgT69i2GMhIREWWB4yBR0WnQQPa8fvwY2LkzTy9hMxsREZV2DEhlnZkZ8Prr8vH33+fpJdqAtGdPEZWJiIjIyBiQCBg2TN5v2wY8eJDr4q1by4Ejr18Hbt4s4rIREREZAQMSAdWrA82by4uuZbymSDYcHADtSYLP08wWGgqEhRX89UREREWFAYmkt9+W999/D2g0uS7+vP2QTp4EWrQA2rXL0+aIiIiKFQMSSf37AyqVbDf7++9cF9cGpH37ZMVTfs2dK4PRvXvAhQv5fz0REVFRYkAiyc4OGDRIPv7uu8zzk5KA114DXnkFSE1F48YyT8XEAKdO5W9T16/LIQK0QkIKXmwiIqKiwIBE6bTNbL/9BkRHp08XAhgxAvj1V3lhtsOHYWEBtG0rZ+e3me3LL2Xtkbm5fP7PP89fdCIiosLEgETpGjSQF2FLTgbWrEmfPm8e8OOP6c937QJQsNP9IyOB4GD5ePp0eR8SIjMYERGRqWBAonQKBTB8uHz83XcytfzxBzB5spymvTDbsyojbUA6cCDvzWyLFwOJiUCTJsDEifK6b5GRwJUrhfYuiIiInhsDEukbNEhehO38eeCbb4DAQBmURo1Kr0U6ehR49Aj+/kCnTkBKirykW25jIsXHA0uXyscffig3ox0ugP2QiIjIlDAgkT4nJ3lGGwCMHAkkJMiqoq+/Bnx8gGrVZAeiffugUAC//ALUqQNERMiwFBWV/aq/+05e4NbfH+jZU05r1Uresx8SERGZEgYkykzbWRuQgejXX+XQ2QDQsaO8f9YPydER2L4dqFABuHoV6N4dePIk8ypTUoAFC+TjiRPTO2i3bi3v2Q+JiIhMCQMSZdaypby98II8a83JKX1exoD0LNF4ewM7dgAuLrL17bXXgNRU/VWuXw/cuSOvi/vmm+nTX3pJZq+7d4Fbt4r0XREREeWZhbELQCZIoZBVOmlp6TVHWq1bA5aWMs1cvw5UrgxAVjRt3SpHxv7zT6BZM8DTM/1lx4/L+3HjZN8jLTs7oHFjedmRf/4B/PyK9q0RERHlBWuQKGsKReZwBAD29vK6bYCumU2rWTPZJ8nMDDh2TAYm7e3BA9kcN3Jk5lVq+yGxozYREZkK1iBR/nXoIC9HsmuXPLstg1dekddZO3Ys88teekm/tU6rdWt56RF21CYiIlOhEIJdY9VqNRwdHREXFweVSmXs4pi+48ehu9bIw4eyye05qNWAs7M8Oe7uXdn1iYiIKDdF+f3NJjbKv/r1AVdXmWyOHn3u1alUcpVA0dciCQFcuiQHCyciIsoOAxLln7k50L69fGzQD6mgirofkhCyqE2aADVq6I9kQEREZIgBiQpGe52RQg5IRVGDdPCgvEpKp07pZ9OtWSPHbSIiIsoKAxIVjDYgPbvsSL6lpQGffy5Pe4McdgmQzV+RkYVTxIcP5cCVLVvK4GVlBYwfL4ciEAL48svC2Y6pePgQOH3a2KUgIiodGJCoYHx90y87sn9//l//1VfAlCnAW28BKSlwdQVq1ZKzDhyQ91euAB9/LDczZIi8yG1eJSUBffrIMZnMzeU1eK9dk5udNk0us2oVEB6e/6KbGiGA1avlkFT16wMTJmQeqJOIiPKHAYkKzuCyI3l2+TLwySfy8dOnwIULANIvO/L113JMpapVgVmzgLAwGQA6dQJiY3NfvRByvKUDB2QH8OPHgZUr5aXkAKBFCznkQFISsHhx3osdGyvL8dtv8lq+T59mvVxysuy/Xhzu35dDKwwZAsTFyWlffQV061awij0iIpIYkKjgtAFp507ZZJYXqanA4MEynWg9GzRJ2w/pwAE5sraZGdC1q2wKU6lkM1nLlnIogJzMnw8EB8vX//ILUK+e/nyFAvjwQ/l42TLg8ePci33pkhzZYMgQoF8/oHZtOQp4xYqyf1PjxnIUcJUKUCrloJh168oasNDQ3HfPnTvyknfvvQe8846s7cqJELIfVa1awLZtsvlw1ix5SRdbW5lZX3pJhksiIso/joMEjoNUYPHx8uJqCQlAUJCsjlEocn7N3LnApEkySfToAaxdK9u/Vq6EWi1rkdLS5PXaAgMBLy/5sjNngC5d5Ijc5cvLa7/VrJl59Vu2AL16yQDx9dfA2LFZF0OjkWezhYXJADZhQvZF/usvYOBAWSvk7S23HxaWXmOTF+XKyb5PDg7602NjgcOHM4c+Z2dgwwb5GkPh4cC77wKbN8vnDRvK5kJtE+Xp00DPnsDt23I3r14t+2JlNTB6UYmIkEExq4FBiQpbaqqsMY2Olv9TlSvL/zkqfhqN/GyMjgZiYvI+pEq1ajkfs7Q04MgROexe48bp04vy+5sBCQxIz+WXX2R6EAL49FPZryg7Fy4ADRrI/5jgYJkW+vWTHWdOnsx1U//9J5vZwsLkF++nn8pQUKWKDC5nz8qroDx5IgPEsmU557Xvv5en+5cvLy8rZ2WlP18IWRv14YfycYsWsnnN3V0+f/hQ9pO6fVu+FVfX9JtGI2txtm0Dtm/PPUyZmwN16simxWPHZN93c3PZXDZ6tHwfQsgaotGj5QePpaXsT/XBB5nH6oyMBPr2lWfwAbK2q0kTICBA3po1kxcXLgqrVwMjRshtbt2afmUaypuUFPk3XBjhMjVV/q2kpMi/y4zXQQTkl87t2/Lv+MoVWevatKms/cxu/NeYmMx/z0LI30vR0em3mBigenX5O8jcPPsyxsbK8jk75y3EJyfLWuZt24Ddu4F797Juevf3T/97DwiQZTH8Hwfkvt6zR/6tHjyY+Qvdygro3FnWHhvWRmfn7l1Zc3zjhqxlrlJFlsfeXs5Xq+WXfWgo8O+/8v+1TZv0k0oy7nshZI1yaKj84VG5slxfpUqZj2dWoqLkZ9CFC/rHJzpafk5l/NxydZWfNYbLKRSZl0tLy7yc9rhrNHnbTxmZmcnj1L27vNWsKffTzp3yWP/1l1x/z57AH3+kv44BqYgxID2nJUuAMWPk4xUrZBuRodRU+dd//LjsILN1q2xXqlBBfio+fpyn//boaNnn5t9/9afb2cl/sMePZa3L9u25D/CdlCSbxR48kDUwgwenz4uKkrVKa9bI52+/DSxdmvUHbG5SUoBDh+QHomFTm1Ipa4AaN5bvAZCd0UeMAH76KX3b06bJC/3+/rucVr++LHOdOtlvNzlZhqdVqzJ/oWX1YaQNk4mJMjBeuSK/pFu3lsvnJjVVhskFC9KnWVsDP/8sa/WehxA5h10h5JflsmWyHBk/yMuVS/+S8vXN+cs6PzQa2QfsyhX5xRUQILeTX5GR8stPezt2TPZva91a/k3265e55lErOlr+YNAGHG1g135ZGR53W9v0/ZKcLL94s/qFb2MDNGokg7RKpb/+6Oj8vT8/P/m3O3Ro+vtITJQ1oKtXyy9A7Reqk1PmY6d9bG8v/+937sy+j5+jo1zu3r3M88zMZFmqVJE3T0857tr+/fot/jmpU0cek1dflevLGAxu35a1waGh8qMtK97ecn+Ghcm/2ayoVDKQ1awpPy5DQ+WPMUMKhfz41L6fjLf4eBkqtm6VZTLGt7ydXdahPCvJyfL65xl5ecnP4YwnnDg5yR9+332XPo0BqYgxIBWCKVPkaftmZrJtqE+f9HmJicDMmXK+k5P8KePtLf9rPT3TvyFeeilPm3r6FJg3T36RXLkif6Vpg0eVKvIDwdk5b8WeN09+qWs/jP76S35o//WX/Mc0NwcWLpQtiLm1HhYmIWTQ+OAD+eVhZibvLSzkrp48Oe9XeNFoZB8q7Rfwv//KfvIZVawof5leuyZr6jJ+KlSvLvtGvf66/OLMyqNHsiJx5075/JNPZFPftm2y7EuXylq9nDx5AuzdK5tP//tP/8snNlb2+9I2vXp6pr/u0CHZ1ysvg4wqlfJ9+vvLdRj+Ks74pezoKMv+6FF6OMgYRq5ela3LGdWqlR46X3pJ/s1om36io2Xz6NWr+oEjIiLnMtvayi+F9u3lF2/G18bE5P6eAfl3nF0/OKVS7o8qVeT/1uHDuXfwt7HJ/P+gDV/a/efgII+lNlA5OsrW9MePZU1ofpqoDbm7y99Z3brJpnJXV1kjqq2BiolJr6EJDZU1sjmdOOHnJ2u6OnfOXLMaESF7AvzxR96bi7Q1wtWqpR8zw+FL/PxkAA0IkOXfuVOedRsVlXl92h9Svr7y8y6/Tfz168uaKTc3/b91w5CnPVaG/xNCZF7OzCzr/x/tTanMe/kAuZ/+/FN+Zuzdm37WcrVq6f9TzZpl/txjQCpiDEiFQAj5DbhypfzPmDYNuHlTpo5z59J/Bvz4I/DGG+mv69ZNppHFi2XbUQGkpMhN3bwpf/W6uub9tXFx8uy2x4/lB3rGDtuNGskA1bZtgYpVKLZvBwYMkB/udevK2qC8VvPnRPthtHWr/DAy/AWtUskvzLCw9H1Srpy8NnHv3vpNIY8eyVquK1fkl+Tq1bLWIzVVnk2o/bX3ySeyWVShkF/Wjx7JL42QEFmOffvy9kve3Fx+kfXtC2zcKP98APln9+67Mkhl/CCPipLBL7vakuyYmclfwTl14rewAF58UX75nzypH0KsreX2cmtuUCjkl3zG5iBbW/mlvHq13K858fHRrz3w85PHShtUnJ3le1Gr5f54+FDem5vL5X189GvVNBp53LXhIilJf/2VK6c3FeUmIUH+y3/1Veb34esrA++bb8oyx8Rk3WSjvT16JL8se/SQNa55qdXUEkKG04xB984d2eLfo4f8EZDbD6BHj2SPglWrZPgyN5dhShsI3N3lZ0ZAgCyf4T569EiG45gY+T+cMeRraTQyzG3dKn8kNGwo11e/vn7gEEL+XWvfy9Wr6eH92jW5b9q3l6GiWzfZjaAkSUiQ+7h8eRnec8KAVMQYkApJWhrQvz+waVPmeW5u8lt05kz9T6Lp04EZM+Sn5OrVxVbUjD78UAYhQFbrvvGGrEavUcMoxcnk1i35YdG7d8Ga+HLz5IlsZoiKSq9JcHOTh0mtln21vv5afmDnxNdXNptkDHDarmnTp8vnPj6y+j82Nutq/4oV5RdW3br6zSu2tjIsrl4tazgyMjcHhg2TNWs5fRFk7G9z9ap8v9l9GcfH67+2fPn0gODvL4egqFJFllf7izYmRtaYaPudZewXo+2j5uaWvo8z3rJrQhNCHvtVq4CLF/WbiLRhRds0a8o0GhnIg4NlmHzjDdnnJj8hx5Q8fSoDiymWPy1N7u/nvIZ4icGAVMQYkApRYqLsg3T/vvw51aiR/Dnl45P1T7Rt2+Q3Yo0auvGQitvTp7LvSs2a8ldXcZ7tVVKkpsr+T4sWZV2j8dJLsqbI3T3r13/7razdMaxNUalkjU+PHvLXbo0auf+Sv3xZ1kr89Zd87bRpMigUpqQkGZQePwZeeCHvNSZaqamyKUSlkrUMRRFsiYgBqcgxIBlRRISsa1YoZHtXdj+lqcS7dUvW4Gj7qbi4lJ1fuURUNIry+5u/lcm4PDxk7dKdO7ITh3Y4bSp1KlYs2FleRETGYIItqFTmNGok748fN245iIiInmFAIuPTDov67JIjRERExsaARMbHgERERCaGAYmMr2FDeX/jRt5HviMiIipCDEhkfM7O6edpsx8SERGZAAYkMg3sqE1ERCaEAYlMQ3b9kOLi5PUqFi8u2CWiiYiICoDjIJFpyCogxccDXbvKq6sC8qJhq1fLaxUQEREVIdYgkWmoX19e2OjePeDBA3n9j1dekeFIpZLXati8WQap8+eNXVoiIirlGJDINNjby0tqAzIU9e0rr6Bqbw/s2gUcPCivhnr1KtC0KfDzz7mvU60GPv9croeIiCgfGJDIdGib2YYNk5dDt7GRlwBv2lTOO3FCXk02IQEYNAh47z15VdCsPHokl50yBXj5ZbnOjJdXJyIiygEDEpkO7ZlscXHpTWqtWqXPL1cO2LEDmDxZPl+4UPZRMhw7KSpKhqJjx9Ivw/7DD/JS8Zs3F/nbICKiko8BiUxHs2by3sIC2LgR6NAh8zLm5sCsWXK+rS2we7esYbp0Sc4PDwfatgVOn5YXwg0NBQ4cAKpUkX2bevUCBgwAHj4srndFREQlEAMSmY769eVZavv3Az165Lxs376yr1KFCsC1azIkBQcDrVsDFy4A3t5ASAhQqxbQooUMTJMmyYD1yy+y+e3x42J5W0REVPIohBDC2IUwNrVaDUdHR8TFxUGlUhm7OJQfUVFAv37AP/+kT/P1BfbtAypVyrz8iRNAt25ARIS837xZhqa8EgJQKJ6/3Np1qdUctoCIqICK8vubNUhUsrm5yWa2d96Rz198UYalrMIRIK/7tmULYG0tO4C//37ethMXBwQFyWY9Pz+gf39g3jwZxOLi8l/ulBRg4EDAyQl47TXg5s38r4OIiIoMa5DAGqRS4/RpGYwcHHJfduNGGXIAYMkSGX6ys2kTMHo0cP9+1vPNzORZddOnZx/MMkpNBd54A1i/Pn2alRUwdizw8ccyNOVXUpLsW7Vrl2xm7NxZ3vz9C6/Gi4jIxBTl9zcDEhiQyqw5c+QZcWZmwNat8oy4jO7eBcaMAf74Qz739wcWLZJh5vhxeZbc8ePArVtyvoWFHE5gyhTghRey3mZaGvDWW8BPPwGWlsBXX8n179kj57u6yqA1cmTem/5SUoBXX00vZ0YVK8omyM8+k7VmRESlCANSEWNAKqOEkIEmOFgOB/Daa3LIgOhoebtxQ47obWEhO3h//HHWIePkSeCTT+TYTQCgVAKjRslaorp1ZQAD5LXkhg+XQw6YmwMbNgC9e8tybN8OTJyYfjZe69bAmjVA+fI5v4e0NLmdn3+W2/32W9m/ascOefZecrJcrkcP4LffZCgjIiolGJCKGANSGZacDHTqBPz9d9bzX3pJho5atXJf18GDwEcfyWCi5eEBdOwom7tCQoCVK2Vg+vlnWeuTUWqq3NYHH8jr0Lm4AN9/L4cmyErGwGVpKZsCu3VLn//kieyEPmwYkJgot7duXf46pRMRmTAGpCLGgFTGxcbKpjOFQjZxaW8eHkDNmuk1QHkhhOwHtHSp7MD95In+fIVCNq8FBma/jmvXZAfu48fl81GjgPnz5cjiGbczdqzsP2VmJocu6Ncv6/X99ZcMWSkpwODBMlDl5z0REZkoBqQixoBERSIpSY7VtGMHsHMncOUKsHy5DCm5SU6WTXrz58vnFSrIwJaSIm9PnwLXr8vAtXq1bGbLye+/yxqktDTZv2npUnbeJqISjwGpiDEgkcnauRN4800gMjLzPIVCBi7tEAe5WbtWBikh5Fl706bJYRKIiEooBqQixoBEJi0mRvZvUihkXyNLS9lxvEIFeZZafnz7LTBihHxsZiYvy/Lqq7KzOMMSEZUwDEhFjAGJypT162XT3YkT6dPMzYF27WRtVe/eckBMIiITx4BUxBiQqEy6cUMONfDrr3KoAi0HB1mrNGQI0Lw5+yoRkcliQCpiDEhU5l27Jsdd+vFH/cue2NjI8ZW0TXvacZS0ncW1NysrQKWSN0dHea9QAAkJskO59t7bG2jVSt4CAgA7u/RtJSXJQTf/+08OcVCzpv6Ze0REBhiQihgDEtEzGo0cx2n1almzZDhMQWGysAAaNJAB7MYNeSmXjB9H5uZAtWpysM169eRI5uXLAz4+sr+UmZlc/t49eYbglStyPba2gKdn+s3DQ94zbBGVOgxIebB06VJ88cUXCA8PR926dbF48WI0adIkT69lQCLKwtOnMrSkpMhBLLX3Quh3Fre0lPPi4gC1Wt60F/C1tZXBxNZWBqGwMHkx4ZAQ4M6dzNu0s5OdzyMjgYcPsy+blZUMPQ8fytqpvFCp0gOTm5scSiFjeePjZejSvicLC7kdBwf92jFHx/TQlTGEubrmbaTylBRZU3blitwfDx/KCyBXqQJUrSrXxWZNojxhQMrFL7/8gjfffBMrVqxA06ZNsXDhQmzYsAFhYWFwd3fP9fUMSERG8N9/wKFDMgy8+KK8lSsnnwshw9mZM/IixGfOyFBx5w4QHp65punFF2W4qFRJNtWFh6ffHjyQ04qDg0P6QKNOTnLcqYwB8/Fj2YSZmprzOvz95Xvy80u/L19eBkht6LSxkfsqPj495KnVcpsuLunlKGmXlxFC7ic7O476TrliQMpF06ZN0bhxYyxZsgQAoNFo4OPjgzFjxmDSpEm5vp4BiagESUmR4en+fRkA/PxyDgFCyOCgDUwREbKGyto6vb+Uo6O8Hp8Q+n2rkpLSA4g2hMTGytdnDGGRkfqhLTc2NrLGqEoVGQpv3pQ1SrduyWbOwuTgIG8FrZXS9i/T7iuVKutrEpqZZa5ZTErS33dqtXx/GZezsJD7WHsdxJgYGSAtLABf3/SAWKGCbPK9c0deSPrOHRl+7ewy1+bZ2OiXw9JSltnWNj1gWlvL8mXsI5eYKEOZ4Wuz2ncZ51taytdpg7D2ptGkh1ntdi0t5XYybjevAd7MTH9dtrbpNbgZg7hCkXk5hSJ9m9rtCpE+X7usuXnmPoZZ/W1rj3fG/WAYaIWQNbWGfRFtbNIDvL195v2bmiqXTU7Wf18pKfK4VaigW5QBKQfJycmwtbXFxo0b0SvDNasGDx6M2NhYbN68Odd1MCAR0XNJS5PBSXuh4+hoGQYMv2xtbIDKlWVn9awu95KUJPtRXb0qQ9PNm/L5zZsyED59Km+GrKzSA4yZmQwZMTH5C21ExmBlJWs8gfTwllMNa4cO8nJOzxTl97dFoa7NCB4+fIi0tDR4eHjoTffw8MDly5ezfE1SUhKSMiT2uGf9JdRqddEVlIhKN0vL9FqM3MTHZz/vhRfkrU2brOdrNOk1H2lpMhQplZmXS0uTtTYxMc/X2T5jX63Hj+V9cnLW20tN1f+1r1Sm12Bpa+kMayhSU2UtkItL+s3JCXj0SDbDas9svH1bvt7bW+6f8uVlbdHTp7JWMCoqvXYwKSnzNrS1NtpbYqIsn7Zmydpa3jSa9Ndpm0gNCZH5vaal6fdf09Y8aY+VtgYlJSW9Bku7XaUybzV8aWmy3NqamKdP02vbMm4bSH+PCQnpNVSWlunb1I51lnG5jMc14zqzCvPa46197zmxtk6vSbO2ltuMjpbbS06WtbA5MayZzPBdrf3eLoq6nhIfkApi9uzZmDFjRqbpPj4+RigNERGVKY8eGWe72pMptCdR5EQbABMTn3+7iYnyVtD3rQ2hgLz4tqNjpkWio6PhmMX051HiA1K5cuVgbm6OiIgIvekRERHwzOaX3OTJkzFhwgTd89jYWFSoUAG3b98u9B1M+aNWq+Hj44M7d+6wudPIeCxMB4+F6eCxMC1xcXHw9fWFi7aZrhCV+IBkZWWFhg0bYu/evbo+SBqNBnv37sXo0aOzfI1SqYQyiyppR0dH/sGbCJVKxWNhIngsTAePhengsTAtZlk1Az6nEh+QAGDChAkYPHgwGjVqhCZNmmDhwoV48uQJ3nrrLWMXjYiIiEqgUhGQXnvtNURFRWHq1KkIDw9HvXr1sGPHjkwdt4mIiIjyolQEJAAYPXp0tk1quVEqlZg2bVqWzW5UvHgsTAePhengsTAdPBampSiPR4kfB4mIiIiosBV+ryYiIiKiEo4BiYiIiMgAAxIRERGRAQYkIiIiIgNlPiAtXboUFStWhLW1NZo2bYqjR48au0il3uzZs9G4cWM4ODjA3d0dvXr1QlhYmN4yiYmJCAoKgqurK+zt7dG3b99Mo6VT4ZszZw4UCgXGjx+vm8ZjUXzu3buH119/Ha6urrCxsUHt2rVx/Phx3XwhBKZOnQovLy/Y2Nigffv2uHr1qhFLXHqlpaVhypQp8PPzg42NDSpVqoTPPvtM75pfPB5F459//kGPHj3g7e0NhUKBP/74Q29+XvZ7TEwMAgMDoVKp4OTkhGHDhiE+p2sgZqFMB6RffvkFEyZMwLRp03Dy5EnUrVsXnTp1QmRkpLGLVqqFhIQgKCgIhw8fxu7du5GSkoKOHTviSYYLar733nvYunUrNmzYgJCQENy/fx99+vQxYqlLv2PHjuGbb75BnTp19KbzWBSPR48eoXnz5rC0tMT27dtx8eJFfPnll3B2dtYtM2/ePCxatAgrVqzAkSNHYGdnh06dOiGxMK6XRXrmzp2L5cuXY8mSJbh06RLmzp2LefPmYfHixbpleDyKxpMnT1C3bl0sXbo0y/l52e+BgYG4cOECdu/ejW3btuGff/7BiBEj8lcQUYY1adJEBAUF6Z6npaUJb29vMXv2bCOWquyJjIwUAERISIgQQojY2FhhaWkpNmzYoFvm0qVLAoAIDQ01VjFLtcePHwt/f3+xe/du0bp1azFu3DghBI9Fcfrwww9FixYtsp2v0WiEp6en+OKLL3TTYmNjhVKpFD///HNxFLFM6datmxg6dKjetD59+ojAwEAhBI9HcQEgNm3apHuel/1+8eJFAUAcO3ZMt8z27duFQqEQ9+7dy/O2y2wNUnJyMk6cOIH27dvrppmZmaF9+/YIDQ01YsnKnrhnV5bWXmzwxIkTSElJ0Ts21apVg6+vL49NEQkKCkK3bt309jnAY1GctmzZgkaNGqF///5wd3dH/fr18e233+rm37x5E+Hh4XrHwtHREU2bNuWxKALNmjXD3r17ceXKFQDAmTNncPDgQXTp0gUAj4ex5GW/h4aGwsnJCY0aNdIt0759e5iZmeHIkSN53lapGUk7vx4+fIi0tLRMlyPx8PDA5cuXjVSqskej0WD8+PFo3rw5atWqBQAIDw+HlZUVnJyc9Jb18PBAeHi4EUpZuq1fvx4nT57EsWPHMs3jsSg+N27cwPLlyzFhwgR89NFHOHbsGMaOHQsrKysMHjxYt7+z+szisSh8kyZNglqtRrVq1WBubo60tDTMnDkTgYGBAMDjYSR52e/h4eFwd3fXm29hYQEXF5d8HZsyG5DINAQFBeH8+fM4ePCgsYtSJt25cwfjxo3D7t27YW1tbezilGkajQaNGjXCrFmzAAD169fH+fPnsWLFCgwePNjIpSt7fv31V6xduxbr1q1DzZo1cfr0aYwfPx7e3t48HmVEmW1iK1euHMzNzTOdjRMREQFPT08jlapsGT16NLZt24b9+/ejfPnyuumenp5ITk5GbGys3vI8NoXvxIkTiIyMRIMGDWBhYQELCwuEhIRg0aJFsLCwgIeHB49FMfHy8kKNGjX0plWvXh23b98GAN3+5mdW8fjf//6HSZMmYcCAAahduzbeeOMNvPfee5g9ezYAHg9jyct+9/T0zHSyVWpqKmJiYvJ1bMpsQLKyskLDhg2xd+9e3TSNRoO9e/ciICDAiCUr/YQQGD16NDZt2oR9+/bBz89Pb37Dhg1haWmpd2zCwsJw+/ZtHptC1q5dO5w7dw6nT5/W3Ro1aoTAwEDdYx6L4tG8efNMw11cuXIFFSpUAAD4+fnB09NT71io1WocOXKEx6IIJCQkwMxM/yvS3NwcGo0GAI+HseRlvwcEBCA2NhYnTpzQLbNv3z5oNBo0bdo07xt77i7mJdj69euFUqkUq1atEhcvXhQjRowQTk5OIjw83NhFK9VGjhwpHB0dxd9//y0ePHiguyUkJOiWeffdd4Wvr6/Yt2+fOH78uAgICBABAQFGLHXZkfEsNiF4LIrL0aNHhYWFhZg5c6a4evWqWLt2rbC1tRVr1qzRLTNnzhzh5OQkNm/eLM6ePSt69uwp/Pz8xNOnT41Y8tJp8ODB4oUXXhDbtm0TN2/eFL///rsoV66c+OCDD3TL8HgUjcePH4tTp06JU6dOCQBiwYIF4tSpU+K///4TQuRtv3fu3FnUr19fHDlyRBw8eFD4+/uLgQMH5qscZTogCSHE4sWLha+vr7CyshJNmjQRhw8fNnaRSj0AWd6Cg4N1yzx9+lSMGjVKODs7C1tbW9G7d2/x4MED4xW6DDEMSDwWxWfr1q2iVq1aQqlUimrVqomVK1fqzddoNGLKlCnCw8NDKJVK0a5dOxEWFmak0pZuarVajBs3Tvj6+gpra2vx4osvio8//lgkJSXpluHxKBr79+/P8jti8ODBQoi87ffo6GgxcOBAYW9vL1QqlXjrrbfE48eP81UOhRAZhgUlIiIiorLbB4mIiIgoOwxIRERERAYYkIiIiIgMMCARERERGWBAIiIiIjLAgERERERkgAGJiIiIyAADEhFRFhQKBf744w9jF4OIjIQBiYhMzpAhQ6BQKDLdOnfubOyiEVEZYWHsAhARZaVz584IDg7Wm6ZUKo1UGiIqa1iDREQmSalUwtPTU+/m7OwMQDZ/LV++HF26dIGNjQ1efPFFbNy4Ue/1586dw8svvwwbGxu4urpixIgRiI+P11vmhx9+QM2aNaFUKuHl5YXRo0frzX/48CF69+4NW1tb+Pv7Y8uWLUX7ponIZDAgEVGJNGXKFPTt2xdnzpxBYGAgBgwYgEuXLgEAnjx5gk6dOsHZ2RnHjh3Dhg0bsGfPHr0AtHz5cgQFBWHEiBE4d+4ctmzZgsqVK+ttY8aMGXj11Vdx9uxZdO3aFYGBgYiJiSnW90lERlI4194lIio8gwcPFubm5sLOzk7vNnPmTCGEEADEu+++q/eapk2bipEjRwohhFi5cqVwdnYW8fHxuvl//vmnMDMzE+Hh4UIIIby9vcXHH3+cbRkAiE8++UT3PD4+XgAQ27dvL7T3SUSmi32QiMgktW3bFsuXL9eb5uLionscEBCgNy8gIACnT58GAFy6dAl169aFnZ2dbn7z5s2h0WgQFhYGhUKB+/fvo127djmWoU6dOrrHdnZ2UKlUiIyMLOhbIqIShAGJiEySnZ1dpiavwmJjY5On5SwtLfWeKxQKaDSaoigSEZkY9kEiohLp8OHDmZ5Xr14dAFC9enWcOXMGT5480c0/dOgQzMzMULVqVTg4OKBixYrYu3dvsZaZiEoO1iARkUlKSkpCeHi43jQLCwuUK1cOALBhwwY0atQILVq0wNq1a3H06FF8//33AIDAwEBMmzYNgwcPxvTp0xEVFYUxY8bgjTfegIeHBwBg+vTpePfdd+Hu7o4uXbrg8ePHOHToEMaMGVO8b5SITBIDEhGZpB07dsDLy0tvWtWqVXH58mUA8gyz9evXY9SoUfDy8sLPP/+MGjVqAABsbW2xc+dOjBs3Do0bN4atrS369u2LBQsW6NY1ePBgJCYm4quvvsLEiRNRrlw59OvXr/jeIBGZNIUQQhi7EERE+aFQKLBp0yb06tXL2EUholKKfZCIiIiIDDAgERERERlgHyQiKnHYM4CIihprkIiIiIgMMCARERERGWBAIiIiIjLAgERERERkgAGJiIiIyAADEhEREZEBBiQiIiIiAwxIRERERAYYkIiIiIgM/B8MMV4u3cBMFgAAAABJRU5ErkJggg==",
            "text/plain": [
              "<Figure size 640x480 with 1 Axes>"
            ]
          },
          "metadata": {},
          "output_type": "display_data"
        }
      ],
      "source": [
        "# choose cross entropy loss function (equation 5.24 in the loss notes)\n",
        "loss_function = nn.CrossEntropyLoss()\n",
        "# construct SGD optimizer and initialize learning rate and momentum\n",
        "optimizer = torch.optim.SGD(model.parameters(), lr = 0.05, momentum=0.9)\n",
        "# object that decreases learning rate by half every 20 epochs\n",
        "scheduler = StepLR(optimizer, step_size=20, gamma=0.5)\n",
        "# create 100 dummy data points and store in data loader class\n",
        "x_train = torch.tensor(train_data_x.transpose().astype('float32'))\n",
        "y_train = torch.tensor(train_data_y.astype('long')).long()\n",
        "x_val= torch.tensor(val_data_x.transpose().astype('float32'))\n",
        "y_val = torch.tensor(val_data_y.astype('long')).long()\n",
        "\n",
        "# load the data into a class that creates the batches\n",
        "data_loader = DataLoader(TensorDataset(x_train,y_train), batch_size=100, shuffle=True, worker_init_fn=np.random.seed(1))\n",
        "\n",
        "# Initialize model weights\n",
        "model.apply(weights_init)\n",
        "\n",
        "# loop over the dataset n_epoch times\n",
        "n_epoch = 100\n",
        "# store the loss and the % correct at each epoch\n",
        "losses_train = np.zeros((n_epoch))\n",
        "errors_train = np.zeros((n_epoch))\n",
        "losses_val = np.zeros((n_epoch))\n",
        "errors_val = np.zeros((n_epoch))\n",
        "\n",
        "for epoch in range(n_epoch):\n",
        "  # loop over batches\n",
        "  for i, data in enumerate(data_loader):\n",
        "    # retrieve inputs and labels for this batch\n",
        "    x_batch, y_batch = data\n",
        "    # zero the parameter gradients\n",
        "    optimizer.zero_grad()\n",
        "    # forward pass -- calculate model output\n",
        "    pred = model(x_batch[:,None,:])\n",
        "    # compute the loss\n",
        "    loss = loss_function(pred, y_batch)\n",
        "    # backward pass\n",
        "    loss.backward()\n",
        "    # SGD update\n",
        "    optimizer.step()\n",
        "\n",
        "  # Run whole dataset to get statistics -- normally wouldn't do this\n",
        "  pred_train = model(x_train[:,None,:])\n",
        "  pred_val = model(x_val[:,None,:])\n",
        "  _, predicted_train_class = torch.max(pred_train.data, 1)\n",
        "  _, predicted_val_class = torch.max(pred_val.data, 1)\n",
        "  errors_train[epoch] = 100 - 100 * (predicted_train_class == y_train).float().sum() / len(y_train)\n",
        "  errors_val[epoch]= 100 - 100 * (predicted_val_class == y_val).float().sum() / len(y_val)\n",
        "  losses_train[epoch] = loss_function(pred_train, y_train).item()\n",
        "  losses_val[epoch]= loss_function(pred_val, y_val).item()\n",
        "  print(f'Epoch {epoch:5d}, train loss {losses_train[epoch]:.6f}, train error {errors_train[epoch]:3.2f},  val loss {losses_val[epoch]:.6f}, percent error {errors_val[epoch]:3.2f}')\n",
        "\n",
        "  # tell scheduler to consider updating learning rate\n",
        "  scheduler.step()\n",
        "\n",
        "# Plot the results\n",
        "fig, ax = plt.subplots()\n",
        "ax.plot(errors_train,'r-',label='train')\n",
        "ax.plot(errors_val,'b-',label='validation')\n",
        "ax.set_ylim(0,100); ax.set_xlim(0,n_epoch)\n",
        "ax.set_xlabel('Epoch'); ax.set_ylabel('Error')\n",
        "ax.set_title('Part I: Validation Result %3.2f'%(errors_val[-1]))\n",
        "ax.legend()\n",
        "plt.show()"
      ]
    }
  ],
  "metadata": {
    "colab": {
      "authorship_tag": "ABX9TyNb46PJB/CC1pcHGfjpUUZg",
      "include_colab_link": true,
      "provenance": []
    },
    "kernelspec": {
      "display_name": ".venv",
      "language": "python",
      "name": "python3"
    },
    "language_info": {
      "codemirror_mode": {
        "name": "ipython",
        "version": 3
      },
      "file_extension": ".py",
      "mimetype": "text/x-python",
      "name": "python",
      "nbconvert_exporter": "python",
      "pygments_lexer": "ipython3",
      "version": "3.12.1"
    }
  },
  "nbformat": 4,
  "nbformat_minor": 0
}