AI_Summer_BYOL_in_CIFAR10.ipynb

{
  "cells": [
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "_OMBQgFNUMyQ"
      },
      "source": [
        "# BYOL CIFAR10 Tutorial Ai Summer\n",
        "\n",
        "## Basic imports"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "bDaWDTnQ4SSj"
      },
      "outputs": [],
      "source": [
        "from tqdm import tqdm\n",
        "import numpy as np\n",
        "import torch\n",
        "import torch.nn as nn\n",
        "import torch.optim as optim\n",
        "import torch.utils.data as data\n",
        "import torchvision.transforms as transforms\n",
        "\n",
        "def reproducibility(SEED):\n",
        "    torch.manual_seed(SEED)\n",
        "    torch.backends.cudnn.deterministic = True\n",
        "    torch.backends.cudnn.benchmark = False\n",
        "    np.random.seed(SEED)\n",
        "    if torch.cuda.is_available():\n",
        "        torch.cuda.manual_seed(SEED)\n",
        "\n",
        "def define_param_groups(model, weight_decay, optimizer_name):\n",
        "    def exclude_from_wd_and_adaptation(name):\n",
        "        if 'bn' in name:\n",
        "            return True\n",
        "        if optimizer_name == 'lars' and 'bias' in name:\n",
        "            return True\n",
        "\n",
        "    param_groups = [\n",
        "        {\n",
        "            'params': [p for name, p in model.named_parameters() if not exclude_from_wd_and_adaptation(name)],\n",
        "            'weight_decay': weight_decay,\n",
        "            'layer_adaptation': True,\n",
        "        },\n",
        "        {\n",
        "            'params': [p for name, p in model.named_parameters() if exclude_from_wd_and_adaptation(name)],\n",
        "            'weight_decay': 0.,\n",
        "            'layer_adaptation': False,\n",
        "        },\n",
        "    ]\n",
        "    return param_groups"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "znYh1nn7b1XG"
      },
      "source": [
        "## Augmentations \n"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "X5wf39mFb0FC"
      },
      "outputs": [],
      "source": [
        "import torch\n",
        "from torchvision import transforms as T\n",
        "import torchvision.transforms.functional as TF\n",
        "import random\n",
        "\n",
        "\n",
        "class Augment:\n",
        "    \"\"\"\n",
        "    A stochastic data augmentation module\n",
        "    Transforms any given data example randomly\n",
        "    resulting in two correlated views of the same example,\n",
        "    denoted x ̃i and x ̃j, which we consider as a positive pair.\n",
        "    \"\"\"\n",
        "    def __init__(self, img_size, s=1):\n",
        "        color_jitter = T.ColorJitter(\n",
        "            0.8 * s, 0.8 * s, 0.8 * s, 0.2 * s\n",
        "        )\n",
        "        blur = T.GaussianBlur((3, 3), (0.1, 2.0))\n",
        "\n",
        "        self.train_transform = T.Compose([\n",
        "            T.ToTensor(),\n",
        "            T.RandomResizedCrop(size=img_size),\n",
        "            T.RandomHorizontalFlip(p=0.5),  # with 0.5 probability\n",
        "            T.RandomApply([color_jitter], p=0.8),\n",
        "            T.RandomApply([blur], p=0.5),\n",
        "            T.RandomGrayscale(p=0.2),\n",
        "            # imagenet stats\n",
        "            T.Normalize(mean=[0.485, 0.456, 0.406], std=[0.229, 0.224, 0.225])            \n",
        "        ])\n",
        "    def __call__(self, x):\n",
        "        return self.train_transform(x), self.train_transform(x), "
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "nDg1RCPFerm6"
      },
      "source": [
        "# Dataloaders"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 86
        },
        "id": "HQPmF7XIes7B",
        "outputId": "cde464af-0f5a-48e0-a31c-a850385c5d55"
      },
      "outputs": [
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Downloading https://www.cs.toronto.edu/~kriz/cifar-10-python.tar.gz to ./cifar-10-python.tar.gz\n"
          ]
        },
        {
          "data": {
            "application/vnd.jupyter.widget-view+json": {
              "model_id": "13da6466c1824e52ad4cd6861c366f23",
              "version_major": 2,
              "version_minor": 0
            },
            "text/plain": [
              "  0%|          | 0/170498071 [00:00<?, ?it/s]"
            ]
          },
          "metadata": {},
          "output_type": "display_data"
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Extracting ./cifar-10-python.tar.gz to ./\n"
          ]
        }
      ],
      "source": [
        "from torchvision.datasets import CIFAR10\n",
        "from torch.utils.data import DataLoader\n",
        "\n",
        "def get_cifar10_dataloader(batch_size, train=True, transform=Augment(224)):\n",
        "    dataset = CIFAR10(root=\"./\", train=train, transform=transform, download=True)\n",
        "    return DataLoader(dataset=dataset, batch_size=batch_size, num_workers=4, drop_last=True)\n",
        "data_transform = Augment(32)\n",
        "dataloader = get_cifar10_dataloader(batch_size=64, train=True, transform=data_transform)"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "TxS14RpVRih5"
      },
      "source": [
        "# BYOL method"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "nGbRO6fWRfZT"
      },
      "outputs": [],
      "source": [
        "import copy\n",
        "import torch\n",
        "from torch import nn\n",
        "import torch.nn.functional as F\n",
        "\n",
        "class MLP(nn.Module):\n",
        "    def __init__(self, dim, embedding_size=256, hidden_size=2048, batch_norm_mlp=False):\n",
        "        super().__init__()\n",
        "        norm = nn.BatchNorm1d(hidden_size) if batch_norm_mlp else nn.Identity()\n",
        "        self.net = nn.Sequential(\n",
        "            nn.Linear(dim, hidden_size),\n",
        "            norm,\n",
        "            nn.ReLU(inplace=True),\n",
        "            nn.Linear(hidden_size, embedding_size)\n",
        "        )\n",
        "\n",
        "    def forward(self, x):\n",
        "        return self.net(x)\n",
        "\n",
        "\n",
        "class AddProjHead(nn.Module):\n",
        "    def __init__(self, model, in_features, layer_name, hidden_size=4096,\n",
        "                 embedding_size=256, batch_norm_mlp=True):\n",
        "        super(AddProjHead, self).__init__()\n",
        "        self.backbone = model\n",
        "        # remove last layer 'fc' or 'classifier'\n",
        "        setattr(self.backbone, layer_name, nn.Identity())\n",
        "        self.backbone.conv1 = torch.nn.Conv2d(3, 64, kernel_size=3, stride=1, padding=1, bias=False)\n",
        "        self.backbone.maxpool = torch.nn.Identity()\n",
        "        # add mlp projection head\n",
        "        self.projection = MLP(in_features, embedding_size, hidden_size=hidden_size, batch_norm_mlp=batch_norm_mlp)\n",
        "\n",
        "    def forward(self, x, return_embedding=False):\n",
        "        embedding = self.backbone(x)\n",
        "        if return_embedding:\n",
        "            return embedding\n",
        "        return self.projection(embedding)\n",
        "\n",
        "\n",
        "def loss_fn(x, y):\n",
        "    # L2 normalization\n",
        "    x = F.normalize(x, dim=-1, p=2)\n",
        "    y = F.normalize(y, dim=-1, p=2)\n",
        "    return 2 - 2 * (x * y).sum(dim=-1)\n",
        "\n",
        "\n",
        "class EMA():\n",
        "    def __init__(self, alpha):\n",
        "        super().__init__()\n",
        "        self.alpha = alpha\n",
        "\n",
        "    def update_average(self, old, new):\n",
        "        if old is None:\n",
        "            return new\n",
        "        return old * self.alpha + (1 - self.alpha) * new\n",
        "\n",
        "\n",
        "\n",
        "class BYOL(nn.Module):\n",
        "    def __init__(\n",
        "            self,\n",
        "            net,\n",
        "            batch_norm_mlp=True,\n",
        "            layer_name='fc',\n",
        "            in_features=512,\n",
        "            projection_size=256,\n",
        "            projection_hidden_size=2048,\n",
        "            moving_average_decay=0.99,\n",
        "            use_momentum=True):\n",
        "        \"\"\"\n",
        "        Args:\n",
        "            net: model to be trained\n",
        "            batch_norm_mlp: whether to use batchnorm1d in the mlp predictor and projector\n",
        "            in_features: the number features that are produced by the backbone net i.e. resnet\n",
        "            projection_size: the size of the output vector of the two identical MLPs\n",
        "            projection_hidden_size: the size of the hidden vector of the two identical MLPs\n",
        "            augment_fn2: apply different augmentation the second view\n",
        "            moving_average_decay: t hyperparameter to control the influence in the target network weight update\n",
        "            use_momentum: whether to update the target network\n",
        "        \"\"\"\n",
        "        super().__init__()\n",
        "        self.net = net\n",
        "        self.student_model = AddProjHead(model=net, in_features=in_features,\n",
        "                                         layer_name=layer_name,\n",
        "                                         embedding_size=projection_size,\n",
        "                                         hidden_size=projection_hidden_size,\n",
        "                                         batch_norm_mlp=batch_norm_mlp)\n",
        "        self.use_momentum = use_momentum\n",
        "        self.teacher_model = self._get_teacher()\n",
        "        self.target_ema_updater = EMA(moving_average_decay)\n",
        "        self.student_predictor = MLP(projection_size, projection_size, projection_hidden_size)\n",
        "    \n",
        "    @torch.no_grad()\n",
        "    def _get_teacher(self):\n",
        "        return copy.deepcopy(self.student_model)\n",
        "    \n",
        "    @torch.no_grad()\n",
        "    def update_moving_average(self):\n",
        "        assert self.use_momentum, 'you do not need to update the moving average, since you have turned off momentum ' \\\n",
        "                                  'for the target encoder '\n",
        "        assert self.teacher_model is not None, 'target encoder has not been created yet'\n",
        "\n",
        "        for student_params, teacher_params in zip(self.student_model.parameters(), self.teacher_model.parameters()):\n",
        "          old_weight, up_weight = teacher_params.data, student_params.data\n",
        "          teacher_params.data = self.target_ema_updater.update_average(old_weight, up_weight)\n",
        "\n",
        "    def forward(\n",
        "            self,\n",
        "            image_one, image_two=None,\n",
        "            return_embedding=False):\n",
        "        if return_embedding or (image_two is None):\n",
        "            return self.student_model(image_one, return_embedding=True)\n",
        "\n",
        "        # student projections: backbone + MLP projection\n",
        "        student_proj_one = self.student_model(image_one)\n",
        "        student_proj_two = self.student_model(image_two)\n",
        "\n",
        "        # additional student's MLP head called predictor\n",
        "        student_pred_one = self.student_predictor(student_proj_one)\n",
        "        student_pred_two = self.student_predictor(student_proj_two)\n",
        "\n",
        "        with torch.no_grad():\n",
        "            # teacher processes the images and makes projections: backbone + MLP\n",
        "            teacher_proj_one = self.teacher_model(image_one).detach_()\n",
        "            teacher_proj_two = self.teacher_model(image_two).detach_()\n",
        "            \n",
        "        loss_one = loss_fn(student_pred_one, teacher_proj_one)\n",
        "        loss_two = loss_fn(student_pred_two, teacher_proj_two)        \n",
        "\n",
        "        return (loss_one + loss_two).mean()\n"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "OTPAkVK3sMzS"
      },
      "source": [
        "# LARS optimizer\n"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "12HgYb4jsVKI"
      },
      "outputs": [],
      "source": [
        "from torch.optim.optimizer import Optimizer, required\n",
        "import torch\n",
        "\n",
        "# almost copy paste from https://github.com/noahgolmant/pytorch-lars/blob/master/lars.py\n",
        "class LARS(Optimizer):\n",
        "    r\"\"\"Implements LARS (Layer-wise Adaptive Rate Scaling).\n",
        "    Args:\n",
        "        params (iterable): iterable of parameters to optimize or dicts defining\n",
        "            parameter groups\n",
        "        lr (float): learning rate\n",
        "        momentum (float, optional): momentum factor (default: 0)\n",
        "        eta (float, optional): LARS coefficient as used in the paper (default: 1e-3)\n",
        "        weight_decay (float, optional): weight decay (L2 penalty) (default: 0)\n",
        "        dampening (float, optional): dampening for momentum (default: 0)\n",
        "        nesterov (bool, optional): enables Nesterov momentum (default: False)\n",
        "        epsilon (float, optional): epsilon to prevent zero division (default: 0)\n",
        "    Example:\n",
        "        >>> optimizer = torch.optim.LARS(model.parameters(), lr=0.1, momentum=0.9)\n",
        "        >>> optimizer.zero_grad()\n",
        "        >>> loss_fn(model(input), target).backward()\n",
        "        >>> optimizer.step()\n",
        "    \"\"\"\n",
        "\n",
        "    def __init__(self, params, lr=required, momentum=0, eta=1e-3, dampening=0,\n",
        "                 weight_decay=0, nesterov=False, epsilon=0):\n",
        "        if lr is not required and lr < 0.0:\n",
        "            raise ValueError(\"Invalid learning rate: {}\".format(lr))\n",
        "        if momentum < 0.0:\n",
        "            raise ValueError(\"Invalid momentum value: {}\".format(momentum))\n",
        "        if weight_decay < 0.0:\n",
        "            raise ValueError(\"Invalid weight_decay value: {}\".format(weight_decay))\n",
        "\n",
        "        defaults = dict(lr=lr, momentum=momentum, eta=eta, dampening=dampening,\n",
        "                        weight_decay=weight_decay, nesterov=nesterov, epsilon=epsilon)\n",
        "        if nesterov and (momentum <= 0 or dampening != 0):\n",
        "            raise ValueError(\"Nesterov momentum requires a momentum and zero dampening\")\n",
        "        super(LARS, self).__init__(params, defaults)\n",
        "\n",
        "    def __setstate__(self, state):\n",
        "        super(LARS, self).__setstate__(state)\n",
        "        for group in self.param_groups:\n",
        "            group.setdefault('nesterov', False)\n",
        "\n",
        "    def step(self, closure=None):\n",
        "        \"\"\"Performs a single optimization step.\n",
        "        Arguments:\n",
        "            closure (callable, optional): A closure that reevaluates the model\n",
        "                and returns the loss.\n",
        "        \"\"\"\n",
        "        loss = None\n",
        "        if closure is not None:\n",
        "            loss = closure()\n",
        "\n",
        "        for group in self.param_groups:\n",
        "            weight_decay = group['weight_decay']\n",
        "            momentum = group['momentum']\n",
        "            eta = group['eta']\n",
        "            dampening = group['dampening']\n",
        "            nesterov = group['nesterov']\n",
        "            epsilon = group['epsilon']\n",
        "\n",
        "            for p in group['params']:\n",
        "                if p.grad is None:\n",
        "                    continue\n",
        "                w_norm = torch.norm(p.data)\n",
        "                g_norm = torch.norm(p.grad.data)\n",
        "                if w_norm * g_norm > 0:\n",
        "                    local_lr = eta * w_norm / (g_norm +\n",
        "                        weight_decay * w_norm + epsilon)\n",
        "                else:\n",
        "                    local_lr = 1\n",
        "                d_p = p.grad.data\n",
        "                if weight_decay != 0:\n",
        "                    d_p.add_(p.data, alpha=weight_decay)\n",
        "                if momentum != 0:\n",
        "                    param_state = self.state[p]\n",
        "                    if 'momentum_buffer' not in param_state:\n",
        "                        buf = param_state['momentum_buffer'] = torch.clone(d_p).detach()\n",
        "                    else:\n",
        "                        buf = param_state['momentum_buffer']\n",
        "                    buf.mul_(momentum).add_(d_p, alpha=1 - dampening)\n",
        "                    if nesterov:\n",
        "                        d_p = d_p.add(momentum, buf)\n",
        "                    else:\n",
        "                        d_p = buf\n",
        "\n",
        "                p.data.add_(d_p, alpha=-local_lr * group['lr'])\n",
        "\n",
        "        return loss"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "cGUGGUYo-rwu"
      },
      "source": [
        "# Pre-train function"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "leycY3ABYxej"
      },
      "outputs": [],
      "source": [
        "def training_step(model, data):\n",
        "    (view1, view2), _ = data\n",
        "    loss = model(view1.cuda(), view2.cuda())\n",
        "    return loss\n",
        "\n",
        "def train_one_epoch(model, train_dataloader, optimizer):\n",
        "    model.train()\n",
        "    total_loss = 0.\n",
        "    num_batches = len(train_dataloader)\n",
        "    for data in train_dataloader:\n",
        "        optimizer.zero_grad()\n",
        "        loss = training_step(model, data)\n",
        "        loss.backward()\n",
        "        optimizer.step()\n",
        "        # EMA update\n",
        "        model.update_moving_average()\n",
        "\n",
        "        total_loss += loss.item()\n",
        "        \n",
        "    \n",
        "    return total_loss/num_batches"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "jtfbIiygSX9_"
      },
      "source": [
        "# KNN evaluation to track classification accuray through ssl pretraining"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "oqeDuGP_Xmkr"
      },
      "outputs": [],
      "source": [
        "import numpy as np\n",
        "import torch\n",
        "from sklearn.model_selection import cross_val_score\n",
        "from sklearn.neighbors import KNeighborsClassifier\n",
        "from torch import nn\n",
        "\n",
        "\n",
        "class KNN():\n",
        "    def __init__(self, model, k, device):\n",
        "        super(KNN, self).__init__()\n",
        "        self.k = k\n",
        "        self.device = device\n",
        "        self.model = model.to(device)\n",
        "        self.model.eval()\n",
        "\n",
        "    def extract_features(self, loader):\n",
        "        \"\"\"\n",
        "        Infer/Extract features from a trained model\n",
        "        Args:\n",
        "            loader: train or test loader\n",
        "        Returns: 3 tensors of all:  input_images, features , labels\n",
        "        \"\"\"\n",
        "        x_lst = []\n",
        "        features = []\n",
        "        label_lst = []\n",
        "\n",
        "        with torch.no_grad():\n",
        "            for input_tensor, label in loader:\n",
        "                h = self.model(input_tensor.to(self.device))\n",
        "                features.append(h)\n",
        "                x_lst.append(input_tensor)\n",
        "                label_lst.append(label)\n",
        "\n",
        "            x_total = torch.stack(x_lst)\n",
        "            h_total = torch.stack(features)\n",
        "            label_total = torch.stack(label_lst)\n",
        "\n",
        "            return x_total, h_total, label_total\n",
        "\n",
        "    def knn(self, features, labels, k=1):\n",
        "        \"\"\"\n",
        "        Evaluating knn accuracy in feature space.\n",
        "        Calculates only top-1 accuracy (returns 0 for top-5)\n",
        "        Args:\n",
        "            features: [... , dataset_size, feat_dim]\n",
        "            labels: [... , dataset_size]\n",
        "            k: nearest neighbours\n",
        "        Returns: train accuracy, or train and test acc\n",
        "        \"\"\"\n",
        "        feature_dim = features.shape[-1]\n",
        "        with torch.no_grad():\n",
        "            features_np = features.cpu().view(-1, feature_dim).numpy()\n",
        "            labels_np = labels.cpu().view(-1).numpy()\n",
        "            # fit\n",
        "            self.cls = KNeighborsClassifier(k, metric=\"cosine\").fit(features_np, labels_np)\n",
        "            acc = self.eval(features, labels)\n",
        "            \n",
        "        return acc\n",
        "    \n",
        "    def eval(self, features, labels):\n",
        "      feature_dim = features.shape[-1]\n",
        "      features = features.cpu().view(-1, feature_dim).numpy()\n",
        "      labels = labels.cpu().view(-1).numpy()\n",
        "      acc = 100 * np.mean(cross_val_score(self.cls, features, labels))\n",
        "      return acc\n",
        "\n",
        "    def _find_best_indices(self, h_query, h_ref):\n",
        "        h_query = h_query / h_query.norm(dim=1).view(-1, 1)\n",
        "        h_ref = h_ref / h_ref.norm(dim=1).view(-1, 1)\n",
        "        scores = torch.matmul(h_query, h_ref.t())  # [query_bs, ref_bs]\n",
        "        score, indices = scores.topk(1, dim=1)  # select top k best\n",
        "        return score, indices\n",
        "\n",
        "    def fit(self, train_loader, test_loader=None):\n",
        "        with torch.no_grad():\n",
        "            x_train, h_train, l_train = self.extract_features(train_loader)\n",
        "            train_acc = self.knn(h_train, l_train, k=self.k)\n",
        "\n",
        "            if test_loader is not None:\n",
        "                x_test, h_test, l_test = self.extract_features(test_loader)\n",
        "                test_acc = self.eval(h_test, l_test)\n",
        "                return train_acc, test_acc"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "Q5JfoJj69vex"
      },
      "source": [
        "# Putting it all together"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "zqnw0lfq-zm0"
      },
      "outputs": [],
      "source": [
        "import torchvision.models as models\n",
        "import torch\n",
        "import numpy as np\n",
        "import os\n",
        "import torch\n",
        "import torchvision.transforms as T\n",
        "import torch.nn as nn\n",
        "import torch.nn.functional as F\n",
        "from torchvision.datasets import CIFAR10\n",
        "from torch.utils.data import DataLoader\n",
        "from google.colab import files\n",
        "\n",
        "load = False\n",
        "model = models.resnet18(pretrained=False)\n",
        "model = BYOL(model, in_features=512, batch_norm_mlp=True)\n",
        "model.cuda()\n",
        "\n",
        "# optimizer and loss\n",
        "lr = 3e-4\n",
        "weight_decay = 0.000001\n",
        "BATCH_SIZE  = 256\n",
        "\n",
        "\n",
        "#param_groups = define_param_groups(model, weight_decay, 'lars')    \n",
        "#optimizer = LARS(param_groups, lr=0.1, momentum=0.9)\n",
        "\n",
        "optimizer = torch.optim.Adam(model.parameters(), lr=lr)\n",
        "\n",
        "# data\n",
        "data_transform = Augment(32)\n",
        "\n",
        "test_transform = T.Compose([\n",
        "            T.ToTensor(),\n",
        "            T.Normalize(mean=[0.485, 0.456, 0.406], std=[0.229, 0.224, 0.225]) ])\n",
        "\n",
        "\n",
        "loader_train = get_cifar10_dataloader(batch_size=BATCH_SIZE, train=True, transform=data_transform)\n",
        "loader_train_plain = get_cifar10_dataloader(batch_size=BATCH_SIZE, train=True, transform=test_transform)\n",
        "loader_test = get_cifar10_dataloader(batch_size=BATCH_SIZE, train=False, transform=test_transform)\n",
        "\n",
        "\n",
        "# general info\n",
        "available_gpus = len([torch.cuda.device(i) for i in range(torch.cuda.device_count())])\n",
        "print('available_gpus:',available_gpus)\n",
        "\n",
        "PATH = 'BYOL_resNet18.ckpt'\n",
        "\n",
        "reproducibility(9999)\n",
        "\n",
        "if load:\n",
        "  model.load_state_dict(torch.load(\"....ckpt\"))\n",
        "\n",
        "\n",
        "epochs = 100\n",
        "mean_losses = []\n",
        "train_knns = []\n",
        "val_knns = []\n",
        "\n",
        "for i in range(epochs):\n",
        "  mean_loss = train_one_epoch(model, loader_train, optimizer)\n",
        "  \n",
        "  mean_losses.append(mean_loss)\n",
        "  if (i%4)==0:\n",
        "    # KNN evaluation\n",
        "    ssl_evaluator = KNN(model=model, k=1, device='cuda')\n",
        "    train_acc, val_acc = ssl_evaluator.fit(loader_train_plain, loader_test)\n",
        "    print(f'\\n Epoch {i}: loss:{mean_loss}')\n",
        "    print(f\"k-nn accuracy k= {ssl_evaluator.k} for train split: {train_acc}\")\n",
        "    print(f\"k-nn accuracy k= {ssl_evaluator.k} for val split: {val_acc} \\n\")\n",
        "    print('-----------------')\n",
        "    train_knns.append(train_acc)\n",
        "    val_knns.append(val_acc)\n",
        "\n",
        "\n",
        "import matplotlib.pyplot as plt\n",
        "\n",
        "plt.plot(train_knns, label='train set KNN acc (%)')\n",
        "plt.plot(val_knns, label='val set KNN acc (%)')\n",
        "plt.ylabel('Accuracy in %')\n",
        "plt.xlabel('Epochs')\n",
        "plt.legend()\n",
        "\n",
        "\n",
        "torch.save(model.state_dict(), PATH)\n",
        "files.download(PATH)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 406
        },
        "id": "3dE_pZtRBXeV",
        "outputId": "8f9396f9-3e5f-4857-8360-4e5c6efd5c85"
      },
      "outputs": [
        {
          "data": {
            "text/plain": [
              "<matplotlib.legend.Legend at 0x7fe3a0292390>"
            ]
          },
          "execution_count": 10,
          "metadata": {},
          "output_type": "execute_result"
        },
        {
          "data": {
            "image/png": "iVBORw0KGgoAAAANSUhEUgAAAmMAAAFzCAYAAABy9g57AAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4yLjIsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+WH4yJAAAgAElEQVR4nOzdd3hc1b3u8e8aadR7s9wL7hhsY4MNBkIzvYWaECAktMMJEHJzckMqJwk3hySckISTEwKhJaG3hA6GmGKKG+62bLlXFctWs6TRlHX/WCNZtmVbtjXaI+n9PM88e2ZP2T+NSfRqVWOtRURERES84fO6ABEREZHeTGFMRERExEMKYyIiIiIeUhgTERER8ZDCmIiIiIiHFMZEREREPJTodQEdUVBQYIcMGeJ1GSIiIiIHNX/+/O3W2sKOvr5bhLEhQ4Ywb948r8sQEREROShjzIZDeb26KUVEREQ8pDAmIiIi4iGFMREREREPKYyJiIiIeEhhTERERMRDCmMiIiIiHlIYExEREfGQwpiIiIiIhxTGRERERDykMCYiIiLiIYUxEREREQ91i70pRURE2hOOWILhCKGIJRiKEIxECIXduWD0GApbgpEIwZB7XShiyU710zc7hYKMZBJ8xusfQ3o5hTEREfFEOGLZsauZyroA2+sDVNYFqGw5Rm9VuwI0BSOEwhGCkd3hqjnszkXskdWQ4DP0yUymODuFvtmp0WNKm2MqRZnJ+BO6b0dSJGLZvLORFWW1rCyrY3t9gIzkRDJSEslMTiQzxb/7cUoimcl+MlISyUhOJCkxNj+3tZZAKEJDc5hdgZA7NodoCESPzSECwQjJfh+p/kTSkhJITUog1Z9AWlICaUmJpPrduVjV2JUUxkREpNNYa6ltCu0OVG3C1fa2Qas+QFV9oN0wlZ6UQGFmMgUZyQwtSCctKZFEnyExwUdSgjsmJhiSEnwk+nz4Ew1+nzvnT/DhTzDR8z780ff5W5/zkeCD6oYg22qaKKtpcsdaF1b+VVJBYzC8Rz3GQGFGMn1zUumblbJXYEulb3YKRVnJJCcmdNG3vH87dzVTUlbHyrJaSsrqKCmrY1V5HQ3Nu3+mrJREdjWHCXcgySYn+siMBrN9Q1vLfT/JiT4amsM0NIfYFXBH93h/Yatj1++IRJ9pDWYutCWS6ve5wNZyrs3zF43vx+jirE65dmdRGBORHi8SsVQ3BlvDQMsxLz2JKcPy6Z+T6nWJ3UJjc5iyWhdgymubKKt1x/LWcy5kNYci+7zXn2AozEimMDOZfjkpHDsgm8JM97jlfEsAS0/27leTtZbaxhDbahv3DGs17vGayno+Wb2dukBon/fmpydRmJlMn6wUiqLHPlnJFGa6Y5+sFAo7qZUtEAqzuqKekm11rCyvaw1g5bWB1tfkpvkZVZzJVZMHMro4k1HFmYzsk0l6cmJry1RtU5D6phD1gRD1TSFqW+8HqYverwuE3P2mIPWBEJt2NLQ+Vx8I7RGqEnyGtKQE0pMSSUuOHqPhenBS2p7nkxNI8yeQlpy4z+vTkxNJTvRFW89CNDaHaQy6cNcYDXmNwTCNbUJfU/R595xrXdteH9jrfSHG9ctWGBMR6QwtAaslXLXt5tpe17zHuR27mgkd4K/wAbmpTBmaz9RheUwdls+A3FSM8W4cUWVdgAUbd7JgUzULNu5kS3Uj6UmJZKX4XYtEtDViz6N7PmOv5zOSEvEdZExUOGKpqg9QXhtwYau2ifKa3WGrJXzVNu0bQDKSEynKSqY4K4UpQ/MozGoTrtqErOxUv6ffaUcZY8hO85Od5j/gL+y6piDltS6obatpYlt1E+V1TVTUNlFRF6CkrJbKuvZb/vLTkyhqDWzJFEXDWlGbENcS2qx1XYx7t3at276rNQQlJfgYXpTBtOEF0dCVxZjiTAozk/f7nRtjSPEnkOJPoCjz8L8vay2NwTCBYITUpASSE31x/e9sbee0xnU2E6+FtTV58mQ7b948r8sQkU4WjljqAyF2Rf/Cbvlre1f0r/S6Ns9V1Tfv0bJVtau53W6OlhaYgmgri7ufREFG9HH0fEFGEttqmvh8bRWz1+5gzvod7NjVDEC/7BSmDHPhbMrQfAbnp8XsF0xzKMLybbV8sWF3+Nq8sxFw3S9H98tiSEE6uwJh6qKtFXWB6LEpdNCuHmMgI2nvAJeIP8FHRV2A8mh42PtzfAYXErJTWsdU9clKoTjaTdcneszwsBUr3oUjlqpdASpqA1TUuZbDlu+7otY9rqhrOmBoC4Qi1LdphRuQm8ro4qzWlq4xfTMZkp9OYjce09YTGWPmW2snd/j1CmMiciRaxghtq2lka3UjW6ubqGkM7jdUtX3cdhzLgaT4feSlJR00YBVmJJOVmnhYwSkSsZRW1DN7nQtns9dVsb3ehbM+WcnRlrN8pgzLY1hB+mFdw1rL1pom1+q10QWvpVtrW7v1+mWnMHFQLhMH5TBxUA5H98smxb//cUjWWpqCEeqagtQ2hXaHtbb3A23P736+ORRp7VLrk7Vv2NIsw67T0jLZEo7L24Q3f4JhVHEmo4uzGNkng8wUv9flSgcojIlIp2oORSivbWJLdWM0cLn7W6t3h6/6dsbPJPpM64ys1lvrQGA3RmSPx8nt3E/2k56c4Mlf/dZa1lTuci1n63Ywe20VFXVuPE5hZjInDM1j6lDXrTm8KKPdcNbQHGLJ5prWFq8FG6tbPyM50cexA7Jd+BqYw8RBuRRnp3TpzygisaEwJtLNWWspq22itLye1RX1lFbUs7qiji07G0lundbtZgylJ+2eIZTeZuZQWnQQbNv7qUm7B8emRs/7DOxsCLK1urE1YG2r2TNsVdQF2Pv/JvLSk+iXk0K/7FT65aTSP8cd+0bP5aT5437syKGy1rK+qiHarekC2raaJsB1J50wNI8pQ/PITPGzcFM1CzbtZMW2utbuvyH5abtbvQbmMrpvZrdeLkFE9i9uwpgxJgX4CEjGTRR40Vp7j3H/73wvcCUQBv5krf3DgT5LYUx6okjEsqW6kdKKOkrLW0KXu7VtacpO9TOyTwYDc9NoDkdaZxK1nTrecj/Qziy2A0n0mX0Gticn+naHq+yUPcJWvxw3lT81yfsp/F6z1rJpRyOfr63i82jX5pZqN9YrIzmRCQNzWrsbJwzMJS89yeOKRaSrHGoYi+XIywBwhrW23hjjB2YZY94CxgADgdHW2ogxpiiGNYi0amx2U8FXlrt1d1aV17GqrI6axiA5aUlkp/rJSfOTm5ZEdpqf3DQ/OalJ5KT5yUlLco+j97NT/R1u1QiFI2zY0UBpeT1rKuspLa+jtMLdbwruDk+FmckML8zgsuP6M6Iog+FFmQwvyqAgI6nDLUzhiJvZ1BDYN6i1vd8S6AKhMAUZyW0CVwp56R2/Xm9mjGFQfhqD8tO46viBAGza0UBTMMywwgyNtxKRDotZGLOuya0++tAfvVngNuAaa20k+rqKWNUgvVMgFGZt5a7WwLWyrJ7Sijo27mho7W5LSvBxVFEGxw/NIz89mZrGINUNzVQ3BllRVktNQ5DqxuABZ6plJidGQ9vuwJaT6kIcxrCmsp7V5fWs276L5vDu0NUvO4XhfTJbxxq54JVBTtqRt5wk+Ezr+CzpegPz0rwuQUS6oZj+P7YxJgGYDwwH/mitnW2MOQq42hjzZaASuNNaWxrLOqRnCoUjrK/axaryelaW1VFaUcfKsjrWVzW0hqgEn2FoQTrj+mVz2cQBjOyTwcjiTAbnpR10UHgkYqlvDlG9K0h1YzM7G6KBrSFIdUOQnQ3N1DS6Y3VDkE07GqhuDFLTGARgUF4awwszOG10ISOKMhlRlMFRRRkKSiIisoeY/law1oaBCcaYHOAVY8w43BiyJmvtZGPMZcBjwCl7v9cYcwtwC8CgQYNiWaZ0A43NYeau38GSLTWsjG7vsbZyd4uTMTA4L42RfTI5b1xfRhZnMrJPBkML0g97ixKfz5CV4icrxc8gOt7iEY5YwhHbI/ZLExGR2Ouy2ZTGmJ8CDcBNwHnW2nXRwfzV1trsA71XA/h7n3DEsmxrDbNWb2dW6XbmbdjZuhZT/5zU1hauUX3c9h5HFWZoULmIiMSFuBnAb4wpBILW2mpjTCowHfgV8A/gdGAd8CVgVaxqkO5lY1WDC1+rK/l0TRXVDa67b3RxJl8/cTDThhcwaXCuFj0UEZEeJZbdlH2BJ6PjxnzA89ba140xs4CnjDHfwQ3wvymGNUgcq25o5tM1Va2tXxt3NABQnJXCWWP6cMqIAk46qoDCzGSPKxUREYmdWM6mXAxMbOd8NXBBrK4r8SsQCjN/w05mlW7nk9XbWbylBmvdmkxTh+XzzWlDOHlEIUcVHt5WMyIiIt2RpnVJzEQilpKyOmatrmTW6irmrKuiKRghwWeYODCHb585gpOHFzB+YI5WIhcRkV5LYUw6XTAc4b/eLOHVRVtaN1oeXpTBV44fxMnDC5gyLE/jvkRERKIUxqRTNYci3PnMAt5eVsYFx/TltFGFnDyigL7ZqV6XJiIiEpcUxqTTBEJhvvXUF7y3ooKfXDiWG08e6nVJIiIicU9hTDpFUzDMrX+bz4erKvnFpeO4bupgr0sSERHpFhTG5Ig1NIe4+a/z+HRNFb+6/BiuPl47JoiIiHSUwpgckfpAiG8+MZd563fw31eO57LjBnhdkoiISLeiMCaHrbYpyA2PzWHR5hp+95WJXDy+n9cliYiIdDsKY3JYahqCXP/YbJZtreV/vjqR847p63VJIiIi3ZLCmByyHbuaue7R2ZSW1/PQtZM4a2wfr0sSERHpthTG5JBsrw9w7V9ms277Lh6+fhKnjSryuiQREZFuTWFMOqyitolr/jKbzTsbeOyG45k2vMDrkkRERLo9hTHpkG01jVzzyGzKa5t44hsnMHVYvtcliYiI9AgKY3JQm3Y0cM1fPqd6V5C/3XgCkwbneV2SiIhIj6EwJge0oWoX1zwym7qmIH+/aQrjB+Z4XZKIiEiPojAm+7Wmsp6vPTKbQCjM0zdPZVz/bK9LEhER6XEUxqRdpeV1fPWR2VhreeaWqYwuzvK6JBERkR5JYUz2sWJbLdf+ZTY+n+GZm6cyok+m1yWJiIj0WD6vC5D4snRLDV995HP8CT6eu0VBTEREJNbUMiatFm6q5vpHZ5OZ4ueZm6cyKD/N65JERER6PIUxAWDe+h3c8Phc8tKTePrmKQzIVRATERHpCuqmFD5bU8X1j82hKDOZ526dqiAmIiLShdQy1stV1DZx45Nz6Z+TylM3TaEoK8XrkkRERHoVtYz1cg/+azXNoQh/+fpkBTEREREPKIz1Ypt2NPDs3I1cffxABuene12OiIhIr6Qw1ov97r1SfMZwxxkjvC5FRESk11IY66VWV9TxyoLNXH/iYIqz1T0pIiLiFYWxXuq3M1aR6k/gttOGe12KiIhIr6Yw1gst3VLDm0vKuPGUYeSlJ3ldjoiISK+mMNYL3f/uSnLS/Nx0ylCvSxEREen1FMZ6mbnrd/DBykpu+9JRZKX4vS5HRESk11MY60Wstfzm7ZUUZiZz/YlDvC5HREREUBjrVT4q3c6c9Tu484zhpCYleF2OiIiIoDDWa1hruf+dlQzITeXq4wd5XY6IiIhEKYz1Eu8sK2PJlhruOmskSYn6ZxcREYkX+q3cC4QjlvvfXcXwogy+PLG/1+WIiIhIGwpjvcA/FmxhdUU9/2f6SBJ8xutyREREpA2FsR6uORThd++vYlz/LM49utjrckRERGQvCmM93HPzNrFpRyP/cfYofGoVExERiTsKYz1YUzDMg++XcvyQXL40stDrckRERKQdMQtjxpgUY8wcY8wiY8wyY8zP9nr+D8aY+lhdX+Cvn62noi7A984ZjTFqFRMREYlHiTH87ABwhrW23hjjB2YZY96y1n5ujJkM5Mbw2r1eXVOQ//1gDaeOLOSEoXlelyMiIiL7EbOWMeu0tHz5ozdrjEkAfgP831hdW+DRWeuobgjyvbNHeV2KiIiIHEBMx4wZYxKMMQuBCmCGtXY2cDvwqrV2Wyyv3Zvt2NXMXz5ex3njijlmQLbX5YiIiMgBxLKbEmttGJhgjMkBXjHGnApcCZx2sPcaY24BbgEYNEjb9xyKhz5cw67mEP9n+kivSxEREZGD6JLZlNbaamAmcDowHFhtjFkPpBljVu/nPQ9baydbaycXFmomYEeV1zbx5Kfr+fLE/ozok+l1OSIiInIQsZxNWRhtEcMYkwpMB+Zba4uttUOstUOABmvt8FjV0Bs9+K9SItbynbPUKiYiItIdxLKbsi/wZHTAvg943lr7egyv1+ttrGrg2Tmb+MoJAxmYl+Z1OSIiItIBMQtj1trFwMSDvCYjVtfvjX73/ioSfIY7zhjhdSkiIiLSQVqBv4coLa/jlQVbuOGkIfTJSvG6HBEREekghbEe4rczVpGelMi/fekor0sRERGRQ6Aw1gMs2VzDW0vLuOmUoeSmJ3ldjoiIiBwChbEe4P53V5Kb5ufGk4d6XYqIiIgcIoWxbm722io+XFXJbacdRWaK3+tyRERE5BApjHVj1lruf3clfbKSuf7EIV6XIyIiIodBYawb+3BVJXPX7+T2M0aQ4k/wuhwRERE5DApj3VQkYvnNOysZmJfK1ZMHel2OiIiIHCaFsW7q7WVlLNtay3fOGklSov4ZRUREuiv9Fu+GwhHLf7+7khFFGVwyob/X5YiIiMgRUBjrhl5ZsIU1lbv47tkjSfAZr8sRERGRI6Aw1s00hyL87r1VHNM/m3OOLva6HBERETlCCmPdzHNzN7J5ZyP/cc4ojFGrmIiISHenMNaNNAXD/OFfqzlhaB6njijwuhwRERHpBApj3cgHKyuprAtw++nD1SomIiLSQyiMdSMzlpeTnernpKPyvS5FREREOonCWDcRCkf4V0k5Z4wuIjFB/2wiIiI9hX6rdxPzN+xkZ0OQ6WP7eF2KiIiIdCKFsW5ixvJykhJ8nDqy0OtSREREpBMpjHUD1lpmrCjnpOH5ZCQnel2OiIiIdCKFsW6gtKKeDVUN6qIUERHpgRTGuoEZy8sBOGuMwpiIiEhPozDWDcxYXs74Adn0yUrxuhQRERHpZApjca6itomFm6rVRSkiItJDKYzFufdWVAAwfaw2BRcREemJFMbi3IzlZQzKS2NknwyvSxEREZEYUBiLY7sCIT5ZU8X0sX20F6WIiEgPpTAWxz5aVUlzKKLxYiIiIj2Ywlgcm7G8nJw0P5MH53pdioiIiMSIwlicCoUj/GtlhTYGFxER6eH0Wz5OzV2/k+qGINO10KuIiEiPpjAWp2YsLycpURuDi4iI9HQKY3HIbQxexrSj8knXxuAiIiI9msJYHFpVXs+mHY1a6FVERKQXUBiLQzOWlwFw1pgijysRERGRWFMYi0MzlpczYWAORdoYXEREpMdTGIsz5bVNLNpco4VeRUREegmFsTgzY3k5AGcrjImIiPQKCmNxZsbycobkpzG8SBuDi4iI9AYKY3GkPhDiM20MLiIi0qtoEas48uHKSprDEc7SqvsiIj2PtRBqgmDjXscmCDXuPoYCu59LTIG8Ye6WWQzx8Id6JAJ1W2F7KVStdsfmXTDqXBg+HfyafHaoYhbGjDEpwEdAcvQ6L1pr7zHGPAVMBoLAHOBWa20wVnV0JzOWl5Gb5meSNgYXEel61kK42QWL5vroce9b/f7vBxuigaqp/dAVajqy+vxpkDsU8lpuw3bfsvqDL6FzvocWTbUubLUErqpS2L4adqxxP2uLpAzwJcLCv0NyNoy5CI65Aoae2vk19VCxbBkLAGdYa+uNMX5gljHmLeAp4Nroa54GbgL+FMM6uoVgOMK/SiqYPrZYG4OLiHSGSBjqK6BmM9Rsih43Q+0WaKppP1hFQh3//MRUSEqDpHTwp7v7iamQludatPypR34M1MKOdbBj7e7j9lIonQHhwO5aEpIgd4gLZrltg9pQyBkECf72f4ZwCKo37G7laglcVaVQX777dcYHOYOhYAQMPQXyh7v7+SNci10kDOs+gCUvwvJ/umCWXgTjLoNxV8CAyfHRqhenYhbGrLUWqI8+9Edv1lr7ZstrjDFzgAGxqqE7mbt+B7VNIS1pISLSUU21uwNWzSYXsvZ4vHXfcJWUCdkDIDUX0gpcwEjKcIGq9ZbRgfvpXdPqk17gQhVn7nm+patwx9q9buth3ccQ3LX7tSbBBbKWcOZPhao1LnztWAeRNp1TafkuaA2fDgXDXdjKH+7el5i8/zoTEmH4We52YSOsegeWvADzHoPZD7mgOO5yOOZKKBrTiV9Qz2BcZorRhxuTAMwHhgN/tNZ+v81zfmA28G1r7cftvPcW4BaAQYMGTdqwYUPM6owHP3ttGU/P3siCn04nLUlD+USkFwqHXIho21oVqIO6sj1btmo2Q80WCNTs+X5fImT2c2Frf7eUbG9+tq5krWsRbBvSdkZb1arWunFpeUftGbYKose0vM6tpbEaSl53LWbrPgQbgaKjXTfmuMshd3DnXi9OGGPmW2snd/j1sQxjrRcxJgd4BbjDWrs0eu4RYJe19q6DvX/y5Ml23rx5Ma7SO9ZaTvn1TEb1yeTRG473uhwRkUNjrWuVqlrjwtP+xlYFGw487upgY6pS86KhamD02H/Pxxl9NEbpYKx1N58Hw2HqK2DZKy6YbZ7jzg2c4lrLxl4KGYVdX1OMHGoY65ImGGtttTFmJnAusNQYcw9QCNzaFdePdyVldWze2cjtpw/3uhQRiRfhEFSugC3z3S+x7AGuSy13MGT29SZ0RMJufFHlKqgsgcqV7rh9lQtV++NP27erLznTjTXauxuw9bVtzmf1c7ek9K77WXsqY7wbu5VRBFNudbed62HpSy6Yvfkf8Nb3YdhprsVs9IWQkuVNjR6J5WzKQiAYDWKpwHTgV8aYm4BzgDOttZFYXb87mbG8HGPgTC1pIQcSaobEJK+rkFiw1nUjbfkiepsP2xa57qT2+PwunOUO3h3QctrcTy88sl+44aAbS9QSuLa3hK7SPVuvMoqhcBRM+Jo75g93Y7Hajqvyp6m1SvaVOwRO+a67lS9zoWzpi/CP2yDhLhh5jpuVmZTu/giw4egxstfjA52PtPO6MBx7NRQf4/U3sIdYtoz1BZ6MjhvzAc9ba183xoSADcBn0YVNX7bW/jyGdcS9GcvLmTgwh8LMAwyOlN4nHHJN+avecTOnKpa5wcdZfV3LSFa/PY+Zfd1z6qrZv0gYGqrcLLH68t1jjLL6upaarlJfsTt0bY0eG3e65xJToO94mPwN6Hcc9D/OLVtQu8W1JlRvgOqNsHODu1/yBjRs3/Pz/WluwHbOoD3DWssxNce9LtgUXbZg5e5WrsqVrrux7aDu7EFQOBKGfsmFrsLRUDBy9+eIHIk+R7vbmT+FzXNdMFv2Mqx4tfOuYXxuIoMvAQYcH3dh7JDGjEXXDkuy1tbGrqR99eQxY9tqGjnxv/7F988dzW2nHeV1OeK1Xdth9XsugK15302/9yXCoBPdranGzaCqK4PabVBftu9sMeNzgWyPwNbXhY7M4t3neko3gLXue6mv2B2y6itgV0U75yrdX9DtScrY87va+5hZ7L7XhEP8GzZQB1sX7g5dW75wg9HB/VsVjnGBq/8kdywau/9lCPZ7jXoX0Ko3uoDWEtRajoG9/i87ORtSs91A+Jbvw/hca0XhaBe4CkZFjyMhWduzSRcLh6BiuWvJaglRrUffXo87cL6Lu2ZjNmYs2r14BZBgjJlrrf3h4RQoe3ovujH49LFFHlfSC+1YC7N+51oC+ox1fykVH+umXXfV2JRIBMoWuZavVe+4X9ZYtz7P6ItgxHQ46vT9zwCLRFzAqNvqwlnrsczdr1oD6z92YWVvSRmuq2vEdDeAtvjY+FwHKByEDZ+4Fpz6ija38t3HtusttfD5XXjKKHItS/0mRh9Hz6UXuSBbt80tgVC3LXp/m7te3bb2g2560V5BrSX0FrtzocZo6FrgjpUlQPSP3pzB7q/yKbe68NV3fOf8t5ac4f4b7jN23+eshabqfQNa40449iu7W7ryh2vldIkfCYnQ91ivq+gy+20ZM8ZcbK19tc3jZ621X4neX2StHd9FNfbolrHrHp3N5p2N/Ou7X9J+lF1leyl8dL9bA8eX6H4hVq5sM03euF9MxcdA8TgXUoqPcb/EO+PfqKkW1s6E0nddCKsvd9fsfxyMOAdGng3F4zt3tlPzrmhr2tY24aPMDbxe96ELHfkjotPNr3BT3r0UCsCamW7xyJVvtAmTxq271BKo2garvc+l5h7Zv1ck4rr/9v7OWgNv9FxTdfvvTyto0+I1yYXB9ILDr0dEuo3ObBk7xhhzI3CPtXYhsNgY8xfcn3jLjrBOAWqbgny+topvTBuqINYVypfDx/fD0pfduJypt8FJd7gWDWtdF0/5Uihb4m5b5rlxCy3SCqIBLdqCVjzOBZiDdVtZG10x+x3X+rXxMxd+krNh+BkugA0/K7bTupPSIf8od9tbww4Xepa8CB/cBx/8F/Sd4FrLxl3mWn26QnOD66Jd8SqsfBua61yL4KgLYOzFLsykFRx6N+Hh8vmi4a4ImHDguutaWiO3uW6Rfse58Vr637WIdMABx4wZY4qBnwMG+AmQCaRaaxd3TXlOT20Ze23RVu54ZgEv/NuJHD+kkxfak922LYaPfg0rXnNdc8ffBCfe3rHw01jtZvqULYHyaEirWOH2rwMX6orGuIDWJxrU+hztxvysn+Vav1a947qFwI0HGjHdBbCBU7ouWHRU7VYXVpe8ANsWAgaGnOxazMZc3PkLQgbq3Pez4lXXShhscCuAj77QBbAhp2oGqYh0O5266KsxJhMIAyOAXwDzgF9ba49wt9ND01PD2LefXcCs0u3M+dFZJPj0F3Sn2zIfPvwNrHoLkrPcOJ2p/37kgSIcdN17ZUuhbPHulrTGHbtfk5DkAltiKgz7Eow424WwnEFHdu2utH21m2q+5AU3Xsvndy14x1wBo847/LFOjdWw6m3XGrf6fTfeK6OPm8Y+5mIYPC3+QqqIyCHotG5KY8y9wAnR18KQp3sAACAASURBVLxqrb3YGHMx8KYx5glr7V+PvNzeKxiOMLOkgnOOLlYQ62wbZ7uWsNXvQUoOnP4jOOGWzpuGn+DfPRV7/NXunLWui6psiQtojdUw7HQYMs3tA9cdFQyH0+6GL33frXm19EVY8pILt/40GHW+C2ZHnXnw1qtdVW7s1/JXYe0HbtmErP4w+Zsw9hIYeIKW4xCRXutAf35eaK2dYNxgpvnA76y1rxpj3gS+1TXl9Vxz1mlj8E63fhZ8+CtY95Hr6jrzHtcl2RVLOBize5XwkefE/npdyRjoN8Hdzvq5G/O25AVY/g8X0FJyXKA65krXqtUy8aC+wnUNL/+n+7exYTebcOpt7vX9jvNmSxYRkThzoDC21BjzMJAKfNhy0lobAn4f68J6uhnLy0nx+zhlRM/Zi8sT1rqZiR/+BjZ+6mbVnf3/3IKZ2jql8/l8rrVvyDQ479fuu1/yort98aRb5mHU+W45hw2fAtbNTD35LtcF2Xe8BrWLiOxlv2HMWnutMeYY3JZGJV1YU49nrWXG8nJOHl5IapK6Zg6LtW7A94e/crMeM/u5cHDc9d23W7C7SUxyrYAjz3FLZ6x6OxrK/goFI1z35thL3AQHBTARkf064ChZa+2SriqkN1m+rZYt1Y3ceaY2Bj9kkQisfBM++o2b7Zc9CC74LUy8FhK1nZRnktJh3OXuFglr/JeIyCHQlCUPtGwMfsZojRfrsFCzW/5g1gNuLbDcIXDx/8D4rxz61jESWwpiIiKHRGHMAzOWl3PcoFxtDN4RtVth3uMw/wm312D+CPjyn90q8Vr+QEREeoAO/TYzxvQHBrd9vbX2o1gV1ZNtqW5k2dZa7j5vtNelxC9r3d6Acx6GFa+7jYxHnA0n3OyWUdAMPBER6UEOGsaMMb8CrgaW4xaABbclksLYYdi9Mbi6KPcRqIfFz8GcR6ByhVsy4cR/h8k3Qt5Qr6sTERGJiY60jF0KjLLWBmJdTG8wY3k5wwrTOaoww+tS4sf2UhfAFj0DgVq37+PF/+MGgyeleV2diIhITHUkjK0F/IDC2BFq2Rj8xlPUykMk7JZCmPOwW5Hd54ejv+y6Igccr6UQRESk1+hIGGsAFhpj3qdNILPW3hmzqnqoD1ZWEopYzu7NXZS7trt1qOY9BjWb3JY4Z/wYjvs6ZBR5XZ2IiEiX60gYezV6kyM0Y3k5BRlJTBiY63UpXW/zfNcKtuxlt4H20FPhnF+61do1K1JERHqxg/4WtNY+2RWF9HTNoQgflFRw/jF9e8/G4MEmF77mPAxbF0BShmsBO/4mKNJsUhEREThAGDPGPG+tvcoYswQ3e3IP1tpjY1pZDzN7XRV1gRBn9fQuSmthy3xY9gosfBoad0DBKDj/fjj26q7ZtFtERKQbOVDL2Lejxwu7opCermVj8JOHF3hdSueLhGHTbFj+qlslv3aLG5A/6lw4/mbXJakB+SIiIu060Ebh26LHDV1XTs9kreW95eWcMqIHbQweDrmFWZf/E0peh/pySEiG4WfCmT+FkedCao7XVYqIiMQ9jZzuAsu21rK1pom7po/0upQjE2qGdR/Bin9CyRvQUAX+NBgxHcZcDCPPgeRMr6sUERHpVhTGusC7y8vxGThzdDdcuiHYBGtnuhawlW9CU40biD/yXBh7CQw/SwuzioiIHIGObId0EfCGtTbSBfX0SDOWlzNpcC75Gd1kY/DmBlg9w40BW/UONNdBSrZbhmLsJTDsdPCneF2liIhIj9CRlrGrgd8ZY14CHrPWlsS4ph5l044GVmyr5Yfnx/lSDoE6F7xWvAqlMyDYAKl5MO7LMOYSNwg/McnrKkVERHqcjqwzdq0xJgv4KvCEMcYCjwPPWGvrYl1gd/f+ipaNwYs9rmQ/Qs3w6u2w7B8QDkBGHxj/VdcCNniaFmQVERGJsQ79prXW1hpjXgRSgbuALwPfM8b8wVr7YCwL7O5mrChneFEGQwvSvS6lfe//DBY/B5NvhGOugIFTwNdDZnyKiIh0Ax0ZM3Yx8A1gOPBX4ARrbYUxJg1YDiiM7UdNY5DZa3dw0ynDvC6lfavfh8/+xwWxC3/rdTUiIiK9Ukdaxi4HHrDWftT2pLW2wRhzY2zK6hk+WFlBKGKZHo+r7tdXwiv/BoWj4ex7va5GRESk1+pIGPtPYFvLA2NMKtDHWrveWvt+rArrCd5dXk5BRjITB8bZ4qfWwj//3S1Tcf0/tDSFiIiIh3wdeM0LQNtlLcLRc3IAgVCYD1dWctaYInzxtjH47D9D6buuRazP0V5XIyIi0qt1JIwlWmubWx5E72uNg4OYVbqd+kAo/rooy5bAjJ+4RVtPuNnrakRERHq9joSxyuggfgCMMZcA22NXUs/w1882UJSZzCkjCr0uZbfmBnjxRkjNhUv+qM27RURE4kBHxoz9G/CUMeZ/AANsAq6PaVXd3OqKej5cVcl3p48kKbEjebeLvPND2L4SrnsF0gu8rkZERETo2KKva4CpxpiM6OP6mFfVzT356XqSEn1cM2WQ16XstvxVmP84nHQnHHWG19WIiIhIVIcWfTXGXAAcDaSYaNeWtfbnMayr26ppDPLSF5u5eHy/+NmLsmYLvHoH9J0AZ/zE62pERESkjYP2oRljHsLtT3kHrpvySmBwjOvqtl6Yt4mG5jA3nDTE61KcSBhevgXCQbjiMe0vKSIiEmc6MqDpJGvt9cBOa+3PgBOBkbEtq3sKRyxPfraeE4bkMa5/ttflOLMegA2z4PzfQP5RXlcjIiIie+lIGGuKHhuMMf2AINA3diV1X++vKGfTjka+MW2I16U4m+bCzF/CuMthwjVeVyMiIiLt6EgYe80YkwP8BvgCWA88fbA3GWNSjDFzjDGLjDHLjDE/i54faoyZbYxZbYx5zhjTY/rNHv9kPf1zUuNjbbGmGnjpRsjqDxf8VstYiIiIxKkDhjFjjA9431pbba19CTdWbLS19qcd+OwAcIa1djwwATjXGDMV+BVur8vhwE6gR+xvWVJWy2drq7juxMEkJsTBchZv/AfUbILL/wKpcbYdk4iIiLQ6YGqw1kaAP7Z5HLDW1nTkg63TsgyGP3qzwBnAi9HzTwKXHmrR8eiJT9aT4vfxleMHel0KLHoWljwPX7obBk3xuhoRERE5gI404bxvjLncmEPv5zLGJBhjFgIVwAxgDVBtrQ1FX7IZ6L+f995ijJlnjJlXWVl5qJfuUjt2NfPKgi18eeIActI87nXdsRbe+C4MOglO/Q9vaxEREZGD6kgYuxW3MXjAGFNrjKkzxtR25MOttWFr7QRgAHACMLqjhVlrH7bWTrbWTi4sjKMthdrx7NyNBEIR7wfuh4Pw0k3gS4DLHnZHERERiWsdWYE/80gvYq2tNsbMxC2LkWOMSYy2jg0Athzp53spGI7wt882cPLwAkb2OeKv6sjM/H+wZT5c+STkxEF3qYiIiBzUQcOYMebU9s5baz86yPsKgWA0iKUC03GD92cCVwDPAl8H/nmoRceTd5eVs62miV9cMs7bQtZ+CLN+B8ddD0f3iGF4IiIivUJHtkP6Xpv7Kbjuxvm4gfgH0hd40hiTgOsOfd5a+7oxZjnwrDHmXmAB8Oihlx0/Hv9kHYPz0zhjdJF3ReyqglduhfzhcO593tUhIiIih6wj3ZQXtX1sjBkI/K4D71sMTGzn/FpcoOv2lmyuYd6Gnfz0wrH4fB6t42Wt23eyoQqueQ6S0r2pQ0RERA5LhzYK38tmYExnF9IdPf7pOtKTErhi8gDvipj3KKx8A875JfQd710dIiIiclg6MmbsQdz6YOC6GyfgVuLv1SrrAry+aBvXTBlEVorfmyLKl8M7P4KjzoQpt3lTg4iIiByRjrSMzWtzPwQ8Y639JEb1dBtPz95IczjC9ScO9qaAYKPb7ig5E778EPjiYNV/EREROWQdCWMvAk3W2jC0LuSaZq1tiG1p8as5FOHvszdw+qhChhVmeFPEjJ9CxXL42kuQ4eHkARERETkiHVqBH0ht8zgVeC825XQPbyzZSmVdgG9MG+pNASvfhjkPw9R/hxFneVODiIiIdIqOhLGUNntMEr2fFruS4pu1lsc/Wc9RhemcMqKg6wuoK4N//jv0OQbO+s+uv76IiIh0qo6EsV3GmONaHhhjJgGNsSspvn2xsZrFm2u4YdpQDmO7ziMTibj1xJob4IpHITG5a68vIiIina4jY8buAl4wxmwFDFAMXB3TquLY45+sIzMlkcsmtru/eWxtmAVrP4Dz74fCUV1/fREREel0HVn0da4xZjTQ8tt/pbU2GNuy4tO2mkbeWlrGN6cNIT35cJZoO0Ilb0BiCky4puuvLSIiIjFx0G5KY8y3gHRr7VJr7VIgwxjz77EvLf78/fMNWGu5/sQhXX9xa10YG3a6VtkXERHpQToyZuxma211ywNr7U7g5tiVFJ+agmGenr2Rs8b0YWCeB/MXyhZDzSYYc2HXX1tERERipiNhLMG0Gake3fg7KXYlxadXF25lZ0PQu+UsSt4A44OR53pzfREREYmJjgx8eht4zhjz5+jjW6Pneg1rLY99so7RxZlMHZbnTRElb8CgEyHdg+U0REREJGY60jL2feBfwG3R2/vA92JZVLyZvW4HJWV1fGPakK5fzgJgxzooXwqjL+j6a4uIiEhMHTSMWWsj1tqHrLVXWGuvAJYDD8a+tPjx+CfryE3zc8kED5azANcqBjDqfG+uLyIiIjHTofUZjDETga8CVwHrgJdjWVQ82bSjgRnLy/m3Lx1Fij/BmyJK3oA+4yDPo/FqIiIiEjP7DWPGmJG4APZVYDvwHGCstad3UW1x4W+fb8AYw3UnDvamgPpK2PQ5nPp/vbm+iIiIxNSBWsZKgI+BC621qwGMMd/pkqriRENziGfnbOS8ccX0zU49+BtiYdXbYCMaLyYiItJDHWjM2GXANmCmMeYRY8yZuO2Qeo2Xv9hCbVOIb0wb4l0RJa9D9iAoPsa7GkRERCRm9hvGrLX/sNZ+BRgNzMTtUVlkjPmTMebsrirQK9Zanvh0PccOyOa4QbneFBGohzUzXauYF7M4RUREJOY6Mptyl7X2aWvtRcAAYAFuuYse7ePS7ayuqOeGkzxazgJgzfsQDqiLUkREpAfryDpjray1O621D1trz4xVQfHiiU/XU5CRzAXH9vWuiJI3IDXPLfYqIiIiPdIhhbHeYt32XfyrpIJrpw4iOdGj5SzCQTd4f9R5kNChFUhERESkG1IYa8eTn67Hn2C4Zsog74rY8Ak01aiLUkREpIdTGNtLXVOQF+dv5qJj+1GUmeJdISteh8RUGNarlnUTERHpdRTG9vLCvM3UB0Lc4OVyFta68WLDz4SkNO/qEBERkZhTGGsjErE8+dl6Jg3O5dgBOd4VsnUB1G1VF6WIiEgvoDDWxsyVFWyoavB2kVdwrWImAUae620dIiIiEnMKY208/sl6irNSOOfoYm8LKXkDBp8EaXne1iEiIiIxpzAWVVpex6zV27nuxMH4Ezz8WqrWQOUKGH2hdzWIiIhIl1EYi3r80/UkJ/r46gkeLmcBbi9KgNHne1uHiIiIdAmFMaCmIcjLX2zm0gn9yUtP8raYkjeg+FjI8TgUioiISJdQGAOenbuRpmDE2+UsAOrKYdMcdVGKiIj0ItpnB7h0Yn+yU/2M6ZvlbSGr3gKslrQQERHpRdQyBvTJSuErXo8VA9dFmTsE+hztdSUiIiLSRRTG4kVTLaz9wHVRGuN1NSIiItJFFMbixer3INysLkoREZFeRmEsXpS8AWn5MHCK15WIiIhIF1IYiwehZih9F0adB74Er6sRERGRLqQwFg/WfwyBWi1pISIi0gvFLIwZYwYaY2YaY5YbY5YZY74dPT/BGPO5MWahMWaeMeaEWNXQbZS8Af50GHaa15WIiIhIF4vlOmMh4LvW2i+MMZnAfGPMDODXwM+stW8ZY86PPj4thnXEt0jEhbHhZ4I/1etqREREpIvFLIxZa7cB26L364wxK4D+gAVaVlfNBrbGqoZuYesXUF+mLkoREZFeqktW4DfGDAEmArOBu4B3jDH347pJT9rPe24BbgEYNCgOFmSNlZLXwSTAyLO9rkREREQ8EPMB/MaYDOAl4C5rbS1wG/Ada+1A4DvAo+29z1r7sLV2srV2cmFhYazL9E7JGzDkZEjN9boSERER8UBMw5gxxo8LYk9Za1+Onv460HL/BaD3DuCvXAXbV6mLUkREpBeL5WxKg2v1WmGt/W2bp7YCX4rePwMojVUNcW/lG+44+nxv6xARERHPxHLM2DTgOmCJMWZh9NwPgZuB3xtjEoEmouPCeqUVr0O/iZA9wOtKRERExCOxnE05C9jfjteTYnXdbqN2G2yZB2f82OtKRERExENagd8rK990R40XExER6dUUxrxS8gbkDYPC0V5XIiIiIh5SGPNCUw2s+whGXwBmfz25IiIi0hsojHmhdAZEgjD6Iq8rEREREY8pjHmh5HVIL4IBk72uRERERDymMNbVQgHXMjbqPPAleF2NiIiIeExhrKut+wia6zWLUkRERACFsa5X8jokZcDQU72uREREROKAwlhXikSg5E0Yfhb4U7yuRkREROKAwlhX2jwXdlXAGM2iFBEREUdhrCuVvA4+P4yY7nUlIiIiEicUxrqKtS6MDT0FUrK9rkZERETihMJYV6lcCTvWulX3RURERKIUxrpKyevuOOp8b+sQERGRuKIw1lVK3oD+kyGrn9eViIiISBxRGOsKNVtg6xfqohQREZF9KIx1hZVvuqNW3RcREZG9KIx1hZLXIX8EFI70uhIRERGJMwpjsda4E9bPUheliIiItEthLNZKZ0AkpC5KERERaZfCWKyteA0yiqH/JK8rERERkTikMBZLwUZY/T6MPh98+qpFRERkX0oIsbT2Qwju0ngxERER2S+FsVgqeR2Ss2DIqV5XIiIiInFKYSxWImFY+RaMmA6JSV5XIyIiInFKYSxWNn4ODdvVRSkiIiIHpDAWK4ufA38ajDjb60pEREQkjimMxUKwEZa9AmMuhuRMr6sRERGROKYwFgsr34RALUz4qteViIiISJxTGIuFhc9A1gDNohQREZGDUhjrbHVlsOZ9OPYqLfQqIiIiB6W00NmWvAA2AhOu8boSERER6QYUxjqTta6Lsv9kKBjhdTUiIiLSDSiMdaayxVCxDMZ/xetKREREpJtQGOtMi56FhCQYd7nXlYiIiEg3oTDWWcJBWPw8jDwX0vK8rkZERES6CYWxzrL6Pbf90XitLSYiIiIdpzDWWRY9A2kFbmNwERERkQ5SGOsMDTtg5VtwzJWQ4Pe6GhEREelGFMY6w7KXIdysWZQiIiJyyGIWxowxA40xM40xy40xy4wx327z3B3GmJLo+V/HqoYus+hZKBoLfcd7XYmIiIh0M4kx/OwQ8F1r7RfGmExgvjFmBtAHuAQYb60NGGOKYlhD7G0vhc1zYfovwBivqxEREZFuJmZhzFq7DdgWvV9njFkB9AduBu6z1gaiz1XEqoYusegZMD63F6WIiIjIIeqSMWPGmCHARGA2MBI4xRgz2xjzoTHm+K6oISYiEVj0HBx1BmQWe12NiIiIdEMxD2PGmAzgJeAua20trjUuD5gKfA943ph9+/eMMbcYY+YZY+ZVVlbGuszDs/5jqN2stcVERETksMU0jBlj/Lgg9pS19uXo6c3Ay9aZA0SAgr3fa6192Fo72Vo7ubCwMJZlHr5Fz0ByFoy+wOtKREREpJuK5WxKAzwKrLDW/rbNU/8ATo++ZiSQBGyPVR0xE6iH5a/C0ZeCP9XrakRERKSbiuVsymnAdcASY8zC6LkfAo8BjxljlgLNwNettTaGdcTGitcguAvGX+N1JSIiItKNxXI25Sxgf2s9XBur63aZRU9D7hAYNNXrSkRERKQb0wr8h6N6E6z72A3c19piIiIicgQUxg7H4ucAq+2PRERE5IgpjB0qa90sysHTXDeliIiIyBFQGDtUm+dB1Wq1iomIiEinUBg7VIuegcRUGHup15WIiIhID6AwdihCAVj6Eoy5EFKyvK5GREREegCFsUOx8i1oqlYXpYiIiHQahbFDsehZyOwLw073uhIRERHpIRTGOqq+ElbPgGOvAl+C19WIiIhID6Ew1lFLXoBIyC30KiIiItJJFMY6atEz0HcCFI3xuhIRERHpQWK5UXjPUb4MyhbDeb/2uhIREYmBYDDI5s2baWpq8roU6UZSUlIYMGAAfr//iD5HYawjFj4NvkQYd4XXlYiISAxs3ryZzMxMhgwZgtGew9IB1lqqqqrYvHkzQ4cOPaLPUjflwYRDbrzYiHMgPd/rakREJAaamprIz89XEJMOM8aQn5/fKa2pCmMHs3Ym1JfDBA3cFxHpyRTE5FB11n8zCmMHs/BpSM11LWMiIiIinUxh7EAaq6HkDTdWLDHJ62pERKSHqq6u5n//938P673nn38+1dXVnVyRs379ep5++un9Pjdu3LjWx4888giTJk1i586d3HDDDfTv359AIADA9u3bGTJkSOv7jDE8+OCDre+9/fbbeeKJJ2LyM+xtwYIF3HjjjQC89NJLHH300ZxyyilUVVUBsGbNGq6++urW1zc3N3PqqacSCoViVpPC2IEs/weEA+qiFBGRmDpQGDtYCHjzzTfJycmJRVkHDGNt/e1vf+PBBx/knXfeITc3F4CEhAQee+yxdl9fVFTE73//e5qbmzu13o745S9/yZ133gnAgw8+yNy5c7n11ltbf84f//jH3Hvvva2vT0pK4swzz+S5556LWU2aTXkgC5+BglHQ7zivKxERkS7ys9eWsXxrbad+5th+Wdxz0dH7ff7uu+9mzZo1TJgwgenTp3PBBRfwk5/8hNzcXEpKSli1ahWXXnopmzZtoqmpiW9/+9vccsstAAwZMoR58+ZRX1/Peeedx8knn8ynn35K//79+ec//0lqauoe13rhhRf42c9+RkJCAtnZ2Xz00UeEw2HuvvtuPvjgAwKBAN/61re49dZbufvuu1mxYgUTJkzg61//Ot/5znf2qf3555/nvvvu4/3336egoKD1/F133cUDDzzAzTffvM97CgsLmTZtGk8++WS7z7d47bXXuPfee2lubiY/P5+nnnqKPn36UF9fzx133MG8efMwxnDPPfdw+eWX8/bbb/PDH/6QcDhMQUEB77///h6fV1dXx+LFixk/fjwAPp+PQCBAQ0MDfr+fjz/+mOLiYkaMGLHH+y699FJ+8IMf8LWvfW2/tR4JhbH92bEWNn0OZ94DGtQpIiIxdN9997F06VIWLlwIwAcffMAXX3zB0qVLW5dNeOyxx8jLy6OxsZHjjz+eyy+/nPz8PWf5l5aW8swzz/DII49w1VVX8dJLL3Httdfu8Zqf//znvPPOO/Tv37+1e/PRRx8lOzubuXPnEggEmDZtGmeffTb33Xcf999/P6+//nq7dW/YsIHbb7+dBQsWUFxcvMdzgwYN4uSTT+Zvf/sbF1100T7v/f73v895553HN7/5zf1+LyeffDKff/45xhj+8pe/8Otf/5r//u//5he/+AXZ2dksWbIEgJ07d1JZWcnNN9/MRx99xNChQ9mxY8c+nzdv3rw9ulZ/8IMfcNZZZ9GvXz/+/ve/c+WVV/Lss8/u875x48Yxd+7c/dZ5pBTG9mfRs4CBY68+6EtFRKTnOFALVlc64YQT9li/6g9/+AOvvPIKAJs2baK0tHSfMDZ06FAmTJgAwKRJk1i/fv0+nztt2jRuuOEGrrrqKi677DIA3n33XRYvXsyLL74IQE1NDaWlpSQlHXi8dGFhIXl5eTz//PPttpr94Ac/4JJLLuGCCy7Y57lhw4YxZcqUA3aDbt68mauvvppt27bR3Nzc+n289957e4Sm3NxcXnvtNU499dTW1+Tl5e3zedu2baOwsLD18fTp05k+fToAf/3rXzn//PNZtWoV999/P7m5ufz+978nLS2NhIQEkpKSqKurIzMz84DfyeHQmLH2RCJu+6Nhp0F2f6+rERGRXig9Pb31/gcffMB7773HZ599xqJFi5g4cWK761slJye33k9ISGh3vNlDDz3Evffey6ZNm5g0aRJVVVVYa3nwwQdZuHAhCxcuZN26dZx99tkHrTEtLY0333yThx56iKeeemqf50eMGMGECRN4/vnn233/D3/4Q371q19hrW33+TvuuIPbb7+dJUuW8Oc///mI1/RKTU1t9zMaGhp44okn+Na3vsU999zDk08+ycknn7zHzxQIBEhJSTmi6++Pwlh7Nn4G1Ru1KbiIiHSJzMxM6urq9vt8TU0Nubm5pKWlUVJSwueff37Y11qzZg1Tpkzh5z//OYWFhWzatIlzzjmHP/3pTwSDQQBWrVrFrl27DloXuMH4LWO13nnnnX2e/9GPfsT999/f7ntHjx7N2LFjee2119p9vqamhv79XaPIk08+2Xp++vTp/PGPf2x9vHPnTqZOncpHH33EunXrANrtphwzZgyrV6/e5/xvfvMb7rzzTvx+P42NjRhj8Pl8NDQ0AFBVVUVBQcERb3u0Pwpj7Vn0NCRlwJgLva5ERER6gfz8fKZNm8a4ceP43ve+t8/z5557LqFQiDFjxnD33XczderUw77W9773PY455hjGjRvHSSedxPjx47npppsYO3Ysxx13HOPGjePWW28lFApx7LHHkpCQwPjx43nggQf2+5lDhw7l1Vdf5Zvf/CZz5szZ47mjjz6a447b/0S4H/3oR2zevLnd5/7zP/+TK6+8kkmTJu0xOeDHP/4xO3fuZNy4cYwfP56ZM2dSWFjIww8/zGWXXcb48eP3WJ6ixejRo6mpqdkjYG7dupU5c+Zw6aWXAq417vjjj+ehhx7immuuAWDmzJntdrV2FrO/psF4MnnyZDtv3ryuuVhzA9w/jd0AbwAAD25JREFUEsZeApf+8eCvFxGRbm/FihWMGTPG6zKkCzzwwANkZmZy0003dfg9l112Gffddx8jR47c57n2/tsxxsy31k7u6OerZWxvJa9Dc53WFhMREemBbrvttj3G1h1Mc3Mzl156abtBrLMojO1t0TOQPQgGneR1JSIiItLJUlJSuO666zr8+qSkJK6//voYVqQwtqfarbD2Axj/FfDpqxEREZHYU+Joa/FzYCMujImIiIh0AYWxFta6hV4HToH/3969B2dRpXkc/z4E2CQDhSigQnAiMyjXTVAK2QUdDety0TKKYBxRRpiFwlJEQAckUBbiVq1bFI4ojOXOIOiiqHEY0HJ1hAkLi6CThFsQVi1ECBd5wciC3EJ89o9uYkIuhIHQb8jvU/VW3j7dffq8OTnJk3NO97nsZ1GXRkRERBoIBWOn7F4Hsa16tpiIiIhcUArGTtnwBiT8HXS9K+qSiIiInFGzZs3OOY/58+eze/fuKvc9+OCDZcsjffvtt/To0YNXXnmF7du3Y2a88MILZcc+8sgjzJ8/v+y8du3acfz4cQD2799PamrqOZe1toYMGcK2bds4fvw4AwYMoFu3bsydO7ds/+jRoykoKCjbfvHFF5k3b94FK19VFIwBnDwBm3Kg0yBIuiTq0oiIiFwQNQVjpxw8eJD+/fszevRoRowYAQRP3X/++ec5ceJEleckJCREEuBs3ryZ0tJSOnTowIcffkjfvn3ZuHEjr732GgAbNmygtLS0wkNoR44cWSGwjIIWCgf44s9w9FtIuy/qkoiISNT+azLs3XR+87yiOwz8t2p3T548mfbt2/Pwww8DwZPnmzVrxpgxY8jMzKS4uJiSkhKeeeYZMjMzq83n+++/55577qGoqIjS0lKmTZtGVlYW+fn5TJgwgcOHD9OqVSvmz5/P6tWrycvLY9iwYSQlJbFmzRqSkpIq5Hf48GEGDhzIfffdx0MPPVSW3rp1a/r06cOCBQsYNWpUpXI89thjPPfcc1XuK+/OO+9k586dHDt2jHHjxjF69GiAsuWVSktLadWqFcuXL+fw4cOMHTuWvLw8zIynnnqKu+++u0J+CxcuLPv+NGnShCNHjlBSUlK29uW0adN46aWXKpyTnJxMamoqn376Kb169aqxvHVFPWMQ3EH50z7ws4yoSyIiIg1QVlZWhcW033rrLbKyskhMTGTx4sUUFBSQm5vLxIkTq11UG4Igpm3btmzYsIHCwkIGDBhASUkJY8eOJScnh/z8fEaOHEl2djZDhgyhZ8+eLFy4kPXr11cKxAAmTJhA3759GT9+fKV9kyZNYubMmZSWllbad9VVV9G3b9+yHqnqzJs3j/z8fPLy8pg9ezYHDhwgFosxatQo3nnnHTZs2MDbb78NwIwZM2jRogWbNm1i48aNZGRU/pu9evVqrr/+eiBYv3L79u307t2bRx99lKVLl3LdddfRtm3bSuf17NmTVatW1VjWuqSeMYAudwQvERGRGnqw6kqPHj3Yt28fu3fvJhaL0bJlS9q3b09JSQlTpkxh5cqVNGrUiF27dvHNN99wxRVXVJlP9+7dmThxIpMmTeL222/nxhtvpLCwkMLCQm699VYASktLufLKK2tVroyMDJYsWcLjjz9OmzZtKuzr0KEDN9xwA6+//nqV5z755JNkZmbWuKbj7NmzWbx4MQA7d+7kiy++IBaLcdNNN3H11VcDcOmllwKwbNkyFi1aVHZuy5YtK+W3Z88eWrduDUDjxo3LylZSUkL//v1ZsmQJEyZMYMeOHQwfPpw77gj+9rdp04atW7fW6ntSFxSMiYiIxIGhQ4eSk5PD3r17yxa5XrhwIbFYjPz8fJo0aUJqairHjh2rNo9rrrmGgoIC3n//faZOnUq/fv2466676Nq1K2vWrDnrMt1777306dOHQYMGkZubS/PmzSvsnzJlCkOGDOEXv/hFpXM7duxIenp6hR6/8lasWMGyZctYs2YNycnJ3HzzzTV+ttpISkqqMo+5c+cyfPhw1q5dS4sWLXjzzTfJyMgoC8aOHTtWZc/ghaJhShERkTiQlZXFokWLyMnJYejQoUAweb5NmzY0adKE3Nxcvv766xrz2L17N8nJydx///088cQTFBQUcO211xKLxcqCsZKSEjZv3gxA8+bNOXToUI15jh8/nn79+jF48OBKE/Y7depEly5dePfdd6s8Nzs7m5kzZ1a57+DBg7Rs2ZLk5GS2bt3K2rVrAejduzcrV67kq6++AoI7OSEYdpwzZ07Z+cXFxZXy7Ny5M19++WWFtOLiYt577z2GDx/OkSNHaNSoEWbG0aNHy475/PPP6datW43fh7qkYExERCQOdO3alUOHDtGuXbuyYcRhw4aRl5dH9+7defXVV+nUqVONeWzatIlevXqRnp7O9OnTmTp1Kk2bNiUnJ4dJkyaRlpZGeno6H3/8MRA8hmLMmDGkp6dXCE5O9+yzz5KSksIDDzzADz/8UGFfdnY2RUVF1X6m8nculjdgwABOnjxJ586dmTx5Mr179waCmwNefvllBg8eTFpaWlkv4dSpUykuLqZbt26kpaWRm5tbKc/bbruNFStWVEh7+umnyc7OplGjRvTv359Vq1bRvXv3CutTrl69umwYNwpW00TAeNGzZ0/Py8uLuhgiInKR2rJlC507d466GHKOjh49yi233MLq1atJSEio1Tnr1q1j1qxZZ7zZoDpV/eyYWb6796xtHnXWM2Zm7c0s18w+M7PNZjbutP0TzczNrFVdlUFEREQajqSkJKZPn86uXbtqfc7+/fuZMWNGHZbqzOpyAv9JYKK7F5hZcyDfzD5y98/MrD3wz8COOry+iIiINDD9+/c/q+OjHJ48pc56xtx9j7sXhO8PAVuAduHu54DfAPE/RioiIg1CfZi2I/HlfP3MXJAJ/GaWCvQAPjGzTGCXu2+4ENcWERE5k8TERA4cOKCATGrN3Tlw4ACJiYnnnFedP2fMzJoB7wCPEQxdTiEYojzTeaOB0RA8yVdERKSupKSkUFRURCwWi7ooUo8kJiaSkpJyzvnU6d2UZtYEeA/40N1nmVl3YDlwJDwkBdgN9HL3vdXlo7spRUREpL4427sp66xnzMwM+AOwxd1nAbj7JqBNuWO2Az3dfX9dlUNEREQkntXlnLE+wANAhpmtD1+D6vB6IiIiIvVOnfWMufv/AHaGY1Lr6voiIiIi9UG9eAK/mcWAmhfkOnetAA2Xxj/VU/2geop/qqP6QfVUP5xeTz9199a1PbleBGMXgpnlnc1kO4mG6ql+UD3FP9VR/aB6qh/OtZ60ULiIiIhIhBSMiYiIiERIwdiPXo66AFIrqqf6QfUU/1RH9YPqqX44p3rSnDERERGRCKlnTERERCRCCsYAMxtgZv9rZl+a2eSoyyNgZu3NLNfMPjOzzWY2Lky/1Mw+MrMvwq8toy6rgJklmNk6M3sv3L7azD4J29SbZtY06jI2dGZ2iZnlmNlWM9tiZv+g9hRfzGx8+Puu0MzeMLNEtaXomdk8M9tnZoXl0qpsOxaYHdbXRjO7rjbXaPDBmJklAHOAgUAX4Jdm1iXaUgnBovIT3b0L0Bt4OKyXycByd+9IsM6pguf4MA7YUm77WeA5d/85UAz8OpJSSXnPAx+4eycgjaC+1J7ihJm1Ax4lWCKwG5AA3IvaUjyYDww4La26tjMQ6Bi+RgO/q80FGnwwBvQCvnT3be5+AlgEZEZcpgbP3fe4e0H4/hDBH452BHWzIDxsAXBnNCWUU8wsBbgN+H24bUAGkBMeonqKmJm1AG4iWC8Ydz/h7t+h9hRvGgNJZtYYSAb2oLYUOXdfCXx7WnJ1bScTeNUDa4FLzOzKM11DwVjwB35nue2iME3ihJmlAj2AT4DL3X1PuGsvcHlExZIf/Rb4DfBDuH0Z8J27nwy31aaidzUQA14Jh5N/b2Y/Qe0pbrj7LmAmsIMgCDsI5KO2FK+qazt/U0yhYEzimpk1A94BHnP3/yu/z4NbgXU7cITM7HZgn7vnR10WqVFj4Drgd+7eA/ie04Yk1Z6iFc45yiQInNsCP6Hy0JjEofPRdhSMwS6gfbntlDBNImZmTQgCsYXu/scw+ZtTXb7h131RlU8A6APcYWbbCYb4MwjmJl0SDrWA2lQ8KAKK3P2TcDuHIDhTe4of/wR85e4xdy8B/kjQvtSW4lN1bedviikUjMFfgY7hHStNCSZMLo24TA1eOO/oD8AWd59VbtdS4Ffh+18BSy502eRH7v6ku6e4eypB2/mLuw8DcoEh4WGqp4i5+15gp5ldGyb1Az5D7Sme7AB6m1ly+PvvVB2pLcWn6trOUmB4eFdlb+BgueHMaumhr4CZDSKY95IAzHP3f424SA2emfUFVgGb+HEu0hSCeWNvAVcBXwP3uPvpEyslAmZ2M/C4u99uZh0IesouBdYB97v78SjL19CZWTrBTRZNgW3ACIJ/yNWe4oSZTQeyCO4mXwf8C8F8I7WlCJnZG8DNQCvgG+Ap4E9U0XbCQPpFgiHmI8AId8874zUUjImIiIhER8OUIiIiIhFSMCYiIiISIQVjIiIiIhFSMCYiIiISIQVjIiIiIhFSMCYi9ZKZlZrZ+nKv87bItZmlmlnh+cpPRKQmjc98iIhIXDrq7ulRF0JE5FypZ0xELipmtt3M/t3MNpnZp2b28zA91cz+YmYbzWy5mV0Vpl9uZovNbEP4+scwqwQz+w8z22xmfzazpPD4R83sszCfRRF9TBG5iCgYE5H6Kum0YcqscvsOunt3gidh/zZMewFY4O5/DywEZofps4H/dvc0gvUaN4fpHYE57t4V+A64O0yfDPQI8xlTVx9ORBoOPYFfROolMzvs7s2qSN8OZLj7tnCx+b3ufpmZ7QeudPeSMH2Pu7cysxiQUn6JGTNLBT5y947h9iSgibs/Y2YfAIcJlkP5k7sfruOPKiIXOfWMicjFyKt5fzbKr/9Xyo9zbG8D5hD0ov3VzDT3VkTOiYIxEbkYZZX7uiZ8/zFwb/h+GMFC9ADLgYcAzCzBzFpUl6mZNQLau3suMAloAVTqnRMRORv6j05E6qskM1tfbvsDdz/1eIuWZraRoHfrl2HaWOAVM3sCiAEjwvRxwMtm9muCHrCHgD3VXDMB+M8wYDNgtrt/d94+kYg0SJozJiIXlXDOWE933x91WUREakPDlCIiIiIRUs+YiIiISITUMyYiIiISIQVjIiIiIhFSMCYiIiISIQVjIiIiIhFSMCYiIiISIQVjIiIiIhH6f2IujdswrtuxAAAAAElFTkSuQmCC\n",
            "text/plain": [
              "<Figure size 720x432 with 1 Axes>"
            ]
          },
          "metadata": {},
          "output_type": "display_data"
        }
      ],
      "source": [
        "indices = range(0,100,4)\n",
        "plt.figure(figsize=(10,6))\n",
        "plt.plot(indices, train_knns, label='train set KNN acc (%)')\n",
        "plt.plot(indices, val_knns, label='val set KNN acc (%)')\n",
        "plt.ylabel('Accuracy in %')\n",
        "plt.xlabel('Epochs')\n",
        "plt.legend()"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 406
        },
        "id": "skRpnHRHfcVC",
        "outputId": "4bd392f2-7735-4518-acd5-20f7e3e14c27"
      },
      "outputs": [
        {
          "data": {
            "text/plain": [
              "<matplotlib.legend.Legend at 0x7f3050c0f150>"
            ]
          },
          "execution_count": 18,
          "metadata": {},
          "output_type": "execute_result"
        },
        {
          "data": {
            "image/png": "iVBORw0KGgoAAAANSUhEUgAAA0AAAAFzCAYAAAAXEmloAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4yLjIsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+WH4yJAAAgAElEQVR4nOzdd3hUZd7/8fdJMimT3ntI6BC6oQkqFhY7LiLYxd7rro8o+lNXH9d1i6uuq4/rurquDbGvrq66Iq5ioQlIFYEQEkjvbTJz//44QwoECEoyKZ/Xdc115sw5c+abbBbzyX2f720ZYxAREREREekL/HxdgIiIiIiISFdRABIRERERkT5DAUhERERERPoMBSAREREREekzFIBERERERKTPUAASEREREZE+I8DXBXREXFycyczM9HUZIiIiIiLSTS1fvrzYGBN/sPN6RADKzMxk2bJlvi5DRERERES6KcuytnfkPE2BExERERGRPkMBSERERERE+gwFIBERERER6TN6xD1A7XG5XOTl5VFfX+/rUqQHCQ4OJi0tDYfD4etSRERERMQHemwAysvLIzw8nMzMTCzL8nU50gMYYygpKSEvL4+srCxflyMiIiIiPtBjp8DV19cTGxur8CMdZlkWsbGxGjUUERER6cN6bAACFH7kkOlnRkRERKRv69EBSERERERE5FAoAP1I5eXl/PnPf/5R7z355JMpLy8/zBXZtm3bxosvvrjfYyNGjGje/8tf/sIRRxxBWVkZ8+bNIzU1lYaGBgCKi4vJzMxsfp9lWTz22GPN773uuut49tlnO+Vr2NvKlSu59NJLAXjttdfIzs7mqKOOoqSkBIAtW7Ywd+7c5vMbGxs5+uijaWpq6pL6RERERKTnUAD6kQ4UgA72i/d7771HVFRUZ5R1wADU2vPPP89jjz3GBx98QHR0NAD+/v4888wz7Z6fkJDAI488QmNj42GttyMeeOABbrjhBgAee+wxvvnmG6688srmr/POO+/k/vvvbz4/MDCQ448/nldeeaXLaxURERGR7q3HdoFr7d53vmNdfuVhvebwlAjuPi17v8fnz5/Pli1bGDNmDNOnT+eUU07hrrvuIjo6mg0bNrBp0ybOOOMMduzYQX19PTfeeCNXXHEFAJmZmSxbtozq6mpOOukkpk6dyhdffEFqaipvvfUWISEhbT7r1Vdf5d5778Xf35/IyEiWLFmC2+1m/vz5LF68mIaGBq699lquvPJK5s+fz/r16xkzZgwXXXQRN9988z61L1y4kAcffJCPP/6YuLi45tdvuukmHn74YS6//PJ93hMfH8+UKVN47rnn2j2+xzvvvMP9999PY2MjsbGxvPDCCyQmJlJdXc3111/PsmXLsCyLu+++mzPPPJP333+fO+64A7fbTVxcHB9//HGb61VVVbF69WpGjx4NgJ+fHw0NDdTW1uJwOPjss89ISkpi0KBBbd53xhlncPvtt3Peeeftt1YRERER6Xt6RQDyhQcffJC1a9eyatUqABYvXsyKFStYu3Ztc4vlZ555hpiYGOrq6hg/fjxnnnkmsbGxba6zefNmXnrpJf7yl78wZ84cXnvtNc4///w25/zqV7/igw8+IDU1tXnq3F//+lciIyP55ptvaGhoYMqUKfzsZz/jwQcf5He/+x3//Oc/2617+/btXHfddaxcuZKkpKQ2xzIyMpg6dSrPP/88p5122j7vve222zjppJO45JJL9vt9mTp1Kl9++SWWZfH000/z0EMP8fvf/5777ruPyMhI1qxZA0BZWRlFRUVcfvnlLFmyhKysLEpLS/e53rJly9pM27v99ts54YQTSElJ4R//+AdnnXUWL7/88j7vGzFiBN98881+6xQRERGRg3O5PdS53NS73NQ32s/rXG7qGt3UN7kZnRZFTGigr8s8JL0iAB1opKYrTZgwoc36Mo8++ihvvPEGADt27GDz5s37BKCsrCzGjBkDwBFHHMG2bdv2ue6UKVOYN28ec+bMYdasWQD8+9//ZvXq1SxatAiAiooKNm/eTGDggX8A4+PjiYmJYeHChe2ODt1+++3MnDmTU045ZZ9j/fv3Z+LEiQecYpeXl8fcuXMpKCigsbGx+fvx0UcftQkq0dHRvPPOOxx99NHN58TExOxzvYKCAuLj45v3p0+fzvTp0wH4+9//zsknn8ymTZv43e9+R3R0NI888ghOpxN/f38CAwOpqqoiPDz8gN8TERERkZ7KGENhVQOlNY3Ue8NJvctNXaNnr/09zz3tvOZuPrfO+94G7/Mmjzng5//j0olMHRR3wHO6m14RgLqL0NDQ5ueLFy/mo48+YunSpTidTqZNm9bu+jNBQUHNz/39/amrq9vnnCeffJKvvvqKd999lyOOOILly5djjOGxxx5jxowZbc5dvHjxAWt0Op289957HHXUUSQkJOwzRWzQoEGMGTOGhQsXtvv+O+64g9mzZ3PMMce0e/z666/nlltu4fTTT2fx4sXcc889B6znYEJCQtr9vtXW1vLss8/ywQcfcOqpp/L666+zaNEiXnjhheYpeg0NDQQHB/+kzxcRERHxtT0hZ1txDdtKathaXMv2khq2FtewvaSWOpe7Q9cJ8LMIcfgTHOhvbx1+3q0/0aGBJAf4ExJo77c+3vY1f0IC/Zr3BySEdfJXf/gpAP1I4eHhVFVV7fd4RUUF0dHROJ1ONmzYwJdffvmjP2vLli1MnDiRiRMn8q9//YsdO3YwY8YMnnjiCY477jgcDgebNm0iNTX1oHWB3dDg/fffZ9q0acTFxe0TohYsWNDuCBDA0KFDGT58OO+88w7jx49v9+tOTU0F4Lnnnmt+ffr06Tz++OP88Y9/BOwpcJMmTeKaa65h69atzVPg9h4FGjZsGL///e/3+Zzf/va33HDDDTgcDurq6rAsCz8/P2prawEoKSkhLi4Oh8NxwO+FiIiISHdgjKGoqqE51GwtqfEGHjvs1Da2hByHv0V6jJOs2FCOHBBHVpyTuLCg5mDTHFQc/gQHtoQch7/6n4EC0I8WGxvLlClTGDFiBCeddNI+geHEE0/kySefZNiwYQwZMoRJkyb96M+69dZb2bx5M8YYjj/+eEaPHs2oUaPYtm0b48aNwxhDfHw8b775JqNGjcLf35/Ro0czb968dqe5gT317u233+bkk09unqa3R3Z2NuPGjWPFihXtvnfBggWMHTu23WP33HMPZ511FtHR0Rx33HFs3boVsDu1XXvttYwYMQJ/f3/uvvtuZs2axVNPPcWsWbPweDwkJCTw4Ycftrne0KFDqaioaDOVLT8/n6+//pq7774bsEedxo8fT1RUFG+++SYAn3zyyX5DnIiIiIgvGGMoqm5gW3Ft82hO6xGd1iEnwM8iI8ZJZlwok/vHkhnnJDM2lKy4UJIjgwlQmPnRLGMOPK+vO8jJyTHLli1r89r69esZNmyYjyqSrvTwww8THh7OZZdd1uH3zJo1iwcffJDBgwfvc0w/OyIiItJZ2oQc7yjO9pJa78hODTV7hZz0GCeZsXbQyYoLpV9sKFmxoaREKeQcKsuylhtjcg52nkaApNu7+uqrefXVVzt8fmNjI2eccUa74UdERETkx6h3uSmqaqC4uoHi6kZ722p/z7HdlfXthpx+sU4mZMV4Q46TrLhQUqNCFHJ8QAFIur3g4GAuuOCCDp8fGBjIhRde2IkViYiISG9Q1+imuLqBouqGlnBT5Q031a3CTlUDVQ3tL3QfHhxAfFgQceFBDEuO4OjB8WTGOukXZ4/kpEaH6N6bbkYBSERERER6jSa3h8KqBgoq6luN2DS0O3rTeqSmtcgQB3FhgcSFBTE8JYL4sCDiw4OaX4vzBp7Y0ECCHf5d/BXKT6UAJCIiIiI9Qr3Lze7Kegoq6tlVYW/t/brm/eLqBtpbuibK6SAuLIj4sCBGpUU1hxl79Mb7PDyI2NAgAgM0YtObKQCJiIiIiM9VNzSxq6KOgj3BpqKegsqWoLOroo6yWtc+7wsPCiApMpikyGAGJ4aTHBlMUmQISZFBJIQHExcWRExooEKNNFMAEhEREZFOY4yhvNZlh5jKupZwU1HPrlajOdXt3GMTGxpIYkQwKZHBjMuIIinCDjrJkSHNoScsSL/OyqHRT0wXCgsLo7q6+idd49lnn+VnP/sZKSkp+xybN28ep556KrNnz6a0tJTjjz+eG264gWOPPZasrCweffRRrr/+egCuu+46cnJymDdvHvPmzePDDz/khx9+ICgoiOLiYnJycti2bdtPqrWjZs+ezUMPPURqaiozZ84kLy+Pa665hmuuuQaAK664gquuuopx48YB8Kc//Qmn08kll1zSJfWJiIhIi8YmD2W1jZRUN9rbmkbKalq2pa0ftfZrTXvNSfOzICHcDjAD48OYOjDOO3ITTFKEHXASIoJ0f410CgWgHubZZ59lxIgR7QagPSoqKpgxYwZXXHEFF198Mdu2bSMhIYFHHnmEK6+8ksDAwH3e4+/vzzPPPMPVV1/dmeXv47vvvsPtdtO/f3/efvttpk6dyh133MGUKVO45ppr+Pbbb3G73c3hB+CSSy5hypQpCkAiIiI/kTGGqoYmSqvtsFJa3RJcSvcOM96As79uaJYFUSEOokMDiQ0NpF+sk3H9ooh22vfXNAecyGDiw4LU/ll8pncEoH/Nh11rDu81k0bCSQ/u9/D8+fNJT0/n2muvBeCee+4hLCyMq666ipkzZ1JWVobL5eL+++9n5syZ+71OTU0Nc+bMIS8vD7fbzV133cXcuXNZvnw5t9xyC9XV1cTFxfHss8/y+eefs2zZMs477zxCQkJYunQpISEhba5XXV3NSSedxLnnntsmzMTHxzNlyhSee+45Lr/88n3quOmmm3j44YfbPdbaGWecwY4dO6ivr+fGG2/kiiuuAOD999/njjvuwO12ExcXx8cff0x1dTXXX389y5Ytw7Is7r77bs4888w213vhhReavz8Oh4Pa2lpcLhd7Fui96667ePLJJ9u8x+l0kpmZyddff82ECRMOWK+IiEhf0uT2UFbrorSmkZKaBspqXJTWNFDiDTB7j9KU1TbicrfTMQAIDPAjNjSQGO+jX6yTaKcdbvaEnJhWj8gQh0KN9Ai9IwD5wNy5c7npppuaA9DChQv54IMPCA4O5o033iAiIoLi4mImTZrE6aefjmVZ7V7n/fffJyUlhXfffRewR29cLhfXX389b731FvHx8bzyyissWLCAZ555hj/96U/87ne/Iyen/UVub7nlFi677DJuvvnmfY7ddtttnHTSSe2OnGRkZDB16lSef/55TjvttP1+3c888wwxMTHU1dUxfvx4zjzzTDweD5dffjlLliwhKyuL0tJSAO677z4iIyNZs8YOp2VlZftc7/PPP+ecc84BYPr06Tz//PNMmjSJW2+9lbfffptx48a1O9qVk5PDZ599pgAkIiK9WkOT2w4u1S2hxQ4zDZR6w03La41U1Lkw7ecZIkMczWElPcbJ6LQoYsICiXF6Q0zr56GBOAP99/v7i0hP1jsC0AFGajrL2LFjKSwsJD8/n6KiIqKjo0lPT8flcnHHHXewZMkS/Pz82LlzJ7t37yYpKand64wcOZJf/OIX3HbbbZx66qkcddRRrF27lrVr1zJ9+nQA3G43ycnJHarruOOO46233uKXv/wlCQkJbY7179+fiRMn8uKLL7b73ttvv52ZM2dyyimn7Pf6jz76KG+88QYAO3bsYPPmzRQVFXH00UeTlZUFQExMDAAfffQRL7/8cvN7o6Oj97leQUEB8fHxAAQEBDTX5nK5mDFjBm+99Ra33HILubm5XHjhhZx++ukAJCQksGHDhg59T0RERLqThiY3W4tryC+vayfYtAScshpXu40BwL6HpvXoy7CkiDb7MXtGaMLs59HOQC3GKeLVOwKQj5x11lksWrSIXbt2MXfuXMCe0lVUVMTy5ctxOBxkZmZSX1+/32sMHjyYFStW8N5773HnnXdy/PHH8/Of/5zs7GyWLl16yDWdffbZTJkyhZNPPplPPvmE8PDwNsfvuOMOZs+ezTHHHLPPewcNGsSYMWNYuHBhu9devHgxH330EUuXLsXpdDJt2rQDfm0dERIS0u41/vznP3PhhRfy5ZdfEhkZySuvvMJxxx3XHIDq6+v3mf4nIiLSnbg9hu0lNWzaXcXGXdX2dncVW4trcO/VFGDv6WaZsc6WEBNqt3GO9YaZGKc93czPT6MzIj+GAtBPMHfuXC6//HKKi4v59NNPAXsKW0JCAg6Hg08++YTt27cf8Br5+fnExMRw/vnnExUVxdNPP838+fMpKipi6dKlTJ48GZfLxaZNm8jOziY8PJyqqqoDXvPmm29m165dzJo1q3lq3R5Dhw5l+PDhvPPOO4wfP36f9y5YsGC/I0AVFRVER0fjdDrZsGEDX375JQCTJk3immuuYevWrc1T4GJiYpg+fTqPP/44f/zjHwF7Ctzeo0DDhg3j+++/JzMzs/m1srIy/vnPf/LBBx/wzjvv4Ofnh2VZ1NXVNZ+zadMmpkyZcsDvg4iISFcwxrCzvK5N0Nm0u4rNhdU0NnkAu0FARoyTwYnhnJidxOCkcNKjQ4gNDSImLJBQTTcT6TIKQD9BdnY2VVVVpKamNk9RO++88zjttNMYOXIkOTk5DB069IDXWLNmDbfeeit+fn44HA6eeOIJAgMDWbRoETfccAMVFRU0NTVx0003kZ2dzbx587jqqqv22wRhj9/85jdcfPHFXHDBBfz6179uc2zBggWMHTt2v1/TuHHjWLFixT7HTjzxRJ588kmGDRvGkCFDmDRpEmA3WHjqqaeYNWsWHo+HhIQEPvzwQ+68806uvfZaRowYgb+/P3fffTezZs1qc81TTjmFxYsXc8IJJzS/9qtf/YoFCxbg5+fHjBkzePzxxxk5ciRXXXVV8zmff/4599xzzwG/tyIiIoeTMYai6gY2765m466q5hGdzbur20xVS/YuyDllYByDEsIYkhTOwIQwnIH6tUukO7DM/u6U60ZycnLMsmXL2ry2fv16hg0b5qOK5HCpq6vj2GOP5fPPP8ffv2O9/leuXMkf/vAHnn/++R/1mfrZERGRg6modbGpsKol6Hi3ZbWu5nNiQgMZnBjGkMRwBieFMyQxnEGJ4USGOHxYuUjfZVnWcmNM+53CWtGfIsSnQkJCuPfee9m5cycZGRkdek9xcTH33XdfJ1cmIiK9mTGG6oYmKupcFFc32lPWdlexcXc1m3ZVsauy5f7UsKAABieGceKIJAYnhjcHnriwIB9+BSLyYykAic/NmDHjkM7f0x1PRET6toYmNxV1Lirrmqisd3mfex/1Tc37FXUuKuvt81qeu9irDwFBAX4MSgzjyIGxdsjxBp2UyGDdnyPSi/ToAGSM0T9Ickh6wpRPEZG+qKrexabdVRRVNTSHmtZhpaKdUNPgbTCwP8EOPyKCHUSGOIgIcRAfHsSA+NDm/cgQBxHBDqKcDgYlhpMR48RfndVEer0eG4CCg4MpKSkhNjZWIUg6xBhDSUkJwcHBvi5FRKTPMsaQV1bH+oJK1hdUsa6ggvUFVeSW1u5zrp8F4c0BJoDIEAcJ4WF7BZgAIvYKNHvODwro2L2lItK39NgAlJaWRl5eHkVFRb4uRXqQ4OBg0tLSfF2GiEifUO9ys3l3NesLKlnnfawvqKSq3u6YZlmQGRvKiNQI5uSkMSw5gqTI4OaAExYYoLVuROSw67EByOFwkJWV5esyREREBCiqamgOOuu9jy1FLQt+OgP9GZoUzumjUxiWHMHwlAiGJIYTGtRjfxURkR5K/+qIiIhIhzW5PWwtrmk1olPFuvxKiqsbms9JiQxmWHIEPxuexPCUCIYlR9AvxqnRHBHpFhSAREREpF2V9S42FFSxLr/Ce79OJZt2VzU3H3D4WwxKCOeYwfHeoBPO8OQIopyBPq5cRGT/FIBERET6sMYmDzvKatleUsP2klq2l9SyraSG7wurySuraz4vJjSQ4ckRXDi5X/MUtgHxYTj8/XxYvYjIoVMAEhER6eXqGt3kltrBZntJDdtKasn1Bp388ro26+GEBQXQL9bJmPQozpmQwfCUCIYnR5AQHqSuqyK9nbsJGiqhvhzqK73P995WtN2f/itIGunryg+JApCIiEgvUFXvajOCsyfobC+pYXdlQ5tzo50O+sWGckS/aGaNSyMz1km/2FAyY53EhAYq6Ij0RM3hpWLfkLK/8LL31rVvO/p9BIRAcAQERdjbpoaDv6ebUQASERHpAYwxlNe6vOHGDjm5zWGnlpKaxjbnx4cHkRnr5KhB8WTGOsnwBpx+MaFEOh0++ipEZB/GQGM1NFR5g0gVNFTstV954GMNVT8uvARFQGSqdz/SfrQ+1mYbCUHhENDz7/FTABIREelGahqa2Fpcw5aiarYU1fBDUbV3ZKeGSu/6OWCvoZMcEUy/2FB+lp3YPILTLzaUjBin2kuLdJWmBqgthbqylkdzMGk1utJeaKmvhMYqMJ6Df05QhB1A9mydMRDdr2X/YOElOAL89ccPUAASERHpch6PoaCyni2F1fxQVM0P3sDzQ1ENBRX1zedZFqRFh5AVF8aY9Cj6xTrJjA0lM85JWrSTYIe/D78KkV7GVd8qxOwVaPYOOK0fBxt5CQhuFVK829D+rcJJ+F7HI9ueGxQBgWHgp4Yjh4sCkIiISCdpbzTnh6Iafiiupt7V8hff8KAA+ieEMbl/LP3jQxkQH0b/+DD6xSrkiBwyY+xgUr0baksOT5Dxc9gjLiHR9iMqA5LHQEiUvd/6WHBUr5sy1tsoAImIiPwEHR3N8bMgLdpJ//hQJg9oHXRCiQ9ThzWRg/J47JGZql32o3rXXs93t2zd+7kx/1CCTEg0hHj3A0PtIVnpFRSAREREOqCu0e0dyWkZzdlSVMPWvUdzggPoH2+P5gxICKN/XKhGc0QOxOOGmmKoKrBHbap2ebcFrUKN9zVP077vD46EsCQIT4SMyRCWCOFJ9jY0TkFG9qEAJCIispfy2ka+y6/ku/wK77aSH4qqm9fL2TOaMyA+lCM1miN9nTHgdkFTvd0QoKm+1cO7X1/ZMjrTOuhU7YKaIjDufa8bEmMHmfAkiBtiB5ywpJbX9gQdR0jXf83So3VaALIsawjwSquX+gP/D/i79/VMYBswxxhT1ll1iIiI7I8xhoKK+jZhZ11+JTvL65rPSY4MJjslgpNHJjMsKZwBCfZoTlCARnOkh6naDTuX2d3H2oSVBnDVtd3fO8AcbIs56MfbLAiNbwkzSSMgPLnVqM2ecJMAAUGd+d2QPqzTApAxZiMwBsCyLH9gJ/AGMB/42BjzoGVZ8737t3VWHSIiImDfq7O1pKY57KzzjuyUetfPsSzIigtlXL9oLpjcj+yUCLJTIokJ1Q3M0kNVF8G2z2Dbf+1t8ab9n+sXYK8RExBkdy3bexsYCs7Y/R/fs3UEe/f3OhYYZgeb0Hi1Yhaf66opcMcDW4wx2y3LmglM877+HLAYBSARETmMGprcbN5d3WYK2/qCSmob7Wk2gf5+DE4KY/qwRLJTI8hOiWBoUoTWzpGeraYEtv8XtnpDT9F6+/XAMPvemLHnQ8aREBrbNqT4B4G/fval7+iqn/azgZe8zxONMQXe57uAxC6qQUREeqHqhibvaE5L2Pm+sAqX256SExroT3ZKJHNy0ptHdQYmhBEYoDU1pIerLYXtn7cEnsLv7NcdoZAxCUbPhcyj7C5nCjgizTr9/w2WZQUCpwO3733MGGMsy2p30qhlWVcAVwBkZGR0ao0iItIzFFc3tNyvs9PebitpWbsjLiyQ4SmRTBsS3xx2+sU48fNTUwLpBerKYPsXLYFn91rAgMMJ6RNh5Jl24EkZq2lmIgfQFX8OOAlYYYzZ7d3fbVlWsjGmwLKsZKCwvTcZY54CngLIycnp6J11IiLSCxhjyK+oZ+1O76iOd7ursmVdnfSYELKTIzlzXJp3GlskCeHqwCa9SH2FHXi2/Re2LoFdawBjT11LnwjHLfAGnnFabFPkEHRFADqHlulvAG8DFwEPerdvdUENIiLSTbVpTuANOmvzKyivdQF2y+kB8WFMHhBLdkoEw1MiyE6OJNKpv3BLL1NfCblL7YYFWz+DXavBeOx7dNInwLTbIesoSD1CHdJEfoJODUCWZYUC04ErW738ILDQsqxLge3AnM6sQUREug+X28Pm3dWs9XZhW7uzgvUFldS0ak4wJCmcE7OTyE6NJDslgmFJEYQEquW09EINVZD7ZUvgKVjlDTyBkDYBjv4fb+DJsburichh0akByBhTA8Tu9VoJdlc4ERHpxeoa3azfVdlmZGfjrioa3R4AnIH+DE+O4KycdIanRDBCzQmkN2ishdpiqC2xu7LVlrTs15ZATbHdvKC2GEq22AuA+jkgLQeO+iVkTrVHe7S4p0inUUsQERH5ySrqXG06sa3dWcGWomo83js4o5wORqREcvGUzOaRnczYUPzVnEC6M4/bbjxQU9xOkNkr3OzZb6pr/1qWv72OTmicvU0YBsNOs+/hSZ8Igc6u/dpE+jAFIBEROST1Ljff5VewMreclbnlrNlZQW5pSye2pIhgslMiOGlkMtkpEYxIjSQlMljNCcS3PB6oL7dHX+pKvaMwJa2ee0dmWoedunJgP32YAsPt9XScsRCWBAnZLfvOWHDGtQo8MRAcZa+2KyI+pwAkIiL7ZYwhr6yOlTvKWbG9jJU7ylmXX9G8xk5qVAhj0qOYOz6dEd6Rnbgw3ZwtncztskdmmsNMyV7Py9q+Xltihx/jaf96fgGtgkssJGa3jNQ4vQGmed/7UBMCkR5LAUhERJrVNjaxOm/P6E4ZK3LLKa5uACDE4c+otEgundqfsRlRjM2IIiFcN2bLYeBugqp8KN8BNYXe0NLeSI033DRU7P9a/kHekBJjPxKzvc9jISSm5VhITMs5QREanRHpQxSARET6KGMM20pqvUGnjJW55WzYVYXbe+NOVlwoRw+KY2y/aMamRzE0KZwAfzUokB/B7YLKnVCe633saPU81z5m3Pu+LzDMG1Si7eASndV+gGkdbBxOhRkROaoRG/AAACAASURBVCAFIBGRPqKq3sW3OypYmWtPZVuZW0aZd62dsKAARqdHcs20AYzNiGJMejQxoVpYUTqoqREq8/YfcKry95p+ZkF4MkRlQMYkiEq3n0emQ3hSS7jRNDMR6QQKQCIivZDHY9hSVM3K3PLm0Z1NhVUY7/3cAxPCmD48kbEZ0YzLiGZgQpg6ssn+NTVARR6Ub99PwCmgTbMAyw/CU+xQkznV3u4JOVEZEJEGAQrYIuIbCkAiIr1ASXUDq3e23Luzakc5VfVNAEQEBzA2I5qTRiYxLiOa0elRRIY4fFyxdEsN1ZC/AvKWwe7vWgJO9a6251n+EJkKUf2g/7SWYLMn5ESkgr9+xkSke1IAEhHpQYwx7CitY12Bvd7Od/mVrMuvZFdlPQB+FgxODOe00SmMTY9ibEY0/eNC8dPojuzN44GSzZD3jfexDArXtUxVi8qA6EwYeEKrgOMNOeEp4K9fIUSkZ9K/XiIi3ZTL7WHz7mrWFbQsMLo+v5KqBntkx8+CAfFhTOofQ3aK3YJ6VHoUYUH6p13aUVsKO5e3CjzLW7qpBUdCag4MPRXSxkPqOPseHBGRXkj/lRQR6QaqG5rYUFDpHdWpYF1BJZt2VdPotv8aH+zwY1hyBKePSSE7JZLhKREMTQon2OHv48qlW3K77Clse0Z2di6Dku/tY5afvWjniFl22EkbD7EDwU8d/kSkb1AAEhHpYkVVDc0jOusK7Cls20pqmhsUxIQGkp0SwcVTMhmeEkF2SgRZcWpSIAdQWdB2Klv+Smiqs4+FJtghZ8x59jZlLASF+bZeEREfUgASEekkHo8ht7S2zajOd/mVFFU1NJ+THhPC8OQIfj42leyUCIanRJAUEYyldUxkf1z1UPBt28BTmWcf8w+E5NGQczGk5diBJzJd6+KIiLSiACQicpgYY1idV8Hb3+azOq+c9QVVVHvv1wnwsxiYEMZRg+Ka79cZlhyhbmyyfx6PfY9OTTHkr2oJPLvWgMdev8leR2cipF1nh52kkVo7R0TkIBSARER+osLKet5YuZNFy/PYXFhNYIAfo1IjmTXOO6qTHMmgxDDdr9NXNTVAXRnUlXu33kd96/32jpXTZm0dR6jdnOBIb9hJzYHwRJ99WSIiPZUCkIjIj9DQ5Obj9YUsWp7Hp5uKcHsM4zKieODnIzllVLJGdnqjpgao2rWfANM6xJS3Pe6q3f81LT+7A1tItP1wxkBM/5b9Pa8ljoD4oWo9LSJyGOhfUhGRDjLGsHZnJa8u38Fbq/KpqHORGBHEFUf3Z/YRaQyI143lvYLHA+XbYPc6e12cwnX285Lvwbjbf49/kB1UQqIhOMpePyckGkKivI9WgSa41X5QhLqviYh0MQUgEZGDKKpq4E3vFLeNu6sIDPBjRnYSs49IY+rAOHVn66mMgerCtiGncB0UbWg7ahOdabeNHnYaRPeDkJhWgcYbZhwhPvsyRETk0CgAiYi0o7HJw3827GbR8jw+2WhPcRuTHsX9Z4zgtFEpRDo1xa1HaaiCwvVtg07hOqgtaTknNB4ShsO4iyBxuB164oeoZbSISC+jACQi0sranRUsWp7HW6t2UlbrIiE8iMuOyuKsI9IYmBDu6/LkYJoaoWRz25BTuA7Kc1vOcYRCwjAYcjIkZtuhJ2E4hMX7rm4REekyCkAi0ucVVzfw1qp8Fi3PY31BJYH+fkwfnsjsnDSOGhhHgL/u0eh2PB6oyG3nPp3N4LFbj+MXALGD7I5p4y60R3QSh0Nkhu67ERHpwxSARKRPcrk9/GeD3cXtkw2FNHkMo9MiuW9mNqeNTiHKGejrEsXjgcqdULYVSn+AUu+2bCuU/ACumpZzIzPscDPkxJagEzsIAvS/o4iItKUAJCJ9yrr8yuYpbiU1jcSFBXHJ1CxmH5HG4ERNcetybpc9PW3vgFP6A5RtB3dDy7l+DrsJQXQWZBxpT2NLzLbbQwdH+O5rEBGRHkUBSER6vdKaRt5aZXdx+y6/Eoe/ZU9xOyKNowfFa4pbZ2us9YaavQJO6Vao2AHG03Kuw2mvgxM3GAafCDFZduCJ6Q+RaeCnxWRFROSnUQASkV6pos7FfzcX8863+Xy8YTcut2FkaiT3np7N6aNTiA7V1KjDqq5sr4DTKvBU72p7bki0HWrSxsOoOXa42RNywhLAUltxERHpPApAItIrGGP4Lr+STzcVsXhjIStyy3F7DHFhgVw0OZPZOWkMTdI0qcOivgJ2fA3bP4ftS6F4ox2AWgtLsgPNwOPbjuLEZNkBSERExEcUgESkx6qodbFkcxGLNxaxZHMRRVX2/SIjUiO4+pgBTBsSz5j0KE1x+6lqimH7F97H57B7rT1tzS8AksdA9s/bBpzoTAgM9XXVIiIi7VIAEpEew+OxR3kWbyxk8aYiVuaW4TEQGeLgqEFxTBuSwNGD40gID/Z1qT1bRZ49srP9czv0FG+0Xw8ItqetHf0/0O9ISMtR0BERkR5HAUhEurXy2kaWbC5m8cZClmwqprjaHuUZmRrJtccOZNqQeEanaZTnRzPGvk9nT9jZ/gWUb7ePBUVAxiQYfTb0mwIpY9VWWkREejwFIBHpVjwew9r8ChZvtO/lWbWjHI+BKKeDowfFM21IPEcNiic+PMjXpfZMHg8UrW+Zzrb9C6jebR9zxtojO5OutreJI9R1TUREeh0FIBHxubKaxpZ7eTYVUVLTiGXBqNRIrjtuUPMoj7+fuoMdMrcLCla3hJ3cpVBfbh+LSIWsY+yw0+9Iu/W0OrCJiEgvpwAkIl3O4zGs3llh38uzsYhv88oxBqKdDo4ebI/yHD0ontgwjfIcMlc97FzeMsKz42tw1djHYgbAsNPs6Wz9joSoDAUeERHpcxSARKRLlNY0ssTbonrJ5mJK94zypEVxg3eUZ5RGeQ5dQxXs+MobeJbCzmXgbgQsSMyGsefZYSfjSAhP9HW1IiIiPqcAJCKdxu0xLN5YyItf5fLJxkI8BmJCAzna27HtqEFxGuU5VLWl9jS2PSM8Bd/aLaktf0gZAxOvtEd40ieCM8bX1YqIiHQ7CkAictjtqqjnlW928Mo3ueRX1BMfHsSVxwxgRnYSo1Ij8dMoT8dV5rdag+cLu4EBgH+Q3ZL6qF96W1KPh6Aw39YqIiLSAygAichh4fYYlmwu4sWvcvnPhkLcHsNRg+K469ThnDA8EYfaVB+cMVC2tW3gKdtqHwsMh4yJMOosezpb6jgI0OiZiIjIoVIAEpGfpLCynoXLdvDS1zvYWV5HbGgglx/Vn3MmpNMvVotkHpDHYy8y2noNnqoC+1hIjD2yM+Fyb0vqkeCvf7JFRER+Kv3XVEQOmcdj+O/3xbz4VS4frd9Nk8dw5IBYbj95KD8bnkRggEZ72uVugl2rW8JO7hdQV2YfC09u6c7W70iIGwJ++j6KiIgcbgpAItJhRVUNvLp8By9/vYPc0lqinQ4umZrF2ePT6R+v+0/24aqH/BXeEZ6ldre2xmr7WHQWDDmlJfBEZ6oltYiISBdQABKRA/J4DEt/KOHFr3L597pduNyGiVkx/OJngzlxRBJBAf6+LrF7qdoFy5+DrZ9C3jJwN9ivJwyH0We3tKSOSPZtnSIiIn2UApCItKu0ppFFy3fw4le5bCupJTLEwYWTMzlnQgYDEzTas4+SLfDFo7DqRXC77JbUe+7fyZisltQiIiLdhAKQiDQzxvDV1lJe/CqX99fuotHtIadfNDccP4iTRyYT7NBozz7yV8Hnf4R1b4GfA8aeD0deDzH9fV2ZiIiItEMBSEQor21k0fI8Xvo6ly1FNYQHB3DuxAzOnZjB4MRwX5fX/RgD2z6D/z4MW/4DQREw5UaYeDWEJ/q6OhERETkABSCRPsoYw7LtZbz4VS7vrimgscnD2Iwofjt7FKeOSiEkUKM9+/B4YOO7dvDZuRxCE+CEeyDnEgiO9HV1IiIi0gEKQCJ9TEWdi9dX5PHiV7lsLqwmPCiAuTnpnDsxg2HJEb4ur3tqaoQ1C+G/f4SSzXbHtlMfhtHngiPY19WJiIjIIVAAEukjPB7DK8t28Ov31lNZ38TotEh+c+ZIThudgjNQ/xS0q6HK7ui29HGoyoekkTD7GRg2U4uSioiI9FD6L7hIH7B5dxV3vLGGb7aVMSErhrtOGc7INE3Z2q+aYvjq/+Drp6C+HDKPgpmPwYDjtVaPiIhID6cAJNKL1bvcPP7J9zz56RacgQE8dOYozspJw9Iv8e0rz4Uv/gQr/g5NdTD0VJh6M6Tl+LoyEREROUwUgER6qc+/L2bBG2vYVlLLrLGp3HHKMOLCgnxdVve0ex18/gisedUe4Rl1Nky5AeKH+LoyEREROcwUgER6mZLqBv733fW8vnInmbFO/nHpRKYOivN1Wd1T7pd2Y4NN/wJHKEy8CiZfA5Fpvq5MREREOokCkEgvYYzh1eV5PPDeemoamrju2IFcd9xALV66N2Ng87/tVta5SyEkBqbdARMuB2eMr6sTERGRTqYAJNILfF9YzYI31vDV1lJy+kXzwKyRWsB0b+4m+O51e8Sn8DuITIcTfwPjLoDAUF9XJyIiIl1EAUikB6t3uXli8RaeWLyFYIcfD84ayZycdPz81OSgWWMtrHoBvnjUbnIQPxR+/n8w4kzwd/i6OhEREelinRqALMuKAp4GRgAGuATYCLwCZALbgDnGmLLOrEOkN/piSzF3vrGWH4prmDkmhTtPGU58uJocAFBdaN/fk7sUVi+E2mJImwAnPQSDZoCfn68rFBERER/p7BGgR4D3jTGzLcsKBJzAHcDHxpgHLcuaD8wHbuvkOkR6jdKaRh54bz2LlueRHhPCc5dM4JjB8b4uy3eMgeLNdtjZ8ZW9Lf3BPhYQDP2nwZQbIWOy1vARERGRzgtAlmVFAkcD8wCMMY1Ao2VZM4Fp3tOeAxajACRyUMYYXl+xk/vfXUdVfRPXTBvA9ccNIiSwjzU5aGqAgm/toJP7pf2oK7WPOWPtoHPExfY2eTQEBPq2XhEREelWOnMEKAsoAv5mWdZoYDlwI5BojCnwnrMLSOzEGkR6hR+KqrnzzbV8saWEcRlRPDBrJEOTInxdVteoK4Md37QEnvwV0FRvH4sZAENOhoyJduCJHahRHhERETmgzgxAAcA44HpjzFeWZT2CPd2tmTHGWJZl2nuzZVlXAFcAZGRkdGKZIt1XQ5Ob//v0B/70yfcEBfhx/xkjOHdCRu9tcmCM3agg90vY4R3dKVxnH/MLsEd0xl8GGZMgfSKEJfi2XhEREelxOjMA5QF5xpivvPuLsAPQbsuyko0xBZZlJQOF7b3ZGPMU8BRATk5OuyFJpDf7emspd7yxhu8Lqzl1VDL/79ThJEQE+7qsw8vjht1rWxoW5H4FVfn2saAISBsP2bPswJN6BAQ6fVuviIiI9HidFoCMMbssy9phWdYQY8xG4HhgnfdxEfCgd/tWZ9Ug0hOV1zby6/c28MqyHaRGhfC3i8dz7JBeMtLRWAN5y1oCT94yaKyyj0WkQr/J9lS2jEmQMBz8+tj9TSIiItLpOrsL3PXAC94OcD8AFwN+wELLsi4FtgNzOrkGkR7BGMNbq/K575/rKK9zceUx/bnx+EE4A3vocl1uFxSuh4JVkL8Kdi6HXWvAuAELErNh1JyWwBOV7uuKRUREpA/o1N+sjDGrgJx2Dh3fmZ8r0tNsL6nhzjfX8tnmYsakR/H8z0cyPKUHNTlwu+x7dfJXtQSe3d+Bu8E+HhgOKWNg6k124EkbDyFRvq1ZRERE+qQe+qdlkd6hscnDXz77gUc/3kygvx/3zczm3In98O/OTQ6aGqFo/V5hZy24G+3jQRF2s4IJl0PKWEgeAzH9tfioiIiIdAsKQCI+smpHOf+z6Fs27a7m5JFJ3H1aNondrclBc9hZ2RJ4dn/XKuxEQvIomHilHXRSxkJ0lsKOiIiIdFsKQCJdzOX28Kf/fM+fPvmexPAg/npRDscP6wbLYTU12tPY9ozqKOyIiIhIL6QAJNKFthRVc8srq/g2r4JZ41K55/RsIoIdXV9Im7DjHd0pXNc27KSMholX2ffuJI9R2BEREZFeQQFIpAsYY3j+y+088N56gh3+/Pm8cZw8MrnrC6kphreugy0ft4Sd4Ej7np09YWfPyI7Vje9DEhEREfmRFIBEOtnuynpuXbSaJZuKOGZwPL+dPco3C5ruWgsvnQPVu2HCFfbCoiljFHZERESkTzmkAGRZVjAQaIyp7KR6RHqVd1cXsODNNdS73Nx3xgjOn5iB5Yuwse5teOMqCI6AS/5lhx8RERGRPqjDAciyrMuA2YC/ZVnfGGPu6LyyRHq2ijoX97z9HW+s3MnotEgenjuG/vFhXV+IxwOf/gY+fRBSc+DsFyA8qevrEBEREekm9huALMs63RjzdquXTjDGnOg99i2gACTSji+2FPPLhd+yu6qBm04YxLXHDsTh74PmAQ3V8MaVsOGfMPpcOPVhcHSzNtsiIiIiXexAI0AjLcu6FLjbGLMKWG1Z1tOAAb7rkupEepB6l5vffbCRp/+7lf5xobx29ZGMSY/yTTFl2+Clc+01fGY8AJOu0X0+IiIiIhwgABlj/teyrCTgV5Z908JdQDgQYoxZ3VUFivQE3+VXcPMrq9i0u5oLJ/fj9pOGERLo75titn4GCy8E44bzFsHA431Th4iIiEg3dLB7gGqAm4BBwFPAMuChzi5KpKdwewxPLfmBP3y4kShnIM9ePJ5pQxJ8U4wx8M3T8P58iBkA57wEsQN8U4uIiIhIN3Wge4DuByZ4z3nbGHO6ZVmnA+9ZlvWsMebvXVWkSHeUW1LLL15dxTfbyjh5ZBL/e8ZIokMDfVNMUyP861ZY/iwMmgFnPm13fBMRERGRNg40AnSqMWaMd/rbcuCPxpi3Lct6D7i2a8oT6X6MMby6LI973/kOP8vi4bmjOWNMqm/aWwNUF9lT3nK/gKk3w3F3gZ+Ppt+JiIiIdHMHCkBrLct6CggBPt3zojGmCXikswsT6Y6Kqxu4/fU1fLhuN5P6x/D7OWNIjQrxXUEFq+Hlc6GmCM78K4yc7btaRERERHqAAzVBON+yrJGAyxizoQtrEumWPlq3m/mvr6ayrok7TxnGJVOy8PPzYWe1796AN6+BkGi45H1IGeu7WkRERER6iAM2QTDGrOmqQkS6q5qGJu5/dx0vfb2DYckRvHDZGIYkhfuuII8HFj8AS34L6RNhzvMQnui7ekRERER6kIN1gRPp05ZvL+XmV75lR1ktVx0zgJunDyIowIf31zRUwetXwsZ3Yez5cMofICDId/WIiIiI9DAKQCLtaGzy8MjHm3hi8RZSokJ45YrJTMiK8W1RpVvhpXOgeBOc+BuYeKUWNxURERE5RB0KQJZlpQL9Wp9vjFnSWUWJ+NLm3VXc9MoqvsuvZE5OGnedOpzwYIdvi/rhU3j1Inutnwteh/7TfFuPiIiISA910ABkWdZvgLnAOsDtfdkACkDSq3g8hme/2MaD728gLCiAJ88/ghNHJPm2KGPg66fg/dshbpC9uGlMf9/WJCIiItKDdWQE6AxgiDGmobOLEfGVgoo6fvnqt3z+fQnHD03g12eOJCE82LdFNTXCe7+AFX+HwSfBrKe0uKmIiIjIT9SRAPQD4AAUgKRXWr69lEueXYbL7eHXs0Zy9vh03y1qukd1IbxyAez4Eo76JRy7APz8fFuTiIiISC/QkQBUC6yyLOtjWoUgY8wNnVaVSBf5ZEMhV7+wnOTIEJ6ZN56suFBflwT5q+Dl86C2BGb/DUbM8nVFIiIiIr1GRwLQ296HSK/y5sqd/PLVbxmSFM5zl0wgLqwbtJNe+xq8eS04Y+HSDyB5tK8rEhEREelVDhqAjDHPdUUhIl3pb59v5d531jExK4anL8rxfZc3jwc+uR8++z1kTIY5f4ewBN/WJCIiItIL7TcAWZa10Bgzx7KsNdhd39owxozq1MpEOoExhoc/3MSj//menw1P5NFzxhLs8OHCpgD1lfD6FbDpXzDuIjj5dxAQ6NuaRERERHqpA40A3ejdntoVhYh0NrfHcPfba/nHl7nMyUnjgZ+PJMDfx40FSrbAy+dC8WY7+Iy/TIubioiIiHSi/QYgY0yBd7u968oR6RyNTR5uXriKd1cXcOUx/Zl/4lDfd3r7/iNYdKkdeC54A/of49t6RERERPqAjjRBEOnRahqauOofy/lsczG3nzSUK48Z4NuCPG5Y/CAs+S0kDIezX4CYLN/WJCIiItJHKABJr1ZW08jFz37D6rxyHpo9ijk56b4tqLoIXrsUtn4KY86Hk38LgU7f1iQiIiLShxw0AFmWdRrwrjHG0wX1iBw2BRV1XPDXr8ktreWJ849gRnaSbwvavhQWXQx1ZXD6n2DcBb6tR0RERKQP6sgd4HOBzZZlPWRZ1tDOLkjkcNhSVM3sJ5ayq6Ke5y6e4NvwYwx8/gg8ewo4QuCyjxR+RERERHykI+sAnW9ZVgRwDvCsZVkG+BvwkjGmqrMLFDlUq/PKmfe3b/Cz4OUrJjEiNdJ3xdSVwZvXwMb3YPhMe+QnOMJ39YiIiIj0cR3qAWyMqQQWAS8DycDPgRWWZV3fibWJHLIvvi/mnKe+JMThz6tXHenb8JO/Ev7vGNj8bzjxN3DWcwo/IiIiIj7WkXuATgcuBgYCfwcmGGMKLctyAuuAxzq3RJGO+deaAm58eRWZcU7+fslEkiKDfVOIMbDsGXh/PoQmwMXvQ/p439QiIiIiIm10pAvcmcDDxpglrV80xtRalnVp55Qlcmhe+jqXBW+sYUx6FM/MG0+UM9A3hTRUwz9vgjWvwsAT4OdPQWisb2oRERERkX10JADdAxTs2bEsKwRINMZsM8Z83FmFiXSEMYYnPt3CQ+9v5JjB8Txx/jicgT7q7l64ARZeCCWb4bg7YeovwK9Ds0xFREREpIt05DfFV4EjW+27va9pTo/4lMdjeOC99Tz9362cPjqF3501msAAHwWO1QvhnRshMBQueBP6H+ObOkRERETkgDoSgAKMMY17dowxjZZl+Wh+kYjN5fZw22ureX3FTi6a3I+7T8vGz8/yQSH19r0+y/8GGUfC7GcgIrnr6xARERGRDulIACqyLOt0Y8zbAJZlzQSKO7cskf2rd7m59oUVfLyhkJtPGMwNxw/EsnwQfkq3wqsXQcG3MOUmOO4u8PfR9DsRERER6ZCO/LZ2FfCCZVl/AixgB3Bhp1Ylsh8VdS4uf24Z32wv5b6Z2VwwOdM3hWx4F9642v5/xDkvw5CTfFOHiIiIiBySjiyEugWYZFlWmHe/utOrEmlHYVU9Fz3zDd8XVvHo2WM5bXRK1xfhdsHH98IXj0HyGJjzHERndn0dIiIiIvKjdGi+jmVZpwDZQPCeqUbGmF91Yl0ibeSW1HLBM19RWNnA0xeN55jB8V1fRGU+LLoEcpfC+MtgxgMQENT1dYiIiIjIj9aRhVCfBJzAscDTwGzg606uS6TZ+oJKLnzma1xuDy9cPpFxGdFdX8SWT+C1y8BVB2f+FUbO7voaREREROQn60jP4CONMRcCZcaYe4HJwODOLUvE9s22Uub831L8LYtXr5zc9eHH44HFv4Hnfw6hcXDFJwo/IiIiIj1YR6bA1Xu3tZZlpQAlgPr8Sqf7z4bdXP2PFaRGhfD3SyeQFu3s2gJqiuH1y2HLf2DU2XDqH+x1fkRERESkx+pIAHrHsqwo4LfACsAAf+nUqqTPe31FHrcuWs2w5HCevXgCcWFdfK9N7lfw6jyoLYHTHoFxF4EvWm2LiIiIyGF1wABkWZYf8LExphx4zbKsfwLBxpiKLqlO+qS//ncr9/1zHZP7x/LUhUcQHuzoug83Br78M3z4/yAyDS77EJJHd93ni4iIiEinOmAAMsZ4LMt6HBjr3W8AGrqiMOmbnli8hd+8v4EZ2Yk8cvZYgh3+Xffh9RXw1rWw/h0YeirMfBxCorru80VERESk03VkCtzHlmWdCbxujDGdXZD0XZ9tLuKhDzZw6qhk/jh3DAH+HenRcZgUrIaFF0LFDvjZ/8LkazXlTURERKQX6kgAuhK4BWiyLKsesABjjIno1MqkTymoqOPGl1cxKCGMh2aP6rrwYwys+Du8dys4Y2Heu5AxqWs+W0RERES63EEDkDEmvCsKkb7L5fZw3YsrqXe5+fN5R+AM7ND6vD9dYy28+wv49kXofyyc+bTd6lpEREREeq2OLIR6dHuvG2OWdOC924AqwA00GWNyLMuKAV4BMoFtwBxjTFnHS5be5jf/2sDy7WU8es5YBiaEdc2HFn9vT3krXAfHzIdj/gf8uvB+IxERERHxiY78qf3WVs+DgQnAcuC4Dn7GscaY4lb787E7yz1oWdZ87/5tHbyW9DLvry3g6f9u5cLJ/Th9dErXfOh3b8Bb14O/A85/DQYe3zWfKyIiIiI+15EpcKe13rcsKx3440/4zJnANO/z54DFKAD1SduKa7j11dWMTotkwSnDOv8Dmxrhw7vgqychbQKc9Te71bWIiIiI9Bk/5maLPKCjv60a4N+WZRng/4wxTwGJxpgC7/FdQOKPqEF6uHqXm2teWIHf/2/vvsOsqq7Gj38XvSoiRaSIRBREERCVaBL9qTExYu9iJ2LsxmiMyWuMKUZjS4zGRGMSK0UFNMbXCPYeQVDsIErxRYpIlzazf3/c6/sSIzLMPXfuzNzv53nmmXvO3FlnjZxnO2v2Xvs0CG4aMoCmjYq8/GzRrNyDTT+cAIPOhH0vh0ZNintNSZIk1TpV6QH6PblCBqAB0A94pYrxv5ZS+jAiOgDjIuLtdb+YUkr54uiLrjsMGAbQrVu3Kl5OdcXPHnyDN+cs4S8nD6TLZi2Ke7Gp42H0aVCxBo68HfocUtzrSZIkqdaqygzQhHVerwWGp5Seq0rwlNKH+c/zImIMuf6h9wQdQAAAHjVJREFUuRHRKaU0JyI6AfPW8723ALcADBw40OcP1SP3TZzNiJdnceZeX2HvXkWcAKysgCevhKevho59csVPu22Kdz1JkiTVelUpgO4DVqaUKgAiomFEtEgprfiyb4qIlkCDlNLS/Ov9gJ8DDwInAVfmPz9QyA+guuXtj5bwX2OnMKhHWy745rbFu9Cy+XD/UHj/Keh3PHznamhS5JkmSZIk1XpVKYAeA/YFluWPmwOPArtv4Ps6AmMi4rPr3JNSeiQiXgZGRcRQYAZwVHUSV92zbNVazrzrFVo3a8wNx/Yv3sNOZ76Y6/f59BM46EYYcEJxriNJkqQ6pyoFULOU0mfFDymlZRGxwT+lp5SmAzt9wfmPAfcdLjMpJS6+/zU++Hg595w2iA6tmxXjIvDCjTDuMmjTDYaOg059s7+OJEmS6qyqFEDLI2JASukVgIjYGfi0uGmpvrnjhRn847U5XPztXgzqsXn2F/h0ETxwFrz9EPQ+EA6+CZptmv11JEmSVKdVpQA6H7g3Iv4HCGAL4OiiZqV6ZdLMT/jlP95kn14dOP0bPbK/wJzXYNSJsHgWfOuK3DbXuaWXkiRJ0r+pyoNQX46IXsB2+VPvpJTWFDct1RefLF/N2fdMouMmzbj2qJ1o0CDDwiQlmHQn/ONCaLE5nPwwdNstu/iSJEmqdzbYhR4RZwEtU0qvp5ReB1pFxJnFT011XWVl4vujJjN/6Sr+MGQAbVpk+ODR1Stg7Jnw4Dmw1e7wvWcsfiRJkrRBVdmG67SU0qLPDlJKnwCnFS8l1Rc3P/UeT74zn0sP3J6+XdpkF3jBNPjzvvDqcNjzR3D8/dCyXXbxJUmSVG9VpQeoYURESilB7jlAQIZ/yld99Py0BVz76DsctNOWHL9bt+wCvzEGHjgHGjbOFT7buKGgJEmSqq4qBdAjwMiI+FP++PT8OekLzV2yknNHTKJH+1b8+rAdiSw2JFi7GsZdCi/9EbrsAkf+DTbtUnhcSZIklZWqFEAXA8OAM/LH44Bbi5aR6rS1FZWcc88klq+qYPhpA2jZtCq32AYsmgX3nQKzX87t8Lbv5dDISUhJkiRtvKrsAlcJ/DH/QUR8Hfg9cFZxU1NddPWj7/CvDxby26P70bNj68IDTh0Po0+DijVw5O3Q55DCY0qSJKlsVenP8xHRHzgWOAp4HxhdzKRUN417cy5/emo6Q3brxiH9OxcWrLICnrwSnr4aOvbJFT/ttskmUUmSJJWt9RZAEbEtuaLnWGABMBKIlNL/q6HcVIfMWriCH4yazA6dN+HSwdsXFmzZfLh/KLz/FPQbAt+5Bpq0yCZRSZIklbUvmwF6G3gGGJxSmgYQEd+vkaxUp6xcU8EZd08E4OYhO9OsccPqB5s9EUYOgU8/gYNuhAEnZJSlJEmS9OUF0GHAMcATEfEIMALIYDsv1Te/eOhNXv9wCbeeOJCubQuYqVk6F4YfA42bwdBx0KlvdklKkiRJfMmDUFNKY1NKxwC9gCeA84EOEXFzROxXUwmqdhs76UPufmkmp+/Zg29u37H6gSorcsveVi2F40ZZ/EiSJKko1lsAfSaltDyldE9K6UCgCzCJ3NbYKnNT5y7lktFT2HXrtly033aFBXv6avjgGTjgWujQO5sEJUmSpM/ZYAG0rpTSJymlW1JK+xQrIdUNy1et5Yy7X6Fl04bceGx/GjXcqFvp301/Krfj207HQf8h2SUpSZIkfU4GT6lUuUkp8eMxU5g+fxl3fXc3OmzSrPrBls6F+78L7baFA67JLklJkiTpC1gAaaPd/dJMHpj8P1y437bs/pV21Q9UWZF7yOmqpXDiA9CkZXZJSpIkSV/AAkgb5bXZi/j5399kr+3ac+ZeBT6Y9Olrcs/6OehG6Fjgs4MkSZKkKiigcUPlZvGKNZx59yu0b92U64/qR4MGBeyK/v7T8OSvoe8x0P/47JKUJEmSvoQzQKqSysrED+6dzNwlKxl1+lfZrGWT6gdbNi/f99Mzt+tb+HgpSZIk1QxngFQltzwznfFvzeMn3+lN/26bVT/QZ30/KxfDkX+Dpq0yy1GSJEnaEGeAtEEvTv+Yq//5Dgf07cRJu3cvLNgz18H0J+Gg30PHPlmkJ0mSJFWZM0D6UvOWruSc4ZPYavMWXHV4X6KQ5WrvPwNPXgE7HgX9T8guSUmSJKmKnAHSeq2tqOS84ZNZunINdw3djVZNC7hdls3P9f20/QoMvt6+H0mSJJWEBZDW6/rx7/LC9I+59sid2G6L1tUPVFmZ7/tZBMffb9+PJEmSSsYCSF/oyXfmcdMT73HMLl05fOcuhQV79lqY/gQc+DvYYodsEpQkSZKqwR4g/Ye5S1ZywahX6bVFa352UIEbFXzwHDxxBex4JAw4KZsEJUmSpGqyANK/qahMnDt8EivXVHDjcQNo1rhh9YMtmw/3D4W2Pez7kSRJUq3gEjj9mxsem8pL7y/k2iN3YpsOBfTqVFbCmGGwYiEMuReaFtBDJEmSJGXEAkj/6/n3FnDD41M5fECXwvt+nrse3ns8N/OzxY7ZJChJkiQVyCVwAmDBslWcP2IyPdq15OcHF9j3M+N5ePyXsMPhsPMp2SQoSZIkZcAZIFFZmbhg1Kss+nQNt5+6Ky0Led7P8gVw36mw2dYw+Lf2/UiSJKlWcQZI/Onp6Tz97nwuO3B7enfapPqBKithzOm5vp8j/wbNCoglSZIkFYEzQGVu4oyFXPPoOxzQtxPH7dqtsGDP/RamjYcDroNOfbNJUJIkScqQM0BlbNGK1Zw7fDKd2zTn14ftSBSyXG3GC7m+nz6HwsBTs0tSkiRJypAzQGUqpcRF973GvKUruf+M3dmkWePqB1v+ca7vp003OPAG+34kSZJUa1kAlam/Pf8B496cy6WDt6dvlzbVD/S/fT8L4Lvj7fuRJElSrWYBVIamzF7MFQ+/xb69O3LqHt0LC/b8DTBtHHznGui0Uyb5SZIkScViD1CZWbpyDWcPf4V2rZpy9RF9C+v7mfkiPPZz2P4Q2OW72SUpSZIkFYkzQGUkpcQlo6cw+5NPGTFsEJu1bFL9YCsW5vt+usJB9v1IkiSpbrAAKiMjXp7FQ6/N4aJvbccu3dtWP1BlJYz5HiyfD0MfhWabZpekJEmSVEQWQGXi7Y+W8LMH3+DrPdtxxp5fKSzYC7+Hqf+E/a+GLftnk6AkSZJUA+wBKgMrVq/l7HsmsUnzxlx3VD8aNCik7+clGH859D4Idj0tuyQlSZKkGuAMUBm47IE3eG/+Mu4auhvtWzetfqB1+34OvtG+H0mSJNU5FkD13OhXZnPvxNmcu/c27LFNu+oHSgnGngHL5tr3I0mSpDrLAqgee2/+Mv5r7Ovs2r0t5+7Ts7BgL9wI7z4C+/8GOg/IJkFJkiSphtkDVE+tXFPB2fdMommjBvzu2H40aljAP/Wsf8H4n0HvA2HXYZnlKEmSJNU0Z4DqqV/94y3emrOEv5w8kE6bNq9+oM/6fjbZEg6y70eSJEl1mwVQPfTwlDnc+eIMhn2jB3v36lj9QCnB2DNh6Ucw9J/QvE12SUqSJEklYAFUz8xauIKL73uNnbq24cL9tiss2As3wbv/Dd++EjrvnE2CkiRJUgnZA1SPrF5bydnDJ0HAjcf2p0mjAv55P5wI4y+DXoNht+9ll6QkSZJUQs4A1SNX//NtXp21iJuHDKBr2xbVD7R6BYweBq06+rwfSZIk1StFnwGKiIYRMSkiHsofbx0RL0XEtIgYGRFNip1DOXjsrbnc+sz7nDBoK/bfsVNhwcZdCh9Pg0P+AM03yyZBSZIkqRaoiSVw5wFvrXN8FXB9Smkb4BNgaA3kUK/NWfwpP7j3VXp32oSfHNC7sGBTx8HLf4ZBZ0GPvbJIT5IkSao1iloARUQX4ADgz/njAPYG7su/5XbgkGLmUN+trajk3OGTWL22kpuO60+zxg2rH2zFQnjgLGjfG/b5aXZJSpIkSbVEsWeAfgv8EKjMH28OLEoprc0fzwY6FzmHeu13j03l5Q8+4YpDd6RH+1bVD5QS/P28XBF02C3QuFl2SUqSJEm1RNEKoIgYDMxLKU2s5vcPi4gJETFh/vz5GWdXPzw7dQE3PjGNI3fuwiH9C6wjXx0Bbz0Ie/8EOvXNJkFJkiSplinmDNAewEER8QEwgtzSt98BbSLis93nugAfftE3p5RuSSkNTCkNbN++fRHTrJvmLV3J+SMn85X2rbj84D6FBftkBjx8EXTbHXY/N5sEJUmSpFqoaAVQSumSlFKXlFJ34Bjg8ZTSEOAJ4Ij8204CHihWDvVVZWXigpGvsnTlGm46bgAtmhSwm3llBYzJP+fn0D9CgwJ6iCRJkqRarhQPQr0YuCAippHrCbqtBDnUaTc/9R7PTlvAzw7qw3ZbtC4s2As3wsznYf+rYLOtsklQkiRJqqVq5EGoKaUngSfzr6cDu9bEdeujf72/kGsffYcDd9qSY3bpWliwj6bAY7+A3gdCv+OySVCSJEmqxUoxA6RqWrh8NecOn0TXti244tAdyO0qXk1rVsLoYdCiLQz+HRQSS5IkSaojamQGSIVLKXHRva+ycPlqRp+5O62bNS4s4OO/gHlvwpD7oOXm2SQpSZIk1XLOANURtz37Po+9PY9LvtOLHTpvWliw6U/len8GDoWe38wmQUmSJKkOsACqAybPWsRVj7zNN7fvyMm7dy8s2KeLYOyZ0PYrsN8vMslPkiRJqitcAlfLLVy+mrPufoUOrZtx9RF9C+v7gdzzfpbOgaHjoEnLbJKUJEmS6ghngGqxNRWVnHn3ROYvW8VNQwbQpkWTwgK+fj9MGQV7/hC67JxNkpIkSVIdYgFUi/3qH2/x4vSF/PrQHenXtU1hwZb8Dzx0AXTeGb5+YTYJSpIkSXWMBVAtNfLlmfzt+Q8Y+rWtOXznLoUFq6zM9f1UrIbDboWGrnyUJElSefI34Vpo4oyF/NfY1/naNu24ZP9ehQd8+VaY/gQMvh42/0rh8SRJkqQ6yhmgWuajxSv53l2v0GnT5tx4XH8aNSzwn2je2zDup9BzP9j5lGySlCRJkuooZ4BqkZVrKjj9zgmsWLWWu7+7W+GbHqxdDWOG5XZ7O+hGKHQHOUmSJKmOswCqJVJK/Hj0FF6dvZg/nbAz23ZsXXjQp66EOa/C0XdB646Fx5MkSZLqOJfA1RK3Pfs+oyd9yPn79uRbfbYoPODMF+HZ66Hf8dD7wMLjSZIkSfWABVAt8MzU+Vzx8Ft8q09Hzt27Z+EBVy2FMafDpl1h/ysLjydJkiTVEy6BK7EZHy/n7Hsm0bNDa649qh8NGmTQp/PIJbBoJpz8MDTNYCmdJEmSVE84A1RCy1at5bQ7JgBwy4k706ppBvXo2/+ASXfCHufDVl8tPJ4kSZJUj1gAlUhlZeKCkZOZNm8ZNx03gK02b1l40GXz4MFzYYsdYa9LCo8nSZIk1TMugSuRGx6fyqNvzuXSwdvztZ7tCg+YEjx4Tq7/57CHoFGBW2hLkiRJ9ZAFUAk88vpH/Hb8VA4f0IVT9+ieTdBXbod3H4FvXwkdemcTU5IkSapnXAJXw97+aAkXjJrMTl025VeH7kBk8XDSj9+DR34MW+8Ju55eeDxJkiSpnrIAqkGLVqxm2B0Tadm0EX86YSDNGjcsPGjF2tyW1w0bwSE3QwP/SSVJkqT1cQlcDVlbUcnZ90zio8UrGT5sEFts2iybwM9eB7NfhsNvg007ZxNTkiRJqqcsgGrIr//7bZ6dtoDfHN6XnbfaLJugH06EJ6+EHY6AHY/IJqYkSZJUj7leqgbcN3E2tz37Pifv3p2jdumaTdDVK2D06dB6CzjgmmxiSpIkSfWcM0BFNnnWIn48Zgpf7bE5Pzkgw93Zxv0UPp4KJz4AzTOaUZIkSZLqOWeAimjekpWcfucEOrRuyk1DBtC4YUb/uaeOh5dvhUFnQY+9sokpSZIklQFngIpk1doKTr9rIks+XcvoM3enbcuMHky6YiE8cBa07w37/DSbmJIkSVKZsAAqgpQSl459nUkzF3HTcQPo3WmTrALD38+DFR/DkHuhcUY7yUmSJEllwiVwRXDHCzMYNWE25+y9DQf07ZRd4FdHwFsPwt4/gU59s4srSZIklQkLoIw9/94Cfv7Qm+zbuwPf33fb7AJ/MgMevgi67Q67n5tdXEmSJKmMWABlaNbCFZx19yts3a4l1x/djwYNIpvAlRUw9ozc60P/CA0aZhNXkiRJKjP2AGVkxeq1nHbHBCoqE7eeOJDWzRpnF/z538OM5+DgP8BmW2UXV5IkSSozFkAZSClx4b2v8u7cpfz1lF3Zul3L7IK/MQbG/wy2Pxj6HZddXEmSJKkMuQQuAzc9MY2Hp3zExd/uxZ7bts8u8NTxcP9p0G0QHPJHiIyW1EmSJEllygKoQOPenMs1j77Lwf22ZNg3emQXeOaLMPJ46NALjh0BTVpkF1uSJEkqUxZABZg2bynfHzmZHTpvwlWH9yWymqGZ8xrcfRRs2hmOHwPN22QTV5IkSSpzFkDVtHjFGk67YyLNGjfglhMG0qxxRjuzLZgGdx0GTVvBCWOhVYZL6iRJkqQy5yYI1VBRmThnxCRmf7KCe04bxJZtmmcTePFsuPMQSJW54qdN12ziSpIkSQIsgKrlN4+8zdPvzueKQ3dkl+5tswm6fAHccQisXAwn/R3aZ/gQVUmSJEmABdBGe2Dyh/zp6ekcP6gbx+3WLZugK5fklr0tngXHj4Yt+2UTV5IkSdK/sQDaCFNmL+aH973Grlu35aeD+2QTdM2nMPwYmPsGHDMcuu+RTVxJkiRJ/8ECaCN88PFytmzTnD8MGUCTRhnsH1GxBkadBDOeh8P/DNvuV3hMSZIkSetlAbQRDtxpS77VZ4tsip/KShh7Bkz9Jwy+HnY8ovCYkiRJkr6U22BvpEyKn5Tg4Qthyr2wz2Uw8NTCY0qSJEnaIAugUnj8FzDhNtjjPPj6BaXORpIkSSobFkA17bkb4JlrYeeTYd/LS52NJEmSVFYsgGrSxNth3KXQ5zA44DqIKHVGkiRJUlmxAKopb4yBv58H23wTDv0TNGhY6owkSZKksmMBVBOmjof7T4Ouu8FRd0CjJqXOSJIkSSpLFkDFNvNFGHk8dOgFx42EJi1KnZEkSZJUtiyAimnOa3D3UbDJlnD8aGjeptQZSZIkSWXNAqhYFkyDuw6Dpq3gxAegVYdSZyRJkiSVPQugYlg8G+48BFIlnDAW2nQtdUaSJEmSKGIBFBHNIuJfEfFqRLwREZfnz28dES9FxLSIGBkR9WtHgOUL4M5DYeXi3LK39tuWOiNJkiRJecWcAVoF7J1S2gnoB3w7IgYBVwHXp5S2AT4BhhYxh5q1cgncdTgsmgnHjoAt+5U6I0mSJEnrKFoBlHKW5Q8b5z8SsDdwX/787cAhxcqhRq35FIYfA3Nfh6PuhO57lDojSZIkSZ9T1B6giGgYEZOBecA44D1gUUppbf4ts4HOxcyhRlSsgVEnwYzncw853Xa/UmckSZIk6QsUtQBKKVWklPoBXYBdgV5V/d6IGBYREyJiwvz584uWY8EqK2HsGTD1nzD4OtjxiFJnJEmSJGk9amQXuJTSIuAJ4KtAm4holP9SF+DD9XzPLSmlgSmlge3bt6+JNDdeSvDwhTDlXtjnMhh4aqkzkiRJkvQlirkLXPuIaJN/3Rz4JvAWuULos2mSk4AHipVD0T3+C5hwG+xxHnz9glJnI0mSJGkDGm34LdXWCbg9IhqSK7RGpZQeiog3gRER8UtgEnBbEXMonudugGeuhQEnwb6XlzobSZIkSVVQtAIopfQa0P8Lzk8n1w9Ud028HcZdCn0OhcHXQ0SpM5IkSZJUBTXSA1SvvDEG/n4ebLMvHHoLNGhY6owkSZIkVZEF0MaYOh7uPw267pZ71k+jJqXOSJIkSdJGsADaGA0bQ5dd4LiR0KRFqbORJEmStJGKuQlC/dNjT9j6G/b8SJIkSXWUM0Aby+JHkiRJqrMsgCRJkiSVDQsgSZIkSWXDAkiSJElS2bAAkiRJklQ2LIAkSZIklQ0LIEmSJEllwwJIkiRJUtmwAJIkSZJUNiyAJEmSJJUNCyBJkiRJZcMCSJIkSVLZsACSJEmSVDYipVTqHDYoIuYDM0qdR147YEGpk1C94f2kLHk/KUveT8qS95OytL77aauUUvsNfXOdKIBqk4iYkFIaWOo8VD94PylL3k/KkveTsuT9pCwVej+5BE6SJElS2bAAkiRJklQ2LIA23i2lTkD1iveTsuT9pCx5PylL3k/KUkH3kz1AkiRJksqGM0CSJEmSyoYF0EaIiG9HxDsRMS0iflTqfFS3RETXiHgiIt6MiDci4rz8+bYRMS4ipuY/b1bqXFU3RETDiJgUEQ/lj7eOiJfyY9TIiGhS6hxVd0REm4i4LyLejoi3IuKrjk+qroj4fv7/da9HxPCIaOYYpaqKiL9ExLyIeH2dc184HkXODfn76rWIGLCh+BZAVRQRDYGbgP2B7YFjI2L70malOmYt8IOU0vbAIOCs/D30I+CxlFJP4LH8sVQV5wFvrXN8FXB9Smkb4BNgaEmyUl31O+CRlFIvYCdy95bjkzZaRHQGzgUGppR2ABoCx+AYpar7G/Dtz51b33i0P9Az/zEMuHlDwS2Aqm5XYFpKaXpKaTUwAji4xDmpDkkpzUkpvZJ/vZTcLxedyd1Ht+ffdjtwSGkyVF0SEV2AA4A/548D2Bu4L/8W7yVVWURsCnwDuA0gpbQ6pbQIxydVXyOgeUQ0AloAc3CMUhWllJ4GFn7u9PrGo4OBO1LOi0CbiOj0ZfEtgKquMzBrnePZ+XPSRouI7kB/4CWgY0ppTv5LHwEdS5SW6pbfAj8EKvPHmwOLUkpr88eOUdoYWwPzgb/ml1X+OSJa4vikakgpfQhcA8wkV/gsBibiGKXCrG882ujf0S2ApBoWEa2A+4HzU0pL1v1aym3L6NaM+lIRMRiYl1KaWOpcVG80AgYAN6eU+gPL+dxyN8cnVVW+N+NgcoX1lkBL/nM5k1RthY5HFkBV9yHQdZ3jLvlzUpVFRGNyxc/dKaXR+dNzP5uqzX+eV6r8VGfsARwUER+QW467N7n+jTb55SbgGKWNMxuYnVJ6KX98H7mCyPFJ1bEv8H5KaX5KaQ0wmty45RilQqxvPNro39EtgKruZaBnfgeTJuSa+R4scU6qQ/I9GrcBb6WUrlvnSw8CJ+VfnwQ8UNO5qW5JKV2SUuqSUupObix6PKU0BHgCOCL/Nu8lVVlK6SNgVkRslz+1D/Amjk+qnpnAoIhokf9/32f3k2OUCrG+8ehB4MT8bnCDgMXrLJX7Qj4IdSNExHfIrbtvCPwlpfSrEqekOiQivgY8A0zh//o2fkyuD2gU0A2YARyVUvp845/0hSJiL+DClNLgiOhBbkaoLTAJOD6ltKqU+anuiIh+5DbVaAJMB04h94dSxydttIi4HDia3A6ok4DvkuvLcIzSBkXEcGAvoB0wF7gMGMsXjEf5IvtGcsssVwCnpJQmfGl8CyBJkiRJ5cIlcJIkSZLKhgWQJEmSpLJhASRJkiSpbFgASZIkSSobFkCSJEmSyoYFkCSpxkVERURMXufjRxnG7h4Rr2cVT5JUvzTa8FskScrcpymlfqVOQpJUfpwBkiTVGhHxQUT8JiKmRMS/ImKb/PnuEfF4RLwWEY9FRLf8+Y4RMSYiXs1/7J4P1TAibo2INyLi0Yhonn//uRHxZj7OiBL9mJKkErIAkiSVQvPPLYE7ep2vLU4p7Ujuyd6/zZ/7PXB7SqkvcDdwQ/78DcBTKaWdgAHAG/nzPYGbUkp9gEXA4fnzPwL65+N8r1g/nCSp9oqUUqlzkCSVmYhYllJq9QXnPwD2TilNj4jGwEcppc0jYgHQKaW0Jn9+TkqpXUTMB7qklFatE6M7MC6l1DN/fDHQOKX0y4h4BFgGjAXGppSWFflHlSTVMs4ASZJqm7Se1xtj1TqvK/i/ntcDgJvIzRa9HBH2wkpSmbEAkiTVNkev8/mF/OvngWPyr4cAz+RfPwacARARDSNi0/UFjYgGQNeU0hPAxcCmwH/MQkmS6jf/8iVJKoXmETF5neNHUkqfbYW9WUS8Rm4W59j8uXOAv0bERcB84JT8+fOAWyJiKLmZnjOAOeu5ZkPgrnyRFMANKaVFmf1EkqQ6wR4gSVKtke8BGphSWlDqXCRJ9ZNL4CRJkiSVDWeAJEmSJJUNZ4AkSZIklQ0LIEmSJEllwwJIkiRJUtmwAJIkSZJUNiyAJEmSJJUNCyBJkiRJZeP/A8xX9y37Hu7fAAAAAElFTkSuQmCC\n",
            "text/plain": [
              "<Figure size 1008x432 with 1 Axes>"
            ]
          },
          "metadata": {},
          "output_type": "display_data"
        }
      ],
      "source": [
        "indices = range(0,100,4)\n",
        "plt.figure(figsize=(14,6))\n",
        "plt.plot(indices, train_knns, label='train set KNN acc (%)')\n",
        "plt.plot(indices, val_knns, label='val set KNN acc (%)')\n",
        "plt.ylabel('Accuracy in %')\n",
        "plt.xlabel('Epochs')\n",
        "plt.legend()"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "03A0Eap7X-9p"
      },
      "source": [
        "# Fine-tune"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "background_save": true,
          "base_uri": "https://localhost:8080/",
          "height": 437
        },
        "id": "X_oUUAdEYAXY",
        "outputId": "8d704968-a2ea-47c5-a4c5-efc20e73f4a2"
      },
      "outputs": [
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Files already downloaded and verified\n",
            "Files already downloaded and verified\n"
          ]
        },
        {
          "ename": "RuntimeError",
          "evalue": "ignored",
          "output_type": "error",
          "traceback": [
            "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
            "\u001b[0;31mRuntimeError\u001b[0m                              Traceback (most recent call last)",
            "\u001b[0;32m<ipython-input-19-70c23dfe95d7>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m()\u001b[0m\n\u001b[1;32m    128\u001b[0m \u001b[0;31m###########################################################################\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    129\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 130\u001b[0;31m \u001b[0mmodel_byol\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mval_accs_1\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mmain\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m'byol'\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    131\u001b[0m \u001b[0m_\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mval_accs_2\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mmain\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m'imagenet'\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    132\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;32m<ipython-input-19-70c23dfe95d7>\u001b[0m in \u001b[0;36mmain\u001b[0;34m(mode)\u001b[0m\n\u001b[1;32m    105\u001b[0m       \u001b[0mmodel_ssl\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mBYOL\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mmodel\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0min_features\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;36m512\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mbatch_norm_mlp\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;32mTrue\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    106\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 107\u001b[0;31m       \u001b[0mmodel_ssl\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mload_state_dict\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mtorch\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mload\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mPATH\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    108\u001b[0m       \u001b[0mmodel\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mmodel_ssl\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mstudent_model\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mbackbone\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    109\u001b[0m       model.fc = nn.Sequential(\n",
            "\u001b[0;32m/usr/local/lib/python3.7/dist-packages/torch/serialization.py\u001b[0m in \u001b[0;36mload\u001b[0;34m(f, map_location, pickle_module, **pickle_load_args)\u001b[0m\n\u001b[1;32m    598\u001b[0m             \u001b[0;31m# reset back to the original position.\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    599\u001b[0m             \u001b[0morig_position\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mopened_file\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mtell\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 600\u001b[0;31m             \u001b[0;32mwith\u001b[0m \u001b[0m_open_zipfile_reader\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mopened_file\u001b[0m\u001b[0;34m)\u001b[0m \u001b[0;32mas\u001b[0m \u001b[0mopened_zipfile\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    601\u001b[0m                 \u001b[0;32mif\u001b[0m \u001b[0m_is_torchscript_zip\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mopened_zipfile\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    602\u001b[0m                     warnings.warn(\"'torch.load' received a zip file that looks like a TorchScript archive\"\n",
            "\u001b[0;32m/usr/local/lib/python3.7/dist-packages/torch/serialization.py\u001b[0m in \u001b[0;36m__init__\u001b[0;34m(self, name_or_buffer)\u001b[0m\n\u001b[1;32m    240\u001b[0m \u001b[0;32mclass\u001b[0m \u001b[0m_open_zipfile_reader\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0m_opener\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    241\u001b[0m     \u001b[0;32mdef\u001b[0m \u001b[0m__init__\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mname_or_buffer\u001b[0m\u001b[0;34m)\u001b[0m \u001b[0;34m->\u001b[0m \u001b[0;32mNone\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 242\u001b[0;31m         \u001b[0msuper\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0m_open_zipfile_reader\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m__init__\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mtorch\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_C\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mPyTorchFileReader\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mname_or_buffer\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    243\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    244\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;31mRuntimeError\u001b[0m: PytorchStreamReader failed reading zip archive: failed finding central directory"
          ]
        }
      ],
      "source": [
        "import torchvision\n",
        "PATH = 'BYOL_resNet18.ckpt'\n",
        "\n",
        "import torch.optim as optim\n",
        "import torchvision\n",
        "\n",
        "def train(model, trainloader, epochs, device, valloader, criterion):\n",
        "  optimizer = optim.SGD(model.parameters(), lr=0.0008, momentum=0.9)\n",
        "  model.to(device)\n",
        "  epoch_loss = []\n",
        "  val_accs = []\n",
        "  train_accs = []\n",
        "  # loop over the dataset multiple times\n",
        "  for epoch in range(epochs):  \n",
        "      running_loss = 0.0\n",
        "      correct = 0\n",
        "      total = 0\n",
        "      model.train()\n",
        "      for i, data in enumerate(trainloader):\n",
        "          # get the inputs; data is a list of [inputs, labels]\n",
        "          inputs, labels = data[0].to(device) , data[1].to(device)\n",
        "          labels = labels.view(-1)\n",
        "\n",
        "          # zero the parameter gradients\n",
        "          optimizer.zero_grad()\n",
        "\n",
        "          # forward + backward + optimize\n",
        "          outputs = model(inputs)\n",
        "          loss = criterion(outputs, labels)\n",
        "          loss.backward()\n",
        "          optimizer.step()\n",
        "\n",
        "          # track statistics\n",
        "          with torch.no_grad():\n",
        "            running_loss += loss.item()\n",
        "            correct += calc_correct(outputs,labels)\n",
        "            total += labels.size(0)\n",
        "\n",
        "      print(f\"\\n Epoch [{epoch + 1} / {epochs}] Loss: {running_loss/len(trainloader)}\")\n",
        "      epoch_loss.append(running_loss)\n",
        "      running_loss = 0.0\n",
        "      train_acc = (100 * correct / total)\n",
        "      val_acc = val(model,device,valloader)\n",
        "      print(f'Train VS Val Accuracy: {train_acc} VS {val_acc}')\n",
        "      val_accs.append(val_acc)\n",
        "      train_accs.append(train_acc)\n",
        "  \n",
        "  print('Finished Training')\n",
        "  \n",
        "  return model,epoch_loss,val_accs,train_accs\n",
        "\n",
        "def val(model, device, valloader):\n",
        "  correct = 0\n",
        "  total = 0\n",
        "  model.eval()\n",
        "  model.to(device)\n",
        "  with torch.no_grad():\n",
        "      for data in valloader:\n",
        "          images, labels = data[0].to(device) , data[1].to(device)\n",
        "          labels = labels.view(-1)\n",
        "          # calculate outputs by running images through the network\n",
        "          outputs = model(images)\n",
        "          correct += calc_correct(outputs,labels)\n",
        "          total += labels.size(0)\n",
        "          \n",
        "  acc =  (100 * correct / total)\n",
        "  return acc\n",
        "\n",
        "def calc_correct(outputs,labels):\n",
        "  _, predicted = torch.max(outputs.data, 1)\n",
        "  \n",
        "  return (predicted == labels).sum().item() \n",
        "\n",
        "def define_augmentation(image_size):\n",
        "  DEFAULT_AUG =  T.Compose([\n",
        "        T.ToTensor(),                    \n",
        "        T.RandomHorizontalFlip(p=0.1),\n",
        "        T.RandomVerticalFlip(p=0.1),\n",
        "        T.RandomResizedCrop((image_size, image_size),scale=(0.5, 1.0)),\n",
        "       T.Normalize(mean=[0.485, 0.456, 0.406], std=[0.229, 0.224, 0.225]) \n",
        "  ])\n",
        "  return DEFAULT_AUG\n",
        "\n",
        "def main(mode='byol'):\n",
        "    reproducibility(77777)   \n",
        "    device = 'cuda'\n",
        "    data_transform = Augment(32)\n",
        "    BATCH_SIZE = 256\n",
        "\n",
        "    test_transform = T.Compose([\n",
        "                T.ToTensor(),\n",
        "                T.Normalize(mean=[0.485, 0.456, 0.406], std=[0.229, 0.224, 0.225]) ])\n",
        "\n",
        "    train_loader = get_cifar10_dataloader(batch_size=BATCH_SIZE, train=True, transform=define_augmentation(32))\n",
        "    test_loader = get_cifar10_dataloader(batch_size=BATCH_SIZE, train=False, transform=test_transform)\n",
        "\n",
        "    if mode == 'random':\n",
        "      model = torchvision.models.resnet18(pretrained=False)\n",
        "      model.fc = nn.Sequential(\n",
        "            nn.Linear(512, 128),\n",
        "            nn.ReLU(),\n",
        "            nn.Linear(128, 10))\n",
        "    elif mode == 'byol':\n",
        "      model = models.resnet18(pretrained=False)\n",
        "      model_ssl = BYOL(model, in_features=512, batch_norm_mlp=True)\n",
        "\n",
        "      model_ssl.load_state_dict(torch.load(PATH))\n",
        "      model = model_ssl.student_model.backbone\n",
        "      model.fc = nn.Sequential(\n",
        "            nn.Linear(512, 128),\n",
        "            nn.ReLU(),\n",
        "            nn.Linear(128, 10))\n",
        "    else:\n",
        "      model = torchvision.models.resnet18(pretrained=True)\n",
        "      model.fc = nn.Sequential(\n",
        "            nn.Linear(512, 128),\n",
        "            nn.ReLU(),\n",
        "            nn.Linear(128, 10))\n",
        "    \n",
        "    criterion = torch.nn.CrossEntropyLoss()\n",
        "\n",
        "    model, epoch_loss, val_accs, train_accs = train(model, train_loader,\n",
        "                                                      epochs=20, device=device, \n",
        "                                                      valloader=test_loader,\n",
        "                                                    criterion=criterion)\n",
        "    return model,val_accs\n",
        "\n",
        "###########################################################################\n",
        "\n",
        "model_byol, val_accs_1 = main('byol')\n",
        "_, val_accs_2 = main('imagenet')\n",
        "\n",
        "import matplotlib.pyplot as plt\n",
        "\n",
        "plt.plot(val_accs_1, label='byol')\n",
        "plt.plot(val_accs_2, label='supervised-imagenet-pretrained-weights')\n",
        "plt.ylabel('Validation Accuracy')\n",
        "plt.xlabel('Epochs')\n",
        "plt.legend()"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "WAZLFl9soeJj"
      },
      "outputs": [],
      "source": [
        "import matplotlib.pyplot as plt\n",
        "\n",
        "plt.plot(val_accs_1, label='byol-simpleCNN')\n",
        "plt.plot(val_accs_2, label='supervised-imagenet-resnet18')\n",
        "plt.plot(val_accs_3, label='random-weights-simpleCNN')\n",
        "plt.ylabel('Validation Accuracy')\n",
        "plt.xlabel('Epochs')\n",
        "plt.legend()"
      ]
    }
  ],
  "metadata": {
    "accelerator": "GPU",
    "colab": {
      "collapsed_sections": [
        "nDg1RCPFerm6",
        "OTPAkVK3sMzS",
        "cGUGGUYo-rwu",
        "jtfbIiygSX9_"
      ],
      "machine_shape": "hm",
      "name": "AI Summer BYOL in CIFAR10.ipynb",
      "provenance": []
    },
    "kernelspec": {
      "display_name": "Python 3",
      "name": "python3"
    },
    "language_info": {
      "name": "python"
    }
  },
  "nbformat": 4,
  "nbformat_minor": 0
}