st-gcn-ad_n_v_all_exper_notebook.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "from IPython.core.display import display, HTML\n",
    "display(HTML(\"<style>.container { width:77% !important; }</style>\"))\n",
    "%matplotlib inline\n",
    "%load_ext autoreload\n",
    "%autoreload 2"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "import os\n",
    "import re\n",
    "import time\n",
    "import pickle\n",
    "import torch\n",
    "import torchvision\n",
    "import torch.nn as nn\n",
    "import torch.optim as optim\n",
    "import numpy as np\n",
    "import pandas as pd\n",
    "from collections import OrderedDict\n",
    "from net.st_gcn import Model\n",
    "from feeder.feeder import Feeder\n",
    "from torchlight.io import IO\n",
    "from ad_utils import map2ind, loader_initializer\n",
    "import matplotlib.pyplot as plt\n",
    "np.set_printoptions(linewidth=100, precision=5, suppress=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Loading of original weights\n",
    "data_dir_path = '/root/sharedfolder/datasets/data_ssd/kinetics-skeleton/st-gcn_kinetics/Kinetics/kinetics-skeleton/'\n",
    "root_path = '/root/sharedfolder/Research/pose_ad/st-gcn/'\n",
    "weights_path =  os.path.join(root_path, 'models/kinetics-st_gcn.pt')\n",
    "ft_weights_path = os.path.join(root_path, 'models/kinetics-st_gcn_ft.pt')\n",
    "io = IO(root_path)\n",
    "model_name = 'net.st_gcn.Model'\n",
    "graph_args = {'layout':'openpose', 'strategy':'spatial'}\n",
    "model_args = {\n",
    "    'edge_importance_weighting':True,\n",
    "    'graph_args':graph_args,\n",
    "    'in_channels':3,\n",
    "    'num_class':4, }"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "model = io.load_model(model_name, **model_args)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Need to turn into a func\n",
    "weights = torch.load(weights_path)\n",
    "weights_list = [[k.split('module.')[-1], v.cpu()] for k, v in weights.items()]\n",
    "weights = OrderedDict(weights_list)\n",
    "model_state_dict = model.state_dict()\n",
    "weights.pop('fcn.bias') # loading all but the Final FC layer's weight and bias\n",
    "weights.pop('fcn.weight')\n",
    "model_state_dict.update(weights)\n",
    "model.load_state_dict(model_state_dict)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "# Freezing of layers for fine tuning\n",
    "for name, param in model.named_parameters():\n",
    "    param.requires_grad = False\n",
    "    if \"fcn\" in name:  # Keeping the last FC layer trainable\n",
    "        param.requires_grad = True\n",
    "#     if name.find('9') != -1:  # For adding an additional graph conv layer to the training\n",
    "#         param.requires_grad = True"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "data_path   = dict()\n",
    "label_path  = dict()\n",
    "data_loader = dict()\n",
    "feeder_args = dict()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "train_data_path = os.path.join(data_dir_path, 'train_data.npy')\n",
    "train_label_path = os.path.join(data_dir_path, 'train_label.pkl')\n",
    "test_data_path = os.path.join(data_dir_path, 'val_data.npy')\n",
    "test_label_path = os.path.join(data_dir_path, 'val_label.pkl')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "normal_classes = [30, 74, 22, 15]\n",
    "# for split in ['normal_train', 'normal_test', 'mixed_test']:\n",
    "#     data_path[split]   =  os.path.join(data_dir_path, 'ad_experiment', split+'_data.npy')\n",
    "#     label_path[split]  =  os.path.join(data_dir_path, 'ad_experiment', split+'_label.pkl')\n",
    "#     feeder_args[split] = {'data_path': data_path[split], 'label_path': label_path[split]}\n",
    "    \n",
    "feeder_args['random_train'] = {'data_path': train_data_path, 'label_path': train_label_path, 'specify_classes':normal_classes}\n",
    "feeder_args['random_test'] = {'data_path': test_data_path, 'label_path': test_label_path}  #, 'specify_classes':normal_classes}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [],
   "source": [
    "batch_size=32\n",
    "# data_loader['normal_train'] = loader_initializer(feeder_args['normal_train'], batch_size=batch_size)\n",
    "# data_loader['normal_test' ] = loader_initializer(feeder_args['normal_test' ], batch_size=batch_size, drop_last=False)\n",
    "# data_loader['mixed_test'  ] = loader_initializer(feeder_args['mixed_test'  ], batch_size=batch_size, suffle=False, drop_last=False)\n",
    "\n",
    "data_loader['random_train' ] = loader_initializer(feeder_args['random_train' ], batch_size=batch_size, drop_last=True)\n",
    "data_loader['random_test'  ] = loader_initializer(feeder_args['random_test'  ], batch_size=batch_size, suffle=True, drop_last=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Model Definitions for Training\n",
    "dev = 'cuda:0'\n",
    "model = model.to(dev)\n",
    "loss_fn = nn.CrossEntropyLoss()\n",
    "# loader = data_loader['normal_train']\n",
    "loader = data_loader['random_train']\n",
    "loss_th = 0.15"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [],
   "source": [
    "optimizer = optim.Adam(model.parameters())\n",
    "# optimizer = optim.SGD(model.parameters(), lr=1e-4, momentum=0.9)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [],
   "source": [
    "model_filename = 'models/kinetics-nc{}_curr.pt'.format(len(normal_classes))\n",
    "curr_weights_path = os.path.join(root_path, model_filename)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Iteration 0 (Epoch: 0), loss is 1.353910207748413\n",
      "Iteration 10 (Epoch: 0), loss is 1.336286187171936\n",
      "Iteration 20 (Epoch: 0), loss is 0.9653661251068115\n",
      "Iteration 30 (Epoch: 0), loss is 0.9475481510162354\n",
      "Iteration 40 (Epoch: 0), loss is 0.8236885070800781\n",
      "Iteration 50 (Epoch: 0), loss is 0.6526684165000916\n",
      "Iteration 60 (Epoch: 0), loss is 0.5776662230491638\n",
      "Iteration 70 (Epoch: 0), loss is 0.5663155913352966\n",
      "Iteration 0 (Epoch: 1), loss is 0.3762049078941345\n",
      "Iteration 10 (Epoch: 1), loss is 0.42019519209861755\n",
      "Iteration 20 (Epoch: 1), loss is 0.4635215699672699\n",
      "Iteration 30 (Epoch: 1), loss is 0.41238880157470703\n",
      "Iteration 40 (Epoch: 1), loss is 0.36050087213516235\n",
      "Iteration 50 (Epoch: 1), loss is 0.3235246539115906\n",
      "Iteration 60 (Epoch: 1), loss is 0.34993988275527954\n",
      "Iteration 70 (Epoch: 1), loss is 0.26971614360809326\n",
      "Iteration 0 (Epoch: 2), loss is 0.275116503238678\n",
      "Iteration 10 (Epoch: 2), loss is 0.3378540575504303\n",
      "Iteration 20 (Epoch: 2), loss is 0.3760155141353607\n",
      "Iteration 30 (Epoch: 2), loss is 0.3310149013996124\n",
      "Iteration 40 (Epoch: 2), loss is 0.30752965807914734\n",
      "Iteration 50 (Epoch: 2), loss is 0.3423963189125061\n",
      "Iteration 60 (Epoch: 2), loss is 0.20038625597953796\n",
      "Iteration 70 (Epoch: 2), loss is 0.32319408655166626\n",
      "Done\n"
     ]
    }
   ],
   "source": [
    "# Training Loop\n",
    "train = 2 # 0 - Not, 1 - Train only, 2 - Train and Save\n",
    "\n",
    "if train >= 1:\n",
    "    for epoch in range(4):\n",
    "        for itern, [data, label] in enumerate(loader):\n",
    "                # get data\n",
    "                data = data.float().to(dev)\n",
    "                label = torch.from_numpy(map2ind(label))\n",
    "                label = label.long().to(dev)\n",
    "\n",
    "                # forward\n",
    "                output = model(data)\n",
    "                loss = loss_fn(output, label)\n",
    "\n",
    "                # backward\n",
    "                optimizer.zero_grad()\n",
    "                loss.backward()\n",
    "                optimizer.step()\n",
    "                if itern%10 == 0: \n",
    "                    print(\"Iteration {} (Epoch: {}), loss is {}\".format(itern, epoch, loss))\n",
    "                if loss < loss_th:\n",
    "                    print(\"Met loss {} in Epoch {}, iteration {}\".format(loss, epoch, itern))\n",
    "                    break\n",
    "        else:\n",
    "            continue\n",
    "        break\n",
    "    print(\"Done\")\n",
    "if train == 2:\n",
    "    torch.save(model.state_dict(), curr_weights_path)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Correct predictions\n",
      " tensor([1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 1, 1, 1, 1, 1, 1, 1,\n",
      "        1, 1, 0, 1, 1, 1, 1, 1], device='cuda:0', dtype=torch.uint8)\n"
     ]
    }
   ],
   "source": [
    "# For output evaluation\n",
    "if train >= 1:\n",
    "    sfmax_vals, cls = torch.max(output, 1)\n",
    "    print(\"Correct predictions\\n {}\".format(cls==label))\n",
    "    softmax = nn.Softmax(dim=1)\n",
    "    osoftmax_gpu = softmax(output)\n",
    "\n",
    "    sfmax_vals, cls = torch.max(osoftmax_gpu, 1)\n",
    "\n",
    "    osoftmax = 100 * osoftmax_gpu.cpu().detach().numpy()\n",
    "#     print(\"Softmax Outputs \\n{} \\nSofmax max values \\n{}\".format(osoftmax, sfmax_vals))\n",
    "    torch.mean(sfmax_vals)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "Model(\n",
       "  (data_bn): BatchNorm1d(54, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
       "  (st_gcn_networks): ModuleList(\n",
       "    (0): st_gcn(\n",
       "      (gcn): ConvTemporalGraphical(\n",
       "        (conv): Conv2d(3, 192, kernel_size=(1, 1), stride=(1, 1))\n",
       "      )\n",
       "      (tcn): Sequential(\n",
       "        (0): BatchNorm2d(64, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
       "        (1): ReLU(inplace)\n",
       "        (2): Conv2d(64, 64, kernel_size=(9, 1), stride=(1, 1), padding=(4, 0))\n",
       "        (3): BatchNorm2d(64, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
       "        (4): Dropout(p=0, inplace)\n",
       "      )\n",
       "      (relu): ReLU(inplace)\n",
       "    )\n",
       "    (1): st_gcn(\n",
       "      (gcn): ConvTemporalGraphical(\n",
       "        (conv): Conv2d(64, 192, kernel_size=(1, 1), stride=(1, 1))\n",
       "      )\n",
       "      (tcn): Sequential(\n",
       "        (0): BatchNorm2d(64, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
       "        (1): ReLU(inplace)\n",
       "        (2): Conv2d(64, 64, kernel_size=(9, 1), stride=(1, 1), padding=(4, 0))\n",
       "        (3): BatchNorm2d(64, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
       "        (4): Dropout(p=0, inplace)\n",
       "      )\n",
       "      (relu): ReLU(inplace)\n",
       "    )\n",
       "    (2): st_gcn(\n",
       "      (gcn): ConvTemporalGraphical(\n",
       "        (conv): Conv2d(64, 192, kernel_size=(1, 1), stride=(1, 1))\n",
       "      )\n",
       "      (tcn): Sequential(\n",
       "        (0): BatchNorm2d(64, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
       "        (1): ReLU(inplace)\n",
       "        (2): Conv2d(64, 64, kernel_size=(9, 1), stride=(1, 1), padding=(4, 0))\n",
       "        (3): BatchNorm2d(64, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
       "        (4): Dropout(p=0, inplace)\n",
       "      )\n",
       "      (relu): ReLU(inplace)\n",
       "    )\n",
       "    (3): st_gcn(\n",
       "      (gcn): ConvTemporalGraphical(\n",
       "        (conv): Conv2d(64, 192, kernel_size=(1, 1), stride=(1, 1))\n",
       "      )\n",
       "      (tcn): Sequential(\n",
       "        (0): BatchNorm2d(64, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
       "        (1): ReLU(inplace)\n",
       "        (2): Conv2d(64, 64, kernel_size=(9, 1), stride=(1, 1), padding=(4, 0))\n",
       "        (3): BatchNorm2d(64, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
       "        (4): Dropout(p=0, inplace)\n",
       "      )\n",
       "      (relu): ReLU(inplace)\n",
       "    )\n",
       "    (4): st_gcn(\n",
       "      (gcn): ConvTemporalGraphical(\n",
       "        (conv): Conv2d(64, 384, kernel_size=(1, 1), stride=(1, 1))\n",
       "      )\n",
       "      (tcn): Sequential(\n",
       "        (0): BatchNorm2d(128, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
       "        (1): ReLU(inplace)\n",
       "        (2): Conv2d(128, 128, kernel_size=(9, 1), stride=(2, 1), padding=(4, 0))\n",
       "        (3): BatchNorm2d(128, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
       "        (4): Dropout(p=0, inplace)\n",
       "      )\n",
       "      (residual): Sequential(\n",
       "        (0): Conv2d(64, 128, kernel_size=(1, 1), stride=(2, 1))\n",
       "        (1): BatchNorm2d(128, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
       "      )\n",
       "      (relu): ReLU(inplace)\n",
       "    )\n",
       "    (5): st_gcn(\n",
       "      (gcn): ConvTemporalGraphical(\n",
       "        (conv): Conv2d(128, 384, kernel_size=(1, 1), stride=(1, 1))\n",
       "      )\n",
       "      (tcn): Sequential(\n",
       "        (0): BatchNorm2d(128, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
       "        (1): ReLU(inplace)\n",
       "        (2): Conv2d(128, 128, kernel_size=(9, 1), stride=(1, 1), padding=(4, 0))\n",
       "        (3): BatchNorm2d(128, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
       "        (4): Dropout(p=0, inplace)\n",
       "      )\n",
       "      (relu): ReLU(inplace)\n",
       "    )\n",
       "    (6): st_gcn(\n",
       "      (gcn): ConvTemporalGraphical(\n",
       "        (conv): Conv2d(128, 384, kernel_size=(1, 1), stride=(1, 1))\n",
       "      )\n",
       "      (tcn): Sequential(\n",
       "        (0): BatchNorm2d(128, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
       "        (1): ReLU(inplace)\n",
       "        (2): Conv2d(128, 128, kernel_size=(9, 1), stride=(1, 1), padding=(4, 0))\n",
       "        (3): BatchNorm2d(128, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
       "        (4): Dropout(p=0, inplace)\n",
       "      )\n",
       "      (relu): ReLU(inplace)\n",
       "    )\n",
       "    (7): st_gcn(\n",
       "      (gcn): ConvTemporalGraphical(\n",
       "        (conv): Conv2d(128, 768, kernel_size=(1, 1), stride=(1, 1))\n",
       "      )\n",
       "      (tcn): Sequential(\n",
       "        (0): BatchNorm2d(256, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
       "        (1): ReLU(inplace)\n",
       "        (2): Conv2d(256, 256, kernel_size=(9, 1), stride=(2, 1), padding=(4, 0))\n",
       "        (3): BatchNorm2d(256, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
       "        (4): Dropout(p=0, inplace)\n",
       "      )\n",
       "      (residual): Sequential(\n",
       "        (0): Conv2d(128, 256, kernel_size=(1, 1), stride=(2, 1))\n",
       "        (1): BatchNorm2d(256, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
       "      )\n",
       "      (relu): ReLU(inplace)\n",
       "    )\n",
       "    (8): st_gcn(\n",
       "      (gcn): ConvTemporalGraphical(\n",
       "        (conv): Conv2d(256, 768, kernel_size=(1, 1), stride=(1, 1))\n",
       "      )\n",
       "      (tcn): Sequential(\n",
       "        (0): BatchNorm2d(256, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
       "        (1): ReLU(inplace)\n",
       "        (2): Conv2d(256, 256, kernel_size=(9, 1), stride=(1, 1), padding=(4, 0))\n",
       "        (3): BatchNorm2d(256, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
       "        (4): Dropout(p=0, inplace)\n",
       "      )\n",
       "      (relu): ReLU(inplace)\n",
       "    )\n",
       "    (9): st_gcn(\n",
       "      (gcn): ConvTemporalGraphical(\n",
       "        (conv): Conv2d(256, 768, kernel_size=(1, 1), stride=(1, 1))\n",
       "      )\n",
       "      (tcn): Sequential(\n",
       "        (0): BatchNorm2d(256, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
       "        (1): ReLU(inplace)\n",
       "        (2): Conv2d(256, 256, kernel_size=(9, 1), stride=(1, 1), padding=(4, 0))\n",
       "        (3): BatchNorm2d(256, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
       "        (4): Dropout(p=0, inplace)\n",
       "      )\n",
       "      (relu): ReLU(inplace)\n",
       "    )\n",
       "  )\n",
       "  (edge_importance): ParameterList(\n",
       "      (0): Parameter containing: [torch.cuda.FloatTensor of size 3x18x18 (GPU 0)]\n",
       "      (1): Parameter containing: [torch.cuda.FloatTensor of size 3x18x18 (GPU 0)]\n",
       "      (2): Parameter containing: [torch.cuda.FloatTensor of size 3x18x18 (GPU 0)]\n",
       "      (3): Parameter containing: [torch.cuda.FloatTensor of size 3x18x18 (GPU 0)]\n",
       "      (4): Parameter containing: [torch.cuda.FloatTensor of size 3x18x18 (GPU 0)]\n",
       "      (5): Parameter containing: [torch.cuda.FloatTensor of size 3x18x18 (GPU 0)]\n",
       "      (6): Parameter containing: [torch.cuda.FloatTensor of size 3x18x18 (GPU 0)]\n",
       "      (7): Parameter containing: [torch.cuda.FloatTensor of size 3x18x18 (GPU 0)]\n",
       "      (8): Parameter containing: [torch.cuda.FloatTensor of size 3x18x18 (GPU 0)]\n",
       "      (9): Parameter containing: [torch.cuda.FloatTensor of size 3x18x18 (GPU 0)]\n",
       "  )\n",
       "  (fcn): Conv2d(256, 4, kernel_size=(1, 1), stride=(1, 1))\n",
       ")"
      ]
     },
     "execution_count": 16,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "dev = 'cuda:0'\n",
    "model_ld = io.load_model(model_name, **model_args)\n",
    "model_ld.load_state_dict(torch.load(curr_weights_path))\n",
    "model_ld.to(dev)\n",
    "model_ld.eval()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "Model(\n",
       "  (data_bn): BatchNorm1d(54, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
       "  (st_gcn_networks): ModuleList(\n",
       "    (0): st_gcn(\n",
       "      (gcn): ConvTemporalGraphical(\n",
       "        (conv): Conv2d(3, 192, kernel_size=(1, 1), stride=(1, 1))\n",
       "      )\n",
       "      (tcn): Sequential(\n",
       "        (0): BatchNorm2d(64, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
       "        (1): ReLU(inplace)\n",
       "        (2): Conv2d(64, 64, kernel_size=(9, 1), stride=(1, 1), padding=(4, 0))\n",
       "        (3): BatchNorm2d(64, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
       "        (4): Dropout(p=0, inplace)\n",
       "      )\n",
       "      (relu): ReLU(inplace)\n",
       "    )\n",
       "    (1): st_gcn(\n",
       "      (gcn): ConvTemporalGraphical(\n",
       "        (conv): Conv2d(64, 192, kernel_size=(1, 1), stride=(1, 1))\n",
       "      )\n",
       "      (tcn): Sequential(\n",
       "        (0): BatchNorm2d(64, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
       "        (1): ReLU(inplace)\n",
       "        (2): Conv2d(64, 64, kernel_size=(9, 1), stride=(1, 1), padding=(4, 0))\n",
       "        (3): BatchNorm2d(64, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
       "        (4): Dropout(p=0, inplace)\n",
       "      )\n",
       "      (relu): ReLU(inplace)\n",
       "    )\n",
       "    (2): st_gcn(\n",
       "      (gcn): ConvTemporalGraphical(\n",
       "        (conv): Conv2d(64, 192, kernel_size=(1, 1), stride=(1, 1))\n",
       "      )\n",
       "      (tcn): Sequential(\n",
       "        (0): BatchNorm2d(64, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
       "        (1): ReLU(inplace)\n",
       "        (2): Conv2d(64, 64, kernel_size=(9, 1), stride=(1, 1), padding=(4, 0))\n",
       "        (3): BatchNorm2d(64, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
       "        (4): Dropout(p=0, inplace)\n",
       "      )\n",
       "      (relu): ReLU(inplace)\n",
       "    )\n",
       "    (3): st_gcn(\n",
       "      (gcn): ConvTemporalGraphical(\n",
       "        (conv): Conv2d(64, 192, kernel_size=(1, 1), stride=(1, 1))\n",
       "      )\n",
       "      (tcn): Sequential(\n",
       "        (0): BatchNorm2d(64, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
       "        (1): ReLU(inplace)\n",
       "        (2): Conv2d(64, 64, kernel_size=(9, 1), stride=(1, 1), padding=(4, 0))\n",
       "        (3): BatchNorm2d(64, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
       "        (4): Dropout(p=0, inplace)\n",
       "      )\n",
       "      (relu): ReLU(inplace)\n",
       "    )\n",
       "    (4): st_gcn(\n",
       "      (gcn): ConvTemporalGraphical(\n",
       "        (conv): Conv2d(64, 384, kernel_size=(1, 1), stride=(1, 1))\n",
       "      )\n",
       "      (tcn): Sequential(\n",
       "        (0): BatchNorm2d(128, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
       "        (1): ReLU(inplace)\n",
       "        (2): Conv2d(128, 128, kernel_size=(9, 1), stride=(2, 1), padding=(4, 0))\n",
       "        (3): BatchNorm2d(128, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
       "        (4): Dropout(p=0, inplace)\n",
       "      )\n",
       "      (residual): Sequential(\n",
       "        (0): Conv2d(64, 128, kernel_size=(1, 1), stride=(2, 1))\n",
       "        (1): BatchNorm2d(128, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
       "      )\n",
       "      (relu): ReLU(inplace)\n",
       "    )\n",
       "    (5): st_gcn(\n",
       "      (gcn): ConvTemporalGraphical(\n",
       "        (conv): Conv2d(128, 384, kernel_size=(1, 1), stride=(1, 1))\n",
       "      )\n",
       "      (tcn): Sequential(\n",
       "        (0): BatchNorm2d(128, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
       "        (1): ReLU(inplace)\n",
       "        (2): Conv2d(128, 128, kernel_size=(9, 1), stride=(1, 1), padding=(4, 0))\n",
       "        (3): BatchNorm2d(128, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
       "        (4): Dropout(p=0, inplace)\n",
       "      )\n",
       "      (relu): ReLU(inplace)\n",
       "    )\n",
       "    (6): st_gcn(\n",
       "      (gcn): ConvTemporalGraphical(\n",
       "        (conv): Conv2d(128, 384, kernel_size=(1, 1), stride=(1, 1))\n",
       "      )\n",
       "      (tcn): Sequential(\n",
       "        (0): BatchNorm2d(128, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
       "        (1): ReLU(inplace)\n",
       "        (2): Conv2d(128, 128, kernel_size=(9, 1), stride=(1, 1), padding=(4, 0))\n",
       "        (3): BatchNorm2d(128, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
       "        (4): Dropout(p=0, inplace)\n",
       "      )\n",
       "      (relu): ReLU(inplace)\n",
       "    )\n",
       "    (7): st_gcn(\n",
       "      (gcn): ConvTemporalGraphical(\n",
       "        (conv): Conv2d(128, 768, kernel_size=(1, 1), stride=(1, 1))\n",
       "      )\n",
       "      (tcn): Sequential(\n",
       "        (0): BatchNorm2d(256, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
       "        (1): ReLU(inplace)\n",
       "        (2): Conv2d(256, 256, kernel_size=(9, 1), stride=(2, 1), padding=(4, 0))\n",
       "        (3): BatchNorm2d(256, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
       "        (4): Dropout(p=0, inplace)\n",
       "      )\n",
       "      (residual): Sequential(\n",
       "        (0): Conv2d(128, 256, kernel_size=(1, 1), stride=(2, 1))\n",
       "        (1): BatchNorm2d(256, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
       "      )\n",
       "      (relu): ReLU(inplace)\n",
       "    )\n",
       "    (8): st_gcn(\n",
       "      (gcn): ConvTemporalGraphical(\n",
       "        (conv): Conv2d(256, 768, kernel_size=(1, 1), stride=(1, 1))\n",
       "      )\n",
       "      (tcn): Sequential(\n",
       "        (0): BatchNorm2d(256, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
       "        (1): ReLU(inplace)\n",
       "        (2): Conv2d(256, 256, kernel_size=(9, 1), stride=(1, 1), padding=(4, 0))\n",
       "        (3): BatchNorm2d(256, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
       "        (4): Dropout(p=0, inplace)\n",
       "      )\n",
       "      (relu): ReLU(inplace)\n",
       "    )\n",
       "    (9): st_gcn(\n",
       "      (gcn): ConvTemporalGraphical(\n",
       "        (conv): Conv2d(256, 768, kernel_size=(1, 1), stride=(1, 1))\n",
       "      )\n",
       "      (tcn): Sequential(\n",
       "        (0): BatchNorm2d(256, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
       "        (1): ReLU(inplace)\n",
       "        (2): Conv2d(256, 256, kernel_size=(9, 1), stride=(1, 1), padding=(4, 0))\n",
       "        (3): BatchNorm2d(256, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
       "        (4): Dropout(p=0, inplace)\n",
       "      )\n",
       "      (relu): ReLU(inplace)\n",
       "    )\n",
       "  )\n",
       "  (edge_importance): ParameterList(\n",
       "      (0): Parameter containing: [torch.cuda.FloatTensor of size 3x18x18 (GPU 0)]\n",
       "      (1): Parameter containing: [torch.cuda.FloatTensor of size 3x18x18 (GPU 0)]\n",
       "      (2): Parameter containing: [torch.cuda.FloatTensor of size 3x18x18 (GPU 0)]\n",
       "      (3): Parameter containing: [torch.cuda.FloatTensor of size 3x18x18 (GPU 0)]\n",
       "      (4): Parameter containing: [torch.cuda.FloatTensor of size 3x18x18 (GPU 0)]\n",
       "      (5): Parameter containing: [torch.cuda.FloatTensor of size 3x18x18 (GPU 0)]\n",
       "      (6): Parameter containing: [torch.cuda.FloatTensor of size 3x18x18 (GPU 0)]\n",
       "      (7): Parameter containing: [torch.cuda.FloatTensor of size 3x18x18 (GPU 0)]\n",
       "      (8): Parameter containing: [torch.cuda.FloatTensor of size 3x18x18 (GPU 0)]\n",
       "      (9): Parameter containing: [torch.cuda.FloatTensor of size 3x18x18 (GPU 0)]\n",
       "  )\n",
       "  (fcn): Conv2d(256, 4, kernel_size=(1, 1), stride=(1, 1))\n",
       ")"
      ]
     },
     "execution_count": 17,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Model Definitions for Testing\n",
    "# loader = data_loader['mixed_test']\n",
    "loader = data_loader['random_test']\n",
    "model = model_ld\n",
    "model.eval()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Iteration 0\n",
      "Iteration 50\n",
      "Iteration 100\n",
      "Iteration 150\n",
      "Iteration 200\n",
      "Iteration 250\n",
      "Iteration 300\n",
      "Iteration 350\n",
      "Iteration 400\n",
      "Iteration 450\n",
      "Iteration 500\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Exception ignored in: <bound method _DataLoaderIter.__del__ of <torch.utils.data.dataloader._DataLoaderIter object at 0x7f8289dd0b00>>\n",
      "Traceback (most recent call last):\n",
      "  File \"/usr/local/lib/python3.5/dist-packages/torch/utils/data/dataloader.py\", line 399, in __del__\n",
      "    self._shutdown_workers()\n",
      "  File \"/usr/local/lib/python3.5/dist-packages/torch/utils/data/dataloader.py\", line 378, in _shutdown_workers\n",
      "    self.worker_result_queue.get()\n",
      "  File \"/usr/lib/python3.5/multiprocessing/queues.py\", line 345, in get\n",
      "    return ForkingPickler.loads(res)\n",
      "  File \"/usr/local/lib/python3.5/dist-packages/torch/multiprocessing/reductions.py\", line 151, in rebuild_storage_fd\n",
      "    fd = df.detach()\n",
      "  File \"/usr/lib/python3.5/multiprocessing/resource_sharer.py\", line 57, in detach\n",
      "    with _resource_sharer.get_connection(self._id) as conn:\n",
      "  File \"/usr/lib/python3.5/multiprocessing/resource_sharer.py\", line 87, in get_connection\n",
      "    c = Client(address, authkey=process.current_process().authkey)\n",
      "  File \"/usr/lib/python3.5/multiprocessing/connection.py\", line 494, in Client\n",
      "    deliver_challenge(c, authkey)\n",
      "  File \"/usr/lib/python3.5/multiprocessing/connection.py\", line 722, in deliver_challenge\n",
      "    response = connection.recv_bytes(256)        # reject large message\n",
      "  File \"/usr/lib/python3.5/multiprocessing/connection.py\", line 216, in recv_bytes\n",
      "    buf = self._recv_bytes(maxlength)\n",
      "  File \"/usr/lib/python3.5/multiprocessing/connection.py\", line 407, in _recv_bytes\n",
      "    buf = self._recv(4)\n",
      "  File \"/usr/lib/python3.5/multiprocessing/connection.py\", line 379, in _recv\n",
      "    chunk = read(handle, remaining)\n",
      "ConnectionResetError: [Errno 104] Connection reset by peer\n",
      "/usr/local/lib/python3.5/dist-packages/numpy/core/fromnumeric.py:2920: RuntimeWarning: Mean of empty slice.\n",
      "  out=out, **kwargs)\n",
      "/usr/local/lib/python3.5/dist-packages/numpy/core/_methods.py:85: RuntimeWarning: invalid value encountered in double_scalars\n",
      "  ret = ret.dtype.type(ret / rcount)\n"
     ]
    }
   ],
   "source": [
    "loss_value = []\n",
    "result_frag = []\n",
    "label_frag = []\n",
    "label_ad = np.empty(0)\n",
    "outputs = []\n",
    "epoch_info = dict()\n",
    "confusion_mat = np.zeros([5,5], dtype=np.int32)\n",
    "evaluation = True\n",
    "softmax = nn.Softmax(dim=1)\n",
    "\n",
    "for itern, [data, label_ld] in enumerate(loader):\n",
    "    # get data\n",
    "    data = data.float().cuda()\n",
    "    label_ad_curr = map2ind(label_ld, from_arr=normal_classes, to_arr=np.ones_like(normal_classes), def_val=0)\n",
    "    label_ad = np.concatenate((label_ad, label_ad_curr))\n",
    "    label_mapped = map2ind(label_ld, from_arr=normal_classes, to_arr=None, def_val=0) # Assign output probs regardless of abnormal classes,\n",
    "    label = torch.from_numpy(label_mapped)\n",
    "\n",
    "    label = label.long().cuda()\n",
    "\n",
    "    # inference\n",
    "    with torch.no_grad():\n",
    "        output = model(data)\n",
    "    result_frag.append(output.data.cpu().numpy())\n",
    "\n",
    "    # get loss\n",
    "    if evaluation:\n",
    "        loss = loss_fn(output, label)\n",
    "#         confusion_mat[label_mapped, np.argmax(output, axis=1)] += 1\n",
    "#         loss_value.append(loss.data[0])\n",
    "        label_frag.append(label.data.cpu().numpy())\n",
    "        outputs.append([itern ,softmax(output).data.cpu().numpy()])\n",
    "\n",
    "    # Track progress\n",
    "    if itern % 50 == 0:\n",
    "        print(\"Iteration {}\".format(itern))\n",
    "    # Stop\n",
    "    if itern == 500:\n",
    "        break\n",
    "\n",
    "result = np.concatenate(result_frag)\n",
    "# np.savetxt(\"kinetics400_confusion.csv\", confusion_mat)\n",
    "if evaluation:\n",
    "    label2 = np.concatenate(label_frag)\n",
    "    epoch_info['mean_loss'] = np.mean(loss_value)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(16032,)"
      ]
     },
     "execution_count": 28,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "label_ad.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0.7936247 0.68401027\n"
     ]
    }
   ],
   "source": [
    "len(outputs)\n",
    "out_sfmax = [np.array(sf) for i, sf in outputs]\n",
    "out_sfmax = np.concatenate(out_sfmax, axis=0)\n",
    "out_sfmax_max_vals = out_sfmax.max(axis=1)\n",
    "split_idx = 198\n",
    "normal_max_sfmax = out_sfmax_max_vals[label_ad == 1]\n",
    "abnormal_max_sfmax = out_sfmax_max_vals[label_ad == 0]\n",
    "print(normal_max_sfmax.mean(), abnormal_max_sfmax.mean())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Correct predictions\n",
      " tensor([0, 0, 1, 0, 0, 0, 0, 0, 1, 1, 0, 1, 0, 0, 0, 0, 0, 0, 1, 0, 0, 1, 0, 1,\n",
      "        0, 0, 0, 0, 0, 0, 0, 1], device='cuda:0', dtype=torch.uint8)\n",
      "Softmax Outputs \n",
      "[[13.81801 34.12931  4.29795 47.75475]\n",
      " [15.31688 42.85121 21.37346 20.45845]\n",
      " [46.35085 10.06052 33.75373  9.8349 ]\n",
      " [14.76985  2.29571  2.74723 80.18721]\n",
      " [ 7.75083  1.66602 86.58537  3.99778]\n",
      " [ 7.68746  7.27262 19.84185 65.19806]\n",
      " [17.89087 26.04573 45.10905 10.95434]\n",
      " [ 7.58863 50.07758  9.81072 32.52309]\n",
      " [59.74096  7.33461 25.20918  7.71524]\n",
      " [87.51398  0.18372  2.35238  9.94992]\n",
      " [15.2642   5.24221 73.47461  6.01898]\n",
      " [47.11449  4.75482 43.70334  4.42736]\n",
      " [20.30817 47.36475 16.68689 15.6402 ]\n",
      " [ 5.08321  8.13469 10.29965 76.48246]\n",
      " [ 3.99515  1.35391 83.54682 11.10411]\n",
      " [ 5.47435 63.51363 19.96676 11.04527]\n",
      " [ 7.19495 50.07774 12.34617 30.38114]\n",
      " [ 2.25397 16.56493 80.10133  1.07978]\n",
      " [52.06236 14.50983  2.33272 31.0951 ]\n",
      " [21.15195 12.51193  1.99634 64.33978]\n",
      " [ 4.70428 74.22443  5.74089 15.3304 ]\n",
      " [87.24993  0.38008 10.01402  2.35598]\n",
      " [ 5.50159 17.47742  3.89415 73.12685]\n",
      " [62.60039  0.49034  0.7399  36.16938]\n",
      " [ 2.47638 27.43074  3.87364 66.21924]\n",
      " [23.02257  0.25282  0.6389  76.0857 ]\n",
      " [17.07602 13.78436 54.24641 14.89321]\n",
      " [ 1.18551 83.49519  9.36699  5.95232]\n",
      " [ 3.92656  5.61965  1.23784 89.21594]\n",
      " [ 1.36564  1.57145  0.39679 96.66613]\n",
      " [ 2.49135  9.80918  4.5986  83.10086]\n",
      " [47.3441   2.89395 46.27153  3.49042]] \n",
      "Sofmax max values \n",
      "tensor([0.4775, 0.4285, 0.4635, 0.8019, 0.8659, 0.6520, 0.4511, 0.5008, 0.5974,\n",
      "        0.8751, 0.7347, 0.4711, 0.4736, 0.7648, 0.8355, 0.6351, 0.5008, 0.8010,\n",
      "        0.5206, 0.6434, 0.7422, 0.8725, 0.7313, 0.6260, 0.6622, 0.7609, 0.5425,\n",
      "        0.8350, 0.8922, 0.9667, 0.8310, 0.4734], device='cuda:0')\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "tensor([0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,\n",
       "        0, 0, 0, 0, 0, 0, 0, 0], device='cuda:0')"
      ]
     },
     "execution_count": 30,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# For output evaluation\n",
    "sfmax_vals, cls = torch.max(output, 1)\n",
    "print(\"Correct predictions\\n {}\".format(cls==label))\n",
    "softmax = nn.Softmax(dim=1)\n",
    "osoftmax_gpu = softmax(output)\n",
    "\n",
    "sfmax_vals, cls = torch.max(osoftmax_gpu, 1)\n",
    "\n",
    "osoftmax = 100 * osoftmax_gpu.cpu().detach().numpy()\n",
    "print(\"Softmax Outputs \\n{} \\nSofmax max values \\n{}\".format(osoftmax, sfmax_vals))\n",
    "torch.mean(sfmax_vals)\n",
    "label\n",
    "# print(\"Correct predictions\\n {}\".format(cls_bn==label))\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.6785634686869959"
      ]
     },
     "execution_count": 31,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "from sklearn.metrics import roc_curve, auc\n",
    "\n",
    "true_labels = label_ad\n",
    "fpr, tpr, thresholds = roc_curve(true_labels, out_sfmax_max_vals)\n",
    "roc_auc = auc(fpr, tpr)\n",
    "\n",
    "roc_auc"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Log To Text File\n",
    "log_str = \"Results for classes {}\\nfpr: {}\\ntpr: {}\\nthr: {}\".format(normal_classes, fpr[::20], tpr[::20], thresholds[::20])\n",
    "# print(log_str)\n",
    "normal_classes_str = [str(i) for i in normal_classes]\n",
    "classes_str = '_'.join(normal_classes_str)\n",
    "time_str = time.strftime(\"%b%d_%H%M\")\n",
    "log_filename = \"{}_{}_auc{}.txt\".format(time_str, classes_str, str(int(100*roc_auc)))\n",
    "log_dirname = 'logs/kinetics_trained_n_v_all/'\n",
    "logfile_path = os.path.join(root_path, log_dirname, log_filename)\n",
    "with open(logfile_path, 'w') as f:\n",
    "    f.write(log_str)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0.3125 0.10219254032258064 0.92117476\n"
     ]
    }
   ],
   "source": [
    "ind=95\n",
    "print(tpr[ind], fpr[ind], thresholds[ind])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 34,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYoAAAEWCAYAAAB42tAoAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDMuMC4wLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvqOYd8AAAIABJREFUeJzs3Xd4FNX6wPHvm0IKoSUUEaS3QGgamihFpQjYQAVRVEQxIja8NEFREK+KIiBVr8jPa8GrgiAdFLBLL1Kkl9BrIKSQcn5/zCa7CamQzWST9/M8eXbmzJmZd0v23Zkzc44YY1BKKaUy42V3AEoppQo2TRRKKaWypIlCKaVUljRRKKWUypImCqWUUlnSRKGUUipLmiiUbUQkQER+EJEoEfkmH/a3SkSedPd+3EFEjIjUyuU67UQk0l0x5WD/r4vI53bt3yWOh0Vkmd1xeDJNFDYRkQMiEisi0SJyXERmiUhQLtavKCKfiMgxEbkoIjtF5A0RKe7OuPPY/UAFIMQY84DdwSj75FUSF5FqjqTqk1JmjPnCGNPxWrddlGmisNddxpggoAnQFBiek5VEJBj4AwgAWhljSgAdgNJATTfFius/Xx6pCuwyxiQWgFjyTEGOTamroYmiADDGHAeWYiUMAESklIh8JiKnROSgiIwUkZT3axBwEXjEGHPAsY3DxpgXjDFbMtqHiNwiIr+LyHkROSwijzvK0/ySE5HHReRXl3kjIs+KyG5gt4hME5H30m17nogMckxfLyLfOeLeLyLPZxLPG8BrQE/HUVU/EfFyPM+DInLS8fxLOeqn/FLsJyKHgJ8y2GYZEVng2Pc5x3TldNVqisgaEbngiDs43fYfE5FDInJaREa4bNtPRCaIyFHH3wQR8XMsaycikSIyVESOA5+6lA1xPJdjInKviHQRkV0iclZEXnHZfnMR+cPx/hwTkckiUiyj1y6D5x0sIp864jonIt9nUm+YiOx1HIFuF5H7XJbVEpHVYp0GPC0iXzvKRUQ+cDyHCyKyVUTCMtl+dcc2LorIcqBsuuUtXT6Dm0WknaN8LHArMNnxWZjsKK8nIssdr9U/IvKgy7YCROR9x2clSkR+FZEA4GdHlfOObbXK4DN9s4isday3VkRudlm2SkTGiMhvjuexTETSPI8iyRijfzb8AQeAOxzTlYGtwESX5Z8B84ASQDVgF9DPsexP4I1c7KsqVmJ5CPAFQoAmjmWrgCdd6j4O/Ooyb4DlQDDWEUwb4DAgjuVlgFjgeqwfHuuxEkAxoAawD+iUSVyvA5+7zD8B7HGsFwTMAf7rWFbNEctnQHEgIIPthQA9gEDH6/YN8L3L8lXAESDMsY3vUvbvsv2PHc+zMRAPhDqWj3a87uWBcsDvwBjHsnZAIvAO4OdYP6XsNcdr/hRwCvjSEVsDx+tW3bGNm4CWgI8jlh3Ai+neh1qZvI4Lga8d74Uv0NYlrkiXeg+4vE89gUtARceyr4ARjmX+wC2O8k6O97Q0IEBoyjoZxPEHMN7xGrTB+sylvL6VgDNAF8c+Ojjmy2XyOSyO9Tnr63hNmgKngfqO5VMc61QCvIGbHftNeR99MvpMY32OzwF9HNt9yDEf4hLHXqCO431cBbxt9/eF3X+2B1BU/7ASRbTjn8kAPwKlHcu8gcsp/xSOsqeBVY7p3UBELvY1HJibybL0/6Cp/1SOeQPc5jIvwCGgjWP+KeAnx3QL4FAG+/40k32/TtpE8SMwwGW+LpCA88vTADVy8bybAOfSPde3XebrO15nb5ftV3ZZvgbo5ZjeC3RxWdYJOOCYbufYjr/L8nZYicDbMV/Csf0WLnXWA/dmEvuLru8ZmSQKoCKQDJTJYFk7XBJFBss3Afc4pj8DPnJ9/o7y27B+pLQEvLLYVhWsxFjcpexLnIliKI6k77J8KfBYJp/DnsAv6erPAEZhJZpYoHEGcaS8j5klij7AmnTr/AE87hLHSJdlA4AlOf3MFdY/PfVkr3uN1b7QDqiH81C9LNYvw4MudQ9i/XoC65dYxVzs5wasL7qrdThlwlj/PbOxfokB9Aa+cExXBa53nFo4LyLngVewGqxz4nqufM4+6dY/TCZEJFBEZjhOR1zAOg1RWkS8M1n/INbr7Hpq4bjLdAzWkU1msV3vMn/KGBOXLqQzxpgkx3Ss4/GEy/LYlO2LSB3HqbLjjtjfShdXZm4AzhpjzmVXUUQeFZFNLu9NmMs+hmD9CFgjIttE5AkAY8xPwGSsX/AnReQjESmZweavx0rKl1zKXF+vqsAD6T4bt5D557gq0CJd/YeB6xwx+3N1n+n072NKnJVc5jP7DBRZmigKAGPMamAWkHLu/zTWL+mqLtWqYJ02AVgB3CfONovsHCbzRu5LWKdqUlyXUYjp5r8C7heRqlhHEd+57Ge/Maa0y18JY0yXHMZ5lCufcyJpv1yz6u74ZayjkBbGmJJYpz/A+gJMcUO67Sdgvd5XE9vRHMaVE9OAnUBtR+yvkDbuzBwGgkWkdFaVHO/Vx8BArNMspYG/U/ZhjDlujHnKGHM91tHrVHFcjmuMmWSMuQnrCKwOMDiDXRwDykjaq+6qpIvzv+k+G8WNMW87lqd//Q4Dq9PVDzLGPIP1fsWR8Wc6u/ch/fuYEueRDOoqB00UBccEoIOINHb8Cv0fMFZESjj+yQcBKdekjwdKAv/nWIaIVBKR8SLSKINtfwHcISIPioiPiISISErD+Sagu+PXeC2gX3aBGmM2Yv2z/gdYaow571i0BrgoVqNugIh4i0iYiDTL4WvwFfCSo1E0COtX9dcm51dFlcD6lX5erEbqURnUeURE6otIIFa7w7cuv/qzi22kiJRzNG6+hvP9yAslgAtAtIjUA57JyUrGmGPAYqwv9jIi4isibTKoWhzrS/QUgIj0xTqiwDH/gDgb/s856iaLSDMRaSEivlg/KuKwTnWlj+MgsA54Q0SKicgtwF0uVT4H7hKRTo7Phb9YDf4p+zyB1TaVYgFQR0T6OJ6TryOWUGNMMjATGC/WxRPejkZrP8fzS063LVeLHNvt7fhf6ImVABdkUl+hiaLAMMacwjpP/Jqj6Dmsf8x9wK9Y53tnOuqexWq8SwD+EpGLWOf3o7Aag9Nv+xBWI+LLwFms5NDYsfgDrPPrJ4D/w3kaKTtfAnc4HlP2kwR0w2ob2I8zmZTK4TZnAv/FOmW0H+tL6bkcrgtWsg1w7PdPYEkGdf6LdfR2HOv0RYZXZWXgTawvwi1YFx5scJTllX9hnca7iPXL/+tcrNsH67OwEziJ1b6RhjFmO/A+1vn4E0BD4DeXKs2wPkvRwHzgBWPMPqwfJB9jJY+DWKc9x2USR2+sI8yzWEn6M5f9HwbuwTpSOoV1xDAY53fQRKyj1HMiMskYcxHoCPTCOgo4jvNiAbBer63AWsf+3sFqQ4kBxgK/OU5ZtUz3OpzB+oy+7HguQ4BuxpicHFUWWSlXriillFIZ0iMKpZRSWdJEoZRSKkuaKJRSSmVJE4VSSqkseVznZWXLljXVqlWzOwyllPIo69evP22MKXc163pcoqhWrRrr1q2zOwyllPIoIpL+jvQc01NPSimlsqSJQimlVJY0USillMqSJgqllFJZ0kShlFIqS5oolFJKZcltiUJEZoo1zu7fmSwXEZkkIntEZIuI3OiuWJRSSl09d95HMQtrZKzPMll+J1Db8dcCa+CWFm6MRymlPENCDBz9HZIT8mRzly9fMYRIrrgtURhjfhaRallUuQf4zDG05p8iUlpEKjoGYlFKqaLp6J+w+BE4fy2jFzsN/qEDG4/mZuTkK9l5Z3Yl0o5fHOkouyJRiEh/oD9AlSpV0i9WSinPsuUj+Ok5SMroiMExRlCZ2lAqsxGMcy6sYVkm/VbtmrbhEV14GGM+Aj4CCA8P15GWlFKeyRj4/TX4M4vBEb39oOlz0PpN8PHLvF4mtm8/xYYNx3jkEWtU5Ee7G9q+HEX16qOvNmpbE8UR0g50Xxkd4FwpVRhdOAQHlliPf411lvf8GSq1vrK+5P46o5iYBN5882fGjfsdb2+hZcvK1KoVjIhQrVrpawje3kQxHxgoIrOxGrGjtH1CKVUoJFyCuHPO+W87wLldaev0/guuawYi17y7xYt38+yzi9i//zwA/frdREhIwDVvN4XbEoWIfAW0A8qKSCTWYOu+AMaY6cAioAuwB4gB+rorFqWUyjfRx2BmHUiIvnLZdc2gfFOo8wBUbH7Nuzpy5AIvvriUb7/dDkCjRhWYPr0rrVrdkM2auePOq54eyma5AZ511/6VUsqtds6GtePAJKUtP7XZOR1UyTldpjZ0X3JV7Q6ZefbZRcyb9w+Bgb6MHt2OF15oiY9P3t8e5xGN2UopVeAszPK3MDToC51n5vluExOTU5PBO+/cga+vN++/35EqVUrl+b5SaKJQSiljrCOBhJic1Y897ZzuNNM6neRKvCGkft7FB0RFxTFy5E/s2nWWJUseRkSoW7cs33zzQJ7uJyOaKJRSauMkWPni1a3b4PE8aZDOjDGGb77ZzosvLuHYsWi8vYVNm47TtOm13USXG5oolFJFy/7FsOxJSIx1lrleoVSxVc631eBRtyaJvXvPMnDgYpYs2QNAq1aVmT69G40aVXDbPjOiiUIpVbjt/Br2LXDO7/g843reftBjKdzQNn/iysZ77/3Oq6+uJC4ukdKl/XnnnTt48skb8fJyX2LKjCYKpVThlZwIC3tlvOz2qVC3p3Pexx98A/MnrhyIiUkgLi6RPn0a8d57HSlfvrhtsWiiUEoVTibZ6lMpxR3TwNfxZetXBqp3Bq+C8xV46tQl/vnnDLfcYvVnN3Roa9q1q0abNlVtjkwThVKqMEpOgv82hdNbrXnfIGgcYW9MmUhONsycuZEhQ5bj4+PFzp0DCQ4OwM/Pp0AkCdBEoZQqTJITYVl/OP4XnLHuViawPNz5X3vjysTff58kImIBv/1mdaTdoUMNYmISCA7Ou+438oImCqVU4RBzElYPhu0uY6WVbwp9NtgXUyYuXbrM6NGrGT/+TxITk6lQoTgTJnSmZ88GiBuvorpamiiUUp7BGDi3G0xixstXvggHlzvnH1x15Y1wBcT993/DkiV7EIEBA8IZO/Z2Spf2tzusTGmiUEp5hlWDYMOE7OsFVoAeS6B8E/fHdJWGDm3NiRPRTJvWlRYtKtsdTrY0USilCr6YU2mTRHBoxvUCykLXL6FEwfnyTUxM5sMP/+LAgfNMnHgnAO3aVWPduv623BNxNTRRKKUKnt1z4NBPzvk93zun++2F0jXyP6arsGbNEZ5+egGbNh0HoH//m2jQoDyAxyQJ0EShlCoIkpPg6B/OMRzm98i4Xu3uHpEkzp+P45VXfmT69HUYA1WrlmLy5C6pScLTaKJQStlvy0fw44Ary9tPdA4L6uULte/L37iuwuzZf/Pii0s4ceISPj5evPxyK159tQ3FixezO7SrpolCKWW/30Y6p6t1sh6rdoAbn7cnnmuwbNleTpy4ROvWNzBtWlcaNszfDvzcQROFUspeyUkQd9aavmNagb2DOjPx8YkcOXKRGjXKAPDuux249dYqPPZYE49qh8hK3o+Zp5RSOZWUAL+/5pyv3d2+WK7CTz/tp1Gj6XTt+iWXL1tDopYtG0jfvk0LTZIATRRKKTvEnIKLR6wBg/56yyoLKGt1t+EBTpyIpk+fudx++2fs2nUGgMjICzZH5T566kkplb82fAgrM2h7uGde/seSS8nJho8/Xs+wYT9y/nwc/v4+jBx5K4MHt6ZYMW+7w3MbTRRKqfyTlJA2SQRdDz4BcPs0qHSzfXHl0H33fc38+f8A0KlTTaZM6ULNmsE2R+V+miiUUnkvMQ42T4fY02nLT21yTvf8GSrfmr9xXaPu3euxZs0RJk7szAMP1C+QHfi5gyYKpVTeSk6CP96ANW9nXqdUdY9IEvPn/0Nk5AUGDGgGwKOPNqZ791BKlPCzObL8pYlCKZW3fh4C68c751uPSbtcvKDWvfkbUy4dOhTF888vZt68f/Dz86Zz51rUqFEGESlySQI0USil8pprkuj5C1S+xb5YcikhIYlJk/5i1KhVXLqUQIkSxXjzzduoWrWU3aHZShOFUuraXYy0LnNN6asJoO9OCK5rX0y59OefkTz99AK2bDkBwAMP1OeDDzpRqVJJmyOznyYKpdTVS4y3Bgv69RXn+NQAxUpAmTr2xXUVXn11JVu2nKB69dJMntyFLl1q2x1SgaGJQimVe8lJkBgL6z9Ie2d1cD1oPhyuawYF/IogYwwXL16mZEmrzWHy5Dv57LPNjBjRhsBAX5ujK1g0USilcicxHv4vDM7vSVse1g9ajYKSN9gTVy78889pBgxYhAgsX94HEaFu3bKMHXu73aEVSJoolFJpRR2AlS9AfFTGyy8cgAsHrWnf4uBXCu6eAxVb5FeEVy0uLpF///sX3n77Ny5fTiIkJIADB85TvXoZu0Mr0DRRKKWc9i2EZU/CpePZ1610K/T62f0x5ZHly/cyYMAi9uyxeqp94okmvPtuB0JCAm2OrOBza6IQkc7ARMAb+I8x5u10y6sA/weUdtQZZoxZ5M6YlFLpJMZD1F4rOczt5iyv3BZufj3z9SqEuz20vGCMoV+/+Xz6qXVXeP365Zg+vSu33lrV5sg8h9sShYh4A1OADkAksFZE5htjtrtUGwn8zxgzTUTqA4uAau6KSSmVga9vheNr05a1GgWNn4Hinj/ojohQrVppAgJ8eO21tgwa1KpQd+DnDu48omgO7DHG7AMQkdnAPYBrojBAykXKpYCjboxHKZXepmnOJBFUGfxKQqP+cOML9sZ1jTZtOs6xYxe5807rEtehQ1vTp08jbYu4Su5MFJWAwy7zkUD61q7XgWUi8hxQHLgjow2JSH+gP0CVKlXyPFClioyTm2Hbp9blrQCbJjuXPXUAvDz7l/bFi/GMGrWKiRP/IiQkgJ07BxIcHICfn48miWtgd2P2Q8AsY8z7ItIK+K+IhBljkl0rGWM+Aj4CCA8PNzbEqZTnOLPDeVVSenO7gUm6svyxvz06SRhj+P77nTz//BIiIy/g5SX07t0QX18dmy0vuDNRHAFcL6iu7Chz1Q/oDGCM+UNE/IGywEk3xqVU4RW1H2bVz75e7R5WYzVA6RpQtoF743KjgwfPM3DgYhYs2AVAePj1zJjRjRtvrGhzZIWHOxPFWqC2iFTHShC9gN7p6hwCbgdmiUgo4A+ccmNMShVecefgPzWc89U6ZVyvxA1w22Tw8fxeUI0x9OjxP9avP0bJkn689dZtRESE4+2tRxJ5yW2JwhiTKCIDgaVYl77ONMZsE5HRwDpjzHzgZeBjEXkJq2H7cWOMnlpSKrf2LYTFjznnW74Grd+wLx43S042eHkJIsJ773Vk+vR1fPBBJypWLGF3aIWSeNr3cnh4uFm3bp3dYShVcCTEwiSXm8Zqd4e7v7MvHjc6cyaGYcNWAPDxx3fbHI1nEZH1xpiruvnF7sZspdS1WjfOOX3r29b9D4WMMYbPPtvMv/61nNOnYyhWzJtRo9pRubJ2AZ4fNFEo5cl2fg2/j7KmfQKh2ZAC32trbu3YcYpnnlnI6tXWlVzt2lVj2rSumiTykSYKpTzV/iWwsJdz/sGfClWSMMbw2msreeed30hISKZs2UDef78jffo0QgrR8/QEmiiUKsgOLIfI1Rkv+2usc/qubz2i99bcEBGOHLlIQkIyTz11I2+/fQfBwQF2h1UkaaJQqqC5cBCij1nT33XMvv7tU6BOD/fGlE+OHr3I6dMxNGpk9TH17rsd6NevKa1ba48MdtJEoVRBcman44a5dFcjtno94zunAytAWN/8iMytkpKSmTZtHSNG/ESlSiXYtCmCYsW8KVs2kLJlNUnYTROFUnY4tgYW9ITLF9KWx511TqecSqraEW4elX+x5bMNG47x9NMLWLfO6hO0TZuqXLgQT9myOk5EQZGjRCEixYAqxpg92VZWSmUt8lera++stBqV9VgQhcCFC/G8+upPTJ68luRkQ+XKJZk0qTP33ltPG6sLmGwThYh0BcYDxYDqItIEGGWMuc/dwSlVaCTEwMHlkBRvHUmkaDkSbnwxbV3xBv/S+RtfPjPG0KbNp2zefAJvb2HQoJa8/no7SpTw/G5FCqOcHFGMxuoefCWAMWaTiNRya1RKFTZ/vAFr301b1mwotBgBPv72xGQjEeGll1oydeo6ZszoRpMm19kdkspCThJFgjHmfLpDQc/q90MpO8WeSZsk6jwAJavCLWM9umvv3Lh8OYnx4//A21sYPLg1AI8+2phHHmmkHfh5gJwkih0i8iDg5egJ9nngT/eGpVQBFn8Blve3xpjOiaO/O6cf3wEh9dwTVwH1yy8HiYhYyPbtp/Dz8+bRRxtToUIQIoK3t7ZFeIKcJIqBwGtAMjAHqzfYV9wZlFIF2v5F8M/XuV+vbs8ilSROn45hyJDlfPrpJgBq1w5m6tSuVKgQZHNkKrdykig6GWOGAkNTCkSkO1bSUKpoSIyzBgUC2PG5s/zBlTlbX7zguuZ5H1cBZIxh1qxNDB68nDNnYilWzJvhw29h2LBb8PfXK/I9UU7etZFcmRRGZFCmVOH1RTM4/XfasjoPwg3tbAmnoPv8862cORPLbbdVZ+rULtStW9bukNQ1yDRRiEgnrGFKK4nIeJdFJbFOQylVNCTGO5NE6Vrg5QO+QdB8iL1xFSAxMQlERcVRsWIJRISpU7uwdu1RHn64od4TUQhkdURxEvgbiAO2uZRfBIa5MyilCpTTW5zTT+wqVD205oXFi3fz7LOLqFGjDMuX90FEqFu3rB5FFCKZJgpjzEZgo4h8YYyJy8eYlCpYLh6xHgPKaZJwceTIBV58cSnffrsdgBIl/DhzJla73iiEctJGUUlExgL1gdQ7g4wxddwWlVIFiZfj36S43hQGVgd+U6asZeTIn7h48TLFi/syenR7nn++BT4+ek9EYZSTRDELeBN4D7gT6IvecKcKu5UvwsbJ1rRxNMmVqmFfPAVEcrKhbdtZ/PbbYQDuvbceEyd2pkqVUjZHptwpJ4ki0BizVETeM8bsBUaKyDrgVTfHplTe2jMPts0Ck4PfOXvnpZ0Xb6h6h1vC8iReXkLHjjU5dCiKyZO7cPfdde0OSeWDnCSKeBHxAvaKSARwBCjh3rCUymPndsO8e3O3jnhBxHHwLwNIkeluw5Uxhv/9bxs+Pl706FEfgKFDWzNoUCuCgorZHJ3KLzlJFC8BxbG67hgLlAKecGdQSuUpY+BLl2FC274HpWpmv15wHQgs5764Cri9e88yYMAili3bS7lygdx2W3XKlAnAz88HP+3ktUjJNlEYY/5yTF4E+gCISCV3BqVUnlr5AsSds6abDYGbBunVS1mIj09k3LjfGTv2F+LiEilTxp+xY2+jVKmi18utsmSZKESkGVAJ+NUYc1pEGmB15XEbUDkf4lMqd7Z9BhsnORugAU5utB4DK8Ctb2uSyMKqVQd45pmF7Nx5GoA+fRrx3nsdKV++uM2RKTtldWf2v4EewGasBuwFwADgHSAif8JTKhf2L4Elj2W8zMsXHlmnSSILSUnJDBhgJYm6dUOYNq0r7dtXtzssVQBkdURxD9DYGBMrIsHAYaChMWZf/oSmVAYuHIToY1eWmySYc6dzvsuXEOxyRU7Q9XofRAaSkw1xcYkEBvri7e3FtGld+fnngwwZ0ho/P+3AT1my+iTEGWNiAYwxZ0VklyYJZauz/8CnoWR7G0+XLyD0oXwJyZNt3XqCiIiF1KsXwief3ANA27bVaNu2mr2BqQInq0RRQ0RSeogVrPGyU3uMNcZ0d2tkSqW3dhypSaJii4zr1OgGob3zLSRPdOnSZUaPXs348X+SmJjM/v3nOHculjJlAuwOTRVQWSWKHunmJ7szEKWyFXfWeqzcBnqutjcWD/XDD/8wcOBiDh2KQgQGDAhn7NjbKV1ar2hSmcuqU8Af8zMQpTJ0fh+cWGdNRzs652v6vH3xeKjExGR69vyWOXN2ANCkyXXMmNGN5s31SneVPW2tUgWXSYbPb4T4qLTl3npHcG75+HhRqpQfQUHFGDOmPQMHNtcO/FSOuTVRiEhnYCLgDfzHGPN2BnUeBF7HOvm82RijJ5gVJCXAZ42dSaJ2d6u/peLXQZXb7I3NQ/z1VyQALVpYtzyNG9eB0aPbU7lySTvDUh4ox4lCRPyMMfG5qO8NTAE6AJHAWhGZb4zZ7lKnNjAcaG2MOSci5XMeuiqUki7D8v5w9HerfyaAWvfB3d/ZG5cHOX8+juHDVzBjxnrq1SvLpk0RFCvmTUiIjhOhrk62x54i0lxEtgK7HfONReTDHGy7ObDHGLPPGHMZmI11b4arp4ApxphzAMaYk7mKXhUuCZdg+dOw7f+cSaJiC7hHh2fPCWMMX365lXr1JjN9+nq8vb24++66JCXpyMXq2uTkiGIS0A34HsAYs1lE2udgvUpYN+mliATSX9NYB0BEfsM6PfW6MWZJDratCpPL0XAxEjZNtroBB+tO6h5LocJNtobmKXbvPsOAAYtYscK61al16xuYPr0bYWF6kK6uXU4ShZcx5mC6AdKT8nD/tYF2WH1H/SwiDY0x510riUh/oD9AlSpV8mjXqkBIjIdPakHMibTlj6yDco3sicnDJCQkcdttnxEZeYHg4ADeffcO+vZtipeXdlei8kZOEsVhEWkOGEe7w3PArhysdwS4wWW+sqPMVSTwlzEmAdgvIruwEsda10rGmI+AjwDCw8N1dD1PlxAL398FUfvh0nFIjLHKy9SFYkHQYYYmiRwwxiAi+Pp6M3bsbaxceYB3372DcuW0Az+Vt8RkM9qXo4F5EpAyvNcKYKAx5nQ26/lgJZTbsRLEWqC3MWabS53OwEPGmMdEpCywEWhijDmT2XbDw8PNunXrsn1iqoDaPB3+ngnH16Ytr3IHPLDcnpg8zIkT0fzrX8upUyeYV19ta3c4ykOIyHpjTPjVrJuTI4pEY0yv3G7YGJMoIgOBpVjtDzONMdtEZDSwzhgz37Gso4hsxzqdNTirJKE8XOSvsOIZ53yZ2tB9sTVdspotIXlRkgl5AAAgAElEQVSS5GTDxx+vZ9iwHzl/Po7Spf158cWWlCihowgp98rJEcVe4B/ga2COMeZifgSWGT2i8FAJsTDJ5fLM2z6EmndByar2xeRBNm8+TkTEQv7807o3onPnWkyZ0oUaNcrYHJnyFG49ojDG1BSRm4FewBsisgmYbYyZfTU7VEXEP9/A0icgKc6aT050Lus6G+r1tCcuD5OQkMTw4T8yYcKfJCUZKlYMYuLEztx/f31Ex9ZQ+SRHN9wZY34HfheR14EJwBdY90UoBae2wl9jITHOWbZ3XsZ1Qx/WJJELPj5ebNx4nORkw3PPNWfMmPY6JKnKd9kmChEJwrpRrhcQCswDbnZzXMqTrBoEh1ZkvOzu76DGXc55b9/8icmDHToURVJSMtWrl0FEmD69K1FR8YSHX293aKqIyskRxd/AD8C7xphf3ByP8kQpSaJ6F2j4lLM8IAQqtQbRzudyIiEhiYkT/2LUqFW0alWZ5cv7ICLUrh1id2iqiMtJoqhhjNE+AFTGTm5yTt8xVRunr9IffxwmImIhW7ZYNx4GBwcQE5NA8eLaU66yX6aJQkTeN8a8DHwnIldcGqUj3BUBq4dkfkopxcmNzukSetd8bp07F8uwYSv46KMNAFSvXpopU7pw5521bY5MKaesjii+djzqyHZFUcIlWDcu5/VbjQK9CidX4uMTadJkBocOReHr68XgwTczYkQbAgO1HUcVLFmNcLfGMRlqjEmTLBw30ukIeIXZts+c04+sz7qutx+E1HdvPIWQn58P/fo15ccf9zNtWlfq1y9nd0hKZSgnN9xtMMbcmK5sozGmqVsjy4TecJcPYk7DNMeXVslq8NR+W8MpLOLiEvn3v3+hbt2y9O7dELCGKPX2Fr0nQrmdW264E5GeWJfEVhcR1wEBSgDnM15LebwDy+C7Ts75zp/aF0shsnz5XgYMWMSePWcpX744991Xj4AAXx2OVHmErNoo1gBnsHp9neJSfhGr8z5V2BxbkzZJNHwKbmhnWziFwfHj0QwatJSvvvobgAYNyjF9ejcCArQdQnmOrNoo9gP7sXqLVUXBH284p2/7EJoOtC8WD5eUlMyMGet55ZUfiYqKJyDAh1Gj2vLSS60oVszb7vCUypWsTj2tNsa0FZFzgGtDhgDGGBPs9uiU+yTGg0k3/tTlC9bjjS9Co6fzP6ZCJCnJ8OGHa4iKiqdLl9pMnnwn1atrB37KM2V16illuNOy+RGIykdbZ8Ly/lcmihS1e2hXG1fh4sV4kpIMpUv7U6yYNx9/fBcnTkTTvXuoNlYrj5ZpS5rL3dg3AN7GmCSgFfA0oENoeaLkRFjaD5b1cyYJn4C0f8GhOrpcLhljmDNnB6GhU3j55aWp5bfcUoUePbSXV+X5ctKFx/dAMxGpCXwKLAC+BLq5MzDlBhsmWaPLpeixDKp1sC+eQuDAgfM899xiFiywRgf+++9TxMUl4u+fo46ZlfIIOfk0JxtjEkSkO/ChMWaSiOhVT54iOQnO74WkeFj9srP80c165HANEhKSGD/+D954YzWxsYmULOnHW2/dRkREON7eesmrKlxyNBSqiDwA9AHudZTpCWxP8cP9sOf7tGWPbNAkcQ1iYhJo2fI/bN16EoBevcIYP74jFSuWsDkypdwjJ4niCWAAVjfj+0SkOvCVe8NS1+z4Wlj2FJza7CwrU8dqqK5gy031hUZgoC/h4dcTE5PA1Kld6dixpt0hKeVW2XbhASAiPkAtx+weY0xiVvXdSbvwyMamqXD2H9g4yVnmXwaeOgjF9Bfv1TDG8Nlnm6lZM5hbbrF6yI2KiqNYMW+9cU55DLeOmS0itwL/BY5g3UNxnYj0Mcb8djU7VHkg7rx1xJBe1F748dm0ZTe+CK3HQLGg/ImtkNmx4xTPPLOQ1asPEhpalk2bIihWzFuHI1VFSk5OPX0AdDHGbAcQkVCsxHFVmUnlgbld4ejvWddpPwGKlYS6PcE3MH/iKkRiYxMYO/YX3n33NxISkilXLpDhw2/B11cbqlXRk5NEUSwlSQAYY3aIiA67ZZfEeGeSKFMbSqQbUU7E6qOp7gP5H1shsWTJHp59dhH79p0D4KmnbuTtt+8gODjA5siUskdOEsUGEZkOfO6YfxjtFNA+R13O+D32N3hrzs5L0dGX6dNnLqdPxxAWVp7p07vSurWO3KeKtpwkigjgeWCIY/4X4EO3RaTSMgYOLIHoo9b8mretRy9fTRJ5JCkpmeRkg6+vN0FBxZg4sTORkRd46aWW+PpqB35KZZkoRKQhUBOYa4x5N39CUmkcXwNzulxZHtY3/2MphNavP8rTTy/gnnvq8uqrbQFSBxVSSlmy6j32FaAfsAGrC4/RxpiZmdVXbrJ5mnM67Anr0ScQmg22J55C4sKFeF599ScmT15LcrLhwoV4hg27RY8glMpAVkcUDwONjDGXRKQcsAjQRJEfjIGlfeHUVji5wSqr2hE6fWJvXIWAMYZvv93OCy8s4dixaLy9hUGDWvLGG+01SSiViawSRbwx5hKAMeaUiOh1gfll7w+w7f/SljUfak8shcjFi/H07PktixfvAaBFi0pMn96NJk2uszkypQq2rBJFDZexsgWo6Tp2tjGmu1sjK2qiDkCM1XcQ8+5xlj+yDvyDoVR1W8IqTIKCihEfn0SpUn68/fYd9O9/E15e2gW4UtnJKlH0SDc/2Z2BFGnH18IXza8s7zwLKtyU7+EUJj//fJCKFYOoXTsEEWHmzLvx9/ehQgW9U12pnMpqzOwf8zOQIuvsrrRJ4rpm1mPp2hD6iD0xFQKnT8cwZMhyPv10E7ffXp3ly/sgIlStWtru0JTyODq6ip1O/w2fNXbOt30Pwl/OvL7KVnKyYdasTQwevJyzZ2MpVsybW2+tQlKSwcdHTzMpdTXc2kAtIp1F5B8R2SMiw7Ko10NEjIgUjf6jLkfDnnnwRQtIGXG20dOaJK7Rtm0nadduFv36zefs2Vhuv706W7c+w6hR7fDx0WsxlLpaOT6iEBE/Y0x8Lup7A1OADkAksFZE5rv2G+WoVwJ4Afgrp9v2eKsGwdaPnfO1e0DbcfbFUwhERcXRsuUnREdfpnz54owf35HevRvqeNVK5YFsf2aJSHMR2Qrsdsw3FpGcdOHRHGvsin3GmMvAbOCeDOqNAd4B4nIetge7cMglSQg0fgY6f6pjRVyllPFUSpXyZ+jQ1kRE3MTOnc/y8MONNEkolUdyckQxCegGfA9gjNksIu1zsF4l4LDLfCTQwrWCiNwI3GCMWSgimd5qLCL9gf4AVap4WAdtO76Avz8FHANEHfrJueypg1DyBlvC8nRHjlzghReWcM89denTx2rnGTHiVk0OSrlBThKFlzHmYLp/wKRr3bHjBr7xwOPZ1TXGfAR8BNYId9e673xzeBUsyuTKpUZPa5K4ComJyUyZsoaRI1cSHX2ZDRuO0bt3Q7y9vTRJKOUmOUkUh0WkOWAc7Q7PAbtysN4RwPWbsLKjLEUJIAxY5fgHvw6YLyJ3G2M8f6zThBj4rpNzvtOnUMLxcnj5wPWt7InLg61de4SIiIVs2HAMgHvvrcekSZ3x9taGaqXcKSeJ4hms009VgBPACkdZdtYCtUWkOlaC6AX0TllojIkCyqbMi8gq4F+FIkkAJMZB0mVrusPHEPa4reF4skuXLjN06AqmTl2LMVClSik+/PBO7r67rt2hKVUkZJsojDEnsb7kc8UYkygiA4GlgDcw0xizTURGA+uMMfNzHa0n+eMN69G/DDR60t5YPJyPjxcrVuzDy0sYNKgVo0a1pXhxHYtDqfySbaIQkY9JbYl1Msb0z25dY8wirF5nXctey6Ruu+y25zGiDsDGSdZ08ettDcVT7d17ltKl/QkJCcTPz4f//vc+/P19aNiwgt2hKVXk5OTk7grgR8ffb0B5IMf3UxQ5iXHw/V3O+V6/2heLB4qPT+TNN38mLGwaQ4euSC1v1qySJgmlbJKTU09fu86LyH8B/fbLzMoXrK45ABo8Bv7at1BOrVp1gGeeWcjOnacB6wqnpKRkbaxWymZX09dTdUB/2mUk8hfY8pE17RsEbXT02Jw4efISgwcv57PPNgNQt24I06Z1pX177VpdqYIgJ20U53C2UXgBZ4FM+20qciJ/tdojkhNhz1xn+ePbIbC8fXF5iNOnYwgNncLZs7H4+XkzYsStDBnSGj8/7a9SqYIiy/9GsW5waIzz/odkk9JngrIs6g0XD6ctaz9Jb6bLobJlA7nnnrpERl5g6tSu1KoVbHdISql0skwUxhgjIouMMWH5FZDHSE6C6CPOJNFkIFRpb51yuiEnPZwUTZcuXWb06NV07VqHNm2qAjB1alf8/Lz1zmqlCqicHN9vEpGmxpiNbo/Gk/yvHRxxadNvNx68fW0LxxP88MM/DBy4mEOHoli4cDdbtjyDl5fg76+nmZQqyDL9DxURH2NMItAUq4vwvcAlrPGzjTHmxnyKseBJiHUmCb9SUP9RTRJZOHw4ihdeWMLcuTsBaNr0OmbM6KbjVSvlIbL6KbcGuBG4O59i8Rwn1jqnnz0LopdvZiQxMZlJk/7itddWculSAkFBxXjzzfY8+2xzHUhIKQ+SVaIQAGPM3nyKxXMkOu43rHCTJoksXLgQz7///SuXLiXQo0coEyZ0pnLlknaHpZTKpawSRTkRGZTZQmPMeDfE4xn+Gms96mBDVzh/Po6AAB/8/HwIDg5gxoxu+Pl507VrHbtDU0pdpawShTcQhOPIokgzBuZ3hyO/WfOxp6zHMvrll8IYw1df/c1LLy1l4MBmvPpqWwC6dw+1OTKl1LXKKlEcM8aMzrdICqItH8PR3yHuHOydl3aZbxA01/sOAXbtOsOAAQv58cf9APz88yGMMXq5q1KFRLZtFEVSQizs+wGWp+sgt1hJ6LfbmvYtAb4B+R9bARIXl8g77/zKW2/9yuXLSQQHBzBuXAcef7yJJgmlCpGsEsXt+RZFQfPbq7D+fed8p5nWY6VbtVsOh+PHo2nT5lN27z4LwOOPN2HcuA6ULRtoc2RKqbyWaaIwxpzNz0AKjIMr0iaJzv8HDR61L54CqkKF4txwQyl8fLyYNq0rbdtWszskpZSb6C2xrs7thm87OOcf3wEh9eyLpwBJTjZ8/PF62revTp06IYgIX37ZnTJlAihWzNvu8JRSbqQ3AaSIPgazb3HO3zFdk4TD5s3Had16JhERCxkwYCEp/UJWqBCkSUKpIkCPKADizsMPD0DMSWu+dndo9JS9MRUA0dGXef31VUyY8CdJSYbrry9BRES43WEppfKZJopLx+E/NSAx1povGwZ3zCjyd1x///1OnntuMZGRF/DyEp57rjlvvnkbJUv62R2aUiqfFd1EYZJhQS/Y9Y2zrNKtcPd3EFjWvrgKgCNHLtCr17fExydx000VmT69G+Hh19sdllLKJkU3UWyenjZJhPWDTv+xLx6bJSQk4ePjhYhQqVJJxo69jWLFvBkwoJmOWa1UEVf0EoUxcGId/PisNe9dzBq2tFQNe+Oy0e+/HyYiYgGDB99Mnz6NAXj55ZttjkopVVAUvZ+KO7+CL5o75x9eB6VrQhG8k/js2ViefvoHWreeydatJ5k6dR060q1SKr2idUSxrD9s/dg5f8u/oVxD++KxiTGGzz/fwssvL+PUqRh8fb0YMqQ1I0bcql1vKKWuUHQSxbE1aZPEPfOgVtEbk+nEiWgeeug7Vq48AEDbtlWZNq0roaHl7A1MKVVgFY1EcWY7fNnCOT8wCvyK5gA6pUv7c+xYNGXLBvLeex149NHGehShlMpS0UgUs291Trd9r8glieXL93LjjRUJCQnEz8+Hb755gIoVgwgJ0Q78lFLZK/yN2QmXIM7Rv2H4v6DxAHvjyUfHjl3koYe+o2PHzxk6dEVqeVhYeU0SSqkcK/xHFJumOqfbjrMvjnyUlJTMjBnrGT78Ry5ciCcgwIe6dUN0MCGl1FUp3Ini9Db4eYg1HVQ07izesOEYERELWLv2KABdu9Zm8uQuVKtW2ubIlFKeqnAnioW9nNN3fm5fHPnkwIHzNG/+MUlJhkqVSjBp0p3cd189PYpQSl0TtyYKEekMTAS8gf8YY95Ot3wQ8CSQCJwCnjDGHMyTnSfEwOm/renGEXBD2zzZbEFWrVpp+vZtQokSfrzxRjtKlNAO/JRS185tjdki4g1MAe4E6gMPiUj9dNU2AuHGmEbAt8C7eRaA63jX7ScVyt5gDxw4z113fcXq1QdSyz766C7Gj++kSUIplWfceUTRHNhjjNkHICKzgXuA7SkVjDErXer/CTySJ3tOiIEdX1jT17cGb9882WxBkZCQxPjxf/DGG6uJjU3k9OkY/vijH4CeZlJK5Tl3JopKwGGX+UigRSZ1AfoBizNaICL9gf4AVapUyX7P/3ztnO6+MPv6HuTXXw8REbGAbdtOAdCrVxjjx3e0OSqlVGFWIBqzReQRIBzIsCHBGPMR8BFAeHh49r3W/fqK9Vi8IviVyqswbXXuXCyDBy/nk082AlCzZhmmTu1Kx441bY5MKVXYuTNRHAFucJmv7ChLQ0TuAEYAbY0x8de811UvW6PWAbQadc2bKyiSkw3z5v2Dr68Xw4bdwvDhtxAQULhOqSmlCiZ3Joq1QG0RqY6VIHoBvV0riEhTYAbQ2Rhz8pr29s83sOUjOOS8A5n6edPkYZedO09TvXpp/Px8CAkJ5IsvulOlSinq1SvaI/AppfKX2y4FMsYkAgOBpcAO4H/GmG0iMlpEUrptHQcEAd+IyCYRmX/VO1zwoDNJiDdEHAPf4tfyFGwTE5PAiBE/0qjRNN5997fU8o4da2qSUErlO7e2URhjFgGL0pW95jJ9R57s6Nxu5/TtU6FaRyh+XZ5sOr8tWbKHAQMWsn//eQBOn46xOSKlVFFXIBqzr1lKmwRAk2fsi+MaHD16kRdfXMI331hXDzdsWJ7p07tx8803ZLOmUkq5V+FIFCc3WI9VO9gbx1XatesM4eEfcfHiZQIDfXn99ba8+GJLfH297Q5NKaUKSaLYu8B6vJA3vX/kt9q1g2nWrBLFi/vy4Yd3UrWqduCnlCo4PD9RJMQ6G7HrPmhvLDl04UI8r722kgEDmlGnTggiwvz5vShevJjdoSml1BU8P1Ese9I5HdbPvjhywBjDt99u54UXlnDsWDQ7d55myRLrEl5NEkqpgsrzE8VFRy8h17eGUtVsDSUr+/adY+DARSxevAeAli0r8847eXPRl1JKuZNnJ4rkJDjyizV961v2xpKJy5eTeO+93xkz5mfi4hIpXdqft9++naeeugkvL+3ATylV8Hl2oki52gkgsIJ9cWTh8OEoRo9eTXx8Eg8/3JD33+9IhQpBdoellFI55tmJ4swO53RwXfviSOfcuVhKl/ZHRKhZM5iJEztTq1Ywt99ew+7QlFIq1zx7NB8ff+uxTG1743BITjbMnLmRWrU+5PPPt6SWP/10uCYJpZTH8uxEcWK99Vj+RnvjALZtO0m7drPo128+Z8/GpjZaK6WUp/PsU0+JsdbjxcNZ13OjmJgExoxZzXvv/UFiYjLlyxfngw868dBDYbbFpJRSecmzE4VxjGFUu7stu9+16wydOn3OgQPnEYGIiJt4663bKVMmwJZ4lFLKHTw7Uez8yno0ybbsvmrVUvj7+9C4cQWmT+9Gy5aVbYlD2SshIYHIyEji4uLsDkUp/P39qVy5Mr6+eTewmecmiqgDEHfGms6nLsUTE5OZPn0dDz0URkhIIH5+PixZ8jCVKpXEx8ezm3vU1YuMjKREiRJUq1YNEb03RtnHGMOZM2eIjIykevXqebZdz/122zjJOV2tk9t3t2bNEZo3/5jnnlvM0KHOUfSqVi2tSaKIi4uLIyQkRJOEsp2IEBISkudHt555RGEMrJ9gTVftAIHl3barqKg4Roz4ialT12IMVKlSinvuKTj3bKiCQZOEKijc8Vn0zERx5BfA0ZDdOMItuzDG8PXX23jppaUcPx6Nj48Xgwa15LXX2moHfkqpIsUzz5l83dY5Xb2rW3axefMJHnroO44fj+bmm29gw4b+vPNOB00SqkDy9vamSZMmhIWFcdddd3H+/PnUZdu2beO2226jbt261K5dmzFjxmBSrhgEFi9eTHh4OPXr16dp06a8/PLLdjyFLG3cuJF+/Qp279D//ve/qVWrFnXr1mXp0qUZ1jHGMGLECOrUqUNoaCiTJlmn0MeNG0eTJk1S30Nvb2/Onj0LwMSJEwkLC6NBgwZMmDAhdVv/+te/+Omnn9z/xFIC96S/mxrVM+Y9rL8tn5i8lJiYlGb+pZeWmI8/Xm+SkpLzdD+qcNm+fbvdIZjixYunTj/66KPmzTffNMYYExMTY2rUqGGWLl1qjDHm0qVLpnPnzmby5MnGGGO2bt1qatSoYXbs2GGMMSYxMdFMnTo1T2NLSEi45m3cf//9ZtOmTfm6z9zYtm2badSokYmLizP79u0zNWrUMImJiVfUmzlzpunTp49JSrK+a06cOHFFnfnz55v27dsbY6z3p0GDBubSpUsmISHB3H777Wb37t3GGGMOHDhgOnTokGE8GX0mgXXmKr93Pe/Uk+v42A2fyLPNrly5nwEDFjFjRjfatKkKwPjx7m8kV4XM+25qq3jZZF/HoVWrVmzZYnUh8+WXX9K6dWs6duwIQGBgIJMnT6Zdu3Y8++yzvPvuu4wYMYJ69eoB1pHJM89cOe58dHQ0zz33HOvWrUNEGDVqFD169CAoKIjo6GgAvv32WxYsWMCsWbN4/PHH8ff3Z+PGjbRu3Zo5c+awadMmSpe2Rm+sXbs2v/76K15eXkRERHDo0CEAJkyYQOvWrdPs++LFi2zZsoXGjRsDsGbNGl544QXi4uIICAjg008/pW7dusyaNYs5c+YQHR1NUlISq1evZty4cfzvf/8jPj6e++67jzfeeAOAe++9l8OHDxMXF8cLL7xA//79c/z6ZmTevHn06tULPz8/qlevTq1atVizZg2tWrVKU2/atGl8+eWXeHlZJ3PKl7+yffWrr77ioYceAmDHjh20aNGCwMBAANq2bcucOXMYMmQIVatW5cyZMxw/fpzrrnPvlZ+elyjiHYfUTZ/Pk82dPHmJwYOX89lnmwEYP/6P1EShlKdJSkrixx9/TD1Ns23bNm666aY0dWrWrEl0dDQXLlzg77//ztGppjFjxlCqVCm2bt0KwLlz57JdJzIykt9//x1vb2+SkpKYO3cuffv25a+//qJq1apUqFCB3r1789JLL3HLLbdw6NAhOnXqxI4dO9JsZ926dYSFOXs6qFevHr/88gs+Pj6sWLGCV155he+++w6ADRs2sGXLFoKDg1m2bBm7d+9mzZo1GGO4++67+fnnn2nTpg0zZ84kODiY2NhYmjVrRo8ePQgJCUmz35deeomVK1de8bx69erFsGHD0pQdOXKEli1bps5XrlyZI0eOXLHu3r17+frrr5k7dy7lypVj0qRJ1K7t7KsuJiaGJUuWMHnyZADCwsIYMWIEZ86cISAggEWLFhEeHp5a/8Ybb+S3336jR48e2b4f18LzEkWKpgOvafXkZMMnn2xg6NAVnDsXh5+fNyNHtmHw4JvzKEBVJOXil39eio2NpUmTJhw5coTQ0FA6dOiQp9tfsWIFs2fPTp0vU6ZMtus88MADeHt7A9CzZ09Gjx5N3759mT17Nj179kzd7vbt21PXuXDhAtHR0QQFObviP3bsGOXKlUudj4qK4rHHHmP37t2ICAkJCanLOnToQHBwMADLli1j2bJlNG3aFLCOinbv3k2bNm2YNGkSc+fOBeDw4cPs3r37ikTxwQcf5OzFyYX4+Hj8/f1Zt24dc+bM4YknnuCXX35JXf7DDz/QunXr1OcQGhrK0KFD6dixI8WLF6dJkyaprylYRyRHjx7N8zjT89xEUbrWVa+6f/85HnlkLr//bvUR1bFjTaZM6UKtWsF5FZ1S+SogIIBNmzYRExNDp06dmDJlCs8//zz169fn559/TlN33759BAUFUbJkSRo0aMD69etTT+vkluulmOmv3S9evHjqdKtWrdizZw+nTp3i+++/Z+TIkQAkJyfz559/4u/vn+Vzc932q6++Svv27Zk7dy4HDhygXbt2Ge7TGMPw4cN5+umn02xv1apVrFixgj/++IPAwEDatWuX4X0HuTmiqFSpEocPO/uci4yMpFKlSlesW7lyZbp3t7ocuu++++jbt2+a5bNnz0497ZSiX79+qUeIr7zyCpUrO3uASDn95m6eedVTQFm4hmuFS5b0Y9euM1x3XRCzZ/dgyZKHNUmoQiEwMJBJkybx/vvvk5iYyMMPP8yvv/7KihXWTaKxsbE8//zzDBkyBIDBgwfz1ltvsWvXLsD64p4+ffoV2+3QoQNTpkxJnU859VShQgV27NhBcnJy6i/0jIgI9913H4MGDSI0NDT113vHjh358MMPU+tt2rTpinVDQ0PZs8fZG3NUVFTql/CsWbMy3WenTp2YOXNmahvKkSNHOHnyJFFRUZQpU4bAwEB27tzJn3/+meH6H3zwAZs2bbriL32SALj77ruZPXs28fHx7N+/n927d9O8efMr6t17772pyWf16tXUqVMnzfNavXo199xzT5p1Tp48CcChQ4eYM2cOvXv3Tl22a9euNKfl3MUzE0Xs6VyvsnTpHuLjEwEICQlk/vxe7Nz5LD17hunNUqpQadq0KY0aNeKrr74iICCAefPm8eabb1K3bl0aNmxIs2bNGDjQOnXbqFEjJkyYwEMPPURoaChhYWHs27fvim2OHDmSc+fOERYWRuPGjVO/7N5++226devGzTffTMWKFbOMq2fPnnz++eepp50AJk2axLp162jUqBH169fPMEnVq1ePqKgoLl68CMCQIUMYPnw4TZs2JTExMdP9dezYkc50zocAAAp0SURBVN69e9OqVSsaNmzI/fffz8WLF+ncuTOJiYmEhoYybNiwNG0LV6tBgwY8+OCD1K9fn86dOzNlypTUU0RdunRJPT00bNgwvvvuOxo2bMjw4cP5z3/+k7qNuXPnpp5ictWjRw/q16/PXXfdxZQpU1IvCEhISGDPnj1p2izcRYyx55zq1Qq/Qcy6V8rDMydyVP/w4Sief34J33+/kzFj2jNyZBs3R6iKmh07dhAaGmp3GIXaBx98QIkSJXjyySftDqXAmDt3Lhs2bGDMmDFXLMvoMyki640xV5VVPPOIomz2h1qJicmMH/8HoaFT+P77nQQFFSM4WLv/VsoTPfPMM/j5+dkdRoGSmJiYbzdHemZjdjYDFf35ZyQREQvYvNk66ujRI5SJEztTqVLJ/IhOKZXH/P396dOnj91hFCgPPPBAvu3LMxNFrfsyXfTXX5HcfPMnGPP/7d17kNVlHcfx94ebC8lAylh4a3W4CCq3CClnNEQFhWAqhssACoOZFDVi9kcDTXQZpTGZSdFWSgdoRAyK2owihxCCYbmU3CQVRMbZYoKQmEZpS/j2x/Os57ScPee3657rfl8zO3PO7/wu3/3OOec5z/Oc832guronS5fewbhx/Zrd37m2YGY+1+VKQj6mE8qzoaD5F+SIEZcxZkwfhg79KAsX3kS3bm23eIdzmVRVVXHy5EkvNe6KzuJ6FNm+btwa5TmZvbMOet8AwKFDJ5k/fwNLloyhX7/wlbtz54wOHfwF6wrDV7hzpaS5Fe4+yGR2efYoul9JQ8N7LF68lYcf3kpDw1mqqjqxdu1kAG8kXEF17ty5TVcTc67U5PVbT5LGSnpN0mFJ5/1KRdIFkp6Pj++QVJ3kvBs3H2PQoBoWLdpMQ8NZZs8eQk3N+LYO3znnHHnsUUjqCDwB3AbUA7sk1ZrZwbTd5gCnzKyPpKnA94Ep558t5c23e3Lr+F8DMGBAL2pqxnsRP+ecy6N89ihGAIfN7IiZ/QdYDUxsss9EYEW8vRYYrRyzgafOdKWqqhMPPXQLe/bc542Ec87lWd4msyVNAsaa2T3x/kzgBjObl7bPgbhPfbz/RtznH03OdS/QWDD+OuBAXoIuP72AltczqUyeixTPRYrnIqW/mXVvzYFlMZltZsuAZQCSdrd25r7SeC5SPBcpnosUz0WKpN2tPTafQ09/Ba5Iu3953JZxH0mdgB7AyTzG5JxzroXy2VDsAvpKukpSF2AqUNtkn1rg7nh7EvAHK7cfdjjnXIXL29CTmb0naR6wAegIPGNmr0j6DmGR71rgaeCnkg4DbxMak1yW5SvmMuS5SPFcpHguUjwXKa3ORdn9Mts551xhlWeZceeccwXjDYVzzrmsSrahyFf5j3KUIBcPSDooaZ+kjZIq9leIuXKRtt/nJZmkiv1qZJJcSJocnxuvSFpV6BgLJcFr5EpJmyS9HF8ndxYjznyT9Iyk4/E3apkel6THYp72SRqW6MRmVnJ/hMnvN4CrgS7AXmBgk32+BNTE21OB54sddxFzMQroFm/Pbc+5iPt1B7YAdcDwYsddxOdFX+Bl4MPx/iXFjruIuVgGzI23BwJHix13nnJxEzAMONDM43cCvyWs1TAS2JHkvKXao8hL+Y8ylTMXZrbJzN6Nd+sIv1mpREmeFwDfJdQNq+S630ly8QXgCTM7BWBmxwscY6EkyYUBjUtc9gD+VsD4CsbMthC+QdqcicBKC+qAnpJ65zpvqTYUlwHp653Wx20Z9zGz94DTwMUFia6wkuQi3RzCJ4ZKlDMXsSt9hZn9ppCBFUGS50U/oJ+kbZLqJI0tWHSFlSQXi4AZkuqB9cBXChNayWnp+wlQJiU8XDKSZgDDgZuLHUsxSOoALAFmFTmUUtGJMPz0aUIvc4uk683sn0WNqjimAcvN7FFJnyT8fus6MztX7MDKQan2KLz8R0qSXCDpVmABMMHMGgoUW6HlykV3QtHIlyQdJYzB1lbohHaS50U9UGtm/zWzN4HXCQ1HpUmSiznAzwDMbDtQRSgY2N4kej9pqlQbCi//kZIzF5KGAk8RGolKHYeGHLkws9Nm1svMqs2smjBfM8HMWl0MrYQleY38ktCbQFIvwlDUkUIGWSBJcvEWMBpA0gBCQ3GioFGWhlrgrvjtp5HAaTM7luugkhx6svyV/yg7CXPxCHAhsCbO579lZhOKFnSeJMxFu5AwFxuA2yUdBM4CXzeziut1J8zF14AfS5pPmNieVYkfLCU9R/hw0CvOx3wL6AxgZjWE+Zk7gcPAu8DsROetwFw555xrQ6U69OScc65EeEPhnHMuK28onHPOZeUNhXPOuay8oXDOOZeVNxSu5Eg6K2lP2l91ln2rm6uU2cJrvhSrj+6NJS/6t+Ic90m6K96eJenStMd+ImlgG8e5S9KQBMfcL6nbB722a7+8oXCl6IyZDUn7O1qg6043s8GEYpOPtPRgM6sxs5Xx7izg0rTH7jGzg20SZSrOJ0kW5/2ANxSu1byhcGUh9hz+KOnP8e9TGfa5VtLO2AvZJ6lv3D4jbftTkjrmuNwWoE88dnRcw2B/rPV/Qdy+WKk1QH4Qty2S9KCkSYSaW8/Ga3aNPYHhsdfx/pt77HksbWWc20kr6CbpR5J2K6w98e247auEBmuTpE1x2+2Stsc8rpF0YY7ruHbOGwpXirqmDTuti9uOA7eZ2TBgCvBYhuPuA35oZkMIb9T1sVzDFODGuP0sMD3H9T8D7JdUBSwHppjZ9YRKBnMlXQx8FrjWzAYB30s/2MzWArsJn/yHmNmZtId/Ho9tNAVY3co4xxLKdDRaYGbDgUHAzZIGmdljhJLao8xsVCzlsRC4NeZyN/BAjuu4dq4kS3i4du9MfLNM1xlYGsfkzxLqFjW1HVgg6XLgF2Z2SNJo4OPArljepCuh0cnkWUlngKOEMtT9gTfN7PX4+Argy8BSwloXT0t6AXgh6T9mZickHYl1dg4B1wDb4nlbEmcXQtmW9DxNlnQv4XXdm7BAz74mx46M27fF63Qh5M25ZnlD4crFfODvwGBCT/i8RYnMbJWkHcA4YL2kLxJW8lphZt9IcI3p6QUEJV2UaadYW2gEocjcJGAecEsL/pfVwGTgVWCdmZnCu3biOIE/EeYnHgc+J+kq4EHgE2Z2StJyQuG7pgS8aGbTWhCva+d86MmVix7Asbh+wExC8bf/I+lq4EgcbvkVYQhmIzBJ0iVxn4uUfE3x14BqSX3i/ZnA5jim38PM1hMasMEZjv0Xoex5JusIK41NIzQatDTOWNDum8BISdcQVm97Bzgt6SPAHc3EUgfc2Pg/SfqQpEy9M+fe5w2FKxdPAndL2ksYrnknwz6TgQOS9hDWpVgZv2m0EPi9pH3Ai4RhmZzM7N+E6pprJO0HzgE1hDfdF+L5tpJ5jH85UNM4md3kvKeAvwAfM7OdcVuL44xzH48SqsLuJayP/SqwijCc1WgZ8DtJm8zsBOEbWc/F62wn5NO5Znn1WOecc1l5j8I551xW3lA455zLyhsK55xzWXlD4ZxzLitvKJxzzmXlDYVzzrmsvKFwzjmX1f8A67DgxuXalvUAAAAASUVORK5CYII=\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "plt.figure()\n",
    "plt.plot(fpr, tpr, color='darkorange', lw=2, label='ROC curve (area = %0.3f)' % roc_auc)\n",
    "plt.plot([0, 1], [0, 1], color='navy', lw=2, linestyle='--')\n",
    "plt.xlim([0.0, 1.0])\n",
    "plt.ylim([0.0, 1.05])\n",
    "plt.xlabel('False Positive Rate')\n",
    "plt.ylabel('True Positive Rate')\n",
    "plt.title('RoC curve for abnormal class detection')\n",
    "plt.legend(loc=\"lower right\")\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 50,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Experiment split 0, normal classes [312  92 260 181]\n",
      "Iteration 0 (Epoch: 0), loss is 1.4863206148147583\n",
      "Iteration 10 (Epoch: 0), loss is 1.198142647743225\n",
      "Iteration 20 (Epoch: 0), loss is 1.000427484512329\n",
      "Iteration 30 (Epoch: 0), loss is 0.8451233506202698\n",
      "Iteration 40 (Epoch: 0), loss is 0.6721122860908508\n",
      "Iteration 0 (Epoch: 1), loss is 0.7509787678718567\n",
      "Iteration 10 (Epoch: 1), loss is 0.6556546092033386\n",
      "Iteration 20 (Epoch: 1), loss is 0.5348088145256042\n",
      "Iteration 30 (Epoch: 1), loss is 0.5415209531784058\n",
      "Iteration 40 (Epoch: 1), loss is 0.5374873280525208\n",
      "Iteration 0 (Epoch: 2), loss is 0.4417767822742462\n",
      "Iteration 10 (Epoch: 2), loss is 0.422802209854126\n",
      "Iteration 20 (Epoch: 2), loss is 0.44954001903533936\n",
      "Iteration 30 (Epoch: 2), loss is 0.49613872170448303\n",
      "Iteration 40 (Epoch: 2), loss is 0.33806145191192627\n",
      "Done\n",
      "Correct predictions\n",
      " tensor([0, 1, 1, 1, 0, 1, 1, 1, 0, 1, 1, 1, 1, 0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0,\n",
      "        1, 1, 1, 1, 1, 1, 1, 1], device='cuda:0', dtype=torch.uint8)\n",
      "Iteration 0\n",
      "Iteration 50\n",
      "Iteration 100\n",
      "Iteration 150\n",
      "Iteration 200\n",
      "Iteration 250\n",
      "Iteration 300\n",
      "Iteration 350\n",
      "Iteration 400\n",
      "Iteration 450\n",
      "Iteration 500\n",
      "Exception ignored in: <bound method _DataLoaderIter.__del__ of <torch.utils.data.dataloader._DataLoaderIter object at 0x7f414f632b38>>\n",
      "Traceback (most recent call last):\n",
      "  File \"/usr/local/lib/python3.5/dist-packages/torch/utils/data/dataloader.py\", line 399, in __del__\n",
      "    self._shutdown_workers()\n",
      "  File \"/usr/local/lib/python3.5/dist-packages/torch/utils/data/dataloader.py\", line 378, in _shutdown_workers\n",
      "    self.worker_result_queue.get()\n",
      "  File \"/usr/lib/python3.5/multiprocessing/queues.py\", line 345, in get\n",
      "    return ForkingPickler.loads(res)\n",
      "  File \"/usr/local/lib/python3.5/dist-packages/torch/multiprocessing/reductions.py\", line 151, in rebuild_storage_fd\n",
      "    fd = df.detach()\n",
      "  File \"/usr/lib/python3.5/multiprocessing/resource_sharer.py\", line 58, in detach\n",
      "    return reduction.recv_handle(conn)\n",
      "  File \"/usr/lib/python3.5/multiprocessing/reduction.py\", line 181, in recv_handle\n",
      "    return recvfds(s, 1)[0]\n",
      "  File \"/usr/lib/python3.5/multiprocessing/reduction.py\", line 152, in recvfds\n",
      "    msg, ancdata, flags, addr = sock.recvmsg(1, socket.CMSG_LEN(bytes_size))\n",
      "ConnectionResetError: [Errno 104] Connection reset by peer\n",
      "Experiment split 1, normal classes [374 321 228 204]\n",
      "Iteration 0 (Epoch: 0), loss is 1.2285661697387695\n",
      "Iteration 10 (Epoch: 0), loss is 1.0150657892227173\n",
      "Iteration 20 (Epoch: 0), loss is 0.9610595703125\n",
      "Iteration 30 (Epoch: 0), loss is 0.8770454525947571\n",
      "Iteration 40 (Epoch: 0), loss is 0.6950671672821045\n",
      "Iteration 50 (Epoch: 0), loss is 0.7338578701019287\n",
      "Iteration 60 (Epoch: 0), loss is 0.7541520595550537\n",
      "Iteration 70 (Epoch: 0), loss is 0.835045576095581\n",
      "Iteration 0 (Epoch: 1), loss is 0.5409736633300781\n",
      "Iteration 10 (Epoch: 1), loss is 0.6473339796066284\n",
      "Iteration 20 (Epoch: 1), loss is 0.6194311380386353\n",
      "Iteration 30 (Epoch: 1), loss is 0.4744812250137329\n",
      "Iteration 40 (Epoch: 1), loss is 0.6890513300895691\n",
      "Iteration 50 (Epoch: 1), loss is 0.5159334540367126\n",
      "Iteration 60 (Epoch: 1), loss is 0.5110463500022888\n",
      "Iteration 70 (Epoch: 1), loss is 0.5898869037628174\n",
      "Iteration 0 (Epoch: 2), loss is 0.4372919797897339\n",
      "Iteration 10 (Epoch: 2), loss is 0.5350305438041687\n",
      "Iteration 20 (Epoch: 2), loss is 0.4176807403564453\n",
      "Iteration 30 (Epoch: 2), loss is 0.5439883470535278\n",
      "Iteration 40 (Epoch: 2), loss is 0.5260605812072754\n",
      "Iteration 50 (Epoch: 2), loss is 0.6155232191085815\n",
      "Iteration 60 (Epoch: 2), loss is 0.4041816294193268\n",
      "Iteration 70 (Epoch: 2), loss is 0.4368404150009155\n",
      "Done\n",
      "Correct predictions\n",
      " tensor([1, 1, 1, 0, 1, 1, 0, 0, 1, 1, 0, 1, 1, 0, 1, 1, 1, 1, 0, 1, 1, 1, 1, 0,\n",
      "        1, 1, 1, 1, 0, 1, 0, 1], device='cuda:0', dtype=torch.uint8)\n",
      "Iteration 0\n",
      "Iteration 50\n",
      "Iteration 100\n",
      "Iteration 150\n",
      "Iteration 200\n",
      "Iteration 250\n",
      "Iteration 300\n",
      "Iteration 350\n",
      "Iteration 400\n",
      "Iteration 450\n",
      "Iteration 500\n",
      "Experiment split 2, normal classes [379 193 199 383]\n",
      "Iteration 0 (Epoch: 0), loss is 1.3718746900558472\n",
      "Iteration 10 (Epoch: 0), loss is 1.2203282117843628\n",
      "Iteration 20 (Epoch: 0), loss is 1.0142853260040283\n",
      "Iteration 30 (Epoch: 0), loss is 0.9500205516815186\n",
      "Iteration 40 (Epoch: 0), loss is 0.851035475730896\n",
      "Iteration 50 (Epoch: 0), loss is 0.8032236099243164\n",
      "Iteration 60 (Epoch: 0), loss is 0.6888030767440796\n",
      "Iteration 70 (Epoch: 0), loss is 0.7341716289520264\n",
      "Iteration 80 (Epoch: 0), loss is 0.7515078783035278\n",
      "Iteration 90 (Epoch: 0), loss is 0.5910401344299316\n",
      "Iteration 100 (Epoch: 0), loss is 0.7095517516136169\n",
      "Iteration 0 (Epoch: 1), loss is 0.5787824392318726\n",
      "Iteration 10 (Epoch: 1), loss is 0.5630432367324829\n",
      "Iteration 20 (Epoch: 1), loss is 0.6100700497627258\n",
      "Iteration 30 (Epoch: 1), loss is 0.6399298310279846\n",
      "Iteration 40 (Epoch: 1), loss is 0.5900307297706604\n",
      "Iteration 50 (Epoch: 1), loss is 0.5924573540687561\n",
      "Iteration 60 (Epoch: 1), loss is 0.5459948182106018\n",
      "Iteration 70 (Epoch: 1), loss is 0.5440285205841064\n",
      "Iteration 80 (Epoch: 1), loss is 0.44778403639793396\n",
      "Iteration 90 (Epoch: 1), loss is 0.48577404022216797\n",
      "Iteration 100 (Epoch: 1), loss is 0.5570332407951355\n",
      "Iteration 0 (Epoch: 2), loss is 0.5239275693893433\n",
      "Iteration 10 (Epoch: 2), loss is 0.4675913453102112\n",
      "Iteration 20 (Epoch: 2), loss is 0.3909758925437927\n",
      "Iteration 30 (Epoch: 2), loss is 0.43266329169273376\n",
      "Iteration 40 (Epoch: 2), loss is 0.46221983432769775\n",
      "Iteration 50 (Epoch: 2), loss is 0.4541403353214264\n",
      "Iteration 60 (Epoch: 2), loss is 0.5446707606315613\n",
      "Iteration 70 (Epoch: 2), loss is 0.48889875411987305\n",
      "Iteration 80 (Epoch: 2), loss is 0.5293354392051697\n",
      "Iteration 90 (Epoch: 2), loss is 0.5111904144287109\n",
      "Iteration 100 (Epoch: 2), loss is 0.38114139437675476\n",
      "Done\n",
      "Correct predictions\n",
      " tensor([0, 1, 1, 1, 1, 1, 1, 0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,\n",
      "        1, 1, 1, 1, 1, 1, 1, 1], device='cuda:0', dtype=torch.uint8)\n",
      "Iteration 0\n",
      "Iteration 50\n",
      "Iteration 100\n",
      "Iteration 150\n",
      "Iteration 200\n",
      "Iteration 250\n",
      "Iteration 300\n",
      "Iteration 350\n",
      "Iteration 400\n",
      "Iteration 450\n",
      "Iteration 500\n",
      "Exception ignored in: <bound method _DataLoaderIter.__del__ of <torch.utils.data.dataloader._DataLoaderIter object at 0x7f414d144518>>\n",
      "Traceback (most recent call last):\n",
      "  File \"/usr/local/lib/python3.5/dist-packages/torch/utils/data/dataloader.py\", line 399, in __del__\n",
      "    self._shutdown_workers()\n",
      "  File \"/usr/local/lib/python3.5/dist-packages/torch/utils/data/dataloader.py\", line 378, in _shutdown_workers\n",
      "    self.worker_result_queue.get()\n",
      "  File \"/usr/lib/python3.5/multiprocessing/queues.py\", line 345, in get\n",
      "    return ForkingPickler.loads(res)\n",
      "  File \"/usr/local/lib/python3.5/dist-packages/torch/multiprocessing/reductions.py\", line 151, in rebuild_storage_fd\n",
      "    fd = df.detach()\n",
      "  File \"/usr/lib/python3.5/multiprocessing/resource_sharer.py\", line 58, in detach\n",
      "    return reduction.recv_handle(conn)\n",
      "  File \"/usr/lib/python3.5/multiprocessing/reduction.py\", line 181, in recv_handle\n",
      "    return recvfds(s, 1)[0]\n",
      "  File \"/usr/lib/python3.5/multiprocessing/reduction.py\", line 152, in recvfds\n",
      "    msg, ancdata, flags, addr = sock.recvmsg(1, socket.CMSG_LEN(bytes_size))\n",
      "ConnectionResetError: [Errno 104] Connection reset by peer\n",
      "Experiment split 3, normal classes [ 16 178 130 136]\n",
      "Iteration 0 (Epoch: 0), loss is 1.506719946861267\n",
      "Iteration 10 (Epoch: 0), loss is 1.276674509048462\n",
      "Iteration 20 (Epoch: 0), loss is 1.1699599027633667\n",
      "Iteration 30 (Epoch: 0), loss is 0.8933062553405762\n",
      "Iteration 40 (Epoch: 0), loss is 0.7654660940170288\n",
      "Iteration 50 (Epoch: 0), loss is 0.6356047987937927\n",
      "Iteration 60 (Epoch: 0), loss is 0.6580831408500671\n",
      "Iteration 70 (Epoch: 0), loss is 0.5966483950614929\n",
      "Iteration 0 (Epoch: 1), loss is 0.5721805691719055\n",
      "Iteration 10 (Epoch: 1), loss is 0.49271532893180847\n",
      "Iteration 20 (Epoch: 1), loss is 0.41319847106933594\n",
      "Iteration 30 (Epoch: 1), loss is 0.5750640034675598\n",
      "Iteration 40 (Epoch: 1), loss is 0.5458282232284546\n",
      "Iteration 50 (Epoch: 1), loss is 0.43138134479522705\n",
      "Iteration 60 (Epoch: 1), loss is 0.3528435528278351\n",
      "Iteration 70 (Epoch: 1), loss is 0.3360084295272827\n",
      "Iteration 0 (Epoch: 2), loss is 0.5209528803825378\n",
      "Iteration 10 (Epoch: 2), loss is 0.39063119888305664\n",
      "Iteration 20 (Epoch: 2), loss is 0.36638930439949036\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Iteration 30 (Epoch: 2), loss is 0.3444449007511139\n",
      "Iteration 40 (Epoch: 2), loss is 0.3522026240825653\n",
      "Iteration 50 (Epoch: 2), loss is 0.344154417514801\n",
      "Iteration 60 (Epoch: 2), loss is 0.32283246517181396\n",
      "Iteration 70 (Epoch: 2), loss is 0.5324721336364746\n",
      "Done\n",
      "Correct predictions\n",
      " tensor([1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 1, 1, 1, 1, 1, 1, 1, 0, 1, 1, 1, 1, 1, 1,\n",
      "        1, 1, 1, 1, 1, 1, 1, 1], device='cuda:0', dtype=torch.uint8)\n",
      "Iteration 0\n",
      "Iteration 50\n",
      "Iteration 100\n",
      "Iteration 150\n",
      "Iteration 200\n",
      "Iteration 250\n",
      "Iteration 300\n",
      "Iteration 350\n",
      "Iteration 400\n",
      "Iteration 450\n",
      "Iteration 500\n",
      "Exception ignored in: <bound method _DataLoaderIter.__del__ of <torch.utils.data.dataloader._DataLoaderIter object at 0x7f41cc5e72e8>>\n",
      "Traceback (most recent call last):\n",
      "  File \"/usr/local/lib/python3.5/dist-packages/torch/utils/data/dataloader.py\", line 399, in __del__\n",
      "    self._shutdown_workers()\n",
      "  File \"/usr/local/lib/python3.5/dist-packages/torch/utils/data/dataloader.py\", line 378, in _shutdown_workers\n",
      "    self.worker_result_queue.get()\n",
      "  File \"/usr/lib/python3.5/multiprocessing/queues.py\", line 345, in get\n",
      "    return ForkingPickler.loads(res)\n",
      "  File \"/usr/local/lib/python3.5/dist-packages/torch/multiprocessing/reductions.py\", line 151, in rebuild_storage_fd\n",
      "    fd = df.detach()\n",
      "  File \"/usr/lib/python3.5/multiprocessing/resource_sharer.py\", line 57, in detach\n",
      "    with _resource_sharer.get_connection(self._id) as conn:\n",
      "  File \"/usr/lib/python3.5/multiprocessing/resource_sharer.py\", line 87, in get_connection\n",
      "    c = Client(address, authkey=process.current_process().authkey)\n",
      "  File \"/usr/lib/python3.5/multiprocessing/connection.py\", line 494, in Client\n",
      "    deliver_challenge(c, authkey)\n",
      "  File \"/usr/lib/python3.5/multiprocessing/connection.py\", line 722, in deliver_challenge\n",
      "    response = connection.recv_bytes(256)        # reject large message\n",
      "  File \"/usr/lib/python3.5/multiprocessing/connection.py\", line 216, in recv_bytes\n",
      "    buf = self._recv_bytes(maxlength)\n",
      "  File \"/usr/lib/python3.5/multiprocessing/connection.py\", line 407, in _recv_bytes\n",
      "    buf = self._recv(4)\n",
      "  File \"/usr/lib/python3.5/multiprocessing/connection.py\", line 379, in _recv\n",
      "    chunk = read(handle, remaining)\n",
      "ConnectionResetError: [Errno 104] Connection reset by peer\n",
      "Experiment split 4, normal classes [328   0 248  28]\n",
      "Iteration 0 (Epoch: 0), loss is 1.3990161418914795\n",
      "Iteration 10 (Epoch: 0), loss is 1.0755020380020142\n",
      "Iteration 20 (Epoch: 0), loss is 1.1714897155761719\n",
      "Iteration 30 (Epoch: 0), loss is 1.0953813791275024\n",
      "Iteration 40 (Epoch: 0), loss is 0.8717703223228455\n",
      "Iteration 50 (Epoch: 0), loss is 0.7059884071350098\n",
      "Iteration 60 (Epoch: 0), loss is 0.77968829870224\n",
      "Iteration 70 (Epoch: 0), loss is 0.782166063785553\n",
      "Iteration 0 (Epoch: 1), loss is 0.5432838797569275\n",
      "Iteration 10 (Epoch: 1), loss is 0.6572031378746033\n",
      "Iteration 20 (Epoch: 1), loss is 0.646757960319519\n",
      "Iteration 30 (Epoch: 1), loss is 0.72713303565979\n",
      "Iteration 40 (Epoch: 1), loss is 0.648741602897644\n",
      "Iteration 50 (Epoch: 1), loss is 0.47577744722366333\n",
      "Iteration 60 (Epoch: 1), loss is 0.5930089354515076\n",
      "Iteration 70 (Epoch: 1), loss is 0.5311587452888489\n",
      "Iteration 0 (Epoch: 2), loss is 0.6038616299629211\n",
      "Iteration 10 (Epoch: 2), loss is 0.42719796299934387\n",
      "Iteration 20 (Epoch: 2), loss is 0.6646344661712646\n",
      "Iteration 30 (Epoch: 2), loss is 0.4702939987182617\n",
      "Iteration 40 (Epoch: 2), loss is 0.36449065804481506\n",
      "Iteration 50 (Epoch: 2), loss is 0.5512346625328064\n",
      "Iteration 60 (Epoch: 2), loss is 0.4513876140117645\n",
      "Iteration 70 (Epoch: 2), loss is 0.39347267150878906\n",
      "Done\n",
      "Correct predictions\n",
      " tensor([0, 1, 1, 0, 1, 0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 1, 1, 0, 1, 1, 0, 1, 1,\n",
      "        1, 1, 1, 0, 0, 1, 1, 0], device='cuda:0', dtype=torch.uint8)\n",
      "Iteration 0\n",
      "Iteration 50\n",
      "Iteration 100\n",
      "Iteration 150\n",
      "Iteration 200\n",
      "Iteration 250\n",
      "Iteration 300\n",
      "Iteration 350\n",
      "Iteration 400\n",
      "Iteration 450\n",
      "Iteration 500\n",
      "Exception ignored in: <bound method _DataLoaderIter.__del__ of <torch.utils.data.dataloader._DataLoaderIter object at 0x7f414f632ba8>>\n",
      "Traceback (most recent call last):\n",
      "  File \"/usr/local/lib/python3.5/dist-packages/torch/utils/data/dataloader.py\", line 399, in __del__\n",
      "    self._shutdown_workers()\n",
      "  File \"/usr/local/lib/python3.5/dist-packages/torch/utils/data/dataloader.py\", line 378, in _shutdown_workers\n",
      "    self.worker_result_queue.get()\n",
      "  File \"/usr/lib/python3.5/multiprocessing/queues.py\", line 345, in get\n",
      "    return ForkingPickler.loads(res)\n",
      "  File \"/usr/local/lib/python3.5/dist-packages/torch/multiprocessing/reductions.py\", line 151, in rebuild_storage_fd\n",
      "    fd = df.detach()\n",
      "  File \"/usr/lib/python3.5/multiprocessing/resource_sharer.py\", line 57, in detach\n",
      "    with _resource_sharer.get_connection(self._id) as conn:\n",
      "  File \"/usr/lib/python3.5/multiprocessing/resource_sharer.py\", line 87, in get_connection\n",
      "    c = Client(address, authkey=process.current_process().authkey)\n",
      "  File \"/usr/lib/python3.5/multiprocessing/connection.py\", line 494, in Client\n",
      "    deliver_challenge(c, authkey)\n",
      "  File \"/usr/lib/python3.5/multiprocessing/connection.py\", line 722, in deliver_challenge\n",
      "    response = connection.recv_bytes(256)        # reject large message\n",
      "  File \"/usr/lib/python3.5/multiprocessing/connection.py\", line 216, in recv_bytes\n",
      "    buf = self._recv_bytes(maxlength)\n",
      "  File \"/usr/lib/python3.5/multiprocessing/connection.py\", line 407, in _recv_bytes\n",
      "    buf = self._recv(4)\n",
      "  File \"/usr/lib/python3.5/multiprocessing/connection.py\", line 379, in _recv\n",
      "    chunk = read(handle, remaining)\n",
      "ConnectionResetError: [Errno 104] Connection reset by peer\n"
     ]
    }
   ],
   "source": [
    "!python3 st-gcn-ad_n_v_all_exper.py"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.5.2"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}