A3C-Pong-GPU.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "# =======================================================================================================\n",
    "#!apt-get update\n",
    "#!apt-get install -y xvfb ffmpeg cmake wget unrar python-opengl x11-utils\n",
    "#!apt-get upgrade --yes \n",
    "#\n",
    "#!pip install --upgrade pip\n",
    "#!pip install gym[atari] opencv-python atari-py pyglet pyvirtualdisplay ale-py lz4 gym-notebook-wrapper --upgrade\n",
    "# =======================================================================================================\n",
    "#!SHELL=/bin/bash\n",
    "#!if [ ! -f 'Roms.rar' ] ; then wget http://www.atarimania.com/roms/Roms.rar ; unrar e Roms.rar ; fi\n",
    "#!python -m atari_py.import_roms . \n",
    "# ======================================================================================================="
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "# =======================================================================================================\n",
    "import os\n",
    "import time\n",
    "import matplotlib\n",
    "import cv2\n",
    "import gym\n",
    "import gnwrapper\n",
    "import time\n",
    "import multiprocessing\n",
    "import pickle\n",
    "import shutil\n",
    "# =======================================================================================================\n",
    "import tensorflow        as tf\n",
    "import numpy             as np\n",
    "import matplotlib.pyplot as plt\n",
    "# =======================================================================================================\n",
    "from time                import gmtime, strftime\n",
    "from collections         import deque\n",
    "from gym                 import envs\n",
    "from gym.wrappers        import FrameStack, GrayScaleObservation, ResizeObservation, AutoResetWrapper\n",
    "from tensorflow          import keras\n",
    "from keras               import backend\n",
    "from threading           import Thread, Lock\n",
    "from queue               import Queue\n",
    "from typing              import List, Tuple\n",
    "# ======================================================================================================="
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "# =======================================================================================================\n",
    "# tf.keras.backend.set_floatx('float32') # set keras default floating point precision type\n",
    "tf_fpp = tf.float32                      # tf floating point precision type \n",
    "np_fpp = np.float32                      # np floating point precision type\n",
    "tf.keras.backend.clear_session()\n",
    "# ======================================================================================================="
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "# =======================================================================================================\n",
    "maximum_number_of_workers = 64           # Single out-of-context variable to specify the maximum number of \n",
    "                                         # threads to run in parallel. This because of the creation of an array\n",
    "                                         # of environments in used in a **static** context in order to apply \n",
    "                                         # the @tf.function annotation and speed up the whole process.\n",
    "            \n",
    "                                         # Otherwise, calls to functions of non-static variables in an OO \n",
    "                                         # context would've required the *self* starting argument, not \n",
    "                                         # convertible to a Tensor, therefore not allowing the conversion to graph.\n",
    "# ======================================================================================================="
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "# =======================================================================================================\n",
    "# CropObservation wrapper crops the image(s) in input to a specified dimension. \n",
    "# =======================================================================================================\n",
    "class CropObservation(gym.ObservationWrapper):\n",
    "\n",
    "    def __init__(self, environment, kwargs):\n",
    "        super().__init__(environment)\n",
    "        self.x_low  = kwargs[\"x_low\"]\n",
    "        self.x_high = kwargs[\"x_high\"]\n",
    "        self.y_low  = kwargs[\"y_low\"]\n",
    "        self.y_high = kwargs[\"y_high\"]\n",
    "\n",
    "    def observation(self, observation):\n",
    "        observation = observation[self.y_low:self.y_high, self.x_low:self.x_high]\n",
    "        return observation\n",
    "# =======================================================================================================\n",
    "\n",
    "# =======================================================================================================\n",
    "# DenoiseObservation wrapper uses computer vision's adaptive threshold function to highlight important features.\n",
    "# Original function documentation can be found here [https://docs.opencv.org/3.4/d7/d4d/tutorial_py_thresholding.html]\n",
    "# =======================================================================================================\n",
    "class DenoiseObservation(gym.ObservationWrapper):\n",
    "  \n",
    "    def __init__(self, env):\n",
    "        super().__init__(env)\n",
    "    \n",
    "    def observation(self, observation):\n",
    "        observation = cv2.adaptiveThreshold(observation, 255., cv2.ADAPTIVE_THRESH_GAUSSIAN_C, cv2.THRESH_BINARY, 3, 2)\n",
    "        return observation\n",
    "# =======================================================================================================\n",
    "  \n",
    "    \n",
    "# =======================================================================================================\n",
    "# NormalizeObservation wrapper divides the uint8 pixels by their max value (255.) in order to scale it to a value\n",
    "# between 0 and 1. This reduces the variance of values in the neural network.\n",
    "# =======================================================================================================\n",
    "class NormalizeObservation(gym.ObservationWrapper):\n",
    "    \n",
    "    def __init__(self, environment):\n",
    "        super().__init__(environment)\n",
    "        \n",
    "    def observation(self, observation):\n",
    "        observation = tf.cast(observation, dtype=tf.float32)\n",
    "        observation = tf.divide(observation, 255.)\n",
    "        return observation\n",
    "# =======================================================================================================\n",
    "\n",
    "# =======================================================================================================\n",
    "# Function that, based on the arguments passed in the last section, automatically wraps the environment \n",
    "# =======================================================================================================\n",
    "def make_env(kwargs):\n",
    "    \n",
    "    env = gym.make(env_name, render_mode=\"rgb_array\")\n",
    "    env = AutoResetWrapper(env)\n",
    "    \n",
    "    if kwargs[\"crop\"]:\n",
    "        env = CropObservation(env, kwargs)\n",
    "    if kwargs[\"grayscale\"]:\n",
    "        env = GrayScaleObservation(env, keep_dim=True)\n",
    "    if kwargs[\"resize\"]:\n",
    "        env = ResizeObservation(env, (kwargs[\"screen_size\"], kwargs[\"screen_size\"]))\n",
    "    if kwargs[\"denoise\"]:\n",
    "        env = DenoiseObservation(env)\n",
    "    if kwargs[\"normalize\"]: \n",
    "        env = NormalizeObservation(env)\n",
    "    if kwargs[\"stack_frames\"]:\n",
    "        env = FrameStack(env, kwargs[\"n_frames\"])\n",
    "\n",
    "    return env\n",
    "# ======================================================================================================="
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "# =======================================================================================================\n",
    "# Actor-Critic model as specified in the original A3C paper in section 8 (Experimental Setup) of the \n",
    "# supplementary material.\n",
    "# A possible extension to the neural network would be to add LSTM cells to this architecture. \n",
    "# Another possible extension would be to directly use ConvLSTM2D layers offered by tensorflow.\n",
    "#\n",
    "# When experiencing fading gradients, it's adviced to switch the activation to leaky_relu.\n",
    "# \n",
    "# Technically, the actor layer's function activation should be a softmax. The reason to postpone the softmax \n",
    "# operation to later stages is intended: as reported in an older documentation, it should be more numerically \n",
    "# stable. Or at least it was. \n",
    "# [https://github.com/tensorflow/tensorflow/blob/4f50b5dc6426f63a8e70b65d3b9e55ed8f7d38e2/tensorflow/python/keras/losses.py#L437]\n",
    "# =======================================================================================================\n",
    "\n",
    "class ActorCriticModel(tf.keras.Model):\n",
    "    \n",
    "    def leaky_relu(self, x):\n",
    "        return tf.keras.activations.relu(x, alpha=self.alpha, max_value=None, threshold=0.0)\n",
    "        \n",
    "    def __init__(self, state_shape, action_size):\n",
    "\n",
    "        self.alpha = 0.125\n",
    "        \n",
    "        self.state_shape     = state_shape\n",
    "        self.action_size     = action_size\n",
    "\n",
    "        inputs           = tf.keras.layers.Input(shape=self.state_shape)\n",
    "        # conv_1         = tf.keras.layers.ConvLSTM2D(16, 8, strides=4, activation='relu', return_sequences=True) (inputs)\n",
    "        # conv_2         = tf.keras.layers.ConvLSTM2D(32, 4, strides=2, activation='relu') (conv_1)\n",
    "        conv_1           = tf.keras.layers.Conv2D(8, 8, strides=2, activation='relu', kernel_initializer='he_uniform') (inputs)\n",
    "        conv_2           = tf.keras.layers.Conv2D(16, 4, strides=2, activation='relu', kernel_initializer='he_uniform') (conv_1)\n",
    "        conv_3           = tf.keras.layers.Conv2D(32, 2, strides=4, activation='relu', kernel_initializer='he_uniform') (conv_2)\n",
    "        flatten          = tf.keras.layers.Flatten() (conv_3)\n",
    "        dense_1          = tf.keras.layers.Dense(units=1024, activation=self.leaky_relu, kernel_initializer='he_uniform') (flatten)\n",
    "        critic           = tf.keras.layers.Dense(units=1) (dense_1)\n",
    "        actor            = tf.keras.layers.Dense(units=action_size) (dense_1) \n",
    "\n",
    "        super(ActorCriticModel, self).__init__(inputs=inputs, outputs=[actor, critic])\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "# =======================================================================================================\n",
    "# GlobalUpdater is the delegate that the asynchronous Worker uses to communicate with Controller.\n",
    "# It's called Global Updater because allows the update of the global model, held by Controller.\n",
    "# =======================================================================================================\n",
    "class GlobalUpdater():\n",
    "    \n",
    "    def __init__(self, controller_queue, worker_queue):\n",
    "        self.controller_queue = controller_queue\n",
    "        self.worker_queue = worker_queue\n",
    "    # =======================================================================================================            \n",
    "    # Operation to signal the controller the completion of an episode. It simply communicates all the variables \n",
    "    # needed to log the episode. \n",
    "    # =======================================================================================================\n",
    "    def episode_completion_update(self):\n",
    "        self.controller_queue.put((\"completion\", ()))\n",
    "    # =======================================================================================================\n",
    "    \n",
    "    \n",
    "    # =======================================================================================================\n",
    "    # Operation to signal the update of the global model. Gradients are sent through the queue to the Controller\n",
    "    # so that he can apply gradients using its optimizer.\n",
    "    # Returns: the updated global model weights to substitute the local model ones with.\n",
    "    # =======================================================================================================\n",
    "    def global_model_update(self, worker_idx, grads):\n",
    "        self.controller_queue.put((\"update\", (worker_idx, grads)))\n",
    "        return self.worker_queue.get()\n",
    "    # =======================================================================================================\n",
    "    \n",
    "    # =======================================================================================================\n",
    "    # Operation to request the global episode variable, held by Controller.\n",
    "    # =======================================================================================================\n",
    "    def get_global_episode(self, worker_idx):\n",
    "        self.controller_queue.put((\"get_episode\", (worker_idx,)))\n",
    "        return self.worker_queue.get()\n",
    "    # =======================================================================================================\n",
    "    \n",
    "    \n",
    "    # =======================================================================================================\n",
    "    # Operation to request the weights of Controller's global model so to initialize every Worker's local model\n",
    "    # to have the correct initial weights.\n",
    "    # =======================================================================================================\n",
    "    def get_weights(self, worker_idx):\n",
    "        self.controller_queue.put((\"get_weights\", (worker_idx,)))\n",
    "        return self.worker_queue.get()\n",
    "    # =======================================================================================================\n",
    "    \n",
    "    # =======================================================================================================\n",
    "    # Operation to signal the Controller the reaching of the termination state of the worker. Termination state\n",
    "    # occurs when the global episode is larger than the maximum number of episodes to train the model\n",
    "    # =======================================================================================================            \n",
    "    def signal_termination(self):\n",
    "        self.controller_queue.put(None)\n",
    "    # =======================================================================================================\n",
    "        "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "# =======================================================================================================\n",
    "# As specified above, this variable holds the static array accessed by Worker in a way that is compatible with\n",
    "# @tf.function annotation. This speeds up the training process by a lot.\n",
    "# =======================================================================================================\n",
    "\n",
    "envs = [None] * maximum_number_of_workers\n",
    "\n",
    "\n",
    "class Worker(Thread):\n",
    "    # =======================================================================================================\n",
    "    def __init__(self, worker_idx, updater, state_shape, kwargs):\n",
    "        super(Worker, self).__init__()\n",
    "        # Worker's identifier\n",
    "        self.worker_idx           = worker_idx\n",
    "        # State shape to initialize the ActorCriticModel class with\n",
    "        self.state_shape          = state_shape\n",
    "        # Updater to communicate with the Controller\n",
    "        self.updater              = updater\n",
    "        # Private Worker environment stored in a static array as specified above\n",
    "        # self.env                  = make_env(kwargs)\n",
    "        envs[self.worker_idx]     = make_env(kwargs)\n",
    "        # Name of the environment to explore\n",
    "        self.env_name             = kwargs[\"env_name\"]\n",
    "        # Set of available actions that the Worker can perform during its exploration\n",
    "        self.available_actions    = kwargs[\"available_actions\"]\n",
    "        self.action_size          = len(self.available_actions)\n",
    "        # Local ActorCriticModel used to explore and process gradients \n",
    "        self.local_model          = ActorCriticModel(self.state_shape, self.action_size)\n",
    "\n",
    "        \n",
    "        # Losses used to train the model. As mentioned above, both actor loss and entropy loss have the \n",
    "        # from_logits variable set to False for better stability\n",
    "        # Entropy loss can be calculated as the crossentropy with labels = true_values = probs\n",
    "        reduction                 = tf.keras.losses.Reduction.NONE\n",
    "        self.actor_loss_fn        = tf.keras.losses.SparseCategoricalCrossentropy(from_logits=True, reduction=reduction)\n",
    "        self.entropy_loss_fn      = tf.keras.losses.CategoricalCrossentropy(from_logits=True, reduction=reduction)\n",
    "        self.critic_loss_fn       = tf.keras.losses.MeanSquaredError(reduction=reduction)\n",
    "\n",
    "        \n",
    "        # Hyperparams\n",
    "        self.max_eps              = kwargs[\"max_eps\"]\n",
    "        self.update_freq          = kwargs[\"update_freq\"]\n",
    "        self.gamma                = kwargs[\"gamma\"]\n",
    "        self.exponential_discount = kwargs[\"exponential_discount\"]\n",
    "        self.entropy_factor       = kwargs[\"entropy_factor\"]\n",
    "        self.critic_loss_factor   = kwargs[\"critic_loss_factor\"]\n",
    "        self.eps                  = kwargs[\"eps\"]\n",
    "        \n",
    "        print(f'Worker [{self.worker_idx}] started.')\n",
    "    # =======================================================================================================\n",
    "\n",
    "        \n",
    "    # =======================================================================================================\n",
    "    # Each of the following static methods is nothing else than a method that explicitly or implicitly use a \n",
    "    # Numpy function and its own tensorflow wrapper. This operation is being done to allow graph mode execution\n",
    "    # via the usage of @tf.function annotation.\n",
    "    # =======================================================================================================\n",
    "    \n",
    "    # =======================================================================================================\n",
    "    @staticmethod\n",
    "    def env_reset(index):\n",
    "        new_state, info = envs[index].reset()\n",
    "        return np.array(new_state, np_fpp)\n",
    "\n",
    "    @staticmethod\n",
    "    def tf_env_reset(index):\n",
    "        return tf.numpy_function(Worker.env_reset, [index], tf_fpp)\n",
    "    # =======================================================================================================\n",
    "    \n",
    "    \n",
    "    # =======================================================================================================\n",
    "    @staticmethod\n",
    "    def env_step(index, action):\n",
    "        new_state, reward, done, truncated, info = envs[index].step(action)\n",
    "        return (np.array(new_state, np_fpp), np.array(reward, np_fpp), np.array(done, np_fpp))\n",
    "\n",
    "    @staticmethod\n",
    "    def tf_env_step(index, action):\n",
    "        return tf.numpy_function(Worker.env_step, [index, action], [tf_fpp, tf_fpp, tf_fpp])\n",
    "    # =======================================================================================================\n",
    "\n",
    "    \n",
    "    # =======================================================================================================\n",
    "    @staticmethod\n",
    "    def get_action(available_actions, action_index):\n",
    "        return available_actions[action_index].astype(np.uint8)\n",
    "\n",
    "    @staticmethod\n",
    "    def tf_get_action(available_actions, action_index):\n",
    "        return tf.numpy_function(Worker.get_action, [available_actions, action_index], tf.uint8)\n",
    "    # =======================================================================================================\n",
    "    \n",
    "    \n",
    "    # =======================================================================================================\n",
    "    # This function explores the environment by playing for n steps. It stores every variable needed to \n",
    "    # calculate the gradient onto their own TensorArray, then proceeds to stack before passing them to the \n",
    "    # loss function. The 'set_shape' methods are used to preserve the variable shape across different \n",
    "    # iterations of the for loop.\n",
    "    #\n",
    "    # The bootstrapped value is calculated as specified in the paper:\n",
    "    # - 0        if state is a terminal state.\n",
    "    # - V(state) if state is not a terminal state.\n",
    "    #\n",
    "    # It's important to notice that this function is working while in a tf.GradientTape context. Therefore,\n",
    "    # the model is only called in this function and values are stored, so there's no need to call once more\n",
    "    # the model while in the function that calculates the total loss. \n",
    "    # =======================================================================================================\n",
    "    def play_steps(self, state):\n",
    "        \n",
    "        logits     = tf.TensorArray(dtype=tf_fpp, size=0, dynamic_size=True)\n",
    "        values     = tf.TensorArray(dtype=tf_fpp, size=0, dynamic_size=True)\n",
    "        rewards    = tf.TensorArray(dtype=tf_fpp, size=0, dynamic_size=True)\n",
    "        dones      = tf.TensorArray(dtype=tf_fpp, size=0, dynamic_size=True)\n",
    "        actions    = tf.TensorArray(dtype=tf.uint8, size=0, dynamic_size=True)\n",
    "        \n",
    "        for t in tf.range(self.update_freq):\n",
    "\n",
    "            logit, value        = self.local_model(state)\n",
    "            \n",
    "            action_index        = tf.cast(tf.squeeze(tf.random.categorical(logit, 1)), tf.uint8)\n",
    "            action              = Worker.tf_get_action(self.available_actions, action_index)\n",
    "            \n",
    "            state, reward, done = Worker.tf_env_step(self.worker_idx, action)\n",
    "\n",
    "            reward              = tf.convert_to_tensor(reward, dtype=tf_fpp)\n",
    "            state               = tf.expand_dims(tf.reshape(state, shape=self.state_shape), axis=0)\n",
    "            \n",
    "            logits              = logits.write(t, logit)\n",
    "            values              = values.write(t, value)\n",
    "            rewards             = rewards.write(t, reward)\n",
    "            dones               = dones.write(t, done)\n",
    "            actions             = actions.write(t, action_index)\n",
    "            \n",
    "        # Bootstrap last state\n",
    "        _, value = self.local_model(state)\n",
    "        values   = values.write(t + 1, value)\n",
    "            \n",
    "        return logits.stack(), values.stack(), rewards.stack(), dones.stack(), actions.stack(), state\n",
    "    # =======================================================================================================\n",
    "    \n",
    "    # =======================================================================================================\n",
    "    # Function to process rewards and calculate Generalized Advantage Estimates as proposed in the \n",
    "    # \"High-Dimensional Continuous Control Using Generalized Advantage Estimation\" paper. Using this should \n",
    "    # improve learning with respect to the standard advantage estimation. \n",
    "    # =======================================================================================================\n",
    "    def get_generalized_advantage_estimate(self, values, dones, rewards): \n",
    "        \n",
    "        next_advantage   = tf.constant(0.0, dtype=tf_fpp)\n",
    "        next_adv_shape   = next_advantage.shape\n",
    "        advantages       = tf.TensorArray(dtype=tf_fpp, size=0, dynamic_size=True)\n",
    "        \n",
    "        values           = tf.squeeze(values, axis=1)\n",
    "        \n",
    "        next_values      = values[1:]\n",
    "        values           = values[:self.update_freq]\n",
    "        \n",
    "        rewards          = rewards[::-1]\n",
    "        next_values      = next_values[::-1]\n",
    "        values           = values[::-1]\n",
    "        \n",
    "        for t in tf.range(self.update_freq):\n",
    "\n",
    "            delta          = rewards[t] + (self.gamma * (1.0 - dones[t]) * next_values[t]) - values[t]\n",
    "            next_advantage = delta + (self.gamma * self.exponential_discount * (1.0 - dones[t]) * next_advantage)\n",
    "            \n",
    "            next_advantage.set_shape(next_adv_shape)\n",
    "            advantages     = advantages.write(t, next_advantage)\n",
    "        \n",
    "        return advantages.stack()[::-1]\n",
    "    # =======================================================================================================\n",
    "\n",
    "    \n",
    "    # =======================================================================================================\n",
    "    # Computes the total loss of the trajectory experienced through the function play_steps. \n",
    "    # Depending on hyperparameters, one can either use standard advantage estimate or the general one. \n",
    "    # Furthermore, it's possible to clip rewards (to values [-1, 0, 1]) and to normalize advantages.\n",
    "    # The single loss functions used in this function do match the expected behavior:\n",
    "    # * actor   = - reduce_mean(reduce_sum(log(probs[action], axis=1))\n",
    "    # * entropy = - reduce_mean(reduce_sum(log(probs) * probs, axis=1))\n",
    "    # * critic  = reduce_mean((value_targets - values) ** 2)\n",
    "    #\n",
    "    # Since the gradient tape wraps the play_steps and the advantage estimation functions, it's mandatory to\n",
    "    # stop gradients earlier. Furthermore, the stop_gradient function has been applied to actions as well \n",
    "    # since the softmax_sparse_categorical_crossentropy_v2 function defined by tensorflow propagates gradients\n",
    "    # to true values as well and it's not clear whether this function is used in the SparseCategoricalCrossEntropy\n",
    "    # wrapper. \n",
    "    # ======================================================================================================= \n",
    "    def compute_loss(self, logits, values, rewards, dones, actions):\n",
    "        \n",
    "        logits         = tf.squeeze(logits, axis=1)\n",
    "        values         = tf.squeeze(values, axis=1)\n",
    "    \n",
    "        advantages     = self.get_generalized_advantage_estimate(values, dones, rewards)\n",
    "        \n",
    "        values         = values[:self.update_freq]\n",
    "        value_targets  = tf.expand_dims(advantages, axis=1) + values    \n",
    "        \n",
    "        actions        = tf.stop_gradient(actions)\n",
    "        advantages     = tf.stop_gradient(advantages)\n",
    "        actor_loss     = self.actor_loss_fn(y_true=actions, y_pred=logits, sample_weight=advantages)\n",
    "\n",
    "        value_targets  = tf.stop_gradient(value_targets)\n",
    "        critic_loss    = self.critic_loss_factor * self.critic_loss_fn(y_true=value_targets, y_pred=values) \n",
    "\n",
    "        probs          = tf.nn.softmax(logits + self.eps)\n",
    "        entropy_factor = self.entropy_factor / (self.episode ** 1/16 + 1)\n",
    "        entropy        = - entropy_factor * self.entropy_loss_fn(y_true=probs, y_pred=logits)  \n",
    "\n",
    "        total_loss     = tf.reduce_mean(actor_loss + entropy + critic_loss)\n",
    "                \n",
    "        return total_loss\n",
    "    # =======================================================================================================\n",
    "\n",
    "    \n",
    "    # =======================================================================================================\n",
    "    # Basic tf.function wrapper to encapsulate both play_steps and compute_loss function\n",
    "    # =======================================================================================================\n",
    "    # @tf.function    \n",
    "    def step(self, state):\n",
    "        with tf.GradientTape() as tape:\n",
    "            (logits, \n",
    "             values, \n",
    "             rewards, \n",
    "             dones,\n",
    "             actions, \n",
    "             state)    = self.play_steps(state)\n",
    "            \n",
    "            total_loss = self.compute_loss(logits, values, rewards, dones, actions)\n",
    "\n",
    "        grads = tape.gradient(total_loss, self.local_model.trainable_weights)\n",
    "            \n",
    "        if tf.reduce_sum(dones) > 0:\n",
    "            done = tf.constant(True, dtype=tf.bool)\n",
    "        else:\n",
    "            done = tf.constant(False, dtype=tf.bool)\n",
    "            \n",
    "        return state, done, grads\n",
    "    # =======================================================================================================\n",
    "    \n",
    "    \n",
    "    # =======================================================================================================\n",
    "    # Function that wraps all the behavior expected from the Worker. It initializes the states and \n",
    "    # communicates to Controller through GlobalUpdater both the gradients and the variables to be logged. \n",
    "    # =======================================================================================================\n",
    "    def run(self):\n",
    "        \n",
    "        # Initialize weights with global model ones\n",
    "        self.local_model.set_weights(self.updater.get_weights(self.worker_idx))\n",
    "        \n",
    "        state = tf.convert_to_tensor(Worker.env_reset(self.worker_idx), dtype=tf_fpp)\n",
    "        state = tf.expand_dims(tf.reshape(state, shape=self.state_shape), axis=0)\n",
    "        \n",
    "        self.episode = self.updater.get_global_episode(self.worker_idx)\n",
    "        \n",
    "        while self.episode < self.max_eps:\n",
    "            state, done, grads = self.step(state)\n",
    "\n",
    "            # Push local gradients to global model\n",
    "            self.local_model.set_weights(self.updater.global_model_update(self.worker_idx, grads))\n",
    "\n",
    "            if done: \n",
    "                # Signal completion to Controller\n",
    "                self.updater.episode_completion_update()\n",
    "                \n",
    "            self.episode = self.updater.get_global_episode(self.worker_idx)\n",
    "\n",
    "        self.updater.signal_termination()\n",
    "    # =======================================================================================================\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "class Saver():\n",
    "    def __init__(self, save_dir, checkpoint):\n",
    "        # Directories and files where the progress is saved into\n",
    "        self.save_dir = save_dir\n",
    "        self.checkpoint = checkpoint\n",
    "        \n",
    "        self.output_file = save_dir + 'output.txt'\n",
    "        self.best_score_file = save_dir + 'best_score.bin'\n",
    "        self.episode_file = save_dir + 'episode.bin'\n",
    "        self.deque_file = save_dir + 'deque.bin'\n",
    "        self.plot_data_file = save_dir + 'plot_data.bin'\n",
    "        \n",
    "        self.cp_manager = tf.train.CheckpointManager(self.checkpoint, directory=save_dir, max_to_keep=2)\n",
    "        \n",
    "        self.lock = Lock()\n",
    "        self.output_lock = Lock()\n",
    "        \n",
    "    def restore_or_reset(self, restore, restore_from_best, moving_average_card):\n",
    "        if restore:\n",
    "            if restore_from_best:\n",
    "                self.checkpoint.restore(self.cp_manager.checkpoints[-2])\n",
    "            else:\n",
    "                self.checkpoint.restore(self.cp_manager.latest_checkpoint)\n",
    "            global_episode, best_score, rdeque, plot_data = self.load()\n",
    "            global_moving_average_reward = np.mean(rdeque)\n",
    "        else:\n",
    "            global_episode, best_score, rdeque, plot_data = 0, float('-inf'), deque(maxlen=moving_average_card), []\n",
    "            global_moving_average_reward = 0\n",
    "            self.reset()\n",
    "            \n",
    "        return global_episode, best_score, rdeque, plot_data, global_moving_average_reward\n",
    "        \n",
    "    def save(self, episode, best_score, deque, plot_data):\n",
    "        with self.lock:\n",
    "            print(f'Saving model to {self.save_dir}, best episode score: {best_score}')\n",
    "            self.cp_manager.save()\n",
    "            self.__store_element(self.episode_file, episode)\n",
    "            self.__store_element(self.best_score_file, best_score)\n",
    "            self.__store_element(self.deque_file, deque)\n",
    "            self.__store_element(self.plot_data_file, plot_data)\n",
    "            shutil.copy2(self.output_file, self.output_file + '.bak')\n",
    "\n",
    "    def load(self):\n",
    "        with self.lock:\n",
    "            episode = self.__load_element(self.episode_file)\n",
    "            best_score = self.__load_element(self.best_score_file)\n",
    "            deque = self.__load_element(self.deque_file)\n",
    "            plot_data = self.__load_element(self.plot_data_file)\n",
    "                 \n",
    "            self.__remove_output_file()\n",
    "            shutil.copy2(self.output_file + '.bak', self.output_file)\n",
    "            self.serialize_output(f' --- RESTORING EPISODE {episode} - BESTSCORE {best_score} ---')\n",
    "        \n",
    "        return episode, best_score, deque, plot_data\n",
    "    \n",
    "    def reset(self):\n",
    "        with self.lock:\n",
    "            self.__make_save_dir()\n",
    "            self.__remove_output_file()\n",
    "            \n",
    "    def serialize_output(self, output):\n",
    "        with self.output_lock:\n",
    "            self.__make_save_dir()\n",
    "            with open(self.output_file, \"a\") as file:\n",
    "                file.write(self.__timestamp(output) + \"\\n\"),\n",
    "            print(output)\n",
    "            \n",
    "    def __store_element(self, element_file, element):\n",
    "        self.__make_save_dir()\n",
    "        with open(element_file, \"wb\") as file:\n",
    "            pickle.dump(element, file)        \n",
    "        \n",
    "    def __load_element(self, element_file):\n",
    "        with open(element_file, \"rb\") as file:\n",
    "            element = pickle.load(file)\n",
    "        return element\n",
    "            \n",
    "    def __remove_output_file(self):\n",
    "        if os.path.exists(self.output_file):\n",
    "            os.remove(self.output_file)\n",
    "            print('Removed ' + self.output_file)\n",
    "            \n",
    "    def __make_save_dir(self):\n",
    "        if not os.path.isdir(self.save_dir):\n",
    "            os.makedirs(self.save_dir)\n",
    "    \n",
    "    def __timestamp(self, string):\n",
    "        return strftime(f'[%Y-%m-%d %H:%M:%S] {string}', gmtime())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [],
   "source": [
    "\n",
    "class Controller():\n",
    "    # =======================================================================================================\n",
    "    def __init__(self, **kwargs):\n",
    "\n",
    "        # Environment name\n",
    "        self.env_name                     = kwargs[\"env_name\"]\n",
    "        # List of available actions\n",
    "        self.available_actions            = kwargs[\"available_actions\"]\n",
    "        self.action_size                  = len(self.available_actions)\n",
    "        # Local environment for playing and displaying features\n",
    "        self.env                          = make_env(kwargs)\n",
    "        # Shape of the state used to initialize the ActorCriticModel neural network\n",
    "        self.state_shape                  = self.env.observation_space.shape\n",
    "        \n",
    "        # Global Model\n",
    "        self.global_model                 = ActorCriticModel(self.state_shape, self.action_size)\n",
    "        # Optimizer\n",
    "        self.opt                          = tf.keras.optimizers.RMSprop(\n",
    "                                                learning_rate=kwargs[\"lr_schedule\"], \n",
    "                                                centered=kwargs[\"centered\"], \n",
    "                                                global_clipnorm=kwargs[\"global_clipnorm\"], \n",
    "                                                rho=kwargs[\"rho\"],\n",
    "                                            )\n",
    "        # Cardinality of the moving average deque\n",
    "        self.moving_average_card          = kwargs[\"moving_average_card\"]\n",
    "        # Interval of episodes after which global model plays a game and prints output\n",
    "        self.play_interval                = kwargs[\"play_interval\"]\n",
    "        \n",
    "        # Checkpoint\n",
    "        self.checkpoint                   = tf.train.Checkpoint(optimizer=self.opt, model=self.global_model)\n",
    "\n",
    "        self.saver                        = Saver(kwargs[\"save_dir\"], tf.train.Checkpoint(optimizer=self.opt, model=self.global_model))\n",
    "        \n",
    "        (self.global_episode, \n",
    "        self.best_score, \n",
    "        self.deque, \n",
    "        self.plot_data, \n",
    "        self.global_moving_average_reward)= self.saver.restore_or_reset(kwargs[\"restore\"], kwargs[\"restore_from_best\"], self.moving_average_card)\n",
    "        \n",
    "        # Number of Workers\n",
    "        self.number_of_workers            = kwargs[\"number_of_workers\"]\n",
    "        # Pool of Worker threads\n",
    "        self.workers                      = []\n",
    "        # Queue used to communicate with Workers (Workers -> Controller)\n",
    "        self.controller_queue             = Queue()\n",
    "        # Queues used to communicate with Workers (Controller -> Worker)\n",
    "        self.worker_queues                = []\n",
    "        \n",
    "        # Initialization of threads\n",
    "        for i in range(self.number_of_workers):\n",
    "            worker_queue = Queue()\n",
    "            self.worker_queues.append(worker_queue)\n",
    "\n",
    "            updater = GlobalUpdater(self.controller_queue, worker_queue)\n",
    "            worker  = Worker(i, updater, self.state_shape, kwargs)\n",
    "            self.workers.append(worker)\n",
    "        \n",
    "    # =======================================================================================================\n",
    "            \n",
    "        \n",
    "    # =======================================================================================================\n",
    "    # Function to start the pool of threads and start working \n",
    "    # =======================================================================================================\n",
    "    def train(self, **kwargs):\n",
    "        \n",
    "        # Since we want the server to be online as soon as possible, each worker is started by a separate thread\n",
    "        # so that the Controller is ready to reply to early responses\n",
    "        Thread(target=self.start_workers, args=()).start()\n",
    "        self.terminated_workers = 0\n",
    "        try:\n",
    "            while True:\n",
    "                msg = self.controller_queue.get()\n",
    "                if msg is not None:\n",
    "                    (task, args) = msg\n",
    "\n",
    "                    # Receive gradients from Worker and update the global model\n",
    "                    if task == \"update\":\n",
    "                        (worker_idx, grads) = args\n",
    "                        self.opt.apply_gradients(zip(grads, self.global_model.trainable_weights))\n",
    "                        self.worker_queues[worker_idx].put(self.global_model.get_weights())\n",
    "                    \n",
    "                    # Communicate weights to Worker to initialize local model weights\n",
    "                    if task == \"get_weights\":\n",
    "                        (worker_idx,) = args\n",
    "                        self.worker_queues[worker_idx].put(self.global_model.get_weights())\n",
    "                    \n",
    "                    # Communicate the current episode to Worker\n",
    "                    if task == \"get_episode\":\n",
    "                        (worker_idx,) = args\n",
    "                        self.worker_queues[worker_idx].put(self.global_episode)\n",
    "\n",
    "                    # Receive completion data from Worker\n",
    "                    if task == \"completion\":\n",
    "                        () = args\n",
    "\n",
    "                        # Update global episode\n",
    "                        self.global_episode += 1\n",
    "                        \n",
    "                        if self.global_episode % self.play_interval == 0:\n",
    "                            \n",
    "                            episode_reward, episode_steps, actions_count = self.global_model_play()\n",
    "                            # Update best score\n",
    "                            if episode_reward >= self.best_score and episode_reward > self.global_moving_average_reward:\n",
    "                                self.best_score = episode_reward\n",
    "                                self.saver.save(self.global_episode, self.best_score, self.deque, self.plot_data)\n",
    "                            # Update global moving average for rewards\n",
    "                            self.deque.append(episode_reward)\n",
    "                            # Compute moving average\n",
    "                            self.global_moving_average_reward = np.mean(self.deque)\n",
    "                            # Add result to list\n",
    "                            self.plot_data.append((self.global_moving_average_reward, episode_reward))\n",
    "                            # Serialize output\n",
    "                            self.saver.serialize_output(\n",
    "                                    f\"Episode: {self.global_episode / self.play_interval:6} | \"\n",
    "                                    f\"MA{self.moving_average_card}: {self.global_moving_average_reward:+6.1f} | \"\n",
    "                                    f\"Episode Reward: {episode_reward:6} | \"\n",
    "                                    f\"Steps: {episode_steps:5} | \"\n",
    "                                    f\"Actions: {actions_count}\"\n",
    "                            )\n",
    "                    \n",
    "                else:\n",
    "                    self.terminated_workers = self.terminated_workers + 1\n",
    "                    if self.terminated_workers == self.number_of_workers:\n",
    "                        break\n",
    "\n",
    "        # When receiving the interruption signal, save state\n",
    "        except KeyboardInterrupt:\n",
    "            print(self.saver.serialize_output(\"Interrupting. Saving checkpoint...\"))\n",
    "            self.saver.save(self.global_episode, self.best_score, self.deque, self.plot_data)\n",
    "    # =======================================================================================================\n",
    "\n",
    "    \n",
    "    # =======================================================================================================\n",
    "    # Debug function to properly show the images fed to the neural network\n",
    "    # =======================================================================================================    \n",
    "    def show_screen(self): \n",
    "        real_env        = gym.make(env_name)\n",
    "        print(self.env.observation_space.shape)\n",
    "\n",
    "        obs             = self.env.reset()\n",
    "        real_obs        = real_env.reset()\n",
    "\n",
    "        for _ in range(120):\n",
    "            obs, _, _, _      = self.env.step(0)\n",
    "            real_obs, _, _, _ = real_env.step(0)\n",
    "\n",
    "        plt.figure()\n",
    "        plt.title('Real Image')\n",
    "        plt.imshow(real_obs)\n",
    "        plt.figure()\n",
    "        plt.title('Edited Image 1')\n",
    "        plt.imshow(obs[0], cmap='gray')\n",
    "        plt.figure()\n",
    "        plt.title('Edited Image 2')\n",
    "        plt.imshow(obs[1], cmap='gray')\n",
    "        plt.figure()\n",
    "        plt.title('Edited Image 3')\n",
    "        plt.imshow(obs[2], cmap='gray')\n",
    "        plt.show()\n",
    "    # =======================================================================================================\n",
    "    \n",
    "    \n",
    "    # =======================================================================================================\n",
    "    # Plot the average rewards graph\n",
    "    # =======================================================================================================\n",
    "    def plot(self): \n",
    "        \n",
    "        print(self.plot_data)\n",
    "        moving_average_rewards = []\n",
    "        rewards                = []\n",
    "        for data in self.plot_data:\n",
    "            moving_average_rewards.append(data[0])\n",
    "            rewards.append(data[1])\n",
    "        \n",
    "        plt.figure()\n",
    "        plt.plot(moving_average_rewards, 'r')\n",
    "        plt.plot(rewards, 'b')\n",
    "        plt.xlabel('Episode')\n",
    "        \n",
    "        plt.show()\n",
    "    # =======================================================================================================\n",
    "    \n",
    "    # =======================================================================================================\n",
    "    # Play the game with global model and record data\n",
    "    # =======================================================================================================\n",
    "    def global_model_play(self):\n",
    "        \n",
    "        done = False\n",
    "        state, _ = self.env.reset()\n",
    "        \n",
    "        episode_reward = 0\n",
    "        episode_steps  = 0\n",
    "        actions_count  = tf.Variable(tf.zeros(self.action_size))\n",
    "        \n",
    "        while not done:\n",
    "\n",
    "            state                      = tf.convert_to_tensor(state)\n",
    "            state                      = tf.expand_dims(tf.reshape(state, shape=self.state_shape), axis=0)\n",
    "            logit, _                   = self.global_model(state)\n",
    "            \n",
    "            action_index               = tf.cast(tf.squeeze(tf.random.categorical(logit, 1)), tf.uint8).numpy()\n",
    "            action                     = self.available_actions[action_index]\n",
    "            state, reward, done, _, _  = self.env.step(action)\n",
    "\n",
    "            action_index               = tf.convert_to_tensor(action_index, dtype=tf.uint8)\n",
    "            action_one_hot             = tf.one_hot(action, self.action_size)\n",
    "            actions_count.assign_add(action_one_hot)\n",
    "            \n",
    "            episode_reward            += reward\n",
    "            episode_steps             += 1\n",
    "        \n",
    "        return episode_reward, episode_steps, actions_count.numpy()\n",
    "    # =======================================================================================================\n",
    "    \n",
    "    # =======================================================================================================\n",
    "    # Play the game according to the neural network and render it\n",
    "    # =======================================================================================================\n",
    "    def play(self):\n",
    "        env   = gnwrapper.Animation(self.env)\n",
    "        state, _ = env.reset()\n",
    "        done  = False\n",
    "        \n",
    "        while not done:\n",
    "            state                = tf.convert_to_tensor(state)\n",
    "            state                = tf.expand_dims(tf.reshape(state, shape=self.state_shape), axis=0)\n",
    "            logits, _            = self.global_model(state)\n",
    "            logits               = tf.squeeze(logits)\n",
    "            probs                = tf.nn.softmax(logits)\n",
    "            \n",
    "            # No need to sample, just go for the best action\n",
    "            action_index         = tf.math.argmax(probs).numpy()\n",
    "            action               = self.available_actions[action_index]\n",
    "            \n",
    "            state, _, done, _, _ = env.step(action)\n",
    "            env.render(mode='rgb_array')\n",
    "    # =======================================================================================================\n",
    "    \n",
    "    \n",
    "    # =======================================================================================================\n",
    "    # Accessory function to start Workers on a separate thread.\n",
    "    # A timer's set for further randomization: we might want to avoid situations where the global model\n",
    "    # is updated with almost simultaneous analogous updates. \n",
    "    # =======================================================================================================\n",
    "    def start_workers(self):\n",
    "        for worker in self.workers:\n",
    "            worker.start()\n",
    "            time.sleep(4)\n",
    "    # ======================================================================================================="
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Possible Actions: [0, 1, 2, 3, 4, 5]\n",
      "Selected Actions: ['NOOP', 'FIRE', 'RIGHT', 'LEFT', 'RIGHTFIRE', 'LEFTFIRE']\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/usr/local/lib/python3.8/dist-packages/ale_py/roms/__init__.py:119: DeprecationWarning: Automatic importing of atari-py roms won't be supported in future releases of ale-py. Please migrate over to using `ale-import-roms` OR an ALE-supported ROM package. To make this warning disappear you can run `ale-import-roms --import-from-pkg atari_py.atari_roms`. For more information see: https://github.com/mgbellemare/Arcade-Learning-Environment#rom-management\n",
      "  path = _resolve_rom(name)\n"
     ]
    }
   ],
   "source": [
    "# =======================================================================================================\n",
    "# General Hyperparameters: here one can, after declaring an environment, put a constraints on the actions \n",
    "# to be used.\n",
    "# =======================================================================================================\n",
    "env_name          = 'ALE/Pong-v5'\n",
    "save_dir          = 'Pong/'\n",
    "temp_env          = gym.make(env_name)\n",
    "actions_meaning   = temp_env.unwrapped.get_action_meanings()\n",
    "possible_actions  = [i for i in range(temp_env.action_space.n)]\n",
    "available_actions = possible_actions\n",
    "selected_actions  = [actions_meaning[available_actions[i]] for i in range(len(available_actions))]\n",
    "print(f\"Possible Actions: {possible_actions}\")\n",
    "print(f\"Selected Actions: {selected_actions}\")\n",
    "# =======================================================================================================\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {
    "scrolled": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Removed Pong/output.txt\n",
      "Worker [0] started.\n",
      "Worker [1] started.\n",
      "Worker [2] started.\n",
      "Worker [3] started.\n",
      "Worker [4] started.\n",
      "Worker [5] started.\n",
      "Worker [6] started.\n",
      "Worker [7] started.\n",
      "Worker [8] started.\n",
      "Worker [9] started.\n",
      "Worker [10] started.\n",
      "Worker [11] started.\n",
      "Worker [12] started.\n",
      "Worker [13] started.\n",
      "Worker [14] started.\n",
      "Worker [15] started.\n"
     ]
    }
   ],
   "source": [
    "# Controller initialization with hyperparameters\n",
    "# =======================================================================================================\n",
    "agent = Controller(\n",
    "    # Environment name\n",
    "    env_name             = env_name,\n",
    "    # Directory where progress is saved\n",
    "    save_dir             = save_dir,\n",
    "    # List of available actions\n",
    "    available_actions    = available_actions,\n",
    "    # Interval of episodes after which global model plays a game and prints output\n",
    "    play_interval        = 10,\n",
    "    # Cardinality of the set upon which the average is computed\n",
    "    moving_average_card  = 50,\n",
    "    \n",
    "    \n",
    "    # Number of threads\n",
    "    number_of_workers    = 16,\n",
    "# =======================================================================================================\n",
    "    # Learning Rate\n",
    "    lr_schedule          = 7.5e-4,\n",
    "    # Global Clipnorm\n",
    "    global_clipnorm      = 1,\n",
    "    # Decay\n",
    "    rho                  = 0.99,\n",
    "    # Centered optimizer\n",
    "    centered             = False,\n",
    "# =======================================================================================================\n",
    "    # Normalize advantages (subtract mean and divide by standard deviation)\n",
    "    normalize_advantages = False,\n",
    "    # Use Generalized Advantage Estimate instead of Standard one\n",
    "    use_gae              = True,\n",
    "# =======================================================================================================\n",
    "    # Max length for training\n",
    "    max_eps              = 10_000,\n",
    "    # Frequency of neural network updates\n",
    "    update_freq          = 16,\n",
    "    # Gamma\n",
    "    gamma                = 0.99,\n",
    "    # Lambda - GAE\n",
    "    exponential_discount = 0.95,\n",
    "    # Entropy factor\n",
    "    entropy_factor       = 0.01,\n",
    "    # Critic loss multiplier\n",
    "    critic_loss_factor   = 0.5,\n",
    "    # Epsilon a variable related to numerical stability in optimizer and normalization computations\n",
    "    # It's useful, for example, to avoid division by zeros or fading entropy (lim (x->0) x * log(x) = 0)\n",
    "    eps                  = 1e-7,\n",
    "# =======================================================================================================\n",
    "    # Grayscale images\n",
    "    grayscale            = True,\n",
    "    # Clip rewards to [-1, 0, 1]\n",
    "    clip_rewards         = True,\n",
    "    # Divide images input fed to neural networks for 255 so to have values in [0,1]\n",
    "    normalize            = True,\n",
    "    # Crop image to specified values\n",
    "    crop                 = True,      # value function needs something to distinguish states -> scores\n",
    "    x_low                = 0,          # default = 0\n",
    "    x_high               = 160,        # default = 160\n",
    "    y_low                = 34,         # default = 0\n",
    "    y_high               = 194,        # default = 210\n",
    "    # Resize screen to specified value\n",
    "    resize               = True,\n",
    "    screen_size          = 84,\n",
    "    # Denoise images\n",
    "    denoise              = True,\n",
    "    # Stack states so to have continuity and let neural network understand directions\n",
    "    stack_frames         = True,\n",
    "    # Number of frames to stack\n",
    "    n_frames             = 4,\n",
    "# =======================================================================================================\n",
    "    # Restore saved progress\n",
    "    restore              = False,\n",
    "    # Either restore from best (True) or latest (False)\n",
    "    restore_from_best    = False, \n",
    "# =======================================================================================================\n",
    ")\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [],
   "source": [
    "# agent.show_screen()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {
    "scrolled": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "WARNING:tensorflow:5 out of the last 5 calls to <function _BaseOptimizer._update_step_xla at 0x7f800c384280> triggered tf.function retracing. Tracing is expensive and the excessive number of tracings could be due to (1) creating @tf.function repeatedly in a loop, (2) passing tensors with different shapes, (3) passing Python objects instead of tensors. For (1), please define your @tf.function outside of the loop. For (2), @tf.function has reduce_retracing=True option that can avoid unnecessary retracing. For (3), please refer to https://www.tensorflow.org/guide/function#controlling_retracing and https://www.tensorflow.org/api_docs/python/tf/function for  more details.\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "WARNING:tensorflow:5 out of the last 5 calls to <function _BaseOptimizer._update_step_xla at 0x7f800c384280> triggered tf.function retracing. Tracing is expensive and the excessive number of tracings could be due to (1) creating @tf.function repeatedly in a loop, (2) passing tensors with different shapes, (3) passing Python objects instead of tensors. For (1), please define your @tf.function outside of the loop. For (2), @tf.function has reduce_retracing=True option that can avoid unnecessary retracing. For (3), please refer to https://www.tensorflow.org/guide/function#controlling_retracing and https://www.tensorflow.org/api_docs/python/tf/function for  more details.\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "WARNING:tensorflow:6 out of the last 6 calls to <function _BaseOptimizer._update_step_xla at 0x7f800c384280> triggered tf.function retracing. Tracing is expensive and the excessive number of tracings could be due to (1) creating @tf.function repeatedly in a loop, (2) passing tensors with different shapes, (3) passing Python objects instead of tensors. For (1), please define your @tf.function outside of the loop. For (2), @tf.function has reduce_retracing=True option that can avoid unnecessary retracing. For (3), please refer to https://www.tensorflow.org/guide/function#controlling_retracing and https://www.tensorflow.org/api_docs/python/tf/function for  more details.\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "WARNING:tensorflow:6 out of the last 6 calls to <function _BaseOptimizer._update_step_xla at 0x7f800c384280> triggered tf.function retracing. Tracing is expensive and the excessive number of tracings could be due to (1) creating @tf.function repeatedly in a loop, (2) passing tensors with different shapes, (3) passing Python objects instead of tensors. For (1), please define your @tf.function outside of the loop. For (2), @tf.function has reduce_retracing=True option that can avoid unnecessary retracing. For (3), please refer to https://www.tensorflow.org/guide/function#controlling_retracing and https://www.tensorflow.org/api_docs/python/tf/function for  more details.\n",
      "Exception in thread Thread-3:\n",
      "Traceback (most recent call last):\n",
      "  File \"/usr/lib/python3.8/threading.py\", line 932, in _bootstrap_inner\n",
      "    self.run()\n",
      "  File \"<ipython-input-8-af6184f58c44>\", line 256, in run\n",
      "  File \"<ipython-input-8-af6184f58c44>\", line 230, in step\n",
      "  File \"/usr/local/lib/python3.8/dist-packages/tensorflow/python/eager/backprop.py\", line 1112, in gradient\n",
      "    flat_grad = imperative_grad.imperative_grad(\n",
      "  File \"/usr/local/lib/python3.8/dist-packages/tensorflow/python/eager/imperative_grad.py\", line 67, in imperative_grad\n",
      "    return pywrap_tfe.TFE_Py_TapeGradient(\n",
      "  File \"/usr/local/lib/python3.8/dist-packages/tensorflow/python/eager/backprop.py\", line 157, in _gradient_function\n",
      "    return grad_fn(mock_op, *out_grads)\n",
      "  File \"/usr/local/lib/python3.8/dist-packages/tensorflow/python/ops/nn_grad.py\", line 413, in _ReluGrad\n",
      "    return gen_nn_ops.relu_grad(grad, op.outputs[0])\n",
      "  File \"/usr/local/lib/python3.8/dist-packages/tensorflow/python/ops/gen_nn_ops.py\", line 10911, in relu_grad\n",
      "    _ops.raise_from_not_ok_status(e, name)\n",
      "  File \"/usr/local/lib/python3.8/dist-packages/tensorflow/python/framework/ops.py\", line 7215, in raise_from_not_ok_status\n",
      "    raise core._status_to_exception(e) from None  # pylint: disable=protected-access\n",
      "tensorflow.python.framework.errors_impl.InvalidArgumentError: {{function_node __wrapped__ReluGrad_device_/job:localhost/replica:0/task:0/device:GPU:0}} Inputs to operation ReluGrad of type ReluGrad must have the same size and shape.  Input 0: [4,18,18,16] != input 1: [1,4,18,18,16] [Op:ReluGrad]\n",
      "Exception in thread Thread-12:\n",
      "Traceback (most recent call last):\n",
      "  File \"/usr/lib/python3.8/threading.py\", line 932, in _bootstrap_inner\n",
      "Exception in thread Thread-8:\n",
      "Traceback (most recent call last):\n",
      "  File \"/usr/lib/python3.8/threading.py\", line 932, in _bootstrap_inner\n",
      "        self.run()\n",
      "  File \"<ipython-input-8-af6184f58c44>\", line 256, in run\n",
      "self.run()\n",
      "  File \"<ipython-input-8-af6184f58c44>\", line 256, in run\n",
      "  File \"<ipython-input-8-af6184f58c44>\", line 228, in step\n",
      "  File \"<ipython-input-8-af6184f58c44>\", line 226, in step\n",
      "  File \"<ipython-input-8-af6184f58c44>\", line 193, in compute_loss\n",
      "  File \"<ipython-input-8-af6184f58c44>\", line 164, in get_generalized_advantage_estimate\n",
      "  File \"<ipython-input-8-af6184f58c44>\", line 116, in play_steps\n",
      "  File \"/usr/local/lib/python3.8/dist-packages/tensorflow/python/framework/ops.py\", line 7327, in __next__\n",
      "  File \"/usr/local/lib/python3.8/dist-packages/tensorflow/python/util/traceback_utils.py\", line 153, in error_handler\n",
      "    raise e.with_traceback(filtered_tb) from None\n",
      "  File \"/usr/local/lib/python3.8/dist-packages/tensorflow/python/framework/ops.py\", line 7215, in raise_from_not_ok_status\n",
      "    result = self._tensor[self._index]\n",
      "  File \"/usr/local/lib/python3.8/dist-packages/tensorflow/python/util/traceback_utils.py\", line 153, in error_handler\n",
      "    raise e.with_traceback(filtered_tb) from None\n",
      "  File \"/usr/local/lib/python3.8/dist-packages/tensorflow/python/framework/ops.py\", line 7215, in raise_from_not_ok_status\n",
      "    raise core._status_to_exception(e) from None  # pylint: disable=protected-access\n",
      "tensorflow.python.framework.errors_impl.InvalidArgumentError: {{function_node __wrapped__StridedSlice_device_/job:localhost/replica:0/task:0/device:GPU:0}} slice index 13 of dimension 0 out of bounds. [Op:StridedSlice] name: strided_slice/\n",
      "    raise core._status_to_exception(e) from None  # pylint: disable=protected-access\n",
      "tensorflow.python.framework.errors_impl.InvalidArgumentError: {{function_node __wrapped__StridedSlice_device_/job:localhost/replica:0/task:0/device:GPU:0}} slice index 13 of dimension 0 out of bounds. [Op:StridedSlice] name: strided_slice/\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "ERROR:tensorflow:==================================\n",
      "Object was never used (type <class 'tensorflow.python.ops.tensor_array_ops.TensorArray'>):\n",
      "<tensorflow.python.ops.tensor_array_ops.TensorArray object at 0x7f800c14ee80>\n",
      "If you want to mark it as used call its \"mark_used()\" method.\n",
      "It was originally created here:\n",
      "  File \"/usr/lib/python3.8/threading.py\", line 942, in _bootstrap_inner\n",
      "    pass  File \"<ipython-input-8-af6184f58c44>\", line 256, in run\n",
      "    state, done, grads = self.step(state)  File \"<ipython-input-8-af6184f58c44>\", line 228, in step\n",
      "    total_loss = self.compute_loss(logits, values, rewards, dones, actions)  File \"<ipython-input-8-af6184f58c44>\", line 116, in play_steps\n",
      "    for t in tf.range(self.update_freq):  File \"/usr/local/lib/python3.8/dist-packages/tensorflow/python/util/tf_should_use.py\", line 243, in wrapped\n",
      "    return _add_should_use_warning(fn(*args, **kwargs),\n",
      "==================================\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "ERROR:tensorflow:==================================\n",
      "Object was never used (type <class 'tensorflow.python.ops.tensor_array_ops.TensorArray'>):\n",
      "<tensorflow.python.ops.tensor_array_ops.TensorArray object at 0x7f800c14ee80>\n",
      "If you want to mark it as used call its \"mark_used()\" method.\n",
      "It was originally created here:\n",
      "  File \"/usr/lib/python3.8/threading.py\", line 942, in _bootstrap_inner\n",
      "    pass  File \"<ipython-input-8-af6184f58c44>\", line 256, in run\n",
      "    state, done, grads = self.step(state)  File \"<ipython-input-8-af6184f58c44>\", line 228, in step\n",
      "    total_loss = self.compute_loss(logits, values, rewards, dones, actions)  File \"<ipython-input-8-af6184f58c44>\", line 116, in play_steps\n",
      "    for t in tf.range(self.update_freq):  File \"/usr/local/lib/python3.8/dist-packages/tensorflow/python/util/tf_should_use.py\", line 243, in wrapped\n",
      "    return _add_should_use_warning(fn(*args, **kwargs),\n",
      "==================================\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "ERROR:tensorflow:==================================\n",
      "Object was never used (type <class 'tensorflow.python.ops.tensor_array_ops.TensorArray'>):\n",
      "<tensorflow.python.ops.tensor_array_ops.TensorArray object at 0x7f800c14e400>\n",
      "If you want to mark it as used call its \"mark_used()\" method.\n",
      "It was originally created here:\n",
      "  File \"/usr/lib/python3.8/threading.py\", line 942, in _bootstrap_inner\n",
      "    pass  File \"<ipython-input-8-af6184f58c44>\", line 256, in run\n",
      "    state, done, grads = self.step(state)  File \"<ipython-input-8-af6184f58c44>\", line 228, in step\n",
      "    total_loss = self.compute_loss(logits, values, rewards, dones, actions)  File \"<ipython-input-8-af6184f58c44>\", line 116, in play_steps\n",
      "    for t in tf.range(self.update_freq):  File \"/usr/local/lib/python3.8/dist-packages/tensorflow/python/util/tf_should_use.py\", line 243, in wrapped\n",
      "    return _add_should_use_warning(fn(*args, **kwargs),\n",
      "==================================\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "ERROR:tensorflow:==================================\n",
      "Object was never used (type <class 'tensorflow.python.ops.tensor_array_ops.TensorArray'>):\n",
      "<tensorflow.python.ops.tensor_array_ops.TensorArray object at 0x7f800c14e400>\n",
      "If you want to mark it as used call its \"mark_used()\" method.\n",
      "It was originally created here:\n",
      "  File \"/usr/lib/python3.8/threading.py\", line 942, in _bootstrap_inner\n",
      "    pass  File \"<ipython-input-8-af6184f58c44>\", line 256, in run\n",
      "    state, done, grads = self.step(state)  File \"<ipython-input-8-af6184f58c44>\", line 228, in step\n",
      "    total_loss = self.compute_loss(logits, values, rewards, dones, actions)  File \"<ipython-input-8-af6184f58c44>\", line 116, in play_steps\n",
      "    for t in tf.range(self.update_freq):  File \"/usr/local/lib/python3.8/dist-packages/tensorflow/python/util/tf_should_use.py\", line 243, in wrapped\n",
      "    return _add_should_use_warning(fn(*args, **kwargs),\n",
      "==================================\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "ERROR:tensorflow:==================================\n",
      "Object was never used (type <class 'tensorflow.python.ops.tensor_array_ops.TensorArray'>):\n",
      "<tensorflow.python.ops.tensor_array_ops.TensorArray object at 0x7f800c14ecd0>\n",
      "If you want to mark it as used call its \"mark_used()\" method.\n",
      "It was originally created here:\n",
      "  File \"/usr/lib/python3.8/threading.py\", line 942, in _bootstrap_inner\n",
      "    pass  File \"<ipython-input-8-af6184f58c44>\", line 256, in run\n",
      "    state, done, grads = self.step(state)  File \"<ipython-input-8-af6184f58c44>\", line 228, in step\n",
      "    total_loss = self.compute_loss(logits, values, rewards, dones, actions)  File \"<ipython-input-8-af6184f58c44>\", line 116, in play_steps\n",
      "    for t in tf.range(self.update_freq):  File \"/usr/local/lib/python3.8/dist-packages/tensorflow/python/util/tf_should_use.py\", line 243, in wrapped\n",
      "    return _add_should_use_warning(fn(*args, **kwargs),\n",
      "==================================\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "ERROR:tensorflow:==================================\n",
      "Object was never used (type <class 'tensorflow.python.ops.tensor_array_ops.TensorArray'>):\n",
      "<tensorflow.python.ops.tensor_array_ops.TensorArray object at 0x7f800c14ecd0>\n",
      "If you want to mark it as used call its \"mark_used()\" method.\n",
      "It was originally created here:\n",
      "  File \"/usr/lib/python3.8/threading.py\", line 942, in _bootstrap_inner\n",
      "    pass  File \"<ipython-input-8-af6184f58c44>\", line 256, in run\n",
      "    state, done, grads = self.step(state)  File \"<ipython-input-8-af6184f58c44>\", line 228, in step\n",
      "    total_loss = self.compute_loss(logits, values, rewards, dones, actions)  File \"<ipython-input-8-af6184f58c44>\", line 116, in play_steps\n",
      "    for t in tf.range(self.update_freq):  File \"/usr/local/lib/python3.8/dist-packages/tensorflow/python/util/tf_should_use.py\", line 243, in wrapped\n",
      "    return _add_should_use_warning(fn(*args, **kwargs),\n",
      "==================================\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "ERROR:tensorflow:==================================\n",
      "Object was never used (type <class 'tensorflow.python.ops.tensor_array_ops.TensorArray'>):\n",
      "<tensorflow.python.ops.tensor_array_ops.TensorArray object at 0x7f800c14ea00>\n",
      "If you want to mark it as used call its \"mark_used()\" method.\n",
      "It was originally created here:\n",
      "  File \"/usr/lib/python3.8/threading.py\", line 942, in _bootstrap_inner\n",
      "    pass  File \"<ipython-input-8-af6184f58c44>\", line 256, in run\n",
      "    state, done, grads = self.step(state)  File \"<ipython-input-8-af6184f58c44>\", line 228, in step\n",
      "    total_loss = self.compute_loss(logits, values, rewards, dones, actions)  File \"<ipython-input-8-af6184f58c44>\", line 116, in play_steps\n",
      "    for t in tf.range(self.update_freq):  File \"/usr/local/lib/python3.8/dist-packages/tensorflow/python/util/tf_should_use.py\", line 243, in wrapped\n",
      "    return _add_should_use_warning(fn(*args, **kwargs),\n",
      "==================================\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "ERROR:tensorflow:==================================\n",
      "Object was never used (type <class 'tensorflow.python.ops.tensor_array_ops.TensorArray'>):\n",
      "<tensorflow.python.ops.tensor_array_ops.TensorArray object at 0x7f800c14ea00>\n",
      "If you want to mark it as used call its \"mark_used()\" method.\n",
      "It was originally created here:\n",
      "  File \"/usr/lib/python3.8/threading.py\", line 942, in _bootstrap_inner\n",
      "    pass  File \"<ipython-input-8-af6184f58c44>\", line 256, in run\n",
      "    state, done, grads = self.step(state)  File \"<ipython-input-8-af6184f58c44>\", line 228, in step\n",
      "    total_loss = self.compute_loss(logits, values, rewards, dones, actions)  File \"<ipython-input-8-af6184f58c44>\", line 116, in play_steps\n",
      "    for t in tf.range(self.update_freq):  File \"/usr/local/lib/python3.8/dist-packages/tensorflow/python/util/tf_should_use.py\", line 243, in wrapped\n",
      "    return _add_should_use_warning(fn(*args, **kwargs),\n",
      "==================================\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "ERROR:tensorflow:==================================\n",
      "Object was never used (type <class 'tensorflow.python.ops.tensor_array_ops.TensorArray'>):\n",
      "<tensorflow.python.ops.tensor_array_ops.TensorArray object at 0x7f800c150df0>\n",
      "If you want to mark it as used call its \"mark_used()\" method.\n",
      "It was originally created here:\n",
      "  File \"/usr/lib/python3.8/threading.py\", line 942, in _bootstrap_inner\n",
      "    pass  File \"<ipython-input-8-af6184f58c44>\", line 256, in run\n",
      "    state, done, grads = self.step(state)  File \"<ipython-input-8-af6184f58c44>\", line 228, in step\n",
      "    total_loss = self.compute_loss(logits, values, rewards, dones, actions)  File \"<ipython-input-8-af6184f58c44>\", line 116, in play_steps\n",
      "    for t in tf.range(self.update_freq):  File \"/usr/local/lib/python3.8/dist-packages/tensorflow/python/util/tf_should_use.py\", line 243, in wrapped\n",
      "    return _add_should_use_warning(fn(*args, **kwargs),\n",
      "==================================\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "ERROR:tensorflow:==================================\n",
      "Object was never used (type <class 'tensorflow.python.ops.tensor_array_ops.TensorArray'>):\n",
      "<tensorflow.python.ops.tensor_array_ops.TensorArray object at 0x7f800c150df0>\n",
      "If you want to mark it as used call its \"mark_used()\" method.\n",
      "It was originally created here:\n",
      "  File \"/usr/lib/python3.8/threading.py\", line 942, in _bootstrap_inner\n",
      "    pass  File \"<ipython-input-8-af6184f58c44>\", line 256, in run\n",
      "    state, done, grads = self.step(state)  File \"<ipython-input-8-af6184f58c44>\", line 228, in step\n",
      "    total_loss = self.compute_loss(logits, values, rewards, dones, actions)  File \"<ipython-input-8-af6184f58c44>\", line 116, in play_steps\n",
      "    for t in tf.range(self.update_freq):  File \"/usr/local/lib/python3.8/dist-packages/tensorflow/python/util/tf_should_use.py\", line 243, in wrapped\n",
      "    return _add_should_use_warning(fn(*args, **kwargs),\n",
      "==================================\n",
      "Exception in thread Thread-6:\n",
      "Traceback (most recent call last):\n",
      "  File \"/usr/lib/python3.8/threading.py\", line 932, in _bootstrap_inner\n",
      "    self.run()\n",
      "  File \"<ipython-input-8-af6184f58c44>\", line 256, in run\n",
      "Exception in thread Thread-7:\n",
      "  File \"<ipython-input-8-af6184f58c44>\", line 230, in step\n",
      "Traceback (most recent call last):\n",
      "  File \"/usr/lib/python3.8/threading.py\", line 932, in _bootstrap_inner\n",
      "  File \"/usr/local/lib/python3.8/dist-packages/tensorflow/python/eager/backprop.py\", line 1112, in gradient\n",
      "    self.run()\n",
      "  File \"<ipython-input-8-af6184f58c44>\", line 256, in run\n",
      "    flat_grad = imperative_grad.imperative_grad(\n",
      "  File \"/usr/local/lib/python3.8/dist-packages/tensorflow/python/eager/imperative_grad.py\", line 67, in imperative_grad\n",
      "    return pywrap_tfe.TFE_Py_TapeGradient(  File \"<ipython-input-8-af6184f58c44>\", line 230, in step\n",
      "\n",
      "  File \"/usr/local/lib/python3.8/dist-packages/tensorflow/python/eager/backprop.py\", line 157, in _gradient_function\n",
      "    return grad_fn(mock_op, *out_grads)\n",
      "  File \"/usr/local/lib/python3.8/dist-packages/tensorflow/python/ops/array_grad.py\", line 801, in _ReshapeGrad\n",
      "    array_ops.reshape(\n",
      "  File \"/usr/local/lib/python3.8/dist-packages/tensorflow/python/util/traceback_utils.py\", line 153, in error_handler\n",
      "  File \"/usr/local/lib/python3.8/dist-packages/tensorflow/python/eager/backprop.py\", line 1112, in gradient\n",
      "    raise e.with_traceback(filtered_tb) from None\n",
      "      File \"/usr/local/lib/python3.8/dist-packages/tensorflow/python/framework/ops.py\", line 7215, in raise_from_not_ok_status\n",
      "flat_grad = imperative_grad.imperative_grad(\n",
      "  File \"/usr/local/lib/python3.8/dist-packages/tensorflow/python/eager/imperative_grad.py\", line 67, in imperative_grad\n",
      "    return pywrap_tfe.TFE_Py_TapeGradient(\n",
      "  File \"/usr/local/lib/python3.8/dist-packages/tensorflow/python/eager/backprop.py\", line 157, in _gradient_function\n",
      "    return grad_fn(mock_op, *out_grads)\n",
      "  File \"/usr/local/lib/python3.8/dist-packages/tensorflow/python/ops/array_grad.py\", line 801, in _ReshapeGrad\n",
      "    array_ops.reshape(\n",
      "  File \"/usr/local/lib/python3.8/dist-packages/tensorflow/python/util/traceback_utils.py\", line 153, in error_handler\n",
      "    raise e.with_traceback(filtered_tb) from None\n",
      "  File \"/usr/local/lib/python3.8/dist-packages/tensorflow/python/framework/ops.py\", line 7215, in raise_from_not_ok_status\n",
      "    raise core._status_to_exception(e) from None  # pylint: disable=protected-access\n",
      "tensorflow.python.framework.errors_impl.InvalidArgumentError: {{function_node __wrapped__Reshape_device_/job:localhost/replica:0/task:0/device:GPU:0}} Input to reshape is a tensor with 48672 values, but the requested shape has 16 [Op:Reshape]\n",
      "    raise core._status_to_exception(e) from None  # pylint: disable=protected-access\n",
      "tensorflow.python.framework.errors_impl.InvalidArgumentError: {{function_node __wrapped__Reshape_device_/job:localhost/replica:0/task:0/device:GPU:0}} Input to reshape is a tensor with 48672 values, but the requested shape has 16 [Op:Reshape]\n",
      "Exception in thread Thread-9:\n",
      "Traceback (most recent call last):\n",
      "  File \"/usr/lib/python3.8/threading.py\", line 932, in _bootstrap_inner\n",
      "    self.run()\n",
      "  File \"<ipython-input-8-af6184f58c44>\", line 256, in run\n",
      "  File \"<ipython-input-8-af6184f58c44>\", line 230, in step\n",
      "  File \"/usr/local/lib/python3.8/dist-packages/tensorflow/python/eager/backprop.py\", line 1112, in gradient\n",
      "    flat_grad = imperative_grad.imperative_grad(\n",
      "  File \"/usr/local/lib/python3.8/dist-packages/tensorflow/python/eager/imperative_grad.py\", line 67, in imperative_grad\n",
      "    return pywrap_tfe.TFE_Py_TapeGradient(\n",
      "  File \"/usr/local/lib/python3.8/dist-packages/tensorflow/python/eager/backprop.py\", line 157, in _gradient_function\n",
      "    return grad_fn(mock_op, *out_grads)\n",
      "  File \"/usr/local/lib/python3.8/dist-packages/tensorflow/python/ops/math_grad.py\", line 1384, in _MulGrad\n",
      "    gy = array_ops.reshape(\n",
      "  File \"/usr/local/lib/python3.8/dist-packages/tensorflow/python/util/traceback_utils.py\", line 153, in error_handler\n",
      "    raise e.with_traceback(filtered_tb) from None\n",
      "  File \"/usr/local/lib/python3.8/dist-packages/tensorflow/python/framework/ops.py\", line 7215, in raise_from_not_ok_status\n",
      "    raise core._status_to_exception(e) from None  # pylint: disable=protected-access\n",
      "tensorflow.python.framework.errors_impl.InvalidArgumentError: {{function_node __wrapped__Reshape_device_/job:localhost/replica:0/task:0/device:GPU:0}} Input to reshape is a tensor with 48672 values, but the requested shape has 16 [Op:Reshape]\n",
      "Exception in thread Thread-10:\n",
      "Traceback (most recent call last):\n",
      "  File \"/usr/lib/python3.8/threading.py\", line 932, in _bootstrap_inner\n",
      "    self.run()\n",
      "  File \"<ipython-input-8-af6184f58c44>\", line 256, in run\n",
      "  File \"<ipython-input-8-af6184f58c44>\", line 230, in step\n",
      "  File \"/usr/local/lib/python3.8/dist-packages/tensorflow/python/eager/backprop.py\", line 1112, in gradient\n",
      "    flat_grad = imperative_grad.imperative_grad(\n",
      "  File \"/usr/local/lib/python3.8/dist-packages/tensorflow/python/eager/imperative_grad.py\", line 67, in imperative_grad\n",
      "    return pywrap_tfe.TFE_Py_TapeGradient(\n",
      "  File \"/usr/local/lib/python3.8/dist-packages/tensorflow/python/eager/backprop.py\", line 157, in _gradient_function\n",
      "    return grad_fn(mock_op, *out_grads)\n",
      "  File \"/usr/local/lib/python3.8/dist-packages/tensorflow/python/ops/nn_grad.py\", line 413, in _ReluGrad\n",
      "    return gen_nn_ops.relu_grad(grad, op.outputs[0])\n",
      "  File \"/usr/local/lib/python3.8/dist-packages/tensorflow/python/ops/gen_nn_ops.py\", line 10911, in relu_grad\n",
      "    _ops.raise_from_not_ok_status(e, name)\n",
      "  File \"/usr/local/lib/python3.8/dist-packages/tensorflow/python/framework/ops.py\", line 7215, in raise_from_not_ok_status\n",
      "    raise core._status_to_exception(e) from None  # pylint: disable=protected-access\n",
      "tensorflow.python.framework.errors_impl.InvalidArgumentError: {{function_node __wrapped__ReluGrad_device_/job:localhost/replica:0/task:0/device:GPU:0}} Inputs to operation ReluGrad of type ReluGrad must have the same size and shape.  Input 0: [4,18,18,16] != input 1: [1,4,18,18,16] [Op:ReluGrad]\n",
      "Exception in thread Thread-4:\n",
      "Traceback (most recent call last):\n",
      "  File \"/usr/lib/python3.8/threading.py\", line 932, in _bootstrap_inner\n",
      "    self.run()\n",
      "  File \"<ipython-input-8-af6184f58c44>\", line 256, in run\n",
      "  File \"<ipython-input-8-af6184f58c44>\", line 226, in step\n",
      "  File \"<ipython-input-8-af6184f58c44>\", line 118, in play_steps\n",
      "  File \"/usr/local/lib/python3.8/dist-packages/keras/utils/traceback_utils.py\", line 70, in error_handler\n",
      "    raise e.with_traceback(filtered_tb) from None\n",
      "  File \"/usr/local/lib/python3.8/dist-packages/tensorflow/python/framework/ops.py\", line 1299, in set_shape\n",
      "    raise ValueError(f\"Tensor's shape {self.shape} is not compatible \"\n",
      "ValueError: Exception encountered when calling layer 'conv2d_8' (type Conv2D).\n",
      "\n",
      "Tensor's shape (4, 5, 5, 32) is not compatible with supplied shape (1, 4, 5, 5, 32).\n",
      "\n",
      "Call arguments received by layer 'conv2d_8' (type Conv2D):\n",
      "  • inputs=tf.Tensor(shape=(1, 4, 18, 18, 16), dtype=float32)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "ERROR:tensorflow:==================================\n",
      "Object was never used (type <class 'tensorflow.python.ops.tensor_array_ops.TensorArray'>):\n",
      "<tensorflow.python.ops.tensor_array_ops.TensorArray object at 0x7f800c4ddb80>\n",
      "If you want to mark it as used call its \"mark_used()\" method.\n",
      "It was originally created here:\n",
      "  File \"/usr/lib/python3.8/threading.py\", line 942, in _bootstrap_inner\n",
      "    pass  File \"<ipython-input-8-af6184f58c44>\", line 256, in run\n",
      "    state, done, grads = self.step(state)  File \"<ipython-input-8-af6184f58c44>\", line 228, in step\n",
      "    total_loss = self.compute_loss(logits, values, rewards, dones, actions)  File \"<ipython-input-8-af6184f58c44>\", line 118, in play_steps\n",
      "    logit, value        = self.local_model(state)  File \"/usr/local/lib/python3.8/dist-packages/tensorflow/python/util/tf_should_use.py\", line 243, in wrapped\n",
      "    return _add_should_use_warning(fn(*args, **kwargs),\n",
      "==================================\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "ERROR:tensorflow:==================================\n",
      "Object was never used (type <class 'tensorflow.python.ops.tensor_array_ops.TensorArray'>):\n",
      "<tensorflow.python.ops.tensor_array_ops.TensorArray object at 0x7f800c4ddb80>\n",
      "If you want to mark it as used call its \"mark_used()\" method.\n",
      "It was originally created here:\n",
      "  File \"/usr/lib/python3.8/threading.py\", line 942, in _bootstrap_inner\n",
      "    pass  File \"<ipython-input-8-af6184f58c44>\", line 256, in run\n",
      "    state, done, grads = self.step(state)  File \"<ipython-input-8-af6184f58c44>\", line 228, in step\n",
      "    total_loss = self.compute_loss(logits, values, rewards, dones, actions)  File \"<ipython-input-8-af6184f58c44>\", line 118, in play_steps\n",
      "    logit, value        = self.local_model(state)  File \"/usr/local/lib/python3.8/dist-packages/tensorflow/python/util/tf_should_use.py\", line 243, in wrapped\n",
      "    return _add_should_use_warning(fn(*args, **kwargs),\n",
      "==================================\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "ERROR:tensorflow:==================================\n",
      "Object was never used (type <class 'tensorflow.python.ops.tensor_array_ops.TensorArray'>):\n",
      "<tensorflow.python.ops.tensor_array_ops.TensorArray object at 0x7f800c475f70>\n",
      "If you want to mark it as used call its \"mark_used()\" method.\n",
      "It was originally created here:\n",
      "  File \"/usr/lib/python3.8/threading.py\", line 942, in _bootstrap_inner\n",
      "    pass  File \"<ipython-input-8-af6184f58c44>\", line 256, in run\n",
      "    state, done, grads = self.step(state)  File \"<ipython-input-8-af6184f58c44>\", line 228, in step\n",
      "    total_loss = self.compute_loss(logits, values, rewards, dones, actions)  File \"<ipython-input-8-af6184f58c44>\", line 118, in play_steps\n",
      "    logit, value        = self.local_model(state)  File \"/usr/local/lib/python3.8/dist-packages/tensorflow/python/util/tf_should_use.py\", line 243, in wrapped\n",
      "    return _add_should_use_warning(fn(*args, **kwargs),\n",
      "==================================\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "ERROR:tensorflow:==================================\n",
      "Object was never used (type <class 'tensorflow.python.ops.tensor_array_ops.TensorArray'>):\n",
      "<tensorflow.python.ops.tensor_array_ops.TensorArray object at 0x7f800c475f70>\n",
      "If you want to mark it as used call its \"mark_used()\" method.\n",
      "It was originally created here:\n",
      "  File \"/usr/lib/python3.8/threading.py\", line 942, in _bootstrap_inner\n",
      "    pass  File \"<ipython-input-8-af6184f58c44>\", line 256, in run\n",
      "    state, done, grads = self.step(state)  File \"<ipython-input-8-af6184f58c44>\", line 228, in step\n",
      "    total_loss = self.compute_loss(logits, values, rewards, dones, actions)  File \"<ipython-input-8-af6184f58c44>\", line 118, in play_steps\n",
      "    logit, value        = self.local_model(state)  File \"/usr/local/lib/python3.8/dist-packages/tensorflow/python/util/tf_should_use.py\", line 243, in wrapped\n",
      "    return _add_should_use_warning(fn(*args, **kwargs),\n",
      "==================================\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "ERROR:tensorflow:==================================\n",
      "Object was never used (type <class 'tensorflow.python.ops.tensor_array_ops.TensorArray'>):\n",
      "<tensorflow.python.ops.tensor_array_ops.TensorArray object at 0x7f800c475bb0>\n",
      "If you want to mark it as used call its \"mark_used()\" method.\n",
      "It was originally created here:\n",
      "  File \"/usr/lib/python3.8/threading.py\", line 942, in _bootstrap_inner\n",
      "    pass  File \"<ipython-input-8-af6184f58c44>\", line 256, in run\n",
      "    state, done, grads = self.step(state)  File \"<ipython-input-8-af6184f58c44>\", line 228, in step\n",
      "    total_loss = self.compute_loss(logits, values, rewards, dones, actions)  File \"<ipython-input-8-af6184f58c44>\", line 118, in play_steps\n",
      "    logit, value        = self.local_model(state)  File \"/usr/local/lib/python3.8/dist-packages/tensorflow/python/util/tf_should_use.py\", line 243, in wrapped\n",
      "    return _add_should_use_warning(fn(*args, **kwargs),\n",
      "==================================\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "ERROR:tensorflow:==================================\n",
      "Object was never used (type <class 'tensorflow.python.ops.tensor_array_ops.TensorArray'>):\n",
      "<tensorflow.python.ops.tensor_array_ops.TensorArray object at 0x7f800c475bb0>\n",
      "If you want to mark it as used call its \"mark_used()\" method.\n",
      "It was originally created here:\n",
      "  File \"/usr/lib/python3.8/threading.py\", line 942, in _bootstrap_inner\n",
      "    pass  File \"<ipython-input-8-af6184f58c44>\", line 256, in run\n",
      "    state, done, grads = self.step(state)  File \"<ipython-input-8-af6184f58c44>\", line 228, in step\n",
      "    total_loss = self.compute_loss(logits, values, rewards, dones, actions)  File \"<ipython-input-8-af6184f58c44>\", line 118, in play_steps\n",
      "    logit, value        = self.local_model(state)  File \"/usr/local/lib/python3.8/dist-packages/tensorflow/python/util/tf_should_use.py\", line 243, in wrapped\n",
      "    return _add_should_use_warning(fn(*args, **kwargs),\n",
      "==================================\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "ERROR:tensorflow:==================================\n",
      "Object was never used (type <class 'tensorflow.python.ops.tensor_array_ops.TensorArray'>):\n",
      "<tensorflow.python.ops.tensor_array_ops.TensorArray object at 0x7f800c3bdee0>\n",
      "If you want to mark it as used call its \"mark_used()\" method.\n",
      "It was originally created here:\n",
      "  File \"/usr/lib/python3.8/threading.py\", line 942, in _bootstrap_inner\n",
      "    pass  File \"<ipython-input-8-af6184f58c44>\", line 256, in run\n",
      "    state, done, grads = self.step(state)  File \"<ipython-input-8-af6184f58c44>\", line 228, in step\n",
      "    total_loss = self.compute_loss(logits, values, rewards, dones, actions)  File \"<ipython-input-8-af6184f58c44>\", line 118, in play_steps\n",
      "    logit, value        = self.local_model(state)  File \"/usr/local/lib/python3.8/dist-packages/tensorflow/python/util/tf_should_use.py\", line 243, in wrapped\n",
      "    return _add_should_use_warning(fn(*args, **kwargs),\n",
      "==================================\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "ERROR:tensorflow:==================================\n",
      "Object was never used (type <class 'tensorflow.python.ops.tensor_array_ops.TensorArray'>):\n",
      "<tensorflow.python.ops.tensor_array_ops.TensorArray object at 0x7f800c3bdee0>\n",
      "If you want to mark it as used call its \"mark_used()\" method.\n",
      "It was originally created here:\n",
      "  File \"/usr/lib/python3.8/threading.py\", line 942, in _bootstrap_inner\n",
      "    pass  File \"<ipython-input-8-af6184f58c44>\", line 256, in run\n",
      "    state, done, grads = self.step(state)  File \"<ipython-input-8-af6184f58c44>\", line 228, in step\n",
      "    total_loss = self.compute_loss(logits, values, rewards, dones, actions)  File \"<ipython-input-8-af6184f58c44>\", line 118, in play_steps\n",
      "    logit, value        = self.local_model(state)  File \"/usr/local/lib/python3.8/dist-packages/tensorflow/python/util/tf_should_use.py\", line 243, in wrapped\n",
      "    return _add_should_use_warning(fn(*args, **kwargs),\n",
      "==================================\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "ERROR:tensorflow:==================================\n",
      "Object was never used (type <class 'tensorflow.python.ops.tensor_array_ops.TensorArray'>):\n",
      "<tensorflow.python.ops.tensor_array_ops.TensorArray object at 0x7f800c3bdfd0>\n",
      "If you want to mark it as used call its \"mark_used()\" method.\n",
      "It was originally created here:\n",
      "  File \"/usr/lib/python3.8/threading.py\", line 942, in _bootstrap_inner\n",
      "    pass  File \"<ipython-input-8-af6184f58c44>\", line 256, in run\n",
      "    state, done, grads = self.step(state)  File \"<ipython-input-8-af6184f58c44>\", line 228, in step\n",
      "    total_loss = self.compute_loss(logits, values, rewards, dones, actions)  File \"<ipython-input-8-af6184f58c44>\", line 118, in play_steps\n",
      "    logit, value        = self.local_model(state)  File \"/usr/local/lib/python3.8/dist-packages/tensorflow/python/util/tf_should_use.py\", line 243, in wrapped\n",
      "    return _add_should_use_warning(fn(*args, **kwargs),\n",
      "==================================\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "ERROR:tensorflow:==================================\n",
      "Object was never used (type <class 'tensorflow.python.ops.tensor_array_ops.TensorArray'>):\n",
      "<tensorflow.python.ops.tensor_array_ops.TensorArray object at 0x7f800c3bdfd0>\n",
      "If you want to mark it as used call its \"mark_used()\" method.\n",
      "It was originally created here:\n",
      "  File \"/usr/lib/python3.8/threading.py\", line 942, in _bootstrap_inner\n",
      "    pass  File \"<ipython-input-8-af6184f58c44>\", line 256, in run\n",
      "    state, done, grads = self.step(state)  File \"<ipython-input-8-af6184f58c44>\", line 228, in step\n",
      "    total_loss = self.compute_loss(logits, values, rewards, dones, actions)  File \"<ipython-input-8-af6184f58c44>\", line 118, in play_steps\n",
      "    logit, value        = self.local_model(state)  File \"/usr/local/lib/python3.8/dist-packages/tensorflow/python/util/tf_should_use.py\", line 243, in wrapped\n",
      "    return _add_should_use_warning(fn(*args, **kwargs),\n",
      "==================================\n",
      "Exception in thread Thread-14:\n",
      "Traceback (most recent call last):\n",
      "  File \"/usr/lib/python3.8/threading.py\", line 932, in _bootstrap_inner\n",
      "    self.run()\n",
      "  File \"<ipython-input-8-af6184f58c44>\", line 256, in run\n",
      "  File \"<ipython-input-8-af6184f58c44>\", line 230, in step\n",
      "  File \"/usr/local/lib/python3.8/dist-packages/tensorflow/python/eager/backprop.py\", line 1112, in gradient\n",
      "    flat_grad = imperative_grad.imperative_grad(\n",
      "  File \"/usr/local/lib/python3.8/dist-packages/tensorflow/python/eager/imperative_grad.py\", line 67, in imperative_grad\n",
      "    return pywrap_tfe.TFE_Py_TapeGradient(\n",
      "  File \"/usr/local/lib/python3.8/dist-packages/tensorflow/python/eager/backprop.py\", line 157, in _gradient_function\n",
      "    return grad_fn(mock_op, *out_grads)\n",
      "  File \"/usr/local/lib/python3.8/dist-packages/tensorflow/python/ops/array_grad.py\", line 801, in _ReshapeGrad\n",
      "    array_ops.reshape(\n",
      "  File \"/usr/local/lib/python3.8/dist-packages/tensorflow/python/util/traceback_utils.py\", line 153, in error_handler\n",
      "    raise e.with_traceback(filtered_tb) from None\n",
      "  File \"/usr/local/lib/python3.8/dist-packages/tensorflow/python/framework/ops.py\", line 7215, in raise_from_not_ok_status\n",
      "    raise core._status_to_exception(e) from None  # pylint: disable=protected-access\n",
      "Exception in thread Thread-17:\n",
      "Traceback (most recent call last):\n",
      "  File \"/usr/lib/python3.8/threading.py\", line 932, in _bootstrap_inner\n",
      "tensorflow.python.framework.errors_impl.InvalidArgumentError: {{function_node __wrapped__Reshape_device_/job:localhost/replica:0/task:0/device:GPU:0}} Input to reshape is a tensor with 48672 values, but the requested shape has 3200 [Op:Reshape]\n",
      "    self.run()\n",
      "  File \"<ipython-input-8-af6184f58c44>\", line 256, in run\n",
      "  File \"<ipython-input-8-af6184f58c44>\", line 230, in step\n",
      "  File \"/usr/local/lib/python3.8/dist-packages/tensorflow/python/eager/backprop.py\", line 1112, in gradient\n",
      "    flat_grad = imperative_grad.imperative_grad(\n",
      "  File \"/usr/local/lib/python3.8/dist-packages/tensorflow/python/eager/imperative_grad.py\", line 67, in imperative_grad\n",
      "    return pywrap_tfe.TFE_Py_TapeGradient(\n",
      "  File \"/usr/local/lib/python3.8/dist-packages/tensorflow/python/eager/backprop.py\", line 157, in _gradient_function\n",
      "    return grad_fn(mock_op, *out_grads)\n",
      "  File \"/usr/local/lib/python3.8/dist-packages/tensorflow/python/ops/array_grad.py\", line 801, in _ReshapeGrad\n",
      "    array_ops.reshape(\n",
      "  File \"/usr/local/lib/python3.8/dist-packages/tensorflow/python/util/traceback_utils.py\", line 153, in error_handler\n",
      "    raise e.with_traceback(filtered_tb) from None\n",
      "  File \"/usr/local/lib/python3.8/dist-packages/tensorflow/python/framework/ops.py\", line 7215, in raise_from_not_ok_status\n",
      "    raise core._status_to_exception(e) from None  # pylint: disable=protected-access\n",
      "tensorflow.python.framework.errors_impl.InvalidArgumentError: {{function_node __wrapped__Reshape_device_/job:localhost/replica:0/task:0/device:GPU:0}} Input to reshape is a tensor with 48672 values, but the requested shape has 3200 [Op:Reshape]\n",
      "Exception in thread Thread-18:\n",
      "Traceback (most recent call last):\n",
      "  File \"/usr/lib/python3.8/threading.py\", line 932, in _bootstrap_inner\n",
      "    self.run()\n",
      "  File \"<ipython-input-8-af6184f58c44>\", line 256, in run\n",
      "  File \"<ipython-input-8-af6184f58c44>\", line 226, in step\n",
      "  File \"<ipython-input-8-af6184f58c44>\", line 118, in play_steps\n",
      "  File \"/usr/local/lib/python3.8/dist-packages/keras/utils/traceback_utils.py\", line 70, in error_handler\n",
      "    raise e.with_traceback(filtered_tb) from None\n",
      "  File \"/usr/local/lib/python3.8/dist-packages/tensorflow/python/framework/ops.py\", line 7215, in raise_from_not_ok_status\n",
      "    raise core._status_to_exception(e) from None  # pylint: disable=protected-access\n",
      "tensorflow.python.framework.errors_impl.InvalidArgumentError: Exception encountered when calling layer 'conv2d_48' (type Conv2D).\n",
      "\n",
      "{{function_node __wrapped__Reshape_device_/job:localhost/replica:0/task:0/device:GPU:0}} Input to reshape is a tensor with 48672 values, but the requested shape has 20736 [Op:Reshape]\n",
      "\n",
      "Call arguments received by layer 'conv2d_48' (type Conv2D):\n",
      "  • inputs=tf.Tensor(shape=(1, 4, 84, 84, 1), dtype=float32)\n",
      "Exception in thread Thread-15:\n",
      "Traceback (most recent call last):\n",
      "  File \"/usr/lib/python3.8/threading.py\", line 932, in _bootstrap_inner\n",
      "    self.run()\n",
      "  File \"<ipython-input-8-af6184f58c44>\", line 256, in run\n",
      "  File \"<ipython-input-8-af6184f58c44>\", line 230, in step\n",
      "  File \"/usr/local/lib/python3.8/dist-packages/tensorflow/python/eager/backprop.py\", line 1112, in gradient\n",
      "    flat_grad = imperative_grad.imperative_grad(\n",
      "  File \"/usr/local/lib/python3.8/dist-packages/tensorflow/python/eager/imperative_grad.py\", line 67, in imperative_grad\n",
      "    return pywrap_tfe.TFE_Py_TapeGradient(\n",
      "  File \"/usr/local/lib/python3.8/dist-packages/tensorflow/python/eager/backprop.py\", line 157, in _gradient_function\n",
      "    return grad_fn(mock_op, *out_grads)\n",
      "  File \"/usr/local/lib/python3.8/dist-packages/tensorflow/python/ops/array_grad.py\", line 801, in _ReshapeGrad\n",
      "    array_ops.reshape(\n",
      "  File \"/usr/local/lib/python3.8/dist-packages/tensorflow/python/util/traceback_utils.py\", line 153, in error_handler\n",
      "    raise e.with_traceback(filtered_tb) from None\n",
      "  File \"/usr/local/lib/python3.8/dist-packages/tensorflow/python/framework/ops.py\", line 7215, in raise_from_not_ok_status\n",
      "    raise core._status_to_exception(e) from None  # pylint: disable=protected-access\n",
      "tensorflow.python.framework.errors_impl.InvalidArgumentError: {{function_node __wrapped__Reshape_device_/job:localhost/replica:0/task:0/device:GPU:0}} Input to reshape is a tensor with 48672 values, but the requested shape has 20736 [Op:Reshape]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "ERROR:tensorflow:==================================\n",
      "Object was never used (type <class 'tensorflow.python.ops.tensor_array_ops.TensorArray'>):\n",
      "<tensorflow.python.ops.tensor_array_ops.TensorArray object at 0x7f800c485280>\n",
      "If you want to mark it as used call its \"mark_used()\" method.\n",
      "It was originally created here:\n",
      "  File \"/usr/lib/python3.8/threading.py\", line 942, in _bootstrap_inner\n",
      "    pass  File \"<ipython-input-8-af6184f58c44>\", line 256, in run\n",
      "    state, done, grads = self.step(state)  File \"<ipython-input-8-af6184f58c44>\", line 228, in step\n",
      "    total_loss = self.compute_loss(logits, values, rewards, dones, actions)  File \"<ipython-input-8-af6184f58c44>\", line 118, in play_steps\n",
      "    logit, value        = self.local_model(state)  File \"/usr/local/lib/python3.8/dist-packages/tensorflow/python/util/tf_should_use.py\", line 243, in wrapped\n",
      "    return _add_should_use_warning(fn(*args, **kwargs),\n",
      "==================================\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "ERROR:tensorflow:==================================\n",
      "Object was never used (type <class 'tensorflow.python.ops.tensor_array_ops.TensorArray'>):\n",
      "<tensorflow.python.ops.tensor_array_ops.TensorArray object at 0x7f800c485280>\n",
      "If you want to mark it as used call its \"mark_used()\" method.\n",
      "It was originally created here:\n",
      "  File \"/usr/lib/python3.8/threading.py\", line 942, in _bootstrap_inner\n",
      "    pass  File \"<ipython-input-8-af6184f58c44>\", line 256, in run\n",
      "    state, done, grads = self.step(state)  File \"<ipython-input-8-af6184f58c44>\", line 228, in step\n",
      "    total_loss = self.compute_loss(logits, values, rewards, dones, actions)  File \"<ipython-input-8-af6184f58c44>\", line 118, in play_steps\n",
      "    logit, value        = self.local_model(state)  File \"/usr/local/lib/python3.8/dist-packages/tensorflow/python/util/tf_should_use.py\", line 243, in wrapped\n",
      "    return _add_should_use_warning(fn(*args, **kwargs),\n",
      "==================================\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "ERROR:tensorflow:==================================\n",
      "Object was never used (type <class 'tensorflow.python.ops.tensor_array_ops.TensorArray'>):\n",
      "<tensorflow.python.ops.tensor_array_ops.TensorArray object at 0x7f800c41ed60>\n",
      "If you want to mark it as used call its \"mark_used()\" method.\n",
      "It was originally created here:\n",
      "  File \"/usr/lib/python3.8/threading.py\", line 942, in _bootstrap_inner\n",
      "    pass  File \"<ipython-input-8-af6184f58c44>\", line 256, in run\n",
      "    state, done, grads = self.step(state)  File \"<ipython-input-8-af6184f58c44>\", line 228, in step\n",
      "    total_loss = self.compute_loss(logits, values, rewards, dones, actions)  File \"<ipython-input-8-af6184f58c44>\", line 118, in play_steps\n",
      "    logit, value        = self.local_model(state)  File \"/usr/local/lib/python3.8/dist-packages/tensorflow/python/util/tf_should_use.py\", line 243, in wrapped\n",
      "    return _add_should_use_warning(fn(*args, **kwargs),\n",
      "==================================\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "ERROR:tensorflow:==================================\n",
      "Object was never used (type <class 'tensorflow.python.ops.tensor_array_ops.TensorArray'>):\n",
      "<tensorflow.python.ops.tensor_array_ops.TensorArray object at 0x7f800c41ed60>\n",
      "If you want to mark it as used call its \"mark_used()\" method.\n",
      "It was originally created here:\n",
      "  File \"/usr/lib/python3.8/threading.py\", line 942, in _bootstrap_inner\n",
      "    pass  File \"<ipython-input-8-af6184f58c44>\", line 256, in run\n",
      "    state, done, grads = self.step(state)  File \"<ipython-input-8-af6184f58c44>\", line 228, in step\n",
      "    total_loss = self.compute_loss(logits, values, rewards, dones, actions)  File \"<ipython-input-8-af6184f58c44>\", line 118, in play_steps\n",
      "    logit, value        = self.local_model(state)  File \"/usr/local/lib/python3.8/dist-packages/tensorflow/python/util/tf_should_use.py\", line 243, in wrapped\n",
      "    return _add_should_use_warning(fn(*args, **kwargs),\n",
      "==================================\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "ERROR:tensorflow:==================================\n",
      "Object was never used (type <class 'tensorflow.python.ops.tensor_array_ops.TensorArray'>):\n",
      "<tensorflow.python.ops.tensor_array_ops.TensorArray object at 0x7f8033407040>\n",
      "If you want to mark it as used call its \"mark_used()\" method.\n",
      "It was originally created here:\n",
      "  File \"/usr/lib/python3.8/threading.py\", line 942, in _bootstrap_inner\n",
      "    pass  File \"<ipython-input-8-af6184f58c44>\", line 256, in run\n",
      "    state, done, grads = self.step(state)  File \"<ipython-input-8-af6184f58c44>\", line 228, in step\n",
      "    total_loss = self.compute_loss(logits, values, rewards, dones, actions)  File \"<ipython-input-8-af6184f58c44>\", line 118, in play_steps\n",
      "    logit, value        = self.local_model(state)  File \"/usr/local/lib/python3.8/dist-packages/tensorflow/python/util/tf_should_use.py\", line 243, in wrapped\n",
      "    return _add_should_use_warning(fn(*args, **kwargs),\n",
      "==================================\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "ERROR:tensorflow:==================================\n",
      "Object was never used (type <class 'tensorflow.python.ops.tensor_array_ops.TensorArray'>):\n",
      "<tensorflow.python.ops.tensor_array_ops.TensorArray object at 0x7f8033407040>\n",
      "If you want to mark it as used call its \"mark_used()\" method.\n",
      "It was originally created here:\n",
      "  File \"/usr/lib/python3.8/threading.py\", line 942, in _bootstrap_inner\n",
      "    pass  File \"<ipython-input-8-af6184f58c44>\", line 256, in run\n",
      "    state, done, grads = self.step(state)  File \"<ipython-input-8-af6184f58c44>\", line 228, in step\n",
      "    total_loss = self.compute_loss(logits, values, rewards, dones, actions)  File \"<ipython-input-8-af6184f58c44>\", line 118, in play_steps\n",
      "    logit, value        = self.local_model(state)  File \"/usr/local/lib/python3.8/dist-packages/tensorflow/python/util/tf_should_use.py\", line 243, in wrapped\n",
      "    return _add_should_use_warning(fn(*args, **kwargs),\n",
      "==================================\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "ERROR:tensorflow:==================================\n",
      "Object was never used (type <class 'tensorflow.python.ops.tensor_array_ops.TensorArray'>):\n",
      "<tensorflow.python.ops.tensor_array_ops.TensorArray object at 0x7f800c5d18e0>\n",
      "If you want to mark it as used call its \"mark_used()\" method.\n",
      "It was originally created here:\n",
      "  File \"/usr/lib/python3.8/threading.py\", line 942, in _bootstrap_inner\n",
      "    pass  File \"<ipython-input-8-af6184f58c44>\", line 256, in run\n",
      "    state, done, grads = self.step(state)  File \"<ipython-input-8-af6184f58c44>\", line 228, in step\n",
      "    total_loss = self.compute_loss(logits, values, rewards, dones, actions)  File \"<ipython-input-8-af6184f58c44>\", line 118, in play_steps\n",
      "    logit, value        = self.local_model(state)  File \"/usr/local/lib/python3.8/dist-packages/tensorflow/python/util/tf_should_use.py\", line 243, in wrapped\n",
      "    return _add_should_use_warning(fn(*args, **kwargs),\n",
      "==================================\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "ERROR:tensorflow:==================================\n",
      "Object was never used (type <class 'tensorflow.python.ops.tensor_array_ops.TensorArray'>):\n",
      "<tensorflow.python.ops.tensor_array_ops.TensorArray object at 0x7f800c5d18e0>\n",
      "If you want to mark it as used call its \"mark_used()\" method.\n",
      "It was originally created here:\n",
      "  File \"/usr/lib/python3.8/threading.py\", line 942, in _bootstrap_inner\n",
      "    pass  File \"<ipython-input-8-af6184f58c44>\", line 256, in run\n",
      "    state, done, grads = self.step(state)  File \"<ipython-input-8-af6184f58c44>\", line 228, in step\n",
      "    total_loss = self.compute_loss(logits, values, rewards, dones, actions)  File \"<ipython-input-8-af6184f58c44>\", line 118, in play_steps\n",
      "    logit, value        = self.local_model(state)  File \"/usr/local/lib/python3.8/dist-packages/tensorflow/python/util/tf_should_use.py\", line 243, in wrapped\n",
      "    return _add_should_use_warning(fn(*args, **kwargs),\n",
      "==================================\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "ERROR:tensorflow:==================================\n",
      "Object was never used (type <class 'tensorflow.python.ops.tensor_array_ops.TensorArray'>):\n",
      "<tensorflow.python.ops.tensor_array_ops.TensorArray object at 0x7f80333e77c0>\n",
      "If you want to mark it as used call its \"mark_used()\" method.\n",
      "It was originally created here:\n",
      "  File \"/usr/lib/python3.8/threading.py\", line 942, in _bootstrap_inner\n",
      "    pass  File \"<ipython-input-8-af6184f58c44>\", line 256, in run\n",
      "    state, done, grads = self.step(state)  File \"<ipython-input-8-af6184f58c44>\", line 228, in step\n",
      "    total_loss = self.compute_loss(logits, values, rewards, dones, actions)  File \"<ipython-input-8-af6184f58c44>\", line 118, in play_steps\n",
      "    logit, value        = self.local_model(state)  File \"/usr/local/lib/python3.8/dist-packages/tensorflow/python/util/tf_should_use.py\", line 243, in wrapped\n",
      "    return _add_should_use_warning(fn(*args, **kwargs),\n",
      "==================================\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "ERROR:tensorflow:==================================\n",
      "Object was never used (type <class 'tensorflow.python.ops.tensor_array_ops.TensorArray'>):\n",
      "<tensorflow.python.ops.tensor_array_ops.TensorArray object at 0x7f80333e77c0>\n",
      "If you want to mark it as used call its \"mark_used()\" method.\n",
      "It was originally created here:\n",
      "  File \"/usr/lib/python3.8/threading.py\", line 942, in _bootstrap_inner\n",
      "    pass  File \"<ipython-input-8-af6184f58c44>\", line 256, in run\n",
      "    state, done, grads = self.step(state)  File \"<ipython-input-8-af6184f58c44>\", line 228, in step\n",
      "    total_loss = self.compute_loss(logits, values, rewards, dones, actions)  File \"<ipython-input-8-af6184f58c44>\", line 118, in play_steps\n",
      "    logit, value        = self.local_model(state)  File \"/usr/local/lib/python3.8/dist-packages/tensorflow/python/util/tf_should_use.py\", line 243, in wrapped\n",
      "    return _add_should_use_warning(fn(*args, **kwargs),\n",
      "==================================\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Episode:    1.0 | MA50:  -18.0 | Episode Reward:  -18.0 | Steps:  1087 | Actions: [205. 217. 146. 227. 188. 104.]\n",
      "Episode:    2.0 | MA50:  -19.5 | Episode Reward:  -21.0 | Steps:   812 | Actions: [ 50. 113.  74.  68.  97. 410.]\n",
      "Interrupting. Saving checkpoint...\n",
      "None\n",
      "Saving model to Pong/, best episode score: -inf\n"
     ]
    }
   ],
   "source": [
    "agent.train()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[(-18.0, -18.0), (-19.5, -21.0)]\n"
     ]
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAjgAAAGwCAYAAACkfh/eAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjYuMiwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy8o6BhiAAAACXBIWXMAAA9hAAAPYQGoP6dpAABZ2UlEQVR4nO3de3zO9f/H8ce1sZnTDGNk5FCWSo4JKWfSSYmcR5JC30LfQgd9k/imbyffvqJkcwipFDmU0MGhSCYJJTRhUticx/b5/fH6bWuFbezaZ9e15/12+9xuPp/rsNf1cbie3keP4zgOIiIiIn4kwO0CRERERHKbAo6IiIj4HQUcERER8TsKOCIiIuJ3FHBERETE7yjgiIiIiN9RwBERERG/U8jtAtyQmprK3r17KVGiBB6Px+1yREREJBscx+HIkSNUrFiRgIDzt9EUyICzd+9eIiMj3S5DRERELsDu3bupVKnSeZ9TIANOiRIlALtBJUuWdLkaERERyY6kpCQiIyPTv8fPp0AGnLRuqZIlSyrgiIiI+JjsDC/RIGMRERHxOwo4IiIi4ncUcERERMTvKOCIiIiI31HAEREREb+jgCMiIiJ+RwFHRERE/I4CjoiIiPgdBRwRERHxOwo4IiIi4ne8FnDGjBlDkyZNKFq0KKVKlTrrc9atW0erVq0oVaoUYWFhtGvXjo0bN573fU+ePMmgQYMoU6YMxYsXp1OnTuzfv98Ln0BERER8ldcCTnJyMp07d+aBBx446+NHjx6lffv2VK5cma+//pqVK1dSokQJ2rVrx+nTp8/5vkOGDGHBggXMnTuXzz//nL1793LnnXd662OIiIiID/I4juN48wfExMTw8MMPc/jw4UzXv/nmGxo2bEh8fDyRkZEAbNq0idq1a/PTTz9Ro0aNv71XYmIi4eHhvP3229x1110AbN26lSuuuII1a9Zw3XXXZaumpKQkQkNDSUxMzPXNNj95Po4bBl5FkeIFch9TERERr8nJ97drY3Bq1qxJmTJlmDJlCsnJyZw4cYIpU6ZwxRVXcOmll571NevXr+f06dO0bt06/VpUVBSVK1dmzZo15/xZp06dIikpKdPhDV++9RM3PXY1TcJ+YHu/sbBpk1d+joiIiJyfawGnRIkSfPbZZ8yYMYOQkBCKFy/OkiVLWLx4MYUKnb31IyEhgaCgoL+N6SlfvjwJCQnn/Fljx44lNDQ0/UhrMcptJ3/ZT5jnMBvO1KbeW4OYXXsM1K8Pr74Kv//ulZ8pIiIif5ejgDN8+HA8Hs95j61bt2brvU6cOEG/fv1o2rQpX331FatWreKqq67i5ptv5sSJExf0Yc5lxIgRJCYmph+7d+/O1fdP0+Zf1xP3c0ma1fqDI5SkG7MZ8O19nHjoMahQAe64Az74AJKTvfLzRURExORooMiwYcPo06fPeZ9TrVq1bL3X22+/za5du1izZg0BAQHp18LCwvjwww/p2rXr314TERFBcnIyhw8fztSKs3//fiIiIs75s4KDgwkODs5WXRerUtXCLN9Yhqefhueec5jsDGBNSEveOXErUR98YAGnbFno3h2io6FuXfB48qQ2ERGRgiJHASc8PJzw8PBc+cHHjx8nICAAz5++3NPOU1NTz/qa+vXrU7hwYZYtW0anTp0A2LZtG/Hx8TRu3DhX6soNhQrBs8/CjTd66NkTNv12GQ1CNjOx+Tv02jAUEhKs2+rVV+Hqqy3o9OgB5wlpIiIikn1eG4MTHx9PXFwc8fHxpKSkEBcXR1xcHEePHgWgTZs2HDp0iEGDBrFlyxY2b95M3759KVSoEC1atABgz549REVFsXbtWgBCQ0Pp168fQ4cOZcWKFaxfv56+ffvSuHHjbM+gyktt2kBcHLRoAcdOBNJ7cTfuabeHY+8tgS5dIDjYBiI/8ghUqgS33AJz58LJk26XLiIi4tscL4mOjnaAvx0rVqxIf84nn3ziNG3a1AkNDXXCwsKcli1bOmvWrEl/fOfOnX97zYkTJ5yBAwc6YWFhTtGiRZ077rjD2bdvX45qS0xMdAAnMTHxYj9mtpw54zhPP+04Ho/jgOPUquU433/vOM7Bg44zcaLjXHedPZB2hIU5zgMPOM7XXztOamqe1CgiIpLf5eT72+vr4ORH3lwH53xWrLChNwkJEBICr70Gffr8/xCcbdsgNhamT4dff814UVSUdWH16gWXXJJntYqIiOQ3Ofn+VsDJw4ADsH+/ZZWlS+28Z0+YOBGKF///J6SkwPLlFnbefx/SZpQFBEDr1hZ2OnaEokXztG4RERG3KeBkwc2AA5CaCuPGwZNP2q9r1oR33oHatf9WqI3JiY2FL7/MuF6ypI3hiY6Gpk01C0tERAoEBZwsuB1w0nz5JXTrBnv22HjjV1+F/v3PkVd+/hmmTbNj166M6zVqQO/edlSpkleli4iI5DkFnCzkl4ADtsBx796weLGdd+0KkyZZI81ZpabCF19Yq87cuXDsWMZjLVpYq06nTn/q8xIREfEPCjhZyE8BByyz/Oc/MGKEDcGpUcO6rOrWzeKFR4/aOJ3YWBu3k6ZYMbjrLgs7N95o43dERER8nAJOFvJbwEmzZo214MTHQ1AQvPgiDByYzSE2v/xiM7BiY2H79ozrVapYE1F0NFSv7rXaRUREvE0BJwv5NeAAHDwIffvC/Pl23qkTvPkm/GV/0XNzHFi92oLOnDk2UDnN9ddb0OnS5Tx9YCIiIvmTAk4W8nPAAcsor7wCjz4Kp09D1aqWVRo2zOEbnThhe1/Fxtq89LQtMEJCbOPPPn2gZUsIDMzlTyAiIpL7FHCykN8DTpp166yxZdcuKFwYnn8eHnroAmeF79kDM2ZY2NmyJeP6JZfYwjzR0baooIiISD6lgJMFXwk4AIcPQ79+NpYY4LbbYOpUKF36At/QcSw5xcbCrFlw6FDGY40aWavO3XdDWNhFVi4iIpK7FHCy4EsBByyT/O9/MHQoJCdD5cowezZc9Abqp07BggUWdhYvtilcYIvy3Habteq0a2fbo4uIiLhMAScLvhZw0nz7rXVZ/fyzZY7nnoNhw3JpFvj+/TBzJsTE2A7naSIioEcPCztXX50LP0hEROTCKOBkwVcDDtikqPvus0HHAB06WANM2bK59AMcB+Li7E1nzrSVCNPUq2dBp3v3XPyBIiIi2aOAkwVfDjhgGeSNN+Af/7BepksuseE0zZrl8g9KTrauq9hY+Ogjm9IFNuL55pttvM5NN9miPSIiIl6mgJMFXw84ab77zrqstm2zmd7PPAPDh3tp4eLff7cUFRsL69dnXC9b1lp0+vSBOnW08aeIiHiNAk4W/CXggO3W8MADNgMcoE0b+3W5cl78od9/b0Fn+nQbu5Pm6qst6PToAeXLe7EAEREpiBRwsuBPAQesyyomBgYNsrX9IiLg7bdt702vOnMGPvnEws4HH1iXFlhzUvv2FnZuvdVmZYmIiFwkBZws+FvASbN5s3VZ/fCDdVM99RQ88UQeLVR86JCNfI6Jga+/zrgeFgbdutng5IYN1YUlIiIXTAEnC/4acACOHYMHH7TFAMFacWbOhAoV8rCIrVth2jQ79uzJuB4VZa06PXvayGgREZEcUMDJgj8HnDTTp9vYnGPHbDzOjBk2PidPpaTA8uXWqjNvnvWfgTUvtW5tYadjR9sbS0REJAs5+f72xnwbyQd69YJvvrFxv7/9ZgsSP/GEDZvJM4GBlqpmzoSEBNsWvVkz2/Tzk09s9lVEBPTvD6tW2WAiERGRXKAWHD9twUlz4gQMGQKTJtl5s2Y2ALlSJReL+vln676KjYVffsm4XqMG9O5tR5Uq7tUnIiL5krqoslCQAk6a2bNtBeQjR6BMGevCuukml4tKTYUvvrCgM3eu9aeladHCurDuvBOKF3etRBERyT/URSV/07Wrrc9Xty788Ydt8fDYYxmLE7siIACaN7cR0QkJFnRatrTHVqywmVcRERZ0PvvMApGIiEg2qAWngLTgpDl5Eh55BF57zc4bN7bWncqV3a0rk19+sSammBjrzkpz6aUZXVjVq7tVnYiIuERdVFkoyAEnzXvvQb9+kJhoS9XExMBtt7ld1V84DqxebS07c+bYTqNprr/eWnY6d4YC+nsoIlLQqItKstSpE3z7ra29d+gQ3H47DB2asRhxvuDxQNOmMHmydWG9/Ta0bWvXV66Ee++1LqyePWHpUpuWLiIiglpwCmwLTprkZBuL8/LLdt6woTWWVK3qalnnt2ePLewTE2OLCqapVMnmx0dHQ82arpUnIiLeoS6qLCjg/N2HH1qPz+HDEBoKb71lE5jyNceBdess6MyebU1Raa67zoLO3XdbH5yIiPg8BZwsKOCc3S+/2Gyrr76y88GD4YUXfGSvzFOnYMECCztLlmR0VwUHW/9bdLR1bxUq5GqZIiJy4RRwsqCAc26nT9uKx88/b+f16lmXVY0a7taVI2njdWJiYNOmjOtp43Wio+Gqq1wrT0RELowCThYUcLK2aJHNxv7jDyhRAt54w3p7fIrjQFyczcKaORN+/z3jsXr1rE+uWzcoW9atCkVEJAcUcLKggJM9v/5q3/8rV9r5gAHw0ks+ujdmcjIsXmytOh99lLEpV+HCcMst1qrToYOdi4hIvqSAkwUFnOw7cwZGjYKxY61BpHZteOcdH5+k9PvvMGuWhZ1vv824XrYs9OhhYadOHZuOLiIi+YYCThYUcHLuk09s+MqBA1CsGLz+up37vE2brAtrxgzYvz/jeu3aFnR69IDy5d2rT0RE0ingZEEB58Ls2wfdu9u2UAD33AMTJkDRoq6WlTvOnLEUFxNjc+bTVjwMDLRdSaOj4dZbfWRKmYiIf1LAyYICzoVLSYHRo+GZZ6zL6sorrcuqVi23K8tFBw/a1LHYWPj664zrYWE2KCk62lZEVBeWiEieUsDJggLOxVu+3HpvEhJs0PFrr9mkJL/7zt+61YLO9Om2gnKaK66woNOzJ1xyiXv1iYgUIAo4WVDAyR3799v3+6ef2nmvXvC//0Hx4u7W5RUpKZbqYmLg/fdtW3aAgABo08bCTseOPjrFTETEN2izTckT5cvDxx/Ds8/a9/z06dZz8913blfmBYGBFmRmzrRmqzfesB3NU1PtJnTvbgsJ3ncfrFpl/XciIuIateCoBSdXfPGFDU/ZuxeKFIFXXoH+/f2wy+qvtm+HadPs+OWXjOuXXWYrJfbqBVWquFefiIgfURdVFhRwvOPAAeupWbzYzrt2hUmToEDc4tRUS3kxMfDuu3DsWMZjLVvajenUyebYi4jIBVEXlbgiPNwWCf73v61HZ/ZsqF8fNmxwu7I8EBAAzZtbwElIsIHJLVrYY8uXW8ApXx769rV59qmpLhYrIuL/1IJTIJoX8t7q1daCs3s3BAXZFg8PPFAAuqz+atcuG5wUGws//5xx/dJLrQurd2+oXt2t6kREfIq6qLKggJM3Dh60qeMLFtj5XXfBm29CaKirZbnDcSz1xcTYwkFJSRmPNWtmLTydOxeQ/jwRkQuTL7qoxowZQ5MmTShatCilSpU663PWrVtHq1atKFWqFGFhYbRr146NGzee932bN2+Ox+PJdNx///1e+ARysUqXtkWBX3zR9rB8912oWxfWrXO7Mhd4PNC0qc2+2rfPZmO1bWvXv/wS7r3XZmH17AlLl9q0dBERuWBeCzjJycl07tyZBx544KyPHz16lPbt21O5cmW+/vprVq5cSYkSJWjXrh2nT58+73v379+fffv2pR/PP/+8Nz6C5AKPB4YMsR3JL70Udu607/lXXinAM6mLFrVp5R9/DPHxtpNpVBScOJERfC69FEaOhG3b3K5WRMQneb2LKiYmhocffpjDhw9nuv7NN9/QsGFD4uPjiYyMBGDTpk3Url2bn376iRo1apz1/Zo3b06dOnV4+eWXL7gmdVG54/Bh6NfP1skDuP12eOsta+kp8BwH1q61sTqzZtnNSnPdddaFdffdtl2EiEgBlS+6qLJSs2ZNypQpw5QpU0hOTubEiRNMmTKFK664gksvvfS8r505cyZly5blqquuYsSIERw/fvy8zz916hRJSUmZDsl7pUpZN9WECTbw+MMPrcvqq6/criwf8HigUSNbCnrfPhunc/PNNh3tq69shHaFChZyFi+2zUFFROScXAs4JUqU4LPPPmPGjBmEhIRQvHhxlixZwuLFiylUqNA5X9e9e3dmzJjBihUrGDFiBNOnT6dnz57n/Vljx44lNDQ0/UhrMZK85/HA4MGwZo1NHoqPtzG248dr5nS6IkVswPFHH8Gvv8ILL8BVV8GpUxZ8OnSAyEj45z/h++/drlZEJF/KURfV8OHD+fe//33e52zZsoWoqKj083N1UZ04cYLmzZsTFRXF4MGDSUlJ4YUXXmDr1q2sW7eOkGzu6bN8+XJatWrF9u3bqX6O6banTp3i1KlT6edJSUlERkaqi8plSUm2s8GcOXZ+8802yahsWVfLyp8cxxYUio2Ft9+G33/PeKx+fevC6tZNN09E/JrXpokfOHCAP/7447zPqVatGkFBQenn5wo4U6ZMYeTIkezbt4+AAGtISk5OJiwsjClTptC1a9ds1XTs2LH01p927dpl6zUag5N/OA5MngwPPWQNFJdcYgsEXn+925XlY8nJsGiRhZ2PPsroripcGG65xcJOhw52LiLiR3Ly/X3uvqCzCA8PJzw8/KKKS3P8+HECAgLw/Gnlt7Tz1Bz0VcTFxQFQoUKFXKlL8pbHAwMG2DjaLl3gxx9tQeDRo+Gxx2yBYPmLoCDbubxjR9sfY9YsCzvffgvz5tkRHm4ztfr0gTp13K1XRMQFXvv6iI+PJy4ujvj4eFJSUoiLiyMuLo6jR48C0KZNGw4dOsSgQYPYsmULmzdvpm/fvhQqVIgW/7/E/Z49e4iKimLt2rUA/Pzzz4wePZr169eza9cu5s+fT+/evbnhhhuoXbu2tz6K5IFrroH1620ZmJQUmyF9003w229uV5bPhYfDP/5hN++772DYMNsS4sABm4tft67d3BdfhP373a5WRCTvOF4SHR3tAH87VqxYkf6cTz75xGnatKkTGhrqhIWFOS1btnTWrFmT/vjOnTszvSY+Pt654YYbnNKlSzvBwcFOjRo1nH/+859OYmJijmpLTEx0gBy/TrwvNdVxpkxxnJAQxwHHqVDBcf70R0ay4/Rpx/noI8fp3NlxgoLsRoLjBAY6zi23OM7cuY5z8qTbVYqI5FhOvr+1VYPG4ORLmzfbRKItW6ybatQoePxxmzUtOXDwoI3ijo2Fr7/OuB4WZoOS+/SBBg0K4CZhIuKLtBdVFhRwfMOxY/DggzB1qp23bGkL/UZEuFuXz9q61YLO9OmwZ0/G9SuusIHJPXvaKG8RkXxKAScLCji+Zfp0W+fu2DEoV85CTuvWblflw1JSYNkyCzvvvw8nT9r1gABo08ZadW6/HbK5VIOISF7xiZWMRbKrVy/45hu4+mobdNy2LTz5pBbzvWCBgXYTZ86EhATbAPT6622lxY8/tq6rChVskaLVqwvwpmEi4svUgqMWHJ9x4gQ8/LCtmwNwww225p16VXLJ9u0wbZq17MTHZ1y/7DLrwurVCypXdq8+ESnw1EWVBQUc3zZ7NvTvD0eP2sK906dD+/ZuV+VHUlPh888t6Lz7rvUNgg1EbtHCwk6nTlCsmLt1ikiBoy4q8Wtdu9qadnXq2I4FN90Ew4fD6dNuV+YnAgIsyMTEWBdWTIydOw4sX24BJyIC+va1IKRNxEQkH1ILjlpwfNbJk/DII/Daa3bepIkt6qteFC/Ztcuay2Jj4eefM65fein07m3HOfaDExHJDeqiyoICjn95913o18827yxd2hocbr3V7ar8mOPAqlUWdObMgSNHMh5r1sxmYd11F+jvlojkMnVRSYFy11220XaDBrau3W232Y4FycluV+anPB6bdfXGG9aFNXOmzcryeODLLy1tRkTYujqffmrT0kVE8phacPS/TL9x6pSNxXn5ZTu/9lobkFy1qqtlFRy//gozZlgT2rZtGdcrVbLuq+houPxy18oTEd+nLqosKOD4tw8/tF6Sw4chNNRWQr7jDrerKkAcB9autS6sWbPsNyLNddfZb87dd0OpUi4VKCK+SgEnCwo4/u+XX2y21Vdf2fmDD8L48RAc7G5dBc7Jk7BggbXqfPxxRndVcDB07GitOm3aQKFCblYpIj5CAScLCjgFw+nTtkHn+PF2Xr++jYnVRB+XpI3XiYmB77/PuB4RYYsIRkfDlVe6Vp6I5H8KOFlQwClYFi60784//oASJeDNN6FLF7erKsAcx0aFx8TYUtR//JHxWP361oXVrRuUKeNWhSKSTyngZEEBp+D59Vf7zly50s7vvx9efFH7SbouORkWLbKws3BhxgZjhQvDLbdY2LnpJjsXkQJPAScLCjgF05kz8NRTMHasndeuDe+8AzVruluX/L8DB2xQcmysLVWdJjwcune3sFOnjlvViUg+oICTBQWcgu3jj23Ix4EDtp3SpEnQo4fbVUkmmzZZ0JkxA/bvz7heu7YFne7doXx518oTEXco4GRBAUf27rVQ89lndt6vH7z6KhQt6mpZ8ldnzlgijY21+f9pqzcGBlrXVZ8+1pWl6XEiBYICThYUcARsxvIzz8Do0Tbu9corrcuqVi23K5OzOnjQpsHFxNg6O2lKl7YBVtHRtpy1x+NaiSLiXQo4WVDAkT9bvtxacxISrAXntdesYUDysS1bYNo0O/buzbh+xRX2m9ezJ1Ss6Fp5IuIdCjhZUMCRv9q/P2PrJLCdBV57DYoXd7cuyUJKCixbZq068+bZwoIAAQG2P1Z0NNx+u6bLifgJBZwsKODI2aSkwLhxNtMqNRWioqzL6uqr3a5MsiUxEebOtbCzalXG9dBQ2xoiOhoaN1YXlogPU8DJggKOnM8XX9iQjr17oUgRmDDBBiHre9GH/PRTRhdWfHzG9csus6DTqxdUruxefSJyQRRwsqCAI1k5cMC6qZYssfNu3Ww6eYkS7tYlOZSaCp9/bq06774Lx4/bdY8HWra0sHPnnbZegIjkewo4WVDAkexITbV9rB5/3LqvatSwLqu6dd2uTC7IkSPw3ns25TxtfQCwgVadO1vYadbMxu+ISL6kgJMFBRzJidWrbWfy3bttuZUXX4QHHlCXlU/btQumT7eWnR07Mq5feqkFnd69oVo1l4oTkXNRwMmCAo7k1B9/QN++sGCBnd91l23aGRrqbl1ykRzHBiTHxFjz3JEjGY/dcIOFnc6d1Tcpkk8o4GRBAUcuhOPASy/BY4/ZArvVqtm6cw0auF2Z5Irjx+GDDyzsfPqp/YaDTTHv1MnCTosWtoqyiLhCAScLCjhyMdautVnHu3bZJtfjx8M//qEuK7/y66/WhRUbC9u2ZVyPjLQZWNHRcPnl7tUnUkAp4GRBAUcu1uHDcM89trYcQMeO8NZbEBbmZlWS6xzHEm1MDMyebb/xaRo3tqBz991QqpRLBYoULAo4WVDAkdzgOLba8bBhtgdklSrWZdWokduViVecPAnz51urzpIlNs0ObOR5x44Wdtq0gUKFXC1TxJ8p4GRBAUdy0/r19p/4n3+277Zx42DIEM029mv79sHMmRZ2vv8+43qFCrbnR3S07d4qIrlKAScLCjiS2xIT4b77bCIOwM0323dfmTLu1iVe5jiwYYN1Yb39tk23S9OggQWdbt30B0EklyjgZEEBR7zBcWy144cfhlOnoFIlG7bRtKnblUmeSE6GhQst2S5caFPtwEai33qrhZ2bbrJzEbkgCjhZUMARb9q4Ebp0gR9/tBnFo0fb1HJ1WRUgBw5Yi05srLXwpAkPhx49LOzUqeNaeSK+SgEnCwo44m1HjthqxzNn2nm7drbvY7ly7tYlLvjuOws6M2fC/v0Z16+5xoJOjx76gyGSTQo4WVDAkbzgODZ1/MEH4cQJG386axbceKPblYkrzpyBjz+28Trz51uXFlgzX4cOFnZuucVmZYnIWSngZEEBR/LS999bl9WWLdZNNWqUbeCpBXELsIMHbYBWbKyts5OmdGkblBwdbYOUtXqkSCYKOFlQwJG8duwYDB5s/3kHaNUKZsyAiAhXy5L8YMsWCzrTp8PevRnXa9WyoNOzJ1Ss6F59IvmIAk4WFHDELdOm2dic48ehfHkbltGqldtVSb6QkmJ7YMXG2hLZJ0/a9YAAaNvWws7tt9veWCIFVE6+vzWvQyQP9e5tCwNedZWNN23TBp56KmNGsRRggYE2Gv3ttyEhASZPtjUGUlNt5eRu3Wwg14ABsGZNxmagInJWasFRC4644MQJeOgheOMNO7/hBhuArJ4I+ZuffrKmv2nTID4+4/pll1mrTq9eULmye/WJ5CF1UWVBAUfyi1mzbAXko0ehbFkbhtG+vdtVSb6UmgqffWZdWO++a/2cYAORW7aEPn3gjjugWDE3qxTxKnVRifiIbt2sy6pOHfj9d1vodsQIdVnJWQQEWJCJjbUurKlToXlz66patsxaciIibJv7L77I2AxUpIBSC45acCQfOHnSdiX/3//svGlTa92JjHS3LvEBO3da019sLOzYkXG9alUb9NW7N1Sr5l59IrkoX7TgjBkzhiZNmlC0aFFKlSp11ucsW7aMJk2aUKJECSIiInjsscc4k8V/XU+ePMmgQYMoU6YMxYsXp1OnTuz/8+qgIj6oSBF47TXbrLNkSVi1ylp1PvrI7cok36ta1Uaqb99uLTf9+kGJEhZ8/vUvqF7dVpd86y1bYlukgPBawElOTqZz58488MADZ31848aNdOjQgfbt27NhwwbmzJnD/PnzGT58+Hnfd8iQISxYsIC5c+fy+eefs3fvXu68805vfASRPNe5s21d1KCBrQV3663WspO26K3IOXk80KwZvPmmdWHNmGHT9DyejOATEWFdWZ9+qi4s8X+Ol02dOtUJDQ392/URI0Y4DRo0yHRt/vz5TpEiRZykpKSzvtfhw4edwoULO3Pnzk2/tmXLFgdw1qxZk+2aEhMTHcBJTEzM9mtE8tLJk47z0EOOYwMsHKdRI8fZudPtqsQnxcc7znPPOU7Nmhl/oMBxIiMdZ+RIx9m2ze0KRbItJ9/frg0yPnXqFEWKFMl0LSQkhJMnT7J+/fqzvmb9+vWcPn2a1q1bp1+LioqicuXKrFmz5rw/KykpKdMhkp8FB8PLL9t6b6VKwddfQ9268MEHLhcmvicy0kaub9li6+fcf7/9odq9G557DmrWhCZNYNIkOHzY7WpFco1rAaddu3asXr2aWbNmkZKSwp49e3jmmWcA2Ldv31lfk5CQQFBQ0N/G9JQvX56EhIRz/qyxY8cSGhqafkRq5Kb4iI4dIS4OGjWy75477rD1c06dcrkw8T0eD1x3HUycCPv2wZw5tslnQEBG8ImIgK5dbWHBlBS3Kxa5KDkKOMOHD8fj8Zz32Lp1a7beq23btowfP57777+f4OBgLr/8cjp06GBFBeRu7hoxYgSJiYnpx+7du3P1/UW8qUoV+PJLeOQRO3/1VZtl9fPP7tYlPqxIEdsBduFC+PVXGD8errzSkvOcObZeQWQkPPoobN7sdrUiFyRH08QPHDjAH3/8cd7nVKtWjaCgoPTzmJgYHn74YQ6fo+nTcRz27dtHWFgYu3btolatWqxdu5aGDRv+7bnLly+nVatWHDp0KFMrTpUqVXj44YcZMmRItj6HpomLr1q40Gb9Hjxos63efNMGJotcNMeBb7+16eZvvw1//re+QQNbSLBrVyhTxrUSRbw2TTw8PJyoqKjzHn8ON9nh8XioWLEiISEhzJo1i8jISOrVq3fW59avX5/ChQuzbNmy9Gvbtm0jPj6exo0b5+jniviim2+2LqumTSEpyf4TPnBgxr6MIhfM44H69a2JcO9eeP9929yzUCH45hsYPNj2wurUCRYsgNOn3a5Y5Ly8NgYnPj6euLg44uPjSUlJIS4ujri4OI4ePZr+nPHjx7Np0yY2b97M6NGjGTduHK+++iqBgYEA7Nmzh6ioKNauXQtAaGgo/fr1Y+jQoaxYsYL169fTt29fGjduzHXXXeetjyKSr0RG2or9I0bY+cSJNrTixx9dLUv8SVCQDfj64APYs8dGvNepY6Hm/ffhttugUiUYOhQ2bnS5WJFz8NZUrujoaAf427FixYr057Ro0cIJDQ11ihQp4jRq1MhZtGhRpvfYuXPn315z4sQJZ+DAgU5YWJhTtGhR54477nD27duXo9o0TVz8xZIljhMebrN+ixd3nJkz3a5I/NrGjY4zdKjjlCuXecr5Ndc4zksvOc7+/W5XKH4uJ9/f2qpBY3DEx+3dC927w+ef2/m998Irr0DRou7WJX7s9Gn4+GMbrzN/fsZKlIUK2QDlPn2sPzU42NUyxf9oN/EsKOCIvzlzBkaPtsNx4KqrbNuHK65wuzLxewcPwuzZFnb+fzgBAKVL226yffrY2B6Px7USxX8o4GRBAUf81bJl0KMH7N9vLTj/+x9ER7tdlRQYW7ZY0Jk+3ZoW09SqZUGnZ08bqCxygRRwsqCAI/4sIcG+R9ImG0ZH20aexYq5W5cUICkptt9VbKwtx502zS8gANq2tbBz++22Ho9IDijgZEEBR/xdSgqMHQujRtmeilFRMHeudV2J5KnEROsvjYmB1aszroeG2ro60dE2DVBdWJINCjhZUMCRguLzz20A8t699p/lCRNsU2l9l4grfvoJpk2zlp0/ryh/+eUWdHr1snUQRM5BAScLCjhSkBw4YKsfL1li5927w+uvQ4kS7tYlBVhqqi3mFBMD770Hx4/bdY8HWrWysHPHHepXlb/x2krGIuJ7wsNti4dx4yAw0Fbhb9DAVkQWcUVAALRsaa05CQkwdSrceKNNAfz0U2vJiYiw5sYvvrDrIjmkFhy14EgBsmqVDXv49VdbouSll2wTaXVZSb6wc6fNwIqNhR07Mq5XrWqtOr1726+lwFIXVRYUcKQg++MPm8Ty0Ud23qULTJ5sYz5F8gXHgZUrrQtr7lw4ciTjsRtvtLBz113qZy2AFHCyoIAjBZ3jWOvNY4/ZIoHVqsGcOdZ1JZKvHD9uU81jY637Ku0rq2hRuPNOS+stWli3l/g9BZwsKOCImK+/hrvvhl9+gcKF4YUX4MEH1WUl+dTu3TBjhrXs/Hl32chI676KjobLLnOtPPE+BZwsKOCIZDh0CO65xzaOBpu8MmUKhIW5WpbIuTmOpfPYWJg1y9baSdO4sbXqdOkCpUq5VaF4iQJOFhRwRDJzHPjvf+GRR2zfxCpVrMuqUSO3KxPJwsmTtuFnTIxtAJqaateDgy2tR0dDmzY2hVB8ngJOFhRwRM5u/Xr7j++OHbYx9LhxMHSouqzER+zbBzNnWtjZvDnjeoUKNvU8Otr2xRKfpYCTBQUckXNLTIT+/W3yCsAtt9j3RZkyrpYlkn2OA99+a39w337bdjxP07ChBZ2uXfWH2gcp4GRBAUfk/BwHJk2Chx+GU6dsDOesWdC0qduVieRQcrKtdBkTA4sW2bRBsFH1t91mYad9ezuXfE8BJwsKOCLZExdnXVY//WRDGJ59Fh59VDNyxUf99pu16MTGZl7Ku1w56NHDws4117hWnmRNAScLCjgi2XfkiK12/Pbbdt6+va2wHx7ubl0iF2XjRgs6M2da8ElTp44Fne7dLfhIvqKAkwUFHJGccRx46y0YPNgmrVSsaF1WN9zgdmUiF+n0aduJNjYWFiywLi2wUfYdOljYueUWCApyt04BFHCypIAjcmG+/966rLZssW6qf/0LRozQDFzxEwcPwuzZNl5n3bqM66VLW4tOdDTUr69phS5SwMmCAo7IhTt2DAYNsv/wArRqZYvLRkS4W5dIrvrhB/tDPmMG7N2bcf3KKy3o9Oxp088lTyngZEEBR+TixcbCwIG2VVD58jaUoVUrt6sSyWUpKbYHVkyMLfd98qRdDwiAdu0s7Nx+OxQp4maVBUZOvr81F0JELkh0NHzzDVx1Fezfb4vFjhpl3wcifiMw0ILMrFm2kOCkSdCkia2YvHixradToYKNxF+zJmMzUHGdWnDUgiNyUY4fh4cegjfftPMbb7QZVxUruluXiFf9+KNNJ5w2zTYBTXP55Zb+e/WyBaQkV6mLKgsKOCK57+23YcAAOHrUppBPn27/8RXxa6mpsGKF9dm+954lfrCByK1aWdi5804oWtTdOv2EuqhEJM917257WV1zDRw4YOvljBiRsXCsiF8KCLAgM20aJCTYego33mhdVZ9+ai05ERHQrx988YW6sPKQWnDUgiOSq06etA06J06086ZNbfiCWuulQNm500JPbKz9Ok21atC7tx1Vq7pXn49SF1UWFHBEvO+dd2zTzqQkW0Zk2jS4+Wa3qxLJY6mpsHKlBZ133rE+3DQ33mhdWHfdBSVKuFejD1EXlYi4rksX29C5fn1bP+2WW+Cf/7SFY0UKjIAAW/J7yhTrwpo+HVq3tjE6n38O99xjXVi9e8OyZRaIJFeoBUctOCJedeqUbdD56qt23qgRzJkDVaq4W5eIq3bvtrATG2szstJUrmzjdqKj4bLL3Ksvn1IXVRYUcETy3gcfQN++cPgwlCoFU6dCx47u1iTiOseBr7+2hQRnz4bExIzHmjSxoNOli/2lEXVRiUj+07EjbNhgLTiHD8Mdd8DDD1sLj0iB5fHAddfB669bF9bs2XDTTda1tXq1rb1QoQJ062abgmolzWxTC45acETyVHIyjBwJ//mPndevb2Mvq1Vzty6RfGXvXtv/JDYWNm/OuF6xou2DFR0NtWq5V59L1EWVBQUcEfd99JH9G33wIJQsaWMw77rL7apE8hnHsQWmYmNtNc2DBzMea9jQ/hJ162ZTFQsABZwsKOCI5A+7d9u/zatW2fnAgdayo30LRc7i1ClYuNDCzqJFGatoBgXBrbdCnz62fHjhwq6W6U0KOFlQwBHJP06fhqeegnHj7LxOHeuy0gQSkfP47Tdr0YmNhbi4jOvlykGPHhZ2atd2qzqvUcDJggKOSP6zZInNjv39dyheHCZPttYdEcnCxo0WdGbMsH1S0tSpY0Gne3fbIM4PKOBkQQFHJH/au9f+Lf78czvv3x9eeQVCQtytS8QnnD5t/1OIjYX58zNW1SxUCDp0sLBz883WpeWjFHCyoIAjkn+dOQPPPAPPPmvjK6+6CubOhagotysT8SF//GFTzmNjYd26jOtlyljTaJ8+UK+eTVP3IQo4WVDAEcn/Pv3UZsPu3w9Fi9rmnb17u12ViA/64QcLOtOnw759GdevvNJmYfXsaWvt+AAFnCwo4Ij4hoQE+7d32TI779MH/vtfKFbM1bJEfNOZM/Y/h9hYmDcvY5XNgACbfRUdDbffnq+nMSrgZEEBR8R3pKTAc8/B00/bPoS1atksqyuvdLsyER92+LD9RYqNtRWT05QqBXffbf+baNQo33VhKeBkQQFHxPd8/rkNHdi3zwYdT5hgGzHns39/RXzPjz/CtGl27N6dcf3yy61Vp1cviIx0r74/UcDJggKOiG/67Tcbh/Pxx3beo4eNzSlRwt26RPxCaiqsWGGtOu+9B8eP23WPB1q1sladO+6wQXEu0WabIuKXypWzBVzHjoXAQNuqp0EDWwZERC5SQIAFmWnTbADcW2/BjTfadMa0Uf8REdCvH3z5pV3Px7wWcMaMGUOTJk0oWrQopc6xzfuyZcto0qQJJUqUICIigscee4wzaUtPn0Pz5s3xeDyZjvvvv98Ln0BE8qOAABg+HD77DCpVstb1Ro1sM+Z8/u+tiO8oUQL69rW/aD//bIPgqlaFI0cs+NxwA9SoYWs67NrlcrFn57WAk5ycTOfOnXnggQfO+vjGjRvp0KED7du3Z8OGDcyZM4f58+czfPjwLN+7f//+7Nu3L/14/vnnc7t8Ecnnrr/eVqi/5RabDPLAA9C1KyQluV2ZiJ+pVg1GjYLt220w3D332HLjO3bY9apVoXlziImBo0fdrjad18fgxMTE8PDDD3P48OFM10eOHMnSpUtZ96cFiBYsWECXLl347bffKHGOTvXmzZtTp04dXn755QuuSWNwRPyH48CLL1qrzpkzUL06zJkD9eu7XZmIHzt2zKaax8TA8uUZzadFi8Jdd9ng5ObNrck1F/nEGJxTp05R5C9z7UNCQjh58iTr168/72tnzpxJ2bJlueqqqxgxYgTH0wZCnednJSUlZTpExD94PDBsmA0JqFLFWtObNLFZVuqyEvGSYsVsTM6nn1oX1ZgxtkPu8eM2hqdVK2jY0NW/hK4FnHbt2rF69WpmzZpFSkoKe/bs4ZlnngFg359XWvyL7t27M2PGDFasWMGIESOYPn06PXv2PO/PGjt2LKGhoelHZD6Z7iYiuee662DDBujYEZKT4R//sP9I/qXxWERyW+XKMHIkbNtma+oMGAChodC0qbvrODg58NhjjznAeY8tW7Zkes3UqVOd0NDQs77ff/7zH6dkyZJOYGCgU7RoUWfs2LEO4MyePTvbNS1btswBnO3bt5/zOSdPnnQSExPTj927dzuAk5iYmO2fIyK+ITXVcV55xXEKF3YccJxLL3Wcr792uyqRAub4ccf5/fdcf9vExMRsf3/naAzOgQMH+OOPP877nGrVqhH0p51KzzUG508Bi3379hEWFsauXbuoVasWa9eupWHDhtmq6dixYxQvXpwlS5bQrl27bL1GY3BE/N8339iCrDt22GbK//43DBmihQFFfFlOvr8L5eSNw8PDCQ8Pv6ji/srj8VCxYkUAZs2aRWRkJPXq1cv26+Pi4gCo4CMbhYlI3mjQAL79Fu69F95918bpfPaZjYksXdrt6kTE27w2Bic+Pp64uDji4+NJSUkhLi6OuLg4jv5pCtn48ePZtGkTmzdvZvTo0YwbN45XX32VwMBAAPbs2UNUVBRr164F4Oeff2b06NGsX7+eXbt2MX/+fHr37s0NN9xA7dq1vfVRRMRHhYbadjv/+x8EB8OCBVCnTuatd0TEP3kt4Dz11FPUrVuXUaNGcfToUerWrUvdunX55ptv0p+zePFimjVrRoMGDVi4cCEffvghHTt2TH/89OnTbNu2LX2WVFBQEJ9++ilt27YlKiqKYcOG0alTJxYsWOCtjyEiPs7jsTVyvvrKJnns3m1rlP3737YyvYj4J+1FpTE4IgXGkSNw//3w9tt2ftNNtu1OLve8i4iX+MQ6OCIiea1ECZgxA954A4oUgcWLrcvqiy/crkxEcpsCjogUKB6PDTxeuxaiomDvXmjRAp59FlJS3K5ORHKLAo6IFEhXX21TyaOjbSzOk09C+/awf7/blYlIblDAEZECq1gxmzYeE2Nb6Hz6qXVZLV/ucmEictEUcESkwIuOhnXr4MorISEBWre2TZLVZSXiuxRwRESAWrVsXM6999r+gM88Y0Fn7163KxORC6GAIyLy/4oWtRlWM2dC8eK28nGdOvDJJ25XJiI5pYAjIvIX3bvD+vVwzTVw4AC0a2ebJZ8543ZlIpJdCjgiImdx+eW2+vEDD9j52LE2nfzXX92tS0SyRwFHROQcihSxfazmzLFFAleutC6rRYvcrkxEsqKAIyKShS5dYMMGqF8f/vgDbr4ZHn0UTp92uzIRORcFHBGRbKheHVatggcftPPx423Tzl9+cbcuETk7BRwRkWwKDoZXX4X334dSpWyMTt268OGHblcmIn+lgCMikkN33GFdVtdeC4cOQceO8PDDkJzsdmUikkYBR0TkAlx6KXz5JQwbZuevvAJNm8KOHa6WJSL/TwFHROQCBQXBCy/A/PlQurRt3lm3Lrz3ntuViYgCjojIRbr1VuuyatIEkpLgrrtg8GA4edLtykQKLgUcEZFcULmybe0wfLidv/aaBZ6ffnK1LJECSwFHRCSXFC5sKx4vXgxly1qrTr16MHu225WJFDwKOCIiuax9e4iLs3Vyjh6Fbt1gwAA4ccLtykQKDgUcEREvuOQSWLYMnngCPB6YPBkaNYKtW92uTKRgUMAREfGSQoVg9Gj45BMoXx42bYIGDWD6dLcrE/F/CjgiIl7WurV1WbVsCceOQe/ecM899msR8Q4FHBGRPBARYS05//oXBATA1Km2EvLmzW5XJuKfFHBERPJIYCA89ZSNzalQAX74ARo2hLfeAsdxuzoR/6KAIyKSx5o3ty6rtm1tZlW/ftZtdfSo25WJ+A8FHBERF5QrZ+vlPPectezMmAH168N337ldmYh/UMAREXFJQACMGGErIF9yCfz4o43LmTRJXVYiF0sBR0TEZddfb11WN98Mp07B/ffb4oBJSW5XJuK7FHBERPKBsmVtV/Lx4239nDlzbJuHb791uzIR36SAIyKSTwQEwCOPwJdfQpUq8PPP0Lgx/Pe/6rISySkFHBGRfOa662yjzttvh+RkePBBuOsuOHzY7cpEfIcCjohIPhQWBvPmwcsv2y7l778PdevC2rVuVybiGxRwRETyKY8HHnoIVq2CqlVh1y4bkPzSS+qyEsmKAo6ISD7XsKF1Wd11F5w+DUOHQseOcPCg25WJ5F8KOCIiPiA0FN55B157DYKCbMZVnTqwZo3blYnkTwo4IiI+wuOBgQPhq6+gRg3YvRuaNYPnn4fUVLerE8lfFHBERHxM3bq2Pk63bpCSAo89BrfcAr//7nZlIvmHAo6IiA8qUQJmzoTJk6FIEdvXqk4dW0NHRBRwRER8lscD/fvb1PGoKNizx3YqHzNGXVYiCjgiIj7u6qth3Tro3duCzRNPQPv2sH+/25WJuEcBR0TEDxQvDrGxMHUqFC0KS5dal9Xy5W5XJuIOBRwRET/Sp4+15lx5JSQkQOvW8PTTNhhZpCBRwBER8TO1atm4nH79bMXjf/0L2rSBffvcrkwk7yjgiIj4oaJF4c03YcYMKFYMVqywLqulS92uTCRveC3g7Nq1i379+lG1alVCQkKoXr06o0aNIjk5OdPzvvvuO5o1a0aRIkWIjIzk+eefz/K94+PjufnmmylatCjlypXjn//8J2fOnPHWRxER8Vk9esD69VC7Nvz2G7RrZ4OQ9U+m+DuvBZytW7eSmprKpEmT2Lx5My+99BKvv/46I0eOTH9OUlISbdu2pUqVKqxfv57x48fz9NNPM3ny5HO+b0pKCjfffDPJycmsXr2a2NhYYmJieOqpp7z1UUREfFrNmrb68f33W5fVmDHQsiX8+qvblYl4j8dx8m5P2vHjxzNx4kR27NgBwMSJE3n88cdJSEggKCgIgOHDh/PBBx+wdevWs77H4sWLueWWW9i7dy/ly5cH4PXXX+exxx7jwIED6e9zPklJSYSGhpKYmEjJkiVz6dOJiOR/77wD994LR45AmTIwbRp06OB2VSLZk5Pv7zwdg5OYmEjp0qXTz9esWcMNN9yQKZS0a9eObdu2cejQobO+x5o1a7j66qvTw03aa5KSkti8efNZX3Pq1CmSkpIyHSIiBVGXLrbNQ7168McfcPPN8Oijtku5iD/Js4Czfft2JkyYwIABA9KvJSQkZAoqQPp5QkLCWd/nQl4zduxYQkND04/IyMgL/hwiIr6uRg1YvRoefNDOx4+HG2+E+Hh36xLJTTkOOMOHD8fj8Zz3+Gv30p49e2jfvj2dO3emf//+uVZ8do0YMYLExMT0Y/fu3Xleg4hIfhIcDK++Cu+9B6GhsGaNzbKaP9/tykRyR6GcvmDYsGH06dPnvM+pVq1a+q/37t1LixYtaNKkyd8GD0dERLD/L2uJp51HRESc9b0jIiJYu3Ztjl4THBxMcHDweWsWESmI7rzTdie/+25bIPD22+Hhh+Hf/4ZsDGkUybdy3IITHh5OVFTUeY+0MTV79uyhefPm1K9fn6lTpxIQkPnHNW7cmC+++ILTf+r8Xbp0KTVr1iQsLOysP79x48Zs2rSJ3377LdNrSpYsSa1atXL6cURECryqVWHlShg61M5ffhmuvx527nS1LJGL4rUxOGnhpnLlyrzwwgscOHCAhISETONkunfvTlBQEP369WPz5s3MmTOHV155haFpf8uAefPmERUVlX7etm1batWqRa9evdi4cSMff/wxTzzxBIMGDVIrjYjIBQoKgv/8x7qowsKsNaduXXj/fbcrE7kwXgs4S5cuZfv27SxbtoxKlSpRoUKF9CNNaGgon3zyCTt37qR+/foMGzaMp556ivvuuy/9OYmJiWzbti39PDAwkI8++ojAwEAaN25Mz5496d27N88884y3PoqISIFx660QFweNG0NiInTqZIORT550uzKRnMnTdXDyC62DIyJyfqdPw5NP2lgcsNacd96xGVgibsm36+CIiIhvKFwYxo2DRYugbFnYsMHWzpkzx+3KRLJHAUdERM7pppusy6pZM1v9uGtX2/LhxAm3KxM5PwUcERE5r0sugeXLbZNOjwcmTYLrroM/DY8UyXcUcEREJEuFCsHo0fDxx1CuHHz3HdSvDzNmuF2ZyNkp4IiISLa1aWNdVi1awLFj0KsX3HMPHD/udmUimSngiIhIjlSoAEuXwr/+BQEBMHUqNGwI59jvWMQVCjgiIpJjgYHw1FOwbBlERMAPP1jImToVCt7iI5IfKeCIiMgFa94cNm6Etm1tZtU990B0NBw96nZlUtAp4IiIyEUpVw4WL4bnnrOWnenToUEDG4gs4hYFHBERuWgBATBiBHz2mU0r37YNGjWCyZPVZSXuUMAREZFcc/31NsuqQwfbv2rAAOjeHZKS3K5MChoFHBERyVVly8KCBTB+vK2fM3u2rZmzYYPblUlBooAjIiK5LiAAHnkEvvgCKleG7dtt9ePXXlOXleQNBRwREfGaxo2t5ea22yA5GQYPhi5d4PBhtysTf6eAIyIiXlW6NHzwAbz8su1S/u67tjP5unVuVyb+TAFHRES8zuOBhx6CVaugalXYuROaNrXQoy4r8QYFHBERyTMNG8K330KnTnD6NAwZAh07wsGDblcm/kYBR0RE8lSpUjB3rg04DgqC+fOhbl1Ys8btysSfKOCIiEie83hg4ED46iuoUQPi4+GGG2xqeWqq29WJP1DAERER19StC+vXQ9eucOYMPPoo3Hor/P6725WJr1PAERERV5UsCW+/bds6FCkCixZBnTrw5ZduVya+TAFHRERc5/FA//7w9ddQsybs2QMtWtgGnuqykguhgCMiIvlG7drwzTfQqxekpMDjj8NNN8Fvv7ldmfgaBRwREclXiheH2Fh46y0ICYFPPrEuq88+c7sy8SUKOCIiku94PNC3r7Xm1KoF+/ZBq1bwr39Zy45IVhRwREQk36pVy7Z0uOceG4vz9NPQtq0FHpHzUcAREZF8rWhRmDIFpk+HYsVg+XLrslq61O3KJD9TwBEREZ/Qs6etmVO7tg06btcOnnjC1s8R+SsFHBER8Rk1a9rqxwMG2CadY8ZAy5Y2rVzkzxRwRETEp4SEwOuvw+zZUKKELQhYpw4sXux2ZZKfKOCIiIhPuvtu25m8Xj3b2qFDB3jsMdulXEQBR0REfFaNGrB6NQwebOfPPw/Nm9vmnVKwKeCIiIhPCw6GCRPg3XchNNQCT506sGCB25WJmxRwRETEL3TqBBs2QMOGcOgQ3HYbDBsGycluVyZuUMARERG/UbUqrFwJQ4bY+YsvQrNmsHOnu3VJ3lPAERERvxIUZMHmww8hLAzWroW6deH9992uTPKSAo6IiPil226DuDho3BgSE60L68EH4dQptyuTvKCAIyIifqtyZfj8c3j0UTv/73+hSRPYvt3dusT7FHBERMSvFS4M//43LFwIZcpkrJ3zzjtuVybepIAjIiIFQocO1mXVrBkcOWILBd5/P5w44XZl4g0KOCIiUmBUqmS7kT/+OHg8MGkSXHcdbNvmdmWS2xRwRESkQClUCJ59Fj7+GMqVg+++g/r1YeZMtyuT3KSAIyIiBVKbNtZl1aIFHDsGPXvCvffC8eNuVya5QQFHREQKrAoVYOlSePpp67KaMgWuvRZ++MHtyuRiKeCIiEiBFhgIo0bBsmUQEQGbN9t2DzExblcmF8NrAWfXrl3069ePqlWrEhISQvXq1Rk1ahTJf9kU5LvvvqNZs2YUKVKEyMhInn/++Szf2+Px/O2YPXu2tz6KiIgUAC1awMaN1nV1/Dj07QvR0XD0qNuVyYXwWsDZunUrqampTJo0ic2bN/PSSy/x+uuvM3LkyPTnJCUl0bZtW6pUqcL69esZP348Tz/9NJMnT87y/adOncq+ffvSj44dO3rro4iISAFRrhwsWQJjxkBAAEybZq05mza5XZnklMdxHCevftj48eOZOHEiO3bsAGDixIk8/vjjJCQkEBQUBMDw4cP54IMP2Lp16znfx+PxMG/evAsONUlJSYSGhpKYmEjJkiUv6D1ERMS/ffkldOsGe/ZAkSLw6qs2CNnjcbuygisn3995OgYnMTGR0qVLp5+vWbOGG264IT3cALRr145t27Zx6NCh877XoEGDKFu2LNdeey1vvfUW58tpp06dIikpKdMhIiJyPs2a2Syrm26Ckyfhvvuge3fQV4hvyLOAs337diZMmMCAAQPSryUkJFC+fPlMz0s7T0hIOOd7PfPMM7zzzjssXbqUTp06MXDgQCZMmHDO548dO5bQ0ND0IzIy8iI/jYiIFARly8JHH8Hzz9tg5Nmzbc2cDRvcrkyykuOAM3z48LMO8v3z8dfupT179tC+fXs6d+5M//79L7roJ598kqZNm1K3bl0ee+wxHn30UcaPH3/O548YMYLExMT0Y/fu3Rddg4iIFAwBAfDPf1qXVeXKtlHnddfB//4HeTfIQ3KqUE5fMGzYMPr06XPe51SrVi3913v37qVFixY0adLkb4OHIyIi2L9/f6ZraecRERHZrqlRo0aMHj2aU6dOERwc/LfHg4ODz3pdREQkuxo3tpabvn1h/nwYNAhWrIA334TQULerk7/KccAJDw8nPDw8W8/ds2cPLVq0oH79+kydOpWAgMwNRo0bN+bxxx/n9OnTFC5cGIClS5dSs2ZNwsLCsl1TXFwcYWFhCjEiIuJVpUvDBx/AK6/Ao4/Cu+/C+vW2M3mDBm5XJ3/mtTE4e/bsoXnz5lSuXJkXXniBAwcOkJCQkGlsTffu3QkKCqJfv35s3ryZOXPm8MorrzB06ND058ybN4+oqKj08wULFvDmm2/y/fffs337diZOnMhzzz3Hgw8+6K2PIiIiks7jgYcfhlWroGpV2LkTmjSx0KMuq/wjxy042bV06VK2b9/O9u3bqVSpUqbH0mY8hYaG8sknnzBo0CDq169P2bJleeqpp7jvvvvSn5uYmMi2P23zWrhwYV577TWGDBmC4zjUqFGDF198MVfG9oiIiGRXw4bw7bc2dfy99yz0fPYZvPUW5KATQrwkT9fByS+0Do6IiOQWx7EBx0OHQnIyVKlis62uu87tyvxPvl0HR0RExN94PDbgeM0aqF4dfvnF1tB54QVITXW7uoJLAUdERCQX1KtnXVZ33w1nztjU8ttug99/d7uygkkBR0REJJeULAmzZsGkSRAcDAsXQt26sHKl25UVPAo4IiIiucjjsW0d1q6FmjXh11+heXMYO1ZdVnlJAUdERMQLateGb76Bnj0hJQVGjrR9rX77ze3KCgYFHBERES8pXhymTbOp4yEh8MknUKeOTScX71LAERER8SKPx7Z3WLcOatWCffugVSt45hlr2RHvUMARERHJA1deaeNy+va1sTijRkHbtvCnBf4lFyngiIiI5JFixay7ato0+/Xy5dZl9emnblfmfxRwRERE8livXjYA+eqrYf9+a8l58klbP0dyhwKOiIiIC6Ki4OuvYcAA2+7h2WdtbM6ePW5X5h8UcERERFwSEgKvv26LA5YoAV98YV1WS5a4XZnvU8ARERFxWdeusH69rXr8+++2Xs7w4XD6tNuV+S4FHBERkXzgsstg9WrbuBPg3/+2FZB373a1LJ+lgCMiIpJPFCkC//0vvPsuhIZa4KlTBxYscLsy36OAIyIiks906mQ7kzdsCAcP2q7kw4ZBcrLblfkOBRwREZF8qFo124X84Yft/MUXoVkz2LXLzap8hwKOiIhIPhUUBC+9BB9+CGFhthJy3bowb57bleV/CjgiIiL53G23wYYNcN11cPgw3Hkn/OMfcOqU25XlXwo4IiIiPqBKFVsn59FH7XzCBGjaFH7+2d268isFHBERER9RuLBNH1+4EMqUsbVz6tWDuXPdriz/UcARERHxMR06QFwcXH89JCVBly4wcCCcPOl2ZfmHAo6IiIgPqlQJVqyAkSPB44GJE22Mzo8/ul1Z/qCAIyIi4qMKFYIxY2zvqvBw2LjRuqxmznS7Mvcp4IiIiPi4tm0t3DRvDseOQc+ecO+9cPy425W5RwFHRETED1SoAJ9+CqNGWZfVlCnQqBFs2eJ2Ze5QwBEREfETgYHw9NMWdCIi4PvvoUEDiI11u7K8p4AjIiLiZ1q2tFlWbdpYN1WfPhAdbd1XBYUCjoiIiB8qX94GHz/7LAQEwLRp1pqzaZPbleUNBRwRERE/FRAAjz9u08krVoStW+Haa+HNN8Fx3K7OuxRwRERE/NwNN1iX1U032WKA/fvbTKsjR9yuzHsUcERERAqA8HD46CPb6iEwEN5+G+rXt+DjjxRwRERECoiAANus84svIDISfvrJVj+eONH/uqwUcERERAqYJk2s5ebWW+HUKdvH6u67ITHR7cpyjwKOiIhIAVS6NHz4Ibz4ou1SPneubfPwzTduV5Y7FHBEREQKKI8HhgyBlSvh0kthxw5r3Xn1Vd/vslLAERERKeCuvRY2bIA774TTp+Ghh+zXhw65XdmFU8ARERERSpWCd9+FCRMgKAg++ADq1oWvv3a7sgujgCMiIiKAdVkNHgxr1kD16vDLL3D99fCf//hel5UCjoiIiGRSrx58+63NrDpzBh55BG67Df74w+3Ksk8BR0RERP6mZEmYNQtefx2Cg22RwDp1YNUqtyvLHgUcEREROSuPBwYMsHE4l18Ov/4KN94I48ZBaqrb1Z2fAo6IiIic1zXXwPr1tn9VSgqMGAEdOsBvv7ld2bkp4IiIiEiWiheHadNgyhQICYGPP7Yuq88/d7uys1PAERERkWzxeOCee2DdOqhVC/btg5YtYfRoa9nJT7wWcHbt2kW/fv2oWrUqISEhVK9enVGjRpGcnJz+nJMnT9KnTx+uvvpqChUqRMeOHbP13gcPHqRHjx6ULFmSUqVK0a9fP44ePeqlTyIiIiJ/duWVsHYt9O1rY3GeegratYOEBLcry+C1gLN161ZSU1OZNGkSmzdv5qWXXuL1119n5MiR6c9JSUkhJCSEf/zjH7Ru3Trb792jRw82b97M0qVL+eijj/jiiy+47777vPExRERE5CyKFYO33rJuq2LFYNky67JatsztyozHcfJu6Z7x48czceJEduzY8bfH+vTpw+HDh/nggw/O+x5btmyhVq1arFu3jgYNGgCwZMkSOnTowK+//krFihWzrCMpKYnQ0FASExMpWbLkBX0WERERMVu3QpcusGmTdWM98QSMGgWBgbn7c3Ly/Z2nY3ASExMpXbr0Rb3HmjVrKFWqVHq4AWjdujUBAQF8fY71pE+dOkVSUlKmQ0RERHJHVJRNJb/vPlvxePRoaN3a9rVyS54FnO3btzNhwgQGDBhwUe+TkJBAuXLlMl0rVKgQpUuXJuEcnX9jx44lNDQ0/YiMjLyoGkRERCSzkBCYNMkWByxeHBo0gMKF3asnxwFn+PDheDye8x5bt27N9Jo9e/bQvn17OnfuTP/+/XOt+OwaMWIEiYmJ6cfu3bvzvAYREZGCoGtX2LgRxoxxt45COX3BsGHD6NOnz3mfU61atfRf7927lxYtWtCkSRMmT56c4wL/KiIigt/+srLQmTNnOHjwIBEREWd9TXBwMMHBwRf9s0VERCRrf4oBrslxwAkPDyc8PDxbz92zZw8tWrSgfv36TJ06lYCAi+8Ra9y4MYcPH2b9+vXUr18fgOXLl5OamkqjRo0u+v1FRETE93ltDM6ePXto3rw5lStX5oUXXuDAgQMkJCT8bZzMDz/8QFxcHAcPHiQxMZG4uDji4uLSH1+7di1RUVHs2bMHgCuuuIL27dvTv39/1q5dy6pVqxg8eDBdu3bN1gwqERER8X85bsHJrqVLl7J9+3a2b99OpUqVMj3255npHTp04Jdffkk/r1u3bqbnHD9+nG3btnH6T0OxZ86cyeDBg2nVqhUBAQF06tSJV1991VsfRURERHxMnq6Dk19oHRwRERHfk2/XwRERERHJCwo4IiIi4ncUcERERMTvKOCIiIiI31HAEREREb+jgCMiIiJ+RwFHRERE/I4CjoiIiPgdBRwRERHxO17bqiE/S1u8OSkpyeVKREREJLvSvrezswlDgQw4R44cASAyMtLlSkRERCSnjhw5Qmho6HmfUyD3okpNTWXv3r2UKFECj8eTq++dlJREZGQku3fv1j5XXqT7nDd0n/OG7nPe0H3OO966147jcOTIESpWrEhAwPlH2RTIFpyAgIC/7XCe20qWLKm/QHlA9zlv6D7nDd3nvKH7nHe8ca+zarlJo0HGIiIi4ncUcERERMTvKODksuDgYEaNGkVwcLDbpfg13ee8ofucN3Sf84buc97JD/e6QA4yFhEREf+mFhwRERHxOwo4IiIi4ncUcERERMTvKOCIiIiI31HAuQCvvfYal156KUWKFKFRo0asXbv2vM+fO3cuUVFRFClShKuvvppFixblUaW+LSf3+Y033qBZs2aEhYURFhZG69ats/x9EZPTP89pZs+ejcfjoWPHjt4t0E/k9D4fPnyYQYMGUaFCBYKDg7n88sv1b0c25PQ+v/zyy9SsWZOQkBAiIyMZMmQIJ0+ezKNqfdMXX3zBrbfeSsWKFfF4PHzwwQdZvuazzz6jXr16BAcHU6NGDWJiYrxeJ47kyOzZs52goCDnrbfecjZv3uz079/fKVWqlLN///6zPn/VqlVOYGCg8/zzzzs//PCD88QTTziFCxd2Nm3alMeV+5ac3ufu3bs7r732mrNhwwZny5YtTp8+fZzQ0FDn119/zePKfUtO73OanTt3OpdcconTrFkz5/bbb8+bYn1YTu/zqVOnnAYNGjgdOnRwVq5c6ezcudP57LPPnLi4uDyu3Lfk9D7PnDnTCQ4OdmbOnOns3LnT+fjjj50KFSo4Q4YMyePKfcuiRYucxx9/3Hn//fcdwJk3b955n79jxw6naNGiztChQ50ffvjBmTBhghMYGOgsWbLEq3Uq4OTQtdde6wwaNCj9PCUlxalYsaIzduzYsz6/S5cuzs0335zpWqNGjZwBAwZ4tU5fl9P7/FdnzpxxSpQo4cTGxnqrRL9wIff5zJkzTpMmTZw333zTiY6OVsDJhpze54kTJzrVqlVzkpOT86pEv5DT+zxo0CCnZcuWma4NHTrUadq0qVfr9CfZCTiPPvqoc+WVV2a6dvfddzvt2rXzYmWOoy6qHEhOTmb9+vW0bt06/VpAQACtW7dmzZo1Z33NmjVrMj0foF27dud8vlzYff6r48ePc/r0aUqXLu2tMn3ehd7nZ555hnLlytGvX7+8KNPnXch9nj9/Po0bN2bQoEGUL1+eq666iueee46UlJS8KtvnXMh9btKkCevXr0/vxtqxYweLFi2iQ4cOeVJzQeHW92CB3GzzQv3++++kpKRQvnz5TNfLly/P1q1bz/qahISEsz4/ISHBa3X6ugu5z3/12GOPUbFixb/9pZIMF3KfV65cyZQpU4iLi8uDCv3DhdznHTt2sHz5cnr06MGiRYvYvn07AwcO5PTp04waNSovyvY5F3Kfu3fvzu+//87111+P4zicOXOG+++/n5EjR+ZFyQXGub4Hk5KSOHHiBCEhIV75uWrBEb8zbtw4Zs+ezbx58yhSpIjb5fiNI0eO0KtXL9544w3Kli3rdjl+LTU1lXLlyjF58mTq16/P3XffzeOPP87rr7/udml+5bPPPuO5557jf//7H99++y3vv/8+CxcuZPTo0W6XJrlALTg5ULZsWQIDA9m/f3+m6/v37yciIuKsr4mIiMjR8+XC7nOaF154gXHjxvHpp59Su3Ztb5bp83J6n3/++Wd27drFrbfemn4tNTUVgEKFCrFt2zaqV6/u3aJ90IX8ea5QoQKFCxcmMDAw/doVV1xBQkICycnJBAUFebVmX3Qh9/nJJ5+kV69e3HvvvQBcffXVHDt2jPvuu4/HH3+cgAC1AeSGc30PlixZ0mutN6AWnBwJCgqifv36LFu2LP1aamoqy5Yto3Hjxmd9TePGjTM9H2Dp0qXnfL5c2H0GeP755xk9ejRLliyhQYMGeVGqT8vpfY6KimLTpk3ExcWlH7fddhstWrQgLi6OyMjIvCzfZ1zIn+emTZuyffv29AAJ8OOPP1KhQgWFm3O4kPt8/Pjxv4WYtFDpaJvGXOPa96BXhzD7odmzZzvBwcFOTEyM88MPPzj33XefU6pUKSchIcFxHMfp1auXM3z48PTnr1q1yilUqJDzwgsvOFu2bHFGjRqlaeLZkNP7PG7cOCcoKMh59913nX379qUfR44ccesj+ISc3ue/0iyq7MnpfY6Pj3dKlCjhDB482Nm2bZvz0UcfOeXKlXOeffZZtz6CT8jpfR41apRTokQJZ9asWc6OHTucTz75xKlevbrTpUsXtz6CTzhy5IizYcMGZ8OGDQ7gvPjii86GDRucX375xXEcxxk+fLjTq1ev9OenTRP/5z//6WzZssV57bXXNE08v5owYYJTuXJlJygoyLn22mudr776Kv2xG2+80YmOjs70/Hfeece5/PLLnaCgIOfKK690Fi5cmMcV+6ac3OcqVao4wN+OUaNG5X3hPianf57/TAEn+3J6n1evXu00atTICQ4OdqpVq+aMGTPGOXPmTB5X7Xtycp9Pnz7tPP3000716tWdIkWKOJGRkc7AgQOdQ4cO5X3hPmTFihVn/fc27d5GR0c7N954499eU6dOHScoKMipVq2aM3XqVK/X6XEctcOJiIiIf9EYHBEREfE7CjgiIiLidxRwRERExO8o4IiIiIjfUcARERERv6OAIyIiIn5HAUdERET8jgKOiIiI+B0FHBHxGbt27cLj8RAXF+e1n9GnTx86duzotfcXkbyhgCMieaZPnz54PJ6/He3bt8/W6yMjI9m3bx9XXXWVlysVEV9XyO0CRKRgad++PVOnTs10LTg4OFuvDQwMJCIiwhtliYifUQuOiOSp4OBgIiIiMh1hYWEAeDweJk6cyE033URISAjVqlXj3XffTX/tX7uoDh06RI8ePQgPDyckJITLLrssU3jatGkTLVu2JCQkhDJlynDfffdx9OjR9MdTUlIYOnQopUqVokyZMjz66KP8dXu+1NRUxo4dS9WqVQkJCeGaa67JVJOI5E8KOCKSrzz55JN06tSJjRs30qNHD7p27cqWLVvO+dwffviBxYsXs2XLFiZOnEjZsmUBOHbsGO3atSMsLIx169Yxd+5cPv30UwYPHpz++v/85z/ExMTw1ltvsXLlSg4ePMi8efMy/YyxY8cybdo0Xn/9dTZv3syQIUPo2bMnn3/+ufdugohcPK/vVy4i8v+io6OdwMBAp1ixYpmOMWPGOI7jOIBz//33Z3pNo0aNnAceeMBxHMfZuXOnAzgbNmxwHMdxbr31Vqdv375n/VmTJ092wsLCnKNHj6ZfW7hwoRMQEOAkJCQ4juM4FSpUcJ5//vn0x0+fPu1UqlTJuf322x3HcZyTJ086RYsWdVavXp3pvfv16+d069btwm+EiHidxuCISJ5q0aIFEydOzHStdOnS6b9u3LhxpscaN258zllTDzzwAJ06deLbb7+lbdu2dOzYkSZNmgCwZcsWrrnmGooVK5b+/KZNm5Kamsq2bdsoUqQI+/bto1GjRumPFypUiAYNGqR3U23fvp3jx4/Tpk2bTD83OTmZunXr5vzDi0ieUcARkTxVrFgxatSokSvvddNNN/HLL7+waNEili5dSqtWrRg0aBAvvPBCrrx/2nidhQsXcskll2R6LLsDo0XEHRqDIyL5yldfffW38yuuuOKczw8PDyc6OpoZM2bw8ssvM3nyZACuuOIKNm7cyLFjx9Kfu2rVKgICAqhZsyahoaFUqFCBr7/+Ov3xM2fOsH79+vTzWrVqERwcTHx8PDVq1Mh0REZG5tZHFhEvUAuOiOSpU6dOkZCQkOlaoUKF0gcHz507lwYNGnD99dczc+ZM1q5dy5QpU876Xk899RT169fnyiuv5NSpU3z00UfpYahHjx6MGjWK6Ohonn76aQ4cOMCDDz5Ir169KF++PAAPPfQQ48aN47LLLiMqKooXX3yRw4cPp79/iRIleOSRRxgyZAipqalcf/31JCYmsmrVKkqWLEl0dLQX7pCI5AYFHBHJU0uWLKFChQqZrtWsWZOtW7cC8K9//YvZs2czcOBAKlSowKxZs6hVq9ZZ3ysoKIgRI0awa9cuQkJCaNasGbNnzwagaNGifPzxxzz00EM0bNiQokWL0qlTJ1588cX01w8bNox9+/YRHR1NQEAA99xzD3fccQeJiYnpzxk9ejTh4eGMHTuWHTt2UKpUKerVq8fIkSNz+9aISC7yOM5fFn0QEXGJx+Nh3rx52ipBRC6axuCIiIiI31HAEREREb+jMTgikm+ox1xEcotacERERMTvKOCIiIiI31HAEREREb+jgCMiIiJ+RwFHRERE/I4CjoiIiPgdBRwRERHxOwo4IiIi4nf+D6J1+hkDT8Q4AAAAAElFTkSuQmCC\n",
      "text/plain": [
       "<Figure size 640x480 with 1 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "agent.plot()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Optimizer direttamente in worker, calcolare i gradienti, sincronizzare ricevendo i pesi dal controller, \n",
    "# Metterli nel local model, applicare i gradienti, rispedendo indietro i pesi attraverso worker queue.\n",
    "\n",
    "# PPO sincronizzando con global model?\n",
    "\n",
    "# LSTM al posto di framestack? come? \n",
    "\n",
    "# GAMARE L'ERRORE CHE COMPARE SE SI TOGLIE @TF FUNCTION"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.10"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}