sorb_disable_training.ipynb

{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "colab": {
      "name": "sorb_disable_training.ipynb",
      "provenance": [],
      "collapsed_sections": []
    },
    "kernelspec": {
      "display_name": "Python 3",
      "language": "python",
      "name": "python3"
    },
    "language_info": {
      "codemirror_mode": {
        "name": "ipython",
        "version": 3
      },
      "file_extension": ".py",
      "mimetype": "text/x-python",
      "name": "python",
      "nbconvert_exporter": "python",
      "pygments_lexer": "ipython3",
      "version": "3.5.2"
    },
    "widgets": {
      "application/vnd.jupyter.widget-state+json": {
        "ea30e0c68b2e43d491cd4011cd3be63a": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "state": {
            "_view_name": "HBoxView",
            "_dom_classes": [],
            "_model_name": "HBoxModel",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "box_style": "",
            "layout": "IPY_MODEL_520c62e857f34806814dbd5339b95e01",
            "_model_module": "@jupyter-widgets/controls",
            "children": [
              "IPY_MODEL_8c2b1a7399914df0869bd1e8e28d20b7",
              "IPY_MODEL_808ab10d324d4eada3ee2f7c95417398"
            ]
          }
        },
        "520c62e857f34806814dbd5339b95e01": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "8c2b1a7399914df0869bd1e8e28d20b7": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "state": {
            "_view_name": "ProgressView",
            "style": "IPY_MODEL_1ed67ab07db543249d2cc4939007eae6",
            "_dom_classes": [],
            "description": "100%",
            "_model_name": "FloatProgressModel",
            "bar_style": "success",
            "max": 100,
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": 100,
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "orientation": "horizontal",
            "min": 0,
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_fd10ff0e6c63427fb9772504a9840952"
          }
        },
        "808ab10d324d4eada3ee2f7c95417398": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "state": {
            "_view_name": "HTMLView",
            "style": "IPY_MODEL_060a0a5419e74eeca9851a7a6ce42f0d",
            "_dom_classes": [],
            "description": "",
            "_model_name": "HTMLModel",
            "placeholder": "​",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": " 100/100 [00:52&lt;00:00,  1.91it/s]",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_feb3500bdb6441e3920ac871bb02ba63"
          }
        },
        "1ed67ab07db543249d2cc4939007eae6": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "ProgressStyleModel",
            "description_width": "initial",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "bar_color": null,
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "fd10ff0e6c63427fb9772504a9840952": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "060a0a5419e74eeca9851a7a6ce42f0d": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "DescriptionStyleModel",
            "description_width": "",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "feb3500bdb6441e3920ac871bb02ba63": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "925c0bcbbad949429ca9dd24e8554797": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "state": {
            "_view_name": "HBoxView",
            "_dom_classes": [],
            "_model_name": "HBoxModel",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "box_style": "",
            "layout": "IPY_MODEL_c3c4d41bd37548638b72492bb52611c9",
            "_model_module": "@jupyter-widgets/controls",
            "children": [
              "IPY_MODEL_b3e335e9111e468db195851f0a69609a",
              "IPY_MODEL_8ea47bfd1b3e47078d832fab1b62e1a3"
            ]
          }
        },
        "c3c4d41bd37548638b72492bb52611c9": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "b3e335e9111e468db195851f0a69609a": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "state": {
            "_view_name": "ProgressView",
            "style": "IPY_MODEL_3ea4276d16734c0a9edb73feb1421bf0",
            "_dom_classes": [],
            "description": "100%",
            "_model_name": "FloatProgressModel",
            "bar_style": "success",
            "max": 100,
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": 100,
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "orientation": "horizontal",
            "min": 0,
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_1ce0d785f5344e67bb888ca005870d75"
          }
        },
        "8ea47bfd1b3e47078d832fab1b62e1a3": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "state": {
            "_view_name": "HTMLView",
            "style": "IPY_MODEL_7aeb56cce93c49238a4b629537dc46e8",
            "_dom_classes": [],
            "description": "",
            "_model_name": "HTMLModel",
            "placeholder": "​",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": " 100/100 [00:21&lt;00:00,  4.58it/s]",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_efa771ec08b4449e9fab37a03fa2b4f7"
          }
        },
        "3ea4276d16734c0a9edb73feb1421bf0": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "ProgressStyleModel",
            "description_width": "initial",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "bar_color": null,
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "1ce0d785f5344e67bb888ca005870d75": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "7aeb56cce93c49238a4b629537dc46e8": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "DescriptionStyleModel",
            "description_width": "",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "efa771ec08b4449e9fab37a03fa2b4f7": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "35d832b8dac443bab1a5fed1ca2db46c": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "state": {
            "_view_name": "HBoxView",
            "_dom_classes": [],
            "_model_name": "HBoxModel",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "box_style": "",
            "layout": "IPY_MODEL_80010960ba4c4f878d112a386f899af9",
            "_model_module": "@jupyter-widgets/controls",
            "children": [
              "IPY_MODEL_6805fe15923e4f13b93ce9a538b2ed30",
              "IPY_MODEL_59bf5cbc0d044cd48e29f3c43c370d8f"
            ]
          }
        },
        "80010960ba4c4f878d112a386f899af9": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "6805fe15923e4f13b93ce9a538b2ed30": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "state": {
            "_view_name": "ProgressView",
            "style": "IPY_MODEL_a5358168528641edb8c848fbda609a78",
            "_dom_classes": [],
            "description": " 91%",
            "_model_name": "FloatProgressModel",
            "bar_style": "danger",
            "max": 100,
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": 91,
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "orientation": "horizontal",
            "min": 0,
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_8e57a6dc5abe48088bbd7f2295921d95"
          }
        },
        "59bf5cbc0d044cd48e29f3c43c370d8f": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "state": {
            "_view_name": "HTMLView",
            "style": "IPY_MODEL_cf237d1de3e54d6caf1316a7a5592659",
            "_dom_classes": [],
            "description": "",
            "_model_name": "HTMLModel",
            "placeholder": "​",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": " 91/100 [00:00&lt;00:00, 136.84it/s]",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_4b318a99f7a948a7923fe72426607271"
          }
        },
        "a5358168528641edb8c848fbda609a78": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "ProgressStyleModel",
            "description_width": "initial",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "bar_color": null,
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "8e57a6dc5abe48088bbd7f2295921d95": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "cf237d1de3e54d6caf1316a7a5592659": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "DescriptionStyleModel",
            "description_width": "",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "4b318a99f7a948a7923fe72426607271": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "9de12f52414c46d0a92e5808f9c4535e": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "state": {
            "_view_name": "HBoxView",
            "_dom_classes": [],
            "_model_name": "HBoxModel",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "box_style": "",
            "layout": "IPY_MODEL_93f0c6efb59649c6a7c9d9ebaec5aa9b",
            "_model_module": "@jupyter-widgets/controls",
            "children": [
              "IPY_MODEL_fb8455b5af524243bda1fdb6c268bbd5",
              "IPY_MODEL_43da1dc528d54982b4484dffbee77b0f"
            ]
          }
        },
        "93f0c6efb59649c6a7c9d9ebaec5aa9b": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "fb8455b5af524243bda1fdb6c268bbd5": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "state": {
            "_view_name": "ProgressView",
            "style": "IPY_MODEL_f3d7366b832a4a54bc8d832d4abbd0d0",
            "_dom_classes": [],
            "description": " 92%",
            "_model_name": "FloatProgressModel",
            "bar_style": "danger",
            "max": 100,
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": 92,
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "orientation": "horizontal",
            "min": 0,
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_7e1bfa38e5be408db65068234c796988"
          }
        },
        "43da1dc528d54982b4484dffbee77b0f": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "state": {
            "_view_name": "HTMLView",
            "style": "IPY_MODEL_f9f0810f26d9410f97cb5043d967e885",
            "_dom_classes": [],
            "description": "",
            "_model_name": "HTMLModel",
            "placeholder": "​",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": " 92/100 [00:01&lt;00:00, 59.84it/s]",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_3c49fdcb269849fab641530b40fecd57"
          }
        },
        "f3d7366b832a4a54bc8d832d4abbd0d0": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "ProgressStyleModel",
            "description_width": "initial",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "bar_color": null,
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "7e1bfa38e5be408db65068234c796988": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "f9f0810f26d9410f97cb5043d967e885": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "DescriptionStyleModel",
            "description_width": "",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "3c49fdcb269849fab641530b40fecd57": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "27440cd240534c869698a5ce8bfdde1e": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "state": {
            "_view_name": "HBoxView",
            "_dom_classes": [],
            "_model_name": "HBoxModel",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "box_style": "",
            "layout": "IPY_MODEL_5e2f2130e96649ada0bdb6bc245a9b73",
            "_model_module": "@jupyter-widgets/controls",
            "children": [
              "IPY_MODEL_8a6ea8da1edb42f893cccdac946e3e06",
              "IPY_MODEL_658771ec879d47e082c2571f179043d2"
            ]
          }
        },
        "5e2f2130e96649ada0bdb6bc245a9b73": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "8a6ea8da1edb42f893cccdac946e3e06": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "state": {
            "_view_name": "ProgressView",
            "style": "IPY_MODEL_7a903ef939f745be85109dbdb495207e",
            "_dom_classes": [],
            "description": "100%",
            "_model_name": "FloatProgressModel",
            "bar_style": "success",
            "max": 1000,
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": 1000,
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "orientation": "horizontal",
            "min": 0,
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_9b6498d13a8f4a6d8feaa793611e02c3"
          }
        },
        "658771ec879d47e082c2571f179043d2": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "state": {
            "_view_name": "HTMLView",
            "style": "IPY_MODEL_8f6bb53c48794c5598b49bad14c9074e",
            "_dom_classes": [],
            "description": "",
            "_model_name": "HTMLModel",
            "placeholder": "​",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": " 1000/1000 [00:00&lt;00:00, 1305.76it/s]",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_370ee068b2ce49af8046f383456287e5"
          }
        },
        "7a903ef939f745be85109dbdb495207e": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "ProgressStyleModel",
            "description_width": "initial",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "bar_color": null,
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "9b6498d13a8f4a6d8feaa793611e02c3": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "8f6bb53c48794c5598b49bad14c9074e": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "DescriptionStyleModel",
            "description_width": "",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "370ee068b2ce49af8046f383456287e5": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        }
      }
    }
  },
  "cells": [
    {
      "cell_type": "markdown",
      "metadata": {
        "colab_type": "text",
        "id": "view-in-github"
      },
      "source": [
        "<a href=\"https://colab.research.google.com/github/alycialee/cs159_sorb/blob/master/sorb_disable_training.ipynb\" target=\"_parent\"><img src=\"https://colab.research.google.com/assets/colab-badge.svg\" alt=\"Open In Colab\"/></a>"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "colab_type": "text",
        "id": "YJhveYqWn-tA"
      },
      "source": [
        "##### Copyright 2019 Google LLC.\n",
        "\n",
        "Licensed under the Apache License, Version 2.0 (the \"License\");"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "cellView": "form",
        "colab_type": "code",
        "id": "pKe5zvMRoAB7",
        "colab": {}
      },
      "source": [
        "#@title License\n",
        "# Licensed under the Apache License, Version 2.0 (the \"License\");\n",
        "# you may not use this file except in compliance with the License.\n",
        "# You may obtain a copy of the License at\n",
        "#\n",
        "# https://www.apache.org/licenses/LICENSE-2.0\n",
        "#\n",
        "# Unless required by applicable law or agreed to in writing, software\n",
        "# distributed under the License is distributed on an \"AS IS\" BASIS,\n",
        "# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n",
        "# See the License for the specific language governing permissions and\n",
        "# limitations under the License."
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "colab_type": "text",
        "id": "Dc1snEt7qwn6"
      },
      "source": [
        "# [_Search on the Replay Buffer_: Bridging Planning and Reinforcement Learning](https://arxiv.org/abs/1906.05253)\n",
        "\n",
        "*Benjamin Eysenbach, Ruslan Salakhutdinov, and Sergey Levine*\n",
        "\n",
        "\n",
        "What is *SoRB*? *SoRB* is a machine learning algorithm that learns to make decisions to reach goals. Typically, these sorts of control algorithms either rely on planning methods or reinforcement learning methods, both of which have limitations. Planning algorithms can reason over long horizons, but cannot deal with high-dimensional observations. Reinforcement learning algorithms fail to plan over long distances. *SoRB* attempts to combine the best of both algorithms.\n",
        "\n",
        "\n",
        "<center><img src=\"http://drive.google.com/uc?export=view&id=1LCqQJb06vyBOHl1uX-bi1p_LvuDMfPIF\" \n",
        "alt=\"search on the replay buffer\" width=\"800px\"/></center>\n",
        "The figure above highlights the basic idea.\n",
        "\n",
        "* (a) Goal-conditioned RL often fails to reach distant goals, but can successfully reach nearby goals (indicated in green).\n",
        "* (b) Our goal is to use observations in our replay buffer (yellow squares) as waypoints leading to the goal.\n",
        "* (c) We automatically find these waypoints by using the agent's value function to predict when two states are nearby, and building the corresponding graph.\n",
        "* (d) We run graph search to find the sequence of waypoints (blue arrows), and then use our goal-conditioned policy to reach each waypoint.\n",
        "\n",
        "<center><img src=\"http://drive.google.com/uc?export=view&id=1RsSVVYEcRADJmd78JYLARR0nGJtrpIN9\" \n",
        "alt=\"search on the replay buffer\" width=\"400px\"/></center>\n",
        "In our paper, we show how this algorithm can be used to solve complex visual navigation tasks, like the one shown above. In this colab notebook, we implement a basic version of *SoRB* on a simple navigation task. Interactive visualizations below allow you to explore the effect of various hyperparameters, as well as train your own agents.\n",
        "\n",
        "### Related Work\n",
        "A number of prior works have proposed methods for learning goal-conditioned policies and combining planning with RL. We encourage you to check out these related works:\n",
        "* Kaelbling, Leslie Pack. \"Learning to achieve goals.\" IJCAI. 1993.\n",
        "* Schaul, Tom, et al. \"Universal value function approximators.\" International conference on machine learning. 2015.\n",
        "* Pong, Vitchyr, et al. \"Temporal difference models: Model-free deep rl for model-based control.\" arXiv preprint arXiv:1802.09081 (2018).\n",
        "* Francis, Anthony, et al. \"Long-Range Indoor Navigation with PRM-RL.\" arXiv preprint arXiv:1902.09458 (2019).\n",
        "* Savinov, Nikolay, Alexey Dosovitskiy, and Vladlen Koltun. \"Semi-parametric topological memory for navigation.\" arXiv preprint arXiv:1803.00653 (2018).\n",
        "\n",
        "\n",
        "If you find this code useful, please consider citing our paper: [https://arxiv.org/abs/1906.05253](https://arxiv.org/abs/1906.05253)\n",
        "```\n",
        "@misc{eysenbach2019,\n",
        "Author = {Benjamin Eysenbach and Ruslan Salakhutdinov and Sergey Levine},\n",
        "Title = {Search on the Replay Buffer: Bridging Planning and Reinforcement Learning},\n",
        "Year = {2019},\n",
        "Eprint = {arXiv:1906.05253},\n",
        "}\n",
        "```\n",
        "\n",
        "### Getting Started\n",
        "To get started, click the \"Connect\" button in the top right corner of the screen. You should see a green checkmark next to some stats about RAM and Disk. Run each of the cells below, either by clicking the \"run cell\" button on the left of each cell, or by pressing [ctrl]+[enter]. **Double click on any cell to see the code.**\n",
        "\n"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "cellView": "both",
        "colab_type": "code",
        "id": "JpFNoJBc7zur",
        "outputId": "52e54ef0-336d-4e9b-fd9a-2548fc12a8d8",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 819
        }
      },
      "source": [
        "#@title Install Dependencies\n",
        "!pip install tensorflow\n",
        "!pip install tf-agents"
      ],
      "execution_count": 2,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Requirement already satisfied: tensorflow in /usr/local/lib/python3.6/dist-packages (2.2.0)\n",
            "Requirement already satisfied: grpcio>=1.8.6 in /usr/local/lib/python3.6/dist-packages (from tensorflow) (1.29.0)\n",
            "Requirement already satisfied: protobuf>=3.8.0 in /usr/local/lib/python3.6/dist-packages (from tensorflow) (3.10.0)\n",
            "Requirement already satisfied: google-pasta>=0.1.8 in /usr/local/lib/python3.6/dist-packages (from tensorflow) (0.2.0)\n",
            "Requirement already satisfied: numpy<2.0,>=1.16.0 in /usr/local/lib/python3.6/dist-packages (from tensorflow) (1.18.4)\n",
            "Requirement already satisfied: keras-preprocessing>=1.1.0 in /usr/local/lib/python3.6/dist-packages (from tensorflow) (1.1.2)\n",
            "Requirement already satisfied: wrapt>=1.11.1 in /usr/local/lib/python3.6/dist-packages (from tensorflow) (1.12.1)\n",
            "Requirement already satisfied: six>=1.12.0 in /usr/local/lib/python3.6/dist-packages (from tensorflow) (1.12.0)\n",
            "Requirement already satisfied: absl-py>=0.7.0 in /usr/local/lib/python3.6/dist-packages (from tensorflow) (0.9.0)\n",
            "Requirement already satisfied: h5py<2.11.0,>=2.10.0 in /usr/local/lib/python3.6/dist-packages (from tensorflow) (2.10.0)\n",
            "Requirement already satisfied: termcolor>=1.1.0 in /usr/local/lib/python3.6/dist-packages (from tensorflow) (1.1.0)\n",
            "Requirement already satisfied: wheel>=0.26; python_version >= \"3\" in /usr/local/lib/python3.6/dist-packages (from tensorflow) (0.34.2)\n",
            "Requirement already satisfied: gast==0.3.3 in /usr/local/lib/python3.6/dist-packages (from tensorflow) (0.3.3)\n",
            "Requirement already satisfied: scipy==1.4.1; python_version >= \"3\" in /usr/local/lib/python3.6/dist-packages (from tensorflow) (1.4.1)\n",
            "Requirement already satisfied: tensorflow-estimator<2.3.0,>=2.2.0 in /usr/local/lib/python3.6/dist-packages (from tensorflow) (2.2.0)\n",
            "Requirement already satisfied: tensorboard<2.3.0,>=2.2.0 in /usr/local/lib/python3.6/dist-packages (from tensorflow) (2.2.1)\n",
            "Requirement already satisfied: astunparse==1.6.3 in /usr/local/lib/python3.6/dist-packages (from tensorflow) (1.6.3)\n",
            "Requirement already satisfied: opt-einsum>=2.3.2 in /usr/local/lib/python3.6/dist-packages (from tensorflow) (3.2.1)\n",
            "Requirement already satisfied: setuptools in /usr/local/lib/python3.6/dist-packages (from protobuf>=3.8.0->tensorflow) (46.3.0)\n",
            "Requirement already satisfied: google-auth-oauthlib<0.5,>=0.4.1 in /usr/local/lib/python3.6/dist-packages (from tensorboard<2.3.0,>=2.2.0->tensorflow) (0.4.1)\n",
            "Requirement already satisfied: google-auth<2,>=1.6.3 in /usr/local/lib/python3.6/dist-packages (from tensorboard<2.3.0,>=2.2.0->tensorflow) (1.7.2)\n",
            "Requirement already satisfied: markdown>=2.6.8 in /usr/local/lib/python3.6/dist-packages (from tensorboard<2.3.0,>=2.2.0->tensorflow) (3.2.2)\n",
            "Requirement already satisfied: werkzeug>=0.11.15 in /usr/local/lib/python3.6/dist-packages (from tensorboard<2.3.0,>=2.2.0->tensorflow) (1.0.1)\n",
            "Requirement already satisfied: requests<3,>=2.21.0 in /usr/local/lib/python3.6/dist-packages (from tensorboard<2.3.0,>=2.2.0->tensorflow) (2.23.0)\n",
            "Requirement already satisfied: tensorboard-plugin-wit>=1.6.0 in /usr/local/lib/python3.6/dist-packages (from tensorboard<2.3.0,>=2.2.0->tensorflow) (1.6.0.post3)\n",
            "Requirement already satisfied: requests-oauthlib>=0.7.0 in /usr/local/lib/python3.6/dist-packages (from google-auth-oauthlib<0.5,>=0.4.1->tensorboard<2.3.0,>=2.2.0->tensorflow) (1.3.0)\n",
            "Requirement already satisfied: pyasn1-modules>=0.2.1 in /usr/local/lib/python3.6/dist-packages (from google-auth<2,>=1.6.3->tensorboard<2.3.0,>=2.2.0->tensorflow) (0.2.8)\n",
            "Requirement already satisfied: cachetools<3.2,>=2.0.0 in /usr/local/lib/python3.6/dist-packages (from google-auth<2,>=1.6.3->tensorboard<2.3.0,>=2.2.0->tensorflow) (3.1.1)\n",
            "Requirement already satisfied: rsa<4.1,>=3.1.4 in /usr/local/lib/python3.6/dist-packages (from google-auth<2,>=1.6.3->tensorboard<2.3.0,>=2.2.0->tensorflow) (4.0)\n",
            "Requirement already satisfied: importlib-metadata; python_version < \"3.8\" in /usr/local/lib/python3.6/dist-packages (from markdown>=2.6.8->tensorboard<2.3.0,>=2.2.0->tensorflow) (1.6.0)\n",
            "Requirement already satisfied: chardet<4,>=3.0.2 in /usr/local/lib/python3.6/dist-packages (from requests<3,>=2.21.0->tensorboard<2.3.0,>=2.2.0->tensorflow) (3.0.4)\n",
            "Requirement already satisfied: certifi>=2017.4.17 in /usr/local/lib/python3.6/dist-packages (from requests<3,>=2.21.0->tensorboard<2.3.0,>=2.2.0->tensorflow) (2020.4.5.1)\n",
            "Requirement already satisfied: urllib3!=1.25.0,!=1.25.1,<1.26,>=1.21.1 in /usr/local/lib/python3.6/dist-packages (from requests<3,>=2.21.0->tensorboard<2.3.0,>=2.2.0->tensorflow) (1.24.3)\n",
            "Requirement already satisfied: idna<3,>=2.5 in /usr/local/lib/python3.6/dist-packages (from requests<3,>=2.21.0->tensorboard<2.3.0,>=2.2.0->tensorflow) (2.9)\n",
            "Requirement already satisfied: oauthlib>=3.0.0 in /usr/local/lib/python3.6/dist-packages (from requests-oauthlib>=0.7.0->google-auth-oauthlib<0.5,>=0.4.1->tensorboard<2.3.0,>=2.2.0->tensorflow) (3.1.0)\n",
            "Requirement already satisfied: pyasn1<0.5.0,>=0.4.6 in /usr/local/lib/python3.6/dist-packages (from pyasn1-modules>=0.2.1->google-auth<2,>=1.6.3->tensorboard<2.3.0,>=2.2.0->tensorflow) (0.4.8)\n",
            "Requirement already satisfied: zipp>=0.5 in /usr/local/lib/python3.6/dist-packages (from importlib-metadata; python_version < \"3.8\"->markdown>=2.6.8->tensorboard<2.3.0,>=2.2.0->tensorflow) (3.1.0)\n",
            "Requirement already satisfied: tf-agents in /usr/local/lib/python3.6/dist-packages (0.4.0)\n",
            "Requirement already satisfied: numpy>=1.13.3 in /usr/local/lib/python3.6/dist-packages (from tf-agents) (1.18.4)\n",
            "Requirement already satisfied: absl-py>=0.6.1 in /usr/local/lib/python3.6/dist-packages (from tf-agents) (0.9.0)\n",
            "Requirement already satisfied: tensorflow-probability>=0.8.0 in /usr/local/lib/python3.6/dist-packages (from tf-agents) (0.10.0)\n",
            "Requirement already satisfied: six>=1.10.0 in /usr/local/lib/python3.6/dist-packages (from tf-agents) (1.12.0)\n",
            "Requirement already satisfied: gin-config==0.1.3 in /usr/local/lib/python3.6/dist-packages (from tf-agents) (0.1.3)\n",
            "Requirement already satisfied: decorator in /usr/local/lib/python3.6/dist-packages (from tensorflow-probability>=0.8.0->tf-agents) (4.4.2)\n",
            "Requirement already satisfied: cloudpickle>=1.2.2 in /usr/local/lib/python3.6/dist-packages (from tensorflow-probability>=0.8.0->tf-agents) (1.3.0)\n",
            "Requirement already satisfied: gast>=0.3.2 in /usr/local/lib/python3.6/dist-packages (from tensorflow-probability>=0.8.0->tf-agents) (0.3.3)\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "cellView": "both",
        "colab_type": "code",
        "id": "bbiO8OnH8Bk7",
        "colab": {}
      },
      "source": [
        "#@title Import dependencies.\n",
        "from __future__ import absolute_import\n",
        "from __future__ import division\n",
        "from __future__ import print_function\n",
        "\n",
        "import collections\n",
        "import random\n",
        "import time\n",
        "import tqdm\n",
        "\n",
        "import gym\n",
        "import gym.spaces\n",
        "import matplotlib.pyplot as plt\n",
        "import networkx as nx\n",
        "import numpy as np\n",
        "import scipy.sparse.csgraph\n",
        "import tensorflow as tf\n",
        "\n",
        "from tf_agents.agents import tf_agent\n",
        "from tf_agents.agents.ddpg import actor_network\n",
        "from tf_agents.agents.ddpg import critic_network\n",
        "from tf_agents.drivers import dynamic_step_driver\n",
        "from tf_agents.environments import gym_wrapper\n",
        "from tf_agents.environments import tf_py_environment\n",
        "from tf_agents.environments import wrappers\n",
        "from tf_agents.eval import metric_utils\n",
        "from tf_agents.metrics import tf_metrics\n",
        "from tf_agents.networks import utils\n",
        "from tf_agents.policies import actor_policy\n",
        "from tf_agents.policies import ou_noise_policy\n",
        "from tf_agents.policies import tf_policy\n",
        "from tf_agents.replay_buffers import tf_uniform_replay_buffer\n",
        "from tf_agents.trajectories import time_step\n",
        "from tf_agents.trajectories import trajectory\n",
        "from tf_agents.utils import common\n",
        "\n",
        "tf.compat.v1.enable_v2_behavior()\n",
        "# tf.enable_eager_execution()\n",
        "# tf.logging.set_verbosity(tf.logging.INFO)"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "cellView": "both",
        "colab_type": "code",
        "id": "esd_jQgISaff",
        "outputId": "096cfc75-3960-477b-feea-612848de347f",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 465
        }
      },
      "source": [
        "#@title Implement the 2D navigation environment and helper functions.\n",
        "WALLS = {\n",
        "    'Small':\n",
        "        np.array([[0, 0, 0, 0],\n",
        "                  [0, 0, 0, 0],\n",
        "                  [0, 0, 0, 0],\n",
        "                  [0, 0, 0, 0]]),\n",
        "    'Cross':\n",
        "        np.array([[0, 0, 0, 0, 0, 0, 0],\n",
        "                  [0, 0, 0, 1, 0, 0, 0],\n",
        "                  [0, 0, 0, 1, 0, 0, 0],\n",
        "                  [0, 1, 1, 1, 1, 1, 0],\n",
        "                  [0, 0, 0, 1, 0, 0, 0],\n",
        "                  [0, 0, 0, 1, 0, 0, 0],\n",
        "                  [0, 0, 0, 0, 0, 0, 0]]),\n",
        "    'FourRooms':\n",
        "        np.array([[0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0],\n",
        "                  [0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0],\n",
        "                  [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0],\n",
        "                  [0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0],\n",
        "                  [0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0],\n",
        "                  [1, 0, 1, 1, 1, 1, 0, 0, 0, 0, 0],\n",
        "                  [0, 0, 0, 0, 0, 1, 1, 1, 0, 1, 1],\n",
        "                  [0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0],\n",
        "                  [0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0],\n",
        "                  [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0],\n",
        "                  [0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0]]),\n",
        "    'Spiral5x5':\n",
        "        np.array([[0, 0, 0, 0, 0],\n",
        "                  [0, 1, 1, 1, 1],\n",
        "                  [0, 1, 0, 0, 1],\n",
        "                  [0, 1, 1, 0, 1],\n",
        "                  [0, 0, 0, 0, 1]]),\n",
        "    'Spiral7x7':\n",
        "        np.array([[1, 1, 1, 1, 1, 1, 1],\n",
        "                  [1, 0, 0, 0, 0, 0, 0],\n",
        "                  [1, 0, 1, 1, 1, 1, 0],\n",
        "                  [1, 0, 1, 0, 0, 1, 0],\n",
        "                  [1, 0, 1, 1, 0, 1, 0],\n",
        "                  [1, 0, 0, 0, 0, 1, 0],\n",
        "                  [1, 1, 1, 1, 1, 1, 0]]),\n",
        "    'Spiral9x9':\n",
        "        np.array([[0, 0, 0, 0, 0, 0, 0, 0, 0],\n",
        "                  [0, 1, 1, 1, 1, 1, 1, 1, 1],\n",
        "                  [0, 1, 0, 0, 0, 0, 0, 0, 1],\n",
        "                  [0, 1, 0, 1, 1, 1, 1, 0, 1],\n",
        "                  [0, 1, 0, 1, 0, 0, 1, 0, 1],\n",
        "                  [0, 1, 0, 1, 1, 0, 1, 0, 1],\n",
        "                  [0, 1, 0, 0, 0, 0, 1, 0, 1],\n",
        "                  [0, 1, 1, 1, 1, 1, 1, 0, 1],\n",
        "                  [0, 0, 0, 0, 0, 0, 0, 0, 1]]),\n",
        "    'Spiral11x11':\n",
        "        np.array([[1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1],\n",
        "                  [1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0],\n",
        "                  [1, 0, 1, 1, 1, 1, 1, 1, 1, 1, 0],\n",
        "                  [1, 0, 1, 0, 0, 0, 0, 0, 0, 1, 0],\n",
        "                  [1, 0, 1, 0, 1, 1, 1, 1, 0, 1, 0],\n",
        "                  [1, 0, 1, 0, 1, 0, 0, 1, 0, 1, 0],\n",
        "                  [1, 0, 1, 0, 1, 1, 0, 1, 0, 1, 0],\n",
        "                  [1, 0, 1, 0, 0, 0, 0, 1, 0, 1, 0],\n",
        "                  [1, 0, 1, 1, 1, 1, 1, 1, 0, 1, 0],\n",
        "                  [1, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0],\n",
        "                  [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0]]),\n",
        "    'Maze3x3':\n",
        "        np.array([[0, 0, 0],\n",
        "                  [1, 1, 0],\n",
        "                  [0, 0, 0]]),\n",
        "    'Maze6x6':\n",
        "        np.array([[0, 0, 1, 0, 0, 0],\n",
        "                  [1, 0, 1, 0, 1, 0],\n",
        "                  [0, 0, 1, 0, 1, 1],\n",
        "                  [0, 1, 1, 0, 0, 1],\n",
        "                  [0, 0, 1, 1, 0, 1],\n",
        "                  [1, 0, 0, 0, 0, 1]]),\n",
        "    'Maze11x11':\n",
        "        np.array([[0, 1, 0, 0, 0, 1, 0, 0, 0, 1, 0],\n",
        "                  [0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0],\n",
        "                  [0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0],\n",
        "                  [0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0],\n",
        "                  [0, 0, 0, 1, 0, 0, 0, 1, 0, 1, 0],\n",
        "                  [1, 1, 1, 1, 1, 1, 1, 1, 0, 1, 0],\n",
        "                  [1, 1, 1, 0, 0, 0, 1, 0, 0, 1, 0],\n",
        "                  [1, 1, 1, 0, 1, 0, 1, 0, 1, 1, 0],\n",
        "                  [0, 0, 0, 0, 1, 0, 0, 0, 1, 1, 0],\n",
        "                  [0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0],\n",
        "                  [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]]),\n",
        "    'Tunnel':\n",
        "        np.array([[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 0, 0, 0, 0, 0, 0, 0],\n",
        "                  [0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 1, 1, 0, 1, 1, 1, 1, 1, 0],\n",
        "                  [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 1, 0, 1, 1, 0, 0, 1, 0],\n",
        "                  [0, 1, 1, 1, 1, 1, 1, 1, 0, 0, 1, 0, 0, 1, 0, 0, 0, 0, 0, 1, 0],\n",
        "                  [0, 1, 1, 0, 0, 0, 0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0],\n",
        "                  [0, 1, 1, 0, 1, 1, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0],\n",
        "                  [0, 1, 1, 0, 1, 1, 0, 1, 0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 1, 0],\n",
        "                  [0, 1, 1, 0, 1, 1, 0, 1, 0, 1, 0, 0, 0, 0, 0, 0, 0, 1, 0, 1, 0],\n",
        "                  [0, 1, 1, 0, 0, 1, 1, 1, 0, 1, 0, 1, 1, 1, 1, 1, 1, 1, 0, 1, 0],\n",
        "                  [0, 1, 1, 0, 0, 1, 1, 1, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0],\n",
        "                  [0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 1, 0, 1, 1, 1, 1, 0, 1, 0],\n",
        "                  [0, 1, 1, 1, 1, 1, 1, 1, 0, 1, 0, 0, 1, 0, 1, 0, 0, 1, 0, 1, 0],\n",
        "                  [0, 1, 0, 1, 1, 0, 0, 1, 0, 1, 0, 0, 0, 0, 0, 0, 0, 1, 0, 1, 0],\n",
        "                  [0, 1, 0, 1, 1, 0, 0, 1, 0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0],\n",
        "                  [0, 1, 0, 1, 1, 0, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0],\n",
        "                  [0, 1, 0, 1, 1, 0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0],\n",
        "                  [0, 1, 0, 1, 1, 0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0],\n",
        "                  [0, 1, 0, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0],\n",
        "                  [0, 1, 0, 1, 1, 1, 1, 1, 0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0],\n",
        "                  [0, 1, 0, 0, 0, 0, 1, 1, 0, 1, 0, 0, 0, 0, 0, 1, 0, 0, 1, 1, 0],\n",
        "                  [0, 1, 1, 1, 1, 0, 1, 1, 0, 1, 1, 1, 1, 1, 0, 1, 0, 0, 1, 1, 0],\n",
        "                  [0, 0, 0, 0, 0, 0, 1, 1, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0]]),\n",
        "    'U':\n",
        "        np.array([[0, 0, 0],\n",
        "                  [0, 1, 0],\n",
        "                  [0, 1, 0],\n",
        "                  [0, 1, 0],\n",
        "                  [1, 1, 0],\n",
        "                  [0, 1, 0],\n",
        "                  [0, 1, 0],\n",
        "                  [0, 1, 0],\n",
        "                  [0, 0, 0]]),\n",
        "    'Tree':\n",
        "        np.array([\n",
        "            [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1],\n",
        "            [1, 1, 1, 1, 0, 0, 0, 0, 1, 1, 0, 1, 1, 0, 0, 0, 0, 1, 1, 1, 1],\n",
        "            [1, 1, 1, 1, 0, 1, 1, 0, 1, 1, 0, 1, 1, 0, 1, 1, 0, 1, 1, 1, 1],\n",
        "            [1, 1, 1, 1, 0, 1, 1, 0, 0, 0, 0, 0, 0, 0, 1, 1, 0, 1, 1, 1, 1],\n",
        "            [1, 1, 1, 1, 0, 1, 1, 1, 1, 1, 0, 1, 1, 1, 1, 1, 0, 1, 1, 1, 1],\n",
        "            [0, 0, 0, 1, 0, 1, 0, 0, 0, 1, 0, 1, 0, 0, 0, 1, 0, 1, 0, 0, 0],\n",
        "            [0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0],\n",
        "            [0, 1, 0, 0, 0, 0, 0, 1, 0, 1, 0, 1, 0, 1, 0, 0, 0, 0, 0, 1, 0],\n",
        "            [0, 1, 1, 1, 0, 1, 1, 1, 0, 1, 1, 1, 0, 1, 1, 1, 0, 1, 1, 1, 0],\n",
        "            [0, 1, 1, 1, 0, 1, 1, 1, 0, 1, 1, 1, 0, 1, 1, 1, 0, 1, 1, 1, 0],\n",
        "            [0, 1, 1, 1, 1, 1, 1, 1, 0, 1, 1, 1, 0, 1, 1, 1, 1, 1, 1, 1, 0],\n",
        "            [0, 1, 1, 1, 1, 1, 1, 1, 0, 1, 1, 1, 0, 1, 1, 1, 1, 1, 1, 1, 0],\n",
        "            [0, 1, 1, 1, 1, 1, 1, 1, 0, 1, 1, 1, 0, 1, 1, 1, 1, 1, 1, 1, 0],\n",
        "        ]),\n",
        "    'UMulti':\n",
        "        np.array([\n",
        "            [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0],\n",
        "            [0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0],\n",
        "            [0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0],\n",
        "            [0, 1, 0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 1, 0],\n",
        "            [0, 1, 0, 1, 0, 0, 0, 0, 0, 0, 0, 1, 0, 1, 0],\n",
        "            [0, 1, 0, 1, 0, 1, 1, 1, 1, 1, 0, 1, 0, 1, 0],\n",
        "            [0, 1, 0, 1, 0, 1, 0, 0, 0, 1, 0, 1, 0, 1, 0],\n",
        "            [0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0],\n",
        "            [0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0],\n",
        "            [0, 0, 0, 0, 0, 0, 0, 1, 0, 1, 0, 1, 0, 1, 0],\n",
        "            [1, 1, 1, 1, 1, 1, 1, 1, 0, 1, 0, 1, 0, 1, 0],\n",
        "            [0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0],\n",
        "            [0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0],\n",
        "            [0, 1, 0, 1, 0, 1, 0, 0, 0, 1, 0, 1, 0, 1, 0],\n",
        "            [0, 1, 0, 1, 0, 1, 1, 1, 1, 1, 0, 1, 0, 1, 0],\n",
        "            [0, 1, 0, 1, 0, 0, 0, 0, 0, 0, 0, 1, 0, 1, 0],\n",
        "            [0, 1, 0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 1, 0],\n",
        "            [0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0],\n",
        "            [0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0],\n",
        "            [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0],\n",
        "         ]),\n",
        "    'FlyTrapSmall':\n",
        "        np.array([\n",
        "            [1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1],\n",
        "            [1, 1, 1, 1, 1, 1, 1, 0, 1, 1, 1, 1, 1, 1, 1],\n",
        "            [0, 1, 0, 0, 0, 0, 1, 0, 1, 0, 0, 0, 0, 1, 0],\n",
        "            [0, 1, 0, 0, 0, 0, 1, 0, 1, 0, 0, 0, 0, 1, 0],\n",
        "            [0, 1, 0, 0, 0, 0, 1, 0, 1, 0, 0, 0, 0, 1, 0],\n",
        "            [0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0],\n",
        "            [0, 1, 1, 1, 1, 0, 0, 0, 0, 0, 1, 1, 1, 1, 0],\n",
        "            [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0],\n",
        "            [0, 1, 1, 1, 1, 0, 0, 0, 0, 0, 1, 1, 1, 1, 0],\n",
        "            [0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0],\n",
        "            [0, 1, 0, 0, 0, 0, 1, 0, 1, 0, 0, 0, 0, 1, 0],\n",
        "            [0, 1, 0, 0, 0, 0, 1, 0, 1, 0, 0, 0, 0, 1, 0],\n",
        "            [0, 1, 0, 0, 0, 0, 1, 0, 1, 0, 0, 0, 0, 1, 0],\n",
        "            [1, 1, 1, 1, 1, 1, 1, 0, 1, 1, 1, 1, 1, 1, 1],\n",
        "            [1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1],\n",
        "         ]),\n",
        "    'FlyTrapBig':\n",
        "        np.array([\n",
        "            [1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1],\n",
        "            [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1],\n",
        "            [0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0],\n",
        "            [0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0],\n",
        "            [0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0],\n",
        "            [0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0],\n",
        "            [0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0],\n",
        "            [0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0],\n",
        "            [0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0],\n",
        "            [0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0],\n",
        "            [0, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 1, 1, 0],\n",
        "            [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0],\n",
        "            [0, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 1, 1, 0],\n",
        "            [0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0],\n",
        "            [0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0],\n",
        "            [0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0],\n",
        "            [0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0],\n",
        "            [0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0],\n",
        "            [0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0],\n",
        "            [0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0],\n",
        "            [0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0],\n",
        "            [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1],\n",
        "            [1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1],\n",
        "         ]),\n",
        "    'Galton':\n",
        "        np.array([\n",
        "            [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0],\n",
        "            [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0],\n",
        "            [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0],\n",
        "            [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0],\n",
        "            [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0],\n",
        "            [0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 1, 0, 1, 0, 1, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0],\n",
        "            [0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 1, 0, 1, 0, 1, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0],\n",
        "            [0, 0, 0, 0, 0, 0, 1, 1, 1, 1, 1, 1, 1, 0, 1, 0, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0],\n",
        "            [0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 1, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0],\n",
        "            [0, 1, 0, 0, 0, 0, 0, 1, 0, 1, 0, 1, 0, 0, 1, 0, 0, 1, 0, 1, 0, 1, 0, 0, 0, 0, 0, 1, 0],\n",
        "            [0, 1, 0, 0, 0, 0, 0, 1, 0, 1, 0, 1, 0, 0, 1, 0, 0, 1, 0, 1, 0, 1, 0, 0, 0, 0, 0, 1, 0],\n",
        "            [0, 1, 1, 1, 1, 1, 1, 1, 0, 1, 0, 1, 1, 0, 1, 0, 1, 1, 0, 1, 0, 1, 1, 1, 1, 1, 1, 1, 0],\n",
        "            [0, 0, 0, 0, 1, 0, 0, 0, 0, 1, 0, 0, 1, 0, 1, 0, 1, 0, 0, 1, 0, 0, 0, 0, 1, 0, 0, 0, 0],\n",
        "        ]),\n",
        "}\n",
        "\n",
        "def resize_walls(walls, factor):\n",
        "  \"\"\"Increase the environment by rescaling.\n",
        "\n",
        "  Args:\n",
        "    walls: 0/1 array indicating obstacle locations.\n",
        "    factor: (int) factor by which to rescale the environment.\"\"\"\n",
        "  (height, width) = walls.shape\n",
        "  row_indices = np.array([i for i in range(height) for _ in range(factor)])\n",
        "  col_indices = np.array([i for i in range(width) for _ in range(factor)])\n",
        "  walls = walls[row_indices]\n",
        "  walls = walls[:, col_indices]\n",
        "  assert walls.shape == (factor * height, factor * width)\n",
        "  return walls\n",
        "\n",
        "\n",
        "\n",
        "class PointEnv(gym.Env):\n",
        "  \"\"\"Abstract class for 2D navigation environments.\"\"\"\n",
        "\n",
        "  def __init__(self, walls=None, resize_factor=1,\n",
        "               action_noise=1.0):\n",
        "    \"\"\"Initialize the point environment.\n",
        "\n",
        "    Args:\n",
        "      walls: (str) name of one of the maps defined above.\n",
        "      resize_factor: (int) Scale the map by this factor.\n",
        "      action_noise: (float) Standard deviation of noise to add to actions. Use 0\n",
        "        to add no noise.\n",
        "    \"\"\"\n",
        "    if resize_factor > 1:\n",
        "      self._walls = resize_walls(WALLS[walls], resize_factor)\n",
        "    else:\n",
        "      self._walls = WALLS[walls]\n",
        "    self._apsp = self._compute_apsp(self._walls)\n",
        "    (height, width) = self._walls.shape\n",
        "    self._height = height\n",
        "    self._width = width\n",
        "    self._action_noise = action_noise\n",
        "    self.action_space = gym.spaces.Box(\n",
        "        low=np.array([-1.0, -1.0]),\n",
        "        high=np.array([1.0, 1.0]),\n",
        "        dtype=np.float32)\n",
        "    self.observation_space = gym.spaces.Box(\n",
        "        low=np.array([0.0, 0.0]),\n",
        "        high=np.array([self._height, self._width]),\n",
        "        dtype=np.float32)\n",
        "    self.reset()\n",
        "\n",
        "  def _sample_empty_state(self):\n",
        "    candidate_states = np.where(self._walls == 0)\n",
        "    num_candidate_states = len(candidate_states[0])\n",
        "    state_index = np.random.choice(num_candidate_states)\n",
        "    state = np.array([candidate_states[0][state_index],\n",
        "                      candidate_states[1][state_index]],\n",
        "                     dtype=np.float)\n",
        "    state += np.random.uniform(size=2)\n",
        "    assert not self._is_blocked(state)\n",
        "    return state\n",
        "\n",
        "  def reset(self):\n",
        "    self.state = self._sample_empty_state()\n",
        "    return self.state.copy()\n",
        "\n",
        "  def _get_distance(self, obs, goal):\n",
        "    \"\"\"Compute the shortest path distance.\n",
        "\n",
        "    Note: This distance is *not* used for training.\"\"\"\n",
        "    (i1, j1) = self._discretize_state(obs)\n",
        "    (i2, j2) = self._discretize_state(goal)\n",
        "    return self._apsp[i1, j1, i2, j2]\n",
        "\n",
        "  def _discretize_state(self, state, resolution=1.0):\n",
        "    (i, j) = np.floor(resolution * state).astype(np.int)\n",
        "    # Round down to the nearest cell if at the boundary.\n",
        "    if i == self._height:\n",
        "      i -= 1\n",
        "    if j == self._width:\n",
        "      j -= 1\n",
        "    return (i, j)\n",
        "\n",
        "  def _is_blocked(self, state):\n",
        "    if not self.observation_space.contains(state):\n",
        "      return True\n",
        "    (i, j) = self._discretize_state(state)\n",
        "    return (self._walls[i, j] == 1)\n",
        "\n",
        "  def step(self, action):\n",
        "    if self._action_noise > 0:\n",
        "      action += np.random.normal(0, self._action_noise)\n",
        "    action = np.clip(action, self.action_space.low, self.action_space.high)\n",
        "    assert self.action_space.contains(action)\n",
        "    num_substeps = 10\n",
        "    dt = 1.0 / num_substeps\n",
        "    num_axis = len(action)\n",
        "    for _ in np.linspace(0, 1, num_substeps):\n",
        "      for axis in range(num_axis):\n",
        "        new_state = self.state.copy()\n",
        "        new_state[axis] += dt * action[axis]\n",
        "        if not self._is_blocked(new_state):\n",
        "          self.state = new_state\n",
        "\n",
        "    done = False\n",
        "    rew = -1.0 * np.linalg.norm(self.state)\n",
        "    return self.state.copy(), rew, done, {}\n",
        "\n",
        "  @property\n",
        "  def walls(self):\n",
        "    return self._walls\n",
        "\n",
        "  def _compute_apsp(self, walls):\n",
        "    (height, width) = walls.shape\n",
        "    g = nx.Graph()\n",
        "    # Add all the nodes\n",
        "    for i in range(height):\n",
        "      for j in range(width):\n",
        "        if walls[i, j] == 0:\n",
        "          g.add_node((i, j))\n",
        "\n",
        "    # Add all the edges\n",
        "    for i in range(height):\n",
        "      for j in range(width):\n",
        "        for di in [-1, 0, 1]:\n",
        "          for dj in [-1, 0, 1]:\n",
        "            if di == dj == 0: continue  # Don't add self loops\n",
        "            if i + di < 0 or i + di > height - 1: continue  # No cell here\n",
        "            if j + dj < 0 or j + dj > width - 1: continue  # No cell here\n",
        "            if walls[i, j] == 1: continue  # Don't add edges to walls\n",
        "            if walls[i + di, j + dj] == 1: continue  # Don't add edges to walls\n",
        "            g.add_edge((i, j), (i + di, j + dj))\n",
        "\n",
        "    # dist[i, j, k, l] is path from (i, j) -> (k, l)\n",
        "    dist = np.full((height, width, height, width), np.float('inf'))\n",
        "    for ((i1, j1), dist_dict) in nx.shortest_path_length(g):\n",
        "      for ((i2, j2), d) in dist_dict.items():\n",
        "        dist[i1, j1, i2, j2] = d\n",
        "    return dist\n",
        "\n",
        "class GoalConditionedPointWrapper(gym.Wrapper):\n",
        "  \"\"\"Wrapper that appends goal to state produced by environment.\"\"\"\n",
        "\n",
        "\n",
        "  def __init__(self, env, prob_constraint=0.8, min_dist=0, max_dist=4,\n",
        "               threshold_distance=1.0):\n",
        "    \"\"\"Initialize the environment.\n",
        "\n",
        "    Args:\n",
        "      env: an environment.\n",
        "      prob_constraint: (float) Probability that the distance constraint is\n",
        "        followed after resetting.\n",
        "      min_dist: (float) When the constraint is enforced, ensure the goal is at\n",
        "        least this far from the initial state.\n",
        "      max_dist: (float) When the constraint is enforced, ensure the goal is at\n",
        "        most this far from the initial state.\n",
        "      threshold_distance: (float) States are considered equivalent if they are\n",
        "        at most this far away from one another.\n",
        "    \"\"\"\n",
        "    self._threshold_distance = threshold_distance\n",
        "    self._prob_constraint = prob_constraint\n",
        "    self._min_dist = min_dist\n",
        "    self._max_dist = max_dist\n",
        "    super(GoalConditionedPointWrapper, self).__init__(env)\n",
        "    self.observation_space = gym.spaces.Dict({\n",
        "        'observation': env.observation_space,\n",
        "        'goal': env.observation_space,\n",
        "    })\n",
        "\n",
        "  def _normalize_obs(self, obs):\n",
        "    return np.array([\n",
        "        obs[0] / float(self.env._height),\n",
        "        obs[1] / float(self.env._width)\n",
        "    ])\n",
        "\n",
        "  def reset(self):\n",
        "    goal = None\n",
        "    count = 0\n",
        "    while goal is None:\n",
        "      obs = self.env.reset()\n",
        "      (obs, goal) = self._sample_goal(obs)\n",
        "      count += 1\n",
        "      if count > 1000:\n",
        "        print('WARNING: Unable to find goal within constraints.')\n",
        "    self._goal = goal\n",
        "    return {'observation': self._normalize_obs(obs),\n",
        "            'goal': self._normalize_obs(self._goal)}\n",
        "\n",
        "  def step(self, action):\n",
        "    obs, _, _, _ = self.env.step(action)\n",
        "    rew = -1.0\n",
        "    done = self._is_done(obs, self._goal)\n",
        "    return {'observation': self._normalize_obs(obs),\n",
        "            'goal': self._normalize_obs(self._goal)}, rew, done, {}\n",
        "\n",
        "  def set_sample_goal_args(self, prob_constraint=None,\n",
        "                           min_dist=None, max_dist=None):\n",
        "    assert prob_constraint is not None\n",
        "    assert min_dist is not None\n",
        "    assert max_dist is not None\n",
        "    assert min_dist >= 0\n",
        "    assert max_dist >= min_dist\n",
        "    self._prob_constraint = prob_constraint\n",
        "    self._min_dist = min_dist\n",
        "    self._max_dist = max_dist\n",
        "\n",
        "  def _is_done(self, obs, goal):\n",
        "    \"\"\"Determines whether observation equals goal.\"\"\"\n",
        "    return np.linalg.norm(obs - goal) < self._threshold_distance\n",
        "\n",
        "  def _sample_goal(self, obs):\n",
        "    \"\"\"Sampled a goal state.\"\"\"\n",
        "    if np.random.random() < self._prob_constraint:\n",
        "      return self._sample_goal_constrained(obs, self._min_dist, self._max_dist)\n",
        "    else:\n",
        "      return self._sample_goal_unconstrained(obs)\n",
        "\n",
        "  def _sample_goal_constrained(self, obs, min_dist, max_dist):\n",
        "    \"\"\"Samples a goal with dist min_dist <= d(obs, goal) <= max_dist.\n",
        "\n",
        "    Args:\n",
        "      obs: observation (without goal).\n",
        "      min_dist: (int) minimum distance to goal.\n",
        "      max_dist: (int) maximum distance to goal.\n",
        "    Returns:\n",
        "      obs: observation (without goal).\n",
        "      goal: a goal state.\n",
        "    \"\"\"\n",
        "    (i, j) = self.env._discretize_state(obs)\n",
        "    mask = np.logical_and(self.env._apsp[i, j] >= min_dist,\n",
        "                          self.env._apsp[i, j] <= max_dist)\n",
        "    mask = np.logical_and(mask, self.env._walls == 0)\n",
        "    candidate_states = np.where(mask)\n",
        "    num_candidate_states = len(candidate_states[0])\n",
        "    if num_candidate_states == 0:\n",
        "      return (obs, None)\n",
        "    goal_index = np.random.choice(num_candidate_states)\n",
        "    goal = np.array([candidate_states[0][goal_index],\n",
        "                     candidate_states[1][goal_index]],\n",
        "                    dtype=np.float)\n",
        "    goal += np.random.uniform(size=2)\n",
        "    dist_to_goal = self.env._get_distance(obs, goal)\n",
        "    assert min_dist <= dist_to_goal <= max_dist\n",
        "    assert not self.env._is_blocked(goal)\n",
        "    return (obs, goal)\n",
        "\n",
        "  def _sample_goal_unconstrained(self, obs):\n",
        "    \"\"\"Samples a goal without any constraints.\n",
        "\n",
        "    Args:\n",
        "      obs: observation (without goal).\n",
        "    Returns:\n",
        "      obs: observation (without goal).\n",
        "      goal: a goal state.\n",
        "    \"\"\"\n",
        "    return (obs, self.env._sample_empty_state())\n",
        "\n",
        "  @property\n",
        "  def max_goal_dist(self):\n",
        "    apsp = self.env._apsp\n",
        "    return np.max(apsp[np.isfinite(apsp)])\n",
        "\n",
        "\n",
        "class NonTerminatingTimeLimit(wrappers.PyEnvironmentBaseWrapper):\n",
        "  \"\"\"Resets the environment without setting done = True.\n",
        "\n",
        "  Resets the environment if either these conditions holds:\n",
        "    1. The base environment returns done = True\n",
        "    2. The time limit is exceeded.\n",
        "  \"\"\"\n",
        "\n",
        "  def __init__(self, env, duration):\n",
        "    super(NonTerminatingTimeLimit, self).__init__(env)\n",
        "    self._duration = duration\n",
        "    self._step_count = None\n",
        "\n",
        "  def _reset(self):\n",
        "    self._step_count = 0\n",
        "    return self._env.reset()\n",
        "\n",
        "  @property\n",
        "  def duration(self):\n",
        "    return self._duration\n",
        "\n",
        "  def _step(self, action):\n",
        "    if self._step_count is None:\n",
        "      return self.reset()\n",
        "\n",
        "    ts = self._env.step(action)\n",
        "\n",
        "    self._step_count += 1\n",
        "    if self._step_count >= self._duration or ts.is_last():\n",
        "      self._step_count = None\n",
        "\n",
        "    return ts\n",
        "\n",
        "def env_load_fn(environment_name,\n",
        "         max_episode_steps=None,\n",
        "         resize_factor=1,\n",
        "         gym_env_wrappers=(GoalConditionedPointWrapper,),\n",
        "         terminate_on_timeout=False):\n",
        "  \"\"\"Loads the selected environment and wraps it with the specified wrappers.\n",
        "\n",
        "  Args:\n",
        "    environment_name: Name for the environment to load.\n",
        "    max_episode_steps: If None the max_episode_steps will be set to the default\n",
        "      step limit defined in the environment's spec. No limit is applied if set\n",
        "      to 0 or if there is no timestep_limit set in the environment's spec.\n",
        "    gym_env_wrappers: Iterable with references to wrapper classes to use\n",
        "      directly on the gym environment.\n",
        "    terminate_on_timeout: Whether to set done = True when the max episode\n",
        "      steps is reached.\n",
        "\n",
        "  Returns:\n",
        "    A PyEnvironmentBase instance.\n",
        "  \"\"\"\n",
        "  gym_env = PointEnv(walls=environment_name,\n",
        "                     resize_factor=resize_factor)\n",
        "\n",
        "  for wrapper in gym_env_wrappers:\n",
        "    gym_env = wrapper(gym_env)\n",
        "  env = gym_wrapper.GymWrapper(\n",
        "      gym_env,\n",
        "      discount=1.0,\n",
        "      auto_reset=True,\n",
        "  )\n",
        "\n",
        "  if max_episode_steps > 0:\n",
        "    if terminate_on_timeout:\n",
        "      env = wrappers.TimeLimit(env, max_episode_steps)\n",
        "    else:\n",
        "      env = NonTerminatingTimeLimit(env, max_episode_steps)\n",
        "\n",
        "  return tf_py_environment.TFPyEnvironment(env)\n",
        "\n",
        "def plot_walls(walls):\n",
        "  walls = walls.T\n",
        "  (height, width) = walls.shape\n",
        "  for (i, j) in zip(*np.where(walls)):\n",
        "    x = np.array([j, j+1]) / float(width)\n",
        "    y0 = np.array([i, i]) / float(height)\n",
        "    y1 = np.array([i+1, i+1]) / float(height)\n",
        "    plt.fill_between(x, y0, y1, color='grey')\n",
        "  plt.xlim([0, 1])\n",
        "  plt.ylim([0, 1])\n",
        "  plt.xticks([])\n",
        "  plt.yticks([])\n",
        "\n",
        "plt.figure(figsize=(12, 7))\n",
        "for index, (name, walls) in enumerate(WALLS.items()):\n",
        "  plt.subplot(3, 6, index + 1)\n",
        "  plt.title(name)\n",
        "  plot_walls(walls)\n",
        "plt.subplots_adjust(wspace=0.1, hspace=0.2)\n",
        "plt.suptitle('Navigation Environments', fontsize=20)\n",
        "plt.show()"
      ],
      "execution_count": 4,
      "outputs": [
        {
          "output_type": "display_data",
          "data": {
            "image/png": "iVBORw0KGgoAAAANSUhEUgAAAq8AAAHACAYAAACMK4GPAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4yLjEsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+j8jraAAAgAElEQVR4nO3deZxkVX3///d7gAbZZsBBW9ZRSNyjmChqMMFEo6h8IUbjAsqYYECDfv2pieI2PSqMIYmCqIH4NQ4CLgiIu+ISVDCo4ChqXBAYGMBWFmdYBhigP78/zmm9U1PVXd1dyz23Xs/Hox7dVXc79566pz733HPOdUQIAAAAKMGiYScAAAAA6BbBKwAAAIpB8AoAAIBiELwCAACgGASvAAAAKAbBKwAAAIpB8Ao0nO21ttcOOx2tbC+3HbaXDzstdWb7oHycJoadFgCoA4JXoAdycBG2r7G9XYd51uZ5th50+oahxKCrElDP9Fo77HRifmwvy3m4ethpATB/I/EjCgzQ3pJeI+ldw05IxV8OOwEdfErSJZJ+NeyEtPFDSed3mLZ+kAmR9F1JD5d004C3CwC1RPAK9M5vJYWkN9r+fxFRi2AjIq4cdhraiYgNkjYMOx0d/CAiJoadCEmKiI2SfjbsdABAXdBsAOidjZLeIWmxpBXdLpRvVZ9r+yrbd9q+1fbFto9oM+/PbG+yvbTDut6Qb4seW/msbZtX24ttn2T7Ott35XW/1vZD2t1atf2Htt9l+1LbN9q+OzeT+E/be7bMu1rSf+e3K1puux9U2e+2bV5t/3E+Jr+pbOcDth/UZt7VeT3LbB9t+0d5f36d07a43bHqlenja3sH2/9q+9qc5l/m/HBl3ifmtH5qhvX9NC+/a37ftvmF7Qvz52O232b753m51ZV5+nocK/u+o+332F6Xv8M/sH1Ynmdr22+2fUVe35XV72ebdT7D9hds35TTfGU+rksWeOwnJF2d3x7Z8p1cnuex7SNtfzt/x+/K+/Rl2y/olGYAg0XNK9Bb75d0rKSjbb83Iq7oYpn/kPQTSd9UuoV+f0nPknSG7YdGxFsr854u6QRJL5J0Spt1HSlpk6SPzrRBp3a5X5f0OElrJJ2lFHS/WdJTOiz2XEnHKAWl387beaSkoyQdYvtPIuL6PO/0LfcjJX1D0oWV9aydJW3PkXSuJEs6R9I1kv5Y0iskHWr7wIi4us2iJ0p6hqTPSrpA0lMlvVzSfpL+YqZt9sA2kr4saXdJX5R0r6TDlJqPbCdppSRFxCW2fy7pWbbvHxE3V1di+wmSHibp3Ii4pcttnyvp8Xm750v6TV7XoI7jNpK+ImlXSZ+WNKb0/TzX9l9JeqWkA3L67pb0fEmn2L4xIj7Rsv8rJE1IukXS5/K+/JGk1ysdsydFxK1ttj/rsVf6Di6R9H+1ZbOQH+S/x0s6TinIPVvpzsCDlI7v8yVtll4AQxIRvHjxWuBLqbnAdfn/5+X357XMszZ/vnXL5/u2Wd+YpK9JukfSHpXP95R0n6RL2yzz+Lz+c9tsd23LZ2/N835Mkiuf7yXpxjxtdcsye0jats12/yqn6T9aPj8or2eiwzFbnqcvr3y2o6Sb8/qe0jL/G/L8F7R8vjp/fq2kvSufb610QRCSntBlPk6n6QdKQVS71zM75OsXJN2v8vkDlNrHrpe0TeXz4/L8x7bZ/vvztENmO45KwVhIulzS0pZpAzmOlX3/bPW7oXQBFEpB6PckLalMe4jShc+alnU9NS/z7er8LfnyngUe+2Vq892uTL9Z0nWStm8zbWm7ZXjx4jX4F80GgB6LiHMk/Y+kv7Z9YBfzb9EmNSI2KQUyW6vS4SoirlMKav/Y9iNbFjsy/z29i2QeKWlK0nEREZX1r5N0Uod0Xh8Rd7f5/AKlmuNndLHd2RyqVIP3iYj4Vsu0f1cKVp5ue+82y749Iq6tpOteSR/Ob58wx3Q8RqnpR7vXMzss8+qIuLOy/d8o1UQulvTQynxnKB37I6sL2x6T9EKl2sYvziGtb40t21cP+ji+pvrdyNu8WtIukt4QEesr066SdLGkR9neqrKOV+e/L6/On5dZrXRBcXiH7Xd77Ltxj1LQv5k2xxjAkBC8Av3xuvz332ab0fbett/v1OZ043Q7PKVbvlKq8axanf8eWVnH9K3a3yjVQs20vZ0l7Svp+ohY22aWizosZ9tH2P5qbg94byWtj26Tzvl4XP779dYJOYj6Zn67f5tlL23z2br8d5c5puP0iHCH12vazL8hIn7ZzfYrFyB/YvsRlXkPUQo4z8r72q3vtvlskMdxfbsLMEk35L+XtZl2vdKF2XjlsycpBY7Ptz3R+lK6G7Gb7fu3rKvrY9+Fs5RqZ//X9irbz2zX1hfAcNHmFeiDiPgf2+dIep7tF0RL275pth+iFHzsIulbSm0MNyjV/CxTClC3bVnsU5JulXSE7eMi4j5Jz1EKfE7qIvDZOf/9dYfpnT5/t9IwYL9SamN4vaTp2q7lkvaZZbvdmA4UOg2fNf35Fp131H4Iq+ljsVWbab3UafisTttfLenpSvn7hvzZXGrOqybbfDbI49hpxIh7pd+NKtFpfdtUPru/0m/SbJ0dp5tETJvrsZ/J/yfpKkkvk/TG/LrX9hckva5DkAxgwAhegf45Tun27Sp37l3+WqUf7ZflW6O/Y/tFarm1LEkRcafts5U6Sj1d0pc0t8BnusPLAztM3+Jz2w9Quq37Y0lPjojb2qS1F6YDnfEO0x/UMl+pqhcgb1L6Dhws6YcR8cO5rKja7KOixOO4QdKiiNh1WAnIF4InSTopf+cPVGrK8XxJj7T9yHZNZwAMFs0GgD7JtTQfkPRgSa/qMNt++e+5bab9+QyrX53/Hml7N6XA5/KI+EHnRX6XrluVapf2sL2szSzt2uk+RKm8uKBN4Lpnnt5qut3gXGq+1uS/B7VOcHoy2fRICN+fwzprJ7fPPFuph/zTJL1YqTJhrrWunZR4HC+RtEubtty91PV3MiJ+ExHnRcTfKjW/2FfSo/qYNgBdIngF+uvtSrc136x0u7PV2vz3oOqHtp+hVLPaVkRcLOkKpZrdY5Ruv66eQ7o+onT+r2oZC3MvpaYBndJ5YLWTje0dJX1Q7e/iTN/abdcpqJPzlXqov8j2E1umvUbpQuCr1Q5FBVud/740v+5VanPZCyUex/fkvx+0vXvrxDyWa+u+zNX0g0S2+E7a3tb2n7b5fBulJjlSGssZwJDRbADoo4i4xfYJSmNntvMBpfZ1n8xtZG9Qqt15plLN3EwDo39E6aEIb9XcA58TlcbCfKGkh9q+QKmd5N8qdeY5TKlH/PR+TNr+eJ7/B5X5ny7pLqWe4I9t2cbPldrFvtD2PUrjjIakMyLimnaJiojbbf+dpE9K+obtTyoN3fTHSkNyTUo6eg77OV+PdctDAVrS2XFatyLiYtu/VLolvY2kz+Ze8gtWo+PYtYj4mu03Slol6YrczvRqpYu+fZTuRFykzqM9dLON221/R9JTbJ8l6RdKtbGfUTo+F+U8uUzp+7qd0nf84ZI+ExE/ne+2AfQOwSvQf+9VGqh9WeuEiLjc9lMlvVPSs5XOyR8qPRBgvWYPXlcqBT6fm0vgk9vNPlWpZvh5Sh1VrlZ6AMK3lILX1sHg/16pucELJP2j0niwn5H0NrVp9hAR99n+a6XB4p8vaSelAfMvUgoMOqXt07kG7E1Kw28tVgq2TpX0joi4odOyPfSY/OpkokfbOV3pAmT6/56pyXGck4j4F9sXK7WvPlDpzsIGpYug/9QsD9/o0kuUanmfqTRCh5XGdv2pUue5p0p6stI5cJukK5Ue7PBfPdg2gB5w+7b+AEaV7ZcrBQrHRMRpw04PAABVBK/AiLK9e2vtWx60/iKl3uj71LF2DgAw2mg2AIyuc3NnlMuUmigsUxovdnulJ28RuAIAaoeaV2BE2X6lUvu/P1BqD3m70hBL74uI84aZNgAAOiF4BQAAQDEY5xUAAADFIHgFAABAMQheAQAAUAyCVwAAABSD4BUAAADFIHgFAABAMQheAQAAUAyCVwAAABSD4BUAAADFIHgFAABAMQheAQAAUAyCVwAAABSD4BUAAADFIHgFAABAMQheAQAAUAyC11nYXmY7bG+d319o+6hhpwsA+sX2F20fuYDl19p+Wi/ThLkhD8tHHnZWdPBq+0Db37a9wfYtti+2/fhhpwtzZ/vFti+1fbvtX+WT9sBhp2sU5ALuznzsp1+793gbq21vyuu+xfZXbD+sl9vAluZbRkbEwRFxeo/SsNz2fS3fr4O6XPZC23dVlvt5L9JUkprk4akt+Xe37du6XPYnLcvea/uzvUhXKWqSh9vafo/tG2z/1vYHbG/T5bJ72P50Tvt1to/pRZoWotjg1fbOkj4n6RRJu0raQ9JKSXcPM12YO9uvlXSSpBMkPVDS3pI+IOnQNvNuPdjUjYxDImLHyuuGXq3Y9lb53xMjYkelc/V6SR/q1TawpX6VkfM8B/+n5ft14RyWPbay3EPnse1i1SUPI+KYav5J+pikT3a57CMry+0kaV23yzZBXfJQ0hsl/YmkR0n6Q0mPk/SWLpc9U9LVSr/Pz5Z0gu2nznH7PVVs8Kp08BURH4uI+yLizoi4ICIuz1f6F+erjPW2r7L95Pz5Otu/qVbF23627TW2b83TJ4a2VyPG9mJJb5f0jxFxXkTcERH3RMRnI+KfbE/YPsf2mbZvlbTc9u62P5OvAn9p++WV9T0h1+DeavvXtt+dP98ur+Pm/J34nu0HDmm3ay9fpZ+Ur9JvyP9vm6ctt31Ry/xhe7/8/2rb/2H7C7bvkLRZIRcRd0o6W9JjK8s/PNeyrc81Nf+nMm2x7Y/YvtH2NbbfYntRJS1zOdefZft/bd9m+3rbr+/D4auLbsrI9+XaoJ/Z/svpBV1pHtVyjG+WNGF7X9tfz+fTTbbPsr1krgnMeXWT7b3y+8fkWiFq5ZPa5aHtHST9jaTT8/t9c1n8uPx+93yuHtRm8T+TtFTSuQs8LiWpSx4eIum9EXFLRNwo6b2S/i6vu2Me2t5R0kGSjs+/zT+UdM70ssNScvD6C0n32T7d9sG2d2mZfoCkyyXdX9JHJX1c0uMl7SfpCEnvy5kiSXdIeqmkJUpXFa+wfdgA9gHSkyRtJ+lTM8xzqNLJskTSWUp5eZ2k3SU9T+kq8C/yvCdLOjkidpa0r1KQJElHSlosaS+l78Qxku7s6Z40y5slPVEpwHyMpCeo+6t0SXqxpOOValpaA90dJL1I0i/z+20kfVbSBZIeIOlVks6yPV3LdopS3j1E0p8rnasvq6xyLuf6hyQdHRE7KdVAfH0O+1SabsrIK5WCiRWSzrO9a4d1HSDpKqWal+MlWdIqpXPw4Urn1cQMadk//7j+wvZbnWuNIuLbkk6TdLrt+ynV8Lw1In5WWXZVXvbiDgFRk9UpD6f9jaQbJX1TkiLiSklvkHSm7e0lfVjS6R1q14+UdG5E3NHFdpqiTnnolv/3tL14ljx0h2UfNcN2+i8iin0pZdZqpUDmXkmfUcrU5ZKuqMz3aEkh6YGVz26W9NgO6z1J0nvy/8vyslvn9xdKOmrY+96Ul6TDJU3OMH1C0jcr7/eSdJ+knSqfrZK0Ov//TaVbMktb1vN3kr4t6Y+Gvc91e0laK+l2Sevz63ylwvRZlXmeIWlt/n+5pIta1hGS9sv/r5b0kZbpqyXdldc/pXQL6o/ytKdImpS0qDL/x3LebyVpk6RHVKYdLenCSlq6PtclXZuX33nYx31AeTtTGXmDJFfm/a6kl+T/f1fO5XmvnWU7h0la0/Kdelr+/yGSHqxUWfJoSf8r6bjKvNtIukzSjyR9qSVNByhdAG2rFPjcJmnfYR/XUcvDlvm+JmmizeefyXl4uaRt20zfXtKtkg4a9jEdxTyU9E5JF0vaTdK4pO8olZUPmi0PlSogTlGqaHqcpFsk/XyYx7TkmldFxE8jYnlE7Kl0FbC7UuApSb+uzHpnnr/1sx0lyfYBtv87V5NvUKqVW9r3HYCUAoulnrn9zrrK/7tLuiUiqp0FrlFqRyRJf690m+ZnTk0DnpM/P0PSlyV93Ok2+InusrH6iDgsIpbk12FKx/mayvRr8mfdWtfms3+LiCVKF4R3SpquWd1d0rqImGrZ3h5K5+E2bdKyR+V91+e6Uq3RsyRdY/sbtp80h30qzixl5PWRf5mymfJ4s/y0/UDbH3dqenGrUo1p2zIzIq6KiKsjYioifqTUTOh5len3KP2wP0rSv1fTFBHfiYjbIuLuSB1XLlbKv5FRhzysLLO30i3kj7SZ/MGcvlMiol17zucqBT3fmGkbTVSTPDxe0hpJP1CqyDlf0j3avPzslIeHK12ArpP0H3k7182wy31XdPBaFek202rNryr7o0pXHHtFxGJJp2rzKnL0z/8oNVyfqZlG9cS+QdKutneqfLa3UgcgRcQVEfEipdvP/yLpHNs7RGqrszIiHiHpyZKeo3T7Ge3dIGmfyvu982dSamaz/fQE2+Ntlo82n6UJEddK+r+STs63im+QtJdzO9bK9q6XdJNSAdualuu73pPNt/29iDhU6ftxvn7frKTx2pSRe9iulnPVPN5i8Zb3J+TPHh2pic4R6r7MjOq8tvdQul36YUn/7ty2uptlR00N8vAlki6OiKuqH+ZmOScpNcuZ6HDb+0ilOzIdy4ZRMKw8jNTW9tiI2CMiHqJUcXTZdKXBTHkYEddExHMiYreIOEApQP5u1zvdB8UGr7YfZvt1tvfM7/dSakd3yTxWt5NSbd5dtp+g1F4PAxARGyS9TdL7bR9me3vb2+S2QSe2mX+d0lXjKqdOWH+kVNt6piTZPsL2bvmEXJ8Xm7L9VNuPdur5fqtSQDTVun78zsckvcX2braXKuXRmXnaDyU90vZjbW+n7trJbSYivqJUQP+D0u2rjZL+Oef9QUqdCz4eEfcpBZjH297J9j6SXltJS9dsj9k+PLfxukfpe9DY70AXZeQDJL06H/PnK93a/EKXq99JqanJhhx8/tMM6TjYuXOkU0est0r6dH5vpR/yDymdx7+S9I48bYntZ+TzfGvbhyt1+PlSt8egdHXJw4qXKuVXq5MlXRoRR0n6vFIFUHU/9lTquNmTYZ9KUpc8dBruancnT1Q6D1dUZumYh04danfKZegRkv5K0ru7TGNfFBu8KrV9OkDSd5x6NF8i6ceSXjePdb1S0tudxq17m0aoNqYOIuLflQKStyh1BFgn6VilmrF2XqR06/kGpY5eKyLiq3naMyX9xPbtSifjCyP1bh9X6vR1q6SfKt26OqMf+9MQ75R0qVLbpx9J+n7+TBHxC6Vbv1+VdIVaOmTNwb9K+melmoJDJB2sVNP6AUkvjd932nmVUm3vVXlbH5X0X/Pc5kskrc232I5Ruh3WVLOVkd+R9AdKx/x4Sc+LiJu7XPdKpbZvG5R+6M6bYd6/lHR5TsMX8rwn5GmvVvrxfmuukXuZpJfZfopSc5F3KpUJNyl9Dw7L379RUZc8VG5is6dahrmyfahSufuK/NFrJT0uX2xMe4nScGlXdpm2JqlLHu6rVPFzh9JFxBsj4gKpqzx8hlL5+1ulcvOZkUYsGBqPeA0+AIwc28uVOoLwIJBCkYflIw/nr+SaVwAAAIwYglcAAAAUg2YDAAAAKAY1rwAAACjGTAPDb2Hp0qWxbNmyPiUFC3HZZZfdFBG7zTYfeVhf5OHvTU5Oampq5lGsFi1apPHxdkPMDk+3eSj1Ph+7OWbozvr162Pjxo2zVu7ssMMOsXjx4nlto47f3ybpV3nKeTY4M52Hcwpely1bpksvvbQ3qUJP2b5m9rnIwzojD39v5cqVXc23YsWK2WcaoG7zUOp9PnZ7zDC70047rasHISxevFhHH330vLdTt+9vk/SrPOU8G5yZzkOaDQAAAKAYBK8AAAAoBsErAAAAikHwCgAAgGIQvAIAAKAYBK8AAAAoBsErAAAAikHwCgAAgGIQvAIAAKAYBK8AAAAoBsErAAAAaiUipjpN23qQCQEAABgVK1asGHYSijUxMbGm0zRqXgEAAFAMglcAAAAUg2YDqLVVq1Zp06ZN81p2bGxMxx13XI9TBADDs5AycZSMj4/vP+w0oH+oeUWtLaSQpoAH0DSUa92xTXzTYGQuAAAAikHwCgAAgGIQvAKonbGxsZ7MAwBoHjpsAagdOtoBADqh5hUAgIqZnuwDYPioeQUAdDSKTwia6ck+AIaPmlcAAAAUg5pXAACAIRuFB1D06uFB1LwCAAAMWdMDV6l3+0jwCgAAgGIQvAIAAKAYBK8AAAAoBh22ANRONx0XetXwHwBQFmpeAdRON436R6FzAwBgSwSvAAAAKAbNBgAAGBElPTFt5cqVw04CaoqaVwAAABSD4BUAAADFIHgFAABAMQheAQAAUAw6bAEAABSmDp3vhtWpjppXAAAAFIPgFQAAAMUgeAUAAEAxCF4BAABQDIJXAAAAFIPgFQAAAMVgqCwAwECsWrVKmzZt6uk6x8bGdNxxx/V0nQDqjZpXAMBA9Dpw7dc6AdQbwSsAAACKQbMBdK0ft/z6bdBP/+AWJoAmWkj5T7mIXqPmFV0rLXAdBo4RgCZaSNlGuYheo+YVAACgweZSc15CTTnBK1CYycnJnjWH6KaQqnNzkbodh/Hx8f17kiAA6KG5lOF1Le+raDYAFGZqaqpn6+qmkCqhIFuoXh0H25SpANBnFLQAAAAoBsErAAAAikHwCgAAgGIQvAIAAKAYBK8AAAAoBsErAAAAikHwCgAAgGIQvAIAAKAYBK9AYRYt6t1pOzY21pN5Ster4xARvXuCBACgLR4PCxRmfHxcK1asGNj2evmM67k8znWQ+9iNbo7DxMTEmgEkBQB69njsElHzCgAAgGJQ84qujY2NjcRz7hdiFG6xAyjXKNfWoTkIXtG1Xt4+7tZCC9q63XoGAAALQ7MBAAAAFIOaVwBAR9xmBlA31LwCAACgGASvAAAAKAbBKwAAAIpB8AoAAIBiELwCAACgGASvAAAAKAbBKwAAwJDV5QmNdUnHTBjnFQAAYMiG8RTLUlHzCgAAgGIQvAIAAKAYBK8AAAAoBsErAAAAikHwCgBAIUroCd6qxDSj3hhtAACAQtAjHaDmFQAAAAUheAUAAEAxCF4BAABQDIJXAAAAFIPgFQAAAMVgtAEAAIA+WLly5bCT0EjUvAIAAKAYBK8A0ACjOhD8qO43MMpoNgAADcDg9QBGBTWvAAam21oyatMAAJ1Q8wpgYKgdBAAsFDWvAAAAKAbBKwAAAIpB8AoAAIBi0OYVtTY2NqZNmzbNe1kAqCsGsAfmh+AVtUYHHwAAUEWzAQAAABSD4BUAAKALNEdbmF4dP5oNAAAAdIGmbPVAzSsAAACKQc0rgIFZtWpVV6NHjI2NUcMBAGiLmlcAA9PtsGfzHR4NANB8BK8AAAAoBsErAAAAikHwCgAAgGIQvAIAAKAYBK8AAAAoBsErAAAAikHwCgAAgGIQvAIAAKAYjojuZ7ZvlHRN/5KDBdgnInabbSbysNYan4fj4+P72571ojkipiYnJ9cMIk091lUeSmXn4wjoKh+33377qSVLlngQCcLcrF+/PjZu3DhrWcN5WGsdz8M5Ba8AAADAMNFsAAAAAMUgeAUAAEAxCF4BAABQDIJXAAAAFIPgFQAAAMUoJni1/UXbRy5g+bW2n9bLNAEAgMT2Mtthe+thpwXNNvDg1faBtr9te4PtW2xfbPvxsy0XEQdHxOk9SsOjbH/Z9k22txgrzPaxti+1fbft1XNc93/a/rntKdvL57LdJskXC5tsL235fE0u3Jb1absvzMd/g+3f2D7d9s5dLnui7XW2b7V9je039SONpRhWHuZtPMT252zfls+XE+ew7NNsf9/2Hbavs/23/UpnHQzxXKMcHTLbt1deU7bvrLw/fNjpQ//lc3y/ls8mbJ85rDQNwkCD1xxEfE7SKZJ2lbSHpJWS7l7geud6lXePpLMl/X2H6TdIeqek/5pHcn4o6ZWSvj+P7TbN1ZJeNP3G9qMlbd/nbV4s6U8jYrGkh0jaWikvu/EhSQ+LiJ0lPVnS4baf259kFmPgeWh7TNJXJH1d0rikPSV1VRDbfoSkj0p6s6TFkh4j6bL+pLRWhnGuUY4OWUTsOP2SdK2kQyqfnTXs9AH9Muia1z+UpIj4WETcFxF3RsQFEXG57eW5FvZ9udbsZ7b/cnpB2xfaPir/Pz3ve2zfLGnC9r62v2775nxFfpbtJe0SERE/j4gPSfpJh+nnRcT5km5unWb7Dba/Mx0w236F7Z/Y3i4v+/6I+Jqku+a63QY6Q9JLK++PlPSR6Te2n51rh27NNZ4TlWnva6lVuHd6uu3dbZ9r+0bbV9t+9fRyEbEuIm6qbPM+Sfvl5fZ1qu1/XGU9N9o+KC/784i4o7Ls1PSyI2zgeShpuaQbIuLdEXFHRNwVEZfn5WbMQ0lvkXRaRHwxIu6NiJsj4sqeH5X6Gca5RjlaU601b265nZ9/T9+Rf0dvs32Bc819Zd4jbV+bf0/fXFnXIttvtH1l/r092/aug99LjLJBB6+/kHSf063cg23v0jL9AElXSloqaYWk82Y4KQ6QdJWkB0o6XpIlrZK0u6SHS9pL0kTP90D6V6Wa4rfY/gNJJ0g6IiK2KGShSyTtbPvhtreS9EJtXoN2h9IP7hJJz5b0CtuHSVJEHFupUThQ0m8lfdrp0aKfVaqZ2UPSX0p6je1nTK/UqWnKBkm3SfobSSfldV4p6Q2SzrS9vaQPSzo9Ii6sLPtG27dLuk7SDkq1eKNsGHn4RElrndq535R/aB+d1zlbHj5Rkmz/yPavbJ85Ij+sQznXFoBydPheLOllkh4gaUzS61umHyjpoUr5/jbbD8+fv0rSYZL+XOn39reS3j+IBAPTBhq8RsStSidESPqgpBttf8b2A/Msv5F0UkTcExGfkPRzpYK2nRsi4pRcu3JnRPwyIr4SEXdHxI2S3q10cvV6H6aUfgReLekzkk6MiBKfwT4o0zVCT5f0U0nXT0+IiAsj4kcRMZVr1j6mljyzvZuk8yW9KuWEl5gAAB5nSURBVB/nx0vaLSLeHhGbIuIqpe/SCyvrvSg3G9hT6UdybWXaByX9UtJ3JD1I6fayKtPfJWknSY/Lad/Qi4NQuEHn4Z75//cq/Th+XimYGsvbnCkP95T0EqWLlj+QdD+lZkqjYODn2nxRjtbChyPiFxFxp1IzjMe2TF+Zf1t/qHQB85j8+TGS3hwR10XE3UqVRM8znbQwQAP/skXET5VuC8r2w5RqB06S9GVJ10dEtQH+NUo/Xu2sq77JAfDJkp6iFHwsUroi7LmIWGv7vyU9S1xxzuYMSd+U9GBVbmNKku0DJL1L0qOUrvy3lfTJyvRtJJ0j6aMR8fH88T6Sdre9vrKqrSR9q3XDEXG97S9J+rhSMDrtg0o/mP+QC9/W5ULSmlzDtFLSa+eyww006Dy8U9JFEfHFvI5/U2oO8HClH1Gpcx7eqfyjnJc9QdJX573nZRnauTYflKNDN1n5f6OkHbucvo+kT9meqky/T+kuKAbvPknbtHy2jVLb8MYa6lBZEfEzSauVClRJ2sO2K7PsrdTov+3iLe9PyJ89One4OUKpKUHP2X62pCdJ+ppSzR46iIhrlDqTPEvSeS2TP6oUgOyVa0pP1eZ5doqkW5UCl2nrJF0dEUsqr50i4lkdkrC1pH2n39jeUeli6UNKbaVnuqW82bKjagh5eLm2PL9/Z5Y8bF12ZHqj1+BcmxPK0b66Q5t32Bvv4brXSTq45XuxXURcP+uS6IdrJS1r+ezBSpV/jTXo0QYeZvt1tvfM7/dS6iF7SZ7lAZJebXsb289Xqmn5Qper30nS7ZI22N5D0j/NkA7njgFj+f12tretTN86T99K0lZ5+nRD96WS/p+ko5Q6RRxi+1mVZcfyspa0TV52UTfbbbC/l/QXsXlnKCnl2S0RcZftJyi1wZIk2T5a6bbm4fkW47TvSrrNqcPH/Wxv5TR0zuPzcofb3jv/v49Se+ivVZY/WdKlEXGU0u3oU/O8i2wfbXuXnE9PkPSPLcuOsoHlodLdmCc6DXm1laTXSLpJ6Va41CEPsw9LepnTUFvbS3qj0ggno2KQ5xrlaH39QNKf2d7b9mJJx/Vw3adKOj6Xr7K9m+1De7h+zM0nlNqO75l/x54m6RClOynNFREDeyk1+j9bqS3WHfnvaZJ2VmpKcLGk9ym1M/yFpL+qLHuhpKPy/8uVbitW1/1IpSFxblc6cV8n6brK9LWSnpb/X6ZUI1N9ra3MO9Fm+kSedp6kUyvzHqxUO3z/Sjpblz2om+026VU93i2fb533e5mk5yldHd6mFGC8T9KZleN4d87P6deb8rTdldrsTSo1DbmkkrfHK3W2uiP//c9K3hyav3O75vc7KrWdPFzpQu5Lkm7J2/qFpDdJ8rCP5ajlYZ7+3Jw3t+b1PHK2PKwsu1LSjfl1hqRdhn0sm5hPohyt1av1e6DUFGN9Pj9eno/T1pXje1Rl3uXKv6mV47t1Zfrv5lcqK1+r1CflNqVO1id0WpZX3/P9fvp9344NSsPL/Z9hp6vfL+edHzqngaiPiogDh50WAAAA1FMxj4cFAAAACF4BAABQjNo0GwAAAABmQ80rAAAAijGnhxTssMMOsXjx4hnnWbRokcbHezmkHLpx2WWX3RQRu80239KlS2PZsmUDSBHmqsQ8nJyc1NTU1Owzjoj169fHxo0bu6oUqFM+YnMlnovYHHlYvpnycE7B6+LFi3X00UfPOt+KFSvmslr0gO2uBiRetmyZLr300n4nB/NQYh6uXLly2EmoldNOO63rB6PUKR+xuRLPRWyOPCzfTHlIswEAAAAUg+AVAAAAxSB4BQAAQDHm1OYVAAB0tmrVKm3atGnYyRiosbExHXfccb973+0xmO9ys60HzUfNKwAAPTJqgau05T53ewzmu9xs60HzEbwCAACgGASvAAAAKAbBKwAAAIoxlA5bo9igvd/Gx8f3H3YaAHRvcnKShzwMGB17gGYYSs0rgWvv2aYWHSgIj9UdPH57gGZgqCwAAPqom0emt6uFr+Oj1rlbgDqgtg4AAADFIHgFAABAMQheAQAAUAzavAIAUIBePj61SaP+jI2NDTsJGDCCVwAACtDLx6eWFLjWseMahotmAwAAACgGNa/AiGrSbcO6iAgGbwWAPiN4BUZUnQLXptwWnJiYWNPtvIsWceMLAOaj9sFrU37U5osBoYFmGh8fp3wbkfKtdT95TC2wMLUPXgEAaJJe3/WoXgQt5IJgvk8CAwaN+1YAAAAoBsErAAAAikHwCgAAgGIQvAIAAKAYdNgCAAC1NZ9OYuPj4/v3ISmoCWpeAQBAo9gmvmkwMhcAAADFIHgFAABAMYpv81ry89l5ygqAmTS9fBsbGyt2/wAMT/HBa8kFX8lpB9B/JZcR3aS9lxfvdXnyUzcB+djY2IBSMzdNeoxtREwNOw3on+KDVwAA6qLUYK+dulw8dfPY2lYTExNr+pAU1ARtXgEAAFAMglcAAAAUg+AVAAAAxaDNK4aiTr2oS+6UsBB16endi84rvfg+jer3APM3OTlZm45ic1Vyx7LW832Q526dfrtK1Ku8InjFUNTp5K9TWgapSYFaL/JwVL8HmL+pqXI7tJd8/reeq4M8dyknFqZXx49mAwAAACgGwSsAAACKQbMBAKipurRLnq9etQelPTKAqpEKXucz0PF8ldqIH0B91DFgG0bZVnIAD6D3Rip4BQAA3fWap8YbdUXwCgBAH3Vz12/QNdrd1GZT4426osMWAAAAikHwCgAAgGIQvAIAAKAYBK8AAMzDokWz/4TW9RGr3Sg57Wg2OmwBADAP4+PjAx2Csd+atC9oNmpeAQAAUAxqXisY9w7AoExOTtbuYSZ1Ld+4fQ2giuC1gnHvAAzK1NTUsJOwhV6Xb9yGBtAPBK8AAPRIN3fw+mlsbGyL7VNzjaYheAUAoEeGfXeujs0+gF6jwxYAAACKQc0rUJg6dvQBAGBQqHkFClPHjj4AAAwKwSsAAACKQbMBAAAK1qtmRHVtjlTXdGF4CF4BAOijbsa7JUADujdSwSuFA4C6WLSofq22Bj0e6LDHRAVQppEKXgGgLsbHx0f+CVQErgDmo36X/gAAAEAHBK8AABSgro957VW65rueuh4X9A/NBgAAKEDTH/3a9P1D71DzCgAAgGIQvAIAAKAYBK8AAAAoBm1eAQAYoG7Ht+3n2ORjY2ObtTGtw5i7vUzD+Pj4/j1ZEWqJmlcAAAZo2EFiuzTUMU0LYZv4psGKr3kdGxsb6EnHkBwABqUOtWH9NOjyG0AzFB+8MrQGgKZqemDXy/Kbx38Do4NqdQAAABSD4BUAAADFIHgFCrNoEadtP9CeHcBsKCcWplfHr/g2r8CoGR8f14oVK4adjJ7pVVvFJh0TjKZefId73fa3mzSNUnvjuvSzmc8xb1IZSRUOAAAAikHwCgAAgGIQvAIAAKAYtHkFAKBmSnpAxXwfNtHPzk8RMdW3lWPoah+8jlJDcACoKv0JVL0qv8fGxmrTUWZQSsr3fufNfDoaTUxMrOlDUlATtQ9eAWBU1TFgG0aFQkmBHID+o80rAAAAikHwCgAAgGLQbAAAgBHTbWepOvY7ae3MNoptokcdwSsAAA3SpCcptdPaBpo20aOHZgMAAAAoBsErAAAAikHwCgAAgGIQvAIAAKAYBK8AANTMfB+d2s9HrgJ1MZTRBkp/5GGvUMgAo2tycrJ2wxAx5FB9kA9AZ0MJXjkpgeFrHStxPnoR7PTiYrYXF4K9OB7j4+P7dzvv1NTUgrbVD1QqACgB47wCI6oXgUov1lGXi9le7IttmmIBQJ9R0AIAAKAY1LwCAFADg2wDXUr7ZvqGoB2CVwAARky3zWSa/qhZlIlmAwAAACgGwSsAAACKQfAKAACAYtDmFQAAtFW3B2kAUp+CV77sADCzRYvqd+Or5J7d3TzsouT9A/B71LwCwBCMj4/Tk7uHShj2CUBv1O/SHwAAAOiA4BUAAADFIHgFAABAMQheAQDokdZOYe06idWh41g36SxFyWnH/NBhCwCAHumm41gdO5fVMU1AJ9S8AgAAoBgErwAAACgGwSsAAACK4Yjoeubtt99+asmSJe5jejBP69evj40bN856MWL7RknXDCBJMxofH9/fdi0uniJianJycs2w0yFpn4jYbbaZepWHvciDGh27BevF8ej2PJTqcy7O1TDO3SF8zwZ6LqIvGp2Hcz0PCy2rO+bhnIJXAAAAYJhqUfMFAAAAdIPgFQAAAMUgeAUAAEAxCF4BAABQDIJXAAAAFIPgFY1me7nti2aY/hTbPx9kmurG9jLbYXvkHhfduu+2L7R91LDT1Usl5q/tU22/ddjpaLoSvxuA1KDg1fbtldeU7Tsr7w8fdvqwuVxg7tfy2YTtM/P/B+V5PtUyz2Py5xf2YrsR8a2IeOh81lUi22tbzo3bJe3e5bJfrCx3j+1Nlfen9jidB9r+tu0Ntm+xfbHtx/dyG01UUP5W0/lb25+3vdf09Ig4JiLe0cttNpntF9r+ju07bP8m//9K23Mal72JF29opsYErxGx4/RL0rWSDql8dtb0fFxhFuVGSU+yff/KZ0dK+sWQ0tMUh7ScLzd0s1BEHFxZ5ixJJ1bWc8z0fAs9x2zvLOlzkk6RtKukPSStlHT3QtY7Qmqdv63plPQgSb9Wym/Mke3XSTpZ0r9KGpf0QEnHSPpTSWNDTBrQN40JXjvJNXjX2X6D7UlJH7a9yPYbbV9p+2bbZ9vetbLME3Otz3rbP7R90PD2YKRtknS+pBdKku2tJL1A6YdV+bMtbnt1qj2w/c387w9zjc8Lpr8ffd2Lgth+vu3LWj57re1Pz7Jc2P5H21dIuiJ/drLtdbZvtX2Z7adU5p+wfY7tT9i+zfb3bT8mT/5DSYqIj0XEfRFxZ0RcEBGX52WX55rY9+Rz9CrbT86fr8s1T0dWtvVs22tyOtbZnujJwSpQTfJ3MxFxl6RzJD2isvxq2++svP9n27+yfYPto9zmzs0osr1Y0tslvTIizomI2yJZExGHR8Td3X7/bR8v6SmS3pfLx/flz59s+3tOd0G+Z/vJlWUutP2OfD7eZvsC20sHsOsYcY0PXrNxpRqcfST9g6RXSTpM0p8r3VL7raT3S5LtPSR9XtI78zKvl3Su7VkfM4e++Iikl+b/nyHpx+qyJqlVRPxZ/vcxuTbpEz1IX9N8RtKDbT+88tlLlPJhNodJOkC/D0K+J+mxSufRRyV90vZ2lfkPlfTJyvTzbW+jVLN+n+3TbR9se5c22zpA0uWS7p+X/bikx0vaT9IRSj/AO+Z571D6Di2R9GxJr7B9WBf700R1yN/N2N5e6aL0knYbtf1MSa+V9DSl/D2oi7SOiidJ2lbSTBcfXX3/I+LNkr4l6dhcPh6bK3U+L+m9SufauyV93pvfDXuxpJdJeoBSTe/rF7xXwCxGJXidkrQiIu6OiDuVbqm8OSKui4i7JU1Iel6uvTtC0hci4gsRMRURX5F0qaRnDSvxoywivi1pV9sPVSqAu/mRxczOzzWW622fX52Qz4dPKJ0Hsv1IScuUbuPPZlVE3JLPMUXEmRFxc0TcGxH/rvQjW21ffFmuLbpH6UdxO0lPjIhbJR0oKSR9UNKNtj9j+4GVZa+OiA9HxH05vXtJens+xy9QqrXfL6fjwoj4UT6fL5f0MaUL16aqdf62plPSBklPV7rt3c7fSvpwRPwkIjYqlddIlkq6KSLunf7Av79reKftP1vg9//Zkq6IiDNyPn9M0s8kHVKZ58MR8Yv8vThb6YIG6KtRCV5vzLempu0j6VPTBbykn0q6T6mt0D6Snl8p/Ncr/ZA+aOCpbrb7JLXWwmwj6Z42854h6VhJT5X0qTbTMTeHRcSS/GpXA3m6pBfbtlKt3Nk56JnNuuob26+3/dN8u3G9pMVKP7ZbzB8RU5KuU+5cFBE/jYjlEbGnpEflz0+qLPvryv/TwVTrZzvmdBxg+79t32h7g9LFa5NvbdY+f6vpVApqj5X0Ddvjbba7e8u217WZZ1TdLGmpK82mIuLJ+bjeLGnRAr//u0u6puWza5TaoU+brPy/Ufm8A/ppVILXaHm/TtLBlQJ+SURsFxHX52lntEzbISLeNfhkN9q1SjU+VQ/WlgWllILXVyrViG9smXZH/rt95bN2P4DoUkRcolRz+RSlW4JndLvo9D+5/eM/K9Wa7ZJ/TDdIqvZ+3qsy/yJJe6pNk5CI+Jmk1UpB7Hx8VOl2+V4RsVjSqS3pGCk1zN/7IuI8pQvaA9ts91d52S3WC/2PUkfGQ2eYZy7f/9bfyhuUKnSq9pZ0/dyTCvTOqASvrU6VdLztfSTJ9m62p0/+MyUdYvsZtreyvZ1Tp549O64N8/EJSW+xvadTB7qnKd2KOqd1xoi4Wuk215vbTLtRqSA9IufX30nad4bt/lrSQ3qxAw33EUnvk3RPRHQcJ3cGO0m6V2nEiK1tv03Szi3z/LHt5+Zao9co/QhfYvthtl83fc45DaH0InVoE9llWm6JiLtsP0EpYBt1Q8vf1hU5OVTSLkp3wVqdLellth+e28cy/msWEeuVRuL4gO3n2d4pl6ePlbRDnm0u3//W8vELkv7Q9ottb237BUptnrtpZgL0zagGrycrXYleYPs2pQL1AEmKiHVKV7FvUiqY10n6J43useqXt0v6tqSLlDrMnSjp8Ij4cbuZI+KiiOjUUevlSnl0s6RH5vV2MiHp9Nwk5G/nmfZRcIZSTeeZ81z+y5K+pNT56hpJd2nL272fVuqo81ul29fPze0jb1M6H79j+w6l8/PHkl43z7S8UtLb87n+NqVgaNQNM3+nfdZpHNpbJR0v6ciI+EnrhiLii0odhv5b0i/1+wCYodMkRcSJSh3a/lkp+Py1pNMkvUGpLJzL9/9kpf4fv7X93oi4WdJzlM69m/M2nhMRN/Vrf4BuOKL1LgGAUWf7fpJ+I+lxEXFFH9Y/IWm/iDii1+vG7ErO3zxSwo8lbVvtqARgdFCbCKCdV0j6Xj8CG9RCUflr+69tb5uHTfsXSZ8lcAVGF0+bArAZ22uVOnSM6liojVZo/h6t1GnvPknfULoVDmBE0WwAAAAAxaDZAAAAAIoxp2YDO+ywQyxevLhfaVmQRYsWaXx8dIf3vOyyy26KiFkfYbt06dJYtmzZAFKEueo2D+d7Ho76OTJXk5OTmpqamtMy69evj40bN3ZVKdBNPpJnvddNvnabj5Sn9dVteYoyzSl4Xbx4sY4++uh+pWXBVqxYMewkDI3tdoP7b2HZsmW69NJL+50czEO3ebiQ83CUz5G5Wrly5ZyXOe2007p++EG3+Uie9VY3+dptPlKe1le35SnKRLMBAAAAFIPgFQAAAMVgqCwAANBTq1at0qZNm/q2/rGxMR133HF9Wz/qjeAVAGpsPm1vB6U1gOh3wIJy9Pt7wPdstI1M8Nr0QnV8fHz/YacBwGhpLVObXMYCqI+RafPa9ELV9sjkJQAAGF0jU/OKzppeK11Hw2qvVYdb0DPtO99FoBnGxsY4l9E3BK+ggBmCUT7mM+37KB8XoEkWenFehwtt1Be3mgFgCBYtovitq4iY26PVAAzUyNS8cgsDQJ2Mj49v9vSsTjVNdXnC1kJqwuqyD1L7/WhN38TExJpBpQfA3I1M8MotDAAAgPJx3woAAADFIHgFAABAMQhegcLQ0ae+6OgDAP3XqDavC2mXynOSt1SnThYl63V76daOPnPZ1iDztB/txOuS/k7poKMPMDgznaM8dbLZqMLJGIkAAIBm4KmTzUbmAgAAoBgErwAAACgGwSsAAGgUOk82W6M6bAEAgNEwUwdOOk82GzWv2djY2IKmAwAAoP96XvPa1OGVeLystGrVKkZlmAVDrgEA0F80G0DXCFxnxzECUBf9rHDgQh3DRLMBAAAaqJ8X01yoY5gIXgEAAFCMgTYbqHObydlugdQ57QAAtBobGxva7xa/meingQavdf4iz5a2OqcdAIBWC2mTutBOxvxmop9oNgAAAIBiELwCAGbVbqzr1s+6mQcAFoqhstATTR3ft5MmjNsLzEU3t6AZOgnT+t3eloui0UbwmnEiAADQG1zIoJ8aFbyOWu0fAADAqKHNKwAAAIrRqJpXAAAwGHVu+z8+Pr7/sNOA/qHmFQAANIpt4psGI3MBAABQjIE2G+BRdaOjScd7tkcHo71O5/tMI3vMVEYwIggAQBpw8Mqj6kZHk453k/ZlkOZzvnORAKAXImJq2GlA/9BhCwAAzFmdh6ecmJhYM+w0oH9o8woAAIBiELwCAACgGD1vNjDftqn97hQzzM5ig0D7HgCol353XK1zZ9JR3nf0X23avPY7sGz6l5z2PQBQL/3+Xatzhcwo7zv6rzbBK4D+6/UTcepS+9GkodkAADMjeMVA1bl3qlTvxx3WUV0CxrqkAwDQfwSvAAD0wbD7WgzzjsQg9n2myobx8fH9+7pxDFWjgtdRrjXjRAWAellok5qSH84z7H23zWhKDdao4HWUcaICzdSLi/K6tE1eCNo1A5hGwAMADdeEoK8J+wCgNwheAQBooLGxsSLX3Yv1M/Z5s9FsAACABiq5qchC087Y581G8IqhGnZv2JIL93YG3bu537Uv3RrkftdlnwFgVM0peF20qH+tDGb7QRj2kCPoj2HmaRO/T00LxrvVhP3uZxnXhIC7X8en9GMzyqPszIQReJptTsHr+Pj40AaZH/aPEwUEgH4adhlXdxwfzAUj8DQbmQsAAIBiELwCAACgGASvAACgURgqq9nm1OZ1cnKyb20/Z+v5zdNVADRZP8u4Joys0a/jU/qxGVY/lLpjqKxmm1PwOjXVvwuZ2QolAtdmGuYoEqX3Mm5n0Bd5dfnhH+R+92uf+5n+JpSf/dqHJhwbYNQwziuGaqFBQKc7AaNaGzHoH+K6/PAPMh112WcAGFUErwAANFDJTVEWmnbGeW02OmwBANBAJTdFWej6Gee12cjchqBnJYBOmtC+uwn7AKA3GtVsYFTbOUr0rASaapTLtape3aLmaYlA+RoVvAIAUBfDHuJxmKO5DHvfuRvZbASvAAD0wbBHplhIbfVCa6gHse8z3ZXgbmSzEbxioOZbIM61Zyu3BgejLu0Qh1nDBAAYLIJXFIHApDea2n6yH0P2cAEEAPVUm+C13zU4w25/02+MaQcA9dLvOwJ1ufPRzijvO/qv58FrXWt2mhy4SoxpBwB1U4dHJw/LKO87+o+ABwAAAMUgeAUAAEAxatPmFQAAlKPOnRrpB9JsAw1eh9lpiqF0BqvXx3uYjfPpGDA/nc73mYY9m6mMmOtwaQBGF/1Amm2gweswg8eF/ujV+QqzjoYdZNS14+Ao6XS+z1QOzHcaAGB0cGUCAACAYhC8AgCARomIqWGnAf1Dhy0AADBndW6eNTExsWbYaUD/NCp4XUi7VDqDAAAA1F+jgteFoDMIAAC90e/RhahwGm0Er+iJfo/GMNeCitEhAGB4+l0hRIXTaCN4RREoqIDhaleT1npR2c08ALBQjDYAAJhVuwvI1s+6mQcAFoqaVwAAGoinWqKpBhq81vnLPNsjQOucdgAAWvFUSzTVQIPXkts9cSICAAAMH21eAQAAUAyCVwAAGmi25nB1XTcwGzpsoWvDbPdbSkFZSjoBNF/JTfWAmfQ8eC21bedsYxEOs9dmXZRUENb5mdsAAGD+aDaQzRaYjnrgCgAAUAc0GwAAAMWZ6U7v+Pj4/gNMCgaMmlcAANAotolvGozMBQAAQDEIXjN6iQMAANRfo9q80sO8t+o0csRso0G0qlPae21ycnLe+1f6calL+julg3Z2QD1ExNSw04D+aVTwiuZitIffm5qiTK4r2tkBgzNThdXExMSaASYFA0ZBCwAAgGIQvAIAAKAYBK8AAAAoxsi0eeXxrgAAAOUbmeCVwBVAnXQ7akRdRlhYiLrvQ2v6GDUCqDeaDQDAEDBqRH0xagRQbyNT84rOxsbGal8z3bSHSDRtf+Zipn0v4bsIYHY01UM/EbxiToP/l4IHVrRX9+NSp+9i3W91A3VG4Ip+GplbI02v6eJpIgAGrbVcbXo5C6AeRqbmtU41Ov3A00SAZqp7bXlVCeUsNeqD0e8mQFwojbaRCV4BAMBglHAhg3KNTLMBAAAAlI/gFQAAAMVwRHQ98/bbbz+1ZMkS9zE98xYRU5OTk6Pc7nOfiNhttpls3yjpmgGkZyDGx8f3bx2TseDvQld5ON/zsODjMhTtvluzWb9+fWzcuLGrZbrJR/Ks97rJ127zsWnlacN0VZ6iTHMKXgEAAIBhotkAAAAAikHwCgAAgGIQvAIAAKAYBK8AAAAoBsErAAAAikHwCgAAgGIQvAIAAKAYBK8AAAAoBsErAAAAivH/A7bwdOuScgUpAAAAAElFTkSuQmCC\n",
            "text/plain": [
              "<Figure size 864x504 with 17 Axes>"
            ]
          },
          "metadata": {
            "tags": []
          }
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "cellView": "both",
        "colab_type": "code",
        "id": "3nLaKviAXrq9",
        "colab": {}
      },
      "source": [
        "#@title Implement the goal-conditioned actor and critic.\n",
        "def disable_training_actor(actor):\n",
        "  for layer in list(actor._mlp_layers)[1:-1]:\n",
        "      layer.trainable = False\n",
        "\n",
        "def disable_training_critic(critic):\n",
        "  for layer in list(critic._joint_layers)[1:-1]:\n",
        "      layer.trainable = False\n",
        "\n",
        "def set_goal(traj, goal):\n",
        "  \"\"\"Sets the goal of a Trajectory or TimeStep.\"\"\"\n",
        "  for obs_field in ['observation', 'goal']:\n",
        "    assert obs_field in traj.observation.keys()\n",
        "  obs = traj.observation['observation']\n",
        "  tf.nest.assert_same_structure(obs, goal)\n",
        "  modified_traj = traj._replace(\n",
        "      observation={'observation': obs, 'goal': goal})\n",
        "  return modified_traj\n",
        "\n",
        "def merge_obs_goal(observations):\n",
        "  \"\"\"Merge the observation and goal fields into a single tensor.\n",
        "\n",
        "  If both are 1D, we concatenate the observation and goal together. If both are\n",
        "  3D, we stack along the third axis, so the resulting tensor has\n",
        "  shape (H x W x 2 * D).\n",
        "\n",
        "  Args:\n",
        "    observations: Dictionary-type observations.\n",
        "  Returns:\n",
        "    a merged observation\n",
        "  \"\"\"\n",
        "  obs = observations['observation']\n",
        "  goal = observations['goal']\n",
        "\n",
        "  assert obs.shape == goal.shape\n",
        "  # For 1D observations, simply concatenate them together.\n",
        "  assert len(obs.shape) == 2\n",
        "  modified_observations = tf.concat([obs, goal], axis=-1)\n",
        "  assert obs.shape[0] == modified_observations.shape[0]\n",
        "  assert modified_observations.shape[1] == obs.shape[1] + goal.shape[1]\n",
        "  return modified_observations\n",
        "\n",
        "\n",
        "class GoalConditionedActorNetwork(actor_network.ActorNetwork):\n",
        "  \"\"\"Actor network that takes observations and goals as inputs.\"\"\"\n",
        "\n",
        "  def __init__(self,\n",
        "               input_tensor_spec,\n",
        "               output_tensor_spec,\n",
        "               **kwargs):\n",
        "    modified_tensor_spec = None\n",
        "    super(GoalConditionedActorNetwork, self).__init__(\n",
        "        modified_tensor_spec, output_tensor_spec,\n",
        "        fc_layer_params=(256, 256),\n",
        "        **kwargs)\n",
        "    self._input_tensor_spec = input_tensor_spec\n",
        "    self.create_variables()\n",
        "\n",
        "  def call(self, observations, step_type=(), network_state=()):\n",
        "    modified_observations = merge_obs_goal(observations)\n",
        "    return_vals = super(GoalConditionedActorNetwork, self).call(\n",
        "        modified_observations, step_type=step_type, network_state=network_state)\n",
        "    return return_vals\n",
        "\n",
        "\n",
        "class GoalConditionedCriticNetwork(critic_network.CriticNetwork):\n",
        "  \"\"\"Actor network that takes observations and goals as inputs.\n",
        "\n",
        "  Further modified so it can make multiple predictions.\n",
        "  \"\"\"\n",
        "\n",
        "  def __init__(self,\n",
        "               input_tensor_spec,\n",
        "               observation_conv_layer_params=None,\n",
        "               observation_fc_layer_params=(256,),\n",
        "               action_fc_layer_params=None,\n",
        "               joint_fc_layer_params=(256,),\n",
        "               activation_fn=tf.nn.relu,\n",
        "               name='CriticNetwork',\n",
        "               output_dim=None):\n",
        "    \"\"\"Creates an instance of `CriticNetwork`.\n",
        "\n",
        "    Args:\n",
        "      input_tensor_spec: A tuple of (observation, action) each a nest of\n",
        "        `tensor_spec.TensorSpec` representing the inputs.\n",
        "      observation_conv_layer_params: Optional list of convolution layer\n",
        "        parameters for observations, where each item is a length-three tuple\n",
        "        indicating (num_units, kernel_size, stride).\n",
        "      observation_fc_layer_params: Optional list of fully connected parameters\n",
        "        for observations, where each item is the number of units in the layer.\n",
        "      action_fc_layer_params: Optional list of fully connected parameters for\n",
        "        actions, where each item is the number of units in the layer.\n",
        "      joint_fc_layer_params: Optional list of fully connected parameters after\n",
        "        merging observations and actions, where each item is the number of units\n",
        "        in the layer.\n",
        "      activation_fn: Activation function, e.g. tf.nn.relu, slim.leaky_relu, ...\n",
        "      name: A string representing name of the network.\n",
        "      output_dim: An integer specifying the number of outputs. If None, output\n",
        "        will be flattened.\n",
        "\n",
        "    \"\"\"\n",
        "    self._output_dim = output_dim\n",
        "    (_, action_spec) = input_tensor_spec\n",
        "    modified_obs_spec = None\n",
        "    modified_tensor_spec = (modified_obs_spec, action_spec)\n",
        "\n",
        "    super(critic_network.CriticNetwork, self).__init__(\n",
        "        input_tensor_spec=modified_tensor_spec,\n",
        "        state_spec=(),\n",
        "        name=name)\n",
        "    self._input_tensor_spec = input_tensor_spec\n",
        "\n",
        "    flat_action_spec = tf.nest.flatten(action_spec)\n",
        "    if len(flat_action_spec) > 1:\n",
        "      raise ValueError('Only a single action is supported by this network')\n",
        "    self._single_action_spec = flat_action_spec[0]\n",
        "\n",
        "    self._observation_layers = utils.mlp_layers(\n",
        "        observation_conv_layer_params,\n",
        "        observation_fc_layer_params,\n",
        "        activation_fn=activation_fn,\n",
        "        kernel_initializer=tf.compat.v1.keras.initializers.VarianceScaling(\n",
        "            scale=1. / 3., mode='fan_in', distribution='uniform'),\n",
        "        name='observation_encoding')\n",
        "\n",
        "    self._action_layers = utils.mlp_layers(\n",
        "        None,\n",
        "        action_fc_layer_params,\n",
        "        activation_fn=activation_fn,\n",
        "        kernel_initializer=tf.compat.v1.keras.initializers.VarianceScaling(\n",
        "            scale=1. / 3., mode='fan_in', distribution='uniform'),\n",
        "        name='action_encoding')\n",
        "\n",
        "    self._joint_layers = utils.mlp_layers(\n",
        "        None,\n",
        "        joint_fc_layer_params,\n",
        "        activation_fn=activation_fn,\n",
        "        kernel_initializer=tf.compat.v1.keras.initializers.VarianceScaling(\n",
        "            scale=1. / 3., mode='fan_in', distribution='uniform'),\n",
        "        name='joint_mlp')\n",
        "\n",
        "    self._joint_layers.append(\n",
        "        tf.keras.layers.Dense(\n",
        "            self._output_dim if self._output_dim is not None else 1,\n",
        "            activation=None,\n",
        "            kernel_initializer=tf.compat.v1.keras.initializers.RandomUniform(\n",
        "                minval=-0.003, maxval=0.003),\n",
        "            name='value'))\n",
        "    self.create_variables()\n",
        "\n",
        "\n",
        "  def call(self, inputs, step_type=(), network_state=()):\n",
        "    observations, actions = inputs\n",
        "    modified_observations = merge_obs_goal(observations)\n",
        "    modified_inputs = (modified_observations, actions)\n",
        "    output = super(GoalConditionedCriticNetwork, self).call(\n",
        "        modified_inputs, step_type=step_type, network_state=network_state)\n",
        "    (predictions, network_state) = output\n",
        "\n",
        "    # We have to reshape the output, which is flattened by default\n",
        "    if self._output_dim is not None:\n",
        "      predictions = tf.reshape(predictions, [-1, self._output_dim])\n",
        "\n",
        "    return predictions, network_state"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "cellView": "code",
        "colab_type": "code",
        "id": "66Rh9rum8qNh",
        "colab": {}
      },
      "source": [
        "#@title Implement the goal-conditioned agent.\n",
        "\n",
        "class UvfAgent(tf_agent.TFAgent):\n",
        "  \"\"\"A UVF Agent.\"\"\"\n",
        "\n",
        "  def __init__(\n",
        "      self,\n",
        "      time_step_spec,\n",
        "      action_spec,\n",
        "      ou_stddev=1.0,\n",
        "      ou_damping=1.0,\n",
        "      target_update_tau=0.05,\n",
        "      target_update_period=5,\n",
        "      max_episode_steps=None,\n",
        "      ensemble_size=3,\n",
        "      combine_ensemble_method='min',\n",
        "      use_distributional_rl=True):\n",
        "    \"\"\"Creates a Uvf Agent.\n",
        "\n",
        "    Args:\n",
        "      time_step_spec: A `TimeStep` spec of the expected time_steps.\n",
        "      action_spec: A nest of BoundedTensorSpec representing the actions.\n",
        "      ou_stddev: Standard deviation for the Ornstein-Uhlenbeck (OU) noise added\n",
        "        in the default collect policy.\n",
        "      ou_damping: Damping factor for the OU noise added in the default collect\n",
        "        policy.\n",
        "      target_update_tau: Factor for soft update of the target networks.\n",
        "      target_update_period: Period for soft update of the target networks.\n",
        "      max_episode_steps: Int indicating number of steps in an episode. Used for\n",
        "        determining the number of bins for distributional RL.\n",
        "      ensemble_size: (int) Number of models in ensemble of critics.\n",
        "      combine_ensemble_method: (str) At test time, how to combine the distances\n",
        "        predicted by each member of the ensemble. Options are 'mean', 'min',\n",
        "        and 'td3'. The 'td3' option is pessimistic w.r.t. the pdf, and then\n",
        "        takes computes the corresponding distance. The 'min' option takes the\n",
        "        minimum q values, corresponding to taking the maximum predicted\n",
        "        distance. Note that we never aggregate predictions during training.\n",
        "      use_distributional_rl: (bool) Whether to use distributional RL.\n",
        "    \"\"\"\n",
        "    tf.Module.__init__(self, name='UvfAgent')\n",
        "\n",
        "    assert max_episode_steps is not None\n",
        "    self._max_episode_steps = max_episode_steps\n",
        "    self._ensemble_size = ensemble_size\n",
        "    self._use_distributional_rl = use_distributional_rl\n",
        "\n",
        "    # Create the actor\n",
        "    self._actor_network = GoalConditionedActorNetwork(\n",
        "        time_step_spec.observation, action_spec)\n",
        "    self._target_actor_network = self._actor_network.copy(\n",
        "        name='TargetActorNetwork')\n",
        "    \n",
        "\n",
        "    # Create a prototypical critic, which we will copy to create the ensemble.\n",
        "    critic_net_input_specs = (time_step_spec.observation, action_spec)\n",
        "    critic_network = GoalConditionedCriticNetwork(\n",
        "        critic_net_input_specs,\n",
        "        output_dim=max_episode_steps if use_distributional_rl else None,\n",
        "    )\n",
        "    self._critic_network_list = []\n",
        "    self._target_critic_network_list = []\n",
        "    for ensemble_index in range(self._ensemble_size):\n",
        "      self._critic_network_list.append(\n",
        "          critic_network.copy(name='CriticNetwork%d' % ensemble_index))\n",
        "      self._target_critic_network_list.append(\n",
        "          critic_network.copy(name='TargetCriticNetwork%d' % ensemble_index))\n",
        "\n",
        "    self._actor_optimizer = tf.optimizers.Adam(learning_rate=3e-4)\n",
        "    self._critic_optimizer = tf.optimizers.Adam(learning_rate=3e-4)\n",
        "\n",
        "    self._ou_stddev = ou_stddev\n",
        "    self._ou_damping = ou_damping\n",
        "    self._target_update_tau = target_update_tau\n",
        "    self._target_update_period = target_update_period\n",
        "\n",
        "    self._update_target = self._get_target_updater(\n",
        "        target_update_tau, target_update_period)\n",
        "\n",
        "    policy = actor_policy.ActorPolicy(\n",
        "        time_step_spec=time_step_spec, action_spec=action_spec,\n",
        "        actor_network=self._actor_network, clip=True)\n",
        "    collect_policy = actor_policy.ActorPolicy(\n",
        "        time_step_spec=time_step_spec, action_spec=action_spec,\n",
        "        actor_network=self._actor_network, clip=False)\n",
        "    collect_policy = ou_noise_policy.OUNoisePolicy(\n",
        "        collect_policy,\n",
        "        ou_stddev=self._ou_stddev,\n",
        "        ou_damping=self._ou_damping,\n",
        "        clip=True)\n",
        "\n",
        "    super(UvfAgent, self).__init__(\n",
        "        time_step_spec,\n",
        "        action_spec,\n",
        "        policy,\n",
        "        collect_policy,\n",
        "        train_sequence_length=2)\n",
        "\n",
        "  def initialize_search(self, active_set, max_search_steps=3,\n",
        "                        combine_ensemble_method='min'):\n",
        "    self._combine_ensemble_method = combine_ensemble_method\n",
        "    self._max_search_steps = max_search_steps\n",
        "    self._active_set_tensor = tf.convert_to_tensor(value=active_set)\n",
        "    pdist = self._get_pairwise_dist(self._active_set_tensor, masked=True,\n",
        "                                    aggregate=combine_ensemble_method)\n",
        "    distances = scipy.sparse.csgraph.floyd_warshall(pdist, directed=True)\n",
        "    self._distances_tensor = tf.convert_to_tensor(value=distances, dtype=tf.float32)\n",
        "\n",
        "  def _get_pairwise_dist(self, obs_tensor, goal_tensor=None, masked=False,\n",
        "                         aggregate='mean'):\n",
        "    \"\"\"Estimates the pairwise distances.\n",
        "\n",
        "    Args:\n",
        "      obs_tensor: Tensor containing observations\n",
        "      goal_tensor: (optional) Tensor containing a second set of observations. If\n",
        "        not specified, computes the pairwise distances between obs_tensor and\n",
        "        itself.\n",
        "      masked: (bool) Whether to ignore edges that are too long, as defined by\n",
        "        max_search_steps.\n",
        "      aggregate: (str) How to combine the predictions from the ensemble. Options\n",
        "        are to take the minimum predicted q value (i.e., the maximum distance),\n",
        "        the mean, or to simply return all the predictions.\n",
        "    \"\"\"\n",
        "    if goal_tensor is None:\n",
        "      goal_tensor = obs_tensor\n",
        "    dist_matrix = []\n",
        "    for obs_index in range(obs_tensor.shape[0]):\n",
        "      obs = obs_tensor[obs_index]\n",
        "      obs_repeat_tensor = tf.ones_like(goal_tensor) * tf.expand_dims(obs, 0)\n",
        "      obs_goal_tensor = {'observation': obs_repeat_tensor,\n",
        "                         'goal': goal_tensor}\n",
        "      pseudo_next_time_steps = time_step.transition(obs_goal_tensor,\n",
        "                                                    reward=0.0,  # Ignored\n",
        "                                                    discount=1.0)\n",
        "      dist = self._get_dist_to_goal(pseudo_next_time_steps, aggregate=aggregate)\n",
        "      dist_matrix.append(dist)\n",
        "\n",
        "    pairwise_dist = tf.stack(dist_matrix)\n",
        "    if aggregate is None:\n",
        "      pairwise_dist = tf.transpose(a=pairwise_dist, perm=[1, 0, 2])\n",
        "\n",
        "    if masked:\n",
        "      mask = (pairwise_dist > self._max_search_steps)\n",
        "      return tf.compat.v1.where(mask, tf.fill(pairwise_dist.shape, np.inf),\n",
        "                        pairwise_dist)\n",
        "    else:\n",
        "      return pairwise_dist\n",
        "\n",
        "  def _get_critic_output(self, critic_net_list, next_time_steps,\n",
        "                         actions=None):\n",
        "    \"\"\"Calls the critic net.\n",
        "\n",
        "    Args:\n",
        "      critic_net_list: (list) List of critic networks.\n",
        "      next_time_steps: time_steps holding the observations and step types\n",
        "      actions: (optional) actions to compute the Q values for. If None, returns\n",
        "      the Q values for the best action.\n",
        "    Returns:\n",
        "      q_values_list: (list) List containing a tensor of q values for each member\n",
        "      of the ensemble. For distributional RL, computes the expectation over the\n",
        "      distribution.\n",
        "    \"\"\"\n",
        "    q_values_list = []\n",
        "    critic_net_input = (next_time_steps.observation, actions)\n",
        "    for critic_index in range(self._ensemble_size):\n",
        "      critic_net = critic_net_list[critic_index]\n",
        "      q_values, _ = critic_net(critic_net_input, next_time_steps.step_type)\n",
        "      q_values_list.append(q_values)\n",
        "    return q_values_list\n",
        "\n",
        "  def _get_expected_q_values(self, next_time_steps, actions=None):\n",
        "    if actions is None:\n",
        "      actions, _ = self._actor_network(next_time_steps.observation,\n",
        "                                        next_time_steps.step_type)\n",
        "\n",
        "    q_values_list = self._get_critic_output(self._critic_network_list,\n",
        "                                            next_time_steps, actions)\n",
        "\n",
        "    expected_q_values_list = []\n",
        "    for q_values in q_values_list:\n",
        "      if self._use_distributional_rl:\n",
        "        q_probs = tf.nn.softmax(q_values, axis=1)\n",
        "        batch_size = q_probs.shape[0]\n",
        "        bin_range = tf.range(1, self._max_episode_steps + 1, dtype=tf.float32)\n",
        "        ### NOTE: We want to compute the value of each bin, which is the\n",
        "        # negative distance. Without properly negating this, the actor is\n",
        "        # optimized to take the *worst* actions.\n",
        "        neg_bin_range = -1.0 * bin_range\n",
        "        tiled_bin_range = tf.tile(tf.expand_dims(neg_bin_range, 0),\n",
        "                                  [batch_size, 1])\n",
        "        assert q_probs.shape == tiled_bin_range.shape\n",
        "\n",
        "        ### Take the inner produce between these two tensors\n",
        "        expected_q_values = tf.reduce_sum(input_tensor=q_probs * tiled_bin_range, axis=1)\n",
        "        expected_q_values_list.append(expected_q_values)\n",
        "      else:\n",
        "        expected_q_values_list.append(q_values)\n",
        "    return tf.stack(expected_q_values_list)\n",
        "\n",
        "  def _get_state_values(self, next_time_steps, actions=None, aggregate='mean'):\n",
        "    \"\"\"Computes the value function, averaging across bins (for distributional RL)\n",
        "    and the ensemble (for bootstrap RL).\n",
        "\n",
        "    Args:\n",
        "      next_time_steps: time_steps holding the observations and step types\n",
        "      actions: actions for which to compute the Q values. If None, uses the\n",
        "      best actions (i.e., returns the value function).\n",
        "    Returns:\n",
        "      state_values: Tensor storing the state values for each sample in the\n",
        "      batch. These values should all be negative.\n",
        "    \"\"\"\n",
        "    with tf.compat.v1.name_scope('state_values'):\n",
        "      expected_q_values = self._get_expected_q_values(next_time_steps, actions)\n",
        "      if aggregate is not None:\n",
        "        if aggregate == 'mean':\n",
        "          expected_q_values = tf.reduce_mean(input_tensor=expected_q_values, axis=0)\n",
        "        elif aggregate == 'min':\n",
        "          expected_q_values = tf.reduce_min(input_tensor=expected_q_values, axis=0)\n",
        "        else:\n",
        "          raise ValueError('Unknown method for combining ensemble: %s' %\n",
        "                           aggregate)\n",
        "\n",
        "      # Clip the q values if not using distributional RL. If using\n",
        "      # distributional RL, the q values are implicitly clipped.\n",
        "      if not self._use_distributional_rl:\n",
        "        min_q_val = -1.0 * self._max_episode_steps\n",
        "        max_q_val = 0.0\n",
        "        expected_q_values = tf.maximum(expected_q_values, min_q_val)\n",
        "        expected_q_values = tf.minimum(expected_q_values, max_q_val)\n",
        "      return expected_q_values\n",
        "\n",
        "  def _get_dist_to_goal(self, next_time_step, aggregate='mean'):\n",
        "    q_values = self._get_state_values(next_time_step, aggregate=aggregate)\n",
        "    return -1.0 * q_values\n",
        "\n",
        "  def _get_waypoint(self, next_time_steps):\n",
        "    obs_tensor = next_time_steps.observation['observation']\n",
        "    goal_tensor = next_time_steps.observation['goal']\n",
        "    obs_to_active_set_dist = self._get_pairwise_dist(\n",
        "        obs_tensor, self._active_set_tensor, masked=True,\n",
        "        aggregate=self._combine_ensemble_method)  # B x A\n",
        "    active_set_to_goal_dist = self._get_pairwise_dist(\n",
        "        self._active_set_tensor, goal_tensor, masked=True,\n",
        "        aggregate=self._combine_ensemble_method)  # A x B\n",
        "\n",
        "    # The search_dist tensor should be (B x A x A)\n",
        "    search_dist = sum([\n",
        "        tf.expand_dims(obs_to_active_set_dist, 2),\n",
        "        tf.expand_dims(self._distances_tensor, 0),\n",
        "        tf.expand_dims(tf.transpose(a=active_set_to_goal_dist), axis=1)\n",
        "    ])\n",
        "\n",
        "    # We assume a batch size of 1.\n",
        "    assert obs_tensor.shape[0] == 1\n",
        "    min_search_dist = tf.reduce_min(input_tensor=search_dist, axis=[1, 2])[0]\n",
        "    waypoint_index = tf.argmin(input=tf.reduce_min(input_tensor=search_dist, axis=[2]), axis=1)[0]\n",
        "    waypoint = self._active_set_tensor[waypoint_index]\n",
        "\n",
        "    return waypoint, min_search_dist\n",
        "\n",
        "  def _initialize(self):\n",
        "    for ensemble_index in range(self._ensemble_size):\n",
        "      common.soft_variables_update(\n",
        "          self._critic_network_list[ensemble_index].variables,\n",
        "          self._target_critic_network_list[ensemble_index].variables,\n",
        "          tau=1.0)\n",
        "    # Caution: actor should only be updated once.\n",
        "    common.soft_variables_update(\n",
        "        self._actor_network.variables,\n",
        "        self._target_actor_network.variables,\n",
        "        tau=1.0)\n",
        "\n",
        "  def _get_target_updater(self, tau=1.0, period=1):\n",
        "    \"\"\"Performs a soft update of the target network parameters.\n",
        "\n",
        "    For each weight w_s in the original network, and its corresponding\n",
        "    weight w_t in the target network, a soft update is:\n",
        "    w_t = (1- tau) x w_t + tau x ws\n",
        "\n",
        "    Args:\n",
        "      tau: A float scalar in [0, 1]. Default `tau=1.0` means hard update.\n",
        "      period: Step interval at which the target networks are updated.\n",
        "    Returns:\n",
        "      An operation that performs a soft update of the target network parameters.\n",
        "    \"\"\"\n",
        "    with tf.compat.v1.name_scope('get_target_updater'):\n",
        "      def update():  # pylint: disable=missing-docstring\n",
        "        critic_update_list = []\n",
        "        for ensemble_index in range(self._ensemble_size):\n",
        "          critic_update = common.soft_variables_update(\n",
        "              self._critic_network_list[ensemble_index].variables,\n",
        "              self._target_critic_network_list[ensemble_index].variables, tau)\n",
        "          critic_update_list.append(critic_update)\n",
        "        actor_update = common.soft_variables_update(\n",
        "            self._actor_network.variables,\n",
        "            self._target_actor_network.variables, tau)\n",
        "        return tf.group(critic_update_list + [actor_update])\n",
        "\n",
        "      return common.Periodically(update, period, 'periodic_update_targets')\n",
        "\n",
        "  def _experience_to_transitions(self, experience):\n",
        "    transitions = trajectory.to_transition(experience)\n",
        "    transitions = tf.nest.map_structure(lambda x: tf.squeeze(x, [1]),\n",
        "                                        transitions)\n",
        "\n",
        "    time_steps, policy_steps, next_time_steps = transitions\n",
        "    actions = policy_steps.action\n",
        "    return time_steps, actions, next_time_steps\n",
        "\n",
        "  def _train(self, experience, weights=None):\n",
        "    del weights\n",
        "    time_steps, actions, next_time_steps = self._experience_to_transitions(\n",
        "        experience)\n",
        "\n",
        "    # Update the critic\n",
        "    critic_vars = []\n",
        "    for ensemble_index in range(self._ensemble_size):\n",
        "      critic_net = self._critic_network_list[ensemble_index]\n",
        "      critic_vars.extend(critic_net.variables)\n",
        "\n",
        "    with tf.GradientTape(watch_accessed_variables=False) as tape:\n",
        "      assert critic_vars\n",
        "      tape.watch(critic_vars)\n",
        "      critic_loss = self.critic_loss(time_steps, actions, next_time_steps)\n",
        "    tf.debugging.check_numerics(critic_loss, 'Critic loss is inf or nan.')\n",
        "    critic_grads = tape.gradient(critic_loss, critic_vars)\n",
        "    self._apply_gradients(critic_grads, critic_vars,\n",
        "                          self._critic_optimizer)\n",
        "\n",
        "    # Update the actor\n",
        "    actor_vars = self._actor_network.variables\n",
        "    with tf.GradientTape(watch_accessed_variables=False) as tape:\n",
        "      assert actor_vars, 'No actor variables to optimize.'\n",
        "      tape.watch(actor_vars)\n",
        "      actor_loss = self.actor_loss(time_steps)\n",
        "    tf.debugging.check_numerics(actor_loss, 'Actor loss is inf or nan.')\n",
        "    actor_grads = tape.gradient(actor_loss, actor_vars)\n",
        "    self._apply_gradients(actor_grads, actor_vars, self._actor_optimizer)\n",
        "\n",
        "    self.train_step_counter.assign_add(1)\n",
        "    self._update_target()\n",
        "    total_loss = actor_loss + critic_loss\n",
        "    return tf_agent.LossInfo(total_loss, (actor_loss, critic_loss))\n",
        "\n",
        "  def _apply_gradients(self, gradients, variables, optimizer):\n",
        "    # Tuple is used for py3, where zip is a generator producing values once.\n",
        "    grads_and_vars = tuple(zip(gradients, variables))\n",
        "    optimizer.apply_gradients(grads_and_vars)\n",
        "\n",
        "  def critic_loss(self,\n",
        "                  time_steps,\n",
        "                  actions,\n",
        "                  next_time_steps):\n",
        "    \"\"\"Computes the critic loss for UvfAgent training.\n",
        "\n",
        "    Args:\n",
        "      time_steps: A batch of timesteps.\n",
        "      actions: A batch of actions.\n",
        "      next_time_steps: A batch of next timesteps.\n",
        "    Returns:\n",
        "      critic_loss: A scalar critic loss.\n",
        "    \"\"\"\n",
        "    with tf.compat.v1.name_scope('critic_loss'):\n",
        "      # We compute the target actions once for all critics.\n",
        "      target_actions, _ = self._target_actor_network(\n",
        "          next_time_steps.observation, next_time_steps.step_type)\n",
        "\n",
        "      critic_loss_list = []\n",
        "      q_values_list = self._get_critic_output(self._critic_network_list,\n",
        "                                              time_steps, actions)\n",
        "      target_q_values_list = self._get_critic_output(\n",
        "          self._target_critic_network_list, next_time_steps, target_actions)\n",
        "      assert len(target_q_values_list) == self._ensemble_size\n",
        "      for ensemble_index in range(self._ensemble_size):\n",
        "        # The target_q_values should be a Batch x ensemble_size tensor.\n",
        "        target_q_values = target_q_values_list[ensemble_index]\n",
        "\n",
        "        if self._use_distributional_rl:\n",
        "          target_q_probs = tf.nn.softmax(target_q_values, axis=1)\n",
        "          batch_size = target_q_probs.shape[0]\n",
        "          one_hot = tf.one_hot(tf.zeros(batch_size, dtype=tf.int32),\n",
        "                               self._max_episode_steps)\n",
        "          ### Calculate the shifted probabilities\n",
        "          # Fist column: Since episode didn't terminate, probability that the\n",
        "          # distance is 1 equals 0.\n",
        "          col_1 = tf.zeros((batch_size, 1))\n",
        "          # Middle columns: Simply the shifted probabilities.\n",
        "          col_middle = target_q_probs[:, :-2]\n",
        "          # Last column: Probability of taking at least n steps is sum of\n",
        "          # last two columns in unshifted predictions:\n",
        "          col_last = tf.reduce_sum(input_tensor=target_q_probs[:, -2:], axis=1,\n",
        "                                   keepdims=True)\n",
        "\n",
        "          shifted_target_q_probs = tf.concat([col_1, col_middle, col_last],\n",
        "                                             axis=1)\n",
        "          assert one_hot.shape == shifted_target_q_probs.shape\n",
        "          td_targets = tf.compat.v1.where(next_time_steps.is_last(),\n",
        "                                one_hot,\n",
        "                                shifted_target_q_probs)\n",
        "          td_targets = tf.stop_gradient(td_targets)\n",
        "        else:\n",
        "          td_targets = tf.stop_gradient(\n",
        "              next_time_steps.reward +\n",
        "              next_time_steps.discount * target_q_values)\n",
        "\n",
        "        q_values = q_values_list[ensemble_index]\n",
        "        if self._use_distributional_rl:\n",
        "          critic_loss = tf.nn.softmax_cross_entropy_with_logits(\n",
        "              labels=td_targets,\n",
        "              logits=q_values\n",
        "              )\n",
        "        else:\n",
        "          critic_loss = common.element_wise_huber_loss(td_targets, q_values)\n",
        "        critic_loss = tf.reduce_mean(input_tensor=critic_loss)\n",
        "        critic_loss_list.append(critic_loss)\n",
        "\n",
        "      critic_loss = tf.reduce_mean(input_tensor=critic_loss_list)\n",
        "\n",
        "      return critic_loss\n",
        "\n",
        "  def actor_loss(self, time_steps):\n",
        "    \"\"\"Computes the actor_loss for UvfAgent training.\n",
        "\n",
        "    Args:\n",
        "      time_steps: A batch of timesteps.\n",
        "    Returns:\n",
        "      actor_loss: A scalar actor loss.\n",
        "    \"\"\"\n",
        "    with tf.compat.v1.name_scope('actor_loss'):\n",
        "      actions, _ = self._actor_network(time_steps.observation,\n",
        "                                       time_steps.step_type)\n",
        "      with tf.GradientTape(watch_accessed_variables=False) as tape:\n",
        "        tape.watch(actions)\n",
        "        avg_expected_q_values = self._get_state_values(time_steps, actions,\n",
        "                                                       aggregate='mean')\n",
        "        actions = tf.nest.flatten(actions)\n",
        "      dqdas = tape.gradient([avg_expected_q_values], actions)\n",
        "\n",
        "      actor_losses = []\n",
        "      for dqda, action in zip(dqdas, actions):\n",
        "        loss = common.element_wise_squared_loss(\n",
        "            tf.stop_gradient(dqda + action), action)\n",
        "        loss = tf.reduce_sum(input_tensor=loss, axis=1)\n",
        "        loss = tf.reduce_mean(input_tensor=loss)\n",
        "        actor_losses.append(loss)\n",
        "\n",
        "      actor_loss = tf.add_n(actor_losses)\n",
        "\n",
        "      with tf.compat.v1.name_scope('Losses/'):\n",
        "        tf.compat.v2.summary.scalar(\n",
        "            name='actor_loss', data=actor_loss, step=self.train_step_counter)\n",
        "\n",
        "    return actor_loss\n",
        "\n",
        "  # Sets the number of trainable layers in this UvfAgent\n",
        "  def set_trainable_layers(self, n_layers, all_trainable=False):\n",
        "    '''\n",
        "    Args:\n",
        "      n_layers = number of layers from top of each network that are trainable\n",
        "      all_trainable = if true, allows full model to train \n",
        "    Ret:\n",
        "      None\n",
        "    Todo:\n",
        "      Add an assert so n_layers not too large\n",
        "      Diff number of trainable layers for each of actor and critic\n",
        "    '''\n",
        "    actors = [self._actor_network, self._target_actor_network]\n",
        "    critics = self._critic_network_list\n",
        "    critics.extend(self._target_critic_network_list)\n",
        "\n",
        "    def set_layers(layers, n_layers):\n",
        "      for layer in layers[:-n_layers]:\n",
        "          layer.trainable = False\n",
        "      for layer in layers[-n_layers:] + [layers[0]]:\n",
        "          layer.trainable = True\n",
        "\n",
        "    # todo use set_layers in first sec\n",
        "    if(all_trainable):\n",
        "      # set all layers to trainable.\n",
        "      for layer in actor._mlp_layers:\n",
        "          layer.trainable = True\n",
        "      for critic in critics:\n",
        "        for layer in critic._joint_layers:\n",
        "          layer.trainable = True\n",
        "    else:\n",
        "      # set layers for actor\n",
        "      for actor in actors:\n",
        "        set_layers(actor._mlp_layers, n_layers)\n",
        "      # set layers for critics\n",
        "      for critic in critics:\n",
        "        set_layers(critic._joint_layers, n_layers)\n",
        "\n",
        "      "
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "cellView": "form",
        "colab_type": "code",
        "id": "MbGXO1TURVf-",
        "colab": {}
      },
      "source": [
        "#@title Training script.\n",
        "def train_eval(\n",
        "    tf_agent,\n",
        "    tf_env,\n",
        "    eval_tf_env,\n",
        "    num_iterations=2000000,\n",
        "    # Params for collect\n",
        "    initial_collect_steps=1000,\n",
        "    batch_size=64,\n",
        "    # Params for eval\n",
        "    num_eval_episodes=100,\n",
        "    eval_interval=10000,\n",
        "    # Params for checkpoints, summaries, and logging\n",
        "    log_interval=1000,\n",
        "    random_seed=0):\n",
        "  \"\"\"A simple train and eval for UVF.  \"\"\"\n",
        "  print('random_seed = %d' % random_seed)\n",
        "  np.random.seed(random_seed)\n",
        "  random.seed(random_seed)\n",
        "  tf.random.set_seed(random_seed)\n",
        "\n",
        "  max_episode_steps = tf_env.pyenv.envs[0]._duration\n",
        "  global_step = tf.compat.v1.train.get_or_create_global_step()\n",
        "  replay_buffer = tf_uniform_replay_buffer.TFUniformReplayBuffer(\n",
        "      tf_agent.collect_data_spec,\n",
        "      batch_size=tf_env.batch_size)\n",
        "\n",
        "  eval_metrics = [\n",
        "    tf_metrics.AverageReturnMetric(buffer_size=num_eval_episodes),\n",
        "  ]\n",
        "\n",
        "  eval_policy = tf_agent.policy\n",
        "  collect_policy = tf_agent.collect_policy\n",
        "  initial_collect_driver = dynamic_step_driver.DynamicStepDriver(\n",
        "      tf_env,\n",
        "      collect_policy,\n",
        "      observers=[replay_buffer.add_batch],\n",
        "      num_steps=initial_collect_steps)\n",
        "\n",
        "  collect_driver = dynamic_step_driver.DynamicStepDriver(\n",
        "      tf_env,\n",
        "      collect_policy,\n",
        "      observers=[replay_buffer.add_batch],\n",
        "      num_steps=1)\n",
        "\n",
        "  initial_collect_driver.run = common.function(initial_collect_driver.run)\n",
        "  collect_driver.run = common.function(collect_driver.run)\n",
        "  tf_agent.train = common.function(tf_agent.train)\n",
        "\n",
        "  initial_collect_driver.run()\n",
        "\n",
        "  time_step = None\n",
        "  policy_state = collect_policy.get_initial_state(tf_env.batch_size)\n",
        "\n",
        "  timed_at_step = global_step.numpy()\n",
        "  time_acc = 0\n",
        "\n",
        "  # Dataset generates trajectories with shape [Bx2x...]\n",
        "  dataset = replay_buffer.as_dataset(\n",
        "      num_parallel_calls=3,\n",
        "      sample_batch_size=batch_size,\n",
        "      num_steps=2).prefetch(3)\n",
        "  iterator = iter(dataset)\n",
        "\n",
        "  for _ in tqdm.tnrange(num_iterations):\n",
        "    start_time = time.time()\n",
        "    time_step, policy_state = collect_driver.run(\n",
        "        time_step=time_step,\n",
        "        policy_state=policy_state,\n",
        "    )\n",
        "\n",
        "    experience, _ = next(iterator)\n",
        "    train_loss = tf_agent.train(experience)\n",
        "    time_acc += time.time() - start_time\n",
        "\n",
        "    if global_step.numpy() % log_interval == 0:\n",
        "      tf.compat.v1.logging.info('step = %d, loss = %f', global_step.numpy(),\n",
        "                    train_loss.loss)\n",
        "      steps_per_sec = log_interval / time_acc\n",
        "      tf.compat.v1.logging.info('%.3f steps/sec', steps_per_sec)\n",
        "      time_acc = 0\n",
        "\n",
        "    if global_step.numpy() % eval_interval == 0:\n",
        "      start = time.time()\n",
        "      tf.compat.v1.logging.info('step = %d' % global_step.numpy())\n",
        "      for dist in [2, 5, 10]:\n",
        "        tf.compat.v1.logging.info('\\t dist = %d' % dist)\n",
        "        eval_tf_env.pyenv.envs[0].gym.set_sample_goal_args(\n",
        "          prob_constraint=1.0, min_dist=dist-1, max_dist=dist+1)\n",
        "\n",
        "        results = metric_utils.eager_compute(\n",
        "            eval_metrics,\n",
        "            eval_tf_env,\n",
        "            eval_policy,\n",
        "            num_episodes=num_eval_episodes,\n",
        "            train_step=global_step,\n",
        "            summary_prefix='Metrics',\n",
        "        )\n",
        "        for (key, value) in results.items():\n",
        "          tf.compat.v1.logging.info('\\t\\t %s = %.2f', key, value.numpy())\n",
        "        # For debugging, it's helpful to check the predicted distances for\n",
        "        # goals of known distance.\n",
        "        pred_dist = []\n",
        "        for _ in range(num_eval_episodes):\n",
        "          ts = eval_tf_env.reset()\n",
        "          dist_to_goal = agent._get_dist_to_goal(ts)[0]\n",
        "          pred_dist.append(dist_to_goal.numpy())\n",
        "        tf.compat.v1.logging.info('\\t\\t predicted_dist = %.1f (%.1f)' %\n",
        "                        (np.mean(pred_dist), np.std(pred_dist)))\n",
        "      tf.compat.v1.logging.info('\\t eval_time = %.2f' % (time.time() - start))\n",
        "\n",
        "  return train_loss"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "colab_type": "text",
        "id": "zxW1bAAPeOH6"
      },
      "source": [
        "----------------\n",
        "## Train the Agent!\n",
        "Now we're going to train the goal-conditioned RL agent. The first cell resets the weights, and the second cell does the actual training. If you want to continue training for longer, simply run the second cell again. Expect training to take about 10 minutes. For some of the complex environments, you may need to increase the `num_iterations` to 100,000."
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab_type": "code",
        "id": "lEBCYVpqU07z",
        "colab": {}
      },
      "source": [
        "# Run this cell before training on a new environment!\n",
        "from tensorflow.python.framework import ops\n",
        "ops.reset_default_graph()"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab_type": "code",
        "id": "4mmoO3zoSGVE",
        "outputId": "5d8ad3ed-3a0c-4c80-c1c6-c7a2044ddb26",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 1000,
          "referenced_widgets": [
            "ea30e0c68b2e43d491cd4011cd3be63a",
            "520c62e857f34806814dbd5339b95e01",
            "8c2b1a7399914df0869bd1e8e28d20b7",
            "808ab10d324d4eada3ee2f7c95417398",
            "1ed67ab07db543249d2cc4939007eae6",
            "fd10ff0e6c63427fb9772504a9840952",
            "060a0a5419e74eeca9851a7a6ce42f0d",
            "feb3500bdb6441e3920ac871bb02ba63"
          ]
        }
      },
      "source": [
        "# If you change the environment parameters below, make sure to run\n",
        "# tf.reset_default_graph() in the cell above before training.\n",
        "max_episode_steps = 20\n",
        "env_name = 'FourRooms'  # Choose one of the environments shown above.\n",
        "resize_factor = 5  # Inflate the environment to increase the difficulty.\n",
        "\n",
        "tf_env = env_load_fn(env_name, max_episode_steps,\n",
        "                     resize_factor=resize_factor,\n",
        "                     terminate_on_timeout=False)\n",
        "eval_tf_env = env_load_fn(env_name, max_episode_steps,\n",
        "                          resize_factor=resize_factor,\n",
        "                          terminate_on_timeout=True)\n",
        "agent = UvfAgent(\n",
        "    tf_env.time_step_spec(),\n",
        "    tf_env.action_spec(),\n",
        "    max_episode_steps=max_episode_steps,\n",
        "    use_distributional_rl=True,\n",
        "    ensemble_size=3)\n",
        "\n",
        "train_eval(\n",
        "    agent,\n",
        "    tf_env,\n",
        "    eval_tf_env,\n",
        "    initial_collect_steps=1000,\n",
        "    eval_interval=10, # 1000\n",
        "    num_eval_episodes=10,\n",
        "    num_iterations=100,#30000,\n",
        ")\n",
        "\n"
      ],
      "execution_count": 9,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "/usr/local/lib/python3.6/dist-packages/gym/logger.py:30: UserWarning: \u001b[33mWARN: Box bound precision lowered by casting to float32\u001b[0m\n",
            "  warnings.warn(colorize('%s: %s'%('WARN', msg % args), 'yellow'))\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "random_seed = 0\n",
            "WARNING:tensorflow:From /usr/local/lib/python3.6/dist-packages/tf_agents/drivers/dynamic_step_driver.py:201: calling while_loop_v2 (from tensorflow.python.ops.control_flow_ops) with back_prop=False is deprecated and will be removed in a future version.\n",
            "Instructions for updating:\n",
            "back_prop=False is deprecated. Consider using tf.stop_gradient instead.\n",
            "Instead of:\n",
            "results = tf.while_loop(c, b, vars, back_prop=False)\n",
            "Use:\n",
            "results = tf.nest.map_structure(tf.stop_gradient, tf.while_loop(c, b, vars))\n",
            "WARNING:tensorflow:From /usr/local/lib/python3.6/dist-packages/tensorflow/python/ops/resource_variable_ops.py:1817: calling BaseResourceVariable.__init__ (from tensorflow.python.ops.resource_variable_ops) with constraint is deprecated and will be removed in a future version.\n",
            "Instructions for updating:\n",
            "If using Keras pass *_constraint arguments to layers.\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "/usr/local/lib/python3.6/dist-packages/ipykernel_launcher.py:65: TqdmDeprecationWarning: Please use `tqdm.notebook.trange` instead of `tqdm.tnrange`\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "display_data",
          "data": {
            "application/vnd.jupyter.widget-view+json": {
              "model_id": "ea30e0c68b2e43d491cd4011cd3be63a",
              "version_minor": 0,
              "version_major": 2
            },
            "text/plain": [
              "HBox(children=(FloatProgress(value=0.0), HTML(value='')))"
            ]
          },
          "metadata": {
            "tags": []
          }
        },
        {
          "output_type": "stream",
          "text": [
            "INFO:tensorflow:step = 10\n",
            "INFO:tensorflow:\t dist = 2\n",
            "INFO:tensorflow:\t\t AverageReturn = -19.90\n",
            "INFO:tensorflow:\t\t predicted_dist = 10.6 (0.0)\n",
            "INFO:tensorflow:\t dist = 5\n",
            "INFO:tensorflow:\t\t AverageReturn = -20.00\n",
            "INFO:tensorflow:\t\t predicted_dist = 10.6 (0.0)\n",
            "INFO:tensorflow:\t dist = 10\n",
            "INFO:tensorflow:\t\t AverageReturn = -20.00\n",
            "INFO:tensorflow:\t\t predicted_dist = 10.6 (0.0)\n",
            "INFO:tensorflow:\t eval_time = 2.24\n",
            "INFO:tensorflow:step = 20\n",
            "INFO:tensorflow:\t dist = 2\n",
            "INFO:tensorflow:\t\t AverageReturn = -15.40\n",
            "INFO:tensorflow:\t\t predicted_dist = 10.7 (0.1)\n",
            "INFO:tensorflow:\t dist = 5\n",
            "INFO:tensorflow:\t\t AverageReturn = -20.00\n",
            "INFO:tensorflow:\t\t predicted_dist = 10.7 (0.1)\n",
            "INFO:tensorflow:\t dist = 10\n",
            "INFO:tensorflow:\t\t AverageReturn = -20.00\n",
            "INFO:tensorflow:\t\t predicted_dist = 10.7 (0.1)\n",
            "INFO:tensorflow:\t eval_time = 1.64\n",
            "INFO:tensorflow:step = 30\n",
            "INFO:tensorflow:\t dist = 2\n",
            "INFO:tensorflow:\t\t AverageReturn = -14.00\n",
            "INFO:tensorflow:\t\t predicted_dist = 11.1 (0.3)\n",
            "INFO:tensorflow:\t dist = 5\n",
            "INFO:tensorflow:\t\t AverageReturn = -20.00\n",
            "INFO:tensorflow:\t\t predicted_dist = 11.1 (0.2)\n",
            "INFO:tensorflow:\t dist = 10\n",
            "INFO:tensorflow:\t\t AverageReturn = -20.00\n",
            "INFO:tensorflow:\t\t predicted_dist = 11.1 (0.2)\n",
            "INFO:tensorflow:\t eval_time = 1.63\n",
            "INFO:tensorflow:step = 40\n",
            "INFO:tensorflow:\t dist = 2\n",
            "INFO:tensorflow:\t\t AverageReturn = -18.40\n",
            "INFO:tensorflow:\t\t predicted_dist = 11.1 (0.3)\n",
            "INFO:tensorflow:\t dist = 5\n",
            "INFO:tensorflow:\t\t AverageReturn = -20.00\n",
            "INFO:tensorflow:\t\t predicted_dist = 11.2 (0.3)\n",
            "INFO:tensorflow:\t dist = 10\n",
            "INFO:tensorflow:\t\t AverageReturn = -20.00\n",
            "INFO:tensorflow:\t\t predicted_dist = 11.1 (0.2)\n",
            "INFO:tensorflow:\t eval_time = 1.70\n",
            "INFO:tensorflow:step = 50\n",
            "INFO:tensorflow:\t dist = 2\n",
            "INFO:tensorflow:\t\t AverageReturn = -14.80\n",
            "INFO:tensorflow:\t\t predicted_dist = 11.3 (0.2)\n",
            "INFO:tensorflow:\t dist = 5\n",
            "INFO:tensorflow:\t\t AverageReturn = -17.80\n",
            "INFO:tensorflow:\t\t predicted_dist = 11.2 (0.2)\n",
            "INFO:tensorflow:\t dist = 10\n",
            "INFO:tensorflow:\t\t AverageReturn = -19.40\n",
            "INFO:tensorflow:\t\t predicted_dist = 11.2 (0.2)\n",
            "INFO:tensorflow:\t eval_time = 1.66\n",
            "INFO:tensorflow:step = 60\n",
            "INFO:tensorflow:\t dist = 2\n",
            "INFO:tensorflow:\t\t AverageReturn = -12.30\n",
            "INFO:tensorflow:\t\t predicted_dist = 11.3 (0.3)\n",
            "INFO:tensorflow:\t dist = 5\n",
            "INFO:tensorflow:\t\t AverageReturn = -20.00\n",
            "INFO:tensorflow:\t\t predicted_dist = 11.4 (0.3)\n",
            "INFO:tensorflow:\t dist = 10\n",
            "INFO:tensorflow:\t\t AverageReturn = -20.00\n",
            "INFO:tensorflow:\t\t predicted_dist = 11.4 (0.2)\n",
            "INFO:tensorflow:\t eval_time = 1.65\n",
            "INFO:tensorflow:step = 70\n",
            "INFO:tensorflow:\t dist = 2\n",
            "INFO:tensorflow:\t\t AverageReturn = -14.80\n",
            "INFO:tensorflow:\t\t predicted_dist = 11.3 (0.2)\n",
            "INFO:tensorflow:\t dist = 5\n",
            "INFO:tensorflow:\t\t AverageReturn = -20.00\n",
            "INFO:tensorflow:\t\t predicted_dist = 11.4 (0.2)\n",
            "INFO:tensorflow:\t dist = 10\n",
            "INFO:tensorflow:\t\t AverageReturn = -20.00\n",
            "INFO:tensorflow:\t\t predicted_dist = 11.3 (0.2)\n",
            "INFO:tensorflow:\t eval_time = 1.65\n",
            "INFO:tensorflow:step = 80\n",
            "INFO:tensorflow:\t dist = 2\n",
            "INFO:tensorflow:\t\t AverageReturn = -16.60\n",
            "INFO:tensorflow:\t\t predicted_dist = 11.4 (0.3)\n",
            "INFO:tensorflow:\t dist = 5\n",
            "INFO:tensorflow:\t\t AverageReturn = -20.00\n",
            "INFO:tensorflow:\t\t predicted_dist = 11.5 (0.3)\n",
            "INFO:tensorflow:\t dist = 10\n",
            "INFO:tensorflow:\t\t AverageReturn = -20.00\n",
            "INFO:tensorflow:\t\t predicted_dist = 11.2 (0.2)\n",
            "INFO:tensorflow:\t eval_time = 1.71\n",
            "INFO:tensorflow:step = 90\n",
            "INFO:tensorflow:\t dist = 2\n",
            "INFO:tensorflow:\t\t AverageReturn = -18.30\n",
            "INFO:tensorflow:\t\t predicted_dist = 11.3 (0.4)\n",
            "INFO:tensorflow:\t dist = 5\n",
            "INFO:tensorflow:\t\t AverageReturn = -18.60\n",
            "INFO:tensorflow:\t\t predicted_dist = 11.4 (0.3)\n",
            "INFO:tensorflow:\t dist = 10\n",
            "INFO:tensorflow:\t\t AverageReturn = -20.00\n",
            "INFO:tensorflow:\t\t predicted_dist = 11.3 (0.3)\n",
            "INFO:tensorflow:\t eval_time = 1.69\n",
            "INFO:tensorflow:step = 100\n",
            "INFO:tensorflow:\t dist = 2\n",
            "INFO:tensorflow:\t\t AverageReturn = -16.40\n",
            "INFO:tensorflow:\t\t predicted_dist = 11.5 (0.3)\n",
            "INFO:tensorflow:\t dist = 5\n",
            "INFO:tensorflow:\t\t AverageReturn = -20.00\n",
            "INFO:tensorflow:\t\t predicted_dist = 11.3 (0.3)\n",
            "INFO:tensorflow:\t dist = 10\n",
            "INFO:tensorflow:\t\t AverageReturn = -19.30\n",
            "INFO:tensorflow:\t\t predicted_dist = 11.3 (0.3)\n",
            "INFO:tensorflow:\t eval_time = 1.66\n",
            "\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "LossInfo(loss=<tf.Tensor: shape=(), dtype=float32, numpy=2.9389083>, extra=(<tf.Tensor: shape=(), dtype=float32, numpy=0.0005192029>, <tf.Tensor: shape=(), dtype=float32, numpy=2.938389>))"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 9
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab_type": "code",
        "id": "N6tOct1DKMyJ",
        "outputId": "3c583add-08ba-4a4d-c6f5-6b94ea2db804",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 1000,
          "referenced_widgets": [
            "925c0bcbbad949429ca9dd24e8554797",
            "c3c4d41bd37548638b72492bb52611c9",
            "b3e335e9111e468db195851f0a69609a",
            "8ea47bfd1b3e47078d832fab1b62e1a3",
            "3ea4276d16734c0a9edb73feb1421bf0",
            "1ce0d785f5344e67bb888ca005870d75",
            "7aeb56cce93c49238a4b629537dc46e8",
            "efa771ec08b4449e9fab37a03fa2b4f7"
          ]
        }
      },
      "source": [
        "# Second run of our agent with disabled training\n",
        "env_name2 = 'Maze11x11'  # Choose one of the environments shown above. Different from first one\n",
        "resize_factor2 = 5 # may need to change resize factor if first env too diff in size\n",
        "\n",
        "tf_env = env_load_fn(env_name2, max_episode_steps,\n",
        "                     resize_factor=resize_factor2,\n",
        "                     terminate_on_timeout=False)\n",
        "eval_tf_env = env_load_fn(env_name2, max_episode_steps,\n",
        "                          resize_factor=resize_factor2,\n",
        "                          terminate_on_timeout=True)\n",
        "agent.set_trainable_layers(1) \n",
        "# limit trainable layers from above cell, train again.\n",
        "train_eval(\n",
        "    agent,\n",
        "    tf_env,\n",
        "    eval_tf_env,\n",
        "    initial_collect_steps=1000,\n",
        "    eval_interval=10, # 1000\n",
        "    num_eval_episodes=10,\n",
        "    num_iterations=100, # 30000\n",
        ")\n",
        "\n"
      ],
      "execution_count": 10,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "/usr/local/lib/python3.6/dist-packages/gym/logger.py:30: UserWarning: \u001b[33mWARN: Box bound precision lowered by casting to float32\u001b[0m\n",
            "  warnings.warn(colorize('%s: %s'%('WARN', msg % args), 'yellow'))\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "random_seed = 0\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "/usr/local/lib/python3.6/dist-packages/ipykernel_launcher.py:65: TqdmDeprecationWarning: Please use `tqdm.notebook.trange` instead of `tqdm.tnrange`\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "display_data",
          "data": {
            "application/vnd.jupyter.widget-view+json": {
              "model_id": "925c0bcbbad949429ca9dd24e8554797",
              "version_minor": 0,
              "version_major": 2
            },
            "text/plain": [
              "HBox(children=(FloatProgress(value=0.0), HTML(value='')))"
            ]
          },
          "metadata": {
            "tags": []
          }
        },
        {
          "output_type": "stream",
          "text": [
            "INFO:tensorflow:step = 110\n",
            "INFO:tensorflow:\t dist = 2\n",
            "INFO:tensorflow:\t\t AverageReturn = -16.30\n",
            "INFO:tensorflow:\t\t predicted_dist = 11.4 (0.3)\n",
            "INFO:tensorflow:\t dist = 5\n",
            "INFO:tensorflow:\t\t AverageReturn = -20.00\n",
            "INFO:tensorflow:\t\t predicted_dist = 11.5 (0.3)\n",
            "INFO:tensorflow:\t dist = 10\n",
            "INFO:tensorflow:\t\t AverageReturn = -18.50\n",
            "INFO:tensorflow:\t\t predicted_dist = 11.4 (0.2)\n",
            "INFO:tensorflow:\t eval_time = 1.77\n",
            "INFO:tensorflow:step = 120\n",
            "INFO:tensorflow:\t dist = 2\n",
            "INFO:tensorflow:\t\t AverageReturn = -16.60\n",
            "INFO:tensorflow:\t\t predicted_dist = 11.6 (0.4)\n",
            "INFO:tensorflow:\t dist = 5\n",
            "INFO:tensorflow:\t\t AverageReturn = -17.50\n",
            "INFO:tensorflow:\t\t predicted_dist = 11.5 (0.3)\n",
            "INFO:tensorflow:\t dist = 10\n",
            "INFO:tensorflow:\t\t AverageReturn = -19.50\n",
            "INFO:tensorflow:\t\t predicted_dist = 11.7 (0.2)\n",
            "INFO:tensorflow:\t eval_time = 1.66\n",
            "INFO:tensorflow:step = 130\n",
            "INFO:tensorflow:\t dist = 2\n",
            "INFO:tensorflow:\t\t AverageReturn = -16.40\n",
            "INFO:tensorflow:\t\t predicted_dist = 11.4 (0.3)\n",
            "INFO:tensorflow:\t dist = 5\n",
            "INFO:tensorflow:\t\t AverageReturn = -20.00\n",
            "INFO:tensorflow:\t\t predicted_dist = 11.5 (0.3)\n",
            "INFO:tensorflow:\t dist = 10\n",
            "INFO:tensorflow:\t\t AverageReturn = -19.30\n",
            "INFO:tensorflow:\t\t predicted_dist = 11.5 (0.3)\n",
            "INFO:tensorflow:\t eval_time = 1.71\n",
            "INFO:tensorflow:step = 140\n",
            "INFO:tensorflow:\t dist = 2\n",
            "INFO:tensorflow:\t\t AverageReturn = -20.00\n",
            "INFO:tensorflow:\t\t predicted_dist = 11.4 (0.3)\n",
            "INFO:tensorflow:\t dist = 5\n",
            "INFO:tensorflow:\t\t AverageReturn = -17.30\n",
            "INFO:tensorflow:\t\t predicted_dist = 11.7 (0.2)\n",
            "INFO:tensorflow:\t dist = 10\n",
            "INFO:tensorflow:\t\t AverageReturn = -19.60\n",
            "INFO:tensorflow:\t\t predicted_dist = 11.8 (0.3)\n",
            "INFO:tensorflow:\t eval_time = 1.72\n",
            "INFO:tensorflow:step = 150\n",
            "INFO:tensorflow:\t dist = 2\n",
            "INFO:tensorflow:\t\t AverageReturn = -13.60\n",
            "INFO:tensorflow:\t\t predicted_dist = 11.5 (0.4)\n",
            "INFO:tensorflow:\t dist = 5\n",
            "INFO:tensorflow:\t\t AverageReturn = -18.50\n",
            "INFO:tensorflow:\t\t predicted_dist = 11.6 (0.2)\n",
            "INFO:tensorflow:\t dist = 10\n",
            "INFO:tensorflow:\t\t AverageReturn = -20.00\n",
            "INFO:tensorflow:\t\t predicted_dist = 11.6 (0.3)\n",
            "INFO:tensorflow:\t eval_time = 1.67\n",
            "INFO:tensorflow:step = 160\n",
            "INFO:tensorflow:\t dist = 2\n",
            "INFO:tensorflow:\t\t AverageReturn = -16.50\n",
            "INFO:tensorflow:\t\t predicted_dist = 11.8 (0.3)\n",
            "INFO:tensorflow:\t dist = 5\n",
            "INFO:tensorflow:\t\t AverageReturn = -17.40\n",
            "INFO:tensorflow:\t\t predicted_dist = 11.8 (0.2)\n",
            "INFO:tensorflow:\t dist = 10\n",
            "INFO:tensorflow:\t\t AverageReturn = -19.50\n",
            "INFO:tensorflow:\t\t predicted_dist = 11.9 (0.3)\n",
            "INFO:tensorflow:\t eval_time = 1.72\n",
            "INFO:tensorflow:step = 170\n",
            "INFO:tensorflow:\t dist = 2\n",
            "INFO:tensorflow:\t\t AverageReturn = -14.60\n",
            "INFO:tensorflow:\t\t predicted_dist = 11.9 (0.2)\n",
            "INFO:tensorflow:\t dist = 5\n",
            "INFO:tensorflow:\t\t AverageReturn = -18.70\n",
            "INFO:tensorflow:\t\t predicted_dist = 11.7 (0.3)\n",
            "INFO:tensorflow:\t dist = 10\n",
            "INFO:tensorflow:\t\t AverageReturn = -20.00\n",
            "INFO:tensorflow:\t\t predicted_dist = 11.8 (0.3)\n",
            "INFO:tensorflow:\t eval_time = 1.69\n",
            "INFO:tensorflow:step = 180\n",
            "INFO:tensorflow:\t dist = 2\n",
            "INFO:tensorflow:\t\t AverageReturn = -20.00\n",
            "INFO:tensorflow:\t\t predicted_dist = 12.0 (0.2)\n",
            "INFO:tensorflow:\t dist = 5\n",
            "INFO:tensorflow:\t\t AverageReturn = -17.30\n",
            "INFO:tensorflow:\t\t predicted_dist = 12.0 (0.3)\n",
            "INFO:tensorflow:\t dist = 10\n",
            "INFO:tensorflow:\t\t AverageReturn = -19.20\n",
            "INFO:tensorflow:\t\t predicted_dist = 12.0 (0.3)\n",
            "INFO:tensorflow:\t eval_time = 1.71\n",
            "INFO:tensorflow:step = 190\n",
            "INFO:tensorflow:\t dist = 2\n",
            "INFO:tensorflow:\t\t AverageReturn = -16.20\n",
            "INFO:tensorflow:\t\t predicted_dist = 11.8 (0.3)\n",
            "INFO:tensorflow:\t dist = 5\n",
            "INFO:tensorflow:\t\t AverageReturn = -15.30\n",
            "INFO:tensorflow:\t\t predicted_dist = 12.0 (0.4)\n",
            "INFO:tensorflow:\t dist = 10\n",
            "INFO:tensorflow:\t\t AverageReturn = -19.10\n",
            "INFO:tensorflow:\t\t predicted_dist = 12.0 (0.3)\n",
            "INFO:tensorflow:\t eval_time = 1.63\n",
            "INFO:tensorflow:step = 200\n",
            "INFO:tensorflow:\t dist = 2\n",
            "INFO:tensorflow:\t\t AverageReturn = -14.50\n",
            "INFO:tensorflow:\t\t predicted_dist = 11.9 (0.3)\n",
            "INFO:tensorflow:\t dist = 5\n",
            "INFO:tensorflow:\t\t AverageReturn = -20.00\n",
            "INFO:tensorflow:\t\t predicted_dist = 12.0 (0.4)\n",
            "INFO:tensorflow:\t dist = 10\n",
            "INFO:tensorflow:\t\t AverageReturn = -19.20\n",
            "INFO:tensorflow:\t\t predicted_dist = 12.0 (0.4)\n",
            "INFO:tensorflow:\t eval_time = 1.71\n",
            "\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "LossInfo(loss=<tf.Tensor: shape=(), dtype=float32, numpy=2.9713323>, extra=(<tf.Tensor: shape=(), dtype=float32, numpy=0.0053511593>, <tf.Tensor: shape=(), dtype=float32, numpy=2.9659812>))"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 10
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "colab_type": "text",
        "id": "KxFFTj3Ynv03"
      },
      "source": [
        "-------\n",
        "## Visualization\n",
        "Now, let's visualize some rollouts from the learned policy. Below, you can change the difficulty of the task, which moves the goals closer or further from the starting location. You can change the distance to the goal using the slider below. Notice that, if the goals are nearby, the agent can always reach them. If the goals are far away, the agent rarely reaches them. If only we could lay down a set of \"breadcrumbs\" that led the agent to the goal..."
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "cellView": "form",
        "colab_type": "code",
        "id": "ZVmiN6xpvNaL",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 440,
          "referenced_widgets": [
            "35d832b8dac443bab1a5fed1ca2db46c",
            "80010960ba4c4f878d112a386f899af9",
            "6805fe15923e4f13b93ce9a538b2ed30",
            "59bf5cbc0d044cd48e29f3c43c370d8f",
            "a5358168528641edb8c848fbda609a78",
            "8e57a6dc5abe48088bbd7f2295921d95",
            "cf237d1de3e54d6caf1316a7a5592659",
            "4b318a99f7a948a7923fe72426607271",
            "9de12f52414c46d0a92e5808f9c4535e",
            "93f0c6efb59649c6a7c9d9ebaec5aa9b",
            "fb8455b5af524243bda1fdb6c268bbd5",
            "43da1dc528d54982b4484dffbee77b0f",
            "f3d7366b832a4a54bc8d832d4abbd0d0",
            "7e1bfa38e5be408db65068234c796988",
            "f9f0810f26d9410f97cb5043d967e885",
            "3c49fdcb269849fab641530b40fecd57"
          ]
        },
        "outputId": "7d321feb-9f81-4e73-e54e-a666ba6a2c8d"
      },
      "source": [
        "#@title Visualize rollouts. {run: \"auto\" }\n",
        "eval_tf_env.pyenv.envs[0]._duration = 100  # We'll give the agent lots of time to try to find the goal.\n",
        "difficulty = 0.6 #@param {min:0, max: 1, step: 0.1, type:\"slider\"}\n",
        "max_goal_dist = eval_tf_env.pyenv.envs[0].gym.max_goal_dist\n",
        "eval_tf_env.pyenv.envs[0].gym.set_sample_goal_args(\n",
        "    prob_constraint=1.0,\n",
        "    min_dist=max(0, max_goal_dist * (difficulty - 0.05)),\n",
        "    max_dist=max_goal_dist * (difficulty + 0.05))\n",
        "\n",
        "\n",
        "def get_rollout(tf_env, policy, seed=None):\n",
        "  np.random.seed(seed)  # Use the same task for both policies.\n",
        "  obs_vec = []\n",
        "  waypoint_vec = []\n",
        "  ts = tf_env.reset()\n",
        "  goal = ts.observation['goal'].numpy()[0]\n",
        "  for _ in tqdm.tnrange(tf_env.pyenv.envs[0]._duration):\n",
        "    obs_vec.append(ts.observation['observation'].numpy()[0])\n",
        "    action = policy.action(ts)\n",
        "    waypoint_vec.append(ts.observation['goal'].numpy()[0])\n",
        "    ts = tf_env.step(action)\n",
        "    if ts.is_last():\n",
        "      break\n",
        "  obs_vec.append(ts.observation['observation'].numpy()[0])\n",
        "  obs_vec = np.array(obs_vec)\n",
        "  waypoint_vec = np.array(waypoint_vec)\n",
        "  return obs_vec, goal, waypoint_vec\n",
        "\n",
        "plt.figure(figsize=(8, 4))\n",
        "for col_index in range(2):\n",
        "  plt.subplot(1, 2, col_index + 1)\n",
        "  plot_walls(eval_tf_env.pyenv.envs[0].env.walls)\n",
        "  obs_vec, goal, _ = get_rollout(eval_tf_env, agent.policy)\n",
        "\n",
        "  plt.plot(obs_vec[:, 0], obs_vec[:, 1], 'b-o', alpha=0.3)\n",
        "  plt.scatter([obs_vec[0, 0]], [obs_vec[0, 1]], marker='+',\n",
        "              color='red', s=200, label='start')\n",
        "  plt.scatter([obs_vec[-1, 0]], [obs_vec[-1, 1]], marker='+',\n",
        "              color='green', s=200, label='end')\n",
        "  plt.scatter([goal[0]], [goal[1]], marker='*',\n",
        "              color='green', s=200, label='goal')\n",
        "  if col_index == 0:\n",
        "    plt.legend(loc='lower left', bbox_to_anchor=(0.3, 1), ncol=3, fontsize=16)\n",
        "plt.show()"
      ],
      "execution_count": 11,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "/usr/local/lib/python3.6/dist-packages/ipykernel_launcher.py:17: TqdmDeprecationWarning: Please use `tqdm.notebook.trange` instead of `tqdm.tnrange`\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "display_data",
          "data": {
            "application/vnd.jupyter.widget-view+json": {
              "model_id": "35d832b8dac443bab1a5fed1ca2db46c",
              "version_minor": 0,
              "version_major": 2
            },
            "text/plain": [
              "HBox(children=(FloatProgress(value=0.0), HTML(value='')))"
            ]
          },
          "metadata": {
            "tags": []
          }
        },
        {
          "output_type": "stream",
          "text": [
            "\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "/usr/local/lib/python3.6/dist-packages/ipykernel_launcher.py:17: TqdmDeprecationWarning: Please use `tqdm.notebook.trange` instead of `tqdm.tnrange`\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "display_data",
          "data": {
            "application/vnd.jupyter.widget-view+json": {
              "model_id": "9de12f52414c46d0a92e5808f9c4535e",
              "version_minor": 0,
              "version_major": 2
            },
            "text/plain": [
              "HBox(children=(FloatProgress(value=0.0), HTML(value='')))"
            ]
          },
          "metadata": {
            "tags": []
          }
        },
        {
          "output_type": "stream",
          "text": [
            "\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "display_data",
          "data": {
            "image/png": "iVBORw0KGgoAAAANSUhEUgAAAdAAAAEPCAYAAADyNZX6AAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4yLjEsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+j8jraAAAgAElEQVR4nO3de3RU5b038O+ey55Lksk95DKEACHhHgwERUmhKAi8XopoLau1crRaXQvbdVitmPN2iXm1BOxbEE+XWo/Lg8eeY7EsFY++lYNUrFSxSCTcJNyaQBISJjcmk8x99vvH4+RGApmdTGaSfD9rZU1m9s6e3+zJs7/zPPsykqIoICIiotBoIl0AERHRSMQAJSIiUoEBSkREpAIDlIiISAUGKBERkQoMUCIiIhV0kS5gNCgvL79dp9NtVBQlHfxQQkQjX0CSpHqfz1daWFi4J9LFRCuJ54EOTnl5+e0Gg+F3OTk5HpPJ5NJoNFyhRDSiBQIByel0GquqqmS3272OIdo39pYGSafTbczJyfHExMQ4GZ5ENBpoNBolJibGmZOT49HpdBsjXU+0YoAOkqIo6SaTyRXpOoiIhprJZHJ9u2uK+sAAHTwNe55ENBp9u21jTvSDK4aIiEgFBigREQB/wB/pEmiEYYBGo/nz8zF/fn6ky+jL559/blq/fn1mQ0ODdqiX/cEHH8StX78+0+8fHRuy+f82P3/+v0Xn+zhUVq9enZOVlTUr0nUMVr2jXpvx24zZDY6h/7+OVqPlvYskBiiF5KuvvjJv27Ytw2azDfk5xH/5y1/itm3blhEIBIZ60UTXtPP4zgRbh02/88TOhEjXQiMHA5Qizu12SwxNiqQ/HP1DcvdbooFggNJVjh49ali6dOnkpKSkAoPBUJiRkTFrxYoVk7Zu3Zry85//PAcAZs2aNVOSpLmSJM2trKyUAWDTpk2pc+bMmRofHz8nLi5uTkFBwdQ//vGP8d2XXVlZKUuSNHfz5s2pjz32mDUtLW22yWQqfPjhh8dv27YtAwBkWZ4bXPawv/hR5osvvjAtWbIk12KxzDEajYWFhYVTP/roo9jg9NWrV+eMGzdu9t/+9jfT3Llz800m0w0TJkyY+fzzz6f2Xtbu3bvjpk+fPs1gMBSOHz9+5m9+85uU4X014dHsbNZUNFTEAsCR+iOxzc7miGwXf//73ydNnDhxhsFgKMzLy5v+n//5n/Hz58/Pn99td05FRYVh6dKlk+Pi4uYYjcbCgoKCqbt27bJ0X87x48cN3/ve9yZmZWXNMhqNhVarddYPf/jDbJvNNmaGp4cLL+VHV7nzzjunWCwW/9atW6tTU1N9Fy9elP/85z/H33fffa3V1dXyiy++mPH666+fz87O9gBAdna2FwCqqqoMDz74YOOkSZPcPp9P2r17d8KaNWtydTrdmXvvvdfe/Tm2bt2aMXv27Pbt27dX+/1+6aabbmrv6OjQvP322yl79uw5pdWyrQ/WgQMHzEuXLs2fPn16x/bt26vNZnPg97//fepdd92Vt2/fvlPFxcUdANDe3q594IEHJj3++OMNubm5l15//fXkDRs2ZE+bNs115513tgFAeXm58b777psyc+bMjtdee+28y+WSysrKMjs6OrRarXbEnMZla7dpz7aclbs/tufsHoteq1e8Aa+k1+qVF798MfX23Nt7/L/mJuZ6UmNSw7Zz/t1337U8/vjjE5csWdK6efPmmsuXL+s2bNiQ7fF4pJycHDcAVFVV6RcvXjw1JiYmsGXLlgsJCQn+l19+Oe3++++f8tZbb535/ve/bweAixcv6q1Wq+fee++9mJyc7Dtz5oxh69atGUuXLjUfOXLkVLhew1jES/kNUkVFRVVBQUHjkC40+Inz73+vHNLlDsClS5d0mZmZBX/4wx/O/vCHP7zSe/qLL76Y/POf/zzn2LFjx2fOnOnubzl+vx+BQACLFy+eYjQaA/v27TsHiB7o1KlTZ02bNq3j+PHj32g0XR/2169fn7lt27YMj8dzWK/Xh+X1DafgAUR/f2T430cAWLBgQV5DQ4P+5MmTJ41GowIAPp8PeXl5MyZNmuT6+OOPz61evTrnnXfeSX7//fdPB8PS6XRKGRkZBStWrGh56623qgHgrrvumvjZZ59Zqqurj1kslgAAnD17Vj9t2rRZaWlp3tra2mOReI2hevyDx7NeOfxKuqyVFb1G37nfoN3b3vmJLUYf0xmU3oBX4/F7pMfmPlb/8h0v14arrhtuuGGqw+HQVlZWngi2ic8++8z8ne98Z1pRUZHj73//e+Wjjz5qff3118cdOXKks+35fD7k5ubONJvN/pMnT37T17K9Xi/+8pe/xC5fvjz/wIEDJ2+55RYnIEYfDh48GHe9966ioiKloKAgZ2hf8ejAHmgk9Xek7aFDsdecHsZgHTdunM9qtbqffvppa319vX7ZsmVts2bN6jcou/vss8/MTz/9dObRo0djWlpadMEPZzk5OVddqWnlypWt3cNzJOvvSNtDdeJ97G96OIPV4XBIhw4dilu3bt0lrVareL3ezmnFxcX29957r3Nfn9FoDATDEwBMJpOSk5Pjqqmp6eyplZeXxy5evPhKMDwBIDc311tYWOioqakxhOt1DLXfrfxdbbwx3v/CwRcyuodmd90fN2gNgQ23bKj79ZJf14erJp/Ph+PHj5vXrVtX371NFBcXd2RlZXmC97/44ou4goICR/cPrjqdDvfcc0/zCy+8kNHc3KxJSkoKuFwu6Zlnnhm3c+fO5EuXLhncbrcUnP/EiRPGYIDS4I2OLRgNGY1Ggz179pyePXt2+3PPPZc1e/bsmVarddaWLVuu2ifW3dmzZ/UrV67Ma21t1W3ZsuXC3r17T3366affFBcX2z0ez1X/ZxkZGd6+lkNDw2az6fx+P7Zv354hy/Lc7j//8R//kWa327XB04UsFstVQ5OyLCtut7vzfWtsbNSnpaX5es+Xmpp61WPRTKvRYvNtm+v//KM/Vyabkr2yVu7z6DVZKysp5hTvnh/tqdx82+Z6rSZ8uxQuXbqk8/l8Ulpa2lVtIiUlpfOxK1euaPuaJz093asoChobG3UA8MQTT2T99re/zbzvvvua33777TP79+//5o033jgHAC6Xi9v8IcQeaCT115OM4BAuAEyfPt3z7rvvVgUCARw8eNC0ffv2tKeeeip74sSJ/fZEd+/eHe9wOLS7du06N3ny5M5G7nQ6+2ywkiT19fCI1F9PMpJDuMnJyX6NRoMHHnjg8kMPPdTU1zyh7GdOSUnxXr58+artRThOZxoO3835bsfpJ04fn/3y7Bm1bbVy7+mp5lTv0cePnkgyJYX98PCMjAyfTqdTLl++fNV+i8bGRn1mZqYHAOLj4/19zVNfX6+XJAkpKSk+ANi9e3fSPffc0/T8889fCs7z3//93zyoIAz4aYT6pdFocPPNNztffvnliwBw7Ngxk8FgUACgo6Ojx/9O8L4sy5071Y8ePWr4+uuvYzFABoMhAAAOh4P/l4NksVgCc+fOdZw4ccJ8yy23dHznO9+56ieU5RUWFjr2798fb7fbO9+bs2fP6svLywf8/kYbo86oNHY09rmzvbGjUW/SmYblABGdToeZM2d2fPDBB4ndT+f67LPPzLW1XeF+8803t1VUVMQEj3oHxPDve++9lzht2rSOpCQR9i6XS6PT6XrU/vrrr/P0nDAYkZ8eKXy+/PJL089+9rPxq1evbsnLy3P5/X5px44dyVqtVlm2bFlbsGG+8MILqQ899FCTLMvK/PnznStWrLCXlpYqa9asmbh+/fqG2tpa/ebNmzPT09M9Az1QbcaMGS4AePbZZ9PvuOOOKzqdTgl1Q09dtm7denHZsmX5xcXFU9auXduYlZXltdlsusOHD8f4/X689NJLAz4o5plnnrl04403Ji5evDjvn//5n+vdbrdUVlaWmZycPKKGcLt779R7Fp1Gp7j9bsmoMwb8Ab+klbSKy+/S6DQ65b3K9yxrZq656kC6cHj66afr7rnnninLli2b/MgjjzTabDbd5s2bM1NSUrzBL6soKSlpePvtt5OXLVuWV1JSUhcfH+9/5ZVXUqurq407d+48E1zWokWLrrzzzjvJmzdvdubl5bl37dqVcPjw4RH7QSea8ZM+9WC1Wr1ZWVmel156adyaNWtyH3rooUn19fXyzp07zxYXF3csWLDAuX79+rqPP/444fbbb5+6aNGiadXV1fp58+a5XnnllX/U1tbKa9asyd2+fXv6xo0ba2688ca26z+r8IMf/KD1Rz/6kW3Hjh2pt91229RFixZNC+drHe0WLlzYceDAgW+SkpL8Tz31VPaqVavyNmzYkH3ixAnTokWLHKEsq7Cw0PWnP/3pjNPp1Dz88MOTnnnmGetjjz3WcMstt9iv/9fR6c2jbya3e9s1Rp0xcP+M+xttv7Qd+f7M7zcatcZAu7dd84eK4buowqpVq+wvv/zyP86ePWt64IEHJr/wwgvpmzZtupiSkuKLi4vzA0BOTo53//79p6ZMmeL85S9/mb127drJra2tup07d/Y4TezVV1+9eOutt17ZtGlT1tq1ayc5HA7tm2++eX64XstYwtNYBmm0ncZCQyfSp7FQ/1w+l5S4JXGOVtLije+9cW719NWdAbTr5C7L2vfWTg4oAbRsaDli0BkispE8d+6cfvr06bN+9rOfXfrNb35z6fp/ER48jaV/HMIlojHHF/Dh3mn3Nm26bVPdeMv4HsPQ906/136j9cZj//Lxv2R6A14YEP6zdBwOh/TTn/50/NKlS+2pqam+s2fPGrZv355uNBoD69atG9oP6DRk2AMdpLD0QIloTHG5XNKdd9456ciRIzGtra06k8kUmDdvnmPLli01RUVFV51HPZzYA+0fe6BERBFmNBqVvXv3not0HRQaHkRERESkAgOUiIhIBQbo4AUCgcDouawOEdG3vt228ct6+8EAHSRJkuqdTqcx0nUQEQ01p9NplCQpbBfSH+kYoIPk8/lKq6qq5Pb2dhN7okQ0GgQCAam9vd1UVVUl+3y+0kjXE614GssQKC8vv12n021UFCUd/FBCRCNfQJKkep/PV1pYWLgn0sVEKwYoERGRCuwtERERqcAAJSIiUoEBSkREpAIDlIiISAUGKBERkQoMUCIiIhVC+jaWlJQUJScnJ0ylEI0ehw8fblQUJTXSdfRHklIUIKfzviwDMTHid40GCAS6fgd63u/oAMxmwOXqmtdo7Ho8eBsUvB8IiOdob+/6O7cb8PsBSQKCZ9QFAuJ3g6FnPVotkJwMtLUBXq+Ypvt2C+bziWWZzWJZTqdYhiyLeaVulziR5atfh9st5tFqAY9H3CqKmC8hQdw6HEBqqqjF6xX3g39jNvd8Xd3XndMp5pOkrmlarfhdlge2zruvO4+n+/so6nR9+4Vner247/f3fL2xsWIZHo9YZ8G69XqxHjQaUWfv981k6nq8o0P8ndcrlhlcv1qteNznE/8HwXUbfB+Dr1dRxK3f3/V8RqNY98H3UK8X8yQmivlTo6AFXasthxSgOTk5+Oqrr4amKqJRTJKk6kjXcG05AL4EoAUAzJ4NLF0qprS2itAAujZuwTBrbQUaGoCcHODCBREisbFAdjZQVQWMG9c1PbjRDT6ekAAsWAB88UXXc1RWAufPiw2q8dsLYtbUiPvz5on7DofY2LrdQG4ucOYMEB8PzJolNrJmM1BeLsIlMVEExPnzXUEJiPAxGkWAFBZe/ToOHxbzpaUBFy+KDbkkib/LzRXPf/EicPPN4vmsVnFfoxEb/uXLe76u7uuuqkqEjtcr1hUgatXpgJkzB7bOu6+7w4fFNK1WLNvnE7UAYr3ExgJ2e1fwTpoEWCxAfX1X7c3NYl0tXSrWZ/f10ft96/1+t7aKv/X5xO9ms3gurRa4805g4kTgo49EPcGgliRRV0uLqLH78506BRw7JtZ1Xp54TxMTgeJi4Cc/CemfOiyu1ZY5hEtEiIsDJk8G0tOBpiZxO3ky0Ngo7nefNnu22JCaTKI3aDSK+7Nn95ze1NTz8fR0sUHv/hwLFogNZmurWJ5eL+YxmURoxMeLDX9Li9gAt7V19aDi4rqep61NBF1qqgi5G28UyzAagblzuzb2kyb1/TosFtEzamsDMjPF436/mP/0aaCuTmzsa2tFPSkpXUEcrLm/dXfTTSLMg+GRkCACzGAY+Drvvu5kWfyYTGJac7MIMZ1OBHNyMpCUJEI/JkZ80HA4RCjGxooPCampXR9geq+P3u9b7/c7MxOw2cTr8Xi6PpgEe/2BgFgn7e1i3c+ZI3rIzc3iPev9fA6HCFmtVgR/enrPEYloFtKViObNm6ewB0p0fZIkHVYUZV6k6+iPJM1TRA9UAbQujCv+H1iyz0NvaYWceBlP3/AqbDbRs1CUruE0q1X0EE+fFhvEYC8nMRHIz796eu/Hbbaey7HZRAgdOwZUV4uN9Ny5YuN+/rzYkDocIjAaGkRQLFwoXkNtrQiK5mbxe0oKUFAgbhsbgQ8/FBvu7GzRq3M6xUa7rzo1GrGMqirxnCkp4rmqq8Vrz80V4WeziVuLRQRTVZUIpN7rp/e602hEaFRVifuTJvUcpuzv73pPC96vqAD+9CfgyhXRszUau4ZMXS4RnHY7MH26CNRTp8TtzJnAtGki3E6fFj3X1auv/74FHw8G25kzXUPdsizmTU0VHzQmTxavob5e9EZjY8V6PH5cvO7ey/3kE/G+NzaKDxVz5oga/X7g0Ucj10aCrtWWGaBEYRD1Aaq5QYF+HyTFCK25FRPv/i9kfveDzun71+6PXHG9vPqqGEpsbBRBYbGIsGho6NrA1tQAu3eL4I2NFaHbO9yKisSGO9TnTkwUw5dpaaIn2z141CxzqATXS2ur6H16PH2Hos0GnDsHZGWJQAuy28WHhFWr1D+33y+WP26cCNfe78mhQ13B39+6evdd0TuWZeCbb8RjMTHA5csijNW+d0PlWm05pH2gRDRKKBK0OgkT0s0wm834P9/7BVat+kWkq+pTaqoIxLS0rseCB/QEWa3A3XeLDXZDg5j24IOD3+impoqNe15e12OyDMyfry54hlJwvSQldT3Wu7aiInEb/IBht3d9wGhtBRYtGtxzWyxiSBcQy+79ngxk/RcVdX34mTIF+OtfRZDedZcIZodDTL/77siFaH+4D5RoDNLIbhhSxNc8Tp7ctaGNRkVFYmNvt4uep90u7veu2WoVwfHoo+J2KDa2A33uSAiltuAHDLO5ayh8MIE0lOule21tbV1Dwm63+PF4xJDxr38teqs1NepqDgcO4RKFQbQP4Wr0sxRj+jtYNncK/vf/jo5AuJaBDgeOtue+ntG4Xl59VQzVnzsnQvnCha4h4oULxWPD2RvlEC4R9SDHt2Li997A7zY8FzVhcC0DHQ4cbc99PaNxvQSHzQsKgL17uw5Iio3tOk/10KHoeE84hEs0BunjWpE690BUbISIugsODwcvQDF+vDj9JTlZHJkdEyN6vdGAATrKBJRApEsgIlKt+z7R4NWYliwRpw1d+O2f8N5P/x++/jo69ocyQEeRpo4mjN82Hs3O5kiXQlEuPzk/qk5VIeoueEDYr34ljsyVZdEbPe/KxEV3GvLzxTDv7t2RDVEG6CjyfuX7qGurw/uV70e6FCKiQeveGz10CJhkqsO82ErY7WJYNyFBPB4pDNBRZEfFDnF7ZEdE6yAiGirB3mhhIbAk4WtkGRs7h3ZjYyO7P5RH4Y4SdrcdB2sOAgAO1hyE3W2HxWCJcFVERENDowE+aZ2Danc6LP8AJkwQQ7uR/MYWBugI1OJswYUrF3o8tu8f+2DQGuDxeyBrZbxW/hpunXhrj3my47ORaEoczlKJiAatpkacytLsi4OiiP2h+/eLK0Q9+GDk6mKAjkCbD2zG858/D6POCFkrdz7e5mnrvC39tBSln5YCADx+D1w+F568+UlsWbolIjUTEQ3Y4sU97h6yLcREvwHxly/hr1gE6aMPkaB1IO3zKlj3/rlrxv37h7VMBugIVHZbGRKMCXj2r8/C7rb3OU/3x006EzYt2YQNCzcMV4lEREPG5k3AOH0zZFzGd7Ef6cnJ0CKABm/S9f84jBigI5BG0qCkuAS3TroVd//xbrS6WuHyua6az6A1IMmUhN0/2I2irCi/VhsRUVCvnmTqu4CjA7CUlWA8AJQ9Jy5ebwYQwS9B4FG4I9j8rPmoXFeJjNiMPqdnxmXi1LpTDE8iGtE6L17vMyOgSFFzUf+QLiZvtVqVRx55JIzlUKi88GILtsAH31XTdNDhKTwFHQcaBkSWZZSUlAzJsqL9YvJsyzTSXLkSh/w3T6EpkIyK2wqQlVWH+Pi2fucfqvY8ZBeTDwR4mbhocx7nofl2IEEPPQIIQIIEH3zQQotzOId85Ee4ypHB4/FEuoRhw7ZMI018fBtuj/kfAIB3uvG68w9He+YQ7ghXgQp44IEOOtyAG/AknkQhCqGDDm64cRRHI10iEdGoxLG9EcwHH87gDAww4D7ch1zkAgBWYiWmYAp2YRdO4zT88EMLbYSrJSIaXRigI5gCBQUowGIsRixie0ybgil4Ak9gP/YjgAADlIhGvDf+6Z8iXUIPDNARTA897sAd/U6PRew1pxMRkXrcB0pERKQCA5SIiEgFBigREZEKDFAiIiIVGKBEREQqMECJiIhUYIASERGpEFKANqEJ/45/D1ctREREIwZ7oERERCowQImIiFQI6VJ+nkuzUP3Ml3gGHtx66wEkJzdd9zvZiIiIRiOV18KVsW/fEiQl2RAT40R29kVkZdUxTImIaMwY1MXkm5uT4fG0AwDi4trgcOQjP7+SIUpERKPeIPeBauDw+FHX7sfnjn/AaHShtjZzaCojorDRaHj4A9FgDfrrzCSNHoYYH2LkTPz4x/egoQF49NGhKI2iUWlpaaRLoCGQnp6OjRs3RroMiiC25cFTGaBdX86cHGtBQZ4F8+dPhcMBpKYOUWVERERRbNDjOKmpwLhxQHo60NoKFBUNRVlERETRbVBDuIsXA1OnAhMmAOPHi/C0WoeoMiIioiimIkAD0Gq1yMgAPvlk6AsiIiIaCUIbwpX80MW3wGoFJk0KU0VEREQjQIgBqkCj8yAmBli5MkwVERERjQAhDeHKOh2mWa1ISQGWLAlXSURERNEvpAAN9jzT04GaGh5xS0REY1dIQ7guF1BZCXg8gM0WrpKIiIiiX0gB6vUC5eXAjh3inE8iIqKxKqQAlSRAUYD6euDLL8NVEhERUfQLaR+oogAmk9gX+s034SqJiIgo+oXUAw0EALtdDOUSERGNZSFfC7e1Faiu5kXjiYhobAspQBVF7Ac1mQCnU5zKQkRENBaFFKBaLZCQAMyZA+j1wKFD4SqLiIgouoV0EJHZDEycCBgMQGIizwUlIqKxK6QeqNstTmFpaRG9UO4HJSKisSqkHmggIHqhKSmATqf+Un5lZWXweDzq/phCJssySkpKhmxZfO+oO7bn4TVU7ZltefBCCtDYWGDhQrH/My1N/Zdn800bXkO5vocqiIdSaWlppEsY09ieh9dQrW+25cELOUCXLxc90YaGcJVEREQU/ULaB9rUJK6DW1HB/Z9ERDS2hRSgHo84deW//ksM4xIREY1VIV+JSKcTF1F4++1wlENERDQyhLQPVKcDsrKA9nbgq6/CVRIREVH0C3kI9/x5wOEIVzlEREQjQ8jXwrXbgYsXgezscJVEREQU/UIawgUAjUYcQBQT0/88NTXiYCObTRytW1Sk/pxRIiKiaBRSgGq1QEaGCEWbrSsoT58GmpvF9XFTUsTl/qxWcbGF9nZg927g7rsZokRENHqEfDH5ggLgyhVxJO7u3YDfL/aLarXiu0JPnBBhOnkykJkpeqrnzgG//jWwbJn6y/8RERFFk5APIjp9WlyFKDtbfLXZpUvi21ni4sTBRSdPArIMuFxAYyPw4YfA5cuA1wt0dIjQvXIlLlyvh4iIaFiE1ANVFPGTkiJOZ4mNBY4eFb1MSRI/MTEiXH0+sa80M1P0TP1+EbIAUFubifj4ynC8HiIiomER8hDuwoWiZ/nRR8Ann4igTEkRP8nJ4ra1VewPvXIFiI8XR+6OHy8CNjYW6Oi4xhFIREREI0BIAarXA9XVYqg2eEUip1MEZlKS+L7QGTOAtjZxANHhwyI8fT6gqkoM7aanA2Zze5heDhER0fAI+ftAGxpET9JgEEfkNjSIU1tcLnGA0fjxXaetzJkDPP+8CNb0dBG0584BFos9XK+HiIhoWIQUoJIkeqEul+hhms3AggXiAKE5c4BHH+05f02NCNPu54TGxgJffz0HdrsFFosddrsFHR0xMJvbe9yXpAAUBQA0kKQAOjqMaG1NhMcjQ5bdMBrdcLkMcLtlABKMRjfi4+1ITm5CVlYd4uPbhmwlERER9RbyQUSJicC4caJHqSiidylJfX+9mc0G5OeL4d3jx8URvBkZAKDgyhULjh+fgezsC0hIsKOlJb7zvl7vxYULEwAAKSkNqKvLgt2egPj4ZrS3x8LjSYbfr4VW64NWqyAQUKDVKnC5DNDrPXA48pGfX8kQJSKisAkpQDMyRI/z9Gmx71NRgJYWIDe37/M7U1PF/tJx44DKSnFAUVMT4HKZUFeXBZ9Ph9raLHi9Bly+nNp5H5Dg92sBKDhzJh9+vwZ+vwZ1dWKa02mEz6eB2eyCVuuHJAVgNrvgcMTC4YhDWpqNR/oSEVFYhRSgsgw8+KA4Averr0TP85ZbgBUr+r7KUFGROO8TEFckSk8XR/CmpV1BfX0iTCYnPB4DtFo/fD5t531AgcHggaIAHo8MrdYHQEFHhxlarQ+KIiEQ0MDpNEKS/DCZvNBonHC5jHC7DZBlDxyO2MGvnVGktLQ00iX0IMsySkpKIl0GDQFZluHxeCJdxpjC9hwdQr4WrtUK/OQn4mcg8959t9gHGgiIHuuSJUBKSjG++EIcVJSQACxYMKPHfUAMDQPi6F2vV1zAXlHEAUtOp7iog14vfuLjgenTE6EowKxZVsyYcSPMZmDVqjtDfXlRJdoayVDiBnf0GIsbzlCN5rYMjN32HPIXams4E0QAAAlPSURBVIfKagVWrQJ+9StgyhTRiw0ERG+0qUnc9r4/ebLoqTY1ATfdJP6mtVUctBQIiJ6vTid+FEWcb3rypLg1GMS8vGQgERGFU8g9ULW690YbGsTpLr/4hThSt/d9mw0oLu668tHy5YDJBBw5IgIyGKQ2mxgaTksDJk4UPdHaWmDlSl64noiIwmvYAhQQodY72Hr3FPvrOS5fDrzxhjiAKTlZBOsXX4iL2H/3u8C8eeJ3u73r9BkiIqJwGdYAHQyr9eoDmKZMAWbNElc+On5c7D+trRX7S4N/E+zR9v5e0t7fWdo574b/i1R9K4r2PMdeLBER9WvEBChw9QFM774rvuFFkoCvvwYOHBAhOmGCCNE//lEcJTxhgjid5o03xHBvc7O4ItLMmWJadXW3efXNcPhN/A5TIiK6phEVoL0FT5NJSBBH58bFiQOJvF4xvNveDvztb6KH6vOJb45JSwOMRnEU76FDIoDPnxf7VMvLAbM3AePkFiBBTGeAEhFRX0Z0gHY/MKm2Vhy9m54uptXUiAs4tLeLwDx0SITohQtiekyMCNHDh0WIWiziqN9aTwrS5BbExoqDm4iIiPoyogMU6HlgUjAIAfEl3q2tIkTz8kQvU6Pp+t5Sn0+EqN0u9qW2toojfI+3T8RFdypiPxFDvERERH0J+3mgw6WoSISg3d73eaZ6vRjKveEGceF7v1/sF01IEPNcvCiGfz1+LXyKhNZWoL5e9GSJiIh6G/E90KDrnWc6c6YIRFkW3wgz/eMXcLx9EhKMdRhvrsHC9gxc8qagqkUPBXYs/rwM8kEfDr3lhjX1QNcT7d8fsddIRETRY9QEKHD980yDp640NADjDY24J+UArMZGAMCr3jswNaYC9XVN8ECPFDkOAUVCgzdpGF8BERGNFKMqQK+nR8A++lyPaanvAo4OILPs2+t6lpXBYQdSzQBW/WJY6yQioug3avaBDlbnPlSfGQFFgt3Oa+oSEVH/GKDfCu5DNWvdaPAmwWzmhRSIiKh/Y2oI93qsVnQdMMRhWyIiugb2QImIiFRggBIREanAIdzeeJ4nERENQEgBWl9fj9LS0nDVMirIsoySkpJIlxH1ZFmOdAljGtvywLA9D8xYbc8hBWggEAhXHaOGx+OJdAl92rhxY6RLoCjCtjwwbM90LdwHSkREpAIDlIiISAUGKBERkQoMUCIiIhUYoERERCowQImIiFRggBIREanAACUiIlKBAUpERKQCA5SIiEgFBigREZEKDFAiIiIVGKBEREQqMECJiIhUYIASERGpwAAlIiJSgQFKRESkgi6UmTUa5u31yLIc6RLCqqysDB6PJ9Jl0CCxLQ/MaG7PbMuDF1KApqenY+PGjeGqhUYANrjRgW2Z2JYHjx9DiYiIVGCAEhERqcAAJaKoEFACkS6BKCQMUCKKuKaOJozfNh7NzuZIl0I0YAxQIoq49yvfR11bHd6vfD/SpRANGAOUiCJuR8UOcXtkR0TrIAoFA5SIIsrutuNgzUEAwMGag7C77RGuiGhgQjoPlIhoMFqcLbhw5UKPx/b9Yx8MWgM8fg9krYzXyl/DrRNv7TFPdnw2Ek2Jw1kq0XUxQIlo2Gw+sBnPf/48jDojZG3XVX7aPG2dt6WflqL001IAgMfvgcvnwpM3P4ktS7dEpGai/jBAiWjYlN1WhgRjAp7967P9DtV2f9ykM2HTkk3YsHDDcJVINGDcB0pEw0YjaVBSXIL9a/cjPTYdRp2xz/kMWgMyYjPw6dpPUVJcAo3ETRVFH/5XEtGwm581H5XrKpERm9Hn9My4TJxadwpFWUXDXBnRwDFAiSgiZK2MS45LfU6rd9TDoDUMc0VEoWGAElFE7D23F7JGHEhk1psha2WYdWYAgF6rx97zeyNZHtF1MUCJKCLePPom7B47zDozHr7hYTQ92YSHCh+CSWeC3W3Hm0ffjHSJRNfEACWiYefxe/DhmQ9hMVjwzv3v4MUVLyJWjsW/rvhXvHP/O7DIFnx4+kN4/d5Il0rULwYoEQ07f8CPBwsexOl1p3F77u09pi3PXY7TT5zGjwt+DF/AF6EKia6P54ES0bAz6U146X+91O/0cbHjrjmdKBpEJEDLysrg8Xgi8dQ0SLIs872jHtieRya25cGLSIDyTRu5SkpKIl3CVUpLSyNdwpjG9jwysS0PHveBEhERqcAAJSIiUoEBSkREpAIDlIiISAUGKBERkQoMUCIiIhUYoERERCowQImIiFRggBIREanAACUiIlKBAUpERKQCA5SIiEgFBigREZEKDFAiIiIVGKBEREQqMECJiIhUYIASERGpwAAlIiJSQReJJ5VlGR6PJxJPPWaVlpZGuoQeZFlGSUlJpMugIcD2PPzYnqNDRAJ0LK5oNaKtkQwlbnBHD7bn6xvNbRkYu+2ZQ7hEREQqMECJiIhUYIASERGpwAAlIiJSgQFKRESkAgOUiIhIBQYoERGRCgxQIiIiFRigREREKjBAiYiIVGCAEhERqcAAJSIiUoEBSkREpAIDlIiISAUGKBERkQoMUCIiIhUYoERERCowQImIiFTQhTJzfX09SktLw1XLqCDLMkpKSiJdBtE1sS0PDNszXUtIPdBAIBCuOkYNj8cT6RKIrotteWDYnulaOIRLRESkAgOUiIhIBQYoERGRCgxQIiIiFRigREREKjBAiYiIVGCAEhERqcAAJSIiUoEBSkREpAIDlIiISAUGKBERkQoMUCIiIhUYoERERCowQImIiFRggBIREanAACUiIlKBAUpERKQCA5SIiEiFkAJUo2HeXo8sy5EuYcwZqnU+lt47tuWBGUv/E9FgKNf3cLx3ulBmTk9Px8aNG8NVC5EqJSUlkS5hxGFbpmg00toyP4YSERGpwAAlIiJSgQFKRESkAgOUiIhIBQYoERGRCgxQIiIiFRigREREKjBAiYiIVGCAEhERqcAAJSIiUoEBSkREpAIDlIiISAUGKBERkQoMUCIiIhUYoERERCowQImIiFRggBIREakgKYoy8JklyQagOnzlUHfp6ek3SJI0Kj/kKIoSqK+v/zrSdYTRBEVRUiNdRH/YlofXaG7LwKhvz/225ZAClIiIiIRR+4mIiIgonBigREREKjBAiYiIVGCAEhERqcAAJSIiUoEBSkREpAIDlIiISAUGKBERkQoMUCIiIhX+P47X9Rbm1IDFAAAAAElFTkSuQmCC\n",
            "text/plain": [
              "<Figure size 576x288 with 2 Axes>"
            ]
          },
          "metadata": {
            "tags": []
          }
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "colab_type": "text",
        "id": "llYSSGRVoIDj"
      },
      "source": [
        "We now will implement the search policy, which automatically finds these waypoints via graph search. The first step is to fill the replay buffer with random data."
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "cellView": "form",
        "colab_type": "code",
        "id": "6-TCcL5Q9Kn_",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 464,
          "referenced_widgets": [
            "27440cd240534c869698a5ce8bfdde1e",
            "5e2f2130e96649ada0bdb6bc245a9b73",
            "8a6ea8da1edb42f893cccdac946e3e06",
            "658771ec879d47e082c2571f179043d2",
            "7a903ef939f745be85109dbdb495207e",
            "9b6498d13a8f4a6d8feaa793611e02c3",
            "8f6bb53c48794c5598b49bad14c9074e",
            "370ee068b2ce49af8046f383456287e5"
          ]
        },
        "outputId": "1e4b2a42-6d8d-4269-ee7b-b930c9e40371"
      },
      "source": [
        "#@title Fill the replay buffer with random data  {vertical-output: true, run: \"auto\" }\n",
        "replay_buffer_size = 1000 #@param {min:100, max: 1000, step: 100, type:\"slider\"}\n",
        "\n",
        "eval_tf_env.pyenv.envs[0].gym.set_sample_goal_args(\n",
        "    prob_constraint=0.0,\n",
        "    min_dist=0,\n",
        "    max_dist=np.inf)\n",
        "rb_vec = []\n",
        "for _ in tqdm.tnrange(replay_buffer_size):\n",
        "  ts = eval_tf_env.reset()\n",
        "  rb_vec.append(ts.observation['observation'].numpy()[0])\n",
        "rb_vec = np.array(rb_vec)\n",
        "\n",
        "plt.figure(figsize=(6, 6))\n",
        "plt.scatter(*rb_vec.T)\n",
        "plot_walls(eval_tf_env.pyenv.envs[0].env.walls)\n",
        "plt.show()"
      ],
      "execution_count": 12,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "/usr/local/lib/python3.6/dist-packages/ipykernel_launcher.py:9: TqdmDeprecationWarning: Please use `tqdm.notebook.trange` instead of `tqdm.tnrange`\n",
            "  if __name__ == '__main__':\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "display_data",
          "data": {
            "application/vnd.jupyter.widget-view+json": {
              "model_id": "27440cd240534c869698a5ce8bfdde1e",
              "version_minor": 0,
              "version_major": 2
            },
            "text/plain": [
              "HBox(children=(FloatProgress(value=0.0, max=1000.0), HTML(value='')))"
            ]
          },
          "metadata": {
            "tags": []
          }
        },
        {
          "output_type": "stream",
          "text": [
            "\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "display_data",
          "data": {
            "image/png": "iVBORw0KGgoAAAANSUhEUgAAAWAAAAFYCAYAAABtSCaMAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4yLjEsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+j8jraAAAgAElEQVR4nO29X2xcR5ro951uHkpNe62WdnXH696x7LsBJEDQtbjSYpToSQ5gBXBscK2Z0W48QHYfNslbJBgEpMQYiQMDUkAMNC95Sh7uwxgD2taEkNcB5AfpYeBc+640JCMQkHDvXFty2mtEWanltdgSD5snD2Q1q+t8X9VXdep0nybr92Kr2X3+Vn311fc3StMUAoFAINB/KoO+gEAgENiqBAEcCAQCAyII4EAgEBgQQQAHAoHAgAgCOBAIBAZEEMCBQCAwIEZsvlwd25GO7PhX3X//cOcY1Mdi65Pe/vZfIOmsZj6PqxXY9/wfob9pLSXQbLVhVQqbq0QRNOo1p2twvQ4uRVyviSLvR8et5iPn3x5o7LA6ns0zbC0l8PXDJe13XJ+N7hoPNHb05f1j5+AgP3NqzFD8cOdY38e1idZSAv/PwyWQn0IEAH/mKJ+K4ObNm/9fmqa71c+tBPDIjn8Ff/rf/qr77ziuwrtvHYCJ8YbVxbx85hPAhkwEADcuvo7+5ujFa7DSamc+/0G9Bp+dedXq/LbXMTvXhOmrd+CbVhteqNdg8vhemBhvkJ8Xdb2+7sc3Ry9egyZyryYaxLMwHY/zDGfnmnD2t7fgT5OO9nuuz4a6RnFP/Xj/1Dl0qM+cGjMY1SiClTSFHyB/K3Jcmxj/xafw/FKS+fyPxmK48fPXBnBFWaIouot9biWAVdpJB6Y+XiSFEMbsXBMqUQQdZNV+oV4jf/cNMdCozznsqMXQamdf3I7axqopJnJ7fSI3W204+9tbcOPuA7h8s5n5HABgYrxRyPWaqI/F8BAZiBCt3Ydp0XBl8vjenmeEEQH0TPRaXIXJ43udjtdsteHoxWvaa5++ekd7PQJ1zGHPRxxP/gy7RvmeqPfcbLXh5TOfeHn2tmNJXJ+4R9tFE5uzrtfiE3TMaz4vE7kEMMDaTYobVYWQihBm2IvUTUiAtYmCDRid0DYRRebPsYncTjrwmy++ztxHO+nA9NU7MDHeKOR6TTwlBE6aAmvRcEX89p0PFtB3GwHA20dehOu377MEv+l4nGvnCIS4EsHS8kpXIB7btzvzfCY/XACIAJJO2nPeC28dgAtvHegKsmoUdd8/AD1eAdYWomarDadm5uH8lUU4/+Z+p+evOwcAQFyN4JnREXjUTnoWE9NiKRMBkAqTei0c8igARSgPtvi+htwCWEUWQiqUVlKNIrhgMGVQWhFHG6JoaVbOoxevwTetNrk9owakmPgmDakIlhLalsdZNPIgfo+9o+1xBQ7v2QXvTRzQHkMd3H/zox/2CEQV3bWbhFO9FsPj5ZUe5eHXn9/LfC9Zzb5ncV6x5cZ2SCcONbTXLmi1E+dFEBtjYqfRIITD0YvX2MJXmCtePvOJ9nvcca3bTZoWZ+q3AGvvEtvJ1mt+7L/yjkHeyflQYKwEcIVSGRVst9+raWq8gYnxBty4+wDe//xeRii6Pghqkkbrx9RRNZhRxHUMesWW0WmTPhD3NvXxYs/2r52sou9HFrj1sRi+f7LSFXjNVhsu32zCiUMNuH77PnmN1JiiFkCx0B+9eA2dtFzEeakd0vXb97sasm4hF993WQSxMXZs3+6uMBPauHxcrqlAFqq6xYwS9BjUs5LnNDWXqd+empmHei2GSgQgr5VxJYLzb+43XhOA3uykCl31PeZVYKwEcKNegx/Ua90Lffx0BR3E1HYk77b8+u375EB2eRCUVm1yStTiKqrhqJrAxHijrwKX0gQE1KIRwYaN2IRpCzYx3oDpq3cy9jf1/agaDWavE4JMOLVsxo5pAcxrsxTn1Skb8vs3ORap43CeN/VMMWFm2hkArI2TE4c2jmtazLhQ98gRarr31WonEFci2FEbgdZSYqXsYM9s8qMFgHRj92OSB3nGkpUAro/FPZ5O9eIB7J0rEQAc25eJzkAx3ajtg1AnKeWUkxG2vuu373e1s7JouOff3A+THy6g22axaGA7iBSAtXhxJjgAbwfEdZLlMenoFkCOIKLgaIeVKOpxtpkci9hConveANnFhdIS5Xc7eXwvOUYEnTSFyzebcHjPru4zvHH3QdeEpQpoLjbPXB1Dpt8mqymMjY7AnGXUA/bMhL2fSx6/Tq5EjInxBlx46wA06jWIYE1D1q2KE+MNOHGoAbIhIwWAyzebMDvXNJ7PdKMuD2JivAGfnXkVvrz4OjyzTb8eRbCxjRdb5Mnje+HLi6/DZ2deHajwBVjXPn/yCjTWn0N13WQk3st7EwfI1ZyzeOkmuIxuB2RzPvk3tmPNxOTxvVCLq9a/iwAy2iF2nE6adp1tp2fm4cbdB3DhrQOwE4lLFSavoxev9cwD6nlPfbwIZ397C5rrpg0hmDlmmonxBjy73ax3ye91dq4Jl282u2NfCGjOnJWZPL4X4grPjKmOIc77arbamWuanWvC0YvX4OUzn2SeL0D+nVBev05uJ5ztNhszI8gvW7fdMmkRXE2awvQyfNt/isD0Pho5zEBc2z6102m22jD+i08hTc3bOoBiTTry7ke18+lIYW0MY8f5ptVGIwZSAHj/83tweM8umPv5a2ynDiVQKXONagcVqO+Wcj6rmOzc56/YhaBOjDfgf/4/bkGyrN/5yO9d9RNsG6lod6ny8/NlklExOTpt6Hsqsi4+ElvV5RVLaEFVwhkoTwwXXDToQcY/uoBpEtxVnKPZAvRqqwC9McAPlxJyAsXVCOq12IuGy0Hsfr66+DpcOnkQ1U4x1Hcu76KorDRh5hE8frrS/VymnXTgnQ8W4CVD5AEGJnzjapR5t9xxLr5HCahWO9HOV5XZuSY8NghfAOi+dyFAxTkeLiXwdGUVfnbkRVIbVpU5044Nmw9xNcpo6uJfjXoNLp08CF952vUWKoAx9Z96+cK2KoNtbyfGG+QgzysMXbalRcX1mrZOruTZytsIbyGUGvUaS7Ns1Gsw/eNXYP7cawMx6UyMN2Du56/Br04e7D4baqHXvXNTMtHsXBMmP1zQanG6mNsI7MKrnhkdyTxHzjiX3yv1HFSw+Sqj+5ugUa/17Cp0ESYUQg5wdmzYfJj+8StdU574zKfQlfEeByyg1H8qeoAyK6gP0TWTjgNnOylTVFyvmKRySNbkhws915gH1628S2gdZ1GMAAaWxqqiiyoAML/zyeN74fTMPLrovFCvwfTVO1oHmIkU1pyt3GSKR4igN4Wvqe/VlIQho3vfprGgPltThAmVzSfkADfqipoP/VAAcglgXYjM1MeLrPhI2YNrelh5Mum4mCag2E7vHIshTQFOz8zD9NU7XiMgzl9ZzEzSZDWF81cWvQ8K28weW+HNsbEVmR2YB5cFh4pXF+Pz9Mx8rmtSNUTXkFCb90j5DTBMuwObeGLq+6LAjikyhvIZPX66wg67LBpnAWwKkaHysNX4SBmTtkGFLlUiKMReSE1A9Vp9pfQKqO1pnsQBDG5YWR5MjlO1PkFZQvoELruF9yYOwOE9u9D7canBIJDng3pdLto6F+wdxpWoJ0Wbcz7beOLJ43th8qOFTFjY9096BSg1bqjEoFY76UamyNmZgxiDkU1X5MOHD6c3btwAAH01KADacE9VwAIwPwBd5aZfnTzYtwlL3fvOsRjGRkdyv0CdA+YrpHKX68AxVfTyNSBVT3aagrE+gUugv+68vieU67FV85IgrkZw8i9/iMZpA6zZYX/501e05+j3/QLYZ3raXuPBqU9RxUMnR1SocR4BwKV1uUEtYOoYdH3GURTdTNP0sPq5swbsWu3LNXAeQL+FKSIcjHrY1D3aFCbSsZOoaoZ56fNosbp36FM7Nr1XrD5B3hC/IrV712OL8ZSspj2RITvHYjj3xlpBHqweBQA/Xb8oJcSXnVTVWrF0aRnMhg1g53DXZeCJMcZJYiliTDlHQVC2nkoUkUWQ67U41wDRCW/f4WBqCIwcZsO1WZq8wvK55IiH1//Nn0Jc7fU8x9UIzr2RzW3nJkdgUPdRH4tzHZeCiuwoonRnEdef59jyeAJYm/y1uAq/OnkQ5n7+WndeNIh3UlY7uS26eYXBDX3UwSlz65q9mXdMOQvgY/t2Axac0klT+P7JSkaA1OJqtziGa4jVxHiDDMHxPUB1D9smXM0kRLABeflmE07+5Q8zoTHY4pVHeE0e35t5TwBrNjbb4jcmXBa0FMA5BM/luZjGpfi7y7PhTt48cdoYRYUzumIrxHw8j8nje1FZBdAbMaH7O0AxioKTABapiZQ9NllN4ZnRETTW1HYFVDn/5n6vA5TCFAKjxg66Lgy6esOcNOc8GsLEeAOeGc1aoXRhUq4LneuCZjs+TNdJCfV3Z2/B6Zl5clyqGqzNOQH4kzdPnLZK3rlWBKZC9eq78fE8JsYb8PaRFzNCuBZX4di+3d1FFfu7LFd2EHO8EkXGFOhK7bld2G+dbMCcQiqP2gnMn8sWxuDYWnToIhNEDV8fDghTDKEvLzQ1IDtpyrIvcYrU6BwHlI0NI89CZ1rQADbSglVcK91hHnSArO1udq6JOr/k85rGvM+GAr5suXnnWhFwCtWr497H88AiU9QC/CnQacZrWXwr6LHVuYrZikee270H+621AJ6da7LCaChtwHVrqAoQXVU2H8Zx2+pbppAYSgjqBiSnRgbnvLqkDlOcbjWKYDVNcy9q3AWNinRxqXR3/soiGbon1zLQ3b/4m+78nJoAPgv0cz3xRWyZ82IKSwQobpFQBTnm/BXCV42wmL56R1slzbhYRxFqbbASwK2ljer9OnQDy7YmMEe4FrHSu2ja1EqtuwfTgBTf1d2/TkMwJXWYzr+apvClQ9NKVUio2gYAPk7y1oyWMWn3rTZdl0Ig0nCp66pGEWthcknqwMDG0umZeTg1M59ZCLjPsp/xr+pzyFOdLy82CxTnesR3bOK8rQTwt989gT8hJiq3QpCtJsARrkWt9CYzA1fT1t2DWGlPaTKk8iwuuqSOl858AvVaDCcONdB2RQBugg97TqK7xT8s/FP3mrbHWaWAOz5MQkOXsm6D+D21UHFNReLveQUbNpaoampc81TRyTgqJpMTgN2444wFm90ndm6bjE6q8QGGlRMu6dA9x7jFKmyN6hzh6iNUhYNrGIrpHlwGuq82Qq12AjP//mv4mx/90Jtzk3pO/7DwT/B0ZWMMPVzfUdk6XbgOs7zCF2AjLExcF1aYxld4GweTUiFfi/osd66Xczw9M991dhUZrkdhcmjajDuTo1H3d8r5u7S8Au/O3upxoplK3crXbDPurDTguIrLazk/nYONJsBZpXza13S4atqce7DJtwdY69zMcTpSSR0yyWpK1uhwWRyo54Fp45g2rxsfPhxm9VoMyysdbRNTALweMVXLIe9ui2sG4GhiarWvifEGvDt7C+29xi2C5RPd++F025CfFbbLMY0Fdfep+goeLiU9CTFiBzcWV9AxozYVtpnLVhrwH20fMYZquKCLVcRiVdUapz5Dd3S4atqcWEbqOxRpCqzwonNv7EdjfVVENIKoa5un7J7tzsNmsk9fvWO0G5qO9/jpCjxFHCoVWFuwdGOoiN2WTbgYJwYds/FSi5ZLyc286N6PrtvG7FwTDk59Cqek3Y+pOzln92nqhAOw9qxGR6roHFVTxG0aQ1gJ4NZS0vMS1fYsLrAGH9bETMGX8NDhGhQ+Mb7WikkMdmyVpxYRKjNKhdo2Tow3YPrHrxiP43PCUc+JKnhuc27d5DUF1QuS1RQ6SKzzjrEY5n6ur0fsO1ECwM60JY8TAGApRLpFq5OmfYmrlzG9H+zehZzgFqTykWCh0monLEXPpjGElQlCLYSutmdxweRkw+qnJqvpQGIZXT3ZVE8t0fRQPj52LG7tV2ogieOO/+JT0hzhc8LpIkjymoqoLXgEG/egq8mrg9Oqx1c0g4ytaUseJxzTBSeErp9VwDihaOo1c3IPBKaSlOqY45h1ADYWO1MRIJsdXe6C7HltRabBV7ZYRhdPdp4wOWzC29Z+FegETNExlzJ5Jjs2oSIAePvIiz1hebqoEgquJu4rUUI+r2voHedaTIuW7/sx4RIFYZrvVLw6Z8HkLAgAawrnOx+YGyNwBTqAZTnKP32hkf4P//1/x/6+LWm65lzifj5MFH0Ptsffis90mMZX2cZLPyh6DI+OjsLZs2e7/9aVSdUJUFOpVCwr9p/+7f+4+vTb/5gx3lvZgCPrTZ0d1MMs20Bxoeh7sD3+VnymwzS+yjZe+kHRY3h5ebn7/1TDz0snD3Z7GVKYwvQwf87Kd/fvYt8trCdcIBAIlBWTWfDYvt1kcXwA3CSis8dHZ797gB0nCOBAILDlMDUj0FV7BMBD/VwyCgttSx8IBAJlRBee5lL5zjWjMAjgQCCw5dDFc5vC9jAHnGu0VhDAgUBgy6HLnqW0Y1GmkkoRxzCFEgYbcCAQ2JJQ8c8utWVc69EEARwIBLYsusgFm4Qh1wzJIIADgcCWxBS5YJsd6PKbYAMOBAJbkkHUQlYJAjgQCGxJylBnJpggAoHAlsRn/0GBbX+9oAEHAoEtie/azjaF9QVBAAcCgS2D3HkHALx20nGxKQcTRCAQ2DIIk4PQTi+8dcBYYJ2Li005aMCBQGBL4jviwSUbLgjgQCCwZfEZ8eBiUw4miEAgsCXAmv/4bEbrkg0XBHAgENgSYB00lpZXYHauiQpJ25AyAPtsuCCAA4HAlqFei3sa2j5cSmDywwWY+ngRWktJtzdcq51ABNAtys4tsG5LsAEHAoEtwzPbsjpnsprCw6Wk2xtOCGjVYlFEmnIQwIFAYMuQ1+nmO005COBAILBlyOt08+m0AwgCOBAIbCGwUDEuedKUKYITLhAIbBnUULEdtRgeL69A0tH1QF5LU+ZEQdgSBHAgENhSqKFis3NNODUzT35faL6+hS9AMEEEAoEtzsR4A3aOxeTfTdEPs3PNniI/uupnKkEABwKBLc+5N/ZDXEUyNdahoh9cSlDKWAngSsVOXmOpf16/b/f1ocL2WWwGqFumnkXRz2gLvoIty8R4A6Z//ApUsXQ5oKMf8rY1srIBP//883Du3Dn0b0cvXkOryzfqNVa5t5fPfIIO+AgAvrz4es9najM9gDU7TZ5anmVB9xwunTxonRppQ7+fK3U+dUALxFg4OPVpTzaTgDvWbK5HZEP5cMJgx/+72g3n4wX8MjHegBt3H8D7n9/rmYO66AdKM8ZkIYY3E0Te/krUCrOjlrXNlKGZXlFQz6E+Fufa6nAo8rlidjLqfDotZHauiQpfAP6gp8CuR01FzfO8seMHysPsXBMu32z2CN8IAE4cous7UPM1Wj+eiVwCWJ5UFUvVXWXy+F6IK9ljPF4vliGfk5poPrNU8hjW80CVtEtT8CocsfsrqkkhZSej3mMnTdFYzcdPV2Dq40XyPJTg5mK6z7yLUT+bPQbsoRbg67fvk7+ZPL4XsFGXrh/PhLMAVidVBzHI2QQuT4w34NntSJ52J+3eiDgnha8slbyGdd1xTUJ9YryBtkl5RGh96qTmnIO6vzrhCc77XG01XXHPqme61U7g4RL+HADwMWgD5z5thKj6LrDdXKA8uCggE+MN0lfAGSvOApjaTlWjyLm/UouYXOJGdFs4n1kqRWzFbYT6xHgDPjvzKnx58XX47MyrMDHeIIVDCtAVtNxzUPeXpuC1SaGAGoiYpivHXI6N2oWpN3IuFJwsKe5ihL2Lx8sr6C4vUA5cOloAABnCRik0Ms4CmJpUq2naIzgwKC3N9AB0K4pPR1ERW/G8Ql0nHISgnfp4kXUO6j4etROvTQoF1HsVx6fOZ/O842qUe6GYGG/AiUMNdEsJYLcYYe876aSwImnputjTQP9x7ZL8hFAKORsy50y4F+o11IZnWi1UT7BcZ3Py+F7UKy4eAHXORr1mLSR0xZZ3KDVDBfIW0rZYc16hLqdQYs+gnXTI3YF6Dt392RaU5qB7r7rzUe9bJYJeU1We679++z66paxGkdViRL1XeVI+SVZDLmqJcOloMTvXhHayiv6NMhvKOGvArqsFpQm+88ECAOjbRLueU8W0Vad8OcsrHTh68Rq8dOYTOD0zb2Ujdt3eyAjThO0mVj0HdX85fVgklF3bJMy4hVN8RirodnY2gp3zXkNERPnAzH86dDtYzhhwXn9dVgsAvT3Q1Cba9ZwqOnPAxHiDtEUvJauwtH79VLFm6lowLTCuRLC0vAIvn/nE6l4ozbBei+Hpyiq5gxBQ90d97gNbzVoOU6tGEdvBZnoPpvNRZ7F1RGLvO7D50O1gOYphrg2Qy3ZVt63kTB4fW2STOYC79eUeF4CuwiS8+jYtT6gt/fk39/ecgxLqruajfqGaqYSzbntc0UZBCGxt9ViChIzLLkt93xWLRSQwPFBzaSyusORU32tBmLaVeYPpOZjMAa41Q00CTN7ePLNtJFMCj+uU023pOVsoX6acIpida8I7HyywozQwbBcSXXQNZS7hhhSKd/HLn76CPvPAcEPlLySrKcsU1ncXgBjI73ywgGoEIoOkyJRik7MPM3UsSdoqhq0A8+GUy/OMtseV7v3XazGcf3O/l2fu0klW/u3Z394iNcVH7aQnHRur5eqykFDPPAJAzWE6RzImqMX11sdi2DZSgUftpPtsFq58YXWtgXIxMd6AqY8XM7JBOIVNY38gPlhxUadn5jM2N5FBUqQA5tiSsZqhPusEFGUGMAlA7D6eruBeXJdzcwUThilV94X1aBf1frD7tVkIbN+FyYcgX5v8PB4uJVCLq3Dp5MHu9xaukLcbGBJM+Qs6vApgm0E/Md4giyD3I2XTVoP05QAUYFp4BADH9u12Oh4ATwByhQd1fN39m45t+r3pvS+tp6XrFkruc5Ax7YhUuLuXPM86MDzkUaa8CWAX7adRcmeQisu2XxU6x/bthuu378M3rTZsj3tN8CkAXL7ZhMN7djlNUM6EdzV9cN6v7tic35ucnw+XEpZGbSv4RBWs33zxNXTSFKpRZCzAwhm3RdXWCPghj7lMxnYBl/HmhHPJ9NI5gwZVDMcnWLzxrz+/1/03FsCdJ+WZM+Fd45E571d3bM7vOc5PzvOxFXyiCpawPXfSFC7fbJJjjuvE9BH7HSgGn/VehFNczmzcNsITrQMtR0l58wEg83AmP1qAg1OfsgRyWYS3a/lB+ZnZ3AtnwmPCI65G8PjpivYcnLqnOsHEGR/qeKAwaZC2gs9WeeAmlpQ52mSrU0S9lyeSQtVqJyyB7k0Au672E+MNmDy+F16o1+CbVhumr95BaxoknRRa7cS4WmEr26mZeRj/xad9F8SuW03xzGxXac6EV4XHzrEYIAXjs+XUPdUJJu74kEO3qOI6cgEi1+cg46o8mML9XDMAA8Xj2zzkKtC92YBd7SCYbZADZdOjtE6u/dAnLgkd8jNzsWWK3+nsWrIt++jFa5kQGuwck8f3sqJWKDu5y/jQZZPpfAy2DlOOTdfVXlhEbY1AfnxHIbkKdG8C2DVKIE+XAOzmdDecxwPtMgE56agVAIAIYDWFjPPHVTOzuT/uOfJGrbiMD/G381cW0eJBpsWI+xyoxeHYvt3dVlsi5BDAPrwuUD7yOM4wKIFeiSLtzttKALeWEjh68Ro7dpZDHo8wtlqZtE6X87nGt4q/UYILAKBajbqJBML5I6Ig+pEyvJaQkXUGYsXD80atuGqDujhlHxEF2OJwbN9uuHyz2X3ntrU/AuUmb1gpFt0kjxeBqHFTqT23CzuOlQ242Wp77xJB9kCrxT12SjXdj1qtTJ50F+GVx2A/Md4gbZnVKNKmIxftxNGV0vvuSZJ5t4NwKnGSM3yg2nSv375v3JmFcLLhxmTHpxzgmG/m8s0mnDjUQLu8tJMOVJ/dhUp2Kw14NcWFRR4tQFdYRj7uu7O3yDhNdTU6cagB/7DwT5ltq6uwoDRqrn2XukdT/V7fyR8qugVkNV3LVLxx9wG8N3GgL9eDoRNyRQp/jnANLYY2L7pdL6WQXb99PyMjBVF1ZBT7PLcN2KXylDqBL7x1wJg+i8VpHt6zptWrD+ryzWY3nM2HsKDKIXKbQFKCiyquLmt1RTpxTAtICgDvf36vJzGk304lygxjWyBdxWTT5zhQi6qfHBg8ul2vzm9CjZu0s7KM/Sa3ALbZAlKriq4GMIDZBED9jVNQmQNVHMa1vKDo7vtwKelx7gD0L050dq6ZOTdGP2pz6KB2D3mFr8mmz3GgutRPVgX/aw7XHygeFyErFnJsvHa+f4Daaq0EsNp63lZYuObGu0QD+LTPUc4nbhNIdcLLppEU8hX1cUVXfFxlkLbOIswenHEon5fShDntt8TvxS5KjaaAkBRXSlyErDwu1fH6V+999wA7j5UAbtRr8IP1hAmXieAaK2eKBig6UiBvyIrJkSSEr24XIPCVv24jVAedOuvb7GETeieKCNkWTsKKygOYdxyBcuAiZF3MdFYCuD4Ws4QEhWtYlUkA+oznw8irhXGEHec7ecs9ynCTRDZj6qztOBTFet7//F5XgJoKJ+WJbw/0F51S40PI6uhrPWBXTZIjAIv2zOd54Bxhx9EyfZY3pN7FiUONbrW2fkQ5DAJT4gV271i3ZN2zDyFqw4FJqXGNC+bOmyi1cCQdPnw4vXHjBvv7GC4XeuHCBVheRp2IQ0Oa4l5z6nPb4wTcWYEq/KcX/kv4v/7wIOMQFQuSbgFtIGY5kUFn4mfbfw9xlI3F3szjZZDXdO7cuZ5/U+8JMwnqiv+bHMVRFN1M0/Swep6BtCSy1aiGXfgC+GsFX7bJtBkYgQ589oesj6SddHrMDhgRbPggZO1JF0UhO11h3wR8ur7joFosiTBNrqAQvHzmE/TaIwD48uLr6G98+RhkVCH3d7V8Slzea5HviesPcIkL5uxM+96UMxAYJjhhejLtpANTHy/2VEID2IgZb9RrcOnkQfjq4usweXwvXL7Z7GZUtdoJmRnp4sC2rYc6DioAACAASURBVFDos0aujGuT2yJQ74n7jCghe/7KIrnTKaQlURErZCCwmXi4lHRbJ+nmBtdRZ4o9pbC1dRfVrkr14Qwa+Z64finqurEiUQLvLYlaS4k3L7yOEKweGHY4QosrjOpjsZMDm1NkSJ7DPttVnZ6Zh1Mz8z2x7eJ6pqYGZ4IQcNL9ZTlUIbJhKbjRQ1YC+NvvnsCfGFbIvBoyWh84BKsH+kwEAP/Fn++C3997lLurCQU3FDBN8b51f/HiDpi+egdOz8yzaj8DrNlAKS1Xp2WbQrXUY5a9dKcp3Z+K4+bCzdS0sgEnHbxyVrPVhqMXr8G7s7dy25DQlxmi1wMFoo6vnWMxXDp5EN7/+/+8x45rA2f7ybWNPmonaD2Uz/7wwHqu6bRcqtrdsX27tfPatNjkbfXjC/GeOdopZR6qRlG3u4ncA06mUa+xFxtvTrhmqw3vf34vd58l7GUGz3+gSKIIupPqVycPwtzPX+uxX3525lX41cmDVo6kpeUVUhiKMoenZubh6Upvdh0G1dRUhTPXdE4n2XEot1DCSnPK5+IsNmWw/UZRb1soXb9F6npX07RbvvLcG/tzl2f1GoZGKaq2aa+2bXwCgbxQYVkCtTaEqO3QWLerquVPqRZY6tZ2VZo0I9UIIAVIVnvD0EQ7KA6muWayJavb8dm5ptHLzylcNOh0doEI2TMlYHCcnj7qlPQlDtjm4XNeZiAwCHRRDddv3894xLEIAp0mm3RS2DkWw9joCBqhwLIXQzbWVb0HcR0moSGEFIWY1+riNKgKfzbowsps7iNvSrJ3Aezj4W8bqXQfjujaGwiUGW4EgUlDbS0lMPfzbNyPjWJicnxxhYbJ7PHw8VN4+cwnXSEua5dlD1XVhZWJhbQflQqdBXBcWTOcqVk7eWoJYCl9T5LVAeTrBQJ2cON0TSY2XUEgAOjWkTbho1uNabFYWm9nhdVPKJvAFYj4Z25YmU2lQhesRFtcrUAEGzUxAfpTpzUdoR0UgUAZ4Mbp6jRZTlyvHJ9qMknkdXzZ+GOGoUlpmm6kjduElXHioF3lYC7d0vdKR73sIHwDZQdz0smRAlShd9mZx5244ju6btsA5vhdE7b+mDJEOuhwjabS+bDyloi1EsBJZ7UnDpB7Ei5U77VAYBDYCi/xN92ElI8pC13xuS6pQsYUbibH76rXcuPuA5aZkNMVRKYskQ4+Me1K8paIddaAfW45xOALwjdQFlw1G1P/QkogUunB1Ll02mYUQbeCGnYtcoU3rsMO88/IlDHSgcPOsRieJKuZbidcx5tr+rYglwkCO4mt1mB6sYHAICiifyF1TJFebHMurX12PXWZih2mCssD6DtAyH/fUYshitaiNsoa6WCiFlfh3Bv7AcDdl+Xa5UeQSwCrJ3HRGkyhLmUpYxfYWhTRv5D6LbXz051LJGdgvxTz0saJJuaqbu6WObqBizBzVqMIThxqZBYYW/L2i3RORY7WTy5j2n5h6AaZSBsMBPqNbS1dAVVPYfL4XvK3VcI7pDvXxHgD3j7yYsZBHVciWFpegZfPfAJLyytr4aISlB9KOA1lylLDwRdpCj21NC7fbMK7s7fIdGQOVPp2oVEQEQC8feTFzElci0abKv0vXHG5ykDAnaL6F1J9+GQbMPdc700cgMN7dvWYBR4vr3TjhB8uJRBXI6jXYnjUTtBylOJc1C607JENNqjrXDvpwK8/v9f9t2twQZ6dgbUA1hmmfRaNHkaDfmDzkCfPn5qQ4rPzVxa72Vbb4woc3rOrR5C6nGt2rgnvfLCQMWcknRSe2TYC8+c2suuwc1GRDpsxskFHv+OZrQTwgcYObaO6+lgMcSXqKSYCAPDSH+u3UgDm+MlAoN8UZfN8urJR1lUU7bnw1gHnbCvhe+Hakqn7CorQGv3U+nOVo1R7SD1cStBB8NkfHsC7s3RRD9EapBZXu7/31Y8qEOCQxw5og4ufxOWYMhwtNq8tczPRT60/VxQE9uJXiVDe33zxNbw3QTvUdAMztCQKFA3W2VhNnPARbpUnbpS6Ft1v40rE1mI3Q5SDiYYhMqTfWr/3OGAKU5KFdmBuLTNUoM+oQ7OddOCdDxbgf73+H+A//L+Pu5/7yAB1jRvVhXjqws2e3T6y6YWqDZ+deZXMPdg5FsO5N/aTz0uuweGSQo6RywRho6pToTamY201J0Cg/2BDs5OmPcJXINrOu5osdGFqOs5fWSR3iLrfthiV07YamLlFdEIBwM1RsrkVAEhTqa7LBkYuDRiLYKhWIuggdogj/3qn9bHEwFy48rs8lxkIeOXhUtIN9bLVil2iK2bnmmT7829abZgYb5BlKoMCg8NpxNlstWHywwVjCVBTqrmOXBowtpL88ievwNE/35X57u/vPdKuBsEJEBhWbJ1oos+c6C1mGuO6YwsB66M/2VYH80Mlqymr/rIu1Vz3/nKXOsdWEuyEnPi6reAECJSTvJX4igxdMqUkA/jpT+bicByG7hccdL3vOOhSzXXvr5BeE9wL2SwvLzD8/PKnr7CKQsUVgGQ1+3mRW33KybZzLO6ZL3kUmNm5Jkx+tNDtcNNstWHyo4Xucanf5KmFWxZMve9MiJ2GLpnlK+K3uQQwJUA5nl6blzc6OgrLy8t5LhUA1rzdg2pxT51b9J0q6vjWx4GtWQB/4cr/Dn89Aj0zgnoW6LN+CjA1VYyv4jWAbiRQz7lTgKmpf+d0zNHRUTh79mz331MfL/a0FwNYy6Kb+njRqpAWttNV5UTZwkpNcdQYVBQE5cf6q7P4cZwFsE6ActKLdV5d9YXLA8WGoxevGetM+D6GbXnNCMwt0X1dmw7sumtxta92+Kmpqb6chwu1EA1qEfd5blWhoeycOvsnZ6eLyYmyhZVq46irUc/CJOrgvDdxoKeI/vTVO3Bs3+5MQ2FdWBtADgGsW/3E5KfMCyavrg90Nh2bc9jadWxXU6ykJ9cs47OORt7K/oGtB2en66Jd9psdtZiUR3GlVwCnAHD55lowgVpEXy7sA7DeUNiAswA2CSadPYrj1c2DyaaDnSOPOUXGRrirwtLWpubD8WK6bu79BHv+cFMnhFC9FpO/4SgAZa+mNjvXhMfLK+TflxAhShXRx75nUmCsBHBrKTG2deYIUI5XNw+6VRfTEGfnmjD54UK3iJCI/xPXY6Nl6rKS6utdBMS2bttIbxSgq03Nh7DLU9l/szhjtjLn39zfMwcA1rS/82/uJ3/DUQBsisIPgumrdzK2bw7ciBnTAmQVB9xstbuFd7AL4G5/qUldsbRvUVknupvGbJrnryxmKrglqymcv7JoHZ+szUpqJz3bklY76cmisbGpiffgq2iRa4YWQDEFZgL9ZWK8AdM/eaVnnE//5BXjAmqKacbGVZlw1dBNmb2C+hi9gwCw1IBXEaFbjSJYTVMrTYxqd72aApyemYcbdx9oC/cAuOXGN+o19Poo+4/43Ca8R5eVpOs6wDV3UMJO561WNeZj+3aTXXFdNOu85otAOfARh4/tzkSDUPEZPPV0wR5w0dCpIvoYJkU5VyYcwJpQFqsfAK+sn9AqsVUkBYD3P79n1Oh0Wlcebc4HVFaSqV4r57opofZwKUGfGaYx//rze6gGbZuhJQh1PAIA9O4MAHrGVZk4tm+3MexSdBWRd8DvTRzo2RlTPCKUO0FuAVyJIpida1pvjSfGG6hGDbAmhE3bV53WZWs22ElsE6jPTVDnbxgEFee6dULt/JXFzGccL3Rec8GgF7xAORg2U9TsXBMu32z2NDaNAODon+/qNcX8+BWYP/daRjGRFRbT3KawMkFUEI21k6Zw9re3euLfBHlaa+fpPgtgt50698b+niwggLVVT7SsdsG164DpuieP74VTRLtxzJTCNQPkMRf4jMYIDC/DZorCFowUAL7653Y3s03UeADYGOeYmQUzq0awpmHrsBLAjXoNUiT6oZ10nJr6cVpr637rKwa2XwLEx3kmxhukAMbg2rjymgtCHY9AnkiaQUDJJrF7p6qaYRXTnt0+ggrzyzebcHhPtjiZwEoA18di+GfLgiWm1to37j6A9z+/1yOEqVAxVXCdONToxuNVowhOHHIXAv0SID7Os3MsRp18mMmEcnjKDNJcIL/Xvy3nPA0wGYYGu/J4o0Jpdc5y8f8yuopp3quh6QqDPElWc7fWxrRCqk4nRBvhcJ007a42XAE3rMkDNiYTTOvWRUH0E/W9DrJWRyA/w2CKkscbFUrrspvX4a0a2u1v/wX+pNWGCCCjsYrJ77ONt4Cq06nSTjrd2F0VLBRLTSWkkgd8C+q8x7Md6K5ad9ELlPpeg/AdHqixUXZTFCZc1VDa81cWUX+K2M3bhq15q4aWdNaSCESVqBQgUw2oiIdvs/K02kk3pEqAadCq2QOAzjrzmeXl63hFD/R+ZLeV1TkT0LPZMh9FKC0AnZosNze1KbYlfkdVQ3MOQxPCV4RlcHoh2fZLEtga8VWbC+XtxFCFgu/QmmEJ1enHdZbVORPQwxkbrnN9EKiJTlhqsmhuqoaK6mplyL+j8NIVmbMiYt/hZr1hxv24EqFmCPm6qH/rUIWCzlPqwrCE6vTjOjkOwkD50I2N2blmJhO0TBqyauPlJjrJzU3V3ee7s7cyldCw32HkSsQQec6cFZHSQjlZb1iCwvRPXiETJVQhSmlaqskRcxrqfuuyqg9L1lg/rlN9r4HhYAeh9W2PK3D2t7fQiICy7PJcE5104/69iQNsWaSSSwB//2QFZueaLG2J+g4n6w0AL/rBbURIZWq9feTF7svYORbDtpEKnJ6Z79kyTR7fi3dFYF63is+ssSK3ef3KbpPfa2DwcMYU5SxtK1FQKmXY5bkUD+KMe9emqLlMEMlqCtNX77ACsPNkvVFwowFM36NMKDfuPoDrt++z7cU+r9mE6ZrzRi7YXCflER/WML+tCte5ZtpWU5Rtl4fhOj9df5e7Kec3rTZcOnnQGICdJ+tNBzcaQPc9yoSCRUrIuF63jwgGzjXntb1xrlO3EHDD/ALlgFuP2rWCWJkSMgD8h9K5/C53MZ4X1ks8mmwrE+MNePvIiyy7a7/RmUcobK/bt7mAe81F296oSfubL74eimiPwAZcx6ttjd96Le5rb0EOnOJh3DmbZ27nKsYjCyGT9J+da3a381RH0UFhu6LbXjd3aydW5GarbXxGNtfMMZW4mguoY5tKbwbKASc1V93pqdtt6ncAAD9bb2BZNkzaPpp9+9ECnL+yCI/aSXeOAGRrQ9js9KyL8fygXrOepOrNdNK0K7wHLXwB8HAoNdtP4NJxmLO1w54RAP1Cba5ZZyqZnWvC//Tb/7un95XNIKIWgmqOllWB/kCNORlZydLZ+qlwQtvyAP3C5I9Cs287aTdDTsyR7bF9FUgZ62I8LgWVy95xl6qXoFa855gdsEHK2drp6vZiz8rHNc/ONTM1JXTnxKAKsGAdA8pgbgpsQI05rMsNZxf3zgcLaKVEYXYqS9Gl2bmmUVnh7NRcq0DK5HbCcRhE8oHtllo2oYjftpNOjyng2L7dMH31DpyemWcXDTr721tQJ6qXydqg6Vlgf8fMPqbCRjKmhoSc96Pz/tpcS6D/UFqgnJorMClRE+MNOE2USVXLOw6a6at3UOEbwUZPx7zNRLk7vb4I4H7XCc2Tq06ZSzjFe6hBum2kYszAMb1w7rOy8cRyit7nOWfZC7MEcLD3zlGidOaosghfAL0DW4zXPFmaNju93FEQHPrdsiZPHYM8Xn3qxT5qJ8YoEZ1nuahnxSl6H9haUGONkyFGzXNuC/d+Qd2L3FZIjezaORZDrGnbXo0iVvszlb5owP2uE5rH5JHHq6/T9E3aoPyMOFEQPtC1NxqLK8aoFlMN52B+GD4o4cEptk7NczGmywK3cLw6Z2fnmuR8EWYbMe4pM6VKlFqsTn/2Z3+W/v3f/z37+4NClMvkfq49FlEk3PbzYWE5rcD7T/4CANbiN4XXd+dYDOfe2E96vWtxtTt5Ka94La7Ak2SVHJhTU1MF350fXMaRt3N7Gl/yexaYInxcF1V1PPzt9hsDmyPnzp3rXpPLvRy9eI00s/zNj36IOp4vvHUA/uov/uxmmqaH1d9ZacCrq6vmL5UA6t26vHNqoNh+PiyMRhvvWC5K/XApgcmPFgDA7JChvOvt9VA3m0p4/UJMTB2mhccXmHCgskgjAOs6GuI+AOwiVPJkiAFsaMZlmCOu90LZhjtpqq0xTtEXG3BguKDmR9JJu5MIQ3zOMfVwK+GViX7USKYytGoxPlWpymQ6OJmrvilL0aW8maji2VWRVcSlZkxfbMCB4UJnlBJamS6qhRvCIyrKDYttuB/hlJSQ973j2qoRKvKiBoBnoprMErqQOwxdS6KgAQcyNDTREWJg6qJaqBKeGM1Wu5tHX3b6USOZDJEiVkWsMtkwdaMYFFgHD1NtCJk8NcZlggAOZJg8vhcNuYmrUVcr0G1hqcJLGBFAd9CXnX6EU1ITG9vyAmw0RRDYCpKtjCkTVWde4tQY55h2ggkikEEMGLk7rBwFIb6jG1jvTRzoyYSrxZWeehMA2doVZY8g6Uc4pS61e+Yfv85kLoqmCKZkoGEy9RSFuovgZKJSn/saC0EAB1B82AixOEp5wKp24rIJX5vqdL6YGG/AjbsP4DdffA2dNIVqFMGJQw14b+IA/MPCP2XapYumCOJ6hqXn4CCQxxc3E1VnXvIxR4IADuTCJp5SHbBUTGUZsK1O5/O8l282u+frpGm3otijdtbeC7DRDJOqcQCwIUi2eoJMBIDeNzc5wzdBAAecyVNzA6DcXZFtq9NhuAi7qY8XSRMCpaXtqMXa5ygESd73pWNYBPsL6+V0hW1XNqkB9C9bVxAEcCDD0YvXWP3ddMKCM3DVQV8mXKrTybgIu9m5Jlo1T/we67xbi6sQRUAKX9lkcvTitULsw2jx8g8XYOrjRWgtJaUSyGIBw97HIELzQhREIIPqQX/7f/t3cHpmvufzyY8WSGFhI0zLEqCvYgotM/3dJWlD97cIIPO8Rasf6j0AQE/n36Lsw2jx8tUUHi4lpY7EKEOLrCCAAz2onuJ20oHP/vAgY1vU1RHeDF0v8lancxF2ur9hT/uZbSMwMd4gQ9TUz4uKY+YWLx+0sMNiqQe98woCONCDj0iEzVDGUo51BtgQZty0XRdhZysIm602vHzmE7JSn/o5J47ZJYmDe92DFnbY2K6PxQNNWgk24IBX6rW4FLY+H1A2QSGkdM4aF6869ZvtcYU0M+gSWNSMRpOjydVJpytrKlO2nVFcjeD7JyvdZ2vrlPTheAwCOOBMXIkgWd0QAbW4Cuff3D/AKyoerpBy8apTvwEA62gRStjrHE2uSRwT4w2Y+nhRa4suSz/AhtRU+PHTlUxctU2Ei4+IkiCAAxnEIN1Ri+G7JwmsImrWz468uCV7vtkIKRevuklAfmNI26biXDnkcdKde2M/2aW76OQVG+R6x1T9Ec79+so4DAI4kEEM0qMXr2U0BIA1M4Oo41uGSdVP+pFpRm1txbOmElhMBdVNUM1jsVKYs3PNTKr6iUMNuH77/tAsyHl6VfoaB8EJFyDR9bjbqviKJKCcXZxiOkUVBaIqri0lqz3nn51rwuSHC5mC/TP/+DVMHt8LX158vSf8razkeY7U+65EkZUjLwjgAEk/yi8OG7pJy40g0AlZTvxwUQXVdQurfP7pq3d6bP8CUbB/WMph5nmOVJhiJ02tYp6DCSJAMqj8+DKjOsp21GKIIoBTM/M91d10ThmdkKVqY6ifF5G1pSukL++GdNtscd9FpDsXQd42S+98sJAJ97OxBQcNOEAyiNY1w4DI3rt08iA8XVnt2k25/cB09kNdUoWLZmnzG10hfXnXo9sBVaOo8LZNZWFivAGrjG7pOoIADmiRhQ0AwOmZ+VJvK/uJrmCPAJuIOtOOLqnilJIObtrq2hZnpwrpq7seqmB/tRKR1z/oJIyiyGumCwI4YMRnl4VhsQ9y4AgVeSKKe2+22qSQ07WDUuHUlrDVRt+bOACXTh407nqe2dZrvXxmtKoVJpvVb5DXIRpswAEjvmIeiyyHOAhMzUfliajeewp0nKxN0oXu/K6hUjq7qHofAGv3GVcr8HhZXw5zM5K3jGUQwAEjvmIeKUF+/soiTF+9A685X+FgwJyUlFDF7l18T47dlb/PKVYfAfS0JJLJE+dKQb1D3YKx2f0G2IKlxnJXas/twn4bBHDAiK+JTAnsVjtZiykdsl2qjfZjs4jZaMLp+vmxcxYRxWK76DbqtU0tfDGwnd7Ic7v3YN8NAjiQQS0042sim7bswwgnjGl2rgmVCHdQUYsYx8En0DWOxPrL5RGI1DvcORbDk2Q1hCwC8e6iCDWRBydcIIPqbAMAL+Fouhq7mxWhDWHCVyegbDRNSohT/eXyOD4pp9O5N/YPXchiUQ5hm3cXNOCAFuFsE3ZKsd1We2pxwLbsS8sr2ipaww6lyVajSCugKE1TTvYA0AvxIlrUm8wuZRa4MkU6hG12elYCuFLBFeY0xYsdU59T2H6fPA4AGVA+7BT9rJfT7DsWXXfVAXt6Zh5u3H3QLczDAWtVL47r6/2XCUobWk3TtUSXCxdgeXk58/fXAFCbODa2F658AQtXst99DQDS7cgzfQowNfU788Vr6F7fU4CFK79Dz19milicBGiz2TRdxb5rJYCff/55OHfuXPffL5/5RFsaD4Cu0OSzohMVGlO2LZDLdXJbt3OfG/XORBlDytlGefHf//weHN6zy4tGFT11OkSpMTkwMeGrw3Z92mwLmi+KrGqH7RK+/u7+Xey7uWzAecq2+azo5BJwzsWnncjlOjkDgvvcZuea5AwWW0nqnVDXIbzweRDZdpuRoiqXBfKRJ4ONIxPkZrOfnXkVVtvfPcCOlcsGjKraCtQNcUN4OG0/ilrNfNuJXK6T0qCqUQSracoO/BYlBLFM0bgaZWJWxfM+tm83TF+9o93pUNcvv7v6WAxpulZxaxhqxfoib6B+oBhcI3soU9yHN+7BV//ctn7HuQSwGjRu4yAQv9ddJEcAuoT4UOcS91FdP14VOW4eO5FLPC01UGzNK1QJQQCAZ0ZHepwoVI8wCuz61d/KjrZhyYDz0fNLPYZY0E7PzMML9drQJZ9sFlwXRsoU99kfNhRcm/GdOwpCnbA+V3pqy/7OBwuZ8n8qNts8VVjIYTsYrpq1y6rrS4PSXTNVB5YTi0pdv+m3vhweReFj94Md49ef3+v+vdlqD13yyWZCyC4ht07PzMP01Tva+cWd+yLDs3ABLOO7Ril1s0IwUsLXFOKjYhP0DuCeyqkKU7EtN714H89VFxpD3Y9usJl6j3EGapkrZPnwknPG1WaM/BgmbBdamxCzVjsh08QFpU7EcBV0IsSHi40gMGmsJgO9XN7xSbIKrXaSu8IYB6qEoLD/YlDPv1GvGdvOcN5dmStk+fArcL4bhO9gsXWM62omU8fXUWoB7Jo5ZTuxud83ZffYlG0sMnIDY2K8AdM/eQXqtbj72c6xGKZ//Ap5P3k8+KZ3V/ZIAB/tmHZIzzpQTmwXWqpmsu3xBaXOhFO37JSzTcZlYnOiOXaOxcZQKZNQle24nNYvvrE1ZeSxP1PmlmGJgvBR/yJot+VnRy1GO39Ti+fsXBOu374PKUDXSd+o1+Dh46ewlGRzLUwLtrUA9u1oMyELjXdnb8H7n98jbb87x2I498Z+6+vhlAA0yH0AoIUn1ieLciCWbVuex/5cRN+yfsFZfExzoaVJsRY2dNiEySfDBLVIYp9jznp5UXZZsK0EcGspGVhBbVFYBBNajfXwnuu377M8mepx5UlEwWnFrovZxUJXbMP2Av3FpjA5Nheo8SBnLeZNCQ7kg1oksc91O1y1VgpXObUSwN9+9wT+pKD8aRO6oibNVrtHM+YuDNgkyqOZUttWyrQhCnL3azfRr91Lv3dJg4ATJRG6Spcfm9h8k73YZcdnJYCTDlpPoi/hRLYhaZyFgQqq1mmmOuGimjKE5osldAC41b1wpV/tgDZb2yEKjvMmZMGVH5tFsogOI1YCOK7iQRO6eqS+Bp9LMW/TwqCrb4BpphzhgnUzsK0FS5HneeaJa7U5L3WeU5JpSHxvmIUSdzIOsx18K0AtkgDFNSaQsauG9tx2iJUtNXUBvsoXCjiRCiqmlYljo5PhCjGducSmfoNMXs3SNa7V9ry64zVbbZj8cAFWAaCznhbdbLVh8qMF4/WXjWBe2BxgygUAoLIrhTVH/7aRirdoHisBXB+L4d23DrC0F9/lC9WVyhSUwJkMtpOIK8R0NWC/vPi69poo8mZmuW6fbM9r2qlg9SiSTgpTHy/ChPZKBo86WU8casD12/eHWpPfymDKxamZeahEAOowFf98uJRALa7CpZMHvbxr6zA0VRBSnRFM5QtdLl7ezo3/4lNtJ4VtI+YcE1sbHVeIFWEryqPB6oolHdu3O7PVku/f9rwuOxWA9WI95YrA6wGbrJdvNo2JOcNuatnMUDtVomZVF5+BB9aZcNxsL52wabbauWvrnntjP8RVOtK91U5Yqb1q3U7dQz22b3cmAwbTmIuoAeuSmSW/K4DebgqNdQ3u8s2m9l3anndivNHtDbaZsM1ctMmKDAyGPMEDvgIPrAUwdyCacqbzDsiJ8QZM//gV7URXrytPcXUsDjkCQLvMykLIV4NCF6FOmYGEjfv67fvGd+lyXrGo/erkQXYqeb3kabu2OwHdPJHHYWBw5NmR+kqYshbANgNxbFQ/+fLWPhATXSeExXXl1UgoYXb99n3ttXE0aw4uQt30rrihVNzzqgscQLab8s+OvJgpChRXIjj/5n7N3Q8e252ALivy1Mx8dxwGBge31gxn1+uKtQ2Ysm9Woqhbes2UMizDVeV19jRTR4nZuSa888GCtrg65Q01Of36WVLRNqTJL2PNWQAAIABJREFUZIv2GUpFRUtceOtAJqLk8J5d6Lssc2NHymFL2dBdwiYD/UWM6fNXFtF6EABr77hIZ6u1AKacLJ00hdMz83BqZt7qeNweTLpQKF0L72P7dsPZ397SFlfHjj/50QJAinvtba9/UJiiPI7t291TIFxwbN9u63PZREsMY2ws5rA9tm83XL7ZRMelqzMy0F/kouyUknb99n2rhClKmcNwjoLALtZ2S8VV5U2TGxvsEQC8feRF1M4pQ3X8TTrmu9HFQNt4v4vylpuiPCjzCfW57toHUd2t36gLx9GL19i1AYK5odxMjDfgNKE8NlttePnMJ+y5jCmLldpzu7DvO5Wj1F2sjp1jMYyNjlgLGk4ONgAuaHSODiFAbe9F1w3CNnGBSlg5NTMPDQ/CWKdtFhHaplLmHUJebGoDHL14DV2kQkeM8qAzG8l+I4DsXNYpIu2kA9Vnd6GT0LkesIuNSy4VKfdh8hF/SwkaXYUy4UzSaXAqpvoNtokLlHMPgJftlkd7dolXVhcMnfDd7JlhNs9v8vhemPxwIWPSCsK3PHDMRthc5jSvjaojo9jnzh0xbLtV1OJKj/C1iUjw3ZmhFlfhlz/d6ASBfaeKtO8BMNtHdVoRFgZn0jaLjDX1FdqGwQ29K0tIlkt4ovXzU4aULo490H/UiB8KYZIQY4XV+6+zsox97qwBq5W/dFvRuBLBhbf+TffftlpinqpSnN9i33n8dAX1jJrso5RWtKMWo6aJ+liszegDcIs19fVsuNciw63yxm173w/EO6N2HbqdBjc1X/UrcPwMgeKg3qnJbATQa5LgdA7vfP8AXdVztSSSL1a+GVP7GRfbYx7POee36ndeIjQyk6mCijyIIkCF5baRirZmMIB9rKmN48tXaJvAxuxg2426X6iLmMmuT9n2OS2oAoOBqgNx/soinH9zP+ncV9GVmwWArh/nr9777gH2d2894WwmchG1EnxDPdSqwWhHaUWUo+9RO4FLJw9q6zXouhb3+zlSESciw87GBl3mCAn52mx2GrNzTZj6eLFnV8NxVgb6C7X4ixIGAPwCYKI1kap0cUxwUcppdrbO4cOH0xs3brC/T4FtPXUXfOHCBVheRk0ohePLSy3XYWB9bnnezeBNH9Q9pCnAv31yuOcz2Yzy8plP0MkXAfRUt7M1qdTiKvw3238Pqyvmdlcmin52m2F8AQCcO3cOAOh3KsDMaJRJQigeOlNUFEU30zQ9rP52IF2RbW2PgxK+AP4GHXUY8nPL826GyTGoe1DPq+46uDsNjkmlXosV09x/5XzdLkLEBZ3g4R5fNsn8bS2/EpcXk1kI253pEptcTaTOURB5CGX6AmVDV+uCG+3AtfO+sN5tRRTmccVkavJl4uH6GqhiV2q0ThkwRXFhz7aIIlt914C3Ss+wwHCh0+S4OzadM0bQaifd6Jq8Y9/kJPLlC+DsAHTzuozOVvG8VXs9gN7v4juNvu8COG/oVCAwCDgTzyR8MdR+eTZzQFdMxmcSDKdzjG5el9XZKteBGNSO3EoA3/72X9g50RQ+QqcCgTLSyBFuJmuMALi2rYtbLVKIcHYAunld9jC8QRaHsm5Lb8qJNjEMIWiBgAumEL2l5RVt0k076cDpD+ZhpBJ1kzTEXLtx9wFZeU0IECGIpz5ehFPr9UTqtbgb15oHk5DSzWs1DXuzRFT4wNkJ51pMvYh2PYGAL/J0TcGcNJdOHoSv1ovyn3tjvzF9P02zGXLtpAO/+eJrY/eS2bkmTH600CPkW+0ETs3Mw/gvPu25F3GfL535BP787P8JLzncr4xxXksCNwjfDXLZgF3MBnnSigOBIvHhINZpirpSriZ09awFWLqz4OFS0mPikO9THDvPzpaa1wBu97tVyCWAXc0Gw1iQO7D56YeD2LWUKxVhIc9Bm8JOVFRCnvtV57VY0IZZ+BbtoHMWwL6KkQcCZaFfDmIXp9SRf70Tfn/vUY/gjCsRLC2vdB3jeQo72X6HQxnDz2woulY3gKUNOK5WtAHIoRV3YJixbbzpim0pVwCAr/653WNfrtdigGjNtCDm2vdPVsgyqoJKFMEOQwdqX/c77JFNnFrdeWWblQa87/k/ghtS7ruK7RYuaMuBMsGJd8WwHceqvVSuHqhr/qqWSlRjf5PVFOq1GKJ1wYzRSVN4vLwCcSVC+x36dIhTmv4goyBs3hXXpMNplhDvfukA9h2viRg2W7iQERcoGy4OYrUDOHccU34Qqu6CmnVGmTBEdT1dhlzSSWHnutCXhbhtyJpJmOlKsw4KG5nDMRVRMk+Vb947YmDYbOF02nIgMCgmxhvw2ZlX4cv10DGdMJqda/YIX0GecWwK5xITm4JqMqvycCmBR4oG/XRllX2dHHMjVTthkNjIHI6piJJ5XPu3VwFsE+MbMuICw8701Ttak4ELpoIvuokt5hr33HkWDq4CZbOgDQrqecnvAiBbuVBnruG+A68mCJstXMiICww7ukn2Qr3m7OPQhWnqzunSZNbm+JzvDaMCpXtXskxrttrdcEBTFAQ30sV7MR5ujK+rwyMQKAvUJItgrXmri4/DJLSpczbqte73dCnRJupj+ggJ03WUXYHCOlfo3hVANmlFrgFMwWlnBDCgesAAxdTWDAT6CWZyiwDg7SMvwvXb9619HBy7KsfMR6VENxjC8fsnK6zQqmEtKYDJHN27cvVVqe/Ae1dkV+QVfsd6yEwgMIzoTG4vE01ddVt0Thgn18xH7URNWlmymrIy4Ya1pAD2XKjMRN27En/T7Vjkc0X/y3+Nek77KoDV0Aw5BCaEoQWGEUrQuWzRuXZV11R+bpNJrh13s5QUoDII62MxjI2OkO/RRyhtX00QptCMEIYW2Cy4bNH7kYknRyVQJgnsfHmqxJUdqlRFmurfo49QWi8CmPty+pmHHggMEhcfR7/tqtzzYbbp0zPzuUtYlgU1Hlr+XPcefUSC5DZB2KjhnNCMsntRA5uTItLibbfoJruq72vk2nE5NRHk4w0bJnORTzOTSm4BbFP/wRSaEVej0ntRA5uT0zPzhQkVG8FJTfaiUvc5iwSnJsL5K4tD55ATuIbE+gilzW2CsFHDhTpfJUIfnhkdGZqXFthc+EwnlvFVIXCQqfscja7VTnrucfLDhaExTUyMN+DEoUZXLlWjCE4cMi9MPkJpc2vAtmq4riA1ZYsJBAYBx5Yna7dyVTOhBfoq8j7IzDNuUoFMsprC+SuLQ6FQzc414fLNZrdwfCdN4fLNJhzes4slhPPcY24BfGzfbvj15/fQzymGNYsmsLUwjUfVLCCHMglNlxJatoJzkHNGCJipjxeNBd9l1HKZZaWITiiq2alSe24X9r3cJojrt+9bfQ5AVxlaWuZl4QQCRRMBGG15nLBKytxmKzgHnXk2Md6AsVF7fW0Y5rPr7oKK/sLMTiPP7d6DHSO3BkxdZLPV7rZKObZvN1y/fb/HQH/hrQNw/spizyopNw6UV57R0VFYXkYz+axwKQTtq3j0oIpQp5Ct4qT9fAu2DE9TgL+r3ch8vnDlBixcoX/3GgCAIkep55f5/CnA1NTvrK7zr0cA0pHe97Zw5QvtNVKMjo7C2bNnrX7jYu7w2U+vKFx2FzqnKLowRxGq7BZmAwaArvSXTRTiQi+8dQCe2TaS2abIjoUNFf5HMPmmnVf15TOfoJk+EQB8qenq4fo7GaqodqNeg8/OvMo6BoXOo64OCoxaXIULbx3o8frr8HHNXKampvpyHhWfCw51LF/n8HWpQqGxidBw6WUnC21xrtfcL7sQXKIZdGYLm4UqtwnCpb+V6UKFkM7jOXbNKvKRjaS7rzxbMpNHnVMEWjx77v2ExJjNi22Ehs6vQyHGmXyusuESzaAzW9jIitwCWL14LroLrUZR7pAbV5sZ9ju5+ywn80f3AvI08jOFItnUcuUunMExunmxDW3T+XUw5PlW9g7JtoXjdYoaOrfSFG034iUVmZNfrkJdaC2udsNBVGy0MdtVTRjUT8/Mw/a4stbcEPDusyYhqhNueWI3Tc4CrrB8Yb12rPx8do7FECsddUWt1M1aA2CrY+t84miv1ShC59tm20npFDwxt+pS9+mUEMDeq6FxYgbVgsaqDYqq6G+rjXFj9LBwolpchUsnD8L01TuknVqXzQQAcMqhzJ0Ok7PA5tmL65TvQbUHHtu3Gy7fbIbGqZsUG+fT7FzTWNRd+BdcG1wOE5w0brnHXlSpoLK2kI4Y6oVhURBYzUwhAJqtduZlFxlyQ23FdHGPJiGqaw3DiS/FXqzJWWD77LFrlv929OI16/jIImoqBIqBGk9i16MqRTrha2rRI59rs0Ta6BQ8rsmlkHrALtkhqhYqwqRSML/cvFDCVBd0/kK9Bu/O3oLffPE1dNIUqlEEf/OjH8J7ExtdXymN9PHTtXhn7H44bc51As5njVbbLWpR9QoCxYCNp5f+uIaOP50w+QqJDsIW4gtvHYDpq3cgelrE3ZSLgTTlzANVcakhrcCnZ+Zza1Wyli0a7In/cqnFVXjpj2s94XWdNO3+WwhhKoOo1cbjnU1tzoVwLUqYqZNmRy1Gs5lsWnHnzSgKFIu6A8XCE0VCCTZHhM9H7XTzeHkFks7a9+XQ08/OvGod/zyMcE0uA+sJp1JkSJpADYWRc79tuPDWAfj8Pz1E//abL77u+TeVQdROOnBqZr7HsVVEm3MuWEjS4+UV1DFn24p7szlgNiu68SeaUcqIsaCOnVY76QpfQVmbLRRVaJ4bZVQaAVxkSJrAZJeRPbiyB1NGdJ+lhDb2uU4AyQuK7nuVKCo0AgF7NkknhWe3j2QiSQAAHbT96OgQKA7d+BPvHosq4to7y7YQvzt7C07PzHtR7lTUKCPvTTm5zhbu9yiHALeYCec8pgGwmqbdbDcsq0yOB6bAcv9N2xE5OYL6XidNC7WnUs+mtZTA3M83cpd0Nmof9VEDg4Maf6IuBmX+4grWMi3EHHNfXgprysl1ttg4ZSgHk1ovQiC/TOw8p2fm4dTMfI8DzyQI5WPK19NstSGK1krsmapB/c2Pfpj5jBMe9k2rDZdOHtR+D0/TtreJY4sV9WzqY3HXI07ZhMV1iZRl07WFSIlygo3TCADePvKi9v1w7J1lW4gHae6TcRLAXGeLrVMGi0t9vLyS+V5c6e2cYWqZMvnhQtcRRsUyYgNEXAunFioWBSEf58bdB92ICQyRHAGgL/uneqVtIw2oRfHEoUZPzC/AWoeS75+sdK9FV15QDFqTk1C3KAcGg1rTeNtIpaemsWlcYYI7rkTw7PYRaC3xj9NPdEK2n5q6lQC+1XxEFpoByN5UXqfM9NU7GWM+AMDoSKXnZZqOJ2uucnib8OzqwtxM9i1OkZ7ZuSbM/Hta+KqxvNNX75ACWGcT5wxwalG8fvt+N0xIaKaPn66wa7pyB61uUS5bkZatgC4JiSswOeGRZUOntT9+utKt5Fj0fVhrwFiShECdhHmLSFOC9fFypyeO1jbLRoS3cap8mYQ7517OX1mEZBUXvpjw150zb5q2blFUtVedrVuGUzsXYG2yaxfv8pgItwy+QgeLDI8sAsosGMHGTq8fcexOURBYLVlsC5+3iLROuMlREC4V2Xw4Drj3otMiscIfunPuHMOjM2zqQHA/5xyTYyME2NC0bK8rUCxFVe4rO6IPnCrHMKfc1MeLhV2Hcxia0CJ1hW7yNq3TCTd5gMjnAeDVTOVOeEq412sxq7gPV4tUz0ndQ5pCrkXNZlGkKsPtHIu77/PSyYOo3VtFZ8opm4NmK1FU5b6yIodOXr99n1UT++FSUthzcA5D427h82xNJsYbZBQEAPRsD7CaElhWDoDdhKfsW+IzLDuPUxjddE6qkM+jdtItEuRib7Ox1/m07el2HGIhc+nsEMiHLkJHaH/DZNvVkabQE+9rMz+LyuZ0EsD91FjOv7lfO0CwB4NFU8iRBdtG7BR/9Ximeg3cwHQq2QNgbYGj7Od57W02v/dl26Ps9CKxJTAYxLOnFvyHS0l33lDhncOCHKKvS6/GcAlNkxXBePdL6DbRWgAX8eB1caGmAcJ9ME+SjdJwVC0GDu/O3kK7QMuLAeea4koE59/cT/69H0kNRcXjYscNSRrlQ35PXGGkKxI1bIj0ao6yZOujUHfBUXVkFPuelSp4oLGDVS3eBk5blInxBlnonfNgbCv/6671fUT4CriF0Rv1Gkz/5BXtc8xrPzdh244m73EBoND7CdgjvyfbeigA5a3vwAVLr/7ZkRe9dJ8eaDlKG7hhMHk0KF9FYkw1UYXgPbZvN5rmCJAtRq9DtcGKwe4js6yoymW64/pevAP5wARENYpgNU3ZceDDWmRdnofqmDy8Z1funeHQlKPkCsc8DqG88cimaxU0W204OPUpPF5eIQW1Tsi5dKRwrcFbVOWyUBFtuDHVQ8GgaluXkQjAKDt8+D24uQleBHAeW6KNcHR9ML7sj5yHyskcw4QRJkhNtmYAd03W16LUr+MG/EJZHHT1UCiGqd6zKWvVFpvuNRjOxXjk3PHvn6x0M71sjfP9cM74CqeaPL4XJj9cILPauLxQr2Ve3NLyCjssRhbgrhpnUc89ONuGA6olEFYPZWK8AS+f+aQUxWv6gU2lR6wI2I27D7qx8eI43spRYrnjKja2xH7lkfvYVkyMN7SFcjiInlvqi7NB1lJcNc6invsw1gUIrLFzLCbfk273hykUw/rObUx6VBGw9z+/B4f37OqROd7KUdoUX+a+lGHKI285CF+1tx33GWKo2mQejbOo5z5M7zOwRi2uwrk39GGRkx8tZIpjxZUIVSiGNUTNxqRHaf4p8M0y1gKYu93YUYs3zUuR4diBTaX4ThMxzSaqUZQJ3SqbxrlZNKGtQKNeY78n8Td5B1ivxXD+zf2k0Dp/Zfiy6GxMejpZUFgUBLf4chRlw1w2Q4NGtPZpNYJnRkfYNVTJ4ue1GJ7ZNoJWnKvFVTJudpAaJ6cZo7jGQLnglBJQF9Rzb+zPvEtKoWi1E7SyWJmxMelNHt+LNjGlvo9hJYC//fZb+Punv8uUDUxTxKifAqTbkc+fwtB3Rf3rEYB0BCn6UwOApwALV36nrWvwmvguIM/uKZBlGReufFF4vYTR0VE4e/Ys67uqvUzXLSMI4OHDVDxfCOYKM4tuGOo+25j0JsbXGi2oMf82TmcrAby6uop+TnlUqc83A75urWzPaHkZddaiDGszxgAPyrQw9fEiPElWu3+zyaIre91njklP3RW8feRFuH77vpOpZeCJGIHhZRibMQY2MHV9oN6vrluLyKJbWl5Bv/dCvba2yysxOpMetiu4fLPpnFYfBHAgA9eRlrcZo3qeMm9NNyNqrQ71Hdt2mumkKfxqvZURlkUnxsLClfKaIE1j33cKfxDAgQyY3e/G3QeZbVaeZoyYJlHmrelmhuq2rabCA6wJ0W0jFVaNbvV4YiyUte4zJwbYd6p9EMCBDNgKj9U/vvDWgUwjT679K08sdMA/WLftyzebcOJQA67fvg/N9ZKV7aQD2+MKxJUIzQiVtcFhigefnWvCOx8sZOzZqnbrO9U+COBAD5Q/BeuVlafCGaYxoNE0gUJQ3zPVbfv67fuZnc7DpQTiKv2ihs3pKjRfTsNb36n2zj3hApsTGwGYZ6JhGkMQvv1Dfta1uKoVPthuJemkUCVeWArQ7bs2DJh2Y2qBIp91rYMGHHAmT3QDt1pUoDjk0oxUxbMX1rPlMHQdJYYpCUenSGDarU/TStCAAyxUXSdvhTNZkwAAUpsKFMeXF1/vmpB03bKphVbuKIExLB0zqPvDUv99EwRwIAM2Ed8+8iLsHNtoIio3Np2da8LRi9d6Wn5zkCe+S0ucgD90W2udcJ4Yb8BnZ14lE5Nk7VKMk7JB3d8vf6pvG+aDYIIIZMAiGwAALt/cEKyisemNuw+MXTt0hGiI8kBtrTnZYabogJ4Qr5KFG5rur8gCU0EABzJgE/HoxWuol/w3X3xtDN3RIWtIIQqivJjsnqboAHmhLeN7pu7PteUXl2CCCLDQOWJsvq8i29/KNikDfEzRAfJ4GOR7tjWV+eqoThE04ACL+liM5vZXIgCsQxM3QiJEQ5Qbm+23Tku2TWsuCnENg25eKwgacIAF5SPbNlLJODDiagSPn66gmobqsAMArSc9MDjE9rvZavfUjXCJ78UcXYOGo8lSioRow+TifJYJGnCAxSMi9/9JsgqXTh7MNGmlCnFjTQxFu6ZAufBVeEZo0e2kU7pwQ9fmtZw2TPLuId790gHs+EEAB1jovNzy1vPoxWsZU4WsaWBNDAFCMZ4y4mP7rTqxyhZu6Nq81rQ4qfcdVUdGseMHARxgwc2BzzNpy+gd38pwCs+4lG8c1HtWZX9ciWBpecVYFxmzbVNtmMQ454ZXBgEc0CJPsPpYDNtGKtred6ZJq3PEBOFbLrBFN4K1d3j04rVMuUpu+cZBveco2mhEKvoXit0axyknzwWqDZMY54U15QxsHdRt1MOlBGpxFS6tF92WvycLabVUoawpY63NA+Vl20ilRwDLJiO1FxpA1ka8oxaTtYMHgWhEevTitcx16ezbHFOKPM65UR9BAAdIOE4YTEjH1QjqtTijKc/ONbN1LQOlQO1unXRW4fGyfgtNvUoheGbnmvB4ecXzlfrB1lRGmRSq65qwKOcpfB3c8MoQhhYg4QxSqlThM9tGeoq9iO9iRbwBQhTEIFHDzVrtxCh8OcecvnqntLsdXXgZhqkinNCIZVPGiUONjagP4jEEARwg4QxSG02C+m4EG1vDQP8poh6HEOhlRVdgCENXMQ3bJZ6/sgiXbzY3TBWE3TsI4EAGEWDebLWNZShtNAlbrSNQPLNzTWdBqfOl6WJ+y6AT2xZWpwQ2FVbXaiesRS0I4EAGWXtJYWOiYYPURpOw1ToCxSJMD1ziSgQ7x+KuwHr7yIva7DaxPZepxVWt4O4nopSmaiqjvosJ7Lyms+CEC2TAkiUa9VrXTKDGforGjaZ6AZyyhoH+YWN6qNdiOP/m/sy7OrxnF9rMEmBtzIikBfl9L1z5wsv1u3D04jXnsUfVusDi47fHFbR2ikoQwAEW30iebax7LrdzwDB1yt3s6GJVd47F0Fqi470FE+MNuHH3QSYkTS7Yrv52kG3pbYvxmKCUCoCsYMYIAjjAQthpfdUHCAweKlZV3u0AbPgEqGLll282e4RvBGsRAGUfD77GLbbIzM41e2Ko09VVNB4v2IADGXR2Wtv4SR8VowLFwLHJmyqioanGAPDrz+/1vG95HJQJX2UlZcQzkxM9oihCZa2VBlypbD15TeWty86pzcTo6ChceDPbkkis8Jz6AIKiuwkE8sGxyZt2PDoBJt632rZqOa3AaLSa+T451yw/p1hOs/KrPhbnsgtjoLZ1HwL4+eefh3PnzrlfmUSRfZZ8IMKwKCJY6yrLvY+y36+MTkvlFuUBCOaKYcBkkzfteEwpt1jbqvef/AUAZE0dujkXGc6lHgsgqwCoxNUIvn9iVw+Cg41WPRCV1qXQs24rW8Q211RAWhRk5tyHz8LWRWO6Vpv4yaK7CQSKh4rRrkQRzM41WYXWuW2rJo/vJXeVsnOLcywAfZRHo16DZ0ZHMpmZedoNCTlkE+dsJYBbS4kXQWfbZ0knFIoSbkLQ1Gtx5m9C4+PeR9F9pXzCuVYqflJdCGsxPrxC4sXwQAnYTpp2tUVTPCyVkKGOg4nxBrx95EUy+WdivIHOR+xYAObMS6rJgIuCIMshlDTN2lzA0gTRbLVhxUMYh49CGLoi3+2kA+98sOB0bTJie0aZD0w1Qal/mz7Pi1qdLE1BW0KSc03NVltbNxWz92LElSgkXgwR4j1jsb5iDopFGNvy1+IqnDjU6LEBi89VZx9nzJ5/cz/bBGbyV3D9GRzzoUnb/vq7+3exv1kJ4NUc7cdlbBw5AG4CTF6hdUJUhvoOZSfj3oft/eYBq04m4CyaOjubvMNQj8EN6n92+0iw/w4ZXGVD59Q7vGcXOf+4ZU9N51Ax+StMf5+da8LUx4usOWTStqOz3z3A/p47Dtglj9zGkQPgXuRb1pI5/ZtsPfbc+7C93zyYBKFp0eSU0cOOwdXmW4zsoED54CoRlLKic/bZOmu5yTwmYa37u86Bh10b9XyErZwitwB2abJnm5JqEmCisSPGN6026wXbDgK10WAnTbupl3lW7bxwBKHuO+q16p6rDLcAdbD/lhtqF8hVIlyifYo00ZmENfV3kyKDORAxgS124pXac7uw4+QWwDZN9sTLabbaRqElYxJgp4jtEcDahOe8YJtBgFXHlx0F1D30Y+vNEYScRoTiWqnQIPUYHM05FN4pL9zttk64usZ9U2M2hbXxZ2Mu9IXtHDLZyqvP7kIvLrcANlUDkoVuBBul6LACxi4rlbgG7IFFAN1oBZMQsRkERcS3+hpQJkEoWmpzg8+5mk/XSXJlEW1Bs3MshnNvZIu5BAYPd7ttUiKoeXFqZh6mr97R2monP1xAi/X7MhfaUiV6vgmWlldgdq7Zcz6drZzqipwrDtik0aihGdTt5A3JwkJlIgB4+8iL3e0T9vdj+3ZrjyFQQ9tctkymOGZfoXRqnO7OsRjqtY0SgsIjzT2XTdwvAMAz29bWdGGaatRr8KuTB2Hu568F4VtSbLbbunHMyYrDxtnEeAOe3U7rgqp86EdYp2ln/3ApQe+H2l2mnZVl7HNnAVyNImMFLJtyd3nsPZiQuHTyILw3caD79xOHGj3xhSkAXL7ZRBMMMOQXbFtY3CWfPs+AEovOC/UatJYSeGbbCFw6eRA+O/MqXL993/pcVNyvfH8Hpz6FUzPz3cWWY5YJlAPT3BPj2jSOTaYt3TgzlW50NRe6wvFtYfdDFm7//gGq4TgJ4FpchV/+9BVn4zpGXueMSUhcv32f7OCqHoN69M1WG8Z/8Skc27dbG1jgAAAHGElEQVTbqrC4ScD6HlC6iVLUuTCzA2cRkTWqMnRK2Iro5p48rk3jmJMVR40zk8BTzYWm75gwZc9yfVvq/VA7xtW2pzA01Wmms11yPeMAveaAIrARPLrrfriUwMw/fg0n//KHPUXIj+3bDdNX78DpmfnMc3DNp3ddlHQTpR/nkjGZZWRb3mYsbjQMUH4DtQi7aRzLPhJq/lDjTCfwVOUmb1gnZUO+cfdBd06bbMC6+7FxuFsJ4B/uHAOAtbCv6at34Ni+3T0ZLqoxnNuaGWBNQy0SnVAd/8WnPcWnTdeddFK4fvt+T4cInVPAJPR8xQnLDk+Mb1ptuHTyoPO5sMWWu33FKKIZZMAebpikbhyrY+NnR140Zr/JUI50zNSZN6yTUlDkovIc4esjqidXKrJaBR8g6zUF6H1QOuFQJJRQTQEy1ZAuvHUALrx1QBveZmrNLj8Hk4D1ESdsqvwEsDZRXM9FLTI7ajFqflDvESMU5SkPmNamClVV4QLYiKrBuqRwW1UB0EoI5WfKE9ZJjTtM5FajCFbTFOpjMTxJOtBONko6bBvJX8ssVyoyJ0hfflCzc00yaaLoAH1xDac/mAfd4ibnt1O9rgDsWrNzhF7eOGGTNqkKfNtzUYvM9rgCtbia+Rsn7MzGRBXoL1TrKUyoUmND3SVSJjqArPlCtHsX9mWfjlybcbeaptKusbeeTqud5A5/K6QlESVMp6/eQYWviNftBxzbuhCcum2I2pode6E7pMpNRSdi6AYUJ9nFBLXItJYSuHTyoJP2rmo9tgW2A/nQ+W84QlVgqhOBCfPTM/Nw4+6DbqQSwIYQKzrGF9O25RwFmRfqNa1yQ8X/q8+WyoTzXg9Yt+3Uqf79CFXihnWJBYQKSds5Fvdc7+TxvRBXspLj8XqwdtHMzjVJB1Y12qg+hnl9ubWUqUW1PhY7m05Uj3EQvv1FjZSZ/GgBDk59Ci+f+cTKVGiKSqDaFr3/+b3MeOtHjC8WqfD2kRfJyCaTqUz9OxaFNPLc7j3Yb60EcMUwQ0yxwdSLMmXT+YJjc1SrJWEv5dwb+3s+owLJk07al5q/1M4CYE2Ln/xoASY/XMiEpb07e4udAII9C9FRIE8CiRw+GOgvqqBLOim02ok2HBCbw6becjrFS50f/SrdqoatvjdxgEw4MplHRbasLq7fS0uiRr0GP9DYTzppqtV++lkVDIMyFUTr+w+bakkqVJUvNYvIpRaGCdPgTDrZKdVOsq1ixOfYlgp7Fo+frmQccHlTsgPlhZqrpnmis7k2W+2elN5+lm5VocyEnGgu2VRis1hYCeD6WAyfnXmVLNASAWTyo2V8ePtt4HpxdVo713ZrGjizc82efHfbWhgu5zbBbRUjUJ8F1eG22Wp7b3QYGByiH5trsanJ43u1EUWm0NVBFnHCqh5SMcKmWHsMJyfc5PG9aDSD2FK4FtXxiY0XV74e16I4WEERufvD+SuLaLERADetUb7OHUSbFhPUQOJqG+SOAjacgqET8nCDNbt0oVqJoMMY//1W0lTUzhzfP1npUZqEwkZFc1Gx9l5aEgkmxhvkilaW2E4bL64gd5Ul1UQu/ZuKlRXYPDf1OlvtBCoAgL5hWLPVQgo9CwC3VYwOrjc5mCXKh7ru7lSEDYA/zXP66h1S+Aqo0NV+ousmI+BklGKLCNWSyDkKgnKclaXgtosxP48HdvrqnYytNemkcP7KIhy9eM34e5vnhl3nKqyljor3Ilcjm/7xKzD9k1cyDgbV8VCvxbA9rsDpmXlW01XMm8wt4B4YLLI/PQKAuZ+/ho4RH4KQ8+7LIDe4mZnftNpGx6Pq5PNWC0JQNluNik3apNji5PHAknGy7cSo/cZVu0aV1LketROYP/ca+TtdRpGr9q9qK9wC7oHyIN5NUZqnySaqloYdFFwlIU9GqYqzBkxV/XF9gdx4VC7UCiXSJrHQqTxVlnIJGMsyYD6qQan4ir80aQaBctGPd2OqkqaWhh0UnPkjnpevBgq5EjFMJSC5+CxILl8btkDo6uHmER6cUnwUyapdvHARQs5X/KXvhTlQHP16N+qYwEpP+k62cIGKdZcbGlx4ay1zz5e8KiQV2ZYiWvwA4FsqXdpknm0F9tul5RVjoWn5/FyK8BT7jL8clBMlYIeP6AYu8pigwhcH7SfgzqujF695k1dRatFUM4qi+wCAevPyMPr8f3aI+tvyt//xps9zxbtfOoD1Z0o7K8vJ/a9u+TxXpfbcrpHndu+hsmCKPr8N6LWm6erKd/fvUg4E3zz//PPjEeNZBfKTpunqt99+OzeIc/dzDhaBo7zak6ZpxtBtJYADgUAg4I+gbQQCgcCACAI4EAgEBkQQwIFAIDAgggAOBAKBAREEcCAQCAyIIIADgUBgQAQBHAgEAgMiCOBAIBAYEEEABwKBwID4/wECO5YSdxEIxQAAAABJRU5ErkJggg==\n",
            "text/plain": [
              "<Figure size 432x432 with 1 Axes>"
            ]
          },
          "metadata": {
            "tags": []
          }
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "colab_type": "text",
        "id": "yhW1jzfNoR4w"
      },
      "source": [
        "As a sanity check, we'll plot the pairwise distances between all observations in the replay buffer. We expect to see a range of values from 1 to 20. Distributional RL implicitly caps the maximum predicted distance by the largest bin. We've used 20 bins, so the critic predicts 20 for all states that are at least 20 steps away from one another."
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "cellView": "form",
        "colab_type": "code",
        "id": "gIo_CYsZu6Qy",
        "colab": {}
      },
      "source": [
        "#@title Compute the pairwise distances { vertical-output: true}\n",
        "pdist = agent._get_pairwise_dist(rb_vec, aggregate=None).numpy()\n",
        "plt.figure(figsize=(6, 3))\n",
        "plt.hist(pdist.flatten(), bins=range(20))\n",
        "plt.xlabel('predicted distance')\n",
        "plt.ylabel('number of (s, g) pairs')\n",
        "plt.show()"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "colab_type": "text",
        "id": "bWVW45bzozca"
      },
      "source": [
        "With these distances, we can construct a graph. Nodes in the graph are observations in our replay buffer. We connect observations with edges whose lengths are equal to the predicted distance between those observations. Since it is hard to visualize the edge lengths, we included a slider that allows you to only show edges whose predicted length is less than some threshold.\n",
        "\n",
        "Our method learns a collection of critics, each of which makes an independent prediction for the distance between two states. Because each network may make bad predictions for pairs of states it hasn't seen before, we act in a *risk-averse* manner by using the maximum predicted distance across our ensemble. That is, we act pessimistically, only adding an edge if *all* critics think that this pair of states is nearby. "
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "cellView": "both",
        "colab_type": "code",
        "id": "30X6CutHy_9W",
        "colab": {}
      },
      "source": [
        "#@title Graph Construction { vertical-output: true, run: \"auto\" }\n",
        "cutoff = 13 #@param {min:0, max: 20, type:\"slider\"}\n",
        "# To make visualization easier, we only display the shortest edges for each\n",
        "# node. We will use all edges for planning.\n",
        "edges_to_display = 8\n",
        "plt.figure(figsize=(6, 6))\n",
        "\n",
        "plot_walls(eval_tf_env.pyenv.envs[0].env.walls)\n",
        "pdist_combined = np.max(pdist, axis=0)\n",
        "plt.scatter(*rb_vec.T)\n",
        "for i, s_i in enumerate(tqdm.tqdm_notebook(rb_vec)):\n",
        "  for count, j in enumerate(np.argsort(pdist_combined[i])):\n",
        "    if count < edges_to_display and pdist_combined[i, j] < cutoff:\n",
        "      s_j = rb_vec[j]\n",
        "      plt.plot([s_i[0], s_j[0]], [s_i[1], s_j[1]], c='k', alpha=0.5)\n",
        "\n",
        "plt.show()"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "colab_type": "text",
        "id": "ZAfUMydwp16_"
      },
      "source": [
        "We can also visualize the predictions from each critic. Note that while each critic may make incorrect decisions for distant states, their predictions in aggregate are correct."
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "cellView": "form",
        "colab_type": "code",
        "id": "x_yH1vFNcuRj",
        "colab": {}
      },
      "source": [
        "#@title Ensemble of Critics { vertical-output: true, run: \"auto\" }\n",
        "cutoff = 5 #@param {min:0, max: 20, type:\"slider\"}\n",
        "edges_to_display = 8\n",
        "plt.figure(figsize=(15, 4))\n",
        "\n",
        "for col_index in range(agent._ensemble_size):\n",
        "  plt.subplot(1, agent._ensemble_size, col_index + 1)\n",
        "  plot_walls(eval_tf_env.pyenv.envs[0].env.walls)\n",
        "  plt.title('critic %d' % (col_index + 1))\n",
        "\n",
        "  plt.scatter(*rb_vec.T)\n",
        "  desc='critic %d / %d' % (col_index + 1, agent._ensemble_size)\n",
        "  for i, s_i in enumerate(tqdm.tqdm_notebook(rb_vec, desc=desc)):\n",
        "    for count, j in enumerate(np.argsort(pdist[col_index, i])):\n",
        "      if count < edges_to_display and pdist[col_index, i, j] < cutoff:\n",
        "        s_j = rb_vec[j]\n",
        "        plt.plot([s_i[0], s_j[0]], [s_i[1], s_j[1]], c='k', alpha=0.5)\n",
        "\n",
        "plt.show()"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "colab_type": "text",
        "id": "NHCpUMJmSDzq"
      },
      "source": [
        "### Search Policy\n",
        "Now, we will combine the goal-conditioned policy and the distance estimates to form a search policy. Internally, this policy performs search over the replay buffer to find a set of waypoints leading to the goal. It then takes actions to reach each waypoint in turn. Because the search happens internally, this policy is a drop-in replacement for any other goal-conditioned policy.\n",
        "\n",
        "We used a *closed loop* policy in our paper, recomputing the search path after each step. Below, we implement both the original closed loop version as well as an *open loop* version, which only performs search once at the start of the episode. The open loop version is much faster, but may perform worse in stochastic environments where replanning is important."
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "cellView": "both",
        "colab_type": "code",
        "id": "tXFDgreyOodM",
        "colab": {}
      },
      "source": [
        "#@title Implement the search policy\n",
        "class SearchPolicy(tf_policy.Base):\n",
        "\n",
        "  def __init__(self, agent, open_loop=False):\n",
        "    self._agent = agent\n",
        "    self._open_loop = open_loop\n",
        "    self._g = self._build_graph()\n",
        "    super(SearchPolicy, self).__init__(agent.policy.time_step_spec,\n",
        "                                       agent.policy.action_spec)\n",
        "\n",
        "  def _build_graph(self):\n",
        "    g = nx.DiGraph()\n",
        "    pdist_combined = np.max(pdist, axis=0)\n",
        "    for i, s_i in enumerate(rb_vec):\n",
        "      for j, s_j in enumerate(rb_vec):\n",
        "        length = pdist_combined[i, j]\n",
        "        if length < self._agent._max_search_steps:\n",
        "          g.add_edge(i, j, weight=length)\n",
        "    return g\n",
        "\n",
        "  def _get_path(self, time_step):\n",
        "    start_to_rb = agent._get_pairwise_dist(ts.observation['observation'],\n",
        "                                           rb_vec,\n",
        "                                           aggregate='min',\n",
        "                                           masked=True).numpy().flatten()\n",
        "    rb_to_goal = agent._get_pairwise_dist(rb_vec,\n",
        "                                          ts.observation['goal'],\n",
        "                                          aggregate='min',\n",
        "                                          masked=True).numpy().flatten()\n",
        "\n",
        "    g2 = self._g.copy()\n",
        "    for i, (dist_from_start, dist_to_goal) in enumerate(zip(start_to_rb, rb_to_goal)):\n",
        "      if dist_from_start < self._agent._max_search_steps:\n",
        "        g2.add_edge('start', i, weight=dist_from_start)\n",
        "      if dist_to_goal < self._agent._max_search_steps:\n",
        "        g2.add_edge(i, 'goal', weight=dist_to_goal)\n",
        "    path = nx.shortest_path(g2, 'start', 'goal')\n",
        "    edge_lengths = []\n",
        "    for (i, j) in zip(path[:-1], path[1:]):\n",
        "      edge_lengths.append(g2[i][j]['weight'])\n",
        "    wypt_to_goal_dist = np.cumsum(edge_lengths[::-1])[::-1]  # Reverse CumSum\n",
        "    waypoint_vec = list(path)[1:-1]\n",
        "    return waypoint_vec, wypt_to_goal_dist[1:]\n",
        "\n",
        "  def _action(self, time_step, policy_state=(), seed=None):\n",
        "    goal = time_step.observation['goal']\n",
        "    dist_to_goal = self._agent._get_dist_to_goal(time_step)[0].numpy()\n",
        "    if self._open_loop:\n",
        "      if time_step.is_first():\n",
        "        self._waypoint_vec, self._wypt_to_goal_dist_vec = self._get_path(time_step)\n",
        "        self._waypoint_counter = 0\n",
        "      waypoint = rb_vec[self._waypoint_vec[self._waypoint_counter]]\n",
        "      time_step.observation['goal'] = waypoint[None]\n",
        "      dist_to_waypoint = self._agent._get_dist_to_goal(time_step)[0].numpy()\n",
        "      if dist_to_waypoint < self._agent._max_search_steps:\n",
        "        self._waypoint_counter = min(self._waypoint_counter + 1,\n",
        "                                   len(self._waypoint_vec) - 1)\n",
        "        waypoint = rb_vec[self._waypoint_vec[self._waypoint_counter]]\n",
        "        time_step.observation['goal'] = waypoint[None]\n",
        "        dist_to_waypoint = self._agent._get_dist_to_goal(time_step._replace())[0].numpy()\n",
        "      dist_to_goal_via_wypt = dist_to_waypoint + self._wypt_to_goal_dist_vec[self._waypoint_counter]\n",
        "    else:\n",
        "      (waypoint, dist_to_goal_via_wypt) = self._agent._get_waypoint(time_step)\n",
        "      dist_to_goal_via_wypt = dist_to_goal_via_wypt.numpy()\n",
        "\n",
        "    if (dist_to_goal_via_wypt < dist_to_goal) or \\\n",
        "        (dist_to_goal > self._agent._max_search_steps):\n",
        "      time_step.observation['goal'] = tf.convert_to_tensor(value=waypoint[None])\n",
        "    else:\n",
        "      time_step.observation['goal'] = goal\n",
        "    return self._agent.policy.action(time_step, policy_state, seed)"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "colab_type": "text",
        "id": "fc5_wBqF5QnD"
      },
      "source": [
        "Let's initialize the search policy:"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab_type": "code",
        "id": "G5kWC2q4UujP",
        "colab": {}
      },
      "source": [
        "agent.initialize_search(rb_vec, max_search_steps=7)\n",
        "search_policy = SearchPolicy(agent, open_loop=True)"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "colab_type": "text",
        "id": "RO6k_DbOzWig"
      },
      "source": [
        "Now, let's plot the search path found by the search policy:"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "cellView": "both",
        "colab_type": "code",
        "id": "9OvZr9VIRmYA",
        "colab": {}
      },
      "source": [
        "#@title Search Path. { vertical-output: false, run: \"auto\"}\n",
        "\n",
        "difficulty = 0.5 #@param {min:0, max: 1, step: 0.1, type:\"slider\"}\n",
        "max_goal_dist = eval_tf_env.pyenv.envs[0].gym.max_goal_dist\n",
        "eval_tf_env.pyenv.envs[0].gym.set_sample_goal_args(\n",
        "    prob_constraint=1.0,\n",
        "    min_dist=max(0, max_goal_dist * (difficulty - 0.05)),\n",
        "    max_dist=max_goal_dist * (difficulty + 0.05))\n",
        "ts = eval_tf_env.reset()\n",
        "start = ts.observation['observation'].numpy()[0]\n",
        "goal = ts.observation['goal'].numpy()[0]\n",
        "search_policy.action(ts)\n",
        "\n",
        "plt.figure(figsize=(6, 6))\n",
        "plot_walls(eval_tf_env.pyenv.envs[0].env.walls)\n",
        "\n",
        "waypoint_vec = [start]\n",
        "for waypoint_index in search_policy._waypoint_vec:\n",
        "  waypoint_vec.append(rb_vec[waypoint_index])\n",
        "waypoint_vec.append(goal)\n",
        "waypoint_vec = np.array(waypoint_vec)\n",
        "\n",
        "plt.scatter([start[0]], [start[1]], marker='+',\n",
        "            color='red', s=200, label='start')\n",
        "plt.scatter([goal[0]], [goal[1]], marker='*',\n",
        "            color='green', s=200, label='goal')\n",
        "plt.plot(waypoint_vec[:, 0], waypoint_vec[:, 1], 'y-s', alpha=0.3, label='waypoint')\n",
        "plt.legend(loc='lower left', bbox_to_anchor=(-0.1, -0.15), ncol=4, fontsize=16)\n",
        "plt.show()"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "ahT21PxlNL8q",
        "colab_type": "text"
      },
      "source": [
        "Code to calculate success rate for agent."
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "bKvWnIRANKxJ",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "\"\"\"\n",
        "This loop does not need many parameter changes. Three things to consider\n",
        "changing are max_steps, the difficulties to try, and number of samples N.\n",
        "Could also add another loop to test how max_steps will affect performance, \n",
        "but the result of adding more steps seems fairly obvious.\n",
        "\"\"\"\n",
        "\n",
        "max_steps = 7\n",
        "\n",
        "agent.initialize_search(rb_vec, max_search_steps=max_steps)\n",
        "search_policy = SearchPolicy(agent, open_loop=True)\n",
        "\n",
        "print(\"Searching with max steps of \", max_steps)\n",
        "print(\"==================================\")\n",
        "difficulties = [0.3, 0.4, 0.5, 0.6, 0.7, 0.8]\n",
        "\n",
        "for difficulty in difficulties:\n",
        "  print(\"Difficulty = \", difficulty)\n",
        "  N = 20\n",
        "  success = 0\n",
        "\n",
        "  for i in range(N):\n",
        "    try:\n",
        "      max_goal_dist = eval_tf_env.pyenv.envs[0].gym.max_goal_dist\n",
        "      eval_tf_env.pyenv.envs[0].gym.set_sample_goal_args(\n",
        "          prob_constraint=1.0,\n",
        "          min_dist=max(0, max_goal_dist * (difficulty - 0.05)),\n",
        "          max_dist=max_goal_dist * (difficulty + 0.05))\n",
        "      ts = eval_tf_env.reset()\n",
        "      start = ts.observation['observation'].numpy()[0]\n",
        "      goal = ts.observation['goal'].numpy()[0]\n",
        "      search_policy.action(ts)\n",
        "\n",
        "      print(\"Path found\")\n",
        "      success += 1\n",
        "    except:\n",
        "      print(\"Path not found\")\n",
        "      continue\n",
        "\n",
        "  print(\"Path success rate is \", success / N)\n",
        "  print(\"==================================\")"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "colab_type": "text",
        "id": "UIq3YzOeqYkW"
      },
      "source": [
        "Now, we'll use that path to guide the agent towards the goal. On the left, we plot rollouts from the baseline goal-conditioned policy. On the right, we use that same policy to reach each of the waypoints leading to the goal. As before, the slider allows you to change the distance to the goal. Note that only the search policy is able to reach distant goals."
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "cellView": "form",
        "colab_type": "code",
        "id": "IK4ukv7TDoWE",
        "colab": {}
      },
      "source": [
        "#@title Rollouts with Search. { vertical-output: true, run: \"auto\"}\n",
        "eval_tf_env.pyenv.envs[0]._duration = 300\n",
        "seed = np.random.randint(0, 1000000)\n",
        "\n",
        "difficulty = 0.8 #@param {min:0, max: 1, step: 0.1, type:\"slider\"}\n",
        "max_goal_dist = eval_tf_env.pyenv.envs[0].gym.max_goal_dist\n",
        "eval_tf_env.pyenv.envs[0].gym.set_sample_goal_args(\n",
        "    prob_constraint=1.0,\n",
        "    min_dist=max(0, max_goal_dist * (difficulty - 0.05)),\n",
        "    max_dist=max_goal_dist * (difficulty + 0.05))\n",
        "\n",
        "\n",
        "plt.figure(figsize=(12, 5))\n",
        "for col_index in range(2):\n",
        "  title = 'no search' if col_index == 0 else 'search'\n",
        "  plt.subplot(1, 2, col_index + 1)\n",
        "  plot_walls(eval_tf_env.pyenv.envs[0].env.walls)\n",
        "  use_search = (col_index == 1)\n",
        "  np.random.seed(seed)\n",
        "  ts = eval_tf_env.reset()\n",
        "  goal = ts.observation['goal'].numpy()[0]\n",
        "  start = ts.observation['observation'].numpy()[0]\n",
        "  obs_vec = []\n",
        "  for _ in tqdm.tnrange(eval_tf_env.pyenv.envs[0]._duration,\n",
        "                        desc='rollout %d / 2' % (col_index + 1)):\n",
        "    if ts.is_last():\n",
        "      break\n",
        "    obs_vec.append(ts.observation['observation'].numpy()[0])\n",
        "    if use_search:\n",
        "      action = search_policy.action(ts)\n",
        "    else:\n",
        "      action = agent.policy.action(ts)\n",
        "\n",
        "    ts = eval_tf_env.step(action)\n",
        "\n",
        "  obs_vec = np.array(obs_vec)\n",
        "\n",
        "  plt.plot(obs_vec[:, 0], obs_vec[:, 1], 'b-o', alpha=0.3)\n",
        "  plt.scatter([obs_vec[0, 0]], [obs_vec[0, 1]], marker='+',\n",
        "              color='red', s=200, label='start')\n",
        "  plt.scatter([obs_vec[-1, 0]], [obs_vec[-1, 1]], marker='+',\n",
        "              color='green', s=200, label='end')\n",
        "  plt.scatter([goal[0]], [goal[1]], marker='*',\n",
        "              color='green', s=200, label='goal')\n",
        "\n",
        "  plt.title(title, fontsize=24)\n",
        "  if use_search:\n",
        "    waypoint_vec = [start]\n",
        "    for waypoint_index in search_policy._waypoint_vec:\n",
        "      waypoint_vec.append(rb_vec[waypoint_index])\n",
        "    waypoint_vec.append(goal)\n",
        "    waypoint_vec = np.array(waypoint_vec)\n",
        "\n",
        "    plt.plot(waypoint_vec[:, 0], waypoint_vec[:, 1], 'y-s', alpha=0.3, label='waypoint')\n",
        "    plt.legend(loc='lower left', bbox_to_anchor=(-0.8, -0.15), ncol=4, fontsize=16)\n",
        "plt.show()"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "colab_type": "text",
        "id": "2JvMnn7X2mrf"
      },
      "source": [
        "-----------------------------\n",
        "## Next Steps and Open Problems\n",
        "We encourage readers to play around with the experiments. To get started, here are a few questions:\n",
        "1. What effect does distributional RL have on learning distances? (Hint: change `use_distributional_rl` when initializing the `UvfAgent`.)\n",
        "2. What is the effect of using more critic networks in the ensemble? (Hint: change `ensemble_size` when initializing the `UvfAgent`)\n",
        "3. While we applied planning *after* training a goal-conditioned policy, can planning be used to accelerate learning of the goal-conditioned policy? (Hint: Set `UvfAgent.collect_policy` to be the `SearchPolicy`)\n",
        "4. Can you be smart about which observations to include in the replay buffer to make search faster? (Hint: Simple behavior cloning may be enough.)\n",
        "7. Can the search policy be distilled into a single neural network policy?\n",
        "5. What tricks are important for learning distances with RL?\n",
        "6. How can more sophisticated planning algorithms be used in a similar framework?\n",
        "8. Can the same idea by applied to other domains such, as manipulation?\n"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab_type": "code",
        "id": "r4FA69AqWJ4t",
        "colab": {}
      },
      "source": [
        ""
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab_type": "code",
        "id": "0_QyWB6JqORM",
        "colab": {}
      },
      "source": [
        ""
      ],
      "execution_count": 0,
      "outputs": []
    }
  ]
}