ANER_dataset_test.ipynb

{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "colab": {
      "name": "ANER_dataset_test.ipynb",
      "provenance": [],
      "collapsed_sections": [
        "PsQxjGC5dqb8"
      ],
      "mount_file_id": "1i2dlrc03WCSJavRcY43nyCyGqPI7K9aO",
      "authorship_tag": "ABX9TyPdPzHLJGnx2817R60dN5Ee",
      "include_colab_link": true
    },
    "kernelspec": {
      "name": "python3",
      "display_name": "Python 3"
    },
    "language_info": {
      "name": "python"
    },
    "widgets": {
      "application/vnd.jupyter.widget-state+json": {
        "df9d5fb273e146d88b3dafe76efe9c20": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "state": {
            "_view_name": "HBoxView",
            "_dom_classes": [],
            "_model_name": "HBoxModel",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "box_style": "",
            "layout": "IPY_MODEL_8c3b573133f043a0a35aad910af4ce93",
            "_model_module": "@jupyter-widgets/controls",
            "children": [
              "IPY_MODEL_70c506c3e18748598a72d869e0716a9b",
              "IPY_MODEL_76e5e582c7f0421f98d8f6ebf508934b"
            ]
          }
        },
        "8c3b573133f043a0a35aad910af4ce93": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "70c506c3e18748598a72d869e0716a9b": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "state": {
            "_view_name": "ProgressView",
            "style": "IPY_MODEL_884b58371df54b02a3661b67a2b43ec7",
            "_dom_classes": [],
            "description": "Downloading: 100%",
            "_model_name": "FloatProgressModel",
            "bar_style": "success",
            "max": 381,
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": 381,
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "orientation": "horizontal",
            "min": 0,
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_46c2ce18ef4144b0856094d7e4a5c770"
          }
        },
        "76e5e582c7f0421f98d8f6ebf508934b": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "state": {
            "_view_name": "HTMLView",
            "style": "IPY_MODEL_f9922ec0e0374677bcbc362443fc6334",
            "_dom_classes": [],
            "description": "",
            "_model_name": "HTMLModel",
            "placeholder": "​",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": " 381/381 [00:00&lt;00:00, 515B/s]",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_e5989cb7e3e44349855fa0e8fe6eaf18"
          }
        },
        "884b58371df54b02a3661b67a2b43ec7": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "ProgressStyleModel",
            "description_width": "initial",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "bar_color": null,
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "46c2ce18ef4144b0856094d7e4a5c770": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "f9922ec0e0374677bcbc362443fc6334": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "DescriptionStyleModel",
            "description_width": "",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "e5989cb7e3e44349855fa0e8fe6eaf18": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "4733c7ac987f403bbc63bdf759df1af4": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "state": {
            "_view_name": "HBoxView",
            "_dom_classes": [],
            "_model_name": "HBoxModel",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "box_style": "",
            "layout": "IPY_MODEL_5d77e52b0c114297aacce461a2691e5d",
            "_model_module": "@jupyter-widgets/controls",
            "children": [
              "IPY_MODEL_15c2ac3a22564393b8c0ef811d148a30",
              "IPY_MODEL_a2b3a067e1744b3aa15bd18d6dd1cba8"
            ]
          }
        },
        "5d77e52b0c114297aacce461a2691e5d": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "15c2ac3a22564393b8c0ef811d148a30": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "state": {
            "_view_name": "ProgressView",
            "style": "IPY_MODEL_9519f24a848a4a869b803cc92ae2cc6d",
            "_dom_classes": [],
            "description": "Downloading: 100%",
            "_model_name": "FloatProgressModel",
            "bar_style": "success",
            "max": 384,
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": 384,
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "orientation": "horizontal",
            "min": 0,
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_d62b56398c65458daa0d85b3a5adb8bb"
          }
        },
        "a2b3a067e1744b3aa15bd18d6dd1cba8": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "state": {
            "_view_name": "HTMLView",
            "style": "IPY_MODEL_a89eae03c1b54bd2ac0e6f9c4e7f242d",
            "_dom_classes": [],
            "description": "",
            "_model_name": "HTMLModel",
            "placeholder": "​",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": " 384/384 [00:23&lt;00:00, 16.3B/s]",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_e5fd0e3f3bf84e4a8840723659b73a5c"
          }
        },
        "9519f24a848a4a869b803cc92ae2cc6d": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "ProgressStyleModel",
            "description_width": "initial",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "bar_color": null,
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "d62b56398c65458daa0d85b3a5adb8bb": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "a89eae03c1b54bd2ac0e6f9c4e7f242d": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "DescriptionStyleModel",
            "description_width": "",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "e5fd0e3f3bf84e4a8840723659b73a5c": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "82de917239834629ba8a095bbf5e6ec5": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "state": {
            "_view_name": "HBoxView",
            "_dom_classes": [],
            "_model_name": "HBoxModel",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "box_style": "",
            "layout": "IPY_MODEL_95dc9a0210954d9ab836f465d00603b1",
            "_model_module": "@jupyter-widgets/controls",
            "children": [
              "IPY_MODEL_5adff5402e5f4e59872b87c41a1735ef",
              "IPY_MODEL_eca35d9563f24c638a9b8cdc9a81cb24"
            ]
          }
        },
        "95dc9a0210954d9ab836f465d00603b1": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "5adff5402e5f4e59872b87c41a1735ef": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "state": {
            "_view_name": "ProgressView",
            "style": "IPY_MODEL_7b55484d972c43a8a009eddfbbe9e905",
            "_dom_classes": [],
            "description": "Downloading: 100%",
            "_model_name": "FloatProgressModel",
            "bar_style": "success",
            "max": 824793,
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": 824793,
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "orientation": "horizontal",
            "min": 0,
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_ab1fc188d2914c099c162337ce51340f"
          }
        },
        "eca35d9563f24c638a9b8cdc9a81cb24": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "state": {
            "_view_name": "HTMLView",
            "style": "IPY_MODEL_501d82c0fe9348919691e41c62531530",
            "_dom_classes": [],
            "description": "",
            "_model_name": "HTMLModel",
            "placeholder": "​",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": " 825k/825k [00:00&lt;00:00, 835kB/s]",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_f219e5df9ad54fc58cde262cd68bd46b"
          }
        },
        "7b55484d972c43a8a009eddfbbe9e905": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "ProgressStyleModel",
            "description_width": "initial",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "bar_color": null,
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "ab1fc188d2914c099c162337ce51340f": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "501d82c0fe9348919691e41c62531530": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "DescriptionStyleModel",
            "description_width": "",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "f219e5df9ad54fc58cde262cd68bd46b": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "c116df5c51564092a690bebb30904422": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "state": {
            "_view_name": "HBoxView",
            "_dom_classes": [],
            "_model_name": "HBoxModel",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "box_style": "",
            "layout": "IPY_MODEL_0f28247d6dc84fbf8d70deeaa62e257e",
            "_model_module": "@jupyter-widgets/controls",
            "children": [
              "IPY_MODEL_252431a5f886477c902f7b396abcfad4",
              "IPY_MODEL_477fd59f6c1d4f1fac368044112a7010"
            ]
          }
        },
        "0f28247d6dc84fbf8d70deeaa62e257e": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "252431a5f886477c902f7b396abcfad4": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "state": {
            "_view_name": "ProgressView",
            "style": "IPY_MODEL_3e7fe1d8bb734b93960be974caa306dd",
            "_dom_classes": [],
            "description": "Downloading: 100%",
            "_model_name": "FloatProgressModel",
            "bar_style": "success",
            "max": 2642362,
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": 2642362,
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "orientation": "horizontal",
            "min": 0,
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_806b784d9b7c4cc9807c6340ca84a4e7"
          }
        },
        "477fd59f6c1d4f1fac368044112a7010": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "state": {
            "_view_name": "HTMLView",
            "style": "IPY_MODEL_08a0d2434eef4f2eb888d1aabaa12515",
            "_dom_classes": [],
            "description": "",
            "_model_name": "HTMLModel",
            "placeholder": "​",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": " 2.64M/2.64M [00:21&lt;00:00, 123kB/s]",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_6e863e924560410188f83baf25ae1131"
          }
        },
        "3e7fe1d8bb734b93960be974caa306dd": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "ProgressStyleModel",
            "description_width": "initial",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "bar_color": null,
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "806b784d9b7c4cc9807c6340ca84a4e7": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "08a0d2434eef4f2eb888d1aabaa12515": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "DescriptionStyleModel",
            "description_width": "",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "6e863e924560410188f83baf25ae1131": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "3936fbd6175e4ca1b082f14d9d3943b5": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "state": {
            "_view_name": "HBoxView",
            "_dom_classes": [],
            "_model_name": "HBoxModel",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "box_style": "",
            "layout": "IPY_MODEL_ea929c45235545f3ab5a5ac5e29625d4",
            "_model_module": "@jupyter-widgets/controls",
            "children": [
              "IPY_MODEL_5ef71b0e60a046ecb4874e1a988ac524",
              "IPY_MODEL_2337733c66a74b47a2e86316c49579c9"
            ]
          }
        },
        "ea929c45235545f3ab5a5ac5e29625d4": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "5ef71b0e60a046ecb4874e1a988ac524": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "state": {
            "_view_name": "ProgressView",
            "style": "IPY_MODEL_ae6173bac98845ffacde6591d2cacef7",
            "_dom_classes": [],
            "description": "Downloading: 100%",
            "_model_name": "FloatProgressModel",
            "bar_style": "success",
            "max": 112,
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": 112,
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "orientation": "horizontal",
            "min": 0,
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_4cd7b11865904ebfbab5c9a611a61eb1"
          }
        },
        "2337733c66a74b47a2e86316c49579c9": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "state": {
            "_view_name": "HTMLView",
            "style": "IPY_MODEL_4d0608fe5bfc4b45b620fbe93dbb03e4",
            "_dom_classes": [],
            "description": "",
            "_model_name": "HTMLModel",
            "placeholder": "​",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": " 112/112 [00:00&lt;00:00, 124B/s]",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_14c9a60d913541f3962210deae19e470"
          }
        },
        "ae6173bac98845ffacde6591d2cacef7": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "ProgressStyleModel",
            "description_width": "initial",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "bar_color": null,
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "4cd7b11865904ebfbab5c9a611a61eb1": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "4d0608fe5bfc4b45b620fbe93dbb03e4": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "DescriptionStyleModel",
            "description_width": "",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "14c9a60d913541f3962210deae19e470": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "ddcf4ff7e10d40e5a2bd30392224f58e": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "state": {
            "_view_name": "HBoxView",
            "_dom_classes": [],
            "_model_name": "HBoxModel",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "box_style": "",
            "layout": "IPY_MODEL_3053455a9b654d07a23c43e6aafcece0",
            "_model_module": "@jupyter-widgets/controls",
            "children": [
              "IPY_MODEL_eda598850eee46babb4d23c65325a857",
              "IPY_MODEL_1ac5725662084e3bb6ff258788a99804"
            ]
          }
        },
        "3053455a9b654d07a23c43e6aafcece0": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "eda598850eee46babb4d23c65325a857": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "state": {
            "_view_name": "ProgressView",
            "style": "IPY_MODEL_b201e0ccb3ee42f094761812a9e0e7f5",
            "_dom_classes": [],
            "description": "Downloading: 100%",
            "_model_name": "FloatProgressModel",
            "bar_style": "success",
            "max": 543490667,
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": 543490667,
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "orientation": "horizontal",
            "min": 0,
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_057b5c1ab4c547ce9e0750659c412934"
          }
        },
        "1ac5725662084e3bb6ff258788a99804": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "state": {
            "_view_name": "HTMLView",
            "style": "IPY_MODEL_6a81b0b3debd4782a8932cafc30886a5",
            "_dom_classes": [],
            "description": "",
            "_model_name": "HTMLModel",
            "placeholder": "​",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": " 543M/543M [00:12&lt;00:00, 43.7MB/s]",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_a8de1de271e249458ee1ddbf567b00af"
          }
        },
        "b201e0ccb3ee42f094761812a9e0e7f5": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "ProgressStyleModel",
            "description_width": "initial",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "bar_color": null,
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "057b5c1ab4c547ce9e0750659c412934": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "6a81b0b3debd4782a8932cafc30886a5": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "DescriptionStyleModel",
            "description_width": "",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "a8de1de271e249458ee1ddbf567b00af": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "985019b915664c3d83ec3240d95e7bba": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "state": {
            "_view_name": "HBoxView",
            "_dom_classes": [],
            "_model_name": "HBoxModel",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "box_style": "",
            "layout": "IPY_MODEL_9edd99038a3046a088876566d64ff9dc",
            "_model_module": "@jupyter-widgets/controls",
            "children": [
              "IPY_MODEL_0ec74a8b82f54907862d8f28113a7f81",
              "IPY_MODEL_5baa743e23bb4a22991bbd8b120eff0a"
            ]
          }
        },
        "9edd99038a3046a088876566d64ff9dc": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "0ec74a8b82f54907862d8f28113a7f81": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "state": {
            "_view_name": "ProgressView",
            "style": "IPY_MODEL_8ccb594c40b941218fbc10652d4ac8d3",
            "_dom_classes": [],
            "description": "100%",
            "_model_name": "FloatProgressModel",
            "bar_style": "success",
            "max": 298,
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": 298,
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "orientation": "horizontal",
            "min": 0,
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_5eb15b030ad1450cb9096c23e08781c6"
          }
        },
        "5baa743e23bb4a22991bbd8b120eff0a": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "state": {
            "_view_name": "HTMLView",
            "style": "IPY_MODEL_0c723fcbd3734a1293e695176659f945",
            "_dom_classes": [],
            "description": "",
            "_model_name": "HTMLModel",
            "placeholder": "​",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": " 298/298 [2:53:15&lt;00:00, 34.89s/it]",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_d6e1256d9f48415b8816b7d390e60951"
          }
        },
        "8ccb594c40b941218fbc10652d4ac8d3": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "ProgressStyleModel",
            "description_width": "initial",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "bar_color": null,
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "5eb15b030ad1450cb9096c23e08781c6": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "0c723fcbd3734a1293e695176659f945": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "DescriptionStyleModel",
            "description_width": "",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "d6e1256d9f48415b8816b7d390e60951": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        }
      }
    }
  },
  "cells": [
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "view-in-github",
        "colab_type": "text"
      },
      "source": [
        "<a href=\"https://colab.research.google.com/github/omarhatem97/ANER_DEV/blob/main/ANER_dataset_test.ipynb\" target=\"_parent\"><img src=\"https://colab.research.google.com/assets/colab-badge.svg\" alt=\"Open In Colab\"/></a>"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "_RPpBgnURDWY"
      },
      "source": [
        "# Fine Tuning Arabert Model to do NER Task"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "Ml3jN74SzSkA"
      },
      "source": [
        "### Environment setup"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "amGAPB1SRlOL",
        "outputId": "7aa93236-ba8d-41b9-e9d3-77dd3d7c0d11"
      },
      "source": [
        "import torch\n",
        "\n",
        "# If there's a GPU available...\n",
        "if torch.cuda.is_available():    \n",
        "\n",
        "    # Tell PyTorch to use the GPU.    \n",
        "    device = torch.device(\"cuda\")\n",
        "\n",
        "    print('There are %d GPU(s) available.' % torch.cuda.device_count())\n",
        "\n",
        "    print('We will use the GPU:', torch.cuda.get_device_name(0))\n",
        "    !nvidia-smi\n",
        "\n",
        "# If not...\n",
        "else:\n",
        "    print('No GPU available, using the CPU instead.')\n",
        "    device = torch.device(\"cpu\")"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "No GPU available, using the CPU instead.\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "BKo_mTT8Q_wK",
        "outputId": "e89ab3da-f190-4add-ac51-211318a55be7"
      },
      "source": [
        "\n",
        "# get the dataset \n",
        "!gdown --id  1bYDqmEYKpVmVyGc9Vmz3W_2bsPMel6UC\n",
        "\n",
        "\n",
        "# The label_lsit file\n",
        "!gdown --id  1th3j28peQf-asgodeaGo-04aIARN0QoL\n",
        "\n",
        "\n",
        "# ANER dataset\n",
        "!gdown --id 10f2KkbvBNDrBAEv_tp1cAR9QmV8CEYv1\n",
        "\n",
        "\n",
        "\n",
        "############\n",
        "#\n",
        "#  Aqmar \n",
        "## aqmar test datast\n",
        "!gdown --id 1-1S27QQYAsFpxXDgs2slPq3D10QvgA20\n",
        "\n",
        "## aqmar label list\n",
        "!gdown --id 1Iv-FwkBt0MXsqNG1Sb7HWhslrxCSk1X4\n",
        "\n",
        "##########\n",
        "\n",
        "\n",
        "\n",
        "\n",
        "!pip install transformers \n",
        "!pip install pyarabic\n",
        "!pip install seqeval\n",
        "!git clone https://github.com/aub-mind/arabert"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Downloading...\n",
            "From: https://drive.google.com/uc?id=1bYDqmEYKpVmVyGc9Vmz3W_2bsPMel6UC\n",
            "To: /content/dataset_1.0.txt\n",
            "6.03MB [00:00, 36.8MB/s]\n",
            "Downloading...\n",
            "From: https://drive.google.com/uc?id=1th3j28peQf-asgodeaGo-04aIARN0QoL\n",
            "To: /content/label_list.txt\n",
            "100% 1.22k/1.22k [00:00<00:00, 1.51MB/s]\n",
            "Downloading...\n",
            "From: https://drive.google.com/uc?id=10f2KkbvBNDrBAEv_tp1cAR9QmV8CEYv1\n",
            "To: /content/ANER_dataset.txt\n",
            "100% 2.06M/2.06M [00:00<00:00, 106MB/s]\n",
            "Downloading...\n",
            "From: https://drive.google.com/uc?id=1-1S27QQYAsFpxXDgs2slPq3D10QvgA20\n",
            "To: /content/aqmar_test.txt\n",
            "100% 31.5k/31.5k [00:00<00:00, 77.1MB/s]\n",
            "Downloading...\n",
            "From: https://drive.google.com/uc?id=1Iv-FwkBt0MXsqNG1Sb7HWhslrxCSk1X4\n",
            "To: /content/aqmar_label_list.txt\n",
            "100% 95.0/95.0 [00:00<00:00, 475kB/s]\n",
            "Collecting transformers\n",
            "  Downloading transformers-4.9.0-py3-none-any.whl (2.6 MB)\n",
            "\u001b[K     |████████████████████████████████| 2.6 MB 13.3 MB/s \n",
            "\u001b[?25hCollecting sacremoses\n",
            "  Downloading sacremoses-0.0.45-py3-none-any.whl (895 kB)\n",
            "\u001b[K     |████████████████████████████████| 895 kB 84.3 MB/s \n",
            "\u001b[?25hRequirement already satisfied: requests in /usr/local/lib/python3.7/dist-packages (from transformers) (2.23.0)\n",
            "Requirement already satisfied: filelock in /usr/local/lib/python3.7/dist-packages (from transformers) (3.0.12)\n",
            "Requirement already satisfied: packaging in /usr/local/lib/python3.7/dist-packages (from transformers) (21.0)\n",
            "Collecting huggingface-hub==0.0.12\n",
            "  Downloading huggingface_hub-0.0.12-py3-none-any.whl (37 kB)\n",
            "Requirement already satisfied: numpy>=1.17 in /usr/local/lib/python3.7/dist-packages (from transformers) (1.19.5)\n",
            "Requirement already satisfied: regex!=2019.12.17 in /usr/local/lib/python3.7/dist-packages (from transformers) (2019.12.20)\n",
            "Collecting pyyaml>=5.1\n",
            "  Downloading PyYAML-5.4.1-cp37-cp37m-manylinux1_x86_64.whl (636 kB)\n",
            "\u001b[K     |████████████████████████████████| 636 kB 56.9 MB/s \n",
            "\u001b[?25hRequirement already satisfied: tqdm>=4.27 in /usr/local/lib/python3.7/dist-packages (from transformers) (4.41.1)\n",
            "Collecting tokenizers<0.11,>=0.10.1\n",
            "  Downloading tokenizers-0.10.3-cp37-cp37m-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_12_x86_64.manylinux2010_x86_64.whl (3.3 MB)\n",
            "\u001b[K     |████████████████████████████████| 3.3 MB 18.4 MB/s \n",
            "\u001b[?25hRequirement already satisfied: importlib-metadata in /usr/local/lib/python3.7/dist-packages (from transformers) (4.6.1)\n",
            "Requirement already satisfied: typing-extensions in /usr/local/lib/python3.7/dist-packages (from huggingface-hub==0.0.12->transformers) (3.7.4.3)\n",
            "Requirement already satisfied: pyparsing>=2.0.2 in /usr/local/lib/python3.7/dist-packages (from packaging->transformers) (2.4.7)\n",
            "Requirement already satisfied: zipp>=0.5 in /usr/local/lib/python3.7/dist-packages (from importlib-metadata->transformers) (3.5.0)\n",
            "Requirement already satisfied: certifi>=2017.4.17 in /usr/local/lib/python3.7/dist-packages (from requests->transformers) (2021.5.30)\n",
            "Requirement already satisfied: urllib3!=1.25.0,!=1.25.1,<1.26,>=1.21.1 in /usr/local/lib/python3.7/dist-packages (from requests->transformers) (1.24.3)\n",
            "Requirement already satisfied: idna<3,>=2.5 in /usr/local/lib/python3.7/dist-packages (from requests->transformers) (2.10)\n",
            "Requirement already satisfied: chardet<4,>=3.0.2 in /usr/local/lib/python3.7/dist-packages (from requests->transformers) (3.0.4)\n",
            "Requirement already satisfied: joblib in /usr/local/lib/python3.7/dist-packages (from sacremoses->transformers) (1.0.1)\n",
            "Requirement already satisfied: six in /usr/local/lib/python3.7/dist-packages (from sacremoses->transformers) (1.15.0)\n",
            "Requirement already satisfied: click in /usr/local/lib/python3.7/dist-packages (from sacremoses->transformers) (7.1.2)\n",
            "Installing collected packages: tokenizers, sacremoses, pyyaml, huggingface-hub, transformers\n",
            "  Attempting uninstall: pyyaml\n",
            "    Found existing installation: PyYAML 3.13\n",
            "    Uninstalling PyYAML-3.13:\n",
            "      Successfully uninstalled PyYAML-3.13\n",
            "Successfully installed huggingface-hub-0.0.12 pyyaml-5.4.1 sacremoses-0.0.45 tokenizers-0.10.3 transformers-4.9.0\n",
            "Collecting pyarabic\n",
            "  Downloading PyArabic-0.6.11-py3-none-any.whl (126 kB)\n",
            "\u001b[K     |████████████████████████████████| 126 kB 13.7 MB/s \n",
            "\u001b[?25hInstalling collected packages: pyarabic\n",
            "Successfully installed pyarabic-0.6.11\n",
            "Collecting seqeval\n",
            "  Downloading seqeval-1.2.2.tar.gz (43 kB)\n",
            "\u001b[K     |████████████████████████████████| 43 kB 1.5 MB/s \n",
            "\u001b[?25hRequirement already satisfied: numpy>=1.14.0 in /usr/local/lib/python3.7/dist-packages (from seqeval) (1.19.5)\n",
            "Requirement already satisfied: scikit-learn>=0.21.3 in /usr/local/lib/python3.7/dist-packages (from seqeval) (0.22.2.post1)\n",
            "Requirement already satisfied: joblib>=0.11 in /usr/local/lib/python3.7/dist-packages (from scikit-learn>=0.21.3->seqeval) (1.0.1)\n",
            "Requirement already satisfied: scipy>=0.17.0 in /usr/local/lib/python3.7/dist-packages (from scikit-learn>=0.21.3->seqeval) (1.4.1)\n",
            "Building wheels for collected packages: seqeval\n",
            "  Building wheel for seqeval (setup.py) ... \u001b[?25l\u001b[?25hdone\n",
            "  Created wheel for seqeval: filename=seqeval-1.2.2-py3-none-any.whl size=16181 sha256=69a4ffc48af3faeac024c1f75549f6a2214648109cc85e59f62e9bb64a0bce8e\n",
            "  Stored in directory: /root/.cache/pip/wheels/05/96/ee/7cac4e74f3b19e3158dce26a20a1c86b3533c43ec72a549fd7\n",
            "Successfully built seqeval\n",
            "Installing collected packages: seqeval\n",
            "Successfully installed seqeval-1.2.2\n",
            "Cloning into 'arabert'...\n",
            "remote: Enumerating objects: 530, done.\u001b[K\n",
            "remote: Counting objects: 100% (316/316), done.\u001b[K\n",
            "remote: Compressing objects: 100% (228/228), done.\u001b[K\n",
            "remote: Total 530 (delta 167), reused 226 (delta 82), pack-reused 214\u001b[K\n",
            "Receiving objects: 100% (530/530), 4.86 MiB | 26.90 MiB/s, done.\n",
            "Resolving deltas: 100% (290/290), done.\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "omPkPFWOR4yI"
      },
      "source": [
        "import pandas as pd\n",
        "import numpy as np\n",
        "from sklearn.metrics import classification_report\n",
        "from  tqdm.notebook import tqdm\n",
        "from sklearn.model_selection import train_test_split\n",
        "\n",
        "import matplotlib.pyplot as plt\n",
        "from collections import Counter\n"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "9O5GEuy9zLdb"
      },
      "source": [
        "from arabert.preprocess import ArabertPreprocessor\n",
        "import numpy as np\n",
        "\n",
        "from seqeval.metrics import accuracy_score, f1_score, precision_score, recall_score\n",
        "from transformers import AutoConfig, AutoModelForTokenClassification, AutoTokenizer, BertForTokenClassification, AdamW, get_linear_schedule_with_warmup\n",
        "from transformers import Trainer , TrainingArguments\n",
        "from transformers.trainer_utils import EvaluationStrategy\n",
        "from transformers.data.processors.utils import InputFeatures\n",
        "from torch.utils.data import Dataset\n",
        "from torch.utils.data import DataLoader\n",
        "from sklearn.utils import resample\n",
        "import logging\n",
        "import torch\n",
        "from torch import nn\n",
        "\n",
        "%load_ext google.colab.data_table"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "QKvjJKH_SLxY"
      },
      "source": [
        "## Preprocessing\n",
        "\n",
        "In this part, we take the dataset, and convert it to list of sentences using the newline token to get the end of each sentence\n"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "LvseDejXSRa5"
      },
      "source": [
        "This function will make a dataset of sentences \n",
        "\n",
        "returns: list of shape: \n",
        "  [no_sentences] *  [ [words] , [tags] ]"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "YGKoNP2iSA8j"
      },
      "source": [
        "def read_dataset(path):\n",
        "  with open(path,'r',encoding='utf-8') as f:\n",
        "    data = []\n",
        "    sentence = []\n",
        "    label = []\n",
        "    for line in f:\n",
        "      # line = line.replace(u'\\u200e','')\n",
        "      # line = line.replace(u'\\ufeff','')\n",
        "      if '[NEWLINE]' in line:\n",
        "        if len(sentence) > 0:\n",
        "          data.append([sentence,label])\n",
        "          sentence = []\n",
        "          label = []\n",
        "        continue\n",
        "      line = line.strip('\\n')\n",
        "      splits = line.split(' ')\n",
        "      sentence.append(splits[0])\n",
        "      label.append(splits[1])\n",
        "    if len(sentence) > 0:\n",
        "      data.append([sentence,label])\n",
        "  return data"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "wn5LyEZ7ZQ1S"
      },
      "source": [
        "Here we can see after converting the dataset into sentences, that we have 15763 Sentence"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "uKlNr89VSbf2",
        "outputId": "8d566260-c12f-4d8f-e14e-ed3c4b09dfbb"
      },
      "source": [
        "\n",
        "data = read_dataset('ANER_dataset.txt')\n",
        "print(len(data))\n"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "4760\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 142
        },
        "id": "_yjFiUUWSz-e",
        "outputId": "5be8ecae-9446-498b-9125-a8ec2ff192ea"
      },
      "source": [
        "df = pd.DataFrame(data, columns=['text', 'tags'])\n",
        "df.head(1)\n"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "application/vnd.google.colaboratory.module+javascript": "\n      import \"https://ssl.gstatic.com/colaboratory/data_table/a6224c040fa35dcf/data_table.js\";\n\n      window.createDataTable({\n        data: [[{\n            'v': 0,\n            'f': \"0\",\n        },\n[\"\\u0641\\u0631\\u0627\\u0646\\u0643\\u0641\\u0648\\u0631\\u062a\", \"(\", \"\\u062f\", \"\\u0628\", \"\\u0627\", \")\", \"\\u0627\\u0639\\u0644\\u0646\", \"\\u0627\\u062a\\u062d\\u0627\\u062f\", \"\\u0635\\u0646\\u0627\\u0639\\u0629\", \"\\u0627\\u0644\\u0633\\u064a\\u0627\\u0631\\u0627\\u062a\", \"\\u0641\\u064a\", \"\\u0627\\u0644\\u0645\\u0627\\u0646\\u064a\\u0627\", \"\\u0627\\u0645\\u0633\", \"\\u0627\\u0644\\u0627\\u0648\\u0644\", \"\\u0627\\u0646\", \"\\u0634\\u0631\\u0643\\u0627\\u062a\", \"\\u0635\\u0646\\u0627\\u0639\\u0629\", \"\\u0627\\u0644\\u0633\\u064a\\u0627\\u0631\\u0627\\u062a\", \"\\u0641\\u064a\", \"\\u0627\\u0644\\u0645\\u0627\\u0646\\u064a\\u0627\", \"\\u062a\\u0648\\u0627\\u062c\\u0647\", \"\\u0639\\u0627\\u0645\\u0627\", \"\\u0635\\u0639\\u0628\\u0627\", \"\\u0641\\u064a\", \"\\u0638\\u0644\", \"\\u0631\\u0643\\u0648\\u062f\", \"\\u0627\\u0644\\u0633\\u0648\\u0642\", \"\\u0627\\u0644\\u062f\\u0627\\u062e\\u0644\\u064a\\u0629\", \"\\u0648\", \"\\u0627\\u0644\\u0635\\u0627\\u062f\\u0631\\u0627\\u062a\", \"\\u0648\", \"\\u0647\\u064a\", \"\\u062a\\u0633\\u0639\\u064a\", \"\\u0644\\u0627\\u0646\", \"\\u064a\\u0628\\u0644\\u063a\", \"\\u0627\\u0644\\u0627\\u0646\\u062a\\u0627\\u062c\", \"\\u062d\\u0648\\u0627\\u0644\\u064a\", \"\\u062e\\u0645\\u0633\\u0629\", \"\\u0645\\u0644\\u0627\\u064a\\u064a\\u0646\", \"\\u0633\\u064a\\u0627\\u0631\\u0629\", \"\\u0641\\u064a\", \"\\u0639\\u0627\\u0645\", \"2002\", \".\"],\n[\"B-Population-Center\", \"O\", \"B-Media\", \"I-Media\", \"I-Media\", \"O\", \"O\", \"B-Government\", \"I-Government\", \"I-Government\", \"O\", \"B-Nation\", \"O\", \"O\", \"O\", \"O\", \"O\", \"O\", \"O\", \"B-Nation\", \"O\", \"O\", \"O\", \"O\", \"O\", \"O\", \"O\", \"O\", \"O\", \"O\", \"O\", \"O\", \"O\", \"O\", \"O\", \"O\", \"O\", \"O\", \"O\", \"O\", \"O\", \"O\", \"O\", \"O\"]]],\n        columns: [[\"number\", \"index\"], [\"string\", \"text\"], [\"string\", \"tags\"]],\n        columnOptions: [{\"width\": \"1px\", \"className\": \"index_column\"}],\n        rowsPerPage: 25,\n        helpUrl: \"https://colab.research.google.com/notebooks/data_table.ipynb\",\n        suppressOutputScrolling: true,\n        minimumWidth: undefined,\n      });\n    ",
            "text/html": [
              "<div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>text</th>\n",
              "      <th>tags</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>0</th>\n",
              "      <td>[فرانكفورت, (, د, ب, ا, ), اعلن, اتحاد, صناعة,...</td>\n",
              "      <td>[B-Population-Center, O, B-Media, I-Media, I-M...</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "</div>"
            ],
            "text/plain": [
              "                                                text                                               tags\n",
              "0  [فرانكفورت, (, د, ب, ا, ), اعلن, اتحاد, صناعة,...  [B-Population-Center, O, B-Media, I-Media, I-M..."
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 7
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "WxJw40QPZuo7"
      },
      "source": [
        "Here we get the list of all classes we have"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "krfSvW1YGQ1d",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "ae87b460-6e99-4aad-ece5-d8b94ee345b9"
      },
      "source": [
        "label_list = list(pd.read_csv('label_list.txt', header=None, index_col=0).T)\n",
        "\n",
        "label_map = { v:index for index, v in enumerate(label_list) }\n",
        "inv_label_map = {i: label for i, label in enumerate(label_list)}\n",
        "\n",
        "print(label_map)\n",
        "print(inv_label_map)"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "{'O': 0, 'B-Artist': 1, 'I-Artist': 2, 'B-Sound': 3, 'I-Sound': 4, 'B-Educational': 5, 'I-Educational': 6, 'B-Building-Grounds': 7, 'I-Building-Grounds': 8, 'B-Population-Center': 9, 'B-Nation': 10, 'B-State-or-Province': 11, 'I-State-or-Province': 12, 'B-Water-Body': 13, 'I-Water-Body': 14, 'B-Land-Region-Natural': 15, 'I-Land-Region-Natural': 16, 'B-Software': 17, 'I-Software': 18, 'B-Scientist': 19, 'B-Book': 20, 'I-Book': 21, 'I-Scientist': 22, 'B-Group': 23, 'B-Celestial': 24, 'B-Police': 25, 'I-Police': 26, 'I-Population-Center': 27, 'I-Celestial': 28, 'B-Engineer': 29, 'I-Engineer': 30, 'B-Projectile': 31, 'B-Government': 32, 'I-Government': 33, 'B-Commercial': 34, 'I-Commercial': 35, 'B-Continent': 36, 'B-Air': 37, 'I-Air': 38, 'B-Other_PER': 39, 'I-Other_PER': 40, 'I-Group': 41, 'B-Politician': 42, 'I-Politician': 43, 'B-Athlete': 44, 'I-Athlete': 45, 'B-Religious_ORG': 46, 'I-Religious_ORG': 47, 'B-Path': 48, 'I-Path': 49, 'B-Media': 50, 'I-Media': 51, 'B-Non-Governmental': 52, 'I-Non-Governmental': 53, 'B-County-or-District': 54, 'I-County-or-District': 55, 'B-Businessperson': 56, 'B-Lawyer': 57, 'I-Lawyer': 58, 'B-GPE-Cluster': 59, 'I-GPE-Cluster': 60, 'I-Nation': 61, 'B-Religious_PER': 62, 'I-Religious_PER': 63, 'I-Businessperson': 64, 'B-Medical-Science': 65, 'I-Medical-Science': 66, 'B-Movie': 67, 'I-Movie': 68, 'B-Water': 69, 'I-Water': 70, 'B-Drug': 71, 'B-Hardware': 72, 'I-Hardware': 73, 'B-Subarea-Facility': 74, 'I-Subarea-Facility': 75, 'B-Blunt': 76, 'B-Airport': 77, 'I-Blunt': 78, 'I-Drug': 79, 'B-Sports': 80, 'I-Sports': 81, 'B-Shooting': 82, 'I-Shooting': 83, 'B-Food': 84, 'I-Food': 85, 'I-Continent': 86, 'B-Nuclear': 87, 'I-Nuclear': 88, 'B-Entertainment': 89, 'I-Entertainment': 90, 'I-Projectile': 91, 'B-Land': 92, 'B-Sharp': 93, 'I-Airport': 94, 'I-Land': 95, 'B-Plant': 96, 'I-Plant': 97, 'B-Exploding': 98, 'I-Exploding': 99, 'B-Chemical': 100, 'I-Chemical': 101}\n",
            "{0: 'O', 1: 'B-Artist', 2: 'I-Artist', 3: 'B-Sound', 4: 'I-Sound', 5: 'B-Educational', 6: 'I-Educational', 7: 'B-Building-Grounds', 8: 'I-Building-Grounds', 9: 'B-Population-Center', 10: 'B-Nation', 11: 'B-State-or-Province', 12: 'I-State-or-Province', 13: 'B-Water-Body', 14: 'I-Water-Body', 15: 'B-Land-Region-Natural', 16: 'I-Land-Region-Natural', 17: 'B-Software', 18: 'I-Software', 19: 'B-Scientist', 20: 'B-Book', 21: 'I-Book', 22: 'I-Scientist', 23: 'B-Group', 24: 'B-Celestial', 25: 'B-Police', 26: 'I-Police', 27: 'I-Population-Center', 28: 'I-Celestial', 29: 'B-Engineer', 30: 'I-Engineer', 31: 'B-Projectile', 32: 'B-Government', 33: 'I-Government', 34: 'B-Commercial', 35: 'I-Commercial', 36: 'B-Continent', 37: 'B-Air', 38: 'I-Air', 39: 'B-Other_PER', 40: 'I-Other_PER', 41: 'I-Group', 42: 'B-Politician', 43: 'I-Politician', 44: 'B-Athlete', 45: 'I-Athlete', 46: 'B-Religious_ORG', 47: 'I-Religious_ORG', 48: 'B-Path', 49: 'I-Path', 50: 'B-Media', 51: 'I-Media', 52: 'B-Non-Governmental', 53: 'I-Non-Governmental', 54: 'B-County-or-District', 55: 'I-County-or-District', 56: 'B-Businessperson', 57: 'B-Lawyer', 58: 'I-Lawyer', 59: 'B-GPE-Cluster', 60: 'I-GPE-Cluster', 61: 'I-Nation', 62: 'B-Religious_PER', 63: 'I-Religious_PER', 64: 'I-Businessperson', 65: 'B-Medical-Science', 66: 'I-Medical-Science', 67: 'B-Movie', 68: 'I-Movie', 69: 'B-Water', 70: 'I-Water', 71: 'B-Drug', 72: 'B-Hardware', 73: 'I-Hardware', 74: 'B-Subarea-Facility', 75: 'I-Subarea-Facility', 76: 'B-Blunt', 77: 'B-Airport', 78: 'I-Blunt', 79: 'I-Drug', 80: 'B-Sports', 81: 'I-Sports', 82: 'B-Shooting', 83: 'I-Shooting', 84: 'B-Food', 85: 'I-Food', 86: 'I-Continent', 87: 'B-Nuclear', 88: 'I-Nuclear', 89: 'B-Entertainment', 90: 'I-Entertainment', 91: 'I-Projectile', 92: 'B-Land', 93: 'B-Sharp', 94: 'I-Airport', 95: 'I-Land', 96: 'B-Plant', 97: 'I-Plant', 98: 'B-Exploding', 99: 'I-Exploding', 100: 'B-Chemical', 101: 'I-Chemical'}\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "szAgZufweY6j"
      },
      "source": [
        "## Dataset splits\n"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "pZ71g7W3Z9SD"
      },
      "source": [
        "We splitthe Dataset into:  90% Train, 10% Valdiation, and 10% Test"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "hOpLO-H4eYKe"
      },
      "source": [
        "test_data = data\n",
        "\n",
        "#  train_data, test_data = train_test_split(data,test_size= 0.2)\n",
        "# # \n",
        "#  test_data , eval_data = train_test_split(test_data,test_size= 0.5)"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "YEi_vRw0HtUD",
        "outputId": "be42f0f4-a75a-45fb-a4f4-01200f9d54dc"
      },
      "source": [
        "# print(len(train_data))\n",
        "print(len(test_data))\n",
        "# print(len(eval_data))"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "4760\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "ViJtYz8GaVop"
      },
      "source": [
        "Here we get the frequency of each class in dataset"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "kmfXS6QOWYZ0",
        "outputId": "96ac8a1a-83cf-40e5-8a87-beb8489c0278"
      },
      "source": [
        "# print(Counter([ label for sentence in train_data for label in sentence[1]]))\n",
        "print(Counter([ label for sentence in test_data for label in sentence[1]]))\n",
        "\n"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Counter({'O': 149978, 'B-Nation': 2232, 'B-Population-Center': 1427, 'B-Politician': 1315, 'I-Politician': 1206, 'I-Non-Governmental': 871, 'B-Athlete': 865, 'B-Non-Governmental': 767, 'B-Group': 685, 'B-Sports': 661, 'I-Athlete': 621, 'B-Artist': 451, 'B-Other_PER': 444, 'I-Government': 412, 'I-Other_PER': 403, 'I-Artist': 400, 'B-Media': 358, 'I-Building-Grounds': 292, 'B-Government': 284, 'I-Media': 275, 'I-Nation': 265, 'I-Population-Center': 261, 'I-Sports': 259, 'B-Building-Grounds': 238, 'B-State-or-Province': 220, 'B-Religious_PER': 175, 'B-Commercial': 169, 'B-Scientist': 151, 'I-Commercial': 147, 'I-Religious_PER': 145, 'I-Book': 138, 'I-Educational': 126, 'B-GPE-Cluster': 125, 'I-GPE-Cluster': 118, 'I-Movie': 116, 'B-Continent': 109, 'B-Water-Body': 102, 'B-Book': 94, 'I-Water-Body': 76, 'B-Educational': 73, 'B-Movie': 73, 'I-Land': 69, 'B-Land': 68, 'I-Scientist': 62, 'B-Police': 57, 'I-Police': 52, 'I-State-or-Province': 52, 'B-County-or-District': 48, 'B-Businessperson': 48, 'I-Businessperson': 42, 'I-Medical-Science': 38, 'I-Group': 33, 'I-County-or-District': 28, 'B-Air': 23, 'B-Medical-Science': 23, 'I-Path': 22, 'I-Entertainment': 22, 'I-Airport': 20, 'I-Land-Region-Natural': 20, 'I-Lawyer': 19, 'I-Subarea-Facility': 18, 'B-Land-Region-Natural': 17, 'B-Airport': 16, 'B-Path': 16, 'I-Air': 13, 'B-Subarea-Facility': 13, 'B-Software': 11, 'B-Lawyer': 11, 'I-Religious_ORG': 11, 'B-Entertainment': 10, 'B-Religious_ORG': 9, 'I-Continent': 8, 'I-Software': 7, 'B-Hardware': 7, 'B-Engineer': 6, 'I-Engineer': 5, 'I-Hardware': 5, 'B-Celestial': 4, 'B-Food': 3, 'B-Projectile': 3, 'I-Projectile': 3, 'B-Shooting': 3, 'B-Drug': 2, 'B-Water': 2, 'I-Food': 1, 'B-Plant': 1, 'I-Shooting': 1, 'I-Water': 1})\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "N9At1fRxaaFa"
      },
      "source": [
        "Here we get a histogram of number of tokens in each sentence, to choose  a suitiable size for BERT MAX_LEN"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 302
        },
        "id": "jqqERV0Dd9ep",
        "outputId": "5f12f0eb-c880-42b5-bbbc-bfdf2c7cbe26"
      },
      "source": [
        "# print(\"Training Sentence Lengths: \")\n",
        "# plt.hist([ len(sentence[0]) for sentence in train_data],bins=range(0,256,2))\n",
        "# plt.show()\n",
        "# print(sum([len(sentence[0]) > 256 for sentence in train_data]))\n",
        "\n",
        "print(\"Testing Sentence Lengths: \")\n",
        "plt.hist([ len(sentence[0]) for sentence in test_data],bins=range(0,256,2))\n",
        "plt.show()\n",
        "print(sum([len(sentence[0]) > 256 for sentence in test_data]))"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Testing Sentence Lengths: \n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "display_data",
          "data": {
            "image/png": "iVBORw0KGgoAAAANSUhEUgAAAXcAAAD7CAYAAACRxdTpAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4yLjIsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+WH4yJAAAPXElEQVR4nO3dUYxc1X3H8e+vNMlDQALqreWA0yWRW4k+1EErQAJFVFEDmAcTqULkIXEjKucBpERKH5zkIbwguVWhIlKKZAqKqQgEKYmwBG2TWJGiSIFkiRwwUBonMcLWYjtNFKgi0QL/Psw1GcyuZ2dnZmf37PcjjebOuXdm/odrfnPm3Dt3U1VIktryB9MuQJI0foa7JDXIcJekBhnuktQgw12SGmS4S1KDBoZ7kq1JvpfkuSTPJvlM1357kuNJDnW3HX3P+XySI0leSHLtJDsgSXqnDDrPPckWYEtV/STJecBTwI3ATcD/VNU/nrH9pcBDwOXA+4DvAn9aVW9MoH5J0iL+cNAGVbUALHTLryZ5HrjoLE/ZCTxcVa8Bv0xyhF7Q/3CpJ2zatKlmZ2eHqVuSNrynnnrqV1U1s9i6geHeL8ks8CHgSeAq4LYknwTmgc9V1W/oBf8TfU87xtk/DJidnWV+fn6YUiRpw0vy4lLrln1ANcm5wDeAz1bVK8A9wAeB7fRG9ncOWdTuJPNJ5k+dOjXMUyVJAywr3JO8i16wP1hV3wSoqhNV9UZVvQncS2/qBeA4sLXv6Rd3bW9TVfuqaq6q5mZmFv1WIUlaoeWcLRPgPuD5qrqrr31L32YfAw53yweAm5O8J8klwDbgR+MrWZI0yHLm3K8CPgE8k+RQ1/YF4ONJtgMFHAU+DVBVzyZ5BHgOeB241TNlJGl1LedsmR8AWWTV42d5zh3AHSPUJUkagb9QlaQGGe6S1CDDXZIaZLhLUoOG+oWqemb3PPbW8tG9N0yxEklanCP3MZnd89jbQl+Spslwl6QGGe6S1CDDXZIa5AHVATx4Kmk9cuQuSQ0y3CWpQYa7JDXIcJekBhnuktQgz5YZgr9AlbReOHKXpAY5cl+Co3RJ65nhPiI/BCStRU7LSFKDDHdJapDhLkkNMtwlqUGGuyQ1yHCXpAYZ7pLUIM9zP4PnrUtqgSN3SWqQ4T5Bs3se85uApKkw3CWpQYa7JDXIcJekBhnuktQgw12SGjTwPPckW4EHgM1AAfuq6u4kFwJfB2aBo8BNVfWbJAHuBnYAvwP+pqp+Mpnyx8MzWiS1Zjkj99eBz1XVpcCVwK1JLgX2AAerahtwsHsMcD2wrbvtBu4Ze9WSpLMaGO5VtXB65F1VrwLPAxcBO4H93Wb7gRu75Z3AA9XzBHB+ki1jr1yStKShLj+QZBb4EPAksLmqFrpVL9ObtoFe8L/U97RjXdsCG4BTPJLWgmUfUE1yLvAN4LNV9Ur/uqoqevPxy5Zkd5L5JPOnTp0a5qmSpAGWFe5J3kUv2B+sqm92zSdOT7d09ye79uPA1r6nX9y1vU1V7auquaqam5mZWWn9kqRFDAz37uyX+4Dnq+quvlUHgF3d8i7g0b72T6bnSuC3fdM3kqRVsJw596uATwDPJDnUtX0B2As8kuQW4EXgpm7d4/ROgzxC71TIT421YknSQAPDvap+AGSJ1R9ZZPsCbh2xLknSCPyFqiQ1yHCXpAYZ7pLUIMNdkhpkuEtSgwx3SWqQ4S5JDTLcJalBQ10VUivTf6XIo3tvmGIlkjYKR+6S1CDDXZIaZLhLUoMMd0lqkOEuSQ0y3CWpQYa7JDXIcJekBhnuktQgw12SGmS4S1KDDPdVNrvnsbdda0aSJsFwl6QGGe6S1CDDXZIaZLhPiXPvkibJcJekBhnuktQgw12SGmS4S1KDNvQfyPaApqRWOXKXpAYZ7pLUIMNdkhpkuEtSgwaGe5L7k5xMcriv7fYkx5Mc6m47+tZ9PsmRJC8kuXZShUuSlrackftXgesWaf+nqtre3R4HSHIpcDPw591z/jnJOeMqVpK0PAPDvaq+D/x6ma+3E3i4ql6rql8CR4DLR6hPkrQCo8y535bk6W7a5oKu7SLgpb5tjnVtkqRVtNJwvwf4ILAdWADuHPYFkuxOMp9k/tSpUyssQ5K0mBWFe1WdqKo3qupN4F5+P/VyHNjat+nFXdtir7Gvquaqam5mZmYlZUiSlrCiyw8k2VJVC93DjwGnz6Q5AHwtyV3A+4BtwI9GrrJh/ZdAOLr3hilWIqklA8M9yUPANcCmJMeALwHXJNkOFHAU+DRAVT2b5BHgOeB14NaqemMypUuSljIw3Kvq44s033eW7e8A7hilKEnSaPyFqiQ1yHCXpAZtuOu5r+VruJ+uzQOrkkblyF2SGmS4S1KDDHdJapDhLkkNMtwlqUGGuyQ1yHCXpAYZ7pLUIMNdkhpkuEtSgwx3SWqQ4S5JDTLcJalBhrskNchwX+Nm9zy2pi9TLGltMtwlqUGGuyQ1aMP9Jab1wGkYSaNy5C5JDTLcJalBhrskNchwl6QGGe6S1CDDXZIaZLhLUoMMd0lqkOEuSQ0y3CWpQYa7JDXIcJekBhnuktQgw12SGjQw3JPcn+RkksN9bRcm+U6Sn3X3F3TtSfLlJEeSPJ3kskkWL0la3HJG7l8FrjujbQ9wsKq2AQe7xwDXA9u6227gnvGUKUkaxsBwr6rvA78+o3knsL9b3g/c2Nf+QPU8AZyfZMu4ipUkLc9K59w3V9VCt/wysLlbvgh4qW+7Y12bJGkVjXxAtaoKqGGfl2R3kvkk86dOnRq1DElSn5X+DdUTSbZU1UI37XKyaz8ObO3b7uKu7R2qah+wD2Bubm7oD4eNqv/vqx7de8MUK5G0lq003A8Au4C93f2jfe23JXkYuAL4bd/0zVT5R6clbSQDwz3JQ8A1wKYkx4Av0Qv1R5LcArwI3NRt/jiwAzgC/A741ARqliQNMDDcq+rjS6z6yCLbFnDrqEVJkkbjL1QlqUGGuyQ1yHCXpAYZ7pLUIMNdkhpkuEtSgwx3SWqQ4S5JDVrp5Qe0yrx8gqRhOHKXpAYZ7pLUIMNdkhpkuEtSgwx3SWqQ4S5JDTLcJalBhrskNchwl6QGGe6S1CDDXZIaZLhLUoMMd0lqkFeFbED/FSOP7r1hipVIWiscuUtSgwx3SWqQ4S5JDWp+zt2/YCRpI3LkLkkNMtwlqUGGuyQ1yHCXpAYZ7pLUoObPlmmZZwJJWoojd0lq0Egj9yRHgVeBN4DXq2ouyYXA14FZ4ChwU1X9ZrQyNQ6nR/pef0Zq3zhG7n9ZVduraq57vAc4WFXbgIPdY62S2T2POV0jaSLTMjuB/d3yfuDGCbyHJOksRg33Ar6d5Kkku7u2zVW10C2/DGwe8T0kSUMa9WyZq6vqeJI/Br6T5D/7V1ZVJanFnth9GOwGeP/73z9iGZKkfiON3KvqeHd/EvgWcDlwIskWgO7+5BLP3VdVc1U1NzMzM0oZkqQzrDjck7w3yXmnl4GPAoeBA8CubrNdwKOjFilJGs4o0zKbgW8lOf06X6uqf0/yY+CRJLcALwI3jV6mJGkYKw73qvoF8BeLtP838JFRitL4eFqktDH5C1VJapDhLkkN8sJhjXI6RtrYHLlLUoMMd0lqkOEuSQ1yzn0D6p+P9/K/UpscuUtSgwx3SWqQ4S5JDTLcBfgXnKTWNHlA1ZCStNE1Ge5aPj8IpTY5LaOBnLKR1h9H7lo2z4+X1g/DXW/jCF1qg9MyktQgw12SGmS4S1KDDHdJapDhLkkNMtwlqUGGu1bEHzZJa1tT57kbNmufP4SSVocjd0lq0LofuTtal6R3Wvfhrula7MPV6RZp+gx3rRnOx0vjY7hr7BzNS9PnAVVNjadTSpPjyF2rwhCXVpfhrqlbbvCf3u5sUzzO20s9hrvWpEmM9A1+bSSGu5owzg+Dxb4hLOdbg7SWTCzck1wH3A2cA/xLVe2d1HtpY1lu0A4byGvtuIAfKBrFRMI9yTnAV4C/Ao4BP05yoKqem8T7aWMadq5+GpY7FWSQa9wmNXK/HDhSVb8ASPIwsBMw3LWmDBv8yz2Hv+VvAdN4rdX68GvpQ3ZS4X4R8FLf42PAFRN6L2loywnfcQT/ON5z2B+FTeqDZZIfWMN+w+nXQhBPQqpq/C+a/DVwXVX9bff4E8AVVXVb3za7gd3dwz8DXljh220CfjVCueuN/W2b/W3buPv7J1U1s9iKSY3cjwNb+x5f3LW9par2AftGfaMk81U1N+rrrBf2t232t22r2d9JXX7gx8C2JJckeTdwM3BgQu8lSTrDREbuVfV6ktuA/6B3KuT9VfXsJN5LkvROEzvPvaoeBx6f1Ov3GXlqZ52xv22zv21btf5O5ICqJGm6vOSvJDVoXYd7kuuSvJDkSJI9065nEpIcTfJMkkNJ5ru2C5N8J8nPuvsLpl3nSiW5P8nJJIf72hbtX3q+3O3vp5NcNr3KV2aJ/t6e5Hi3jw8l2dG37vNdf19Icu10ql6ZJFuTfC/Jc0meTfKZrr3J/XuW/k5n/1bVurzRO1D7c+ADwLuBnwKXTruuCfTzKLDpjLZ/APZ0y3uAv592nSP078PAZcDhQf0DdgD/BgS4Enhy2vWPqb+3A3+3yLaXdv+u3wNc0v17P2fafRiir1uAy7rl84D/6vrU5P49S3+nsn/X88j9rUscVNX/AqcvcbAR7AT2d8v7gRunWMtIqur7wK/PaF6qfzuBB6rnCeD8JFtWp9LxWKK/S9kJPFxVr1XVL4Ej9P7drwtVtVBVP+mWXwWep/fr9Sb371n6u5SJ7t/1HO6LXeLgbP8h16sCvp3kqe5XvQCbq2qhW34Z2Dyd0iZmqf61vM9v66Yi7u+bZmumv0lmgQ8BT7IB9u8Z/YUp7N/1HO4bxdVVdRlwPXBrkg/3r6ze97tmT3lqvX+de4APAtuBBeDO6ZYzXknOBb4BfLaqXulf1+L+XaS/U9m/6zncB17ioAVVdby7Pwl8i97XthOnv6529yenV+FELNW/Jvd5VZ2oqjeq6k3gXn7/1Xzd9zfJu+gF3YNV9c2uudn9u1h/p7V/13O4N3+JgyTvTXLe6WXgo8Bhev3c1W22C3h0OhVOzFL9OwB8sjur4krgt31f79etM+aVP0ZvH0OvvzcneU+SS4BtwI9Wu76VShLgPuD5qrqrb1WT+3ep/k5t/077CPOIR6d30Dsi/XPgi9OuZwL9+wC9o+k/BZ493Ufgj4CDwM+A7wIXTrvWEfr4EL2vqv9Hb87xlqX6R+8siq90+/sZYG7a9Y+pv//a9efp7n/4LX3bf7Hr7wvA9dOuf8i+Xk1vyuVp4FB329Hq/j1Lf6eyf/2FqiQ1aD1Py0iSlmC4S1KDDHdJapDhLkkNMtwlqUGGuyQ1yHCXpAYZ7pLUoP8HTnLPmrZHZhkAAAAASUVORK5CYII=\n",
            "text/plain": [
              "<Figure size 432x288 with 1 Axes>"
            ]
          },
          "metadata": {
            "tags": [],
            "needs_background": "light"
          }
        },
        {
          "output_type": "stream",
          "text": [
            "3\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "bmGGv2yNyvCj"
      },
      "source": [
        "## Training\n"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "5qVNRCj4wg1a"
      },
      "source": [
        "### Model Config\n",
        "\n",
        "Here, we define some constants to use "
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "ZCkpPVJOyuQZ",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 262,
          "referenced_widgets": [
            "df9d5fb273e146d88b3dafe76efe9c20",
            "8c3b573133f043a0a35aad910af4ce93",
            "70c506c3e18748598a72d869e0716a9b",
            "76e5e582c7f0421f98d8f6ebf508934b",
            "884b58371df54b02a3661b67a2b43ec7",
            "46c2ce18ef4144b0856094d7e4a5c770",
            "f9922ec0e0374677bcbc362443fc6334",
            "e5989cb7e3e44349855fa0e8fe6eaf18",
            "4733c7ac987f403bbc63bdf759df1af4",
            "5d77e52b0c114297aacce461a2691e5d",
            "15c2ac3a22564393b8c0ef811d148a30",
            "a2b3a067e1744b3aa15bd18d6dd1cba8",
            "9519f24a848a4a869b803cc92ae2cc6d",
            "d62b56398c65458daa0d85b3a5adb8bb",
            "a89eae03c1b54bd2ac0e6f9c4e7f242d",
            "e5fd0e3f3bf84e4a8840723659b73a5c",
            "82de917239834629ba8a095bbf5e6ec5",
            "95dc9a0210954d9ab836f465d00603b1",
            "5adff5402e5f4e59872b87c41a1735ef",
            "eca35d9563f24c638a9b8cdc9a81cb24",
            "7b55484d972c43a8a009eddfbbe9e905",
            "ab1fc188d2914c099c162337ce51340f",
            "501d82c0fe9348919691e41c62531530",
            "f219e5df9ad54fc58cde262cd68bd46b",
            "c116df5c51564092a690bebb30904422",
            "0f28247d6dc84fbf8d70deeaa62e257e",
            "252431a5f886477c902f7b396abcfad4",
            "477fd59f6c1d4f1fac368044112a7010",
            "3e7fe1d8bb734b93960be974caa306dd",
            "806b784d9b7c4cc9807c6340ca84a4e7",
            "08a0d2434eef4f2eb888d1aabaa12515",
            "6e863e924560410188f83baf25ae1131",
            "3936fbd6175e4ca1b082f14d9d3943b5",
            "ea929c45235545f3ab5a5ac5e29625d4",
            "5ef71b0e60a046ecb4874e1a988ac524",
            "2337733c66a74b47a2e86316c49579c9",
            "ae6173bac98845ffacde6591d2cacef7",
            "4cd7b11865904ebfbab5c9a611a61eb1",
            "4d0608fe5bfc4b45b620fbe93dbb03e4",
            "14c9a60d913541f3962210deae19e470"
          ]
        },
        "outputId": "2d65fe90-0ca5-4575-e1bc-7d1fa48380ea"
      },
      "source": [
        "DATASET_NAME = 'Fine_GRATED'\n",
        "MODEL_NAME = 'aubmindlab/bert-base-arabertv02'\n",
        "TASK_NAME = 'tokenclassification'\n",
        "\n",
        "#######  TODO: Make it 256 again\n",
        "MAX_LEN = 256\n",
        "TRAIN_BATCH_SIZE = 16\n",
        "VALID_BATCH_SIZE = 16\n",
        "TEST_BATCH_SIZE = 16\n",
        "EPOCHS = 10\n",
        "MODEL_PATH = \"model\"\n",
        "\n",
        "TOKENIZER = AutoTokenizer.from_pretrained(MODEL_NAME)"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "display_data",
          "data": {
            "application/vnd.jupyter.widget-view+json": {
              "model_id": "df9d5fb273e146d88b3dafe76efe9c20",
              "version_minor": 0,
              "version_major": 2
            },
            "text/plain": [
              "HBox(children=(FloatProgress(value=0.0, description='Downloading', max=381.0, style=ProgressStyle(description_…"
            ]
          },
          "metadata": {
            "tags": []
          }
        },
        {
          "output_type": "stream",
          "text": [
            "\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "display_data",
          "data": {
            "application/vnd.jupyter.widget-view+json": {
              "model_id": "4733c7ac987f403bbc63bdf759df1af4",
              "version_minor": 0,
              "version_major": 2
            },
            "text/plain": [
              "HBox(children=(FloatProgress(value=0.0, description='Downloading', max=384.0, style=ProgressStyle(description_…"
            ]
          },
          "metadata": {
            "tags": []
          }
        },
        {
          "output_type": "stream",
          "text": [
            "\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "display_data",
          "data": {
            "application/vnd.jupyter.widget-view+json": {
              "model_id": "82de917239834629ba8a095bbf5e6ec5",
              "version_minor": 0,
              "version_major": 2
            },
            "text/plain": [
              "HBox(children=(FloatProgress(value=0.0, description='Downloading', max=824793.0, style=ProgressStyle(descripti…"
            ]
          },
          "metadata": {
            "tags": []
          }
        },
        {
          "output_type": "stream",
          "text": [
            "\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "display_data",
          "data": {
            "application/vnd.jupyter.widget-view+json": {
              "model_id": "c116df5c51564092a690bebb30904422",
              "version_minor": 0,
              "version_major": 2
            },
            "text/plain": [
              "HBox(children=(FloatProgress(value=0.0, description='Downloading', max=2642362.0, style=ProgressStyle(descript…"
            ]
          },
          "metadata": {
            "tags": []
          }
        },
        {
          "output_type": "stream",
          "text": [
            "\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "display_data",
          "data": {
            "application/vnd.jupyter.widget-view+json": {
              "model_id": "3936fbd6175e4ca1b082f14d9d3943b5",
              "version_minor": 0,
              "version_major": 2
            },
            "text/plain": [
              "HBox(children=(FloatProgress(value=0.0, description='Downloading', max=112.0, style=ProgressStyle(description_…"
            ]
          },
          "metadata": {
            "tags": []
          }
        },
        {
          "output_type": "stream",
          "text": [
            "\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "ffD7OI29wp5g"
      },
      "source": [
        "### Dataset\n",
        "\n",
        "\n",
        "Here we define the Dataset class, and how to process batches before training\n"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "htos49v6hlzW"
      },
      "source": [
        "class NERDataset:\n",
        "  def __init__(self, texts, tags, label_list, model_name, max_length):\n",
        "    self.texts = texts\n",
        "    self.tags = tags\n",
        "    self.label_map = {label: i for i, label in enumerate(label_list)}\n",
        "    self.preprocessor = ArabertPreprocessor(model_name.split(\"/\")[-1])    \n",
        "    self.pad_token_label_id = torch.nn.CrossEntropyLoss().ignore_index\n",
        "    # Use cross entropy ignore_index as padding label id so that only\n",
        "    # real label ids contribute to the loss later.\n",
        "    self.tokenizer = AutoTokenizer.from_pretrained(model_name)\n",
        "    self.max_length = max_length\n",
        "\n",
        "     \n",
        "  def __len__(self):\n",
        "    return len(self.texts)\n",
        "  \n",
        "  def __getitem__(self, item):\n",
        "    textlist = self.texts[item]\n",
        "    tags = self.tags[item]\n",
        "\n",
        "    tokens = []\n",
        "    label_ids = []\n",
        "    for word, label in zip(textlist, tags):      \n",
        "      clean_word = self.preprocessor.preprocess(word)  \n",
        "      word_tokens = self.tokenizer.tokenize(clean_word)\n",
        "\n",
        "  \n",
        "      if len(word_tokens) > 0:\n",
        "        tokens.extend(word_tokens)    \n",
        "        # Use the real label id for the first token of the word, and padding ids for the remaining tokens\n",
        "        label_ids.extend([self.label_map[label]] + [self.pad_token_label_id] * (len(word_tokens) - 1))\n",
        " \n",
        "    # Account for [CLS] and [SEP] with \"- 2\" and with \"- 3\" for RoBERTa.\n",
        "    special_tokens_count = self.tokenizer.num_special_tokens_to_add()\n",
        "    if len(tokens) > self.max_length - special_tokens_count:\n",
        "      tokens = tokens[: (self.max_length - special_tokens_count)]\n",
        "      label_ids = label_ids[: (self.max_length - special_tokens_count)]\n",
        "  \n",
        "    #Add the [SEP] token\n",
        "    tokens += [self.tokenizer.sep_token]\n",
        "    label_ids += [self.pad_token_label_id]\n",
        "    token_type_ids = [0] * len(tokens)\n",
        "\n",
        "    #Add the [CLS] TOKEN\n",
        "    tokens = [self.tokenizer.cls_token] + tokens\n",
        "    label_ids = [self.pad_token_label_id] + label_ids\n",
        "    token_type_ids = [0] + token_type_ids\n",
        "\n",
        "    input_ids = self.tokenizer.convert_tokens_to_ids(tokens)\n",
        "\n",
        "    # The mask has 1 for real tokens and 0 for padding tokens. Only real\n",
        "    # tokens are attended to.\n",
        "    attention_mask = [1] * len(input_ids)\n",
        "\n",
        "    # ana we ant {pad}\n",
        "    # 1    1  1   0  0\n",
        "    # [cls] B-loc O O B-loc [sep] [pad]\n",
        "    #  -100    2   0  0      -100 \n",
        "\n",
        "    # Zero-pad up to the sequence length.\n",
        "    padding_length = self.max_length - len(input_ids)\n",
        "\n",
        "    input_ids += [self.tokenizer.pad_token_id] * padding_length\n",
        "    attention_mask += [0] * padding_length\n",
        "    token_type_ids += [0] * padding_length\n",
        "    label_ids += [self.pad_token_label_id] * padding_length\n",
        "\n",
        "    assert len(input_ids) == self.max_length\n",
        "    assert len(attention_mask) == self.max_length\n",
        "    assert len(token_type_ids) == self.max_length\n",
        "    assert len(label_ids) == self.max_length\n",
        "\n",
        "    # if item < 5:\n",
        "    #   print(\"*** Example ***\")\n",
        "    #   print(\"tokens:\", \" \".join([str(x) for x in tokens]))\n",
        "    #   print(\"input_ids:\", \" \".join([str(x) for x in input_ids]))\n",
        "    #   print(\"attention_mask:\", \" \".join([str(x) for x in attention_mask]))\n",
        "    #   print(\"token_type_ids:\", \" \".join([str(x) for x in token_type_ids]))\n",
        "    #   print(\"label_ids:\", \" \".join([str(x) for x in label_ids]))\n",
        "    \n",
        "    return {\n",
        "        'input_ids' : torch.tensor(input_ids, dtype=torch.long),\n",
        "        'attention_mask' : torch.tensor(attention_mask, dtype=torch.long),\n",
        "        'token_type_ids' : torch.tensor(token_type_ids, dtype=torch.long),\n",
        "        'labels' : torch.tensor(label_ids, dtype=torch.long)       \n",
        "    }"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "z6MC0SRIb-Pr"
      },
      "source": [
        "Here we create The datasets for train,val, and test"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "HGBV47Nz1HwU"
      },
      "source": [
        "\n",
        "\n",
        "# train_dataset = NERDataset(\n",
        "#     texts=[x[0] for x in train_data],\n",
        "#     tags=[x[1] for x in train_data],\n",
        "#     label_list=label_list,\n",
        "#     model_name=MODEL_NAME,\n",
        "#     max_length=MAX_LEN\n",
        "#     )\n",
        "\n",
        "test_dataset = NERDataset(\n",
        "    texts=[x[0] for x in test_data],\n",
        "    tags=[x[1] for x in test_data],\n",
        "    label_list=label_list,\n",
        "    model_name=MODEL_NAME,\n",
        "    max_length=MAX_LEN\n",
        "    )\n",
        "\n",
        "# eval_dataset = NERDataset(\n",
        "#     texts=[x[0] for x in eval_data],\n",
        "#     tags=[x[1] for x in eval_data],\n",
        "#     label_list=label_list,\n",
        "#     model_name=MODEL_NAME,\n",
        "#     max_length=MAX_LEN\n",
        "#     )"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "7J4XmsTRw4HF"
      },
      "source": [
        "### Model init and Data Loaders \n",
        "\n",
        "Here we initalize the model as a Brt model for token classification"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "yNRpiuLQFNIU"
      },
      "source": [
        "\n",
        "def model_init():\n",
        "    return BertForTokenClassification.from_pretrained(MODEL_NAME,\n",
        "                                                      return_dict=True,\n",
        "                                                      num_labels=len(label_map),output_attentions = False,\n",
        "                                                      output_hidden_states = False)\n",
        "\n",
        "\n",
        "\n"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 171,
          "referenced_widgets": [
            "ddcf4ff7e10d40e5a2bd30392224f58e",
            "3053455a9b654d07a23c43e6aafcece0",
            "eda598850eee46babb4d23c65325a857",
            "1ac5725662084e3bb6ff258788a99804",
            "b201e0ccb3ee42f094761812a9e0e7f5",
            "057b5c1ab4c547ce9e0750659c412934",
            "6a81b0b3debd4782a8932cafc30886a5",
            "a8de1de271e249458ee1ddbf567b00af"
          ]
        },
        "id": "XdWKbbci6lro",
        "outputId": "8210413c-cae5-488a-c24c-73bf6816bab7"
      },
      "source": [
        "my_model = model_init()"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "display_data",
          "data": {
            "application/vnd.jupyter.widget-view+json": {
              "model_id": "ddcf4ff7e10d40e5a2bd30392224f58e",
              "version_minor": 0,
              "version_major": 2
            },
            "text/plain": [
              "HBox(children=(FloatProgress(value=0.0, description='Downloading', max=543490667.0, style=ProgressStyle(descri…"
            ]
          },
          "metadata": {
            "tags": []
          }
        },
        {
          "output_type": "stream",
          "text": [
            "\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "Some weights of the model checkpoint at aubmindlab/bert-base-arabertv02 were not used when initializing BertForTokenClassification: ['cls.seq_relationship.weight', 'cls.predictions.transform.dense.bias', 'cls.predictions.decoder.bias', 'cls.predictions.transform.dense.weight', 'cls.seq_relationship.bias', 'cls.predictions.bias', 'cls.predictions.decoder.weight', 'cls.predictions.transform.LayerNorm.bias', 'cls.predictions.transform.LayerNorm.weight']\n",
            "- This IS expected if you are initializing BertForTokenClassification from the checkpoint of a model trained on another task or with another architecture (e.g. initializing a BertForSequenceClassification model from a BertForPreTraining model).\n",
            "- This IS NOT expected if you are initializing BertForTokenClassification from the checkpoint of a model that you expect to be exactly identical (initializing a BertForSequenceClassification model from a BertForSequenceClassification model).\n",
            "Some weights of BertForTokenClassification were not initialized from the model checkpoint at aubmindlab/bert-base-arabertv02 and are newly initialized: ['classifier.bias', 'classifier.weight']\n",
            "You should probably TRAIN this model on a down-stream task to be able to use it for predictions and inference.\n"
          ],
          "name": "stderr"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "fs-4YX3e6rN1",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "53d2cdf6-040d-4913-d5ce-8d74450ee151"
      },
      "source": [
        "print(my_model)"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "BertForTokenClassification(\n",
            "  (bert): BertModel(\n",
            "    (embeddings): BertEmbeddings(\n",
            "      (word_embeddings): Embedding(64000, 768, padding_idx=0)\n",
            "      (position_embeddings): Embedding(512, 768)\n",
            "      (token_type_embeddings): Embedding(2, 768)\n",
            "      (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
            "      (dropout): Dropout(p=0.1, inplace=False)\n",
            "    )\n",
            "    (encoder): BertEncoder(\n",
            "      (layer): ModuleList(\n",
            "        (0): BertLayer(\n",
            "          (attention): BertAttention(\n",
            "            (self): BertSelfAttention(\n",
            "              (query): Linear(in_features=768, out_features=768, bias=True)\n",
            "              (key): Linear(in_features=768, out_features=768, bias=True)\n",
            "              (value): Linear(in_features=768, out_features=768, bias=True)\n",
            "              (dropout): Dropout(p=0.1, inplace=False)\n",
            "            )\n",
            "            (output): BertSelfOutput(\n",
            "              (dense): Linear(in_features=768, out_features=768, bias=True)\n",
            "              (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
            "              (dropout): Dropout(p=0.1, inplace=False)\n",
            "            )\n",
            "          )\n",
            "          (intermediate): BertIntermediate(\n",
            "            (dense): Linear(in_features=768, out_features=3072, bias=True)\n",
            "          )\n",
            "          (output): BertOutput(\n",
            "            (dense): Linear(in_features=3072, out_features=768, bias=True)\n",
            "            (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
            "            (dropout): Dropout(p=0.1, inplace=False)\n",
            "          )\n",
            "        )\n",
            "        (1): BertLayer(\n",
            "          (attention): BertAttention(\n",
            "            (self): BertSelfAttention(\n",
            "              (query): Linear(in_features=768, out_features=768, bias=True)\n",
            "              (key): Linear(in_features=768, out_features=768, bias=True)\n",
            "              (value): Linear(in_features=768, out_features=768, bias=True)\n",
            "              (dropout): Dropout(p=0.1, inplace=False)\n",
            "            )\n",
            "            (output): BertSelfOutput(\n",
            "              (dense): Linear(in_features=768, out_features=768, bias=True)\n",
            "              (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
            "              (dropout): Dropout(p=0.1, inplace=False)\n",
            "            )\n",
            "          )\n",
            "          (intermediate): BertIntermediate(\n",
            "            (dense): Linear(in_features=768, out_features=3072, bias=True)\n",
            "          )\n",
            "          (output): BertOutput(\n",
            "            (dense): Linear(in_features=3072, out_features=768, bias=True)\n",
            "            (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
            "            (dropout): Dropout(p=0.1, inplace=False)\n",
            "          )\n",
            "        )\n",
            "        (2): BertLayer(\n",
            "          (attention): BertAttention(\n",
            "            (self): BertSelfAttention(\n",
            "              (query): Linear(in_features=768, out_features=768, bias=True)\n",
            "              (key): Linear(in_features=768, out_features=768, bias=True)\n",
            "              (value): Linear(in_features=768, out_features=768, bias=True)\n",
            "              (dropout): Dropout(p=0.1, inplace=False)\n",
            "            )\n",
            "            (output): BertSelfOutput(\n",
            "              (dense): Linear(in_features=768, out_features=768, bias=True)\n",
            "              (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
            "              (dropout): Dropout(p=0.1, inplace=False)\n",
            "            )\n",
            "          )\n",
            "          (intermediate): BertIntermediate(\n",
            "            (dense): Linear(in_features=768, out_features=3072, bias=True)\n",
            "          )\n",
            "          (output): BertOutput(\n",
            "            (dense): Linear(in_features=3072, out_features=768, bias=True)\n",
            "            (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
            "            (dropout): Dropout(p=0.1, inplace=False)\n",
            "          )\n",
            "        )\n",
            "        (3): BertLayer(\n",
            "          (attention): BertAttention(\n",
            "            (self): BertSelfAttention(\n",
            "              (query): Linear(in_features=768, out_features=768, bias=True)\n",
            "              (key): Linear(in_features=768, out_features=768, bias=True)\n",
            "              (value): Linear(in_features=768, out_features=768, bias=True)\n",
            "              (dropout): Dropout(p=0.1, inplace=False)\n",
            "            )\n",
            "            (output): BertSelfOutput(\n",
            "              (dense): Linear(in_features=768, out_features=768, bias=True)\n",
            "              (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
            "              (dropout): Dropout(p=0.1, inplace=False)\n",
            "            )\n",
            "          )\n",
            "          (intermediate): BertIntermediate(\n",
            "            (dense): Linear(in_features=768, out_features=3072, bias=True)\n",
            "          )\n",
            "          (output): BertOutput(\n",
            "            (dense): Linear(in_features=3072, out_features=768, bias=True)\n",
            "            (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
            "            (dropout): Dropout(p=0.1, inplace=False)\n",
            "          )\n",
            "        )\n",
            "        (4): BertLayer(\n",
            "          (attention): BertAttention(\n",
            "            (self): BertSelfAttention(\n",
            "              (query): Linear(in_features=768, out_features=768, bias=True)\n",
            "              (key): Linear(in_features=768, out_features=768, bias=True)\n",
            "              (value): Linear(in_features=768, out_features=768, bias=True)\n",
            "              (dropout): Dropout(p=0.1, inplace=False)\n",
            "            )\n",
            "            (output): BertSelfOutput(\n",
            "              (dense): Linear(in_features=768, out_features=768, bias=True)\n",
            "              (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
            "              (dropout): Dropout(p=0.1, inplace=False)\n",
            "            )\n",
            "          )\n",
            "          (intermediate): BertIntermediate(\n",
            "            (dense): Linear(in_features=768, out_features=3072, bias=True)\n",
            "          )\n",
            "          (output): BertOutput(\n",
            "            (dense): Linear(in_features=3072, out_features=768, bias=True)\n",
            "            (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
            "            (dropout): Dropout(p=0.1, inplace=False)\n",
            "          )\n",
            "        )\n",
            "        (5): BertLayer(\n",
            "          (attention): BertAttention(\n",
            "            (self): BertSelfAttention(\n",
            "              (query): Linear(in_features=768, out_features=768, bias=True)\n",
            "              (key): Linear(in_features=768, out_features=768, bias=True)\n",
            "              (value): Linear(in_features=768, out_features=768, bias=True)\n",
            "              (dropout): Dropout(p=0.1, inplace=False)\n",
            "            )\n",
            "            (output): BertSelfOutput(\n",
            "              (dense): Linear(in_features=768, out_features=768, bias=True)\n",
            "              (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
            "              (dropout): Dropout(p=0.1, inplace=False)\n",
            "            )\n",
            "          )\n",
            "          (intermediate): BertIntermediate(\n",
            "            (dense): Linear(in_features=768, out_features=3072, bias=True)\n",
            "          )\n",
            "          (output): BertOutput(\n",
            "            (dense): Linear(in_features=3072, out_features=768, bias=True)\n",
            "            (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
            "            (dropout): Dropout(p=0.1, inplace=False)\n",
            "          )\n",
            "        )\n",
            "        (6): BertLayer(\n",
            "          (attention): BertAttention(\n",
            "            (self): BertSelfAttention(\n",
            "              (query): Linear(in_features=768, out_features=768, bias=True)\n",
            "              (key): Linear(in_features=768, out_features=768, bias=True)\n",
            "              (value): Linear(in_features=768, out_features=768, bias=True)\n",
            "              (dropout): Dropout(p=0.1, inplace=False)\n",
            "            )\n",
            "            (output): BertSelfOutput(\n",
            "              (dense): Linear(in_features=768, out_features=768, bias=True)\n",
            "              (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
            "              (dropout): Dropout(p=0.1, inplace=False)\n",
            "            )\n",
            "          )\n",
            "          (intermediate): BertIntermediate(\n",
            "            (dense): Linear(in_features=768, out_features=3072, bias=True)\n",
            "          )\n",
            "          (output): BertOutput(\n",
            "            (dense): Linear(in_features=3072, out_features=768, bias=True)\n",
            "            (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
            "            (dropout): Dropout(p=0.1, inplace=False)\n",
            "          )\n",
            "        )\n",
            "        (7): BertLayer(\n",
            "          (attention): BertAttention(\n",
            "            (self): BertSelfAttention(\n",
            "              (query): Linear(in_features=768, out_features=768, bias=True)\n",
            "              (key): Linear(in_features=768, out_features=768, bias=True)\n",
            "              (value): Linear(in_features=768, out_features=768, bias=True)\n",
            "              (dropout): Dropout(p=0.1, inplace=False)\n",
            "            )\n",
            "            (output): BertSelfOutput(\n",
            "              (dense): Linear(in_features=768, out_features=768, bias=True)\n",
            "              (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
            "              (dropout): Dropout(p=0.1, inplace=False)\n",
            "            )\n",
            "          )\n",
            "          (intermediate): BertIntermediate(\n",
            "            (dense): Linear(in_features=768, out_features=3072, bias=True)\n",
            "          )\n",
            "          (output): BertOutput(\n",
            "            (dense): Linear(in_features=3072, out_features=768, bias=True)\n",
            "            (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
            "            (dropout): Dropout(p=0.1, inplace=False)\n",
            "          )\n",
            "        )\n",
            "        (8): BertLayer(\n",
            "          (attention): BertAttention(\n",
            "            (self): BertSelfAttention(\n",
            "              (query): Linear(in_features=768, out_features=768, bias=True)\n",
            "              (key): Linear(in_features=768, out_features=768, bias=True)\n",
            "              (value): Linear(in_features=768, out_features=768, bias=True)\n",
            "              (dropout): Dropout(p=0.1, inplace=False)\n",
            "            )\n",
            "            (output): BertSelfOutput(\n",
            "              (dense): Linear(in_features=768, out_features=768, bias=True)\n",
            "              (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
            "              (dropout): Dropout(p=0.1, inplace=False)\n",
            "            )\n",
            "          )\n",
            "          (intermediate): BertIntermediate(\n",
            "            (dense): Linear(in_features=768, out_features=3072, bias=True)\n",
            "          )\n",
            "          (output): BertOutput(\n",
            "            (dense): Linear(in_features=3072, out_features=768, bias=True)\n",
            "            (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
            "            (dropout): Dropout(p=0.1, inplace=False)\n",
            "          )\n",
            "        )\n",
            "        (9): BertLayer(\n",
            "          (attention): BertAttention(\n",
            "            (self): BertSelfAttention(\n",
            "              (query): Linear(in_features=768, out_features=768, bias=True)\n",
            "              (key): Linear(in_features=768, out_features=768, bias=True)\n",
            "              (value): Linear(in_features=768, out_features=768, bias=True)\n",
            "              (dropout): Dropout(p=0.1, inplace=False)\n",
            "            )\n",
            "            (output): BertSelfOutput(\n",
            "              (dense): Linear(in_features=768, out_features=768, bias=True)\n",
            "              (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
            "              (dropout): Dropout(p=0.1, inplace=False)\n",
            "            )\n",
            "          )\n",
            "          (intermediate): BertIntermediate(\n",
            "            (dense): Linear(in_features=768, out_features=3072, bias=True)\n",
            "          )\n",
            "          (output): BertOutput(\n",
            "            (dense): Linear(in_features=3072, out_features=768, bias=True)\n",
            "            (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
            "            (dropout): Dropout(p=0.1, inplace=False)\n",
            "          )\n",
            "        )\n",
            "        (10): BertLayer(\n",
            "          (attention): BertAttention(\n",
            "            (self): BertSelfAttention(\n",
            "              (query): Linear(in_features=768, out_features=768, bias=True)\n",
            "              (key): Linear(in_features=768, out_features=768, bias=True)\n",
            "              (value): Linear(in_features=768, out_features=768, bias=True)\n",
            "              (dropout): Dropout(p=0.1, inplace=False)\n",
            "            )\n",
            "            (output): BertSelfOutput(\n",
            "              (dense): Linear(in_features=768, out_features=768, bias=True)\n",
            "              (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
            "              (dropout): Dropout(p=0.1, inplace=False)\n",
            "            )\n",
            "          )\n",
            "          (intermediate): BertIntermediate(\n",
            "            (dense): Linear(in_features=768, out_features=3072, bias=True)\n",
            "          )\n",
            "          (output): BertOutput(\n",
            "            (dense): Linear(in_features=3072, out_features=768, bias=True)\n",
            "            (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
            "            (dropout): Dropout(p=0.1, inplace=False)\n",
            "          )\n",
            "        )\n",
            "        (11): BertLayer(\n",
            "          (attention): BertAttention(\n",
            "            (self): BertSelfAttention(\n",
            "              (query): Linear(in_features=768, out_features=768, bias=True)\n",
            "              (key): Linear(in_features=768, out_features=768, bias=True)\n",
            "              (value): Linear(in_features=768, out_features=768, bias=True)\n",
            "              (dropout): Dropout(p=0.1, inplace=False)\n",
            "            )\n",
            "            (output): BertSelfOutput(\n",
            "              (dense): Linear(in_features=768, out_features=768, bias=True)\n",
            "              (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
            "              (dropout): Dropout(p=0.1, inplace=False)\n",
            "            )\n",
            "          )\n",
            "          (intermediate): BertIntermediate(\n",
            "            (dense): Linear(in_features=768, out_features=3072, bias=True)\n",
            "          )\n",
            "          (output): BertOutput(\n",
            "            (dense): Linear(in_features=3072, out_features=768, bias=True)\n",
            "            (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
            "            (dropout): Dropout(p=0.1, inplace=False)\n",
            "          )\n",
            "        )\n",
            "      )\n",
            "    )\n",
            "  )\n",
            "  (dropout): Dropout(p=0.1, inplace=False)\n",
            "  (classifier): Linear(in_features=768, out_features=102, bias=True)\n",
            ")\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "KwPHlLOwPGhy"
      },
      "source": [
        "# train_data_loader = DataLoader(dataset=train_dataset,batch_size=TRAIN_BATCH_SIZE,shuffle= True)\n",
        "# eval_data_loader = DataLoader(dataset=eval_dataset,batch_size=VALID_BATCH_SIZE,shuffle= True)\n",
        "test_data_loader = DataLoader(dataset=test_dataset,batch_size=TEST_BATCH_SIZE,shuffle= True)"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "ll13A3prxJzV"
      },
      "source": [
        "### Train, Eval, Test Functions"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "G_AeVh6DeJqa"
      },
      "source": [
        "#### Helper Funvtions"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "-ux11kosx37u"
      },
      "source": [
        "\n",
        "def align_predictions(predictions, label_ids):\n",
        "    \"\"\"\n",
        "        Takes batch of senteces, and the logits for every word in every sentences \n",
        "        it will exclude the padding tokens \n",
        "        \n",
        "        predictions:  [ [ [logits] ] ]\n",
        "        label_ids: [[Sentence1], [Sentence2]]\n",
        "        \n",
        "        returns : predicition label list of shape (no_sentences, no_words_in_sentece)\n",
        "                  true label list of shape (no_sentences, no_words_in_sentece)\n",
        "                  \n",
        "                  output example: y_true = [['O', 'O', 'O', 'B-MISC', 'I-MISC', 'I-MISC', 'O'], ['B-PER', 'I-PER', 'O']]\n",
        "                                  y_pred = [['O', 'O', 'B-MISC', 'I-MISC', 'I-MISC', 'I-MISC', 'O'], ['B-PER', 'I-PER', 'O']]\n",
        "    \"\"\"\n",
        "   \n",
        "    preds = np.argmax(predictions, axis=2)\n",
        "    \n",
        "   \n",
        "\n",
        "    batch_size, seq_len = preds.shape\n",
        "\n",
        "    assert(preds.shape == label_ids.shape)\n",
        "    \n",
        "    out_label_list = [[] for _ in range(batch_size)]\n",
        "    preds_list = [[] for _ in range(batch_size)]\n",
        "\n",
        "    for i in range(batch_size):\n",
        "        for j in range(seq_len):\n",
        "            \n",
        "            if label_ids[i] [j] != torch.nn.CrossEntropyLoss().ignore_index:\n",
        "                \n",
        "                out_label_list[i].append(inv_label_map[label_ids[i][j]])\n",
        "                preds_list[i].append(inv_label_map[preds[i][j]])\n",
        "\n",
        "    return preds_list, out_label_list"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "_JUuvw2g4JDg"
      },
      "source": [
        "def compute_metrics(predictions, labels , generate_report = False):\n",
        "    \n",
        "    preds_list, out_label_list = align_predictions(predictions,labels)\n",
        "    \n",
        "    \n",
        "    if generate_report:\n",
        "        try:         \n",
        "            print(classification_report(out_label_list, preds_list))\n",
        "        except:\n",
        "            print('There Was an error while generating the classification report!')\n",
        "            \n",
        "    return {\n",
        "        \"accuracy_score\": accuracy_score(out_label_list, preds_list),\n",
        "        \"precision\": precision_score(out_label_list, preds_list),\n",
        "        \"recall\": recall_score(out_label_list, preds_list),\n",
        "        \"f1\": f1_score(out_label_list, preds_list),\n",
        "    }\n"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "PsQxjGC5dqb8"
      },
      "source": [
        "#### Test Function"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "U2K4mVzK8h-x"
      },
      "source": [
        "def model_test(test_dl,model,device):\n",
        "\n",
        "    ###########################################\n",
        "    #                                         #\n",
        "    #           TEST STARTS HERE              #\n",
        "    #                                         #\n",
        "    ###########################################\n",
        "    with torch.no_grad():\n",
        "        model.to(device)\n",
        "        model.eval()\n",
        "        final_loss = 0\n",
        "        all_predictions = []\n",
        "        all_labels = []\n",
        "        for data in tqdm(test_dl, total = len(test_dl)):\n",
        "\n",
        "            input_ids = data['input_ids'].to(device)\n",
        "            attention_mask = data['attention_mask'].to(device)\n",
        "            token_type_ids = data['token_type_ids'].to(device)\n",
        "            labels = data['labels'].to(device)\n",
        "\n",
        "\n",
        "\n",
        "            outputs = model(input_ids = input_ids,\n",
        "                            token_type_ids=token_type_ids,\n",
        "                            attention_mask=attention_mask,\n",
        "                            labels=labels)\n",
        "            \n",
        "            \n",
        "            loss = outputs.loss\n",
        "\n",
        "            predictions = outputs.logits\n",
        "\n",
        "            all_labels.extend(labels.to('cpu').numpy())\n",
        "            all_predictions.extend(outputs.logits.to('cpu').numpy())\n",
        "\n",
        "            final_loss += loss.item()\n",
        "\n",
        "            #print(np.array(all_predictions).shape)\n",
        "\n",
        "            # all_labels=  torch.tensor(all_labels).to('cpu').numpy()\n",
        "            # all_predictions = torch.tensor(all_predictions).to('cpu').numpy()\n",
        "\n",
        "            \n",
        "            \n",
        "        metrics = \\\n",
        "        compute_metrics(predictions=np.asarray(all_predictions), labels=np.asarray(all_labels))\n",
        "\n",
        "        accuracy_score = metrics['accuracy_score']\n",
        "        precision= metrics['precision']\n",
        "        recall= metrics['recall']\n",
        "        f1= metrics['f1']\n",
        "\n",
        "        print(f' Accuracy: {accuracy_score}')\n",
        "        print(f' Precision: {precision}')\n",
        "        print(f' Recall: {recall}')\n",
        "        print(f' F1: {f1}')\n",
        "\n",
        "\n",
        "        final_loss /= len(test_dl) \n",
        "\n",
        "        print(f'Final Test Loss is: {final_loss}' )\n"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "FP53AdCsyKTt"
      },
      "source": [
        "### Testing"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "pwR0FribNXMa"
      },
      "source": [
        ""
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "OXQRksMkG4-9"
      },
      "source": [
        "! cp drive/MyDrive/Models/full_model_v2 ."
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "x2taxTzeGOb7",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 151,
          "referenced_widgets": [
            "985019b915664c3d83ec3240d95e7bba",
            "9edd99038a3046a088876566d64ff9dc",
            "0ec74a8b82f54907862d8f28113a7f81",
            "5baa743e23bb4a22991bbd8b120eff0a",
            "8ccb594c40b941218fbc10652d4ac8d3",
            "5eb15b030ad1450cb9096c23e08781c6",
            "0c723fcbd3734a1293e695176659f945",
            "d6e1256d9f48415b8816b7d390e60951"
          ]
        },
        "outputId": "e4a0c5c8-7058-4ffa-f7bc-5f887c7a43cd"
      },
      "source": [
        "############ TESTING ###########\n",
        "\n",
        "\n",
        "# test_model = model_init()\n",
        "# test_model.load_state_dict(torch.load('model_v1', map_location='cpu'))\n",
        "\n",
        "test_model = torch.load('full_model_v2' ,map_location='cpu')\n",
        "\n",
        "model_test(test_dl= test_data_loader,model=test_model, device= device)"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "display_data",
          "data": {
            "application/vnd.jupyter.widget-view+json": {
              "model_id": "985019b915664c3d83ec3240d95e7bba",
              "version_minor": 0,
              "version_major": 2
            },
            "text/plain": [
              "HBox(children=(FloatProgress(value=0.0, max=298.0), HTML(value='')))"
            ]
          },
          "metadata": {
            "tags": []
          }
        },
        {
          "output_type": "stream",
          "text": [
            "\n",
            " Accuracy: 0.9677850108496078\n",
            " Precision: 0.7739946147832885\n",
            " Recall: 0.7814610190300798\n",
            " F1: 0.777709897015186\n",
            "Final Test Loss is: 0.22581731898072582\n"
          ],
          "name": "stdout"
        }
      ]
    }
  ]
}