Multilingual-NER-App.ipynb

{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.10.12","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"![](https://cdn-images-1.medium.com/max/1067/1*6p7Jfyu0o4J8w1ZDfLTc9Q.png)\n\nIn this notebook, I'll walk you through how to build a multilinguil NER app with HuggingFace, Gradio, and Comet ML.","metadata":{}},{"cell_type":"markdown","source":"# <b><div style='padding:15px;background-color:#850E35;color:white;border-radius:2px;font-size:110%;text-align: center'>Installing Required Libraries </div></b>","metadata":{}},{"cell_type":"code","source":"!pip install -q comet_ml datasets seqeval evaluate gradio ","metadata":{"execution":{"iopub.status.busy":"2023-10-30T17:33:32.295713Z","iopub.execute_input":"2023-10-30T17:33:32.296014Z","iopub.status.idle":"2023-10-30T17:33:57.593349Z","shell.execute_reply.started":"2023-10-30T17:33:32.295988Z","shell.execute_reply":"2023-10-30T17:33:57.592103Z"},"trusted":true},"execution_count":1,"outputs":[{"name":"stdout","text":"\u001b[31mERROR: pip's dependency resolver does not currently take into account all the packages that are installed. This behaviour is the source of the following dependency conflicts.\njupyterlab-lsp 4.2.0 requires jupyter-lsp>=2.0.0, but you have jupyter-lsp 1.5.1 which is incompatible.\u001b[0m\u001b[31m\n\u001b[0m","output_type":"stream"}]},{"cell_type":"code","source":"import comet_ml\n\ncomet_ml.init(project_name=\"Multilingual-NER-App\")","metadata":{"execution":{"iopub.status.busy":"2023-10-30T17:34:38.247539Z","iopub.execute_input":"2023-10-30T17:34:38.247987Z","iopub.status.idle":"2023-10-30T17:34:47.994668Z","shell.execute_reply.started":"2023-10-30T17:34:38.247946Z","shell.execute_reply":"2023-10-30T17:34:47.993749Z"},"trusted":true},"execution_count":2,"outputs":[{"name":"stdout","text":"Please paste your Comet API key from https://www.comet.com/api/my/settings/\n(api key may not show as you type)\n","output_type":"stream"},{"output_type":"stream","name":"stdin","text":"Comet API key:  ·························\n"},{"name":"stderr","text":"\u001b[1;38;5;39mCOMET INFO:\u001b[0m Valid Comet API Key saved in /root/.comet.config (set COMET_CONFIG to change where it is saved).\n","output_type":"stream"}]},{"cell_type":"code","source":"from huggingface_hub import notebook_login\n\nnotebook_login()","metadata":{"execution":{"iopub.status.busy":"2023-10-30T17:34:55.806857Z","iopub.execute_input":"2023-10-30T17:34:55.807202Z","iopub.status.idle":"2023-10-30T17:34:55.999211Z","shell.execute_reply.started":"2023-10-30T17:34:55.807175Z","shell.execute_reply":"2023-10-30T17:34:55.998310Z"},"trusted":true},"execution_count":3,"outputs":[{"output_type":"display_data","data":{"text/plain":"VBox(children=(HTML(value='<center> <img\\nsrc=https://huggingface.co/front/assets/huggingface_logo-noborder.sv…","application/vnd.jupyter.widget-view+json":{"version_major":2,"version_minor":0,"model_id":"bb41039458ac4364b857dc5f2e7dfa4c"}},"metadata":{}}]},{"cell_type":"markdown","source":"# <b><div style='padding:15px;background-color:#850E35;color:white;border-radius:2px;font-size:110%;text-align: center'>Loading the Dataset </div></b>","metadata":{}},{"cell_type":"code","source":"from datasets import load_dataset\nfrom collections import defaultdict\nfrom datasets import DatasetDict\n\n\nlangs = [\"en\", \"de\", \"tr\"]\nfracs= [0.5, 0.5, 0.5]\n\npanx_ds = defaultdict(DatasetDict)\n \nfor lang, frac in zip(langs, fracs):\n    dataset = load_dataset(\"xtreme\", name=f\"PAN-X.{lang}\")\n    for split in dataset:\n        panx_ds[lang][split] = (dataset[split].shuffle(seed=42).select(range(int(frac*dataset[split].num_rows))))","metadata":{"execution":{"iopub.status.busy":"2023-10-30T17:36:51.567277Z","iopub.execute_input":"2023-10-30T17:36:51.568036Z","iopub.status.idle":"2023-10-30T17:37:24.631571Z","shell.execute_reply.started":"2023-10-30T17:36:51.568003Z","shell.execute_reply":"2023-10-30T17:37:24.630552Z"},"trusted":true},"execution_count":4,"outputs":[{"output_type":"display_data","data":{"text/plain":"Downloading builder script:   0%|          | 0.00/9.09k [00:00<?, ?B/s]","application/vnd.jupyter.widget-view+json":{"version_major":2,"version_minor":0,"model_id":"3834ade716de402a92b09e71b5b8c7bb"}},"metadata":{}},{"output_type":"display_data","data":{"text/plain":"Downloading metadata:   0%|          | 0.00/23.1k [00:00<?, ?B/s]","application/vnd.jupyter.widget-view+json":{"version_major":2,"version_minor":0,"model_id":"0d10caeaeb294b79a4fc272e617863cd"}},"metadata":{}},{"name":"stdout","text":"Downloading and preparing dataset xtreme/PAN-X.en (download: 223.17 MiB, generated: 7.30 MiB, post-processed: Unknown size, total: 230.47 MiB) to /root/.cache/huggingface/datasets/xtreme/PAN-X.en/1.0.0/349258adc25bb45e47de193222f95e68a44f7a7ab53c4283b3f007208a11bf7e...\n","output_type":"stream"},{"output_type":"display_data","data":{"text/plain":"Downloading data:   0%|          | 0.00/234M [00:00<?, ?B/s]","application/vnd.jupyter.widget-view+json":{"version_major":2,"version_minor":0,"model_id":"7009ec2fab5c4d5b9403223faf7917d1"}},"metadata":{}},{"output_type":"display_data","data":{"text/plain":"Generating train split:   0%|          | 0/20000 [00:00<?, ? examples/s]","application/vnd.jupyter.widget-view+json":{"version_major":2,"version_minor":0,"model_id":""}},"metadata":{}},{"output_type":"display_data","data":{"text/plain":"Generating validation split:   0%|          | 0/10000 [00:00<?, ? examples/s]","application/vnd.jupyter.widget-view+json":{"version_major":2,"version_minor":0,"model_id":""}},"metadata":{}},{"output_type":"display_data","data":{"text/plain":"Generating test split:   0%|          | 0/10000 [00:00<?, ? examples/s]","application/vnd.jupyter.widget-view+json":{"version_major":2,"version_minor":0,"model_id":""}},"metadata":{}},{"name":"stdout","text":"Dataset xtreme downloaded and prepared to /root/.cache/huggingface/datasets/xtreme/PAN-X.en/1.0.0/349258adc25bb45e47de193222f95e68a44f7a7ab53c4283b3f007208a11bf7e. Subsequent calls will reuse this data.\n","output_type":"stream"},{"output_type":"display_data","data":{"text/plain":"  0%|          | 0/3 [00:00<?, ?it/s]","application/vnd.jupyter.widget-view+json":{"version_major":2,"version_minor":0,"model_id":"197f1938156448ea9f37936f181e052c"}},"metadata":{}},{"name":"stdout","text":"Downloading and preparing dataset xtreme/PAN-X.de (download: 223.17 MiB, generated: 9.08 MiB, post-processed: Unknown size, total: 232.25 MiB) to /root/.cache/huggingface/datasets/xtreme/PAN-X.de/1.0.0/349258adc25bb45e47de193222f95e68a44f7a7ab53c4283b3f007208a11bf7e...\n","output_type":"stream"},{"output_type":"display_data","data":{"text/plain":"Generating train split:   0%|          | 0/20000 [00:00<?, ? examples/s]","application/vnd.jupyter.widget-view+json":{"version_major":2,"version_minor":0,"model_id":""}},"metadata":{}},{"output_type":"display_data","data":{"text/plain":"Generating validation split:   0%|          | 0/10000 [00:00<?, ? examples/s]","application/vnd.jupyter.widget-view+json":{"version_major":2,"version_minor":0,"model_id":""}},"metadata":{}},{"output_type":"display_data","data":{"text/plain":"Generating test split:   0%|          | 0/10000 [00:00<?, ? examples/s]","application/vnd.jupyter.widget-view+json":{"version_major":2,"version_minor":0,"model_id":""}},"metadata":{}},{"name":"stdout","text":"Dataset xtreme downloaded and prepared to /root/.cache/huggingface/datasets/xtreme/PAN-X.de/1.0.0/349258adc25bb45e47de193222f95e68a44f7a7ab53c4283b3f007208a11bf7e. Subsequent calls will reuse this data.\n","output_type":"stream"},{"output_type":"display_data","data":{"text/plain":"  0%|          | 0/3 [00:00<?, ?it/s]","application/vnd.jupyter.widget-view+json":{"version_major":2,"version_minor":0,"model_id":"d9deb93290e842e5add1d5beae725101"}},"metadata":{}},{"name":"stdout","text":"Downloading and preparing dataset xtreme/PAN-X.tr (download: 223.17 MiB, generated: 7.25 MiB, post-processed: Unknown size, total: 230.42 MiB) to /root/.cache/huggingface/datasets/xtreme/PAN-X.tr/1.0.0/349258adc25bb45e47de193222f95e68a44f7a7ab53c4283b3f007208a11bf7e...\n","output_type":"stream"},{"output_type":"display_data","data":{"text/plain":"Generating train split:   0%|          | 0/20000 [00:00<?, ? examples/s]","application/vnd.jupyter.widget-view+json":{"version_major":2,"version_minor":0,"model_id":""}},"metadata":{}},{"output_type":"display_data","data":{"text/plain":"Generating validation split:   0%|          | 0/10000 [00:00<?, ? examples/s]","application/vnd.jupyter.widget-view+json":{"version_major":2,"version_minor":0,"model_id":""}},"metadata":{}},{"output_type":"display_data","data":{"text/plain":"Generating test split:   0%|          | 0/10000 [00:00<?, ? examples/s]","application/vnd.jupyter.widget-view+json":{"version_major":2,"version_minor":0,"model_id":""}},"metadata":{}},{"name":"stdout","text":"Dataset xtreme downloaded and prepared to /root/.cache/huggingface/datasets/xtreme/PAN-X.tr/1.0.0/349258adc25bb45e47de193222f95e68a44f7a7ab53c4283b3f007208a11bf7e. Subsequent calls will reuse this data.\n","output_type":"stream"},{"output_type":"display_data","data":{"text/plain":"  0%|          | 0/3 [00:00<?, ?it/s]","application/vnd.jupyter.widget-view+json":{"version_major":2,"version_minor":0,"model_id":"57d5d6ece337499082750e7a670f0222"}},"metadata":{}}]},{"cell_type":"markdown","source":"Let's take a look at the number of training examples of all datasets:","metadata":{}},{"cell_type":"code","source":"import pandas as pd\n\npd.DataFrame({lang: [panx_ds[lang][\"train\"].num_rows] for lang in langs}, index=[\"Number of Training Samples\"])","metadata":{"execution":{"iopub.status.busy":"2023-10-30T17:37:24.633491Z","iopub.execute_input":"2023-10-30T17:37:24.634571Z","iopub.status.idle":"2023-10-30T17:37:24.657580Z","shell.execute_reply.started":"2023-10-30T17:37:24.634529Z","shell.execute_reply":"2023-10-30T17:37:24.656670Z"},"trusted":true},"execution_count":5,"outputs":[{"execution_count":5,"output_type":"execute_result","data":{"text/plain":"                               en     de     tr\nNumber of Training Samples  10000  10000  10000","text/html":"<div>\n<style scoped>\n    .dataframe tbody tr th:only-of-type {\n        vertical-align: middle;\n    }\n\n    .dataframe tbody tr th {\n        vertical-align: top;\n    }\n\n    .dataframe thead th {\n        text-align: right;\n    }\n</style>\n<table border=\"1\" class=\"dataframe\">\n  <thead>\n    <tr style=\"text-align: right;\">\n      <th></th>\n      <th>en</th>\n      <th>de</th>\n      <th>tr</th>\n    </tr>\n  </thead>\n  <tbody>\n    <tr>\n      <th>Number of Training Samples</th>\n      <td>10000</td>\n      <td>10000</td>\n      <td>10000</td>\n    </tr>\n  </tbody>\n</table>\n</div>"},"metadata":{}}]},{"cell_type":"markdown","source":"# <b><div style='padding:15px;background-color:#850E35;color:white;border-radius:2px;font-size:110%;text-align: center'>Understanding the Dataset </div></b>\n\n![](https://cdn-images-1.medium.com/max/1067/1*l7MZX_RcgQSY2ddTYOzUTg.png)","metadata":{}},{"cell_type":"code","source":"row = panx_ds[\"en\"][\"train\"][0]\n\nfor key, value in row.items():\n    print(f\"{key}: {value}\")","metadata":{"execution":{"iopub.status.busy":"2023-10-30T17:37:24.658824Z","iopub.execute_input":"2023-10-30T17:37:24.659216Z","iopub.status.idle":"2023-10-30T17:37:24.668727Z","shell.execute_reply.started":"2023-10-30T17:37:24.659165Z","shell.execute_reply":"2023-10-30T17:37:24.667578Z"},"trusted":true},"execution_count":6,"outputs":[{"name":"stdout","text":"tokens: [\"''\", 'January', '21', \"''\", '–', 'Nanny', 'and', 'the', 'Professor']\nner_tags: [0, 0, 0, 0, 0, 1, 2, 2, 2]\nlangs: ['en', 'en', 'en', 'en', 'en', 'en', 'en', 'en', 'en']\n","output_type":"stream"}]},{"cell_type":"code","source":"for key, value in panx_ds[\"en\"][\"train\"].features.items():\n    print(f\"{key}: {value}\")","metadata":{"execution":{"iopub.status.busy":"2023-10-30T17:37:24.671535Z","iopub.execute_input":"2023-10-30T17:37:24.671850Z","iopub.status.idle":"2023-10-30T17:37:24.681187Z","shell.execute_reply.started":"2023-10-30T17:37:24.671822Z","shell.execute_reply":"2023-10-30T17:37:24.680238Z"},"trusted":true},"execution_count":7,"outputs":[{"name":"stdout","text":"tokens: Sequence(feature=Value(dtype='string', id=None), length=-1, id=None)\nner_tags: Sequence(feature=ClassLabel(num_classes=7, names=['O', 'B-PER', 'I-PER', 'B-ORG', 'I-ORG', 'B-LOC', 'I-LOC'], id=None), length=-1, id=None)\nlangs: Sequence(feature=Value(dtype='string', id=None), length=-1, id=None)\n","output_type":"stream"}]},{"cell_type":"code","source":"labels = panx_ds[\"en\"][\"train\"].features[\"ner_tags\"].feature\n\ndef create_label_names(example):\n    return {\"ner_tags_str\": [labels.int2str(idx) for idx in example[\"ner_tags\"]]}\n\npanx_en = panx_ds[\"en\"].map(create_label_names)\nen_example = panx_en[\"train\"][0]\npd.DataFrame([en_example[\"tokens\"], en_example[\"ner_tags_str\"]], [\"Tokens\", \"Labels\"])","metadata":{"execution":{"iopub.status.busy":"2023-10-30T17:37:24.682288Z","iopub.execute_input":"2023-10-30T17:37:24.682649Z","iopub.status.idle":"2023-10-30T17:37:27.769013Z","shell.execute_reply.started":"2023-10-30T17:37:24.682616Z","shell.execute_reply":"2023-10-30T17:37:27.767832Z"},"trusted":true},"execution_count":8,"outputs":[{"output_type":"display_data","data":{"text/plain":"  0%|          | 0/10000 [00:00<?, ?ex/s]","application/vnd.jupyter.widget-view+json":{"version_major":2,"version_minor":0,"model_id":"7bafd4c0c5d0433cb9bc94eb1f78e835"}},"metadata":{}},{"output_type":"display_data","data":{"text/plain":"  0%|          | 0/5000 [00:00<?, ?ex/s]","application/vnd.jupyter.widget-view+json":{"version_major":2,"version_minor":0,"model_id":"234604ae638149c4a3f1b59f846b6fe3"}},"metadata":{}},{"output_type":"display_data","data":{"text/plain":"  0%|          | 0/5000 [00:00<?, ?ex/s]","application/vnd.jupyter.widget-view+json":{"version_major":2,"version_minor":0,"model_id":"f79dfc81fa27491b98a087077c8ded7b"}},"metadata":{}},{"execution_count":8,"output_type":"execute_result","data":{"text/plain":"         0        1   2   3  4      5      6      7          8\nTokens  ''  January  21  ''  –  Nanny    and    the  Professor\nLabels   O        O   O   O  O  B-PER  I-PER  I-PER      I-PER","text/html":"<div>\n<style scoped>\n    .dataframe tbody tr th:only-of-type {\n        vertical-align: middle;\n    }\n\n    .dataframe tbody tr th {\n        vertical-align: top;\n    }\n\n    .dataframe thead th {\n        text-align: right;\n    }\n</style>\n<table border=\"1\" class=\"dataframe\">\n  <thead>\n    <tr style=\"text-align: right;\">\n      <th></th>\n      <th>0</th>\n      <th>1</th>\n      <th>2</th>\n      <th>3</th>\n      <th>4</th>\n      <th>5</th>\n      <th>6</th>\n      <th>7</th>\n      <th>8</th>\n    </tr>\n  </thead>\n  <tbody>\n    <tr>\n      <th>Tokens</th>\n      <td>''</td>\n      <td>January</td>\n      <td>21</td>\n      <td>''</td>\n      <td>–</td>\n      <td>Nanny</td>\n      <td>and</td>\n      <td>the</td>\n      <td>Professor</td>\n    </tr>\n    <tr>\n      <th>Labels</th>\n      <td>O</td>\n      <td>O</td>\n      <td>O</td>\n      <td>O</td>\n      <td>O</td>\n      <td>B-PER</td>\n      <td>I-PER</td>\n      <td>I-PER</td>\n      <td>I-PER</td>\n    </tr>\n  </tbody>\n</table>\n</div>"},"metadata":{}}]},{"cell_type":"code","source":"from collections import Counter\n\nsplit2freqs = defaultdict(Counter)\n\nfor split, dataset in panx_en.items():\n    for row in dataset[\"ner_tags_str\"]:\n        for tag in row:\n            if tag.startswith(\"B\"):\n                tag_type = tag.split(\"-\")[1]\n                split2freqs[split][tag_type] +=1\n\npd.DataFrame.from_dict(split2freqs, orient = \"index\").plot(kind=\"barh\")","metadata":{"execution":{"iopub.status.busy":"2023-10-30T17:37:27.770704Z","iopub.execute_input":"2023-10-30T17:37:27.771322Z","iopub.status.idle":"2023-10-30T17:37:28.525040Z","shell.execute_reply.started":"2023-10-30T17:37:27.771286Z","shell.execute_reply":"2023-10-30T17:37:28.524155Z"},"trusted":true},"execution_count":9,"outputs":[{"execution_count":9,"output_type":"execute_result","data":{"text/plain":"<Axes: >"},"metadata":{}},{"output_type":"display_data","data":{"text/plain":"<Figure size 640x480 with 1 Axes>","image/png":"iVBORw0KGgoAAAANSUhEUgAAAl0AAAGdCAYAAAAogsYCAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjcuMiwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy8pXeV/AAAACXBIWXMAAA9hAAAPYQGoP6dpAAAnrElEQVR4nO3de1hVZaLH8d/msjfgdrNRUbyAYCBeMW8hNtNUal4aT2MzacqZE+ljF62pKaa0GsVq0ilz8lQ6lyw756ROZZpnMh00pfTxfsBLEl4G1EbUzARJBYX3/OHjnnZiacK7Ab+f59nPI3u/rPWuvR7i21prLxzGGCMAAADUqqBATwAAAOBqQHQBAABYQHQBAABYQHQBAABYQHQBAABYQHQBAABYQHQBAABYQHQBAABYEBLoCTRkVVVVOnjwoBo3biyHwxHo6QAAgEtgjNGJEyfUqlUrBQXV3PEpoqsWHTx4ULGxsYGeBgAA+AEOHDigNm3a1NjyiK5a1LhxY0nndprH4wnwbAAAwKUoLS1VbGys7/d4TSG6atH5U4oej4foAgCgnqnpS4O4kB4AAMACogsAAMACogsAAMACrukCAKCeMsbo7NmzqqysDPRU6p3g4GCFhIRYvaUT0QUAQD1UUVGh4uJinTx5MtBTqbciIiLUsmVLOZ1OK+sjugAAqGeqqqpUWFio4OBgtWrVSk6nk5twXwZjjCoqKvTFF1+osLBQSUlJNXoT1IshugAAqGcqKipUVVWl2NhYRUREBHo69VJ4eLhCQ0O1b98+VVRUKCwsrNbXyYX0AADUUzaOzjRktt8/9hYAAIAFRBcAAIAFXNMFAEADEj/hA6vrK5p2q9X11Wcc6QIAANZkZGTI4XDI4XDI6XQqMTFRTz/9tM6ePavVq1f7Xvv249ChQ5KkrKws33PBwcGKjY3VPffco2PHjgV4y74fR7oAAIBVgwYN0htvvKHy8nItXbpU48ePV2hoqNLS0iRJBQUF8ng8ft/TvHlz3787d+6sFStWqLKyUvn5+Ro9erRKSkr017/+1ep2XC6iCwAAWOVyuRQTEyNJuv/++7Vo0SItWbLEF13NmzeX1+u96PeHhIT4vr9169a644479MYbb9T6vK8UpxcBAEBAhYeHq6Ki4gd9b1FRkZYvX27trvJXgiNdAAAgIIwxWrlypZYvX64HH3zQ93ybNm38xrVt21affvqp7+vt27fL7XarsrJSp0+fliTNmDHDzqSvANEFAACs+tvf/ia3260zZ86oqqpKo0aNUlZWljZt2iRJ+uSTT9S4cWPf+NDQUL/vT05O1pIlS3T69Gn9z//8j/Ly8vyira4iugAAgFU33XSTZs+eLafTqVatWikkxD9HEhISvvOarvOfepSkadOm6dZbb9WUKVP0zDPP1Oa0rxjXdAEAAKsaNWqkxMRExcXFXRBcP8RTTz2l6dOn6+DBgzUwu9pDdAEAgDrlyJEjOnTokN/jzJkzFx2flpamlJQUPffccxZnefk4vQgAQAPSEO4Qn5ycfMFz69atU58+fS76Pb/+9a+VkZGhxx9/XLGxsbU5vR/MYYwxgZ5EQ1VaWqrIyEiVlJRccJM3AAB+qNOnT6uwsFAJCQkKCwsL9HTqrYu9j7X1+5vTiwAAABYQXQAAABYQXQAAABYQXQAAABYQXQAAABZwywgL+szro+Dw4EBPA0A1tt+1PdBTAHCV4EgXAACABUQXAACABUQXAACABVzTBQBAQ5IVaXl9JZc1PCMjQ8ePH9fixYsveO3UqVOaNm2a5s+fr3379qlx48a66aablJWVpc6dO/uNLS0t1e9//3stXLhQRUVF8nq96tKli8aNG6dhw4bJ4XBcyVbVCqILAAAEXHl5ufr376/9+/frxRdfVGpqqg4fPqypU6cqNTVVK1as8P3txePHj+tHP/qRSkpK9Oyzz6p3794KCQlRTk6OHnvsMd18883yer2B3aBqEF0AACDgXnrpJa1bt065ubnq1q2bJKlt27ZauHChUlNTNWbMGO3YsUMOh0NPPPGEioqKtGvXLrVq1cq3jPbt22vkyJF19u9Rck0XAAAIuHnz5mnAgAG+4DovKChIv/71r7Vz505t3bpVVVVVWrBggdLT0/2C6zy3262QkLp5TInoAgAAAbdr1y517Nix2tfOP79r1y4dPXpUX331lTp06GBzejWC6AIAAHWCMaZGxtRVRBcAAAi49u3bKz8/v9rXzj/fvn17RUdHy+v16rPPPrM5vRpBdAEAgIC78847tWLFCm3dutXv+aqqKv3hD39Qp06d1K1bNwUFBenOO+/UW2+9pYMHD16wnLKyMp09e9bWtC8L0QUAAKwqKSlRXl6e3+Pf//3fdd1112no0KF65513tH//fm3atEk///nPlZ+frzlz5vjuvfW73/1OsbGxSk1N1X/9139p586d2r17t15//XV1795dZWVlAd7C6tXNy/sBAECDtXr1anXv3t3vuTFjxuijjz7Sc889pyeeeMLv5qjr169Xly5dfGObNGmi9evXa9q0aXr22We1b98+RUVFqWvXrnrhhRcUGWn5BrGXyGHq8xVpdVxpaakiIyPVcXZHBYcHB3o6AKqx/a7tgZ4CcNlOnz6twsJCJSQk1Nl7UtUHF3sfz//+LikpkcfjqbH1cXoRAADAAqILAADAAqILAADAAqILAADAAqILAADAAqILAADAAu7TZcH6fZ/L43IEehoAskoCPQMAVzGOdAEAAFhAdAEAAFhAdAEAAFjANV0AADQgXd/sanV9P+RPaR04cECTJ0/WsmXLdPToUbVs2VI/+9nPNGnSJDVt2lSSdOONNyonJ0eS5HK5FBcXp7vvvlsTJkzw/eHr8xYuXKhXX31Vubm5On36tOLi4nT99dfrwQcfvOBvPAYSR7oAAIA1//jHP9SrVy/t3r1b8+fP1549e/THP/5RK1euVFpamo4dO+YbO3bsWBUXF6ugoEATJ07UpEmT9Mc//tFveY8//rhGjBiha6+9VkuWLFFBQYHmzZundu3aaeLEibY37zs1yCNdN954o6699lq99NJLNbK8jIwMHT9+XIsXL66R5QEAcLUaP368nE6n/v73vys8PFySFBcXp+7du+uaa67Rk08+qdmzZ0uSIiIiFBMTI0m6++679corryg7O1v333+/JGn9+vV6/vnnNXPmTP3qV7/yrSMuLk49e/aUMcby1n03jnQBAAArjh07puXLl2vcuHG+4DovJiZG6enp+utf/3pBLBlj9Mknn+izzz6T0+n0PT9//ny53W6NGzeu2vV9+zRkoDW46MrIyFBOTo5mzpwph8Mhh8OhoqIi7dixQ4MHD5bb7VaLFi30y1/+UkePHvV937vvvquuXbsqPDxcTZs2Vf/+/fX1118rKytLb775pt5//33f8lavXh24DQQAoJ7avXu3jDHq2LFjta937NhRX331lb744gtJ0qxZs+R2u+VyuXTDDTeoqqrK74jWrl271K5dO4WE/OvE3YwZM+R2u32PkpK6c3++BhddM2fOVFpamu88cHFxsRo3bqybb75Z3bt31+bNm7Vs2TIdPnxYw4cPlyQVFxdr5MiRGj16tPLz87V69WrdfvvtMsYoMzNTw4cP16BBg3zL69u3b7XrLi8vV2lpqd8DAAD4u9TTfunp6crLy9PatWs1ePBgPfnkkxf9HXze6NGjlZeXpz/96U/6+uuv69QpxgZ3TVdkZKScTqffeeBnn31W3bt313PPPecb9/rrrys2Nla7du1SWVmZzp49q9tvv11t27aVJHXt+q9Pf4SHh6u8vNy3vIuZOnWqpkyZUgtbBQBA/ZeYmCiHw6H8/HwNGzbsgtfz8/MVFRWl6OhoSed+pycmJkqS3n77bSUmJqpPnz7q37+/JCkpKUlr1qzRmTNnFBoaKknyer3yer36/PPPLW3VpWtwR7qqs3XrVq1atcrvcGOHDh0kSXv37lW3bt3Ur18/de3aVXfccYf+8pe/6Kuvvrrs9UycOFElJSW+x4EDB2p6UwAAqLeaNm2qAQMGaNasWTp16pTfa4cOHdJbb72lESNGVHstltvt1kMPPaTMzEzf0auRI0eqrKxMs2bNsjL/K3VVRFdZWZmGDh2qvLw8v8fu3bt1ww03KDg4WNnZ2frwww/VqVMnvfzyy0pOTlZhYeFlrcflcsnj8fg9AADAv7zyyisqLy/XwIED9fHHH+vAgQNatmyZBgwYoNatW+t3v/vdRb/33nvv1a5du7Rw4UJJUlpamh599FE9+uijeuSRR7RmzRrt27dP69ev15w5c+RwOBQUVHdSp+7MpAY5nU5VVlb6vu7Ro4c+/fRTxcfHKzEx0e/RqFEjSec+4XD99ddrypQpys3NldPp1KJFi6pdHgAA+GGSkpK0efNmtWvXTsOHD9c111yje+65RzfddJPWrVunJk2aXPR7mzRpov/4j/9QVlaWqqqqJEnTp0/XvHnzlJubq5/+9KdKSkrSHXfcoaqqKq1bt65OHQBpcNd0SVJ8fLw2bNigoqIiud1ujR8/Xn/5y180cuRIPfbYY2rSpIn27NmjBQsW6LXXXtPmzZu1cuVK3XLLLWrevLk2bNigL774wvfpivj4eC1fvlwFBQVq2rSpIiMjfeeOAQCoS37IHeJta9u2rebOnfudYy52p4Bv3xxVkoYPH+77cFxd1iCPdGVmZio4OFidOnVSdHS0KioqtHbtWlVWVuqWW25R165d9fDDD8vr9SooKEgej0cff/yxhgwZovbt2+upp57Siy++qMGDB0s6d0fc5ORk9erVS9HR0Vq7dm2AtxAAANQ3DlOXPkvZwJSWlioyMlIlExrL46pbN2gDrkpZded+PcCVOH36tAoLC5WQkKCwsLBAT6feutj76Pv9XVJSo6cnG+SRLgAAgLqG6AIAALCA6AIAALCA6AIAoJ7isuwrY/v9a5C3jKhrupyeoyATEehpAJjwge+fRdNuDeBEgCtz/rZFJ0+eVHh4eIBnU3+dPHlSkqzdBoroAgCgngkODpbX69WRI0ckSREREdX+6RxUzxijkydP6siRI/J6vQoODrayXqILAIB6KCYmRpJ84YXL5/V6fe+jDUQXAAD1kMPhUMuWLdW8eXOdOXMm0NOpd0JDQ60d4TqP6AIAoB4LDg62Hg/4Yfj0IgAAgAVEFwAAgAVEFwAAgAVEFwAAgAVEFwAAgAVEFwAAgAVEFwAAgAVEFwAAgAVEFwAAgAVEFwAAgAVEFwAAgAVEFwAAgAVEFwAAgAVEFwAAgAVEFwAAgAUhgZ7A1WDHlIHyeDyBngYAAAggjnQBAABYQHQBAABYQHQBAABYQHQBAABYQHQBAABYQHQBAABYQHQBAABYQHQBAABYQHQBAABYQHQBAABYQHQBAABYQHQBAABYQHQBAABYQHQBAABYQHQBAABYQHQBAABYQHQBAABYQHQBAABYQHQBAABYQHQBAABYQHQBAABYQHQBAABYQHQBAABYQHQBAABYQHQBAABYQHQBAABYQHQBAABYQHQBAABYQHQBAABYQHQBAABYQHQBAABYQHQBAABYQHQBAABYQHQBAABYQHQBAABYQHQBAABYQHQBAABYQHQBAABYQHQBAABYQHQBAABYQHQBAABYQHQBAABYQHQBAABYQHQBAABYQHQBAABYQHQBAABYQHQBAABYQHQBAABYQHQBAABYQHQBAABYQHQBAABYQHQBAABYQHQBAABYQHQBAABYQHQBAABYQHQBAABYQHQBAABYQHQBAABYQHQBAABYQHQBAABYQHQBAABYEBLoCVwN+szro+Dw4EBPA8BFbL9re6CnAOAqwJEuAAAAC4guAAAAC4guAAAAC4guAAAAC4guAAAAC4guAAAAC4guAAAAC4guAAAAC4guAAAAC4guAAAAC4guAAAAC4guAAAAC4guAAAAC4guAAAAC4guAAAAC4guAAAAC4guAAAAC4guAAAAC4guAAAAC0ICPYGrwfp9n8vjcgR6GgC+Lask0DMAcBXhSBcAAIAFRBcAAIAFRBcAAIAFRBcAAIAFVqMrPj5eL730ku9rh8OhxYsXX3R8UVGRHA6H8vLyrmi9NbUcAACAHyqgn14sLi5WVFRUjS4zIyNDx48f94u52NhYFRcXq1mzZjW6LgAAgEsV0OiKiYmxsp7g4GBr6wIAAKjOJZ9e/POf/6xWrVqpqqrK7/nbbrtNo0eP1t69e3XbbbepRYsWcrvd6t27t1asWPGdy/z26cWNGzeqe/fuCgsLU69evZSbm+s3vrKyUmPGjFFCQoLCw8OVnJysmTNn+l7PysrSm2++qffff18Oh0MOh0OrV6+u9vRiTk6OrrvuOrlcLrVs2VITJkzQ2bNnfa/feOON+tWvfqXHHntMTZo0UUxMjLKysi717QIAAPBzydF1xx136Msvv9SqVat8zx07dkzLli1Tenq6ysrKNGTIEK1cuVK5ubkaNGiQhg4dqv3791/S8svKyvTTn/5UnTp10pYtW5SVlaXMzEy/MVVVVWrTpo3eeecd7dy5U5MmTdITTzyht99+W5KUmZmp4cOHa9CgQSouLlZxcbH69u17wbr++c9/asiQIerdu7e2bt2q2bNna86cOXr22Wf9xr355ptq1KiRNmzYoOeff15PP/20srOzL7oN5eXlKi0t9XsAAABIl3F6MSoqSoMHD9a8efPUr18/SdK7776rZs2a6aabblJQUJC6devmG//MM89o0aJFWrJkiR544IHvXf68efNUVVWlOXPmKCwsTJ07d9bnn3+u+++/3zcmNDRUU6ZM8X2dkJCgdevW6e2339bw4cPldrsVHh6u8vLy7zydOGvWLMXGxuqVV16Rw+FQhw4ddPDgQT3++OOaNGmSgoLOtWhKSoomT54sSUpKStIrr7yilStXasCAAdUud+rUqX7zAwAAOO+yPr2Ynp6uhQsXqry8XJL01ltv6c4771RQUJDKysqUmZmpjh07yuv1yu12Kz8//5KPdOXn5yslJUVhYWG+59LS0i4Y9+qrr6pnz56Kjo6W2+3Wn//850texzfXlZaWJofjX3+a5/rrr1dZWZk+//xz33MpKSl+39eyZUsdOXLkosudOHGiSkpKfI8DBw5c1rwAAEDDdVkX0g8dOlTGGH3wwQfq3bu3PvnkE/3hD3+QdO7UXnZ2tqZPn67ExESFh4frF7/4hSoqKmpssgsWLFBmZqZefPFFpaWlqXHjxnrhhRe0YcOGGlvHN4WGhvp97XA4Lrim7ZtcLpdcLletzAUAANRvlxVdYWFhuv322/XWW29pz549Sk5OVo8ePSRJa9euVUZGhoYNGybp3DVaRUVFl7zsjh076r//+791+vRp39Gu9evX+41Zu3at+vbtq3Hjxvme27t3r98Yp9OpysrK713XwoULZYzxHe1au3atGjdurDZt2lzynAEAAC7VZd8cNT09XR988IFef/11paen+55PSkrSe++9p7y8PG3dulWjRo36zqNC3zZq1Cg5HA6NHTtWO3fu1NKlSzV9+nS/MUlJSdq8ebOWL1+uXbt26be//a02bdrkNyY+Pl7btm1TQUGBjh49qjNnzlywrnHjxunAgQN68MEH9dlnn+n999/X5MmT9cgjj/iu5wIAAKhJl10YN998s5o0aaKCggKNGjXK9/yMGTMUFRWlvn37aujQoRo4cKDvKNilcLvd+t///V9t375d3bt315NPPqnf//73fmPuvfde3X777RoxYoRSU1P15Zdf+h31kqSxY8cqOTlZvXr1UnR0tNauXXvBulq3bq2lS5dq48aN6tatm+677z6NGTNGTz311GW+GwAAAJfGYYwxgZ5EQ1VaWqrIyEiVTGgsj8vx/d8AwK6skkDPAEAd5Pv9XVIij8dTY8vlXBoAAIAFRBcAAIAFRBcAAIAFRBcAAIAFl3WfLvwwXU7PUZCJCPQ0AHxD0bRbAz0FAFcZjnQBAABYQHQBAABYQHQBAABYQHQBAABYQHQBAABYQHQBAABYQHQBAABYQHQBAABYQHQBAABYQHQBAABYQHQBAABYQHQBAABYQHQBAABYQHQBAABYQHQBAABYQHQBAABYQHQBAABYQHQBAABYQHQBAABYEBLoCVwNdkwZKI/HE+hpAACAAOJIFwAAgAVEFwAAgAVEFwAAgAVEFwAAgAVEFwAAgAVEFwAAgAVEFwAAgAVEFwAAgAVEFwAAgAVEFwAAgAVEFwAAgAVEFwAAgAVEFwAAgAVEFwAAgAVEFwAAgAVEFwAAgAVEFwAAgAVEFwAAgAVEFwAAgAVEFwAAgAVEFwAAgAVEFwAAgAVEFwAAgAVEFwAAgAVEFwAAgAVEFwAAgAVEFwAAgAVEFwAAgAVEFwAAgAVEFwAAgAVEFwAAgAVEFwAAgAVEFwAAgAVEFwAAgAVEFwAAgAVEFwAAgAVEFwAAgAVEFwAAgAVEFwAAgAVEFwAAgAVEFwAAgAVEFwAAgAVEFwAAgAVEFwAAgAVEFwAAgAVEFwAAgAVEFwAAgAVEFwAAgAVEFwAAgAVEFwAAgAVEFwAAgAVEFwAAgAVEFwAAgAVEFwAAgAVEFwAAgAVEFwAAgAVEFwAAgAVEFwAAgAVEFwAAgAVEFwAAgAVEFwAAgAUhgZ7A1aDPvD4KDg8O9DQAAKjztt+1PdBTqDUc6QIAALCA6AIAALCA6AIAALCA6AIAALCA6AIAALCA6AIAALCA6AIAALCA6AIAALCA6AIAALCA6AIAALCA6AIAALCA6AIAALCA6AIAALCA6AIAALCA6AIAALCA6AIAALCA6AIAALCA6AIAALCA6AIAALAgJNATuBqs3/e5PC5HoKcBAEDdlxV5GWNLam8etYAjXQAAABYQXQAAABYQXQAAABYQXQAAABYQXQAAABYQXd8QHx+vl156KdDTAAAADVC9v2XEjTfeqGuvvbZGYmnTpk1q1KjRlU8KAADgW+p9dH0fY4wqKysVEvL9mxodHW1hRgAA4GpUr08vZmRkKCcnRzNnzpTD4ZDD4dDcuXPlcDj04YcfqmfPnnK5XFqzZo327t2r2267TS1atJDb7Vbv3r21YsUKv+V9+/Siw+HQa6+9pmHDhikiIkJJSUlasmSJ5a0EAAANQb2OrpkzZyotLU1jx45VcXGxiouLFRsbK0maMGGCpk2bpvz8fKWkpKisrExDhgzRypUrlZubq0GDBmno0KHav3//d65jypQpGj58uLZt26YhQ4YoPT1dx44dq3ZseXm5SktL/R4AAABSPY+uyMhIOZ1ORUREKCYmRjExMQoODpYkPf300xowYICuueYaNWnSRN26ddO9996rLl26KCkpSc8884yuueaa7z1ylZGRoZEjRyoxMVHPPfecysrKtHHjxmrHTp06VZGRkb7H+QAEAACo19H1XXr16uX3dVlZmTIzM9WxY0d5vV653W7l5+d/75GulJQU378bNWokj8ejI0eOVDt24sSJKikp8T0OHDhw5RsCAAAahAZ7If23P4WYmZmp7OxsTZ8+XYmJiQoPD9cvfvELVVRUfOdyQkND/b52OByqqqqqdqzL5ZLL5bqyiQMAgAap3keX0+lUZWXl945bu3atMjIyNGzYMEnnjnwVFRXV8uwAAADOqfenF+Pj47VhwwYVFRXp6NGjFz0KlZSUpPfee095eXnaunWrRo0addGxAAAANa3eR1dmZqaCg4PVqVMnRUdHX/QarRkzZigqKkp9+/bV0KFDNXDgQPXo0cPybAEAwNXKYYwxgZ5EQ1VaWqrIyEiVTGgsj8sR6OkAANCwZJXUymJ9v79LSuTxeGpsufX+SBcAAEB9QHQBAABYQHQBAABYUO9vGVEfdDk9R0EmItDTAACgVhRNuzXQU6gXONIFAABgAdEFAABgAdEFAABgAdEFAABgAdEFAABgAdEFAABgAdEFAABgAdEFAABgAdEFAABgAdEFAABgAdEFAABgAdEFAABgAdEFAABgAdEFAABgAdEFAABgAdEFAABgAdEFAABgAdEFAABgAdEFAABgQUigJ3A12DFloDweT6CnAQAAAogjXQAAABYQXQAAABYQXQAAABYQXQAAABYQXQAAABYQXQAAABYQXQAAABYQXQAAABYQXQAAABYQXQAAABYQXQAAABYQXQAAABYQXQAAABYQXQAAABYQXQAAABYQXQAAABYQXQAAABYQXQAAABYQXQAAABYQXQAAABYQXQAAABYQXQAAABYQXQAAABYQXQAAABYQXQAAABYQXQAAABYQXQAAABYQXQAAABYQXQAAABaEBHoCDZkxRpJUWloa4JkAAIBLdf739vnf4zWF6KpFX375pSQpNjY2wDMBAACX68SJE4qMjKyx5RFdtahJkyaSpP3799foTsPlKS0tVWxsrA4cOCCPxxPo6Vy12A91A/uhbmA/1A0X2w/GGJ04cUKtWrWq0fURXbUoKOjcJXORkZH8UNUBHo+H/VAHsB/qBvZD3cB+qBuq2w+1cbCEC+kBAAAsILoAAAAsILpqkcvl0uTJk+VyuQI9lasa+6FuYD/UDeyHuoH9UDfY3g8OU9OfhwQAAMAFONIFAABgAdEFAABgAdEFAABgAdEFAABgAdFVi1599VXFx8crLCxMqamp2rhxY6CnVG99/PHHGjp0qFq1aiWHw6HFixf7vW6M0aRJk9SyZUuFh4erf//+2r17t9+YY8eOKT09XR6PR16vV2PGjFFZWZnfmG3btunHP/6xwsLCFBsbq+eff762N61emTp1qnr37q3GjRurefPm+tnPfqaCggK/MadPn9b48ePVtGlTud1u/fznP9fhw4f9xuzfv1+33nqrIiIi1Lx5c/3mN7/R2bNn/casXr1aPXr0kMvlUmJioubOnVvbm1dvzJ49WykpKb4bOqalpenDDz/0vc4+CIxp06bJ4XDo4Ycf9j3Hvqh9WVlZcjgcfo8OHTr4Xq9T+8CgVixYsMA4nU7z+uuvm08//dSMHTvWeL1ec/jw4UBPrV5aunSpefLJJ817771nJJlFixb5vT5t2jQTGRlpFi9ebLZu3Wr+7d/+zSQkJJhTp075xgwaNMh069bNrF+/3nzyyScmMTHRjBw50vd6SUmJadGihUlPTzc7duww8+fPN+Hh4eZPf/qTrc2s8wYOHGjeeOMNs2PHDpOXl2eGDBli4uLiTFlZmW/MfffdZ2JjY83KlSvN5s2bTZ8+fUzfvn19r589e9Z06dLF9O/f3+Tm5pqlS5eaZs2amYkTJ/rG/OMf/zARERHmkUceMTt37jQvv/yyCQ4ONsuWLbO6vXXVkiVLzAcffGB27dplCgoKzBNPPGFCQ0PNjh07jDHsg0DYuHGjiY+PNykpKeahhx7yPc++qH2TJ082nTt3NsXFxb7HF1984Xu9Lu0DoquWXHfddWb8+PG+rysrK02rVq3M1KlTAzirhuHb0VVVVWViYmLMCy+84Hvu+PHjxuVymfnz5xtjjNm5c6eRZDZt2uQb8+GHHxqHw2H++c9/GmOMmTVrlomKijLl5eW+MY8//rhJTk6u5S2qv44cOWIkmZycHGPMufc9NDTUvPPOO74x+fn5RpJZt26dMeZcQAcFBZlDhw75xsyePdt4PB7fe//YY4+Zzp07+61rxIgRZuDAgbW9SfVWVFSUee2119gHAXDixAmTlJRksrOzzU9+8hNfdLEv7Jg8ebLp1q1bta/VtX3A6cVaUFFRoS1btqh///6+54KCgtS/f3+tW7cugDNrmAoLC3Xo0CG/9zsyMlKpqam+93vdunXyer3q1auXb0z//v0VFBSkDRs2+MbccMMNcjqdvjEDBw5UQUGBvvrqK0tbU7+UlJRI+tcfd9+yZYvOnDnjty86dOiguLg4v33RtWtXtWjRwjdm4MCBKi0t1aeffuob881lnB/Dz8+FKisrtWDBAn399ddKS0tjHwTA+PHjdeutt17wfrEv7Nm9e7datWqldu3aKT09Xfv375dU9/YB0VULjh49qsrKSr8dKEktWrTQoUOHAjSrhuv8e/pd7/ehQ4fUvHlzv9dDQkLUpEkTvzHVLeOb68C/VFVV6eGHH9b111+vLl26SDr3PjmdTnm9Xr+x394X3/c+X2xMaWmpTp06VRubU+9s375dbrdbLpdL9913nxYtWqROnTqxDyxbsGCB/u///k9Tp0694DX2hR2pqamaO3euli1bptmzZ6uwsFA//vGPdeLEiTq3D0Iud+MAQDr3f/c7duzQmjVrAj2Vq1JycrLy8vJUUlKid999V3fddZdycnICPa2ryoEDB/TQQw8pOztbYWFhgZ7OVWvw4MG+f6ekpCg1NVVt27bV22+/rfDw8ADO7EIc6aoFzZo1U3Bw8AWfjjh8+LBiYmICNKuG6/x7+l3vd0xMjI4cOeL3+tmzZ3Xs2DG/MdUt45vrwDkPPPCA/va3v2nVqlVq06aN7/mYmBhVVFTo+PHjfuO/vS++732+2BiPx1Pn/iMaKE6nU4mJierZs6emTp2qbt26aebMmewDi7Zs2aIjR46oR48eCgkJUUhIiHJycvSf//mfCgkJUYsWLdgXAeD1etW+fXvt2bOnzv08EF21wOl0qmfPnlq5cqXvuaqqKq1cuVJpaWkBnFnDlJCQoJiYGL/3u7S0VBs2bPC932lpaTp+/Li2bNniG/PRRx+pqqpKqampvjEff/yxzpw54xuTnZ2t5ORkRUVFWdqaus0YowceeECLFi3SRx99pISEBL/Xe/bsqdDQUL99UVBQoP379/vti+3bt/tFcHZ2tjwejzp16uQb881lnB/Dz8/FVVVVqby8nH1gUb9+/bR9+3bl5eX5Hr169VJ6errv3+wL+8rKyrR37161bNmy7v08XNZl97hkCxYsMC6Xy8ydO9fs3LnT3HPPPcbr9fp9OgKX7sSJEyY3N9fk5uYaSWbGjBkmNzfX7Nu3zxhz7pYRXq/XvP/++2bbtm3mtttuq/aWEd27dzcbNmwwa9asMUlJSX63jDh+/Lhp0aKF+eUvf2l27NhhFixYYCIiIrhlxDfcf//9JjIy0qxevdrv49knT570jbnvvvtMXFyc+eijj8zmzZtNWlqaSUtL871+/uPZt9xyi8nLyzPLli0z0dHR1X48+ze/+Y3Jz883r776Kh+R/4YJEyaYnJwcU1hYaLZt22YmTJhgHA6H+fvf/26MYR8E0jc/vWgM+8KGRx991KxevdoUFhaatWvXmv79+5tmzZqZI0eOGGPq1j4gumrRyy+/bOLi4ozT6TTXXXedWb9+faCnVG+tWrXKSLrgcddddxljzt024re//a1p0aKFcblcpl+/fqagoMBvGV9++aUZOXKkcbvdxuPxmLvvvtucOHHCb8zWrVvNj370I+NyuUzr1q3NtGnTbG1ivVDdPpBk3njjDd+YU6dOmXHjxpmoqCgTERFhhg0bZoqLi/2WU1RUZAYPHmzCw8NNs2bNzKOPPmrOnDnjN2bVqlXm2muvNU6n07Rr185vHVe70aNHm7Zt2xqn02mio6NNv379fMFlDPsgkL4dXeyL2jdixAjTsmVL43Q6TevWrc2IESPMnj17fK/XpX3gMMaYyzs2BgAAgMvFNV0AAAAWEF0AAAAWEF0AAAAWEF0AAAAWEF0AAAAWEF0AAAAWEF0AAAAWEF0AAAAWEF0AAAAWEF0AAAAWEF0AAAAWEF0AAAAW/D+bnh9kbcA2QwAAAABJRU5ErkJggg=="},"metadata":{}}]},{"cell_type":"markdown","source":"# <b><div style='padding:15px;background-color:#850E35;color:white;border-radius:2px;font-size:110%;text-align: center'>Loading the XLM-Roberta Tokenizer & Model</div></b>","metadata":{}},{"cell_type":"code","source":"from transformers import AutoTokenizer\n\nxlmr_tokenizer = AutoTokenizer.from_pretrained(\"xlm-roberta-base\")","metadata":{"execution":{"iopub.status.busy":"2023-10-30T17:37:28.526109Z","iopub.execute_input":"2023-10-30T17:37:28.526381Z","iopub.status.idle":"2023-10-30T17:37:31.966773Z","shell.execute_reply.started":"2023-10-30T17:37:28.526357Z","shell.execute_reply":"2023-10-30T17:37:31.965821Z"},"trusted":true},"execution_count":10,"outputs":[{"output_type":"display_data","data":{"text/plain":"Downloading (…)lve/main/config.json:   0%|          | 0.00/615 [00:00<?, ?B/s]","application/vnd.jupyter.widget-view+json":{"version_major":2,"version_minor":0,"model_id":"83d76b55439146749389b1315e203cc4"}},"metadata":{}},{"output_type":"display_data","data":{"text/plain":"Downloading (…)tencepiece.bpe.model:   0%|          | 0.00/5.07M [00:00<?, ?B/s]","application/vnd.jupyter.widget-view+json":{"version_major":2,"version_minor":0,"model_id":"ec0f3e2d6a0343e3bf0184af262977b3"}},"metadata":{}},{"output_type":"display_data","data":{"text/plain":"Downloading (…)/main/tokenizer.json:   0%|          | 0.00/9.10M [00:00<?, ?B/s]","application/vnd.jupyter.widget-view+json":{"version_major":2,"version_minor":0,"model_id":"bdb7199902c24e38b6c5913a6567f5cc"}},"metadata":{}}]},{"cell_type":"code","source":"text = \"My name is Sarah and I live in London\"\n\nxlmr_tokens = xlmr_tokenizer(text).tokens()\n\npd.DataFrame([xlmr_tokens], index=[\"tokens\"])","metadata":{"execution":{"iopub.status.busy":"2023-10-30T17:37:31.968055Z","iopub.execute_input":"2023-10-30T17:37:31.968761Z","iopub.status.idle":"2023-10-30T17:37:31.989531Z","shell.execute_reply.started":"2023-10-30T17:37:31.968724Z","shell.execute_reply":"2023-10-30T17:37:31.988557Z"},"trusted":true},"execution_count":11,"outputs":[{"execution_count":11,"output_type":"execute_result","data":{"text/plain":"         0    1      2    3       4     5   6      7    8        9     10\ntokens  <s>  ▁My  ▁name  ▁is  ▁Sarah  ▁and  ▁I  ▁live  ▁in  ▁London  </s>","text/html":"<div>\n<style scoped>\n    .dataframe tbody tr th:only-of-type {\n        vertical-align: middle;\n    }\n\n    .dataframe tbody tr th {\n        vertical-align: top;\n    }\n\n    .dataframe thead th {\n        text-align: right;\n    }\n</style>\n<table border=\"1\" class=\"dataframe\">\n  <thead>\n    <tr style=\"text-align: right;\">\n      <th></th>\n      <th>0</th>\n      <th>1</th>\n      <th>2</th>\n      <th>3</th>\n      <th>4</th>\n      <th>5</th>\n      <th>6</th>\n      <th>7</th>\n      <th>8</th>\n      <th>9</th>\n      <th>10</th>\n    </tr>\n  </thead>\n  <tbody>\n    <tr>\n      <th>tokens</th>\n      <td>&lt;s&gt;</td>\n      <td>▁My</td>\n      <td>▁name</td>\n      <td>▁is</td>\n      <td>▁Sarah</td>\n      <td>▁and</td>\n      <td>▁I</td>\n      <td>▁live</td>\n      <td>▁in</td>\n      <td>▁London</td>\n      <td>&lt;/s&gt;</td>\n    </tr>\n  </tbody>\n</table>\n</div>"},"metadata":{}}]},{"cell_type":"code","source":"import torch\nfrom transformers import XLMRobertaForTokenClassification\n\nxlmr_model_name = \"xlm-roberta-base\"\nindex2label = {idx: label for idx, label in enumerate(labels.names)}\nlabel2index = {label: idx for idx, label in enumerate(labels.names)}\nnum_labels = labels.num_classes\n\ndevice = torch.device(\"cuda\" if torch.cuda.is_available() else \"cpu\")\n\nxlmr_model = XLMRobertaForTokenClassification.from_pretrained(\n    xlmr_model_name,\n    num_labels = num_labels,\n    id2label=index2label,\n    label2id=label2index\n).to(device)","metadata":{"execution":{"iopub.status.busy":"2023-10-30T17:37:31.990780Z","iopub.execute_input":"2023-10-30T17:37:31.991172Z","iopub.status.idle":"2023-10-30T17:37:58.142456Z","shell.execute_reply.started":"2023-10-30T17:37:31.991128Z","shell.execute_reply":"2023-10-30T17:37:58.141520Z"},"trusted":true},"execution_count":12,"outputs":[{"name":"stderr","text":"/opt/conda/lib/python3.10/site-packages/scipy/__init__.py:146: UserWarning: A NumPy version >=1.16.5 and <1.23.0 is required for this version of SciPy (detected version 1.23.5\n  warnings.warn(f\"A NumPy version >={np_minversion} and <{np_maxversion}\"\n","output_type":"stream"},{"output_type":"display_data","data":{"text/plain":"Downloading model.safetensors:   0%|          | 0.00/1.12G [00:00<?, ?B/s]","application/vnd.jupyter.widget-view+json":{"version_major":2,"version_minor":0,"model_id":"808dc773f22d4041b569c1fe8b7a0e57"}},"metadata":{}},{"name":"stderr","text":"Some weights of XLMRobertaForTokenClassification were not initialized from the model checkpoint at xlm-roberta-base and are newly initialized: ['classifier.bias', 'classifier.weight']\nYou should probably TRAIN this model on a down-stream task to be able to use it for predictions and inference.\n","output_type":"stream"}]},{"cell_type":"code","source":"def label_text(text, labels, model, tokenizer):\n    tokens = tokenizer(text).tokens()\n    input_ids = xlmr_tokenizer.encode(\n        text, return_tensors = \"pt\").to(device)\n    outputs = model(input_ids)[0]\n    predictions = torch.argmax(outputs, dim=2)\n    preds = [labels.names[p] for p in predictions[0].cpu().numpy()]\n    return pd.DataFrame([tokens, preds], index = [\"Tokens\", \"Tags\"])","metadata":{"execution":{"iopub.status.busy":"2023-10-30T17:37:58.145937Z","iopub.execute_input":"2023-10-30T17:37:58.146224Z","iopub.status.idle":"2023-10-30T17:37:58.152614Z","shell.execute_reply.started":"2023-10-30T17:37:58.146199Z","shell.execute_reply":"2023-10-30T17:37:58.151709Z"},"trusted":true},"execution_count":13,"outputs":[]},{"cell_type":"code","source":"label_text(text, labels, xlmr_model, xlmr_tokenizer)","metadata":{"execution":{"iopub.status.busy":"2023-10-30T17:37:58.153855Z","iopub.execute_input":"2023-10-30T17:37:58.154240Z","iopub.status.idle":"2023-10-30T17:38:04.875608Z","shell.execute_reply.started":"2023-10-30T17:37:58.154181Z","shell.execute_reply":"2023-10-30T17:38:04.874690Z"},"trusted":true},"execution_count":14,"outputs":[{"execution_count":14,"output_type":"execute_result","data":{"text/plain":"           0      1      2      3       4      5      6      7      8   \\\nTokens    <s>    ▁My  ▁name    ▁is  ▁Sarah   ▁and     ▁I  ▁live    ▁in   \nTags    B-ORG  B-ORG  B-ORG  B-ORG   B-ORG  B-ORG  B-ORG  B-ORG  B-ORG   \n\n             9      10  \nTokens  ▁London   </s>  \nTags      B-ORG  B-ORG  ","text/html":"<div>\n<style scoped>\n    .dataframe tbody tr th:only-of-type {\n        vertical-align: middle;\n    }\n\n    .dataframe tbody tr th {\n        vertical-align: top;\n    }\n\n    .dataframe thead th {\n        text-align: right;\n    }\n</style>\n<table border=\"1\" class=\"dataframe\">\n  <thead>\n    <tr style=\"text-align: right;\">\n      <th></th>\n      <th>0</th>\n      <th>1</th>\n      <th>2</th>\n      <th>3</th>\n      <th>4</th>\n      <th>5</th>\n      <th>6</th>\n      <th>7</th>\n      <th>8</th>\n      <th>9</th>\n      <th>10</th>\n    </tr>\n  </thead>\n  <tbody>\n    <tr>\n      <th>Tokens</th>\n      <td>&lt;s&gt;</td>\n      <td>▁My</td>\n      <td>▁name</td>\n      <td>▁is</td>\n      <td>▁Sarah</td>\n      <td>▁and</td>\n      <td>▁I</td>\n      <td>▁live</td>\n      <td>▁in</td>\n      <td>▁London</td>\n      <td>&lt;/s&gt;</td>\n    </tr>\n    <tr>\n      <th>Tags</th>\n      <td>B-ORG</td>\n      <td>B-ORG</td>\n      <td>B-ORG</td>\n      <td>B-ORG</td>\n      <td>B-ORG</td>\n      <td>B-ORG</td>\n      <td>B-ORG</td>\n      <td>B-ORG</td>\n      <td>B-ORG</td>\n      <td>B-ORG</td>\n      <td>B-ORG</td>\n    </tr>\n  </tbody>\n</table>\n</div>"},"metadata":{}}]},{"cell_type":"markdown","source":"# <b><div style='padding:15px;background-color:#850E35;color:white;border-radius:2px;font-size:110%;text-align: center'>Tokenizer for NER Analysis </div></b>","metadata":{}},{"cell_type":"code","source":"def tokenize_and_align_labels(examples):\n    tokenized_inputs = xlmr_tokenizer(examples[\"tokens\"], truncation = True, is_split_into_words = True)\n    labels = []\n    for idx, label in enumerate(examples[\"ner_tags\"]):\n        word_ids = tokenized_inputs.word_ids(batch_index=idx)\n        previous_word_idx = None\n        label_ids = []\n        for word_idx in word_ids:\n            if word_idx is None or word_idx == previous_word_idx:\n                label_ids.append(-100)\n            else:\n                label_ids.append(label[word_idx])\n            previous_word_idx = word_idx\n        labels.append(label_ids)\n    tokenized_inputs[\"labels\"] = labels\n    return tokenized_inputs        ","metadata":{"execution":{"iopub.status.busy":"2023-10-30T17:38:04.877179Z","iopub.execute_input":"2023-10-30T17:38:04.877537Z","iopub.status.idle":"2023-10-30T17:38:04.884082Z","shell.execute_reply.started":"2023-10-30T17:38:04.877503Z","shell.execute_reply":"2023-10-30T17:38:04.883174Z"},"trusted":true},"execution_count":15,"outputs":[]},{"cell_type":"code","source":"def encode_panx_ds(corpus):\n    return corpus.map(tokenize_and_align_labels,\n                      batched=True,\n                      remove_columns=[\"langs\", \"ner_tags\", \"tokens\"])\n\npanx_en_encoded = encode_panx_ds(panx_ds[\"en\"])","metadata":{"execution":{"iopub.status.busy":"2023-10-30T17:38:04.885275Z","iopub.execute_input":"2023-10-30T17:38:04.885564Z","iopub.status.idle":"2023-10-30T17:38:07.190495Z","shell.execute_reply.started":"2023-10-30T17:38:04.885538Z","shell.execute_reply":"2023-10-30T17:38:07.189434Z"},"trusted":true},"execution_count":16,"outputs":[{"output_type":"display_data","data":{"text/plain":"  0%|          | 0/10 [00:00<?, ?ba/s]","application/vnd.jupyter.widget-view+json":{"version_major":2,"version_minor":0,"model_id":"3d6b5c6076d4465e81920efe1f11328c"}},"metadata":{}},{"output_type":"display_data","data":{"text/plain":"  0%|          | 0/5 [00:00<?, ?ba/s]","application/vnd.jupyter.widget-view+json":{"version_major":2,"version_minor":0,"model_id":"9afa29aedd024533987cac5feac5e85e"}},"metadata":{}},{"output_type":"display_data","data":{"text/plain":"  0%|          | 0/5 [00:00<?, ?ba/s]","application/vnd.jupyter.widget-view+json":{"version_major":2,"version_minor":0,"model_id":"8bf07626fb8748af87585c65c0216261"}},"metadata":{}}]},{"cell_type":"code","source":"tokenized_input = xlmr_tokenizer(panx_ds[\"en\"][\"train\"][0][\"tokens\"], is_split_into_words=True)\ntokens = xlmr_tokenizer.convert_ids_to_tokens(tokenized_input[\"input_ids\"])\nword_ids = tokenized_input.word_ids()\ntags = panx_en_encoded[\"train\"][0][\"labels\"]\n\npd.DataFrame([tokens, word_ids, tags], index=[\"Tokens\", \"Word IDs\", \"Tags\"])","metadata":{"execution":{"iopub.status.busy":"2023-10-30T17:38:07.191974Z","iopub.execute_input":"2023-10-30T17:38:07.192282Z","iopub.status.idle":"2023-10-30T17:38:07.211930Z","shell.execute_reply.started":"2023-10-30T17:38:07.192256Z","shell.execute_reply":"2023-10-30T17:38:07.210625Z"},"trusted":true},"execution_count":17,"outputs":[{"execution_count":17,"output_type":"execute_result","data":{"text/plain":"            0    1         2    3    4   5     6     7     8     9   \\\nTokens     <s>  ▁''  ▁January  ▁21  ▁''  ▁–  ▁Nan    ny  ▁and  ▁the   \nWord IDs  None    0         1    2    3   4     5     5     6     7   \nTags      -100    0         0    0    0   0     1  -100     2     2   \n\n                  10    11  \nTokens    ▁Professor  </s>  \nWord IDs           8  None  \nTags               2  -100  ","text/html":"<div>\n<style scoped>\n    .dataframe tbody tr th:only-of-type {\n        vertical-align: middle;\n    }\n\n    .dataframe tbody tr th {\n        vertical-align: top;\n    }\n\n    .dataframe thead th {\n        text-align: right;\n    }\n</style>\n<table border=\"1\" class=\"dataframe\">\n  <thead>\n    <tr style=\"text-align: right;\">\n      <th></th>\n      <th>0</th>\n      <th>1</th>\n      <th>2</th>\n      <th>3</th>\n      <th>4</th>\n      <th>5</th>\n      <th>6</th>\n      <th>7</th>\n      <th>8</th>\n      <th>9</th>\n      <th>10</th>\n      <th>11</th>\n    </tr>\n  </thead>\n  <tbody>\n    <tr>\n      <th>Tokens</th>\n      <td>&lt;s&gt;</td>\n      <td>▁''</td>\n      <td>▁January</td>\n      <td>▁21</td>\n      <td>▁''</td>\n      <td>▁–</td>\n      <td>▁Nan</td>\n      <td>ny</td>\n      <td>▁and</td>\n      <td>▁the</td>\n      <td>▁Professor</td>\n      <td>&lt;/s&gt;</td>\n    </tr>\n    <tr>\n      <th>Word IDs</th>\n      <td>None</td>\n      <td>0</td>\n      <td>1</td>\n      <td>2</td>\n      <td>3</td>\n      <td>4</td>\n      <td>5</td>\n      <td>5</td>\n      <td>6</td>\n      <td>7</td>\n      <td>8</td>\n      <td>None</td>\n    </tr>\n    <tr>\n      <th>Tags</th>\n      <td>-100</td>\n      <td>0</td>\n      <td>0</td>\n      <td>0</td>\n      <td>0</td>\n      <td>0</td>\n      <td>1</td>\n      <td>-100</td>\n      <td>2</td>\n      <td>2</td>\n      <td>2</td>\n      <td>-100</td>\n    </tr>\n  </tbody>\n</table>\n</div>"},"metadata":{}}]},{"cell_type":"markdown","source":"# <b><div style='padding:15px;background-color:#850E35;color:white;border-radius:2px;font-size:110%;text-align: center'>Model Metrics </div></b>","metadata":{}},{"cell_type":"code","source":"import evaluate\nimport numpy as np\n\nseqeval = evaluate.load(\"seqeval\")\n\nlabel_list = label_list = panx_ds[\"en\"][\"train\"].features[\"ner_tags\"].feature.names\n\ndef compute_metrics(p):\n    predictions, labels = p\n    predictions = np.argmax(predictions, axis=2)\n    true_predictions = [\n        [label_list[p] for (p, l) in zip(prediction, label) if l != -100]\n        for prediction, label in zip(predictions, labels)\n    ]\n    true_labels = [\n        [label_list[l] for (p, l) in zip(prediction, label) if l != -100]\n        for prediction, label in zip(predictions, labels)\n    ]\n    results = seqeval.compute(predictions=true_predictions, references=true_labels)\n    return {\n        \"precision\": results[\"overall_precision\"],\n        \"recall\": results[\"overall_recall\"],\n        \"f1\": results[\"overall_f1\"],\n        \"accuracy\": results[\"overall_accuracy\"],\n    }","metadata":{"execution":{"iopub.status.busy":"2023-10-30T17:38:07.213628Z","iopub.execute_input":"2023-10-30T17:38:07.214277Z","iopub.status.idle":"2023-10-30T17:38:10.054983Z","shell.execute_reply.started":"2023-10-30T17:38:07.214239Z","shell.execute_reply":"2023-10-30T17:38:10.054165Z"},"trusted":true},"execution_count":18,"outputs":[{"output_type":"display_data","data":{"text/plain":"Downloading builder script:   0%|          | 0.00/6.34k [00:00<?, ?B/s]","application/vnd.jupyter.widget-view+json":{"version_major":2,"version_minor":0,"model_id":"cb8f5d98c7654b36b24b623a4f0f65a4"}},"metadata":{}}]},{"cell_type":"markdown","source":"# <b><div style='padding:15px;background-color:#850E35;color:white;border-radius:2px;font-size:110%;text-align: center'>Model Training </div></b>","metadata":{}},{"cell_type":"code","source":"from transformers import TrainingArguments\n\ntraining_args = TrainingArguments(\n    output_dir = \"roberta-base-NER\",\n    learning_rate=2e-5,\n    per_device_train_batch_size = 12,\n    per_device_eval_batch_size = 12,\n    num_train_epochs = 4,\n    weight_decay = 0.01,\n    evaluation_strategy = \"epoch\", \n    save_strategy=\"epoch\",\n    load_best_model_at_end=True,\n    log_level=\"error\",\n    report_to= [\"comet_ml\"],\n    push_to_hub = True,\n)","metadata":{"execution":{"iopub.status.busy":"2023-10-30T17:38:10.056179Z","iopub.execute_input":"2023-10-30T17:38:10.056474Z","iopub.status.idle":"2023-10-30T17:38:10.065759Z","shell.execute_reply.started":"2023-10-30T17:38:10.056447Z","shell.execute_reply":"2023-10-30T17:38:10.064220Z"},"trusted":true},"execution_count":19,"outputs":[]},{"cell_type":"code","source":"from transformers import DataCollatorForTokenClassification\n\ndata_collator = DataCollatorForTokenClassification(xlmr_tokenizer)","metadata":{"execution":{"iopub.status.busy":"2023-10-30T17:38:10.067547Z","iopub.execute_input":"2023-10-30T17:38:10.067914Z","iopub.status.idle":"2023-10-30T17:38:10.081557Z","shell.execute_reply.started":"2023-10-30T17:38:10.067881Z","shell.execute_reply":"2023-10-30T17:38:10.080370Z"},"trusted":true},"execution_count":20,"outputs":[]},{"cell_type":"code","source":"from transformers import Trainer\n\ntrainer = Trainer(\n    model = xlmr_model,\n    args = training_args,\n    data_collator = data_collator,\n    compute_metrics = compute_metrics,\n    train_dataset = panx_en_encoded[\"train\"],\n    eval_dataset = panx_en_encoded[\"validation\"],\n    tokenizer = xlmr_tokenizer,\n)\n\ntrainer.train()","metadata":{"execution":{"iopub.status.busy":"2023-10-30T17:38:10.083045Z","iopub.execute_input":"2023-10-30T17:38:10.083469Z","iopub.status.idle":"2023-10-30T17:53:19.468720Z","shell.execute_reply.started":"2023-10-30T17:38:10.083430Z","shell.execute_reply":"2023-10-30T17:53:19.467808Z"},"trusted":true},"execution_count":21,"outputs":[{"name":"stderr","text":"\u001b[1;38;5;214mCOMET WARNING:\u001b[0m As you are running in a Jupyter environment, you will need to call `experiment.end()` when finished to ensure all metrics and code are logged before exiting.\n\u001b[1;38;5;39mCOMET INFO:\u001b[0m Couldn't find a Git repository in '/kaggle/working' nor in any parent directory. Set `COMET_GIT_DIRECTORY` if your Git Repository is elsewhere.\n\u001b[1;38;5;39mCOMET INFO:\u001b[0m Experiment is live on comet.com https://www.comet.com/tirendaz-academy/multilingual-ner-app/8bd60f9e6a00421c9c337badf58c1f35\n\n/opt/conda/lib/python3.10/site-packages/torch/nn/parallel/_functions.py:68: UserWarning: Was asked to gather along dimension 0, but all input tensors were scalars; will instead unsqueeze and return a vector.\n  warnings.warn('Was asked to gather along dimension 0, but all '\n","output_type":"stream"},{"output_type":"display_data","data":{"text/plain":"<IPython.core.display.HTML object>","text/html":"\n    <div>\n      \n      <progress value='1668' max='1668' style='width:300px; height:20px; vertical-align: middle;'></progress>\n      [1668/1668 14:58, Epoch 4/4]\n    </div>\n    <table border=\"1\" class=\"dataframe\">\n  <thead>\n <tr style=\"text-align: left;\">\n      <th>Epoch</th>\n      <th>Training Loss</th>\n      <th>Validation Loss</th>\n      <th>Precision</th>\n      <th>Recall</th>\n      <th>F1</th>\n      <th>Accuracy</th>\n    </tr>\n  </thead>\n  <tbody>\n    <tr>\n      <td>1</td>\n      <td>No log</td>\n      <td>0.339822</td>\n      <td>0.730150</td>\n      <td>0.773457</td>\n      <td>0.751180</td>\n      <td>0.900526</td>\n    </tr>\n    <tr>\n      <td>2</td>\n      <td>0.524000</td>\n      <td>0.298459</td>\n      <td>0.767286</td>\n      <td>0.791068</td>\n      <td>0.778996</td>\n      <td>0.911189</td>\n    </tr>\n    <tr>\n      <td>3</td>\n      <td>0.285800</td>\n      <td>0.286975</td>\n      <td>0.774360</td>\n      <td>0.805015</td>\n      <td>0.789390</td>\n      <td>0.916447</td>\n    </tr>\n    <tr>\n      <td>4</td>\n      <td>0.222500</td>\n      <td>0.288900</td>\n      <td>0.797644</td>\n      <td>0.810792</td>\n      <td>0.804164</td>\n      <td>0.918495</td>\n    </tr>\n  </tbody>\n</table><p>"},"metadata":{}},{"name":"stderr","text":"/opt/conda/lib/python3.10/site-packages/torch/nn/parallel/_functions.py:68: UserWarning: Was asked to gather along dimension 0, but all input tensors were scalars; will instead unsqueeze and return a vector.\n  warnings.warn('Was asked to gather along dimension 0, but all '\n/opt/conda/lib/python3.10/site-packages/torch/nn/parallel/_functions.py:68: UserWarning: Was asked to gather along dimension 0, but all input tensors were scalars; will instead unsqueeze and return a vector.\n  warnings.warn('Was asked to gather along dimension 0, but all '\n/opt/conda/lib/python3.10/site-packages/torch/nn/parallel/_functions.py:68: UserWarning: Was asked to gather along dimension 0, but all input tensors were scalars; will instead unsqueeze and return a vector.\n  warnings.warn('Was asked to gather along dimension 0, but all '\n/opt/conda/lib/python3.10/site-packages/torch/nn/parallel/_functions.py:68: UserWarning: Was asked to gather along dimension 0, but all input tensors were scalars; will instead unsqueeze and return a vector.\n  warnings.warn('Was asked to gather along dimension 0, but all '\n/opt/conda/lib/python3.10/site-packages/torch/nn/parallel/_functions.py:68: UserWarning: Was asked to gather along dimension 0, but all input tensors were scalars; will instead unsqueeze and return a vector.\n  warnings.warn('Was asked to gather along dimension 0, but all '\n\u001b[1;38;5;39mCOMET INFO:\u001b[0m ---------------------------------------------------------------------------------------\n\u001b[1;38;5;39mCOMET INFO:\u001b[0m Comet.ml Experiment Summary\n\u001b[1;38;5;39mCOMET INFO:\u001b[0m ---------------------------------------------------------------------------------------\n\u001b[1;38;5;39mCOMET INFO:\u001b[0m   Data:\n\u001b[1;38;5;39mCOMET INFO:\u001b[0m     display_summary_level : 1\n\u001b[1;38;5;39mCOMET INFO:\u001b[0m     url                   : https://www.comet.com/tirendaz-academy/multilingual-ner-app/8bd60f9e6a00421c9c337badf58c1f35\n\u001b[1;38;5;39mCOMET INFO:\u001b[0m   Metrics [count] (min, max):\n\u001b[1;38;5;39mCOMET INFO:\u001b[0m     epoch [8]                   : (1.0, 4.0)\n\u001b[1;38;5;39mCOMET INFO:\u001b[0m     eval_accuracy [4]           : (0.900525757164367, 0.9184952978056427)\n\u001b[1;38;5;39mCOMET INFO:\u001b[0m     eval_f1 [4]                 : (0.7511801327221728, 0.8041640466708587)\n\u001b[1;38;5;39mCOMET INFO:\u001b[0m     eval_loss [4]               : (0.2869746685028076, 0.3398217558860779)\n\u001b[1;38;5;39mCOMET INFO:\u001b[0m     eval_precision [4]          : (0.7301502859422796, 0.7976437976437977)\n\u001b[1;38;5;39mCOMET INFO:\u001b[0m     eval_recall [4]             : (0.7734573119188504, 0.8107917723302339)\n\u001b[1;38;5;39mCOMET INFO:\u001b[0m     eval_runtime [4]            : (28.8952, 29.0612)\n\u001b[1;38;5;39mCOMET INFO:\u001b[0m     eval_samples_per_second [4] : (172.051, 173.039)\n\u001b[1;38;5;39mCOMET INFO:\u001b[0m     eval_steps_per_second [4]   : (7.192, 7.233)\n\u001b[1;38;5;39mCOMET INFO:\u001b[0m     learning_rate [3]           : (2.0143884892086333e-06, 1.4004796163069546e-05)\n\u001b[1;38;5;39mCOMET INFO:\u001b[0m     loss [170]                  : (0.03431648388504982, 2.329225778579712)\n\u001b[1;38;5;39mCOMET INFO:\u001b[0m     total_flos                  : 795839743276512.0\n\u001b[1;38;5;39mCOMET INFO:\u001b[0m     train_loss                  : 0.33039388725225877\n\u001b[1;38;5;39mCOMET INFO:\u001b[0m     train_runtime               : 887.0845\n\u001b[1;38;5;39mCOMET INFO:\u001b[0m     train_samples_per_second    : 45.092\n\u001b[1;38;5;39mCOMET INFO:\u001b[0m     train_steps_per_second      : 1.88\n\u001b[1;38;5;39mCOMET INFO:\u001b[0m   Parameters:\n\u001b[1;38;5;39mCOMET INFO:\u001b[0m     args/_n_gpu                             : 2\n\u001b[1;38;5;39mCOMET INFO:\u001b[0m     args/_no_sync_in_gradient_accumulation  : True\n\u001b[1;38;5;39mCOMET INFO:\u001b[0m     args/_setup_devices                     : cuda:0\n\u001b[1;38;5;39mCOMET INFO:\u001b[0m     args/adafactor                          : False\n\u001b[1;38;5;39mCOMET INFO:\u001b[0m     args/adam_beta1                         : 0.9\n\u001b[1;38;5;39mCOMET INFO:\u001b[0m     args/adam_beta2                         : 0.999\n\u001b[1;38;5;39mCOMET INFO:\u001b[0m     args/adam_epsilon                       : 1e-08\n\u001b[1;38;5;39mCOMET INFO:\u001b[0m     args/auto_find_batch_size               : False\n\u001b[1;38;5;39mCOMET INFO:\u001b[0m     args/bf16                               : False\n\u001b[1;38;5;39mCOMET INFO:\u001b[0m     args/bf16_full_eval                     : False\n\u001b[1;38;5;39mCOMET INFO:\u001b[0m     args/data_seed                          : None\n\u001b[1;38;5;39mCOMET INFO:\u001b[0m     args/dataloader_drop_last               : False\n\u001b[1;38;5;39mCOMET INFO:\u001b[0m     args/dataloader_num_workers             : 0\n\u001b[1;38;5;39mCOMET INFO:\u001b[0m     args/dataloader_pin_memory              : True\n\u001b[1;38;5;39mCOMET INFO:\u001b[0m     args/ddp_backend                        : None\n\u001b[1;38;5;39mCOMET INFO:\u001b[0m     args/ddp_broadcast_buffers              : None\n\u001b[1;38;5;39mCOMET INFO:\u001b[0m     args/ddp_bucket_cap_mb                  : None\n\u001b[1;38;5;39mCOMET INFO:\u001b[0m     args/ddp_find_unused_parameters         : None\n\u001b[1;38;5;39mCOMET INFO:\u001b[0m     args/ddp_timeout                        : 1800\n\u001b[1;38;5;39mCOMET INFO:\u001b[0m     args/ddp_timeout_delta                  : 0:30:00\n\u001b[1;38;5;39mCOMET INFO:\u001b[0m     args/debug                              : []\n\u001b[1;38;5;39mCOMET INFO:\u001b[0m     args/deepspeed                          : None\n\u001b[1;38;5;39mCOMET INFO:\u001b[0m     args/deepspeed_plugin                   : None\n\u001b[1;38;5;39mCOMET INFO:\u001b[0m     args/default_optim                      : adamw_torch\n\u001b[1;38;5;39mCOMET INFO:\u001b[0m     args/device                             : cuda:0\n\u001b[1;38;5;39mCOMET INFO:\u001b[0m     args/disable_tqdm                       : False\n\u001b[1;38;5;39mCOMET INFO:\u001b[0m     args/dispatch_batches                   : None\n\u001b[1;38;5;39mCOMET INFO:\u001b[0m     args/distributed_state                  : Distributed environment: NO\nNum processes: 1\nProcess index: 0\nLocal process index: 0\nDevice: cuda\n\n\u001b[1;38;5;39mCOMET INFO:\u001b[0m     args/do_eval                            : True\n\u001b[1;38;5;39mCOMET INFO:\u001b[0m     args/do_predict                         : False\n\u001b[1;38;5;39mCOMET INFO:\u001b[0m     args/do_train                           : False\n\u001b[1;38;5;39mCOMET INFO:\u001b[0m     args/eval_accumulation_steps            : None\n\u001b[1;38;5;39mCOMET INFO:\u001b[0m     args/eval_batch_size                    : 24\n\u001b[1;38;5;39mCOMET INFO:\u001b[0m     args/eval_delay                         : 0\n\u001b[1;38;5;39mCOMET INFO:\u001b[0m     args/eval_steps                         : None\n\u001b[1;38;5;39mCOMET INFO:\u001b[0m     args/evaluation_strategy                : IntervalStrategy.EPOCH\n\u001b[1;38;5;39mCOMET INFO:\u001b[0m     args/fp16                               : False\n\u001b[1;38;5;39mCOMET INFO:\u001b[0m     args/fp16_backend                       : auto\n\u001b[1;38;5;39mCOMET INFO:\u001b[0m     args/fp16_full_eval                     : False\n\u001b[1;38;5;39mCOMET INFO:\u001b[0m     args/fp16_opt_level                     : O1\n\u001b[1;38;5;39mCOMET INFO:\u001b[0m     args/framework                          : pt\n\u001b[1;38;5;39mCOMET INFO:\u001b[0m     args/fsdp                               : []\n\u001b[1;38;5;39mCOMET INFO:\u001b[0m     args/fsdp_config                        : {'min_num_params': 0, 'xla': False, 'xla_fsdp_grad_ckpt': False}\n\u001b[1;38;5;39mCOMET INFO:\u001b[0m     args/fsdp_min_num_params                : 0\n\u001b[1;38;5;39mCOMET INFO:\u001b[0m     args/fsdp_transformer_layer_cls_to_wrap : None\n\u001b[1;38;5;39mCOMET INFO:\u001b[0m     args/full_determinism                   : False\n\u001b[1;38;5;39mCOMET INFO:\u001b[0m     args/gradient_accumulation_steps        : 1\n\u001b[1;38;5;39mCOMET INFO:\u001b[0m     args/gradient_checkpointing             : False\n\u001b[1;38;5;39mCOMET INFO:\u001b[0m     args/greater_is_better                  : False\n\u001b[1;38;5;39mCOMET INFO:\u001b[0m     args/group_by_length                    : False\n\u001b[1;38;5;39mCOMET INFO:\u001b[0m     args/half_precision_backend             : auto\n\u001b[1;38;5;39mCOMET INFO:\u001b[0m     args/hub_always_push                    : False\n\u001b[1;38;5;39mCOMET INFO:\u001b[0m     args/hub_model_id                       : None\n\u001b[1;38;5;39mCOMET INFO:\u001b[0m     args/hub_private_repo                   : False\n\u001b[1;38;5;39mCOMET INFO:\u001b[0m     args/hub_strategy                       : HubStrategy.EVERY_SAVE\n\u001b[1;38;5;39mCOMET INFO:\u001b[0m     args/hub_token                          : None\n\u001b[1;38;5;39mCOMET INFO:\u001b[0m     args/ignore_data_skip                   : False\n\u001b[1;38;5;39mCOMET INFO:\u001b[0m     args/include_inputs_for_metrics         : False\n\u001b[1;38;5;39mCOMET INFO:\u001b[0m     args/jit_mode_eval                      : False\n\u001b[1;38;5;39mCOMET INFO:\u001b[0m     args/label_names                        : None\n\u001b[1;38;5;39mCOMET INFO:\u001b[0m     args/label_smoothing_factor             : 0.0\n\u001b[1;38;5;39mCOMET INFO:\u001b[0m     args/learning_rate                      : 2e-05\n\u001b[1;38;5;39mCOMET INFO:\u001b[0m     args/length_column_name                 : length\n\u001b[1;38;5;39mCOMET INFO:\u001b[0m     args/load_best_model_at_end             : True\n\u001b[1;38;5;39mCOMET INFO:\u001b[0m     args/local_process_index                : 0\n\u001b[1;38;5;39mCOMET INFO:\u001b[0m     args/local_rank                         : 0\n\u001b[1;38;5;39mCOMET INFO:\u001b[0m     args/log_level                          : error\n\u001b[1;38;5;39mCOMET INFO:\u001b[0m     args/log_level_replica                  : warning\n\u001b[1;38;5;39mCOMET INFO:\u001b[0m     args/log_on_each_node                   : True\n\u001b[1;38;5;39mCOMET INFO:\u001b[0m     args/logging_dir                        : roberta-base-NER/runs/Oct30_17-38-10_03308ab2df33\n\u001b[1;38;5;39mCOMET INFO:\u001b[0m     args/logging_first_step                 : False\n\u001b[1;38;5;39mCOMET INFO:\u001b[0m     args/logging_nan_inf_filter             : True\n\u001b[1;38;5;39mCOMET INFO:\u001b[0m     args/logging_steps                      : 500\n\u001b[1;38;5;39mCOMET INFO:\u001b[0m     args/logging_strategy                   : IntervalStrategy.STEPS\n\u001b[1;38;5;39mCOMET INFO:\u001b[0m     args/lr_scheduler_type                  : SchedulerType.LINEAR\n\u001b[1;38;5;39mCOMET INFO:\u001b[0m     args/max_grad_norm                      : 1.0\n\u001b[1;38;5;39mCOMET INFO:\u001b[0m     args/max_steps                          : -1\n\u001b[1;38;5;39mCOMET INFO:\u001b[0m     args/metric_for_best_model              : loss\n\u001b[1;38;5;39mCOMET INFO:\u001b[0m     args/mp_parameters                      : \n\u001b[1;38;5;39mCOMET INFO:\u001b[0m     args/n_gpu                              : 2\n\u001b[1;38;5;39mCOMET INFO:\u001b[0m     args/no_cuda                            : False\n\u001b[1;38;5;39mCOMET INFO:\u001b[0m     args/num_train_epochs                   : 4\n\u001b[1;38;5;39mCOMET INFO:\u001b[0m     args/optim                              : OptimizerNames.ADAMW_TORCH\n\u001b[1;38;5;39mCOMET INFO:\u001b[0m     args/optim_args                         : None\n\u001b[1;38;5;39mCOMET INFO:\u001b[0m     args/output_dir                         : roberta-base-NER\n\u001b[1;38;5;39mCOMET INFO:\u001b[0m     args/overwrite_output_dir               : False\n\u001b[1;38;5;39mCOMET INFO:\u001b[0m     args/parallel_mode                      : ParallelMode.NOT_DISTRIBUTED\n\u001b[1;38;5;39mCOMET INFO:\u001b[0m     args/past_index                         : -1\n\u001b[1;38;5;39mCOMET INFO:\u001b[0m     args/per_device_eval_batch_size         : 12\n\u001b[1;38;5;39mCOMET INFO:\u001b[0m     args/per_device_train_batch_size        : 12\n\u001b[1;38;5;39mCOMET INFO:\u001b[0m     args/per_gpu_eval_batch_size            : None\n\u001b[1;38;5;39mCOMET INFO:\u001b[0m     args/per_gpu_train_batch_size           : None\n\u001b[1;38;5;39mCOMET INFO:\u001b[0m     args/place_model_on_device              : True\n\u001b[1;38;5;39mCOMET INFO:\u001b[0m     args/prediction_loss_only               : False\n\u001b[1;38;5;39mCOMET INFO:\u001b[0m     args/process_index                      : 0\n\u001b[1;38;5;39mCOMET INFO:\u001b[0m     args/push_to_hub                        : False\n\u001b[1;38;5;39mCOMET INFO:\u001b[0m     args/push_to_hub_model_id               : None\n\u001b[1;38;5;39mCOMET INFO:\u001b[0m     args/push_to_hub_organization           : None\n\u001b[1;38;5;39mCOMET INFO:\u001b[0m     args/push_to_hub_token                  : None\n\u001b[1;38;5;39mCOMET INFO:\u001b[0m     args/ray_scope                          : last\n\u001b[1;38;5;39mCOMET INFO:\u001b[0m     args/remove_unused_columns              : True\n\u001b[1;38;5;39mCOMET INFO:\u001b[0m     args/report_to                          : ['comet_ml']\n\u001b[1;38;5;39mCOMET INFO:\u001b[0m     args/resume_from_checkpoint             : None\n\u001b[1;38;5;39mCOMET INFO:\u001b[0m     args/run_name                           : roberta-base-NER\n\u001b[1;38;5;39mCOMET INFO:\u001b[0m     args/save_on_each_node                  : False\n\u001b[1;38;5;39mCOMET INFO:\u001b[0m     args/save_safetensors                   : False\n\u001b[1;38;5;39mCOMET INFO:\u001b[0m     args/save_steps                         : 500\n\u001b[1;38;5;39mCOMET INFO:\u001b[0m     args/save_strategy                      : IntervalStrategy.EPOCH\n\u001b[1;38;5;39mCOMET INFO:\u001b[0m     args/save_total_limit                   : None\n\u001b[1;38;5;39mCOMET INFO:\u001b[0m     args/seed                               : 42\n\u001b[1;38;5;39mCOMET INFO:\u001b[0m     args/sharded_ddp                        : []\n\u001b[1;38;5;39mCOMET INFO:\u001b[0m     args/should_log                         : True\n\u001b[1;38;5;39mCOMET INFO:\u001b[0m     args/should_save                        : True\n\u001b[1;38;5;39mCOMET INFO:\u001b[0m     args/skip_memory_metrics                : True\n\u001b[1;38;5;39mCOMET INFO:\u001b[0m     args/tf32                               : None\n\u001b[1;38;5;39mCOMET INFO:\u001b[0m     args/torch_compile                      : False\n\u001b[1;38;5;39mCOMET INFO:\u001b[0m     args/torch_compile_backend              : None\n\u001b[1;38;5;39mCOMET INFO:\u001b[0m     args/torch_compile_mode                 : None\n\u001b[1;38;5;39mCOMET INFO:\u001b[0m     args/torchdynamo                        : None\n\u001b[1;38;5;39mCOMET INFO:\u001b[0m     args/tpu_metrics_debug                  : False\n\u001b[1;38;5;39mCOMET INFO:\u001b[0m     args/tpu_num_cores                      : None\n\u001b[1;38;5;39mCOMET INFO:\u001b[0m     args/train_batch_size                   : 24\n\u001b[1;38;5;39mCOMET INFO:\u001b[0m     args/use_cpu                            : False\n\u001b[1;38;5;39mCOMET INFO:\u001b[0m     args/use_ipex                           : False\n\u001b[1;38;5;39mCOMET INFO:\u001b[0m     args/use_legacy_prediction_loop         : False\n\u001b[1;38;5;39mCOMET INFO:\u001b[0m     args/use_mps_device                     : False\n\u001b[1;38;5;39mCOMET INFO:\u001b[0m     args/warmup_ratio                       : 0.0\n\u001b[1;38;5;39mCOMET INFO:\u001b[0m     args/warmup_steps                       : 0\n\u001b[1;38;5;39mCOMET INFO:\u001b[0m     args/weight_decay                       : 0.01\n\u001b[1;38;5;39mCOMET INFO:\u001b[0m     args/world_size                         : 1\n\u001b[1;38;5;39mCOMET INFO:\u001b[0m     config/_auto_class                      : None\n\u001b[1;38;5;39mCOMET INFO:\u001b[0m     config/_commit_hash                     : 77de1f7a7e5e737aead1cd880979d4f1b3af6668\n\u001b[1;38;5;39mCOMET INFO:\u001b[0m     config/_name_or_path                    : xlm-roberta-base\n\u001b[1;38;5;39mCOMET INFO:\u001b[0m     config/add_cross_attention              : False\n\u001b[1;38;5;39mCOMET INFO:\u001b[0m     config/architectures                    : ['XLMRobertaForMaskedLM']\n\u001b[1;38;5;39mCOMET INFO:\u001b[0m     config/attention_probs_dropout_prob     : 0.1\n\u001b[1;38;5;39mCOMET INFO:\u001b[0m     config/attribute_map                    : {}\n\u001b[1;38;5;39mCOMET INFO:\u001b[0m     config/bad_words_ids                    : None\n\u001b[1;38;5;39mCOMET INFO:\u001b[0m     config/begin_suppress_tokens            : None\n\u001b[1;38;5;39mCOMET INFO:\u001b[0m     config/bos_token_id                     : 0\n\u001b[1;38;5;39mCOMET INFO:\u001b[0m     config/chunk_size_feed_forward          : 0\n\u001b[1;38;5;39mCOMET INFO:\u001b[0m     config/classifier_dropout               : None\n\u001b[1;38;5;39mCOMET INFO:\u001b[0m     config/cross_attention_hidden_size      : None\n\u001b[1;38;5;39mCOMET INFO:\u001b[0m     config/decoder_start_token_id           : None\n\u001b[1;38;5;39mCOMET INFO:\u001b[0m     config/diversity_penalty                : 0.0\n\u001b[1;38;5;39mCOMET INFO:\u001b[0m     config/do_sample                        : False\n\u001b[1;38;5;39mCOMET INFO:\u001b[0m     config/early_stopping                   : False\n\u001b[1;38;5;39mCOMET INFO:\u001b[0m     config/encoder_no_repeat_ngram_size     : 0\n\u001b[1;38;5;39mCOMET INFO:\u001b[0m     config/eos_token_id                     : 2\n\u001b[1;38;5;39mCOMET INFO:\u001b[0m     config/exponential_decay_length_penalty : None\n\u001b[1;38;5;39mCOMET INFO:\u001b[0m     config/finetuning_task                  : None\n\u001b[1;38;5;39mCOMET INFO:\u001b[0m     config/forced_bos_token_id              : None\n\u001b[1;38;5;39mCOMET INFO:\u001b[0m     config/forced_eos_token_id              : None\n\u001b[1;38;5;39mCOMET INFO:\u001b[0m     config/hidden_act                       : gelu\n\u001b[1;38;5;39mCOMET INFO:\u001b[0m     config/hidden_dropout_prob              : 0.1\n\u001b[1;38;5;39mCOMET INFO:\u001b[0m     config/hidden_size                      : 768\n\u001b[1;38;5;39mCOMET INFO:\u001b[0m     config/id2label                         : {0: 'O', 1: 'B-PER', 2: 'I-PER', 3: 'B-ORG', 4: 'I-ORG', 5: 'B-LOC', 6: 'I-LOC'}\n\u001b[1;38;5;39mCOMET INFO:\u001b[0m     config/initializer_range                : 0.02\n\u001b[1;38;5;39mCOMET INFO:\u001b[0m     config/intermediate_size                : 3072\n\u001b[1;38;5;39mCOMET INFO:\u001b[0m     config/is_composition                   : False\n\u001b[1;38;5;39mCOMET INFO:\u001b[0m     config/is_decoder                       : False\n\u001b[1;38;5;39mCOMET INFO:\u001b[0m     config/is_encoder_decoder               : False\n\u001b[1;38;5;39mCOMET INFO:\u001b[0m     config/label2id                         : {'O': 0, 'B-PER': 1, 'I-PER': 2, 'B-ORG': 3, 'I-ORG': 4, 'B-LOC': 5, 'I-LOC': 6}\n\u001b[1;38;5;39mCOMET INFO:\u001b[0m     config/layer_norm_eps                   : 1e-05\n\u001b[1;38;5;39mCOMET INFO:\u001b[0m     config/length_penalty                   : 1.0\n\u001b[1;38;5;39mCOMET INFO:\u001b[0m     config/max_length                       : 20\n\u001b[1;38;5;39mCOMET INFO:\u001b[0m     config/max_position_embeddings          : 514\n\u001b[1;38;5;39mCOMET INFO:\u001b[0m     config/min_length                       : 0\n\u001b[1;38;5;39mCOMET INFO:\u001b[0m     config/model_type                       : xlm-roberta\n\u001b[1;38;5;39mCOMET INFO:\u001b[0m     config/name_or_path                     : xlm-roberta-base\n\u001b[1;38;5;39mCOMET INFO:\u001b[0m     config/no_repeat_ngram_size             : 0\n\u001b[1;38;5;39mCOMET INFO:\u001b[0m     config/num_attention_heads              : 12\n\u001b[1;38;5;39mCOMET INFO:\u001b[0m     config/num_beam_groups                  : 1\n\u001b[1;38;5;39mCOMET INFO:\u001b[0m     config/num_beams                        : 1\n\u001b[1;38;5;39mCOMET INFO:\u001b[0m     config/num_hidden_layers                : 12\n\u001b[1;38;5;39mCOMET INFO:\u001b[0m     config/num_labels                       : 7\n\u001b[1;38;5;39mCOMET INFO:\u001b[0m     config/num_return_sequences             : 1\n\u001b[1;38;5;39mCOMET INFO:\u001b[0m     config/output_attentions                : False\n\u001b[1;38;5;39mCOMET INFO:\u001b[0m     config/output_hidden_states             : False\n\u001b[1;38;5;39mCOMET INFO:\u001b[0m     config/output_past                      : True\n\u001b[1;38;5;39mCOMET INFO:\u001b[0m     config/output_scores                    : False\n\u001b[1;38;5;39mCOMET INFO:\u001b[0m     config/pad_token_id                     : 1\n\u001b[1;38;5;39mCOMET INFO:\u001b[0m     config/position_embedding_type          : absolute\n\u001b[1;38;5;39mCOMET INFO:\u001b[0m     config/prefix                           : None\n\u001b[1;38;5;39mCOMET INFO:\u001b[0m     config/problem_type                     : None\n\u001b[1;38;5;39mCOMET INFO:\u001b[0m     config/pruned_heads                     : {}\n\u001b[1;38;5;39mCOMET INFO:\u001b[0m     config/remove_invalid_values            : False\n\u001b[1;38;5;39mCOMET INFO:\u001b[0m     config/repetition_penalty               : 1.0\n\u001b[1;38;5;39mCOMET INFO:\u001b[0m     config/return_dict                      : True\n\u001b[1;38;5;39mCOMET INFO:\u001b[0m     config/return_dict_in_generate          : False\n\u001b[1;38;5;39mCOMET INFO:\u001b[0m     config/sep_token_id                     : None\n\u001b[1;38;5;39mCOMET INFO:\u001b[0m     config/suppress_tokens                  : None\n\u001b[1;38;5;39mCOMET INFO:\u001b[0m     config/task_specific_params             : None\n\u001b[1;38;5;39mCOMET INFO:\u001b[0m     config/temperature                      : 1.0\n\u001b[1;38;5;39mCOMET INFO:\u001b[0m     config/tf_legacy_loss                   : False\n\u001b[1;38;5;39mCOMET INFO:\u001b[0m     config/tie_encoder_decoder              : False\n\u001b[1;38;5;39mCOMET INFO:\u001b[0m     config/tie_word_embeddings              : True\n\u001b[1;38;5;39mCOMET INFO:\u001b[0m     config/tokenizer_class                  : None\n\u001b[1;38;5;39mCOMET INFO:\u001b[0m     config/top_k                            : 50\n\u001b[1;38;5;39mCOMET INFO:\u001b[0m     config/top_p                            : 1.0\n\u001b[1;38;5;39mCOMET INFO:\u001b[0m     config/torch_dtype                      : None\n\u001b[1;38;5;39mCOMET INFO:\u001b[0m     config/torchscript                      : False\n\u001b[1;38;5;39mCOMET INFO:\u001b[0m     config/transformers_version             : 4.17.0.dev0\n\u001b[1;38;5;39mCOMET INFO:\u001b[0m     config/type_vocab_size                  : 1\n\u001b[1;38;5;39mCOMET INFO:\u001b[0m     config/typical_p                        : 1.0\n\u001b[1;38;5;39mCOMET INFO:\u001b[0m     config/use_bfloat16                     : False\n\u001b[1;38;5;39mCOMET INFO:\u001b[0m     config/use_cache                        : True\n\u001b[1;38;5;39mCOMET INFO:\u001b[0m     config/use_return_dict                  : True\n\u001b[1;38;5;39mCOMET INFO:\u001b[0m     config/vocab_size                       : 250002\n\u001b[1;38;5;39mCOMET INFO:\u001b[0m   Uploads:\n\u001b[1;38;5;39mCOMET INFO:\u001b[0m     conda-environment-definition : 1\n\u001b[1;38;5;39mCOMET INFO:\u001b[0m     conda-info                   : 1\n\u001b[1;38;5;39mCOMET INFO:\u001b[0m     conda-specification          : 1\n\u001b[1;38;5;39mCOMET INFO:\u001b[0m     environment details          : 1\n\u001b[1;38;5;39mCOMET INFO:\u001b[0m     filename                     : 1\n\u001b[1;38;5;39mCOMET INFO:\u001b[0m     installed packages           : 1\n\u001b[1;38;5;39mCOMET INFO:\u001b[0m     model graph                  : 1\n\u001b[1;38;5;39mCOMET INFO:\u001b[0m     notebook                     : 1\n\u001b[1;38;5;39mCOMET INFO:\u001b[0m     os packages                  : 1\n\u001b[1;38;5;39mCOMET INFO:\u001b[0m     source_code                  : 1\n\u001b[1;38;5;39mCOMET INFO:\u001b[0m \n\u001b[1;38;5;39mCOMET INFO:\u001b[0m Please wait for metadata to finish uploading (timeout is 3600 seconds)\n","output_type":"stream"},{"execution_count":21,"output_type":"execute_result","data":{"text/plain":"TrainOutput(global_step=1668, training_loss=0.33039388725225877, metrics={'train_runtime': 887.0845, 'train_samples_per_second': 45.092, 'train_steps_per_second': 1.88, 'total_flos': 795839743276512.0, 'train_loss': 0.33039388725225877, 'epoch': 4.0})"},"metadata":{}}]},{"cell_type":"code","source":"trainer.push_to_hub()","metadata":{"execution":{"iopub.status.busy":"2023-10-30T17:53:19.470125Z","iopub.execute_input":"2023-10-30T17:53:19.470499Z","iopub.status.idle":"2023-10-30T17:53:19.475086Z","shell.execute_reply.started":"2023-10-30T17:53:19.470463Z","shell.execute_reply":"2023-10-30T17:53:19.474139Z"},"trusted":true},"execution_count":22,"outputs":[]},{"cell_type":"code","source":"text = \"My name is Sarah and I live in London\"\n\nlabel_text(text, labels, trainer.model, xlmr_tokenizer)","metadata":{"execution":{"iopub.status.busy":"2023-10-30T17:53:19.476346Z","iopub.execute_input":"2023-10-30T17:53:19.476645Z","iopub.status.idle":"2023-10-30T17:53:19.514828Z","shell.execute_reply.started":"2023-10-30T17:53:19.476618Z","shell.execute_reply":"2023-10-30T17:53:19.513967Z"},"trusted":true},"execution_count":23,"outputs":[{"execution_count":23,"output_type":"execute_result","data":{"text/plain":"           0    1      2    3       4     5   6      7    8        9      10\nTokens    <s>  ▁My  ▁name  ▁is  ▁Sarah  ▁and  ▁I  ▁live  ▁in  ▁London   </s>\nTags    B-ORG    O      O    O   B-PER     O   O      O    O    B-LOC  B-ORG","text/html":"<div>\n<style scoped>\n    .dataframe tbody tr th:only-of-type {\n        vertical-align: middle;\n    }\n\n    .dataframe tbody tr th {\n        vertical-align: top;\n    }\n\n    .dataframe thead th {\n        text-align: right;\n    }\n</style>\n<table border=\"1\" class=\"dataframe\">\n  <thead>\n    <tr style=\"text-align: right;\">\n      <th></th>\n      <th>0</th>\n      <th>1</th>\n      <th>2</th>\n      <th>3</th>\n      <th>4</th>\n      <th>5</th>\n      <th>6</th>\n      <th>7</th>\n      <th>8</th>\n      <th>9</th>\n      <th>10</th>\n    </tr>\n  </thead>\n  <tbody>\n    <tr>\n      <th>Tokens</th>\n      <td>&lt;s&gt;</td>\n      <td>▁My</td>\n      <td>▁name</td>\n      <td>▁is</td>\n      <td>▁Sarah</td>\n      <td>▁and</td>\n      <td>▁I</td>\n      <td>▁live</td>\n      <td>▁in</td>\n      <td>▁London</td>\n      <td>&lt;/s&gt;</td>\n    </tr>\n    <tr>\n      <th>Tags</th>\n      <td>B-ORG</td>\n      <td>O</td>\n      <td>O</td>\n      <td>O</td>\n      <td>B-PER</td>\n      <td>O</td>\n      <td>O</td>\n      <td>O</td>\n      <td>O</td>\n      <td>B-LOC</td>\n      <td>B-ORG</td>\n    </tr>\n  </tbody>\n</table>\n</div>"},"metadata":{}}]},{"cell_type":"markdown","source":"# <b><div style='padding:15px;background-color:#850E35;color:white;border-radius:2px;font-size:110%;text-align: center'>Cross-Lingual Transfer</div></b>","metadata":{}},{"cell_type":"code","source":"def get_f1_score(trainer, dataset):\n    return trainer.predict(dataset).metrics[\"test_f1\"]","metadata":{"execution":{"iopub.status.busy":"2023-10-30T17:53:19.515823Z","iopub.execute_input":"2023-10-30T17:53:19.516068Z","iopub.status.idle":"2023-10-30T17:53:19.520627Z","shell.execute_reply.started":"2023-10-30T17:53:19.516046Z","shell.execute_reply":"2023-10-30T17:53:19.519558Z"},"trusted":true},"execution_count":24,"outputs":[]},{"cell_type":"code","source":"def evaluate_lang_performance(lang, trainer):\n    panx_dataset = encode_panx_ds(panx_ds[lang])\n    return get_f1_score(trainer, panx_dataset[\"test\"])","metadata":{"execution":{"iopub.status.busy":"2023-10-30T17:53:19.521881Z","iopub.execute_input":"2023-10-30T17:53:19.522239Z","iopub.status.idle":"2023-10-30T17:53:19.531329Z","shell.execute_reply.started":"2023-10-30T17:53:19.522206Z","shell.execute_reply":"2023-10-30T17:53:19.530436Z"},"trusted":true},"execution_count":25,"outputs":[]},{"cell_type":"code","source":"f1_scores_on_en = evaluate_lang_performance(\"en\", trainer)\nf1_scores_on_de = evaluate_lang_performance(\"de\", trainer)\nf1_scores_on_tr = evaluate_lang_performance(\"tr\", trainer)\n\npd.DataFrame([f1_scores_on_en,f1_scores_on_de,f1_scores_on_tr],\n             index=[\"English\",\"German\",\"Turkish\"]).plot(kind=\"barh\")","metadata":{"execution":{"iopub.status.busy":"2023-10-30T17:53:19.532460Z","iopub.execute_input":"2023-10-30T17:53:19.532722Z","iopub.status.idle":"2023-10-30T17:54:54.367691Z","shell.execute_reply.started":"2023-10-30T17:53:19.532700Z","shell.execute_reply":"2023-10-30T17:54:54.366828Z"},"trusted":true},"execution_count":26,"outputs":[{"output_type":"display_data","data":{"text/plain":"  0%|          | 0/10 [00:00<?, ?ba/s]","application/vnd.jupyter.widget-view+json":{"version_major":2,"version_minor":0,"model_id":"8e043a67eb8a46c69ff72c346daf710d"}},"metadata":{}},{"output_type":"display_data","data":{"text/plain":"  0%|          | 0/5 [00:00<?, ?ba/s]","application/vnd.jupyter.widget-view+json":{"version_major":2,"version_minor":0,"model_id":"902e4ab46f424bb4b0722444a9a197be"}},"metadata":{}},{"output_type":"display_data","data":{"text/plain":"  0%|          | 0/5 [00:00<?, ?ba/s]","application/vnd.jupyter.widget-view+json":{"version_major":2,"version_minor":0,"model_id":"9bb30af9d91c4025bc9d3a8e50acf7eb"}},"metadata":{}},{"name":"stderr","text":"/opt/conda/lib/python3.10/site-packages/torch/nn/parallel/_functions.py:68: UserWarning: Was asked to gather along dimension 0, but all input tensors were scalars; will instead unsqueeze and return a vector.\n  warnings.warn('Was asked to gather along dimension 0, but all '\n","output_type":"stream"},{"output_type":"display_data","data":{"text/plain":"<IPython.core.display.HTML object>","text/html":""},"metadata":{}},{"output_type":"display_data","data":{"text/plain":"  0%|          | 0/10 [00:00<?, ?ba/s]","application/vnd.jupyter.widget-view+json":{"version_major":2,"version_minor":0,"model_id":"61b336c0c36048a0a134d83ba8ebb681"}},"metadata":{}},{"output_type":"display_data","data":{"text/plain":"  0%|          | 0/5 [00:00<?, ?ba/s]","application/vnd.jupyter.widget-view+json":{"version_major":2,"version_minor":0,"model_id":"485f34af2d4d46ad9bae87bdceef887b"}},"metadata":{}},{"output_type":"display_data","data":{"text/plain":"  0%|          | 0/5 [00:00<?, ?ba/s]","application/vnd.jupyter.widget-view+json":{"version_major":2,"version_minor":0,"model_id":"ae9579798335436eaa19bf069a7a927c"}},"metadata":{}},{"output_type":"display_data","data":{"text/plain":"<IPython.core.display.HTML object>","text/html":""},"metadata":{}},{"output_type":"display_data","data":{"text/plain":"  0%|          | 0/10 [00:00<?, ?ba/s]","application/vnd.jupyter.widget-view+json":{"version_major":2,"version_minor":0,"model_id":"2d74580d6dcb45b799a96763e30ffff4"}},"metadata":{}},{"output_type":"display_data","data":{"text/plain":"  0%|          | 0/5 [00:00<?, ?ba/s]","application/vnd.jupyter.widget-view+json":{"version_major":2,"version_minor":0,"model_id":"880c543761ad4a90b0630b23d393998b"}},"metadata":{}},{"output_type":"display_data","data":{"text/plain":"  0%|          | 0/5 [00:00<?, ?ba/s]","application/vnd.jupyter.widget-view+json":{"version_major":2,"version_minor":0,"model_id":"44310e29136d465f95dfdf88289c1358"}},"metadata":{}},{"output_type":"display_data","data":{"text/plain":"<IPython.core.display.HTML object>","text/html":""},"metadata":{}},{"execution_count":26,"output_type":"execute_result","data":{"text/plain":"<Axes: >"},"metadata":{}},{"output_type":"display_data","data":{"text/plain":"<Figure size 640x480 with 1 Axes>","image/png":"iVBORw0KGgoAAAANSUhEUgAAAkUAAAGdCAYAAAAc+wceAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjcuMiwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy8pXeV/AAAACXBIWXMAAA9hAAAPYQGoP6dpAAAgqUlEQVR4nO3de3CV9Z348fdJQhJyB8EkaCDlaiogVIQhoKKFwUWtdOpSV5cVu63sArtayiylWJFSA4O0MtOKNxRca8HLMjsOImLT0lmVLgsSRQQqtwrbhovWBHRNgHx/f/THGVMi5iTn5Mb7NXNmyJPnPM/nS4S8fc5zQiSEEJAkSTrPJbX2AJIkSW2BUSRJkoRRJEmSBBhFkiRJgFEkSZIEGEWSJEmAUSRJkgQYRZIkSQCktPYA7UVdXR1//OMfyc7OJhKJtPY4kiSpEUIIHD9+nB49epCUdO5rQUZRI/3xj3+kqKiotceQJElNcPDgQS6++OJz7mMUNVJ2djbwl9/UnJycVp5GkiQ1RnV1NUVFRdHv4+diFDXSmZfMcnJyjCJJktqZxtz64o3WkiRJGEWSJEmAUSRJkgR4T5EkSR3e6dOnOXnyZGuPkTCdOnUiOTm52ccxiiRJ6sBOnDjBoUOHCCG09igJE4lEuPjii8nKymrWcYwiSZI6qNOnT3Po0CEyMjLo3r17h/zhwyEEjh49yqFDh+jXr1+zrhgZRZIkdVAnT54khED37t3p3Llza4+TMN27d+fAgQOcPHmyWVHkjdaSJHVwHfEK0WfFa31GkSRJEkaRJEkS4D1FkiSdd4q//1KLnu/Aoutb9HxN5ZUiSZLUJj300EMUFxeTnp7OiBEj2Lx5c0LPZxRJkqQ259lnn2XmzJnMmzePN998k8suu4zx48dz5MiRhJ3TKJIkSW3OT3/6U77zne9wxx138OUvf5lHHnmEjIwMnnzyyYSd0yiSJEltSm1tLVu3bmXs2LHRbUlJSYwdO5ZNmzYl7LxGkSRJalOOHTvG6dOnyc/Pr7c9Pz+fysrKhJ3XKJIkScIokiRJbUy3bt1ITk7m8OHD9bYfPnyYgoKChJ3XKJIkSW1Kamoql19+OeXl5dFtdXV1lJeXM3LkyISd1x/eKEmS2pyZM2dy++23M2zYMIYPH87SpUv5+OOPueOOOxJ2TqNIkqTzTHv4CdPf/OY3OXr0KPfeey+VlZUMGTKE9evXn3XzdTwZRZIkqU2aMWMGM2bMaLHzeU+RJEkSRpEkSRJgFEmSJAFGkSRJEmAUSZLU4YUQWnuEhIrX+nz3WYwGznuFpLSM1h5DkqQvlJOWxOJx3alJ/TNJaTVfuP/gi/MSP1QC1NbWApCcnNys4xhFkiR1UMdr6th++FNyM/9MRl4yRCLn3P/TTz9tocnip66ujqNHj5KRkUFKSvOyxiiSJKmDCsCq7cfplduJLv/3KXDuKEr9v84tMle8JSUl0bNnTyJfEH1fxCiSJKkD+/DTOn5QfoxuGckkf8GdxOXfG9MiM8VbamoqSUnNv03aKJIkqYM7FaDy49NfuF96enoLTNN2+e4zSZIkjCJJkiTAKJIkSQKMIkmSJMAokiRJAowiSZIkwCiSJEkCjCJJkiTAKJIkSQKMIkmSJMAokiRJAowiSZIkwCiSJEkCjCJJkiTAKJIkSQKMIkmSJMAokiRJAowiSZIkwCiSJEkCjCJJkiTAKJIkSQLaeBQVFxezdOnSz/38mDFjuPvuuxt1rFj2lSRJ55+4RFEkEjnn47777ovHac6yZs0aFixYkJBjS5Kk80tKPA7ypz/9KfrrZ599lnvvvZfdu3dHt2VlZcV0vNraWlJTU79wv65du8Z0XEmSpM8TlytFBQUF0Udubi6RSCT68SOPPMLo0aPr7b906VKKi4ujH0+ZMoWJEydy//3306NHDwYMGNDgeZYvX05eXh7l5eXA2S+JLVu2jH79+pGenk5+fj4333xzvefX1dXxb//2b3Tt2pWCgoKEXcGSJEntT1yuFMVDeXk5OTk5vPrqqw1+fvHixSxevJgNGzYwfPjwsz6/ZcsW/vVf/5Wnn36a0tJSPvzwQ/7rv/6r3j5PPfUUM2fO5L//+7/ZtGkTU6ZMYdSoUYwbN+6s49XU1FBTUxP9uLq6upkrlCRJbVmbiaLMzEyWL1/e4Mtms2fP5umnn+a3v/0tl156aYPPf//998nMzOSGG24gOzubXr16MXTo0Hr7DB48mHnz5gHQr18/fv7zn1NeXt5gFC1cuJD58+fHYWWSJKk9aDPvPhs0aFCDQfSTn/yExx9/nNdee+1zgwhg3Lhx9OrVi969ezN58mSeeeYZPvnkk3r7DB48uN7HhYWFHDlypMHjzZkzh6qqqujj4MGDTViVJElqLxIeRUlJSYQQ6m07efLkWftlZmY2+Pwrr7yS06dP89xzz53zPNnZ2bz55pusWrWKwsJC7r33Xi677DI++uij6D6dOnWq95xIJEJdXV2Dx0tLSyMnJ6feQ5IkdVwJj6Lu3btTWVlZL4wqKioa/fzhw4fz8ssvU1ZWxpIlS865b0pKCmPHjmXx4sW8/fbbHDhwgF//+tdNHV2SJJ1HEn5P0ZgxYzh69CiLFy/m5ptvZv369bz88ssxXXkpLS1l3bp1/M3f/A0pKSkN/hDGtWvXsm/fPq666iq6dOnCunXrqKur+9x3skmSJH1Wwq8UlZSUsGzZMh566CEuu+wyNm/ezKxZs2I+zujRo3nppZe45557+NnPfnbW5/Py8lizZg3XXnstJSUlPPLII6xateqc9yFJkiSdEQl/fcOPGlRdXU1ubi5Fdz9HUlpGa48jSVLcHVh0fWuPEHdnvn9XVVV94atUbebdZ5IkSa3JKJIkScIokiRJAowiSZIkwCiSJEkCjCJJkiTAKJIkSQKMIkmSJMAokiRJAowiSZIkwCiSJEkCjCJJkiTAKJIkSQKMIkmSJMAokiRJAowiSZIkwCiSJEkCjCJJkiTAKJIkSQKMIkmSJMAokiRJAowiSZIkwCiSJEkCjCJJkiQAUlp7gPbmnfnjycnJae0xJElSnHmlSJIkCaNIkiQJMIokSZIAo0iSJAkwiiRJkgCjSJIkCTCKJEmSAKNIkiQJMIokSZIAo0iSJAkwiiRJkgCjSJIkCTCKJEmSAKNIkiQJMIokSZIAo0iSJAkwiiRJkgCjSJIkCTCKJEmSAKNIkiQJMIokSZIAo0iSJAkwiiRJkgCjSJIkCTCKJEmSAKNIkiQJMIokSZIAo0iSJAkwiiRJkgCjSJIkCTCKJEmSAKNIkiQJMIokSZIAo0iSJAkwiiRJkgCjSJIkCTCKJEmSAKNIkiQJMIokSZIAo0iSJAkwiiRJkgCjSJIkCTCKJEmSAKNIkiQJMIokSZIAo0iSJAkwiiRJkgCjSJIkCTCKJEmSAKNIkiQJMIokSZIAo0iSJAkwiiRJkgCjSJIkCTCKJEmSAKNIkiQJMIokSZIAo0iSJAkwiiRJkgCjSJIkCTCKJEmSAEhp7QHam4HzXiEpLaO1x5AkqUkOLLq+tUdos7xSJEmShFEkSZIEGEWSJEmAUSRJkgQYRZIkSYBRJEmSBBhFkiRJgFEkSZIEGEWSJEmAUSRJkgQYRZIkSYBRJEmSBBhFkiRJgFEkSZIEGEWSJEmAUSRJkgQYRZIkSYBRJEmSBBhFkiRJgFEkSZIEGEWSJEmAUSRJkgQYRZIkSYBRJEmSBBhFkiRJQBOjqLKykrvuuou+ffuSnp5Ofn4+o0aN4uGHH+aTTz6J94ySJEkJlxLrE/bt28eoUaPIy8ujrKyMQYMGkZaWxvbt23nssce46KKL+NrXvhbzILW1taSmpsb8PEmSpHiI+UrRtGnTSElJYcuWLUyaNImSkhJ69+7NTTfdxEsvvcSNN94IwEcffcS3v/1tunfvTk5ODtdeey1vvfVW9Dj33XcfQ4YMYfny5XzpS18iPT0dgEgkwqOPPsoNN9xARkYGJSUlbNq0iT179jBmzBgyMzMpLS1l79690WPt3buXm266ifz8fLKysrjiiiv41a9+VW/u4uJiysrK+Na3vkV2djY9e/bksccea9JvmiRJ6nhiiqIPPviADRs2MH36dDIzMxvcJxKJAPC3f/u3HDlyhJdffpmtW7fyla98ha9+9at8+OGH0X337NnDf/zHf7BmzRoqKiqi2xcsWMA//MM/UFFRwSWXXMKtt97K1KlTmTNnDlu2bCGEwIwZM6L7nzhxggkTJlBeXs62bdu47rrruPHGG3n//ffrzfaTn/yEYcOGsW3bNqZNm8Y///M/s3v37gbXUVNTQ3V1db2HJEnquGKKoj179hBCYMCAAfW2d+vWjaysLLKyspg9ezavvfYamzdv5vnnn2fYsGH069ePJUuWkJeXxwsvvBB9Xm1tLf/+7//O0KFDGTx4cHT7HXfcwaRJk+jfvz+zZ8/mwIED3HbbbYwfP56SkhLuuusuNm7cGN3/sssuY+rUqQwcOJB+/fqxYMEC+vTpw4svvlhvzgkTJjBt2jT69u3L7Nmz6datG7/5zW8aXOvChQvJzc2NPoqKimL5rZIkSe1MXN59tnnzZioqKrj00kupqanhrbfe4sSJE1xwwQXRWMrKymL//v31Xvbq1asX3bt3P+t4nw2k/Px8AAYNGlRv26effhq9enPixAlmzZpFSUkJeXl5ZGVlsXPnzrOuFH32uJFIhIKCAo4cOdLgmubMmUNVVVX0cfDgwSb8zkiSpPYiphut+/btSyQSOeslp969ewPQuXNn4C+RUlhYWO9qzhl5eXnRX3/eS3CdOnWK/vrMy3ENbaurqwNg1qxZvPrqqyxZsoS+ffvSuXNnbr75Zmpraz/3uGeOc+YYfy0tLY20tLQGPydJkjqemKLoggsuYNy4cfz85z/nX/7lXz43ar7yla9QWVlJSkoKxcXF8ZjznF5//XWmTJnC17/+deAvUXbgwIGEn1eSJHUcMb98tmzZMk6dOsWwYcN49tln2blzJ7t37+YXv/gFu3btIjk5mbFjxzJy5EgmTpzIhg0bOHDgAG+88QZz585ly5YtcV9Ev379ojdrv/XWW9x6662fewVIkiSpITH/nKI+ffqwbds2ysrKmDNnDocOHSItLY0vf/nLzJo1i2nTphGJRFi3bh1z587ljjvu4OjRoxQUFHDVVVdF7xGKp5/+9Kd861vforS0lG7dujF79mzfLSZJkmISCSGE1h6iPaiurv7Lu9Dufo6ktIzWHkeSpCY5sOj61h6hRZ35/l1VVUVOTs459/XfPpMkScIokiRJAowiSZIkwCiSJEkCjCJJkiTAKJIkSQKMIkmSJMAokiRJAowiSZIkwCiSJEkCjCJJkiTAKJIkSQKMIkmSJMAokiRJAowiSZIkwCiSJEkCjCJJkiTAKJIkSQKMIkmSJMAokiRJAowiSZIkwCiSJEkCjCJJkiTAKJIkSQIgpbUHaG/emT+enJyc1h5DkiTFmVeKJEmSMIokSZIAo0iSJAkwiiRJkgCjSJIkCTCKJEmSAKNIkiQJMIokSZIAo0iSJAkwiiRJkgCjSJIkCTCKJEmSAKNIkiQJMIokSZIAo0iSJAkwiiRJkgCjSJIkCTCKJEmSAKNIkiQJMIokSZIAo0iSJAkwiiRJkgCjSJIkCTCKJEmSAKNIkiQJMIokSZIAo0iSJAkwiiRJkgCjSJIkCTCKJEmSAKNIkiQJMIokSZIAo0iSJAkwiiRJkgCjSJIkCTCKJEmSAKNIkiQJMIokSZIAo0iSJAkwiiRJkgCjSJIkCTCKJEmSAKNIkiQJMIokSZIAo0iSJAkwiiRJkgCjSJIkCTCKJEmSAKNIkiQJMIokSZIAo0iSJAkwiiRJkgCjSJIkCTCKJEmSAKNIkiQJMIokSZIAo0iSJAkwiiRJkgCjSJIkCTCKJEmSAKNIkiQJgJTWHqC9GTjvFZLSMlp7DEmSOpQDi65v7RG8UiRJkgRGkSRJEmAUSZIkAUaRJEkSYBRJkiQBRpEkSRJgFEmSJAFGkSRJEmAUSZIkAUaRJEkSYBRJkiQBRpEkSRJgFEmSJAFGkSRJEmAUSZIkAUaRJEkSYBRJkiQBRpEkSRJgFEmSJAFGkSRJEmAUSZIkAUaRJEkSYBRJkiQBRpEkSRLQjqJo5cqV5OXlRT++7777GDJkSKOeG8u+kiTp/BSXKJoyZQqRSOSsx3XXXRePwzdo1qxZlJeXJ+z4kiTp/JISrwNdd911rFixot62tLS0eB3+LFlZWWRlZSXs+JIk6fwSt5fP0tLSKCgoqPfo0qULAJFIhOXLl/P1r3+djIwM+vXrx4svvljv+S+++CL9+vUjPT2da665hqeeeopIJMJHH33U4Pn++iWxjRs3Mnz4cDIzM8nLy2PUqFH84Q9/qPecp59+muLiYnJzc7nllls4fvx4vJYvSZLauRa7p2j+/PlMmjSJt99+mwkTJnDbbbfx4YcfArB//35uvvlmJk6cyFtvvcXUqVOZO3duo4996tQpJk6cyNVXX83bb7/Npk2buPPOO4lEItF99u7dy3/+53+ydu1a1q5dy29/+1sWLVoU93VKkqT2KW5RtHbt2uhLWmceZWVl0c9PmTKFv/u7v6Nv376UlZVx4sQJNm/eDMCjjz7KgAEDeOCBBxgwYAC33HILU6ZMafS5q6urqaqq4oYbbqBPnz6UlJRw++2307Nnz+g+dXV1rFy5koEDB3LllVcyefLkc96TVFNTQ3V1db2HJEnquOJ2T9E111zDww8/XG9b165do78ePHhw9NeZmZnk5ORw5MgRAHbv3s0VV1xR77nDhw9v9Lm7du3KlClTGD9+POPGjWPs2LFMmjSJwsLC6D7FxcVkZ2dHPy4sLIyevyELFy5k/vz5jZ5BkiS1b3G7UpSZmUnfvn3rPT4bRZ06daq3fyQSoa6uLl6nZ8WKFWzatInS0lKeffZZ+vfvz+9+97smn3/OnDlUVVVFHwcPHozbrJIkqe1pEz+naMCAAWzZsqXetv/5n/+J+ThDhw5lzpw5vPHGGwwcOJBf/vKXTZ4pLS2NnJyceg9JktRxxS2KampqqKysrPc4duxYo547depUdu3axezZs/n973/Pc889x8qVKwHq3Sz9efbv38+cOXPYtGkTf/jDH9iwYQPvvfceJSUlzVmSJEk6j8QtitavX09hYWG9x+jRoxv13C996Uu88MILrFmzhsGDB/Pwww9H333WmJ91lJGRwa5du/jGN75B//79ufPOO5k+fTpTp05t1pokSdL5IxJCCK09REPuv/9+HnnkkTZzL091dTW5ubkU3f0cSWkZrT2OJEkdyoFF1yfkuGe+f1dVVX3hrTBxe/dZcy1btowrrriCCy64gNdff50HHniAGTNmtPZYkiTpPNFmoui9997jxz/+MR9++CE9e/bke9/7HnPmzGntsSRJ0nmizUTRgw8+yIMPPtjaY0iSpPNUm3hLviRJUmsziiRJkjCKJEmSAKNIkiQJMIokSZIAo0iSJAkwiiRJkgCjSJIkCTCKJEmSAKNIkiQJMIokSZIAo0iSJAkwiiRJkgCjSJIkCTCKJEmSAKNIkiQJMIokSZIAo0iSJAkwiiRJkgCjSJIkCTCKJEmSAEhp7QHam3fmjycnJ6e1x5AkSXHmlSJJkiSMIkmSJMAokiRJAowiSZIkwCiSJEkCjCJJkiTAKJIkSQKMIkmSJMAokiRJAowiSZIkwCiSJEkCjCJJkiTAKJIkSQKMIkmSJMAokiRJAowiSZIkwCiSJEkCjCJJkiTAKJIkSQKMIkmSJMAokiRJAowiSZIkwCiSJEkCjCJJkiTAKJIkSQKMIkmSJMAokiRJAowiSZIkwCiSJEkCIKW1B2gvQggAVFdXt/IkkiSpsc583z7zffxcjKJG+uCDDwAoKipq5UkkSVKsjh8/Tm5u7jn3MYoaqWvXrgC8//77X/ib2hFUV1dTVFTEwYMHycnJae1xEsq1dlzn03rPp7XC+bVe19o8IQSOHz9Ojx49vnBfo6iRkpL+cvtVbm5uh/+P8rNycnLOm/W61o7rfFrv+bRWOL/W61qbrrEXM7zRWpIkCaNIkiQJMIoaLS0tjXnz5pGWltbao7SI82m9rrXjOp/Wez6tFc6v9brWlhMJjXmPmiRJUgfnlSJJkiSMIkmSJMAokiRJAowiSZIkwCiq56GHHqK4uJj09HRGjBjB5s2bz7n/888/zyWXXEJ6ejqDBg1i3bp1LTRpfMSy3h07dvCNb3yD4uJiIpEIS5cubblB4yCWtT7++ONceeWVdOnShS5dujB27Ngv/G+hLYllrWvWrGHYsGHk5eWRmZnJkCFDePrpp1tw2uaL9c/tGatXryYSiTBx4sTEDhhHsax15cqVRCKReo/09PQWnLZ5Yv26fvTRR0yfPp3CwkLS0tLo379/u/o7OZb1jhkz5qyvbSQS4frrr2/BiZsu1q/t0qVLGTBgAJ07d6aoqIjvfve7fPrpp4kZLiiEEMLq1atDampqePLJJ8OOHTvCd77znZCXlxcOHz7c4P6vv/56SE5ODosXLw7vvvtuuOeee0KnTp3C9u3bW3jypol1vZs3bw6zZs0Kq1atCgUFBeHBBx9s2YGbIda13nrrreGhhx4K27ZtCzt37gxTpkwJubm54dChQy08eexiXetvfvObsGbNmvDuu++GPXv2hKVLl4bk5OSwfv36Fp68aWJd7xn79+8PF110UbjyyivDTTfd1DLDNlOsa12xYkXIyckJf/rTn6KPysrKFp66aWJda01NTRg2bFiYMGFCeO2118L+/fvDxo0bQ0VFRQtP3jSxrveDDz6o93V95513QnJyclixYkXLDt4Esa71mWeeCWlpaeGZZ54J+/fvD6+88kooLCwM3/3udxMyn1H0/w0fPjxMnz49+vHp06dDjx49wsKFCxvcf9KkSeH666+vt23EiBFh6tSpCZ0zXmJd72f16tWrXUVRc9YaQginTp0K2dnZ4amnnkrUiHHT3LWGEMLQoUPDPffck4jx4q4p6z116lQoLS0Ny5cvD7fffnu7iaJY17pixYqQm5vbQtPFV6xrffjhh0Pv3r1DbW1tS40YV839c/vggw+G7OzscOLEiUSNGDexrnX69Onh2muvrbdt5syZYdSoUQmZz5fPgNraWrZu3crYsWOj25KSkhg7diybNm1q8DmbNm2qtz/A+PHjP3f/tqQp622v4rHWTz75hJMnT0b/UeC2qrlrDSFQXl7O7t27ueqqqxI5alw0db0/+tGPuPDCC/nHf/zHlhgzLpq61hMnTtCrVy+Kioq46aab2LFjR0uM2yxNWeuLL77IyJEjmT59Ovn5+QwcOJCysjJOnz7dUmM3WTz+jnriiSe45ZZbyMzMTNSYcdGUtZaWlrJ169boS2z79u1j3bp1TJgwISEz+g/CAseOHeP06dPk5+fX256fn8+uXbsafE5lZWWD+1dWViZsznhpynrbq3isdfbs2fTo0eOsCG5rmrrWqqoqLrroImpqakhOTmbZsmWMGzcu0eM2W1PW+9prr/HEE09QUVHRAhPGT1PWOmDAAJ588kkGDx5MVVUVS5YsobS0lB07dnDxxRe3xNhN0pS17tu3j1//+tfcdtttrFu3jj179jBt2jROnjzJvHnzWmLsJmvu31GbN2/mnXfe4YknnkjUiHHTlLXeeuutHDt2jNGjRxNC4NSpU/zTP/0TP/jBDxIyo1EkncOiRYtYvXo1GzdubFc3qcYiOzubiooKTpw4QXl5OTNnzqR3796MGTOmtUeLq+PHjzN58mQef/xxunXr1trjJNzIkSMZOXJk9OPS0lJKSkp49NFHWbBgQStOFn91dXVceOGFPPbYYyQnJ3P55Zfzv//7vzzwwANtPoqa64knnmDQoEEMHz68tUdJiI0bN1JWVsayZcsYMWIEe/bs4a677mLBggX88Ic/jPv5jCKgW7duJCcnc/jw4XrbDx8+TEFBQYPPKSgoiGn/tqQp622vmrPWJUuWsGjRIn71q18xePDgRI4ZF01da1JSEn379gVgyJAh7Ny5k4ULF7b5KIp1vXv37uXAgQPceOON0W11dXUApKSksHv3bvr06ZPYoZsoHn9mO3XqxNChQ9mzZ08iRoybpqy1sLCQTp06kZycHN1WUlJCZWUltbW1pKamJnTm5mjO1/bjjz9m9erV/OhHP0rkiHHTlLX+8Ic/ZPLkyXz7298GYNCgQXz88cfceeedzJ07l6Sk+N4F5D1FQGpqKpdffjnl5eXRbXV1dZSXl9f7P63PGjlyZL39AV599dXP3b8tacp626umrnXx4sUsWLCA9evXM2zYsJYYtdni9XWtq6ujpqYmESPGVazrveSSS9i+fTsVFRXRx9e+9jWuueYaKioqKCoqasnxYxKPr+3p06fZvn07hYWFiRozLpqy1lGjRrFnz55o5AL8/ve/p7CwsE0HETTva/v8889TU1PD3//93yd6zLhoylo/+eSTs8LnTPyGRPzTrQm5fbsdWr16dUhLSwsrV64M7777brjzzjtDXl5e9C2skydPDt///vej+7/++ushJSUlLFmyJOzcuTPMmzev3b0lP5b11tTUhG3btoVt27aFwsLCMGvWrLBt27bw3nvvtdYSGi3WtS5atCikpqaGF154od7bXo8fP95aS2i0WNdaVlYWNmzYEPbu3RvefffdsGTJkpCSkhIef/zx1lpCTGJd719rT+8+i3Wt8+fPD6+88krYu3dv2Lp1a7jllltCenp62LFjR2stodFiXev7778fsrOzw4wZM8Lu3bvD2rVrw4UXXhh+/OMft9YSYtLU/45Hjx4dvvnNb7b0uM0S61rnzZsXsrOzw6pVq8K+ffvChg0bQp8+fcKkSZMSMp9R9Bk/+9nPQs+ePUNqamoYPnx4+N3vfhf93NVXXx1uv/32evs/99xzoX///iE1NTVceuml4aWXXmrhiZsnlvXu378/AGc9rr766pYfvAliWWuvXr0aXOu8efNafvAmiGWtc+fODX379g3p6emhS5cuYeTIkWH16tWtMHXTxfrn9rPaUxSFENta77777ui++fn5YcKECeHNN99shambJtav6xtvvBFGjBgR0tLSQu/evcP9998fTp061cJTN12s6921a1cAwoYNG1p40uaLZa0nT54M9913X+jTp09IT08PRUVFYdq0aeHPf/5zQmaLhJCI60+SJEnti/cUSZIkYRRJkiQBRpEkSRJgFEmSJAFGkSRJEmAUSZIkAUaRJEkSYBRJkiQBRpEkSRJgFEmSJAFGkSRJEmAUSZIkAfD/AMHoEHDXtGrHAAAAAElFTkSuQmCC"},"metadata":{}}]},{"cell_type":"markdown","source":"# <b><div style='padding:15px;background-color:#850E35;color:white;border-radius:2px;font-size:110%;text-align: center'>Model Deployment</div></b>","metadata":{}},{"cell_type":"code","source":"from transformers import pipeline\n\nimport gradio as gr\n\nner_pipeline = pipeline(\"ner\", model = \"Tirendaz/roberta-base-NER\")\n\nexamples = [\n    \"My name is Tim and I live in California.\",\n    \"Ich arbeite bei Google in Berlin\",\n    \"Ali, Ankara'lı mı?\"\n]\n\ndef ner(text):\n    output = ner_pipeline(text, aggregation_strategy=\"simple\")\n    return {\"text\": text, \"entities\": output}    \n\ndemo = gr.Interface(ner,\n             gr.Textbox(placeholder=\"Enter sentence here...\"), \n             gr.HighlightedText(),\n             examples=examples)\n\ndemo.launch(inline=False, share=True)\n\n\nexperiment = comet_ml.Experiment()\nexperiment.add_tag(\"Ner-Analysis\")\n\ndemo.integrate(comet_ml=experiment)","metadata":{"execution":{"iopub.status.busy":"2023-10-30T17:54:54.369247Z","iopub.execute_input":"2023-10-30T17:54:54.369562Z"},"trusted":true},"execution_count":null,"outputs":[{"output_type":"display_data","data":{"text/plain":"Downloading (…)lve/main/config.json:   0%|          | 0.00/988 [00:00<?, ?B/s]","application/vnd.jupyter.widget-view+json":{"version_major":2,"version_minor":0,"model_id":"bf5768162692484686d8ad59f114079a"}},"metadata":{}},{"output_type":"display_data","data":{"text/plain":"Downloading pytorch_model.bin:   0%|          | 0.00/1.11G [00:00<?, ?B/s]","application/vnd.jupyter.widget-view+json":{"version_major":2,"version_minor":0,"model_id":"38eadc06f18f4114a291d3ea4c874393"}},"metadata":{}},{"output_type":"display_data","data":{"text/plain":"Downloading (…)okenizer_config.json:   0%|          | 0.00/418 [00:00<?, ?B/s]","application/vnd.jupyter.widget-view+json":{"version_major":2,"version_minor":0,"model_id":"e542f88e459545b382e6662b554f8ad2"}},"metadata":{}},{"output_type":"display_data","data":{"text/plain":"Downloading (…)tencepiece.bpe.model:   0%|          | 0.00/5.07M [00:00<?, ?B/s]","application/vnd.jupyter.widget-view+json":{"version_major":2,"version_minor":0,"model_id":"9b3f99dcca1f429eac273ee8ec0fff91"}},"metadata":{}},{"output_type":"display_data","data":{"text/plain":"Downloading tokenizer.json:   0%|          | 0.00/17.1M [00:00<?, ?B/s]","application/vnd.jupyter.widget-view+json":{"version_major":2,"version_minor":0,"model_id":"31e718136777434fb995280aeed0b0e5"}},"metadata":{}},{"output_type":"display_data","data":{"text/plain":"Downloading (…)cial_tokens_map.json:   0%|          | 0.00/280 [00:00<?, ?B/s]","application/vnd.jupyter.widget-view+json":{"version_major":2,"version_minor":0,"model_id":"d2557d0580d0477a8b858f23226ac0ff"}},"metadata":{}},{"name":"stdout","text":"Running on local URL:  http://127.0.0.1:7860\n","output_type":"stream"},{"name":"stderr","text":"\u001b[1;38;5;214mCOMET WARNING:\u001b[0m As you are running in a Jupyter environment, you will need to call `experiment.end()` when finished to ensure all metrics and code are logged before exiting.\n","output_type":"stream"},{"name":"stdout","text":"Running on public URL: https://9136211a04f74e0902.gradio.live\n\nThis share link expires in 72 hours. For free permanent hosting and GPU upgrades, run `gradio deploy` from Terminal to deploy to Spaces (https://huggingface.co/spaces)\n","output_type":"stream"},{"name":"stderr","text":"\u001b[1;38;5;39mCOMET INFO:\u001b[0m Couldn't find a Git repository in '/kaggle/working' nor in any parent directory. Set `COMET_GIT_DIRECTORY` if your Git Repository is elsewhere.\n\u001b[1;38;5;39mCOMET INFO:\u001b[0m Experiment is live on comet.com https://www.comet.com/tirendaz-academy/multilingual-ner-app/de58f882658a4c6ab3dd2e9cd1a025eb\n\n\u001b[1;38;5;214mCOMET WARNING:\u001b[0m Experiment was interrupted by user while waiting for the initial data logger to be flushed.\n\u001b[1;38;5;214mCOMET WARNING:\u001b[0m Error exporting current conda environment\n\u001b[1;38;5;214mCOMET WARNING:\u001b[0m Error retrieving Conda package as an explicit file\n\u001b[1;38;5;39mCOMET INFO:\u001b[0m ---------------------------------------------------------------------------------------\n\u001b[1;38;5;39mCOMET INFO:\u001b[0m Comet.ml Experiment Summary\n\u001b[1;38;5;39mCOMET INFO:\u001b[0m ---------------------------------------------------------------------------------------\n\u001b[1;38;5;39mCOMET INFO:\u001b[0m   Data:\n\u001b[1;38;5;39mCOMET INFO:\u001b[0m     display_summary_level : 1\n\u001b[1;38;5;39mCOMET INFO:\u001b[0m     url                   : https://www.comet.com/tirendaz-academy/multilingual-ner-app/de58f882658a4c6ab3dd2e9cd1a025eb\n\u001b[1;38;5;39mCOMET INFO:\u001b[0m   Others:\n\u001b[1;38;5;39mCOMET INFO:\u001b[0m     Created from : Gradio\n\u001b[1;38;5;39mCOMET INFO:\u001b[0m   Uploads:\n\u001b[1;38;5;214mCOMET WARNING:\u001b[0m Error retrieving Conda information\n\u001b[1;38;5;39mCOMET INFO:\u001b[0m     environment details : 1\n\u001b[1;38;5;39mCOMET INFO:\u001b[0m     filename            : 1\n\u001b[1;38;5;39mCOMET INFO:\u001b[0m     installed packages  : 1\n\u001b[1;38;5;39mCOMET INFO:\u001b[0m     notebook            : 1\n\u001b[1;38;5;39mCOMET INFO:\u001b[0m     os packages         : 1\n\u001b[1;38;5;39mCOMET INFO:\u001b[0m     source_code         : 1\n\u001b[1;38;5;39mCOMET INFO:\u001b[0m     text-sample         : 1\n\u001b[1;38;5;39mCOMET INFO:\u001b[0m \n\u001b[1;38;5;39mCOMET INFO:\u001b[0m Uploading 1 metrics, params and output messages\n","output_type":"stream"}]},{"cell_type":"markdown","source":"![](https://cdn-images-1.medium.com/max/1067/1*QmxYlXQdBJO4SHYBynuiiQ.gif)","metadata":{}},{"cell_type":"markdown","source":"## Resource\n\n- [NLP with Transformers](https://github.com/nlp-with-transformers/notebooks/blob/main/04_multilingual-ner.ipynb)\n\nThanks for reading. If you like this notebook, don't forget to upvote ☺️\n\nLet's connect [YouTube](http://youtube.com/tirendazacademy) | [Medium](http://tirendazacademy.medium.com) | [X](http://x.com/tirendazacademy) | [Linkedin](https://www.linkedin.com/in/tirendaz-academy) 😎","metadata":{}}]}