summarization.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "fafc4027",
   "metadata": {
    "collapsed": true,
    "pycharm": {
     "name": "#%%\n"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Collecting gdown\n",
      "  Downloading gdown-4.5.1.tar.gz (14 kB)\n",
      "  Installing build dependencies: started\n",
      "  Installing build dependencies: finished with status 'done'\n",
      "  Getting requirements to build wheel: started\n",
      "  Getting requirements to build wheel: finished with status 'done'\n",
      "  Preparing metadata (pyproject.toml): started\n",
      "  Preparing metadata (pyproject.toml): finished with status 'done'\n",
      "Requirement already satisfied: beautifulsoup4 in c:\\users\\admin\\appdata\\local\\programs\\python\\python310\\lib\\site-packages (from gdown) (4.11.1)\n",
      "Requirement already satisfied: tqdm in c:\\users\\admin\\appdata\\local\\programs\\python\\python310\\lib\\site-packages (from gdown) (4.64.0)\n",
      "Requirement already satisfied: six in c:\\users\\admin\\appdata\\local\\programs\\python\\python310\\lib\\site-packages (from gdown) (1.16.0)\n",
      "Requirement already satisfied: requests[socks] in c:\\users\\admin\\appdata\\local\\programs\\python\\python310\\lib\\site-packages (from gdown) (2.28.1)\n",
      "Collecting filelock\n",
      "  Downloading filelock-3.7.1-py3-none-any.whl (10 kB)\n",
      "Requirement already satisfied: soupsieve>1.2 in c:\\users\\admin\\appdata\\local\\programs\\python\\python310\\lib\\site-packages (from beautifulsoup4->gdown) (2.3.2.post1)\n",
      "Requirement already satisfied: idna<4,>=2.5 in c:\\users\\admin\\appdata\\local\\programs\\python\\python310\\lib\\site-packages (from requests[socks]->gdown) (3.3)\n",
      "Requirement already satisfied: certifi>=2017.4.17 in c:\\users\\admin\\appdata\\local\\programs\\python\\python310\\lib\\site-packages (from requests[socks]->gdown) (2022.6.15)\n",
      "Requirement already satisfied: urllib3<1.27,>=1.21.1 in c:\\users\\admin\\appdata\\local\\programs\\python\\python310\\lib\\site-packages (from requests[socks]->gdown) (1.26.10)\n",
      "Requirement already satisfied: charset-normalizer<3,>=2 in c:\\users\\admin\\appdata\\local\\programs\\python\\python310\\lib\\site-packages (from requests[socks]->gdown) (2.1.0)\n",
      "Collecting PySocks!=1.5.7,>=1.5.6\n",
      "  Downloading PySocks-1.7.1-py3-none-any.whl (16 kB)\n",
      "Requirement already satisfied: colorama in c:\\users\\admin\\appdata\\local\\programs\\python\\python310\\lib\\site-packages (from tqdm->gdown) (0.4.5)\n",
      "Building wheels for collected packages: gdown\n",
      "  Building wheel for gdown (pyproject.toml): started\n",
      "  Building wheel for gdown (pyproject.toml): finished with status 'done'\n",
      "  Created wheel for gdown: filename=gdown-4.5.1-py3-none-any.whl size=14933 sha256=47a18cf31f1cd6204e450387e0af1b6e47b2d9779999ab5579950b19e68da9cd\n",
      "  Stored in directory: c:\\users\\admin\\appdata\\local\\pip\\cache\\wheels\\38\\72\\83\\dab5cb8321023eb1549d7a7d7f564b479ed4cf71edb70ddad6\n",
      "Successfully built gdown\n",
      "Installing collected packages: PySocks, filelock, gdown\n",
      "Successfully installed PySocks-1.7.1 filelock-3.7.1 gdown-4.5.1\n",
      "Collecting pyarabic\n",
      "  Downloading PyArabic-0.6.15-py3-none-any.whl (126 kB)\n",
      "     ------------------------------------- 126.4/126.4 kB 63.5 kB/s eta 0:00:00\n",
      "Requirement already satisfied: six>=1.14.0 in c:\\users\\admin\\appdata\\local\\programs\\python\\python310\\lib\\site-packages (from pyarabic) (1.16.0)\n",
      "Installing collected packages: pyarabic\n",
      "Successfully installed pyarabic-0.6.15\n",
      "Collecting farasapy\n",
      "  Downloading farasapy-0.0.14-py3-none-any.whl (11 kB)\n",
      "Requirement already satisfied: requests in c:\\users\\admin\\appdata\\local\\programs\\python\\python310\\lib\\site-packages (from farasapy) (2.28.1)\n",
      "Requirement already satisfied: tqdm in c:\\users\\admin\\appdata\\local\\programs\\python\\python310\\lib\\site-packages (from farasapy) (4.64.0)\n",
      "Requirement already satisfied: charset-normalizer<3,>=2 in c:\\users\\admin\\appdata\\local\\programs\\python\\python310\\lib\\site-packages (from requests->farasapy) (2.1.0)\n",
      "Requirement already satisfied: idna<4,>=2.5 in c:\\users\\admin\\appdata\\local\\programs\\python\\python310\\lib\\site-packages (from requests->farasapy) (3.3)\n",
      "Requirement already satisfied: urllib3<1.27,>=1.21.1 in c:\\users\\admin\\appdata\\local\\programs\\python\\python310\\lib\\site-packages (from requests->farasapy) (1.26.10)\n",
      "Requirement already satisfied: certifi>=2017.4.17 in c:\\users\\admin\\appdata\\local\\programs\\python\\python310\\lib\\site-packages (from requests->farasapy) (2022.6.15)\n",
      "Requirement already satisfied: colorama in c:\\users\\admin\\appdata\\local\\programs\\python\\python310\\lib\\site-packages (from tqdm->farasapy) (0.4.5)\n",
      "Installing collected packages: farasapy\n",
      "Successfully installed farasapy-0.0.14\n"
     ]
    }
   ],
   "source": [
    "!pip install gdown\n",
    "!pip install pyarabic\n",
    "!pip install farasapy"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "028d5e54",
   "metadata": {
    "collapsed": true,
    "pycharm": {
     "name": "#%%\n"
    }
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\ProgramData\\Anaconda3\\lib\\site-packages\\gdown\\cli.py:127: FutureWarning: Option `--id` was deprecated in version 4.3.1 and will be removed in 5.0. You don't need to pass it anymore to use a file ID.\n",
      "  warnings.warn(\n",
      "Downloading...\n",
      "From: https://drive.google.com/uc?id=165kzfZDsRTZAAfZKedeZiUlKzMcHNgPd\n",
      "To: D:\\Courses\\Uottawa\\DTI5125[EG] Data Science Applications [LEC] 20225 - Sunday\\Project\\summary\\abstractive_summarizer-master\\Arabic_stop_words.txt\n",
      "\n",
      "  0%|          | 0.00/6.48k [00:00<?, ?B/s]\n",
      "100%|##########| 6.48k/6.48k [00:00<00:00, 6.49MB/s]\n"
     ]
    }
   ],
   "source": [
    "!gdown --id 165kzfZDsRTZAAfZKedeZiUlKzMcHNgPd"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "096fc993",
   "metadata": {
    "pycharm": {
     "name": "#%%\n"
    }
   },
   "outputs": [],
   "source": [
    "arabic_stop_words=[]\n",
    "with open ('./Arabic_stop_words.txt',encoding='utf-8') as f :\n",
    "    for i in f.readlines() :\n",
    "        arabic_stop_words.append(i)\n",
    "        arabic_stop_words[-1]=arabic_stop_words[-1][:-1]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Requirement already satisfied: transformers in c:\\users\\admin\\appdata\\local\\programs\\python\\python310\\lib\\site-packages (4.21.0)\n",
      "Requirement already satisfied: tokenizers!=0.11.3,<0.13,>=0.11.1 in c:\\users\\admin\\appdata\\local\\programs\\python\\python310\\lib\\site-packages (from transformers) (0.12.1)\n",
      "Requirement already satisfied: regex!=2019.12.17 in c:\\users\\admin\\appdata\\local\\programs\\python\\python310\\lib\\site-packages (from transformers) (2022.7.25)\n",
      "Requirement already satisfied: requests in c:\\users\\admin\\appdata\\local\\programs\\python\\python310\\lib\\site-packages (from transformers) (2.28.1)\n",
      "Requirement already satisfied: filelock in c:\\users\\admin\\appdata\\local\\programs\\python\\python310\\lib\\site-packages (from transformers) (3.7.1)\n",
      "Requirement already satisfied: huggingface-hub<1.0,>=0.1.0 in c:\\users\\admin\\appdata\\local\\programs\\python\\python310\\lib\\site-packages (from transformers) (0.8.1)\n",
      "Requirement already satisfied: tqdm>=4.27 in c:\\users\\admin\\appdata\\local\\programs\\python\\python310\\lib\\site-packages (from transformers) (4.64.0)\n",
      "Requirement already satisfied: pyyaml>=5.1 in c:\\users\\admin\\appdata\\local\\programs\\python\\python310\\lib\\site-packages (from transformers) (6.0)\n",
      "Requirement already satisfied: packaging>=20.0 in c:\\users\\admin\\appdata\\local\\programs\\python\\python310\\lib\\site-packages (from transformers) (21.3)\n",
      "Requirement already satisfied: numpy>=1.17 in c:\\users\\admin\\appdata\\local\\programs\\python\\python310\\lib\\site-packages (from transformers) (1.23.0)\n",
      "Requirement already satisfied: typing-extensions>=3.7.4.3 in c:\\users\\admin\\appdata\\local\\programs\\python\\python310\\lib\\site-packages (from huggingface-hub<1.0,>=0.1.0->transformers) (4.3.0)\n",
      "Requirement already satisfied: pyparsing!=3.0.5,>=2.0.2 in c:\\users\\admin\\appdata\\local\\programs\\python\\python310\\lib\\site-packages (from packaging>=20.0->transformers) (3.0.9)\n",
      "Requirement already satisfied: colorama in c:\\users\\admin\\appdata\\local\\programs\\python\\python310\\lib\\site-packages (from tqdm>=4.27->transformers) (0.4.5)\n",
      "Requirement already satisfied: idna<4,>=2.5 in c:\\users\\admin\\appdata\\local\\programs\\python\\python310\\lib\\site-packages (from requests->transformers) (3.3)\n",
      "Requirement already satisfied: urllib3<1.27,>=1.21.1 in c:\\users\\admin\\appdata\\local\\programs\\python\\python310\\lib\\site-packages (from requests->transformers) (1.26.10)\n",
      "Requirement already satisfied: certifi>=2017.4.17 in c:\\users\\admin\\appdata\\local\\programs\\python\\python310\\lib\\site-packages (from requests->transformers) (2022.6.15)\n",
      "Requirement already satisfied: charset-normalizer<3,>=2 in c:\\users\\admin\\appdata\\local\\programs\\python\\python310\\lib\\site-packages (from requests->transformers) (2.1.0)\n"
     ]
    }
   ],
   "source": [
    "!pip install transformers"
   ],
   "metadata": {
    "collapsed": false,
    "pycharm": {
     "name": "#%%\n"
    }
   }
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "2ec59800",
   "metadata": {
    "pycharm": {
     "name": "#%%\n"
    }
   },
   "outputs": [],
   "source": [
    "import numpy as np\n",
    "import pandas as pd\n",
    "import re\n",
    "from nltk.stem.isri import ISRIStemmer\n",
    "import logging\n",
    "import os\n",
    "logging.basicConfig(level=logging.WARNING)\n",
    "logger = logging.getLogger(__name__)\n",
    "import tensorflow as tf\n",
    "from transformers import BertTokenizer"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "fa34b442",
   "metadata": {
    "pycharm": {
     "name": "#%%\n"
    }
   },
   "outputs": [],
   "source": [
    "def data_preprocessing(data_frame):\n",
    "    # clean-up: remove #tags, http links and special symbols\n",
    "    data_frame['text']= data_frame['text'].apply(lambda x: re.sub(r'http\\S+', '', x))\n",
    "    data_frame['text'] = data_frame['text'].apply(lambda x: re.sub(r'[@|#]\\S*', '', x))\n",
    "    data_frame['text'] = data_frame['text'].apply(lambda x: re.sub(r'\"+', '', x))\n",
    "\n",
    "    # Remove arabic signs\n",
    "    data_frame['text'] = data_frame['text'].apply(lambda x: re.sub(r'([@A-Za-z0-9_ـــــــــــــ]+)|[^\\w\\s]|#|http\\S+', '', x))\n",
    "\n",
    "    # Remove repeated letters like \"الللللللللللللللله\" to \"الله\"\n",
    "    data_frame['text'] = data_frame['text'].apply(lambda x: x[0:2] + ''.join([x[i] for i in range(2, len(x)) if x[i]!=x[i-1] or x[i]!=x[i-2]]))\n",
    "\n",
    "    # remove stop words\n",
    "    data_frame['text'] = data_frame['text'].apply(lambda x: '' if x in arabic_stop_words else x)\n",
    "\n",
    "   \n",
    "    data_frame['text']=data_frame['text'].apply(lambda x:ISRIStemmer().stem(x))\n",
    "\n",
    "    return data_frame"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "57cff632",
   "metadata": {
    "pycharm": {
     "name": "#%%\n"
    }
   },
   "outputs": [],
   "source": [
    "def read_text(dir_path, fin):\n",
    "    list_txt = []\n",
    "    file_path = dir_path + \"\\\\\" + fin \n",
    "    with open(file_path, 'r', encoding='utf-8') as file:\n",
    "        for line in file.readlines():\n",
    "            list_txt.append(line)\n",
    "    df = pd.DataFrame()\n",
    "    df[\"text\"] = list_txt\n",
    "    df = data_preprocessing(df)\n",
    "    list_txt = df.values.tolist()\n",
    "    for article in list_txt:\n",
    "        with open(f\"{dir_path}\\cleaned_{fin}\", 'a', encoding='utf-8') as file:\n",
    "            file.write(f\"{article[0]}\\n\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "id": "bf4fbea6",
   "metadata": {
    "pycharm": {
     "name": "#%%\n"
    }
   },
   "outputs": [],
   "source": [
    "direct = 'arabic'\n",
    "file_list = os.listdir(direct)\n",
    "for file in file_list:\n",
    "    read_text(direct,file)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "id": "b25aa315",
   "metadata": {
    "pycharm": {
     "name": "#%%\n"
    }
   },
   "outputs": [],
   "source": [
    "direct = 'arabic'\n",
    "file_list = os.listdir(direct)\n",
    "files_list = [file for file in file_list if \"cleaned\" in file]\n",
    "def tokenized_text(dir_path,file):\n",
    "    content_list=list()\n",
    "    tokenz_list = list()\n",
    "    tok = BertTokenizer.from_pretrained('UBC-NLP/MARBERT')\n",
    "    fin = dir_path + \"\\\\\" + file\n",
    "    fin = open(fin, 'r',encoding=\"utf-8\")\n",
    "    for line in fin.readlines():\n",
    "        word_pieces = tok.tokenize(line.strip())\n",
    "        tokenz_list.append(word_pieces)\n",
    "        new_line = \" \".join(word_pieces)\n",
    "        content_list.append(new_line)\n",
    "    df = pd.DataFrame()\n",
    "    df['content'] = content_list\n",
    "    df['tokenized_content'] = tokenz_list\n",
    "    return tok,df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "id": "0a3d0a09",
   "metadata": {
    "scrolled": true,
    "pycharm": {
     "name": "#%%\n"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": "['cleaned_cleaned_cleaned_test.src',\n 'cleaned_cleaned_cleaned_test.tgt',\n 'cleaned_cleaned_cleaned_train.src',\n 'cleaned_cleaned_cleaned_train.tgt',\n 'cleaned_cleaned_cleaned_val.src',\n 'cleaned_cleaned_cleaned_val.tgt',\n 'cleaned_cleaned_test.src',\n 'cleaned_cleaned_test.tgt',\n 'cleaned_cleaned_train.src',\n 'cleaned_cleaned_train.tgt',\n 'cleaned_cleaned_val.src',\n 'cleaned_cleaned_val.tgt',\n 'cleaned_test.src',\n 'cleaned_test.tgt',\n 'cleaned_train.src',\n 'cleaned_train.tgt',\n 'cleaned_val.src',\n 'cleaned_val.tgt']"
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "files_list"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "id": "2df46be9",
   "metadata": {
    "pycharm": {
     "name": "#%%\n"
    }
   },
   "outputs": [],
   "source": [
    "document_tokenizer, document = tokenized_text(direct,file = 'cleaned_train.src')\n",
    "summary_tokenizer, summary = tokenized_text(direct,file = 'cleaned_train.tgt')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "id": "21fdc053",
   "metadata": {
    "pycharm": {
     "name": "#%%\n"
    }
   },
   "outputs": [],
   "source": [
    "document[\"token_ids\"] = document['tokenized_content'].apply(lambda row:document_tokenizer.convert_tokens_to_ids(row))\n",
    "summary[\"token_ids\"] = summary['tokenized_content'].apply(lambda row:summary_tokenizer.convert_tokens_to_ids(row))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "id": "6d43731c",
   "metadata": {
    "pycharm": {
     "name": "#%%\n"
    }
   },
   "outputs": [],
   "source": [
    "inputs = document[\"token_ids\"].values.tolist()\n",
    "targets= summary[\"token_ids\"].values.tolist()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "id": "8e586596",
   "metadata": {
    "pycharm": {
     "name": "#%%\n"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": "                                                  content  \\\n0       عد الصدق هو الحل الافضل في مثل تلك الحالات لان...   \n1       خنازير الغي ##نية مخلوقات اجتماعية لذلك اذا كا...   \n2       حتا ##ج الى ادخال مسار الملف للتو ##جه الى صفح...   \n3       ب ان تجعل القاري مهتما بالشخص ##ية الرييسية وي...   \n4       مثل احدى الطرق الرحيمة لمنع تزايد اسراب الاوز ...   \n...                                                   ...   \n102200                                                      \n102201  من الافضل ان يصاحب ##ك في عملية الولادة طبيب ا...   \n102202                                                      \n102203  قد يتسبب العفن الموجود في المرح ##اض في الكثير...   \n102204                                                      \n\n                                        tokenized_content  \\\n0       [عد, الصدق, هو, الحل, الافضل, في, مثل, تلك, ال...   \n1       [خنازير, الغي, ##نية, مخلوقات, اجتماعية, لذلك,...   \n2       [حتا, ##ج, الى, ادخال, مسار, الملف, للتو, ##جه...   \n3       [ب, ان, تجعل, القاري, مهتما, بالشخص, ##ية, الر...   \n4       [مثل, احدى, الطرق, الرحيمة, لمنع, تزايد, اسراب...   \n...                                                   ...   \n102200                                                 []   \n102201  [من, الافضل, ان, يصاحب, ##ك, في, عملية, الولاد...   \n102202                                                 []   \n102203  [قد, يتسبب, العفن, الموجود, في, المرح, ##اض, ف...   \n102204                                                 []   \n\n                                                token_ids  \n0       [2353, 8573, 2098, 5864, 5937, 1947, 2614, 410...  \n1       [48590, 6314, 2608, 41760, 24655, 5474, 2140, ...  \n2       [26640, 1020, 2127, 26048, 16962, 14436, 20360...  \n3       [118, 1946, 3188, 12053, 65318, 27900, 1988, 2...  \n4       [2614, 7575, 7398, 89222, 25372, 32084, 75081,...  \n...                                                   ...  \n102200                                                 []  \n102201  [1939, 5937, 1946, 42194, 1012, 1947, 5921, 25...  \n102202                                                 []  \n102203  [2167, 26989, 45804, 15139, 1947, 7477, 2430, ...  \n102204                                                 []  \n\n[102205 rows x 3 columns]",
      "text/html": "<div>\n<style scoped>\n    .dataframe tbody tr th:only-of-type {\n        vertical-align: middle;\n    }\n\n    .dataframe tbody tr th {\n        vertical-align: top;\n    }\n\n    .dataframe thead th {\n        text-align: right;\n    }\n</style>\n<table border=\"1\" class=\"dataframe\">\n  <thead>\n    <tr style=\"text-align: right;\">\n      <th></th>\n      <th>content</th>\n      <th>tokenized_content</th>\n      <th>token_ids</th>\n    </tr>\n  </thead>\n  <tbody>\n    <tr>\n      <th>0</th>\n      <td>عد الصدق هو الحل الافضل في مثل تلك الحالات لان...</td>\n      <td>[عد, الصدق, هو, الحل, الافضل, في, مثل, تلك, ال...</td>\n      <td>[2353, 8573, 2098, 5864, 5937, 1947, 2614, 410...</td>\n    </tr>\n    <tr>\n      <th>1</th>\n      <td>خنازير الغي ##نية مخلوقات اجتماعية لذلك اذا كا...</td>\n      <td>[خنازير, الغي, ##نية, مخلوقات, اجتماعية, لذلك,...</td>\n      <td>[48590, 6314, 2608, 41760, 24655, 5474, 2140, ...</td>\n    </tr>\n    <tr>\n      <th>2</th>\n      <td>حتا ##ج الى ادخال مسار الملف للتو ##جه الى صفح...</td>\n      <td>[حتا, ##ج, الى, ادخال, مسار, الملف, للتو, ##جه...</td>\n      <td>[26640, 1020, 2127, 26048, 16962, 14436, 20360...</td>\n    </tr>\n    <tr>\n      <th>3</th>\n      <td>ب ان تجعل القاري مهتما بالشخص ##ية الرييسية وي...</td>\n      <td>[ب, ان, تجعل, القاري, مهتما, بالشخص, ##ية, الر...</td>\n      <td>[118, 1946, 3188, 12053, 65318, 27900, 1988, 2...</td>\n    </tr>\n    <tr>\n      <th>4</th>\n      <td>مثل احدى الطرق الرحيمة لمنع تزايد اسراب الاوز ...</td>\n      <td>[مثل, احدى, الطرق, الرحيمة, لمنع, تزايد, اسراب...</td>\n      <td>[2614, 7575, 7398, 89222, 25372, 32084, 75081,...</td>\n    </tr>\n    <tr>\n      <th>...</th>\n      <td>...</td>\n      <td>...</td>\n      <td>...</td>\n    </tr>\n    <tr>\n      <th>102200</th>\n      <td></td>\n      <td>[]</td>\n      <td>[]</td>\n    </tr>\n    <tr>\n      <th>102201</th>\n      <td>من الافضل ان يصاحب ##ك في عملية الولادة طبيب ا...</td>\n      <td>[من, الافضل, ان, يصاحب, ##ك, في, عملية, الولاد...</td>\n      <td>[1939, 5937, 1946, 42194, 1012, 1947, 5921, 25...</td>\n    </tr>\n    <tr>\n      <th>102202</th>\n      <td></td>\n      <td>[]</td>\n      <td>[]</td>\n    </tr>\n    <tr>\n      <th>102203</th>\n      <td>قد يتسبب العفن الموجود في المرح ##اض في الكثير...</td>\n      <td>[قد, يتسبب, العفن, الموجود, في, المرح, ##اض, ف...</td>\n      <td>[2167, 26989, 45804, 15139, 1947, 7477, 2430, ...</td>\n    </tr>\n    <tr>\n      <th>102204</th>\n      <td></td>\n      <td>[]</td>\n      <td>[]</td>\n    </tr>\n  </tbody>\n</table>\n<p>102205 rows × 3 columns</p>\n</div>"
     },
     "execution_count": 11,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "document"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "id": "249c1e3f",
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 34
    },
    "colab_type": "code",
    "executionInfo": {
     "elapsed": 1240,
     "status": "ok",
     "timestamp": 1588996264681,
     "user": {
      "displayName": "rohan jagtap",
      "photoUrl": "https://lh3.googleusercontent.com/a-/AOh14GjFFnpJjw-7WaiTzz7xrkIJjBBwMs5i3OwVVYALIg=s64",
      "userId": "07173842849534370372"
     },
     "user_tz": -330
    },
    "id": "kVyErXAei5_b",
    "outputId": "41027c54-ce96-46d2-85c1-ff09bfe5d7dd",
    "pycharm": {
     "name": "#%%\n"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": "['عد', 'الصدق', 'هو', 'الحل', 'الافضل', 'في', 'مثل']"
     },
     "execution_count": 12,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "document_tokenizer.convert_ids_to_tokens([2353, 8573, 2098, 5864, 5937, 1947, 2614])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "id": "fa0cb39f",
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 34
    },
    "colab_type": "code",
    "executionInfo": {
     "elapsed": 1311,
     "status": "ok",
     "timestamp": 1588996326633,
     "user": {
      "displayName": "rohan jagtap",
      "photoUrl": "https://lh3.googleusercontent.com/a-/AOh14GjFFnpJjw-7WaiTzz7xrkIJjBBwMs5i3OwVVYALIg=s64",
      "userId": "07173842849534370372"
     },
     "user_tz": -330
    },
    "id": "KoizyBvLKv8h",
    "outputId": "62198745-1a53-41ef-f10c-5c9ca315fbe4",
    "scrolled": true,
    "pycharm": {
     "name": "#%%\n"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": "(100000, 100000)"
     },
     "execution_count": 13,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "encoder_vocab_size = document_tokenizer.vocab_size\n",
    "decoder_vocab_size = summary_tokenizer.vocab_size\n",
    "\n",
    "# vocab_size\n",
    "encoder_vocab_size, decoder_vocab_size"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "6bbb50ea",
   "metadata": {
    "colab_type": "text",
    "id": "mZden_q9_eZr",
    "pycharm": {
     "name": "#%% md\n"
    }
   },
   "source": [
    "#### Obtaining insights on lengths for defining maxlen"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "id": "c282b2d9",
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "ma4o2nGdK5Xb",
    "pycharm": {
     "name": "#%%\n"
    }
   },
   "outputs": [],
   "source": [
    "document_lengths = pd.Series([len(x) for x in document['tokenized_content']])\n",
    "summary_lengths = pd.Series([len(x) for x in summary['tokenized_content']])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "id": "af1341ee",
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 173
    },
    "colab_type": "code",
    "executionInfo": {
     "elapsed": 1365,
     "status": "ok",
     "timestamp": 1588996333925,
     "user": {
      "displayName": "rohan jagtap",
      "photoUrl": "https://lh3.googleusercontent.com/a-/AOh14GjFFnpJjw-7WaiTzz7xrkIJjBBwMs5i3OwVVYALIg=s64",
      "userId": "07173842849534370372"
     },
     "user_tz": -330
    },
    "id": "iXZlO99C-UXK",
    "outputId": "8b1480e7-5481-43f0-b685-e1deecda4548",
    "scrolled": true,
    "pycharm": {
     "name": "#%%\n"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": "count    102205.000000\nmean        232.570050\nstd         269.364991\nmin           0.000000\n25%           0.000000\n50%         171.000000\n75%         386.000000\nmax        2685.000000\ndtype: float64"
     },
     "execution_count": 15,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "document_lengths.describe()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "id": "c9e398c5",
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 173
    },
    "colab_type": "code",
    "executionInfo": {
     "elapsed": 1139,
     "status": "ok",
     "timestamp": 1588996333927,
     "user": {
      "displayName": "rohan jagtap",
      "photoUrl": "https://lh3.googleusercontent.com/a-/AOh14GjFFnpJjw-7WaiTzz7xrkIJjBBwMs5i3OwVVYALIg=s64",
      "userId": "07173842849534370372"
     },
     "user_tz": -330
    },
    "id": "ALMwKMx--ZF7",
    "outputId": "df49e521-f058-466e-ec7c-f667c215c748",
    "scrolled": true,
    "pycharm": {
     "name": "#%%\n"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": "count    102205.000000\nmean         20.262091\nstd          23.084321\nmin           0.000000\n25%           0.000000\n50%          17.000000\n75%          33.000000\nmax         500.000000\ndtype: float64"
     },
     "execution_count": 16,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "summary_lengths.describe()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "id": "ad72dd6e",
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "cVeMilXr-bpC",
    "pycharm": {
     "name": "#%%\n"
    }
   },
   "outputs": [],
   "source": [
    "# maxlen\n",
    "# taking values > and round figured to 75th percentile\n",
    "# at the same time not leaving high variance\n",
    "encoder_maxlen = 256\n",
    "decoder_maxlen = 128"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "beb214ca",
   "metadata": {
    "colab_type": "text",
    "id": "_SWap3YJBk-D",
    "pycharm": {
     "name": "#%% md\n"
    }
   },
   "source": [
    "#### Padding/Truncating sequences for identical sequence lengths"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "id": "a0746517",
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "vEyUBeu7ACRt",
    "pycharm": {
     "name": "#%%\n"
    }
   },
   "outputs": [],
   "source": [
    "inputs = tf.keras.preprocessing.sequence.pad_sequences(inputs, maxlen=encoder_maxlen, padding='post', truncating='post')\n",
    "targets = tf.keras.preprocessing.sequence.pad_sequences(targets, maxlen=decoder_maxlen, padding='post', truncating='post')"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "4749603e",
   "metadata": {
    "colab_type": "text",
    "id": "wIP0kIIcB8Rm",
    "pycharm": {
     "name": "#%% md\n"
    }
   },
   "source": [
    "### Creating dataset pipeline"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "id": "f191206b",
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "LzO6l3-AB7hJ",
    "pycharm": {
     "name": "#%%\n"
    }
   },
   "outputs": [],
   "source": [
    "inputs = tf.cast(inputs, dtype=tf.int32)\n",
    "targets = tf.cast(targets, dtype=tf.int32)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "id": "d8f34690",
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "slZ5f4P4DurS",
    "pycharm": {
     "name": "#%%\n"
    }
   },
   "outputs": [],
   "source": [
    "BUFFER_SIZE = 20000\n",
    "BATCH_SIZE = 16"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "id": "fdeabba6",
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "wI-fV7eABWN6",
    "pycharm": {
     "name": "#%%\n"
    }
   },
   "outputs": [],
   "source": [
    "dataset = tf.data.Dataset.from_tensor_slices((inputs, targets)).shuffle(BUFFER_SIZE).batch(BATCH_SIZE)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "355edcc2",
   "metadata": {
    "colab_type": "text",
    "id": "isN1CpAXLfsl",
    "pycharm": {
     "name": "#%% md\n"
    }
   },
   "source": [
    "### Positional Encoding for adding notion of position among words as unlike RNN this is non-directional"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "id": "52989c5e",
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "Purv7oyhETDZ",
    "pycharm": {
     "name": "#%%\n"
    }
   },
   "outputs": [],
   "source": [
    "def get_angles(position, i, d_model):\n",
    "    angle_rates = 1 / np.power(10000, (2 * (i // 2)) / np.float32(d_model))\n",
    "    return position * angle_rates"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "id": "1739e779",
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "40J2pc2NEXp5",
    "pycharm": {
     "name": "#%%\n"
    }
   },
   "outputs": [],
   "source": [
    "def positional_encoding(position, d_model):\n",
    "    angle_rads = get_angles(\n",
    "        np.arange(position)[:, np.newaxis],\n",
    "        np.arange(d_model)[np.newaxis, :],\n",
    "        d_model\n",
    "    )\n",
    "\n",
    "    # apply sin to even indices in the array; 2i\n",
    "    angle_rads[:, 0::2] = np.sin(angle_rads[:, 0::2])\n",
    "\n",
    "    # apply cos to odd indices in the array; 2i+1\n",
    "    angle_rads[:, 1::2] = np.cos(angle_rads[:, 1::2])\n",
    "\n",
    "    pos_encoding = angle_rads[np.newaxis, ...]\n",
    "\n",
    "    return tf.cast(pos_encoding, dtype=tf.float32)\n"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "a80fa0bd",
   "metadata": {
    "colab_type": "text",
    "id": "24Pe01DMMWHc",
    "pycharm": {
     "name": "#%% md\n"
    }
   },
   "source": [
    "### Masking\n",
    "\n",
    "- Padding mask for masking \"pad\" sequences\n",
    "- Lookahead mask for masking future words from contributing in prediction of current words in self attention"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "id": "b7000a77",
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "hN1wVQAdMVYy",
    "pycharm": {
     "name": "#%%\n"
    }
   },
   "outputs": [],
   "source": [
    "def create_padding_mask(seq):\n",
    "    seq = tf.cast(tf.math.equal(seq, 0), tf.float32)\n",
    "    return seq[:, tf.newaxis, tf.newaxis, :]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "id": "31fb5287",
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "UmjAPLWuMREE",
    "pycharm": {
     "name": "#%%\n"
    }
   },
   "outputs": [],
   "source": [
    "def create_look_ahead_mask(size):\n",
    "    mask = 1 - tf.linalg.band_part(tf.ones((size, size)), -1, 0)\n",
    "    return mask"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "dc9d6428",
   "metadata": {
    "colab_type": "text",
    "id": "n8DqUBc4NFOy",
    "pycharm": {
     "name": "#%% md\n"
    }
   },
   "source": [
    "### Building the Model"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "0b5521e7",
   "metadata": {
    "colab_type": "text",
    "id": "WfknVF7hNKf7",
    "pycharm": {
     "name": "#%% md\n"
    }
   },
   "source": [
    "#### Scaled Dot Product"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "id": "eedcc174",
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "w_B6M9OBNBKB",
    "pycharm": {
     "name": "#%%\n"
    }
   },
   "outputs": [],
   "source": [
    "def scaled_dot_product_attention(q, k, v, mask):\n",
    "    matmul_qk = tf.matmul(q, k, transpose_b=True)\n",
    "\n",
    "    dk = tf.cast(tf.shape(k)[-1], tf.float32)\n",
    "    scaled_attention_logits = matmul_qk / tf.math.sqrt(dk)\n",
    "\n",
    "    if mask is not None:\n",
    "        scaled_attention_logits += (mask * -1e9)  \n",
    "\n",
    "    attention_weights = tf.nn.softmax(scaled_attention_logits, axis=-1)\n",
    "\n",
    "    output = tf.matmul(attention_weights, v)\n",
    "    return output, attention_weights"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "1b3f08bd",
   "metadata": {
    "colab_type": "text",
    "id": "Rf7_a5uQOfJk",
    "pycharm": {
     "name": "#%% md\n"
    }
   },
   "source": [
    "#### Multi-Headed Attention"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "id": "ed716430",
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "iIuFrdXnNZEC",
    "pycharm": {
     "name": "#%%\n"
    }
   },
   "outputs": [],
   "source": [
    "class MultiHeadAttention(tf.keras.layers.Layer):\n",
    "    def __init__(self, d_model, num_heads):\n",
    "        super(MultiHeadAttention, self).__init__()\n",
    "        self.num_heads = num_heads\n",
    "        self.d_model = d_model\n",
    "\n",
    "        assert d_model % self.num_heads == 0\n",
    "\n",
    "        self.depth = d_model // self.num_heads\n",
    "\n",
    "        self.wq = tf.keras.layers.Dense(d_model)\n",
    "        self.wk = tf.keras.layers.Dense(d_model)\n",
    "        self.wv = tf.keras.layers.Dense(d_model)\n",
    "\n",
    "        self.dense = tf.keras.layers.Dense(d_model)\n",
    "        \n",
    "    def split_heads(self, x, batch_size):\n",
    "        x = tf.reshape(x, (batch_size, -1, self.num_heads, self.depth))\n",
    "        return tf.transpose(x, perm=[0, 2, 1, 3])\n",
    "    \n",
    "    def call(self, v, k, q, mask):\n",
    "        batch_size = tf.shape(q)[0]\n",
    "\n",
    "        q = self.wq(q)\n",
    "        k = self.wk(k)\n",
    "        v = self.wv(v)\n",
    "\n",
    "        q = self.split_heads(q, batch_size)\n",
    "        k = self.split_heads(k, batch_size)\n",
    "        v = self.split_heads(v, batch_size)\n",
    "\n",
    "        scaled_attention, attention_weights = scaled_dot_product_attention(\n",
    "            q, k, v, mask)\n",
    "\n",
    "        scaled_attention = tf.transpose(scaled_attention, perm=[0, 2, 1, 3])\n",
    "\n",
    "        concat_attention = tf.reshape(scaled_attention, (batch_size, -1, self.d_model))\n",
    "        output = self.dense(concat_attention)\n",
    "            \n",
    "        return output, attention_weights"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "027297c2",
   "metadata": {
    "colab_type": "text",
    "id": "A49tXMVvOkOZ",
    "pycharm": {
     "name": "#%% md\n"
    }
   },
   "source": [
    "### Feed Forward Network"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "id": "3c762a31",
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "d9-qoKuTNwKq",
    "pycharm": {
     "name": "#%%\n"
    }
   },
   "outputs": [],
   "source": [
    "def point_wise_feed_forward_network(d_model, dff):\n",
    "    return tf.keras.Sequential([\n",
    "        tf.keras.layers.Dense(dff, activation='relu'),\n",
    "        tf.keras.layers.Dense(d_model)\n",
    "    ])"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "f2bf42eb",
   "metadata": {
    "colab_type": "text",
    "id": "B2RRmn2bOpW9",
    "pycharm": {
     "name": "#%% md\n"
    }
   },
   "source": [
    "#### Fundamental Unit of Transformer encoder"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "id": "fdcff822",
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "HNuoJoFWO335",
    "pycharm": {
     "name": "#%%\n"
    }
   },
   "outputs": [],
   "source": [
    "class EncoderLayer(tf.keras.layers.Layer):\n",
    "    def __init__(self, d_model, num_heads, dff, rate=0.1):\n",
    "        super(EncoderLayer, self).__init__()\n",
    "\n",
    "        self.mha = MultiHeadAttention(d_model, num_heads)\n",
    "        self.ffn = point_wise_feed_forward_network(d_model, dff)\n",
    "\n",
    "        self.layernorm1 = tf.keras.layers.LayerNormalization(epsilon=1e-6)\n",
    "        self.layernorm2 = tf.keras.layers.LayerNormalization(epsilon=1e-6)\n",
    "\n",
    "        self.dropout1 = tf.keras.layers.Dropout(rate)\n",
    "        self.dropout2 = tf.keras.layers.Dropout(rate)\n",
    "    \n",
    "    def call(self, x, training, mask):\n",
    "        attn_output, _ = self.mha(x, x, x, mask)\n",
    "        attn_output = self.dropout1(attn_output, training=training)\n",
    "        out1 = self.layernorm1(x + attn_output)\n",
    "\n",
    "        ffn_output = self.ffn(out1)\n",
    "        ffn_output = self.dropout2(ffn_output, training=training)\n",
    "        out2 = self.layernorm2(out1 + ffn_output)\n",
    "\n",
    "        return out2\n"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "78d3908e",
   "metadata": {
    "colab_type": "text",
    "id": "9i6Zh8gnPqdW",
    "pycharm": {
     "name": "#%% md\n"
    }
   },
   "source": [
    "#### Fundamental Unit of Transformer decoder"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "id": "e954e141",
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "7CVmvs6dPMRC",
    "pycharm": {
     "name": "#%%\n"
    }
   },
   "outputs": [],
   "source": [
    "class DecoderLayer(tf.keras.layers.Layer):\n",
    "    def __init__(self, d_model, num_heads, dff, rate=0.1):\n",
    "        super(DecoderLayer, self).__init__()\n",
    "\n",
    "        self.mha1 = MultiHeadAttention(d_model, num_heads)\n",
    "        self.mha2 = MultiHeadAttention(d_model, num_heads)\n",
    "\n",
    "        self.ffn = point_wise_feed_forward_network(d_model, dff)\n",
    "\n",
    "        self.layernorm1 = tf.keras.layers.LayerNormalization(epsilon=1e-6)\n",
    "        self.layernorm2 = tf.keras.layers.LayerNormalization(epsilon=1e-6)\n",
    "        self.layernorm3 = tf.keras.layers.LayerNormalization(epsilon=1e-6)\n",
    "\n",
    "        self.dropout1 = tf.keras.layers.Dropout(rate)\n",
    "        self.dropout2 = tf.keras.layers.Dropout(rate)\n",
    "        self.dropout3 = tf.keras.layers.Dropout(rate)\n",
    "    \n",
    "    \n",
    "    def call(self, x, enc_output, training, look_ahead_mask, padding_mask):\n",
    "        attn1, attn_weights_block1 = self.mha1(x, x, x, look_ahead_mask)\n",
    "        attn1 = self.dropout1(attn1, training=training)\n",
    "        out1 = self.layernorm1(attn1 + x)\n",
    "\n",
    "        attn2, attn_weights_block2 = self.mha2(enc_output, enc_output, out1, padding_mask)\n",
    "        attn2 = self.dropout2(attn2, training=training)\n",
    "        out2 = self.layernorm2(attn2 + out1)\n",
    "\n",
    "        ffn_output = self.ffn(out2)\n",
    "        ffn_output = self.dropout3(ffn_output, training=training)\n",
    "        out3 = self.layernorm3(ffn_output + out2)\n",
    "\n",
    "        return out3, attn_weights_block1, attn_weights_block2\n"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "5bcb4c58",
   "metadata": {
    "colab_type": "text",
    "id": "6zt5MUc_QNid",
    "pycharm": {
     "name": "#%% md\n"
    }
   },
   "source": [
    "#### Encoder consisting of multiple EncoderLayer(s)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "id": "12a5226a",
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "BrbnTwijQJ-h",
    "pycharm": {
     "name": "#%%\n"
    }
   },
   "outputs": [],
   "source": [
    "class Encoder(tf.keras.layers.Layer):\n",
    "    def __init__(self, num_layers, d_model, num_heads, dff, input_vocab_size, maximum_position_encoding, rate=0.1):\n",
    "        super(Encoder, self).__init__()\n",
    "\n",
    "        self.d_model = d_model\n",
    "        self.num_layers = num_layers\n",
    "\n",
    "        self.embedding = tf.keras.layers.Embedding(input_vocab_size, d_model)\n",
    "        self.pos_encoding = positional_encoding(maximum_position_encoding, self.d_model)\n",
    "\n",
    "        self.enc_layers = [EncoderLayer(d_model, num_heads, dff, rate) for _ in range(num_layers)]\n",
    "\n",
    "        self.dropout = tf.keras.layers.Dropout(rate)\n",
    "        \n",
    "    def call(self, x, training, mask):\n",
    "        seq_len = tf.shape(x)[1]\n",
    "\n",
    "        x = self.embedding(x)\n",
    "        x *= tf.math.sqrt(tf.cast(self.d_model, tf.float32))\n",
    "        x += self.pos_encoding[:, :seq_len, :]\n",
    "\n",
    "        x = self.dropout(x, training=training)\n",
    "    \n",
    "        for i in range(self.num_layers):\n",
    "            x = self.enc_layers[i](x, training, mask)\n",
    "    \n",
    "        return x\n"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "190952ec",
   "metadata": {
    "colab_type": "text",
    "id": "4N5LrNrvRexg",
    "pycharm": {
     "name": "#%% md\n"
    }
   },
   "source": [
    "#### Decoder consisting of multiple DecoderLayer(s)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "id": "8282d066",
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "UmeqkZrIRbSB",
    "pycharm": {
     "name": "#%%\n"
    }
   },
   "outputs": [],
   "source": [
    "class Decoder(tf.keras.layers.Layer):\n",
    "    def __init__(self, num_layers, d_model, num_heads, dff, target_vocab_size, maximum_position_encoding, rate=0.1):\n",
    "        super(Decoder, self).__init__()\n",
    "\n",
    "        self.d_model = d_model\n",
    "        self.num_layers = num_layers\n",
    "\n",
    "        self.embedding = tf.keras.layers.Embedding(target_vocab_size, d_model)\n",
    "        self.pos_encoding = positional_encoding(maximum_position_encoding, d_model)\n",
    "\n",
    "        self.dec_layers = [DecoderLayer(d_model, num_heads, dff, rate) for _ in range(num_layers)]\n",
    "        self.dropout = tf.keras.layers.Dropout(rate)\n",
    "    \n",
    "    def call(self, x, enc_output, training, look_ahead_mask, padding_mask):\n",
    "        seq_len = tf.shape(x)[1]\n",
    "        attention_weights = {}\n",
    "\n",
    "        x = self.embedding(x)\n",
    "        x *= tf.math.sqrt(tf.cast(self.d_model, tf.float32))\n",
    "        x += self.pos_encoding[:, :seq_len, :]\n",
    "\n",
    "        x = self.dropout(x, training=training)\n",
    "\n",
    "        for i in range(self.num_layers):\n",
    "            x, block1, block2 = self.dec_layers[i](x, enc_output, training, look_ahead_mask, padding_mask)\n",
    "\n",
    "            attention_weights['decoder_layer{}_block1'.format(i+1)] = block1\n",
    "            attention_weights['decoder_layer{}_block2'.format(i+1)] = block2\n",
    "    \n",
    "        return x, attention_weights\n"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "08dabf2d",
   "metadata": {
    "colab_type": "text",
    "id": "lbMNK_bzSHnh",
    "pycharm": {
     "name": "#%% md\n"
    }
   },
   "source": [
    "#### Finally, the Transformer"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "id": "d3b54cd6",
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "FXHRG-o4R9Mc",
    "pycharm": {
     "name": "#%%\n"
    }
   },
   "outputs": [],
   "source": [
    "class Transformer(tf.keras.Model):\n",
    "    def __init__(self, num_layers, d_model, num_heads, dff, input_vocab_size, target_vocab_size, pe_input, pe_target, rate=0.1):\n",
    "        super(Transformer, self).__init__()\n",
    "\n",
    "        self.encoder = Encoder(num_layers, d_model, num_heads, dff, input_vocab_size, pe_input, rate)\n",
    "\n",
    "        self.decoder = Decoder(num_layers, d_model, num_heads, dff, target_vocab_size, pe_target, rate)\n",
    "\n",
    "        self.final_layer = tf.keras.layers.Dense(target_vocab_size)\n",
    "    \n",
    "    def call(self, inp, tar, training, enc_padding_mask, look_ahead_mask, dec_padding_mask):\n",
    "        enc_output = self.encoder(inp, training, enc_padding_mask)\n",
    "\n",
    "        dec_output, attention_weights = self.decoder(tar, enc_output, training, look_ahead_mask, dec_padding_mask)\n",
    "\n",
    "        final_output = self.final_layer(dec_output)\n",
    "\n",
    "        return final_output, attention_weights\n"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "bb53e0b3",
   "metadata": {
    "colab_type": "text",
    "id": "UndsMPZXTdSr",
    "pycharm": {
     "name": "#%% md\n"
    }
   },
   "source": [
    "### Training"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 34,
   "id": "a7d294a5",
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "lMTZJdIoSbuy",
    "pycharm": {
     "name": "#%%\n"
    }
   },
   "outputs": [],
   "source": [
    "# hyper-params\n",
    "num_layers = 4\n",
    "d_model = 128\n",
    "dff = 512\n",
    "num_heads = 8\n",
    "EPOCHS = 50"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "57839068",
   "metadata": {
    "colab_type": "text",
    "id": "uOGvkYDNTjIj",
    "pycharm": {
     "name": "#%% md\n"
    }
   },
   "source": [
    "#### Adam optimizer with custom learning rate scheduling"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 35,
   "id": "b79f308e",
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "tfiynCLlTL8C",
    "pycharm": {
     "name": "#%%\n"
    }
   },
   "outputs": [],
   "source": [
    "class CustomSchedule(tf.keras.optimizers.schedules.LearningRateSchedule):\n",
    "    def __init__(self, d_model, warmup_steps=4000):\n",
    "        super(CustomSchedule, self).__init__()\n",
    "\n",
    "        self.d_model = d_model\n",
    "        self.d_model = tf.cast(self.d_model, tf.float32)\n",
    "\n",
    "        self.warmup_steps = warmup_steps\n",
    "    \n",
    "    def __call__(self, step):\n",
    "        arg1 = tf.math.rsqrt(step)\n",
    "        arg2 = step * (self.warmup_steps ** -1.5)\n",
    "\n",
    "        return tf.math.rsqrt(self.d_model) * tf.math.minimum(arg1, arg2)\n"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "7b9427d6",
   "metadata": {
    "colab_type": "text",
    "id": "DsVdrENTUERY",
    "pycharm": {
     "name": "#%% md\n"
    }
   },
   "source": [
    "#### Defining losses and other metrics "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 36,
   "id": "5f3b77b8",
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "Ip1-943kTXXK",
    "pycharm": {
     "name": "#%%\n"
    }
   },
   "outputs": [],
   "source": [
    "learning_rate = CustomSchedule(d_model)\n",
    "\n",
    "optimizer = tf.keras.optimizers.Adam(learning_rate, beta_1=0.9, beta_2=0.98, epsilon=1e-9)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 37,
   "id": "133eab8c",
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "ktKwyvKtTvF6",
    "pycharm": {
     "name": "#%%\n"
    }
   },
   "outputs": [],
   "source": [
    "loss_object = tf.keras.losses.SparseCategoricalCrossentropy(from_logits=True, reduction='none')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 38,
   "id": "43d1cca0",
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "uW4LA_45T4Aa",
    "pycharm": {
     "name": "#%%\n"
    }
   },
   "outputs": [],
   "source": [
    "def loss_function(real, pred):\n",
    "    mask = tf.math.logical_not(tf.math.equal(real, 0))\n",
    "    loss_ = loss_object(real, pred)\n",
    "\n",
    "    mask = tf.cast(mask, dtype=loss_.dtype)\n",
    "    loss_ *= mask\n",
    "\n",
    "    return tf.reduce_sum(loss_)/tf.reduce_sum(mask)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 39,
   "id": "e7f71cec",
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "Ze0u6xxXT7dI",
    "pycharm": {
     "name": "#%%\n"
    }
   },
   "outputs": [],
   "source": [
    "train_loss = tf.keras.metrics.Mean(name='train_loss')"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "5d7fa2f2",
   "metadata": {
    "colab_type": "text",
    "id": "9XvKy3v6ULnO",
    "pycharm": {
     "name": "#%% md\n"
    }
   },
   "source": [
    "#### Transformer"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 40,
   "id": "e802146e",
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "d5-RcxqFUCuk",
    "pycharm": {
     "name": "#%%\n"
    }
   },
   "outputs": [],
   "source": [
    "transformer = Transformer(\n",
    "    num_layers, \n",
    "    d_model, \n",
    "    num_heads, \n",
    "    dff,\n",
    "    encoder_vocab_size, \n",
    "    decoder_vocab_size, \n",
    "    pe_input=encoder_vocab_size, \n",
    "    pe_target=decoder_vocab_size,\n",
    ")"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "4133810f",
   "metadata": {
    "colab_type": "text",
    "id": "f56BGiVXU_Dk",
    "pycharm": {
     "name": "#%% md\n"
    }
   },
   "source": [
    "#### Masks"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 41,
   "id": "97258449",
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "FZxHuyZxU5Pa",
    "pycharm": {
     "name": "#%%\n"
    }
   },
   "outputs": [],
   "source": [
    "def create_masks(inp, tar):\n",
    "    enc_padding_mask = create_padding_mask(inp)\n",
    "    dec_padding_mask = create_padding_mask(inp)\n",
    "\n",
    "    look_ahead_mask = create_look_ahead_mask(tf.shape(tar)[1])\n",
    "    dec_target_padding_mask = create_padding_mask(tar)\n",
    "    combined_mask = tf.maximum(dec_target_padding_mask, look_ahead_mask)\n",
    "  \n",
    "    return enc_padding_mask, combined_mask, dec_padding_mask\n"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "b4097be7",
   "metadata": {
    "colab_type": "text",
    "id": "SYIotvaBVI0d",
    "pycharm": {
     "name": "#%% md\n"
    }
   },
   "source": [
    "#### Checkpoints"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 42,
   "id": "e1b0b159",
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 34
    },
    "colab_type": "code",
    "executionInfo": {
     "elapsed": 7786,
     "status": "ok",
     "timestamp": 1588996357748,
     "user": {
      "displayName": "rohan jagtap",
      "photoUrl": "https://lh3.googleusercontent.com/a-/AOh14GjFFnpJjw-7WaiTzz7xrkIJjBBwMs5i3OwVVYALIg=s64",
      "userId": "07173842849534370372"
     },
     "user_tz": -330
    },
    "id": "tOc1_3c-VGaL",
    "outputId": "eeab15d7-f887-4f37-dfec-c980948f97d6",
    "pycharm": {
     "name": "#%%\n"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Latest checkpoint restored!!\n"
     ]
    }
   ],
   "source": [
    "checkpoint_path = \"checkpoints\"\n",
    "\n",
    "ckpt = tf.train.Checkpoint(transformer=transformer, optimizer=optimizer)\n",
    "\n",
    "ckpt_manager = tf.train.CheckpointManager(ckpt, checkpoint_path, max_to_keep=5)\n",
    "\n",
    "if ckpt_manager.latest_checkpoint:\n",
    "    ckpt.restore(ckpt_manager.latest_checkpoint)\n",
    "    print ('Latest checkpoint restored!!')"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "04102e6d",
   "metadata": {
    "colab_type": "text",
    "id": "WfpI0gS4c06c",
    "pycharm": {
     "name": "#%% md\n"
    }
   },
   "source": [
    "#### Training steps"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 148,
   "id": "3d68d3c3",
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "xmVOMzkrczgl",
    "pycharm": {
     "name": "#%%\n"
    }
   },
   "outputs": [],
   "source": [
    "@tf.function\n",
    "def train_step(inp, tar):\n",
    "    tar_inp = tar[:, :-1]\n",
    "    tar_real = tar[:, 1:]\n",
    "\n",
    "    enc_padding_mask, combined_mask, dec_padding_mask = create_masks(inp, tar_inp)\n",
    "\n",
    "    with tf.GradientTape() as tape:\n",
    "        predictions, _ = transformer(\n",
    "            inp, tar_inp, \n",
    "            True, \n",
    "            enc_padding_mask, \n",
    "            combined_mask, \n",
    "            dec_padding_mask\n",
    "        )\n",
    "        loss = loss_function(tar_real, predictions)\n",
    "\n",
    "    gradients = tape.gradient(loss, transformer.trainable_variables)    \n",
    "    optimizer.apply_gradients(zip(gradients, transformer.trainable_variables))\n",
    "\n",
    "    train_loss(loss)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 149,
   "id": "c71fc7a7",
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 1000
    },
    "colab_type": "code",
    "executionInfo": {
     "elapsed": 6000532,
     "status": "ok",
     "timestamp": 1589005387811,
     "user": {
      "displayName": "rohan jagtap",
      "photoUrl": "https://lh3.googleusercontent.com/a-/AOh14GjFFnpJjw-7WaiTzz7xrkIJjBBwMs5i3OwVVYALIg=s64",
      "userId": "07173842849534370372"
     },
     "user_tz": -330
    },
    "id": "xORKpv69dSW5",
    "outputId": "572d3232-3af4-4bd6-ebc7-a151f6ad000d",
    "pycharm": {
     "name": "#%%\n"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 1 Batch 0 Loss 11.5222\n",
      "Epoch 1 Batch 429 Loss 10.3292\n",
      "Epoch 1 Batch 858 Loss 9.3860\n",
      "Epoch 1 Loss 8.9886\n",
      "Time taken for 1 epoch: 182.31050944328308 secs\n",
      "\n",
      "Epoch 2 Batch 0 Loss 7.6642\n",
      "Epoch 2 Batch 429 Loss 7.7766\n",
      "Epoch 2 Batch 858 Loss 7.7102\n",
      "Epoch 2 Loss 7.6157\n",
      "Time taken for 1 epoch: 174.18780636787415 secs\n",
      "\n",
      "Epoch 3 Batch 0 Loss 6.4352\n",
      "Epoch 3 Batch 429 Loss 7.0391\n",
      "Epoch 3 Batch 858 Loss 6.9624\n",
      "Epoch 3 Loss 6.8937\n",
      "Time taken for 1 epoch: 175.47943949699402 secs\n",
      "\n",
      "Epoch 4 Batch 0 Loss 6.3374\n",
      "Epoch 4 Batch 429 Loss 6.4686\n",
      "Epoch 4 Batch 858 Loss 6.4506\n",
      "Epoch 4 Loss 6.4175\n",
      "Time taken for 1 epoch: 174.64511561393738 secs\n",
      "\n",
      "Epoch 5 Batch 0 Loss 6.2885\n",
      "Epoch 5 Batch 429 Loss 5.9738\n",
      "Epoch 5 Batch 858 Loss 5.9861\n",
      "Saving checkpoint for epoch 5 at checkpoints\\ckpt-1\n",
      "Epoch 5 Loss 5.9814\n",
      "Time taken for 1 epoch: 177.04201316833496 secs\n",
      "\n",
      "Epoch 6 Batch 0 Loss 5.9953\n",
      "Epoch 6 Batch 429 Loss 5.6155\n",
      "Epoch 6 Batch 858 Loss 5.6555\n",
      "Epoch 6 Loss 5.6738\n",
      "Time taken for 1 epoch: 175.93753957748413 secs\n",
      "\n",
      "Epoch 7 Batch 0 Loss 5.7344\n",
      "Epoch 7 Batch 429 Loss 5.3598\n",
      "Epoch 7 Batch 858 Loss 5.4139\n",
      "Epoch 7 Loss 5.4435\n",
      "Time taken for 1 epoch: 174.61379265785217 secs\n",
      "\n",
      "Epoch 8 Batch 0 Loss 5.2067\n",
      "Epoch 8 Batch 429 Loss 5.1271\n",
      "Epoch 8 Batch 858 Loss 5.1987\n",
      "Epoch 8 Loss 5.2584\n",
      "Time taken for 1 epoch: 174.14170050621033 secs\n",
      "\n",
      "Epoch 9 Batch 0 Loss 4.8456\n",
      "Epoch 9 Batch 429 Loss 4.9922\n",
      "Epoch 9 Batch 858 Loss 5.0622\n",
      "Epoch 9 Loss 5.1267\n",
      "Time taken for 1 epoch: 173.66842079162598 secs\n",
      "\n",
      "Epoch 10 Batch 0 Loss 4.9102\n",
      "Epoch 10 Batch 429 Loss 4.8868\n",
      "Epoch 10 Batch 858 Loss 4.9704\n",
      "Saving checkpoint for epoch 10 at checkpoints\\ckpt-2\n",
      "Epoch 10 Loss 5.0359\n",
      "Time taken for 1 epoch: 174.4938840866089 secs\n",
      "\n",
      "Epoch 11 Batch 0 Loss 4.6052\n",
      "Epoch 11 Batch 429 Loss 4.7655\n",
      "Epoch 11 Batch 858 Loss 4.8807\n",
      "Epoch 11 Loss 4.9668\n",
      "Time taken for 1 epoch: 173.55874490737915 secs\n",
      "\n",
      "Epoch 12 Batch 0 Loss 4.4683\n",
      "Epoch 12 Batch 429 Loss 4.7158\n",
      "Epoch 12 Batch 858 Loss 4.8207\n",
      "Epoch 12 Loss 4.8951\n",
      "Time taken for 1 epoch: 173.64703583717346 secs\n",
      "\n",
      "Epoch 13 Batch 0 Loss 4.4472\n",
      "Epoch 13 Batch 429 Loss 4.6400\n",
      "Epoch 13 Batch 858 Loss 4.7457\n",
      "Epoch 13 Loss 4.8186\n",
      "Time taken for 1 epoch: 173.9060800075531 secs\n",
      "\n",
      "Epoch 14 Batch 0 Loss 4.3021\n",
      "Epoch 14 Batch 429 Loss 4.5875\n",
      "Epoch 14 Batch 858 Loss 4.6818\n",
      "Epoch 14 Loss 4.7501\n",
      "Time taken for 1 epoch: 173.95075011253357 secs\n",
      "\n",
      "Epoch 15 Batch 0 Loss 4.5755\n",
      "Epoch 15 Batch 429 Loss 4.5069\n",
      "Epoch 15 Batch 858 Loss 4.6093\n",
      "Saving checkpoint for epoch 15 at checkpoints\\ckpt-3\n",
      "Epoch 15 Loss 4.6924\n",
      "Time taken for 1 epoch: 176.3740198612213 secs\n",
      "\n",
      "Epoch 16 Batch 0 Loss 4.1661\n",
      "Epoch 16 Batch 429 Loss 4.4404\n",
      "Epoch 16 Batch 858 Loss 4.5452\n",
      "Epoch 16 Loss 4.6301\n",
      "Time taken for 1 epoch: 175.225084066391 secs\n",
      "\n",
      "Epoch 17 Batch 0 Loss 4.3144\n",
      "Epoch 17 Batch 429 Loss 4.3987\n",
      "Epoch 17 Batch 858 Loss 4.4964\n",
      "Epoch 17 Loss 4.5786\n",
      "Time taken for 1 epoch: 175.07189750671387 secs\n",
      "\n",
      "Epoch 18 Batch 0 Loss 4.1526\n",
      "Epoch 18 Batch 429 Loss 4.3403\n",
      "Epoch 18 Batch 858 Loss 4.4531\n",
      "Epoch 18 Loss 4.5346\n",
      "Time taken for 1 epoch: 175.17649364471436 secs\n",
      "\n",
      "Epoch 19 Batch 0 Loss 4.2428\n",
      "Epoch 19 Batch 429 Loss 4.2942\n",
      "Epoch 19 Batch 858 Loss 4.4091\n",
      "Epoch 19 Loss 4.4885\n",
      "Time taken for 1 epoch: 175.1970248222351 secs\n",
      "\n",
      "Epoch 20 Batch 0 Loss 3.7939\n",
      "Epoch 20 Batch 429 Loss 4.2492\n",
      "Epoch 20 Batch 858 Loss 4.3585\n",
      "Saving checkpoint for epoch 20 at checkpoints\\ckpt-4\n",
      "Epoch 20 Loss 4.4446\n",
      "Time taken for 1 epoch: 175.79089426994324 secs\n",
      "\n",
      "Epoch 21 Batch 0 Loss 4.1091\n",
      "Epoch 21 Batch 429 Loss 4.1853\n",
      "Epoch 21 Batch 858 Loss 4.3129\n",
      "Epoch 21 Loss 4.4036\n",
      "Time taken for 1 epoch: 174.65212678909302 secs\n",
      "\n",
      "Epoch 22 Batch 0 Loss 3.5612\n",
      "Epoch 22 Batch 429 Loss 4.1621\n",
      "Epoch 22 Batch 858 Loss 4.2791\n",
      "Epoch 22 Loss 4.3663\n",
      "Time taken for 1 epoch: 174.51134371757507 secs\n",
      "\n",
      "Epoch 23 Batch 0 Loss 4.4027\n",
      "Epoch 23 Batch 429 Loss 4.1167\n",
      "Epoch 23 Batch 858 Loss 4.2416\n",
      "Epoch 23 Loss 4.3295\n",
      "Time taken for 1 epoch: 174.99176216125488 secs\n",
      "\n",
      "Epoch 24 Batch 0 Loss 3.9103\n",
      "Epoch 24 Batch 429 Loss 4.0741\n",
      "Epoch 24 Batch 858 Loss 4.2042\n",
      "Epoch 24 Loss 4.2923\n",
      "Time taken for 1 epoch: 184.7098524570465 secs\n",
      "\n",
      "Epoch 25 Batch 0 Loss 4.0695\n",
      "Epoch 25 Batch 429 Loss 4.0640\n",
      "Epoch 25 Batch 858 Loss 4.1890\n",
      "Saving checkpoint for epoch 25 at checkpoints\\ckpt-5\n",
      "Epoch 25 Loss 4.2630\n",
      "Time taken for 1 epoch: 185.35453391075134 secs\n",
      "\n",
      "Epoch 26 Batch 0 Loss 4.0509\n",
      "Epoch 26 Batch 429 Loss 4.0081\n",
      "Epoch 26 Batch 858 Loss 4.1346\n",
      "Epoch 26 Loss 4.2299\n",
      "Time taken for 1 epoch: 171.38398337364197 secs\n",
      "\n",
      "Epoch 27 Batch 0 Loss 3.6412\n",
      "Epoch 27 Batch 429 Loss 3.9901\n",
      "Epoch 27 Batch 858 Loss 4.1171\n",
      "Epoch 27 Loss 4.2028\n",
      "Time taken for 1 epoch: 168.42593359947205 secs\n",
      "\n",
      "Epoch 28 Batch 0 Loss 4.0313\n",
      "Epoch 28 Batch 429 Loss 3.9569\n",
      "Epoch 28 Batch 858 Loss 4.0839\n",
      "Epoch 28 Loss 4.1764\n",
      "Time taken for 1 epoch: 168.26643323898315 secs\n",
      "\n",
      "Epoch 29 Batch 0 Loss 3.8332\n",
      "Epoch 29 Batch 429 Loss 3.9336\n",
      "Epoch 29 Batch 858 Loss 4.0499\n",
      "Epoch 29 Loss 4.1453\n",
      "Time taken for 1 epoch: 169.37164497375488 secs\n",
      "\n",
      "Epoch 30 Batch 0 Loss 3.7518\n",
      "Epoch 30 Batch 429 Loss 3.8998\n",
      "Epoch 30 Batch 858 Loss 4.0246\n",
      "Saving checkpoint for epoch 30 at checkpoints\\ckpt-6\n",
      "Epoch 30 Loss 4.1201\n",
      "Time taken for 1 epoch: 168.6941738128662 secs\n",
      "\n",
      "Epoch 31 Batch 0 Loss 3.7715\n",
      "Epoch 31 Batch 429 Loss 3.8806\n",
      "Epoch 31 Batch 858 Loss 4.0030\n",
      "Epoch 31 Loss 4.0913\n",
      "Time taken for 1 epoch: 167.47118163108826 secs\n",
      "\n",
      "Epoch 32 Batch 0 Loss 3.5910\n",
      "Epoch 32 Batch 429 Loss 3.8442\n",
      "Epoch 32 Batch 858 Loss 3.9740\n",
      "Epoch 32 Loss 4.0702\n",
      "Time taken for 1 epoch: 166.71031069755554 secs\n",
      "\n",
      "Epoch 33 Batch 0 Loss 3.7495\n",
      "Epoch 33 Batch 429 Loss 3.8576\n",
      "Epoch 33 Batch 858 Loss 3.9578\n",
      "Epoch 33 Loss 4.0454\n",
      "Time taken for 1 epoch: 166.40939021110535 secs\n",
      "\n",
      "Epoch 34 Batch 0 Loss 3.5291\n",
      "Epoch 34 Batch 429 Loss 3.8139\n",
      "Epoch 34 Batch 858 Loss 3.9298\n",
      "Epoch 34 Loss 4.0241\n",
      "Time taken for 1 epoch: 166.75570273399353 secs\n",
      "\n",
      "Epoch 35 Batch 0 Loss 3.0183\n",
      "Epoch 35 Batch 429 Loss 3.7844\n",
      "Epoch 35 Batch 858 Loss 3.9106\n",
      "Saving checkpoint for epoch 35 at checkpoints\\ckpt-7\n",
      "Epoch 35 Loss 4.0031\n",
      "Time taken for 1 epoch: 168.89463472366333 secs\n",
      "\n",
      "Epoch 36 Batch 0 Loss 3.5141\n",
      "Epoch 36 Batch 429 Loss 3.7542\n",
      "Epoch 36 Batch 858 Loss 3.8968\n",
      "Epoch 36 Loss 3.9838\n",
      "Time taken for 1 epoch: 167.70160055160522 secs\n",
      "\n",
      "Epoch 37 Batch 0 Loss 3.7652\n",
      "Epoch 37 Batch 429 Loss 3.7378\n",
      "Epoch 37 Batch 858 Loss 3.8645\n",
      "Epoch 37 Loss 3.9610\n",
      "Time taken for 1 epoch: 167.41204285621643 secs\n",
      "\n",
      "Epoch 38 Batch 0 Loss 3.4153\n",
      "Epoch 38 Batch 429 Loss 3.7258\n",
      "Epoch 38 Batch 858 Loss 3.8421\n",
      "Epoch 38 Loss 3.9435\n",
      "Time taken for 1 epoch: 167.13021802902222 secs\n",
      "\n",
      "Epoch 39 Batch 0 Loss 3.2741\n",
      "Epoch 39 Batch 429 Loss 3.6986\n",
      "Epoch 39 Batch 858 Loss 3.8333\n",
      "Epoch 39 Loss 3.9239\n",
      "Time taken for 1 epoch: 168.1245560646057 secs\n",
      "\n",
      "Epoch 40 Batch 0 Loss 3.5935\n",
      "Epoch 40 Batch 429 Loss 3.6893\n",
      "Epoch 40 Batch 858 Loss 3.8166\n",
      "Saving checkpoint for epoch 40 at checkpoints\\ckpt-8\n",
      "Epoch 40 Loss 3.9052\n",
      "Time taken for 1 epoch: 168.45179772377014 secs\n",
      "\n",
      "Epoch 41 Batch 0 Loss 3.3781\n",
      "Epoch 41 Batch 429 Loss 3.6797\n",
      "Epoch 41 Batch 858 Loss 3.7982\n",
      "Epoch 41 Loss 3.8896\n",
      "Time taken for 1 epoch: 167.23592734336853 secs\n",
      "\n",
      "Epoch 42 Batch 0 Loss 3.2907\n",
      "Epoch 42 Batch 429 Loss 3.6455\n",
      "Epoch 42 Batch 858 Loss 3.7791\n",
      "Epoch 42 Loss 3.8719\n",
      "Time taken for 1 epoch: 166.90601301193237 secs\n",
      "\n",
      "Epoch 43 Batch 0 Loss 3.4181\n",
      "Epoch 43 Batch 429 Loss 3.6483\n",
      "Epoch 43 Batch 858 Loss 3.7635\n",
      "Epoch 43 Loss 3.8546\n",
      "Time taken for 1 epoch: 167.4060881137848 secs\n",
      "\n",
      "Epoch 44 Batch 0 Loss 3.6935\n",
      "Epoch 44 Batch 429 Loss 3.6167\n",
      "Epoch 44 Batch 858 Loss 3.7462\n",
      "Epoch 44 Loss 3.8450\n",
      "Time taken for 1 epoch: 167.877295255661 secs\n",
      "\n",
      "Epoch 45 Batch 0 Loss 3.4499\n",
      "Epoch 45 Batch 429 Loss 3.6028\n",
      "Epoch 45 Batch 858 Loss 3.7291\n",
      "Saving checkpoint for epoch 45 at checkpoints\\ckpt-9\n",
      "Epoch 45 Loss 3.8274\n",
      "Time taken for 1 epoch: 169.30084085464478 secs\n",
      "\n",
      "Epoch 46 Batch 0 Loss 3.5166\n",
      "Epoch 46 Batch 429 Loss 3.5744\n",
      "Epoch 46 Batch 858 Loss 3.7096\n",
      "Epoch 46 Loss 3.8095\n",
      "Time taken for 1 epoch: 167.4787471294403 secs\n",
      "\n",
      "Epoch 47 Batch 0 Loss 3.5012\n",
      "Epoch 47 Batch 429 Loss 3.5856\n",
      "Epoch 47 Batch 858 Loss 3.7062\n",
      "Epoch 47 Loss 3.7957\n",
      "Time taken for 1 epoch: 166.90326404571533 secs\n",
      "\n",
      "Epoch 48 Batch 0 Loss 3.2166\n",
      "Epoch 48 Batch 429 Loss 3.5430\n",
      "Epoch 48 Batch 858 Loss 3.6916\n",
      "Epoch 48 Loss 3.7830\n",
      "Time taken for 1 epoch: 167.04971551895142 secs\n",
      "\n",
      "Epoch 49 Batch 0 Loss 3.4534\n",
      "Epoch 49 Batch 429 Loss 3.5407\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 49 Batch 858 Loss 3.6656\n",
      "Epoch 49 Loss 3.7723\n",
      "Time taken for 1 epoch: 167.59003138542175 secs\n",
      "\n",
      "Epoch 50 Batch 0 Loss 3.4673\n",
      "Epoch 50 Batch 429 Loss 3.5264\n",
      "Epoch 50 Batch 858 Loss 3.6600\n",
      "Saving checkpoint for epoch 50 at checkpoints\\ckpt-10\n",
      "Epoch 50 Loss 3.7555\n",
      "Time taken for 1 epoch: 169.83916211128235 secs\n",
      "\n"
     ]
    }
   ],
   "source": [
    "import time\n",
    "for epoch in range(EPOCHS):\n",
    "    start = time.time()\n",
    "\n",
    "    train_loss.reset_states()\n",
    "  \n",
    "    for (batch, (inp, tar)) in enumerate(dataset):\n",
    "        train_step(inp, tar)\n",
    "    \n",
    "        # 55k samples\n",
    "        # we display 3 batch results -- 0th, middle and last one (approx)\n",
    "        # 55k / 64 ~ 858; 858 / 2 = 429\n",
    "        if batch % 429 == 0:\n",
    "            print ('Epoch {} Batch {} Loss {:.4f}'.format(epoch + 1, batch, train_loss.result()))\n",
    "      \n",
    "    if (epoch + 1) % 5 == 0:\n",
    "        ckpt_save_path = ckpt_manager.save()\n",
    "        print ('Saving checkpoint for epoch {} at {}'.format(epoch+1, ckpt_save_path))\n",
    "    \n",
    "    print ('Epoch {} Loss {:.4f}'.format(epoch + 1, train_loss.result()))\n",
    "\n",
    "    print ('Time taken for 1 epoch: {} secs\\n'.format(time.time() - start))\n"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "dc7446b8",
   "metadata": {
    "colab_type": "text",
    "id": "PVbEUCZagJ0G",
    "pycharm": {
     "name": "#%% md\n"
    }
   },
   "source": [
    "### Inference"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "4e46b57e",
   "metadata": {
    "colab_type": "text",
    "id": "YMbqGTixu1cl",
    "pycharm": {
     "name": "#%% md\n"
    }
   },
   "source": [
    "#### Predicting one word at a time at the decoder and appending it to the output; then taking the complete sequence as an input to the decoder and repeating until maxlen or stop keyword appears"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 43,
   "id": "e27a296c",
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "F5D5cv2Jd8-6",
    "pycharm": {
     "name": "#%%\n"
    }
   },
   "outputs": [],
   "source": [
    "def evaluate(input_document):\n",
    "    \n",
    "    input_document = document_tokenizer.convert_tokens_to_ids(input_document)\n",
    "    input_document = [input_document]\n",
    "    \n",
    "    input_document = tf.keras.preprocessing.sequence.pad_sequences(input_document, maxlen=encoder_maxlen, padding='post', truncating='post')\n",
    "    \n",
    "    encoder_input = tf.expand_dims(input_document[0], 0)\n",
    "    \n",
    "    decoder_input = [summary_tokenizer.convert_tokens_to_ids(\"[CLS]\")]\n",
    "    output = tf.expand_dims(decoder_input, 0)\n",
    "    \n",
    "    for i in range(decoder_maxlen):\n",
    "        enc_padding_mask, combined_mask, dec_padding_mask = create_masks(encoder_input, output)\n",
    "\n",
    "        predictions, attention_weights = transformer(\n",
    "            encoder_input, \n",
    "            output,\n",
    "            False,\n",
    "            enc_padding_mask,\n",
    "            combined_mask,\n",
    "            dec_padding_mask\n",
    "        )\n",
    "\n",
    "        predictions = predictions[: ,-1:, :]\n",
    "        predicted_id = tf.cast(tf.argmax(predictions, axis=-1), tf.int32)\n",
    "\n",
    "        if predicted_id == summary_tokenizer.convert_tokens_to_ids(\"[SEP]\"):\n",
    "            return tf.squeeze(output, axis=0), attention_weights\n",
    "\n",
    "        output = tf.concat([output, predicted_id], axis=-1)\n",
    "\n",
    "    return tf.squeeze(output, axis=0), attention_weights\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 44,
   "id": "cdefc461",
   "metadata": {
    "pycharm": {
     "name": "#%%\n"
    }
   },
   "outputs": [],
   "source": [
    "document_test_tokenizer, document_test = tokenized_text(direct,file = 'cleaned_test.src')\n",
    "# summary_test_tokenizer, summary = tokenized_text(direct,file = 'cleaned_test.tgt')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 45,
   "id": "83318ce0",
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "UkpdiW6wnmiS",
    "pycharm": {
     "name": "#%%\n"
    }
   },
   "outputs": [],
   "source": [
    "def summarize(input_document):\n",
    "    summarized = evaluate(input_document=input_document)[0].numpy()\n",
    "    summarized = summary_tokenizer.convert_ids_to_tokens(summarized)\n",
    "    summarized = summary_tokenizer.convert_tokens_to_string(summarized[1:])\n",
    "    return  summarized "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 46,
   "id": "a48989ca",
   "metadata": {
    "pycharm": {
     "name": "#%%\n"
    }
   },
   "outputs": [],
   "source": [
    "# save tokenizers models for loading them in inference\n",
    "d = {\n",
    "    \"document_tokenizer\": document_tokenizer,\n",
    "    \"summary_tokenizer\": summary_tokenizer\n",
    "}\n",
    "import pickle\n",
    "with open(\"objects.pickle\",\"wb\") as obj:\n",
    "    pickle.dump(d, obj, protocol=pickle.HIGHEST_PROTOCOL)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 47,
   "outputs": [],
   "source": [
    "fin = open(\"arabic\\\\cleaned_train.tgt\",\"r\",encoding=\"utf-8\")\n",
    "l = list()\n",
    "for line in fin.readlines():\n",
    "    l.append(line)\n",
    "test_summary = pd.DataFrame()\n",
    "test_summary[\"actual_summary\"] = l"
   ],
   "metadata": {
    "collapsed": false,
    "pycharm": {
     "name": "#%%\n"
    }
   }
  },
  {
   "cell_type": "code",
   "execution_count": 48,
   "outputs": [
    {
     "data": {
      "text/plain": "                                       actual_summary\n0   ل الحقيقة جربي أن تستخدمي طريقة الساندويتش كون...\n1   قه بالقرب من رفاقه لفه وفر له كل الاحتياجات ال...\n2   خل مسار الملف للتوجه إلى صفحة معينة على الموقع...\n3   عل قارئك يميز الشخصية الرئيسية كن مستعدا لحدوث...\n4   لم منع فقس البيض حدد مواقع الأعشاش استخدم زيت ...\n5   رف الفرق بين حزم اللغة وحزم واجهة اللغة  اذهب ...\n6   رف سرعتك الحالية ابحث عن مكان مناسب ضع جدولا ض...\n7   نبي الأطعمة المالحة والغنية بالصوديوم تناولي ا...\n8   ل تطبيقا لهاتفك المحمول اضبط التطبيق فعل المكا...\n9   تخدم الزراعة المائية للحصول على محصول أكثر بال...\n10   الماء في قدر ليغلي ضع حفنة من الملح في الماء ...\n11  ر مقابلة تجريبية مع مستشارك المهني أو أي شخص ت...\n12  دأ بالأماكن العادية من الجسد أولا مرر يدك على ...\n13   بزيارة الطبيب جرب العلاج النفسي اشترك في مجمو...\n14  تح شاشة ابدأ  افتح الإعدادات  انقر على خيار ال...\n15   تتشاجر ما لم تكن مضطرا إلى ذلك ابق عناصر حماي...\n16  تهدف موطن القراد المفضل قص النباتات قم بجز الح...\n17      جع واسألها كن مبتكرا واسألها بطريقة مريحة ل\\n\n18  رع العشب والشجيرات أضف الغطاء العضوي الواقي أو...\n19  ن الفرن على درجة حرارة  مئوية أضف ملعقة كبيرة ...\n20  د طريقة بدء زراعة البذور اختر أكثر منطقة مشمسة...\n21  خل أسطوانة دي في دي فارغة في جهاز الكمبيوتر ان...\n22   لنفسك عادات تمارسها قبل النوم اقرأ كتابا اطفئ...\n23   بغسل الملابس التي قمت بشرائها من متاجر الملاب...\n24  فظي على مرونة جلدك بتناول ما يكفي من البروتين ...\n25  هم سبب خطورة الانسحاب من الألبرازولام بدون إشر...\n26  ل خاصية العدسات لكي تقدر على الوصول إلى خيارات...\n27   أزرار المعطف عند الجلوس ارتد معطفا خارجيا إذا...\n28  تح تطبيق إيمو اضغط على تبويب  الموجود في أعلى ...\n29   ثلجا على منطقة الإصابة ابق هادئا تناول مسكنات...\n30  ر بحثك المسبق عن الشركة قبل المقابلة دون الإجا...\n31  حث واتصل بأندية تربية الكلاب تحدث إلى طبيب بيط...\n32  ديه هدايا قيمة اتركي له ملاحظات لطيفة اذهبا في...\n33   إحدى الأنصاف على وعاء مع مواجهة جانب البذور ب...\n34  رفي باحترافية أظهري شخصيتك استخدمي لغة الجسد ا...\n35   بإذابة البيض في الثلاجة طوال الليل استخدم الب...\n36   كمية صغيرة من بخاخ الشعر مباشرة على العلكة اك...\n37   مستعدا غض البصر ركز على نشاط أو هواية مفيدة ا...\n38  ركي زيت الأطفال حول خط الشعر قبل صبغ شعرك تجنب...\n39  تزم بقاعدة اللون الأبيض لممثل المايم أضف المكي...",
      "text/html": "<div>\n<style scoped>\n    .dataframe tbody tr th:only-of-type {\n        vertical-align: middle;\n    }\n\n    .dataframe tbody tr th {\n        vertical-align: top;\n    }\n\n    .dataframe thead th {\n        text-align: right;\n    }\n</style>\n<table border=\"1\" class=\"dataframe\">\n  <thead>\n    <tr style=\"text-align: right;\">\n      <th></th>\n      <th>actual_summary</th>\n    </tr>\n  </thead>\n  <tbody>\n    <tr>\n      <th>0</th>\n      <td>ل الحقيقة جربي أن تستخدمي طريقة الساندويتش كون...</td>\n    </tr>\n    <tr>\n      <th>1</th>\n      <td>قه بالقرب من رفاقه لفه وفر له كل الاحتياجات ال...</td>\n    </tr>\n    <tr>\n      <th>2</th>\n      <td>خل مسار الملف للتوجه إلى صفحة معينة على الموقع...</td>\n    </tr>\n    <tr>\n      <th>3</th>\n      <td>عل قارئك يميز الشخصية الرئيسية كن مستعدا لحدوث...</td>\n    </tr>\n    <tr>\n      <th>4</th>\n      <td>لم منع فقس البيض حدد مواقع الأعشاش استخدم زيت ...</td>\n    </tr>\n    <tr>\n      <th>5</th>\n      <td>رف الفرق بين حزم اللغة وحزم واجهة اللغة  اذهب ...</td>\n    </tr>\n    <tr>\n      <th>6</th>\n      <td>رف سرعتك الحالية ابحث عن مكان مناسب ضع جدولا ض...</td>\n    </tr>\n    <tr>\n      <th>7</th>\n      <td>نبي الأطعمة المالحة والغنية بالصوديوم تناولي ا...</td>\n    </tr>\n    <tr>\n      <th>8</th>\n      <td>ل تطبيقا لهاتفك المحمول اضبط التطبيق فعل المكا...</td>\n    </tr>\n    <tr>\n      <th>9</th>\n      <td>تخدم الزراعة المائية للحصول على محصول أكثر بال...</td>\n    </tr>\n    <tr>\n      <th>10</th>\n      <td>الماء في قدر ليغلي ضع حفنة من الملح في الماء ...</td>\n    </tr>\n    <tr>\n      <th>11</th>\n      <td>ر مقابلة تجريبية مع مستشارك المهني أو أي شخص ت...</td>\n    </tr>\n    <tr>\n      <th>12</th>\n      <td>دأ بالأماكن العادية من الجسد أولا مرر يدك على ...</td>\n    </tr>\n    <tr>\n      <th>13</th>\n      <td>بزيارة الطبيب جرب العلاج النفسي اشترك في مجمو...</td>\n    </tr>\n    <tr>\n      <th>14</th>\n      <td>تح شاشة ابدأ  افتح الإعدادات  انقر على خيار ال...</td>\n    </tr>\n    <tr>\n      <th>15</th>\n      <td>تتشاجر ما لم تكن مضطرا إلى ذلك ابق عناصر حماي...</td>\n    </tr>\n    <tr>\n      <th>16</th>\n      <td>تهدف موطن القراد المفضل قص النباتات قم بجز الح...</td>\n    </tr>\n    <tr>\n      <th>17</th>\n      <td>جع واسألها كن مبتكرا واسألها بطريقة مريحة ل\\n</td>\n    </tr>\n    <tr>\n      <th>18</th>\n      <td>رع العشب والشجيرات أضف الغطاء العضوي الواقي أو...</td>\n    </tr>\n    <tr>\n      <th>19</th>\n      <td>ن الفرن على درجة حرارة  مئوية أضف ملعقة كبيرة ...</td>\n    </tr>\n    <tr>\n      <th>20</th>\n      <td>د طريقة بدء زراعة البذور اختر أكثر منطقة مشمسة...</td>\n    </tr>\n    <tr>\n      <th>21</th>\n      <td>خل أسطوانة دي في دي فارغة في جهاز الكمبيوتر ان...</td>\n    </tr>\n    <tr>\n      <th>22</th>\n      <td>لنفسك عادات تمارسها قبل النوم اقرأ كتابا اطفئ...</td>\n    </tr>\n    <tr>\n      <th>23</th>\n      <td>بغسل الملابس التي قمت بشرائها من متاجر الملاب...</td>\n    </tr>\n    <tr>\n      <th>24</th>\n      <td>فظي على مرونة جلدك بتناول ما يكفي من البروتين ...</td>\n    </tr>\n    <tr>\n      <th>25</th>\n      <td>هم سبب خطورة الانسحاب من الألبرازولام بدون إشر...</td>\n    </tr>\n    <tr>\n      <th>26</th>\n      <td>ل خاصية العدسات لكي تقدر على الوصول إلى خيارات...</td>\n    </tr>\n    <tr>\n      <th>27</th>\n      <td>أزرار المعطف عند الجلوس ارتد معطفا خارجيا إذا...</td>\n    </tr>\n    <tr>\n      <th>28</th>\n      <td>تح تطبيق إيمو اضغط على تبويب  الموجود في أعلى ...</td>\n    </tr>\n    <tr>\n      <th>29</th>\n      <td>ثلجا على منطقة الإصابة ابق هادئا تناول مسكنات...</td>\n    </tr>\n    <tr>\n      <th>30</th>\n      <td>ر بحثك المسبق عن الشركة قبل المقابلة دون الإجا...</td>\n    </tr>\n    <tr>\n      <th>31</th>\n      <td>حث واتصل بأندية تربية الكلاب تحدث إلى طبيب بيط...</td>\n    </tr>\n    <tr>\n      <th>32</th>\n      <td>ديه هدايا قيمة اتركي له ملاحظات لطيفة اذهبا في...</td>\n    </tr>\n    <tr>\n      <th>33</th>\n      <td>إحدى الأنصاف على وعاء مع مواجهة جانب البذور ب...</td>\n    </tr>\n    <tr>\n      <th>34</th>\n      <td>رفي باحترافية أظهري شخصيتك استخدمي لغة الجسد ا...</td>\n    </tr>\n    <tr>\n      <th>35</th>\n      <td>بإذابة البيض في الثلاجة طوال الليل استخدم الب...</td>\n    </tr>\n    <tr>\n      <th>36</th>\n      <td>كمية صغيرة من بخاخ الشعر مباشرة على العلكة اك...</td>\n    </tr>\n    <tr>\n      <th>37</th>\n      <td>مستعدا غض البصر ركز على نشاط أو هواية مفيدة ا...</td>\n    </tr>\n    <tr>\n      <th>38</th>\n      <td>ركي زيت الأطفال حول خط الشعر قبل صبغ شعرك تجنب...</td>\n    </tr>\n    <tr>\n      <th>39</th>\n      <td>تزم بقاعدة اللون الأبيض لممثل المايم أضف المكي...</td>\n    </tr>\n  </tbody>\n</table>\n</div>"
     },
     "execution_count": 48,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "test_summary = test_summary.iloc[0:40]\n",
    "test_summary"
   ],
   "metadata": {
    "collapsed": false,
    "pycharm": {
     "name": "#%%\n"
    }
   }
  },
  {
   "cell_type": "code",
   "execution_count": 49,
   "id": "067f06ae",
   "metadata": {
    "pycharm": {
     "name": "#%%\n"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": "                                              content  \\\n0   عد الصدق هو الحل الافضل في مثل تلك الحالات لان...   \n1   خنازير الغي ##نية مخلوقات اجتماعية لذلك اذا كا...   \n2   حتا ##ج الى ادخال مسار الملف للتو ##جه الى صفح...   \n3   ب ان تجعل القاري مهتما بالشخص ##ية الرييسية وي...   \n4   مثل احدى الطرق الرحيمة لمنع تزايد اسراب الاوز ...   \n5   ترجم حزم اللغة العادية اغلبية عناصر واجهة المس...   \n6   ل ان تزيد من سرع ##تك عليك معرفة سرعة ركض ##ك ...   \n7   مك يحتاج الصو ##ديو ##م بكميات صغيرة ولكن استه...   \n8   كنك استخدام تطبيقات لاظهار الهاتف وكانه يتلقى ...   \n9   ني الزراعة المايية زراعة النباتات من دون تربة ...   \n10  لا قدرا كبيرا من الماء وضعه على الموق ##د على ...   \n11  جد في العديد من الجامعات حديثا منصب المستشار ا...   \n12  الافضل دايما ان تبدا الملا ##طفة بين الزوجين م...   \n13  ظننت انك تعاني من القلق الاجتماعي فقم بزيارة ا...   \n14  قر على شعار ويندوز الموجود اسفل الجهة اليسرى م...   \n15  دما يواجه ##ك خصم اكبر منك حجما فلا حرج من الت...   \n16  دة ما يوجد القرا ##د خارج المنزل في المناطق ال...   \n17  لما لم تقبل فتاة من قبل فقد تجد صعوبة في قراءة...   \n18  هل جرف التربة الجر ##داء بفعل الرياح والماء وه...   \n19  تختلف درجات حرارة الفرن حسب انواع الاف ##ران ل...   \n20  ا كنت تعيش في مناخ صيفي وشتاء خفيف سيكون من ال...   \n21  كد من قدرة جهازك على حرق اسط ##وانات دي في دي ...   \n22  م تهيية جسدك ليعرف موعد محدد للنوم عن طريق الت...   \n23  ا احضر ##ت ملابس من متجر ملابس مستعملة قم بغسل...   \n24  تخدم جسمك البروتين للتع ##افي وهذا يشمل تضرر ا...   \n25  برا ##زول ##ام او زان ##اكس يستخدم لعلاج اضطرا...   \n26  يح لك خاصية العدسات تاثيرات الوجه في سناب شات ...   \n27  سترا ##ت الرياضية متاحة بزر ##ين او بثلاثة ازر...   \n28  غط على ايقونة التطبيق التي تتمثل في كلمة بداخل...   \n29  ب ان تكون هذه هي الخطوة الاولى عند التعرض لهذه...   \n30  يك البحث المتع ##مق عن الشركة التي تتقدم لها ف...   \n31  دا بالبحث على شبكة الانترنت عن نوا ##دي تربية ...   \n32  تكتفي بشراء مشروبات ##ه المفضلة واصط ##حابه لت...   \n33  سك النصف من الجانب او ضعه على راحة يدك المفر #...   \n34  كنت محظوظة وحصلت على لقاء من وكيل اعمال فمن ال...   \n35  اء اكان البيض ني ##يا ام مط ##هي ##ا يفضل اذاب...   \n36  تي يجب ان تتصل ##ب بسبب بخاخ الشعر العلك ##ة ا...   \n37  ا تركت نفسك غير متاه ##ب للتعامل مع المشكلة وق...   \n38  ي ملعقة كبيرة من زيت الاطفال على يدك ثم اغ ##م...   \n39  م التعرف على مودي الماي ##م من خلال المكياج ال...   \n\n                                    tokenized_content  \\\n0   [عد, الصدق, هو, الحل, الافضل, في, مثل, تلك, ال...   \n1   [خنازير, الغي, ##نية, مخلوقات, اجتماعية, لذلك,...   \n2   [حتا, ##ج, الى, ادخال, مسار, الملف, للتو, ##جه...   \n3   [ب, ان, تجعل, القاري, مهتما, بالشخص, ##ية, الر...   \n4   [مثل, احدى, الطرق, الرحيمة, لمنع, تزايد, اسراب...   \n5   [ترجم, حزم, اللغة, العادية, اغلبية, عناصر, واج...   \n6   [ل, ان, تزيد, من, سرع, ##تك, عليك, معرفة, سرعة...   \n7   [مك, يحتاج, الصو, ##ديو, ##م, بكميات, صغيرة, و...   \n8   [كنك, استخدام, تطبيقات, لاظهار, الهاتف, وكانه,...   \n9   [ني, الزراعة, المايية, زراعة, النباتات, من, دو...   \n10  [لا, قدرا, كبيرا, من, الماء, وضعه, على, الموق,...   \n11  [جد, في, العديد, من, الجامعات, حديثا, منصب, ال...   \n12  [الافضل, دايما, ان, تبدا, الملا, ##طفة, بين, ا...   \n13  [ظننت, انك, تعاني, من, القلق, الاجتماعي, فقم, ...   \n14  [قر, على, شعار, ويندوز, الموجود, اسفل, الجهة, ...   \n15  [دما, يواجه, ##ك, خصم, اكبر, منك, حجما, فلا, ح...   \n16  [دة, ما, يوجد, القرا, ##د, خارج, المنزل, في, ا...   \n17  [لما, لم, تقبل, فتاة, من, قبل, فقد, تجد, صعوبة...   \n18  [هل, جرف, التربة, الجر, ##داء, بفعل, الرياح, و...   \n19  [تختلف, درجات, حرارة, الفرن, حسب, انواع, الاف,...   \n20  [ا, كنت, تعيش, في, مناخ, صيفي, وشتاء, خفيف, سي...   \n21  [كد, من, قدرة, جهازك, على, حرق, اسط, ##وانات, ...   \n22  [م, تهيية, جسدك, ليعرف, موعد, محدد, للنوم, عن,...   \n23  [ا, احضر, ##ت, ملابس, من, متجر, ملابس, مستعملة...   \n24  [تخدم, جسمك, البروتين, للتع, ##افي, وهذا, يشمل...   \n25  [برا, ##زول, ##ام, او, زان, ##اكس, يستخدم, لعل...   \n26  [يح, لك, خاصية, العدسات, تاثيرات, الوجه, في, س...   \n27  [سترا, ##ت, الرياضية, متاحة, بزر, ##ين, او, بث...   \n28  [غط, على, ايقونة, التطبيق, التي, تتمثل, في, كل...   \n29  [ب, ان, تكون, هذه, هي, الخطوة, الاولى, عند, ال...   \n30  [يك, البحث, المتع, ##مق, عن, الشركة, التي, تتق...   \n31  [دا, بالبحث, على, شبكة, الانترنت, عن, نوا, ##د...   \n32  [تكتفي, بشراء, مشروبات, ##ه, المفضلة, واصط, ##...   \n33  [سك, النصف, من, الجانب, او, ضعه, على, راحة, يد...   \n34  [كنت, محظوظة, وحصلت, على, لقاء, من, وكيل, اعما...   \n35  [اء, اكان, البيض, ني, ##يا, ام, مط, ##هي, ##ا,...   \n36  [تي, يجب, ان, تتصل, ##ب, بسبب, بخاخ, الشعر, ال...   \n37  [ا, تركت, نفسك, غير, متاه, ##ب, للتعامل, مع, ا...   \n38  [ي, ملعقة, كبيرة, من, زيت, الاطفال, على, يدك, ...   \n39  [م, التعرف, على, مودي, الماي, ##م, من, خلال, ا...   \n\n                                            token_ids  \n0   [2353, 8573, 2098, 5864, 5937, 1947, 2614, 410...  \n1   [48590, 6314, 2608, 41760, 24655, 5474, 2140, ...  \n2   [26640, 1020, 2127, 26048, 16962, 14436, 20360...  \n3   [118, 1946, 3188, 12053, 65318, 27900, 1988, 2...  \n4   [2614, 7575, 7398, 89222, 25372, 32084, 75081,...  \n5   [54463, 25100, 8368, 29642, 42803, 13797, 3412...  \n6   [140, 1946, 7685, 1939, 28225, 2035, 2373, 946...  \n7   [2452, 4574, 91402, 2769, 1017, 71505, 7441, 2...  \n8   [18110, 8281, 20797, 65289, 15790, 12389, 3173...  \n9   [4488, 28985, 36329, 20512, 43469, 1939, 2675,...  \n10  [1956, 22727, 9521, 1939, 5631, 18858, 1977, 2...  \n11  [3109, 1947, 14022, 1939, 14438, 15126, 19062,...  \n12  [5937, 2650, 1946, 6145, 5244, 39720, 2199, 26...  \n13  [16715, 2237, 13683, 1939, 11214, 8033, 67674,...  \n14  [2973, 1977, 8227, 40205, 15139, 21077, 26146,...  \n15  [5351, 13862, 1012, 10027, 2418, 2661, 88876, ...  \n16  [6113, 1962, 4359, 30023, 1024, 4934, 3436, 19...  \n17  [2273, 2134, 4445, 7756, 1939, 2331, 2911, 373...  \n18  [2572, 49731, 60887, 39509, 5101, 18704, 13453...  \n19  [12657, 6982, 16943, 12166, 4295, 5397, 3711, ...  \n20  [117, 2303, 4385, 1947, 66139, 29131, 68877, 1...  \n21  [10287, 1939, 11765, 26623, 1977, 10600, 6778,...  \n22  [141, 58600, 18262, 40291, 5481, 21972, 21960,...  \n23  [117, 15448, 1009, 12731, 1939, 16558, 12731, ...  \n24  [24608, 21273, 71387, 20213, 3410, 4223, 22254...  \n25  [3702, 11758, 1985, 2050, 18580, 18454, 11285,...  \n26  [2314, 2053, 33613, 75144, 77046, 6320, 1947, ...  \n27  [42191, 1009, 9039, 41673, 25446, 1943, 2050, ...  \n28  [31140, 1977, 37316, 11308, 2367, 57128, 1947,...  \n29  [118, 1946, 2594, 2413, 2215, 21820, 4013, 212...  \n30  [2727, 8466, 15577, 54811, 1986, 11646, 2367, ...  \n31  [2472, 37886, 1977, 9347, 10028, 1986, 5072, 1...  \n32  [21374, 40582, 52359, 1021, 15624, 13022, 3696...  \n33  [4289, 17424, 1939, 13764, 2050, 89740, 1977, ...  \n34  [2303, 59094, 62970, 1977, 4444, 1939, 8975, 4...  \n35  [53681, 42455, 14131, 4488, 1971, 2031, 2755, ...  \n36  [4779, 3790, 1946, 28468, 1016, 3463, 85638, 5...  \n37  [117, 11140, 2681, 2197, 86533, 1016, 32069, 2...  \n38  [146, 37296, 5290, 1939, 11939, 5339, 1977, 86...  \n39  [141, 32075, 1977, 34639, 19469, 1017, 1939, 3...  ",
      "text/html": "<div>\n<style scoped>\n    .dataframe tbody tr th:only-of-type {\n        vertical-align: middle;\n    }\n\n    .dataframe tbody tr th {\n        vertical-align: top;\n    }\n\n    .dataframe thead th {\n        text-align: right;\n    }\n</style>\n<table border=\"1\" class=\"dataframe\">\n  <thead>\n    <tr style=\"text-align: right;\">\n      <th></th>\n      <th>content</th>\n      <th>tokenized_content</th>\n      <th>token_ids</th>\n    </tr>\n  </thead>\n  <tbody>\n    <tr>\n      <th>0</th>\n      <td>عد الصدق هو الحل الافضل في مثل تلك الحالات لان...</td>\n      <td>[عد, الصدق, هو, الحل, الافضل, في, مثل, تلك, ال...</td>\n      <td>[2353, 8573, 2098, 5864, 5937, 1947, 2614, 410...</td>\n    </tr>\n    <tr>\n      <th>1</th>\n      <td>خنازير الغي ##نية مخلوقات اجتماعية لذلك اذا كا...</td>\n      <td>[خنازير, الغي, ##نية, مخلوقات, اجتماعية, لذلك,...</td>\n      <td>[48590, 6314, 2608, 41760, 24655, 5474, 2140, ...</td>\n    </tr>\n    <tr>\n      <th>2</th>\n      <td>حتا ##ج الى ادخال مسار الملف للتو ##جه الى صفح...</td>\n      <td>[حتا, ##ج, الى, ادخال, مسار, الملف, للتو, ##جه...</td>\n      <td>[26640, 1020, 2127, 26048, 16962, 14436, 20360...</td>\n    </tr>\n    <tr>\n      <th>3</th>\n      <td>ب ان تجعل القاري مهتما بالشخص ##ية الرييسية وي...</td>\n      <td>[ب, ان, تجعل, القاري, مهتما, بالشخص, ##ية, الر...</td>\n      <td>[118, 1946, 3188, 12053, 65318, 27900, 1988, 2...</td>\n    </tr>\n    <tr>\n      <th>4</th>\n      <td>مثل احدى الطرق الرحيمة لمنع تزايد اسراب الاوز ...</td>\n      <td>[مثل, احدى, الطرق, الرحيمة, لمنع, تزايد, اسراب...</td>\n      <td>[2614, 7575, 7398, 89222, 25372, 32084, 75081,...</td>\n    </tr>\n    <tr>\n      <th>5</th>\n      <td>ترجم حزم اللغة العادية اغلبية عناصر واجهة المس...</td>\n      <td>[ترجم, حزم, اللغة, العادية, اغلبية, عناصر, واج...</td>\n      <td>[54463, 25100, 8368, 29642, 42803, 13797, 3412...</td>\n    </tr>\n    <tr>\n      <th>6</th>\n      <td>ل ان تزيد من سرع ##تك عليك معرفة سرعة ركض ##ك ...</td>\n      <td>[ل, ان, تزيد, من, سرع, ##تك, عليك, معرفة, سرعة...</td>\n      <td>[140, 1946, 7685, 1939, 28225, 2035, 2373, 946...</td>\n    </tr>\n    <tr>\n      <th>7</th>\n      <td>مك يحتاج الصو ##ديو ##م بكميات صغيرة ولكن استه...</td>\n      <td>[مك, يحتاج, الصو, ##ديو, ##م, بكميات, صغيرة, و...</td>\n      <td>[2452, 4574, 91402, 2769, 1017, 71505, 7441, 2...</td>\n    </tr>\n    <tr>\n      <th>8</th>\n      <td>كنك استخدام تطبيقات لاظهار الهاتف وكانه يتلقى ...</td>\n      <td>[كنك, استخدام, تطبيقات, لاظهار, الهاتف, وكانه,...</td>\n      <td>[18110, 8281, 20797, 65289, 15790, 12389, 3173...</td>\n    </tr>\n    <tr>\n      <th>9</th>\n      <td>ني الزراعة المايية زراعة النباتات من دون تربة ...</td>\n      <td>[ني, الزراعة, المايية, زراعة, النباتات, من, دو...</td>\n      <td>[4488, 28985, 36329, 20512, 43469, 1939, 2675,...</td>\n    </tr>\n    <tr>\n      <th>10</th>\n      <td>لا قدرا كبيرا من الماء وضعه على الموق ##د على ...</td>\n      <td>[لا, قدرا, كبيرا, من, الماء, وضعه, على, الموق,...</td>\n      <td>[1956, 22727, 9521, 1939, 5631, 18858, 1977, 2...</td>\n    </tr>\n    <tr>\n      <th>11</th>\n      <td>جد في العديد من الجامعات حديثا منصب المستشار ا...</td>\n      <td>[جد, في, العديد, من, الجامعات, حديثا, منصب, ال...</td>\n      <td>[3109, 1947, 14022, 1939, 14438, 15126, 19062,...</td>\n    </tr>\n    <tr>\n      <th>12</th>\n      <td>الافضل دايما ان تبدا الملا ##طفة بين الزوجين م...</td>\n      <td>[الافضل, دايما, ان, تبدا, الملا, ##طفة, بين, ا...</td>\n      <td>[5937, 2650, 1946, 6145, 5244, 39720, 2199, 26...</td>\n    </tr>\n    <tr>\n      <th>13</th>\n      <td>ظننت انك تعاني من القلق الاجتماعي فقم بزيارة ا...</td>\n      <td>[ظننت, انك, تعاني, من, القلق, الاجتماعي, فقم, ...</td>\n      <td>[16715, 2237, 13683, 1939, 11214, 8033, 67674,...</td>\n    </tr>\n    <tr>\n      <th>14</th>\n      <td>قر على شعار ويندوز الموجود اسفل الجهة اليسرى م...</td>\n      <td>[قر, على, شعار, ويندوز, الموجود, اسفل, الجهة, ...</td>\n      <td>[2973, 1977, 8227, 40205, 15139, 21077, 26146,...</td>\n    </tr>\n    <tr>\n      <th>15</th>\n      <td>دما يواجه ##ك خصم اكبر منك حجما فلا حرج من الت...</td>\n      <td>[دما, يواجه, ##ك, خصم, اكبر, منك, حجما, فلا, ح...</td>\n      <td>[5351, 13862, 1012, 10027, 2418, 2661, 88876, ...</td>\n    </tr>\n    <tr>\n      <th>16</th>\n      <td>دة ما يوجد القرا ##د خارج المنزل في المناطق ال...</td>\n      <td>[دة, ما, يوجد, القرا, ##د, خارج, المنزل, في, ا...</td>\n      <td>[6113, 1962, 4359, 30023, 1024, 4934, 3436, 19...</td>\n    </tr>\n    <tr>\n      <th>17</th>\n      <td>لما لم تقبل فتاة من قبل فقد تجد صعوبة في قراءة...</td>\n      <td>[لما, لم, تقبل, فتاة, من, قبل, فقد, تجد, صعوبة...</td>\n      <td>[2273, 2134, 4445, 7756, 1939, 2331, 2911, 373...</td>\n    </tr>\n    <tr>\n      <th>18</th>\n      <td>هل جرف التربة الجر ##داء بفعل الرياح والماء وه...</td>\n      <td>[هل, جرف, التربة, الجر, ##داء, بفعل, الرياح, و...</td>\n      <td>[2572, 49731, 60887, 39509, 5101, 18704, 13453...</td>\n    </tr>\n    <tr>\n      <th>19</th>\n      <td>تختلف درجات حرارة الفرن حسب انواع الاف ##ران ل...</td>\n      <td>[تختلف, درجات, حرارة, الفرن, حسب, انواع, الاف,...</td>\n      <td>[12657, 6982, 16943, 12166, 4295, 5397, 3711, ...</td>\n    </tr>\n    <tr>\n      <th>20</th>\n      <td>ا كنت تعيش في مناخ صيفي وشتاء خفيف سيكون من ال...</td>\n      <td>[ا, كنت, تعيش, في, مناخ, صيفي, وشتاء, خفيف, سي...</td>\n      <td>[117, 2303, 4385, 1947, 66139, 29131, 68877, 1...</td>\n    </tr>\n    <tr>\n      <th>21</th>\n      <td>كد من قدرة جهازك على حرق اسط ##وانات دي في دي ...</td>\n      <td>[كد, من, قدرة, جهازك, على, حرق, اسط, ##وانات, ...</td>\n      <td>[10287, 1939, 11765, 26623, 1977, 10600, 6778,...</td>\n    </tr>\n    <tr>\n      <th>22</th>\n      <td>م تهيية جسدك ليعرف موعد محدد للنوم عن طريق الت...</td>\n      <td>[م, تهيية, جسدك, ليعرف, موعد, محدد, للنوم, عن,...</td>\n      <td>[141, 58600, 18262, 40291, 5481, 21972, 21960,...</td>\n    </tr>\n    <tr>\n      <th>23</th>\n      <td>ا احضر ##ت ملابس من متجر ملابس مستعملة قم بغسل...</td>\n      <td>[ا, احضر, ##ت, ملابس, من, متجر, ملابس, مستعملة...</td>\n      <td>[117, 15448, 1009, 12731, 1939, 16558, 12731, ...</td>\n    </tr>\n    <tr>\n      <th>24</th>\n      <td>تخدم جسمك البروتين للتع ##افي وهذا يشمل تضرر ا...</td>\n      <td>[تخدم, جسمك, البروتين, للتع, ##افي, وهذا, يشمل...</td>\n      <td>[24608, 21273, 71387, 20213, 3410, 4223, 22254...</td>\n    </tr>\n    <tr>\n      <th>25</th>\n      <td>برا ##زول ##ام او زان ##اكس يستخدم لعلاج اضطرا...</td>\n      <td>[برا, ##زول, ##ام, او, زان, ##اكس, يستخدم, لعل...</td>\n      <td>[3702, 11758, 1985, 2050, 18580, 18454, 11285,...</td>\n    </tr>\n    <tr>\n      <th>26</th>\n      <td>يح لك خاصية العدسات تاثيرات الوجه في سناب شات ...</td>\n      <td>[يح, لك, خاصية, العدسات, تاثيرات, الوجه, في, س...</td>\n      <td>[2314, 2053, 33613, 75144, 77046, 6320, 1947, ...</td>\n    </tr>\n    <tr>\n      <th>27</th>\n      <td>سترا ##ت الرياضية متاحة بزر ##ين او بثلاثة ازر...</td>\n      <td>[سترا, ##ت, الرياضية, متاحة, بزر, ##ين, او, بث...</td>\n      <td>[42191, 1009, 9039, 41673, 25446, 1943, 2050, ...</td>\n    </tr>\n    <tr>\n      <th>28</th>\n      <td>غط على ايقونة التطبيق التي تتمثل في كلمة بداخل...</td>\n      <td>[غط, على, ايقونة, التطبيق, التي, تتمثل, في, كل...</td>\n      <td>[31140, 1977, 37316, 11308, 2367, 57128, 1947,...</td>\n    </tr>\n    <tr>\n      <th>29</th>\n      <td>ب ان تكون هذه هي الخطوة الاولى عند التعرض لهذه...</td>\n      <td>[ب, ان, تكون, هذه, هي, الخطوة, الاولى, عند, ال...</td>\n      <td>[118, 1946, 2594, 2413, 2215, 21820, 4013, 212...</td>\n    </tr>\n    <tr>\n      <th>30</th>\n      <td>يك البحث المتع ##مق عن الشركة التي تتقدم لها ف...</td>\n      <td>[يك, البحث, المتع, ##مق, عن, الشركة, التي, تتق...</td>\n      <td>[2727, 8466, 15577, 54811, 1986, 11646, 2367, ...</td>\n    </tr>\n    <tr>\n      <th>31</th>\n      <td>دا بالبحث على شبكة الانترنت عن نوا ##دي تربية ...</td>\n      <td>[دا, بالبحث, على, شبكة, الانترنت, عن, نوا, ##د...</td>\n      <td>[2472, 37886, 1977, 9347, 10028, 1986, 5072, 1...</td>\n    </tr>\n    <tr>\n      <th>32</th>\n      <td>تكتفي بشراء مشروبات ##ه المفضلة واصط ##حابه لت...</td>\n      <td>[تكتفي, بشراء, مشروبات, ##ه, المفضلة, واصط, ##...</td>\n      <td>[21374, 40582, 52359, 1021, 15624, 13022, 3696...</td>\n    </tr>\n    <tr>\n      <th>33</th>\n      <td>سك النصف من الجانب او ضعه على راحة يدك المفر #...</td>\n      <td>[سك, النصف, من, الجانب, او, ضعه, على, راحة, يد...</td>\n      <td>[4289, 17424, 1939, 13764, 2050, 89740, 1977, ...</td>\n    </tr>\n    <tr>\n      <th>34</th>\n      <td>كنت محظوظة وحصلت على لقاء من وكيل اعمال فمن ال...</td>\n      <td>[كنت, محظوظة, وحصلت, على, لقاء, من, وكيل, اعما...</td>\n      <td>[2303, 59094, 62970, 1977, 4444, 1939, 8975, 4...</td>\n    </tr>\n    <tr>\n      <th>35</th>\n      <td>اء اكان البيض ني ##يا ام مط ##هي ##ا يفضل اذاب...</td>\n      <td>[اء, اكان, البيض, ني, ##يا, ام, مط, ##هي, ##ا,...</td>\n      <td>[53681, 42455, 14131, 4488, 1971, 2031, 2755, ...</td>\n    </tr>\n    <tr>\n      <th>36</th>\n      <td>تي يجب ان تتصل ##ب بسبب بخاخ الشعر العلك ##ة ا...</td>\n      <td>[تي, يجب, ان, تتصل, ##ب, بسبب, بخاخ, الشعر, ال...</td>\n      <td>[4779, 3790, 1946, 28468, 1016, 3463, 85638, 5...</td>\n    </tr>\n    <tr>\n      <th>37</th>\n      <td>ا تركت نفسك غير متاه ##ب للتعامل مع المشكلة وق...</td>\n      <td>[ا, تركت, نفسك, غير, متاه, ##ب, للتعامل, مع, ا...</td>\n      <td>[117, 11140, 2681, 2197, 86533, 1016, 32069, 2...</td>\n    </tr>\n    <tr>\n      <th>38</th>\n      <td>ي ملعقة كبيرة من زيت الاطفال على يدك ثم اغ ##م...</td>\n      <td>[ي, ملعقة, كبيرة, من, زيت, الاطفال, على, يدك, ...</td>\n      <td>[146, 37296, 5290, 1939, 11939, 5339, 1977, 86...</td>\n    </tr>\n    <tr>\n      <th>39</th>\n      <td>م التعرف على مودي الماي ##م من خلال المكياج ال...</td>\n      <td>[م, التعرف, على, مودي, الماي, ##م, من, خلال, ا...</td>\n      <td>[141, 32075, 1977, 34639, 19469, 1017, 1939, 3...</td>\n    </tr>\n  </tbody>\n</table>\n</div>"
     },
     "execution_count": 49,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "document_test= document.iloc[0:40]\n",
    "document_test"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 50,
   "id": "4453e321",
   "metadata": {
    "pycharm": {
     "name": "#%%\n"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "842.113249540329\n"
     ]
    }
   ],
   "source": [
    "import time\n",
    "t=time.time()\n",
    "test_df = pd.DataFrame()\n",
    "test_df[\"tokenized_content\"] = document_test[\"tokenized_content\"]\n",
    "test_df[\"tokenized_content\"] = test_df[\"tokenized_content\"].apply(lambda row: summarize(row) )\n",
    "print(time.time()-t)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 90,
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Collecting rouge-score\n",
      "  Downloading rouge_score-0.1.2.tar.gz (17 kB)\n",
      "  Preparing metadata (setup.py): started\n",
      "  Preparing metadata (setup.py): finished with status 'done'\n",
      "Requirement already satisfied: absl-py in c:\\users\\admin\\appdata\\local\\programs\\python\\python310\\lib\\site-packages (from rouge-score) (1.1.0)\n",
      "Requirement already satisfied: nltk in c:\\users\\admin\\appdata\\local\\programs\\python\\python310\\lib\\site-packages (from rouge-score) (3.7)\n",
      "Requirement already satisfied: numpy in c:\\users\\admin\\appdata\\local\\programs\\python\\python310\\lib\\site-packages (from rouge-score) (1.23.0)\n",
      "Requirement already satisfied: six>=1.14.0 in c:\\users\\admin\\appdata\\local\\programs\\python\\python310\\lib\\site-packages (from rouge-score) (1.16.0)\n",
      "Requirement already satisfied: regex>=2021.8.3 in c:\\users\\admin\\appdata\\local\\programs\\python\\python310\\lib\\site-packages (from nltk->rouge-score) (2022.7.25)\n",
      "Requirement already satisfied: click in c:\\users\\admin\\appdata\\local\\programs\\python\\python310\\lib\\site-packages (from nltk->rouge-score) (8.1.3)\n",
      "Requirement already satisfied: joblib in c:\\users\\admin\\appdata\\local\\programs\\python\\python310\\lib\\site-packages (from nltk->rouge-score) (1.1.0)\n",
      "Requirement already satisfied: tqdm in c:\\users\\admin\\appdata\\local\\programs\\python\\python310\\lib\\site-packages (from nltk->rouge-score) (4.64.0)\n",
      "Requirement already satisfied: colorama in c:\\users\\admin\\appdata\\local\\programs\\python\\python310\\lib\\site-packages (from click->nltk->rouge-score) (0.4.5)\n",
      "Building wheels for collected packages: rouge-score\n",
      "  Building wheel for rouge-score (setup.py): started\n",
      "  Building wheel for rouge-score (setup.py): finished with status 'done'\n",
      "  Created wheel for rouge-score: filename=rouge_score-0.1.2-py3-none-any.whl size=24955 sha256=4442f885f50f2694657b221a15831005308c574efc5ba04981a7b4eff0f2b583\n",
      "  Stored in directory: c:\\users\\admin\\appdata\\local\\pip\\cache\\wheels\\5f\\dd\\89\\461065a73be61a532ff8599a28e9beef17985c9e9c31e541b4\n",
      "Successfully built rouge-score\n",
      "Installing collected packages: rouge-score\n",
      "Successfully installed rouge-score-0.1.2\n"
     ]
    }
   ],
   "source": [
    "!pip install rouge-score"
   ],
   "metadata": {
    "collapsed": false,
    "pycharm": {
     "name": "#%%\n"
    }
   }
  },
  {
   "cell_type": "code",
   "execution_count": 51,
   "id": "e4af6919",
   "metadata": {
    "pycharm": {
     "name": "#%%\n"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0.0\n"
     ]
    }
   ],
   "source": [
    "from rouge_score import rouge_scorer\n",
    "scorer = rouge_scorer.RougeScorer(['rouge1'])\n",
    "m=0\n",
    "for i in range(40):\n",
    "    rog=scorer.score(test_df['tokenized_content'][i],test_summary[\"actual_summary\"].iloc[0:40][i],)['rouge1'].fmeasure\n",
    "    m+=rog\n",
    "print(m)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "22bbe552",
   "metadata": {
    "pycharm": {
     "name": "#%%\n"
    }
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "87630727",
   "metadata": {
    "pycharm": {
     "name": "#%%\n"
    }
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "3eb2326f",
   "metadata": {
    "pycharm": {
     "name": "#%%\n"
    }
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "675ed132",
   "metadata": {
    "pycharm": {
     "name": "#%%\n"
    }
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "76bee45b",
   "metadata": {
    "pycharm": {
     "name": "#%%\n"
    }
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "fa11331a",
   "metadata": {
    "pycharm": {
     "name": "#%%\n"
    }
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "d8fb1003",
   "metadata": {
    "pycharm": {
     "name": "#%%\n"
    }
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "686ceaea",
   "metadata": {
    "pycharm": {
     "name": "#%%\n"
    }
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "97625f35",
   "metadata": {
    "pycharm": {
     "name": "#%%\n"
    }
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "39358845",
   "metadata": {
    "pycharm": {
     "name": "#%%\n"
    }
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "a34f44b7",
   "metadata": {
    "pycharm": {
     "name": "#%%\n"
    }
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "d9add8c6",
   "metadata": {
    "pycharm": {
     "name": "#%%\n"
    }
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "eeda7cc3",
   "metadata": {
    "pycharm": {
     "name": "#%%\n"
    }
   },
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.9.7"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}