tashkeel_tf2.ipynb

{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "colab": {
      "name": "arabic tashkeel tf2",
      "provenance": [],
      "collapsed_sections": [],
      "authorship_tag": "ABX9TyPbXxZ5YF5rBjkkCXauCjd8",
      "include_colab_link": true
    },
    "kernelspec": {
      "name": "python3",
      "display_name": "Python 3"
    },
    "accelerator": "GPU"
  },
  "cells": [
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "view-in-github",
        "colab_type": "text"
      },
      "source": [
        "<a href=\"https://colab.research.google.com/github/ojuba-org/arabic-ml-data/blob/master/tashkeel_tf2.ipynb\" target=\"_parent\"><img src=\"https://colab.research.google.com/assets/colab-badge.svg\" alt=\"Open In Colab\"/></a>"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "K7Nzhoae6oyN"
      },
      "source": [
        "Tensorflow documentation\n",
        "\n",
        "* [v2](https://www.tensorflow.org/tutorials/text/text_generation)\n",
        "* [v1](https://github.com/tensorflow/docs/blob/master/site/en/r1/tutorials/sequences/text_generation.ipynb)"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "6bB-XTwj9io8"
      },
      "source": [
        "%tensorflow_version 2.x"
      ],
      "execution_count": 1,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "jOeUIdNu_AU-"
      },
      "source": [
        "import sys\n",
        "import os\n",
        "import time\n",
        "import random"
      ],
      "execution_count": 2,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "60OfCkAv-mMZ"
      },
      "source": [
        "import re\n",
        "import unicodedata"
      ],
      "execution_count": 3,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "ApcMwBdi9sgg"
      },
      "source": [
        "import numpy as np"
      ],
      "execution_count": 4,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "33-0GY_z7RIr"
      },
      "source": [
        "import tensorflow as tf"
      ],
      "execution_count": 5,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "K = tf.keras\n",
        "KL = K.layers"
      ],
      "metadata": {
        "id": "HN-KMNSTTSRI"
      },
      "execution_count": 77,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "Bst7e-d8KIGS"
      },
      "source": [
        "#tf.enable_eager_execution()"
      ],
      "execution_count": 6,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "3vCqr4qy7STc"
      },
      "source": [
        "#import tensorflow.compat.v1 as tf"
      ],
      "execution_count": 7,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "vFID0M6u9xbf",
        "outputId": "8ce5164b-ea3a-4167-e5a7-f5d384aca6a9"
      },
      "source": [
        "print(\"tf version = {} and py version = {}\".format(tf.__version__, sys.version))"
      ],
      "execution_count": 8,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "tf version = 2.7.0 and py version = 3.7.12 (default, Jan 15 2022, 18:48:18) \n",
            "[GCC 7.5.0]\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "6XJ6ovEI-AED"
      },
      "source": [
        "assert sys.version_info.major == 3, 'please use python 3'\n",
        "assert tf.test.gpu_device_name()!='', 'no GPU, please enable GPU'"
      ],
      "execution_count": 9,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "TrUDr7WY_cuu"
      },
      "source": [
        "#from tensorflow.keras.layers.experimental import preprocessing\n"
      ],
      "execution_count": 10,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "HnXXpLEm5cCp"
      },
      "source": [
        "! curl -sSLO https://github.com/ojuba-org/arabic-ml-data/archive/refs/heads/master.zip"
      ],
      "execution_count": 40,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "JuBEml2pKA9U"
      },
      "source": [
        "! unzip -o -q master.zip \"arabic-ml-data-master/corpora/quran/*.txt\""
      ],
      "execution_count": 48,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "BUnBxW1qKflI",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "92c72abd-b9aa-491d-f1ee-6c90a2dc3ae5"
      },
      "source": [
        "! cat arabic-ml-data-master/corpora/quran/*.txt > data.txt; wc -l data.txt"
      ],
      "execution_count": 49,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "6236 data.txt\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "xEa_wjUSMcIR"
      },
      "source": [
        "lines=[l.strip() for l in open('data.txt', 'r') if l.strip()]\n",
        "s=set()\n",
        "for l in lines: s.update(l)"
      ],
      "execution_count": 50,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "z8Rt4cHw4cmQ"
      },
      "source": [
        "alpha=sorted(s)"
      ],
      "execution_count": 51,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "Q8aljYj77xyp"
      },
      "source": [
        "tashkeel_set = { ch for ch in alpha if unicodedata.category(ch)=='Mn' }"
      ],
      "execution_count": 52,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "7yfmBqVTOiD3"
      },
      "source": [
        "def tokenize_tashkeel(line):\n",
        "  offs=[ ix for ix, ch in enumerate(line) if ch not in tashkeel_set ]\n",
        "  offs.append(len(line))\n",
        "  return [ line[off: offs[ix+1]] for ix, off in enumerate(offs[:-1]) ]"
      ],
      "execution_count": 53,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "I9kbSBAjSGXc"
      },
      "source": [
        "(\n",
        "  VOC_PAD,\n",
        "  VOC_START,\n",
        "  VOC_UNKNOWN,\n",
        "  VOC_UNKNOWN_TASHKEEL, VOC_UNKNOWN_SHADDA,\n",
        "  VOC_UNKNOWN_LETTER, VOC_UNKNOWN_HAMZA, VOC_UNKNOWN_TEH_OR_HAA, VOC_UNKNOWN_MAQSORA,\n",
        "  VOC_FATHATAN, VOC_DAMMATAN, VOC_KASRATAN, \n",
        "  VOC_FATHA, VOC_DAMMA, VOC_KASRA,\n",
        "  VOC_SHADDA_FATHA, VOC_SHADDA_DAMMA, VOC_SHADDA_KASRA,\n",
        "  VOC_SUKUN,\n",
        "  VOC_STOP,\n",
        ") = range(20)"
      ],
      "execution_count": 54,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "loDjt3YCWf82"
      },
      "source": [
        "chars = [ ch for ch in alpha if unicodedata.category(ch)!='Mn']"
      ],
      "execution_count": 55,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "tKHNejyiW-lu"
      },
      "source": [
        "ch2id = { ch: ix+VOC_STOP+1 for ix, ch in enumerate(chars)}"
      ],
      "execution_count": 56,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "ArhRFFZrXnCR"
      },
      "source": [
        "ch2id.update({\n",
        "  \"\": VOC_START,\n",
        "  \"\\n\": VOC_STOP,\n",
        "  \"\\u064B\": VOC_FATHATAN, \"\\u064C\": VOC_DAMMATAN, \"\\u064D\": VOC_KASRATAN,\n",
        "  \"\\u064E\": VOC_FATHA, \"\\u064F\": VOC_DAMMA, \"\\u0650\": VOC_KASRA,\n",
        "  \"\\u0651\\u064E\": VOC_SHADDA_FATHA, \"\\u0651\\u064F\": VOC_SHADDA_DAMMA, \"\\u0651\\u0650\": VOC_SHADDA_KASRA,\n",
        "  \"\\u0651\": VOC_SUKUN,\n",
        "  \"\\u061f\": VOC_UNKNOWN,\n",
        "  \"\\u0640\\u061f\\u0640\": VOC_UNKNOWN_TASHKEEL,\n",
        "  \"\\u0651\\u061f\": VOC_UNKNOWN_SHADDA,\n",
        "})"
      ],
      "execution_count": 57,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "cmo6gshSYkii"
      },
      "source": [
        "id2ch = { v: k for k, v in ch2id.items() }"
      ],
      "execution_count": 58,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "id2ch2 = dict(id2ch)\n",
        "id2ch2.update({\n",
        "    VOC_START: \"\", \"\\n\": VOC_STOP, VOC_UNKNOWN: \"\", VOC_UNKNOWN_TASHKEEL: \"\", VOC_UNKNOWN_SHADDA: \"ّ\",\n",
        "})"
      ],
      "metadata": {
        "id": "JaVrtHZuZrEa"
      },
      "execution_count": 100,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "-inoKgGpbszJ"
      },
      "source": [
        "voc_size = max(ch2id.values())+1"
      ],
      "execution_count": 59,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "--8Fnu5VanXN"
      },
      "source": [
        "id2ch.update({ ix:'\\u061f' for ix in range(voc_size) if ix not in id2ch})"
      ],
      "execution_count": 60,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "id_repr = { globals()[i]: i for i in dir() if i.startswith('VOC_')}\n",
        "id_repr.update({i: unicodedata.name(id2ch[i]) for i in range(VOC_STOP+1, voc_size)})"
      ],
      "metadata": {
        "id": "vtTMHnZNsrw2"
      },
      "execution_count": 61,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "def tokenize_one_id(sub):\n",
        "  return (ch2id.get(sub[0], VOC_UNKNOWN),\n",
        "          ch2id.get(sub[1:] or \"\\u0640\\u061f\\u0640\", VOC_UNKNOWN_TASHKEEL),\n",
        "  )\n",
        "\n",
        "def tokenize_tashkeel_id(line):\n",
        "  return [(VOC_START, VOC_UNKNOWN_TASHKEEL)]+[tokenize_one_id(sub) for sub in tokenize_tashkeel(line.strip()+\"\\n\")]\n",
        "\n",
        "def tr_all_unknown(seq):\n",
        "  return [ (con_id, VOC_UNKNOWN_TASHKEEL) for con_id, vo_id in seq ]\n",
        "\n",
        "def pad_one(tuples, size):\n",
        "  l = len(tuples)\n",
        "  return (tuples + [(VOC_PAD, VOC_PAD)]*(size-l)) if l<size else tuples\n"
      ],
      "metadata": {
        "id": "1BwKPnqvQ5Zt"
      },
      "execution_count": 62,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "line=lines[0]\n",
        "print(line)\n",
        "print(len(line))\n",
        "ll = [ len(tokenize_tashkeel(l)) for l in lines ]\n",
        "print(max(ll))\n",
        "print(np.percentile(ll, (80, 90, 95, 99)))\n"
      ],
      "metadata": {
        "id": "Gs1so3aYRM05",
        "outputId": "0539232d-2ded-4a4e-b358-5a895b1c4154",
        "colab": {
          "base_uri": "https://localhost:8080/"
        }
      },
      "execution_count": 63,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "بِسْمِ اللَّهِ الرَّحْمَنِ الرَّحِيمِ\n",
            "37\n",
            "679\n",
            "[ 94.  125.5 154.  242. ]\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "# consonant, no tashkeel\n",
        "max_con = 200\n",
        "filtered = [ tokenize_tashkeel_id(l) for l in lines if len(tokenize_tashkeel_id(l))<max_con ]\n",
        "random.shuffle(filtered)\n",
        "\n",
        "tokens=tokenize_tashkeel(line)\n",
        "print(tokens)\n",
        "print([ len(token) for token in tokens])\n",
        "print(voc_size)"
      ],
      "metadata": {
        "id": "JEn8o6nPSQCS",
        "outputId": "85e9131e-555b-4b8e-a1a0-d7d613b653d0",
        "colab": {
          "base_uri": "https://localhost:8080/"
        }
      },
      "execution_count": 69,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "['بِ', 'سْ', 'مِ', ' ', 'ا', 'ل', 'لَّ', 'هِ', ' ', 'ا', 'ل', 'رَّ', 'حْ', 'مَ', 'نِ', ' ', 'ا', 'ل', 'رَّ', 'حِ', 'ي', 'مِ']\n",
            "[2, 2, 2, 1, 1, 1, 3, 2, 1, 1, 1, 3, 2, 2, 2, 1, 1, 1, 3, 2, 1, 2]\n",
            "57\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "for i, (con_ix, vol_id) in enumerate(tokenize_tashkeel_id(line)):\n",
        "  print(i, con_ix, id_repr[con_ix], vol_id, id_repr[vol_id])"
      ],
      "metadata": {
        "id": "U4CgSyNeTEGk",
        "outputId": "d79595ac-f909-44d4-aad2-069dc26d17f3",
        "colab": {
          "base_uri": "https://localhost:8080/"
        }
      },
      "execution_count": 70,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "0 1 VOC_START 3 VOC_UNKNOWN_TASHKEEL\n",
            "1 28 ARABIC LETTER BEH 14 VOC_KASRA\n",
            "2 39 ARABIC LETTER SEEN 3 VOC_UNKNOWN_TASHKEEL\n",
            "3 51 ARABIC LETTER MEEM 14 VOC_KASRA\n",
            "4 20 SPACE 3 VOC_UNKNOWN_TASHKEEL\n",
            "5 27 ARABIC LETTER ALEF 3 VOC_UNKNOWN_TASHKEEL\n",
            "6 50 ARABIC LETTER LAM 3 VOC_UNKNOWN_TASHKEEL\n",
            "7 50 ARABIC LETTER LAM 15 VOC_SHADDA_FATHA\n",
            "8 53 ARABIC LETTER HEH 14 VOC_KASRA\n",
            "9 20 SPACE 3 VOC_UNKNOWN_TASHKEEL\n",
            "10 27 ARABIC LETTER ALEF 3 VOC_UNKNOWN_TASHKEEL\n",
            "11 50 ARABIC LETTER LAM 3 VOC_UNKNOWN_TASHKEEL\n",
            "12 37 ARABIC LETTER REH 15 VOC_SHADDA_FATHA\n",
            "13 33 ARABIC LETTER HAH 3 VOC_UNKNOWN_TASHKEEL\n",
            "14 51 ARABIC LETTER MEEM 12 VOC_FATHA\n",
            "15 52 ARABIC LETTER NOON 14 VOC_KASRA\n",
            "16 20 SPACE 3 VOC_UNKNOWN_TASHKEEL\n",
            "17 27 ARABIC LETTER ALEF 3 VOC_UNKNOWN_TASHKEEL\n",
            "18 50 ARABIC LETTER LAM 3 VOC_UNKNOWN_TASHKEEL\n",
            "19 37 ARABIC LETTER REH 15 VOC_SHADDA_FATHA\n",
            "20 33 ARABIC LETTER HAH 14 VOC_KASRA\n",
            "21 56 ARABIC LETTER YEH 3 VOC_UNKNOWN_TASHKEEL\n",
            "22 51 ARABIC LETTER MEEM 14 VOC_KASRA\n",
            "23 19 VOC_STOP 3 VOC_UNKNOWN_TASHKEEL\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "for i, (con_ix, vol_id) in enumerate(tr_all_unknown(tokenize_tashkeel_id(line))):\n",
        "  print(i, con_ix, id_repr[con_ix], vol_id, id_repr[vol_id])"
      ],
      "metadata": {
        "id": "mxkRHuOgS6cW",
        "outputId": "c4c55fa5-ca23-4a01-9c5a-edd9cd49429e",
        "colab": {
          "base_uri": "https://localhost:8080/"
        }
      },
      "execution_count": 71,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "0 1 VOC_START 3 VOC_UNKNOWN_TASHKEEL\n",
            "1 28 ARABIC LETTER BEH 3 VOC_UNKNOWN_TASHKEEL\n",
            "2 39 ARABIC LETTER SEEN 3 VOC_UNKNOWN_TASHKEEL\n",
            "3 51 ARABIC LETTER MEEM 3 VOC_UNKNOWN_TASHKEEL\n",
            "4 20 SPACE 3 VOC_UNKNOWN_TASHKEEL\n",
            "5 27 ARABIC LETTER ALEF 3 VOC_UNKNOWN_TASHKEEL\n",
            "6 50 ARABIC LETTER LAM 3 VOC_UNKNOWN_TASHKEEL\n",
            "7 50 ARABIC LETTER LAM 3 VOC_UNKNOWN_TASHKEEL\n",
            "8 53 ARABIC LETTER HEH 3 VOC_UNKNOWN_TASHKEEL\n",
            "9 20 SPACE 3 VOC_UNKNOWN_TASHKEEL\n",
            "10 27 ARABIC LETTER ALEF 3 VOC_UNKNOWN_TASHKEEL\n",
            "11 50 ARABIC LETTER LAM 3 VOC_UNKNOWN_TASHKEEL\n",
            "12 37 ARABIC LETTER REH 3 VOC_UNKNOWN_TASHKEEL\n",
            "13 33 ARABIC LETTER HAH 3 VOC_UNKNOWN_TASHKEEL\n",
            "14 51 ARABIC LETTER MEEM 3 VOC_UNKNOWN_TASHKEEL\n",
            "15 52 ARABIC LETTER NOON 3 VOC_UNKNOWN_TASHKEEL\n",
            "16 20 SPACE 3 VOC_UNKNOWN_TASHKEEL\n",
            "17 27 ARABIC LETTER ALEF 3 VOC_UNKNOWN_TASHKEEL\n",
            "18 50 ARABIC LETTER LAM 3 VOC_UNKNOWN_TASHKEEL\n",
            "19 37 ARABIC LETTER REH 3 VOC_UNKNOWN_TASHKEEL\n",
            "20 33 ARABIC LETTER HAH 3 VOC_UNKNOWN_TASHKEEL\n",
            "21 56 ARABIC LETTER YEH 3 VOC_UNKNOWN_TASHKEEL\n",
            "22 51 ARABIC LETTER MEEM 3 VOC_UNKNOWN_TASHKEEL\n",
            "23 19 VOC_STOP 3 VOC_UNKNOWN_TASHKEEL\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "max_len = 2*max_con\n",
        "max_w_pad = max_len+1\n",
        "\n",
        "latent_dim = voc_size\n",
        "num_encoder_tokens = voc_size\n",
        "num_decoder_tokens = voc_size"
      ],
      "metadata": {
        "id": "QZxI_uiYTOBZ"
      },
      "execution_count": 72,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "input_output_a = [ (pad_one(tr_all_unknown(l), max_con), pad_one(l, max_con)) for l in filtered ]\n",
        "input_output_a = np.array(input_output_a).reshape(-1, 2, max_len)\n",
        "input_output_a = np.pad(input_output_a, ((0,0), (0,0), (0,1)))\n",
        "print(input_output_a.shape)\n"
      ],
      "metadata": {
        "id": "_hTTeeYQTtvO",
        "outputId": "71c861b7-50fa-41c1-92b1-3237197e37bd",
        "colab": {
          "base_uri": "https://localhost:8080/"
        }
      },
      "execution_count": 75,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "(6102, 2, 401)\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "encoder_input_data = K.utils.to_categorical(input_output_a[:,0,:], voc_size)\n",
        "print(encoder_input_data.shape)\n",
        "decoder_input_data_ = input_output_a[:,1,:]\n",
        "decoder_input_data = K.utils.to_categorical(decoder_input_data_, voc_size)\n",
        "print(decoder_input_data.shape)\n",
        "decoder_target_data = K.utils.to_categorical(np.pad(decoder_input_data_[:,1:], ((0,0), (0,1))), voc_size)\n",
        "print(decoder_target_data.shape)"
      ],
      "metadata": {
        "id": "riKikdIHT3lr",
        "outputId": "9c0e2be4-301c-4a9a-d635-fcf208d8263e",
        "colab": {
          "base_uri": "https://localhost:8080/"
        }
      },
      "execution_count": 78,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "(6102, 401, 57)\n",
            "(6102, 401, 57)\n",
            "(6102, 401, 57)\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "# Define an input sequence and process it.\n",
        "encoder_inputs = KL.Input(shape=(None, num_encoder_tokens))\n",
        "encoder = KL.LSTM(latent_dim, return_state=True)\n",
        "encoder_outputs, state_h, state_c = encoder(encoder_inputs)\n",
        "# We discard `encoder_outputs` and only keep the states.\n",
        "encoder_states = [state_h, state_c]\n",
        "\n",
        "# Set up the decoder, using `encoder_states` as initial state.\n",
        "decoder_inputs = KL.Input(shape=(None, num_decoder_tokens))\n",
        "# We set up our decoder to return full output sequences,\n",
        "# and to return internal states as well. We don't use the\n",
        "# return states in the training model, but we will use them in inference.\n",
        "decoder_lstm = KL.LSTM(latent_dim, return_sequences=True, return_state=True)\n",
        "decoder_outputs, _, _ = decoder_lstm(decoder_inputs,\n",
        "                                     initial_state=encoder_states)\n",
        "decoder_dense = KL.Dense(num_decoder_tokens, activation='softmax')\n",
        "decoder_outputs = decoder_dense(decoder_outputs)\n",
        "\n",
        "# Define the model that will turn\n",
        "# `encoder_input_data` & `decoder_input_data` into `decoder_target_data`\n",
        "model = K.Model([encoder_inputs, decoder_inputs], decoder_outputs)\n",
        "model.summary()"
      ],
      "metadata": {
        "id": "9jSY-T-sTYKl",
        "outputId": "1690ffb6-8c4e-4c03-e4f3-eaabf6f0547f",
        "colab": {
          "base_uri": "https://localhost:8080/"
        }
      },
      "execution_count": 79,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Model: \"model_1\"\n",
            "__________________________________________________________________________________________________\n",
            " Layer (type)                   Output Shape         Param #     Connected to                     \n",
            "==================================================================================================\n",
            " input_3 (InputLayer)           [(None, None, 57)]   0           []                               \n",
            "                                                                                                  \n",
            " input_4 (InputLayer)           [(None, None, 57)]   0           []                               \n",
            "                                                                                                  \n",
            " lstm (LSTM)                    [(None, 57),         26220       ['input_3[0][0]']                \n",
            "                                 (None, 57),                                                      \n",
            "                                 (None, 57)]                                                      \n",
            "                                                                                                  \n",
            " lstm_1 (LSTM)                  [(None, None, 57),   26220       ['input_4[0][0]',                \n",
            "                                 (None, 57),                      'lstm[0][1]',                   \n",
            "                                 (None, 57)]                      'lstm[0][2]']                   \n",
            "                                                                                                  \n",
            " dense_1 (Dense)                (None, None, 57)     3306        ['lstm_1[0][0]']                 \n",
            "                                                                                                  \n",
            "==================================================================================================\n",
            "Total params: 55,746\n",
            "Trainable params: 55,746\n",
            "Non-trainable params: 0\n",
            "__________________________________________________________________________________________________\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "model.compile(optimizer='rmsprop', loss='categorical_crossentropy')"
      ],
      "metadata": {
        "id": "NIiq2r4LULkz"
      },
      "execution_count": 80,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "weights_fn = 'lstm-weights.h5'\n",
        "if os.path.exists(weights_fn):\n",
        "    model.load_weights(weights_fn)"
      ],
      "metadata": {
        "id": "jHPP5dweURJ_"
      },
      "execution_count": 81,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "cb = K.callbacks.ModelCheckpoint(filepath=weights_fn, save_weights_only=True, verbose=0)"
      ],
      "metadata": {
        "id": "qSE-oOhsUwG8"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "batch_size = 512\n",
        "epochs = 30"
      ],
      "metadata": {
        "id": "1U2Yk0cyUXFT"
      },
      "execution_count": 92,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "source": [
        "# training part"
      ],
      "metadata": {
        "id": "DHPf_TT8U-KP"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "# epochs=epochs, steps_per_epoch=steps_per_epoch,\n",
        "model.fit(\n",
        "  [encoder_input_data, decoder_input_data],\n",
        "  decoder_target_data, batch_size=batch_size, epochs=epochs, validation_split=0.2,\n",
        "  callbacks=[cb]\n",
        ")"
      ],
      "metadata": {
        "id": "nLvAP5Y-UUXK",
        "outputId": "9b880514-2562-463d-bff9-eefb28925e64",
        "colab": {
          "base_uri": "https://localhost:8080/"
        }
      },
      "execution_count": 93,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Epoch 1/30\n",
            "10/10 [==============================] - 3s 318ms/step - loss: 0.3487 - val_loss: 0.3608\n",
            "Epoch 2/30\n",
            "10/10 [==============================] - 3s 266ms/step - loss: 0.3485 - val_loss: 0.3607\n",
            "Epoch 3/30\n",
            "10/10 [==============================] - 3s 264ms/step - loss: 0.3481 - val_loss: 0.3599\n",
            "Epoch 4/30\n",
            "10/10 [==============================] - 3s 268ms/step - loss: 0.3476 - val_loss: 0.3606\n",
            "Epoch 5/30\n",
            "10/10 [==============================] - 3s 268ms/step - loss: 0.3472 - val_loss: 0.3597\n",
            "Epoch 6/30\n",
            "10/10 [==============================] - 3s 264ms/step - loss: 0.3468 - val_loss: 0.3589\n",
            "Epoch 7/30\n",
            "10/10 [==============================] - 3s 259ms/step - loss: 0.3465 - val_loss: 0.3590\n",
            "Epoch 8/30\n",
            "10/10 [==============================] - 3s 266ms/step - loss: 0.3457 - val_loss: 0.3586\n",
            "Epoch 9/30\n",
            "10/10 [==============================] - 3s 266ms/step - loss: 0.3457 - val_loss: 0.3573\n",
            "Epoch 10/30\n",
            "10/10 [==============================] - 3s 263ms/step - loss: 0.3451 - val_loss: 0.3574\n",
            "Epoch 11/30\n",
            "10/10 [==============================] - 3s 264ms/step - loss: 0.3448 - val_loss: 0.3569\n",
            "Epoch 12/30\n",
            "10/10 [==============================] - 3s 270ms/step - loss: 0.3442 - val_loss: 0.3568\n",
            "Epoch 13/30\n",
            "10/10 [==============================] - 3s 267ms/step - loss: 0.3437 - val_loss: 0.3557\n",
            "Epoch 14/30\n",
            "10/10 [==============================] - 3s 262ms/step - loss: 0.3438 - val_loss: 0.3550\n",
            "Epoch 15/30\n",
            "10/10 [==============================] - 3s 270ms/step - loss: 0.3433 - val_loss: 0.3554\n",
            "Epoch 16/30\n",
            "10/10 [==============================] - 3s 266ms/step - loss: 0.3425 - val_loss: 0.3548\n",
            "Epoch 17/30\n",
            "10/10 [==============================] - 3s 268ms/step - loss: 0.3425 - val_loss: 0.3539\n",
            "Epoch 18/30\n",
            "10/10 [==============================] - 3s 264ms/step - loss: 0.3418 - val_loss: 0.3542\n",
            "Epoch 19/30\n",
            "10/10 [==============================] - 3s 265ms/step - loss: 0.3416 - val_loss: 0.3535\n",
            "Epoch 20/30\n",
            "10/10 [==============================] - 3s 263ms/step - loss: 0.3411 - val_loss: 0.3540\n",
            "Epoch 21/30\n",
            "10/10 [==============================] - 3s 267ms/step - loss: 0.3412 - val_loss: 0.3529\n",
            "Epoch 22/30\n",
            "10/10 [==============================] - 3s 265ms/step - loss: 0.3404 - val_loss: 0.3519\n",
            "Epoch 23/30\n",
            "10/10 [==============================] - 3s 262ms/step - loss: 0.3401 - val_loss: 0.3530\n",
            "Epoch 24/30\n",
            "10/10 [==============================] - 3s 269ms/step - loss: 0.3398 - val_loss: 0.3524\n",
            "Epoch 25/30\n",
            "10/10 [==============================] - 3s 269ms/step - loss: 0.3393 - val_loss: 0.3518\n",
            "Epoch 26/30\n",
            "10/10 [==============================] - 3s 267ms/step - loss: 0.3391 - val_loss: 0.3506\n",
            "Epoch 27/30\n",
            "10/10 [==============================] - 3s 267ms/step - loss: 0.3386 - val_loss: 0.3508\n",
            "Epoch 28/30\n",
            "10/10 [==============================] - 3s 272ms/step - loss: 0.3383 - val_loss: 0.3508\n",
            "Epoch 29/30\n",
            "10/10 [==============================] - 3s 262ms/step - loss: 0.3379 - val_loss: 0.3503\n",
            "Epoch 30/30\n",
            "10/10 [==============================] - 3s 264ms/step - loss: 0.3377 - val_loss: 0.3496\n"
          ]
        },
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "<keras.callbacks.History at 0x7f5669880cd0>"
            ]
          },
          "metadata": {},
          "execution_count": 93
        }
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "# Inference model"
      ],
      "metadata": {
        "id": "euxQrgNYU5Gu"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "# Define sampling models\n",
        "encoder_model = K.Model(encoder_inputs, encoder_states)\n",
        "\n",
        "decoder_state_input_h = KL.Input(shape=(latent_dim,))\n",
        "decoder_state_input_c = KL.Input(shape=(latent_dim,))\n",
        "decoder_states_inputs = [decoder_state_input_h, decoder_state_input_c]\n",
        "decoder_outputs, state_h, state_c = decoder_lstm(\n",
        "    decoder_inputs, initial_state=decoder_states_inputs)\n",
        "decoder_states = [state_h, state_c]\n",
        "decoder_outputs = decoder_dense(decoder_outputs)\n",
        "decoder_model = K.Model(\n",
        "    [decoder_inputs] + decoder_states_inputs,\n",
        "    [decoder_outputs] + decoder_states)\n"
      ],
      "metadata": {
        "id": "x1O7BbCrUpDq"
      },
      "execution_count": 87,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "encoder_model.summary()\n",
        "decoder_model.summary()"
      ],
      "metadata": {
        "id": "_RM_H2oFWBRJ",
        "outputId": "00158d85-0c4a-4c38-9c5b-9592912ac71d",
        "colab": {
          "base_uri": "https://localhost:8080/"
        }
      },
      "execution_count": 88,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Model: \"model_2\"\n",
            "_________________________________________________________________\n",
            " Layer (type)                Output Shape              Param #   \n",
            "=================================================================\n",
            " input_3 (InputLayer)        [(None, None, 57)]        0         \n",
            "                                                                 \n",
            " lstm (LSTM)                 [(None, 57),              26220     \n",
            "                              (None, 57),                        \n",
            "                              (None, 57)]                        \n",
            "                                                                 \n",
            "=================================================================\n",
            "Total params: 26,220\n",
            "Trainable params: 26,220\n",
            "Non-trainable params: 0\n",
            "_________________________________________________________________\n",
            "Model: \"model_3\"\n",
            "__________________________________________________________________________________________________\n",
            " Layer (type)                   Output Shape         Param #     Connected to                     \n",
            "==================================================================================================\n",
            " input_4 (InputLayer)           [(None, None, 57)]   0           []                               \n",
            "                                                                                                  \n",
            " input_5 (InputLayer)           [(None, 57)]         0           []                               \n",
            "                                                                                                  \n",
            " input_6 (InputLayer)           [(None, 57)]         0           []                               \n",
            "                                                                                                  \n",
            " lstm_1 (LSTM)                  [(None, None, 57),   26220       ['input_4[0][0]',                \n",
            "                                 (None, 57),                      'input_5[0][0]',                \n",
            "                                 (None, 57)]                      'input_6[0][0]']                \n",
            "                                                                                                  \n",
            " dense_1 (Dense)                (None, None, 57)     3306        ['lstm_1[1][0]']                 \n",
            "                                                                                                  \n",
            "==================================================================================================\n",
            "Total params: 29,526\n",
            "Trainable params: 29,526\n",
            "Non-trainable params: 0\n",
            "__________________________________________________________________________________________________\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "I = np.identity(voc_size)\n",
        "\n",
        "def my_to_categorical(v):\n",
        "  return np.array([ I[ix] for ix in v])\n"
      ],
      "metadata": {
        "id": "2fnpdmIBV2et"
      },
      "execution_count": 89,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "def tashkeel(line):\n",
        "  my_input_seq = np.array(tokenize_tashkeel_id(line)).reshape(-1)\n",
        "  max_len = len(my_input_seq)\n",
        "  \n",
        "  my_input = my_to_categorical(my_input_seq)\n",
        "  my_input = np.expand_dims(my_input, 0)\n",
        "  \n",
        "  sampled = []\n",
        "  sampled_ix = VOC_START\n",
        "  states_value = encoder_model.predict(my_input)\n",
        "  for i, ix in enumerate(my_input_seq):\n",
        "    target_seq = np.zeros((1, 1, num_decoder_tokens))\n",
        "    if ix>VOC_STOP: sampled_ix = ix\n",
        "    print(id_repr[sampled_ix])\n",
        "    sampled.append(sampled_ix)\n",
        "    target_seq[0, 0, sampled_ix] = 1.\n",
        "    output_tokens, h, c = decoder_model.predict([target_seq] + states_value)\n",
        "    states_value = [h, c]\n",
        "    # Sample a token\n",
        "    sampled_ix = np.argmax(output_tokens[0, -1, :])\n",
        "    if sampled_ix == VOC_STOP or len(sampled) >= max_len: break\n",
        "  return \"\".join([ id2ch2.get(ix, \"\") for ix in sampled ])"
      ],
      "metadata": {
        "id": "i97gRtUxbK_A"
      },
      "execution_count": 106,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "line = \"ذهب إلى السوق\"\n",
        "print(tashkeel(line))"
      ],
      "metadata": {
        "id": "9xV3lboAXZ_V",
        "outputId": "8e6e7d87-3252-466d-9453-7102f1304667",
        "colab": {
          "base_uri": "https://localhost:8080/"
        }
      },
      "execution_count": 110,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "VOC_START\n",
            "VOC_UNKNOWN_TASHKEEL\n",
            "ARABIC LETTER THAL\n",
            "VOC_FATHA\n",
            "ARABIC LETTER HEH\n",
            "VOC_DAMMA\n",
            "ARABIC LETTER BEH\n",
            "VOC_UNKNOWN_TASHKEEL\n",
            "SPACE\n",
            "VOC_UNKNOWN_TASHKEEL\n",
            "ARABIC LETTER ALEF WITH HAMZA BELOW\n",
            "VOC_KASRA\n",
            "ARABIC LETTER LAM\n",
            "VOC_SHADDA_FATHA\n",
            "ARABIC LETTER ALEF MAKSURA\n",
            "VOC_UNKNOWN_TASHKEEL\n",
            "SPACE\n",
            "VOC_UNKNOWN_TASHKEEL\n",
            "ARABIC LETTER ALEF\n",
            "VOC_UNKNOWN_TASHKEEL\n",
            "ARABIC LETTER LAM\n",
            "VOC_UNKNOWN_TASHKEEL\n",
            "ARABIC LETTER SEEN\n",
            "VOC_SHADDA_FATHA\n",
            "ARABIC LETTER WAW\n",
            "VOC_FATHA\n",
            "ARABIC LETTER QAF\n",
            "VOC_FATHA\n",
            "ARABIC LETTER TEH MARBUTA\n",
            "VOC_FATHA\n",
            "ذَهُب إِلَّى السَّوَقَةَ\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "ch2id"
      ],
      "metadata": {
        "id": "Q60N4xVzZbB3",
        "outputId": "391def5c-4bfd-41eb-e93a-6b589278a2e9",
        "colab": {
          "base_uri": "https://localhost:8080/"
        }
      },
      "execution_count": 99,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "{'': 1,\n",
              " '\\n': 19,\n",
              " ' ': 20,\n",
              " '؟': 2,\n",
              " 'ء': 21,\n",
              " 'آ': 22,\n",
              " 'أ': 23,\n",
              " 'ؤ': 24,\n",
              " 'إ': 25,\n",
              " 'ئ': 26,\n",
              " 'ا': 27,\n",
              " 'ب': 28,\n",
              " 'ة': 29,\n",
              " 'ت': 30,\n",
              " 'ث': 31,\n",
              " 'ج': 32,\n",
              " 'ح': 33,\n",
              " 'خ': 34,\n",
              " 'د': 35,\n",
              " 'ذ': 36,\n",
              " 'ر': 37,\n",
              " 'ز': 38,\n",
              " 'س': 39,\n",
              " 'ش': 40,\n",
              " 'ص': 41,\n",
              " 'ض': 42,\n",
              " 'ط': 43,\n",
              " 'ظ': 44,\n",
              " 'ع': 45,\n",
              " 'غ': 46,\n",
              " 'ـ؟ـ': 3,\n",
              " 'ف': 47,\n",
              " 'ق': 48,\n",
              " 'ك': 49,\n",
              " 'ل': 50,\n",
              " 'م': 51,\n",
              " 'ن': 52,\n",
              " 'ه': 53,\n",
              " 'و': 54,\n",
              " 'ى': 55,\n",
              " 'ي': 56,\n",
              " 'ً': 9,\n",
              " 'ٌ': 10,\n",
              " 'ٍ': 11,\n",
              " 'َ': 12,\n",
              " 'ُ': 13,\n",
              " 'ِ': 14,\n",
              " 'ّ': 18,\n",
              " 'ّ؟': 4,\n",
              " 'َّ': 15,\n",
              " 'ُّ': 16,\n",
              " 'ِّ': 17}"
            ]
          },
          "metadata": {},
          "execution_count": 99
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "\n",
        "\n",
        "more = True\n",
        "\n",
        "while more:\n",
        "    output_tokens, h, c = decoder_model.predict(\n",
        "            [target_seq] + states_value)\n",
        "\n",
        "    # Sample a token\n",
        "    sampled_token_index = np.argmax(output_tokens[0, -1, :])\n",
        "    print(id_repr[sampled_token_index])\n",
        "    sampled.append(sampled_token_index)\n",
        "\n",
        "    more = sampled_token_index != VOC_STOP and len(sampled) < max_len\n",
        "    # Update the target sequence (of length 1).\n",
        "    target_seq = np.zeros((1, 1, num_decoder_tokens))\n",
        "    target_seq[0, 0, sampled_token_index] = 1.\n",
        "    # Update states\n",
        "    states_value = [h, c]"
      ],
      "metadata": {
        "id": "x4OGNIjfVnrq",
        "outputId": "e8adc5d9-b429-4d77-ed52-2afd67fcc01d",
        "colab": {
          "base_uri": "https://localhost:8080/"
        }
      },
      "execution_count": 94,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "shape: (18,)\n",
            "my input shape: (18, 57)\n",
            "my input shape: (1, 18, 57)\n",
            "my target shape: (1, 1, 57)\n",
            "VOC_UNKNOWN_TASHKEEL\n",
            "ARABIC LETTER WAW\n",
            "VOC_FATHA\n",
            "ARABIC LETTER ALEF\n",
            "VOC_UNKNOWN_TASHKEEL\n",
            "ARABIC LETTER LAM\n",
            "VOC_UNKNOWN_TASHKEEL\n",
            "ARABIC LETTER LAM\n",
            "VOC_SHADDA_FATHA\n",
            "ARABIC LETTER HEH\n",
            "VOC_FATHA\n",
            "SPACE\n",
            "VOC_UNKNOWN_TASHKEEL\n",
            "ARABIC LETTER WAW\n",
            "VOC_FATHA\n",
            "ARABIC LETTER ALEF\n",
            "VOC_UNKNOWN_TASHKEEL\n",
            "ARABIC LETTER LAM\n",
            "VOC_UNKNOWN_TASHKEEL\n",
            "ARABIC LETTER LAM\n"
          ]
        }
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "# Below cells are not yet used"
      ],
      "metadata": {
        "id": "CyhAbdhhUeN2"
      }
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "swG6KFt5RyLV"
      },
      "source": [
        "import functools"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "On5_BfGKRlau"
      },
      "source": [
        "rnn = functools.partial(tf.keras.layers.GRU, recurrent_activation='sigmoid')"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "tf.keras.utils.to_categorical([0, 1, 2, 3], num_classes=voc_size, dtype='float32')"
      ],
      "metadata": {
        "id": "iHhiTIcT6Wb8",
        "outputId": "c8d260ef-d752-441d-8b1c-479bcfe28f11",
        "colab": {
          "base_uri": "https://localhost:8080/"
        }
      },
      "execution_count": 44,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "array([[1., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0.,\n",
              "        0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0.,\n",
              "        0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0.,\n",
              "        0., 0., 0., 0., 0., 0., 0.],\n",
              "       [0., 1., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0.,\n",
              "        0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0.,\n",
              "        0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0.,\n",
              "        0., 0., 0., 0., 0., 0., 0.],\n",
              "       [0., 0., 1., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0.,\n",
              "        0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0.,\n",
              "        0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0.,\n",
              "        0., 0., 0., 0., 0., 0., 0.],\n",
              "       [0., 0., 0., 1., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0.,\n",
              "        0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0.,\n",
              "        0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0.,\n",
              "        0., 0., 0., 0., 0., 0., 0.]], dtype=float32)"
            ]
          },
          "metadata": {},
          "execution_count": 44
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "jgZWBmYGBPuI"
      },
      "source": [
        "def build_model(vocab_size, embedding_dim, rnn_units):\n",
        "  model = tf.keras.Sequential([\n",
        "    rnn(rnn_units,\n",
        "        return_sequences=True,\n",
        "        recurrent_initializer='glorot_uniform',\n",
        "        stateful=True),\n",
        "  ])\n",
        "  return model\n"
      ],
      "execution_count": null,
      "outputs": []
    }
  ]
}