Deep_learning.ipynb

{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "colab": {
      "name": "Deep learning",
      "provenance": [],
      "collapsed_sections": []
    },
    "kernelspec": {
      "name": "python3",
      "display_name": "Python 3"
    }
  },
  "cells": [
    {
      "cell_type": "code",
      "metadata": {
        "id": "oT80cU3qpvfa"
      },
      "source": [
        "from __future__ import print_function\n",
        "import time\n",
        "import numpy as np\n",
        "\n",
        "from sklearn.decomposition import PCA\n",
        "from sklearn.manifold import TSNE\n",
        "\n",
        "%matplotlib inline\n",
        "import matplotlib.pyplot as plt\n",
        "from mpl_toolkits.mplot3d import Axes3D\n",
        "import seaborn as sns\n",
        "\n",
        "!pip install scikit-fuzzy\n",
        "import skfuzzy as fuzz\n",
        "from sklearn.cluster import MiniBatchKMeans\n",
        "\n",
        "from sklearn.ensemble import ExtraTreesClassifier, RandomForestClassifier, GradientBoostingClassifier\n",
        "from sklearn.preprocessing import LabelEncoder, StandardScaler, MinMaxScaler, PowerTransformer, QuantileTransformer, Normalizer\n",
        "from sklearn.model_selection import train_test_split\n",
        "\n",
        "import nltk\n",
        "from nltk.cluster import KMeansClusterer\n",
        "from sklearn import cluster, metrics\n",
        "nltk.download('punkt')\n",
        "nltk.download('stopwords')\n",
        "\n",
        "from gensim.models import Word2Vec\n",
        "from gensim.models.doc2vec import Doc2Vec, TaggedDocument\n",
        "\n",
        "from keras.preprocessing.text import Tokenizer\n",
        "from keras.preprocessing.sequence import pad_sequences\n",
        "from tensorflow import keras\n",
        "from keras.utils import to_categorical\n",
        "\n",
        "from keras.models import Sequential\n",
        "from keras.layers import Dense, Embedding, LSTM, Bidirectional\n",
        "\n",
        "from sklearn.metrics import roc_auc_score\n",
        "from scipy import interp\n",
        "from sklearn.metrics import precision_recall_fscore_support\n",
        "\n",
        "from sklearn.metrics import roc_curve, auc, accuracy_score, confusion_matrix\n",
        "from sklearn.preprocessing import LabelBinarizer\n",
        "\n",
        "from sklearn.multiclass import OneVsRestClassifier\n",
        "from sklearn.svm import SVC\n",
        "from sklearn import metrics\n",
        "from sklearn.feature_selection import SelectFromModel\n",
        "from sklearn.linear_model import LogisticRegression\n",
        "from sklearn.naive_bayes import MultinomialNB, GaussianNB\n",
        "from sklearn.model_selection import train_test_split\n",
        "from sklearn.svm import SVC\n",
        "from sklearn.ensemble import RandomForestClassifier, ExtraTreesClassifier, GradientBoostingClassifier\n",
        "from sklearn.neural_network import MLPClassifier\n",
        "\n",
        "import math\n",
        "import tensorflow as tf\n",
        "\n",
        "\n",
        "from matplotlib.colors import ListedColormap\n",
        "from matplotlib.cm import hsv\n",
        "\n",
        "\n",
        "from wordcloud import WordCloud, STOPWORDS, ImageColorGenerator\n",
        "\n",
        "\n",
        "from matplotlib import rc\n",
        "from matplotlib import font_manager as fm, rcParams \n",
        "import matplotlib.pyplot as plt \n",
        "plt.rc('xtick',labelsize=15)\n",
        "plt.rc('ytick',labelsize=15)\n",
        "plt.rcParams['figure.dpi'] = 120\n",
        "plt.rcParams.update({\n",
        "    \"text.usetex\": True,\n",
        "    \"font.family\": \"serif\",\n",
        "    \"font.serif\": [\"Palatino\"],\n",
        "})\n",
        "\n",
        "#LatX fonts\n",
        "rc('text', usetex=True)\n",
        "plt.rcParams['text.latex.preamble'] = [r'\\usepackage{amsmath}']\n",
        "!apt install texlive-fonts-recommended texlive-fonts-extra cm-super dvipng"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "fn6lZ9tEpl-0"
      },
      "source": [
        "!pip install -U -q PyDrive\n",
        "from pydrive.auth import GoogleAuth\n",
        "from pydrive.drive import GoogleDrive\n",
        "from google.colab import auth\n",
        "from oauth2client.client import GoogleCredentials\n",
        "import warnings\n",
        "import pandas as pd\n",
        "from sklearn.feature_extraction.text import CountVectorizer, TfidfTransformer,TfidfVectorizer\n",
        "from keras.layers import Dense, Flatten, Dropout, LSTM, SpatialDropout1D, Conv1D, MaxPool1D, GRU, Input, Concatenate, GlobalMaxPooling1D\n",
        "from keras.layers import Dense, Flatten, LSTM, Conv1D, MaxPooling1D, Dropout, Activation\n",
        "from sklearn.decomposition import TruncatedSVD\n",
        "!pip install dataframe_image\n",
        "import dataframe_image as dfi\n",
        "warnings.filterwarnings('ignore')\n",
        "import nltk\n",
        "from nltk import tokenize, ne_chunk\n",
        "from nltk.stem import RSLPStemmer\n",
        "from nltk.corpus import floresta, mac_morpho, masc_tagged\n",
        "from pickle import load\n",
        "import numpy as np\n",
        "from keras.preprocessing.text import Tokenizer\n",
        "from keras.preprocessing.sequence import pad_sequences\n",
        "from nltk import RegexpParser, Tree\n",
        "from nltk.corpus import wordnet as wn\n",
        "from nltk import RegexpParser\n",
        "from nltk.stem import SnowballStemmer\n",
        "from keras.initializers import Constant\n",
        "\n",
        "# Authenticate and create the PyDrive client.\n",
        "auth.authenticate_user()\n",
        "gauth = GoogleAuth()\n",
        "gauth.credentials = GoogleCredentials.get_application_default()\n",
        "drive = GoogleDrive(gauth)\n",
        "\n",
        "\n",
        "def build_df(file_id, file_name, index_col=None):\n",
        "  downloaded = drive.CreateFile({'id':file_id})\n",
        "  downloaded.GetContentFile(file_name)\n",
        "  if index_col is not None :\n",
        "    return pd.read_csv(file_name, index_col=index_col)\n",
        "  return pd.read_csv(file_name)\n",
        "\n",
        "df = build_df('1LEbg13-KBZzHmIWAv9SnthGUzjvcbJuA','stylo-data.csv')\n",
        "news_df = build_df('15zZXsks6cI0FY4vpL555lBQSBr1QyjT4', 'news-data.csv' )\n",
        "varela_df = build_df('1JoC7GlAmdQO9WPZeJZKMbv4djyQzeezC',  'varela-stylo-data.csv')\n",
        "varela_df.pop('Subject')\n",
        "\n",
        "#Some clean-up on line breaks\n",
        "def clean_df(df):\n",
        "  df = df.replace(to_replace ='\\r', value = ' ', regex = True) \n",
        "  df = df.replace(to_replace ='\\n ', value = '\\n', regex = True) \n",
        "  df = df.replace(to_replace ='\\n', value = ' ', regex = True)\n",
        "  df = df.replace(to_replace ='\\n{2,*}', value = ' ', regex = True)\n",
        "  return df\n",
        "\n",
        "#Read textual datasets\n",
        "raw_df_test = build_df('1B-Ozck3R_cMeREw3yQQcODcz8UZ8Hvp7', 'raw_data.csv')\n",
        "raw_df = clean_df(build_df('1B-Ozck3R_cMeREw3yQQcODcz8UZ8Hvp7', 'raw_data.csv'))\n",
        "raw_news_df = clean_df(build_df('1P0UF5IRI3VNUuAulldEn84ntVfFLGiIq', 'raw_news_data.csv'))\n",
        "raw_varela_df = clean_df(build_df('1s-qn2puLikOtN1YGOX2WZfZrNMxA6K4e', 'raw_varela_data.csv'))\n",
        "\n",
        "\n",
        "stylo_classic_score_df = build_df('1g4zAbrxtVSqJ1GwwXKJufqMHXpWSrLsC', 'stylo_classic_score.csv', index_col=0)\n",
        "tfidf_classic_score_df = build_df('17O4S-8P66KMzlKKReOcfsKO4LDDMSrYJ', 'idf_classic_score.csv', index_col=0)\n",
        "raw_news_df.pop('Link')\n",
        "raw_varela_df.pop('Subject')\n",
        "\n",
        "\n",
        "def remove_varela_authors(df):\n",
        "  regex_baleia = 'baleia - \\d{1,2}\\/\\d{1,2}\\/\\d{2,4}'\n",
        "  regex_ana = 'ana cristina cavalcante\\s*\\d{1,2} \\w{1,4} \\d{2,4} - \\S{3,5}(min)*'\n",
        "  regex_adriano = 'adriano gambarini - \\d{1,2}\\/\\d{1,2}\\/\\d{2,4}'\n",
        "  regex_ivolnildo = 'ivon(i)*l(d)*o lavôr(\\s)*\\d{1,2} \\w{1,4} \\d{2,4} - \\S{4,5}(min)*'\n",
        "  regex_mario = 'mário pinto(\\s)*\\d{1,2} \\w{1,4} \\d{2,4} - \\S{4,5}(min)*'\n",
        "  regex_julio = \"mais sobr julio preuss - \\d{1,2}\\/\\d{1,2}\\/\\d{2,4}(julio preussescreveu o livro 'fotografia digital: da compra da câmera à impressão das fotos)*\"\n",
        "  regex_roberto = 'roberto linsker - \\d{1,2}\\/\\d{1,2}\\/\\d{2,4}'\n",
        "  df['Text'] = df['Text'].str.replace(regex_baleia, ' ')\n",
        "  df['Text'] = df['Text'].str.replace(regex_ana, ' ')\n",
        "  df['Text'] = df['Text'].str.replace(regex_adriano, ' ')\n",
        "  df['Text'] = df['Text'].str.replace(regex_ivolnildo, ' ')\n",
        "  df['Text'] = df['Text'].str.replace(regex_mario, ' ')\n",
        "  df['Text'] = df['Text'].str.replace(regex_julio, ' ')\n",
        "  df['Text'] = df['Text'].str.replace(regex_roberto, ' ')\n",
        "  return df\n",
        "\n",
        "raw_varela_df = remove_varela_authors(raw_varela_df)\n",
        "\n",
        "# Remove few class entries (We need at least 3 samples of an author) and encode Author column \n",
        "\n",
        "def remove_entries(df, size):\n",
        "  return df.groupby('Author').filter(lambda x: len(x) > size)\n",
        "\n",
        "def encode_target_column(df, column):\n",
        "  le = LabelEncoder()\n",
        "  y = df.pop(column)\n",
        "  encoded_Y = le.fit_transform(y)\n",
        "  return encoded_Y\n",
        "\n",
        "df = remove_entries(df, 2)\n",
        "raw_df = remove_entries(raw_df, 2)\n",
        "\n",
        "encoded_Y = encode_target_column(df, 'Author')\n",
        "encoded_raw_Y  = encode_target_column(raw_df, 'Author')\n",
        "\n",
        "encoded_Y_news  = encode_target_column(news_df, 'Author')\n",
        "raw_news_df.pop('Author')\n",
        "\n",
        "varela_Y_backup = varela_df['Author']\n",
        "encoded_varela_Y = encode_target_column(varela_df, 'Author')\n",
        "raw_varela_df.pop('Author')\n",
        "\n",
        "\n",
        "corpus = raw_df['Text']\n",
        "corpus_news = raw_news_df['Text']\n",
        "corpus_varela = raw_varela_df['Text']"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "aE-wJTwsqANg"
      },
      "source": [
        "# Download word embedding for portuguese-BR (50/100 dim of several types)\n",
        "auth.authenticate_user()\n",
        "gauth = GoogleAuth()\n",
        "gauth.credentials = GoogleCredentials.get_application_default()\n",
        "drive = GoogleDrive(gauth)\n",
        "\n",
        "glove_50_link = 'https://drive.google.com/open?id=10MW2F53DYYxizSXLbvfsphs77Fu1U9Gz'\n",
        "glove_50_id = '18TekAQRy2PxvHse_lfoL2Tt0pPcoptl-'\n",
        "\n",
        "glove_100_link = 'https://drive.google.com/file/d/1GDsyY-TmS0gj_TyM3h_tnuBo9HBZk-80/view?usp=sharing'\n",
        "glove_100_id = '1GDsyY-TmS0gj_TyM3h_tnuBo9HBZk-80'\n",
        "\n",
        "\n",
        "w2vec_cbow_50 = 'https://drive.google.com/file/d/1h3JtxhSXvmRw4_qIpJ1jqhCR_aCEUfK7/view?usp=sharing'\n",
        "w2vec_cbow_50_id = '1h3JtxhSXvmRw4_qIpJ1jqhCR_aCEUfK7'\n",
        "\n",
        "w2vec_cbow_100 = 'https://drive.google.com/file/d/1H_PXtUi98MqFhpB4sWIt2J2J2ftjidj4/view?usp=sharing'\n",
        "w2vec_cbow_100_id = '1H_PXtUi98MqFhpB4sWIt2J2J2ftjidj4'\n",
        "\n",
        "fasttext_skip_50 = 'https://drive.google.com/file/d/1L2gR3GEb_od-ooxW-kaNuqXglnK4TMHc/view?usp=sharing'\n",
        "fasttext_skip_50_id = '1L2gR3GEb_od-ooxW-kaNuqXglnK4TMHc'\n",
        "\n",
        "fasttext_skip_100 = 'https://drive.google.com/file/d/1jm3cgIX7GyVmtH1Tr-IF5sxxfq0DT9qm/view?usp=sharing'\n",
        "fasttext_skip_100_id = '1jm3cgIX7GyVmtH1Tr-IF5sxxfq0DT9qm'\n",
        "\n",
        "\n",
        "def load_embedding(id, filename):\n",
        "  downloaded = drive.CreateFile({'id':id})\n",
        "  downloaded.GetContentFile(filename)\n",
        "  word_embedding_dict = {}\n",
        "  with open(filename,'r', encoding='utf-8') as f:\n",
        "    for line in f:\n",
        "      word, coefs = line.split(maxsplit=1)  \n",
        "      coefs = np.fromstring(coefs, 'f', sep=' ')\n",
        "      word_embedding_dict[word] = coefs\n",
        "  return word_embedding_dict\n",
        "\n",
        "\n",
        "glove_50_embedding = load_embedding(glove_50_id, 'glove_50')\n",
        "glove_100_embedding = load_embedding(glove_100_id, 'glove_100')\n",
        "w2vec_cbow_50_embedding = load_embedding(w2vec_cbow_50_id, 'w2v_50')\n",
        "w2vec_cbow_100_embedding = load_embedding(w2vec_cbow_100_id, 'w2v_100')\n",
        "fasttext_skip_50_embedding = load_embedding(fasttext_skip_50_id, 'ft_50')\n",
        "fasttext_skip_100_embedding = load_embedding(fasttext_skip_100_id, 'ft_100')\n"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "X1es1ax72cT9"
      },
      "source": [
        "def build_matrix(corpus, word_dict):\n",
        "  dim_size = len(word_dict.get('a'))\n",
        "  tokenizer = Tokenizer(num_words=25000)\n",
        "  tokenizer.fit_on_texts(corpus)\n",
        "  # Convert strings to their corresponding integer value created by the Tokenizer\n",
        "  embedded_sentences = tokenizer.texts_to_sequences(corpus)\n",
        "  max_words_text = max(corpus, key=lambda sentence: len(nltk.word_tokenize(sentence, language='portuguese')))\n",
        "  max_sentence_len = len(nltk.word_tokenize(max_words_text, language='portuguese'))\n",
        "  padded_sentences = pad_sequences(embedded_sentences, max_sentence_len, padding='post')\n",
        "  vocab_len = len(tokenizer.word_index) + 1\n",
        "\n",
        "  # Create an embedding matrix, in other words, use the weights from gloVe vector against all our corpus\n",
        "  # In the end we will have a matrix of all our vocabulary x 50 dimensions (size of glove dim)\n",
        "\n",
        "  # initialize with all zeros\n",
        "  embedding_matrix = np.zeros((vocab_len, dim_size))\n",
        "  for word, index in tokenizer.word_index.items():\n",
        "    embedding_vector = word_dict.get(word)\n",
        "    # if the word is in gloVe, we will use its weigths \n",
        "    if embedding_vector is not None and embedding_vector.any():\n",
        "      embedding_matrix[index] = embedding_vector[:dim_size]\n",
        "    else:\n",
        "      embedding_matrix[index] = np.random.randn(dim_size)\n",
        "  return embedding_matrix\n",
        "\n",
        "def build_matrixes(data, embedding):\n",
        "  result = []\n",
        "  for corpus in data:\n",
        "    result.append(build_matrix(corpus, embedding))\n",
        "  return result\n",
        "\n",
        "\n",
        "student_glove_50, news_glove_50, varela_glove_50  = build_matrixes([corpus, corpus_news, corpus_varela], glove_50_embedding)\n",
        "student_glove_100, news_glove_100, varela_glove_100  = build_matrixes([corpus, corpus_news, corpus_varela], glove_100_embedding)\n",
        "\n",
        "student_w2v_50, news_w2v_50, varela_w2v_50  = build_matrixes([corpus, corpus_news, corpus_varela], w2vec_cbow_50_embedding)\n",
        "student_w2v_100, news_w2v_100, varela_w2v_100  = build_matrixes([corpus, corpus_news, corpus_varela], w2vec_cbow_100_embedding)\n",
        "\n",
        "student_ft_50, news_ft_50, varela_ft_50  = build_matrixes([corpus, corpus_news, corpus_varela], fasttext_skip_50_embedding)\n",
        "student_ft_100, news_ft_100, varela_ft_100  = build_matrixes([corpus, corpus_news, corpus_varela], fasttext_skip_100_embedding)"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "c_ufkRuScQfK"
      },
      "source": [
        "embedding_matrixes = [[student_ft_50, news_ft_50, varela_ft_50]\n",
        "      [student_glove_50, news_glove_50, varela_glove_50],\n",
        "      [student_glove_100, news_glove_100, varela_glove_100],\n",
        "      [student_w2v_50, news_w2v_50, varela_w2v_50],\n",
        "      [student_w2v_100, news_w2v_100, varela_w2v_100],\n",
        "      [student_ft_50, news_ft_50, varela_ft_50],\n",
        "      [student_ft_100, news_ft_100, varela_ft_100]]\n",
        "      "
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "AuPy1UQBLG_L"
      },
      "source": [
        "vocab_size = 10000\n",
        "oov_token = \"<OOV>\"\n",
        "max_length = 50\n",
        "padding_type = \"post\"\n",
        "trunction_type=\"post\"\n",
        "\n",
        "X_train, X_test , y_train, y_test = train_test_split(corpus, encoded_Y, test_size = 0.20)\n",
        "\n",
        "tokenizer = Tokenizer(num_words = vocab_size, oov_token=oov_token)\n",
        "tokenizer.fit_on_texts(corpus)"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "bfgQG53qP7iK"
      },
      "source": [
        "max_features = 20000\n",
        "def convert_corpus_to_number(corpus):\n",
        "  tokenizer = Tokenizer(num_words=max_features)\n",
        "  tokenizer.fit_on_texts(corpus)\n",
        "\n",
        "  embedded_sentences = tokenizer.texts_to_sequences(corpus)\n",
        "\n",
        "  longest_sentence = max(corpus, key=lambda sentence: len(nltk.word_tokenize(sentence, language='portuguese')))\n",
        "  max_sentence_len = len(nltk.word_tokenize(longest_sentence, language='portuguese'))\n",
        "\n",
        "  padded_sentences = pad_sequences(embedded_sentences, max_sentence_len, padding='post')\n",
        "\n",
        "  return tokenizer, padded_sentences, max_sentence_len\n",
        "\n",
        "tokenizer, padded_sentences, max_sent_size = convert_corpus_to_number(corpus)\n",
        "\n",
        "word_index = tokenizer.word_index"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "APajtBHd4D8O",
        "outputId": "4a2a080c-f34e-4da0-969c-0776b6fce306"
      },
      "source": [
        "num_words = min(max_features, len(word_index)) + 1\n",
        "print(num_words)"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "5527\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "ZWt0xMb3AWoB"
      },
      "source": [
        "def split_and_train(corpus, encoded_Y, emd_matrix):\n",
        "  tokenizer, padded_sentences, max_sent_size = convert_corpus_to_number(corpus)\n",
        "  X_train, X_test , y_train, y_test = train_test_split(padded_sentences, encoded_Y, test_size = 0.30, random_state=0)\n",
        "  y_train = to_categorical(y_train, len(set(encoded_Y)))\n",
        "  y_test = to_categorical(y_test, len(set(encoded_Y)))\n",
        "  return build_and_train_lstm(X_train, y_train , X_test, y_test, len(tokenizer.word_index) + 1, emd_matrix.shape[1], max_sent_size, emd_matrix, len(set(encoded_Y)))\n",
        "\n",
        "def build_and_train_lstm(X_train, Y_train, X_test, Y_test, vocab_size, embd_size, max_sent_size, embd_matrix, num_classes, epochs=100):\n",
        "  model = Sequential()\n",
        "  embedding_layer = Embedding(vocab_size, embd_size, weights=[embd_matrix],\n",
        "                              input_length=max_sent_size, trainable=False)\n",
        "  model.add(embedding_layer)\n",
        "  model.add(Dropout(0.2))\n",
        "  model.add(Conv1D(64, 5, activation='relu'))\n",
        "  model.add(MaxPooling1D(pool_size=4))\n",
        "  model.add(LSTM(16, return_sequences=True))\n",
        "  model.add(Dropout(0.20))\n",
        "  model.add(Flatten())\n",
        "  model.add(Dense(num_classes))\n",
        "  model.compile(optimizer='rmsprop',loss='categorical_crossentropy', metrics=['accuracy'])\n",
        "  print('fitting model')\n",
        "  history = model.fit(X_train, Y_train, validation_data=(X_test, Y_test), epochs=epochs, verbose=0)\n",
        "  print('latest acc{} '.format(history.history['val_accuracy'][-1]))\n",
        "  return max(history.history['val_accuracy'])\n",
        "\n",
        "#split_and_train(corpus, encoded_Y, student_glove_50)\n",
        "#build_and_train_lstm(X_train, y_train , X_test, y_test, len(tokenizer.word_index) + 1, student_glove_50.shape[1], max_sent_size, student_glove_50, len(set(encoded_Y)))\n"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "mUX0AzaVEbUo",
        "outputId": "b4fbb138-5aed-4cec-d458-aeab66ea205f"
      },
      "source": [
        "def split_and_train_2(corpuses, Y, emd_matrix):\n",
        "  results = {}\n",
        "  #for idx, matrix in enumerate(emd_matrix):\n",
        "  for i, corpus in enumerate(corpuses):\n",
        "    acc = split_and_train(corpus, Y[i], emd_matrix[i])\n",
        "    print(acc)\n",
        "    results[i] = acc\n",
        "    print(results)\n",
        "  return results\n",
        "\n",
        "split_and_train_2(\n",
        "                [corpus, corpus_news, corpus_varela], \n",
        "                [encoded_Y, encoded_Y_news, encoded_varela_Y], \n",
        "                embedding_matrixes)"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "fitting model\n",
            "latest acc0.07692307978868484 \n",
            "0.07692307978868484\n",
            "{0: 0.07692307978868484}\n",
            "fitting model\n",
            "latest acc0.20000000298023224 \n",
            "0.30000001192092896\n",
            "{0: 0.07692307978868484, 1: 0.30000001192092896}\n",
            "fitting model\n",
            "latest acc0.024444444105029106 \n",
            "0.05444444343447685\n",
            "{0: 0.07692307978868484, 1: 0.30000001192092896, 2: 0.05444444343447685}\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "{0: 0.07692307978868484, 1: 0.30000001192092896, 2: 0.05444444343447685}"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 19
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "n7RSrOXAFVRk",
        "outputId": "1845d55c-c9f8-44c1-8c7d-0e63a457b1e6"
      },
      "source": [
        "def split_and_train_2(corpuses, Y, emd_matrix):\n",
        "  results = {}\n",
        "  #for idx, matrix in enumerate(emd_matrix):\n",
        "  for i, corpus in enumerate(corpuses):\n",
        "    acc = split_and_train(corpus, Y[i], emd_matrix[i])\n",
        "    print(acc)\n",
        "    results[idx] = acc\n",
        "    print(results)\n",
        "  return results\n",
        "\n",
        "split_and_train_2(\n",
        "                [corpus, corpus_news, corpus_varela], \n",
        "                [encoded_Y, encoded_Y_news, encoded_varela_Y], \n",
        "                embedding_matrixes)"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "fitting model\n",
            "latest acc0.11538461595773697 \n",
            "{0: 0.11538461595773697}\n",
            "fitting model\n",
            "latest acc0.13333334028720856 \n",
            "{0: 0.20000000298023224}\n",
            "fitting model\n",
            "latest acc0.036666665226221085 \n",
            "{0: 0.04444444552063942}\n",
            "fitting model\n",
            "latest acc0.07692307978868484 \n",
            "{0: 0.04444444552063942, 1: 0.07692307978868484}\n",
            "fitting model\n",
            "latest acc0.1666666716337204 \n",
            "{0: 0.04444444552063942, 1: 0.23333333432674408}\n",
            "fitting model\n",
            "latest acc0.036666665226221085 \n",
            "{0: 0.04444444552063942, 1: 0.03888889029622078}\n",
            "fitting model\n",
            "latest acc0.11538461595773697 \n",
            "{0: 0.04444444552063942, 1: 0.03888889029622078, 2: 0.11538461595773697}\n",
            "fitting model\n",
            "latest acc0.10000000149011612 \n",
            "{0: 0.04444444552063942, 1: 0.03888889029622078, 2: 0.2666666805744171}\n",
            "fitting model\n",
            "latest acc0.02222222276031971 \n",
            "{0: 0.04444444552063942, 1: 0.03888889029622078, 2: 0.06666667014360428}\n",
            "fitting model\n",
            "latest acc0.07692307978868484 \n",
            "{0: 0.04444444552063942, 1: 0.03888889029622078, 2: 0.06666667014360428, 3: 0.07692307978868484}\n",
            "fitting model\n",
            "latest acc0.06666667014360428 \n",
            "{0: 0.04444444552063942, 1: 0.03888889029622078, 2: 0.06666667014360428, 3: 0.20000000298023224}\n",
            "fitting model\n",
            "latest acc0.009999999776482582 \n",
            "{0: 0.04444444552063942, 1: 0.03888889029622078, 2: 0.06666667014360428, 3: 0.058888889849185944}\n",
            "fitting model\n",
            "latest acc0.07692307978868484 \n",
            "{0: 0.04444444552063942, 1: 0.03888889029622078, 2: 0.06666667014360428, 3: 0.058888889849185944, 4: 0.11538461595773697}\n",
            "fitting model\n",
            "latest acc0.10000000149011612 \n",
            "{0: 0.04444444552063942, 1: 0.03888889029622078, 2: 0.06666667014360428, 3: 0.058888889849185944, 4: 0.10000000149011612}\n",
            "fitting model\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "Sjcek4YPd2Mj"
      },
      "source": [
        "embedding_matrixes = [\n",
        "      [student_glove_50, news_glove_50, varela_glove_50],\n",
        "      [student_glove_100, news_glove_100, varela_glove_100],\n",
        "      [student_w2v_50, news_w2v_50, varela_w2v_50],\n",
        "      [student_w2v_100, news_w2v_100, varela_w2v_100],\n",
        "      [student_ft_50, news_ft_50, varela_ft_50],\n",
        "      [student_ft_100, news_ft_100, varela_ft_100]\n",
        "]"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "_y9VTybbLJY8",
        "outputId": "7343cbd2-d9dc-4e2b-f32e-7075ba88a006"
      },
      "source": [
        "num_words = min(vocab_size, len(word_index)) + 1\n",
        "print(num_words)\n",
        "\n",
        "\n",
        "embedding_dim = 50\n",
        "\n",
        "# first create a matrix of zeros, this is our embedding matrix\n",
        "embedding_matrix = np.zeros((num_words, embedding_dim))\n",
        "\n",
        "# for each word in out tokenizer lets try to find that work in our w2v model\n",
        "for word, i in word_index.items():\n",
        "    if i > vocab_size:\n",
        "        continue\n",
        "    embedding_vector = word_embedding_dict.get(word)\n",
        "    if embedding_vector is not None:\n",
        "        # we found the word - add that words vector to the matrix\n",
        "        embedding_matrix[i] = embedding_vector[:embedding_dim]\n",
        "    else:\n",
        "        # doesn't exist, assign a random vector\n",
        "        embedding_matrix[i] = np.random.randn(embedding_dim)"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "5528\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "e785oT2NtJUl"
      },
      "source": [
        "X_train_padded = pad_sequences(tokenizer.texts_to_sequences(X_train),maxlen=len(tokenizer.word_index), padding=padding_type, truncating=trunction_type)\n",
        "X_test_padded = pad_sequences(tokenizer.texts_to_sequences(X_test),maxlen=len(tokenizer.word_index), padding=padding_type, truncating=trunction_type)\n",
        "\n",
        "y_train = to_categorical(y_train, len(set(encoded_Y)))\n",
        "y_test = to_categorical(y_test, len(set(encoded_Y)))"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "L69K3kAetlrv"
      },
      "source": [
        "from keras.initializers import Constant\n",
        "model = Sequential()\n",
        "\n",
        "model.add(Embedding(num_words,\n",
        "                    50,\n",
        "                    embeddings_initializer=Constant(embedding_matrix),\n",
        "                    input_length=sequence_length,\n",
        "                    trainable=True))\n",
        "\n",
        "model.add(Embedding(num_words,\n",
        "                    50,\n",
        "                    embeddings_initializer=Constant(student_glove_50),\n",
        "                    input_length=sequence_length,\n",
        "                    trainable=True))\n",
        "\n",
        "model.add(LSTM(embedding_dim, return_sequences=True))\n",
        "\n",
        "model.add(Dense(16, activation='softmax'))\n",
        "\n",
        "model.compile(loss='categorical_crossentropy',optimizer='adam', metrics=['accuracy'])\n",
        "embedding_layer = Embedding(len(word_index),\n",
        "                            max_length,\n",
        "                            weights=[student_glove_50],\n",
        "                            input_length=6584,\n",
        "                            trainable=False)\n",
        "\n",
        "model = Sequential()\n",
        "\n",
        "model.add(Embedding(num_words,\n",
        "                    50,\n",
        "                    embeddings_initializer=Constant(embedding_matrix),\n",
        "                    input_length=sequence_length,\n",
        "                    trainable=True))\n",
        "\n",
        "model.add(LSTM(embedding_dim, return_sequences=True))\n",
        "\n",
        "model.add(Dense(16, activation='softmax'))\n",
        "\n",
        "model.compile(loss='categorical_crossentropy',optimizer='adam', metrics=['accuracy'])"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "D4f1AJeHzCc2"
      },
      "source": [
        "model.summary()"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "cvZdS7mCuCsU"
      },
      "source": [
        "tokenizer.word_index"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "gArUzDd2qFA7"
      },
      "source": [
        "def convert_docs_to_embeddings(corpus, word_dict, dim_size):\n",
        "  tokenizer = Tokenizer(num_words=40000)\n",
        "  tokenizer.fit_on_texts(corpus)\n",
        "  \n",
        "  max_words_text = max(corpus, key=lambda sentence: len(nltk.word_tokenize(sentence, language='portuguese')))\n",
        "  max_sentence_len = len(nltk.word_tokenize(max_words_text, language='portuguese'))\n",
        "  vocab_len = len(tokenizer.word_index) + 1\n",
        "\n",
        "  docs = np.zeros((len(corpus), vocab_len, dim_size))\n",
        "  \n",
        "  for index, doc in enumerate(corpus):\n",
        "    current_doc = np.zeros((vocab_len, dim_size))\n",
        "    # NOTE: I realized that nltk.word_tokenize has a different behavior than split ''\n",
        "    for word in doc.split(' '):\n",
        "      embedding_vector = word_dict.get(word)\n",
        "      if word.isalnum() and embedding_vector is not None and embedding_vector.any():\n",
        "        idx = tokenizer.word_index[word]\n",
        "        current_doc[idx] = embedding_vector[:dim_size]\n",
        "    docs[index] = current_doc\n",
        "  return docs\n",
        "    \n",
        "\n",
        "student_embedded_docs = convert_docs_to_embeddings(corpus, word_embedding_dict, 50)\n",
        "#news_embedded_docs = convert_docs_to_embeddings(corpus_news, word_embedding_dict, 50)\n",
        "\n",
        "# Crashing out of RAM\n",
        "#varela_embedded_docs = convert_docs_to_embeddings(corpus_varela_sw, word_embedding_dict, 50)"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "5aQBbq_KqL1a"
      },
      "source": [
        "# cleaning up stopwords from Varela to see if memory can handle now\n",
        "corpus_varela.shape\n",
        "\n",
        "corpus_varela_sw = corpus_varela.apply(lambda x: [str(item) for item in x.split(' ') if item not in stopwords])\n",
        "corpus_varela_sw = corpus_varela_sw.apply(lambda x: ' '.join(x))\n",
        "corpus_varela_sw"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "9ktLbQDkG7WG"
      },
      "source": [
        "## Model optimization"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "lMkZrwxsG-Xn"
      },
      "source": [
        "# We have 7 classic models + 1 deep model\n",
        "# Lets do GridSearchCV with each one + data normalization/scaling (optimization notebook)"
      ],
      "execution_count": null,
      "outputs": []
    }
  ]
}