BERT+Kermit_NLI_model.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import os\n",
    "os.environ['CUDA_LAUNCH_BLOCKING'] = \"1\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "import warnings\n",
    "warnings.filterwarnings(\"ignore\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "!export CUDA_LAUNCH_BLOCKING=1\n",
    "#import sys\n",
    "#sys.path.append('../')\n",
    "#from models import model1 as m1\n",
    "import torch\n",
    "import pickle\n",
    "import copy\n",
    "import transformers\n",
    "from torchtext import data as datx\n",
    "from torch import nn\n",
    "from torch import optim\n",
    "from tqdm import tqdm\n",
    "import numpy as np\n",
    "import random\n",
    "\n",
    "torch.cuda.is_available()\n",
    "torch.cuda.get_device_name(0)\n",
    "device_num = 0\n",
    "torch.cuda.set_device(device_num)\n",
    "\n",
    "\n",
    "#seed definition\n",
    "seed = 23"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Reproducibility"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "torch.manual_seed(seed)\n",
    "torch.cuda.manual_seed(seed)\n",
    "#torch.cuda.manual_seed_all(seed)  # if you are using multi-GPU.\n",
    "np.random.seed(seed)  # Numpy module.\n",
    "random.seed(seed)  # Python random module.\n",
    "torch.manual_seed(seed)\n",
    "torch.backends.cudnn.benchmark = False\n",
    "torch.backends.cudnn.deterministic = True\n",
    "\n",
    "def _init_fn(worker_id):\n",
    "    np.random.seed(int(seed))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Defining sentences training data and DTK trees for sentence 1 and 2 columns"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "dataPath = ''\n",
    "\n",
    "#MultiNLI training dataset \n",
    "dataset_train = 'training_full.csv'\n",
    "dataset_test = 'heuristics_evaluation_set.txt'\n",
    "\n",
    "#DTK trees generated from sentence 1 column of training dataset\n",
    "nameTree1_train = './dtk_trees_multiNLI_train_sencence_1_tot.pkl' #train_s1\n",
    "nameTree1_test = './dtk_trees_hans_sentence1.pkl' #train_s1\n",
    "\n",
    "#DTK trees generated from sentence 2 column of training dataset\n",
    "nameTree2_train = './dtk_trees_multiNLI_train_sentence_2_tot.pkl' #train_s2\n",
    "nameTree2_test = './dtk_trees_hans_sentence2.pkl' #train_s1\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Selecting GPU device if available"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Define a function to transform text label to numeric value"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "def from_label_to_int(e):\n",
    "    d = {'neutral':0,'entailment':1,'contradiction':0}\n",
    "    return d[e]\n",
    "\n",
    "def from_label_to_int_heuristic(e):\n",
    "    d = {'neutral':0,'entailment':1,'non-entailment':0}\n",
    "    return d[e]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Data load step"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### First we load training data in a DataFrame, then we build a new DataFram that has the following structure: \n",
    "- id -> id of the sentence pair \n",
    "- sentence -> single sentence derived from sentence 1 and sentence 2 merge, particularly: **SENTENCE1**[SEP]**SENTENCE2** \n",
    "- label -> class label "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0    260754\n",
       "1    130411\n",
       "Name: label, dtype: int64"
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "import pandas as pd\n",
    "\n",
    "data = pd.read_csv(dataPath+dataset_train)\n",
    "gl = data[\"gold_label\"]\n",
    "s = data[\"sentence1\"] + \" [SEP] \" + data[\"sentence2\"]\n",
    "l = data['gold_label'].apply(from_label_to_int)\n",
    "train = pd.concat([l,s],axis=1)\n",
    "train.columns = [\"label\",\"sentence\"]\n",
    "train.to_csv(\"training_labeled.csv\")\n",
    "train.head()\n",
    "\n",
    "train[\"label\"].value_counts()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "data = pd.read_csv(dataPath+dataset_test, delimiter=\"\\t\", error_bad_lines=False, )\n",
    "data.drop(data[data[\"gold_label\"]==\"neutral\"].index, inplace=True)\n",
    "gl = data[\"gold_label\"]\n",
    "s = data[\"sentence1\"] + \" [SEP] \" + data[\"sentence2\"]\n",
    "l = data['gold_label'].apply(from_label_to_int_heuristic)\n",
    "test = pd.concat([l,s],axis=1)\n",
    "test.columns = [\"label\",\"sentence\"]\n",
    "test.to_csv(\"test_labeled.csv\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### This function truncate the sentences representation in order to respect the maximum length of bert input\n",
    "We invoke this function in preprocessing step"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "def _truncate_seq_pair(tokens_a, tokens_b, max_length):\n",
    "    \"\"\"Truncates a sequence pair in place to the maximum length.\"\"\"\n",
    "\n",
    "    # This is a simple heuristic which will always truncate the longer sequence\n",
    "    # one token at a time. This makes more sense than truncating an equal percent\n",
    "    # of tokens from each, since if one sequence is very short then each token\n",
    "    # that's truncated likely contains more information than a longer sequence.\n",
    "    while True:\n",
    "        total_length = len(tokens_a) + len(tokens_b)\n",
    "        if total_length <= max_length:\n",
    "            break\n",
    "        if len(tokens_a) > len(tokens_b):\n",
    "            tokens_a.pop()\n",
    "        else:\n",
    "            tokens_b.pop()\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Preprocessing step, the input of this function is the numeric representation of the sentence.\n",
    "#### 102 tag is appended to separate the two sentences numeric representation like bert 2 sentence classification input guidelines"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [],
   "source": [
    "def pre_processing(x):\n",
    "    splitted = []\n",
    "    tokenized = []\n",
    "    for s in range(len(x)):\n",
    "        if x[s]==102:\n",
    "            splitted.append(x[:s])\n",
    "            splitted.append(x[s+1:])\n",
    "    \n",
    "    _truncate_seq_pair(splitted[0],splitted[1],MAX_LEN-3)\n",
    "    for s1 in splitted[0]:\n",
    "        tokenized.append(s1)\n",
    "    \n",
    "    tokenized.append(102)\n",
    "    \n",
    "    for s2 in splitted[1]:\n",
    "        tokenized.append(s2)\n",
    "    return tokenized\n",
    "            "
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Defining Training and Test Batch sizes "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [],
   "source": [
    "BATCH_SIZE = 32\n",
    "BATCH_SIZE_test = 32"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Legacy function used to read DTK tree representation"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [],
   "source": [
    "class TreeField(datx.Field):\n",
    "\t\tdef __init__(self, *args, **kwargs):\n",
    "\t\t\t\tsuper().__init__(*args, **kwargs)\n",
    "\n",
    "\t\tdef preprocess(self, x):\n",
    "\t\t\t\treturn x\n",
    "\n",
    "\t\tdef process(self, batch, device=None):\n",
    "\t\t\t\treturn torch.stack(batch)\n",
    "\n",
    "\n",
    "def unplickle_trees(path_tree_file):\n",
    "    print('--->read DTKs')\n",
    "    dt_trees = []\n",
    "    with open(path_tree_file, 'rb') as fr:\n",
    "        try:\n",
    "            while True:\n",
    "                dt_trees.append(pickle.load(fr))\n",
    "        except EOFError:\n",
    "            pass\n",
    "    return [torch.FloatTensor(i) for i in dt_trees]\n",
    "\n",
    "def add_parsed_tree(test, test_tree_list, field):\n",
    "\t\ttest_Examples_tree_list = []\n",
    "\t\tfor tr in test_tree_list:\n",
    "\t\t\t\ttree = datx.Example.fromlist([tr], [('Tree', field)])\n",
    "\t\t\t\ttest_Examples_tree_list.append(tree)\n",
    "\t\ttest.fields['Tree'] = field\n",
    "\t\tnew_test_examples_list = []\n",
    "\t\tfor example, tree_ex in zip(test.examples, test_Examples_tree_list):\n",
    "\t\t\t\tto_append = example\n",
    "\t\t\t\tto_append.Tree = tree_ex.Tree\n",
    "\t\t\t\tnew_test_examples_list.append(to_append)\n",
    "\t\ttest.examples = new_test_examples_list\n",
    "\t\treturn test\n",
    "    \n",
    "def first_tree(test, test_tree_list, field):\n",
    "        test_Examples_tree_list = []\n",
    "        tr = test_tree_list[0]\n",
    "        tree = datx.Example.fromlist([tr], [('Tree', field)])\n",
    "        test_Examples_tree_list.append(tree)\n",
    "        test.fields['Tree'] = field\n",
    "        new_test_examples_list = []\n",
    "        for example, tree_ex in zip(test.examples, test_Examples_tree_list):\n",
    "                to_append = example\n",
    "                to_append.Tree = tree_ex.Tree\n",
    "                new_test_examples_list.append(to_append)\n",
    "        test.examples = new_test_examples_list\n",
    "        return test"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [],
   "source": [
    "class UnprField(datx.Field):\n",
    "    def __init__(self, *args, **kwargs):\n",
    "        super().__init__(*args, **kwargs)\n",
    "\n",
    "    def preprocess(self, x):\n",
    "        return x\n",
    "\n",
    "    def process(self, batch, device=None):\n",
    "        return batch\n",
    "\n",
    "def Aduplicate_field(test, dataset_number):\n",
    "    #field = UnprField(sequential=False, use_vocab=False, batch_first=True)\n",
    "    field = TreeField(sequential=False, use_vocab=False, batch_first=True)\n",
    "    test.fields['Text_pntr'] = field\n",
    "    new_test_examples_list = []\n",
    "    for index, example in enumerate(test.examples):\n",
    "        to_append = example\n",
    "        to_append.Text_pntr = torch.tensor([dataset_number, index])\n",
    "        new_test_examples_list.append(to_append)\n",
    "    test.examples = new_test_examples_list\n",
    "    return test"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Tokenize step\n",
    "\n",
    "### Defining Bert Tokenizer, preprocessing steps and Max length"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [],
   "source": [
    "MAX_LEN = 125\n",
    "\n",
    "tokenizer = transformers.AutoTokenizer.from_pretrained('bert-base-uncased')\n",
    "pad_index = tokenizer.convert_tokens_to_ids(tokenizer.pad_token)\n",
    "\n",
    "TEXT = datx.Field(use_vocab=False,fix_length=MAX_LEN, tokenize=tokenizer.encode, sequential=True,\n",
    "                  pad_token=pad_index, batch_first=True)\n",
    "\n",
    "TEXT.preprocessing = pre_processing\n",
    "\n",
    "LABEL = datx.Field(sequential=False, use_vocab=False, batch_first=True)\n",
    "TREE = TreeField(sequential=False, use_vocab=False, batch_first=True)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### This function transforms input dataset to train ready data structure"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [],
   "source": [
    "def dataset_to_train(dataset, dataPath,tree1, tree2, extra_trees = [], exclude = 0):\n",
    "    \"\"\"\n",
    "    Prende in unput la stringa del dataset e restituisce tupla (train_iter, test_iter, vocab)\n",
    "    \"\"\"\n",
    "\n",
    "    LABEL = datx.Field(sequential=False, use_vocab=False, batch_first=True)\n",
    "\n",
    "    \n",
    "    #fields=[('gold_label', LABEL),('sentence1', None),('sentence2', None)]\n",
    "    fields=[(\"id\",None),('Label', LABEL),('Text', TEXT)]\n",
    "\n",
    "    \n",
    "    test = datx.TabularDataset(path=f'{dataset}.csv', format='csv',fields=fields, skip_header=True)\n",
    "\n",
    "    test_trees_list_sentence1 = unplickle_trees(f''+dataPath+''+tree1+'')\n",
    "    test_trees_list_sentence2 = unplickle_trees(f''+dataPath+''+tree2+'')\n",
    "    \n",
    "    print(len(test_trees_list_sentence1))\n",
    "    print(len(test_trees_list_sentence2))\n",
    "\n",
    "    \n",
    "    if len(extra_trees) != 0:\n",
    "        for index in range(len(extra_trees[0])):\n",
    "            test_trees_list_sentence1.append(extra_trees[0][index])\n",
    "            test_trees_list_sentence2.append(extra_trees[1][index])\n",
    "    \n",
    "    if exclude != 0:\n",
    "        test_trees_list_sentence1 = test_trees_list_sentence1[exclude:]\n",
    "        test_trees_list_sentence2 = test_trees_list_sentence2[exclude:]\n",
    "\n",
    "\n",
    "    \n",
    "    #test = duplicate_field(test, dataset_number)\n",
    "    tree_vectors = []\n",
    "    \n",
    "    ## concatenating DTK trees vectors of the two sentences\n",
    "    for vec in range(len(test_trees_list_sentence1)):\n",
    "        tree_vectors.append(torch.cat((test_trees_list_sentence1[vec],test_trees_list_sentence2[vec])))\n",
    "        \n",
    "    train =  add_parsed_tree(test, tree_vectors, TREE)\n",
    "    \n",
    "    train_iter, a = datx.Iterator.splits(\n",
    "            (train, _), sort_key=lambda x: len(x.Text),\n",
    "            batch_sizes=(BATCH_SIZE, 1))\n",
    "\n",
    "    return (train_iter, a)\n",
    "\n",
    "def dataset_to_test(dataset, dataPath, nameTree):\n",
    "    \"\"\"\n",
    "    Prende in unput la stringa del dataset e restituisce tupla (train_iter, test_iter, vocab)\n",
    "    \"\"\"\n",
    "\n",
    "    LABEL = datx.Field(sequential=False, use_vocab=False, batch_first=True)\n",
    "\n",
    "    \n",
    "    fields=[(\"id\",None),('Label', LABEL),('Text', TEXT)]\n",
    "    \n",
    "    \n",
    "    test = datx.TabularDataset(path=f'{dataset}.csv', format='csv',fields=fields, skip_header=False)\n",
    "\n",
    "    test_trees_list_sentence1 = unplickle_trees(f''+dataPath+''+nameTree1+'')\n",
    "    test_trees_list_sentence2 = unplickle_trees(f''+dataPath+''+nameTree2+'')\n",
    "\n",
    "    \n",
    "    dataset_number = 1\n",
    "    tree_vectors = []\n",
    "    \n",
    "    for vec in range(len(test_trees_list_sentence1)):\n",
    "        tree_vectors.append(torch.cat((test_trees_list_sentence1[vec],test_trees_list_sentence2[vec])))\n",
    "        \n",
    "    test = add_parsed_tree(test, tree_vectors, TREE)\n",
    "    train = test\n",
    "\n",
    "    train_iter, a = datx.Iterator.splits(\n",
    "            (train, _), sort_key=lambda x: len(x.Text),\n",
    "            batch_sizes=(BATCH_SIZE_test, 1), repeat=True)\n",
    "\n",
    "    return (train_iter, a)\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [],
   "source": [
    "def get_layer_pred(x_synth):\n",
    "\n",
    "    x_sem = torch.zeros(1, 51).cuda()\n",
    "    x_sem = torch.tensor(x_sem).to(torch.int64)\n",
    "    pred_value = model.get_activation(x_sem, x_synth)\n",
    "        \n",
    "        \n",
    "    return pred_value\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "--->read DTKs\n",
      "--->read DTKs\n",
      "60000\n",
      "60000\n"
     ]
    }
   ],
   "source": [
    "datasets_test = [ dataPath+'test_labeled']\n",
    "\n",
    "test_list = [] #each element is a tuple (train, test)\n",
    "\n",
    "\n",
    "for dat in datasets_test:\n",
    "    test_list.append(dataset_to_train(dat, dataPath, nameTree1_test, nameTree2_test))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [],
   "source": [
    "#MODEL TYPES\n",
    "BERT_ONLY = \"Bert_Only\"\n",
    "BERT_KERMIT = \"Bert+Kermit\"\n",
    "BERT_KERMIT_NO_REL = \"Bert+Kermit_NO_REL\"\n",
    "BERT_KERMIT_MULTILAYER = \"Bert+Kermit_Multi_Layer\""
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Training function\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [],
   "source": [
    "import numpy as np\n",
    "\n",
    "def train(train_iter, dataset_name, EPOCH, ext, model_name= \"Bert+Kermit\"):\n",
    "    \n",
    "    contEp = 0\n",
    "    lung = len(train_iter) # Usata per calcolare l'accuracy\n",
    "    print(lung)\n",
    "    accs = []\n",
    "    total, correct = 0, 0\n",
    "    for epoc in (range(EPOCH)):\n",
    "        f = open(f\"results_{model_name + ext}.txt\",\"a\")\n",
    "        contEp += 1\n",
    "        running_loss = 0\n",
    "        train_acc = 0\n",
    "        tot = []\n",
    "        for elem in tqdm(iter(train_iter)):\n",
    "            x_sem = elem.Text.cuda()\n",
    "            x_synth = elem.Tree.cuda()\n",
    "            target = elem.Label.cuda()\n",
    "            \n",
    "            if model_name == BERT_ONLY:\n",
    "                target_hat = model(x_sem)\n",
    "            else:\n",
    "                target_hat = model(x_sem, x_synth)\n",
    "                \n",
    "            loss = criterion(target_hat, target)\n",
    "            optimizer.zero_grad()\n",
    "            loss.backward()\n",
    "            optimizer.step()\n",
    "            running_loss += loss.item()\n",
    "          \n",
    "            correct += (torch.exp(target_hat).argmax(1) == target).sum().item()\n",
    "            \n",
    "            target_hat = list(torch.exp(target_hat).argmax(1))\n",
    "            #BINARY FUNCTIONS\n",
    "            d = []\n",
    "            total += target.size(0)      \n",
    "        \n",
    "        print(\"Epoch: \" , contEp)\n",
    "        print(\"Loss: \" + str(running_loss / lung))\n",
    "        print(f\"Accuracy: { 100 * correct / total}\")\n",
    "        #TEST\n",
    "        print(\"===================================\")        \n",
    "        print(\"Measure on test\\n\")        \n",
    "        test_accuracies_NO_mem = []\n",
    "        \n",
    "        for elem, dataset_name in zip(test_list, datasets_test):\n",
    "            print(f\"Testing dataset: {dataset_name}\")\n",
    "            test_accuracies_NO_mem.append(infer(elem[0], model, dataset_name, model_name = model_name, ext = ext))\n",
    "        print(\"===================================\")        \n",
    "        ########\n",
    "        \n",
    "        f.write(f'Epoch: {contEp}\\tLoss: {str(running_loss / lung)}\\tAccuracy: { 100 * correct / total}\\n')\n",
    "        f.close()\n",
    "    return accs"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Inference function"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [],
   "source": [
    "def infer(test_iter, neural_model,dataset_name , ext,EPOCH=1, L=30, lambda_norm=0.001, model_name= \"Bert+Kermit\"):\n",
    "    \n",
    "    running_loss = 0\n",
    "    train_acc = 0\n",
    "    lung = len(test_iter)\n",
    "    tot = []\n",
    "    preds = []\n",
    "    targets = []\n",
    "    indices = []\n",
    "    \n",
    "    #Creo copia del modello addestrato\n",
    "    neural_model.cuda()\n",
    "    f = open(f\"results_{model_name + ext}.txt\",\"a\")\n",
    "\n",
    "    criterion = nn.NLLLoss()\n",
    "    #TODO change learning rate for inference time \n",
    "    optimizer = optim.AdamW(neural_model.parameters(), lr=2e-3)\n",
    "    total, correct = 0, 0\n",
    "\n",
    "    for elem in tqdm(iter(test_iter)):\n",
    "        x_sem = elem.Text.cuda()\n",
    "        x_synth = elem.Tree.cuda()\n",
    "        target = elem.Label.cuda()\n",
    "        \n",
    "        with torch.torch.no_grad():\n",
    "            if model_name == BERT_ONLY:\n",
    "                target_hat = neural_model(x_sem)\n",
    "            else:\n",
    "                target_hat = neural_model(x_sem, x_synth)\n",
    "            loss = criterion(target_hat, target)\n",
    "            running_loss += loss.item()\n",
    "            targets.append(target.tolist())\n",
    "            preds.append(target_hat.exp().argmax(dim=1).tolist())\n",
    "            np.concatenate(targets, axis=0)\n",
    "            np.concatenate(preds, axis=0)\n",
    "\n",
    "\n",
    "        \n",
    "        #train_acc += (torch.exp(target_hat).argmax(1) == target).sum().item()\n",
    "        #res = [1 if x == True else 0 for x in list(torch.exp(target_hat).argmax(1) == target)]      \n",
    "        #BINARY FUNCTIONS\n",
    "                    \n",
    "        correct += (torch.exp(target_hat).argmax(1) == target).sum().item()\n",
    "            \n",
    "        target_hat = list(torch.exp(target_hat).argmax(1))\n",
    "        #BINARY FUNCTIONS\n",
    "        d = []\n",
    "        total += target.size(0)\n",
    "\n",
    "\n",
    "        '''       \n",
    "         d = []\n",
    "        for e in (torch.round(target_hat)==target):\n",
    "            if e[0]:\n",
    "                d.append(1)\n",
    "            else:\n",
    "                d.append(0)\n",
    "        d = torch.tensor(d)\n",
    "        \n",
    "        train_acc += d.sum().item()\n",
    "        res = d\n",
    "        tot += res.tolist()\n",
    "        '''\n",
    "    print(\"Loss: \" + str(running_loss / lung))\n",
    "    print(f\"Accuracy: { 100 * correct / total}\")\n",
    "\n",
    "    f.write(f'TEST:\\nLoss: {str(running_loss / lung)}\\tAccuracy: { 100 * correct / total}\\n')\n",
    "    f.close()\n",
    "    return(100 * correct / total), preds, targets\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Defining Bert Model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "import torch\n",
    "from torch import nn\n",
    "import torch.nn.functional as F\n",
    "\n",
    "class DTBertPlusLayer(nn.Module):\n",
    "    def __init__(self, input_dim_bert, input_dim_dt, output_dim):\n",
    "        super().__init__()\n",
    "        self.bert = transformers.BertModel.from_pretrained('bert-base-cased', force_download= True).to(\"cuda\" if torch.cuda.is_available() else \"cpu\")\n",
    "        #self.synth_sem_linear = nn.Linear(input_dim_bert + input_dim_dt, 1)\n",
    "        \n",
    "        #DEEP CLASSIFIER\n",
    "        self.synth_sem_linear = nn.Linear(input_dim_bert + input_dim_dt, 256)\n",
    "        self.fc1 = nn.Linear(in_features= 256, out_features=124)\n",
    "        self.fc2 = nn.Linear(in_features=124, out_features=28)\n",
    "        self.out = nn.Linear(in_features=28, out_features=1)\n",
    "        \n",
    "        #self.activation = {}\n",
    "        \n",
    "    def forward(self, x_sem, x_synth):\n",
    "\n",
    "        x_sem = self.bert(x_sem)[0][:, 0, :]\n",
    "        x_tot = torch.cat((x_sem, x_synth), 1)\n",
    "        \n",
    "        #x_tot = self.synth_sem_linear(x_tot)\n",
    "        #out = F.sigmoid(x_tot, dim=1)\n",
    "        \n",
    "        #return out\n",
    "        \n",
    "        #DEEP CLASSIFIER\n",
    "        t = F.relu(self.synth_sem_linear(x_tot))\n",
    "        t = F.relu(self.fc1(t))\n",
    "        t = F.relu(self.fc2(t))\n",
    "        t = F.sigmoid(self.out(t))\n",
    "        return t\n",
    "        \n",
    "       \n",
    "        \n",
    "    def get_activation(self, x_sem, x_synth):\n",
    "        with torch.no_grad():\n",
    "            x_sem = self.bert(x_sem)[0][:, 0, :]\n",
    "            x_tot = torch.cat((x_sem, x_synth), 1)\n",
    "            x_tot = self.synth_sem_linear(x_tot)\n",
    "            out = F.sigmoid(x_tot, dim=1)\n",
    "        return out\n",
    "    \n",
    "class DTBert(nn.Module):\n",
    "    def __init__(self, input_dim_bert, input_dim_dt, output_dim):\n",
    "        super().__init__()\n",
    "        self.bert = transformers.BertModel.from_pretrained('bert-base-uncased', force_download= True).to(\"cuda\" if torch.cuda.is_available() else \"cpu\")\n",
    "        #self.synth_sem_linear = nn.Linear(input_dim_bert + input_dim_dt, 1)\n",
    "        \n",
    "        #DEEP CLASSIFIER\n",
    "        self.synth_sem_linear = nn.Linear(input_dim_bert + input_dim_dt, 2)\n",
    "        \n",
    "        #self.activation = {}\n",
    "        \n",
    "    def forward(self, x_sem, x_synth):\n",
    "        x_sem = self.bert(x_sem)[0][:, 0, :]\n",
    "        x_tot = torch.cat((x_sem, x_synth), 1)\n",
    "        \n",
    "        #x_tot = self.synth_sem_linear(x_tot)\n",
    "        #out = F.sigmoid(x_tot, dim=1)\n",
    "        \n",
    "        #return out\n",
    "        \n",
    "        #DEEP CLASSIFIER\n",
    "        t = F.log_softmax(self.synth_sem_linear(x_tot))\n",
    "        return t\n",
    "        \n",
    "    def get_activation(self, x_sem, x_synth):\n",
    "        with torch.no_grad():\n",
    "            x_sem = self.bert(x_sem)[0][:, 0, :]\n",
    "            x_tot = torch.cat((x_sem, x_synth), 1)\n",
    "            x_tot = self.synth_sem_linear(x_tot)\n",
    "            out = F.log_softmax(x_tot, dim=1)\n",
    "        return out\n",
    "\n",
    "class Bert(nn.Module):\n",
    "    def __init__(self, input_dim_bert, output_dim):\n",
    "        super().__init__()\n",
    "        self.bert = transformers.BertModel.from_pretrained('bert-base-uncased', force_download = True).to(\"cuda\" if torch.cuda.is_available() else \"cpu\")\n",
    "        self.synth_sem_linear = nn.Linear(input_dim_bert, 2)\n",
    "        \n",
    "    def forward(self, x_sem):\n",
    "        x_sem = self.bert(x_sem)[0][:, 0, :]\n",
    "        x_tot = self.synth_sem_linear(x_sem)\n",
    "        out = F.log_softmax((x_tot), dim=1)\n",
    "        return out\n",
    "        "
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Defining Bert vector size and tree vector size (tree1+tree2)\n",
    "### Loading choosed Bert Model\n",
    "### Defining Cost function and optimizer"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "e603a36e7d574c7bb12e823055db87be",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=0, description='Downloading', max=433, style=ProgressStyle(description_width=…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "be79c5dbdc6f4a348059e17e24bce173",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=0, description='Downloading', max=440473133, style=ProgressStyle(description_…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "BERT_DIM = 768\n",
    "TREE_DIM = 8000\n",
    "\n",
    "OUTPUT_DIM = 2\n",
    "\n",
    "\n",
    "#model = DTBert(BERT_DIM, TREE_DIM, OUTPUT_DIM)        \n",
    "\n",
    "# Defining bert model\n",
    "model = DTBert(BERT_DIM, TREE_DIM, OUTPUT_DIM)        \n",
    "#model = DTBert(BERT_DIM, TREE_DIM,1)        \n",
    "#model = Bert(BERT_DIM, 2)        \n",
    "\n",
    "model.cuda()\n",
    "\n",
    "# Loss function\n",
    "#criterion = nn.NLLoss()\n",
    "\n",
    "#BINARY LOSS\n",
    "criterion = nn.NLLLoss()\n",
    "\n",
    "# Optimizer\n",
    "optimizer = optim.AdamW(model.parameters(), lr=5e-5)\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Training Step"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Now we transform initial dataset to train ready data structure"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "'''\n",
    "If you want to add additional trees to training set from another pickle\n",
    "you can use this function. \n",
    "\n",
    "You can also take only a subrange with  subrange input\n",
    "'''\n",
    "\n",
    "def add_tree_data(tree_path1, tree_path2, subrange=0):  \n",
    "    et1 = unplickle_trees(tree_path1)\n",
    "    et2 = unplickle_trees(tree_path2)\n",
    "    if subrange == 0:\n",
    "        subrange = len(et1)\n",
    "    \n",
    "    extra_trees = [et1[:subrange], et2[:subrange]]\n",
    "    return extra_trees\n",
    "\n",
    "#extra_trees = add_tree_data(f''+dataPath+''+nameTree1_test+'',f''+dataPath+''+nameTree2_test+'', 6000)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "datasets_train = [ dataPath+'training_labeled']\n",
    "\n",
    "train_list = [] #each element is a tuple (train, test)\n",
    "\n",
    "\n",
    "for dat in datasets_train:\n",
    "    train_list.append(dataset_to_train(dat, dataPath,nameTree1_train, nameTree2_train))\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "EPOCH = 4\n",
    "\n",
    "test_accuracies_NO_mem = []\n",
    "\n",
    "SEEDS = [46, 68, 95, 335, 660]\n",
    "#SEEDS = [68]\n",
    "\n",
    "for seed in SEEDS:\n",
    "    \n",
    "    torch.manual_seed(seed)\n",
    "    torch.cuda.manual_seed(seed)\n",
    "    #torch.cuda.manual_seed_all(seed)  # if you are using multi-GPU.\n",
    "    np.random.seed(seed)  # Numpy module.\n",
    "    random.seed(seed)  # Python random module.\n",
    "    torch.manual_seed(seed)\n",
    "    torch.backends.cudnn.benchmark = True\n",
    "    torch.backends.cudnn.deterministic = True\n",
    "    \n",
    "    # download bert model \n",
    "    model = DTBert(BERT_DIM,TREE_DIM, 2)        \n",
    "    model.cuda()\n",
    "    # Loss function\n",
    "    criterion = nn.NLLLoss()\n",
    "    # Optimizer\n",
    "    optimizer = optim.AdamW(model.parameters(), lr=5e-5)\n",
    "    \n",
    "    for elem, dataset_name in zip(train_list, datasets_train):\n",
    "        print(f\"Training dataset: {dataset_name}\")\n",
    "        train(elem[0], dataset_name, EPOCH, f\"_reproducibility_seed_{seed}\", BERT_KERMIT_NO_REL)\n",
    "    \n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# num of train epochs\n",
    "#EPOCH = 4\n",
    "#test_accuracies_NO_mem = []\n",
    "\n",
    "#SEEDS = [5, 7, 9, 23, 31]\n",
    "\n",
    "#train loop\n",
    "#for elem, dataset_name in zip(train_list, datasets_train):\n",
    "#    for seed in SEEDS:\n",
    "        # download bert model \n",
    "#        model = DTBert(BERT_DIM,TREE_DIM, 2)        \n",
    "#        model.cuda()\n",
    "        # Loss function\n",
    "#        criterion = nn.NLLLoss()\n",
    "        # Optimizer\n",
    "#        optimizer = optim.AdamW(model.parameters(), lr=5e-5)\n",
    "\n",
    "#        accuracies = train(elem[0], dataset_name, EPOCH, f\"_reproducibility_seed_{seed}\", BERT_KERMIT_NO_REL)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Train the model\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "# num of train epochs\n",
    "EPOCH = 4\n",
    "test_accuracies_NO_mem = []\n",
    "\n",
    "SEEDS = [5, 7, 9, 23, 31]\n",
    "\n",
    "#train loop\n",
    "for elem, dataset_name in zip(train_list, datasets_train):\n",
    "    for seed in SEEDS:\n",
    "        \n",
    "        # download bert model \n",
    "        model = DTBert(BERT_DIM,TREE_DIM, 2)        \n",
    "        model.cuda()\n",
    "        # Loss function\n",
    "        criterion = nn.NLLLoss()\n",
    "        # Optimizer\n",
    "        optimizer = optim.AdamW(model.parameters(), lr=5e-5)\n",
    "            \n",
    "        accuracies = train(elem[0], dataset_name, EPOCH, f\"_reproducibility_seed_{seed}\", BERT_KERMIT_NO_REL)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "!pip install seaborn\n",
    "import seaborn as sns\n",
    "import matplotlib as mpl\n",
    "import matplotlib.pyplot as plt"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "plt.plot(accuracies)\n",
    "plt.title('model accuracy')\n",
    "plt.ylabel('accuracy')\n",
    "plt.xlabel('epoch')\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def get_layer_pred(x_synth):\n",
    "\n",
    "    x_sem = torch.zeros(1, 51).cuda()\n",
    "    x_sem = torch.tensor(x_sem).to(torch.int64)\n",
    "    pred_value = model.get_activation(x_sem, x_synth)\n",
    "        \n",
    "        \n",
    "    return pred_value\n",
    "\n",
    "x_synt1 = torch.rand(1, 8000).cuda()\n",
    "\n",
    "print(get_layer_pred(x_synt1))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "datasets_test = [ dataPath+'test_labeled']\n",
    "\n",
    "test_list = [] #each element is a tuple (train, test)\n",
    "\n",
    "\n",
    "for dat in datasets_test:\n",
    "    test_list.append(dataset_to_train(dat, dataPath, nameTree1_test, nameTree2_test))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "test_accuracies_NO_mem = []\n",
    "\n",
    "for elem, dataset_name in zip(test_list, datasets_test):\n",
    "    print(f\"Testing dataset: {dataset_name}\")\n",
    "    test_accuracies_NO_mem.append(infer(elem[0], model, dataset_name))\n",
    "print(\"===================================\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def get_preds(data, name):\n",
    "    predictions = []\n",
    "    l = []\n",
    "    index = 0\n",
    "    acc, preds, targets = infer(data, model, name) #acc, preds, target\n",
    "    preds = np.concatenate(preds, axis=0).tolist()\n",
    "    targets = np.concatenate(targets, axis=0).tolist()\n",
    "\n",
    "    for el in preds:\n",
    "        predictions.append(f'Index: {index}\\tCorrect: {targets[index]}\\tPrediction: {el}\\n')\n",
    "        l.append(el)\n",
    "        index += 1\n",
    "            \n",
    "    return predictions, l, targets"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "for elem, dataset_name in zip(test_list, datasets_test):\n",
    "    ps, labels, targets = get_preds(elem[0], \"Test\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "np.unique(np.array(labels), return_counts=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "test_data = pd.read_csv(dataset_test, delimiter = \"\\t\")\n",
    "classes = {0:\"non-entailment\", 1:\"entailment\"}\n",
    "labels = list(map(lambda x: classes[x], labels))\n",
    "test_data[\"prediction\"] = labels\n",
    "test_data[\"gl\"] = list(map(lambda x: classes[x], targets))\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "test_data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "data_to_ann = pd.read_csv(\"data_to_annotate.csv\")\n",
    "data_to_ann_to_test = "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "np.unique(test_data[\"prediction\"].values, return_counts=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def to_visualize(df):\n",
    "    subsequence = df[df.heuristic==\"subsequence\"].sample(2)\n",
    "    constituent = df[df.heuristic==\"constituent\"].sample(2)\n",
    "    lexical_overlap = df[df.heuristic==\"lexical_overlap\"].sample(2)\n",
    "    return subsequence.append([constituent, lexical_overlap])\n",
    "\n",
    "\n",
    "moredata = to_visualize(test_data[test_data[\"prediction\"] == \"non_entailment\"])\n",
    "#viz_data.to_csv(\"data_to_visualize.csv\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "id": "QbiYXgZ03jGw",
    "outputId": "a6976cc0-a44d-4be6-a8d6-ec3a8a594170"
   },
   "outputs": [],
   "source": [
    "data_to_analyze = test_data[test_data.heuristic != \"subsequence\"]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "data_to_analyze.sample(200).to_csv(\"data_to_annotate.csv\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "test_data.values[20001]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def save_weights(model):\n",
    "    torch.save(model, f'Weights_{BERT_ONLY}.pt')\n",
    "#model = torch.load('./modelli/BERT_DT.pt')\n",
    "\n",
    "save_weights(model)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "test.head(30).values[2]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "print(test_accuracies_NO_mem)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "with open('pred_DTBERT.txt', mode=\"w\") as outfile: \n",
    "    for s in test_accuracies_NO_mem[0][1]:\n",
    "        outfile.write(\"%s\\n\" % s)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "with open('accuracy_BERT+DT_70k_2.txt', mode=\"w\") as outfile: \n",
    "    for s in test_accuracies_NO_mem:\n",
    "        outfile.write(\"%s\\n\" % s)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#torch.save(model, 'BERT+DT.pt')\n",
    "\n",
    "#model = torch.load('./modelli/BERT_DT.pt')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "model = torch.load('./Weights_Bert+Kermit.pt')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.7"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}