Day4_4_Word2Vec_Modeling.ipynb

{"nbformat":4,"nbformat_minor":0,"metadata":{"colab":{"name":"Day4_Word2Vec_Modeling.ipynb","version":"0.3.2","provenance":[],"collapsed_sections":[]},"kernelspec":{"name":"python3","display_name":"Python 3"}},"cells":[{"cell_type":"code","metadata":{"id":"5oseaIbcX8vq","colab_type":"code","colab":{}},"source":["\n","import pandas as pd\n","import numpy as np\n","import re\n","import json\n","import gensim\n","from tensorflow.python.keras.preprocessing.sequence import pad_sequences"],"execution_count":0,"outputs":[]},{"cell_type":"code","metadata":{"id":"48iU4cJtXH6i","colab_type":"code","outputId":"6285a088-0969-4139-c69f-09b6d2ab2b96","executionInfo":{"status":"ok","timestamp":1561177999068,"user_tz":-540,"elapsed":8769,"user":{"displayName":"ChangWook Jun","photoUrl":"https://lh6.googleusercontent.com/-NHLAhAbNfPg/AAAAAAAAAAI/AAAAAAAAAEc/NDLQ8z7-qns/s64/photo.jpg","userId":"00685987924881157185"}},"colab":{"base_uri":"https://localhost:8080/","height":54}},"source":["from google.colab import auth\n","auth.authenticate_user()\n","\n","from google.colab import drive\n","drive.mount('/content/gdrive')"],"execution_count":0,"outputs":[{"output_type":"stream","text":["Drive already mounted at /content/gdrive; to attempt to forcibly remount, call drive.mount(\"/content/gdrive\", force_remount=True).\n"],"name":"stdout"}]},{"cell_type":"code","metadata":{"id":"GMBtPn2sXgde","colab_type":"code","colab":{}},"source":["DATA_IN_PATH = './data_in/'\n","DATA_OUT_PATH = './data_out/'"],"execution_count":0,"outputs":[]},{"cell_type":"markdown","metadata":{"id":"rJBFJfe7EpuL","colab_type":"text"},"source":[""]},{"cell_type":"code","metadata":{"id":"WdaXMFkdX-Ei","colab_type":"code","colab":{}},"source":["FILTERS = \"([~.,!?\\\"':;)(])\"\n","MAX_SEQUENCE_LENGTH = 31"],"execution_count":0,"outputs":[]},{"cell_type":"code","metadata":{"id":"lq4JvcjuX-Hi","colab_type":"code","colab":{}},"source":["train_data = pd.read_csv(DATA_IN_PATH + 'train.csv', encoding='utf-8')"],"execution_count":0,"outputs":[]},{"cell_type":"code","metadata":{"id":"oFI749C_X-KM","colab_type":"code","colab":{}},"source":["train_pos_data = train_data.loc[train_data['is_duplicate'] == 1]\n","train_neg_data = train_data.loc[train_data['is_duplicate'] == 0]\n","\n","class_difference = len(train_neg_data) - len(train_pos_data)\n","sample_frac = 1 - (class_difference / len(train_neg_data))\n","train_neg_data = train_neg_data.sample(frac = sample_frac)"],"execution_count":0,"outputs":[]},{"cell_type":"code","metadata":{"id":"BoiMLFIDX-Mr","colab_type":"code","colab":{}},"source":["train_data = pd.concat([train_neg_data, train_pos_data])"],"execution_count":0,"outputs":[]},{"cell_type":"code","metadata":{"id":"2_b0ipf2X-O7","colab_type":"code","colab":{}},"source":["change_filter = re.compile(FILTERS)\n","\n","questions1 = [str(s) for s in train_data['question1']]\n","questions2 = [str(s) for s in train_data['question2']]\n","\n","filtered_questions1 = list()\n","filtered_questions2 = list()\n","\n","for q in questions1:\n","     filtered_questions1.append(re.sub(change_filter, \"\", q).lower())\n","        \n","for q in questions2:\n","     filtered_questions2.append(re.sub(change_filter, \"\", q).lower())"],"execution_count":0,"outputs":[]},{"cell_type":"code","metadata":{"id":"mOFCN6TmawZD","colab_type":"code","colab":{}},"source":["def extract_questions(df):\n","    for data in df:\n","       yield gensim.utils.simple_preprocess(data)"],"execution_count":0,"outputs":[]},{"cell_type":"code","metadata":{"id":"Y49NKqDNa04i","colab_type":"code","colab":{}},"source":["documents_1 = list(extract_questions(filtered_questions1))\n","documents_2 = list(extract_questions(filtered_questions2))"],"execution_count":0,"outputs":[]},{"cell_type":"code","metadata":{"id":"NDVMGfKXcpkh","colab_type":"code","outputId":"e9c70915-18dc-450d-c782-3a45f01b675f","executionInfo":{"status":"ok","timestamp":1561178020853,"user_tz":-540,"elapsed":15685,"user":{"displayName":"ChangWook Jun","photoUrl":"https://lh6.googleusercontent.com/-NHLAhAbNfPg/AAAAAAAAAAI/AAAAAAAAAEc/NDLQ8z7-qns/s64/photo.jpg","userId":"00685987924881157185"}},"colab":{"base_uri":"https://localhost:8080/","height":34}},"source":["len(documents_1)"],"execution_count":0,"outputs":[{"output_type":"execute_result","data":{"text/plain":["298526"]},"metadata":{"tags":[]},"execution_count":11}]},{"cell_type":"code","metadata":{"id":"pkbWON4ScvFB","colab_type":"code","outputId":"59670d09-052f-4d61-e157-025eb7d3a271","executionInfo":{"status":"ok","timestamp":1561178020853,"user_tz":-540,"elapsed":15261,"user":{"displayName":"ChangWook Jun","photoUrl":"https://lh6.googleusercontent.com/-NHLAhAbNfPg/AAAAAAAAAAI/AAAAAAAAAEc/NDLQ8z7-qns/s64/photo.jpg","userId":"00685987924881157185"}},"colab":{"base_uri":"https://localhost:8080/","height":34}},"source":["len(documents_2)"],"execution_count":0,"outputs":[{"output_type":"execute_result","data":{"text/plain":["298526"]},"metadata":{"tags":[]},"execution_count":12}]},{"cell_type":"code","metadata":{"id":"uZZ9udYua07Z","colab_type":"code","outputId":"ec807fe9-390f-4c27-e3b6-70da7ef0f1b9","executionInfo":{"status":"ok","timestamp":1561178020854,"user_tz":-540,"elapsed":14768,"user":{"displayName":"ChangWook Jun","photoUrl":"https://lh6.googleusercontent.com/-NHLAhAbNfPg/AAAAAAAAAAI/AAAAAAAAAEc/NDLQ8z7-qns/s64/photo.jpg","userId":"00685987924881157185"}},"colab":{"base_uri":"https://localhost:8080/","height":34}},"source":["documents  = documents_1 + documents_2\n","len(documents)"],"execution_count":0,"outputs":[{"output_type":"execute_result","data":{"text/plain":["597052"]},"metadata":{"tags":[]},"execution_count":13}]},{"cell_type":"code","metadata":{"id":"qjJ7ZrJGa0-K","colab_type":"code","colab":{}},"source":["model = gensim.models.Word2Vec(documents, size=128)"],"execution_count":0,"outputs":[]},{"cell_type":"code","metadata":{"id":"8DtS4P3IlqVL","colab_type":"code","colab":{}},"source":["word_vectors = model.wv"],"execution_count":0,"outputs":[]},{"cell_type":"code","metadata":{"id":"WYENTaFBlta7","colab_type":"code","outputId":"95885254-2561-42cf-d436-05dc60a23ed2","executionInfo":{"status":"ok","timestamp":1561178107360,"user_tz":-540,"elapsed":84968,"user":{"displayName":"ChangWook Jun","photoUrl":"https://lh6.googleusercontent.com/-NHLAhAbNfPg/AAAAAAAAAAI/AAAAAAAAAEc/NDLQ8z7-qns/s64/photo.jpg","userId":"00685987924881157185"}},"colab":{"base_uri":"https://localhost:8080/","height":34}},"source":["print(\"Number of word vectors: {}\".format(len(word_vectors.vocab)))"],"execution_count":0,"outputs":[{"output_type":"stream","text":["Number of word vectors: 24259\n"],"name":"stdout"}]},{"cell_type":"code","metadata":{"id":"FuYtUeWol3bC","colab_type":"code","colab":{}},"source":["MAX_NB_WORDS = len(word_vectors.vocab)"],"execution_count":0,"outputs":[]},{"cell_type":"code","metadata":{"id":"76kWSnfBa1Ay","colab_type":"code","outputId":"4b6b0e56-1c9e-4854-a4c1-e3a869f3378f","executionInfo":{"status":"ok","timestamp":1561178321023,"user_tz":-540,"elapsed":188017,"user":{"displayName":"ChangWook Jun","photoUrl":"https://lh6.googleusercontent.com/-NHLAhAbNfPg/AAAAAAAAAAI/AAAAAAAAAEc/NDLQ8z7-qns/s64/photo.jpg","userId":"00685987924881157185"}},"colab":{"base_uri":"https://localhost:8080/","height":87}},"source":["model.train(documents, total_examples=len(documents), epochs=10)"],"execution_count":0,"outputs":[{"output_type":"stream","text":["WARNING: Logging before flag parsing goes to stderr.\n","W0622 04:35:33.249613 140049650235264 base_any2vec.py:1182] Effective 'alpha' higher than previous training cycles\n"],"name":"stderr"},{"output_type":"execute_result","data":{"text/plain":["(43165332, 60749550)"]},"metadata":{"tags":[]},"execution_count":18}]},{"cell_type":"code","metadata":{"id":"WcsdrsAWc5n3","colab_type":"code","outputId":"87e2cd4c-3071-4bbe-928b-2fedbf27ec86","executionInfo":{"status":"ok","timestamp":1561178321507,"user_tz":-540,"elapsed":184077,"user":{"displayName":"ChangWook Jun","photoUrl":"https://lh6.googleusercontent.com/-NHLAhAbNfPg/AAAAAAAAAAI/AAAAAAAAAEc/NDLQ8z7-qns/s64/photo.jpg","userId":"00685987924881157185"}},"colab":{"base_uri":"https://localhost:8080/","height":70}},"source":["model.save(DATA_IN_PATH+\"Quora-Question-Pairs.w2v\")"],"execution_count":0,"outputs":[{"output_type":"stream","text":["/usr/local/lib/python3.6/dist-packages/smart_open/smart_open_lib.py:398: UserWarning: This function is deprecated, use smart_open.open instead. See the migration notes for details: https://github.com/RaRe-Technologies/smart_open/blob/master/README.rst#migrating-to-the-new-open-function\n","  'See the migration notes for details: %s' % _MIGRATION_NOTES_URL\n"],"name":"stderr"}]},{"cell_type":"code","metadata":{"id":"kOOmj12et6gW","colab_type":"code","outputId":"0032bafc-7065-421d-aa39-44475d5ba8ba","executionInfo":{"status":"ok","timestamp":1561178322007,"user_tz":-540,"elapsed":175291,"user":{"displayName":"ChangWook Jun","photoUrl":"https://lh6.googleusercontent.com/-NHLAhAbNfPg/AAAAAAAAAAI/AAAAAAAAAEc/NDLQ8z7-qns/s64/photo.jpg","userId":"00685987924881157185"}},"colab":{"base_uri":"https://localhost:8080/","height":70}},"source":["vocab = list(model.wv.vocab)\n","word_list = model[vocab]"],"execution_count":0,"outputs":[{"output_type":"stream","text":["/usr/local/lib/python3.6/dist-packages/ipykernel_launcher.py:2: DeprecationWarning: Call to deprecated `__getitem__` (Method will be removed in 4.0.0, use self.wv.__getitem__() instead).\n","  \n"],"name":"stderr"}]},{"cell_type":"code","metadata":{"id":"6BOz4k3fw59G","colab_type":"code","outputId":"2ce68a29-dd36-4c19-829e-c52a058cb118","executionInfo":{"status":"ok","timestamp":1561178322008,"user_tz":-540,"elapsed":157159,"user":{"displayName":"ChangWook Jun","photoUrl":"https://lh6.googleusercontent.com/-NHLAhAbNfPg/AAAAAAAAAAI/AAAAAAAAAEc/NDLQ8z7-qns/s64/photo.jpg","userId":"00685987924881157185"}},"colab":{"base_uri":"https://localhost:8080/","height":34}},"source":["word_list[0].shape"],"execution_count":0,"outputs":[{"output_type":"execute_result","data":{"text/plain":["(128,)"]},"metadata":{"tags":[]},"execution_count":21}]},{"cell_type":"code","metadata":{"id":"jKhdHtDk1Kyn","colab_type":"code","outputId":"bda146cd-95bc-43b7-c7a2-047d9245c666","executionInfo":{"status":"ok","timestamp":1561133897121,"user_tz":-540,"elapsed":1118,"user":{"displayName":"ChangWook Jun","photoUrl":"https://lh6.googleusercontent.com/-NHLAhAbNfPg/AAAAAAAAAAI/AAAAAAAAAEc/NDLQ8z7-qns/s64/photo.jpg","userId":"00685987924881157185"}},"colab":{"base_uri":"https://localhost:8080/","height":34}},"source":["word_list.shape"],"execution_count":0,"outputs":[{"output_type":"execute_result","data":{"text/plain":["(24349, 128)"]},"metadata":{"tags":[]},"execution_count":41}]},{"cell_type":"code","metadata":{"id":"RuxNodJodoTj","colab_type":"code","outputId":"ff797cf9-e5f5-4683-9df0-f36280fcd4e8","executionInfo":{"status":"ok","timestamp":1561133684138,"user_tz":-540,"elapsed":88548,"user":{"displayName":"ChangWook Jun","photoUrl":"https://lh6.googleusercontent.com/-NHLAhAbNfPg/AAAAAAAAAAI/AAAAAAAAAEc/NDLQ8z7-qns/s64/photo.jpg","userId":"00685987924881157185"}},"colab":{"base_uri":"https://localhost:8080/","height":185}},"source":["!ls -al ./data_in/"],"execution_count":0,"outputs":[{"output_type":"stream","text":["total 570416\n","drwxr-xr-x 2 root root      4096 Jun 21 16:14 .\n","drwxr-xr-x 1 root root      4096 Jun 21 16:11 ..\n","-rw-r--r-- 1 root root  38993494 Jun 21 16:14 Quora-Question-Pairs.w2v\n","-rw-r--r-- 1 root root  22346871 Jun 21 16:11 sample_submission.csv\n","-rw------- 1 root root   5190322 Jun 21 16:11 sample_submission.csv.zip\n","-rw-r--r-- 1 root root 314015126 Jun 21 16:11 test.csv\n","-rw------- 1 root root 117931806 Jun 21 16:11 test.csv.zip\n","-rw-r--r-- 1 root root  63399110 Jun 21 16:11 train.csv\n","-rw------- 1 root root  22202741 Jun 21 16:11 train.csv.zip\n"],"name":"stdout"}]},{"cell_type":"code","metadata":{"id":"Vk8DSh3odtAt","colab_type":"code","outputId":"1463a63e-0f67-4da2-bb8f-1c044e07d1a5","executionInfo":{"status":"ok","timestamp":1561133684139,"user_tz":-540,"elapsed":86363,"user":{"displayName":"ChangWook Jun","photoUrl":"https://lh6.googleusercontent.com/-NHLAhAbNfPg/AAAAAAAAAAI/AAAAAAAAAEc/NDLQ8z7-qns/s64/photo.jpg","userId":"00685987924881157185"}},"colab":{"base_uri":"https://localhost:8080/","height":34}},"source":["len(model.wv.vocab)"],"execution_count":0,"outputs":[{"output_type":"execute_result","data":{"text/plain":["24349"]},"metadata":{"tags":[]},"execution_count":23}]},{"cell_type":"code","metadata":{"id":"UPQ0cg5Hgu4n","colab_type":"code","colab":{}},"source":["word2index = {token: token_index for token_index, token in enumerate(model.wv.index2word)}"],"execution_count":0,"outputs":[]},{"cell_type":"code","metadata":{"id":"paJazF1ogdsH","colab_type":"code","colab":{}},"source":["def sequences(documents):\n","  sequence_output_index = []\n","  for sequence in documents:\n","    #print(sequence)\n","    sequence_index = []\n","    for word in sequence:\n","      if word not in model.wv.vocab:\n","        continue\n","      sequence_index.append(word2index[word])\n","\n","    if len(sequence_index) > MAX_SEQUENCE_LENGTH:\n","      sequence_index = sequence_index[:MAX_SEQUENCE_LENGTH]\n","      \n","    #sequence_index += (MAX_SEQUENCE_LENGTH - len(sequence_index)) * [PAD]\n","    #print(sequence_index)\n","    sequence_output_index.append(sequence_index)    \n","  return np.asarray(sequence_output_index)\n","                     "],"execution_count":0,"outputs":[]},{"cell_type":"code","metadata":{"id":"Tq4arXTkgdvl","colab_type":"code","outputId":"bc68ae50-bebc-437b-8827-4dce50821a04","executionInfo":{"status":"error","timestamp":1561178323789,"user_tz":-540,"elapsed":57734,"user":{"displayName":"ChangWook Jun","photoUrl":"https://lh6.googleusercontent.com/-NHLAhAbNfPg/AAAAAAAAAAI/AAAAAAAAAEc/NDLQ8z7-qns/s64/photo.jpg","userId":"00685987924881157185"}},"colab":{"base_uri":"https://localhost:8080/","height":297}},"source":["q1_data = sequences(documents_1)\n","q2_data = sequences(documents_2)"],"execution_count":0,"outputs":[{"output_type":"error","ename":"NameError","evalue":"ignored","traceback":["\u001b[0;31m---------------------------------------------------------------------------\u001b[0m","\u001b[0;31mNameError\u001b[0m                                 Traceback (most recent call last)","\u001b[0;32m<ipython-input-23-4d228f3bec2e>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m()\u001b[0m\n\u001b[0;32m----> 1\u001b[0;31m \u001b[0mq1_data\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0msequences\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mdocuments_1\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m      2\u001b[0m \u001b[0mq2_data\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0msequences\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mdocuments_2\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n","\u001b[0;32m<ipython-input-22-0c2eeb237b5d>\u001b[0m in \u001b[0;36msequences\u001b[0;34m(documents)\u001b[0m\n\u001b[1;32m      7\u001b[0m       \u001b[0;32mif\u001b[0m \u001b[0mword\u001b[0m \u001b[0;32mnot\u001b[0m \u001b[0;32min\u001b[0m \u001b[0mmodel\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mwv\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mvocab\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m      8\u001b[0m         \u001b[0;32mcontinue\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m----> 9\u001b[0;31m       \u001b[0msequence_index\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mappend\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mword2index\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0mword\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m     10\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     11\u001b[0m     \u001b[0;32mif\u001b[0m \u001b[0mlen\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0msequence_index\u001b[0m\u001b[0;34m)\u001b[0m \u001b[0;34m>\u001b[0m \u001b[0mMAX_SEQUENCE_LENGTH\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n","\u001b[0;31mNameError\u001b[0m: name 'word2index' is not defined"]}]},{"cell_type":"code","metadata":{"id":"hGThdsgZrzFF","colab_type":"code","colab":{}},"source":["q1_data"],"execution_count":0,"outputs":[]},{"cell_type":"code","metadata":{"id":"yrh_981h6F5s","colab_type":"code","colab":{}},"source":["q1_data = pad_sequences(q1_data, maxlen=MAX_SEQUENCE_LENGTH, padding='post')\n","q2_data = pad_sequences(q2_data, maxlen=MAX_SEQUENCE_LENGTH, padding='post')"],"execution_count":0,"outputs":[]},{"cell_type":"code","metadata":{"id":"XC-81FgiqfNk","colab_type":"code","colab":{}},"source":["labels = np.array(train_data['is_duplicate'], dtype=int)"],"execution_count":0,"outputs":[]},{"cell_type":"code","metadata":{"id":"AuIpJyj_YPGT","colab_type":"code","colab":{}},"source":["data_configs = {}\n","data_configs['vocab'] = word2index\n","data_configs['vocab_size'] = len(word2index)"],"execution_count":0,"outputs":[]},{"cell_type":"code","metadata":{"id":"RiJ4LTWoYPJt","colab_type":"code","colab":{}},"source":["TRAIN_Q1_DATA = 'train_q1.npy'\n","TRAIN_Q2_DATA = 'train_q2.npy'\n","TRAIN_LABEL_DATA = 'train_label.npy'\n","DATA_CONFIGS = 'data_configs.json'\n","\n","np.save(open(DATA_IN_PATH + TRAIN_Q1_DATA, 'wb'), q1_data)\n","np.save(open(DATA_IN_PATH + TRAIN_Q2_DATA , 'wb'), q2_data)\n","np.save(open(DATA_IN_PATH + TRAIN_LABEL_DATA , 'wb'), labels)\n","\n","json.dump(data_configs, open(DATA_IN_PATH + DATA_CONFIGS, 'w'))"],"execution_count":0,"outputs":[]},{"cell_type":"code","metadata":{"id":"rtpO91sPq3FL","colab_type":"code","colab":{}},"source":["import sys\n","import tensorflow as tf\n","import numpy as np\n","import os\n","import pandas as pd\n","\n","from sklearn.model_selection import train_test_split\n","\n","import json"],"execution_count":0,"outputs":[]},{"cell_type":"code","metadata":{"id":"I2nZ2tQ1q9Ww","colab_type":"code","colab":{}},"source":["DATA_IN_PATH = './data_in/'\n","DATA_OUT_PATH = './data_out/'"],"execution_count":0,"outputs":[]},{"cell_type":"code","metadata":{"id":"L_pFS85Eq9Zj","colab_type":"code","outputId":"cf3ce21e-9e39-4e89-b9f4-ea801a63f954","executionInfo":{"status":"ok","timestamp":1561135655451,"user_tz":-540,"elapsed":4255,"user":{"displayName":"ChangWook Jun","photoUrl":"https://lh6.googleusercontent.com/-NHLAhAbNfPg/AAAAAAAAAAI/AAAAAAAAAEc/NDLQ8z7-qns/s64/photo.jpg","userId":"00685987924881157185"}},"colab":{"base_uri":"https://localhost:8080/","height":168}},"source":["!ls -al"],"execution_count":0,"outputs":[{"output_type":"stream","text":["total 40\n","drwxr-xr-x 1 root root 4096 Jun 21 16:11 .\n","drwxr-xr-x 1 root root 4096 Jun 21 16:01 ..\n","-rw-r--r-- 1 root root 2487 Jun 21 16:11 adc.json\n","drwxr-xr-x 1 root root 4096 Jun 21 16:11 .config\n","drwxr-xr-x 2 root root 4096 Jun 21 16:15 data_in\n","drwxr-xr-x 3 root root 4096 Jun 21 16:18 data_out\n","drwx------ 3 root root 4096 Jun 21 16:11 gdrive\n","drwxr-xr-x 1 root root 4096 Jun 18 16:14 sample_data\n"],"name":"stdout"}]},{"cell_type":"code","metadata":{"id":"ehwnQ-_Jq9cQ","colab_type":"code","colab":{}},"source":["TRAIN_Q1_DATA_FILE = 'train_q1.npy'\n","TRAIN_Q2_DATA_FILE = 'train_q2.npy'\n","TRAIN_LABEL_DATA_FILE = 'train_label.npy'\n","NB_WORDS_DATA_FILE = 'data_configs.json'\n","\n","## 학습에 필요한 파라메터들에 대해서 지정하는 부분이다.\n","\n","BATCH_SIZE = 4096\n","EPOCH = 2\n","HIDDEN = 64\n","BUFFER_SIZE = 10000\n","\n","NUM_LAYERS = 3\n","DROPOUT_RATIO = 0.3\n","\n","TEST_SPLIT = 0.1\n","RNG_SEED = 13371447\n","EMBEDDING_DIM = 128\n","MAX_SEQ_LEN = 31"],"execution_count":0,"outputs":[]},{"cell_type":"code","metadata":{"id":"krJ5Zychq9eo","colab_type":"code","colab":{}},"source":["## 데이터를 불러오는 부분이다. 효과적인 데이터 불러오기를 위해, 미리 넘파이 형태로 저장시킨 데이터를 로드한다.\n","\n","q1_data = np.load(open(DATA_IN_PATH + TRAIN_Q1_DATA_FILE, 'rb'))\n","q2_data = np.load(open(DATA_IN_PATH + TRAIN_Q2_DATA_FILE, 'rb'))\n","labels = np.load(open(DATA_IN_PATH + TRAIN_LABEL_DATA_FILE, 'rb'))\n","prepro_configs = None\n","\n","with open(DATA_IN_PATH + NB_WORDS_DATA_FILE, 'r') as f:\n","    prepro_configs = json.load(f)"],"execution_count":0,"outputs":[]},{"cell_type":"code","metadata":{"id":"oWl_E5Zjq9hC","colab_type":"code","outputId":"7804cc33-646f-47b6-ef43-df9af14b0218","executionInfo":{"status":"ok","timestamp":1561135657370,"user_tz":-540,"elapsed":538,"user":{"displayName":"ChangWook Jun","photoUrl":"https://lh6.googleusercontent.com/-NHLAhAbNfPg/AAAAAAAAAAI/AAAAAAAAAEc/NDLQ8z7-qns/s64/photo.jpg","userId":"00685987924881157185"}},"colab":{"base_uri":"https://localhost:8080/","height":34}},"source":["VOCAB_SIZE = prepro_configs['vocab_size']\n","VOCAB_SIZE"],"execution_count":0,"outputs":[{"output_type":"execute_result","data":{"text/plain":["24349"]},"metadata":{"tags":[]},"execution_count":83}]},{"cell_type":"code","metadata":{"id":"Ty8bnq6_q9jX","colab_type":"code","colab":{}},"source":["q1_data_len = np.array([min(len(x), MAX_SEQ_LEN) for x in q1_data])\n","q2_data_len = np.array([min(len(x), MAX_SEQ_LEN) for x in q2_data])"],"execution_count":0,"outputs":[]},{"cell_type":"code","metadata":{"id":"OqiHfNTdq9lw","colab_type":"code","colab":{}},"source":["## 데이터를 나누어 저장하자. sklearn의 train_test_split을 사용하면 유용하다. 하지만, 쿼라 데이터의 경우는\n","## 입력이 1개가 아니라 2개이다. 따라서, np.stack을 사용하여 두개를 하나로 쌓은다음 활용하여 분류한다.\n","\n","X = np.stack((q1_data, q2_data), axis=1)\n","y = labels\n","train_X, test_X, train_y, test_y = train_test_split(X, y, test_size=TEST_SPLIT, random_state=RNG_SEED)\n","\n","train_Q1 = train_X[:,0]\n","train_Q2 = train_X[:,1]\n","test_Q1 = test_X[:,0]\n","test_Q2 = test_X[:,1]"],"execution_count":0,"outputs":[]},{"cell_type":"code","metadata":{"id":"1_wxXI7x0qCm","colab_type":"code","outputId":"95c67877-496a-49ec-b482-b3a21c8a554c","executionInfo":{"status":"ok","timestamp":1561135662552,"user_tz":-540,"elapsed":710,"user":{"displayName":"ChangWook Jun","photoUrl":"https://lh6.googleusercontent.com/-NHLAhAbNfPg/AAAAAAAAAAI/AAAAAAAAAEc/NDLQ8z7-qns/s64/photo.jpg","userId":"00685987924881157185"}},"colab":{"base_uri":"https://localhost:8080/","height":134}},"source":["train_Q1"],"execution_count":0,"outputs":[{"output_type":"execute_result","data":{"text/plain":["array([[   3,    2,   53, ...,    0,    0,    0],\n","       [   2,  132, 1636, ...,    0,    0,    0],\n","       [   1,   56, 2486, ...,    0,    0,    0],\n","       ...,\n","       [  13,  715,   15, ...,    0,    0,    0],\n","       [   3,    6,   24, ...,    0,    0,    0],\n","       [   1,    6,   12, ...,    0,    0,    0]], dtype=int32)"]},"metadata":{"tags":[]},"execution_count":86}]},{"cell_type":"code","metadata":{"id":"78wN7Cvlq9oc","colab_type":"code","colab":{}},"source":["\n","def rearrange(base, hypothesis, labels):\n","    features = {\"base\": base, \"hypothesis\": hypothesis}\n","    return features, labels\n","\n","def train_input_fn():\n","    dataset = tf.data.Dataset.from_tensor_slices((train_Q1, train_Q2, train_y))\n","    dataset = dataset.shuffle(buffer_size=len(train_Q1))\n","    dataset = dataset.batch(BATCH_SIZE) #4096\n","    dataset = dataset.map(rearrange)\n","    dataset = dataset.repeat(EPOCH) # 2\n","    iterator = dataset.make_one_shot_iterator()\n","    \n","    return iterator.get_next()\n","\n","def eval_input_fn():\n","    dataset = tf.data.Dataset.from_tensor_slices((test_Q1, test_Q2, test_y))\n","    dataset = dataset.batch(BATCH_SIZE) #4096\n","    dataset = dataset.map(rearrange)\n","    iterator = dataset.make_one_shot_iterator()\n","    \n","    return iterator.get_next()"],"execution_count":0,"outputs":[]},{"cell_type":"code","metadata":{"id":"si8ol35hq9q1","colab_type":"code","colab":{}},"source":["\n","from tensorflow.python.keras.layers import Embedding\n","\n","def Malstm(features, labels, mode):\n","        \n","    TRAIN = mode == tf.estimator.ModeKeys.TRAIN\n","    EVAL = mode == tf.estimator.ModeKeys.EVAL\n","    PREDICT = mode == tf.estimator.ModeKeys.PREDICT\n","    # VOCAB_SIZE : 24349   EMBEDDING_DIM : 128   \n","    embedding = Embedding(VOCAB_SIZE,\n","                     EMBEDDING_DIM,\n","                     mask_zero=False,\n","                     weights=[word_list],\n","                     input_length=MAX_SEQUENCE_LENGTH,\n","                     trainable=False)\n","    \n","    base_embedded_matrix = embedding(features['base']) \n","    \n","    hypothesis_embedded_matrix = embedding(features['hypothesis']) \n","    \n","    q_lstm_cell = tf.nn.rnn_cell.BasicLSTMCell(num_units = HIDDEN, activation = tf.nn.tanh , state_is_tuple=False) \n","\n","    q, q_output_states = tf.nn.dynamic_rnn(cell = q_lstm_cell, \n","                                           inputs = base_embedded_matrix, \n","                                           dtype = tf.float32,\n","                                           scope='query')       \n","\n","    s_lstm_cell = tf.nn.rnn_cell.BasicLSTMCell(num_units = HIDDEN, activation = tf.nn.tanh, state_is_tuple=False) \n","    s, s_output_states = tf.nn.dynamic_rnn(cell = s_lstm_cell, \n","                                           inputs = hypothesis_embedded_matrix, \n","                                           dtype = tf.float32,\n","                                           scope='sim_query')     \n","\n","    with tf.variable_scope('output_layer'):\n","        logit_layer = tf.exp(-tf.reduce_sum(tf.abs(q_output_states - s_output_states), axis=1, keepdims=True)) \n","    \n","        logit_layer = tf.squeeze(logit_layer, axis=-1) \n","       \n","    if PREDICT:\n","        return tf.estimator.EstimatorSpec(\n","                  mode=mode,\n","                  predictions={\n","                      'is_duplicate':logit_layer\n","                  })\n","    \n","    #prediction 진행 시, None\n","    if labels is not None:\n","        labels = tf.to_float(labels)\n","    \n","    loss = tf.losses.mean_squared_error(labels=labels, predictions=logit_layer)\n","\n","    if EVAL:\n","        accuracy = tf.metrics.accuracy(labels, tf.round(logit_layer))\n","        eval_metric_ops = {'acc': accuracy}\n","        return tf.estimator.EstimatorSpec(\n","                  mode=mode,\n","                  eval_metric_ops= eval_metric_ops,\n","                  loss=loss)\n","\n","    elif TRAIN:\n","\n","        global_step = tf.train.get_global_step()\n","        train_op = tf.train.AdamOptimizer(1e-3).minimize(loss, global_step)\n","\n","        return tf.estimator.EstimatorSpec(\n","                  mode=mode,\n","                  train_op=train_op,\n","                  loss=loss)"],"execution_count":0,"outputs":[]},{"cell_type":"code","metadata":{"id":"WW0NTbCJq9th","colab_type":"code","colab":{}},"source":["!rm -rf /checkpoint/malstm/"],"execution_count":0,"outputs":[]},{"cell_type":"code","metadata":{"id":"pVCpnYMbrfkS","colab_type":"code","colab":{}},"source":["model_dir = os.path.join(os.getcwd(), DATA_OUT_PATH + \"/checkpoint/malstm/\")\n","os.makedirs(model_dir, exist_ok=True)\n","\n","config_tf = tf.estimator.RunConfig()\n","\n","lstm_est = tf.estimator.Estimator(Malstm, model_dir=model_dir)"],"execution_count":0,"outputs":[]},{"cell_type":"code","metadata":{"id":"0Y4uHTOgrg-7","colab_type":"code","outputId":"b729327a-a308-4f3f-c1fa-b678ff9587ca","executionInfo":{"status":"ok","timestamp":1561136047537,"user_tz":-540,"elapsed":378007,"user":{"displayName":"ChangWook Jun","photoUrl":"https://lh6.googleusercontent.com/-NHLAhAbNfPg/AAAAAAAAAAI/AAAAAAAAAEc/NDLQ8z7-qns/s64/photo.jpg","userId":"00685987924881157185"}},"colab":{"base_uri":"https://localhost:8080/","height":373}},"source":["lstm_est.train(train_input_fn)"],"execution_count":0,"outputs":[{"output_type":"stream","text":["embedding:  <tensorflow.python.keras.layers.embeddings.Embedding object at 0x7f33a7df9f60>\n"],"name":"stdout"},{"output_type":"stream","text":["W0621 16:47:52.906411 139861576632192 rnn_cell_impl.py:697] <tensorflow.python.ops.rnn_cell_impl.BasicLSTMCell object at 0x7f33a7dc9048>: Using a concatenated state is slower and will soon be deprecated.  Use state_is_tuple=True.\n","W0621 16:47:53.026030 139861576632192 rnn_cell_impl.py:697] <tensorflow.python.ops.rnn_cell_impl.BasicLSTMCell object at 0x7f33a7d9bf28>: Using a concatenated state is slower and will soon be deprecated.  Use state_is_tuple=True.\n"],"name":"stderr"},{"output_type":"stream","text":["base_embedded_matrix:  Tensor(\"embedding/embedding_lookup/Identity_1:0\", shape=(?, 31, 128), dtype=float32)\n","hypothesis_embedded_matrix:  Tensor(\"embedding_1/embedding_lookup/Identity_1:0\", shape=(?, 31, 128), dtype=float32)\n","q_lstm_cell:  <tensorflow.python.ops.rnn_cell_impl.BasicLSTMCell object at 0x7f33a7dc9048>\n","q:  Tensor(\"query/transpose_1:0\", shape=(?, 31, 64), dtype=float32)\n","q_output_states:  Tensor(\"query/while/Exit_3:0\", shape=(?, 128), dtype=float32)\n","s_lstm_cell:  <tensorflow.python.ops.rnn_cell_impl.BasicLSTMCell object at 0x7f33a7d9bf28>\n","s:  Tensor(\"sim_query/transpose_1:0\", shape=(?, 31, 64), dtype=float32)\n","s_output_states:  Tensor(\"sim_query/while/Exit_3:0\", shape=(?, 128), dtype=float32)\n","logit_layer:  Tensor(\"output_layer/Exp:0\", shape=(?, 1), dtype=float32)\n","logit_layer:  Tensor(\"output_layer/Squeeze:0\", shape=(?,), dtype=float32)\n"],"name":"stdout"},{"output_type":"stream","text":["W0621 16:47:54.611497 139861576632192 deprecation.py:323] From /usr/local/lib/python3.6/dist-packages/tensorflow/python/training/saver.py:1276: checkpoint_exists (from tensorflow.python.training.checkpoint_management) is deprecated and will be removed in a future version.\n","Instructions for updating:\n","Use standard file APIs to check for files with this prefix.\n","W0621 16:47:54.884833 139861576632192 deprecation.py:323] From /usr/local/lib/python3.6/dist-packages/tensorflow/python/training/saver.py:1066: get_checkpoint_mtimes (from tensorflow.python.training.checkpoint_management) is deprecated and will be removed in a future version.\n","Instructions for updating:\n","Use standard file utilities to get mtimes.\n"],"name":"stderr"},{"output_type":"execute_result","data":{"text/plain":["<tensorflow_estimator.python.estimator.estimator.Estimator at 0x7f33a7a4aeb8>"]},"metadata":{"tags":[]},"execution_count":91}]},{"cell_type":"code","metadata":{"id":"riaczfTdri3K","colab_type":"code","outputId":"285e165c-7eda-470c-d39a-947109117327","executionInfo":{"status":"ok","timestamp":1561136054212,"user_tz":-540,"elapsed":382039,"user":{"displayName":"ChangWook Jun","photoUrl":"https://lh6.googleusercontent.com/-NHLAhAbNfPg/AAAAAAAAAAI/AAAAAAAAAEc/NDLQ8z7-qns/s64/photo.jpg","userId":"00685987924881157185"}},"colab":{"base_uri":"https://localhost:8080/","height":272}},"source":["lstm_est.evaluate(eval_input_fn)"],"execution_count":0,"outputs":[{"output_type":"stream","text":["W0621 16:54:06.948600 139861576632192 rnn_cell_impl.py:697] <tensorflow.python.ops.rnn_cell_impl.BasicLSTMCell object at 0x7f33a36b9780>: Using a concatenated state is slower and will soon be deprecated.  Use state_is_tuple=True.\n"],"name":"stderr"},{"output_type":"stream","text":["embedding:  <tensorflow.python.keras.layers.embeddings.Embedding object at 0x7f33a79c5ef0>\n","base_embedded_matrix:  Tensor(\"embedding/embedding_lookup/Identity_1:0\", shape=(?, 31, 128), dtype=float32)\n","hypothesis_embedded_matrix:  Tensor(\"embedding_1/embedding_lookup/Identity_1:0\", shape=(?, 31, 128), dtype=float32)\n","q_lstm_cell:  <tensorflow.python.ops.rnn_cell_impl.BasicLSTMCell object at 0x7f33a36b9780>\n"],"name":"stdout"},{"output_type":"stream","text":["W0621 16:54:07.066392 139861576632192 rnn_cell_impl.py:697] <tensorflow.python.ops.rnn_cell_impl.BasicLSTMCell object at 0x7f33a497be10>: Using a concatenated state is slower and will soon be deprecated.  Use state_is_tuple=True.\n"],"name":"stderr"},{"output_type":"stream","text":["q:  Tensor(\"query/transpose_1:0\", shape=(?, 31, 64), dtype=float32)\n","q_output_states:  Tensor(\"query/while/Exit_3:0\", shape=(?, 128), dtype=float32)\n","s_lstm_cell:  <tensorflow.python.ops.rnn_cell_impl.BasicLSTMCell object at 0x7f33a497be10>\n","s:  Tensor(\"sim_query/transpose_1:0\", shape=(?, 31, 64), dtype=float32)\n","s_output_states:  Tensor(\"sim_query/while/Exit_3:0\", shape=(?, 128), dtype=float32)\n","logit_layer:  Tensor(\"output_layer/Exp:0\", shape=(?, 1), dtype=float32)\n","logit_layer:  Tensor(\"output_layer/Squeeze:0\", shape=(?,), dtype=float32)\n"],"name":"stdout"},{"output_type":"execute_result","data":{"text/plain":["{'acc': 0.70883995, 'global_step': 132, 'loss': 0.19398028}"]},"metadata":{"tags":[]},"execution_count":92}]},{"cell_type":"code","metadata":{"id":"9hvO4v69rk4K","colab_type":"code","colab":{}},"source":[""],"execution_count":0,"outputs":[]}]}