lecture_chap11_homework_pub.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# 第11回講義 宿題"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 課題. Attentionを用いたキャプション生成モデルを実装せよ"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 注意"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "- homework関数を完成させて提出してください\n",
    "    - 訓練データのtrain_X, train_yのみが与えられます\n",
    "    - train_Xとtrain_yをtrain_X, valid_xとtrain_y, valid_yに分けるなどしてモデルを学習させてください\n",
    "    - **cost関数を戻り値**としてください\n",
    "- **test_X, test_yに対する交差エントロピー(負の対数尤度)の平均で評価**します\n",
    "- **lecture_chap11_imgcap_exercise_pub.ipynb**でダウンロードしたデータを使用します\n",
    "- 全体の実行時間がiLect上で120分を超えないようにしてください\n",
    "- homework関数の外には何も書かないでください (必要なものは全てhomework関数に入れてください)\n",
    "- 解答提出時には Answer Cell の内容のみを提出してください"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "次のセルのhomework関数を完成させて提出してください"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Answer Cell"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def homework(train_X, train_y):\n",
    "    global vocab_size, sess, x, d # 下記Checker Cell内 で定義した各変数を利用可能.\n",
    "    \n",
    "    # WRITE ME!\n",
    "    #  Hint: 下記Checker Cell内でimportした pad_sequences, VGG16 は利用可能\n",
    "     \n",
    "    \n",
    "    train_y_lens = [len(com) for com in train_y]\n",
    "\n",
    "    sorted_train_indexes = sorted(range(len(train_y_lens)), key=lambda x: -train_y_lens[x])\n",
    "\n",
    "    train_X = [train_X[ind] for ind in sorted_train_indexes]\n",
    "    train_y = [train_y[ind] for ind in sorted_train_indexes]\n",
    "\n",
    "    num = 0\n",
    "    \n",
    "    class Conv:\n",
    "        def __init__(self, filter_shape, W, b, function=lambda x: x, strides=[1,1,1,1], padding='VALID'):\n",
    "            # Xavier\n",
    "            fan_in = np.prod(filter_shape[:3])\n",
    "            fan_out = np.prod(filter_shape[:2]) * filter_shape[3]\n",
    "            self.W = tf.Variable(W, trainable=False, name='W')\n",
    "            self.b = tf.Variable(b, trainable=False, name='b')\n",
    "            self.function = function\n",
    "            self.strides = strides\n",
    "            self.padding = padding\n",
    "\n",
    "        def f_prop(self, x):\n",
    "            conv_out = tf.nn.conv2d(x, self.W, strides=self.strides, padding=self.padding)\n",
    "            return self.function(tf.nn.bias_add(conv_out, self.b)) # broadcasting\n",
    "        \n",
    "    class Pooling:\n",
    "        def __init__(self, ksize=[1,2,2,1], strides=[1,2,2,1], padding='VALID'):\n",
    "            self.ksize = ksize\n",
    "            self.strides = strides\n",
    "            self.padding = padding\n",
    "\n",
    "        def f_prop(self, x):\n",
    "            return tf.nn.max_pool(x, ksize=self.ksize, strides=self.strides, padding=self.padding)\n",
    "        \n",
    "    # 入力xが行列([batch_size, hid_dim])の全結合層\n",
    "    class Dense2d:\n",
    "        def __init__(self, in_dim, out_dim, function=lambda x: x):\n",
    "            # Xavier\n",
    "            self.W = tf.Variable(rng.uniform(\n",
    "                            low=-np.sqrt(6/(in_dim + out_dim)),\n",
    "                            high=np.sqrt(6/(in_dim + out_dim)),\n",
    "                            size=(in_dim, out_dim)\n",
    "                        ).astype('float32'), name='W')\n",
    "            self.b = tf.Variable(np.zeros([out_dim]).astype('float32'))\n",
    "            self.function = function\n",
    "\n",
    "        def f_prop(self, x):\n",
    "            return self.function(tf.matmul(x, self.W) + self.b)\n",
    "        \n",
    "    class Embedding:\n",
    "        def __init__(self, vocab_size, emb_dim, scale=0.08):\n",
    "            self.V = tf.Variable(rng.randn(vocab_size, emb_dim).astype('float32') * scale, name='V')\n",
    "\n",
    "        def f_prop(self, x):\n",
    "            return tf.nn.embedding_lookup(self.V, x)\n",
    "\n",
    "        def f_prop_test(self, x_t):\n",
    "            return tf.nn.embedding_lookup(self.V, x_t)\n",
    "        \n",
    "    class LSTM:\n",
    "        def __init__(self, in_dim, hid_dim, h_0=None, c_0=None):\n",
    "            self.in_dim = in_dim\n",
    "            self.hid_dim = hid_dim\n",
    "\n",
    "            # input gate\n",
    "            self.W_xi = tf.Variable(tf.random_uniform([in_dim, hid_dim], minval=-0.08, maxval=0.08, dtype=tf.float32), name='W_xi')\n",
    "            self.W_hi = tf.Variable(tf.random_uniform([hid_dim, hid_dim], minval=-0.08, maxval=0.08, dtype=tf.float32), name='W_hi')\n",
    "            self.b_i  = tf.Variable(tf.random_uniform([hid_dim], minval=-0.08, maxval=0.08, dtype=tf.float32), name='b_i')\n",
    "\n",
    "            # forget gate\n",
    "            self.W_xf = tf.Variable(tf.random_uniform([in_dim, hid_dim], minval=-0.08, maxval=0.08, dtype=tf.float32), name='W_xf')\n",
    "            self.W_hf = tf.Variable(tf.random_uniform([hid_dim, hid_dim], minval=-0.08, maxval=0.08, dtype=tf.float32), name='W_hf')\n",
    "            self.b_f  = tf.Variable(tf.random_uniform([hid_dim], minval=-0.08, maxval=0.08, dtype=tf.float32), name='b_f')\n",
    "\n",
    "            # cell state\n",
    "            self.W_xc = tf.Variable(tf.random_uniform([in_dim, hid_dim], minval=-0.08, maxval=0.08, dtype=tf.float32), name='W_xc')\n",
    "            self.W_hc = tf.Variable(tf.random_uniform([hid_dim, hid_dim], minval=-0.08, maxval=0.08, dtype=tf.float32), name='W_hc')\n",
    "            self.b_c  = tf.Variable(tf.random_uniform([hid_dim], minval=-0.08, maxval=0.08, dtype=tf.float32), name='b_c')\n",
    "\n",
    "            # output gate\n",
    "            self.W_xo = tf.Variable(tf.random_uniform([in_dim, hid_dim], minval=-0.08, maxval=0.08, dtype=tf.float32), name='W_xo')\n",
    "            self.W_ho = tf.Variable(tf.random_uniform([hid_dim, hid_dim], minval=-0.08, maxval=0.08, dtype=tf.float32), name='W_ho')\n",
    "            self.b_o  = tf.Variable(tf.random_uniform([hid_dim], minval=-0.08, maxval=0.08, dtype=tf.float32), name='b_o')\n",
    "\n",
    "            # initial state\n",
    "            self.h_0 = h_0\n",
    "            self.c_0 = c_0\n",
    "\n",
    "        def f_prop(self, x):\n",
    "            def fn(tm1, x_t):\n",
    "                h_tm1 = tm1[0]\n",
    "                c_tm1 = tm1[1]\n",
    "                # input gate\n",
    "                i_t = tf.nn.sigmoid(tf.matmul(x_t, self.W_xi) + tf.matmul(h_tm1, self.W_hi) + self.b_i)\n",
    "\n",
    "                # forget gate\n",
    "                f_t = tf.nn.sigmoid(tf.matmul(x_t, self.W_xf) + tf.matmul(h_tm1, self.W_hf) + self.b_f)\n",
    "\n",
    "                # cell state\n",
    "                c_t = f_t * c_tm1 + i_t * tf.nn.tanh(tf.matmul(x_t, self.W_xc) + tf.matmul(h_tm1, self.W_hc) + self.b_c)\n",
    "\n",
    "                # output gate\n",
    "                o_t = tf.nn.sigmoid(tf.matmul(x_t, self.W_xo) + tf.matmul(h_tm1, self.W_ho) + self.b_o)\n",
    "\n",
    "                # hidden state\n",
    "                h_t = o_t * tf.nn.tanh(c_t)\n",
    "\n",
    "                return [h_t, c_t]\n",
    "\n",
    "            _x = tf.transpose(x, perm=[1, 0, 2])\n",
    "\n",
    "            if self.h_0 == None:\n",
    "                self.h_0 = tf.matmul(x[:, 0, :], tf.zeros([self.in_dim, self.hid_dim]))\n",
    "            if self.c_0 == None:\n",
    "                self.c_0 = tf.matmul(x[:, 0, :], tf.zeros([self.in_dim, self.hid_dim]))\n",
    "\n",
    "            h, c = tf.scan(fn=fn, elems=_x, initializer=[self.h_0, self.c_0])\n",
    "            return tf.transpose(h, perm=[1, 0, 2]), tf.transpose(c, perm=[1, 0, 2])\n",
    "\n",
    "        def f_prop_test(self, x_t):\n",
    "            # input gate\n",
    "            i_t = tf.nn.sigmoid(tf.matmul(x_t, self.W_xi) + tf.matmul(self.h_0, self.W_hi) + self.b_i)\n",
    "\n",
    "            # forget gate\n",
    "            f_t = tf.nn.sigmoid(tf.matmul(x_t, self.W_xf) + tf.matmul(self.h_0, self.W_hf) + self.b_f)\n",
    "\n",
    "            # output gate\n",
    "            o_t = tf.nn.sigmoid(tf.matmul(x_t, self.W_xo) + tf.matmul(self.h_0, self.W_ho) + self.b_o)\n",
    "\n",
    "            # cell state\n",
    "            c_t = f_t * self.c_0 + i_t * tf.nn.tanh(tf.matmul(x_t, self.W_xc) + tf.matmul(self.h_0, self.W_hc) + self.b_c)\n",
    "\n",
    "            # hidden state\n",
    "            h_t = o_t * tf.nn.tanh(c_t)\n",
    "\n",
    "            return [h_t, c_t]\n",
    "    \n",
    "    # 入力xが3階テンソル([batch_size, sentence_length, hid_dim])の全結合層\n",
    "    class Dense:\n",
    "        def __init__(self, in_dim, out_dim, function=lambda x: x):\n",
    "            # Xavier\n",
    "            self.W = tf.Variable(rng.uniform(\n",
    "                            low=-np.sqrt(6/(in_dim + out_dim)),\n",
    "                            high=np.sqrt(6/(in_dim + out_dim)),\n",
    "                            size=(in_dim, out_dim)\n",
    "                        ).astype('float32'), name='W')\n",
    "            self.b = tf.Variable(tf.zeros([out_dim], dtype=tf.float32), name='b')\n",
    "            self.function = function\n",
    "\n",
    "        def f_prop(self, x):\n",
    "            return self.function(tf.einsum('ijk,kl->ijl', x, self.W) + self.b)\n",
    "\n",
    "        def f_prop_test(self, x_t):\n",
    "            return self.function(tf.matmul(x_t, self.W) + self.b)\n",
    "\n",
    "    class Attention:\n",
    "        def __init__(self, cnn_dim, rnn_dim, out_dim, u):\n",
    "            self.W_cc = tf.Variable(rng.uniform(low=-0.08, high=0.08, size=[cnn_dim, out_dim]).astype('float32'), name='W_cc')\n",
    "            self.W_ch = tf.Variable(rng.uniform(low=-0.08, high=0.08, size=[rnn_dim, out_dim]).astype('float32'), name='W_ch')\n",
    "            self.W_a = tf.Variable(rng.uniform(low=-0.08, high=0.08, size=[cnn_dim, rnn_dim]).astype('float32'), name='W_a')\n",
    "            self.b = tf.Variable(np.zeros(out_dim).astype('float32'), name='b')\n",
    "            self.u = u\n",
    "\n",
    "        def f_prop(self, h_dec):\n",
    "            u = tf.einsum('ijk,kl->ijl', self.u, self.W_a)\n",
    "            score = tf.einsum('ijk,ilk->ijl', h_dec, u) # Attention score\n",
    "            self.a = tf.nn.softmax(score) # Attention weight\n",
    "            c = tf.einsum('ijk,ikl->ijl', self.a, self.u) # Context vector\n",
    "            return tf.nn.tanh(tf.einsum('ijk,kl->ijl', c, self.W_cc) + tf.einsum('ijk,kl->ijl', h_dec, self.W_ch) + self.b)\n",
    "\n",
    "        def f_prop_test(self, h_dec_t):\n",
    "            u = tf.einsum('ijk,kl->ijl', self.u, self.W_a)\n",
    "            score = tf.einsum('ij,ikj->ik', h_dec_t, u) # Attention score\n",
    "            self.a = tf.nn.softmax(score) # Attention weight\n",
    "            c = tf.einsum('ij,ijk->ik', self.a, self.u) # Context vector\n",
    "            return tf.nn.tanh(tf.matmul(c, self.W_cc) + tf.matmul(h_dec_t, self.W_ch) + self.b)\n",
    "    \n",
    "    model = VGG16()\n",
    "\n",
    "    weights = [com.get_weights() for com in model.layers[1:]] # 各層の重み(W, b)を取得.\n",
    "    \n",
    "    \n",
    "    import sys\n",
    "\n",
    "   \n",
    "    emb_dim = 64\n",
    "    rnn_dim = 64\n",
    "    att_dim = 64\n",
    "    hid_dim = 64\n",
    "    mlp_dim = 64\n",
    "    cnn_dim = 512\n",
    "\n",
    "    x = tf.placeholder(tf.float32, [None, 224, 224, 3])\n",
    "    d = tf.placeholder(tf.int32, [None, None], name='d')\n",
    "    d_in = d[:, :-1]\n",
    "    d_out = d[:, 1:]\n",
    "    d_out_one_hot = tf.one_hot(d_out, depth=vocab_size, dtype=tf.float32)\n",
    "\n",
    "\n",
    "    # 16-layer VGG\n",
    "    cnn_layers = [\n",
    "        Conv((3, 3, 3, 64), weights[0][0], weights[0][1], tf.nn.relu, padding='SAME'), # 224x224x3 -> 224x224x64\n",
    "        Conv((3, 3, 64, 64), weights[1][0], weights[1][1], tf.nn.relu, padding='SAME'), # 224x224x64 -> 224x224x64\n",
    "        Pooling((1, 2, 2, 1)), # 224x224x64 -> 112x112x64\n",
    "        Conv((3, 3, 64, 128), weights[3][0], weights[3][1], tf.nn.relu, padding='SAME'), # 112x112x64 -> 112x112x128\n",
    "        Conv((3, 3, 128, 128), weights[4][0], weights[4][1], tf.nn.relu, padding='SAME'), # 112x112x128 -> 112x112x128\n",
    "        Pooling((1, 2, 2, 1)), # 112x112x128 -> 56x56x128\n",
    "        Conv((3, 3, 128, 256), weights[6][0], weights[6][1], tf.nn.relu, padding='SAME'), # 56x56x128 -> 56x56x256\n",
    "        Conv((3, 3, 256, 256), weights[7][0], weights[7][1], tf.nn.relu, padding='SAME'), # 56x56x256 -> 56x56x256\n",
    "        Conv((3, 3, 256, 256), weights[8][0], weights[8][1], tf.nn.relu, padding='SAME'), # 56x56x256 -> 56x56x256\n",
    "        Pooling((1, 2, 2, 1)), # 56x56x256 -> 28x28x256\n",
    "        Conv((3, 3, 256, 512), weights[10][0], weights[10][1], tf.nn.relu, padding='SAME'), # 28x28x256 -> 28x28x512\n",
    "        Conv((3, 3, 512, 512), weights[11][0], weights[11][1], tf.nn.relu, padding='SAME'), # 28x28x512 -> 28x28x512\n",
    "        Conv((3, 3, 512, 512), weights[12][0], weights[12][1], tf.nn.relu, padding='SAME'), # 28x28x512 -> 28x28x512\n",
    "        Pooling((1, 2, 2, 1)), # 28x28x512 -> 14x14x512\n",
    "        Conv((3, 3, 512, 512), weights[14][0], weights[14][1], tf.nn.relu, padding='SAME'), # 14x14x512 -> 14x14x512\n",
    "        Conv((3, 3, 512, 512), weights[15][0], weights[15][1], tf.nn.relu, padding='SAME'), # 14x14x512 -> 14x14x512\n",
    "        Conv((3, 3, 512, 512), weights[16][0], weights[16][1], tf.nn.relu, padding='SAME'), # 14x14x512 -> 14x14x512\n",
    "    ]\n",
    "\n",
    "    def f_props(layers, x):\n",
    "        for layer in layers:\n",
    "            x = layer.f_prop(x)\n",
    "        return x\n",
    "\n",
    "    u_ = f_props(cnn_layers, x)\n",
    "    u = tf.reshape(u_, [-1, 196, 512]) # 14x14x512 -> 196x512\n",
    "\n",
    "    u_mean = tf.reduce_mean(u, axis=1)\n",
    "\n",
    "    # LSTMのセルの初期値を獲得するネットワーク\n",
    "    mlp_layers_c = [\n",
    "        Dense2d(512, rnn_dim, tf.nn.tanh),\n",
    "    ]\n",
    "\n",
    "    # LSTMの隠れ層ベクトルの初期値を獲得するネットワーク\n",
    "    mlp_layers_h = [\n",
    "        Dense2d(512, rnn_dim, tf.nn.tanh),\n",
    "    ]\n",
    "\n",
    "    c_init = f_props(mlp_layers_c, u_mean) # LSTMのセルの初期値\n",
    "    h_init = f_props(mlp_layers_h, u_mean) # LSTMの隠れ層ベクトルの初期値\n",
    "\n",
    "    decoder_pre = [\n",
    "        Embedding(vocab_size, emb_dim),\n",
    "        LSTM(emb_dim, hid_dim, h_init, c_init)\n",
    "    ]\n",
    "\n",
    "    decoder_post = [\n",
    "        Attention(cnn_dim, rnn_dim, hid_dim, u),\n",
    "        Dense(hid_dim, vocab_size, tf.nn.softmax)\n",
    "    ]\n",
    "\n",
    "    h_dec, c_dec = f_props(decoder_pre, d_in)\n",
    "    y = f_props(decoder_post, h_dec)\n",
    "\n",
    "    cost = -tf.reduce_mean(tf.reduce_sum(d_out_one_hot * tf.log(tf.clip_by_value(y, 1e-10, 1.0)), axis=[1,2]))\n",
    "\n",
    "    train = tf.train.AdamOptimizer().minimize(cost)\n",
    "    \n",
    "    sess = tf.Session()\n",
    "    sess.run(tf.global_variables_initializer())\n",
    "\n",
    "    n_epochs = 40\n",
    "    batch_size = 32\n",
    "    n_batches_train = len(train_X)//batch_size\n",
    "\n",
    "\n",
    "    sess.run(tf.global_variables_initializer())\n",
    "    for epoch in range(n_epochs):\n",
    "        # Train\n",
    "        train_costs = []\n",
    "        for i in range(n_batches_train):\n",
    "            start = i * batch_size\n",
    "            end = start + batch_size\n",
    "\n",
    "            train_X_mb = train_X[start:end]\n",
    "            train_y_mb = np.array(pad_sequences(train_y[start:end], padding='post', value=-1))\n",
    "\n",
    "            _, train_cost = sess.run([train, cost], feed_dict={x: train_X_mb, d: train_y_mb})\n",
    "\n",
    "            train_costs.append(train_cost)\n",
    "\n",
    "\n",
    "\n",
    "        print('EPOCH: %i, Training cost: %.3f' % (epoch+1, np.mean(train_costs)))\n",
    "\n",
    "    \n",
    "    return cost # 返り値のcostは,tensorflowの計算グラフのcostを返す."
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Checker Cell (for student)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "import sys\n",
    "\n",
    "from sklearn.utils import shuffle\n",
    "from sklearn.metrics import f1_score\n",
    "from sklearn.model_selection import train_test_split\n",
    "import numpy as np\n",
    "import tensorflow as tf\n",
    "from keras.preprocessing.sequence import pad_sequences\n",
    "from keras.applications.vgg16 import VGG16\n",
    "rng = np.random.RandomState(42)\n",
    "random_state = 42\n",
    "\n",
    "\n",
    "sys.modules['keras'] = None\n",
    "\n",
    "def build_vocab(file_path, target):\n",
    "    vocab = set()\n",
    "    for line in open(file_path, encoding='utf-8'):\n",
    "        words = line.strip().split()\n",
    "        vocab.update(words)\n",
    "\n",
    "    if target:\n",
    "        w2i = {w: np.int32(i+2) for i, w in enumerate(vocab)}\n",
    "        w2i['<s>'], w2i['</s>'] = np.int32(0), np.int32(1)\n",
    "    else:\n",
    "        w2i = {w: np.int32(i) for i, w in enumerate(vocab)}\n",
    "\n",
    "    return w2i\n",
    "\n",
    "def encode(sentence, w2i):\n",
    "    encoded_sentence = []\n",
    "    for w in sentence:\n",
    "        encoded_sentence.append(w2i[w])\n",
    "    return encoded_sentence\n",
    "\n",
    "def load_data(file_path, vocab=None, w2i=None, target=True):\n",
    "    if vocab is None and w2i is None:\n",
    "        w2i = build_vocab(file_path, target)\n",
    "    \n",
    "    data = []\n",
    "    for line in open(file_path, encoding='utf-8'):\n",
    "        s = line.strip().split()\n",
    "        if target:\n",
    "            s = ['<s>'] + s + ['</s>']\n",
    "        enc = encode(s, w2i)\n",
    "        data.append(enc)\n",
    "    i2w = {i: w for w, i in w2i.items()}\n",
    "    return data, w2i, i2w\n",
    "\n",
    "def validate_homework():\n",
    "    global vocab_size, sess, x, d\n",
    "    \n",
    "    train_y, w2i, i2w = load_data('./mscoco_captions_10000.txt', target=True)\n",
    "    train_X = np.load('./mscoco_images_10000.npy')\n",
    "    \n",
    "    vocab_size = len(i2w)\n",
    "    \n",
    "    train_X, test_X, train_y, test_y = train_test_split(train_X, train_y, test_size=0.1, random_state=42)\n",
    "    \n",
    "    # validate for small dataset\n",
    "    train_X_mini = train_X[:100]\n",
    "    train_y_mini = train_y[:100]\n",
    "    test_X_mini = test_X[:100]\n",
    "    test_y_mini = test_y[:100]\n",
    "    \n",
    "    tf.reset_default_graph()\n",
    "\n",
    "    with tf.Session() as sess:\n",
    "        x = tf.placeholder(tf.float32, [None, 224, 224, 3])\n",
    "        d = tf.placeholder(tf.int32, [None, None], name='d')\n",
    "\n",
    "        cost = homework(train_X_mini, train_y_mini)\n",
    "        # sess, x, d はグローバル変数で定義されているので homework関数内で利用可能.\n",
    "        # 返り値のcostは,tensorflowの計算グラフのcostを返す.\n",
    "        \n",
    "        test_batch_size = 32\n",
    "        n_batches_test = -(-len(test_X_mini) // test_batch_size)\n",
    "        test_costs = []\n",
    "\n",
    "        for i in range(n_batches_test):\n",
    "            start = i * test_batch_size\n",
    "            end = start + test_batch_size if (start + test_batch_size) < len(test_X_mini) else len(test_X_mini)\n",
    "\n",
    "            test_X_mb = test_X_mini[start:end]\n",
    "            test_y_mb = np.array(pad_sequences(test_y_mini[start:end], padding='post', value=-1))\n",
    "\n",
    "            test_cost = sess.run(cost, feed_dict={x: test_X_mb, d: test_y_mb})\n",
    "\n",
    "            test_costs.append(test_cost)\n",
    "            \n",
    "    print(np.mean(test_costs))\n",
    "\n",
    "def score_homework():\n",
    "    global vocab_size, sess, x, d\n",
    "    \n",
    "    train_y, w2i, i2w = load_data('mscoco_captions_10000.txt', target=True)\n",
    "    train_X = np.load('mscoco_images_10000.npy')\n",
    "    \n",
    "    vocab_size = len(i2w)\n",
    "    \n",
    "    train_X, test_X, train_y, test_y = train_test_split(train_X, train_y, test_size=0.1, random_state=42)\n",
    "    \n",
    "    tf.reset_default_graph()\n",
    "\n",
    "    with tf.Session() as sess:\n",
    "        x = tf.placeholder(tf.float32, [None, 224, 224, 3])\n",
    "        d = tf.placeholder(tf.int32, [None, None], name='d')\n",
    "\n",
    "        cost = homework(train_X, train_y)\n",
    "        # sess, x, d はグローバル変数で定義されているので homework関数内で利用可能.\n",
    "        # 返り値のcostは,tensorflowの計算グラフのcostを返す.\n",
    "        \n",
    "        test_batch_size = 32\n",
    "        n_batches_test = -(-len(test_X) // test_batch_size)\n",
    "        test_costs = []\n",
    "\n",
    "        for i in range(n_batches_test):\n",
    "            start = i * test_batch_size\n",
    "            end = start + test_batch_size if (start + test_batch_size) < len(test_X) else len(test_X)\n",
    "\n",
    "            test_X_mb = test_X[start:end]\n",
    "            test_y_mb = np.array(pad_sequences(test_y[start:end], padding='post', value=-1))\n",
    "\n",
    "            test_cost = sess.run(cost, feed_dict={x: test_X_mb, d: test_y_mb})\n",
    "\n",
    "            test_costs.append(test_cost)\n",
    "            \n",
    "    print(np.mean(test_costs))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "EPOCH: 1, Training cost: 103.402\n",
      "EPOCH: 2, Training cost: 100.862\n",
      "EPOCH: 3, Training cost: 98.034\n",
      "EPOCH: 4, Training cost: 94.789\n",
      "EPOCH: 5, Training cost: 91.310\n",
      "EPOCH: 6, Training cost: 87.787\n",
      "EPOCH: 7, Training cost: 84.254\n",
      "EPOCH: 8, Training cost: 80.800\n",
      "EPOCH: 9, Training cost: 77.621\n",
      "EPOCH: 10, Training cost: 74.655\n",
      "EPOCH: 11, Training cost: 71.998\n",
      "EPOCH: 12, Training cost: 69.654\n",
      "EPOCH: 13, Training cost: 67.596\n",
      "EPOCH: 14, Training cost: 65.793\n",
      "EPOCH: 15, Training cost: 64.232\n",
      "EPOCH: 16, Training cost: 62.875\n",
      "EPOCH: 17, Training cost: 61.707\n",
      "EPOCH: 18, Training cost: 60.716\n",
      "EPOCH: 19, Training cost: 59.875\n",
      "EPOCH: 20, Training cost: 59.179\n",
      "EPOCH: 21, Training cost: 58.599\n",
      "EPOCH: 22, Training cost: 58.104\n",
      "EPOCH: 23, Training cost: 57.699\n",
      "EPOCH: 24, Training cost: 57.358\n",
      "EPOCH: 25, Training cost: 57.066\n",
      "EPOCH: 26, Training cost: 56.815\n",
      "EPOCH: 27, Training cost: 56.617\n",
      "EPOCH: 28, Training cost: 56.447\n",
      "EPOCH: 29, Training cost: 56.290\n",
      "EPOCH: 30, Training cost: 56.127\n",
      "EPOCH: 31, Training cost: 55.980\n",
      "EPOCH: 32, Training cost: 55.832\n",
      "EPOCH: 33, Training cost: 55.671\n",
      "EPOCH: 34, Training cost: 55.524\n",
      "EPOCH: 35, Training cost: 55.342\n",
      "EPOCH: 36, Training cost: 55.206\n",
      "EPOCH: 37, Training cost: 55.039\n",
      "EPOCH: 38, Training cost: 54.874\n",
      "EPOCH: 39, Training cost: 54.724\n",
      "EPOCH: 40, Training cost: 54.533\n",
      "69.1555\n"
     ]
    }
   ],
   "source": [
    "validate_homework()\n",
    "# score_homework()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "EPOCH: 1, Training cost: 68.927\n",
      "EPOCH: 2, Training cost: 60.592\n",
      "EPOCH: 3, Training cost: 58.881\n",
      "EPOCH: 4, Training cost: 57.217\n",
      "EPOCH: 5, Training cost: 55.169\n",
      "EPOCH: 6, Training cost: 53.097\n",
      "EPOCH: 7, Training cost: 51.574\n",
      "EPOCH: 8, Training cost: 50.381\n",
      "EPOCH: 9, Training cost: 49.269\n",
      "EPOCH: 10, Training cost: 48.331\n",
      "EPOCH: 11, Training cost: 47.646\n",
      "EPOCH: 12, Training cost: 46.837\n",
      "EPOCH: 13, Training cost: 45.998\n",
      "EPOCH: 14, Training cost: 45.389\n",
      "EPOCH: 15, Training cost: 44.742\n",
      "EPOCH: 16, Training cost: 44.113\n",
      "EPOCH: 17, Training cost: 43.504\n",
      "EPOCH: 18, Training cost: 43.011\n",
      "EPOCH: 19, Training cost: 42.565\n",
      "EPOCH: 20, Training cost: 42.085\n",
      "EPOCH: 21, Training cost: 41.645\n",
      "EPOCH: 22, Training cost: 41.156\n",
      "EPOCH: 23, Training cost: 40.780\n",
      "EPOCH: 24, Training cost: 40.704\n",
      "EPOCH: 25, Training cost: 40.342\n",
      "EPOCH: 26, Training cost: 40.043\n",
      "EPOCH: 27, Training cost: 39.668\n",
      "EPOCH: 28, Training cost: 39.416\n",
      "EPOCH: 29, Training cost: 39.104\n",
      "EPOCH: 30, Training cost: 38.746\n",
      "EPOCH: 31, Training cost: 38.648\n",
      "EPOCH: 32, Training cost: 38.274\n",
      "EPOCH: 33, Training cost: 37.983\n",
      "EPOCH: 34, Training cost: 37.822\n",
      "EPOCH: 35, Training cost: 37.602\n",
      "EPOCH: 36, Training cost: 37.520\n",
      "EPOCH: 37, Training cost: 37.287\n",
      "EPOCH: 38, Training cost: 37.173\n",
      "EPOCH: 39, Training cost: 36.934\n",
      "EPOCH: 40, Training cost: 36.991\n",
      "48.5522\n"
     ]
    }
   ],
   "source": [
    "score_homework()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.5.3"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 1
}