lecture_chap8_homework_wzh.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# 第8回講義 宿題"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 課題. Tensorflowを用いて, CIFAR-10を畳み込みニューラルネットワーク(CNN)で学習せよ"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 注意"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "- homework関数を完成させて提出してください\n",
    "    - 訓練データはtrain_X, train_y, テストデータはtest_Xで与えられます\n",
    "    - train_Xとtrain_yをtrain_X, train_yとvalid_X, valid_yに分けるなどしてモデルを学習させてください\n",
    "    - test_Xに対して予想ラベルpred_yを作り, homework関数の戻り値としてください\\\n",
    "- pred_yのtest_yに対する精度(F値)で評価します\n",
    "- 全体の実行時間がiLect上で60分を超えないようにしてください\n",
    "- homework関数の外には何も書かないでください (必要なものは全てhomework関数に入れてください)\n",
    "- 解答提出時には Answer Cell の内容のみを提出してください"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "**`tf` の以下のモジュール及び `keras` はこの回では使用できないように制限されています. 注意してください.**\n",
    "\n",
    "```python\n",
    "tf.app\n",
    "tf.compat\n",
    "tf.contrib\n",
    "tf.erros\n",
    "tf.gfile\n",
    "tf.graph_util\n",
    "tf.image\n",
    "tf.layers\n",
    "tf.logging\n",
    "tf.losses\n",
    "tf.metrics\n",
    "tf.python_io\n",
    "tf.resource_loader\n",
    "tf.saved_model\n",
    "tf.sdca\n",
    "tf.sets\n",
    "tf.summary\n",
    "tf.sysconfig\n",
    "tf.test\n",
    "```"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "次のセルのhomework関数を完成させて提出してください"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Answer Cell"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def homework(train_X, train_y, test_X):\n",
    "    rng = np.random.RandomState(1234)\n",
    "    random_state = 42\n",
    "    train_X, valid_X, train_y, valid_y = train_test_split(train_X, train_y,test_size=0.2, random_state=42)\n",
    "    flip_train_X = train_X[:, :, ::-1, :]\n",
    "    padded = np.pad(train_X, ((0, 0), (4, 4), (4, 4), (0, 0)), mode='constant')\n",
    "    crops = rng.randint(8, size=(len(train_X), 2))\n",
    "    cropped_train_X = [padded[i, c[0]:(c[0]+32), c[1]:(c[1]+32), :] for i, c in enumerate(crops)]\n",
    "    cropped_train_X = np.array(cropped_train_X)\n",
    "    train_X = np.concatenate((train_X, flip_train_X))\n",
    "    train_X = np.concatenate((train_X,  cropped_train_X ))\n",
    "    temp = train_y\n",
    "    train_y = np.concatenate((train_y, train_y))\n",
    "    train_y = np.concatenate((train_y, temp))\n",
    "    import matplotlib.pyplot as plt\n",
    "\n",
    "    # グラフをipython notebook内に表示\n",
    "    #%matplotlib inline\n",
    "    def gcn(x):\n",
    "        mean = np.mean(x, axis=(1, 2, 3), keepdims=True)\n",
    "        std = np.std(x, axis=(1, 2, 3), keepdims=True)\n",
    "        return (x - mean)/std\n",
    "    \n",
    "    class ZCAWhitening:\n",
    "        def __init__(self, epsilon=1e-4):\n",
    "            self.epsilon = epsilon\n",
    "            self.mean = None\n",
    "            self.ZCA_matrix = None\n",
    "\n",
    "        def fit(self, x):\n",
    "            x = x.reshape(x.shape[0], -1)\n",
    "            self.mean = np.mean(x, axis=0)\n",
    "            x -= self.mean\n",
    "            cov_matrix = np.dot(x.T, x) / x.shape[0]\n",
    "            A, d, _ = np.linalg.svd(cov_matrix)\n",
    "            self.ZCA_matrix = np.dot(np.dot(A, np.diag(1. / np.sqrt(d + self.epsilon))), A.T)\n",
    "\n",
    "        def transform(self, x):\n",
    "            shape = x.shape\n",
    "            x = x.reshape(x.shape[0], -1)\n",
    "            x -= self.mean\n",
    "            x = np.dot(x, self.ZCA_matrix.T)\n",
    "            return x.reshape(shape)\n",
    "    \n",
    "    class BatchNorm:\n",
    "        def __init__(self, shape, epsilon=np.float32(1e-5)):\n",
    "            self.gamma = tf.Variable(np.ones(shape, dtype='float32'), name='gamma')\n",
    "            self.beta  = tf.Variable(np.zeros(shape, dtype='float32'), name='beta')\n",
    "            self.epsilon = epsilon\n",
    "\n",
    "        def f_prop(self, x):\n",
    "            if len(x.get_shape()) == 2:\n",
    "                mean, var = tf.nn.moments(x, axes=0, keepdims=True)\n",
    "                std = tf.sqrt(var + self.epsilon)\n",
    "            elif len(x.get_shape()) == 4:\n",
    "                mean, var = tf.nn.moments(x, axes=(0,1,2), keep_dims=True)\n",
    "                std = tf.sqrt(var + self.epsilon)\n",
    "            normalized_x = (x - mean) / std\n",
    "            return self.gamma * normalized_x + self.beta\n",
    "        \n",
    "    class Conv:\n",
    "        def __init__(self, filter_shape, function=lambda x: x, strides=[1,1,1,1], padding='VALID'):\n",
    "            # Xavier\n",
    "            fan_in = np.prod(filter_shape[:3])\n",
    "            fan_out = np.prod(filter_shape[:2]) * filter_shape[3]\n",
    "            self.W = tf.Variable(rng.uniform(\n",
    "                            low=-np.sqrt(6/(fan_in + fan_out)),\n",
    "                            high=np.sqrt(6/(fan_in + fan_out)),\n",
    "                            size=filter_shape\n",
    "                        ).astype('float32'), name='W')\n",
    "            self.b = tf.Variable(np.zeros((filter_shape[3]), dtype='float32'), name='b') # バイアスはフィルタごと\n",
    "            self.function = function\n",
    "            self.strides = strides\n",
    "            self.padding = padding\n",
    "\n",
    "        def f_prop(self, x):\n",
    "            u = tf.nn.conv2d(x, self.W, strides=self.strides, padding=self.padding) + self.b\n",
    "            return self.function(u)\n",
    "        \n",
    "    class Pooling:\n",
    "        def __init__(self, ksize=[1,2,2,1], strides=[1,2,2,1], padding='VALID'):\n",
    "            self.ksize = ksize\n",
    "            self.strides = strides\n",
    "            self.padding = padding\n",
    "\n",
    "        def f_prop(self, x):\n",
    "            return tf.nn.max_pool(x, ksize=self.ksize, strides=self.strides, padding=self.padding)\n",
    "    \n",
    "    class Flatten:\n",
    "        def f_prop(self, x):\n",
    "            return tf.reshape(x, (-1, np.prod(x.get_shape().as_list()[1:])))\n",
    "        \n",
    "    class Dense:\n",
    "        def __init__(self, in_dim, out_dim, function=lambda x: x):\n",
    "            # Xavier\n",
    "            self.W = tf.Variable(rng.uniform(\n",
    "                            low=-np.sqrt(6/(in_dim + out_dim)),\n",
    "                            high=np.sqrt(6/(in_dim + out_dim)),\n",
    "                            size=(in_dim, out_dim)\n",
    "                        ).astype('float32'), name='W')\n",
    "            self.b = tf.Variable(np.zeros([out_dim]).astype('float32'))\n",
    "            self.function = function\n",
    "\n",
    "        def f_prop(self, x):\n",
    "            return self.function(tf.matmul(x, self.W) + self.b)\n",
    "        \n",
    "    class Activation:\n",
    "        def __init__(self, function=lambda x: x):\n",
    "            self.function = function\n",
    "\n",
    "        def f_prop(self, x):\n",
    "            return self.function(x)\n",
    "        \n",
    "    layers = [ # (縦の次元数)x(横の次元数)x(チャネル数)\n",
    "        Conv((3, 3, 3, 32)), # 32x32x3 -> 30x30x32\n",
    "        BatchNorm((30, 30, 32)),\n",
    "        Activation(tf.nn.relu),\n",
    "        Pooling((1, 2, 2, 1)), # 30x30x32 -> 15x15x32\n",
    "        Conv((3, 3, 32, 64)), # 15x15x32 -> 13x13x64\n",
    "        BatchNorm((13, 13, 64)),\n",
    "        Pooling(((1, 2, 2, 1))), # 13x13x64 -> 6x6x64\n",
    "        Conv((3, 3, 64, 128)), # 6x6x64 -> 4x4x128\n",
    "        BatchNorm((4, 4, 128)),\n",
    "        Activation(tf.nn.relu),\n",
    "        Pooling((1, 2, 2, 1)), # 4x4x128 -> 2x2x128\n",
    "        Flatten(),\n",
    "        Dense(2*2*128, 256, tf.nn.relu),\n",
    "#        Dense(256, 128, tf.nn.relu6),\n",
    "        Dense(256, 10, tf.nn.softmax)\n",
    "    ]\n",
    "\n",
    "    x = tf.placeholder(tf.float32, [None, 32, 32, 3])\n",
    "    t = tf.placeholder(tf.float32, [None, 10])\n",
    "\n",
    "    def f_props(layers, x):\n",
    "        for layer in layers:\n",
    "            x = layer.f_prop(x)\n",
    "        return x\n",
    "\n",
    "    y = f_props(layers, x)\n",
    "\n",
    "    cost = -tf.reduce_mean(tf.reduce_sum(t * tf.log(tf.clip_by_value(y, 1e-10, 1.0)), axis=1))\n",
    "    train = tf.train.GradientDescentOptimizer(0.1).minimize(cost)\n",
    "\n",
    "    valid = tf.argmax(y, 1)\n",
    "    \n",
    "    import time\n",
    "    start_time=time.time()\n",
    "    \n",
    "    zca = ZCAWhitening()\n",
    "    zca.fit(gcn(train_X))\n",
    "    zca_train_X = zca.transform(gcn(train_X))\n",
    "    zca_valid_X = zca.transform(gcn(valid_X))\n",
    "    zca_train_y = train_y[:]\n",
    "    zca_test_X = zca.transform(gcn(test_X))\n",
    "    \n",
    "    print(\"ZCAGCN time\",time.time()-start_time)\n",
    "    \n",
    "    n_epochs = 70\n",
    "    batch_size = 100\n",
    "    n_batches = train_X.shape[0]//batch_size\n",
    "\n",
    "    sess = tf.Session()\n",
    "    init = tf.global_variables_initializer()\n",
    "    sess.run(init)\n",
    "    y_train = []\n",
    "    y_valid = []\n",
    "    for epoch in range(n_epochs):\n",
    "        train_X, train_y = shuffle(train_X, train_y, random_state=random_state)\n",
    "        valid_cost=0\n",
    "        for i in range(n_batches):\n",
    "            start = i * batch_size\n",
    "            end = start + batch_size\n",
    "            _,_cost=sess.run([train, cost], feed_dict={x: train_X[start:end], t: train_y[start:end]})\n",
    "            valid_cost += _cost\n",
    "        xx_cost = sess.run([cost], feed_dict={x: valid_X, t: valid_y})\n",
    "        y_train.append(valid_cost/(n_batches+1))\n",
    "        y_valid.append(xx_cost)\n",
    "        time_cost=time.time()-start_time\n",
    "        #print('EPOCH:: %i, Validation cost: %.3f %.3f' % (epoch + 1, valid_cost, time_cost))\n",
    "    #plt.figure(1)\n",
    "    #plt.plot(y_train)  \n",
    "    #plt.plot(y_valid)\n",
    "    #plt.show()\n",
    "    pred_y_m = sess.run([valid], feed_dict={x: test_X})\n",
    "    pred_y=pred_y_m[0]\n",
    "    sess.close()\n",
    "    return pred_y"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "- 以下のvalidate_homework関数を用いてエラーが起きないか動作確認をして下さい。\n",
    "- 提出に際して、以下のscore_homework関数で60分で実行が終わることを確認して下さい。\n",
    "- 評価は以下のscore_homework関数で行われますが、random_stateの値は変更されます。"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Checker Cell (for student)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Using TensorFlow backend.\n"
     ]
    }
   ],
   "source": [
    "import sys\n",
    "\n",
    "from keras.datasets import cifar10\n",
    "from sklearn.utils import shuffle\n",
    "from sklearn.metrics import f1_score\n",
    "from sklearn.datasets import fetch_mldata\n",
    "from sklearn.model_selection import train_test_split\n",
    "\n",
    "import numpy as np\n",
    "import tensorflow as tf\n",
    "\n",
    "del [\n",
    "    tf.app,\n",
    "    tf.compat,\n",
    "    tf.contrib,\n",
    "    tf.errors,\n",
    "    tf.gfile,\n",
    "    tf.graph_util,\n",
    "    tf.image,\n",
    "    tf.layers,\n",
    "    tf.logging,\n",
    "    tf.losses,\n",
    "    tf.metrics,\n",
    "    tf.python_io,\n",
    "    tf.resource_loader,\n",
    "    tf.saved_model,\n",
    "    tf.sdca,\n",
    "    tf.sets,\n",
    "    tf.summary,\n",
    "    tf.sysconfig,\n",
    "    tf.test\n",
    "]\n",
    "\n",
    "sys.modules['keras'] = None\n",
    "\n",
    "def load_cifar():\n",
    "    import time\n",
    "    start_time = time.time()\n",
    "    (cifar_X_1, cifar_y_1), (cifar_X_2, cifar_y_2) = cifar10.load_data()\n",
    "\n",
    "    cifar_X = np.r_[cifar_X_1, cifar_X_2]\n",
    "    cifar_y = np.r_[cifar_y_1, cifar_y_2]\n",
    "\n",
    "    cifar_X = cifar_X.astype('float32') / 255\n",
    "    cifar_y = np.eye(10)[cifar_y.astype('int32').flatten()]\n",
    "\n",
    "    train_X, test_X, train_y, test_y = train_test_split(cifar_X, cifar_y,\n",
    "                                                        test_size=10000,\n",
    "                                                        random_state=42)\n",
    "    print(\"load time:\",time.time()-start_time)\n",
    "    return (train_X, test_X, train_y, test_y)\n",
    "\n",
    "def validate_homework():\n",
    "    train_X, test_X, train_y, test_y = load_cifar()\n",
    "\n",
    "    # validate for small dataset\n",
    "    train_X_mini = train_X[:100]\n",
    "    train_y_mini = train_y[:100]\n",
    "    test_X_mini = test_X[:100]\n",
    "    test_y_mini = test_y[:100]\n",
    "\n",
    "    pred_y = homework(train_X_mini, train_y_mini, test_X_mini)\n",
    "    print(f1_score(np.argmax(test_y_mini, 1), pred_y, average='macro'))\n",
    "\n",
    "def score_homework():\n",
    "    train_X, test_X, train_y, test_y = load_cifar()\n",
    "    pred_y = homework(train_X, train_y, test_X)\n",
    "    print(f1_score(np.argmax(test_y, 1), pred_y, average='macro'))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "load time: 2.2555477619171143\n",
      "ZCAGCN time 13.558365106582642\n",
      "0.214524143458\n"
     ]
    }
   ],
   "source": [
    "validate_homework()\n",
    "# score_homework()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "load time: 2.250060558319092\n"
     ]
    }
   ],
   "source": [
    "score_homework()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "#no-ZCA 50epochs 0.798885025257 2193.520\n",
    "#ZCAGCN 50epochs 0.799183103712 2238.661\n",
    "#ZCAGCN 100epochs 0.800 4200\n",
    "#relu>relu6>tahn\n",
    "#ZCAGCN relu 75epochs 0.800 3240\n",
    "#ZCAGCN relu6 75epochs 0.796 3271.951\n",
    "#ZCAGCN dense 128*2*2->128 relu6 0.78726"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.5.2"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 1
}