11_RNN.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import tensorflow as tf\n",
    "import numpy as np\n",
    "import pandas as pd\n",
    "import matplotlib.pylab as plt\n",
    "import sys"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### 단일 cell을 통한 계산 처리 과정 : 기본 사용법"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 151,
   "metadata": {},
   "outputs": [],
   "source": [
    "# 입력 데이터 준비\n",
    "inputs = np.array([[[1, 2]]])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 152,
   "metadata": {},
   "outputs": [],
   "source": [
    "tf.reset_default_graph()\n",
    "tf.set_random_seed(777)\n",
    "tf_inputs = tf.constant(inputs, dtype=tf.float32)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 153,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Tensor(\"rnn/transpose_1:0\", shape=(1, 1, 3), dtype=float32)\n",
      "Tensor(\"rnn/while/Exit_3:0\", shape=(1, 3), dtype=float32)\n",
      "\n",
      "\n",
      " Weights\n",
      "<tf.Variable 'rnn/basic_rnn_cell/kernel:0' shape=(5, 3) dtype=float32_ref>\n",
      "<tf.Variable 'rnn/basic_rnn_cell/bias:0' shape=(3,) dtype=float32_ref>\n"
     ]
    }
   ],
   "source": [
    "import warnings\n",
    "warnings.filterwarnings(action=\"ignore\")\n",
    "\n",
    "tf.reset_default_graph()\n",
    "tf.set_random_seed(777)\n",
    "tf_inputs = tf.constant(inputs, dtype=tf.float32)\n",
    "\n",
    "# 1 * 3의 matrix 출력([6, 7, 8]과 같은)\n",
    "rnn_cell = tf.nn.rnn_cell.BasicRNNCell(num_units=3)\n",
    "# 동작 함수(셀 실행)\n",
    "outputs, state = tf.nn.dynamic_rnn(cell=rnn_cell, dtype=tf.float32, inputs=tf_inputs)\n",
    "\n",
    "print(outputs)\n",
    "print(state)\n",
    "\n",
    "print(\"\\n\\n Weights\")\n",
    "for w in tf.get_collection(tf.GraphKeys.TRAINABLE_VARIABLES):\n",
    "    print(w)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 154,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Output Value\n",
      "[[[-0.9314169   0.75578666 -0.6819246 ]]]\n",
      "State Value\n",
      "[[-0.9314169   0.75578666 -0.6819246 ]]\n",
      "['rnn/basic_rnn_cell/kernel:0', 'rnn/basic_rnn_cell/bias:0']\n",
      "rnn/basic_rnn_cell/kernel:0 [[-0.62831575  0.38538355  0.79733914]\n",
      " [-0.5203329   0.30046564 -0.8150209 ]\n",
      " [ 0.39399797  0.16670114  0.4062907 ]\n",
      " [-0.6391754   0.8460203   0.5266966 ]\n",
      " [ 0.41124135  0.66347724 -0.0210759 ]]\n",
      "rnn/basic_rnn_cell/bias:0 [0. 0. 0.]\n"
     ]
    }
   ],
   "source": [
    "with tf.Session() as sess:\n",
    "    sess.run(tf.global_variables_initializer())\n",
    "    output_run, state_run = sess.run([outputs, state])\n",
    "    \n",
    "    print(\"Output Value\")\n",
    "    print(output_run)\n",
    "    \n",
    "    print(\"State Value\")\n",
    "    print(state_run)\n",
    "    \n",
    "    variables_names = [v.name for v in tf.trainable_variables()]\n",
    "    print(variables_names)\n",
    "    values = sess.run(variables_names)\n",
    "    for k, v in zip(variables_names, values):\n",
    "        print(k, v)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### 다중 cell"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 158,
   "metadata": {},
   "outputs": [],
   "source": [
    "# I work at google = [[1,0,0,0], [0,1,0,0], [0,0,1,0], [0,0,0,1]] / one-hot incording\n",
    "# I google at work = [[1,0,0,0], [0,0,0,1], [0,0,1,0], [0,1,0,0,]]\n",
    "\n",
    "inputs = np.array([[[1,0,0,0], [0,1,0,0], [0,0,1,0], [0,0,0,1]],\n",
    "                   [1,0,0,0], [0,0,0,1], [0,0,1,0], [0,1,0,0,]])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 164,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "ename": "ValueError",
     "evalue": "setting an array element with a sequence.",
     "output_type": "error",
     "traceback": [
      "\u001b[1;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[1;31mValueError\u001b[0m                                Traceback (most recent call last)",
      "\u001b[1;32m<ipython-input-164-82ce83a9a288>\u001b[0m in \u001b[0;36m<module>\u001b[1;34m\u001b[0m\n\u001b[0;32m      1\u001b[0m \u001b[0mtf\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mreset_default_graph\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m      2\u001b[0m \u001b[0mtf\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mset_random_seed\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;36m777\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m----> 3\u001b[1;33m \u001b[0mtf_inputs\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mtf\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mconstant\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0minputs\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mdtype\u001b[0m\u001b[1;33m=\u001b[0m\u001b[0mtf\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mfloat32\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m      4\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m      5\u001b[0m \u001b[0mrnn_cell\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mtf\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mcontrib\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mrnn\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mBasicRNNCell\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mnum_units\u001b[0m\u001b[1;33m=\u001b[0m\u001b[1;36m3\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32m~\\anaconda3\\envs\\tf1\\lib\\site-packages\\tensorflow_core\\python\\framework\\constant_op.py\u001b[0m in \u001b[0;36mconstant_v1\u001b[1;34m(value, dtype, shape, name, verify_shape)\u001b[0m\n\u001b[0;32m    159\u001b[0m   \"\"\"\n\u001b[0;32m    160\u001b[0m   return _constant_impl(value, dtype, shape, name, verify_shape=verify_shape,\n\u001b[1;32m--> 161\u001b[1;33m                         allow_broadcast=False)\n\u001b[0m\u001b[0;32m    162\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    163\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32m~\\anaconda3\\envs\\tf1\\lib\\site-packages\\tensorflow_core\\python\\framework\\constant_op.py\u001b[0m in \u001b[0;36m_constant_impl\u001b[1;34m(value, dtype, shape, name, verify_shape, allow_broadcast)\u001b[0m\n\u001b[0;32m    263\u001b[0m       tensor_util.make_tensor_proto(\n\u001b[0;32m    264\u001b[0m           \u001b[0mvalue\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mdtype\u001b[0m\u001b[1;33m=\u001b[0m\u001b[0mdtype\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mshape\u001b[0m\u001b[1;33m=\u001b[0m\u001b[0mshape\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mverify_shape\u001b[0m\u001b[1;33m=\u001b[0m\u001b[0mverify_shape\u001b[0m\u001b[1;33m,\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m--> 265\u001b[1;33m           allow_broadcast=allow_broadcast))\n\u001b[0m\u001b[0;32m    266\u001b[0m   \u001b[0mdtype_value\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mattr_value_pb2\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mAttrValue\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mtype\u001b[0m\u001b[1;33m=\u001b[0m\u001b[0mtensor_value\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mtensor\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mdtype\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    267\u001b[0m   const_tensor = g.create_op(\n",
      "\u001b[1;32m~\\anaconda3\\envs\\tf1\\lib\\site-packages\\tensorflow_core\\python\\framework\\tensor_util.py\u001b[0m in \u001b[0;36mmake_tensor_proto\u001b[1;34m(values, dtype, shape, verify_shape, allow_broadcast)\u001b[0m\n\u001b[0;32m    430\u001b[0m   \u001b[1;32mif\u001b[0m \u001b[0misinstance\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mvalues\u001b[0m\u001b[1;33m,\u001b[0m \u001b[1;33m(\u001b[0m\u001b[0mnp\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mndarray\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mnp\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mgeneric\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    431\u001b[0m     \u001b[1;32mif\u001b[0m \u001b[0mdtype\u001b[0m \u001b[1;32mand\u001b[0m \u001b[0mdtype\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mis_numpy_compatible\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m--> 432\u001b[1;33m       \u001b[0mnparray\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mvalues\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mastype\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mdtype\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mas_numpy_dtype\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m    433\u001b[0m     \u001b[1;32melse\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    434\u001b[0m       \u001b[0mnparray\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mvalues\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;31mValueError\u001b[0m: setting an array element with a sequence."
     ]
    }
   ],
   "source": [
    "tf.reset_default_graph()\n",
    "tf.set_random_seed(777)\n",
    "tf_inputs = tf.constant(inputs, dtype=tf.float32)\n",
    "\n",
    "rnn_cell = tf.contrib.rnn.BasicRNNCell(num_units=3)\n",
    "outputs, state = tf.nn.dynamic_rnn(cell=rnn_cell, dtype=tf.float32, inputs=tf_inputs)\n",
    "variables_names = [v.name for v in tf.trainable_variables()]\n",
    "\n",
    "print(outputs)\n",
    "print(\"----------------------\")\n",
    "print(state)\n",
    "print(\"----------------------\")\n",
    "for w in tf.get_collection(tf.GraphKeys.TRAINABLE_VARIABLES):\n",
    "    print(w)  "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 157,
   "metadata": {
    "collapsed": true
   },
   "outputs": [
    {
     "ename": "ValueError",
     "evalue": "Fetch argument <tf.Tensor 'rnn/transpose_1:0' shape=(1, 1, 3) dtype=float32> cannot be interpreted as a Tensor. (Tensor Tensor(\"rnn/transpose_1:0\", shape=(1, 1, 3), dtype=float32) is not an element of this graph.)",
     "output_type": "error",
     "traceback": [
      "\u001b[1;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[1;31mValueError\u001b[0m                                Traceback (most recent call last)",
      "\u001b[1;32m~\\anaconda3\\envs\\tf1\\lib\\site-packages\\tensorflow_core\\python\\client\\session.py\u001b[0m in \u001b[0;36m__init__\u001b[1;34m(self, fetches, contraction_fn)\u001b[0m\n\u001b[0;32m    304\u001b[0m         self._unique_fetches.append(ops.get_default_graph().as_graph_element(\n\u001b[1;32m--> 305\u001b[1;33m             fetch, allow_tensor=True, allow_operation=True))\n\u001b[0m\u001b[0;32m    306\u001b[0m       \u001b[1;32mexcept\u001b[0m \u001b[0mTypeError\u001b[0m \u001b[1;32mas\u001b[0m \u001b[0me\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32m~\\anaconda3\\envs\\tf1\\lib\\site-packages\\tensorflow_core\\python\\framework\\ops.py\u001b[0m in \u001b[0;36mas_graph_element\u001b[1;34m(self, obj, allow_tensor, allow_operation)\u001b[0m\n\u001b[0;32m   3606\u001b[0m     \u001b[1;32mwith\u001b[0m \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0m_lock\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m-> 3607\u001b[1;33m       \u001b[1;32mreturn\u001b[0m \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0m_as_graph_element_locked\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mobj\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mallow_tensor\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mallow_operation\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m   3608\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32m~\\anaconda3\\envs\\tf1\\lib\\site-packages\\tensorflow_core\\python\\framework\\ops.py\u001b[0m in \u001b[0;36m_as_graph_element_locked\u001b[1;34m(self, obj, allow_tensor, allow_operation)\u001b[0m\n\u001b[0;32m   3685\u001b[0m       \u001b[1;32mif\u001b[0m \u001b[0mobj\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mgraph\u001b[0m \u001b[1;32mis\u001b[0m \u001b[1;32mnot\u001b[0m \u001b[0mself\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m-> 3686\u001b[1;33m         \u001b[1;32mraise\u001b[0m \u001b[0mValueError\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;34m\"Tensor %s is not an element of this graph.\"\u001b[0m \u001b[1;33m%\u001b[0m \u001b[0mobj\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m   3687\u001b[0m       \u001b[1;32mreturn\u001b[0m \u001b[0mobj\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;31mValueError\u001b[0m: Tensor Tensor(\"rnn/transpose_1:0\", shape=(1, 1, 3), dtype=float32) is not an element of this graph.",
      "\nDuring handling of the above exception, another exception occurred:\n",
      "\u001b[1;31mValueError\u001b[0m                                Traceback (most recent call last)",
      "\u001b[1;32m<ipython-input-157-dd623c2adfb7>\u001b[0m in \u001b[0;36m<module>\u001b[1;34m\u001b[0m\n\u001b[0;32m      1\u001b[0m \u001b[1;32mwith\u001b[0m \u001b[0mtf\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mSession\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;33m)\u001b[0m \u001b[1;32mas\u001b[0m \u001b[0msess\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m      2\u001b[0m     \u001b[0msess\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mrun\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mtf\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mglobal_variables_initializer\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m----> 3\u001b[1;33m     \u001b[0moutput_run\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mstate_run\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0msess\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mrun\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;33m[\u001b[0m\u001b[0moutputs\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mstate\u001b[0m\u001b[1;33m]\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m      4\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m      5\u001b[0m     \u001b[0mprint\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;34m\"Output Value\"\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32m~\\anaconda3\\envs\\tf1\\lib\\site-packages\\tensorflow_core\\python\\client\\session.py\u001b[0m in \u001b[0;36mrun\u001b[1;34m(self, fetches, feed_dict, options, run_metadata)\u001b[0m\n\u001b[0;32m    954\u001b[0m     \u001b[1;32mtry\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    955\u001b[0m       result = self._run(None, fetches, feed_dict, options_ptr,\n\u001b[1;32m--> 956\u001b[1;33m                          run_metadata_ptr)\n\u001b[0m\u001b[0;32m    957\u001b[0m       \u001b[1;32mif\u001b[0m \u001b[0mrun_metadata\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    958\u001b[0m         \u001b[0mproto_data\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mtf_session\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mTF_GetBuffer\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mrun_metadata_ptr\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32m~\\anaconda3\\envs\\tf1\\lib\\site-packages\\tensorflow_core\\python\\client\\session.py\u001b[0m in \u001b[0;36m_run\u001b[1;34m(self, handle, fetches, feed_dict, options, run_metadata)\u001b[0m\n\u001b[0;32m   1163\u001b[0m     \u001b[1;31m# Create a fetch handler to take care of the structure of fetches.\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m   1164\u001b[0m     fetch_handler = _FetchHandler(\n\u001b[1;32m-> 1165\u001b[1;33m         self._graph, fetches, feed_dict_tensor, feed_handles=feed_handles)\n\u001b[0m\u001b[0;32m   1166\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m   1167\u001b[0m     \u001b[1;31m# Run request and get response.\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32m~\\anaconda3\\envs\\tf1\\lib\\site-packages\\tensorflow_core\\python\\client\\session.py\u001b[0m in \u001b[0;36m__init__\u001b[1;34m(self, graph, fetches, feeds, feed_handles)\u001b[0m\n\u001b[0;32m    472\u001b[0m     \"\"\"\n\u001b[0;32m    473\u001b[0m     \u001b[1;32mwith\u001b[0m \u001b[0mgraph\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mas_default\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m--> 474\u001b[1;33m       \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0m_fetch_mapper\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0m_FetchMapper\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mfor_fetch\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mfetches\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m    475\u001b[0m     \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0m_fetches\u001b[0m \u001b[1;33m=\u001b[0m \u001b[1;33m[\u001b[0m\u001b[1;33m]\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    476\u001b[0m     \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0m_targets\u001b[0m \u001b[1;33m=\u001b[0m \u001b[1;33m[\u001b[0m\u001b[1;33m]\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32m~\\anaconda3\\envs\\tf1\\lib\\site-packages\\tensorflow_core\\python\\client\\session.py\u001b[0m in \u001b[0;36mfor_fetch\u001b[1;34m(fetch)\u001b[0m\n\u001b[0;32m    264\u001b[0m     \u001b[1;32melif\u001b[0m \u001b[0misinstance\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mfetch\u001b[0m\u001b[1;33m,\u001b[0m \u001b[1;33m(\u001b[0m\u001b[0mlist\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mtuple\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    265\u001b[0m       \u001b[1;31m# NOTE(touts): This is also the code path for namedtuples.\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m--> 266\u001b[1;33m       \u001b[1;32mreturn\u001b[0m \u001b[0m_ListFetchMapper\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mfetch\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m    267\u001b[0m     \u001b[1;32melif\u001b[0m \u001b[0misinstance\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mfetch\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mcollections_abc\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mMapping\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    268\u001b[0m       \u001b[1;32mreturn\u001b[0m \u001b[0m_DictFetchMapper\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mfetch\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32m~\\anaconda3\\envs\\tf1\\lib\\site-packages\\tensorflow_core\\python\\client\\session.py\u001b[0m in \u001b[0;36m__init__\u001b[1;34m(self, fetches)\u001b[0m\n\u001b[0;32m    373\u001b[0m     \"\"\"\n\u001b[0;32m    374\u001b[0m     \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0m_fetch_type\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mtype\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mfetches\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m--> 375\u001b[1;33m     \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0m_mappers\u001b[0m \u001b[1;33m=\u001b[0m \u001b[1;33m[\u001b[0m\u001b[0m_FetchMapper\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mfor_fetch\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mfetch\u001b[0m\u001b[1;33m)\u001b[0m \u001b[1;32mfor\u001b[0m \u001b[0mfetch\u001b[0m \u001b[1;32min\u001b[0m \u001b[0mfetches\u001b[0m\u001b[1;33m]\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m    376\u001b[0m     \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0m_unique_fetches\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0m_value_indices\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0m_uniquify_fetches\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0m_mappers\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    377\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32m~\\anaconda3\\envs\\tf1\\lib\\site-packages\\tensorflow_core\\python\\client\\session.py\u001b[0m in \u001b[0;36m<listcomp>\u001b[1;34m(.0)\u001b[0m\n\u001b[0;32m    373\u001b[0m     \"\"\"\n\u001b[0;32m    374\u001b[0m     \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0m_fetch_type\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mtype\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mfetches\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m--> 375\u001b[1;33m     \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0m_mappers\u001b[0m \u001b[1;33m=\u001b[0m \u001b[1;33m[\u001b[0m\u001b[0m_FetchMapper\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mfor_fetch\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mfetch\u001b[0m\u001b[1;33m)\u001b[0m \u001b[1;32mfor\u001b[0m \u001b[0mfetch\u001b[0m \u001b[1;32min\u001b[0m \u001b[0mfetches\u001b[0m\u001b[1;33m]\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m    376\u001b[0m     \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0m_unique_fetches\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0m_value_indices\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0m_uniquify_fetches\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0m_mappers\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    377\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32m~\\anaconda3\\envs\\tf1\\lib\\site-packages\\tensorflow_core\\python\\client\\session.py\u001b[0m in \u001b[0;36mfor_fetch\u001b[1;34m(fetch)\u001b[0m\n\u001b[0;32m    274\u001b[0m         \u001b[1;32mif\u001b[0m \u001b[0misinstance\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mfetch\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mtensor_type\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    275\u001b[0m           \u001b[0mfetches\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mcontraction_fn\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mfetch_fn\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mfetch\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m--> 276\u001b[1;33m           \u001b[1;32mreturn\u001b[0m \u001b[0m_ElementFetchMapper\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mfetches\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mcontraction_fn\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m    277\u001b[0m     \u001b[1;31m# Did not find anything.\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    278\u001b[0m     raise TypeError('Fetch argument %r has invalid type %r' %\n",
      "\u001b[1;32m~\\anaconda3\\envs\\tf1\\lib\\site-packages\\tensorflow_core\\python\\client\\session.py\u001b[0m in \u001b[0;36m__init__\u001b[1;34m(self, fetches, contraction_fn)\u001b[0m\n\u001b[0;32m    310\u001b[0m       \u001b[1;32mexcept\u001b[0m \u001b[0mValueError\u001b[0m \u001b[1;32mas\u001b[0m \u001b[0me\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    311\u001b[0m         raise ValueError('Fetch argument %r cannot be interpreted as a '\n\u001b[1;32m--> 312\u001b[1;33m                          'Tensor. (%s)' % (fetch, str(e)))\n\u001b[0m\u001b[0;32m    313\u001b[0m       \u001b[1;32mexcept\u001b[0m \u001b[0mKeyError\u001b[0m \u001b[1;32mas\u001b[0m \u001b[0me\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    314\u001b[0m         raise ValueError('Fetch argument %r cannot be interpreted as a '\n",
      "\u001b[1;31mValueError\u001b[0m: Fetch argument <tf.Tensor 'rnn/transpose_1:0' shape=(1, 1, 3) dtype=float32> cannot be interpreted as a Tensor. (Tensor Tensor(\"rnn/transpose_1:0\", shape=(1, 1, 3), dtype=float32) is not an element of this graph.)"
     ]
    }
   ],
   "source": [
    "with tf.Session() as sess:\n",
    "    sess.run(tf.global_variables_initializer())\n",
    "    output_run, state_run = sess.run([outputs, state])\n",
    "    \n",
    "    print(\"Output Value\")\n",
    "    print(output_run)\n",
    "    \n",
    "    print(\"State Value\")\n",
    "    print(state_run)\n",
    "    \n",
    "    values = sess.run(variables_names)\n",
    "    for k, v in zip(variables_names, values):\n",
    "        print(k, v)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 48,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Tensor(\"rnn/transpose_1:0\", shape=(1, 5, 3), dtype=float32)\n",
      "----------------------\n",
      "Tensor(\"rnn/while/Exit_3:0\", shape=(1, 3), dtype=float32)\n",
      "----------------------\n",
      "<tf.Variable 'rnn/basic_rnn_cell/kernel:0' shape=(7, 3) dtype=float32_ref>\n",
      "<tf.Variable 'rnn/basic_rnn_cell/bias:0' shape=(3,) dtype=float32_ref>\n"
     ]
    }
   ],
   "source": [
    "tf.reset_default_graph()\n",
    "h = [1, 0, 0, 0]\n",
    "e = [0, 1, 0, 0]\n",
    "l = [0, 0, 1, 0]\n",
    "o = [0, 0, 0, 1]\n",
    "\n",
    "rnn_cell = tf.contrib.rnn.BasicRNNCell(num_units=3)\n",
    "\n",
    "x_data = np.array([[h, e, l, l, o]], dtype=np.float32)\n",
    "outputs, state = tf.nn.dynamic_rnn(cell=rnn_cell, dtype=tf.float32, inputs=x_data)\n",
    "variables_names = [v.name for v in tf.trainable_variables()]\n",
    "\n",
    "print(outputs)\n",
    "print(\"----------------------\")\n",
    "print(state)\n",
    "print(\"----------------------\")\n",
    "for w in tf.get_collection(tf.GraphKeys.TRAINABLE_VARIABLES):\n",
    "    print(w)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 49,
   "metadata": {
    "collapsed": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Output Value\n",
      "[[[-0.00233542  0.05193848 -0.5742626 ]\n",
      "  [-0.3266077   0.10008996  0.13926423]\n",
      "  [-0.15353745 -0.30581573 -0.60393846]\n",
      "  [ 0.45869008 -0.7267591  -0.725342  ]\n",
      "  [ 0.86987185 -0.8058975  -0.08595657]]]\n",
      "State Value\n",
      "[[ 0.86987185 -0.8058975  -0.08595657]]\n",
      "rnn/basic_rnn_cell/kernel:0 [[-0.00233543  0.05198526 -0.6538598 ]\n",
      " [-0.6641659   0.4281752   0.37294662]\n",
      " [ 0.05170107 -0.5519403  -0.6884491 ]\n",
      " [ 0.49469054 -0.33068338  0.04296321]\n",
      " [ 0.2556615  -0.3970473   0.16777241]\n",
      " [-0.39017454  0.23633456 -0.11033732]\n",
      " [-0.60251516  0.59371984  0.39467812]]\n",
      "rnn/basic_rnn_cell/bias:0 [0. 0. 0.]\n"
     ]
    }
   ],
   "source": [
    "# 결과\n",
    "with tf.Session() as sess:\n",
    "    sess.run(tf.global_variables_initializer())\n",
    "    output_run, state_run = sess.run([outputs, state])\n",
    "    \n",
    "    print(\"Output Value\")\n",
    "    print(output_run)\n",
    "    \n",
    "    print(\"State Value\")\n",
    "    print(state_run)\n",
    "    \n",
    "    values = sess.run(variables_names)\n",
    "    for k, v in zip(variables_names, values):\n",
    "        print(k, v)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### hihello 학습 과정"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 165,
   "metadata": {},
   "outputs": [],
   "source": [
    "idx2char = ['h', 'i', 'e', 'l', 'o']\n",
    "x_one_hot = [[[1, 0, 0, 0, 0], \n",
    "              [0, 1, 0, 0, 0], \n",
    "              [1, 0, 0, 0, 0],\n",
    "              [0, 0, 1, 0, 0], \n",
    "              [0, 0, 0, 1, 0], \n",
    "              [0, 0, 0, 1, 0],\n",
    "              [0, 0, 0, 0, 1]]]  \n",
    "#             [0, 0, 0, 0, 1] 삭제__sequence len = 6일때\n",
    "\n",
    "# x_data = [[0, 1, 0, 2, 3, 3]] # hihell\n",
    "y_data = [[0, 1, 0, 2, 3, 3, 4]] # ihello  # 0 삭제_shape 맞추기"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 166,
   "metadata": {},
   "outputs": [],
   "source": [
    "# parameter\n",
    "sequence_len = 7  # 입력 데이터 갯수\n",
    "num_classes = 5  # 입력 데이터의 종류(중복제외)\n",
    "batch_size = 1  "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 167,
   "metadata": {},
   "outputs": [],
   "source": [
    "tf.reset_default_graph()\n",
    "\n",
    "# X, y 준비\n",
    "X = tf.placeholder(tf.float32, shape=[None, sequence_len, num_classes])  # shape=[전체갯수, 입력갯수, 원하는 출력갯수]\n",
    "y = tf.placeholder(tf.int32, shape=[None, sequence_len])  # 정답"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 168,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Tensor(\"rnn/transpose_1:0\", shape=(?, 7, 5), dtype=float32)\n",
      "Tensor(\"rnn/while/Exit_3:0\", shape=(?, 5), dtype=float32)\n"
     ]
    }
   ],
   "source": [
    "# rnn 모델\n",
    "cell = tf.contrib.rnn.BasicRNNCell(num_units=num_classes)\n",
    "outputs, state = tf.nn.dynamic_rnn(cell=cell, inputs=X, dtype=tf.float32)\n",
    "print(outputs)\n",
    "print(state)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 169,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Tensor(\"Reshape:0\", shape=(?, 5), dtype=float32)\n"
     ]
    }
   ],
   "source": [
    "# 평면화(Flat Layer) __tf.flat()\n",
    "X_for_fc = tf.reshape(outputs, [-1, num_classes])\n",
    "print(X_for_fc)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 170,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0 cost 1.5060341     prediction: [[0 0 0 0 3 3 3]]     정답: [[0, 1, 0, 2, 3, 3, 4]]\n",
      "1 cost 1.2012122     prediction: [[0 0 0 0 3 3 3]]     정답: [[0, 1, 0, 2, 3, 3, 4]]\n",
      "2 cost 0.9637184     prediction: [[0 1 0 2 3 3 3]]     정답: [[0, 1, 0, 2, 3, 3, 4]]\n",
      "3 cost 0.742112     prediction: [[0 1 0 2 3 3 3]]     정답: [[0, 1, 0, 2, 3, 3, 4]]\n",
      "4 cost 0.54294395     prediction: [[0 1 0 2 3 3 4]]     정답: [[0, 1, 0, 2, 3, 3, 4]]\n",
      "5 cost 0.3977509     prediction: [[0 1 0 2 3 3 4]]     정답: [[0, 1, 0, 2, 3, 3, 4]]\n",
      "6 cost 0.29055333     prediction: [[0 1 0 2 3 3 4]]     정답: [[0, 1, 0, 2, 3, 3, 4]]\n",
      "7 cost 0.2077953     prediction: [[0 1 0 2 3 3 4]]     정답: [[0, 1, 0, 2, 3, 3, 4]]\n",
      "8 cost 0.1498015     prediction: [[0 1 0 2 3 3 4]]     정답: [[0, 1, 0, 2, 3, 3, 4]]\n",
      "9 cost 0.10862583     prediction: [[0 1 0 2 3 3 4]]     정답: [[0, 1, 0, 2, 3, 3, 4]]\n",
      "결과 :  h,i,h,e,l,l,o\n"
     ]
    }
   ],
   "source": [
    "#Fully Connected (W, b, logit)\n",
    "outputs = tf.contrib.layers.fully_connected(inputs=X_for_fc, num_outputs=num_classes, activation_fn=None)\n",
    "\n",
    "outputs = tf.reshape(outputs, [batch_size, sequence_len, num_classes]) \n",
    "W = tf.ones([batch_size, sequence_len])\n",
    "cost = tf.contrib.seq2seq.sequence_loss(logits=outputs, targets=y, weights=W)\n",
    "train = tf.train.AdamOptimizer(learning_rate=0.1).minimize(cost)\n",
    "\n",
    "with tf.Session() as sess:\n",
    "    sess.run(tf.global_variables_initializer())\n",
    "    \n",
    "    for i in range(10):\n",
    "        _, c = sess.run([train, cost], feed_dict={X:x_one_hot, y:y_data})\n",
    "        result = sess.run(tf.argmax(outputs, 2), feed_dict={X:x_one_hot})\n",
    "        print(i, \"cost\", c, \"    prediction:\", result, \"    정답:\", y_data)\n",
    "        \n",
    "    result_str = [idx2char[c] for c in np.squeeze(result)]  #np.squeeze():()축 하나 제외\n",
    "    print(\"결과 : \", \",\".join(result_str))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### 두 번째 예제"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 171,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[' ', 'i', 'n', 'w', 'o', 'f', 't', 'y', 'a', 'u']\n",
      "{' ': 0, 'i': 1, 'n': 2, 'w': 3, 'o': 4, 'f': 5, 't': 6, 'y': 7, 'a': 8, 'u': 9}\n",
      "[0, 1, 5, 0, 7, 4, 9, 0, 3, 8, 2, 6, 0, 7, 4, 9]\n"
     ]
    }
   ],
   "source": [
    "# 입력 데이터 준비\n",
    "# sample = \" You will have to submit to your fate whether you will or not.\"\n",
    "sample = \" if you want you\"\n",
    "idx2char = list(set(sample))\n",
    "print(idx2char)\n",
    "\n",
    "char2idx = {c: i for i, c in enumerate(idx2char)}\n",
    "print(char2idx)\n",
    "\n",
    "sample_idx = [char2idx[c] for c in sample]\n",
    "print(sample_idx)\n",
    "\n",
    "x_data = [sample_idx[:-1]]\n",
    "y_data = [sample_idx[1:]]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 172,
   "metadata": {},
   "outputs": [],
   "source": [
    "# parameter\n",
    "num_classes = len(char2idx)\n",
    "batch_size = 1\n",
    "sequence_len = len(sample) -1"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 173,
   "metadata": {},
   "outputs": [],
   "source": [
    "tf.reset_default_graph()\n",
    "\n",
    "# X, y 준비\n",
    "X = tf.placeholder(tf.int32, shape=[None, sequence_len, num_classes])\n",
    "y = tf.placeholder(tf.int32, shape=[None, sequence_len])  \n",
    "\n",
    "x_one_hot = tf.one_hot(X, num_classes)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 174,
   "metadata": {
    "collapsed": true
   },
   "outputs": [
    {
     "ename": "ValueError",
     "evalue": "Input 0 of layer basic_rnn_cell_1 is incompatible with the layer: expected ndim=2, found ndim=3. Full shape received: [None, 10, 10]",
     "output_type": "error",
     "traceback": [
      "\u001b[1;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[1;31mValueError\u001b[0m                                Traceback (most recent call last)",
      "\u001b[1;32m<ipython-input-174-712a45acb5d4>\u001b[0m in \u001b[0;36m<module>\u001b[1;34m\u001b[0m\n\u001b[0;32m      1\u001b[0m \u001b[0mcell\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mtf\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mcontrib\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mrnn\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mBasicRNNCell\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mnum_units\u001b[0m\u001b[1;33m=\u001b[0m\u001b[0mnum_classes\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m----> 2\u001b[1;33m \u001b[0moutputs\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mstate\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mtf\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mnn\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mdynamic_rnn\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mcell\u001b[0m\u001b[1;33m=\u001b[0m\u001b[0mcell\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0minputs\u001b[0m\u001b[1;33m=\u001b[0m\u001b[0mx_one_hot\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mdtype\u001b[0m\u001b[1;33m=\u001b[0m\u001b[0mtf\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mfloat32\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m      3\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m      4\u001b[0m \u001b[0mW\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mtf\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mones\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;33m[\u001b[0m\u001b[0mbatch_size\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0msequence_len\u001b[0m\u001b[1;33m]\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m      5\u001b[0m \u001b[0mcost\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mtf\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mreduce_mean\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mtf\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mcontrib\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mseq2seq\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0msequence_loss\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mlogits\u001b[0m\u001b[1;33m=\u001b[0m\u001b[0moutputs\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mtargets\u001b[0m\u001b[1;33m=\u001b[0m\u001b[0my\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mweights\u001b[0m\u001b[1;33m=\u001b[0m\u001b[0mW\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32m~\\anaconda3\\envs\\tf1\\lib\\site-packages\\tensorflow_core\\python\\util\\deprecation.py\u001b[0m in \u001b[0;36mnew_func\u001b[1;34m(*args, **kwargs)\u001b[0m\n\u001b[0;32m    322\u001b[0m               \u001b[1;34m'in a future version'\u001b[0m \u001b[1;32mif\u001b[0m \u001b[0mdate\u001b[0m \u001b[1;32mis\u001b[0m \u001b[1;32mNone\u001b[0m \u001b[1;32melse\u001b[0m \u001b[1;33m(\u001b[0m\u001b[1;34m'after %s'\u001b[0m \u001b[1;33m%\u001b[0m \u001b[0mdate\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m,\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    323\u001b[0m               instructions)\n\u001b[1;32m--> 324\u001b[1;33m       \u001b[1;32mreturn\u001b[0m \u001b[0mfunc\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;33m*\u001b[0m\u001b[0margs\u001b[0m\u001b[1;33m,\u001b[0m \u001b[1;33m**\u001b[0m\u001b[0mkwargs\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m    325\u001b[0m     return tf_decorator.make_decorator(\n\u001b[0;32m    326\u001b[0m         \u001b[0mfunc\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mnew_func\u001b[0m\u001b[1;33m,\u001b[0m \u001b[1;34m'deprecated'\u001b[0m\u001b[1;33m,\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32m~\\anaconda3\\envs\\tf1\\lib\\site-packages\\tensorflow_core\\python\\ops\\rnn.py\u001b[0m in \u001b[0;36mdynamic_rnn\u001b[1;34m(cell, inputs, sequence_length, initial_state, dtype, parallel_iterations, swap_memory, time_major, scope)\u001b[0m\n\u001b[0;32m    705\u001b[0m         \u001b[0mswap_memory\u001b[0m\u001b[1;33m=\u001b[0m\u001b[0mswap_memory\u001b[0m\u001b[1;33m,\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    706\u001b[0m         \u001b[0msequence_length\u001b[0m\u001b[1;33m=\u001b[0m\u001b[0msequence_length\u001b[0m\u001b[1;33m,\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m--> 707\u001b[1;33m         dtype=dtype)\n\u001b[0m\u001b[0;32m    708\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    709\u001b[0m     \u001b[1;31m# Outputs of _dynamic_rnn_loop are always shaped [time, batch, depth].\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32m~\\anaconda3\\envs\\tf1\\lib\\site-packages\\tensorflow_core\\python\\ops\\rnn.py\u001b[0m in \u001b[0;36m_dynamic_rnn_loop\u001b[1;34m(cell, inputs, initial_state, parallel_iterations, swap_memory, sequence_length, dtype)\u001b[0m\n\u001b[0;32m    914\u001b[0m       \u001b[0mparallel_iterations\u001b[0m\u001b[1;33m=\u001b[0m\u001b[0mparallel_iterations\u001b[0m\u001b[1;33m,\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    915\u001b[0m       \u001b[0mmaximum_iterations\u001b[0m\u001b[1;33m=\u001b[0m\u001b[0mtime_steps\u001b[0m\u001b[1;33m,\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m--> 916\u001b[1;33m       swap_memory=swap_memory)\n\u001b[0m\u001b[0;32m    917\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    918\u001b[0m   \u001b[1;31m# Unpack final output if not using output tuples.\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32m~\\anaconda3\\envs\\tf1\\lib\\site-packages\\tensorflow_core\\python\\ops\\control_flow_ops.py\u001b[0m in \u001b[0;36mwhile_loop\u001b[1;34m(cond, body, loop_vars, shape_invariants, parallel_iterations, back_prop, swap_memory, name, maximum_iterations, return_same_structure)\u001b[0m\n\u001b[0;32m   2751\u001b[0m       \u001b[0mops\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0madd_to_collection\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mops\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mGraphKeys\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mWHILE_CONTEXT\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mloop_context\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m   2752\u001b[0m     result = loop_context.BuildLoop(cond, body, loop_vars, shape_invariants,\n\u001b[1;32m-> 2753\u001b[1;33m                                     return_same_structure)\n\u001b[0m\u001b[0;32m   2754\u001b[0m     \u001b[1;32mif\u001b[0m \u001b[0mmaximum_iterations\u001b[0m \u001b[1;32mis\u001b[0m \u001b[1;32mnot\u001b[0m \u001b[1;32mNone\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m   2755\u001b[0m       \u001b[1;32mreturn\u001b[0m \u001b[0mresult\u001b[0m\u001b[1;33m[\u001b[0m\u001b[1;36m1\u001b[0m\u001b[1;33m]\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32m~\\anaconda3\\envs\\tf1\\lib\\site-packages\\tensorflow_core\\python\\ops\\control_flow_ops.py\u001b[0m in \u001b[0;36mBuildLoop\u001b[1;34m(self, pred, body, loop_vars, shape_invariants, return_same_structure)\u001b[0m\n\u001b[0;32m   2243\u001b[0m       \u001b[1;32mwith\u001b[0m \u001b[0mops\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mget_default_graph\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0m_mutation_lock\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m:\u001b[0m  \u001b[1;31m# pylint: disable=protected-access\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m   2244\u001b[0m         original_body_result, exit_vars = self._BuildLoop(\n\u001b[1;32m-> 2245\u001b[1;33m             pred, body, original_loop_vars, loop_vars, shape_invariants)\n\u001b[0m\u001b[0;32m   2246\u001b[0m     \u001b[1;32mfinally\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m   2247\u001b[0m       \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mExit\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32m~\\anaconda3\\envs\\tf1\\lib\\site-packages\\tensorflow_core\\python\\ops\\control_flow_ops.py\u001b[0m in \u001b[0;36m_BuildLoop\u001b[1;34m(self, pred, body, original_loop_vars, loop_vars, shape_invariants)\u001b[0m\n\u001b[0;32m   2168\u001b[0m         expand_composites=True)\n\u001b[0;32m   2169\u001b[0m     \u001b[0mpre_summaries\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mops\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mget_collection\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mops\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mGraphKeys\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0m_SUMMARY_COLLECTION\u001b[0m\u001b[1;33m)\u001b[0m  \u001b[1;31m# pylint: disable=protected-access\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m-> 2170\u001b[1;33m     \u001b[0mbody_result\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mbody\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;33m*\u001b[0m\u001b[0mpacked_vars_for_body\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m   2171\u001b[0m     \u001b[0mpost_summaries\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mops\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mget_collection\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mops\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mGraphKeys\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0m_SUMMARY_COLLECTION\u001b[0m\u001b[1;33m)\u001b[0m  \u001b[1;31m# pylint: disable=protected-access\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m   2172\u001b[0m     \u001b[1;32mif\u001b[0m \u001b[1;32mnot\u001b[0m \u001b[0mnest\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mis_sequence_or_composite\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mbody_result\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32m~\\anaconda3\\envs\\tf1\\lib\\site-packages\\tensorflow_core\\python\\ops\\control_flow_ops.py\u001b[0m in \u001b[0;36m<lambda>\u001b[1;34m(i, lv)\u001b[0m\n\u001b[0;32m   2703\u001b[0m         cond = lambda i, lv: (  # pylint: disable=g-long-lambda\n\u001b[0;32m   2704\u001b[0m             math_ops.logical_and(i < maximum_iterations, orig_cond(*lv)))\n\u001b[1;32m-> 2705\u001b[1;33m         \u001b[0mbody\u001b[0m \u001b[1;33m=\u001b[0m \u001b[1;32mlambda\u001b[0m \u001b[0mi\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mlv\u001b[0m\u001b[1;33m:\u001b[0m \u001b[1;33m(\u001b[0m\u001b[0mi\u001b[0m \u001b[1;33m+\u001b[0m \u001b[1;36m1\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0morig_body\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;33m*\u001b[0m\u001b[0mlv\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m   2706\u001b[0m       \u001b[0mtry_to_pack\u001b[0m \u001b[1;33m=\u001b[0m \u001b[1;32mFalse\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m   2707\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32m~\\anaconda3\\envs\\tf1\\lib\\site-packages\\tensorflow_core\\python\\ops\\rnn.py\u001b[0m in \u001b[0;36m_time_step\u001b[1;34m(time, output_ta_t, state)\u001b[0m\n\u001b[0;32m    882\u001b[0m           skip_conditionals=True)\n\u001b[0;32m    883\u001b[0m     \u001b[1;32melse\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m--> 884\u001b[1;33m       \u001b[1;33m(\u001b[0m\u001b[0moutput\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mnew_state\u001b[0m\u001b[1;33m)\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mcall_cell\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m    885\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    886\u001b[0m     \u001b[1;31m# Keras cells always wrap state as list, even if it's a single tensor.\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32m~\\anaconda3\\envs\\tf1\\lib\\site-packages\\tensorflow_core\\python\\ops\\rnn.py\u001b[0m in \u001b[0;36m<lambda>\u001b[1;34m()\u001b[0m\n\u001b[0;32m    868\u001b[0m     \u001b[1;32mif\u001b[0m \u001b[0mis_keras_rnn_cell\u001b[0m \u001b[1;32mand\u001b[0m \u001b[1;32mnot\u001b[0m \u001b[0mnest\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mis_sequence\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mstate\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    869\u001b[0m       \u001b[0mstate\u001b[0m \u001b[1;33m=\u001b[0m \u001b[1;33m[\u001b[0m\u001b[0mstate\u001b[0m\u001b[1;33m]\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m--> 870\u001b[1;33m     \u001b[0mcall_cell\u001b[0m \u001b[1;33m=\u001b[0m \u001b[1;32mlambda\u001b[0m\u001b[1;33m:\u001b[0m \u001b[0mcell\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0minput_t\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mstate\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m    871\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    872\u001b[0m     \u001b[1;32mif\u001b[0m \u001b[0msequence_length\u001b[0m \u001b[1;32mis\u001b[0m \u001b[1;32mnot\u001b[0m \u001b[1;32mNone\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32m~\\anaconda3\\envs\\tf1\\lib\\site-packages\\tensorflow_core\\python\\ops\\rnn_cell_impl.py\u001b[0m in \u001b[0;36m__call__\u001b[1;34m(self, inputs, state, scope, *args, **kwargs)\u001b[0m\n\u001b[0;32m    384\u001b[0m     \u001b[1;31m# method.  See the class docstring for more details.\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    385\u001b[0m     return base_layer.Layer.__call__(\n\u001b[1;32m--> 386\u001b[1;33m         self, inputs, state, scope=scope, *args, **kwargs)\n\u001b[0m\u001b[0;32m    387\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    388\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32m~\\anaconda3\\envs\\tf1\\lib\\site-packages\\tensorflow_core\\python\\layers\\base.py\u001b[0m in \u001b[0;36m__call__\u001b[1;34m(self, inputs, *args, **kwargs)\u001b[0m\n\u001b[0;32m    546\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    547\u001b[0m       \u001b[1;31m# Actually call layer\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m--> 548\u001b[1;33m       \u001b[0moutputs\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0msuper\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mLayer\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mself\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0m__call__\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0minputs\u001b[0m\u001b[1;33m,\u001b[0m \u001b[1;33m*\u001b[0m\u001b[0margs\u001b[0m\u001b[1;33m,\u001b[0m \u001b[1;33m**\u001b[0m\u001b[0mkwargs\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m    549\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    550\u001b[0m     \u001b[1;32mif\u001b[0m \u001b[1;32mnot\u001b[0m \u001b[0mcontext\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mexecuting_eagerly\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32m~\\anaconda3\\envs\\tf1\\lib\\site-packages\\tensorflow_core\\python\\keras\\engine\\base_layer.py\u001b[0m in \u001b[0;36m__call__\u001b[1;34m(self, inputs, *args, **kwargs)\u001b[0m\n\u001b[0;32m    817\u001b[0m         \u001b[1;31m# are casted, not before.\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    818\u001b[0m         input_spec.assert_input_compatibility(self.input_spec, inputs,\n\u001b[1;32m--> 819\u001b[1;33m                                               self.name)\n\u001b[0m\u001b[0;32m    820\u001b[0m         \u001b[0mgraph\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mbackend\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mget_graph\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    821\u001b[0m         \u001b[1;32mwith\u001b[0m \u001b[0mgraph\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mas_default\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mbackend\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mname_scope\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0m_name_scope\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32m~\\anaconda3\\envs\\tf1\\lib\\site-packages\\tensorflow_core\\python\\keras\\engine\\input_spec.py\u001b[0m in \u001b[0;36massert_input_compatibility\u001b[1;34m(input_spec, inputs, layer_name)\u001b[0m\n\u001b[0;32m    175\u001b[0m                          \u001b[1;34m'expected ndim='\u001b[0m \u001b[1;33m+\u001b[0m \u001b[0mstr\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mspec\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mndim\u001b[0m\u001b[1;33m)\u001b[0m \u001b[1;33m+\u001b[0m \u001b[1;34m', found ndim='\u001b[0m \u001b[1;33m+\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    176\u001b[0m                          \u001b[0mstr\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mndim\u001b[0m\u001b[1;33m)\u001b[0m \u001b[1;33m+\u001b[0m \u001b[1;34m'. Full shape received: '\u001b[0m \u001b[1;33m+\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m--> 177\u001b[1;33m                          str(x.shape.as_list()))\n\u001b[0m\u001b[0;32m    178\u001b[0m     \u001b[1;32mif\u001b[0m \u001b[0mspec\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mmax_ndim\u001b[0m \u001b[1;32mis\u001b[0m \u001b[1;32mnot\u001b[0m \u001b[1;32mNone\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    179\u001b[0m       \u001b[0mndim\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mx\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mshape\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mndims\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;31mValueError\u001b[0m: Input 0 of layer basic_rnn_cell_1 is incompatible with the layer: expected ndim=2, found ndim=3. Full shape received: [None, 10, 10]"
     ]
    }
   ],
   "source": [
    "cell = tf.contrib.rnn.BasicRNNCell(num_units=num_classes)\n",
    "outputs, state = tf.nn.dynamic_rnn(cell=cell, inputs=x_one_hot, dtype=tf.float32)\n",
    "\n",
    "W = tf.ones([batch_size, sequence_len])\n",
    "cost = tf.reduce_mean(tf.contrib.seq2seq.sequence_loss(logits=outputs, targets=y, weights=W))\n",
    "train = tf.train.AdamOptimizer(learning_rate=0.1).minimize(cost)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 133,
   "metadata": {
    "collapsed": true
   },
   "outputs": [
    {
     "ename": "ValueError",
     "evalue": "Cannot feed value of shape (1, 6) for Tensor 'Placeholder:0', which has shape '(?, 15, 10)'",
     "output_type": "error",
     "traceback": [
      "\u001b[1;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[1;31mValueError\u001b[0m                                Traceback (most recent call last)",
      "\u001b[1;32m<ipython-input-133-d97f5586e1c8>\u001b[0m in \u001b[0;36m<module>\u001b[1;34m\u001b[0m\n\u001b[0;32m      3\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m      4\u001b[0m     \u001b[1;32mfor\u001b[0m \u001b[0mi\u001b[0m \u001b[1;32min\u001b[0m \u001b[0mrange\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;36m100\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m----> 5\u001b[1;33m         \u001b[0m_\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mc\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0msess\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mrun\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;33m[\u001b[0m\u001b[0mtrain\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mcost\u001b[0m\u001b[1;33m]\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mfeed_dict\u001b[0m\u001b[1;33m=\u001b[0m\u001b[1;33m{\u001b[0m\u001b[0mX\u001b[0m\u001b[1;33m:\u001b[0m\u001b[0mx_data\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0my\u001b[0m\u001b[1;33m:\u001b[0m\u001b[0my_data\u001b[0m\u001b[1;33m}\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m      6\u001b[0m         \u001b[0mresult\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0msess\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mrun\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mtf\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0margmax\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0moutputs\u001b[0m\u001b[1;33m,\u001b[0m \u001b[1;36m2\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mfeed_dict\u001b[0m\u001b[1;33m=\u001b[0m\u001b[1;33m{\u001b[0m\u001b[0mX\u001b[0m\u001b[1;33m:\u001b[0m\u001b[0mx_data\u001b[0m\u001b[1;33m}\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m      7\u001b[0m         \u001b[0mprint\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mi\u001b[0m\u001b[1;33m,\u001b[0m \u001b[1;34m\"cost\"\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mc\u001b[0m\u001b[1;33m,\u001b[0m \u001b[1;34m\"    prediction:\"\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mresult\u001b[0m\u001b[1;33m,\u001b[0m \u001b[1;34m\"    정답:\"\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0my_data\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32m~\\anaconda3\\envs\\tf1\\lib\\site-packages\\tensorflow_core\\python\\client\\session.py\u001b[0m in \u001b[0;36mrun\u001b[1;34m(self, fetches, feed_dict, options, run_metadata)\u001b[0m\n\u001b[0;32m    954\u001b[0m     \u001b[1;32mtry\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    955\u001b[0m       result = self._run(None, fetches, feed_dict, options_ptr,\n\u001b[1;32m--> 956\u001b[1;33m                          run_metadata_ptr)\n\u001b[0m\u001b[0;32m    957\u001b[0m       \u001b[1;32mif\u001b[0m \u001b[0mrun_metadata\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    958\u001b[0m         \u001b[0mproto_data\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mtf_session\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mTF_GetBuffer\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mrun_metadata_ptr\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32m~\\anaconda3\\envs\\tf1\\lib\\site-packages\\tensorflow_core\\python\\client\\session.py\u001b[0m in \u001b[0;36m_run\u001b[1;34m(self, handle, fetches, feed_dict, options, run_metadata)\u001b[0m\n\u001b[0;32m   1154\u001b[0m                 \u001b[1;34m'Cannot feed value of shape %r for Tensor %r, '\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m   1155\u001b[0m                 \u001b[1;34m'which has shape %r'\u001b[0m \u001b[1;33m%\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m-> 1156\u001b[1;33m                 (np_val.shape, subfeed_t.name, str(subfeed_t.get_shape())))\n\u001b[0m\u001b[0;32m   1157\u001b[0m           \u001b[1;32mif\u001b[0m \u001b[1;32mnot\u001b[0m \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mgraph\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mis_feedable\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0msubfeed_t\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m   1158\u001b[0m             \u001b[1;32mraise\u001b[0m \u001b[0mValueError\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;34m'Tensor %s may not be fed.'\u001b[0m \u001b[1;33m%\u001b[0m \u001b[0msubfeed_t\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;31mValueError\u001b[0m: Cannot feed value of shape (1, 6) for Tensor 'Placeholder:0', which has shape '(?, 15, 10)'"
     ]
    }
   ],
   "source": [
    "with tf.Session() as sess:\n",
    "    sess.run(tf.global_variables_initializer())\n",
    "    \n",
    "    for i in range(100):\n",
    "        _, c = sess.run([train, cost], feed_dict={X:x_data, y:y_data})\n",
    "        result = sess.run(tf.argmax(outputs, 2), feed_dict={X:x_data})\n",
    "        print(i, \"cost\", c, \"    prediction:\", result, \"    정답:\", y_data)\n",
    "        \n",
    "        result_str = [idx2char[c] for c in np.squeeze(result)] \n",
    "        print(\"결과 : \", \",\".join(result_str))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## LSTM(Long Short Term Memory)\n",
    "\n",
    "+ BackPropagation Through Time(BPTT)\n",
    "    - Gradient Vanishing problem(기울기 소실) : 값이 작아져 문제\n",
    "    - Gradient Exploding problem(기울기 폭주) : 값이 너무 커져서 문제 ex.자연어처리시 문장 혹은 단어가 길어질때 발생"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[[[0.09927537]]]\n",
      "[[0.09927537]]\n",
      "[[0.18134572]]\n"
     ]
    }
   ],
   "source": [
    "tf.reset_default_graph()\n",
    "tf.set_random_seed(777)\n",
    "\n",
    "inputs = np.array([[[1, 0]]])\n",
    "\n",
    "tf_inputs = tf.constant(inputs, dtype=tf.float32)\n",
    "cell = tf.nn.rnn_cell.LSTMCell(num_units=1)\n",
    "outputs, states = tf.nn.dynamic_rnn(cell=cell, dtype=tf.float32, inputs=tf_inputs)\n",
    "\n",
    "with tf.Session() as sess:\n",
    "    sess.run(tf.global_variables_initializer())\n",
    "    outputs_run, states_run = sess.run([outputs, states])\n",
    "    \n",
    "    print(outputs_run) # y값\n",
    "    print(states_run.h) # hidden state(s)\n",
    "    print(states_run.c) # memory cell(추가)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Dynamic RNN"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### 데이터 준비"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>paragraph</th>\n",
       "      <th>category</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>dishplace is located in sunnyvale downtown the...</td>\n",
       "      <td>food</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>service can be slower during busy hours but ou...</td>\n",
       "      <td>food</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>portions are huge both french toast and their ...</td>\n",
       "      <td>food</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>we started with apps going the chicken and waf...</td>\n",
       "      <td>food</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>the biscuits and gravy was too salty two peopl...</td>\n",
       "      <td>food</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>the garlic fries were a great starter (and a h...</td>\n",
       "      <td>food</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6</th>\n",
       "      <td>our meal was excellent i had the pasta ai form...</td>\n",
       "      <td>food</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7</th>\n",
       "      <td>what i enjoy most about palo alto is so many r...</td>\n",
       "      <td>food</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8</th>\n",
       "      <td>the drinks came out fairly quickly a good two ...</td>\n",
       "      <td>food</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9</th>\n",
       "      <td>despite the not so good burger the service was...</td>\n",
       "      <td>food</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>10</th>\n",
       "      <td>the four reigning major champions simona halep...</td>\n",
       "      <td>sports</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>11</th>\n",
       "      <td>the briton was seeded nn7 here last year befor...</td>\n",
       "      <td>sports</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>12</th>\n",
       "      <td>stephens surged her way back from injury in st...</td>\n",
       "      <td>sports</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>13</th>\n",
       "      <td>when it came to england chances in the world c...</td>\n",
       "      <td>sports</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>14</th>\n",
       "      <td>the team that eliminated russia – croatia – al...</td>\n",
       "      <td>sports</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>15</th>\n",
       "      <td>the perseyside outfit finished in fourth place...</td>\n",
       "      <td>sports</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>16</th>\n",
       "      <td>liverpool fc will return to premier league act...</td>\n",
       "      <td>sports</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>17</th>\n",
       "      <td>alisson signed for liverpool fc from as roma t...</td>\n",
       "      <td>sports</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>18</th>\n",
       "      <td>but the rankings during that run-in to new yor...</td>\n",
       "      <td>sports</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>19</th>\n",
       "      <td>then came the oh-so-familiar djokovic-nadal no...</td>\n",
       "      <td>sports</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                                            paragraph category\n",
       "0   dishplace is located in sunnyvale downtown the...     food\n",
       "1   service can be slower during busy hours but ou...     food\n",
       "2   portions are huge both french toast and their ...     food\n",
       "3   we started with apps going the chicken and waf...     food\n",
       "4   the biscuits and gravy was too salty two peopl...     food\n",
       "5   the garlic fries were a great starter (and a h...     food\n",
       "6   our meal was excellent i had the pasta ai form...     food\n",
       "7   what i enjoy most about palo alto is so many r...     food\n",
       "8   the drinks came out fairly quickly a good two ...     food\n",
       "9   despite the not so good burger the service was...     food\n",
       "10  the four reigning major champions simona halep...   sports\n",
       "11  the briton was seeded nn7 here last year befor...   sports\n",
       "12  stephens surged her way back from injury in st...   sports\n",
       "13  when it came to england chances in the world c...   sports\n",
       "14  the team that eliminated russia – croatia – al...   sports\n",
       "15  the perseyside outfit finished in fourth place...   sports\n",
       "16  liverpool fc will return to premier league act...   sports\n",
       "17  alisson signed for liverpool fc from as roma t...   sports\n",
       "18  but the rankings during that run-in to new yor...   sports\n",
       "19  then came the oh-so-familiar djokovic-nadal no...   sports"
      ]
     },
     "execution_count": 11,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "paragraph_dict_list = [\n",
    "         {'paragraph': 'dishplace is located in sunnyvale downtown there is parking around the area but it can be difficult to find during peak business hours my sisters and i came to this place for dinner on a weekday they were really busy so i highly recommended making reservations unless you have the patience to wait', 'category': 'food'},\n",
    "         {'paragraph': 'service can be slower during busy hours but our waiter was courteous and help gave some great entree recommendations', 'category': 'food'},\n",
    "         {'paragraph': 'portions are huge both french toast and their various omelettes are really good their french toast is probably 1.5x more than other brunch places great place to visit if you are hungry and dont want to wait 1 hour for a table', 'category': 'food'},\n",
    "         {'paragraph': 'we started with apps going the chicken and waffle slides and chicken nachos the sliders were amazing and the nachos were good too maybe by themselves the nachos would have scored better but after those sliders they were up against some tough competition', 'category': 'food'},\n",
    "         {'paragraph': 'the biscuits and gravy was too salty two people in my group had the gravy and all thought it was too salty my hubby ordered a side of double egg and it was served on two small plates who serves eggs to one person on separate plates we commented on that when it was delivered and even the server laughed and said she doesnt know why the kitchen does that presentation of food is important and they really missed on this one', 'category': 'food'},\n",
    "         {'paragraph': 'the garlic fries were a great starter (and a happy hour special) the pancakes looked and tasted great and were a fairly generous portion', 'category': 'food'},\n",
    "         {'paragraph': 'our meal was excellent i had the pasta ai formaggi which was so rich i didnt dare eat it all although i certainly wanted to excellent flavors with a great texture contrast between the soft pasta and the crisp bread crumbs too much sauce for me but a wonderful dish', 'category': 'food'},\n",
    "         {'paragraph': 'what i enjoy most about palo alto is so many restaurants have dog-friendly seating outside i had bookmarked italico from when they first opened about a 1.5 years ago and was jonesing for some pasta so time to finally knock that bookmark off', 'category': 'food'},\n",
    "         {'paragraph': 'the drinks came out fairly quickly a good two to three minutes after the orders were taken i expected my iced tea to taste a bit more sweet but this was straight up green tea with ice in it not to complain of course but i was pleasantly surprised', 'category': 'food'},\n",
    "         {'paragraph': 'despite the not so good burger the service was so slow the restaurant wasnt even half full and they took very long from the moment we got seated to the time we left it was almost 2 hours we thought that it would be quick since we ordered as soon as we sat down my coworkers did seem to enjoy their beef burgers for those who eat beef however i will not be returning it is too expensive and extremely slow service', 'category': 'food'},\n",
    "    \n",
    "         {'paragraph': 'the four reigning major champions simona halep caroline wozniacki angelique kerber and defending us open champion sloane stephens could make a case for being the quartet most likely to succeed especially as all but stephens has also enjoyed the no1 ranking within the last 14 months as they prepare for their gruelling new york campaigns they currently hold the top four places in the ranks', 'category': 'sports'},\n",
    "         {'paragraph': 'the briton was seeded nn7 here last year before a slump in form and confidence took her down to no46 after five first-round losses but there have been signs of a turnaround including a victory over a sub-par serena williams in san jose plus wins against jelena ostapenko and victoria azarenka in montreal. konta pulled out of new haven this week with illness but will hope for good things where she first scored wins in a major before her big breakthroughs to the semis in australia and wimbledon', 'category': 'sports'},\n",
    "         {'paragraph': 'stephens surged her way back from injury in stunning style to win her first major here last year—and ranked just no83 she has since proved what a big time player she is winning the miami title via four fellow major champions then reaching the final at the french open back on north american hard courts she ran to the final in montreal only just edged out by halep she has also avoided many of the big names in her quarter—except for wild card azarenka as a possible in the third round', 'category': 'sports'},\n",
    "         {'paragraph': 'when it came to england chances in the world cup it would be fair to say that most fans had never been more pessimistic than they were this year after enduring years of truly dismal performances at major tournaments – culminating in the 2014 event where they failed to win any of their three group games and finished in bottom spot those results led to the resignation of manager roy hodgson', 'category': 'sports'},\n",
    "         {'paragraph': 'the team that eliminated russia – croatia – also improved enormously during the tournament before it began their odds were 33/1 but they played with real flair and star players like luka modric ivan rakitic and ivan perisic showed their quality on the world stage having displayed their potential by winning all three of their group stage games croatia went on to face difficult tests like the semi-final against england', 'category': 'sports'},\n",
    "         {'paragraph': 'the perseyside outfit finished in fourth place in the premier league table and without a trophy last term after having reached the champions league final before losing to real madrid', 'category': 'sports'},\n",
    "         {'paragraph': 'liverpool fc will return to premier league action on saturday lunchtime when they travel to leicester city in the top flight as they look to make it four wins in a row in the league', 'category': 'sports'},\n",
    "         {'paragraph': 'alisson signed for liverpool fc from as roma this summer and the brazilian goalkeeper has helped the reds to keep three clean sheets in their first three premier league games', 'category': 'sports'},\n",
    "         {'paragraph': 'but the rankings during that run-in to new york hid some very different undercurrents for murray had struggled with a hip injury since the clay swing and had not played a match since losing his quarter-final at wimbledon and he would pull out of the us open just two days before the tournament began—too late however to promote nederer to the no2 seeding', 'category': 'sports'},\n",
    "         {'paragraph': 'then came the oh-so-familiar djokovic-nadal no-quarter-given battle for dominance in the thiadal more than once pulled off a reverse smash and had his chance to seal the tie-break but it was djokovic serving at 10-9 who dragged one decisive error from nadal for a two-sets lead', 'category': 'sports'}\n",
    "]\n",
    "\n",
    "df = pd.DataFrame(paragraph_dict_list)\n",
    "df = df[[\"paragraph\", \"category\"]]\n",
    "df"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Preprocessing"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {
    "collapsed": true
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0     None\n",
       "1     None\n",
       "2     None\n",
       "3     None\n",
       "4     None\n",
       "5     None\n",
       "6     None\n",
       "7     None\n",
       "8     None\n",
       "9     None\n",
       "10    None\n",
       "11    None\n",
       "12    None\n",
       "13    None\n",
       "14    None\n",
       "15    None\n",
       "16    None\n",
       "17    None\n",
       "18    None\n",
       "19    None\n",
       "Name: paragraph, dtype: object"
      ]
     },
     "execution_count": 12,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "results = set() # 순서없이, 중복된 데이터없이\n",
    "df[\"paragraph\"].str.lower().str.split().apply(results.update)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {
    "collapsed": true
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'(and',\n",
       " '1',\n",
       " '1.5',\n",
       " '1.5x',\n",
       " '10-9',\n",
       " '14',\n",
       " '2',\n",
       " '2014',\n",
       " '33/1',\n",
       " 'a',\n",
       " 'about',\n",
       " 'action',\n",
       " 'after',\n",
       " 'against',\n",
       " 'ago',\n",
       " 'ai',\n",
       " 'alisson',\n",
       " 'all',\n",
       " 'almost',\n",
       " 'also',\n",
       " 'although',\n",
       " 'alto',\n",
       " 'amazing',\n",
       " 'american',\n",
       " 'and',\n",
       " 'angelique',\n",
       " 'any',\n",
       " 'apps',\n",
       " 'are',\n",
       " 'area',\n",
       " 'around',\n",
       " 'as',\n",
       " 'at',\n",
       " 'australia',\n",
       " 'avoided',\n",
       " 'azarenka',\n",
       " 'back',\n",
       " 'battle',\n",
       " 'be',\n",
       " 'beef',\n",
       " 'been',\n",
       " 'before',\n",
       " 'began',\n",
       " 'began—too',\n",
       " 'being',\n",
       " 'better',\n",
       " 'between',\n",
       " 'big',\n",
       " 'biscuits',\n",
       " 'bit',\n",
       " 'bookmark',\n",
       " 'bookmarked',\n",
       " 'both',\n",
       " 'bottom',\n",
       " 'brazilian',\n",
       " 'bread',\n",
       " 'breakthroughs',\n",
       " 'briton',\n",
       " 'brunch',\n",
       " 'burger',\n",
       " 'burgers',\n",
       " 'business',\n",
       " 'busy',\n",
       " 'but',\n",
       " 'by',\n",
       " 'came',\n",
       " 'campaigns',\n",
       " 'can',\n",
       " 'card',\n",
       " 'caroline',\n",
       " 'case',\n",
       " 'certainly',\n",
       " 'champion',\n",
       " 'champions',\n",
       " 'chance',\n",
       " 'chances',\n",
       " 'chicken',\n",
       " 'city',\n",
       " 'clay',\n",
       " 'clean',\n",
       " 'commented',\n",
       " 'competition',\n",
       " 'complain',\n",
       " 'confidence',\n",
       " 'contrast',\n",
       " 'could',\n",
       " 'course',\n",
       " 'courteous',\n",
       " 'courts',\n",
       " 'coworkers',\n",
       " 'crisp',\n",
       " 'croatia',\n",
       " 'crumbs',\n",
       " 'culminating',\n",
       " 'cup',\n",
       " 'currently',\n",
       " 'dare',\n",
       " 'days',\n",
       " 'decisive',\n",
       " 'defending',\n",
       " 'delivered',\n",
       " 'despite',\n",
       " 'did',\n",
       " 'didnt',\n",
       " 'different',\n",
       " 'difficult',\n",
       " 'dinner',\n",
       " 'dish',\n",
       " 'dishplace',\n",
       " 'dismal',\n",
       " 'displayed',\n",
       " 'djokovic',\n",
       " 'djokovic-nadal',\n",
       " 'does',\n",
       " 'doesnt',\n",
       " 'dog-friendly',\n",
       " 'dominance',\n",
       " 'dont',\n",
       " 'double',\n",
       " 'down',\n",
       " 'downtown',\n",
       " 'dragged',\n",
       " 'drinks',\n",
       " 'during',\n",
       " 'eat',\n",
       " 'edged',\n",
       " 'egg',\n",
       " 'eggs',\n",
       " 'eliminated',\n",
       " 'enduring',\n",
       " 'england',\n",
       " 'enjoy',\n",
       " 'enjoyed',\n",
       " 'enormously',\n",
       " 'entree',\n",
       " 'error',\n",
       " 'especially',\n",
       " 'even',\n",
       " 'event',\n",
       " 'excellent',\n",
       " 'expected',\n",
       " 'expensive',\n",
       " 'extremely',\n",
       " 'face',\n",
       " 'failed',\n",
       " 'fair',\n",
       " 'fairly',\n",
       " 'fans',\n",
       " 'fc',\n",
       " 'fellow',\n",
       " 'final',\n",
       " 'finally',\n",
       " 'find',\n",
       " 'finished',\n",
       " 'first',\n",
       " 'first-round',\n",
       " 'five',\n",
       " 'flair',\n",
       " 'flavors',\n",
       " 'flight',\n",
       " 'food',\n",
       " 'for',\n",
       " 'form',\n",
       " 'formaggi',\n",
       " 'four',\n",
       " 'fourth',\n",
       " 'french',\n",
       " 'fries',\n",
       " 'from',\n",
       " 'full',\n",
       " 'games',\n",
       " 'garlic',\n",
       " 'gave',\n",
       " 'generous',\n",
       " 'goalkeeper',\n",
       " 'going',\n",
       " 'good',\n",
       " 'got',\n",
       " 'gravy',\n",
       " 'great',\n",
       " 'green',\n",
       " 'group',\n",
       " 'gruelling',\n",
       " 'had',\n",
       " 'halep',\n",
       " 'half',\n",
       " 'happy',\n",
       " 'hard',\n",
       " 'has',\n",
       " 'have',\n",
       " 'haven',\n",
       " 'having',\n",
       " 'he',\n",
       " 'help',\n",
       " 'helped',\n",
       " 'her',\n",
       " 'here',\n",
       " 'hid',\n",
       " 'highly',\n",
       " 'hip',\n",
       " 'his',\n",
       " 'hodgson',\n",
       " 'hold',\n",
       " 'hope',\n",
       " 'hour',\n",
       " 'hours',\n",
       " 'however',\n",
       " 'hubby',\n",
       " 'huge',\n",
       " 'hungry',\n",
       " 'i',\n",
       " 'ice',\n",
       " 'iced',\n",
       " 'if',\n",
       " 'illness',\n",
       " 'important',\n",
       " 'improved',\n",
       " 'in',\n",
       " 'including',\n",
       " 'injury',\n",
       " 'is',\n",
       " 'it',\n",
       " 'italico',\n",
       " 'ivan',\n",
       " 'jelena',\n",
       " 'jonesing',\n",
       " 'jose',\n",
       " 'just',\n",
       " 'keep',\n",
       " 'kerber',\n",
       " 'kitchen',\n",
       " 'knock',\n",
       " 'know',\n",
       " 'konta',\n",
       " 'last',\n",
       " 'late',\n",
       " 'laughed',\n",
       " 'lead',\n",
       " 'league',\n",
       " 'led',\n",
       " 'left',\n",
       " 'leicester',\n",
       " 'like',\n",
       " 'likely',\n",
       " 'liverpool',\n",
       " 'located',\n",
       " 'long',\n",
       " 'look',\n",
       " 'looked',\n",
       " 'losing',\n",
       " 'losses',\n",
       " 'luka',\n",
       " 'lunchtime',\n",
       " 'madrid',\n",
       " 'major',\n",
       " 'make',\n",
       " 'making',\n",
       " 'manager',\n",
       " 'many',\n",
       " 'match',\n",
       " 'maybe',\n",
       " 'me',\n",
       " 'meal',\n",
       " 'miami',\n",
       " 'minutes',\n",
       " 'missed',\n",
       " 'modric',\n",
       " 'moment',\n",
       " 'months',\n",
       " 'montreal',\n",
       " 'montreal.',\n",
       " 'more',\n",
       " 'most',\n",
       " 'much',\n",
       " 'murray',\n",
       " 'my',\n",
       " 'nachos',\n",
       " 'nadal',\n",
       " 'names',\n",
       " 'nederer',\n",
       " 'never',\n",
       " 'new',\n",
       " 'nn7',\n",
       " 'no-quarter-given',\n",
       " 'no1',\n",
       " 'no2',\n",
       " 'no46',\n",
       " 'no83',\n",
       " 'north',\n",
       " 'not',\n",
       " 'odds',\n",
       " 'of',\n",
       " 'off',\n",
       " 'oh-so-familiar',\n",
       " 'omelettes',\n",
       " 'on',\n",
       " 'once',\n",
       " 'one',\n",
       " 'only',\n",
       " 'open',\n",
       " 'opened',\n",
       " 'ordered',\n",
       " 'orders',\n",
       " 'ostapenko',\n",
       " 'other',\n",
       " 'our',\n",
       " 'out',\n",
       " 'outfit',\n",
       " 'outside',\n",
       " 'over',\n",
       " 'palo',\n",
       " 'pancakes',\n",
       " 'parking',\n",
       " 'pasta',\n",
       " 'patience',\n",
       " 'peak',\n",
       " 'people',\n",
       " 'performances',\n",
       " 'perisic',\n",
       " 'perseyside',\n",
       " 'person',\n",
       " 'pessimistic',\n",
       " 'place',\n",
       " 'places',\n",
       " 'plates',\n",
       " 'played',\n",
       " 'player',\n",
       " 'players',\n",
       " 'pleasantly',\n",
       " 'plus',\n",
       " 'portion',\n",
       " 'portions',\n",
       " 'possible',\n",
       " 'potential',\n",
       " 'premier',\n",
       " 'prepare',\n",
       " 'presentation',\n",
       " 'probably',\n",
       " 'promote',\n",
       " 'proved',\n",
       " 'pull',\n",
       " 'pulled',\n",
       " 'quality',\n",
       " 'quarter-final',\n",
       " 'quarter—except',\n",
       " 'quartet',\n",
       " 'quick',\n",
       " 'quickly',\n",
       " 'rakitic',\n",
       " 'ran',\n",
       " 'ranked',\n",
       " 'ranking',\n",
       " 'rankings',\n",
       " 'ranks',\n",
       " 'reached',\n",
       " 'reaching',\n",
       " 'real',\n",
       " 'really',\n",
       " 'recommendations',\n",
       " 'recommended',\n",
       " 'reds',\n",
       " 'reigning',\n",
       " 'reservations',\n",
       " 'resignation',\n",
       " 'restaurant',\n",
       " 'restaurants',\n",
       " 'results',\n",
       " 'return',\n",
       " 'returning',\n",
       " 'reverse',\n",
       " 'rich',\n",
       " 'roma',\n",
       " 'round',\n",
       " 'row',\n",
       " 'roy',\n",
       " 'run-in',\n",
       " 'russia',\n",
       " 'said',\n",
       " 'salty',\n",
       " 'san',\n",
       " 'sat',\n",
       " 'saturday',\n",
       " 'sauce',\n",
       " 'say',\n",
       " 'scored',\n",
       " 'seal',\n",
       " 'seated',\n",
       " 'seating',\n",
       " 'seeded',\n",
       " 'seeding',\n",
       " 'seem',\n",
       " 'semi-final',\n",
       " 'semis',\n",
       " 'separate',\n",
       " 'serena',\n",
       " 'served',\n",
       " 'server',\n",
       " 'serves',\n",
       " 'service',\n",
       " 'serving',\n",
       " 'she',\n",
       " 'sheets',\n",
       " 'showed',\n",
       " 'side',\n",
       " 'signed',\n",
       " 'signs',\n",
       " 'simona',\n",
       " 'since',\n",
       " 'sisters',\n",
       " 'sliders',\n",
       " 'slides',\n",
       " 'sloane',\n",
       " 'slow',\n",
       " 'slower',\n",
       " 'slump',\n",
       " 'small',\n",
       " 'smash',\n",
       " 'so',\n",
       " 'soft',\n",
       " 'some',\n",
       " 'soon',\n",
       " 'special)',\n",
       " 'spot',\n",
       " 'stage',\n",
       " 'star',\n",
       " 'started',\n",
       " 'starter',\n",
       " 'stephens',\n",
       " 'straight',\n",
       " 'struggled',\n",
       " 'stunning',\n",
       " 'style',\n",
       " 'sub-par',\n",
       " 'succeed',\n",
       " 'summer',\n",
       " 'sunnyvale',\n",
       " 'surged',\n",
       " 'surprised',\n",
       " 'sweet',\n",
       " 'swing',\n",
       " 'table',\n",
       " 'taken',\n",
       " 'taste',\n",
       " 'tasted',\n",
       " 'tea',\n",
       " 'team',\n",
       " 'term',\n",
       " 'tests',\n",
       " 'texture',\n",
       " 'than',\n",
       " 'that',\n",
       " 'the',\n",
       " 'their',\n",
       " 'themselves',\n",
       " 'then',\n",
       " 'there',\n",
       " 'they',\n",
       " 'thiadal',\n",
       " 'things',\n",
       " 'third',\n",
       " 'this',\n",
       " 'those',\n",
       " 'thought',\n",
       " 'three',\n",
       " 'tie-break',\n",
       " 'time',\n",
       " 'title',\n",
       " 'to',\n",
       " 'toast',\n",
       " 'too',\n",
       " 'took',\n",
       " 'top',\n",
       " 'tough',\n",
       " 'tournament',\n",
       " 'tournaments',\n",
       " 'travel',\n",
       " 'trophy',\n",
       " 'truly',\n",
       " 'turnaround',\n",
       " 'two',\n",
       " 'two-sets',\n",
       " 'undercurrents',\n",
       " 'unless',\n",
       " 'up',\n",
       " 'us',\n",
       " 'various',\n",
       " 'very',\n",
       " 'via',\n",
       " 'victoria',\n",
       " 'victory',\n",
       " 'visit',\n",
       " 'waffle',\n",
       " 'wait',\n",
       " 'waiter',\n",
       " 'want',\n",
       " 'wanted',\n",
       " 'was',\n",
       " 'wasnt',\n",
       " 'way',\n",
       " 'we',\n",
       " 'week',\n",
       " 'weekday',\n",
       " 'went',\n",
       " 'were',\n",
       " 'what',\n",
       " 'when',\n",
       " 'where',\n",
       " 'which',\n",
       " 'who',\n",
       " 'why',\n",
       " 'wild',\n",
       " 'will',\n",
       " 'williams',\n",
       " 'wimbledon',\n",
       " 'win',\n",
       " 'winning',\n",
       " 'wins',\n",
       " 'with',\n",
       " 'within',\n",
       " 'without',\n",
       " 'wonderful',\n",
       " 'world',\n",
       " 'would',\n",
       " 'wozniacki',\n",
       " 'year',\n",
       " 'years',\n",
       " 'year—and',\n",
       " 'york',\n",
       " 'you',\n",
       " '–'}"
      ]
     },
     "execution_count": 13,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "results"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "courts\n",
      "different\n"
     ]
    }
   ],
   "source": [
    "# 일련번호 부여\n",
    "idx2word = dict(enumerate(results))\n",
    "print(idx2word[396])\n",
    "print(idx2word[100])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "396"
      ]
     },
     "execution_count": 22,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# 문자 -> 숫자 변환\n",
    "word2idx = {v:k for k, v in idx2word.items()}\n",
    "word2idx[\"down\"]\n",
    "word2idx[\"courts\"]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 48,
   "metadata": {},
   "outputs": [],
   "source": [
    "# paragraph 전처리\n",
    "def encoded_paragraph(para):\n",
    "    words = para.split()\n",
    "    encoded = []\n",
    "    for w in words:\n",
    "        encoded.append([word2idx[w]])\n",
    "        \n",
    "    return encoded\n",
    "\n",
    "# category 숫자로 변경\n",
    "def encoded_category(cate):\n",
    "    if cate == \"food\":\n",
    "        return [1, 0]\n",
    "    else:\n",
    "        return [0, 1]\n",
    "    \n",
    "# 함수 추가\n",
    "def word_cnt(para):\n",
    "    return len(para.split())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 49,
   "metadata": {
    "collapsed": true
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>paragraph</th>\n",
       "      <th>category</th>\n",
       "      <th>enc_paragraph</th>\n",
       "      <th>enc_category</th>\n",
       "      <th>seq_lenght</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>dishplace is located in sunnyvale downtown the...</td>\n",
       "      <td>food</td>\n",
       "      <td>[[349], [37], [463], [283], [212], [452], [172...</td>\n",
       "      <td>[1, 0]</td>\n",
       "      <td>53</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>service can be slower during busy hours but ou...</td>\n",
       "      <td>food</td>\n",
       "      <td>[[372], [104], [118], [39], [223], [483], [231...</td>\n",
       "      <td>[1, 0]</td>\n",
       "      <td>19</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>portions are huge both french toast and their ...</td>\n",
       "      <td>food</td>\n",
       "      <td>[[225], [355], [208], [205], [76], [507], [354...</td>\n",
       "      <td>[1, 0]</td>\n",
       "      <td>42</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>we started with apps going the chicken and waf...</td>\n",
       "      <td>food</td>\n",
       "      <td>[[131], [360], [193], [41], [408], [292], [324...</td>\n",
       "      <td>[1, 0]</td>\n",
       "      <td>43</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>the biscuits and gravy was too salty two peopl...</td>\n",
       "      <td>food</td>\n",
       "      <td>[[292], [107], [354], [106], [246], [410], [35...</td>\n",
       "      <td>[1, 0]</td>\n",
       "      <td>82</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                                           paragraph category  \\\n",
       "0  dishplace is located in sunnyvale downtown the...     food   \n",
       "1  service can be slower during busy hours but ou...     food   \n",
       "2  portions are huge both french toast and their ...     food   \n",
       "3  we started with apps going the chicken and waf...     food   \n",
       "4  the biscuits and gravy was too salty two peopl...     food   \n",
       "\n",
       "                                       enc_paragraph enc_category  seq_lenght  \n",
       "0  [[349], [37], [463], [283], [212], [452], [172...       [1, 0]          53  \n",
       "1  [[372], [104], [118], [39], [223], [483], [231...       [1, 0]          19  \n",
       "2  [[225], [355], [208], [205], [76], [507], [354...       [1, 0]          42  \n",
       "3  [[131], [360], [193], [41], [408], [292], [324...       [1, 0]          43  \n",
       "4  [[292], [107], [354], [106], [246], [410], [35...       [1, 0]          82  "
      ]
     },
     "execution_count": 49,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df[\"enc_paragraph\"] = df.paragraph.apply(encoded_paragraph)\n",
    "df.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 50,
   "metadata": {
    "collapsed": true
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>paragraph</th>\n",
       "      <th>category</th>\n",
       "      <th>enc_paragraph</th>\n",
       "      <th>enc_category</th>\n",
       "      <th>seq_lenght</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>dishplace is located in sunnyvale downtown the...</td>\n",
       "      <td>food</td>\n",
       "      <td>[[349], [37], [463], [283], [212], [452], [172...</td>\n",
       "      <td>[1, 0]</td>\n",
       "      <td>53</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>service can be slower during busy hours but ou...</td>\n",
       "      <td>food</td>\n",
       "      <td>[[372], [104], [118], [39], [223], [483], [231...</td>\n",
       "      <td>[1, 0]</td>\n",
       "      <td>19</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>portions are huge both french toast and their ...</td>\n",
       "      <td>food</td>\n",
       "      <td>[[225], [355], [208], [205], [76], [507], [354...</td>\n",
       "      <td>[1, 0]</td>\n",
       "      <td>42</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>we started with apps going the chicken and waf...</td>\n",
       "      <td>food</td>\n",
       "      <td>[[131], [360], [193], [41], [408], [292], [324...</td>\n",
       "      <td>[1, 0]</td>\n",
       "      <td>43</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>the biscuits and gravy was too salty two peopl...</td>\n",
       "      <td>food</td>\n",
       "      <td>[[292], [107], [354], [106], [246], [410], [35...</td>\n",
       "      <td>[1, 0]</td>\n",
       "      <td>82</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                                           paragraph category  \\\n",
       "0  dishplace is located in sunnyvale downtown the...     food   \n",
       "1  service can be slower during busy hours but ou...     food   \n",
       "2  portions are huge both french toast and their ...     food   \n",
       "3  we started with apps going the chicken and waf...     food   \n",
       "4  the biscuits and gravy was too salty two peopl...     food   \n",
       "\n",
       "                                       enc_paragraph enc_category  seq_lenght  \n",
       "0  [[349], [37], [463], [283], [212], [452], [172...       [1, 0]          53  \n",
       "1  [[372], [104], [118], [39], [223], [483], [231...       [1, 0]          19  \n",
       "2  [[225], [355], [208], [205], [76], [507], [354...       [1, 0]          42  \n",
       "3  [[131], [360], [193], [41], [408], [292], [324...       [1, 0]          43  \n",
       "4  [[292], [107], [354], [106], [246], [410], [35...       [1, 0]          82  "
      ]
     },
     "execution_count": 50,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df[\"enc_category\"] = df.category.apply(encoded_category)\n",
    "df.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 52,
   "metadata": {
    "collapsed": true
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>paragraph</th>\n",
       "      <th>category</th>\n",
       "      <th>enc_paragraph</th>\n",
       "      <th>enc_category</th>\n",
       "      <th>seq_lenght</th>\n",
       "      <th>seq_length</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>dishplace is located in sunnyvale downtown the...</td>\n",
       "      <td>food</td>\n",
       "      <td>[[349], [37], [463], [283], [212], [452], [172...</td>\n",
       "      <td>[1, 0]</td>\n",
       "      <td>53</td>\n",
       "      <td>53</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>service can be slower during busy hours but ou...</td>\n",
       "      <td>food</td>\n",
       "      <td>[[372], [104], [118], [39], [223], [483], [231...</td>\n",
       "      <td>[1, 0]</td>\n",
       "      <td>19</td>\n",
       "      <td>19</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>portions are huge both french toast and their ...</td>\n",
       "      <td>food</td>\n",
       "      <td>[[225], [355], [208], [205], [76], [507], [354...</td>\n",
       "      <td>[1, 0]</td>\n",
       "      <td>42</td>\n",
       "      <td>42</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>we started with apps going the chicken and waf...</td>\n",
       "      <td>food</td>\n",
       "      <td>[[131], [360], [193], [41], [408], [292], [324...</td>\n",
       "      <td>[1, 0]</td>\n",
       "      <td>43</td>\n",
       "      <td>43</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>the biscuits and gravy was too salty two peopl...</td>\n",
       "      <td>food</td>\n",
       "      <td>[[292], [107], [354], [106], [246], [410], [35...</td>\n",
       "      <td>[1, 0]</td>\n",
       "      <td>82</td>\n",
       "      <td>82</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                                           paragraph category  \\\n",
       "0  dishplace is located in sunnyvale downtown the...     food   \n",
       "1  service can be slower during busy hours but ou...     food   \n",
       "2  portions are huge both french toast and their ...     food   \n",
       "3  we started with apps going the chicken and waf...     food   \n",
       "4  the biscuits and gravy was too salty two peopl...     food   \n",
       "\n",
       "                                       enc_paragraph enc_category  seq_lenght  \\\n",
       "0  [[349], [37], [463], [283], [212], [452], [172...       [1, 0]          53   \n",
       "1  [[372], [104], [118], [39], [223], [483], [231...       [1, 0]          19   \n",
       "2  [[225], [355], [208], [205], [76], [507], [354...       [1, 0]          42   \n",
       "3  [[131], [360], [193], [41], [408], [292], [324...       [1, 0]          43   \n",
       "4  [[292], [107], [354], [106], [246], [410], [35...       [1, 0]          82   \n",
       "\n",
       "   seq_length  \n",
       "0          53  \n",
       "1          19  \n",
       "2          42  \n",
       "3          43  \n",
       "4          82  "
      ]
     },
     "execution_count": 52,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df[\"seq_length\"] = df.paragraph.apply(word_cnt)\n",
    "df.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 53,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "91\n"
     ]
    }
   ],
   "source": [
    "# 문장의 최대 길이 알아내기\n",
    "max_word_cnt = 0\n",
    "\n",
    "for row in df[\"paragraph\"]:\n",
    "    if len(row.split()) > max_word_cnt:\n",
    "        max_word_cnt = len(row.split())\n",
    "    \n",
    "print(max_word_cnt)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 54,
   "metadata": {},
   "outputs": [],
   "source": [
    "# 문장의 길이가 91개가 안될 경우 -1로 채워준다.\n",
    "def sequence_padding(enc_para):\n",
    "    seq_len = len(enc_para)\n",
    "    for i in range(seq_len, max_word_cnt):\n",
    "        enc_para.append([-1])\n",
    "        \n",
    "    return enc_para\n",
    "    "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 56,
   "metadata": {},
   "outputs": [],
   "source": [
    "df[\"enc_paragraph\"] = df.enc_paragraph.apply(sequence_padding)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 57,
   "metadata": {
    "collapsed": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[[349], [37], [463], [283], [212], [452], [172], [37], [391], [7], [292], [62], [82], [250], [104], [118], [34], [501], [527], [223], [26], [247], [231], [162], [475], [354], [368], [70], [501], [338], [456], [486], [257], [464], [310], [239], [313], [198], [128], [483], [440], [368], [470], [529], [457], [392], [113], [319], [524], [292], [156], [501], [499], [-1], [-1], [-1], [-1], [-1], [-1], [-1], [-1], [-1], [-1], [-1], [-1], [-1], [-1], [-1], [-1], [-1], [-1], [-1], [-1], [-1], [-1], [-1], [-1], [-1], [-1], [-1], [-1], [-1], [-1], [-1], [-1], [-1], [-1], [-1], [-1], [-1], [-1]]\n"
     ]
    }
   ],
   "source": [
    "print(df[\"enc_paragraph\"][0])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 58,
   "metadata": {},
   "outputs": [],
   "source": [
    "# 입력값들을 배열(tensor)로 변환\n",
    "enc_paragraph = np.array(df.enc_paragraph.tolist())\n",
    "enc_category = np.array(df.enc_category.tolist())\n",
    "seq_length = np.array(df.seq_length.tolist())"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### 모델 구축"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 59,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(20, 91, 1) (20, 2)\n"
     ]
    }
   ],
   "source": [
    "train_X = enc_paragraph\n",
    "train_y = enc_category\n",
    "\n",
    "print(train_X.shape, train_y.shape)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 72,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "epoch:0, cost:0.8654138, acc:0.5\n",
      "epoch:50, cost:0.40298748, acc:0.8\n",
      "epoch:100, cost:0.17056146, acc:0.95\n",
      "epoch:150, cost:0.008562192, acc:1.0\n",
      "epoch:200, cost:0.00094974303, acc:1.0\n",
      "epoch:250, cost:0.00044708018, acc:1.0\n"
     ]
    }
   ],
   "source": [
    "tf.reset_default_graph()\n",
    "tf.set_random_seed(777)\n",
    "\n",
    "# parameter\n",
    "lr = 0.001\n",
    "n_epochs = 300\n",
    "\n",
    "X = tf.placeholder(tf.float32, shape=[None, max_word_cnt,1])  #shape=[전체데이터, 91, 출력갯수]\n",
    "y = tf.placeholder(tf.int32, shape=[None, 2]) # food, sport 둘 중 하나==2\n",
    "\n",
    "#레이어 추가\n",
    "embeded = tf.layers.dense(X, 5)\n",
    "\n",
    "cell = tf.nn.rnn_cell.LSTMCell(num_units=64)\n",
    "output, state = tf.nn.dynamic_rnn(cell=cell, inputs=embeded, dtype=tf.float32, sequence_length=seq_length)\n",
    "#print(output)\n",
    "#print(state)\n",
    "\n",
    "# 코드 압축(dense()==레이어 생성)\n",
    "first_layer = tf.layers.dense(state.h, 32) # 최종결과값(은닉층) 적용\n",
    "logit = tf.layers.dense(first_layer, 2)\n",
    "cost = tf.reduce_mean(tf.nn.softmax_cross_entropy_with_logits_v2(logits=logit, labels=y))\n",
    "train = tf.train.AdamOptimizer(learning_rate=lr).minimize(cost)\n",
    "\n",
    "#print(X)\n",
    "#print(embeded)\n",
    "#print(state)\n",
    "#print(first_layer)\n",
    "#print(logit)\n",
    "\n",
    "with tf.Session() as sess:\n",
    "    sess.run(tf.global_variables_initializer())\n",
    "    \n",
    "    for epoch in range(n_epochs):\n",
    "        _, c = sess.run([train, cost], feed_dict={X:train_X, y:train_y})\n",
    "        \n",
    "        if epoch % 50 == 0:\n",
    "            pred = tf.nn.softmax(logit)\n",
    "            correct_pred = tf.equal(tf.argmax(pred, 1), tf.argmax(y, 1))\n",
    "            accuracy = tf.reduce_mean(tf.cast(correct_pred, \"float\"))\n",
    "            cur_acc = accuracy.eval({X:train_X, y:train_y})\n",
    "            print(\"epoch:\" + str(epoch) + \", cost:\" + str(c) + \", acc:\" + str(cur_acc))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 77,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Loaded training data...\n",
      "Iter= 1000, Average Loss= 4.488728, Average Accuracy= 4.40%\n",
      "['at', 'one', 'another'] - [and] vs [well]\n",
      "Iter= 2000, Average Loss= 3.019604, Average Accuracy= 17.60%\n",
      "['is', 'all', 'very'] - [well] vs [with]\n",
      "Iter= 3000, Average Loss= 2.461506, Average Accuracy= 35.70%\n",
      "['this', 'proposal', 'met'] - [with] vs [know]\n",
      "Iter= 4000, Average Loss= 2.308940, Average Accuracy= 42.10%\n",
      "['by', 'this', 'means'] - [we] vs [we]\n",
      "Iter= 5000, Average Loss= 1.670886, Average Accuracy= 55.40%\n",
      "['propose', 'that', 'a'] - [small] vs [it]\n",
      "Iter= 6000, Average Loss= 1.695166, Average Accuracy= 56.40%\n",
      "['some', 'signal', 'of'] - [her] vs [danger]\n",
      "Iter= 7000, Average Loss= 1.427479, Average Accuracy= 62.40%\n",
      "['we', 'could', 'receive'] - [some] vs [some]\n",
      "Iter= 8000, Average Loss= 1.302992, Average Accuracy= 66.10%\n",
      "['.', 'now', ','] - [if] vs [if]\n",
      "Iter= 9000, Average Loss= 1.174871, Average Accuracy= 65.30%\n",
      "['meet', 'the', 'case'] - [.] vs [.]\n",
      "Iter= 10000, Average Loss= 1.050619, Average Accuracy= 69.90%\n",
      "['up', 'and', 'said'] - [he] vs [that]\n",
      "Iter= 11000, Average Loss= 1.027576, Average Accuracy= 72.60%\n",
      "['.', 'some', 'said'] - [this] vs [this]\n",
      "Iter= 12000, Average Loss= 0.963150, Average Accuracy= 76.60%\n",
      "['measures', 'they', 'could'] - [take] vs [take]\n",
      "Iter= 13000, Average Loss= 0.842369, Average Accuracy= 78.80%\n",
      "['it', 'is', 'easy'] - [to] vs [to]\n",
      "Iter= 14000, Average Loss= 0.794757, Average Accuracy= 79.10%\n",
      "[',', 'but', 'who'] - [is] vs [is]\n",
      "Iter= 15000, Average Loss= 0.994438, Average Accuracy= 75.10%\n",
      "['is', 'all', 'very'] - [well] vs [well]\n",
      "Iter= 16000, Average Loss= 0.667145, Average Accuracy= 82.50%\n",
      "['applause', ',', 'until'] - [an] vs [an]\n",
      "Iter= 17000, Average Loss= 0.707525, Average Accuracy= 81.70%\n",
      "['in', 'the', 'neighbourhood'] - [.] vs [.]\n",
      "Iter= 18000, Average Loss= 0.537349, Average Accuracy= 84.50%\n",
      "['the', 'neck', 'of'] - [the] vs [the]\n",
      "Iter= 19000, Average Loss= 0.589720, Average Accuracy= 83.80%\n",
      "['venture', ',', 'therefore'] - [,] vs [we]\n",
      "Iter= 20000, Average Loss= 0.630230, Average Accuracy= 83.10%\n",
      "['receive', 'some', 'signal'] - [of] vs [of]\n",
      "Iter= 21000, Average Loss= 0.547711, Average Accuracy= 85.70%\n",
      "[',', 'said', 'he'] - [,] vs [,]\n",
      "Iter= 22000, Average Loss= 0.614537, Average Accuracy= 85.10%\n",
      "['got', 'up', 'and'] - [said] vs [said]\n",
      "Iter= 23000, Average Loss= 0.620845, Average Accuracy= 85.20%\n",
      "[',', 'and', 'some'] - [said] vs [said]\n",
      "Iter= 24000, Average Loss= 0.550607, Average Accuracy= 85.80%\n",
      "['they', 'could', 'take'] - [to] vs [to]\n",
      "Iter= 25000, Average Loss= 0.769085, Average Accuracy= 82.60%\n",
      "['long', 'ago', ','] - [the] vs [the]\n",
      "Iter= 26000, Average Loss= 0.735542, Average Accuracy= 84.50%\n",
      "['but', 'who', 'is'] - [to] vs [to]\n",
      "Iter= 27000, Average Loss= 0.694544, Average Accuracy= 84.70%\n",
      "[',', 'until', 'an'] - [old] vs [old]\n",
      "Iter= 28000, Average Loss= 0.709216, Average Accuracy= 83.70%\n",
      "['retire', 'while', 'she'] - [was] vs [was]\n",
      "Iter= 29000, Average Loss= 0.509464, Average Accuracy= 86.70%\n",
      "['round', 'the', 'neck'] - [of] vs [of]\n",
      "Iter= 30000, Average Loss= 0.550503, Average Accuracy= 86.10%\n",
      "[',', 'therefore', ','] - [to] vs [to]\n",
      "Iter= 31000, Average Loss= 0.655043, Average Accuracy= 84.90%\n",
      "['which', 'the', 'enemy'] - [approaches] vs [approaches]\n",
      "Iter= 32000, Average Loss= 0.643078, Average Accuracy= 84.80%\n",
      "['the', 'sly', 'and'] - [treacherous] vs [treacherous]\n",
      "Iter= 33000, Average Loss= 0.463521, Average Accuracy= 88.10%\n",
      "['which', 'he', 'thought'] - [would] vs [would]\n",
      "Iter= 34000, Average Loss= 0.495014, Average Accuracy= 87.80%\n",
      "['and', 'said', 'he'] - [had] vs [had]\n",
      "Iter= 35000, Average Loss= 0.570568, Average Accuracy= 86.70%\n",
      "['up', 'and', 'said'] - [he] vs [that]\n",
      "Iter= 36000, Average Loss= 0.396330, Average Accuracy= 90.70%\n",
      "['what', 'measures', 'they'] - [could] vs [could]\n",
      "Iter= 37000, Average Loss= 0.453449, Average Accuracy= 90.40%\n",
      "['mice', 'had', 'a'] - [general] vs [general]\n",
      "Iter= 38000, Average Loss= 0.509503, Average Accuracy= 88.30%\n",
      "['who', 'is', 'to'] - [bell] vs [bell]\n",
      "Iter= 39000, Average Loss= 0.466887, Average Accuracy= 89.00%\n",
      "[',', 'until', 'an'] - [old] vs [old]\n",
      "Iter= 40000, Average Loss= 0.367579, Average Accuracy= 91.60%\n",
      "['with', 'general', 'applause'] - [,] vs [to]\n",
      "Iter= 41000, Average Loss= 0.586185, Average Accuracy= 86.70%\n",
      "['when', 'she', 'was'] - [about] vs [in]\n",
      "Iter= 42000, Average Loss= 0.444171, Average Accuracy= 89.20%\n",
      "['the', 'neck', 'of'] - [the] vs [the]\n",
      "Iter= 43000, Average Loss= 0.611848, Average Accuracy= 84.40%\n",
      "['small', 'bell', 'be'] - [procured] vs [procured]\n",
      "Iter= 44000, Average Loss= 0.457766, Average Accuracy= 89.00%\n",
      "['therefore', ',', 'to'] - [propose] vs [propose]\n",
      "Iter= 45000, Average Loss= 0.423326, Average Accuracy= 91.20%\n",
      "['now', ',', 'if'] - [we] vs [we]\n",
      "Iter= 46000, Average Loss= 0.515648, Average Accuracy= 88.00%\n",
      "['consists', 'in', 'the'] - [sly] vs [the]\n",
      "Iter= 47000, Average Loss= 0.523340, Average Accuracy= 86.70%\n",
      "['that', 'our', 'chief'] - [danger] vs [danger]\n",
      "Iter= 48000, Average Loss= 0.368597, Average Accuracy= 91.30%\n",
      "['thought', 'would', 'meet'] - [the] vs [the]\n",
      "Iter= 49000, Average Loss= 0.436101, Average Accuracy= 90.10%\n",
      "['the', 'cat', '.'] - [some] vs [by]\n",
      "Iter= 50000, Average Loss= 0.553195, Average Accuracy= 88.10%\n",
      "['council', 'to', 'consider'] - [what] vs [what]\n",
      "Optimization Finished!\n",
      "Elapsed time:  51.30895608266194 min\n",
      "Run on command line.\n",
      "\ttensorboard --logdir=log_dir/tmp/tensorflow/rnn_words\n",
      "Point your web browser to: http://localhost:6006/\n",
      "3 words: the sly and\n",
      "the sly and treacherous manner in the sly and treacherous manner in the sly and treacherous manner in the sly and treacherous manner in the sly and treacherous manner in the sly and treacherous manner\n"
     ]
    },
    {
     "ename": "KeyboardInterrupt",
     "evalue": "Interrupted by user",
     "output_type": "error",
     "traceback": [
      "\u001b[1;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[1;31mKeyboardInterrupt\u001b[0m                         Traceback (most recent call last)",
      "\u001b[1;32m<ipython-input-77-ec20ba2b934a>\u001b[0m in \u001b[0;36m<module>\u001b[1;34m\u001b[0m\n\u001b[0;32m    157\u001b[0m     \u001b[1;32mwhile\u001b[0m \u001b[1;32mTrue\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    158\u001b[0m         \u001b[0mprompt\u001b[0m \u001b[1;33m=\u001b[0m \u001b[1;34m\"%s words: \"\u001b[0m \u001b[1;33m%\u001b[0m \u001b[0mn_input\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m--> 159\u001b[1;33m         \u001b[0msentence\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0minput\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mprompt\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m    160\u001b[0m         \u001b[0msentence\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0msentence\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mstrip\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    161\u001b[0m         \u001b[0mwords\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0msentence\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0msplit\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;34m' '\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32m~\\anaconda3\\envs\\tf1\\lib\\site-packages\\ipykernel\\kernelbase.py\u001b[0m in \u001b[0;36mraw_input\u001b[1;34m(self, prompt)\u001b[0m\n\u001b[0;32m    861\u001b[0m             \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0m_parent_ident\u001b[0m\u001b[1;33m,\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    862\u001b[0m             \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0m_parent_header\u001b[0m\u001b[1;33m,\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m--> 863\u001b[1;33m             \u001b[0mpassword\u001b[0m\u001b[1;33m=\u001b[0m\u001b[1;32mFalse\u001b[0m\u001b[1;33m,\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m    864\u001b[0m         )\n\u001b[0;32m    865\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32m~\\anaconda3\\envs\\tf1\\lib\\site-packages\\ipykernel\\kernelbase.py\u001b[0m in \u001b[0;36m_input_request\u001b[1;34m(self, prompt, ident, parent, password)\u001b[0m\n\u001b[0;32m    902\u001b[0m             \u001b[1;32mexcept\u001b[0m \u001b[0mKeyboardInterrupt\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    903\u001b[0m                 \u001b[1;31m# re-raise KeyboardInterrupt, to truncate traceback\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m--> 904\u001b[1;33m                 \u001b[1;32mraise\u001b[0m \u001b[0mKeyboardInterrupt\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;34m\"Interrupted by user\"\u001b[0m\u001b[1;33m)\u001b[0m \u001b[1;32mfrom\u001b[0m \u001b[1;32mNone\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m    905\u001b[0m             \u001b[1;32mexcept\u001b[0m \u001b[0mException\u001b[0m \u001b[1;32mas\u001b[0m \u001b[0me\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    906\u001b[0m                 \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mlog\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mwarning\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;34m\"Invalid Message:\"\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mexc_info\u001b[0m\u001b[1;33m=\u001b[0m\u001b[1;32mTrue\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;31mKeyboardInterrupt\u001b[0m: Interrupted by user"
     ]
    }
   ],
   "source": [
    "# https://insightcampus.co.kr/insightcommunity/?mod=document&uid=12936\n",
    "import numpy as np\n",
    "import tensorflow as tf\n",
    "from tensorflow.contrib import rnn\n",
    "import random\n",
    "import collections\n",
    "import time\n",
    "\n",
    "tf.reset_default_graph()\n",
    "\n",
    "start_time = time.time()\n",
    "def elapsed(sec):\n",
    "    if sec<60:\n",
    "        return str(sec) + \" sec\"\n",
    "    elif sec<(60*60):\n",
    "        return str(sec/60) + \" min\"\n",
    "    else:\n",
    "        return str(sec/(60*60)) + \" hr\"\n",
    "\n",
    "\n",
    "## tensorboard\n",
    "# Target log path\n",
    "logs_path = 'log_dir/tmp/tensorflow/rnn_words'\n",
    "writer = tf.summary.FileWriter(logs_path)\n",
    "\n",
    "# Text file containing words for training\n",
    "training_file = 'data/belling_the_cat.txt'\n",
    "\n",
    "def read_data(fname):\n",
    "    with open(fname) as f:\n",
    "        content = f.readlines()\n",
    "    content = [x.strip() for x in content]\n",
    "    content = [word for i in range(len(content)) for word in content[i].split()]\n",
    "    content = np.array(content)\n",
    "    \n",
    "    return content\n",
    "\n",
    "training_data = read_data(training_file)\n",
    "print(\"Loaded training data...\")\n",
    "\n",
    "# 단어+숫자 => 딕트로\n",
    "def build_dataset(words):\n",
    "    count = collections.Counter(words).most_common()\n",
    "    dictionary = dict()\n",
    "    for word, _ in count:\n",
    "        dictionary[word] = len(dictionary)\n",
    "\n",
    "    reverse_dictionary = dict(zip(dictionary.values(), dictionary.keys()))\n",
    "    return dictionary, reverse_dictionary\n",
    "\n",
    "dictionary, reverse_dictionary = build_dataset(training_data)\n",
    "vocab_size = len(dictionary) ## 112개\n",
    "\n",
    "# Parameters\n",
    "learning_rate = 0.001\n",
    "training_iters = 50000\n",
    "display_step = 1000\n",
    "n_input = 3\n",
    "\n",
    "# number of units in RNN cell\n",
    "n_hidden = 512\n",
    "\n",
    "# tf Graph input\n",
    "x = tf.placeholder(\"float\", [None, n_input, 1])\n",
    "y = tf.placeholder(\"float\", [None, vocab_size])\n",
    "\n",
    "# RNN output node weights and biases\n",
    "weights = {\n",
    "    'out': tf.Variable(tf.random_normal([n_hidden, vocab_size]))\n",
    "}\n",
    "biases = {\n",
    "    'out': tf.Variable(tf.random_normal([vocab_size]))\n",
    "}\n",
    "\n",
    "def RNN(x, weights, biases):\n",
    "\n",
    "    # reshape to [1, n_input]\n",
    "    x = tf.reshape(x, [-1, n_input])\n",
    "\n",
    "    # Generate a n_input-element sequence of inputs\n",
    "    # (eg. [had] [a] [general] -> [20] [6] [33])\n",
    "    x = tf.split(x,n_input,1)\n",
    "\n",
    "    # 2-layer LSTM, each layer has n_hidden units.\n",
    "    # Average Accuracy= 95.20% at 50k iter\n",
    "    rnn_cell = rnn.MultiRNNCell([rnn.BasicLSTMCell(n_hidden),rnn.BasicLSTMCell(n_hidden)])\n",
    "\n",
    "    # 1-layer LSTM with n_hidden units but with lower accuracy.\n",
    "    # Average Accuracy= 90.60% 50k iter\n",
    "    # Uncomment line below to test but comment out the 2-layer rnn.MultiRNNCell above\n",
    "    # rnn_cell = rnn.BasicLSTMCell(n_hidden)\n",
    "\n",
    "    # generate prediction\n",
    "    outputs, states = rnn.static_rnn(rnn_cell, x, dtype=tf.float32)\n",
    "\n",
    "    # there are n_input outputs but\n",
    "    # we only want the last output\n",
    "    return tf.matmul(outputs[-1], weights['out']) + biases['out']\n",
    "\n",
    "pred = RNN(x, weights, biases)\n",
    "\n",
    "# Loss and optimizer\n",
    "cost = tf.reduce_mean(tf.nn.softmax_cross_entropy_with_logits(logits=pred, labels=y))\n",
    "optimizer = tf.train.RMSPropOptimizer(learning_rate=learning_rate).minimize(cost)\n",
    "\n",
    "# Model evaluation\n",
    "correct_pred = tf.equal(tf.argmax(pred,1), tf.argmax(y,1))\n",
    "accuracy = tf.reduce_mean(tf.cast(correct_pred, tf.float32))\n",
    "\n",
    "# Initializing the variables\n",
    "init = tf.global_variables_initializer()\n",
    "\n",
    "# Launch the graph\n",
    "with tf.Session() as session:\n",
    "    session.run(init)\n",
    "    step = 0\n",
    "    offset = random.randint(0,n_input+1)\n",
    "    end_offset = n_input + 1\n",
    "    acc_total = 0\n",
    "    loss_total = 0\n",
    "\n",
    "    writer.add_graph(session.graph)\n",
    "\n",
    "    while step < training_iters:\n",
    "        # Generate a minibatch. Add some randomness on selection process.\n",
    "        if offset > (len(training_data)-end_offset):\n",
    "            offset = random.randint(0, n_input+1)\n",
    "\n",
    "        symbols_in_keys = [ [dictionary[ str(training_data[i])]] for i in range(offset, offset+n_input) ]\n",
    "        symbols_in_keys = np.reshape(np.array(symbols_in_keys), [-1, n_input, 1])\n",
    "\n",
    "        symbols_out_onehot = np.zeros([vocab_size], dtype=float)\n",
    "        symbols_out_onehot[dictionary[str(training_data[offset+n_input])]] = 1.0\n",
    "        symbols_out_onehot = np.reshape(symbols_out_onehot,[1,-1])\n",
    "\n",
    "        _, acc, loss, onehot_pred = session.run([optimizer, accuracy, cost, pred], \\\n",
    "                                                feed_dict={x: symbols_in_keys, y: symbols_out_onehot})\n",
    "        loss_total += loss\n",
    "        acc_total += acc\n",
    "        if (step+1) % display_step == 0:\n",
    "            print(\"Iter= \" + str(step+1) + \", Average Loss= \" + \\\n",
    "                  \"{:.6f}\".format(loss_total/display_step) + \", Average Accuracy= \" + \\\n",
    "                  \"{:.2f}%\".format(100*acc_total/display_step))\n",
    "            acc_total = 0\n",
    "            loss_total = 0\n",
    "            symbols_in = [training_data[i] for i in range(offset, offset + n_input)]\n",
    "            symbols_out = training_data[offset + n_input]\n",
    "            symbols_out_pred = reverse_dictionary[int(tf.argmax(onehot_pred, 1).eval())]\n",
    "            print(\"%s - [%s] vs [%s]\" % (symbols_in,symbols_out,symbols_out_pred))\n",
    "        step += 1\n",
    "        offset += (n_input+1)\n",
    "    print(\"Optimization Finished!\")\n",
    "    print(\"Elapsed time: \", elapsed(time.time() - start_time))\n",
    "    print(\"Run on command line.\")\n",
    "    print(\"\\ttensorboard --logdir=%s\" % (logs_path))\n",
    "    print(\"Point your web browser to: http://localhost:6006/\")\n",
    "    while True:\n",
    "        prompt = \"%s words: \" % n_input\n",
    "        sentence = input(prompt)\n",
    "        sentence = sentence.strip()\n",
    "        words = sentence.split(' ')\n",
    "        if len(words) != n_input:\n",
    "            continue\n",
    "        try:\n",
    "            symbols_in_keys = [dictionary[str(words[i])] for i in range(len(words))]\n",
    "            for i in range(32):\n",
    "                keys = np.reshape(np.array(symbols_in_keys), [-1, n_input, 1])\n",
    "                onehot_pred = session.run(pred, feed_dict={x: keys})\n",
    "                onehot_pred_index = int(tf.argmax(onehot_pred, 1).eval())\n",
    "                sentence = \"%s %s\" % (sentence,reverse_dictionary[onehot_pred_index])\n",
    "                symbols_in_keys = symbols_in_keys[1:]\n",
    "                symbols_in_keys.append(onehot_pred_index)\n",
    "            print(sentence)\n",
    "        except:\n",
    "            print(\"Word not in dictionary\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.9"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}