NN_Part1.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "import numpy as np"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "#Nonlinearity, here sigmoid func and derivative\n",
    "#Could be someother function as well\n",
    "\n",
    "def nonlin(x, deriv=False):\n",
    "    if(deriv==True):\n",
    "        return(x*(1-x))\n",
    "    else:\n",
    "        return( 1/(1 + np.exp(-x)) )"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.5"
      ]
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "#Tests\n",
    "nonlin(0)#x=0, nonlin(0.5, deriv=True)#Here x=0, sigmoid=0.5"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "#Input dataset\n",
    "\n",
    "X = np.array([\n",
    "                [0,0,1],\n",
    "                [0,1,1],\n",
    "                [1,0,1],\n",
    "                [1,1,1],\n",
    "])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(array([0, 1, 1]), 1)"
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "X[1] , X[1][1]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([[0, 0, 1],\n",
       "       [0, 1, 1],\n",
       "       [1, 0, 1],\n",
       "       [1, 1, 1]])"
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "X"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "#Output Dataset\n",
    "y = np.array([[0,0,1,1]]).T"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([[0],\n",
       "       [0],\n",
       "       [1],\n",
       "       [1]])"
      ]
     },
     "execution_count": 9,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "y"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([0])"
      ]
     },
     "execution_count": 10,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "y[0]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "np.random.seed(1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([[-0.62747958],\n",
       "       [-0.30887855],\n",
       "       [-0.20646505]])"
      ]
     },
     "execution_count": 18,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "#Populate 1st layer with random weights\n",
    "#We need to shift spread (0 to 1) to (-1 to 1)\n",
    "syn0 = 2*np.random.random((3,1)) - 1\n",
    "syn0"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[[-0.44856632]\n",
      " [-0.3739417 ]\n",
      " [ 0.69718835]\n",
      " [ 0.7581976 ]]\n",
      "****************\n",
      "[[-0.47034194]\n",
      " [-0.40704614]\n",
      " [ 0.61302834]\n",
      " [ 0.67205143]]\n",
      "****************\n",
      "[[-0.03195016]\n",
      " [-0.02573781]\n",
      " [ 0.02088504]\n",
      " [ 0.02595728]]\n",
      "****************\n",
      "[[-0.02216442]\n",
      " [-0.01792967]\n",
      " [ 0.01457753]\n",
      " [ 0.01803509]]\n",
      "****************\n",
      "[[-0.01794692]\n",
      " [-0.01454433]\n",
      " [ 0.01183495]\n",
      " [ 0.01461309]]\n",
      "****************\n",
      "[[-0.01546616]\n",
      " [-0.01254728]\n",
      " [ 0.01021487]\n",
      " [ 0.01259809]]\n",
      "****************\n",
      "[[-0.01378727]\n",
      " [-0.01119329]\n",
      " [ 0.00911555]\n",
      " [ 0.0112335 ]]\n",
      "****************\n",
      "[[-0.0125552 ]\n",
      " [-0.01019839]\n",
      " [ 0.00830731]\n",
      " [ 0.01023162]]\n",
      "****************\n",
      "[[-0.01160183]\n",
      " [-0.00942781]\n",
      " [ 0.00768104]\n",
      " [ 0.0094561 ]]\n",
      "****************\n",
      "[[-0.010836  ]\n",
      " [-0.00880835]\n",
      " [ 0.00717741]\n",
      " [ 0.00883296]]\n",
      "****************\n",
      "[[-0.01020343]\n",
      " [-0.00829637]\n",
      " [ 0.00676105]\n",
      " [ 0.00831814]]\n",
      "****************\n",
      "Output after Training:\n",
      "[[ 0.00967004]\n",
      " [ 0.00786445]\n",
      " [ 0.99359029]\n",
      " [ 0.99211604]]\n"
     ]
    }
   ],
   "source": [
    "for iter in xrange(10000):\n",
    "    \n",
    "    #Fwd prop\n",
    "    l0 = X #Input layer\n",
    "    l1 = nonlin(np.dot(l0,syn0)) #L1 gives predictions\n",
    "    #if (iter == 1) or (iter%1000 == 0):\n",
    "    #    print(l1)\n",
    "    #    print(\"****************\")\n",
    "    \n",
    "    #Find error between expected and predicted \n",
    "    l1_error = y - l1\n",
    "    if (iter == 1) or (iter%1000 == 0):\n",
    "        print(l1_error)\n",
    "        print(\"****************\")\n",
    "    \n",
    "    #What fraction of input to move towards \n",
    "    l1_delta = l1_error * nonlin(l1, deriv = True)\n",
    "    \n",
    "    #Move weights towards product of inputs and faction movement\n",
    "    syn0 = syn0 + np.dot(l0.T, l1_delta)\n",
    "    \n",
    "    \n",
    "print(\"Output after Training:\")\n",
    "print(l1) "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 43,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Error:0.496410031903\n",
      "[[-0.00953041]\n",
      " [ 0.01222792]\n",
      " [ 0.01055641]\n",
      " [-0.01422542]]\n"
     ]
    }
   ],
   "source": [
    "import numpy as np\n",
    "\n",
    "def nonlin(x, deriv=False):\n",
    "    if deriv == True:\n",
    "        return(x*(1-x))\n",
    "    else:\n",
    "        return(1/(1+np.exp(-x)))\n",
    "    \n",
    "    \n",
    "X = np.array([\n",
    "        \n",
    "        [0,0,1],\n",
    "        [0,1,1],\n",
    "        [1,0,1],\n",
    "        [1,1,1]\n",
    "    ])\n",
    "\n",
    "y = np.array([\n",
    "        \n",
    "        [0],\n",
    "        [1],\n",
    "        [1],\n",
    "        [0] \n",
    "    ])\n",
    "\n",
    "np.random.seed(1)\n",
    "\n",
    "\n",
    "#Init the 2 layer weights\n",
    "#     o\n",
    "# o   o\n",
    "# o   o    o\n",
    "# o   o\n",
    "# I   H    O\n",
    "syn0 = 2*np.random.random((3,4)) - 1\n",
    "syn1 = 2*np.random.random((4,1)) - 1\n",
    "\n",
    "\n",
    "for j in xrange(6000):\n",
    "    \n",
    "    #Fwd pass activations\n",
    "    l0 = X\n",
    "    l1 = nonlin(np.dot(l0,syn0))\n",
    "    l2 = nonlin(np.dot(l1,syn1))\n",
    "\n",
    "    #Error, start from last layer\n",
    "    \n",
    "    l2_error = y - l2\n",
    "    l2_delta = l2_error*nonlin(l2,deriv=True)\n",
    "    \n",
    "    \n",
    "    l1_error =  l2_delta.dot(syn1.T)\n",
    "    l1_delta =  l1_error*nonlin(l1,deriv=True)\n",
    "    \n",
    "    syn1 = syn1 + np.dot(l1.T,l2_delta)\n",
    "    syn0 = syn0 + np.dot(l0.T,l1_delta)\n",
    "    \n",
    "    if (j% 10000) == 0:\n",
    "        print \"Error:\" + str(np.mean(np.abs(l2_error)))\n",
    "    \n",
    "print(l2_error) \n",
    "    \n",
    "    \n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Error:0.496410031903\n",
      "Error:0.00858452565325\n",
      "Error:0.00578945986251\n",
      "Error:0.00462917677677\n",
      "Error:0.00395876528027\n",
      "Error:0.00351012256786\n"
     ]
    }
   ],
   "source": [
    "import numpy as np\n",
    "\n",
    "def nonlin(x, deriv=False):\n",
    "    if deriv == True:\n",
    "        return(x*(1-x))\n",
    "    else:\n",
    "        return(1/(1+np.exp(-x)))\n",
    "    \n",
    "    \n",
    "X = np.array([\n",
    "        \n",
    "        [0,0,1],\n",
    "        [0,1,1],\n",
    "        [1,0,1],\n",
    "        [1,1,1]\n",
    "    ])\n",
    "\n",
    "y = np.array([\n",
    "        \n",
    "        [0],\n",
    "        [1],\n",
    "        [1],\n",
    "        [0] \n",
    "    ])\n",
    "\n",
    "np.random.seed(1)\n",
    "\n",
    "\n",
    "#Init the 2 layer weights\n",
    "#     o\n",
    "# o   o\n",
    "# o   o    o\n",
    "# o   o\n",
    "# I   H    O\n",
    "syn0 = 2*np.random.random((3,4)) - 1\n",
    "syn1 = 2*np.random.random((4,1)) - 1\n",
    "\n",
    "for j in xrange(60000):\n",
    "\n",
    "\t# Feed forward through layers 0, 1, and 2\n",
    "    \n",
    "        #Fwd pass activations\n",
    "    l0 = X\n",
    "    l1 = nonlin(np.dot(l0,syn0))\n",
    "    l2 = nonlin(np.dot(l1,syn1))\n",
    "\n",
    "    #Error, start from last layer\n",
    "    \n",
    "    l2_error = y - l2\n",
    "    l2_delta = l2_error*nonlin(l2,deriv=True)\n",
    "    \n",
    "    \n",
    "    l1_error =  l2_delta.dot(syn1.T)\n",
    "    l1_delta =  l1_error*nonlin(l1,deriv=True)\n",
    "    \n",
    "    syn1 = syn1 + l1.T.dot(l2_delta)\n",
    "    syn0 = syn0 + l0.T.dot(l1_delta)\n",
    "    \n",
    "    \n",
    "    if (j% 10000) == 0:\n",
    "        print \"Error:\" + str(np.mean(np.abs(l2_error)))\n",
    "        \n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[[ 1.]\n",
      " [ 1.]\n",
      " [ 1.]\n",
      " [ 1.]\n",
      " [ 1.]\n",
      " [ 1.]]\n",
      "******\n",
      "[[ 1.]\n",
      " [ 0.]\n",
      " [ 1.]\n",
      " [ 0.]\n",
      " [ 0.]\n",
      " [ 1.]]\n",
      "******\n",
      "[[ 1.]\n",
      " [ 0.]\n",
      " [ 1.]\n",
      " [ 0.]\n",
      " [ 0.]\n",
      " [ 1.]]\n",
      "******\n",
      "[[ 1.]\n",
      " [ 0.]\n",
      " [ 1.]\n",
      " [ 0.]\n",
      " [ 0.]\n",
      " [ 1.]]\n",
      "******\n",
      "[[ 1.]\n",
      " [ 0.]\n",
      " [ 1.]\n",
      " [ 0.]\n",
      " [ 0.]\n",
      " [ 1.]]\n",
      "******\n",
      "[[ 1.]\n",
      " [ 0.]\n",
      " [ 1.]\n",
      " [ 0.]\n",
      " [ 0.]\n",
      " [ 1.]]\n",
      "******\n",
      "[[ 1.]\n",
      " [ 0.]\n",
      " [ 1.]\n",
      " [ 0.]\n",
      " [ 0.]\n",
      " [ 1.]]\n",
      "******\n"
     ]
    }
   ],
   "source": [
    "import numpy as np\n",
    "\n",
    "def nonlin(x, deriv=False):\n",
    "    if deriv == True:\n",
    "        return(x*(1-x))\n",
    "    return(1/(1+np.exp(-x)))\n",
    "\n",
    "\n",
    "X = np.array([\n",
    "        \n",
    "        [0,1,1,0],\n",
    "        [0,0,0,0],\n",
    "        [0,1,1,1],\n",
    "        [0,0,1,0],\n",
    "        [0,1,0,0],\n",
    "        [1,1,1,1]\n",
    "    ])\n",
    "\n",
    "y = np.array([\n",
    "        [1],\n",
    "        [0],\n",
    "        [1],\n",
    "        [0],\n",
    "        [0],\n",
    "        [1]       \n",
    "    ])\n",
    "    \n",
    "np.random.seed(2)\n",
    "\n",
    "\n",
    "#4 ->  10  -> 1\n",
    "#In -> Hid -> Out\n",
    "\n",
    "num_egs = X.shape[0]\n",
    "num_in = X.shape[1]\n",
    "num_hidden = 10\n",
    "num_out = y.shape[1]\n",
    "\n",
    "syn0 = 2*np.random.random((num_in,num_hidden)) - 1\n",
    "syn1 = 2*np.random.random((num_hidden,num_out)) - 1\n",
    "\n",
    "for iter in xrange(70000):\n",
    "    \n",
    "    \n",
    "    #Fwd pass\n",
    "    l0 = X                       #6x4\n",
    "    l1 = nonlin(np.dot(l0,syn0)) #6x10\n",
    "    l2 = nonlin(np.dot(l1,syn1)) #6x1\n",
    "    \n",
    "    #Backprop from last layer\n",
    "    \n",
    "    l2_error = y - l2                         #6x1\n",
    "    l2_delta = l2_error*nonlin(l2 ,deriv=True)#6x1\n",
    "    \n",
    "    l1_error = np.dot(l2_delta, syn1.T)                            #6x1\n",
    "    l1_delta = l1_error*nonlin(l1, deriv=True)#6x10\n",
    "    \n",
    "    syn1 = syn1 + np.dot(l1.T, l2_delta)   #10x1\n",
    "    syn0 = syn0 + np.dot(l0.T,l1_delta)   #4x10\n",
    "    \n",
    "    if iter%10000 == 0 :\n",
    "        print(np.round(l2))\n",
    "        print(\"******\")\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "6"
      ]
     },
     "execution_count": 11,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 2",
   "language": "python",
   "name": "python2"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 2
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython2",
   "version": "2.7.11"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 1
}