Weight Initialization, GNum class and NFData instances

.gitignore

@@ -1,4 +1,18 @@
+cabal.project.local
 .cabal-sandbox/
 cabal.sandbox.config
 dist/
-
+dist-newstyle/
+.stack-work/
+/.dir-locals.el
+/TAGS
+/stack.yaml.lock
+grenade.cabal
+/examples/TAGS
+/.emacs.desktop.lock
+/.emacs.desktop
+/cbits/TAGS
+/emacs.desktop
+/examples/grenade-examples.cabal
+/examples/main/mnist_test.csv
+/examples/main/mnist_train.csv

.travis.yml

@@ -1,31 +1,92 @@
-# NB: don't set `language: haskell` here
+# Adapted from https://github.com/commercialhaskell/stack
+language: nix
+sudo: false
 
-# The following enables several GHC versions to be tested; often it's enough to test only against the last release in a major GHC version. Feel free to omit lines listings versions you don't need/want testing for.
-env:
- - CABALVER=1.24 GHCVER=8.0.2
- - CABALVER=2.0 GHCVER=8.2.2
- - CABALVER=2.0 GHCVER=8.4.4
- - CABALVER=2.0 GHCVER=8.6.3
+cache:
+  directories:
+  - $HOME/.ghc
+  - $HOME/.cabal
+  - $HOME/.stack
+  - $TRAVIS_BUILD_DIR/.stack-work
+
+matrix:
+  fast_finish: true
+  include:
+    # Add build targets here
+  - env: BUILD=stack ARGS=""
+    compiler: ": #stack default"
+    addons: {apt: {packages: [ libblas3,liblapack3,liblapack-dev,libblas-dev,pkg-config]}}
+
+  - env: BUILD=stack ARGS="--resolver lts-15.8"
+    compiler: ": #stack 8.8.3"
+    addons: {apt: {packages: [ libblas3,liblapack3,libblas-dev,liblapack-dev,pkg-config]}}
+
+  - env: BUILD=stack ARGS="--resolver lts-15.3"
+    compiler: ": #stack 8.8.2"
+    addons: {apt: {packages: [ libblas3,liblapack3,libblas-dev,liblapack-dev,pkg-config]}}
+
+  - env: BUILD=stack ARGS="--resolver lts-14.27"
+    compiler: ": #stack 8.6.5"
+    addons: {apt: {packages: [ libblas3,liblapack3,libblas-dev,liblapack-dev,pkg-config]}}
+
+  - env: BUILD=stack ARGS="--resolver nightly"
+    compiler: ": #stack nightly"
+    addons: {apt: {packages: [ libblas3,liblapack3,libblas-dev,liblapack-dev,pkg-config]}}
+
+  allow_failures:
+  - env: BUILD=stack ARGS="--resolver nightly"
+  - env: BUILD=stack ARGS="--resolver lts-14.27"
 
-# Note: the distinction between `before_install` and `install` is not important.
 before_install:
- - travis_retry sudo add-apt-repository -y ppa:hvr/ghc
- - travis_retry sudo apt-get update
- - travis_retry sudo apt-get install cabal-install-$CABALVER ghc-$GHCVER happy-1.19.5 alex-3.1.7 libblas-dev liblapack-dev
- - export PATH=/opt/alex/3.1.7/bin:/opt/happy/1.19.5/bin:/opt/ghc/$GHCVER/bin:/opt/cabal/$CABALVER/bin:$HOME/.cabal/bin:$PATH
+ # Using compiler above sets CC to an invalid value, so unset it
+ - unset CC
+
+ # We want to always allow newer versions of packages when building on GHC HEAD
+ - CABALARGS=""
+ - if [ "x$GHCVER" = "xhead" ]; then CABALARGS=--allow-newer; fi
+
+ # Download and unpack the stack executable
+ - export PATH=/opt/ghc/$GHCVER/bin:/opt/cabal/$CABALVER/bin:$HOME/.local/bin:/opt/alex/$ALEXVER/bin:/opt/happy/$HAPPYVER/bin:$HOME/.cabal/bin:$PATH
+ - mkdir -p ~/.local/bin
+ - |
+   if [ `uname` = "Darwin" ]
+   then
+     travis_retry curl --insecure -L https://get.haskellstack.org/stable/osx-x86_64.tar.gz | tar xz --strip-components=1 --include '*/stack' -C ~/.local/bin
+   else
+     travis_retry curl -L https://get.haskellstack.org/stable/linux-x86_64.tar.gz | tar xz --wildcards --strip-components=1 -C ~/.local/bin '*/stack'
+   fi
+
+   # Use the more reliable S3 mirror of Hackage
+   mkdir -p $HOME/.cabal
+   echo 'remote-repo: hackage.haskell.org:http://hackage.fpcomplete.com/' > $HOME/.cabal/config
+   echo 'remote-repo-cache: $HOME/.cabal/packages' >> $HOME/.cabal/config
+
+ # Install blas and lapack
+ # - travis_retry sudo apt-get update
+ # - travis_retry sudo apt-get install libblas-dev liblapack-dev
 
 install:
- - cabal --version
  - echo "$(ghc --version) [$(ghc --print-project-git-commit-id 2> /dev/null || echo '?')]"
- - travis_retry cabal update
- - cabal sandbox init
- - cabal install --enable-tests --enable-benchmarks
+ - if [ -f configure.ac ]; then autoreconf -i; fi
+ - |
+   stack --no-terminal --install-ghc $ARGS test --bench --only-dependencies
+
 
-notifications:
-  email: false
+# script:
+#  - |
+#    PKG_CONFIG_PATH="${VIRTUAL_ENV}/lib/pkgconfig:${PKG_CONFIG_PATH}"
+#    echo $PKG_CONFIG_PATH
+#    export PKG_CONFIG_PATH
+#    stack --no-terminal $ARGS test --bench --no-run-benchmarks --haddock --no-haddock-deps
 
-# Here starts the actual work to be performed for the package under test; any command which exits with a non-zero exit code causes the build to fail.
 script:
- - cabal build
- - cabal test --show-details=streaming
+ - echo "$(ghc --version) [$(ghc --print-project-git-commit-id 2> /dev/null || echo '?')]"
+ - |
+   set -ex
+   # Run tests --ghc-options=-Werror
+   # Werror on grenade only!
+   stack build grenade --no-terminal $ARGS --ghc-options=-Werror
+   stack test --no-terminal $ARGS --ghc-options=-Wno-unused-imports
+   stack test --no-terminal $ARGS --ghc-options=-Wno-unused-imports
+   set +ex
 

README.md

@@ -1,10 +1,80 @@
 Grenade
 =======
 
-[![Build Status](https://api.travis-ci.org/HuwCampbell/grenade.svg?branch=master)](https://travis-ci.org/HuwCampbell/grenade)
-[![Hackage page (downloads and API reference)][hackage-png]][hackage]
-[![Hackage-Deps][hackage-deps-png]][hackage-deps]
-
+[![Build Status](https://api.travis-ci.org/schnecki/grenade.svg?branch=master)](https://travis-ci.org/schnecki/grenade)
+<!-- [![Hackage page (downloads and API reference)][hackage-png]][hackage] -->
+<!-- [![Hackage-Deps][hackage-deps-png]][hackage-deps] -->
+
+This is a fork of the original Grenade library found at https://github.com/HuwCampbell/grenade,
+but includes additional features:
+
+ 1. **Optimizer Support**. The code has been restructured to be able to easily implement more
+    optimizers than just SGD with momentum and regularization. Currently we support *Adam* for
+    feedforward neural networks also!
+
+ 2. **Weight Initialization**. Initializing the weights in different ways. Currently implemented:
+    Uniform, HeEtAl, Xavier. The default is Uniform! See chapter 6 of this [seminar
+    report](docs/seminar_report_ANN_analysis_2018.pdf "Seminar Report ANN Analysis") for a small
+    evaluation of the implemented weight initialization methods.
+
+ 3. **Data Type Representation**: You can easily switch from `Double` to `Float` vectors and
+    matrices. Just provide the corresponding flag (`use-float`) when compiling (to all packages
+    that):
+
+        stack clean && stack build --flag=grenade-examples:use-float --flag=grenade:use-float && stack bench
+
+    Ensure you clean before changing the flags, as otherwise you might in the best case get a
+    compile error and in the worst case a Segmentation Fault!
+
+    Clearly `Float`s are less precise but more efficient, both in terms of *time and memory*. In
+    case of small ANNs `Float` should be sufficient, as long as you keep the values of the weights
+    small (which you should always do). This feature uses an [adapted
+    version](http://github.com/schnecki/hmatrix-float "github repository") of
+    [hmatrix](https://hackage.haskell.org/package/hmatrix-0.20.0.0 "stackage") which was especially
+    adapted for this project.
+
+ 4. **Runtime Networks**. Dynamically specifying and build networks at runtime. This is not only a
+    required tool when storing the network architecture to the disk, like in a DB, and reloading it,
+    but it could also be a starting point for developing algorithms that adapt the network to find
+    the best architecture for the underlying problem. You can do that with this feature without
+    knowing its structure by deserializing the network specification and then feed the deserialized
+    network weights into the net.
+
+    However, currently this works only for feedforward networks composed of fully-connected, dropout,
+    deconvolution and convolution layers plus all activation functions. Example (also see
+    `feedforward-netinit` in example folder):
+    ```haskell
+       let spec :: SpecNet
+           spec = specFullyConnected 40 30 |=> specRelu1D 30 |=> specFullyConnected 30 20 |=> specNil1D 20
+       SpecConcreteNetwork1D1D (net0 :: Network layers shapes) <- networkFromSpecificationWith HeEtAl spec
+    ```
+
+    However, Beware! It is important to get the specification right, as otherwise the program will halt
+    abruptly. So at best do not use it manually, but write functions for creating specifications!
+
+    Or probably better, use the simple interface:
+
+    ```haskell
+       buildNetViaInterface :: IO SpecConcreteNetwork
+       buildNetViaInterface =
+         buildModel $
+         inputLayer1D 2 >>
+         fullyConnected 10 >> dropout 0.89 >> relu >>
+         fullyConnected 4 >> relu >>
+         networkLayer (
+           inputLayer1D 4 >> fullyConnected 10 >> relu >> fullyConnected 4 >> sinusoid) >>
+         fullyConnected 1 >> tanhLayer
+    ```
+ 5. **Gradient Clipping**. You can clip gradients using the function `clipByGlobalNorm`.
+
+ 6. **More Activation Functions**. This branch supports `Dropout` (which is unimplemented in the
+    original code), `LeakyRelu` and `Gelu` activation functions.
+
+
+The following is mostly (except the installation procedure) the original description of grenade:
+
+Description
+===========
 
 ```
 First shalt thou take out the Holy Pin, then shalt thou count to three, no more, no less.
@@ -144,22 +214,21 @@ elu, tanh, and fully connected.
 
 Build Instructions
 ------------------
-Grenade is most easily built with the [mafia](https://github.com/ambiata/mafia)
-script that is located in the repository. You will also need the `lapack` and
+This version of Grenade is most easily built with the. You will also need the `lapack` and
 `blas` libraries and development tools. Once you have all that, Grenade can be
 build using:
 
 ```
-./mafia build
+stack build
 ```
 
 and the tests run using:
 
 ```
-./mafia test
+stack test
 ```
 
-Grenade builds with ghc 7.10, 8.0, 8.2 and 8.4.
+This version of Grenade builds with GHC 8.8.
 
 Thanks
 ------

bench/bench-lstm.hs

@@ -1,7 +1,7 @@
-{-# LANGUAGE DataKinds             #-}
-{-# LANGUAGE BangPatterns          #-}
-{-# LANGUAGE ScopedTypeVariables   #-}
-import Criterion.Main
+{-# LANGUAGE BangPatterns        #-}
+{-# LANGUAGE DataKinds           #-}
+{-# LANGUAGE ScopedTypeVariables #-}
+import           Criterion.Main
 
 import           Grenade
 import           Grenade.Recurrent
@@ -26,5 +26,5 @@ type R = Recurrent
 type RecNet = RecurrentNetwork '[ R (LSTM 40 512), R (LSTM 512 40) ]
                                '[ 'D1 40, 'D1 512, 'D1 40 ]
 
-lp :: LearningParameters
-lp = LearningParameters 0.1 0 0
+lp :: Optimizer 'SGD
+lp = defOptimizer