Add ReLU activation function

core/include/jml/math/activation_functions.hpp

@@ -47,4 +47,24 @@ class Sigmoid : private ActivationFunction {
     double df(double x);
 };
 
+class ReLU : private ActivationFunction {
+  public:
+    ReLU();
+
+    double f(double x);
+    double df(double x);
+};
+
+class LeakyReLU : private ActivationFunction {
+  public:
+    LeakyReLU();
+    LeakyReLU(double x);
+
+    double f(double x);
+    double df(double x);
+
+  private:
+    double leak;
+};
+
 } // namespace jml

core/src/math/activation_functions.cpp

@@ -1,10 +1,12 @@
 #include "jml/math/activation_functions.hpp"
-#include <cmath>
 
+#include <cmath>
 #include <iostream>
 
 namespace jml {
 
+// FastSigmoid
+//
 FastSigmoid::FastSigmoid() {}
 FastSigmoid::FastSigmoid(double x) { this->precomputed = (1 + std::fabs(x)); }
 
@@ -26,6 +28,8 @@ double FastSigmoid::df_precomp() {
     return 5.0 * (this->precomputed + this->x) / this->precomputed;
 }
 
+// Sigmoid
+//
 Sigmoid::Sigmoid() {}
 
 double Sigmoid::f(double x) { return 1.0 / (1.0 + std::exp(-x)); }
@@ -35,4 +39,21 @@ double Sigmoid::df(double x) {
     return s * (1 - s);
 }
 
+// ReLU
+//
+ReLU::ReLU() {}
+
+double ReLU::f(double x) { return std::fmax(0, x); }
+
+double ReLU::df(double x) { return x > 0; }
+
+// LeakyReLU
+//
+LeakyReLU::LeakyReLU() { this->leak = 0.01; }
+LeakyReLU::LeakyReLU(double l) { this->leak = l; }
+
+double LeakyReLU::f(double x) { return std::fmax(this->leak * x, x); }
+
+double LeakyReLU::df(double x) { return (x > 0 ? 1 : this->leak); }
+
 } // namespace jml