Add a general-purpose autoencoder class AE to serve as a building b…

…lock for other autoencoder models (#932)

* Port the AE from `@voidvoxel/auto-encoder`

* Rewrite class `AutoEncoder` from scratch

* Remove unused property

* Use shallow clone

* Add unit tests

* Replace `shallowClone` with `deepClone`

* Rename private properties

* Remove the space in the word "autoencoder"

[Wikipedia said "Autoencoder" and not "Auto encoder" or "Auto-encoder"](https://en.wikipedia.org/wiki/Autoencoder)

* Rename class `Autoencoder` to `AE`

The other classes in this library use their respective acronyms with the exceptions of `FeedForward`, `NeuralNetwork`, and `NeuralNetworkGPU`. Furthermore, `AE` variants of existing classes will likely be made, so an acronym equivalent would be desirable. I'm considering the naming conventions for classes such as `LSTMTimeStep`. For example maybe a future `VAE` class could be made to represent variational autoencoders.

* Add `AE` usage to README.md

* Update references to `AE`

* Use Partial<T> instead of nullable properties

* Update autoencoder.ts

* Minor improvements

* Fix "@rollup/plugin-typescript TS2807"
* Choose a more accurate name for `includesAnomalies` (`likelyIncludesAnomalies`, as it makes no guarantees that anomalies are truly present and only provides an intuitive guess)

README.md

@@ -17,7 +17,7 @@ GPU accelerated Neural networks in JavaScript for Browsers and Node.js
   ![CI](https://github.com/BrainJS/brain.js/workflows/CI/badge.svg)
   [![codecov](https://codecov.io/gh/BrainJS/brain.js/branch/master/graph/badge.svg?token=3SJIBJ1679)](https://codecov.io/gh/BrainJS/brain.js)
   <a href="https://twitter.com/brainjsfnd"><img src="https://img.shields.io/twitter/follow/brainjsfnd?label=Twitter&style=social" alt="Twitter"></a>
-  
+
   [![NPM](https://nodei.co/npm/brain.js.png?compact=true)](https://nodei.co/npm/brain.js/)
 
 </p>
@@ -43,6 +43,7 @@ GPU accelerated Neural networks in JavaScript for Browsers and Node.js
     - [For training with NeuralNetwork](#for-training-with-neuralnetwork)
     - [For training with `RNNTimeStep`, `LSTMTimeStep` and `GRUTimeStep`](#for-training-with-rnntimestep-lstmtimestep-and-grutimestep)
     - [For training with `RNN`, `LSTM` and `GRU`](#for-training-with-rnn-lstm-and-gru)
+    - [For training with `AE`](#for-training-with-ae)
   - [Training Options](#training-options)
   - [Async Training](#async-training)
   - [Cross Validation](#cross-validation)
@@ -317,6 +318,54 @@ net.train([
 const output = net.run('I feel great about the world!'); // 'happy'
 ```
 
+#### For training with `AE`
+
+Each training pattern can either:
+
+- Be an array of numbers
+- Be an array of arrays of numbers
+
+Training an autoencoder to compress the values of a XOR calculation:
+
+```javascript
+const net = new brain.AE(
+  {
+    hiddenLayers: [ 5, 2, 5 ]
+  }
+);
+
+net.train([
+  [ 0, 0, 0 ],
+  [ 0, 1, 1 ],
+  [ 1, 0, 1 ],
+  [ 1, 1, 0 ]
+]);
+```
+
+Encoding/decoding:
+
+```javascript
+const input = [ 0, 1, 1 ];
+
+const encoded = net.encode(input);
+const decoded = net.decode(encoded);
+```
+
+Denoise noisy data:
+
+```javascript
+const noisyData = [ 0, 1, 0 ];
+
+const data = net.denoise(noisyData);
+```
+
+Test for anomalies in data samples:
+
+```javascript
+const shouldBeFalse = net.includesAnomalies([0, 1, 1]);
+const shouldBeTrue = net.includesAnomalies([0, 1, 0]);
+```
+
 ### Training Options
 
 `train()` takes a hash of options as its second argument:
@@ -595,6 +644,7 @@ The user interface used:
 
 - [`brain.NeuralNetwork`](src/neural-network.ts) - [Feedforward Neural Network](https://en.wikipedia.org/wiki/Feedforward_neural_network) with backpropagation
 - [`brain.NeuralNetworkGPU`](src/neural-network-gpu.ts) - [Feedforward Neural Network](https://en.wikipedia.org/wiki/Feedforward_neural_network) with backpropagation, GPU version
+- [`brain.AE`](src/autoencoder.ts) - [Autoencoder or "AE"](https://en.wikipedia.org/wiki/Autoencoder) with backpropogation and GPU support
 - [`brain.recurrent.RNNTimeStep`](src/recurrent/rnn-time-step.ts) - [Time Step Recurrent Neural Network or "RNN"](https://en.wikipedia.org/wiki/Recurrent_neural_network)
 - [`brain.recurrent.LSTMTimeStep`](src/recurrent/lstm-time-step.ts) - [Time Step Long Short Term Memory Neural Network or "LSTM"](https://en.wikipedia.org/wiki/Long_short-term_memory)
 - [`brain.recurrent.GRUTimeStep`](src/recurrent/gru-time-step.ts) - [Time Step Gated Recurrent Unit or "GRU"](https://en.wikipedia.org/wiki/Gated_recurrent_unit)

src/autoencoder.test.ts

@@ -0,0 +1,79 @@
+import AE from "./autoencoder";
+
+const trainingData = [
+  [0, 0, 0],
+  [0, 1, 1],
+  [1, 0, 1],
+  [1, 1, 0]
+];
+
+const xornet = new AE<number[], number[]>(
+  {
+    decodedSize: 3,
+    hiddenLayers: [ 5, 2, 5 ]
+  }
+);
+
+const errorThresh = 0.011;
+
+const result = xornet.train(
+  trainingData, {
+    iterations: 100000,
+    errorThresh
+  }
+);
+
+test(
+  "denoise a data sample",
+  async () => {
+    expect(result.error).toBeLessThanOrEqual(errorThresh);
+
+    function xor(...args: number[]) {
+      return Math.round(xornet.denoise(args)[2]);
+    }
+
+    const run1 = xor(0, 0, 0);
+    const run2 = xor(0, 1, 1);
+    const run3 = xor(1, 0, 1);
+    const run4 = xor(1, 1, 0);
+
+    expect(run1).toBe(0);
+    expect(run2).toBe(1);
+    expect(run3).toBe(1);
+    expect(run4).toBe(0);
+  }
+);
+
+test(
+  "encode and decode a data sample",
+  async () => {
+    expect(result.error).toBeLessThanOrEqual(errorThresh);
+
+    const run1$input = [0, 0, 0];
+    const run1$encoded = xornet.encode(run1$input);
+    const run1$decoded = xornet.decode(run1$encoded);
+
+    const run2$input = [0, 1, 1];
+    const run2$encoded = xornet.encode(run2$input);
+    const run2$decoded = xornet.decode(run2$encoded);
+
+    for (let i = 0; i < 3; i++) expect(Math.round(run1$decoded[i])).toBe(run1$input[i]);
+    for (let i = 0; i < 3; i++) expect(Math.round(run2$decoded[i])).toBe(run2$input[i]);
+  }
+);
+
+test(
+  "test a data sample for anomalies",
+  async () => {
+    expect(result.error).toBeLessThanOrEqual(errorThresh);
+
+    function includesAnomalies(...args: number[]) {
+      expect(xornet.likelyIncludesAnomalies(args)).toBe(false);
+    }
+
+    includesAnomalies(0, 0, 0);
+    includesAnomalies(0, 1, 1);
+    includesAnomalies(1, 0, 1);
+    includesAnomalies(1, 1, 0);
+  }
+);

src/autoencoder.ts

@@ -0,0 +1,189 @@
+import { KernelOutput, Texture, TextureArrayOutput } from "gpu.js";
+import { IJSONLayer, INeuralNetworkData, INeuralNetworkDatum, INeuralNetworkTrainOptions } from "./neural-network";
+import { INeuralNetworkGPUOptions, NeuralNetworkGPU } from "./neural-network-gpu";
+import { INeuralNetworkState } from "./neural-network-types";
+import { UntrainedNeuralNetworkError } from "./errors/untrained-neural-network-error";
+
+export interface IAEOptions {
+  binaryThresh: number;
+  decodedSize: number;
+  hiddenLayers: number[];
+}
+
+/**
+ * An autoencoder learns to compress input data down to relevant features and reconstruct input data from its compressed representation.
+ */
+export class AE<DecodedData extends INeuralNetworkData, EncodedData extends INeuralNetworkData> {
+  private decoder?: NeuralNetworkGPU<EncodedData, DecodedData>;
+  private denoiser: NeuralNetworkGPU<DecodedData, DecodedData>;
+
+  constructor (
+    options?: Partial<IAEOptions>
+  ) {
+    // Create default options for the autoencoder.
+    options ??= {};
+
+    // Create default options for the autoencoder's denoiser subnet.
+    const denoiserOptions: Partial<INeuralNetworkGPUOptions> = {};
+
+    // Inherit the binary threshold of the parent autoencoder.
+    denoiserOptions.binaryThresh = options.binaryThresh;
+    // Inherit the hidden layers of the parent autoencoder.
+    denoiserOptions.hiddenLayers = options.hiddenLayers;
+
+    // Define the denoiser subnet's input and output sizes.
+    if (options.decodedSize) denoiserOptions.inputSize = denoiserOptions.outputSize = options.decodedSize;
+
+    // Create the denoiser subnet of the autoencoder.
+    this.denoiser = new NeuralNetworkGPU<DecodedData, DecodedData>(options);
+  }
+
+  /**
+   * Denoise input data, removing any anomalies from the data.
+   * @param {DecodedData} input
+   * @returns {DecodedData}
+   */
+  denoise(input: DecodedData): DecodedData {
+    // Run the input through the generic denoiser.
+    // This isn't the best denoiser implementation, but it's efficient.
+    // Efficiency is important here because training should focus on
+    // optimizing for feature extraction as quickly as possible rather than
+    // denoising and anomaly detection; there are other specialized topologies
+    // better suited for these tasks anyways, many of which can be implemented
+    // by using an autoencoder.
+    return this.denoiser.run(input);
+  }
+
+  /**
+   * Decode `EncodedData` into an approximation of its original form.
+   *
+   * @param {EncodedData} input
+   * @returns {DecodedData}
+   */
+  decode(input: EncodedData): DecodedData {
+    // If the decoder has not been trained yet, throw an error.
+    if (!this.decoder) throw new UntrainedNeuralNetworkError(this);
+
+    // Decode the encoded input.
+    return this.decoder.run(input);
+  }
+
+  /**
+   * Encode data to extract features, reduce dimensionality, etc.
+   *
+   * @param {DecodedData} input
+   * @returns {EncodedData}
+   */
+  encode(input: DecodedData): EncodedData {
+    // If the decoder has not been trained yet, throw an error.
+    if (!this.denoiser) throw new UntrainedNeuralNetworkError(this);
+
+    // Process the input.
+    this.denoiser.run(input);
+
+    // Get the auto-encoded input.
+    let encodedInput: TextureArrayOutput = this.encodedLayer as TextureArrayOutput;
+
+    // If the encoded input is a `Texture`, convert it into an `Array`.
+    if (encodedInput instanceof Texture) encodedInput = encodedInput.toArray();
+    else encodedInput = encodedInput.slice(0);
+
+    // Return the encoded input.
+    return encodedInput as EncodedData;
+  }
+
+  /**
+   * Test whether or not a data sample likely contains anomalies.
+   * If anomalies are likely present in the sample, returns `true`.
+   * Otherwise, returns `false`.
+   *
+   * @param {DecodedData} input
+   * @returns {boolean}
+   */
+  likelyIncludesAnomalies(input: DecodedData, anomalyThreshold: number = 0.2): boolean {
+    // Create the anomaly vector.
+    const anomalies: number[] = [];
+
+    // Attempt to denoise the input.
+    const denoised = this.denoise(input);
+
+    // Calculate the anomaly vector.
+    for (let i = 0; i < (input.length ?? 0); i++) {
+      anomalies[i] = Math.abs((input as number[])[i] - (denoised as number[])[i]);
+    }
+
+    // Calculate the sum of all anomalies within the vector.
+    const sum = anomalies.reduce(
+      (previousValue, value) => previousValue + value
+    );
+
+    // Calculate the mean anomaly.
+    const mean = sum / (input as number[]).length;
+
+    // Return whether or not the mean anomaly rate is greater than the anomaly threshold.
+    return mean > anomalyThreshold;
+  }
+
+  /**
+   * Train the auto encoder.
+   *
+   * @param {DecodedData[]} data
+   * @param {Partial<INeuralNetworkTrainOptions>} options
+   * @returns {INeuralNetworkState}
+   */
+  train(data: DecodedData[], options?: Partial<INeuralNetworkTrainOptions>): INeuralNetworkState {
+    const preprocessedData: INeuralNetworkDatum<Partial<DecodedData>, Partial<DecodedData>>[] = [];
+
+    for (let datum of data) {
+      preprocessedData.push( { input: datum, output: datum } );
+    }
+
+    const results = this.denoiser.train(preprocessedData, options);
+
+    this.decoder = this.createDecoder();
+
+    return results;
+  }
+
+  /**
+   * Create a new decoder from the trained denoiser.
+   *
+   * @returns {NeuralNetworkGPU<EncodedData, DecodedData>}
+   */
+  private createDecoder() {
+    const json = this.denoiser.toJSON();
+
+    const layers: IJSONLayer[] = [];
+    const sizes: number[] = [];
+
+    for (let i = this.encodedLayerIndex; i < this.denoiser.sizes.length; i++) {
+      layers.push(json.layers[i]);
+      sizes.push(json.sizes[i]);
+    }
+
+    json.layers = layers;
+    json.sizes = sizes;
+
+    json.options.inputSize = json.sizes[0];
+
+    const decoder = new NeuralNetworkGPU().fromJSON(json);
+
+    return decoder as unknown as NeuralNetworkGPU<EncodedData, DecodedData>;
+  }
+
+  /**
+   * Get the layer containing the encoded representation.
+   */
+  private get encodedLayer(): KernelOutput {
+    return this.denoiser.outputs[this.encodedLayerIndex];
+  }
+
+  /**
+   * Get the offset of the encoded layer.
+   */
+  private get encodedLayerIndex(): number {
+    return Math.round(this.denoiser.outputs.length * 0.5) - 1;
+  }
+}
+
+export default AE;

src/errors/untrained-neural-network-error.ts

@@ -0,0 +1,7 @@
+export class UntrainedNeuralNetworkError extends Error {
+  constructor (
+    neuralNetwork: any
+  ) {
+    super(`Cannot run a ${neuralNetwork.constructor.name} before it is trained.`);
+  }
+}

src/index.ts

@@ -1,4 +1,5 @@
 import * as activation from './activation';
+import { AE } from './autoencoder';
 import CrossValidate from './cross-validate';
 import { FeedForward } from './feed-forward';
 import * as layer from './layer';
@@ -53,6 +54,7 @@ const utilities = {
 
 export {
   activation,
+  AE,
   CrossValidate,
   likely,
   layer,