2848 lines
91 KiB
JavaScript
2848 lines
91 KiB
JavaScript
/*!
|
|
* The MIT License (MIT)
|
|
*
|
|
* Copyright (c) 2016 Juan Cazala - juancazala.com
|
|
*
|
|
* Permission is hereby granted, free of charge, to any person obtaining a copy
|
|
* of this software and associated documentation files (the "Software"), to deal
|
|
* in the Software without restriction, including without limitation the rights
|
|
* to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
|
* copies of the Software, and to permit persons to whom the Software is
|
|
* furnished to do so, subject to the following conditions:
|
|
*
|
|
* The above copyright notice and this permission notice shall be included in
|
|
* all copies or substantial portions of the Software.
|
|
*
|
|
* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
|
* IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
|
* FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
|
* AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
|
* LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
|
* OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
|
|
* THE SOFTWARE
|
|
*
|
|
*
|
|
*
|
|
* ********************************************************************************************
|
|
* SYNAPTIC (v1.0.8)
|
|
* ********************************************************************************************
|
|
*
|
|
* Synaptic is a javascript neural network library for node.js and the browser, its generalized
|
|
* algorithm is architecture-free, so you can build and train basically any type of first order
|
|
* or even second order neural network architectures.
|
|
*
|
|
* http://en.wikipedia.org/wiki/Recurrent_neural_network#Second_Order_Recurrent_Neural_Network
|
|
*
|
|
* The library includes a few built-in architectures like multilayer perceptrons, multilayer
|
|
* long-short term memory networks (LSTM) or liquid state machines, and a trainer capable of
|
|
* training any given network, and includes built-in training tasks/tests like solving an XOR,
|
|
* passing a Distracted Sequence Recall test or an Embeded Reber Grammar test.
|
|
*
|
|
* The algorithm implemented by this library has been taken from Derek D. Monner's paper:
|
|
*
|
|
*
|
|
* A generalized LSTM-like training algorithm for second-order recurrent neural networks
|
|
* http://www.overcomplete.net/papers/nn2012.pdf
|
|
*
|
|
* There are references to the equations in that paper commented through the source code.
|
|
*
|
|
*/
|
|
/******/ (function(modules) { // webpackBootstrap
|
|
/******/ // The module cache
|
|
/******/ var installedModules = {};
|
|
|
|
/******/ // The require function
|
|
/******/ function __webpack_require__(moduleId) {
|
|
|
|
/******/ // Check if module is in cache
|
|
/******/ if(installedModules[moduleId])
|
|
/******/ return installedModules[moduleId].exports;
|
|
|
|
/******/ // Create a new module (and put it into the cache)
|
|
/******/ var module = installedModules[moduleId] = {
|
|
/******/ exports: {},
|
|
/******/ id: moduleId,
|
|
/******/ loaded: false
|
|
/******/ };
|
|
|
|
/******/ // Execute the module function
|
|
/******/ modules[moduleId].call(module.exports, module, module.exports, __webpack_require__);
|
|
|
|
/******/ // Flag the module as loaded
|
|
/******/ module.loaded = true;
|
|
|
|
/******/ // Return the exports of the module
|
|
/******/ return module.exports;
|
|
/******/ }
|
|
|
|
|
|
/******/ // expose the modules object (__webpack_modules__)
|
|
/******/ __webpack_require__.m = modules;
|
|
|
|
/******/ // expose the module cache
|
|
/******/ __webpack_require__.c = installedModules;
|
|
|
|
/******/ // __webpack_public_path__
|
|
/******/ __webpack_require__.p = "";
|
|
|
|
/******/ // Load entry module and return exports
|
|
/******/ return __webpack_require__(0);
|
|
/******/ })
|
|
/************************************************************************/
|
|
/******/ ([
|
|
/* 0 */
|
|
/***/ function(module, exports, __webpack_require__) {
|
|
|
|
var __WEBPACK_AMD_DEFINE_ARRAY__, __WEBPACK_AMD_DEFINE_RESULT__;var Synaptic = {
|
|
Neuron: __webpack_require__(1),
|
|
Layer: __webpack_require__(3),
|
|
Network: __webpack_require__(4),
|
|
Trainer: __webpack_require__(5),
|
|
Architect: __webpack_require__(6)
|
|
};
|
|
|
|
// CommonJS & AMD
|
|
if (true)
|
|
{
|
|
!(__WEBPACK_AMD_DEFINE_ARRAY__ = [], __WEBPACK_AMD_DEFINE_RESULT__ = function(){ return Synaptic }.apply(exports, __WEBPACK_AMD_DEFINE_ARRAY__), __WEBPACK_AMD_DEFINE_RESULT__ !== undefined && (module.exports = __WEBPACK_AMD_DEFINE_RESULT__));
|
|
}
|
|
|
|
// Node.js
|
|
if (typeof module !== 'undefined' && module.exports)
|
|
{
|
|
module.exports = Synaptic;
|
|
}
|
|
|
|
// Browser
|
|
if (typeof window == 'object')
|
|
{
|
|
(function(){
|
|
var oldSynaptic = window['synaptic'];
|
|
Synaptic.ninja = function(){
|
|
window['synaptic'] = oldSynaptic;
|
|
return Synaptic;
|
|
};
|
|
})();
|
|
|
|
window['synaptic'] = Synaptic;
|
|
}
|
|
|
|
|
|
/***/ },
|
|
/* 1 */
|
|
/***/ function(module, exports, __webpack_require__) {
|
|
|
|
/* WEBPACK VAR INJECTION */(function(module) {// export
|
|
if (module) module.exports = Neuron;
|
|
|
|
/******************************************************************************************
|
|
NEURON
|
|
*******************************************************************************************/
|
|
|
|
function Neuron() {
|
|
this.ID = Neuron.uid();
|
|
this.label = null;
|
|
this.connections = {
|
|
inputs: {},
|
|
projected: {},
|
|
gated: {}
|
|
};
|
|
this.error = {
|
|
responsibility: 0,
|
|
projected: 0,
|
|
gated: 0
|
|
};
|
|
this.trace = {
|
|
elegibility: {},
|
|
extended: {},
|
|
influences: {}
|
|
};
|
|
this.state = 0;
|
|
this.old = 0;
|
|
this.activation = 0;
|
|
this.selfconnection = new Neuron.connection(this, this, 0); // weight = 0 -> not connected
|
|
this.squash = Neuron.squash.LOGISTIC;
|
|
this.neighboors = {};
|
|
this.bias = Math.random() * .2 - .1;
|
|
}
|
|
|
|
Neuron.prototype = {
|
|
|
|
// activate the neuron
|
|
activate: function(input) {
|
|
// activation from enviroment (for input neurons)
|
|
if (typeof input != 'undefined') {
|
|
this.activation = input;
|
|
this.derivative = 0;
|
|
this.bias = 0;
|
|
return this.activation;
|
|
}
|
|
|
|
// old state
|
|
this.old = this.state;
|
|
|
|
// eq. 15
|
|
this.state = this.selfconnection.gain * this.selfconnection.weight *
|
|
this.state + this.bias;
|
|
|
|
for (var i in this.connections.inputs) {
|
|
var input = this.connections.inputs[i];
|
|
this.state += input.from.activation * input.weight * input.gain;
|
|
}
|
|
|
|
// eq. 16
|
|
this.activation = this.squash(this.state);
|
|
|
|
// f'(s)
|
|
this.derivative = this.squash(this.state, true);
|
|
|
|
// update traces
|
|
var influences = [];
|
|
for (var id in this.trace.extended) {
|
|
// extended elegibility trace
|
|
var neuron = this.neighboors[id];
|
|
|
|
// if gated neuron's selfconnection is gated by this unit, the influence keeps track of the neuron's old state
|
|
var influence = neuron.selfconnection.gater == this ? neuron.old : 0;
|
|
|
|
// index runs over all the incoming connections to the gated neuron that are gated by this unit
|
|
for (var incoming in this.trace.influences[neuron.ID]) { // captures the effect that has an input connection to this unit, on a neuron that is gated by this unit
|
|
influence += this.trace.influences[neuron.ID][incoming].weight *
|
|
this.trace.influences[neuron.ID][incoming].from.activation;
|
|
}
|
|
influences[neuron.ID] = influence;
|
|
}
|
|
|
|
for (var i in this.connections.inputs) {
|
|
var input = this.connections.inputs[i];
|
|
|
|
// elegibility trace - Eq. 17
|
|
this.trace.elegibility[input.ID] = this.selfconnection.gain * this.selfconnection
|
|
.weight * this.trace.elegibility[input.ID] + input.gain * input.from
|
|
.activation;
|
|
|
|
for (var id in this.trace.extended) {
|
|
// extended elegibility trace
|
|
var xtrace = this.trace.extended[id];
|
|
var neuron = this.neighboors[id];
|
|
var influence = influences[neuron.ID];
|
|
|
|
// eq. 18
|
|
xtrace[input.ID] = neuron.selfconnection.gain * neuron.selfconnection
|
|
.weight * xtrace[input.ID] + this.derivative * this.trace.elegibility[
|
|
input.ID] * influence;
|
|
}
|
|
}
|
|
|
|
// update gated connection's gains
|
|
for (var connection in this.connections.gated) {
|
|
this.connections.gated[connection].gain = this.activation;
|
|
}
|
|
|
|
return this.activation;
|
|
},
|
|
|
|
// back-propagate the error
|
|
propagate: function(rate, target) {
|
|
// error accumulator
|
|
var error = 0;
|
|
|
|
// whether or not this neuron is in the output layer
|
|
var isOutput = typeof target != 'undefined';
|
|
|
|
// output neurons get their error from the enviroment
|
|
if (isOutput)
|
|
this.error.responsibility = this.error.projected = target - this.activation; // Eq. 10
|
|
|
|
else // the rest of the neuron compute their error responsibilities by backpropagation
|
|
{
|
|
// error responsibilities from all the connections projected from this neuron
|
|
for (var id in this.connections.projected) {
|
|
var connection = this.connections.projected[id];
|
|
var neuron = connection.to;
|
|
// Eq. 21
|
|
error += neuron.error.responsibility * connection.gain * connection.weight;
|
|
}
|
|
|
|
// projected error responsibility
|
|
this.error.projected = this.derivative * error;
|
|
|
|
error = 0;
|
|
// error responsibilities from all the connections gated by this neuron
|
|
for (var id in this.trace.extended) {
|
|
var neuron = this.neighboors[id]; // gated neuron
|
|
var influence = neuron.selfconnection.gater == this ? neuron.old : 0; // if gated neuron's selfconnection is gated by this neuron
|
|
|
|
// index runs over all the connections to the gated neuron that are gated by this neuron
|
|
for (var input in this.trace.influences[id]) { // captures the effect that the input connection of this neuron have, on a neuron which its input/s is/are gated by this neuron
|
|
influence += this.trace.influences[id][input].weight * this.trace.influences[
|
|
neuron.ID][input].from.activation;
|
|
}
|
|
// eq. 22
|
|
error += neuron.error.responsibility * influence;
|
|
}
|
|
|
|
// gated error responsibility
|
|
this.error.gated = this.derivative * error;
|
|
|
|
// error responsibility - Eq. 23
|
|
this.error.responsibility = this.error.projected + this.error.gated;
|
|
}
|
|
|
|
// learning rate
|
|
rate = rate || .1;
|
|
|
|
// adjust all the neuron's incoming connections
|
|
for (var id in this.connections.inputs) {
|
|
var input = this.connections.inputs[id];
|
|
|
|
// Eq. 24
|
|
var gradient = this.error.projected * this.trace.elegibility[input.ID];
|
|
for (var id in this.trace.extended) {
|
|
var neuron = this.neighboors[id];
|
|
gradient += neuron.error.responsibility * this.trace.extended[
|
|
neuron.ID][input.ID];
|
|
}
|
|
input.weight += rate * gradient; // adjust weights - aka learn
|
|
}
|
|
|
|
// adjust bias
|
|
this.bias += rate * this.error.responsibility;
|
|
},
|
|
|
|
project: function(neuron, weight) {
|
|
// self-connection
|
|
if (neuron == this) {
|
|
this.selfconnection.weight = 1;
|
|
return this.selfconnection;
|
|
}
|
|
|
|
// check if connection already exists
|
|
var connected = this.connected(neuron);
|
|
if (connected && connected.type == "projected") {
|
|
// update connection
|
|
if (typeof weight != 'undefined')
|
|
connected.connection.weight = weight;
|
|
// return existing connection
|
|
return connected.connection;
|
|
} else {
|
|
// create a new connection
|
|
var connection = new Neuron.connection(this, neuron, weight);
|
|
}
|
|
|
|
// reference all the connections and traces
|
|
this.connections.projected[connection.ID] = connection;
|
|
this.neighboors[neuron.ID] = neuron;
|
|
neuron.connections.inputs[connection.ID] = connection;
|
|
neuron.trace.elegibility[connection.ID] = 0;
|
|
|
|
for (var id in neuron.trace.extended) {
|
|
var trace = neuron.trace.extended[id];
|
|
trace[connection.ID] = 0;
|
|
}
|
|
|
|
return connection;
|
|
},
|
|
|
|
gate: function(connection) {
|
|
// add connection to gated list
|
|
this.connections.gated[connection.ID] = connection;
|
|
|
|
var neuron = connection.to;
|
|
if (!(neuron.ID in this.trace.extended)) {
|
|
// extended trace
|
|
this.neighboors[neuron.ID] = neuron;
|
|
var xtrace = this.trace.extended[neuron.ID] = {};
|
|
for (var id in this.connections.inputs) {
|
|
var input = this.connections.inputs[id];
|
|
xtrace[input.ID] = 0;
|
|
}
|
|
}
|
|
|
|
// keep track
|
|
if (neuron.ID in this.trace.influences)
|
|
this.trace.influences[neuron.ID].push(connection);
|
|
else
|
|
this.trace.influences[neuron.ID] = [connection];
|
|
|
|
// set gater
|
|
connection.gater = this;
|
|
},
|
|
|
|
// returns true or false whether the neuron is self-connected or not
|
|
selfconnected: function() {
|
|
return this.selfconnection.weight !== 0;
|
|
},
|
|
|
|
// returns true or false whether the neuron is connected to another neuron (parameter)
|
|
connected: function(neuron) {
|
|
var result = {
|
|
type: null,
|
|
connection: false
|
|
};
|
|
|
|
if (this == neuron) {
|
|
if (this.selfconnected()) {
|
|
result.type = 'selfconnection';
|
|
result.connection = this.selfconnection;
|
|
return result;
|
|
} else
|
|
return false;
|
|
}
|
|
|
|
for (var type in this.connections) {
|
|
for (var connection in this.connections[type]) {
|
|
var connection = this.connections[type][connection];
|
|
if (connection.to == neuron) {
|
|
result.type = type;
|
|
result.connection = connection;
|
|
return result;
|
|
} else if (connection.from == neuron) {
|
|
result.type = type;
|
|
result.connection = connection;
|
|
return result;
|
|
}
|
|
}
|
|
}
|
|
|
|
return false;
|
|
},
|
|
|
|
// clears all the traces (the neuron forgets it's context, but the connections remain intact)
|
|
clear: function() {
|
|
|
|
for (var trace in this.trace.elegibility)
|
|
this.trace.elegibility[trace] = 0;
|
|
|
|
for (var trace in this.trace.extended)
|
|
for (var extended in this.trace.extended[trace])
|
|
this.trace.extended[trace][extended] = 0;
|
|
|
|
this.error.responsibility = this.error.projected = this.error.gated = 0;
|
|
},
|
|
|
|
// all the connections are randomized and the traces are cleared
|
|
reset: function() {
|
|
this.clear();
|
|
|
|
for (var type in this.connections)
|
|
for (var connection in this.connections[type])
|
|
this.connections[type][connection].weight = Math.random() * .2 - .1;
|
|
this.bias = Math.random() * .2 - .1;
|
|
|
|
this.old = this.state = this.activation = 0;
|
|
},
|
|
|
|
// hardcodes the behaviour of the neuron into an optimized function
|
|
optimize: function(optimized, layer) {
|
|
|
|
optimized = optimized || {};
|
|
var store_activation = [];
|
|
var store_trace = [];
|
|
var store_propagation = [];
|
|
var varID = optimized.memory || 0;
|
|
var neurons = optimized.neurons || 1;
|
|
var inputs = optimized.inputs || [];
|
|
var targets = optimized.targets || [];
|
|
var outputs = optimized.outputs || [];
|
|
var variables = optimized.variables || {};
|
|
var activation_sentences = optimized.activation_sentences || [];
|
|
var trace_sentences = optimized.trace_sentences || [];
|
|
var propagation_sentences = optimized.propagation_sentences || [];
|
|
var layers = optimized.layers || { __count: 0, __neuron: 0 };
|
|
|
|
// allocate sentences
|
|
var allocate = function(store){
|
|
var allocated = layer in layers && store[layers.__count];
|
|
if (!allocated)
|
|
{
|
|
layers.__count = store.push([]) - 1;
|
|
layers[layer] = layers.__count;
|
|
}
|
|
};
|
|
allocate(activation_sentences);
|
|
allocate(trace_sentences);
|
|
allocate(propagation_sentences);
|
|
var currentLayer = layers.__count;
|
|
|
|
// get/reserve space in memory by creating a unique ID for a variablel
|
|
var getVar = function() {
|
|
var args = Array.prototype.slice.call(arguments);
|
|
|
|
if (args.length == 1) {
|
|
if (args[0] == 'target') {
|
|
var id = 'target_' + targets.length;
|
|
targets.push(varID);
|
|
} else
|
|
var id = args[0];
|
|
if (id in variables)
|
|
return variables[id];
|
|
return variables[id] = {
|
|
value: 0,
|
|
id: varID++
|
|
};
|
|
} else {
|
|
var extended = args.length > 2;
|
|
if (extended)
|
|
var value = args.pop();
|
|
|
|
var unit = args.shift();
|
|
var prop = args.pop();
|
|
|
|
if (!extended)
|
|
var value = unit[prop];
|
|
|
|
var id = prop + '_';
|
|
for (var property in args)
|
|
id += args[property] + '_';
|
|
id += unit.ID;
|
|
if (id in variables)
|
|
return variables[id];
|
|
|
|
return variables[id] = {
|
|
value: value,
|
|
id: varID++
|
|
};
|
|
}
|
|
};
|
|
|
|
// build sentence
|
|
var buildSentence = function() {
|
|
var args = Array.prototype.slice.call(arguments);
|
|
var store = args.pop();
|
|
var sentence = "";
|
|
for (var i in args)
|
|
if (typeof args[i] == 'string')
|
|
sentence += args[i];
|
|
else
|
|
sentence += 'F[' + args[i].id + ']';
|
|
|
|
store.push(sentence + ';');
|
|
};
|
|
|
|
// helper to check if an object is empty
|
|
var isEmpty = function(obj) {
|
|
for (var prop in obj) {
|
|
if (obj.hasOwnProperty(prop))
|
|
return false;
|
|
}
|
|
return true;
|
|
};
|
|
|
|
// characteristics of the neuron
|
|
var noProjections = isEmpty(this.connections.projected);
|
|
var noGates = isEmpty(this.connections.gated);
|
|
var isInput = layer == 'input' ? true : isEmpty(this.connections.inputs);
|
|
var isOutput = layer == 'output' ? true : noProjections && noGates;
|
|
|
|
// optimize neuron's behaviour
|
|
var rate = getVar('rate');
|
|
var activation = getVar(this, 'activation');
|
|
if (isInput)
|
|
inputs.push(activation.id);
|
|
else {
|
|
activation_sentences[currentLayer].push(store_activation);
|
|
trace_sentences[currentLayer].push(store_trace);
|
|
propagation_sentences[currentLayer].push(store_propagation);
|
|
var old = getVar(this, 'old');
|
|
var state = getVar(this, 'state');
|
|
var bias = getVar(this, 'bias');
|
|
if (this.selfconnection.gater)
|
|
var self_gain = getVar(this.selfconnection, 'gain');
|
|
if (this.selfconnected())
|
|
var self_weight = getVar(this.selfconnection, 'weight');
|
|
buildSentence(old, ' = ', state, store_activation);
|
|
if (this.selfconnected())
|
|
if (this.selfconnection.gater)
|
|
buildSentence(state, ' = ', self_gain, ' * ', self_weight, ' * ',
|
|
state, ' + ', bias, store_activation);
|
|
else
|
|
buildSentence(state, ' = ', self_weight, ' * ', state, ' + ',
|
|
bias, store_activation);
|
|
else
|
|
buildSentence(state, ' = ', bias, store_activation);
|
|
for (var i in this.connections.inputs) {
|
|
var input = this.connections.inputs[i];
|
|
var input_activation = getVar(input.from, 'activation');
|
|
var input_weight = getVar(input, 'weight');
|
|
if (input.gater)
|
|
var input_gain = getVar(input, 'gain');
|
|
if (this.connections.inputs[i].gater)
|
|
buildSentence(state, ' += ', input_activation, ' * ',
|
|
input_weight, ' * ', input_gain, store_activation);
|
|
else
|
|
buildSentence(state, ' += ', input_activation, ' * ',
|
|
input_weight, store_activation);
|
|
}
|
|
var derivative = getVar(this, 'derivative');
|
|
switch (this.squash) {
|
|
case Neuron.squash.LOGISTIC:
|
|
buildSentence(activation, ' = (1 / (1 + Math.exp(-', state, ')))',
|
|
store_activation);
|
|
buildSentence(derivative, ' = ', activation, ' * (1 - ',
|
|
activation, ')', store_activation);
|
|
break;
|
|
case Neuron.squash.TANH:
|
|
var eP = getVar('aux');
|
|
var eN = getVar('aux_2');
|
|
buildSentence(eP, ' = Math.exp(', state, ')', store_activation);
|
|
buildSentence(eN, ' = 1 / ', eP, store_activation);
|
|
buildSentence(activation, ' = (', eP, ' - ', eN, ') / (', eP, ' + ', eN, ')', store_activation);
|
|
buildSentence(derivative, ' = 1 - (', activation, ' * ', activation, ')', store_activation);
|
|
break;
|
|
case Neuron.squash.IDENTITY:
|
|
buildSentence(activation, ' = ', state, store_activation);
|
|
buildSentence(derivative, ' = 1', store_activation);
|
|
break;
|
|
case Neuron.squash.HLIM:
|
|
buildSentence(activation, ' = +(', state, ' > 0)', store_activation);
|
|
buildSentence(derivative, ' = 1', store_activation);
|
|
case Neuron.squash.RELU:
|
|
buildSentence(activation, ' = ', state, ' > 0 ? ', state, ' : 0', store_activation);
|
|
buildSentence(derivative, ' = ', state, ' > 0 ? 1 : 0', store_activation);
|
|
break;
|
|
}
|
|
|
|
for (var id in this.trace.extended) {
|
|
// calculate extended elegibility traces in advance
|
|
|
|
var neuron = this.neighboors[id];
|
|
var influence = getVar('influences[' + neuron.ID + ']');
|
|
var neuron_old = getVar(neuron, 'old');
|
|
var initialized = false;
|
|
if (neuron.selfconnection.gater == this)
|
|
{
|
|
buildSentence(influence, ' = ', neuron_old, store_trace);
|
|
initialized = true;
|
|
}
|
|
for (var incoming in this.trace.influences[neuron.ID]) {
|
|
var incoming_weight = getVar(this.trace.influences[neuron.ID]
|
|
[incoming], 'weight');
|
|
var incoming_activation = getVar(this.trace.influences[neuron.ID]
|
|
[incoming].from, 'activation');
|
|
|
|
if (initialized)
|
|
buildSentence(influence, ' += ', incoming_weight, ' * ', incoming_activation, store_trace);
|
|
else {
|
|
buildSentence(influence, ' = ', incoming_weight, ' * ', incoming_activation, store_trace);
|
|
initialized = true;
|
|
}
|
|
}
|
|
}
|
|
|
|
for (var i in this.connections.inputs) {
|
|
var input = this.connections.inputs[i];
|
|
if (input.gater)
|
|
var input_gain = getVar(input, 'gain');
|
|
var input_activation = getVar(input.from, 'activation');
|
|
var trace = getVar(this, 'trace', 'elegibility', input.ID, this.trace
|
|
.elegibility[input.ID]);
|
|
if (this.selfconnected()) {
|
|
if (this.selfconnection.gater) {
|
|
if (input.gater)
|
|
buildSentence(trace, ' = ', self_gain, ' * ', self_weight,
|
|
' * ', trace, ' + ', input_gain, ' * ', input_activation,
|
|
store_trace);
|
|
else
|
|
buildSentence(trace, ' = ', self_gain, ' * ', self_weight,
|
|
' * ', trace, ' + ', input_activation, store_trace);
|
|
} else {
|
|
if (input.gater)
|
|
buildSentence(trace, ' = ', self_weight, ' * ', trace, ' + ',
|
|
input_gain, ' * ', input_activation, store_trace);
|
|
else
|
|
buildSentence(trace, ' = ', self_weight, ' * ', trace, ' + ',
|
|
input_activation, store_trace);
|
|
}
|
|
} else {
|
|
if (input.gater)
|
|
buildSentence(trace, ' = ', input_gain, ' * ', input_activation,
|
|
store_trace);
|
|
else
|
|
buildSentence(trace, ' = ', input_activation, store_trace);
|
|
}
|
|
for (var id in this.trace.extended) {
|
|
// extended elegibility trace
|
|
var neuron = this.neighboors[id];
|
|
var influence = getVar('influences[' + neuron.ID + ']');
|
|
|
|
var trace = getVar(this, 'trace', 'elegibility', input.ID, this.trace
|
|
.elegibility[input.ID]);
|
|
var xtrace = getVar(this, 'trace', 'extended', neuron.ID, input.ID,
|
|
this.trace.extended[neuron.ID][input.ID]);
|
|
if (neuron.selfconnected())
|
|
var neuron_self_weight = getVar(neuron.selfconnection, 'weight');
|
|
if (neuron.selfconnection.gater)
|
|
var neuron_self_gain = getVar(neuron.selfconnection, 'gain');
|
|
if (neuron.selfconnected())
|
|
if (neuron.selfconnection.gater)
|
|
buildSentence(xtrace, ' = ', neuron_self_gain, ' * ',
|
|
neuron_self_weight, ' * ', xtrace, ' + ', derivative, ' * ',
|
|
trace, ' * ', influence, store_trace);
|
|
else
|
|
buildSentence(xtrace, ' = ', neuron_self_weight, ' * ',
|
|
xtrace, ' + ', derivative, ' * ', trace, ' * ',
|
|
influence, store_trace);
|
|
else
|
|
buildSentence(xtrace, ' = ', derivative, ' * ', trace, ' * ',
|
|
influence, store_trace);
|
|
}
|
|
}
|
|
for (var connection in this.connections.gated) {
|
|
var gated_gain = getVar(this.connections.gated[connection], 'gain');
|
|
buildSentence(gated_gain, ' = ', activation, store_activation);
|
|
}
|
|
}
|
|
if (!isInput) {
|
|
var responsibility = getVar(this, 'error', 'responsibility', this.error
|
|
.responsibility);
|
|
if (isOutput) {
|
|
var target = getVar('target');
|
|
buildSentence(responsibility, ' = ', target, ' - ', activation,
|
|
store_propagation);
|
|
for (var id in this.connections.inputs) {
|
|
var input = this.connections.inputs[id];
|
|
var trace = getVar(this, 'trace', 'elegibility', input.ID, this.trace
|
|
.elegibility[input.ID]);
|
|
var input_weight = getVar(input, 'weight');
|
|
buildSentence(input_weight, ' += ', rate, ' * (', responsibility,
|
|
' * ', trace, ')', store_propagation);
|
|
}
|
|
outputs.push(activation.id);
|
|
} else {
|
|
if (!noProjections && !noGates) {
|
|
var error = getVar('aux');
|
|
for (var id in this.connections.projected) {
|
|
var connection = this.connections.projected[id];
|
|
var neuron = connection.to;
|
|
var connection_weight = getVar(connection, 'weight');
|
|
var neuron_responsibility = getVar(neuron, 'error',
|
|
'responsibility', neuron.error.responsibility);
|
|
if (connection.gater) {
|
|
var connection_gain = getVar(connection, 'gain');
|
|
buildSentence(error, ' += ', neuron_responsibility, ' * ',
|
|
connection_gain, ' * ', connection_weight,
|
|
store_propagation);
|
|
} else
|
|
buildSentence(error, ' += ', neuron_responsibility, ' * ',
|
|
connection_weight, store_propagation);
|
|
}
|
|
var projected = getVar(this, 'error', 'projected', this.error.projected);
|
|
buildSentence(projected, ' = ', derivative, ' * ', error,
|
|
store_propagation);
|
|
buildSentence(error, ' = 0', store_propagation);
|
|
for (var id in this.trace.extended) {
|
|
var neuron = this.neighboors[id];
|
|
var influence = getVar('aux_2');
|
|
var neuron_old = getVar(neuron, 'old');
|
|
if (neuron.selfconnection.gater == this)
|
|
buildSentence(influence, ' = ', neuron_old, store_propagation);
|
|
else
|
|
buildSentence(influence, ' = 0', store_propagation);
|
|
for (var input in this.trace.influences[neuron.ID]) {
|
|
var connection = this.trace.influences[neuron.ID][input];
|
|
var connection_weight = getVar(connection, 'weight');
|
|
var neuron_activation = getVar(connection.from, 'activation');
|
|
buildSentence(influence, ' += ', connection_weight, ' * ',
|
|
neuron_activation, store_propagation);
|
|
}
|
|
var neuron_responsibility = getVar(neuron, 'error',
|
|
'responsibility', neuron.error.responsibility);
|
|
buildSentence(error, ' += ', neuron_responsibility, ' * ',
|
|
influence, store_propagation);
|
|
}
|
|
var gated = getVar(this, 'error', 'gated', this.error.gated);
|
|
buildSentence(gated, ' = ', derivative, ' * ', error,
|
|
store_propagation);
|
|
buildSentence(responsibility, ' = ', projected, ' + ', gated,
|
|
store_propagation);
|
|
for (var id in this.connections.inputs) {
|
|
var input = this.connections.inputs[id];
|
|
var gradient = getVar('aux');
|
|
var trace = getVar(this, 'trace', 'elegibility', input.ID, this
|
|
.trace.elegibility[input.ID]);
|
|
buildSentence(gradient, ' = ', projected, ' * ', trace,
|
|
store_propagation);
|
|
for (var id in this.trace.extended) {
|
|
var neuron = this.neighboors[id];
|
|
var neuron_responsibility = getVar(neuron, 'error',
|
|
'responsibility', neuron.error.responsibility);
|
|
var xtrace = getVar(this, 'trace', 'extended', neuron.ID,
|
|
input.ID, this.trace.extended[neuron.ID][input.ID]);
|
|
buildSentence(gradient, ' += ', neuron_responsibility, ' * ',
|
|
xtrace, store_propagation);
|
|
}
|
|
var input_weight = getVar(input, 'weight');
|
|
buildSentence(input_weight, ' += ', rate, ' * ', gradient,
|
|
store_propagation);
|
|
}
|
|
|
|
} else if (noGates) {
|
|
buildSentence(responsibility, ' = 0', store_propagation);
|
|
for (var id in this.connections.projected) {
|
|
var connection = this.connections.projected[id];
|
|
var neuron = connection.to;
|
|
var connection_weight = getVar(connection, 'weight');
|
|
var neuron_responsibility = getVar(neuron, 'error',
|
|
'responsibility', neuron.error.responsibility);
|
|
if (connection.gater) {
|
|
var connection_gain = getVar(connection, 'gain');
|
|
buildSentence(responsibility, ' += ', neuron_responsibility,
|
|
' * ', connection_gain, ' * ', connection_weight,
|
|
store_propagation);
|
|
} else
|
|
buildSentence(responsibility, ' += ', neuron_responsibility,
|
|
' * ', connection_weight, store_propagation);
|
|
}
|
|
buildSentence(responsibility, ' *= ', derivative,
|
|
store_propagation);
|
|
for (var id in this.connections.inputs) {
|
|
var input = this.connections.inputs[id];
|
|
var trace = getVar(this, 'trace', 'elegibility', input.ID, this
|
|
.trace.elegibility[input.ID]);
|
|
var input_weight = getVar(input, 'weight');
|
|
buildSentence(input_weight, ' += ', rate, ' * (',
|
|
responsibility, ' * ', trace, ')', store_propagation);
|
|
}
|
|
} else if (noProjections) {
|
|
buildSentence(responsibility, ' = 0', store_propagation);
|
|
for (var id in this.trace.extended) {
|
|
var neuron = this.neighboors[id];
|
|
var influence = getVar('aux');
|
|
var neuron_old = getVar(neuron, 'old');
|
|
if (neuron.selfconnection.gater == this)
|
|
buildSentence(influence, ' = ', neuron_old, store_propagation);
|
|
else
|
|
buildSentence(influence, ' = 0', store_propagation);
|
|
for (var input in this.trace.influences[neuron.ID]) {
|
|
var connection = this.trace.influences[neuron.ID][input];
|
|
var connection_weight = getVar(connection, 'weight');
|
|
var neuron_activation = getVar(connection.from, 'activation');
|
|
buildSentence(influence, ' += ', connection_weight, ' * ',
|
|
neuron_activation, store_propagation);
|
|
}
|
|
var neuron_responsibility = getVar(neuron, 'error',
|
|
'responsibility', neuron.error.responsibility);
|
|
buildSentence(responsibility, ' += ', neuron_responsibility,
|
|
' * ', influence, store_propagation);
|
|
}
|
|
buildSentence(responsibility, ' *= ', derivative,
|
|
store_propagation);
|
|
for (var id in this.connections.inputs) {
|
|
var input = this.connections.inputs[id];
|
|
var gradient = getVar('aux');
|
|
buildSentence(gradient, ' = 0', store_propagation);
|
|
for (var id in this.trace.extended) {
|
|
var neuron = this.neighboors[id];
|
|
var neuron_responsibility = getVar(neuron, 'error',
|
|
'responsibility', neuron.error.responsibility);
|
|
var xtrace = getVar(this, 'trace', 'extended', neuron.ID,
|
|
input.ID, this.trace.extended[neuron.ID][input.ID]);
|
|
buildSentence(gradient, ' += ', neuron_responsibility, ' * ',
|
|
xtrace, store_propagation);
|
|
}
|
|
var input_weight = getVar(input, 'weight');
|
|
buildSentence(input_weight, ' += ', rate, ' * ', gradient,
|
|
store_propagation);
|
|
}
|
|
}
|
|
}
|
|
buildSentence(bias, ' += ', rate, ' * ', responsibility,
|
|
store_propagation);
|
|
}
|
|
return {
|
|
memory: varID,
|
|
neurons: neurons + 1,
|
|
inputs: inputs,
|
|
outputs: outputs,
|
|
targets: targets,
|
|
variables: variables,
|
|
activation_sentences: activation_sentences,
|
|
trace_sentences: trace_sentences,
|
|
propagation_sentences: propagation_sentences,
|
|
layers: layers
|
|
}
|
|
}
|
|
}
|
|
|
|
|
|
// represents a connection between two neurons
|
|
Neuron.connection = function Connection(from, to, weight) {
|
|
|
|
if (!from || !to)
|
|
throw new Error("Connection Error: Invalid neurons");
|
|
|
|
this.ID = Neuron.connection.uid();
|
|
this.from = from;
|
|
this.to = to;
|
|
this.weight = typeof weight == 'undefined' ? Math.random() * .2 - .1 :
|
|
weight;
|
|
this.gain = 1;
|
|
this.gater = null;
|
|
}
|
|
|
|
|
|
// squashing functions
|
|
Neuron.squash = {};
|
|
|
|
// eq. 5 & 5'
|
|
Neuron.squash.LOGISTIC = function(x, derivate) {
|
|
if (!derivate)
|
|
return 1 / (1 + Math.exp(-x));
|
|
var fx = Neuron.squash.LOGISTIC(x);
|
|
return fx * (1 - fx);
|
|
};
|
|
Neuron.squash.TANH = function(x, derivate) {
|
|
if (derivate)
|
|
return 1 - Math.pow(Neuron.squash.TANH(x), 2);
|
|
var eP = Math.exp(x);
|
|
var eN = 1 / eP;
|
|
return (eP - eN) / (eP + eN);
|
|
};
|
|
Neuron.squash.IDENTITY = function(x, derivate) {
|
|
return derivate ? 1 : x;
|
|
};
|
|
Neuron.squash.HLIM = function(x, derivate) {
|
|
return derivate ? 1 : x > 0 ? 1 : 0;
|
|
};
|
|
Neuron.squash.RELU = function(x, derivate) {
|
|
if (derivate)
|
|
return x > 0 ? 1 : 0;
|
|
return x > 0 ? x : 0;
|
|
};
|
|
|
|
// unique ID's
|
|
(function() {
|
|
var neurons = 0;
|
|
var connections = 0;
|
|
Neuron.uid = function() {
|
|
return neurons++;
|
|
}
|
|
Neuron.connection.uid = function() {
|
|
return connections++;
|
|
}
|
|
Neuron.quantity = function() {
|
|
return {
|
|
neurons: neurons,
|
|
connections: connections
|
|
}
|
|
}
|
|
})();
|
|
|
|
/* WEBPACK VAR INJECTION */}.call(exports, __webpack_require__(2)(module)))
|
|
|
|
/***/ },
|
|
/* 2 */
|
|
/***/ function(module, exports) {
|
|
|
|
module.exports = function(module) {
|
|
if(!module.webpackPolyfill) {
|
|
module.deprecate = function() {};
|
|
module.paths = [];
|
|
// module.parent = undefined by default
|
|
module.children = [];
|
|
module.webpackPolyfill = 1;
|
|
}
|
|
return module;
|
|
}
|
|
|
|
|
|
/***/ },
|
|
/* 3 */
|
|
/***/ function(module, exports, __webpack_require__) {
|
|
|
|
/* WEBPACK VAR INJECTION */(function(module) {// export
|
|
if (module) module.exports = Layer;
|
|
|
|
// import
|
|
var Neuron = __webpack_require__(1)
|
|
, Network = __webpack_require__(4)
|
|
|
|
/*******************************************************************************************
|
|
LAYER
|
|
*******************************************************************************************/
|
|
|
|
function Layer(size, label) {
|
|
this.size = size | 0;
|
|
this.list = [];
|
|
this.label = label || null;
|
|
this.connectedTo = [];
|
|
|
|
while (size--) {
|
|
var neuron = new Neuron();
|
|
this.list.push(neuron);
|
|
}
|
|
}
|
|
|
|
Layer.prototype = {
|
|
|
|
// activates all the neurons in the layer
|
|
activate: function(input) {
|
|
|
|
var activations = [];
|
|
|
|
if (typeof input != 'undefined') {
|
|
if (input.length != this.size)
|
|
throw new Error("INPUT size and LAYER size must be the same to activate!");
|
|
|
|
for (var id in this.list) {
|
|
var neuron = this.list[id];
|
|
var activation = neuron.activate(input[id]);
|
|
activations.push(activation);
|
|
}
|
|
} else {
|
|
for (var id in this.list) {
|
|
var neuron = this.list[id];
|
|
var activation = neuron.activate();
|
|
activations.push(activation);
|
|
}
|
|
}
|
|
return activations;
|
|
},
|
|
|
|
// propagates the error on all the neurons of the layer
|
|
propagate: function(rate, target) {
|
|
|
|
if (typeof target != 'undefined') {
|
|
if (target.length != this.size)
|
|
throw new Error("TARGET size and LAYER size must be the same to propagate!");
|
|
|
|
for (var id = this.list.length - 1; id >= 0; id--) {
|
|
var neuron = this.list[id];
|
|
neuron.propagate(rate, target[id]);
|
|
}
|
|
} else {
|
|
for (var id = this.list.length - 1; id >= 0; id--) {
|
|
var neuron = this.list[id];
|
|
neuron.propagate(rate);
|
|
}
|
|
}
|
|
},
|
|
|
|
// projects a connection from this layer to another one
|
|
project: function(layer, type, weights) {
|
|
|
|
if (layer instanceof Network)
|
|
layer = layer.layers.input;
|
|
|
|
if (layer instanceof Layer) {
|
|
if (!this.connected(layer))
|
|
return new Layer.connection(this, layer, type, weights);
|
|
} else
|
|
throw new Error("Invalid argument, you can only project connections to LAYERS and NETWORKS!");
|
|
|
|
|
|
},
|
|
|
|
// gates a connection betwenn two layers
|
|
gate: function(connection, type) {
|
|
|
|
if (type == Layer.gateType.INPUT) {
|
|
if (connection.to.size != this.size)
|
|
throw new Error("GATER layer and CONNECTION.TO layer must be the same size in order to gate!");
|
|
|
|
for (var id in connection.to.list) {
|
|
var neuron = connection.to.list[id];
|
|
var gater = this.list[id];
|
|
for (var input in neuron.connections.inputs) {
|
|
var gated = neuron.connections.inputs[input];
|
|
if (gated.ID in connection.connections)
|
|
gater.gate(gated);
|
|
}
|
|
}
|
|
} else if (type == Layer.gateType.OUTPUT) {
|
|
if (connection.from.size != this.size)
|
|
throw new Error("GATER layer and CONNECTION.FROM layer must be the same size in order to gate!");
|
|
|
|
for (var id in connection.from.list) {
|
|
var neuron = connection.from.list[id];
|
|
var gater = this.list[id];
|
|
for (var projected in neuron.connections.projected) {
|
|
var gated = neuron.connections.projected[projected];
|
|
if (gated.ID in connection.connections)
|
|
gater.gate(gated);
|
|
}
|
|
}
|
|
} else if (type == Layer.gateType.ONE_TO_ONE) {
|
|
if (connection.size != this.size)
|
|
throw new Error("The number of GATER UNITS must be the same as the number of CONNECTIONS to gate!");
|
|
|
|
for (var id in connection.list) {
|
|
var gater = this.list[id];
|
|
var gated = connection.list[id];
|
|
gater.gate(gated);
|
|
}
|
|
}
|
|
connection.gatedfrom.push({layer: this, type: type});
|
|
},
|
|
|
|
// true or false whether the whole layer is self-connected or not
|
|
selfconnected: function() {
|
|
|
|
for (var id in this.list) {
|
|
var neuron = this.list[id];
|
|
if (!neuron.selfconnected())
|
|
return false;
|
|
}
|
|
return true;
|
|
},
|
|
|
|
// true of false whether the layer is connected to another layer (parameter) or not
|
|
connected: function(layer) {
|
|
// Check if ALL to ALL connection
|
|
var connections = 0;
|
|
for (var here in this.list) {
|
|
for (var there in layer.list) {
|
|
var from = this.list[here];
|
|
var to = layer.list[there];
|
|
var connected = from.connected(to);
|
|
if (connected.type == 'projected')
|
|
connections++;
|
|
}
|
|
}
|
|
if (connections == this.size * layer.size)
|
|
return Layer.connectionType.ALL_TO_ALL;
|
|
|
|
// Check if ONE to ONE connection
|
|
connections = 0;
|
|
for (var neuron in this.list) {
|
|
var from = this.list[neuron];
|
|
var to = layer.list[neuron];
|
|
var connected = from.connected(to);
|
|
if (connected.type == 'projected')
|
|
connections++;
|
|
}
|
|
if (connections == this.size)
|
|
return Layer.connectionType.ONE_TO_ONE;
|
|
},
|
|
|
|
// clears all the neuorns in the layer
|
|
clear: function() {
|
|
for (var id in this.list) {
|
|
var neuron = this.list[id];
|
|
neuron.clear();
|
|
}
|
|
},
|
|
|
|
// resets all the neurons in the layer
|
|
reset: function() {
|
|
for (var id in this.list) {
|
|
var neuron = this.list[id];
|
|
neuron.reset();
|
|
}
|
|
},
|
|
|
|
// returns all the neurons in the layer (array)
|
|
neurons: function() {
|
|
return this.list;
|
|
},
|
|
|
|
// adds a neuron to the layer
|
|
add: function(neuron) {
|
|
this.neurons[neuron.ID] = neuron || new Neuron();
|
|
this.list.push(neuron);
|
|
this.size++;
|
|
},
|
|
|
|
set: function(options) {
|
|
options = options || {};
|
|
|
|
for (var i in this.list) {
|
|
var neuron = this.list[i];
|
|
if (options.label)
|
|
neuron.label = options.label + '_' + neuron.ID;
|
|
if (options.squash)
|
|
neuron.squash = options.squash;
|
|
if (options.bias)
|
|
neuron.bias = options.bias;
|
|
}
|
|
return this;
|
|
}
|
|
}
|
|
|
|
// represents a connection from one layer to another, and keeps track of its weight and gain
|
|
Layer.connection = function LayerConnection(fromLayer, toLayer, type, weights) {
|
|
this.ID = Layer.connection.uid();
|
|
this.from = fromLayer;
|
|
this.to = toLayer;
|
|
this.selfconnection = toLayer == fromLayer;
|
|
this.type = type;
|
|
this.connections = {};
|
|
this.list = [];
|
|
this.size = 0;
|
|
this.gatedfrom = [];
|
|
|
|
if (typeof this.type == 'undefined')
|
|
{
|
|
if (fromLayer == toLayer)
|
|
this.type = Layer.connectionType.ONE_TO_ONE;
|
|
else
|
|
this.type = Layer.connectionType.ALL_TO_ALL;
|
|
}
|
|
|
|
if (this.type == Layer.connectionType.ALL_TO_ALL ||
|
|
this.type == Layer.connectionType.ALL_TO_ELSE) {
|
|
for (var here in this.from.list) {
|
|
for (var there in this.to.list) {
|
|
var from = this.from.list[here];
|
|
var to = this.to.list[there];
|
|
if(this.type == Layer.connectionType.ALL_TO_ELSE && from == to)
|
|
continue;
|
|
var connection = from.project(to, weights);
|
|
|
|
this.connections[connection.ID] = connection;
|
|
this.size = this.list.push(connection);
|
|
}
|
|
}
|
|
} else if (this.type == Layer.connectionType.ONE_TO_ONE) {
|
|
|
|
for (var neuron in this.from.list) {
|
|
var from = this.from.list[neuron];
|
|
var to = this.to.list[neuron];
|
|
var connection = from.project(to, weights);
|
|
|
|
this.connections[connection.ID] = connection;
|
|
this.size = this.list.push(connection);
|
|
}
|
|
}
|
|
|
|
fromLayer.connectedTo.push(this);
|
|
}
|
|
|
|
// types of connections
|
|
Layer.connectionType = {};
|
|
Layer.connectionType.ALL_TO_ALL = "ALL TO ALL";
|
|
Layer.connectionType.ONE_TO_ONE = "ONE TO ONE";
|
|
Layer.connectionType.ALL_TO_ELSE = "ALL TO ELSE";
|
|
|
|
// types of gates
|
|
Layer.gateType = {};
|
|
Layer.gateType.INPUT = "INPUT";
|
|
Layer.gateType.OUTPUT = "OUTPUT";
|
|
Layer.gateType.ONE_TO_ONE = "ONE TO ONE";
|
|
|
|
(function() {
|
|
var connections = 0;
|
|
Layer.connection.uid = function() {
|
|
return connections++;
|
|
}
|
|
})();
|
|
|
|
/* WEBPACK VAR INJECTION */}.call(exports, __webpack_require__(2)(module)))
|
|
|
|
/***/ },
|
|
/* 4 */
|
|
/***/ function(module, exports, __webpack_require__) {
|
|
|
|
/* WEBPACK VAR INJECTION */(function(module) {// export
|
|
if (module) module.exports = Network;
|
|
|
|
// import
|
|
var Neuron = __webpack_require__(1)
|
|
, Layer = __webpack_require__(3)
|
|
, Trainer = __webpack_require__(5)
|
|
|
|
/*******************************************************************************************
|
|
NETWORK
|
|
*******************************************************************************************/
|
|
|
|
function Network(layers) {
|
|
if (typeof layers != 'undefined') {
|
|
this.layers = layers || {
|
|
input: null,
|
|
hidden: {},
|
|
output: null
|
|
};
|
|
this.optimized = null;
|
|
}
|
|
}
|
|
Network.prototype = {
|
|
|
|
// feed-forward activation of all the layers to produce an ouput
|
|
activate: function(input) {
|
|
|
|
if (this.optimized === false)
|
|
{
|
|
this.layers.input.activate(input);
|
|
for (var layer in this.layers.hidden)
|
|
this.layers.hidden[layer].activate();
|
|
return this.layers.output.activate();
|
|
}
|
|
else
|
|
{
|
|
if (this.optimized == null)
|
|
this.optimize();
|
|
return this.optimized.activate(input);
|
|
}
|
|
},
|
|
|
|
// back-propagate the error thru the network
|
|
propagate: function(rate, target) {
|
|
|
|
if (this.optimized === false)
|
|
{
|
|
this.layers.output.propagate(rate, target);
|
|
var reverse = [];
|
|
for (var layer in this.layers.hidden)
|
|
reverse.push(this.layers.hidden[layer]);
|
|
reverse.reverse();
|
|
for (var layer in reverse)
|
|
reverse[layer].propagate(rate);
|
|
}
|
|
else
|
|
{
|
|
if (this.optimized == null)
|
|
this.optimize();
|
|
this.optimized.propagate(rate, target);
|
|
}
|
|
},
|
|
|
|
// project a connection to another unit (either a network or a layer)
|
|
project: function(unit, type, weights) {
|
|
|
|
if (this.optimized)
|
|
this.optimized.reset();
|
|
|
|
if (unit instanceof Network)
|
|
return this.layers.output.project(unit.layers.input, type, weights);
|
|
|
|
if (unit instanceof Layer)
|
|
return this.layers.output.project(unit, type, weights);
|
|
|
|
throw new Error("Invalid argument, you can only project connections to LAYERS and NETWORKS!");
|
|
},
|
|
|
|
// let this network gate a connection
|
|
gate: function(connection, type) {
|
|
if (this.optimized)
|
|
this.optimized.reset();
|
|
this.layers.output.gate(connection, type);
|
|
},
|
|
|
|
// clear all elegibility traces and extended elegibility traces (the network forgets its context, but not what was trained)
|
|
clear: function() {
|
|
|
|
this.restore();
|
|
|
|
var inputLayer = this.layers.input,
|
|
outputLayer = this.layers.output;
|
|
|
|
inputLayer.clear();
|
|
for (var layer in this.layers.hidden) {
|
|
var hiddenLayer = this.layers.hidden[layer];
|
|
hiddenLayer.clear();
|
|
}
|
|
outputLayer.clear();
|
|
|
|
if (this.optimized)
|
|
this.optimized.reset();
|
|
},
|
|
|
|
// reset all weights and clear all traces (ends up like a new network)
|
|
reset: function() {
|
|
|
|
this.restore();
|
|
|
|
var inputLayer = this.layers.input,
|
|
outputLayer = this.layers.output;
|
|
|
|
inputLayer.reset();
|
|
for (var layer in this.layers.hidden) {
|
|
var hiddenLayer = this.layers.hidden[layer];
|
|
hiddenLayer.reset();
|
|
}
|
|
outputLayer.reset();
|
|
|
|
if (this.optimized)
|
|
this.optimized.reset();
|
|
},
|
|
|
|
// hardcodes the behaviour of the whole network into a single optimized function
|
|
optimize: function() {
|
|
|
|
var that = this;
|
|
var optimized = {};
|
|
var neurons = this.neurons();
|
|
|
|
for (var i in neurons) {
|
|
var neuron = neurons[i].neuron;
|
|
var layer = neurons[i].layer;
|
|
while (neuron.neuron)
|
|
neuron = neuron.neuron;
|
|
optimized = neuron.optimize(optimized, layer);
|
|
}
|
|
for (var i in optimized.propagation_sentences)
|
|
optimized.propagation_sentences[i].reverse();
|
|
optimized.propagation_sentences.reverse();
|
|
|
|
var hardcode = "";
|
|
hardcode += "var F = Float64Array ? new Float64Array(" + optimized.memory +
|
|
") : []; ";
|
|
for (var i in optimized.variables)
|
|
hardcode += "F[" + optimized.variables[i].id + "] = " + (optimized.variables[
|
|
i].value || 0) + "; ";
|
|
hardcode += "var activate = function(input){\n";
|
|
for (var i in optimized.inputs)
|
|
hardcode += "F[" + optimized.inputs[i] + "] = input[" + i + "]; ";
|
|
for (var currentLayer in optimized.activation_sentences) {
|
|
if (optimized.activation_sentences[currentLayer].length > 0) {
|
|
for (var currentNeuron in optimized.activation_sentences[currentLayer]) {
|
|
hardcode += optimized.activation_sentences[currentLayer][currentNeuron].join(" ");
|
|
hardcode += optimized.trace_sentences[currentLayer][currentNeuron].join(" ");
|
|
}
|
|
}
|
|
}
|
|
hardcode += " var output = []; "
|
|
for (var i in optimized.outputs)
|
|
hardcode += "output[" + i + "] = F[" + optimized.outputs[i] + "]; ";
|
|
hardcode += "return output; }; "
|
|
hardcode += "var propagate = function(rate, target){\n";
|
|
hardcode += "F[" + optimized.variables.rate.id + "] = rate; ";
|
|
for (var i in optimized.targets)
|
|
hardcode += "F[" + optimized.targets[i] + "] = target[" + i + "]; ";
|
|
for (var currentLayer in optimized.propagation_sentences)
|
|
for (var currentNeuron in optimized.propagation_sentences[currentLayer])
|
|
hardcode += optimized.propagation_sentences[currentLayer][currentNeuron].join(" ") + " ";
|
|
hardcode += " };\n";
|
|
hardcode +=
|
|
"var ownership = function(memoryBuffer){\nF = memoryBuffer;\nthis.memory = F;\n};\n";
|
|
hardcode +=
|
|
"return {\nmemory: F,\nactivate: activate,\npropagate: propagate,\nownership: ownership\n};";
|
|
hardcode = hardcode.split(";").join(";\n");
|
|
|
|
var constructor = new Function(hardcode);
|
|
|
|
var network = constructor();
|
|
network.data = {
|
|
variables: optimized.variables,
|
|
activate: optimized.activation_sentences,
|
|
propagate: optimized.propagation_sentences,
|
|
trace: optimized.trace_sentences,
|
|
inputs: optimized.inputs,
|
|
outputs: optimized.outputs,
|
|
check_activation: this.activate,
|
|
check_propagation: this.propagate
|
|
}
|
|
|
|
network.reset = function() {
|
|
if (that.optimized) {
|
|
that.optimized = null;
|
|
that.activate = network.data.check_activation;
|
|
that.propagate = network.data.check_propagation;
|
|
}
|
|
}
|
|
|
|
this.optimized = network;
|
|
this.activate = network.activate;
|
|
this.propagate = network.propagate;
|
|
},
|
|
|
|
// restores all the values from the optimized network the their respective objects in order to manipulate the network
|
|
restore: function() {
|
|
if (!this.optimized)
|
|
return;
|
|
|
|
var optimized = this.optimized;
|
|
|
|
var getValue = function() {
|
|
var args = Array.prototype.slice.call(arguments);
|
|
|
|
var unit = args.shift();
|
|
var prop = args.pop();
|
|
|
|
var id = prop + '_';
|
|
for (var property in args)
|
|
id += args[property] + '_';
|
|
id += unit.ID;
|
|
|
|
var memory = optimized.memory;
|
|
var variables = optimized.data.variables;
|
|
|
|
if (id in variables)
|
|
return memory[variables[id].id];
|
|
return 0;
|
|
}
|
|
|
|
var list = this.neurons();
|
|
|
|
// link id's to positions in the array
|
|
var ids = {};
|
|
for (var i in list) {
|
|
var neuron = list[i].neuron;
|
|
while (neuron.neuron)
|
|
neuron = neuron.neuron;
|
|
|
|
neuron.state = getValue(neuron, 'state');
|
|
neuron.old = getValue(neuron, 'old');
|
|
neuron.activation = getValue(neuron, 'activation');
|
|
neuron.bias = getValue(neuron, 'bias');
|
|
|
|
for (var input in neuron.trace.elegibility)
|
|
neuron.trace.elegibility[input] = getValue(neuron, 'trace',
|
|
'elegibility', input);
|
|
|
|
for (var gated in neuron.trace.extended)
|
|
for (var input in neuron.trace.extended[gated])
|
|
neuron.trace.extended[gated][input] = getValue(neuron, 'trace',
|
|
'extended', gated, input);
|
|
}
|
|
|
|
// get connections
|
|
for (var i in list) {
|
|
var neuron = list[i].neuron;
|
|
while (neuron.neuron)
|
|
neuron = neuron.neuron;
|
|
|
|
for (var j in neuron.connections.projected) {
|
|
var connection = neuron.connections.projected[j];
|
|
connection.weight = getValue(connection, 'weight');
|
|
connection.gain = getValue(connection, 'gain');
|
|
}
|
|
}
|
|
},
|
|
|
|
// returns all the neurons in the network
|
|
neurons: function() {
|
|
|
|
var neurons = [];
|
|
|
|
var inputLayer = this.layers.input.neurons(),
|
|
outputLayer = this.layers.output.neurons();
|
|
|
|
for (var neuron in inputLayer)
|
|
neurons.push({
|
|
neuron: inputLayer[neuron],
|
|
layer: 'input'
|
|
});
|
|
|
|
for (var layer in this.layers.hidden) {
|
|
var hiddenLayer = this.layers.hidden[layer].neurons();
|
|
for (var neuron in hiddenLayer)
|
|
neurons.push({
|
|
neuron: hiddenLayer[neuron],
|
|
layer: layer
|
|
});
|
|
}
|
|
for (var neuron in outputLayer)
|
|
neurons.push({
|
|
neuron: outputLayer[neuron],
|
|
layer: 'output'
|
|
});
|
|
|
|
return neurons;
|
|
},
|
|
|
|
// returns number of inputs of the network
|
|
inputs: function() {
|
|
return this.layers.input.size;
|
|
},
|
|
|
|
// returns number of outputs of hte network
|
|
outputs: function() {
|
|
return this.layers.output.size;
|
|
},
|
|
|
|
// sets the layers of the network
|
|
set: function(layers) {
|
|
|
|
this.layers = layers;
|
|
if (this.optimized)
|
|
this.optimized.reset();
|
|
},
|
|
|
|
setOptimize: function(bool){
|
|
this.restore();
|
|
if (this.optimized)
|
|
this.optimized.reset();
|
|
this.optimized = bool? null : false;
|
|
},
|
|
|
|
// returns a json that represents all the neurons and connections of the network
|
|
toJSON: function(ignoreTraces) {
|
|
|
|
this.restore();
|
|
|
|
var list = this.neurons();
|
|
var neurons = [];
|
|
var connections = [];
|
|
|
|
// link id's to positions in the array
|
|
var ids = {};
|
|
for (var i in list) {
|
|
var neuron = list[i].neuron;
|
|
while (neuron.neuron)
|
|
neuron = neuron.neuron;
|
|
ids[neuron.ID] = i;
|
|
|
|
var copy = {
|
|
trace: {
|
|
elegibility: {},
|
|
extended: {}
|
|
},
|
|
state: neuron.state,
|
|
old: neuron.old,
|
|
activation: neuron.activation,
|
|
bias: neuron.bias,
|
|
layer: list[i].layer
|
|
};
|
|
|
|
copy.squash = neuron.squash == Neuron.squash.LOGISTIC ? "LOGISTIC" :
|
|
neuron.squash == Neuron.squash.TANH ? "TANH" :
|
|
neuron.squash == Neuron.squash.IDENTITY ? "IDENTITY" :
|
|
neuron.squash == Neuron.squash.HLIM ? "HLIM" :
|
|
null;
|
|
|
|
neurons.push(copy);
|
|
}
|
|
|
|
// get connections
|
|
for (var i in list) {
|
|
var neuron = list[i].neuron;
|
|
while (neuron.neuron)
|
|
neuron = neuron.neuron;
|
|
|
|
for (var j in neuron.connections.projected) {
|
|
var connection = neuron.connections.projected[j];
|
|
connections.push({
|
|
from: ids[connection.from.ID],
|
|
to: ids[connection.to.ID],
|
|
weight: connection.weight,
|
|
gater: connection.gater ? ids[connection.gater.ID] : null,
|
|
});
|
|
}
|
|
if (neuron.selfconnected())
|
|
connections.push({
|
|
from: ids[neuron.ID],
|
|
to: ids[neuron.ID],
|
|
weight: neuron.selfconnection.weight,
|
|
gater: neuron.selfconnection.gater ? ids[neuron.selfconnection.gater.ID] : null,
|
|
});
|
|
}
|
|
|
|
return {
|
|
neurons: neurons,
|
|
connections: connections
|
|
}
|
|
},
|
|
|
|
// export the topology into dot language which can be visualized as graphs using dot
|
|
/* example: ... console.log(net.toDotLang());
|
|
$ node example.js > example.dot
|
|
$ dot example.dot -Tpng > out.png
|
|
*/
|
|
toDot: function(edgeConnection) {
|
|
if (! typeof edgeConnection)
|
|
edgeConnection = false;
|
|
var code = "digraph nn {\n rankdir = BT\n";
|
|
var layers = [this.layers.input].concat(this.layers.hidden, this.layers.output);
|
|
for (var layer in layers) {
|
|
for (var to in layers[layer].connectedTo) { // projections
|
|
var connection = layers[layer].connectedTo[to];
|
|
var layerTo = connection.to;
|
|
var size = connection.size;
|
|
var layerID = layers.indexOf(layers[layer]);
|
|
var layerToID = layers.indexOf(layerTo);
|
|
/* http://stackoverflow.com/questions/26845540/connect-edges-with-graph-dot
|
|
* DOT does not support edge-to-edge connections
|
|
* This workaround produces somewhat weird graphs ...
|
|
*/
|
|
if ( edgeConnection) {
|
|
if (connection.gatedfrom.length) {
|
|
var fakeNode = "fake" + layerID + "_" + layerToID;
|
|
code += " " + fakeNode +
|
|
" [label = \"\", shape = point, width = 0.01, height = 0.01]\n";
|
|
code += " " + layerID + " -> " + fakeNode + " [label = " + size + ", arrowhead = none]\n";
|
|
code += " " + fakeNode + " -> " + layerToID + "\n";
|
|
} else
|
|
code += " " + layerID + " -> " + layerToID + " [label = " + size + "]\n";
|
|
for (var from in connection.gatedfrom) { // gatings
|
|
var layerfrom = connection.gatedfrom[from].layer;
|
|
var layerfromID = layers.indexOf(layerfrom);
|
|
code += " " + layerfromID + " -> " + fakeNode + " [color = blue]\n";
|
|
}
|
|
} else {
|
|
code += " " + layerID + " -> " + layerToID + " [label = " + size + "]\n";
|
|
for (var from in connection.gatedfrom) { // gatings
|
|
var layerfrom = connection.gatedfrom[from].layer;
|
|
var layerfromID = layers.indexOf(layerfrom);
|
|
code += " " + layerfromID + " -> " + layerToID + " [color = blue]\n";
|
|
}
|
|
}
|
|
}
|
|
}
|
|
code += "}\n";
|
|
return {
|
|
code: code,
|
|
link: "https://chart.googleapis.com/chart?chl=" + escape(code.replace("/ /g", "+")) + "&cht=gv"
|
|
}
|
|
},
|
|
|
|
// returns a function that works as the activation of the network and can be used without depending on the library
|
|
standalone: function() {
|
|
if (!this.optimized)
|
|
this.optimize();
|
|
|
|
var data = this.optimized.data;
|
|
|
|
// build activation function
|
|
var activation = "function (input) {\n";
|
|
|
|
// build inputs
|
|
for (var i in data.inputs)
|
|
activation += "F[" + data.inputs[i] + "] = input[" + i + "];\n";
|
|
|
|
// build network activation
|
|
for (var neuron in data.activate) { // shouldn't this be layer?
|
|
for (var sentence in data.activate[neuron])
|
|
activation += data.activate[neuron][sentence].join('') + "\n";
|
|
}
|
|
|
|
// build outputs
|
|
activation += "var output = [];\n";
|
|
for (var i in data.outputs)
|
|
activation += "output[" + i + "] = F[" + data.outputs[i] + "];\n";
|
|
activation += "return output;\n}";
|
|
|
|
// reference all the positions in memory
|
|
var memory = activation.match(/F\[(\d+)\]/g);
|
|
var dimension = 0;
|
|
var ids = {};
|
|
for (var address in memory) {
|
|
var tmp = memory[address].match(/\d+/)[0];
|
|
if (!(tmp in ids)) {
|
|
ids[tmp] = dimension++;
|
|
}
|
|
}
|
|
var hardcode = "F = {\n";
|
|
for (var i in ids)
|
|
hardcode += ids[i] + ": " + this.optimized.memory[i] + ",\n";
|
|
hardcode = hardcode.substring(0, hardcode.length - 2) + "\n};\n";
|
|
hardcode = "var run = " + activation.replace(/F\[(\d+)]/g, function(
|
|
index) {
|
|
return 'F[' + ids[index.match(/\d+/)[0]] + ']'
|
|
}).replace("{\n", "{\n" + hardcode + "") + ";\n";
|
|
hardcode += "return run";
|
|
|
|
// return standalone function
|
|
return new Function(hardcode)();
|
|
},
|
|
|
|
|
|
// Return a HTML5 WebWorker specialized on training the network stored in `memory`.
|
|
// Train based on the given dataSet and options.
|
|
// The worker returns the updated `memory` when done.
|
|
worker: function(memory, set, options) {
|
|
|
|
// Copy the options and set defaults (options might be different for each worker)
|
|
var workerOptions = {};
|
|
if(options) workerOptions = options;
|
|
workerOptions.rate = options.rate || .2;
|
|
workerOptions.iterations = options.iterations || 100000;
|
|
workerOptions.error = options.error || .005;
|
|
workerOptions.cost = options.cost || null;
|
|
workerOptions.crossValidate = options.crossValidate || null;
|
|
|
|
// Cost function might be different for each worker
|
|
costFunction = "var cost = " + (options && options.cost || this.cost || Trainer.cost.MSE) + ";\n";
|
|
var workerFunction = Network.getWorkerSharedFunctions();
|
|
workerFunction = workerFunction.replace(/var cost = options && options\.cost \|\| this\.cost \|\| Trainer\.cost\.MSE;/g, costFunction);
|
|
|
|
// Set what we do when training is finished
|
|
workerFunction = workerFunction.replace('return results;',
|
|
'postMessage({action: "done", message: results, memoryBuffer: F}, [F.buffer]);');
|
|
|
|
// Replace log with postmessage
|
|
workerFunction = workerFunction.replace("console.log('iterations', iterations, 'error', error, 'rate', currentRate)",
|
|
"postMessage({action: 'log', message: {\n" +
|
|
"iterations: iterations,\n" +
|
|
"error: error,\n" +
|
|
"rate: currentRate\n" +
|
|
"}\n" +
|
|
"})");
|
|
|
|
// Replace schedule with postmessage
|
|
workerFunction = workerFunction.replace("abort = this.schedule.do({ error: error, iterations: iterations, rate: currentRate })",
|
|
"postMessage({action: 'schedule', message: {\n" +
|
|
"iterations: iterations,\n" +
|
|
"error: error,\n" +
|
|
"rate: currentRate\n" +
|
|
"}\n" +
|
|
"})");
|
|
|
|
if (!this.optimized)
|
|
this.optimize();
|
|
|
|
var hardcode = "var inputs = " + this.optimized.data.inputs.length + ";\n";
|
|
hardcode += "var outputs = " + this.optimized.data.outputs.length + ";\n";
|
|
hardcode += "var F = new Float64Array([" + this.optimized.memory.toString() + "]);\n";
|
|
hardcode += "var activate = " + this.optimized.activate.toString() + ";\n";
|
|
hardcode += "var propagate = " + this.optimized.propagate.toString() + ";\n";
|
|
hardcode +=
|
|
"onmessage = function(e) {\n" +
|
|
"if (e.data.action == 'startTraining') {\n" +
|
|
"train(" + JSON.stringify(set) + "," + JSON.stringify(workerOptions) + ");\n" +
|
|
"}\n" +
|
|
"}";
|
|
|
|
var workerSourceCode = workerFunction + '\n' + hardcode;
|
|
var blob = new Blob([workerSourceCode]);
|
|
var blobURL = window.URL.createObjectURL(blob);
|
|
|
|
return new Worker(blobURL);
|
|
},
|
|
|
|
// returns a copy of the network
|
|
clone: function() {
|
|
return Network.fromJSON(this.toJSON());
|
|
}
|
|
};
|
|
|
|
/**
|
|
* Creates a static String to store the source code of the functions
|
|
* that are identical for all the workers (train, _trainSet, test)
|
|
*
|
|
* @return {String} Source code that can train a network inside a worker.
|
|
* @static
|
|
*/
|
|
Network.getWorkerSharedFunctions = function() {
|
|
// If we already computed the source code for the shared functions
|
|
if(typeof Network._SHARED_WORKER_FUNCTIONS !== 'undefined')
|
|
return Network._SHARED_WORKER_FUNCTIONS;
|
|
|
|
// Otherwise compute and return the source code
|
|
// We compute them by simply copying the source code of the train, _trainSet and test functions
|
|
// using the .toString() method
|
|
|
|
// Load and name the train function
|
|
var train_f = Trainer.prototype.train.toString();
|
|
train_f = train_f.replace('function (set', 'function train(set') + '\n';
|
|
|
|
// Load and name the _trainSet function
|
|
var _trainSet_f = Trainer.prototype._trainSet.toString().replace(/this.network./g, '');
|
|
_trainSet_f = _trainSet_f.replace('function (set', 'function _trainSet(set') + '\n';
|
|
_trainSet_f = _trainSet_f.replace('this.crossValidate', 'crossValidate');
|
|
_trainSet_f = _trainSet_f.replace('crossValidate = true', 'crossValidate = { }');
|
|
|
|
// Load and name the test function
|
|
var test_f = Trainer.prototype.test.toString().replace(/this.network./g, '');
|
|
test_f = test_f.replace('function (set', 'function test(set') + '\n';
|
|
|
|
return Network._SHARED_WORKER_FUNCTIONS = train_f + _trainSet_f + test_f;
|
|
};
|
|
|
|
// rebuild a network that has been stored in a json using the method toJSON()
|
|
Network.fromJSON = function(json) {
|
|
|
|
var neurons = [];
|
|
|
|
var layers = {
|
|
input: new Layer(),
|
|
hidden: [],
|
|
output: new Layer()
|
|
};
|
|
|
|
for (var i in json.neurons) {
|
|
var config = json.neurons[i];
|
|
|
|
var neuron = new Neuron();
|
|
neuron.trace.elegibility = {};
|
|
neuron.trace.extended = {};
|
|
neuron.state = config.state;
|
|
neuron.old = config.old;
|
|
neuron.activation = config.activation;
|
|
neuron.bias = config.bias;
|
|
neuron.squash = config.squash in Neuron.squash ? Neuron.squash[config.squash] : Neuron.squash.LOGISTIC;
|
|
neurons.push(neuron);
|
|
|
|
if (config.layer == 'input')
|
|
layers.input.add(neuron);
|
|
else if (config.layer == 'output')
|
|
layers.output.add(neuron);
|
|
else {
|
|
if (typeof layers.hidden[config.layer] == 'undefined')
|
|
layers.hidden[config.layer] = new Layer();
|
|
layers.hidden[config.layer].add(neuron);
|
|
}
|
|
}
|
|
|
|
for (var i in json.connections) {
|
|
var config = json.connections[i];
|
|
var from = neurons[config.from];
|
|
var to = neurons[config.to];
|
|
var weight = config.weight;
|
|
var gater = neurons[config.gater];
|
|
|
|
var connection = from.project(to, weight);
|
|
if (gater)
|
|
gater.gate(connection);
|
|
}
|
|
|
|
return new Network(layers);
|
|
};
|
|
|
|
/* WEBPACK VAR INJECTION */}.call(exports, __webpack_require__(2)(module)))
|
|
|
|
/***/ },
|
|
/* 5 */
|
|
/***/ function(module, exports, __webpack_require__) {
|
|
|
|
/* WEBPACK VAR INJECTION */(function(module) {// export
|
|
if (module) module.exports = Trainer;
|
|
|
|
/*******************************************************************************************
|
|
TRAINER
|
|
*******************************************************************************************/
|
|
|
|
function Trainer(network, options) {
|
|
options = options || {};
|
|
this.network = network;
|
|
this.rate = options.rate || .2;
|
|
this.iterations = options.iterations || 100000;
|
|
this.error = options.error || .005;
|
|
this.cost = options.cost || null;
|
|
this.crossValidate = options.crossValidate || null;
|
|
}
|
|
|
|
Trainer.prototype = {
|
|
|
|
// trains any given set to a network
|
|
train: function(set, options) {
|
|
|
|
var error = 1;
|
|
var iterations = bucketSize = 0;
|
|
var abort = false;
|
|
var currentRate;
|
|
var cost = options && options.cost || this.cost || Trainer.cost.MSE;
|
|
var crossValidate = false, testSet, trainSet;
|
|
|
|
var start = Date.now();
|
|
|
|
if (options) {
|
|
if (options.shuffle) {
|
|
//+ Jonas Raoni Soares Silva
|
|
//@ http://jsfromhell.com/array/shuffle [v1.0]
|
|
function shuffle(o) { //v1.0
|
|
for (var j, x, i = o.length; i; j = Math.floor(Math.random() * i), x = o[--i], o[i] = o[j], o[j] = x);
|
|
return o;
|
|
};
|
|
}
|
|
if (options.iterations)
|
|
this.iterations = options.iterations;
|
|
if (options.error)
|
|
this.error = options.error;
|
|
if (options.rate)
|
|
this.rate = options.rate;
|
|
if (options.cost)
|
|
this.cost = options.cost;
|
|
if (options.schedule)
|
|
this.schedule = options.schedule;
|
|
if (options.customLog){
|
|
// for backward compatibility with code that used customLog
|
|
console.log('Deprecated: use schedule instead of customLog')
|
|
this.schedule = options.customLog;
|
|
}
|
|
if (this.crossValidate || options.crossValidate) {
|
|
if(!this.crossValidate) this.crossValidate = {};
|
|
crossValidate = true;
|
|
if (options.crossValidate.testSize)
|
|
this.crossValidate.testSize = options.crossValidate.testSize;
|
|
if (options.crossValidate.testError)
|
|
this.crossValidate.testError = options.crossValidate.testError;
|
|
}
|
|
}
|
|
|
|
currentRate = this.rate;
|
|
if(Array.isArray(this.rate)) {
|
|
var bucketSize = Math.floor(this.iterations / this.rate.length);
|
|
}
|
|
|
|
if(crossValidate) {
|
|
var numTrain = Math.ceil((1 - this.crossValidate.testSize) * set.length);
|
|
trainSet = set.slice(0, numTrain);
|
|
testSet = set.slice(numTrain);
|
|
}
|
|
|
|
var lastError = 0;
|
|
while ((!abort && iterations < this.iterations && error > this.error)) {
|
|
if (crossValidate && error <= this.crossValidate.testError) {
|
|
break;
|
|
}
|
|
|
|
var currentSetSize = set.length;
|
|
error = 0;
|
|
iterations++;
|
|
|
|
if(bucketSize > 0) {
|
|
var currentBucket = Math.floor(iterations / bucketSize);
|
|
currentRate = this.rate[currentBucket] || currentRate;
|
|
}
|
|
|
|
if(typeof this.rate === 'function') {
|
|
currentRate = this.rate(iterations, lastError);
|
|
}
|
|
|
|
if (crossValidate) {
|
|
this._trainSet(trainSet, currentRate, cost);
|
|
error += this.test(testSet).error;
|
|
currentSetSize = 1;
|
|
} else {
|
|
error += this._trainSet(set, currentRate, cost);
|
|
currentSetSize = set.length;
|
|
}
|
|
|
|
// check error
|
|
error /= currentSetSize;
|
|
lastError = error;
|
|
|
|
if (options) {
|
|
if (this.schedule && this.schedule.every && iterations %
|
|
this.schedule.every == 0)
|
|
abort = this.schedule.do({ error: error, iterations: iterations, rate: currentRate });
|
|
else if (options.log && iterations % options.log == 0) {
|
|
console.log('iterations', iterations, 'error', error, 'rate', currentRate);
|
|
};
|
|
if (options.shuffle)
|
|
shuffle(set);
|
|
}
|
|
}
|
|
|
|
var results = {
|
|
error: error,
|
|
iterations: iterations,
|
|
time: Date.now() - start
|
|
};
|
|
|
|
return results;
|
|
},
|
|
|
|
// trains any given set to a network, using a WebWorker (only for the browser). Returns a Promise of the results.
|
|
trainAsync: function(set, options) {
|
|
var train = this.workerTrain.bind(this);
|
|
return new Promise(function(resolve, reject) {
|
|
try {
|
|
train(set, resolve, options, true)
|
|
} catch(e) {
|
|
reject(e)
|
|
}
|
|
})
|
|
},
|
|
|
|
// preforms one training epoch and returns the error (private function used in this.train)
|
|
_trainSet: function(set, currentRate, costFunction) {
|
|
var errorSum = 0;
|
|
for (var train in set) {
|
|
var input = set[train].input;
|
|
var target = set[train].output;
|
|
|
|
var output = this.network.activate(input);
|
|
this.network.propagate(currentRate, target);
|
|
|
|
errorSum += costFunction(target, output);
|
|
}
|
|
return errorSum;
|
|
},
|
|
|
|
// tests a set and returns the error and elapsed time
|
|
test: function(set, options) {
|
|
|
|
var error = 0;
|
|
var input, output, target;
|
|
var cost = options && options.cost || this.cost || Trainer.cost.MSE;
|
|
|
|
var start = Date.now();
|
|
|
|
for (var test in set) {
|
|
input = set[test].input;
|
|
target = set[test].output;
|
|
output = this.network.activate(input);
|
|
error += cost(target, output);
|
|
}
|
|
|
|
error /= set.length;
|
|
|
|
var results = {
|
|
error: error,
|
|
time: Date.now() - start
|
|
};
|
|
|
|
return results;
|
|
},
|
|
|
|
// trains any given set to a network using a WebWorker [deprecated: use trainAsync instead]
|
|
workerTrain: function(set, callback, options, suppressWarning) {
|
|
|
|
if (!suppressWarning) {
|
|
console.warn('Deprecated: do not use `workerTrain`, use `trainAsync` instead.')
|
|
}
|
|
var that = this;
|
|
|
|
if (!this.network.optimized)
|
|
this.network.optimize();
|
|
|
|
// Create a new worker
|
|
var worker = this.network.worker(this.network.optimized.memory, set, options);
|
|
|
|
// train the worker
|
|
worker.onmessage = function(e) {
|
|
switch(e.data.action) {
|
|
case 'done':
|
|
var iterations = e.data.message.iterations;
|
|
var error = e.data.message.error;
|
|
var time = e.data.message.time;
|
|
|
|
that.network.optimized.ownership(e.data.memoryBuffer);
|
|
|
|
// Done callback
|
|
callback({
|
|
error: error,
|
|
iterations: iterations,
|
|
time: time
|
|
});
|
|
|
|
// Delete the worker and all its associated memory
|
|
worker.terminate();
|
|
break;
|
|
|
|
case 'log':
|
|
console.log(e.data.message);
|
|
|
|
case 'schedule':
|
|
if (options && options.schedule && typeof options.schedule.do === 'function') {
|
|
var scheduled = options.schedule.do
|
|
scheduled(e.data.message)
|
|
}
|
|
break;
|
|
}
|
|
};
|
|
|
|
// Start the worker
|
|
worker.postMessage({action: 'startTraining'});
|
|
},
|
|
|
|
// trains an XOR to the network
|
|
XOR: function(options) {
|
|
|
|
if (this.network.inputs() != 2 || this.network.outputs() != 1)
|
|
throw new Error("Incompatible network (2 inputs, 1 output)");
|
|
|
|
var defaults = {
|
|
iterations: 100000,
|
|
log: false,
|
|
shuffle: true,
|
|
cost: Trainer.cost.MSE
|
|
};
|
|
|
|
if (options)
|
|
for (var i in options)
|
|
defaults[i] = options[i];
|
|
|
|
return this.train([{
|
|
input: [0, 0],
|
|
output: [0]
|
|
}, {
|
|
input: [1, 0],
|
|
output: [1]
|
|
}, {
|
|
input: [0, 1],
|
|
output: [1]
|
|
}, {
|
|
input: [1, 1],
|
|
output: [0]
|
|
}], defaults);
|
|
},
|
|
|
|
// trains the network to pass a Distracted Sequence Recall test
|
|
DSR: function(options) {
|
|
options = options || {};
|
|
|
|
var targets = options.targets || [2, 4, 7, 8];
|
|
var distractors = options.distractors || [3, 5, 6, 9];
|
|
var prompts = options.prompts || [0, 1];
|
|
var length = options.length || 24;
|
|
var criterion = options.success || 0.95;
|
|
var iterations = options.iterations || 100000;
|
|
var rate = options.rate || .1;
|
|
var log = options.log || 0;
|
|
var schedule = options.schedule || {};
|
|
var cost = options.cost || this.cost || Trainer.cost.CROSS_ENTROPY;
|
|
|
|
var trial, correct, i, j, success;
|
|
trial = correct = i = j = success = 0;
|
|
var error = 1,
|
|
symbols = targets.length + distractors.length + prompts.length;
|
|
|
|
var noRepeat = function(range, avoid) {
|
|
var number = Math.random() * range | 0;
|
|
var used = false;
|
|
for (var i in avoid)
|
|
if (number == avoid[i])
|
|
used = true;
|
|
return used ? noRepeat(range, avoid) : number;
|
|
};
|
|
|
|
var equal = function(prediction, output) {
|
|
for (var i in prediction)
|
|
if (Math.round(prediction[i]) != output[i])
|
|
return false;
|
|
return true;
|
|
};
|
|
|
|
var start = Date.now();
|
|
|
|
while (trial < iterations && (success < criterion || trial % 1000 != 0)) {
|
|
// generate sequence
|
|
var sequence = [],
|
|
sequenceLength = length - prompts.length;
|
|
for (i = 0; i < sequenceLength; i++) {
|
|
var any = Math.random() * distractors.length | 0;
|
|
sequence.push(distractors[any]);
|
|
}
|
|
var indexes = [],
|
|
positions = [];
|
|
for (i = 0; i < prompts.length; i++) {
|
|
indexes.push(Math.random() * targets.length | 0);
|
|
positions.push(noRepeat(sequenceLength, positions));
|
|
}
|
|
positions = positions.sort();
|
|
for (i = 0; i < prompts.length; i++) {
|
|
sequence[positions[i]] = targets[indexes[i]];
|
|
sequence.push(prompts[i]);
|
|
}
|
|
|
|
//train sequence
|
|
var distractorsCorrect;
|
|
var targetsCorrect = distractorsCorrect = 0;
|
|
error = 0;
|
|
for (i = 0; i < length; i++) {
|
|
// generate input from sequence
|
|
var input = [];
|
|
for (j = 0; j < symbols; j++)
|
|
input[j] = 0;
|
|
input[sequence[i]] = 1;
|
|
|
|
// generate target output
|
|
var output = [];
|
|
for (j = 0; j < targets.length; j++)
|
|
output[j] = 0;
|
|
|
|
if (i >= sequenceLength) {
|
|
var index = i - sequenceLength;
|
|
output[indexes[index]] = 1;
|
|
}
|
|
|
|
// check result
|
|
var prediction = this.network.activate(input);
|
|
|
|
if (equal(prediction, output))
|
|
if (i < sequenceLength)
|
|
distractorsCorrect++;
|
|
else
|
|
targetsCorrect++;
|
|
else {
|
|
this.network.propagate(rate, output);
|
|
}
|
|
|
|
error += cost(output, prediction);
|
|
|
|
if (distractorsCorrect + targetsCorrect == length)
|
|
correct++;
|
|
}
|
|
|
|
// calculate error
|
|
if (trial % 1000 == 0)
|
|
correct = 0;
|
|
trial++;
|
|
var divideError = trial % 1000;
|
|
divideError = divideError == 0 ? 1000 : divideError;
|
|
success = correct / divideError;
|
|
error /= length;
|
|
|
|
// log
|
|
if (log && trial % log == 0)
|
|
console.log("iterations:", trial, " success:", success, " correct:",
|
|
correct, " time:", Date.now() - start, " error:", error);
|
|
if (schedule.do && schedule.every && trial % schedule.every == 0)
|
|
schedule.do({
|
|
iterations: trial,
|
|
success: success,
|
|
error: error,
|
|
time: Date.now() - start,
|
|
correct: correct
|
|
});
|
|
}
|
|
|
|
return {
|
|
iterations: trial,
|
|
success: success,
|
|
error: error,
|
|
time: Date.now() - start
|
|
}
|
|
},
|
|
|
|
// train the network to learn an Embeded Reber Grammar
|
|
ERG: function(options) {
|
|
|
|
options = options || {};
|
|
var iterations = options.iterations || 150000;
|
|
var criterion = options.error || .05;
|
|
var rate = options.rate || .1;
|
|
var log = options.log || 500;
|
|
var cost = options.cost || this.cost || Trainer.cost.CROSS_ENTROPY;
|
|
|
|
// gramar node
|
|
var Node = function() {
|
|
this.paths = [];
|
|
};
|
|
Node.prototype = {
|
|
connect: function(node, value) {
|
|
this.paths.push({
|
|
node: node,
|
|
value: value
|
|
});
|
|
return this;
|
|
},
|
|
any: function() {
|
|
if (this.paths.length == 0)
|
|
return false;
|
|
var index = Math.random() * this.paths.length | 0;
|
|
return this.paths[index];
|
|
},
|
|
test: function(value) {
|
|
for (var i in this.paths)
|
|
if (this.paths[i].value == value)
|
|
return this.paths[i];
|
|
return false;
|
|
}
|
|
};
|
|
|
|
var reberGrammar = function() {
|
|
|
|
// build a reber grammar
|
|
var output = new Node();
|
|
var n1 = (new Node()).connect(output, "E");
|
|
var n2 = (new Node()).connect(n1, "S");
|
|
var n3 = (new Node()).connect(n1, "V").connect(n2, "P");
|
|
var n4 = (new Node()).connect(n2, "X");
|
|
n4.connect(n4, "S");
|
|
var n5 = (new Node()).connect(n3, "V");
|
|
n5.connect(n5, "T");
|
|
n2.connect(n5, "X");
|
|
var n6 = (new Node()).connect(n4, "T").connect(n5, "P");
|
|
var input = (new Node()).connect(n6, "B");
|
|
|
|
return {
|
|
input: input,
|
|
output: output
|
|
}
|
|
};
|
|
|
|
// build an embeded reber grammar
|
|
var embededReberGrammar = function() {
|
|
var reber1 = reberGrammar();
|
|
var reber2 = reberGrammar();
|
|
|
|
var output = new Node();
|
|
var n1 = (new Node).connect(output, "E");
|
|
reber1.output.connect(n1, "T");
|
|
reber2.output.connect(n1, "P");
|
|
var n2 = (new Node).connect(reber1.input, "P").connect(reber2.input,
|
|
"T");
|
|
var input = (new Node).connect(n2, "B");
|
|
|
|
return {
|
|
input: input,
|
|
output: output
|
|
}
|
|
|
|
};
|
|
|
|
// generate an ERG sequence
|
|
var generate = function() {
|
|
var node = embededReberGrammar().input;
|
|
var next = node.any();
|
|
var str = "";
|
|
while (next) {
|
|
str += next.value;
|
|
next = next.node.any();
|
|
}
|
|
return str;
|
|
};
|
|
|
|
// test if a string matches an embeded reber grammar
|
|
var test = function(str) {
|
|
var node = embededReberGrammar().input;
|
|
var i = 0;
|
|
var ch = str.charAt(i);
|
|
while (i < str.length) {
|
|
var next = node.test(ch);
|
|
if (!next)
|
|
return false;
|
|
node = next.node;
|
|
ch = str.charAt(++i);
|
|
}
|
|
return true;
|
|
};
|
|
|
|
// helper to check if the output and the target vectors match
|
|
var different = function(array1, array2) {
|
|
var max1 = 0;
|
|
var i1 = -1;
|
|
var max2 = 0;
|
|
var i2 = -1;
|
|
for (var i in array1) {
|
|
if (array1[i] > max1) {
|
|
max1 = array1[i];
|
|
i1 = i;
|
|
}
|
|
if (array2[i] > max2) {
|
|
max2 = array2[i];
|
|
i2 = i;
|
|
}
|
|
}
|
|
|
|
return i1 != i2;
|
|
};
|
|
|
|
var iteration = 0;
|
|
var error = 1;
|
|
var table = {
|
|
"B": 0,
|
|
"P": 1,
|
|
"T": 2,
|
|
"X": 3,
|
|
"S": 4,
|
|
"E": 5
|
|
};
|
|
|
|
var start = Date.now();
|
|
while (iteration < iterations && error > criterion) {
|
|
var i = 0;
|
|
error = 0;
|
|
|
|
// ERG sequence to learn
|
|
var sequence = generate();
|
|
|
|
// input
|
|
var read = sequence.charAt(i);
|
|
// target
|
|
var predict = sequence.charAt(i + 1);
|
|
|
|
// train
|
|
while (i < sequence.length - 1) {
|
|
var input = [];
|
|
var target = [];
|
|
for (var j = 0; j < 6; j++) {
|
|
input[j] = 0;
|
|
target[j] = 0;
|
|
}
|
|
input[table[read]] = 1;
|
|
target[table[predict]] = 1;
|
|
|
|
var output = this.network.activate(input);
|
|
|
|
if (different(output, target))
|
|
this.network.propagate(rate, target);
|
|
|
|
read = sequence.charAt(++i);
|
|
predict = sequence.charAt(i + 1);
|
|
|
|
error += cost(target, output);
|
|
}
|
|
error /= sequence.length;
|
|
iteration++;
|
|
if (iteration % log == 0) {
|
|
console.log("iterations:", iteration, " time:", Date.now() - start,
|
|
" error:", error);
|
|
}
|
|
}
|
|
|
|
return {
|
|
iterations: iteration,
|
|
error: error,
|
|
time: Date.now() - start,
|
|
test: test,
|
|
generate: generate
|
|
}
|
|
},
|
|
|
|
timingTask: function(options){
|
|
|
|
if (this.network.inputs() != 2 || this.network.outputs() != 1)
|
|
throw new Error("Invalid Network: must have 2 inputs and one output");
|
|
|
|
if (typeof options == 'undefined')
|
|
options = {};
|
|
|
|
// helper
|
|
function getSamples (trainingSize, testSize){
|
|
|
|
// sample size
|
|
var size = trainingSize + testSize;
|
|
|
|
// generate samples
|
|
var t = 0;
|
|
var set = [];
|
|
for (var i = 0; i < size; i++) {
|
|
set.push({ input: [0,0], output: [0] });
|
|
}
|
|
while(t < size - 20) {
|
|
var n = Math.round(Math.random() * 20);
|
|
set[t].input[0] = 1;
|
|
for (var j = t; j <= t + n; j++){
|
|
set[j].input[1] = n / 20;
|
|
set[j].output[0] = 0.5;
|
|
}
|
|
t += n;
|
|
n = Math.round(Math.random() * 20);
|
|
for (var k = t+1; k <= (t + n) && k < size; k++)
|
|
set[k].input[1] = set[t].input[1];
|
|
t += n;
|
|
}
|
|
|
|
// separate samples between train and test sets
|
|
var trainingSet = []; var testSet = [];
|
|
for (var l = 0; l < size; l++)
|
|
(l < trainingSize ? trainingSet : testSet).push(set[l]);
|
|
|
|
// return samples
|
|
return {
|
|
train: trainingSet,
|
|
test: testSet
|
|
}
|
|
}
|
|
|
|
var iterations = options.iterations || 200;
|
|
var error = options.error || .005;
|
|
var rate = options.rate || [.03, .02];
|
|
var log = options.log === false ? false : options.log || 10;
|
|
var cost = options.cost || this.cost || Trainer.cost.MSE;
|
|
var trainingSamples = options.trainSamples || 7000;
|
|
var testSamples = options.trainSamples || 1000;
|
|
|
|
// samples for training and testing
|
|
var samples = getSamples(trainingSamples, testSamples);
|
|
|
|
// train
|
|
var result = this.train(samples.train, {
|
|
rate: rate,
|
|
log: log,
|
|
iterations: iterations,
|
|
error: error,
|
|
cost: cost
|
|
});
|
|
|
|
return {
|
|
train: result,
|
|
test: this.test(samples.test)
|
|
}
|
|
}
|
|
};
|
|
|
|
// Built-in cost functions
|
|
Trainer.cost = {
|
|
// Eq. 9
|
|
CROSS_ENTROPY: function(target, output)
|
|
{
|
|
var crossentropy = 0;
|
|
for (var i in output)
|
|
crossentropy -= (target[i] * Math.log(output[i]+1e-15)) + ((1-target[i]) * Math.log((1+1e-15)-output[i])); // +1e-15 is a tiny push away to avoid Math.log(0)
|
|
return crossentropy;
|
|
},
|
|
MSE: function(target, output)
|
|
{
|
|
var mse = 0;
|
|
for (var i in output)
|
|
mse += Math.pow(target[i] - output[i], 2);
|
|
return mse / output.length;
|
|
},
|
|
BINARY: function(target, output){
|
|
var misses = 0;
|
|
for (var i in output)
|
|
misses += Math.round(target[i] * 2) != Math.round(output[i] * 2);
|
|
return misses;
|
|
}
|
|
}
|
|
|
|
/* WEBPACK VAR INJECTION */}.call(exports, __webpack_require__(2)(module)))
|
|
|
|
/***/ },
|
|
/* 6 */
|
|
/***/ function(module, exports, __webpack_require__) {
|
|
|
|
/* WEBPACK VAR INJECTION */(function(module) {// import
|
|
var Layer = __webpack_require__(3)
|
|
, Network = __webpack_require__(4)
|
|
, Trainer = __webpack_require__(5)
|
|
|
|
/*******************************************************************************************
|
|
ARCHITECT
|
|
*******************************************************************************************/
|
|
|
|
// Collection of useful built-in architectures
|
|
var Architect = {
|
|
|
|
// Multilayer Perceptron
|
|
Perceptron: function Perceptron() {
|
|
|
|
var args = Array.prototype.slice.call(arguments); // convert arguments to Array
|
|
if (args.length < 3)
|
|
throw new Error("not enough layers (minimum 3) !!");
|
|
|
|
var inputs = args.shift(); // first argument
|
|
var outputs = args.pop(); // last argument
|
|
var layers = args; // all the arguments in the middle
|
|
|
|
var input = new Layer(inputs);
|
|
var hidden = [];
|
|
var output = new Layer(outputs);
|
|
|
|
var previous = input;
|
|
|
|
// generate hidden layers
|
|
for (var level in layers) {
|
|
var size = layers[level];
|
|
var layer = new Layer(size);
|
|
hidden.push(layer);
|
|
previous.project(layer);
|
|
previous = layer;
|
|
}
|
|
previous.project(output);
|
|
|
|
// set layers of the neural network
|
|
this.set({
|
|
input: input,
|
|
hidden: hidden,
|
|
output: output
|
|
});
|
|
|
|
// trainer for the network
|
|
this.trainer = new Trainer(this);
|
|
},
|
|
|
|
// Multilayer Long Short-Term Memory
|
|
LSTM: function LSTM() {
|
|
|
|
var args = Array.prototype.slice.call(arguments); // convert arguments to array
|
|
if (args.length < 3)
|
|
throw new Error("not enough layers (minimum 3) !!");
|
|
|
|
var last = args.pop();
|
|
var option = {
|
|
peepholes: Layer.connectionType.ALL_TO_ALL,
|
|
hiddenToHidden: false,
|
|
outputToHidden: false,
|
|
outputToGates: false,
|
|
inputToOutput: true,
|
|
};
|
|
if (typeof last != 'number') {
|
|
var outputs = args.pop();
|
|
if (last.hasOwnProperty('peepholes'))
|
|
option.peepholes = last.peepholes;
|
|
if (last.hasOwnProperty('hiddenToHidden'))
|
|
option.hiddenToHidden = last.hiddenToHidden;
|
|
if (last.hasOwnProperty('outputToHidden'))
|
|
option.outputToHidden = last.outputToHidden;
|
|
if (last.hasOwnProperty('outputToGates'))
|
|
option.outputToGates = last.outputToGates;
|
|
if (last.hasOwnProperty('inputToOutput'))
|
|
option.inputToOutput = last.inputToOutput;
|
|
} else
|
|
var outputs = last;
|
|
|
|
var inputs = args.shift();
|
|
var layers = args;
|
|
|
|
var inputLayer = new Layer(inputs);
|
|
var hiddenLayers = [];
|
|
var outputLayer = new Layer(outputs);
|
|
|
|
var previous = null;
|
|
|
|
// generate layers
|
|
for (var layer in layers) {
|
|
// generate memory blocks (memory cell and respective gates)
|
|
var size = layers[layer];
|
|
|
|
var inputGate = new Layer(size).set({
|
|
bias: 1
|
|
});
|
|
var forgetGate = new Layer(size).set({
|
|
bias: 1
|
|
});
|
|
var memoryCell = new Layer(size);
|
|
var outputGate = new Layer(size).set({
|
|
bias: 1
|
|
});
|
|
|
|
hiddenLayers.push(inputGate);
|
|
hiddenLayers.push(forgetGate);
|
|
hiddenLayers.push(memoryCell);
|
|
hiddenLayers.push(outputGate);
|
|
|
|
// connections from input layer
|
|
var input = inputLayer.project(memoryCell);
|
|
inputLayer.project(inputGate);
|
|
inputLayer.project(forgetGate);
|
|
inputLayer.project(outputGate);
|
|
|
|
// connections from previous memory-block layer to this one
|
|
if (previous != null) {
|
|
var cell = previous.project(memoryCell);
|
|
previous.project(inputGate);
|
|
previous.project(forgetGate);
|
|
previous.project(outputGate);
|
|
}
|
|
|
|
// connections from memory cell
|
|
var output = memoryCell.project(outputLayer);
|
|
|
|
// self-connection
|
|
var self = memoryCell.project(memoryCell);
|
|
|
|
// hidden to hidden recurrent connection
|
|
if (option.hiddenToHidden)
|
|
memoryCell.project(memoryCell, Layer.connectionType.ALL_TO_ELSE);
|
|
|
|
// out to hidden recurrent connection
|
|
if (option.outputToHidden)
|
|
outputLayer.project(memoryCell);
|
|
|
|
// out to gates recurrent connection
|
|
if (option.outputToGates) {
|
|
outputLayer.project(inputGate);
|
|
outputLayer.project(outputGate);
|
|
outputLayer.project(forgetGate);
|
|
}
|
|
|
|
// peepholes
|
|
memoryCell.project(inputGate, option.peepholes);
|
|
memoryCell.project(forgetGate, option.peepholes);
|
|
memoryCell.project(outputGate, option.peepholes);
|
|
|
|
// gates
|
|
inputGate.gate(input, Layer.gateType.INPUT);
|
|
forgetGate.gate(self, Layer.gateType.ONE_TO_ONE);
|
|
outputGate.gate(output, Layer.gateType.OUTPUT);
|
|
if (previous != null)
|
|
inputGate.gate(cell, Layer.gateType.INPUT);
|
|
|
|
previous = memoryCell;
|
|
}
|
|
|
|
// input to output direct connection
|
|
if (option.inputToOutput)
|
|
inputLayer.project(outputLayer);
|
|
|
|
// set the layers of the neural network
|
|
this.set({
|
|
input: inputLayer,
|
|
hidden: hiddenLayers,
|
|
output: outputLayer
|
|
});
|
|
|
|
// trainer
|
|
this.trainer = new Trainer(this);
|
|
},
|
|
|
|
// Liquid State Machine
|
|
Liquid: function Liquid(inputs, hidden, outputs, connections, gates) {
|
|
|
|
// create layers
|
|
var inputLayer = new Layer(inputs);
|
|
var hiddenLayer = new Layer(hidden);
|
|
var outputLayer = new Layer(outputs);
|
|
|
|
// make connections and gates randomly among the neurons
|
|
var neurons = hiddenLayer.neurons();
|
|
var connectionList = [];
|
|
|
|
for (var i = 0; i < connections; i++) {
|
|
// connect two random neurons
|
|
var from = Math.random() * neurons.length | 0;
|
|
var to = Math.random() * neurons.length | 0;
|
|
var connection = neurons[from].project(neurons[to]);
|
|
connectionList.push(connection);
|
|
}
|
|
|
|
for (var j = 0; j < gates; j++) {
|
|
// pick a random gater neuron
|
|
var gater = Math.random() * neurons.length | 0;
|
|
// pick a random connection to gate
|
|
var connection = Math.random() * connectionList.length | 0;
|
|
// let the gater gate the connection
|
|
neurons[gater].gate(connectionList[connection]);
|
|
}
|
|
|
|
// connect the layers
|
|
inputLayer.project(hiddenLayer);
|
|
hiddenLayer.project(outputLayer);
|
|
|
|
// set the layers of the network
|
|
this.set({
|
|
input: inputLayer,
|
|
hidden: [hiddenLayer],
|
|
output: outputLayer
|
|
});
|
|
|
|
// trainer
|
|
this.trainer = new Trainer(this);
|
|
},
|
|
|
|
Hopfield: function Hopfield(size) {
|
|
|
|
var inputLayer = new Layer(size);
|
|
var outputLayer = new Layer(size);
|
|
|
|
inputLayer.project(outputLayer, Layer.connectionType.ALL_TO_ALL);
|
|
|
|
this.set({
|
|
input: inputLayer,
|
|
hidden: [],
|
|
output: outputLayer
|
|
});
|
|
|
|
var trainer = new Trainer(this);
|
|
|
|
var proto = Architect.Hopfield.prototype;
|
|
|
|
proto.learn = proto.learn || function(patterns)
|
|
{
|
|
var set = [];
|
|
for (var p in patterns)
|
|
set.push({
|
|
input: patterns[p],
|
|
output: patterns[p]
|
|
});
|
|
|
|
return trainer.train(set, {
|
|
iterations: 500000,
|
|
error: .00005,
|
|
rate: 1
|
|
});
|
|
};
|
|
|
|
proto.feed = proto.feed || function(pattern)
|
|
{
|
|
var output = this.activate(pattern);
|
|
|
|
var pattern = [];
|
|
for (var i in output)
|
|
pattern[i] = output[i] > .5 ? 1 : 0;
|
|
|
|
return pattern;
|
|
}
|
|
}
|
|
}
|
|
|
|
// Extend prototype chain (so every architectures is an instance of Network)
|
|
for (var architecture in Architect) {
|
|
Architect[architecture].prototype = new Network();
|
|
Architect[architecture].prototype.constructor = Architect[architecture];
|
|
}
|
|
|
|
// export
|
|
if (module) module.exports = Architect;
|
|
/* WEBPACK VAR INJECTION */}.call(exports, __webpack_require__(2)(module)))
|
|
|
|
/***/ }
|
|
/******/ ]); |