I surveyed a number of implementations and was dismayed that I could not find a good one. I was even more dismayed by the number of blog posts and code articles that had deeply flawed implementations, where someone couldn't expect to get these implementations to work in test, much less production.
So... because I couldn't find a good implementation, I built my own library into my Big Data application. I am going to play the implementation details close to the chest, but I wanted to share one of my unit tests that I developed for the Backpropagation algorithm since it was time consuming to work out an example in Microsoft Excel. I could not find any other worked examples online, so I built an example by hand in Microsoft Excel for the following network:
I use both weights on the adjacencies and a threshold/bias factor for each node and I used the hyperbolic tangent function for the activation function. Below is the Visual Studio Unit Test showing the worked example. For simplicity, I round everything to 6 decimal points for test purposes.
[TestMethod]
public void TestBackpropagationMultilayerUntrained()
{
//Arrange
NeuralNetwork n = new NeuralNetwork("BackProp3LayerTest", "Back Propagation Test for 2 Processing Layers", new HyperbolicTangentActivationFunction(), 3, 3, 2, 3);
n.layers[0].Adjacencies[0][0].Weight = .75D;
n.layers[0].Adjacencies[1][0].Weight = -.75D;
n.layers[0].Adjacencies[2][0].Weight = .75D;
n.layers[0].Adjacencies[0][1].Weight = -.8D;
n.layers[0].Adjacencies[1][1].Weight = .8D;
n.layers[0].Adjacencies[2][1].Weight = -.9D;
n.layers[1].Nodes[0].Threshold = .5D;
n.layers[1].Nodes[1].Threshold = -.5D;
n.layers[1].Adjacencies[0][0].Weight = .4D;
n.layers[1].Adjacencies[0][1].Weight = -.5D;
n.layers[1].Adjacencies[0][2].Weight = .6D;
n.layers[1].Adjacencies[1][0].Weight = -.4D;
n.layers[1].Adjacencies[1][1].Weight = .5D;
n.layers[1].Adjacencies[1][2].Weight = -.6D;
n.layers[2].Nodes[0].Threshold = .1D;
n.layers[2].Nodes[1].Threshold = -.1D;
n.layers[2].Nodes[2].Threshold = 0D;
BackPropagationLearning l = new BackPropagationLearning(n);
double[] input = { .5D, -.5D, .75D };
double[] output = { 1D, -1D, 1D };
double error;
//Act
error = l.Step(new List<double>(input), new List<double>(output));
//Assert
//Total error E
Assert.AreEqual(.085699D, Math.Round(error, 6));
///////////////////////Verify Deltas///////////////////////////
//Input layer (0)
Assert.AreEqual( .018949D, Math.Round(l.deltas[0][0],6));
Assert.AreEqual(-.018949D, Math.Round(l.deltas[0][1], 6));
Assert.AreEqual( .019989D, Math.Round(l.deltas[0][2], 6));
//Hidden layer (1)
Assert.AreEqual( .014179D, Math.Round(l.deltas[1][0], 6));
Assert.AreEqual(-.010394D, Math.Round(l.deltas[1][1], 6));
//Output layer (2)
Assert.AreEqual( .154514D, Math.Round(l.deltas[2][0], 6));
Assert.AreEqual(-.083378D, Math.Round(l.deltas[2][1], 6));
Assert.AreEqual( .061175D, Math.Round(l.deltas[2][2], 6));
///////////////////////Verify Threshold Updates/////////////////
//No threshold updates for input layer
Assert.AreEqual(.0D, l.ThresholdUpdates[0][0]);
Assert.AreEqual(.0D, l.ThresholdUpdates[0][1]);
Assert.AreEqual(.0D, l.ThresholdUpdates[0][2]);
//Hidden Layer (1)
Assert.AreEqual( .001418D, Math.Round(l.ThresholdUpdates[1][0], 6));
Assert.AreEqual(-.001039D, Math.Round(l.ThresholdUpdates[1][1], 6));
//Output Layer (2)
Assert.AreEqual(.015451D, Math.Round(l.ThresholdUpdates[2][0], 6));
Assert.AreEqual(-.008338D, Math.Round(l.ThresholdUpdates[2][1], 6));
Assert.AreEqual(.006117D, Math.Round(l.ThresholdUpdates[2][2], 6));
///////////////////////Verify Weight Updates/////////////////
//Input layer (0) - Hidden Layer (1)
Assert.AreEqual(.000709D, Math.Round(l.WeightUpdates[0][0][0], 6));
Assert.AreEqual(-.000520D, Math.Round(l.WeightUpdates[0][0][1], 6));
Assert.AreEqual(-.000709D, Math.Round(l.WeightUpdates[0][1][0], 6));
Assert.AreEqual(.000520, Math.Round(l.WeightUpdates[0][1][1], 6));
Assert.AreEqual(.001063D, Math.Round(l.WeightUpdates[0][2][0], 6));
Assert.AreEqual(-.000780D, Math.Round(l.WeightUpdates[0][2][1], 6));
//Hidden Layer (1) - Output Layer (2)
Assert.AreEqual( .014649D, Math.Round(l.WeightUpdates[1][0][0], 6));
Assert.AreEqual(-.007905D, Math.Round(l.WeightUpdates[1][0][1], 6));
Assert.AreEqual( .005800D, Math.Round(l.WeightUpdates[1][0][2], 6));
Assert.AreEqual(-.014868D, Math.Round(l.WeightUpdates[1][1][0], 6));
Assert.AreEqual( .008023D, Math.Round(l.WeightUpdates[1][1][1], 6));
Assert.AreEqual(-.005886D, Math.Round(l.WeightUpdates[1][1][2], 6));
///////////////////////Verify Final Weights and Thresholds/////////////////
//Input layer (0) - Hidden Layer (1)
Assert.AreEqual(0.750709D, Math.Round(n.layers[0].Adjacencies[0][0].Weight, 6));
Assert.AreEqual(-0.800520D, Math.Round(n.layers[0].Adjacencies[0][1].Weight, 6));
Assert.AreEqual(-0.750709D, Math.Round(n.layers[0].Adjacencies[1][0].Weight, 6));
Assert.AreEqual(0.800520D, Math.Round(n.layers[0].Adjacencies[1][1].Weight, 6));
Assert.AreEqual(0.751063D, Math.Round(n.layers[0].Adjacencies[2][0].Weight, 6));
Assert.AreEqual(-0.900780D, Math.Round(n.layers[0].Adjacencies[2][1].Weight, 6));
//Hidden Layer (1)
Assert.AreEqual(0.501418D, Math.Round(n.layers[1].Nodes[0].Threshold, 6));
Assert.AreEqual(-0.501039D, Math.Round(n.layers[1].Nodes[1].Threshold, 6));
//Hidden Layer (1) - Output layer (2)
Assert.AreEqual(0.414649D, Math.Round(n.layers[1].Adjacencies[0][0].Weight, 6));
Assert.AreEqual(-0.507905D, Math.Round(n.layers[1].Adjacencies[0][1].Weight, 6));
Assert.AreEqual(0.605800D, Math.Round(n.layers[1].Adjacencies[0][2].Weight, 6));
Assert.AreEqual(-0.414868D, Math.Round(n.layers[1].Adjacencies[1][0].Weight, 6));
Assert.AreEqual(0.508023D, Math.Round(n.layers[1].Adjacencies[1][1].Weight, 6));
Assert.AreEqual(-0.605886D, Math.Round(n.layers[1].Adjacencies[1][2].Weight, 6));
//Output Layer (2)
Assert.AreEqual(0.115451D, Math.Round(n.layers[2].Nodes[0].Threshold, 6));
Assert.AreEqual(-0.108338D, Math.Round(n.layers[2].Nodes[1].Threshold, 6));
Assert.AreEqual(0.006117D, Math.Round(n.layers[2].Nodes[2].Threshold, 6));
///////////////////////Verify Consistent Movement/////////////////
double lasterr = error;
for (int i = 0; i < 10; i++)
{
lasterr = error;
error = l.Step(new List<double>(input), new List<double>(output));
Assert.IsTrue(lasterr >= error);
}
}
-->
