JavaScript: checking a gradient numerically
When you implement backpropagation by hand it is easy to make a mistake in a derivative, and the network may still appear to train. A gradient check compares the analytic gradient with a numerical estimate from the definition of the derivative.
JavaScript
const sigmoid = (z) => 1 / (1 + Math.exp(-z));
const x = 1.5;
const target = 1;
const b = -0.3;
function loss(w) {
const a = sigmoid(w * x + b);
return (a - target) ** 2;
}
function analyticGradient(w) {
const a = sigmoid(w * x + b);
return 2 * (a - target) * a * (1 - a) * x;
}
const w = 0.4;
const h = 1e-5;
const numeric = (loss(w + h) - loss(w - h)) / (2 * h);
const analytic = analyticGradient(w);
console.log("analytic: " + analytic.toFixed(8));
console.log("numeric: " + numeric.toFixed(8));
console.log("close enough: " + (Math.abs(analytic - numeric) < 1e-8));