C#: activation functions and their ranges
An activation function adds non-linearity after a weighted sum. Without it, a stack of layers is just one linear function. The sigmoid maps to (0, 1) and is used for probabilities. The hyperbolic tangent maps to (−1, 1) and is centred on zero. ReLU, max(0, x), is cheap and does not saturate for positive inputs.
C#
using System;
using System.Globalization;
class Program {
static string F(double v) {
return v.ToString("F3", CultureInfo.InvariantCulture);
}
static void Main() {
Console.WriteLine("x sigmoid tanh relu");
foreach (double x in new[] { -2.0, 0.0, 2.0 }) {
double sigmoid = 1.0 / (1.0 + Math.Exp(-x));
double tanh = Math.Tanh(x);
double relu = Math.Max(0.0, x);
Console.WriteLine(F(x).PadLeft(6) + F(sigmoid).PadLeft(9) + F(tanh).PadLeft(8) + F(relu).PadLeft(8));
}
}
}