C: softmax and why the largest score is subtracted
Softmax turns a vector of scores into probabilities: each output is exp(z_i) divided by the sum of exp(z_j). The outputs are positive and sum to 1. A network usually ends in softmax when it must choose one of several classes.
C
#include <math.h>
#include <stdio.h>
static void softmax_naive(const double *z, double *out, int n) {
double total = 0.0;
for (int i = 0; i < n; i++) {
out[i] = exp(z[i]);
total += out[i];
}
for (int i = 0; i < n; i++) {
out[i] /= total;
}
}
static void softmax_stable(const double *z, double *out, int n) {
double largest = z[0];
for (int i = 1; i < n; i++) {
if (z[i] > largest) {
largest = z[i];
}
}
double total = 0.0;
for (int i = 0; i < n; i++) {
out[i] = exp(z[i] - largest);
total += out[i];
}
for (int i = 0; i < n; i++) {
out[i] /= total;
}
}
int main(void) {
double small[3] = {2.0, 1.0, 0.1};
double out[3];
softmax_stable(small, out, 3);
printf("stable: %.3f %.3f %.3f\n", out[0], out[1], out[2]);
softmax_naive(small, out, 3);
printf("naive: %.3f %.3f %.3f\n", out[0], out[1], out[2]);
double big[2] = {1000.0, 1001.0};
double big_out[2];
softmax_naive(big, big_out, 2);
printf("naive on large scores is NaN: %s\n", isnan(big_out[0]) ? "yes" : "no");
softmax_stable(big, big_out, 2);
printf("stable on large scores: %.3f %.3f\n", big_out[0], big_out[1]);
return 0;
}