C++: stable softmax and cross-entropy loss
A classifier network ends with a vector of raw scores called logits. Softmax turns them into probabilities that are positive and sum to 1: each probability is exp(z_i) divided by the sum of exp(z_j). The cross-entropy loss for the true class is the negative natural logarithm of the probability assigned to it, so it is small when the model is confident and right and large when it is confident and wrong.
C++
#include <algorithm>
#include <cmath>
#include <iomanip>
#include <iostream>
#include <vector>
std::vector<double> softmax(const std::vector<double>& z) {
double largest = *std::max_element(z.begin(), z.end());
std::vector<double> p(z.size());
double total = 0.0;
for (std::size_t i = 0; i < z.size(); ++i) {
p[i] = std::exp(z[i] - largest);
total += p[i];
}
for (double& v : p) {
v /= total;
}
return p;
}
int main() {
std::cout << std::fixed << std::setprecision(3);
std::vector<double> p = softmax({2.0, 1.0, 0.1});
std::cout << p[0] << " " << p[1] << " " << p[2] << "\n";
std::cout << "loss if class 0 is true: " << -std::log(p[0]) << "\n";
std::cout << "loss if class 2 is true: " << -std::log(p[2]) << "\n";
std::vector<double> big = softmax({1000.0, 1001.0});
std::cout << big[0] << " " << big[1] << "\n";
}