Evaluating classifiers: accuracy, precision, recall, F1
Machine learning builds models from data, so judging a model is as important as building it. For a binary classifier the confusion matrix counts four outcomes: true positives (predicted positive and actually positive), false positives, false negatives, and true negatives. Every common metric is computed from these four counts.
Python
def confusion(actual, predicted):
tp = sum(a == 1 and p == 1 for a, p in zip(actual, predicted))
fp = sum(a == 0 and p == 1 for a, p in zip(actual, predicted))
fn = sum(a == 1 and p == 0 for a, p in zip(actual, predicted))
tn = sum(a == 0 and p == 0 for a, p in zip(actual, predicted))
return tp, fp, fn, tn
def metrics(actual, predicted):
tp, fp, fn, tn = confusion(actual, predicted)
accuracy = (tp + tn) / len(actual)
precision = tp / (tp + fp) if tp + fp else 0.0
recall = tp / (tp + fn) if tp + fn else 0.0
f1 = 2 * precision * recall / (precision + recall) if precision + recall else 0.0
return accuracy, precision, recall, f1
actual = [1, 1, 1, 1, 0, 0, 0, 0, 0, 0]
predicted = [1, 1, 0, 0, 0, 0, 0, 0, 1, 0]
tp, fp, fn, tn = confusion(actual, predicted)
print(f"tp={tp} fp={fp} fn={fn} tn={tn}")
print(*[round(value, 3) for value in metrics(actual, predicted)])
rare = [1] * 10 + [0] * 990
always_negative = [0] * 1000
accuracy, precision, recall, f1 = metrics(rare, always_negative)
print(accuracy, recall)