C#: entropy and information gain for decision trees
A decision tree chooses the feature that best separates the labels. Entropy measures the impurity of a set: for a probability p of the positive class it is −p·log2(p) − (1 − p)·log2(1 − p). It is 0 for a pure set and 1 for an even split of two classes.
C#
using System;
using System.Globalization;
class Program {
static double Entropy(int yes, int no) {
double total = yes + no;
double result = 0;
foreach (int count in new[] { yes, no }) {
if (count > 0) {
double p = count / total;
result -= p * Math.Log2(p);
}
}
return result;
}
static string F(double v) {
return v.ToString("F3", CultureInfo.InvariantCulture);
}
static void Main() {
double whole = Entropy(9, 5);
int[][] outlook = { new[] { 2, 3 }, new[] { 4, 0 }, new[] { 3, 2 } };
double remainder = 0;
foreach (int[] group in outlook) {
remainder += (group[0] + group[1]) / 14.0 * Entropy(group[0], group[1]);
}
Console.WriteLine("entropy of the data: " + F(whole));
Console.WriteLine("entropy after splitting on Outlook: " + F(remainder));
Console.WriteLine("information gain: " + F(whole - remainder));
}
}