Measuring Error: Choosing a Loss Function

# Chapter 3: one number for "how wrong". Regression losses first, then probabilities and cross-entropy.
import math

# --- regression: the ride log from Chapter 1, plus one flat-tyre day ------------
rides_km  = [2, 5, 8, 12, 20]
rides_min = [11, 21, 28, 42, 75]            # the 20 km ride had a puncture


def predict(km, w=3, b=5):
    return w * km + b


def mae(errors):  return sum(abs(e) for e in errors) / len(errors)
def mse(errors):  return sum(e * e for e in errors) / len(errors)
def rmse(errors): return math.sqrt(mse(errors))


errors = [predict(x) - y for x, y in zip(rides_km, rides_min)]
print("errors:", errors)
for name, n in [("first 4 rides", 4), ("with flat tyre", 5)]:
    e = errors[:n]
    print(f"{name:>15}: MAE={mae(e):.3f}  MSE={mse(e):.3f}  RMSE={rmse(e):.3f}")
share = errors[4] ** 2 / sum(e * e for e in errors)
print(f"share of squared error from the flat-tyre ride: {share:.0%}")

# The best constant guess: mean for MSE, median for MAE
times = sorted(rides_min)
mean = sum(times) / len(times)
median = times[len(times) // 2]
print(f"\nbest constant under MSE = mean = {mean}, under MAE = median = {median}")

# --- classification: will the picnic go well? ------------------------------------
def sigmoid(z):
    return 1 / (1 + math.exp(-z))


def bce(p, y):
    """Binary cross-entropy for one example: the 'surprise' at what actually happened."""
    return -(y * math.log(p) + (1 - y) * math.log(1 - p))


w, b = [2.0, -3.0, -1.0], 0.5               # the picnic neuron from Chapter 2
history = [                                 # (sun, rain, wind) -> 1 = the picnic went well
    ([0.7, 0.2, 0.5], 1),
    ([0.1, 0.9, 0.3], 0),
    ([0.9, 0.0, 0.1], 1),
    ([0.4, 0.3, 0.9], 0),
    ([0.6, 0.1, 0.2], 1),
    ([0.5, 0.4, 0.4], 0),
]
print("\n  z       p(good)  label  loss")
total = 0.0
for x, y in history:
    z = sum(wi * xi for wi, xi in zip(w, x)) + b
    p = sigmoid(z)
    total += bce(p, y)
    print(f"{z:+.2f}    {p:.3f}     {y}     {bce(p, y):.3f}")
print(f"mean cross-entropy: {total / len(history):.4f}")

# --- confident and wrong: cross-entropy vs squared error ---------------------------
print("\np(good) when it WAS good | squared error | cross-entropy")
for p in [0.9, 0.5, 0.1, 0.01]:
    print(f"{p:>24} | {(1 - p) ** 2:>13.3f} | {bce(p, 1):>13.3f}")

# --- why not train on accuracy? Nudge the bias and watch both ----------------------
def scores(bias):
    acc, ce = 0, 0.0
    for x, y in history:
        p = sigmoid(sum(wi * xi for wi, xi in zip(w, x)) + bias)
        acc += int((p > 0.5) == bool(y))
        ce += bce(p, y)
    return acc / len(history), ce / len(history)

print("\nbias   accuracy   cross-entropy")
for bias in [0.3, 0.4, 0.5, 0.6, 0.7]:
    acc, ce = scores(bias)
    print(f"{bias:.1f}    {acc:.3f}      {ce:.4f}")