Which Way Is Downhill? Derivatives by Hand and by Nudging

# Chapter 4: which way is downhill? Derivatives by hand, checked numerically.
import math

rides_km  = [2, 5, 8, 12]
rides_min = [11, 21, 28, 42]
N = len(rides_km)


def loss(w, b):
    """Mean squared error of the line w*km + b on the ride log."""
    return sum((w * x + b - y) ** 2 for x, y in zip(rides_km, rides_min)) / N


def grad_by_hand(w, b):
    """dL/dw and dL/db, derived with the chain rule (see the maths section)."""
    errors = [w * x + b - y for x, y in zip(rides_km, rides_min)]
    dw = 2 / N * sum(e * x for e, x in zip(errors, rides_km))
    db = 2 / N * sum(errors)
    return dw, db


def grad_numeric(w, b, h=1e-5):
    """Central differences: nudge each knob up and down, measure the change."""
    dw = (loss(w + h, b) - loss(w - h, b)) / (2 * h)
    db = (loss(w, b + h) - loss(w, b - h)) / (2 * h)
    return dw, db


w, b = 3.0, 5.0
print(f"loss at w={w}, b={b}: {loss(w, b)}")
print("by hand :", [round(g, 6) for g in grad_by_hand(w, b)])
print("numeric :", [round(g, 6) for g in grad_numeric(w, b)])

# --- a gradient check, the way you'll use it in every later chapter -------------
def rel_error(a, b):
    return abs(a - b) / max(1e-12, abs(a) + abs(b))

for point in [(3.0, 5.0), (1.0, 0.0), (3.3, 4.1)]:
    exact, approx = grad_by_hand(*point), grad_numeric(*point)
    worst = max(rel_error(e, a) for e, a in zip(exact, approx))
    print(f"gradient check at {point}: worst relative error {worst:.1e}")

# --- forward vs central difference, and what happens when h gets too small ------
f = math.sin            # true derivative at x = 1 is cos(1)
x, true = 1.0, math.cos(1.0)
print("\n      h        forward error   central error")
for h in [1e-1, 1e-2, 1e-4, 1e-6, 1e-8, 1e-10, 1e-12]:
    fwd = (f(x + h) - f(x)) / h
    cen = (f(x + h) - f(x - h)) / (2 * h)
    print(f"{h:>8.0e}     {abs(fwd - true):.2e}        {abs(cen - true):.2e}")

# --- the gradient points uphill, so one small step against it lowers the loss -----
lr = 0.01
dw, db = grad_by_hand(w, b)
w2, b2 = w - lr * dw, b - lr * db
print(f"\none step downhill: ({w}, {b}) -> ({w2:.4f}, {b2:.4f}), loss {loss(w, b):.4f} -> {loss(w2, b2):.4f}")

# --- setting dL/db = 0 recovers Chapter 1's answer -------------------------------
w = 3.0
b_star = sum(y - w * x for x, y in zip(rides_km, rides_min)) / N
print(f"at w=3, dL/db = 0 when b = {b_star}; check: dL/db there = {grad_by_hand(w, b_star)[1]:.1e}")

# --- sigmoid: its derivative is s(1 - s) -------------------------------------------
def sigmoid(z): return 1 / (1 + math.exp(-z))
z = 0.8
s = sigmoid(z)
numeric = (sigmoid(z + 1e-5) - sigmoid(z - 1e-5)) / 2e-5
print(f"\nsigmoid'(0.8): formula s(1-s) = {s * (1 - s):.6f}, numeric = {numeric:.6f}")