import numpy as np
import matplotlib.pyplot as plt

def sigmoid(x):       return 1 / (1 + np.exp(-x))
def sigmoid_grad(x):  s = sigmoid(x); return s * (1 - s)
def tanh_grad(x):     return 1 - np.tanh(x) ** 2
def relu(x):           return np.maximum(0, x)
def relu_grad(x):      return (x > 0).astype(float)
def leaky_relu(x, a=0.01):      return np.where(x >= 0, x, a * x)
def leaky_relu_grad(x, a=0.01): return np.where(x >= 0, 1.0, a)

x = np.linspace(-6, 6, 400)

fig, axes = plt.subplots(2, 2, figsize=(10, 8))
axes[0,0].plot(x, sigmoid(x), label="sigmoid"); axes[0,0].plot(x, sigmoid_grad(x), '--', label="sigmoid'")
axes[0,1].plot(x, np.tanh(x), label="tanh");     axes[0,1].plot(x, tanh_grad(x), '--', label="tanh'")
axes[1,0].plot(x, relu(x), label="ReLU");        axes[1,0].plot(x, relu_grad(x), '--', label="ReLU'")
axes[1,1].plot(x, leaky_relu(x), label="Leaky ReLU"); axes[1,1].plot(x, leaky_relu_grad(x), '--', label="Leaky ReLU'")
for ax in axes.flat:
    ax.legend(); ax.axhline(0, color='gray', lw=0.5); ax.axvline(0, color='gray', lw=0.5)
plt.tight_layout()
plt.savefig("activations.png", dpi=120)
print("Saved activations.png")

# ---- numeric facts referenced in the lecture ----
print("\nsigmoid'(5)      =", round(sigmoid_grad(5), 4))       # ~0.0067
print("1 - tanh(4)^2    =", round(tanh_grad(4), 6))            # vanishing at large x

import math
def normal_cdf(x, mu=0, sigma=1):
    return 0.5 * (1 + math.erf((x - mu) / (sigma * math.sqrt(2))))

print("P(x<0), mu=0     =", round(normal_cdf(0, mu=0), 4))     # 0.5, standard normal symmetry
print("P(x<0), mu=-1    =", round(normal_cdf(0, mu=-1), 4))    # ~0.8413

# ---- Section 4: sigmoid vs tanh weight-update comparison ----
w = np.array([0.5, 0.5]); x_in = np.array([1, -1]); eta, g = 0.1, 1.0
z = w * x_in                              # elementwise local pre-activations
a_sig  = sigmoid(z)
a_tanh = np.tanh(z)
w_sig_new  = w - eta * g * a_sig
w_tanh_new = w - eta * g * a_tanh
print("\nSigmoid activations:", np.round(a_sig, 2), " -> updated w:", np.round(w_sig_new, 2))
print("Tanh activations:   ", np.round(a_tanh, 2), " -> updated w:", np.round(w_tanh_new, 2))
print("(Sigmoid: both weights move the SAME direction. Tanh: weights split in OPPOSITE directions.)")