Bài 1: Giải tích (Calculus) — Bộ não của Quá trình Học

import numpy as np

def loss(w, x=2.0, y=4.0):
    """MSE loss: L(w) = (y - w*x)^2"""
    return (y - w * x) ** 2

def grad(w, x=2.0, y=4.0):
    """Đạo hàm phân tích: dL/dw = -2x(y - wx)"""
    return -2 * x * (y - w * x)

# Kiểm tra bằng numerical gradient (xấp xỉ số)
def numerical_grad(w, eps=1e-5):
    return (loss(w + eps) - loss(w - eps)) / (2 * eps)

w = 1.0
print(f"Analytic  gradient: {grad(w):.6f}")       # → 8.000000
print(f"Numerical gradient: {numerical_grad(w):.6f}")  # → 8.000000 ✓

import torch

# PyTorch tự tính đạo hàm riêng qua autograd
x = torch.tensor([2.0, 3.0])
y = torch.tensor(10.0)
w = torch.tensor([1.0, 1.0], requires_grad=True)

loss = (y - (w * x).sum()) ** 2
loss.backward()

print(f"∂L/∂w1 = {w.grad[0]:.2f}")  # → -20.00
print(f"∂L/∂w2 = {w.grad[1]:.2f}")  # → -30.00

import numpy as np
import matplotlib.pyplot as plt

# Gradient Descent đơn giản
def gradient_descent(w_init=5.0, lr=0.1, n_steps=50):
    """
    Bài toán: minimize L(w) = w^2
    Gradient: dL/dw = 2w
    Optimal: w* = 0
    """
    w = w_init
    history = [w]
    
    for step in range(n_steps):
        grad = 2 * w          # đạo hàm L(w) = w²
        w = w - lr * grad     # cập nhật tham số
        history.append(w)
        
        if step % 10 == 0:
            print(f"Step {step:3d}: w={w:.4f}, L={w**2:.4f}")
    
    return history

history = gradient_descent()
# Step  0: w=4.0000, L=16.0000
# Step 10: w=0.4295, L=0.1845
# Step 20: w=0.0461, L=0.0021
# Step 30: w=0.0049, L=0.0000
# Step 40: w=0.0005, L=0.0000  → hội tụ về 0 ✓

import numpy as np

class TinyNet:
    """2-layer network với chain rule viết tay"""
    
    def __init__(self):
        np.random.seed(42)
        self.W1 = np.random.randn(3, 4) * 0.1  # (input=3, hidden=4)
        self.W2 = np.random.randn(4, 1) * 0.1  # (hidden=4, output=1)
    
    def relu(self, z):
        return np.maximum(0, z)
    
    def relu_grad(self, z):
        return (z > 0).astype(float)
    
    def forward(self, x):
        """x: shape (batch, 3)"""
        self.x  = x
        self.z1 = x @ self.W1            # (batch, 4)
        self.a1 = self.relu(self.z1)     # (batch, 4)
        self.z2 = self.a1 @ self.W2      # (batch, 1)
        return self.z2
    
    def backward(self, y):
        """MSE loss backward — chain rule viết tay"""
        batch = self.x.shape[0]
        
        # dL/dz2 = 2/N * (z2 - y)
        dz2 = 2 / batch * (self.z2 - y)          # (batch, 1)
        
        # dL/dW2 = a1.T @ dz2  [chain rule: ∂L/∂W2 = ∂L/∂z2 · ∂z2/∂W2]
        dW2 = self.a1.T @ dz2                     # (4, 1)
        
        # dL/da1 = dz2 @ W2.T  [chain rule: ∂L/∂a1 = ∂L/∂z2 · ∂z2/∂a1]
        da1 = dz2 @ self.W2.T                     # (batch, 4)
        
        # dL/dz1 = da1 * relu'(z1)  [chain rule qua ReLU]
        dz1 = da1 * self.relu_grad(self.z1)       # (batch, 4)
        
        # dL/dW1 = x.T @ dz1
        dW1 = self.x.T @ dz1                      # (3, 4)
        
        return dW1, dW2

# Demo
X = np.random.randn(10, 3)
y = np.random.randn(10, 1)

net = TinyNet()
pred = net.forward(X)
dW1, dW2 = net.backward(y)

print(f"dW1 shape: {dW1.shape}")  # (3, 4)
print(f"dW2 shape: {dW2.shape}")  # (4, 1)
print(f"Gradients computed via chain rule ✓")

# Minh họa vanishing gradient
import numpy as np

def sigmoid(x):
    return 1 / (1 + np.exp(-x))

def sigmoid_grad(x):
    s = sigmoid(x)
    return s * (1 - s)  # max = 0.25 tại x=0!

# Chain rule qua 10 layer sigmoid
gradient = 1.0
for layer in range(10):
    x = 0.0  # giả sử input = 0
    local_grad = sigmoid_grad(x)  # = 0.25
    gradient *= local_grad
    print(f"Layer {layer+1:2d}: gradient = {gradient:.2e}")

# Layer  1: gradient = 2.50e-01
# Layer  2: gradient = 6.25e-02
# Layer  5: gradient = 9.77e-04
# Layer 10: gradient = 9.54e-07  ← gần như = 0!

# Adam viết tay để hiểu cơ chế
def adam_step(grad, m, v, t, lr=1e-3, beta1=0.9, beta2=0.999, eps=1e-8):
    m = beta1 * m + (1 - beta1) * grad          # update 1st moment
    v = beta2 * v + (1 - beta2) * grad ** 2     # update 2nd moment
    
    m_hat = m / (1 - beta1 ** t)                # bias correction
    v_hat = v / (1 - beta2 ** t)                # bias correction
    
    update = lr * m_hat / (np.sqrt(v_hat) + eps)
    return update, m, v

# Demo
w, m, v = 5.0, 0.0, 0.0
for t in range(1, 101):
    grad = 2 * w  # gradient của L = w²
    update, m, v = adam_step(grad, m, v, t)
    w -= update
    if t % 20 == 0:
        print(f"Step {t:3d}: w={w:.4f}, L={w**2:.6f}")

# Step  20: w=4.9413, L=24.4165  ← Adam chậm hơn GD thuần ở đầu
# Step  40: w=4.6134, L=21.2835
# Step  60: w=3.9476, L=15.5836
# Step  80: w=2.9068, L=8.4495
# Step 100: w=1.6014, L=2.5645