AI & Machine Learning
with Python — From First Principles to Real Models

from collections import deque

# 0 = free cell, 1 = wall. The agent starts top-left and must reach bottom-right.
maze = [
    [0, 0, 1, 0, 0],
    [1, 0, 1, 0, 1],
    [0, 0, 0, 0, 0],
    [0, 1, 1, 1, 0],
    [0, 0, 0, 1, 0],
]
start, goal = (0, 0), (4, 4)

def solve(maze, start, goal):
    queue = deque([start])
    came_from = {start: None}
    while queue:
        r, c = queue.popleft()
        if (r, c) == goal:
            break
        for dr, dc in [(1, 0), (-1, 0), (0, 1), (0, -1)]:
            nr, nc = r + dr, c + dc
            inside = 0 <= nr < len(maze) and 0 <= nc < len(maze[0])
            if inside and maze[nr][nc] == 0 and (nr, nc) not in came_from:
                came_from[(nr, nc)] = (r, c)
                queue.append((nr, nc))
    if goal not in came_from:
        return None
    path, node = [], goal
    while node is not None:
        path.append(node)
        node = came_from[node]
    return path[::-1]

path = solve(maze, start, goal)
print("Shortest path length:", len(path) - 1, "steps")
print("Path:", path)

import numpy as np

a = np.array([2.0, 1.0, 3.0])
b = np.array([1.0, 0.0, -1.0])

print("a + b      =", a + b)
print("3 * a      =", 3 * a)
print("dot(a, b)  =", np.dot(a, b))
print("length |a| =", round(float(np.linalg.norm(a)), 4))

cos_sim = np.dot(a, b) / (np.linalg.norm(a) * np.linalg.norm(b))
print("cosine similarity =", round(float(cos_sim), 4))

import numpy as np

# 4 samples, 3 features each: [size_m2, bedrooms, age_years]
X = np.array([[ 50, 1, 20],
              [ 80, 2, 10],
              [120, 3,  5],
              [200, 4,  1]], dtype=float)

w = np.array([1.5, 10.0, -0.8])     # one weight per feature
b = 20.0

scores = X @ w + b                  # (4, 3) @ (3,) -> (4,)
print("X shape:", X.shape, "| w shape:", w.shape)
print("scores:", scores)

# A "layer" of 2 neurons: the weight matrix has shape (3 inputs, 2 outputs)
W = np.array([[ 0.01, -0.02],
              [ 0.50,  0.10],
              [-0.04,  0.03]])
layer_out = X @ W                   # (4, 3) @ (3, 2) -> (4, 2)
print("layer output shape:", layer_out.shape)
print(layer_out.round(2))

# f(x) = (x - 3)^2 has its lowest point at x = 3.
def f(x):
    return (x - 3) ** 2

def numerical_derivative(func, x, h=1e-6):
    return (func(x + h) - func(x - h)) / (2 * h)

def analytic_derivative(x):
    return 2 * (x - 3)

for x in [0.0, 2.0, 3.0, 5.0]:
    print(f"x={x:>4}  numeric={numerical_derivative(f, x):8.4f}  analytic={analytic_derivative(x):8.4f}")

# Gradient descent: repeatedly step AGAINST the slope
x = 10.0
learning_rate = 0.2
for step in range(11):
    if step % 2 == 0:
        print(f"step {step:2d}: x={x:.4f}  f(x)={f(x):.4f}")
    x -= learning_rate * analytic_derivative(x)

import numpy as np

def sigmoid(z):
    return 1 / (1 + np.exp(-z))

h = 1e-6
z = 0.7
s = sigmoid(z)
analytic = s * (1 - s)                                   # the formula used in the neuron program
numeric = (sigmoid(z + h) - sigmoid(z - h)) / (2 * h)
print("sigmoid(0.7)         =", round(float(s), 6))
print("analytic derivative  =", round(float(analytic), 6))
print("numerical derivative =", round(float(numeric), 6))

# Chain rule: d/dw sigmoid(w * x) = sigmoid'(w * x) * x
w, x = 0.5, 2.0
chain = sigmoid(w * x) * (1 - sigmoid(w * x)) * x
numeric_chain = (sigmoid((w + h) * x) - sigmoid((w - h) * x)) / (2 * h)
print("chain rule result    =", round(float(chain), 6))
print("numerical check      =", round(float(numeric_chain), 6))

import numpy as np

rng = np.random.default_rng(42)
heights = rng.normal(loc=165, scale=8, size=1000)      # simulated heights in cm

print("mean     :", round(float(heights.mean()), 2))
print("median   :", round(float(np.median(heights)), 2))
print("variance :", round(float(heights.var()), 2))
print("std dev  :", round(float(heights.std()), 2))

dist = np.abs(heights - heights.mean())
print("within 1 std:", round(float(np.mean(dist < heights.std())), 3), "(theory: about 0.683)")
print("within 2 std:", round(float(np.mean(dist < 2 * heights.std())), 3), "(theory: about 0.954)")

# Law of large numbers: the share of heads in fair coin flips approaches 0.5
flips = rng.integers(0, 2, size=100000)
for n in [10, 100, 1000, 100000]:
    print(f"{n:>6} flips -> proportion of heads = {flips[:n].mean():.4f}")

# Bayes' theorem: P(A|B) = P(B|A) * P(A) / P(B)
p_fraud = 0.01                 # prior: 1% of transactions are fraud
p_flag_given_fraud = 0.99      # the detector catches 99% of fraud (recall)
p_flag_given_ok = 0.05         # but it also flags 5% of honest transactions

p_flag = p_flag_given_fraud * p_fraud + p_flag_given_ok * (1 - p_fraud)
p_fraud_given_flag = p_flag_given_fraud * p_fraud / p_flag

print(f"P(flagged)         = {p_flag:.4f}")
print(f"P(fraud | flagged) = {p_fraud_given_flag:.4f}")
print("Out of 1,000 flagged transactions, about", round(1000 * p_fraud_given_flag), "are truly fraud.")

import numpy as np
import pandas as pd

rng = np.random.default_rng(7)
n = 200
df = pd.DataFrame({
    "age": rng.integers(18, 70, size=n),
    "income": rng.normal(35000, 12000, size=n).clip(5000).round(0),
    "city": rng.choice(["Manila", "Cebu", "Davao", "Zamboanga"], size=n),
    "plan": rng.choice(["free", "basic", "pro"], size=n, p=[0.5, 0.3, 0.2]),
})
df["churned"] = ((df["income"] < 30000) & (rng.random(n) < 0.6)).astype(int)
df.loc[rng.choice(n, 12, replace=False), "income"] = np.nan    # inject missing values

print(df.head())
print("\nShape:", df.shape)
print("\nMissing values per column:")
print(df.isna().sum())
print("\nNumeric summary:")
print(df.describe().round(1))
print("\nPlan counts:")
print(df["plan"].value_counts())
print("\nChurn rate by plan:")
print(df.groupby("plan")["churned"].mean().round(3))
print("\nCorrelation with churn:")
print(df[["age", "income", "churned"]].corr()["churned"].round(3))

import numpy as np
import pandas as pd
from sklearn.model_selection import train_test_split
from sklearn.preprocessing import StandardScaler

rng = np.random.default_rng(0)
df = pd.DataFrame({
    "income": rng.normal(35000, 12000, 8).round(0),
    "city": ["Manila", "Cebu", "Davao", "Cebu", "Manila", "Davao", "Cebu", "Manila"],
    "plan": ["free", "basic", "pro", "free", "pro", "basic", "free", "basic"],
})

# 1) Nominal categories (no natural order) -> one-hot encoding
encoded = pd.get_dummies(df, columns=["city"], dtype=int)

# 2) Ordinal categories (a real order exists) -> integers that respect the order
encoded["plan"] = encoded["plan"].map({"free": 0, "basic": 1, "pro": 2})
print(encoded)

# 3) Scaling: learn the mean/std from TRAIN only, then apply to TEST (prevents leakage)
train, test = train_test_split(encoded, test_size=0.25, random_state=1)
scaler = StandardScaler().fit(train[["income"]])
print("\nmean and std learned from train:", scaler.mean_.round(1), scaler.scale_.round(1))
print("scaled train mean:", round(float(scaler.transform(train[["income"]]).mean()), 3) + 0.0)
print("scaled test mean :", round(float(scaler.transform(test[["income"]]).mean()), 3), "(not exactly 0, and that is normal)")

import numpy as np
from sklearn.feature_selection import SelectKBest, f_classif
from sklearn.linear_model import LogisticRegression
from sklearn.model_selection import RepeatedStratifiedKFold, cross_val_score
from sklearn.pipeline import make_pipeline

rng = np.random.default_rng(0)
cv = RepeatedStratifiedKFold(n_splits=5, n_repeats=5, random_state=0)
X = rng.normal(size=(100, 2000))      # 2,000 features of pure noise
y = rng.integers(0, 2, size=100)      # random labels: there is NOTHING to learn

# WRONG: choose the "best" features using ALL the data, then cross-validate
X_leaky = SelectKBest(f_classif, k=20).fit_transform(X, y)
leaky = cross_val_score(LogisticRegression(max_iter=1000), X_leaky, y, cv=cv).mean()

# RIGHT: do the feature selection inside each training fold with a Pipeline
pipe = make_pipeline(SelectKBest(f_classif, k=20), LogisticRegression(max_iter=1000))
honest = cross_val_score(pipe, X, y, cv=cv).mean()

print(f"Leaky CV accuracy : {leaky:.2f}   <- looks impressive but is meaningless")
print(f"Honest CV accuracy: {honest:.2f}   <- near 0.50, as it must be for random labels")

import pandas as pd
import numpy as np
from sklearn.model_selection import train_test_split
from sklearn.preprocessing import StandardScaler
from sklearn.datasets import make_classification

# Simulate a raw dataset, the way you might receive it from a CSV export
X_raw, y_raw = make_classification(n_samples=500, n_features=6, n_informative=4, random_state=42)
data = pd.DataFrame(X_raw, columns=[f"feature_{i}" for i in range(6)])
data["target"] = y_raw

# Simulate a few missing values, just like real-world data
rng = np.random.default_rng(42)
missing_idx = rng.choice(data.index, size=10, replace=False)
data.loc[missing_idx, "feature_0"] = np.nan
data.fillna(data.mean(numeric_only=True), inplace=True)

X = data.drop("target", axis=1)
y = data["target"]

X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.2, random_state=42)

scaler = StandardScaler()
X_train_scaled = scaler.fit_transform(X_train)
X_test_scaled = scaler.transform(X_test)

print("Training samples:", X_train_scaled.shape[0])
print("Test samples:", X_test_scaled.shape[0])

from sklearn.ensemble import RandomForestClassifier

model = RandomForestClassifier(n_estimators=100, max_depth=10, random_state=42)
model.fit(X_train_scaled, y_train)
print("Model trained on", X_train_scaled.shape[0], "samples.")

from sklearn.metrics import classification_report, confusion_matrix

y_pred = model.predict(X_test_scaled)
print(classification_report(y_test, y_pred))
print("Confusion Matrix:", confusion_matrix(y_test, y_pred))

from sklearn.model_selection import GridSearchCV
from sklearn.ensemble import RandomForestClassifier

params = {"n_estimators": [50, 100, 200], "max_depth": [5, 10, 20]}
grid = GridSearchCV(RandomForestClassifier(random_state=42), params, cv=5)
grid.fit(X_train_scaled, y_train)
print("Best params:", grid.best_params_)

import joblib

joblib.dump(model, "model.pkl")
joblib.dump(scaler, "scaler.pkl")

loaded_model = joblib.load("model.pkl")
loaded_scaler = joblib.load("scaler.pkl")

# A new sample must have the same 6 features as the training data
new_sample = [[0.5, -0.3, 1.2, 0.1, -0.7, 0.4]]
new_prediction = loaded_model.predict(loaded_scaler.transform(new_sample))
print("Prediction for new sample:", new_prediction)

def chatbot_reply(message):
    text = message.lower().strip()

    if "hello" in text or "hi" in text:
        return "Hello! Ask me about Python or machine learning."
    elif "your name" in text:
        return "I'm PoeticBot, a rule-based chatbot with zero learning ability."
    elif "machine learning" in text:
        return "Machine learning is when a program learns patterns from data instead of being told exact rules."
    elif "bye" in text:
        return "Goodbye! Keep practising Python."
    else:
        return "I don't have a rule for that yet. Try 'hello', 'your name', or 'machine learning'."

# Simulate a short conversation
conversation = ["Hi there", "What is machine learning?", "What's your name?", "See you later"]

for user_message in conversation:
    print(f"User: {user_message}")
    print(f"Bot:  {chatbot_reply(user_message)}\n")

import numpy as np

# Training data: every combination of two binary inputs, and the correct AND output
X = np.array([[0, 0], [0, 1], [1, 0], [1, 1]])
y = np.array([0, 0, 0, 1])

# Start with random weights and a random bias — the neuron knows nothing yet
np.random.seed(1)
weights = np.random.uniform(-1, 1, size=2)
bias = np.random.uniform(-1, 1)
learning_rate = 0.1

def sigmoid(z):
    return 1 / (1 + np.exp(-z))

# Train for 2,000 passes over the tiny dataset
for epoch in range(2000):
    for inputs, target in zip(X, y):
        z = np.dot(inputs, weights) + bias
        prediction = sigmoid(z)

        error = target - prediction
        gradient = error * prediction * (1 - prediction)

        weights += learning_rate * gradient * inputs
        bias += learning_rate * gradient

# Test the trained neuron on the same four cases
print("Learned weights:", weights)
print("Learned bias:", bias)
for inputs in X:
    z = np.dot(inputs, weights) + bias
    output = sigmoid(z)
    print(f"{inputs} -> {output:.3f} -> rounded: {round(output)}")

import numpy as np

# Generate noisy data that roughly follows y = 3x + 5
np.random.seed(0)
X = np.linspace(0, 10, 40)
noise = np.random.normal(0, 2, size=X.shape)
y = 3 * X + 5 + noise

# Start with a bad guess for slope (m) and intercept (b)
m, b = 0.0, 0.0
learning_rate = 0.01
n = len(X)

for epoch in range(1000):
    y_pred = m * X + b
    error = y_pred - y

    # Gradients of the mean-squared-error loss with respect to m and b
    grad_m = (2 / n) * np.dot(error, X)
    grad_b = (2 / n) * np.sum(error)

    m -= learning_rate * grad_m
    b -= learning_rate * grad_b

    if epoch % 200 == 0:
        mse = np.mean(error ** 2)
        print(f"Epoch {epoch:4d} | m={m:.3f} b={b:.3f} | MSE={mse:.3f}")

print(f"\nFinal line: y = {m:.2f}x + {b:.2f}  (true relationship: y = 3x + 5)")

import numpy as np
from collections import Counter

# Labelled training points: 2D coordinates and a class (0 or 1)
X_train = np.array([[1, 2], [2, 3], [3, 3], [6, 5], [7, 7], [8, 6]])
y_train = np.array([0, 0, 0, 1, 1, 1])

def euclidean_distance(a, b):
    return np.sqrt(np.sum((a - b) ** 2))

def knn_predict(point, X_train, y_train, k=3):
    distances = [euclidean_distance(point, row) for row in X_train]
    nearest_indices = np.argsort(distances)[:k]
    nearest_labels = y_train[nearest_indices]
    vote = Counter(nearest_labels).most_common(1)[0][0]
    return vote

# Classify two brand-new, unlabelled points
new_points = np.array([[2, 2], [7, 6]])
for point in new_points:
    predicted_class = knn_predict(point, X_train, y_train, k=3)
    print(f"Point {point} -> predicted class {predicted_class}")

import numpy as np

np.random.seed(3)

# Two clusters of points: class 0 around (-1, -1) and class 1 around (+1, +1)
n = 100
X0 = np.random.randn(n, 2) + np.array([-1.0, -1.0])
X1 = np.random.randn(n, 2) + np.array([1.0, 1.0])
X = np.vstack([X0, X1])
y = np.array([0] * n + [1] * n)

def sigmoid(z):
    return 1 / (1 + np.exp(-z))

w = np.zeros(2)
b = 0.0
learning_rate = 0.1

for epoch in range(501):
    p = sigmoid(X @ w + b)                                        # predicted probabilities
    loss = -np.mean(y * np.log(p + 1e-9) + (1 - y) * np.log(1 - p + 1e-9))

    grad_w = X.T @ (p - y) / len(y)
    grad_b = np.mean(p - y)

    w -= learning_rate * grad_w
    b -= learning_rate * grad_b

    if epoch % 100 == 0:
        accuracy = np.mean((p >= 0.5) == y)
        print(f"epoch {epoch:3d} | loss={loss:.4f} | accuracy={accuracy:.3f}")

print("weights:", w.round(3), "| bias:", round(float(b), 3))
new_point = np.array([0.8, 1.2])
print("P(class 1) for", new_point, "=", round(float(sigmoid(new_point @ w + b)), 3))

import numpy as np

# Features: [hours_studied, hours_slept]    Label: 1 = passed, 0 = failed
X = np.array([[1, 8], [2, 6], [3, 9], [4, 7], [2, 4], [6, 8], [7, 6],
              [8, 7], [9, 9], [6, 3], [8, 4], [9, 2], [5, 6], [7, 5]], dtype=float)
y = np.array([0, 0, 0, 0, 0, 1, 1, 1, 1, 0, 0, 0, 1, 1])
feature_names = ["hours_studied", "hours_slept"]

def gini(labels):
    if len(labels) == 0:
        return 0.0
    p1 = np.mean(labels)
    return 1.0 - p1 ** 2 - (1 - p1) ** 2

def best_split(X, y):
    best = (None, None, 1.0)                    # (feature index, threshold, weighted gini)
    for f in range(X.shape[1]):
        for t in np.unique(X[:, f]):
            left, right = y[X[:, f] <= t], y[X[:, f] > t]
            if len(left) == 0 or len(right) == 0:
                continue
            score = (len(left) * gini(left) + len(right) * gini(right)) / len(y)
            if score < best[2]:
                best = (f, t, score)
    return best

def build_tree(X, y, depth=0, max_depth=2):
    if depth == max_depth or gini(y) == 0:
        return {"leaf": int(round(np.mean(y)))}
    f, t, _ = best_split(X, y)
    if f is None:
        return {"leaf": int(round(np.mean(y)))}
    mask = X[:, f] <= t
    return {"feature": f, "threshold": t,
            "left": build_tree(X[mask], y[mask], depth + 1, max_depth),
            "right": build_tree(X[~mask], y[~mask], depth + 1, max_depth)}

def predict(tree, row):
    while "leaf" not in tree:
        branch = "left" if row[tree["feature"]] <= tree["threshold"] else "right"
        tree = tree[branch]
    return tree["leaf"]

def show(tree, indent=""):
    if "leaf" in tree:
        print(indent + "->", "PASS" if tree["leaf"] else "FAIL")
        return
    name = feature_names[tree["feature"]]
    print(f"{indent}if {name} <= {tree['threshold']}:")
    show(tree["left"], indent + "    ")
    print(f"{indent}else:")
    show(tree["right"], indent + "    ")

tree = build_tree(X, y)
show(tree)
preds = np.array([predict(tree, row) for row in X])
print("training accuracy:", round(float(np.mean(preds == y)), 3))
print("prediction for [6 hours studied, 5 hours slept]:", predict(tree, np.array([6.0, 5.0])))

import numpy as np

np.random.seed(5)

# Three blobs of 40 points each
true_centers = np.array([[0, 0], [6, 6], [0, 7]])
X = np.vstack([np.random.randn(40, 2) + c for c in true_centers])

k = 3
centroids = X[np.random.choice(len(X), k, replace=False)]    # random starting centroids

for iteration in range(10):
    # 1) Assignment step: every point joins its nearest centroid
    distances = np.linalg.norm(X[:, None, :] - centroids[None, :, :], axis=2)
    labels = np.argmin(distances, axis=1)

    # 2) Update step: move each centroid to the mean of its points
    new_centroids = np.array([X[labels == j].mean(axis=0) for j in range(k)])
    shift = np.linalg.norm(new_centroids - centroids)
    centroids = new_centroids

    inertia = np.sum((X - centroids[labels]) ** 2)
    print(f"iteration {iteration + 1:2d} | centroid shift={shift:.4f} | inertia={inertia:.2f}")
    if shift < 1e-6:
        print("Converged.")
        break

print("Final centroids:")
print(centroids.round(2))
print("Cluster sizes:", np.bincount(labels))

import math
from collections import Counter, defaultdict

train = [
    ("win free money now", "spam"),
    ("free prize claim now", "spam"),
    ("limited offer win cash", "spam"),
    ("meeting at noon tomorrow", "ham"),
    ("lunch tomorrow with the team", "ham"),
    ("project report due today", "ham"),
]

class_counts = Counter(label for _, label in train)
word_counts = defaultdict(Counter)
for text, label in train:
    word_counts[label].update(text.split())

vocab = {w for counts in word_counts.values() for w in counts}

def predict(text):
    scores = {}
    for label in class_counts:
        score = math.log(class_counts[label] / len(train))            # log prior
        total_words = sum(word_counts[label].values())
        for word in text.split():
            # Laplace smoothing: add 1 so an unseen word never makes the probability 0
            p_word = (word_counts[label][word] + 1) / (total_words + len(vocab))
            score += math.log(p_word)
        scores[label] = score
    return max(scores, key=scores.get), scores

for message in ["free cash prize", "team meeting today", "win a free lunch"]:
    label, scores = predict(message)
    rounded = {k: round(v, 2) for k, v in scores.items()}
    print(f"{message!r:22} -> {label:4}  log-scores: {rounded}")

import numpy as np

np.random.seed(1)
X = np.array([[0, 0], [0, 1], [1, 0], [1, 1]], dtype=float)
y = np.array([[0], [1], [1], [0]], dtype=float)      # XOR: one neuron alone can never learn this

def sigmoid(z):
    return 1 / (1 + np.exp(-z))

hidden = 4
W1 = np.random.randn(2, hidden)
b1 = np.zeros((1, hidden))
W2 = np.random.randn(hidden, 1)
b2 = np.zeros((1, 1))
lr = 0.5

for epoch in range(5001):
    # ---- forward pass ----
    h = np.tanh(X @ W1 + b1)                  # hidden layer
    out = sigmoid(h @ W2 + b2)                # output layer
    loss = -np.mean(y * np.log(out + 1e-9) + (1 - y) * np.log(1 - out + 1e-9))

    # ---- backward pass (chain rule, from the output back to the input) ----
    d_z2 = (out - y) / len(X)                 # gradient at the output (log-loss + sigmoid)
    dW2 = h.T @ d_z2
    db2 = d_z2.sum(axis=0, keepdims=True)
    d_h = d_z2 @ W2.T
    d_z1 = d_h * (1 - h ** 2)                 # derivative of tanh
    dW1 = X.T @ d_z1
    db1 = d_z1.sum(axis=0, keepdims=True)

    # ---- gradient descent update ----
    W1 -= lr * dW1
    b1 -= lr * db1
    W2 -= lr * dW2
    b2 -= lr * db2

    if epoch % 1000 == 0:
        print(f"epoch {epoch:4d} | loss={loss:.4f}")

print("\nPredictions after training:")
for inputs, p in zip(X, out):
    print(f"{inputs.astype(int)} -> {p[0]:.3f} -> {int(p[0] > 0.5)}")

import warnings
import numpy as np

warnings.simplefilter("ignore")           # very high-degree fits trigger numerical warnings
rng = np.random.default_rng(1)

def make_data(n):
    x = np.sort(rng.uniform(0, 1, n))
    y = np.sin(2 * np.pi * x) + rng.normal(0, 0.25, n)     # true signal + noise
    return x, y

x_train, y_train = make_data(20)
x_test, y_test = make_data(200)

def rmse(y_true, y_pred):
    return float(np.sqrt(np.mean((y_true - y_pred) ** 2)))

print("degree | train RMSE | test RMSE")
for degree in [1, 3, 5, 9, 15]:
    coeffs = np.polyfit(x_train, y_train, degree)
    train_err = rmse(y_train, np.polyval(coeffs, x_train))
    test_err = rmse(y_test, np.polyval(coeffs, x_test))
    print(f"{degree:6d} | {train_err:10.3f} | {test_err:9.3f}")

import numpy as np
from sklearn.linear_model import LinearRegression, Ridge, Lasso
from sklearn.model_selection import train_test_split
from sklearn.metrics import r2_score

rng = np.random.default_rng(0)
n, p = 60, 40                                    # few samples, many features
X = rng.normal(size=(n, p))
true_w = np.zeros(p)
true_w[:5] = [3, -2, 1.5, 4, -1]                 # only 5 features actually matter
y = X @ true_w + rng.normal(0, 2.0, size=n)

X_tr, X_te, y_tr, y_te = train_test_split(X, y, test_size=0.4, random_state=0)

models = {
    "Linear (no regularisation)": LinearRegression(),
    "Ridge (L2, alpha=10)": Ridge(alpha=10),
    "Lasso (L1, alpha=0.3)": Lasso(alpha=0.3),
}
for name, m in models.items():
    m.fit(X_tr, y_tr)
    train_r2 = r2_score(y_tr, m.predict(X_tr))
    test_r2 = r2_score(y_te, m.predict(X_te))
    print(f"{name:28s} train R2={train_r2:5.2f}   test R2={test_r2:5.2f}")

lasso = models["Lasso (L1, alpha=0.3)"]
print("Features Lasso kept:", int(np.sum(lasso.coef_ != 0)), "of", p)

from sklearn.datasets import load_iris
from sklearn.ensemble import RandomForestClassifier
from sklearn.model_selection import cross_val_score

iris = load_iris()
X, y = iris.data, iris.target

model = RandomForestClassifier(n_estimators=50, random_state=42)
scores = cross_val_score(model, X, y, cv=5)
print(f"Mean CV accuracy: {scores.mean():.3f}")

model.fit(X, y)
print("Predictions for new sample:", model.predict([[5.1, 3.5, 1.4, 0.2]]))

from sklearn.datasets import load_diabetes
from sklearn.model_selection import train_test_split
from sklearn.ensemble import GradientBoostingRegressor
from sklearn.metrics import mean_squared_error, r2_score
import numpy as np

diabetes = load_diabetes()
X, y = diabetes.data, diabetes.target

X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.2, random_state=42)

model = GradientBoostingRegressor(n_estimators=100, learning_rate=0.1, random_state=42)
model.fit(X_train, y_train)

y_pred = model.predict(X_test)
print(f"RMSE: {np.sqrt(mean_squared_error(y_test, y_pred)):.3f}")
print(f"R²: {r2_score(y_test, y_pred):.3f}")

import numpy as np
from sklearn.linear_model import LinearRegression

rng = np.random.default_rng(2)
size = rng.uniform(30, 200, 150)
bedrooms = rng.integers(1, 5, 150)
price = 800 * size + 15000 * bedrooms + 20000 + rng.normal(0, 15000, 150)

X = np.column_stack([size, bedrooms])
model = LinearRegression().fit(X, price)

print("learned coefficients:", model.coef_.round(1), "(true values: [800, 15000])")
print("learned intercept   :", round(float(model.intercept_), 1), "(true value: 20000)")
print("R2 on training data :", round(model.score(X, price), 3))
print("Prediction for 100 m2, 3 bedrooms ->", round(float(model.predict([[100, 3]])[0])))

from sklearn.datasets import load_iris
from sklearn.tree import DecisionTreeClassifier, export_text

iris = load_iris()
tree = DecisionTreeClassifier(max_depth=2, random_state=0).fit(iris.data, iris.target)
print(export_text(tree, feature_names=list(iris.feature_names)))
print("classes:", ", ".join(iris.target_names))

from sklearn.datasets import load_breast_cancer
from sklearn.model_selection import cross_val_score
from sklearn.pipeline import make_pipeline
from sklearn.preprocessing import StandardScaler
from sklearn.linear_model import LogisticRegression
from sklearn.neighbors import KNeighborsClassifier
from sklearn.tree import DecisionTreeClassifier
from sklearn.ensemble import RandomForestClassifier
from sklearn.svm import SVC
from sklearn.naive_bayes import GaussianNB

X, y = load_breast_cancer(return_X_y=True)

models = {
    "Logistic Regression": make_pipeline(StandardScaler(), LogisticRegression(max_iter=1000)),
    "k-Nearest Neighbors": make_pipeline(StandardScaler(), KNeighborsClassifier(n_neighbors=5)),
    "Decision Tree": DecisionTreeClassifier(max_depth=4, random_state=0),
    "Random Forest": RandomForestClassifier(n_estimators=100, random_state=0),
    "SVM (RBF kernel)": make_pipeline(StandardScaler(), SVC()),
    "Gaussian Naive Bayes": GaussianNB(),
}

print(f"{'Model':22s} mean accuracy (5-fold CV)")
for name, model in models.items():
    scores = cross_val_score(model, X, y, cv=5)
    print(f"{name:22s} {scores.mean():.3f}  (+/- {scores.std():.3f})")

from sklearn.cluster import KMeans
from sklearn.datasets import make_blobs

X, _ = make_blobs(n_samples=300, centers=4, cluster_std=0.6, random_state=42)

kmeans = KMeans(n_clusters=4, random_state=42)
labels = kmeans.fit_predict(X)

print("Cluster centers:", kmeans.cluster_centers_)
print("Cluster labels:", labels[:10])

from sklearn.decomposition import PCA
from sklearn.datasets import load_digits

digits = load_digits()
X, y = digits.data, digits.target

pca = PCA(n_components=2)
X_2d = pca.fit_transform(X)

print("Explained variance ratio:", pca.explained_variance_ratio_)
print("Reduced shape:", X_2d.shape)

from sklearn.cluster import KMeans
from sklearn.datasets import make_blobs
from sklearn.metrics import silhouette_score

X, _ = make_blobs(n_samples=300, centers=4, cluster_std=0.7, random_state=42)

print("k | inertia  | silhouette")
for k in range(2, 8):
    km = KMeans(n_clusters=k, n_init=10, random_state=42).fit(X)
    print(f"{k} | {km.inertia_:8.1f} | {silhouette_score(X, km.labels_):.3f}")

from sklearn.cluster import KMeans, DBSCAN
from sklearn.datasets import make_moons
from sklearn.metrics import adjusted_rand_score

X, y_true = make_moons(n_samples=300, noise=0.07, random_state=0)

km_labels = KMeans(n_clusters=2, n_init=10, random_state=0).fit_predict(X)
db_labels = DBSCAN(eps=0.2, min_samples=5).fit_predict(X)

print("K-Means agreement with the true moons:", round(adjusted_rand_score(y_true, km_labels), 3))
print("DBSCAN  agreement with the true moons:", round(adjusted_rand_score(y_true, db_labels), 3))
print("DBSCAN clusters found:", len(set(db_labels) - {-1}), "| noise points:", int((db_labels == -1).sum()))

import numpy as np
from sklearn.ensemble import IsolationForest

rng = np.random.default_rng(0)
normal = rng.normal(0, 1, size=(300, 2))
outliers = rng.uniform(-6, 6, size=(10, 2))
X = np.vstack([normal, outliers])

iso = IsolationForest(contamination=10 / 310, random_state=0).fit(X)
flags = iso.predict(X)                    # -1 = anomaly, 1 = normal

print("Points flagged as anomalies:", int((flags == -1).sum()))
print("Of the 10 planted outliers, caught:", int((flags[-10:] == -1).sum()))

import pandas as pd
from sklearn.datasets import load_iris
from sklearn.model_selection import train_test_split
from sklearn.ensemble import RandomForestClassifier
from sklearn.metrics import confusion_matrix

iris = load_iris()
X_train, X_test, y_train, y_test = train_test_split(
    iris.data, iris.target, test_size=0.3, random_state=0, stratify=iris.target)

model = RandomForestClassifier(n_estimators=100, random_state=0)
model.fit(X_train, y_train)

y_pred = model.predict(X_test)
names = list(iris.target_names)
cm = pd.DataFrame(confusion_matrix(y_test, y_pred),
                  index=["true " + n for n in names],
                  columns=["pred " + n for n in names])
print(cm)

import numpy as np
from sklearn.metrics import precision_score, recall_score, f1_score

y_true = np.array([1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 1, 0])
y_pred = np.array([1, 1, 0, 1, 0, 0, 1, 0, 0, 0, 0, 0])

tp = int(np.sum((y_pred == 1) & (y_true == 1)))
fp = int(np.sum((y_pred == 1) & (y_true == 0)))
fn = int(np.sum((y_pred == 0) & (y_true == 1)))
tn = int(np.sum((y_pred == 0) & (y_true == 0)))
print("TP, FP, FN, TN =", tp, fp, fn, tn)

precision = tp / (tp + fp)
recall = tp / (tp + fn)
f1 = 2 * precision * recall / (precision + recall)
print(f"from scratch : precision={precision:.3f}  recall={recall:.3f}  f1={f1:.3f}")
print(f"scikit-learn : precision={precision_score(y_true, y_pred):.3f}  "
      f"recall={recall_score(y_true, y_pred):.3f}  f1={f1_score(y_true, y_pred):.3f}")

from sklearn.datasets import make_classification
from sklearn.dummy import DummyClassifier
from sklearn.ensemble import RandomForestClassifier
from sklearn.metrics import accuracy_score, precision_score, recall_score, f1_score, roc_auc_score
from sklearn.model_selection import train_test_split

X, y = make_classification(n_samples=5000, n_features=10, n_informative=5,
                           weights=[0.98, 0.02], flip_y=0, class_sep=1.0, random_state=0)
print("share of the positive class:", round(float(y.mean()), 3))

X_tr, X_te, y_tr, y_te = train_test_split(X, y, test_size=0.3, stratify=y, random_state=0)

def report(name, model):
    model.fit(X_tr, y_tr)
    pred = model.predict(X_te)
    auc = roc_auc_score(y_te, model.predict_proba(X_te)[:, 1])
    print(f"{name:16s} acc={accuracy_score(y_te, pred):.3f} precision={precision_score(y_te, pred, zero_division=0):.3f} "
          f"recall={recall_score(y_te, pred):.3f} f1={f1_score(y_te, pred):.3f} auc={auc:.3f}")

report("Always say 'no'", DummyClassifier(strategy="most_frequent"))
report("Random Forest", RandomForestClassifier(n_estimators=100, random_state=0))
report("RF (balanced)", RandomForestClassifier(n_estimators=100, class_weight="balanced", random_state=0))

import numpy as np

z = np.array([-3.0, -1.0, 0.0, 1.0, 3.0])
sigmoid = 1 / (1 + np.exp(-z))
tanh = np.tanh(z)
relu = np.maximum(0, z)
leaky = np.where(z > 0, z, 0.01 * z)

print("z         :", z)
print("sigmoid   :", sigmoid.round(3))
print("tanh      :", tanh.round(3))
print("ReLU      :", relu)
print("LeakyReLU :", leaky.round(2))

# softmax turns raw scores (logits) into probabilities that add up to 1
logits = np.array([2.0, 1.0, 0.1])
exps = np.exp(logits - logits.max())          # subtracting the max keeps the numbers stable
softmax = exps / exps.sum()
print("softmax   :", softmax.round(3), "| sum =", round(float(softmax.sum()), 3))

from sklearn.datasets import load_digits
from sklearn.model_selection import train_test_split
from sklearn.neural_network import MLPClassifier
from sklearn.pipeline import make_pipeline
from sklearn.preprocessing import StandardScaler

digits = load_digits()
X_tr, X_te, y_tr, y_te = train_test_split(
    digits.data, digits.target, test_size=0.25, random_state=0, stratify=digits.target)

net = make_pipeline(StandardScaler(),
                    MLPClassifier(hidden_layer_sizes=(64, 32), max_iter=500, random_state=0))
net.fit(X_tr, y_tr)

print("Test accuracy:", round(net.score(X_te, y_te), 3))
print("Weight matrix shapes:", [w.shape for w in net[-1].coefs_])
print("Predicted:", net.predict(X_te[:10]))
print("True     :", y_te[:10])

from collections import Counter
import numpy as np

docs = ["the cat sat on the mat", "the dog sat on the log", "stock prices fell sharply today"]
tokens = [d.split() for d in docs]
vocab = sorted({w for t in tokens for w in t})
print("vocabulary:", vocab)

vectors = []
for d, t in zip(docs, tokens):
    counts = Counter(t)
    vec = np.array([counts[w] for w in vocab])
    vectors.append(vec)
    print(f"{d!r:36} -> {vec}")

def cosine(u, v):
    return float(u @ v / (np.linalg.norm(u) * np.linalg.norm(v)))

print("\ncosine(cat-doc, dog-doc)   =", round(cosine(vectors[0], vectors[1]), 3))
print("cosine(cat-doc, stock-doc) =", round(cosine(vectors[0], vectors[2]), 3))

import numpy as np
from sklearn.feature_extraction.text import TfidfVectorizer
from sklearn.linear_model import LogisticRegression
from sklearn.pipeline import make_pipeline

reviews = [
    ("great movie loved the acting", 1), ("fantastic story and wonderful music", 1),
    ("absolutely brilliant and moving", 1), ("a joy to watch loved it", 1),
    ("really enjoyable and funny", 1), ("wonderful performances great fun", 1),
    ("terrible plot and bad acting", 0), ("boring and far too long", 0),
    ("awful script i hated it", 0), ("a waste of time terrible", 0),
    ("dull boring and predictable", 0), ("bad movie awful ending", 0),
]
texts, labels = zip(*reviews)

model = make_pipeline(TfidfVectorizer(), LogisticRegression(C=10))
model.fit(texts, labels)

tests = ["wonderful acting loved it", "boring and terrible", "the plot was great fun"]
for t, p in zip(tests, model.predict_proba(tests)[:, 1]):
    print(f"{t!r:30} P(positive) = {p:.2f}")

vec, clf = model[0], model[1]
vocab = np.array(vec.get_feature_names_out())
order = np.argsort(clf.coef_[0])
print("most negative words:", list(vocab[order[:4]]))
print("most positive words:", list(vocab[order[-4:]]))

import numpy as np

def softmax(x):
    e = np.exp(x - x.max(axis=-1, keepdims=True))
    return e / e.sum(axis=-1, keepdims=True)

np.random.seed(0)
tokens = ["the", "cat", "sat"]
d = 4                                      # tiny embedding size
X = np.random.randn(len(tokens), d)        # one vector per token

Wq, Wk, Wv = (np.random.randn(d, d) for _ in range(3))
Q, K, V = X @ Wq, X @ Wk, X @ Wv

scores = Q @ K.T / np.sqrt(d)              # how strongly each token "looks at" every other token
weights = softmax(scores)                  # each row sums to 1
output = weights @ V                       # every token becomes a weighted mix of the value vectors

print("attention weights (row = token looking, column = token looked at):")
for tok, row in zip(tokens, weights.round(2)):
    print(f"{tok:>4}: {row}")
print("row sums:", weights.sum(axis=1).round(2))
print("output shape:", output.shape)

import numpy as np

np.random.seed(0)
SIZE = 4
GOAL, TRAP = (3, 3), (1, 2)
ACTIONS = [(-1, 0), (1, 0), (0, -1), (0, 1)]          # up, down, left, right
LETTERS = ["U", "D", "L", "R"]

def step(state, action):
    r, c = state
    dr, dc = ACTIONS[action]
    r, c = min(max(r + dr, 0), SIZE - 1), min(max(c + dc, 0), SIZE - 1)
    if (r, c) == GOAL:
        return (r, c), 10, True
    if (r, c) == TRAP:
        return (r, c), -10, True
    return (r, c), -1, False

Q = np.zeros((SIZE, SIZE, 4))                          # one value per (row, col, action)
alpha, gamma, epsilon = 0.1, 0.95, 0.2

for episode in range(2000):
    state, done, steps = (0, 0), False, 0
    while not done and steps < 200:
        if np.random.rand() < epsilon:
            action = np.random.randint(4)              # explore
        else:
            action = int(np.argmax(Q[state]))          # exploit
        next_state, reward, done = step(state, action)
        best_next = 0 if done else np.max(Q[next_state])
        Q[state][action] += alpha * (reward + gamma * best_next - Q[state][action])
        state = next_state
        steps += 1

state, path = (0, 0), [(0, 0)]
while state not in (GOAL, TRAP) and len(path) < 20:
    state, _, _ = step(state, int(np.argmax(Q[state])))
    path.append(state)
print("Learned path:", path)
print("Reached the goal:", path[-1] == GOAL)

print("\nBest action in each cell (G = goal, X = trap):")
for r in range(SIZE):
    cells = []
    for c in range(SIZE):
        if (r, c) == GOAL:
            cells.append("G")
        elif (r, c) == TRAP:
            cells.append("X")
        else:
            cells.append(LETTERS[int(np.argmax(Q[r, c]))])
    print(" ".join(cells))

import numpy as np
from sklearn.linear_model import LogisticRegression
from sklearn.model_selection import train_test_split

rng = np.random.default_rng(0)
# Stylised example: Group A is large, Group B is small, and the rule that links x to y
# is different for B (its positives start at a higher x than group A's do)
xa = rng.normal(0, 1, 900)
ya = (xa + rng.normal(0, 0.5, 900) > 0).astype(int)
xb = rng.normal(0, 1, 100)
yb = (xb + rng.normal(0, 0.5, 100) > 1.0).astype(int)

X = np.concatenate([xa, xb]).reshape(-1, 1)
y = np.concatenate([ya, yb])
group = np.array(["A"] * 900 + ["B"] * 100)

X_tr, X_te, y_tr, y_te, g_tr, g_te = train_test_split(
    X, y, group, test_size=0.3, random_state=0, stratify=group)
pred = LogisticRegression().fit(X_tr, y_tr).predict(X_te)

print("Overall accuracy:", round(float(np.mean(pred == y_te)), 3))
for g in ["A", "B"]:
    mask = g_te == g
    print(f"Group {g} accuracy: {np.mean(pred[mask] == y_te[mask]):.3f}  (n={mask.sum()})")

from sklearn.datasets import load_breast_cancer
from sklearn.ensemble import RandomForestClassifier
from sklearn.inspection import permutation_importance
from sklearn.model_selection import train_test_split

data = load_breast_cancer()
X_tr, X_te, y_tr, y_te = train_test_split(data.data, data.target, test_size=0.3, random_state=0)
model = RandomForestClassifier(n_estimators=60, random_state=0).fit(X_tr, y_tr)
print("test accuracy:", round(model.score(X_te, y_te), 3))

result = permutation_importance(model, X_te, y_te, n_repeats=5, random_state=0)
top = result.importances_mean.argsort()[::-1][:5]
print("Top 5 features by permutation importance:")
for i in top:
    print(f"  {data.feature_names[i]:25s} {result.importances_mean[i]:.4f}")

import numpy as np
from scipy.stats import ks_2samp

rng = np.random.default_rng(0)
train_income = rng.normal(35000, 8000, 2000)
this_week = rng.normal(35000, 8000, 500)        # same distribution as training
after_campaign = rng.normal(42000, 8000, 500)   # the customer base has changed

for name, sample in [("this week", this_week), ("after campaign", after_campaign)]:
    stat, p = ks_2samp(train_income, sample)
    status = "DRIFT - investigate or retrain" if p < 0.01 else "ok"
    print(f"{name:15s} KS statistic={stat:.3f}  p-value={p:.4f}  -> {status}")

import numpy as np
import pandas as pd
from sklearn.ensemble import GradientBoostingRegressor, RandomForestRegressor
from sklearn.linear_model import LinearRegression
from sklearn.metrics import mean_absolute_error, r2_score
from sklearn.model_selection import train_test_split

rng = np.random.default_rng(10)
n = 1000
df = pd.DataFrame({
    "area_m2": rng.uniform(25, 250, n),
    "bedrooms": rng.integers(1, 6, n),
    "age_years": rng.integers(0, 50, n),
    "distance_km": rng.uniform(0.5, 25, n),
})
# A non-linear pricing rule plus noise (a stand-in for a real housing dataset)
df["price"] = (1200 * df["area_m2"] + 8000 * df["bedrooms"] - 900 * df["age_years"]
               - 6000 * np.sqrt(df["distance_km"]) * (df["area_m2"] / 100)
               + rng.normal(0, 12000, n))

X, y = df.drop(columns="price"), df["price"]
X_tr, X_te, y_tr, y_te = train_test_split(X, y, test_size=0.2, random_state=0)

models = {
    "Linear Regression": LinearRegression(),
    "Random Forest": RandomForestRegressor(n_estimators=80, random_state=0),
    "Gradient Boosting": GradientBoostingRegressor(random_state=0),
}
print(f"{'Model':18s} {'MAE':>9s} {'RMSE':>9s} {'R2':>6s}")
for name, m in models.items():
    pred = m.fit(X_tr, y_tr).predict(X_te)
    rmse = float(np.sqrt(np.mean((y_te - pred) ** 2)))
    print(f"{name:18s} {mean_absolute_error(y_te, pred):9.0f} {rmse:9.0f} {r2_score(y_te, pred):6.3f}")

rf = models["Random Forest"]
print("\nRandom Forest feature importances:")
for name, imp in sorted(zip(X.columns, rf.feature_importances_), key=lambda t: -t[1]):
    print(f"  {name:12s} {imp:.3f}")

from sklearn.feature_extraction.text import TfidfVectorizer
from sklearn.metrics import classification_report
from sklearn.model_selection import train_test_split
from sklearn.naive_bayes import MultinomialNB
from sklearn.pipeline import make_pipeline

spam = [
    "win a free prize now", "claim your free cash reward today", "urgent your account needs verification",
    "limited offer buy now and save", "congratulations you won a free gift card", "free entry win cash now",
    "cheap loans approved instantly", "act now exclusive deal just for you", "you have been selected for a free holiday",
    "earn money fast from home", "final notice claim your prize", "get rich quick with this offer",
]
ham = [
    "are we still meeting for lunch tomorrow", "please review the attached project report", "can you send me the notes from class",
    "dinner at my place on saturday", "the meeting is moved to three pm", "thanks for helping me with the assignment",
    "see you at the library later", "happy birthday hope you have a great day", "i will call you after work",
    "reminder to submit the form by friday", "did you finish reading the chapter", "let us plan the trip next week",
]
texts = spam + ham
labels = [1] * len(spam) + [0] * len(ham)

X_tr, X_te, y_tr, y_te = train_test_split(texts, labels, test_size=0.3, random_state=3, stratify=labels)
model = make_pipeline(TfidfVectorizer(), MultinomialNB())
model.fit(X_tr, y_tr)

print(classification_report(y_te, model.predict(X_te), target_names=["ham", "spam"], zero_division=0))
for msg in ["claim your free prize now", "can we move the meeting to friday", "exclusive cash offer just for you"]:
    p = model.predict_proba([msg])[0, 1]
    print(f"{msg!r:40} P(spam) = {p:.2f}")

import numpy as np
import pandas as pd
from sklearn.cluster import KMeans
from sklearn.metrics import silhouette_score
from sklearn.preprocessing import StandardScaler

rng = np.random.default_rng(4)

def group(n, recency, frequency, monetary):
    return pd.DataFrame({
        "recency_days": rng.normal(recency, 8, n).clip(1),
        "orders": rng.poisson(frequency, n),
        "spend": rng.normal(monetary, monetary * 0.2, n).clip(5),
    })

customers = pd.concat([group(150, 10, 12, 900),     # frequent, big spenders
                       group(200, 40, 4, 250),      # occasional buyers
                       group(120, 120, 1, 60)],     # lapsed customers
                      ignore_index=True)

X = StandardScaler().fit_transform(customers)       # scale so no feature dominates
km = KMeans(n_clusters=3, n_init=10, random_state=0).fit(X)
customers["segment"] = km.labels_

print("silhouette score:", round(silhouette_score(X, km.labels_), 3))
profile = customers.groupby("segment").agg(
    customers=("spend", "size"),
    recency_days=("recency_days", "mean"),
    orders=("orders", "mean"),
    spend=("spend", "mean"),
).round(1)
print(profile)

import numpy as np
from sklearn.datasets import load_digits
from sklearn.metrics import confusion_matrix
from sklearn.model_selection import train_test_split
from sklearn.neural_network import MLPClassifier
from sklearn.pipeline import make_pipeline
from sklearn.preprocessing import StandardScaler

digits = load_digits()
X_tr, X_te, y_tr, y_te = train_test_split(
    digits.data, digits.target, test_size=0.25, random_state=0, stratify=digits.target)

net = make_pipeline(StandardScaler(),
                    MLPClassifier(hidden_layer_sizes=(64,), max_iter=500, random_state=0))
net.fit(X_tr, y_tr)
pred = net.predict(X_te)
print("Test accuracy:", round(float(np.mean(pred == y_te)), 3))

# Error analysis: which pairs of digits does the network mix up most?
cm = confusion_matrix(y_te, pred)
np.fill_diagonal(cm, 0)
worst = np.dstack(np.unravel_index(np.argsort(cm.ravel())[::-1][:3], cm.shape))[0]
for true_digit, pred_digit in worst:
    print(f"true {true_digit} predicted as {pred_digit}: {cm[true_digit, pred_digit]} times")

# Draw one test image as text (pixel values range from 0 to 16)
def ascii_digit(pixels):
    for row in pixels.reshape(8, 8):
        print("".join("#" if v > 8 else "." for v in row))

print("\nA test image the network read as", pred[0], "(true label:", y_te[0], ")")
ascii_digit(X_te[0])