%matplotlib inline
import numpy as np
import matplotlib.pyplot as plt

plt.rcParams["figure.figsize"] = (6, 4)
plt.rcParams["axes.grid"] = True

# Three 2D vectors (like the book's example)
a = np.array([2, 3])
b = np.array([-2, 5])
c = np.array([1, 0])

# A tiny matrix: 2 rows x 3 columns
W = np.array([[1, 2, 3],
              [4, 5, 6]])

print("vector a =", a, "   a[0] =", a[0], "(the 1st attribute)")
print("vector b =", b)
print("vector c =", c)
print("matrix W (2x3) =\n", W)
print("W.shape =", W.shape, "-> 2 rows, 3 columns")

# Plot the same vectors two ways: as arrows AND as points at their tips
fig, ax = plt.subplots()
origin = np.array([0, 0])
for vec, name, col in [(a, "a = [2, 3]", "tab:red"),
                       (b, "b = [-2, 5]", "tab:blue"),
                       (c, "c = [1, 0]", "tab:green")]:
    ax.annotate("", xy=vec, xytext=origin,
                arrowprops=dict(arrowstyle="->", color=col, lw=2))  # arrow view
    ax.scatter(*vec, color=col, zorder=5)                            # point view
    ax.text(vec[0] + 0.1, vec[1] + 0.1, name, color=col, fontsize=11)
ax.axhline(0, color="gray", lw=0.5)
ax.axvline(0, color="gray", lw=0.5)
ax.set_xlim(-3, 4)
ax.set_ylim(-1, 6)
ax.set_title("Vectors shown as arrows (and their tips as points)")
ax.set_xlabel("dimension 1")
ax.set_ylabel("dimension 2")
plt.show()

vector a = [2 3]    a[0] = 2 (the 1st attribute)
vector b = [-2  5]
vector c = [1 0]
matrix W (2x3) =
 [[1 2 3]
 [4 5 6]]
W.shape = (2, 3) -> 2 rows, 3 columns

# Dot product: combine two same-length vectors into one scalar
w = np.array([1, 2, 3])
x = np.array([4, 5, 6])

dot = np.dot(w, x)            # 1*4 + 2*5 + 3*6
print("w =", w)
print("x =", x)
print("dot product w·x =", dot, "  (manual check:", 1*4 + 2*5 + 3*6, ")")

# Norms: measure a vector's "length"
v = np.array([3, 4])
l2 = np.linalg.norm(v)                 # default = 2-norm: sqrt(3^2 + 4^2) = 5
l1 = np.linalg.norm(v, ord=1)          # 1-norm: |3| + |4| = 7
print("\nvector v =", v)
print("2-norm (Euclidean) ||v||_2 =", l2)
print("1-norm (taxicab)    ||v||_1 =", l1)

# Visual: a straight-line walk vs a grid walk for v = [3, 4]
fig, ax = plt.subplots()
ax.plot([0, v[0]], [0, v[1]], color="tab:blue", lw=2, label="2-norm path (straight)")
ax.plot([0, v[0], v[0]], [0, 0, v[1]], color="tab:orange", lw=2, ls="--",
        label="1-norm path (taxicab)")
ax.scatter(*v, color="black", zorder=5)
ax.text(v[0] + 0.1, v[1] + 0.1, "v = [3, 4]", fontsize=11)
ax.set_xlim(-0.5, 4.5)
ax.set_ylim(-0.5, 5)
ax.set_title("Two ways to measure the length of v = [3, 4]")
ax.set_xlabel("dimension 1")
ax.set_ylabel("dimension 2")
ax.legend()
plt.show()

w = [1 2 3]
x = [4 5 6]
dot product w·x = 32   (manual check: 32 )

vector v = [3 4]
2-norm (Euclidean) ||v||_2 = 5.0
1-norm (taxicab)    ||v||_1 = 7.0

# A continuous random variable: X ~ Normal(mean=5, std=2)
true_mean, true_std = 5.0, 2.0
rng = np.random.default_rng(42)

# Draw a sample of 5000 values
sample = rng.normal(loc=true_mean, scale=true_std, size=5000)

fig, axes = plt.subplots(1, 2, figsize=(11, 4))

# Left: a histogram of the samples approximates the PDF
axes[0].hist(sample, bins=40, density=True, color="tab:blue", alpha=0.7)
axes[0].axvline(true_mean, color="tab:red", ls="--", label=f"true mean = {true_mean}")
axes[0].set_title("Histogram of samples (approximates the PDF)")
axes[0].set_xlabel("value")
axes[0].set_ylabel("density")
axes[0].legend()

# Right: the running sample mean homes in on the true mean (Law of Large Numbers)
n_vals = np.arange(1, len(sample) + 1)
running_mean = np.cumsum(sample) / n_vals
axes[1].plot(n_vals, running_mean, color="tab:blue", lw=1, label="sample mean so far")
axes[1].axhline(true_mean, color="tab:red", ls="--", label=f"true mean = {true_mean}")
axes[1].set_title("Sample mean approaches the true mean as n grows")
axes[1].set_xlabel("number of samples so far (n)")
axes[1].set_ylabel("running mean")
axes[1].legend()
plt.tight_layout()
plt.show()

print(f"Sample mean over all 5000 draws: {sample.mean():.3f}   (true mean: {true_mean})")
print(f"Sample std : {sample.std():.3f}   (true std : {true_std})")

Sample mean over all 5000 draws: 4.960   (true mean: 5.0)
Sample std : 1.999   (true std : 2.0)

# Show the sample MEAN is unbiased, and that variance is biased (÷N) vs unbiased (÷N-1)
true_mean, true_std = 5.0, 2.0
true_var = true_std ** 2
rng = np.random.default_rng(7)

N = 10                 # each sample is small
repeats = 8000         # but we take many of them
sample_means = np.empty(repeats)
biased_vars = np.empty(repeats)
unbiased_vars = np.empty(repeats)

for r in range(repeats):
    s = rng.normal(loc=true_mean, scale=true_std, size=N)
    sample_means[r] = s.mean()
    biased_vars[r] = s.var(ddof=0)       # divide by N      -> biased
    unbiased_vars[r] = s.var(ddof=1)     # divide by (N-1)  -> unbiased

print(f"true mean = {true_mean},   true variance = {true_var}")
print(f"avg of sample means       = {sample_means.mean():.4f}   (unbiased: matches true mean)")
print(f"avg of biased  variance   = {biased_vars.mean():.4f}   (too low)")
print(f"avg of unbiased variance  = {unbiased_vars.mean():.4f}   (matches true variance)")

fig, ax = plt.subplots()
ax.hist(sample_means, bins=40, density=True, color="tab:blue", alpha=0.7)
ax.axvline(true_mean, color="tab:red", ls="--", label=f"true mean = {true_mean}")
ax.axvline(sample_means.mean(), color="tab:green", ls=":",
           label=f"avg sample mean = {sample_means.mean():.3f}")
ax.set_title("Distribution of the sample mean over many small samples")
ax.set_xlabel("sample mean value")
ax.set_ylabel("density")
ax.legend()
plt.show()

true mean = 5.0,   true variance = 4.0
avg of sample means       = 4.9972   (unbiased: matches true mean)
avg of biased  variance   = 3.5897   (too low)
avg of unbiased variance  = 3.9886   (matches true variance)

# The classic base-rate example: a rare disease and a pretty good test
p_disease = 0.01                       # prior: 1% of people have the disease
p_no_disease = 1 - p_disease
p_pos_given_disease = 0.99             # sensitivity (true positive rate)
p_pos_given_no_disease = 1 - 0.95      # 1 - specificity = false positive rate (5%)

# Evidence: overall probability of getting a positive test
p_positive = (p_pos_given_disease * p_disease
              + p_pos_given_no_disease * p_no_disease)

# Bayes' rule: probability you actually have the disease given a positive test
p_disease_given_pos = (p_pos_given_disease * p_disease) / p_positive

print("Given a POSITIVE test:")
print(f"  P(disease)        = {p_disease:.2f}       (prior)")
print(f"  P(+ | disease)    = {p_pos_given_disease:.2f}       (likelihood / sensitivity)")
print(f"  P(+)              = {p_positive:.4f}    (evidence)")
print(f"  P(disease | +)    = {p_disease_given_pos:.4f}  <-- posterior")
print(f"\nSo a positive test means only a {p_disease_given_pos*100:.1f}% chance you're actually sick.")

fig, ax = plt.subplots(figsize=(5, 3.5))
ax.bar(["prior\nP(disease)", "posterior\nP(disease | +)"],
       [p_disease, p_disease_given_pos], color=["tab:gray", "tab:red"])
ax.set_ylim(0, 1)
ax.set_ylabel("probability")
ax.set_title("Bayes' rule: a 1% prior becomes a 16% posterior")
for i, v in enumerate([p_disease, p_disease_given_pos]):
    ax.text(i, v + 0.02, f"{v*100:.1f}%", ha="center")
plt.show()

Given a POSITIVE test:
  P(disease)        = 0.01       (prior)
  P(+ | disease)    = 0.99       (likelihood / sensitivity)
  P(+)              = 0.0594    (evidence)
  P(disease | +)    = 0.1667  <-- posterior

So a positive test means only a 16.7% chance you're actually sick.

# Estimate the mean of a Gaussian via MLE vs MAP
true_mean, true_std = 7.0, 2.0
rng = np.random.default_rng(3)
N = 5                                      # a TINY sample, so the prior will matter
data = rng.normal(loc=true_mean, scale=true_std, size=N)

# MLE for the mean of a Gaussian = the sample mean
mle = data.mean()

# MAP with a Gaussian prior on the mean: prior ~ N(prior_mean, prior_sd^2)
prior_mean, prior_sd = 0.0, 3.0
# For a Gaussian likelihood + Gaussian prior, the MAP estimate is a
# precision-weighted average of the MLE and the prior mean:
lik_prec = N / (true_std ** 2)             # how strongly the data speaks
prior_prec = 1 / (prior_sd ** 2)           # how strongly the prior speaks
map_est = (lik_prec * mle + prior_prec * prior_mean) / (lik_prec + prior_prec)

print(f"true mean             = {true_mean}")
print(f"sample (N={N})         = {np.round(data, 2)}")
print(f"MLE mean (data only)  = {mle:.3f}")
print(f"MAP mean (+ prior)    = {map_est:.3f}   (pulled toward prior mean {prior_mean})")

# Plot the log-likelihood and log-posterior across candidate means
grid = np.linspace(-2, 10, 300)
loglik = -0.5 * np.sum((data[None, :] - grid[:, None]) ** 2 / true_std ** 2, axis=1)
logpost = loglik - 0.5 * ((grid - prior_mean) ** 2 / prior_sd ** 2)

fig, ax = plt.subplots()
ax.plot(grid, loglik - loglik.max(), color="tab:blue", label="log-likelihood (MLE)")
ax.plot(grid, logpost - logpost.max(), color="tab:orange", label="log-posterior (MAP)")
ax.axvline(mle, color="tab:blue", ls="--", label=f"MLE = {mle:.2f}")
ax.axvline(map_est, color="tab:orange", ls="--", label=f"MAP = {map_est:.2f}")
ax.axvline(prior_mean, color="tab:green", ls=":", label=f"prior mean = {prior_mean}")
ax.set_title("MLE vs MAP for the mean of a Gaussian (tiny sample)")
ax.set_xlabel("candidate mean μ")
ax.set_ylabel("log value (normalized)")
ax.legend()
plt.show()

true mean             = 7.0
sample (N=5)         = [11.08  1.89  7.84  5.86  6.09]
MLE mean (data only)  = 6.553
MAP mean (+ prior)    = 6.018   (pulled toward prior mean 0.0)

from sklearn.linear_model import LogisticRegression, LinearRegression
from sklearn.datasets import make_classification, make_regression

SEED = 42

# --- Classification: labels are categories (0 / 1) ---
Xc, yc = make_classification(n_samples=120, n_features=2, n_redundant=0,
                             n_informative=2, n_clusters_per_class=1,
                             random_state=SEED)
clf = LogisticRegression(max_iter=200)
clf.fit(Xc, yc)

# --- Regression: labels are real numbers ---
Xr, yr = make_regression(n_samples=120, n_features=1, noise=15.0, random_state=SEED)
reg = LinearRegression()
reg.fit(Xr, yr)

fig, axes = plt.subplots(1, 2, figsize=(11, 4))

# Left: classification with a straight decision boundary
axes[0].scatter(Xc[:, 0][yc == 0], Xc[:, 1][yc == 0], color="tab:blue", label="class 0")
axes[0].scatter(Xc[:, 0][yc == 1], Xc[:, 1][yc == 1], color="tab:orange", label="class 1")
xx = np.linspace(Xc[:, 0].min(), Xc[:, 0].max(), 50)
w0, w1 = clf.coef_[0]
b = clf.intercept_[0]
axes[0].plot(xx, -(w0 * xx + b) / w1, "k-", label="decision boundary")
axes[0].set_title("Classification: LogisticRegression (label = category)")
axes[0].set_xlabel("feature 1")
axes[0].set_ylabel("feature 2")
axes[0].legend()

# Right: regression with a fitted line
axes[1].scatter(Xr.ravel(), yr, color="tab:blue", alpha=0.7, label="data")
axes[1].plot(Xr.ravel(), reg.predict(Xr), color="tab:red", lw=2, label="fitted line")
axes[1].set_title("Regression: LinearRegression (label = number)")
axes[1].set_xlabel("feature")
axes[1].set_ylabel("target (real number)")
axes[1].legend()
plt.tight_layout()
plt.show()

from sklearn.neighbors import KNeighborsClassifier
from sklearn.linear_model import LogisticRegression
from sklearn.datasets import make_moons

# A dataset that is NOT linearly separable: two interleaving moons
X, y = make_moons(n_samples=200, noise=0.20, random_state=5)

# Model-based: logistic regression (keeps only w, b at predict time)
log = LogisticRegression(max_iter=200)
log.fit(X, y)

# Instance-based: k-NN (keeps the whole dataset)
knn = KNeighborsClassifier(n_neighbors=5, n_jobs=1)
knn.fit(X, y)

# Build a grid so we can color the decision regions
x_min, x_max = X[:, 0].min() - 0.5, X[:, 0].max() + 0.5
y_min, y_max = X[:, 1].min() - 0.5, X[:, 1].max() + 0.5
xx, yy = np.meshgrid(np.linspace(x_min, x_max, 150),
                     np.linspace(y_min, y_max, 150))
grid = np.c_[xx.ravel(), yy.ravel()]

fig, axes = plt.subplots(1, 2, figsize=(11, 4))
for ax, model, title in [(axes[0], log, "Model-based: LogisticRegression"),
                         (axes[1], knn, "Instance-based: k-NN (k=5)")]:
    Z = model.predict(grid).reshape(xx.shape)
    ax.contourf(xx, yy, Z, alpha=0.3, cmap="coolwarm")
    ax.scatter(X[:, 0][y == 0], X[:, 1][y == 0], color="tab:blue",
               edgecolor="k", s=30, label="class 0")
    ax.scatter(X[:, 0][y == 1], X[:, 1][y == 1], color="tab:red",
               edgecolor="k", s=30, label="class 1")
    ax.set_title(title)
    ax.set_xlabel("feature 1")
    ax.set_ylabel("feature 2")
    ax.legend()
plt.tight_layout()
plt.show()

from sklearn.linear_model import LogisticRegression
from sklearn.neural_network import MLPClassifier
from sklearn.datasets import make_moons

# Same two-moons dataset (curved, not linearly separable)
X, y = make_moons(n_samples=200, noise=0.20, random_state=5)

# Shallow: logistic regression -> straight-line boundary
shallow = LogisticRegression(max_iter=200)
shallow.fit(X, y)

# Deep: a small multi-layer perceptron (2 hidden layers) -> can bend the boundary
deep = MLPClassifier(hidden_layer_sizes=(10, 10), max_iter=500,
                     random_state=5)
deep.fit(X, y)

x_min, x_max = X[:, 0].min() - 0.5, X[:, 0].max() + 0.5
y_min, y_max = X[:, 1].min() - 0.5, X[:, 1].max() + 0.5
xx, yy = np.meshgrid(np.linspace(x_min, x_max, 150),
                     np.linspace(y_min, y_max, 150))
grid = np.c_[xx.ravel(), yy.ravel()]

fig, axes = plt.subplots(1, 2, figsize=(11, 4))
for ax, model, title in [(axes[0], shallow, "Shallow: LogisticRegression (linear)"),
                         (axes[1], deep, "Deep: MLPClassifier (2 hidden layers)")]:
    Z = model.predict(grid).reshape(xx.shape)
    ax.contourf(xx, yy, Z, alpha=0.3, cmap="coolwarm")
    ax.scatter(X[:, 0][y == 0], X[:, 1][y == 0], color="tab:blue",
               edgecolor="k", s=30, label="class 0")
    ax.scatter(X[:, 0][y == 1], X[:, 1][y == 1], color="tab:red",
               edgecolor="k", s=30, label="class 1")
    ax.set_title(title)
    ax.set_xlabel("feature 1")
    ax.set_ylabel("feature 2")
    ax.legend()
plt.tight_layout()
plt.show()

print(f"Shallow training accuracy: {shallow.score(X, y):.2f}")
print(f"Deep training accuracy   : {deep.score(X, y):.2f}")

C:\Users\DELL\anaconda3\Lib\site-packages\sklearn\neural_network\_multilayer_perceptron.py:691: ConvergenceWarning: Stochastic Optimizer: Maximum iterations (500) reached and the optimization hasn't converged yet.
  warnings.warn(

Shallow training accuracy: 0.88
Deep training accuracy   : 0.94

# Exercise 8: dot product and norms
import numpy as np
w = np.array([1, 2, 3])
x = np.array([4, 5, 6])
v = np.array([3, 4])

# TODO: dot product with np.dot AND by hand (sum of products)
dot = 0  # placeholder

# TODO: 2-norm and 1-norm of v
l2 = 0  # placeholder
l1 = 0  # placeholder

print("dot =", dot, " (expected 32)")
print("||v||_2 =", l2, " (expected 5)")
print("||v||_1 =", l1, " (expected 7)")

# Exercise 9: Law of Large Numbers -- running sample mean
import numpy as np
import matplotlib.pyplot as plt

rng = np.random.default_rng(42)
sample = rng.normal(loc=5.0, scale=2.0, size=5000)

# TODO: compute the running sample mean
n_vals = np.arange(1, len(sample) + 1)
running_mean = np.full(len(sample), 5.0)  # placeholder

# TODO: plot running_mean vs n_vals with a horizontal line at the true mean 5.0
plt.show()

# Exercise 10: sample mean is unbiased, variance-by-N is biased
import numpy as np
rng = np.random.default_rng(7)
true_mean, true_std, true_var = 5.0, 2.0, 4.0
N, repeats = 10, 8000
sample_means = np.empty(repeats)
biased_vars = np.empty(repeats)
unbiased_vars = np.empty(repeats)

for r in range(repeats):
    s = rng.normal(loc=true_mean, scale=true_std, size=N)
    sample_means[r] = s.mean()
    # TODO: biased_vars[r] = s.var(ddof=0)  and  unbiased_vars[r] = s.var(ddof=1)
    biased_vars[r] = 0      # placeholder
    unbiased_vars[r] = 0    # placeholder

print("true variance =", true_var)
print("avg biased variance (ddof=0)   =", biased_vars.mean())
print("avg unbiased variance (ddof=1) =", unbiased_vars.mean())

# Exercise 11: Bayes' rule for a diagnostic test
def posterior_disease_given_positive(p_disease, sensitivity, specificity):
    # TODO: compute P(disease | +) using Bayes' rule
    # evidence P(+) = sensitivity*p_disease + (1-specificity)*(1-p_disease)
    return 0.0  # placeholder

sensitivity, specificity = 0.99, 0.95
print("posterior (1% prior):", posterior_disease_given_positive(0.01, sensitivity, specificity))

# TODO: call it again with p_disease = 0.10 and print the result

# Exercise 12: model-based vs instance-based decision regions
import numpy as np
import matplotlib.pyplot as plt
from sklearn.linear_model import LogisticRegression
from sklearn.neighbors import KNeighborsClassifier
from sklearn.datasets import make_moons

X, y = make_moons(n_samples=200, noise=0.20, random_state=5)

# TODO: fit LogisticRegression (model-based) and KNeighborsClassifier(n_neighbors=5)
log = None   # placeholder
knn = None   # placeholder

# TODO: build a meshgrid over the data range and plot each model's decision regions
# Hint: np.meshgrid + np.c_[xx.ravel(), yy.ravel()] -> predict -> reshape -> contourf
plt.show()

# Exercise 13: shallow vs deep on the moons
import numpy as np
import matplotlib.pyplot as plt
from sklearn.linear_model import LogisticRegression
from sklearn.neural_network import MLPClassifier
from sklearn.datasets import make_moons

X, y = make_moons(n_samples=200, noise=0.20, random_state=5)

# TODO: fit a shallow LogisticRegression and a deep MLPClassifier(hidden_layer_sizes=(10,10), max_iter=500, random_state=5)
shallow = None  # placeholder
deep = None     # placeholder

# TODO: plot both decision regions and print both training accuracies (.score(X, y))
plt.show()

	Classification	Regression
Label type	a category from a finite set	a real number
Example	spam / not-spam; dog / cat / bird	house price; temperature; wait time
Output	a class label (or a probability per class)	a continuous value
Variants	binary (2 classes) or multiclass (3+)	—

	Model-based	Instance-based
What it stores	a small set of parameters	the whole dataset
Prediction cost	cheap (one formula)	costlier (compare to all data)
Boundary shape	fixed by the model (e.g. a line)	flexible / local

Chapter 2 — Notation and Definitions¶

2.1 Notation — Scalars, Vectors, and Sets¶

Reading the plot¶

2.1 (continued) — Dot Product and Norms¶

Reading the output¶

2.2 Random Variables¶

Reading the plots¶

2.3 Unbiased Estimators¶

Reading the output¶

2.4 Bayes' Rule¶

Reading the result¶

2.5 Parameter Estimation — MLE and MAP¶

Reading the plot¶

2.6 Classification vs Regression¶

Reading the plots¶

2.7 Model-Based vs Instance-Based Learning¶

Reading the plots¶

2.8 Shallow vs Deep Learning¶

Reading the plots¶

Key Takeaways¶

What's Next¶

Exercises¶

Hands-On Coding Problems¶