Support_Vector_Machine_3D.py

import matplotlib.pyplot as plt
import numpy as np

rand = np.random.default_rng(42)
# Feature 1 (X)
x1 = rand.uniform(0, 5, size=25)
x2 = rand.uniform(6, 10, size=25)

# Feature 2 (Y)
y1 = rand.uniform(0, 5, size=25)
y2 = rand.uniform(6, 10, size=25)

# Feature 3 (Z) - New dimension
z1 = rand.uniform(0, 5, size=25)
z2 = rand.uniform(6, 10, size=25)

class_1 = np.column_stack((x1, y1, z1))
class_2 = np.column_stack((x2, y2, z2))

X = np.vstack((class_1, class_2))
y = np.concatenate((-np.ones(25), np.ones(25)))

indices = np.random.permutation(len(X))
X = X[indices]
y = y[indices]

train_size = int(0.8 * len(X))
X_train, y_train = X[:train_size], y[:train_size]
X_test, y_test = X[train_size:], y[train_size:]

class SupportVectorMachine3D:
    def __init__(self, learning_rate=0.01, lambda_param=0.01):
        self.lr = learning_rate
        self.lambda_param = lambda_param
        self.weight = np.zeros(X.shape[1])
        self.bias = 0.0
    
    def forward(self, x):
        return np.dot(x, self.weight) + self.bias
    
    def compute_loss(self, y_pred, y_true):
        sample_losses = np.maximum(0, 1 - (y_true * y_pred))
        hinge_loss = np.mean(sample_losses)
        reg_loss = self.lambda_param * np.sum(self.weight ** 2)
        return hinge_loss + reg_loss
    
    def backpropagation(self, X, y_true, y_pred):
        num_samples = X.shape[0]
        margin_violation = (y_true * y_pred) < 1
        
        dw = 2 * self.lambda_param * self.weight
        db = 0.0
        
        if np.any(margin_violation):
            X_violators = X[margin_violation]
            y_violators = y_true[margin_violation]
            
            dw -= np.dot(y_violators, X_violators) / num_samples
            db -= np.sum(y_violators) / num_samples
            
        self.weight -= self.lr * dw
        self.bias -= self.lr * db

    def predict(self, x):
        preds = self.forward(x)
        return np.where(preds >= 0, 1, -1)

epochs = 10000
model = SupportVectorMachine3D(learning_rate=0.001, lambda_param=0.01)
train_loss_hist, test_loss_hist = [], []

for epoch in range(epochs):
    train_pred = model.forward(X_train)
    train_loss = model.compute_loss(y_pred=train_pred, y_true=y_train)
    train_loss_hist.append(train_loss)

    model.backpropagation(X_train, y_true=y_train, y_pred=train_pred)

    test_pred = model.forward(X_test)
    test_loss = model.compute_loss(y_pred=test_pred, y_true=y_test)
    test_loss_hist.append(test_loss)

    if epoch % 300 == 0:
        print(f'Epoch: {epoch} | Train loss: {train_loss:.4f} | Test loss: {test_loss:.4f}')

final_test_pred = model.predict(X_test)
accuracy = np.mean(final_test_pred == y_test) * 100
print(f"\nFinal Test Accuracy: {accuracy}%")

# Plotting
fig = plt.figure(figsize=(15, 7))

# Subplot 1: Interactive 3D Hyperplane and Margins
ax = fig.add_subplot(1, 2, 1, projection='3d')
ax.scatter(x1, y1, z1, color='blue', label='Class -1', s=40)
ax.scatter(x2, y2, z2, color='red', label='Class 1', s=40)

# Create a 2D meshgrid spanning X and Y space to evaluate the plane equations
x_mesh, y_mesh = np.meshgrid(np.linspace(-1, 11, 10), np.linspace(-1, 11, 10))

# Planar algebra transformation: w0*x + w1*y + w2*z + b = Target  =>  z = (Target - w0*x - w1*y - b) / w2
z_boundary = (0 - model.weight[0]*x_mesh - model.weight[1]*y_mesh - model.bias) / model.weight[2]
z_margin_plus = (1 - model.weight[0]*x_mesh - model.weight[1]*y_mesh - model.bias) / model.weight[2]
z_margin_minus = (-1 - model.weight[0]*x_mesh - model.weight[1]*y_mesh - model.bias) / model.weight[2]

# Plot Surface planes
ax.plot_surface(x_mesh, y_mesh, z_boundary, color='black', alpha=0.4, rstride=100, cstride=100)
ax.plot_surface(x_mesh, y_mesh, z_margin_plus, color='gray', alpha=0.15, rstride=100, cstride=100)
ax.plot_surface(x_mesh, y_mesh, z_margin_minus, color='gray', alpha=0.15, rstride=100, cstride=100)

# Calculate and circle 3D Support Vectors
train_preds = model.forward(X_train)
sv_idx = (y_train * train_preds) <= 1.05
ax.scatter(X_train[sv_idx, 0], X_train[sv_idx, 1], X_train[sv_idx, 2], 
           s=120, facecolors='none', edgecolors='green', linewidths=2.5, label='Support Vectors')

ax.set_xlim(-1, 11)
ax.set_ylim(-1, 11)
ax.set_zlim(-1, 11)
ax.set_title('3D SVM Hyperplane & Margin Planes')
ax.set_xlabel('Feature X')
ax.set_ylabel('Feature Y')
ax.set_zlabel('Feature Z')
ax.legend()

# Subplot 2: Loss Plots
plt.subplot(1, 2, 2)
plt.plot(train_loss_hist, color='blue', label='Train Loss')
plt.plot(test_loss_hist, color='red', label='Test Loss')
plt.title('Loss Curve Over Epochs')
plt.xlabel('Epoch')
plt.ylabel('Loss (MSE / Hinge)')
plt.legend()
plt.grid(True, alpha=0.3)

plt.tight_layout()
plt.show()