import numpy as np
import torch.nn as nn
from sklearn.base import RegressorMixin, ClassifierMixin
from sklearn.utils import check_array
from sklearn.utils.extmath import softmax
from sklearn.utils.validation import check_is_fitted
from .base import ReLUDNN
from ..base import ModelBaseRegressor, ModelBaseClassifier
[docs]
class MoReLUDNNRegressor(RegressorMixin, ReLUDNN, ModelBaseRegressor):
"""A deep neural network regressor using ReLU activation functions.
This model implements a multi-layer neural network for regression tasks, using ReLU
activation functions and supporting early stopping, L1 regularization, and batch training.
Parameters
----------
name : str, default=None
Optional identifier for the model instance.
hidden_layer_sizes : tuple of int, default=(40, 40)
Architecture of the neural network specified as a tuple of integers, where each
integer represents the number of neurons in a hidden layer.
max_epochs : int, default=1000
Maximum number of complete passes through the training dataset.
learning_rate : float, default=0.001
Step size used for gradient updates during optimization.
batch_size : int, default=500
Number of training samples used in each gradient update.
l1_reg : float, default=1e-5
Strength of L1 regularization applied to model weights.
val_ratio : float, default=0.2
Proportion of training data to use for validation in early stopping.
n_epoch_no_change : int, default=20
Number of epochs with no improvement after which training will be stopped.
device : string, default=None
Computing device to use for training ('cpu', 'cuda', etc.).
n_jobs : int, default=10
Number of parallel processes for computation (-1 for using all available cores).
verbose : bool, default=False
If True, prints training progress and statistics.
random_state : int, default=0
Seed for reproducible random number generation.
Attributes
----------
net_ : torch.nn.Module
Trained neural network model.
train_epoch_loss_ : list of float
History of training loss values for each epoch.
validation_epoch_loss_ : list of float
History of validation loss values for each epoch.
"""
def __init__(self,
name: str = None,
hidden_layer_sizes=(40, 40),
max_epochs=1000,
learning_rate=0.001,
batch_size=500,
l1_reg=1e-5,
val_ratio=0.2,
n_epoch_no_change=20,
device=None,
n_jobs=10,
verbose=False,
random_state=0):
self.name = name
super().__init__(
hidden_layer_sizes=hidden_layer_sizes,
loss_fn=nn.MSELoss(),
max_epochs=max_epochs,
learning_rate=learning_rate,
batch_size=batch_size,
l1_reg=l1_reg,
val_ratio=val_ratio,
n_epoch_no_change=n_epoch_no_change,
verbose=verbose,
device=device,
n_jobs=n_jobs,
random_state=random_state)
def _predict(self, X):
"""Generate regression predictions for input samples.
Processes input features through the trained neural network to produce
continuous-valued predictions.
Parameters
----------
X : np.ndarray of shape (n_samples, n_features)
Input samples for which to generate predictions.
Returns
-------
pred : np.ndarray of shape (n_samples,)
Predicted values for each input sample.
"""
X = check_array(X)
pred = self.get_raw_output(X).detach().cpu().numpy().ravel()
return pred
[docs]
class MoReLUDNNClassifier(ClassifierMixin, ReLUDNN, ModelBaseClassifier):
"""A deep neural network classifier using ReLU activation functions.
This model implements a multi-layer neural network for binary classification tasks,
using ReLU activation functions and supporting early stopping, L1 regularization,
and batch training.
Parameters
----------
name : str, default=None
Optional identifier for the model instance.
hidden_layer_sizes : tuple of int, default=(40, 40)
Architecture of the neural network specified as a tuple of integers, where each
integer represents the number of neurons in a hidden layer.
max_epochs : int, default=1000
Maximum number of complete passes through the training dataset.
learning_rate : float, default=0.001
Step size used for gradient updates during optimization.
batch_size : int, default=500
Number of training samples used in each gradient update.
l1_reg : float, default=1e-5
Strength of L1 regularization applied to model weights.
val_ratio : float, default=0.2
Proportion of training data to use for validation in early stopping.
n_epoch_no_change : int, default=20
Number of epochs with no improvement after which training will be stopped.
device : string, default=None
Computing device to use for training ('cpu', 'cuda', etc.).
n_jobs : int, default=10
Number of parallel processes for computation (-1 for using all available cores).
verbose : bool, default=False
If True, prints training progress and statistics.
random_state : int, default=0
Seed for reproducible random number generation.
Attributes
----------
net_ : torch.nn.Module
Trained neural network model.
train_epoch_loss_ : list of float
History of training loss values for each epoch.
validation_epoch_loss_ : list of float
History of validation loss values for each epoch.
"""
def __init__(self,
name: str = None,
hidden_layer_sizes=(40, 40),
max_epochs=1000,
learning_rate=0.001,
batch_size=500,
l1_reg=1e-5,
val_ratio=0.2,
n_epoch_no_change=20,
device=None,
n_jobs=10,
verbose=False,
random_state=0):
self.name = name
super().__init__(
hidden_layer_sizes=hidden_layer_sizes,
loss_fn=nn.BCEWithLogitsLoss(),
max_epochs=max_epochs,
learning_rate=learning_rate,
batch_size=batch_size,
l1_reg=l1_reg,
val_ratio=val_ratio,
n_epoch_no_change=n_epoch_no_change,
verbose=verbose,
device=device,
n_jobs=n_jobs,
random_state=random_state)
def _decision_function(self, X):
"""Computes raw decision scores for samples.
Returns the raw decision values before sigmoid transformation, representing the
model's confidence in predicting the positive class. Higher positive values indicate
stronger confidence in class 1, while lower negative values indicate stronger
confidence in class 0.
Parameters
----------
X : array-like of shape (n_samples, n_features)
Input samples for prediction.
Returns
-------
array-like of shape (n_samples,)
Decision function values, where larger values indicate higher confidence
in the positive class.
"""
X = check_array(X)
check_is_fitted(self)
pred = self.get_raw_output(X).detach().cpu().numpy().ravel()
return pred
def _predict_proba(self, X):
"""Predicts class probabilities for samples.
Computes probability estimates for each class by applying softmax transformation
to the decision function outputs. Returns probabilities for both classes, where
the probabilities sum to 1 for each sample.
Parameters
----------
X : array-like of shape (n_samples, n_features)
Input samples for prediction.
Returns
-------
array-like of shape (n_samples, 2)
Probability estimates for each class, where [:, 0] contains probabilities
for class 0 and [:, 1] contains probabilities for class 1.
"""
pred = self._decision_function(X)
pred_proba = softmax(np.vstack([-pred, pred]).T / 2, copy=False)
return pred_proba