Tuning with optuna (Experimental)

To run this code, you need to have optuna installed.

Installation

# To install the required package, use the following command:
# !pip install modeva

Authentication

# To get authentication, use the following command: (To get full access please replace the token to your own token)
# from modeva.utils.authenticate import authenticate
# authenticate(auth_code='eaaa4301-b140-484c-8e93-f9f633c8bacb')

Import required modules

from modeva import DataSet
from modeva import TestSuite
from modeva.models import MoLGBMClassifier
from modeva.models import ModelTuneOptuna
from scipy.stats import uniform, randint

Load Dataset

ds = DataSet()
ds.load(name="SimuCredit")
ds.set_random_split()

Run HPO

param_distributions = {"max_depth": [1, 2, 3],
                       "learning_rate": uniform(0.01, 0.3),
                       "n_estimators": randint(1, 100),
                      }

model = MoLGBMClassifier(verbose=-1)
hpo = ModelTuneOptuna(dataset=ds, model=model)
result = hpo.run(param_distributions=param_distributions,
                 sampler="tpe", # "grid", "random", "tpe", "gs", "cma-es", "qmc"
                 metric=("AUC", "ACC", "LogLoss"),
                 cv=5)
result.table

	max_depth	learning_rate	n_estimators	AUC	ACC	LogLoss	AUC_rank	ACC_rank	LogLoss_rank	mean_fit_time
5	2	0.2112	79	0.8381	0.7575	0.4923	1	1	1	0.0372
3	3	0.0354	54	0.8277	0.7476	0.5197	2	8	7	0.0329
1	1	0.2360	91	0.8270	0.7539	0.5102	3	2	2	0.0331
8	1	0.2287	84	0.8264	0.7532	0.5113	4	3	3	0.0256
2	1	0.2938	57	0.8258	0.7522	0.5123	5	4	4	0.0226
0	1	0.3088	50	0.8252	0.7507	0.5131	6	5	5	0.0268
4	1	0.2341	52	0.8233	0.7491	0.5167	7	6	6	0.0230
7	1	0.0881	95	0.8200	0.7484	0.5252	8	7	8	0.0231
6	1	0.1059	31	0.8061	0.7328	0.5564	9	9	9	0.0152
9	1	0.0534	18	0.7729	0.7170	0.6066	10	10	10	0.0125

result.plot("parallel", figsize=(8, 6))

result.plot(("max_depth", "AUC"))

result.plot(("learning_rate", "AUC"))

result.plot(("n_estimators", "AUC"))

Retrain model with best hyperparameter

model_tuned = MoLGBMClassifier(**result.value["params"][0],
                               name="LGBM-Tuned",
                               verbose=-1)
model_tuned.fit(ds.train_x, ds.train_y)
model_tuned

MoLGBMClassifier(boosting_type='gbdt', class_weight=None, colsample_bytree=1.0,
                 importance_type='split', learning_rate=0.3088427504212384,
                 max_depth=1, min_child_samples=20, min_child_weight=0.001,
                 min_split_gain=0.0, n_estimators=50, n_jobs=None,
                 num_leaves=31, objective=None, random_state=None,
                 reg_alpha=0.0, reg_lambda=0.0, subsample=1.0,
                 subsample_for_bin=200000, subsample_freq=0, verbose=-1)

In a Jupyter environment, please rerun this cell to show the HTML representation or trust the notebook.
On GitHub, the HTML representation is unable to render, please try loading this page with nbviewer.org.

Diagnose the tuned model

ts = TestSuite(ds, model_tuned)
result = ts.diagnose_accuracy_table()
result.table

	AUC	ACC	F1	LogLoss	Precision	Recall	Brier
train	0.8288	0.7542	7.8598e-01	0.5089	0.7572	0.8171	1.6843e-01
test	0.8275	0.7588	7.8608e-01	0.5097	0.7646	0.8089	1.6842e-01
GAP	-0.0013	0.0046	9.5997e-05	0.0007	0.0074	-0.0082	-1.8027e-05