Overfitting Analysis (Classification)

This example demonstrates how to analyze model overfitting across different data slices for classification problems using various slicing methods and metrics.

Installation

# To install the required package, use the following command:
# !pip install modeva

Authentication

# To get authentication, use the following command: (To get full access please replace the token to your own token)
# from modeva.utils.authenticate import authenticate
# authenticate(auth_code='eaaa4301-b140-484c-8e93-f9f633c8bacb')

Import required module

from modeva import DataSet
from modeva import TestSuite
from modeva.models import MoLGBMClassifier
from modeva.models import MoXGBClassifier
from modeva.testsuite.utils.slicing_utils import get_data_info

Load and prepare dataset

ds = DataSet()
ds.load(name="TaiwanCredit")
ds.set_random_split()

Train models

model1 = MoXGBClassifier(max_depth=1)
model1.fit(ds.train_x, ds.train_y)

model2 = MoLGBMClassifier(max_depth=2, verbose=-1, random_state=0)
model2.fit(ds.train_x, ds.train_y.ravel())

MoLGBMClassifier(boosting_type='gbdt', class_weight=None, colsample_bytree=1.0,
                 importance_type='split', learning_rate=0.1, max_depth=2,
                 min_child_samples=20, min_child_weight=0.001,
                 min_split_gain=0.0, n_estimators=100, n_jobs=None,
                 num_leaves=31, objective=None, random_state=0, reg_alpha=0.0,
                 reg_lambda=0.0, subsample=1.0, subsample_for_bin=200000,
                 subsample_freq=0, verbose=-1)

In a Jupyter environment, please rerun this cell to show the HTML representation or trust the notebook.
On GitHub, the HTML representation is unable to render, please try loading this page with nbviewer.org.

Conduct slicing analysis for overfit regions

ts = TestSuite(ds, model1)
results = ts.diagnose_slicing_overfit(
    train_dataset="train",
    test_dataset="test",
    features="PAY_1",
    metric="AUC"
)
results.table

	Feature	Segment	train-Size	train-AUC	test-Size	test-AUC	GAP	Threshold	Weak
5	PAY_1	[3.50, 4.40)	62	0.5774	14	0.4500	-0.1274	-0.0049	True
3	PAY_1	[1.70, 2.60)	2154	0.5845	513	0.5323	-0.0522	-0.0049	True
1	PAY_1	[-1.00, -0.10)	4561	0.7070	1125	0.6733	-0.0338	-0.0049	True
2	PAY_1	[0.80, 1.70)	2936	0.6773	752	0.6584	-0.0189	-0.0049	True
0	PAY_1	[-0.10, 0.80)	13975	0.6781	3521	0.6615	-0.0165	-0.0049	True
4	PAY_1	[2.60, 3.50)	253	0.6542	69	0.8119	0.1577	-0.0049	False
6	PAY_1	[4.40, 5.30)	25	0.6154	1	NaN	NaN	-0.0049	False
7	PAY_1	[5.30, 6.20)	10	0.3600	1	NaN	NaN	-0.0049	False
8	PAY_1	[6.20, 7.10)	9	0.5000	0	NaN	NaN	-0.0049	False
9	PAY_1	[7.10, 8.00]	15	0.8571	4	NaN	NaN	-0.0049	False

Visualize the results

results.plot()

Analyze data drift between samples above and under the threshold

data_info = get_data_info(res_value=results.value)["PAY_1"]
data_results = ds.data_drift_test(
    **data_info,
    distance_metric="PSI",
    psi_method="uniform",
    psi_bins=10
)
data_results.plot("summary")

Single feature density plot

data_results.plot(("density", "PAY_1"))

Batch mode 1D slicing analysis

results = ts.diagnose_slicing_overfit(
    train_dataset="train",
    test_dataset="test",
    features=(("PAY_1", ), ("PAY_2",), ("PAY_3", )),
    method="auto-xgb1",
    metric="AUC",
    threshold=0.0,
)
results.table

	Feature	Segment	train-Size	train-AUC	test-Size	test-AUC	GAP	Threshold	Weak
18	PAY_3	[7.00, 8.00)	23	0.8158	4	0.6667	-0.1491	0.0	True
16	PAY_3	[3.00, 4.00)	191	0.7040	49	0.6097	-0.0943	0.0	True
17	PAY_3	[4.00, 7.00)	101	0.6198	19	0.5256	-0.0942	0.0	True
3	PAY_1	[2.00, 3.00)	2154	0.5845	513	0.5323	-0.0522	0.0	True
14	PAY_3	[-1.00, -0.00)	4747	0.6986	1191	0.6643	-0.0343	0.0	True
1	PAY_1	[-1.00, -0.00)	4561	0.7070	1125	0.6733	-0.0338	0.0	True
11	PAY_2	[3.00, 6.00)	376	0.5908	74	0.5621	-0.0287	0.0	True
9	PAY_2	[-1.00, -0.00)	4862	0.6825	1188	0.6543	-0.0282	0.0	True
2	PAY_1	[1.00, 2.00)	2936	0.6773	752	0.6584	-0.0189	0.0	True
0	PAY_1	[-0.00, 1.00)	13975	0.6781	3521	0.6615	-0.0165	0.0	True
8	PAY_2	[-0.00, 1.00)	15556	0.7384	3956	0.7309	-0.0075	0.0	True
13	PAY_3	[-0.00, 2.00)	15857	0.7515	3996	0.7520	0.0005	0.0	False
15	PAY_3	[2.00, 3.00)	3079	0.7341	740	0.7465	0.0123	0.0	False
10	PAY_2	[1.00, 3.00)	3179	0.7133	776	0.7308	0.0175	0.0	False
4	PAY_1	[3.00, 5.00)	315	0.6416	83	0.7743	0.1327	0.0	False
12	PAY_2	[6.00, 7.00]	27	0.7841	6	1.0000	0.2159	0.0	False
5	PAY_1	[5.00, 7.00)	35	0.5359	2	NaN	NaN	0.0	False
6	PAY_1	[7.00, 8.00)	9	0.5000	0	NaN	NaN	0.0	False
7	PAY_1	[8.00, 8.00]	15	0.8571	4	NaN	NaN	0.0	False
19	PAY_3	[8.00, 8.00]	2	1.0000	1	NaN	NaN	0.0	False

Batch mode 1D Slicing (all features by setting features=None)

results = ts.diagnose_slicing_overfit(
    train_dataset="train",
    test_dataset="test",
    features=None,
    method="auto-xgb1",
    metric="AUC",
    threshold=0.0,
)
results.table

	Feature	Segment	train-Size	train-AUC	test-Size	test-AUC	GAP	Threshold	Weak
43	EDUCATION	0.0	364	0.6616	104	0.4362	-0.2254	0.0	True
77	PAY_3	[7.00, 8.00)	23	0.8158	4	0.6667	-0.1491	0.0	True
75	PAY_3	[3.00, 4.00)	191	0.7040	49	0.6097	-0.0943	0.0	True
76	PAY_3	[4.00, 7.00)	101	0.6198	19	0.5256	-0.0942	0.0	True
92	PAY_6	[3.00, 6.00)	198	0.7157	48	0.6252	-0.0905	0.0	True
...	...	...	...	...	...	...	...	...	...
64	PAY_1	[5.00, 7.00)	35	0.5359	2	NaN	NaN	0.0	False
65	PAY_1	[7.00, 8.00)	9	0.5000	0	NaN	NaN	0.0	False
66	PAY_1	[8.00, 8.00]	15	0.8571	4	NaN	NaN	0.0	False
78	PAY_3	[8.00, 8.00]	2	1.0000	1	NaN	NaN	0.0	False
88	PAY_5	[8.00, 8.00]	1	NaN	0	NaN	NaN	0.0	False

134 rows × 9 columns

Analyze data drift for ‘PAY_1’ feature

data_info = get_data_info(res_value=results.value)["PAY_1"]
data_results = ds.data_drift_test(
    **data_info,
    distance_metric="PSI",
    psi_method="uniform",
    psi_bins=10
)
data_results.plot("summary")

2D feature interaction analysis

results = ts.diagnose_slicing_overfit(
    train_dataset="train",
    test_dataset="test",
    features=("PAY_1", "PAY_2"),
    method="uniform",
    metric="AUC",
    threshold=-0.1
)
results.table

	Feature1	Segment1	Feature2	Segment2	train-Size	train-AUC	test-Size	test-AUC	GAP	Threshold	Weak
26	PAY_1	[0.80, 1.70)	PAY_2	[3.80, 4.60)	26	0.8105	6	0.1250	-0.6855	-0.1	True
45	PAY_1	[2.60, 3.50)	PAY_2	[3.00, 3.80)	32	0.6182	9	0.1250	-0.4932	-0.1	True
55	PAY_1	[3.50, 4.40)	PAY_2	[3.00, 3.80)	47	0.5625	11	0.3000	-0.2625	-0.1	True
10	PAY_1	[-1.00, -0.10)	PAY_2	[-0.20, 0.60)	477	0.7393	126	0.6453	-0.0940	-0.1	False
33	PAY_1	[1.70, 2.60)	PAY_2	[1.40, 2.20)	1282	0.5932	309	0.5170	-0.0762	-0.1	False
...	...	...	...	...	...	...	...	...	...	...	...
95	PAY_1	[7.10, 8.00]	PAY_2	[3.00, 3.80)	0	NaN	0	NaN	NaN	-0.1	False
96	PAY_1	[7.10, 8.00]	PAY_2	[3.80, 4.60)	0	NaN	0	NaN	NaN	-0.1	False
97	PAY_1	[7.10, 8.00]	PAY_2	[4.60, 5.40)	0	NaN	0	NaN	NaN	-0.1	False
98	PAY_1	[7.10, 8.00]	PAY_2	[5.40, 6.20)	0	NaN	0	NaN	NaN	-0.1	False
99	PAY_1	[7.10, 8.00]	PAY_2	[6.20, 7.00]	15	0.8571	4	NaN	NaN	-0.1	False

100 rows × 11 columns

Analyze data drift for feature interaction

data_info = get_data_info(res_value=results.value)[("PAY_1", "PAY_2")]
data_results = ds.data_drift_test(
    **data_info,
    distance_metric="PSI",
    psi_method="uniform",
    psi_bins=10
)
data_results.plot("summary")

Model comparison

tsc = TestSuite(ds, models=[model1, model2])
results = tsc.compare_slicing_overfit(
    train_dataset="train",
    test_dataset="test",
    features="PAY_1",
    method="quantile",
    bins=10,
    metric="AUC"
)
results.table

	MoXGBClassifier							MoLGBMClassifier
	Feature	Segment	train-Size	train-AUC	test-Size	test-AUC	GAP	Feature	Segment	train-Size	train-AUC	test-Size	test-AUC	GAP
0	PAY_1	[-1.00, 0.00)	4561	0.7070	1125	0.6733	-0.0338	PAY_1	[2.00, 8.00]	2528	0.6381	602	0.5756	-0.0626
1	PAY_1	[2.00, 8.00]	2528	0.5945	602	0.5620	-0.0325	PAY_1	[-1.00, 0.00)	4561	0.7111	1125	0.6775	-0.0336
2	PAY_1	[1.00, 2.00)	2936	0.6773	752	0.6584	-0.0189	PAY_1	[1.00, 2.00)	2936	0.6877	752	0.6560	-0.0317
3	PAY_1	[0.00, 1.00)	13975	0.6781	3521	0.6615	-0.0165	PAY_1	[0.00, 1.00)	13975	0.6866	3521	0.6615	-0.0251