Overfitting Analysis (Classification)

This example demonstrates how to analyze model overfitting across different data slices for classification problems using various slicing methods and metrics.

Installation

# To install the required package, use the following command:
# !pip install modeva

Authentication

# To get authentication, use the following command: (To get full access please replace the token to your own token)
# from modeva.utils.authenticate import authenticate
# authenticate(auth_code='eaaa4301-b140-484c-8e93-f9f633c8bacb')

Import required module

from modeva import DataSet
from modeva import TestSuite
from modeva.models import MoLGBMClassifier
from modeva.models import MoXGBClassifier
from modeva.testsuite.utils.slicing_utils import get_data_info

Load and prepare dataset

ds = DataSet()
ds.load(name="TaiwanCredit")
ds.set_random_split()

Train models

model1 = MoXGBClassifier(max_depth=1)
model1.fit(ds.train_x, ds.train_y)

model2 = MoLGBMClassifier(max_depth=2, verbose=-1, random_state=0)
model2.fit(ds.train_x, ds.train_y.ravel())

MoLGBMClassifier(boosting_type='gbdt', class_weight=None, colsample_bytree=1.0,
                 importance_type='split', learning_rate=0.1, max_depth=2,
                 min_child_samples=20, min_child_weight=0.001,
                 min_split_gain=0.0, n_estimators=100, n_jobs=None,
                 num_leaves=31, objective=None, random_state=0, reg_alpha=0.0,
                 reg_lambda=0.0, subsample=1.0, subsample_for_bin=200000,
                 subsample_freq=0, verbose=-1)

In a Jupyter environment, please rerun this cell to show the HTML representation or trust the notebook.
On GitHub, the HTML representation is unable to render, please try loading this page with nbviewer.org.

Conduct slicing analysis for overfit regions

ts = TestSuite(ds, model1)
results = ts.diagnose_slicing_overfit(
    train_dataset="train",
    test_dataset="test",
    features="PAY_1",
    metric="AUC"
)
results.table

	Feature	Segment	train-Size	train-AUC	test-Size	test-AUC	GAP	Threshold	Weak
5	PAY_1	[3.50, 4.40)	62	0.5714	14	0.4500	-0.1214	-0.0051	True
3	PAY_1	[1.70, 2.60)	2154	0.5843	513	0.5327	-0.0516	-0.0051	True
1	PAY_1	[-1.00, -0.10)	4561	0.7069	1125	0.6744	-0.0325	-0.0051	True
2	PAY_1	[0.80, 1.70)	2936	0.6773	752	0.6584	-0.0188	-0.0051	True
0	PAY_1	[-0.10, 0.80)	13975	0.6780	3521	0.6599	-0.0181	-0.0051	True
4	PAY_1	[2.60, 3.50)	253	0.6548	69	0.8136	0.1588	-0.0051	False
6	PAY_1	[4.40, 5.30)	25	0.6282	1	NaN	NaN	-0.0051	False
7	PAY_1	[5.30, 6.20)	10	0.4000	1	NaN	NaN	-0.0051	False
8	PAY_1	[6.20, 7.10)	9	0.5000	0	NaN	NaN	-0.0051	False
9	PAY_1	[7.10, 8.00]	15	0.8571	4	NaN	NaN	-0.0051	False

Visualize the results

results.plot()

Analyze data drift between samples above and under the threshold

data_info = get_data_info(res_value=results.value)["PAY_1"]
data_results = ds.data_drift_test(
    **data_info,
    distance_metric="PSI",
    psi_method="uniform",
    psi_bins=10
)
data_results.plot("summary")

Single feature density plot

data_results.plot(("density", "PAY_1"))

Batch mode 1D slicing analysis

results = ts.diagnose_slicing_overfit(
    train_dataset="train",
    test_dataset="test",
    features=(("PAY_1", ), ("PAY_2",), ("PAY_3", )),
    method="auto-xgb1",
    metric="AUC",
    threshold=0.0,
)
results.table

	Feature	Segment	train-Size	train-AUC	test-Size	test-AUC	GAP	Threshold	Weak
18	PAY_3	[7.00, 8.00)	23	0.8158	4	0.6667	-0.1491	0.0	True
17	PAY_3	[4.00, 7.00)	101	0.6238	19	0.5256	-0.0982	0.0	True
16	PAY_3	[3.00, 4.00)	191	0.7034	49	0.6114	-0.0920	0.0	True
3	PAY_1	[2.00, 3.00)	2154	0.5843	513	0.5327	-0.0516	0.0	True
14	PAY_3	[-1.00, -0.00)	4747	0.6986	1191	0.6648	-0.0337	0.0	True
1	PAY_1	[-1.00, -0.00)	4561	0.7069	1125	0.6744	-0.0325	0.0	True
11	PAY_2	[3.00, 6.00)	376	0.5908	74	0.5621	-0.0287	0.0	True
9	PAY_2	[-1.00, -0.00)	4862	0.6829	1188	0.6558	-0.0271	0.0	True
2	PAY_1	[1.00, 2.00)	2936	0.6773	752	0.6584	-0.0188	0.0	True
0	PAY_1	[-0.00, 1.00)	13975	0.6780	3521	0.6599	-0.0181	0.0	True
8	PAY_2	[-0.00, 1.00)	15556	0.7381	3956	0.7297	-0.0084	0.0	True
13	PAY_3	[-0.00, 2.00)	15857	0.7514	3996	0.7512	-0.0002	0.0	True
15	PAY_3	[2.00, 3.00)	3079	0.7341	740	0.7464	0.0124	0.0	False
10	PAY_2	[1.00, 3.00)	3179	0.7131	776	0.7308	0.0176	0.0	False
4	PAY_1	[3.00, 5.00)	315	0.6411	83	0.7754	0.1343	0.0	False
12	PAY_2	[6.00, 7.00]	27	0.7955	6	1.0000	0.2045	0.0	False
5	PAY_1	[5.00, 7.00)	35	0.5588	2	NaN	NaN	0.0	False
6	PAY_1	[7.00, 8.00)	9	0.5000	0	NaN	NaN	0.0	False
7	PAY_1	[8.00, 8.00]	15	0.8571	4	NaN	NaN	0.0	False
19	PAY_3	[8.00, 8.00]	2	1.0000	1	NaN	NaN	0.0	False

Batch mode 1D Slicing (all features by setting features=None)

results = ts.diagnose_slicing_overfit(
    train_dataset="train",
    test_dataset="test",
    features=None,
    method="auto-xgb1",
    metric="AUC",
    threshold=0.0,
)
results.table

	Feature	Segment	train-Size	train-AUC	test-Size	test-AUC	GAP	Threshold	Weak
44	EDUCATION	0.0	364	0.6615	104	0.4336	-0.2279	0.0	True
77	PAY_3	[7.00, 8.00)	23	0.8158	4	0.6667	-0.1491	0.0	True
76	PAY_3	[4.00, 7.00)	101	0.6238	19	0.5256	-0.0982	0.0	True
75	PAY_3	[3.00, 4.00)	191	0.7034	49	0.6114	-0.0920	0.0	True
92	PAY_6	[3.00, 5.00)	187	0.7058	46	0.6161	-0.0897	0.0	True
...	...	...	...	...	...	...	...	...	...
64	PAY_1	[5.00, 7.00)	35	0.5588	2	NaN	NaN	0.0	False
65	PAY_1	[7.00, 8.00)	9	0.5000	0	NaN	NaN	0.0	False
66	PAY_1	[8.00, 8.00]	15	0.8571	4	NaN	NaN	0.0	False
78	PAY_3	[8.00, 8.00]	2	1.0000	1	NaN	NaN	0.0	False
88	PAY_5	[8.00, 8.00]	1	NaN	0	NaN	NaN	0.0	False

135 rows × 9 columns

Analyze data drift for ‘PAY_1’ feature

data_info = get_data_info(res_value=results.value)["PAY_1"]
data_results = ds.data_drift_test(
    **data_info,
    distance_metric="PSI",
    psi_method="uniform",
    psi_bins=10
)
data_results.plot("summary")

2D feature interaction analysis

results = ts.diagnose_slicing_overfit(
    train_dataset="train",
    test_dataset="test",
    features=("PAY_1", "PAY_2"),
    method="uniform",
    metric="AUC",
    threshold=-0.1
)
results.table

	Feature1	Segment1	Feature2	Segment2	train-Size	train-AUC	test-Size	test-AUC	GAP	Threshold	Weak
26	PAY_1	[0.80, 1.70)	PAY_2	[3.80, 4.60)	26	0.8170	6	0.1250	-0.6920	-0.1	True
45	PAY_1	[2.60, 3.50)	PAY_2	[3.00, 3.80)	32	0.6227	9	0.1250	-0.4977	-0.1	True
55	PAY_1	[3.50, 4.40)	PAY_2	[3.00, 3.80)	47	0.5685	11	0.3000	-0.2685	-0.1	True
10	PAY_1	[-1.00, -0.10)	PAY_2	[-0.20, 0.60)	477	0.7347	126	0.6467	-0.0880	-0.1	False
33	PAY_1	[1.70, 2.60)	PAY_2	[1.40, 2.20)	1282	0.5925	309	0.5161	-0.0765	-0.1	False
...	...	...	...	...	...	...	...	...	...	...	...
95	PAY_1	[7.10, 8.00]	PAY_2	[3.00, 3.80)	0	NaN	0	NaN	NaN	-0.1	False
96	PAY_1	[7.10, 8.00]	PAY_2	[3.80, 4.60)	0	NaN	0	NaN	NaN	-0.1	False
97	PAY_1	[7.10, 8.00]	PAY_2	[4.60, 5.40)	0	NaN	0	NaN	NaN	-0.1	False
98	PAY_1	[7.10, 8.00]	PAY_2	[5.40, 6.20)	0	NaN	0	NaN	NaN	-0.1	False
99	PAY_1	[7.10, 8.00]	PAY_2	[6.20, 7.00]	15	0.8571	4	NaN	NaN	-0.1	False

100 rows × 11 columns

Analyze data drift for feature interaction

data_info = get_data_info(res_value=results.value)[("PAY_1", "PAY_2")]
data_results = ds.data_drift_test(
    **data_info,
    distance_metric="PSI",
    psi_method="uniform",
    psi_bins=10
)
data_results.plot("summary")

Model comparison

tsc = TestSuite(ds, models=[model1, model2])
results = tsc.compare_slicing_overfit(
    train_dataset="train",
    test_dataset="test",
    features="PAY_1",
    method="quantile",
    bins=10,
    metric="AUC"
)
results.table

	MoXGBClassifier							MoLGBMClassifier
	Feature	Segment	train-Size	train-AUC	test-Size	test-AUC	GAP	Feature	Segment	train-Size	train-AUC	test-Size	test-AUC	GAP
0	PAY_1	[-1.00, 0.00)	4561	0.7069	1125	0.6744	-0.0325	PAY_1	[2.00, 8.00]	2528	0.6381	602	0.5756	-0.0626
1	PAY_1	[2.00, 8.00]	2528	0.5942	602	0.5623	-0.0319	PAY_1	[-1.00, 0.00)	4561	0.7111	1125	0.6775	-0.0336
2	PAY_1	[1.00, 2.00)	2936	0.6773	752	0.6584	-0.0188	PAY_1	[1.00, 2.00)	2936	0.6877	752	0.6560	-0.0317
3	PAY_1	[0.00, 1.00)	13975	0.6780	3521	0.6599	-0.0181	PAY_1	[0.00, 1.00)	13975	0.6866	3521	0.6615	-0.0251