Sliced Performance (Classification)

This example demonstrates how to analyze model performance across different data slices for classification problems using various slicing methods and metrics.

Installation

# To install the required package, use the following command:
# !pip install modeva

Authentication

# To get authentication, use the following command: (To get full access please replace the token to your own token)
# from modeva.utils.authenticate import authenticate
# authenticate(auth_code='eaaa4301-b140-484c-8e93-f9f633c8bacb')

Import modeva modules

from modeva import DataSet
from modeva import TestSuite
from modeva.models import MoLGBMClassifier
from modeva.models import MoXGBClassifier
from modeva.testsuite.utils.slicing_utils import get_data_info

Load and prepare data

ds = DataSet(name="TaiwanCredit")
ds.load("TaiwanCredit")
ds.set_target("FlagDefault")
ds.set_inactive_features(["SEX", "MARRIAGE", "AGE"])
ds.set_random_split()

Fit a XGBoost model

model1 = MoXGBClassifier()
model1.fit(ds.train_x, ds.train_y)

MoXGBClassifier(base_score=None, booster=None, callbacks=None,
                colsample_bylevel=None, colsample_bynode=None,
                colsample_bytree=None, device=None, early_stopping_rounds=None,
                enable_categorical=False, eval_metric=None, feature_types=None,
                gamma=None, grow_policy=None, importance_type=None,
                interaction_constraints=None, learning_rate=None, max_bin=None,
                max_cat_threshold=None, max_cat_to_onehot=None,
                max_delta_step=None, max_depth=None, max_leaves=None,
                min_child_weight=None, missing=nan, monotone_constraints=None,
                multi_strategy=None, n_estimators=None, n_jobs=None,
                num_parallel_tree=None, objective='binary:logistic', ...)

In a Jupyter environment, please rerun this cell to show the HTML representation or trust the notebook.
On GitHub, the HTML representation is unable to render, please try loading this page with nbviewer.org.

Fit a LGBM model

model2 = MoLGBMClassifier(max_depth=2, verbose=-1, random_state=0)
model2.fit(ds.train_x, ds.train_y.ravel().astype(float))

MoLGBMClassifier(boosting_type='gbdt', class_weight=None, colsample_bytree=1.0,
                 importance_type='split', learning_rate=0.1, max_depth=2,
                 min_child_samples=20, min_child_weight=0.001,
                 min_split_gain=0.0, n_estimators=100, n_jobs=None,
                 num_leaves=31, objective=None, random_state=0, reg_alpha=0.0,
                 reg_lambda=0.0, subsample=1.0, subsample_for_bin=200000,
                 subsample_freq=0, verbose=-1)

In a Jupyter environment, please rerun this cell to show the HTML representation or trust the notebook.
On GitHub, the HTML representation is unable to render, please try loading this page with nbviewer.org.

Basic slice accuracy analysis

Analyze residual feature importance

ts = TestSuite(ds, model1)

Categorical feature slicing

results = ts.diagnose_slicing_accuracy(features="EDUCATION", metric="AUC",
                                       threshold=0.65)
results.table

	Feature	Segment	Size	AUC	Threshold	Weak
0	EDUCATION	0.0	104	0.6146	0.65	True
2	EDUCATION	2.0	2809	0.7578	0.65	False
3	EDUCATION	3.0	987	0.7589	0.65	False
1	EDUCATION	1.0	2100	0.7714	0.65	False

Uniform binning (Numerical feature)

results = ts.diagnose_slicing_accuracy(features=(("LIMIT_BAL", ), ("PAY_1", )),
                                       method="uniform",
                                       bins=10, metric="AUC",
                                       threshold=0.65)
results.plot(figsize=(6, 5))

Quantile binning (Numerical feature)

results = ts.diagnose_slicing_accuracy(features="LIMIT_BAL",
                                       method="quantile",
                                       bins=10, metric="AUC",
                                       threshold=0.65)
results.plot(figsize=(6, 5))

auto-xgb1 binning (Numerical feature)

results = ts.diagnose_slicing_accuracy(features="LIMIT_BAL",
                                       method="auto-xgb1",
                                       bins=10,
                                       metric="AUC",
                                       threshold=0.75)
results.plot(figsize=(6, 5))

Custom binning (Numerical feature)

results = ts.diagnose_slicing_accuracy(features="LIMIT_BAL",
                                       method="precompute",
                                       bins={"LIMIT_BAL": (0.0, 50000, 1000000)},
                                       metric="AUC")
results.table

	Feature	Segment	Size	AUC	Threshold	Weak
0	LIMIT_BAL	[0.00, 50000.00)	866	0.7369	0.7658	True
1	LIMIT_BAL	[50000.00, 1000000.00]	5134	0.7554	0.7658	True

Advanced slicing analysis

Batch mode 1D Slicing

results = ts.diagnose_slicing_accuracy(features=(("PAY_1", ), ("BILL_AMT1",), ("PAY_AMT1", )),
                                       method="quantile", metric="AUC", threshold=0.6)
results.table

	Feature	Segment	Size	AUC	Threshold	Weak
0	PAY_1	[2.00, 8.00]	602	0.5797	0.6	True
1	PAY_1	[1.00, 2.00)	752	0.6261	0.6	False
2	PAY_1	[-1.00, 0.00)	1125	0.6460	0.6	False
3	BILL_AMT1	[-4.03, 2.30)	597	0.6509	0.6	False
4	PAY_1	[0.00, 1.00)	3521	0.6531	0.6	False
5	BILL_AMT1	[2.30, 3.25)	603	0.6539	0.6	False
6	PAY_AMT1	[2.48, 3.10)	600	0.7069	0.6	False
7	PAY_AMT1	[4.02, 5.61]	600	0.7113	0.6	False
8	PAY_AMT1	[3.10, 3.23)	603	0.7170	0.6	False
9	PAY_AMT1	[3.78, 4.02)	600	0.7170	0.6	False
10	PAY_AMT1	[0.00, 2.48)	1197	0.7305	0.6	False
11	BILL_AMT1	[5.14, 5.98]	600	0.7372	0.6	False
12	BILL_AMT1	[3.25, 3.76)	600	0.7465	0.6	False
13	PAY_AMT1	[3.63, 3.78)	600	0.7575	0.6	False
14	PAY_AMT1	[3.23, 3.32)	578	0.7669	0.6	False
15	PAY_AMT1	[3.32, 3.48)	492	0.7690	0.6	False
16	BILL_AMT1	[4.91, 5.14)	600	0.7854	0.6	False
17	BILL_AMT1	[3.76, 4.11)	600	0.7896	0.6	False
18	BILL_AMT1	[4.35, 4.56)	600	0.7918	0.6	False
19	BILL_AMT1	[4.56, 4.71)	600	0.7956	0.6	False
20	PAY_AMT1	[3.48, 3.63)	730	0.7996	0.6	False
21	BILL_AMT1	[4.71, 4.91)	600	0.8050	0.6	False
22	BILL_AMT1	[4.11, 4.35)	600	0.8069	0.6	False

Batch mode 1D Slicing (all features by setting features=None)

results = ts.diagnose_slicing_accuracy(features=None,
                                       method="quantile", metric="AUC", threshold=0.6)
results.table

	Feature	Segment	Size	AUC	Threshold	Weak
0	PAY_1	[2.00, 8.00]	602	0.5797	0.6	True
1	BILL_AMT2	[-4.42, 0.00)	132	0.5963	0.6	True
2	BILL_AMT3	[-4.79, 0.00)	126	0.6104	0.6	False
3	EDUCATION	0.0	104	0.6146	0.6	False
4	PAY_1	[1.00, 2.00)	752	0.6261	0.6	False
...	...	...	...	...	...	...
137	BILL_AMT2	[4.53, 4.70)	600	0.8203	0.6	False
138	BILL_AMT3	[4.49, 4.69)	600	0.8277	0.6	False
139	BILL_AMT6	[4.59, 4.80)	600	0.8286	0.6	False
140	BILL_AMT3	[4.07, 4.30)	600	0.8312	0.6	False
141	BILL_AMT4	[4.06, 4.28)	601	0.8458	0.6	False

142 rows × 6 columns

Slicing with 2 features

results = ts.diagnose_slicing_accuracy(features=("PAY_1", "PAY_AMT1"),
                                       method="uniform",
                                       bins=10,
                                       metric="AUC",
                                       threshold=0.5)
results.table

	Feature1	Segment1	Feature2	Segment2	Size	AUC	Threshold	Weak
9	PAY_1	[-1.00, -0.10)	PAY_AMT1	[5.05, 5.61]	6	0.2500	0.5	True
50	PAY_1	[3.50, 4.40)	PAY_AMT1	[0.00, 0.56)	11	0.3000	0.5	True
24	PAY_1	[0.80, 1.70)	PAY_AMT1	[2.24, 2.80)	28	0.4167	0.5	True
36	PAY_1	[1.70, 2.60)	PAY_AMT1	[3.36, 3.93)	199	0.4895	0.5	True
22	PAY_1	[0.80, 1.70)	PAY_AMT1	[1.12, 1.68)	5	0.5000	0.5	True
...	...	...	...	...	...	...	...	...
95	PAY_1	[7.10, 8.00]	PAY_AMT1	[2.80, 3.36)	0	NaN	0.5	False
96	PAY_1	[7.10, 8.00]	PAY_AMT1	[3.36, 3.93)	0	NaN	0.5	False
97	PAY_1	[7.10, 8.00]	PAY_AMT1	[3.93, 4.49)	0	NaN	0.5	False
98	PAY_1	[7.10, 8.00]	PAY_AMT1	[4.49, 5.05)	0	NaN	0.5	False
99	PAY_1	[7.10, 8.00]	PAY_AMT1	[5.05, 5.61]	0	NaN	0.5	False

100 rows × 8 columns

Test distributional difference between weak samples and the rest

data_info = get_data_info(res_value=results.value)
data_results = ds.data_drift_test(**data_info[("PAY_1", "PAY_AMT1")],
                                  distance_metric="PSI",
                                  psi_method="uniform",
                                  psi_bins=10)
data_results.plot("summary")

Get the list of available figure names in the result object

data_results.get_figure_names()

['summary', ('density', 'LIMIT_BAL'), ('density', 'EDUCATION'), ('density', 'PAY_1'), ('density', 'PAY_2'), ('density', 'PAY_3'), ('density', 'PAY_4'), ('density', 'PAY_5'), ('density', 'PAY_6'), ('density', 'BILL_AMT1'), ('density', 'BILL_AMT2'), ('density', 'BILL_AMT3'), ('density', 'BILL_AMT4'), ('density', 'BILL_AMT5'), ('density', 'BILL_AMT6'), ('density', 'PAY_AMT1'), ('density', 'PAY_AMT2'), ('density', 'PAY_AMT3'), ('density', 'PAY_AMT4'), ('density', 'PAY_AMT5'), ('density', 'PAY_AMT6'), ('density', 'FlagDefault')]

Generate a plot in the result object using the figure name

data_results.plot(('density', 'PAY_AMT6'))

Model comparison

Model Comparison of 1D slicing accuracy (Numerical feature)

tsc = TestSuite(ds, models=[model1, model2])
results = tsc.compare_slicing_accuracy(features="PAY_AMT1", method="quantile",
                                       bins=10, metric="AUC")
results.plot(figsize=(6, 5))

Model Comparison of 1D slicing accuracy (Categorical feature)

results = tsc.compare_slicing_accuracy(features="EDUCATION", metric="AUC",
                                       threshold=0.6)
results.plot(figsize=(6, 5))