Wrapping Scored Classifier

This example requires full licence, and the program will break if you use the trial licence.

Installation

# To install the required package, use the following command:
# !pip install modeva

Authentication

# To get authentication, use the following command: (To get full access please replace the token to your own token)
# from modeva.utils.authenticate import authenticate
# authenticate(auth_code='eaaa4301-b140-484c-8e93-f9f633c8bacb')

Import modeva modules

import numpy as np
import pandas as pd
from modeva import DataSet
from modeva import TestSuite
from modeva.models import MoXGBClassifier
from modeva.models import MoScoredClassifier
from sklearn.datasets import make_classification
from sklearn.model_selection import train_test_split

Build a model and save the prediction

X, y = make_classification(n_samples=10000, n_features=2,
                           n_redundant=0, random_state=42)
X_train, X_test, y_train, y_test, train_indices, test_indices = train_test_split(
    X, y, np.arange((len(X))), test_size=0.2, random_state=42)

model1 = MoXGBClassifier(max_depth=1)
model1.fit(X_train, y_train)
proba1 = model1.predict_proba(X)[:, 1]

model2 = MoXGBClassifier(max_depth=2)
model2.fit(X_train, y_train)
proba2 = model2.predict_proba(X)[:, 1]

data = pd.DataFrame(np.concatenate([X, y.reshape(-1, 1),
                                    proba1.reshape(-1, 1), proba2.reshape(-1, 1)], 1),
                    columns=['X' + str(i) for i in range(X.shape[1])] + ['Y', "proba_XGB1", "proba_XGB2"])

Wrap the data into Modeva

ds = DataSet(name="scored-test-demo")
ds.load_dataframe(data)
ds.set_train_idx(train_idx=train_indices)
ds.set_test_idx(test_idx=test_indices)
ds.set_target(feature="Y")
ds.set_inactive_features(("proba_XGB1", "proba_XGB2"))

Convert the model into Modeva

scored_model1 = MoScoredClassifier(dataset=ds, prediction_proba_name="proba_XGB1")
scored_model2 = MoScoredClassifier(dataset=ds, prediction_proba_name="proba_XGB2")

Create test suite for diagnostics

Note that the robustness test is not available for scored model

ts = TestSuite(ds, scored_model1)

Run accuracy test without the model object

results = ts.diagnose_accuracy_table()
results.table

	AUC	ACC	F1	LogLoss	Precision	Recall	Brier
train	0.9642	0.9324	0.9332	0.2241	0.9226	0.9441	0.0583
test	0.9612	0.9175	0.9168	0.2383	0.9182	0.9154	0.0639
GAP	-0.0030	-0.0149	-0.0164	0.0142	-0.0045	-0.0287	0.0056

Run residual analysis test without the model object

results = ts.diagnose_residual_analysis(features="X1")
results.table

	X1	Residual
0	-1.2623	-0.0593
1	1.6823	0.1063
2	0.2985	0.1765
3	-0.1532	-0.3667
4	-0.7708	-0.1150
...	...	...
1995	3.1704	0.0908
1996	-0.3181	0.7462
1997	1.2174	0.0333
1998	-0.8111	-0.0497
1999	0.8588	0.1487

2000 rows × 2 columns

Run reliability test without the model object

results = ts.diagnose_reliability()
results.table

	Avg.Width	Avg.Coverage
0	0.985	0.917

Run resilience test without the model object

results = ts.diagnose_resilience()
results.table

	AUC
0.1	0.0634
0.2	0.5436
0.3	0.7263
0.4	0.8224
0.5	0.8723
0.6	0.9039
0.7	0.9255
0.8	0.9406
0.9	0.9521
1.0	0.9612

Run slicing accuracy test without the model object

results = ts.diagnose_slicing_accuracy(features="X1",
                                       dataset="main",
                                       metric="AUC",
                                       threshold=0)
results.table

	Feature	Segment	Size	AUC	Threshold	Weak
2	X1	[-3.18, -2.24)	298	0.3547	0	False
9	X1	[3.38, 4.31]	23	0.4000	0	False
3	X1	[-2.24, -1.31)	1363	0.4476	0	False
7	X1	[1.50, 2.44)	1196	0.6436	0	False
8	X1	[2.44, 3.38)	205	0.6710	0	False
6	X1	[0.57, 1.50)	2356	0.7815	0	False
4	X1	[-1.31, -0.37)	2511	0.8428	0	False
5	X1	[-0.37, 0.57)	1953	0.9333	0	False
0	X1	[-5.05, -4.11)	10	NaN	0	False
1	X1	[-4.11, -3.18)	85	NaN	0	False

Run slicing overfit test without the model object

results = ts.diagnose_slicing_overfit(features="X1",
                                      train_dataset="train",
                                      test_dataset="test",
                                      metric="LogLoss")
results.table

	Feature	Segment	train-Size	train-LogLoss	test-Size	test-LogLoss	GAP	Threshold	Weak
0	X1	[-0.60, 0.29)	1515	0.3323	371	0.4043	0.0720	0.0142	True
2	X1	[-2.38, -1.49)	771	0.1176	223	0.1798	0.0622	0.0142	True
8	X1	[2.07, 2.96)	376	0.1154	92	0.1411	0.0257	0.0142	True
1	X1	[-1.49, -0.60)	1913	0.1970	481	0.2140	0.0170	0.0142	True
4	X1	[-4.16, -3.27)	62	0.0143	11	0.0150	0.0007	0.0142	False
6	X1	[0.29, 1.18)	1821	0.2673	424	0.2570	-0.0102	0.0142	False
7	X1	[1.18, 2.07)	1303	0.2029	323	0.1769	-0.0260	0.0142	False
3	X1	[-3.27, -2.38)	180	0.0570	57	0.0117	-0.0452	0.0142	False
9	X1	[2.96, 3.85]	49	0.3287	18	0.2005	-0.1282	0.0142	False
5	X1	[-5.05, -4.16)	10	0.0033	0	NaN	NaN	0.0142	False

Compare two scored models

tsc = TestSuite(ds, models=[scored_model1, scored_model2])

Run accuracy test without the model object

results = tsc.compare_accuracy_table()
results.table

	MoScoredClassifier
	AUC	ACC	F1	LogLoss	Precision	Recall	Brier
train	0.9788	0.9449	0.9457	0.1737	0.9329	0.9588	0.0467
test	0.9728	0.9295	0.9291	0.2005	0.9277	0.9305	0.0544
GAP	-0.0061	-0.0154	-0.0166	0.0269	-0.0052	-0.0283	0.0078

Run slicing accuracy test without the model object

results = tsc.compare_slicing_accuracy(features="X1",
                                       dataset="test",
                                       metric="AUC")
results.table

	MoScoredClassifier
	Feature	Segment	Size	AUC
0	X1	[3.49, 4.31]	8	0.5000
1	X1	[1.86, 2.67)	124	0.5785
2	X1	[1.04, 1.86)	345	0.7765
3	X1	[-2.23, -1.42)	254	0.8031
4	X1	[-1.42, -0.60)	431	0.8527
5	X1	[0.22, 1.04)	383	0.9142
6	X1	[-0.60, 0.22)	342	0.9169
7	X1	[-3.05, -2.23)	69	0.9926
8	X1	[-3.87, -3.05)	18	NaN
9	X1	[2.67, 3.49)	26	NaN