Wrapping Scored Regressor

This example requires full licence, and the program will break if you use the trial licence.

Installation

# To install the required package, use the following command:
# !pip install modeva

Authentication

# To get authentication, use the following command: (To get full access please replace the token to your own token)
# from modeva.utils.authenticate import authenticate
# authenticate(auth_code='eaaa4301-b140-484c-8e93-f9f633c8bacb')

Import modeva modules

import numpy as np
import pandas as pd
from modeva import DataSet
from modeva import TestSuite
from modeva.models import MoXGBRegressor
from modeva.models import MoScoredRegressor
from sklearn.datasets import make_friedman1
from sklearn.model_selection import train_test_split

Build a model and save the prediction

X, y = make_friedman1(n_samples=10000, n_features=10, noise=0.1, random_state=2024)
X_train, X_test, y_train, y_test, train_indices, test_indices = train_test_split(
    X, y, np.arange((len(X))), test_size=0.2, random_state=42)

model1 = MoXGBRegressor(max_depth=1)
model1.fit(X_train, y_train)
prediction1 = model1.predict(X)

model2 = MoXGBRegressor(max_depth=2)
model2.fit(X_train, y_train)
prediction2 = model2.predict(X)

data = pd.DataFrame(np.concatenate([X, y.reshape(-1, 1),
                                    prediction1.reshape(-1, 1),
                                    prediction2.reshape(-1, 1)], 1),
                    columns=['X' + str(i) for i in range(X.shape[1])] + ['Y', "pred_XGB1", "pred_XGB2"])

Wrap the data into Modeva

ds = DataSet(name="scored-test-demo")
ds.load_dataframe(data)
ds.set_train_idx(train_idx=train_indices)
ds.set_test_idx(test_idx=test_indices)
ds.set_target(feature="Y")
ds.set_inactive_features(("pred_XGB1", "pred_XGB2"))

Convert the model into Modeva

scored_model1 = MoScoredRegressor(dataset=ds, prediction_name="pred_XGB1")
scored_model2 = MoScoredRegressor(dataset=ds, prediction_name="pred_XGB2")

Create test suite for diagnostics

Note that the robustness test is not available for scored model

ts = TestSuite(ds, scored_model1)

Run accuracy test without the model object

results = ts.diagnose_accuracy_table()
results.table

	MSE	MAE	R2
train	1.9714	1.1095	0.9161
test	1.9887	1.1226	0.9174
GAP	0.0173	0.0131	0.0013

Run residual analysis test without the model object

results = ts.diagnose_residual_analysis(features="X1")
results.table

	X1	Residual
0	0.6991	1.2913
1	0.7403	0.9713
2	0.3797	-0.4222
3	0.8710	1.7552
4	0.8201	-1.2072
...	...	...
1995	0.4417	-0.6784
1996	0.8035	0.7619
1997	0.8085	-1.5558
1998	0.9068	-0.7691
1999	0.0849	-1.3383

2000 rows × 2 columns

Run reliability test without the model object

results = ts.diagnose_reliability()
results.table

	Avg.Width	Avg.Coverage
0	3.3759	0.912

Run resilience test without the model object

results = ts.diagnose_resilience()
results.table

	MSE
0.1	9.5238
0.2	6.4691
0.3	5.1016
0.4	4.2515
0.5	3.6352
0.6	3.1641
0.7	2.7847
0.8	2.4703
0.9	2.2078
1.0	1.9887

Run slicing accuracy test without the model object

results = ts.diagnose_slicing_accuracy(features="X1",
                                       dataset="main",
                                       metric="MAE",
                                       threshold=0)
results.table

	Feature	Segment	Size	MAE	Threshold	Weak
9	X1	[0.90, 1.00]	996	1.8278	0	True
0	X1	[0.00, 0.10)	989	1.5007	0	True
8	X1	[0.80, 0.90)	926	1.2352	0	True
4	X1	[0.40, 0.50)	976	1.0533	0	True
5	X1	[0.50, 0.60)	1011	0.9973	0	True
1	X1	[0.10, 0.20)	1019	0.9530	0	True
3	X1	[0.30, 0.40)	1003	0.9523	0	True
7	X1	[0.70, 0.80)	1016	0.9244	0	True
6	X1	[0.60, 0.70)	1049	0.9131	0	True
2	X1	[0.20, 0.30)	1015	0.8011	0	True

Run slicing overfit test without the model object

results = ts.diagnose_slicing_overfit(features="X1",
                                      train_dataset="train",
                                      test_dataset="test",
                                      metric="MAE")
results.table

	Feature	Segment	train-Size	train-MAE	test-Size	test-MAE	GAP	Threshold	Weak
2	X1	[0.20, 0.30)	812	0.7886	203	0.8513	0.0627	0.0131	True
3	X1	[0.30, 0.40)	805	0.9427	198	0.9912	0.0485	0.0131	True
7	X1	[0.70, 0.80)	789	0.9149	227	0.9576	0.0427	0.0131	True
6	X1	[0.60, 0.70)	848	0.9070	201	0.9389	0.0319	0.0131	True
0	X1	[0.00, 0.10)	780	1.4956	209	1.5199	0.0243	0.0131	True
1	X1	[0.10, 0.20)	830	0.9494	189	0.9685	0.0191	0.0131	True
8	X1	[0.80, 0.90)	747	1.2342	179	1.2396	0.0054	0.0131	False
5	X1	[0.50, 0.60)	802	0.9984	209	0.9930	-0.0054	0.0131	False
9	X1	[0.90, 1.00]	800	1.8350	196	1.7981	-0.0369	0.0131	False
4	X1	[0.40, 0.50)	787	1.0681	189	0.9920	-0.0761	0.0131	False

Compare two scored models

tsc = TestSuite(ds, models=[scored_model1, scored_model2])

Run accuracy test without the model object

results = tsc.compare_accuracy_table()
results.table

	MoScoredRegressor
	MSE	MAE	R2
train	0.4907	0.5574	0.9791
test	0.5554	0.5945	0.9769
GAP	0.0647	0.0371	-0.0022

Run slicing accuracy test without the model object

results = tsc.compare_slicing_accuracy(features="X1",
                                       dataset="main",
                                       metric="MAE",
                                       threshold=0)
results.table

	MoScoredRegressor
	Feature	Segment	Size	MAE
0	X1	[0.90, 1.00]	996	0.7049
1	X1	[0.80, 0.90)	926	0.6790
2	X1	[0.10, 0.20)	1019	0.6158
3	X1	[0.60, 0.70)	1049	0.5929
4	X1	[0.40, 0.50)	976	0.5430
5	X1	[0.50, 0.60)	1011	0.5427
6	X1	[0.20, 0.30)	1015	0.5321
7	X1	[0.30, 0.40)	1003	0.5234
8	X1	[0.00, 0.10)	989	0.4713
9	X1	[0.70, 0.80)	1016	0.4513