Missing Value Indicator

Impute missing and special (sentinel) values while adding binary indicator features that flag which rows were affected. The indicators are ordinary model features, so they flow thr

Open In Colab 

# %%
# Installation

# To install the required package, use the following command:
# !pip install modeva

# %%
# Authentication

# To get authentication, use the following command: (To get full access please replace the token to your own token)
# from modeva.utils.authenticate import authenticate
# authenticate(auth_code='YOUR_LICENSE_KEY')

# %%
# Import required modules
import warnings
warnings.filterwarnings("ignore")

import numpy as np
from modeva import DataSet, TestSuite
from modeva.models import MoXGBClassifier

# %%
# Inject missing and special values
# ----------------------------------------------------------
# TaiwanCredit is fully observed, so we inject synthetic missingness (``NaN``)
# and a sentinel special value (``-999``) into two numeric features to
# demonstrate the imputation workflow.
ds = DataSet()
ds.load(name="TaiwanCredit")
df = ds.data.copy()
df.loc[500:599, "LIMIT_BAL"] = np.nan     # missing values
df.loc[600:699, "BILL_AMT1"] = -999       # special sentinel value

ds = DataSet()
ds.load_dataframe(df)
ds.set_target("FlagDefault")

# %%
# Data summary
# ----------------------------------------------------------
ds.summary().table["summary"]

# %%
# Impute with indicators
# ----------------------------------------------------------
# ``impute_missing`` fills missing values (and, when given, ``special_values``)
# and, with ``add_indicators=True``, appends a binary column marking the
# affected rows. The steps are lazy; ``preprocess`` applies them.
ds.reset_preprocess()
ds.impute_missing(features=("LIMIT_BAL",), method="mean",
                  add_indicators=True)
ds.impute_missing(features=("BILL_AMT1",), method="mean",
                  special_values=[-999], add_indicators=True)
ds.preprocess()

# %%
# The new indicator columns
# ----------------------------------------------------------
# Indicators are named ``<feature>_missing_<missing_value>`` and
# ``<feature>_special_<value>``.
indicators = [c for c in ds.data.columns if "_missing_" in c or "_special_" in c]
indicators

# %%
# EDA of an indicator column
# ----------------------------------------------------------
result = ds.eda_1d(feature="BILL_AMT1_special_-999", plot_type="histogram")
result.plot()

# %%
# Train a model on the augmented feature set
# ----------------------------------------------------------
# The indicator columns are part of ``feature_names``, so any model trains on
# them alongside the original features.
ds.set_random_split()
model = MoXGBClassifier(name="XGB", max_depth=2, random_state=0, verbosity=0)
model.fit(ds.train_x, ds.train_y.ravel())

ts = TestSuite(ds, model)
ts.diagnose_accuracy_table().table

# %%
# Indicators participate in interpretation
# ----------------------------------------------------------
results = ts.interpret_fi()
results.plot()

# %%
# Effect of the special-value indicator
# ----------------------------------------------------------
results = ts.interpret_effects(features="BILL_AMT1_special_-999")
results.plot()