Outlier Detection

Installation

# To install the required package, use the following command:
# !pip install modeva

Authentication

# To get authentication, use the following command: (To get full access please replace the token to your own token)
# from modeva.utils.authenticate import authenticate
# authenticate(auth_code='eaaa4301-b140-484c-8e93-f9f633c8bacb')

Import modeva modules

from modeva import DataSet

Load a simulated Friedman data

from sklearn.datasets import make_friedman1

ds = DataSet()
ds.load("BikeSharing")

Outlier detection by CBLOF

results = ds.detect_outlier_cblof(dataset="main", method="kmeans", threshold=0.9)
results.plot()

Outlier detection by Isolation forest

results = ds.detect_outlier_isolation_forest()
results.plot()

Outlier detection by PCA

results = ds.detect_outlier_pca(dataset="main", method="reconst_error")
outliers_sample_index = results.table['outliers'].index
results.plot()

View and use outlier detection results

Outliers table

results.table['outliers']

	season	yr	mnth	hr	holiday	weekday	workingday	weathersit	temp	atemp	hum	windspeed
479	1.0	0.0	1.0	0.0	0.0	6.0	0.0	1.0	0.04	0.0303	0.45	0.2537
585	1.0	0.0	1.0	16.0	0.0	3.0	1.0	4.0	0.22	0.1970	0.93	0.3284
608	1.0	0.0	1.0	14.0	0.0	5.0	1.0	3.0	0.22	0.2727	0.80	0.0000
966	1.0	0.0	2.0	21.0	0.0	6.0	0.0	1.0	0.26	0.3030	0.41	0.0000
1017	1.0	0.0	2.0	1.0	0.0	2.0	1.0	1.0	0.30	0.2424	0.42	0.7761
...	...	...	...	...	...	...	...	...	...	...	...	...
17041	4.0	1.0	12.0	20.0	0.0	1.0	1.0	2.0	0.42	0.4242	0.94	0.2537
17042	4.0	1.0	12.0	21.0	0.0	1.0	1.0	2.0	0.42	0.4242	0.94	0.1343
17043	4.0	1.0	12.0	22.0	0.0	1.0	1.0	2.0	0.42	0.4242	0.94	0.1343
17273	1.0	1.0	12.0	14.0	0.0	4.0	1.0	2.0	0.26	0.2121	0.56	0.5224
17320	1.0	1.0	12.0	13.0	0.0	6.0	0.0	3.0	0.20	0.2424	1.00	0.0000

174 rows × 12 columns

non-outliers table

results.table['non-outliers']

	season	yr	mnth	hr	holiday	weekday	workingday	weathersit	temp	atemp	hum	windspeed
0	1.0	0.0	1.0	0.0	0.0	6.0	0.0	1.0	0.24	0.2879	0.81	0.0000
1	1.0	0.0	1.0	1.0	0.0	6.0	0.0	1.0	0.22	0.2727	0.80	0.0000
2	1.0	0.0	1.0	2.0	0.0	6.0	0.0	1.0	0.22	0.2727	0.80	0.0000
3	1.0	0.0	1.0	3.0	0.0	6.0	0.0	1.0	0.24	0.2879	0.75	0.0000
4	1.0	0.0	1.0	4.0	0.0	6.0	0.0	1.0	0.24	0.2879	0.75	0.0000
...	...	...	...	...	...	...	...	...	...	...	...	...
17374	1.0	1.0	12.0	19.0	0.0	1.0	1.0	2.0	0.26	0.2576	0.60	0.1642
17375	1.0	1.0	12.0	20.0	0.0	1.0	1.0	2.0	0.26	0.2576	0.60	0.1642
17376	1.0	1.0	12.0	21.0	0.0	1.0	1.0	1.0	0.26	0.2576	0.60	0.1642
17377	1.0	1.0	12.0	22.0	0.0	1.0	1.0	1.0	0.26	0.2727	0.56	0.1343
17378	1.0	1.0	12.0	23.0	0.0	1.0	1.0	1.0	0.26	0.2727	0.65	0.1343

17205 rows × 12 columns

Evaluate outlier scores of samples

results.func(results.table['outliers'])

array([ 169.58882597, 5716.67438175,  175.5147215 ,  211.82036584,
        156.00876838,  209.52172788,  211.17825986,  214.92474565,
        239.16633547,  623.07944085,  608.94065677,  609.91112063,
        320.07697435,  368.99053694,  240.4681839 ,  172.96689805,
        165.52386691,  162.84602065,  206.58841076,  203.91086655,
        155.45777351,  154.05596207,  180.44108005,  209.24923191,
        218.76039605,  372.39428464,  361.90776689,  269.1591745 ,
        172.23424385,  153.35886571,  231.59150096,  152.63892622,
        184.89930816,  259.19771062,  240.19020921,  227.20752182,
        218.939138  ,  222.60859736,  451.23560828,  388.36562916,
        188.49949932,  407.01298297,  360.30498762,  367.06170815,
        357.93806097,  352.01777678,  427.53512096,  367.55526208,
        363.6298708 ,  393.45578498,  399.40051357,  418.6672334 ,
        349.30810555,  347.19708841,  365.46464218,  364.75811973,
        175.68152997,  236.09213148,  306.45287301,  151.32737983,
        146.92705194,  205.70441192,  472.51060806,  322.84469629,
        287.25353302,  240.57641591,  148.1966921 ,  148.29998244,
        149.84270706,  147.19673312,  161.93986566,  225.83784574,
        147.08417148,  232.12214548,  459.74119898,  636.66331111,
        345.66441815,  220.28825422,  313.7384874 , 5903.9481895 ,
        281.21843381,  318.40896224,  185.05606342,  152.77418419,
       5899.8619246 ,  231.2259397 ,  524.08340998,  170.01159481,
        498.49249195,  909.64644051,  154.09315457, 1645.64722795,
        191.451988  ,  296.60068917,  899.47145906,  167.28130404,
        193.38635254,  415.5796733 ,  267.29455895,  329.55199525,
        150.29613823,  149.74019125,  187.51951345,  184.1196592 ,
        179.43012181,  225.23881815,  221.72775697,  176.70591259,
        158.48404935,  203.44794474,  192.0860051 ,  346.13769586,
        264.06263672,  358.77115248,  364.63718738,  362.42867552,
        465.20379622,  433.96696482,  348.16941222,  433.59185708,
        441.129473  ,  265.75201011,  395.13694044,  614.22353982,
        280.46657904,  283.22709317, 1741.5758366 ,  243.2773644 ,
        219.1375133 ,  259.32104367,  235.93337067,  248.48869879,
        271.86569063,  206.62127493,  308.25864488,  505.91531027,
        178.63710481,  301.36711332,  292.26237496,  383.08939452,
        176.62591967,  173.77918172,  355.93341379,  288.52711163,
        615.72256946, 1099.06939929,  180.69795192,  271.60859436,
        273.36111761,  574.02353691,  592.15696817,  223.791897  ,
        239.82352321,  213.68282627,  479.60502388,  239.90643424,
        372.65863047,  241.82300612,  194.37222561, 1225.4607848 ,
        223.34497118,  476.15083477,  238.42588847,  156.97176857,
        629.90012033,  366.52888139,  393.36175679,  539.39035252,
        940.57779066,  255.00866784,  404.23887668, 1245.40228736,
        192.05348493,  699.30473822])

Evaluate outlier scores of samples

results.func(results.table['non-outliers'])

array([ 8.11991488,  7.31292663, 15.14645936, ...,  9.80182776,
        6.99212203,  9.52262687])

Apply outlier removal

ds.set_inactive_samples(dataset="main", sample_idx=outliers_sample_index)
ds.x.shape

(17205, 12)