{
  "cells": [
    {
      "cell_type": "markdown",
      "metadata": {},
      "source": [
        "\n# ModelZoo\n\nThis example demonstrates how to use ModelZoo to manage multiple models,\ntrain them, and perform various analyses using TestSuite.\n"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {},
      "source": [
        "Installation\n\n"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "collapsed": false
      },
      "outputs": [],
      "source": [
        "# To install the required package, use the following command:\n# !pip install modeva"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {},
      "source": [
        "Authentication\n\n"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "collapsed": false
      },
      "outputs": [],
      "source": [
        "# To get authentication, use the following command: (To get full access please replace the token to your own token)\n# from modeva.utils.authenticate import authenticate\n# authenticate(auth_code='eaaa4301-b140-484c-8e93-f9f633c8bacb')"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {},
      "source": [
        "Import required modules\n\n"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "collapsed": false
      },
      "outputs": [],
      "source": [
        "from lightgbm import LGBMClassifier\n\nfrom modeva import DataSet\nfrom modeva import ModelZoo\nfrom modeva import TestSuite\nfrom modeva.utils.mlflow import set_mlflow_home, get_mlflow_home\nfrom modeva.models.local_model_zoo import LocalModelZoo\nfrom modeva.models.wrappers.api import modeva_sklearn_classifier\n\n# Import model classes\nfrom modeva.models import (\n    MoLogisticRegression, MoDecisionTreeClassifier,\n    MoLGBMClassifier, MoXGBClassifier, MoCatBoostClassifier,\n    MoRandomForestClassifier, MoGradientBoostingClassifier,\n    MoGAMINetClassifier, MoReLUDNNClassifier,\n    MoGLMTreeBoostClassifier, MoNeuralTreeClassifier\n)"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {},
      "source": [
        "## Configure MLflow settings\n\n"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "collapsed": false
      },
      "outputs": [],
      "source": [
        "set_mlflow_home(mlflow_home=\"~/modeva_mlflow\")\nmlflow_home = get_mlflow_home()"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {},
      "source": [
        "## Load and prepare dataset\n\n"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "collapsed": false
      },
      "outputs": [],
      "source": [
        "ds = DataSet()\nds.load(name=\"TaiwanCredit\")\nds.set_random_split()"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {},
      "source": [
        "## Initialize ModelZoo\n\n"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "collapsed": false
      },
      "outputs": [],
      "source": [
        "mz = LocalModelZoo(name=\"TaiwanCredit-Exp\", dataset=ds)\nprint(f\"Experiment name: {mz.experiment_name}\")\nprint(f\"Experiment ID: {mz.experiment_id}\")"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {},
      "source": [
        "## Add traditional ML models\n\n"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "collapsed": false
      },
      "outputs": [],
      "source": [
        "mz.add_model(model=MoLGBMClassifier(name=\"LGBM2\", max_depth=2, verbose=-1))\nmz.add_model(model=MoXGBClassifier(name=\"XGB2\", max_depth=2))\nmz.add_model(model=MoCatBoostClassifier(name=\"CatBoost2\", max_depth=2, silent=True))\nmz.add_model(model=MoRandomForestClassifier(name=\"RF2\", max_depth=2))\nmz.add_model(model=MoGradientBoostingClassifier(name=\"GBDT2\", max_depth=2))\nmz.add_model(model=MoLogisticRegression(\n    name=\"LR\", \n    feature_names=ds.feature_names, \n    feature_types=ds.feature_types\n))\nmz.add_model(model=MoDecisionTreeClassifier(name=\"DT\", max_depth=8))\nmz.add_model(model=MoReLUDNNClassifier(name=\"ReLUDNN\"))"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {},
      "source": [
        "## Add advanced ML models\n\n"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "collapsed": false
      },
      "outputs": [],
      "source": [
        "mz.add_model(model=MoNeuralTreeClassifier(\n    name=\"NeuralTree\",\n    nn_temperature=0.001,\n    nn_max_epochs=100,\n    verbose=False,\n    random_state=0\n))"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {},
      "source": [
        "## Add wrapped scikit-learn model\n\n"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "collapsed": false
      },
      "outputs": [],
      "source": [
        "wrap_estimator = modeva_sklearn_classifier(\n    name=\"LGBM-wrapped\",\n    estimator=LGBMClassifier(verbose=-1)\n)\nmz.add_model(model=wrap_estimator)"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {},
      "source": [
        "## Train all models and show leaderboard\n\n"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "collapsed": false
      },
      "outputs": [],
      "source": [
        "mz.train_all()\nmz.leaderboard(order_by=\"test AUC\")"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {},
      "source": [
        "## Model interpretation examples\nFeature importance analysis\n\n"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "collapsed": false
      },
      "outputs": [],
      "source": [
        "model = mz.get_model(\"ReLUDNN\")\nts = TestSuite(ds, model)\nresults = ts.interpret_fi()\nresults.plot()"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {},
      "source": [
        "Feature effects analysis for different models\n\n"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "collapsed": false
      },
      "outputs": [],
      "source": [
        "model = mz.get_model(\"LGBM2\")\nts = TestSuite(ds, model)\nresults = ts.interpret_effects(features=\"PAY_1\")\nresults.plot()"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {},
      "source": [
        "## Model registration and loading\nRegister all models\n\n"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "collapsed": false
      },
      "outputs": [],
      "source": [
        "for name in mz.models.keys():\n    mz.register(name)\n\n# List registered models\nregistered_models = mz.list_registered_models()\nprint(\"Registered models:\", registered_models)"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {},
      "source": [
        "## Load and verify registered models\n\n"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "collapsed": false
      },
      "outputs": [],
      "source": [
        "ds_new = DataSet()\nds_new.load(name=\"TaiwanCredit\")\nds_new.set_random_split()\nmz_new = ModelZoo(name=\"TaiwanCredit-Exp\", dataset=ds_new)\n\n# Verify predictions from loaded models\nfor name in mz.models.keys():\n    loaded_model = mz_new.load_registered_model(name)\n    predictions = loaded_model.predict_proba(ds_new.train_x)\n    print(f\"Model {name} predictions shape: {predictions.shape}\")"
      ]
    }
  ],
  "metadata": {
    "kernelspec": {
      "display_name": "Python 3",
      "language": "python",
      "name": "python3"
    },
    "language_info": {
      "codemirror_mode": {
        "name": "ipython",
        "version": 3
      },
      "file_extension": ".py",
      "mimetype": "text/x-python",
      "name": "python",
      "nbconvert_exporter": "python",
      "pygments_lexer": "ipython3",
      "version": "3.11.11"
    }
  },
  "nbformat": 4,
  "nbformat_minor": 0
}