{
  "cells": [
    {
      "cell_type": "markdown",
      "metadata": {},
      "source": [
        "\n# Full Feature Engineering Pipeline\n\nThis example demonstrates a comprehensive feature engineering pipeline\ncombining multiple transformer categories: bivector interactions,\nwhitening, phase encoding, and density estimation.\n"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {},
      "source": [
        "## Setup\nImport libraries and suppress warnings.\n\n"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "collapsed": false
      },
      "outputs": [],
      "source": [
        "import warnings\nwarnings.filterwarnings(\"ignore\")\n\nimport numpy as np\nfrom modeva import DataSet"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {},
      "source": [
        "## Load Dataset\n\n"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "collapsed": false
      },
      "outputs": [],
      "source": [
        "ds = DataSet()\nds.load(name=\"CaliforniaHousing\")\nds.set_random_split()\n\n# Keep a copy of the raw (un-engineered) data; the fitted pipeline is applied\n# to raw-feature input below, not to the already-engineered frame.\nraw_df = ds.to_df()\n\noriginal_n_features = len(ds.feature_names)\nprint(f\"Original features ({original_n_features}): {ds.feature_names}\")"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {},
      "source": [
        "## Build Pipeline\nQueue multiple feature engineering steps across different categories.\n\n"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "collapsed": false
      },
      "outputs": [],
      "source": [
        "# Pairwise interactions\nds.fe_bivector(max_features=30, selection=\"variance\")\n\n# Coordinate transform\nds.fe_whitening()\n\n# Pairwise shear\nds.fe_shear(threshold=0.01)\n\n# Phase encoding\nds.fe_phase_encoder(n_thresholds=3, include_original=False)\n\n# Local density\nds.fe_density(bandwidth=\"median\", n_reference=300)"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {},
      "source": [
        "## Execute Pipeline\n\n"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "collapsed": false
      },
      "outputs": [],
      "source": [
        "ds.engineer_features()\n\nnew_n_features = len(ds.all_feature_names)\nprint(f\"Features: {original_n_features} -> {new_n_features}\")\nprint(f\"New features added: {new_n_features - original_n_features}\")"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {},
      "source": [
        "## Verify Transform on New Data\nApply the fitted pipeline to a fresh batch of data.\n\n"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "collapsed": false
      },
      "outputs": [],
      "source": [
        "test_df = raw_df.iloc[:3]\ntransformed = ds.transform_features(test_df)\n\nprint(f\"Input shape:  {test_df.shape}\")\nprint(f\"Output shape: {transformed.shape}\")\nprint(f\"All columns match: {list(transformed.columns) == list(ds.to_df().columns)}\")"
      ]
    }
  ],
  "metadata": {
    "kernelspec": {
      "display_name": "Python 3",
      "language": "python",
      "name": "python3"
    },
    "language_info": {
      "codemirror_mode": {
        "name": "ipython",
        "version": 3
      },
      "file_extension": ".py",
      "mimetype": "text/x-python",
      "name": "python",
      "nbconvert_exporter": "python",
      "pygments_lexer": "ipython3",
      "version": "3.11.11"
    }
  },
  "nbformat": 4,
  "nbformat_minor": 0
}