Data Filling and Cleaning Techniques/DATA CLEANING-5 Scikit Learn Module.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Method 2,3,4 -> Using scikit learn module"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "import numpy as np\n",
    "import pandas as pd\n",
    "from sklearn.impute import SimpleImputer"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "shape of train df =  (1460, 81)\n",
      "shape of test df =  (1459, 80)\n"
     ]
    }
   ],
   "source": [
    "train=pd.read_csv(r\"C:\\Users\\lalit\\Documents\\JUPYTER NOTEBOOK\\house-prices-advanced-regression-techniques\\train.csv\")\n",
    "test=pd.read_csv(r\"C:\\Users\\lalit\\Documents\\JUPYTER NOTEBOOK\\house-prices-advanced-regression-techniques\\test.csv\")\n",
    "print(\"shape of train df = \",train.shape)\n",
    "print(\"shape of test df = \",test.shape)\n",
    "# dataset link -> https://drive.google.com/drive/folders/1gdfNSb3ptz0WvShshUeiM-kqfMYzQ5tm?usp=sharing"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Id</th>\n",
       "      <th>MSSubClass</th>\n",
       "      <th>MSZoning</th>\n",
       "      <th>LotFrontage</th>\n",
       "      <th>LotArea</th>\n",
       "      <th>Street</th>\n",
       "      <th>Alley</th>\n",
       "      <th>LotShape</th>\n",
       "      <th>LandContour</th>\n",
       "      <th>Utilities</th>\n",
       "      <th>...</th>\n",
       "      <th>PoolArea</th>\n",
       "      <th>PoolQC</th>\n",
       "      <th>Fence</th>\n",
       "      <th>MiscFeature</th>\n",
       "      <th>MiscVal</th>\n",
       "      <th>MoSold</th>\n",
       "      <th>YrSold</th>\n",
       "      <th>SaleType</th>\n",
       "      <th>SaleCondition</th>\n",
       "      <th>SalePrice</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>1</td>\n",
       "      <td>60</td>\n",
       "      <td>RL</td>\n",
       "      <td>65.0</td>\n",
       "      <td>8450</td>\n",
       "      <td>Pave</td>\n",
       "      <td>NaN</td>\n",
       "      <td>Reg</td>\n",
       "      <td>Lvl</td>\n",
       "      <td>AllPub</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0</td>\n",
       "      <td>2</td>\n",
       "      <td>2008</td>\n",
       "      <td>WD</td>\n",
       "      <td>Normal</td>\n",
       "      <td>208500</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>2</td>\n",
       "      <td>20</td>\n",
       "      <td>RL</td>\n",
       "      <td>80.0</td>\n",
       "      <td>9600</td>\n",
       "      <td>Pave</td>\n",
       "      <td>NaN</td>\n",
       "      <td>Reg</td>\n",
       "      <td>Lvl</td>\n",
       "      <td>AllPub</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0</td>\n",
       "      <td>5</td>\n",
       "      <td>2007</td>\n",
       "      <td>WD</td>\n",
       "      <td>Normal</td>\n",
       "      <td>181500</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>3</td>\n",
       "      <td>60</td>\n",
       "      <td>RL</td>\n",
       "      <td>68.0</td>\n",
       "      <td>11250</td>\n",
       "      <td>Pave</td>\n",
       "      <td>NaN</td>\n",
       "      <td>IR1</td>\n",
       "      <td>Lvl</td>\n",
       "      <td>AllPub</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0</td>\n",
       "      <td>9</td>\n",
       "      <td>2008</td>\n",
       "      <td>WD</td>\n",
       "      <td>Normal</td>\n",
       "      <td>223500</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>4</td>\n",
       "      <td>70</td>\n",
       "      <td>RL</td>\n",
       "      <td>60.0</td>\n",
       "      <td>9550</td>\n",
       "      <td>Pave</td>\n",
       "      <td>NaN</td>\n",
       "      <td>IR1</td>\n",
       "      <td>Lvl</td>\n",
       "      <td>AllPub</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0</td>\n",
       "      <td>2</td>\n",
       "      <td>2006</td>\n",
       "      <td>WD</td>\n",
       "      <td>Abnorml</td>\n",
       "      <td>140000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>5</td>\n",
       "      <td>60</td>\n",
       "      <td>RL</td>\n",
       "      <td>84.0</td>\n",
       "      <td>14260</td>\n",
       "      <td>Pave</td>\n",
       "      <td>NaN</td>\n",
       "      <td>IR1</td>\n",
       "      <td>Lvl</td>\n",
       "      <td>AllPub</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0</td>\n",
       "      <td>12</td>\n",
       "      <td>2008</td>\n",
       "      <td>WD</td>\n",
       "      <td>Normal</td>\n",
       "      <td>250000</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>5 rows × 81 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "   Id  MSSubClass MSZoning  LotFrontage  LotArea Street Alley LotShape  \\\n",
       "0   1          60       RL         65.0     8450   Pave   NaN      Reg   \n",
       "1   2          20       RL         80.0     9600   Pave   NaN      Reg   \n",
       "2   3          60       RL         68.0    11250   Pave   NaN      IR1   \n",
       "3   4          70       RL         60.0     9550   Pave   NaN      IR1   \n",
       "4   5          60       RL         84.0    14260   Pave   NaN      IR1   \n",
       "\n",
       "  LandContour Utilities  ... PoolArea PoolQC Fence MiscFeature MiscVal MoSold  \\\n",
       "0         Lvl    AllPub  ...        0    NaN   NaN         NaN       0      2   \n",
       "1         Lvl    AllPub  ...        0    NaN   NaN         NaN       0      5   \n",
       "2         Lvl    AllPub  ...        0    NaN   NaN         NaN       0      9   \n",
       "3         Lvl    AllPub  ...        0    NaN   NaN         NaN       0      2   \n",
       "4         Lvl    AllPub  ...        0    NaN   NaN         NaN       0     12   \n",
       "\n",
       "  YrSold  SaleType  SaleCondition  SalePrice  \n",
       "0   2008        WD         Normal     208500  \n",
       "1   2007        WD         Normal     181500  \n",
       "2   2008        WD         Normal     223500  \n",
       "3   2006        WD        Abnorml     140000  \n",
       "4   2008        WD         Normal     250000  \n",
       "\n",
       "[5 rows x 81 columns]"
      ]
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "train.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "shape of X_train df =  (1460, 80)\n",
      "shape of y_train df =  (1460,)\n"
     ]
    }
   ],
   "source": [
    "# X -> Matrix form   (Capital)\n",
    "# y -> vector form (Small)\n",
    "X_train=train.drop(columns=\"SalePrice\")\n",
    "y_train=train[\"SalePrice\"]\n",
    "print(\"shape of X_train df = \",X_train.shape)\n",
    "print(\"shape of y_train df = \",y_train.shape)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Numerical Missing Value Imputation"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "Index(['Id', 'MSSubClass', 'LotFrontage', 'LotArea', 'OverallQual',\n",
       "       'OverallCond', 'YearBuilt', 'YearRemodAdd', 'MasVnrArea', 'BsmtFinSF1',\n",
       "       'BsmtFinSF2', 'BsmtUnfSF', 'TotalBsmtSF', '1stFlrSF', '2ndFlrSF',\n",
       "       'LowQualFinSF', 'GrLivArea', 'BsmtFullBath', 'BsmtHalfBath', 'FullBath',\n",
       "       'HalfBath', 'BedroomAbvGr', 'KitchenAbvGr', 'TotRmsAbvGrd',\n",
       "       'Fireplaces', 'GarageYrBlt', 'GarageCars', 'GarageArea', 'WoodDeckSF',\n",
       "       'OpenPorchSF', 'EnclosedPorch', '3SsnPorch', 'ScreenPorch', 'PoolArea',\n",
       "       'MiscVal', 'MoSold', 'YrSold'],\n",
       "      dtype='object')"
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# select numerical columns only \n",
    "num_vars=X_train.select_dtypes(include=[\"int64\",\"float64\"]).columns\n",
    "num_vars"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "Id                 0\n",
       "MSSubClass         0\n",
       "LotFrontage      259\n",
       "LotArea            0\n",
       "OverallQual        0\n",
       "OverallCond        0\n",
       "YearBuilt          0\n",
       "YearRemodAdd       0\n",
       "MasVnrArea         8\n",
       "BsmtFinSF1         0\n",
       "BsmtFinSF2         0\n",
       "BsmtUnfSF          0\n",
       "TotalBsmtSF        0\n",
       "1stFlrSF           0\n",
       "2ndFlrSF           0\n",
       "LowQualFinSF       0\n",
       "GrLivArea          0\n",
       "BsmtFullBath       0\n",
       "BsmtHalfBath       0\n",
       "FullBath           0\n",
       "HalfBath           0\n",
       "BedroomAbvGr       0\n",
       "KitchenAbvGr       0\n",
       "TotRmsAbvGrd       0\n",
       "Fireplaces         0\n",
       "GarageYrBlt       81\n",
       "GarageCars         0\n",
       "GarageArea         0\n",
       "WoodDeckSF         0\n",
       "OpenPorchSF        0\n",
       "EnclosedPorch      0\n",
       "3SsnPorch          0\n",
       "ScreenPorch        0\n",
       "PoolArea           0\n",
       "MiscVal            0\n",
       "MoSold             0\n",
       "YrSold             0\n",
       "dtype: int64"
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "X_train[num_vars].isnull().sum()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [],
   "source": [
    "# find the mean value\n",
    "imputer_mean = SimpleImputer(strategy='mean')\n",
    "#imputer_mean = SimpleImputer(strategy='constant', fill_value=99) \n",
    "# will fill 99 in the missing places"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "SimpleImputer()"
      ]
     },
     "execution_count": 12,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# store mean value of the numerical values from the X_train dataset\n",
    "imputer_mean.fit(X_train[num_vars])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([7.30500000e+02, 5.68972603e+01, 7.00499584e+01, 1.05168281e+04,\n",
       "       6.09931507e+00, 5.57534247e+00, 1.97126781e+03, 1.98486575e+03,\n",
       "       1.03685262e+02, 4.43639726e+02, 4.65493151e+01, 5.67240411e+02,\n",
       "       1.05742945e+03, 1.16262671e+03, 3.46992466e+02, 5.84452055e+00,\n",
       "       1.51546370e+03, 4.25342466e-01, 5.75342466e-02, 1.56506849e+00,\n",
       "       3.82876712e-01, 2.86643836e+00, 1.04657534e+00, 6.51780822e+00,\n",
       "       6.13013699e-01, 1.97850616e+03, 1.76712329e+00, 4.72980137e+02,\n",
       "       9.42445205e+01, 4.66602740e+01, 2.19541096e+01, 3.40958904e+00,\n",
       "       1.50609589e+01, 2.75890411e+00, 4.34890411e+01, 6.32191781e+00,\n",
       "       2.00781575e+03])"
      ]
     },
     "execution_count": 13,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# find the mean values \n",
    "imputer_mean.statistics_"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([[1.000e+00, 6.000e+01, 6.500e+01, ..., 0.000e+00, 2.000e+00,\n",
       "        2.008e+03],\n",
       "       [2.000e+00, 2.000e+01, 8.000e+01, ..., 0.000e+00, 5.000e+00,\n",
       "        2.007e+03],\n",
       "       [3.000e+00, 6.000e+01, 6.800e+01, ..., 0.000e+00, 9.000e+00,\n",
       "        2.008e+03],\n",
       "       ...,\n",
       "       [1.458e+03, 7.000e+01, 6.600e+01, ..., 2.500e+03, 5.000e+00,\n",
       "        2.010e+03],\n",
       "       [1.459e+03, 2.000e+01, 6.800e+01, ..., 0.000e+00, 4.000e+00,\n",
       "        2.010e+03],\n",
       "       [1.460e+03, 2.000e+01, 7.500e+01, ..., 0.000e+00, 6.000e+00,\n",
       "        2.008e+03]])"
      ]
     },
     "execution_count": 14,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# transform the mean value to X_train datatset\n",
    "imputer_mean.transform(X_train[num_vars])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [],
   "source": [
    "# put data in the respective data field\n",
    "X_train[num_vars] = imputer_mean.transform(X_train[num_vars])\n",
    "test[num_vars] = imputer_mean.transform(test[num_vars])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "Id               0\n",
       "MSSubClass       0\n",
       "LotFrontage      0\n",
       "LotArea          0\n",
       "OverallQual      0\n",
       "OverallCond      0\n",
       "YearBuilt        0\n",
       "YearRemodAdd     0\n",
       "MasVnrArea       0\n",
       "BsmtFinSF1       0\n",
       "BsmtFinSF2       0\n",
       "BsmtUnfSF        0\n",
       "TotalBsmtSF      0\n",
       "1stFlrSF         0\n",
       "2ndFlrSF         0\n",
       "LowQualFinSF     0\n",
       "GrLivArea        0\n",
       "BsmtFullBath     0\n",
       "BsmtHalfBath     0\n",
       "FullBath         0\n",
       "HalfBath         0\n",
       "BedroomAbvGr     0\n",
       "KitchenAbvGr     0\n",
       "TotRmsAbvGrd     0\n",
       "Fireplaces       0\n",
       "GarageYrBlt      0\n",
       "GarageCars       0\n",
       "GarageArea       0\n",
       "WoodDeckSF       0\n",
       "OpenPorchSF      0\n",
       "EnclosedPorch    0\n",
       "3SsnPorch        0\n",
       "ScreenPorch      0\n",
       "PoolArea         0\n",
       "MiscVal          0\n",
       "MoSold           0\n",
       "YrSold           0\n",
       "dtype: int64"
      ]
     },
     "execution_count": 16,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "X_train[num_vars].isnull().sum()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "Id               0\n",
       "MSSubClass       0\n",
       "LotFrontage      0\n",
       "LotArea          0\n",
       "OverallQual      0\n",
       "OverallCond      0\n",
       "YearBuilt        0\n",
       "YearRemodAdd     0\n",
       "MasVnrArea       0\n",
       "BsmtFinSF1       0\n",
       "BsmtFinSF2       0\n",
       "BsmtUnfSF        0\n",
       "TotalBsmtSF      0\n",
       "1stFlrSF         0\n",
       "2ndFlrSF         0\n",
       "LowQualFinSF     0\n",
       "GrLivArea        0\n",
       "BsmtFullBath     0\n",
       "BsmtHalfBath     0\n",
       "FullBath         0\n",
       "HalfBath         0\n",
       "BedroomAbvGr     0\n",
       "KitchenAbvGr     0\n",
       "TotRmsAbvGrd     0\n",
       "Fireplaces       0\n",
       "GarageYrBlt      0\n",
       "GarageCars       0\n",
       "GarageArea       0\n",
       "WoodDeckSF       0\n",
       "OpenPorchSF      0\n",
       "EnclosedPorch    0\n",
       "3SsnPorch        0\n",
       "ScreenPorch      0\n",
       "PoolArea         0\n",
       "MiscVal          0\n",
       "MoSold           0\n",
       "YrSold           0\n",
       "dtype: int64"
      ]
     },
     "execution_count": 17,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "test[num_vars].isnull().sum()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Categorical Missing Value Imputation"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "Index(['MSZoning', 'Street', 'Alley', 'LotShape', 'LandContour', 'Utilities',\n",
       "       'LotConfig', 'LandSlope', 'Neighborhood', 'Condition1', 'Condition2',\n",
       "       'BldgType', 'HouseStyle', 'RoofStyle', 'RoofMatl', 'Exterior1st',\n",
       "       'Exterior2nd', 'MasVnrType', 'ExterQual', 'ExterCond', 'Foundation',\n",
       "       'BsmtQual', 'BsmtCond', 'BsmtExposure', 'BsmtFinType1', 'BsmtFinType2',\n",
       "       'Heating', 'HeatingQC', 'CentralAir', 'Electrical', 'KitchenQual',\n",
       "       'Functional', 'FireplaceQu', 'GarageType', 'GarageFinish', 'GarageQual',\n",
       "       'GarageCond', 'PavedDrive', 'PoolQC', 'Fence', 'MiscFeature',\n",
       "       'SaleType', 'SaleCondition'],\n",
       "      dtype='object')"
      ]
     },
     "execution_count": 18,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "cat_vars=X_train.select_dtypes(include=[\"O\"]).columns #O -> object\n",
    "cat_vars"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "MSZoning            0\n",
       "Street              0\n",
       "Alley            1369\n",
       "LotShape            0\n",
       "LandContour         0\n",
       "Utilities           0\n",
       "LotConfig           0\n",
       "LandSlope           0\n",
       "Neighborhood        0\n",
       "Condition1          0\n",
       "Condition2          0\n",
       "BldgType            0\n",
       "HouseStyle          0\n",
       "RoofStyle           0\n",
       "RoofMatl            0\n",
       "Exterior1st         0\n",
       "Exterior2nd         0\n",
       "MasVnrType          8\n",
       "ExterQual           0\n",
       "ExterCond           0\n",
       "Foundation          0\n",
       "BsmtQual           37\n",
       "BsmtCond           37\n",
       "BsmtExposure       38\n",
       "BsmtFinType1       37\n",
       "BsmtFinType2       38\n",
       "Heating             0\n",
       "HeatingQC           0\n",
       "CentralAir          0\n",
       "Electrical          1\n",
       "KitchenQual         0\n",
       "Functional          0\n",
       "FireplaceQu       690\n",
       "GarageType         81\n",
       "GarageFinish       81\n",
       "GarageQual         81\n",
       "GarageCond         81\n",
       "PavedDrive          0\n",
       "PoolQC           1453\n",
       "Fence            1179\n",
       "MiscFeature      1406\n",
       "SaleType            0\n",
       "SaleCondition       0\n",
       "dtype: int64"
      ]
     },
     "execution_count": 19,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "X_train[cat_vars].isnull().sum()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "SimpleImputer(strategy='most_frequent')"
      ]
     },
     "execution_count": 20,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "imputer_mode = SimpleImputer(strategy='most_frequent') #most_frequent -> mode\n",
    "#imputer_mean = SimpleImputer(strategy='constant', fill_value=99)\n",
    "imputer_mode"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "SimpleImputer(strategy='most_frequent')"
      ]
     },
     "execution_count": 21,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "imputer_mode.fit(X_train[cat_vars])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array(['RL', 'Pave', 'Grvl', 'Reg', 'Lvl', 'AllPub', 'Inside', 'Gtl',\n",
       "       'NAmes', 'Norm', 'Norm', '1Fam', '1Story', 'Gable', 'CompShg',\n",
       "       'VinylSd', 'VinylSd', 'None', 'TA', 'TA', 'PConc', 'TA', 'TA',\n",
       "       'No', 'Unf', 'Unf', 'GasA', 'Ex', 'Y', 'SBrkr', 'TA', 'Typ', 'Gd',\n",
       "       'Attchd', 'Unf', 'TA', 'TA', 'Y', 'Gd', 'MnPrv', 'Shed', 'WD',\n",
       "       'Normal'], dtype=object)"
      ]
     },
     "execution_count": 22,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "imputer_mode.statistics_"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {},
   "outputs": [],
   "source": [
    "X_train[cat_vars] = imputer_mode.transform(X_train[cat_vars])\n",
    "test[cat_vars] = imputer_mode.transform(test[cat_vars])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "MSZoning         0\n",
       "Street           0\n",
       "Alley            0\n",
       "LotShape         0\n",
       "LandContour      0\n",
       "Utilities        0\n",
       "LotConfig        0\n",
       "LandSlope        0\n",
       "Neighborhood     0\n",
       "Condition1       0\n",
       "Condition2       0\n",
       "BldgType         0\n",
       "HouseStyle       0\n",
       "RoofStyle        0\n",
       "RoofMatl         0\n",
       "Exterior1st      0\n",
       "Exterior2nd      0\n",
       "MasVnrType       0\n",
       "ExterQual        0\n",
       "ExterCond        0\n",
       "Foundation       0\n",
       "BsmtQual         0\n",
       "BsmtCond         0\n",
       "BsmtExposure     0\n",
       "BsmtFinType1     0\n",
       "BsmtFinType2     0\n",
       "Heating          0\n",
       "HeatingQC        0\n",
       "CentralAir       0\n",
       "Electrical       0\n",
       "KitchenQual      0\n",
       "Functional       0\n",
       "FireplaceQu      0\n",
       "GarageType       0\n",
       "GarageFinish     0\n",
       "GarageQual       0\n",
       "GarageCond       0\n",
       "PavedDrive       0\n",
       "PoolQC           0\n",
       "Fence            0\n",
       "MiscFeature      0\n",
       "SaleType         0\n",
       "SaleCondition    0\n",
       "dtype: int64"
      ]
     },
     "execution_count": 24,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "X_train[cat_vars].isnull().sum()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "MSZoning         0\n",
       "Street           0\n",
       "Alley            0\n",
       "LotShape         0\n",
       "LandContour      0\n",
       "Utilities        0\n",
       "LotConfig        0\n",
       "LandSlope        0\n",
       "Neighborhood     0\n",
       "Condition1       0\n",
       "Condition2       0\n",
       "BldgType         0\n",
       "HouseStyle       0\n",
       "RoofStyle        0\n",
       "RoofMatl         0\n",
       "Exterior1st      0\n",
       "Exterior2nd      0\n",
       "MasVnrType       0\n",
       "ExterQual        0\n",
       "ExterCond        0\n",
       "Foundation       0\n",
       "BsmtQual         0\n",
       "BsmtCond         0\n",
       "BsmtExposure     0\n",
       "BsmtFinType1     0\n",
       "BsmtFinType2     0\n",
       "Heating          0\n",
       "HeatingQC        0\n",
       "CentralAir       0\n",
       "Electrical       0\n",
       "KitchenQual      0\n",
       "Functional       0\n",
       "FireplaceQu      0\n",
       "GarageType       0\n",
       "GarageFinish     0\n",
       "GarageQual       0\n",
       "GarageCond       0\n",
       "PavedDrive       0\n",
       "PoolQC           0\n",
       "Fence            0\n",
       "MiscFeature      0\n",
       "SaleType         0\n",
       "SaleCondition    0\n",
       "dtype: int64"
      ]
     },
     "execution_count": 26,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "test[cat_vars].isnull().sum()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0"
      ]
     },
     "execution_count": 27,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# whole dataset in now filled\n",
    "X_train.isnull().sum().sum()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.5"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}