Predicting Credit Card Approvals.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>0</th>\n",
       "      <th>1</th>\n",
       "      <th>2</th>\n",
       "      <th>3</th>\n",
       "      <th>4</th>\n",
       "      <th>5</th>\n",
       "      <th>6</th>\n",
       "      <th>7</th>\n",
       "      <th>8</th>\n",
       "      <th>9</th>\n",
       "      <th>10</th>\n",
       "      <th>11</th>\n",
       "      <th>12</th>\n",
       "      <th>13</th>\n",
       "      <th>14</th>\n",
       "      <th>15</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>b</td>\n",
       "      <td>30.83</td>\n",
       "      <td>0.000</td>\n",
       "      <td>u</td>\n",
       "      <td>g</td>\n",
       "      <td>w</td>\n",
       "      <td>v</td>\n",
       "      <td>1.25</td>\n",
       "      <td>t</td>\n",
       "      <td>t</td>\n",
       "      <td>1</td>\n",
       "      <td>f</td>\n",
       "      <td>g</td>\n",
       "      <td>00202</td>\n",
       "      <td>0</td>\n",
       "      <td>+</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>a</td>\n",
       "      <td>58.67</td>\n",
       "      <td>4.460</td>\n",
       "      <td>u</td>\n",
       "      <td>g</td>\n",
       "      <td>q</td>\n",
       "      <td>h</td>\n",
       "      <td>3.04</td>\n",
       "      <td>t</td>\n",
       "      <td>t</td>\n",
       "      <td>6</td>\n",
       "      <td>f</td>\n",
       "      <td>g</td>\n",
       "      <td>00043</td>\n",
       "      <td>560</td>\n",
       "      <td>+</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>a</td>\n",
       "      <td>24.50</td>\n",
       "      <td>0.500</td>\n",
       "      <td>u</td>\n",
       "      <td>g</td>\n",
       "      <td>q</td>\n",
       "      <td>h</td>\n",
       "      <td>1.50</td>\n",
       "      <td>t</td>\n",
       "      <td>f</td>\n",
       "      <td>0</td>\n",
       "      <td>f</td>\n",
       "      <td>g</td>\n",
       "      <td>00280</td>\n",
       "      <td>824</td>\n",
       "      <td>+</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>b</td>\n",
       "      <td>27.83</td>\n",
       "      <td>1.540</td>\n",
       "      <td>u</td>\n",
       "      <td>g</td>\n",
       "      <td>w</td>\n",
       "      <td>v</td>\n",
       "      <td>3.75</td>\n",
       "      <td>t</td>\n",
       "      <td>t</td>\n",
       "      <td>5</td>\n",
       "      <td>t</td>\n",
       "      <td>g</td>\n",
       "      <td>00100</td>\n",
       "      <td>3</td>\n",
       "      <td>+</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>b</td>\n",
       "      <td>20.17</td>\n",
       "      <td>5.625</td>\n",
       "      <td>u</td>\n",
       "      <td>g</td>\n",
       "      <td>w</td>\n",
       "      <td>v</td>\n",
       "      <td>1.71</td>\n",
       "      <td>t</td>\n",
       "      <td>f</td>\n",
       "      <td>0</td>\n",
       "      <td>f</td>\n",
       "      <td>s</td>\n",
       "      <td>00120</td>\n",
       "      <td>0</td>\n",
       "      <td>+</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>685</th>\n",
       "      <td>b</td>\n",
       "      <td>21.08</td>\n",
       "      <td>10.085</td>\n",
       "      <td>y</td>\n",
       "      <td>p</td>\n",
       "      <td>e</td>\n",
       "      <td>h</td>\n",
       "      <td>1.25</td>\n",
       "      <td>f</td>\n",
       "      <td>f</td>\n",
       "      <td>0</td>\n",
       "      <td>f</td>\n",
       "      <td>g</td>\n",
       "      <td>00260</td>\n",
       "      <td>0</td>\n",
       "      <td>-</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>686</th>\n",
       "      <td>a</td>\n",
       "      <td>22.67</td>\n",
       "      <td>0.750</td>\n",
       "      <td>u</td>\n",
       "      <td>g</td>\n",
       "      <td>c</td>\n",
       "      <td>v</td>\n",
       "      <td>2.00</td>\n",
       "      <td>f</td>\n",
       "      <td>t</td>\n",
       "      <td>2</td>\n",
       "      <td>t</td>\n",
       "      <td>g</td>\n",
       "      <td>00200</td>\n",
       "      <td>394</td>\n",
       "      <td>-</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>687</th>\n",
       "      <td>a</td>\n",
       "      <td>25.25</td>\n",
       "      <td>13.500</td>\n",
       "      <td>y</td>\n",
       "      <td>p</td>\n",
       "      <td>ff</td>\n",
       "      <td>ff</td>\n",
       "      <td>2.00</td>\n",
       "      <td>f</td>\n",
       "      <td>t</td>\n",
       "      <td>1</td>\n",
       "      <td>t</td>\n",
       "      <td>g</td>\n",
       "      <td>00200</td>\n",
       "      <td>1</td>\n",
       "      <td>-</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>688</th>\n",
       "      <td>b</td>\n",
       "      <td>17.92</td>\n",
       "      <td>0.205</td>\n",
       "      <td>u</td>\n",
       "      <td>g</td>\n",
       "      <td>aa</td>\n",
       "      <td>v</td>\n",
       "      <td>0.04</td>\n",
       "      <td>f</td>\n",
       "      <td>f</td>\n",
       "      <td>0</td>\n",
       "      <td>f</td>\n",
       "      <td>g</td>\n",
       "      <td>00280</td>\n",
       "      <td>750</td>\n",
       "      <td>-</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>689</th>\n",
       "      <td>b</td>\n",
       "      <td>35.00</td>\n",
       "      <td>3.375</td>\n",
       "      <td>u</td>\n",
       "      <td>g</td>\n",
       "      <td>c</td>\n",
       "      <td>h</td>\n",
       "      <td>8.29</td>\n",
       "      <td>f</td>\n",
       "      <td>f</td>\n",
       "      <td>0</td>\n",
       "      <td>t</td>\n",
       "      <td>g</td>\n",
       "      <td>00000</td>\n",
       "      <td>0</td>\n",
       "      <td>-</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>690 rows × 16 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "    0      1       2  3  4   5   6     7  8  9   10 11 12     13   14 15\n",
       "0    b  30.83   0.000  u  g   w   v  1.25  t  t   1  f  g  00202    0  +\n",
       "1    a  58.67   4.460  u  g   q   h  3.04  t  t   6  f  g  00043  560  +\n",
       "2    a  24.50   0.500  u  g   q   h  1.50  t  f   0  f  g  00280  824  +\n",
       "3    b  27.83   1.540  u  g   w   v  3.75  t  t   5  t  g  00100    3  +\n",
       "4    b  20.17   5.625  u  g   w   v  1.71  t  f   0  f  s  00120    0  +\n",
       "..  ..    ...     ... .. ..  ..  ..   ... .. ..  .. .. ..    ...  ... ..\n",
       "685  b  21.08  10.085  y  p   e   h  1.25  f  f   0  f  g  00260    0  -\n",
       "686  a  22.67   0.750  u  g   c   v  2.00  f  t   2  t  g  00200  394  -\n",
       "687  a  25.25  13.500  y  p  ff  ff  2.00  f  t   1  t  g  00200    1  -\n",
       "688  b  17.92   0.205  u  g  aa   v  0.04  f  f   0  f  g  00280  750  -\n",
       "689  b  35.00   3.375  u  g   c   h  8.29  f  f   0  t  g  00000    0  -\n",
       "\n",
       "[690 rows x 16 columns]"
      ]
     },
     "execution_count": 21,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Import pandas\n",
    "import pandas as pd\n",
    "\n",
    "# Load dataset\n",
    "cc_apps = pd.read_csv(\"datasets/Predicting Credit Card Approvals/cc_approvals.TXT\",header= None)\n",
    "\n",
    "# Inspect data\n",
    "cc_apps"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 2. Inspecting the applications\n",
    "The output may appear a bit confusing at its first sight, but let's try to figure out the most important features of a credit card application. The features of this dataset have been anonymized to protect the privacy, but this blog gives us a pretty good overview of the probable features. The probable features in a typical credit card application are Gender, Age, Debt, Married, BankCustomer, EducationLevel, Ethnicity, YearsEmployed, PriorDefault, Employed, CreditScore, DriversLicense, Citizen, ZipCode, Income and finally the ApprovalStatus. This gives us a pretty good starting point, and we can map these features with respect to the columns in the output.\n",
    "\n",
    "As we can see from our first glance at the data, the dataset has a mixture of numerical and non-numerical features. This can be fixed with some preprocessing, but before we do that, let's learn about the dataset a bit more to see if there are other dataset issues that need to be fixed."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "               2           7          10             14\n",
      "count  690.000000  690.000000  690.00000     690.000000\n",
      "mean     4.758725    2.223406    2.40000    1017.385507\n",
      "std      4.978163    3.346513    4.86294    5210.102598\n",
      "min      0.000000    0.000000    0.00000       0.000000\n",
      "25%      1.000000    0.165000    0.00000       0.000000\n",
      "50%      2.750000    1.000000    0.00000       5.000000\n",
      "75%      7.207500    2.625000    3.00000     395.500000\n",
      "max     28.000000   28.500000   67.00000  100000.000000\n",
      "\n",
      "\n",
      "<class 'pandas.core.frame.DataFrame'>\n",
      "RangeIndex: 690 entries, 0 to 689\n",
      "Data columns (total 16 columns):\n",
      " #   Column  Non-Null Count  Dtype  \n",
      "---  ------  --------------  -----  \n",
      " 0   0       690 non-null    object \n",
      " 1   1       690 non-null    object \n",
      " 2   2       690 non-null    float64\n",
      " 3   3       690 non-null    object \n",
      " 4   4       690 non-null    object \n",
      " 5   5       690 non-null    object \n",
      " 6   6       690 non-null    object \n",
      " 7   7       690 non-null    float64\n",
      " 8   8       690 non-null    object \n",
      " 9   9       690 non-null    object \n",
      " 10  10      690 non-null    int64  \n",
      " 11  11      690 non-null    object \n",
      " 12  12      690 non-null    object \n",
      " 13  13      690 non-null    object \n",
      " 14  14      690 non-null    int64  \n",
      " 15  15      690 non-null    object \n",
      "dtypes: float64(2), int64(2), object(12)\n",
      "memory usage: 86.4+ KB\n",
      "None\n",
      "\n",
      "\n"
     ]
    }
   ],
   "source": [
    "# Print summary statistics\n",
    "cc_apps_description = cc_apps.describe()\n",
    "print(cc_apps_description)\n",
    "\n",
    "print('\\n')\n",
    "\n",
    "# Print DataFrame information\n",
    "cc_apps_info = cc_apps.info()\n",
    "print(cc_apps_info)\n",
    "\n",
    "print('\\n')\n",
    "\n",
    "# Inspect missing values in the dataset\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 3. Splitting the dataset into train and test sets\n",
    "Now, we will split our data into train set and test set to prepare our data for two different phases of machine learning modeling: training and testing. Ideally, no information from the test data should be used to preprocess the training data or should be used to direct the training process of a machine learning model. Hence, we first split the data and then preprocess it.\n",
    "\n",
    "Also, features like DriversLicense and ZipCode are not as important as the other features in the dataset for predicting credit card approvals. To get a better sense, we can measure their statistical correlation to the labels of the dataset. But this is out of scope for this project. We should drop them to design our machine learning model with the best set of features. In Data Science literature, this is often referred to as feature selection."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>0</th>\n",
       "      <th>1</th>\n",
       "      <th>2</th>\n",
       "      <th>3</th>\n",
       "      <th>4</th>\n",
       "      <th>5</th>\n",
       "      <th>6</th>\n",
       "      <th>7</th>\n",
       "      <th>8</th>\n",
       "      <th>9</th>\n",
       "      <th>10</th>\n",
       "      <th>12</th>\n",
       "      <th>14</th>\n",
       "      <th>15</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>b</td>\n",
       "      <td>30.83</td>\n",
       "      <td>0.000</td>\n",
       "      <td>u</td>\n",
       "      <td>g</td>\n",
       "      <td>w</td>\n",
       "      <td>v</td>\n",
       "      <td>1.25</td>\n",
       "      <td>t</td>\n",
       "      <td>t</td>\n",
       "      <td>1</td>\n",
       "      <td>g</td>\n",
       "      <td>0</td>\n",
       "      <td>+</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>a</td>\n",
       "      <td>58.67</td>\n",
       "      <td>4.460</td>\n",
       "      <td>u</td>\n",
       "      <td>g</td>\n",
       "      <td>q</td>\n",
       "      <td>h</td>\n",
       "      <td>3.04</td>\n",
       "      <td>t</td>\n",
       "      <td>t</td>\n",
       "      <td>6</td>\n",
       "      <td>g</td>\n",
       "      <td>560</td>\n",
       "      <td>+</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>a</td>\n",
       "      <td>24.50</td>\n",
       "      <td>0.500</td>\n",
       "      <td>u</td>\n",
       "      <td>g</td>\n",
       "      <td>q</td>\n",
       "      <td>h</td>\n",
       "      <td>1.50</td>\n",
       "      <td>t</td>\n",
       "      <td>f</td>\n",
       "      <td>0</td>\n",
       "      <td>g</td>\n",
       "      <td>824</td>\n",
       "      <td>+</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>b</td>\n",
       "      <td>27.83</td>\n",
       "      <td>1.540</td>\n",
       "      <td>u</td>\n",
       "      <td>g</td>\n",
       "      <td>w</td>\n",
       "      <td>v</td>\n",
       "      <td>3.75</td>\n",
       "      <td>t</td>\n",
       "      <td>t</td>\n",
       "      <td>5</td>\n",
       "      <td>g</td>\n",
       "      <td>3</td>\n",
       "      <td>+</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>b</td>\n",
       "      <td>20.17</td>\n",
       "      <td>5.625</td>\n",
       "      <td>u</td>\n",
       "      <td>g</td>\n",
       "      <td>w</td>\n",
       "      <td>v</td>\n",
       "      <td>1.71</td>\n",
       "      <td>t</td>\n",
       "      <td>f</td>\n",
       "      <td>0</td>\n",
       "      <td>s</td>\n",
       "      <td>0</td>\n",
       "      <td>+</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>685</th>\n",
       "      <td>b</td>\n",
       "      <td>21.08</td>\n",
       "      <td>10.085</td>\n",
       "      <td>y</td>\n",
       "      <td>p</td>\n",
       "      <td>e</td>\n",
       "      <td>h</td>\n",
       "      <td>1.25</td>\n",
       "      <td>f</td>\n",
       "      <td>f</td>\n",
       "      <td>0</td>\n",
       "      <td>g</td>\n",
       "      <td>0</td>\n",
       "      <td>-</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>686</th>\n",
       "      <td>a</td>\n",
       "      <td>22.67</td>\n",
       "      <td>0.750</td>\n",
       "      <td>u</td>\n",
       "      <td>g</td>\n",
       "      <td>c</td>\n",
       "      <td>v</td>\n",
       "      <td>2.00</td>\n",
       "      <td>f</td>\n",
       "      <td>t</td>\n",
       "      <td>2</td>\n",
       "      <td>g</td>\n",
       "      <td>394</td>\n",
       "      <td>-</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>687</th>\n",
       "      <td>a</td>\n",
       "      <td>25.25</td>\n",
       "      <td>13.500</td>\n",
       "      <td>y</td>\n",
       "      <td>p</td>\n",
       "      <td>ff</td>\n",
       "      <td>ff</td>\n",
       "      <td>2.00</td>\n",
       "      <td>f</td>\n",
       "      <td>t</td>\n",
       "      <td>1</td>\n",
       "      <td>g</td>\n",
       "      <td>1</td>\n",
       "      <td>-</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>688</th>\n",
       "      <td>b</td>\n",
       "      <td>17.92</td>\n",
       "      <td>0.205</td>\n",
       "      <td>u</td>\n",
       "      <td>g</td>\n",
       "      <td>aa</td>\n",
       "      <td>v</td>\n",
       "      <td>0.04</td>\n",
       "      <td>f</td>\n",
       "      <td>f</td>\n",
       "      <td>0</td>\n",
       "      <td>g</td>\n",
       "      <td>750</td>\n",
       "      <td>-</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>689</th>\n",
       "      <td>b</td>\n",
       "      <td>35.00</td>\n",
       "      <td>3.375</td>\n",
       "      <td>u</td>\n",
       "      <td>g</td>\n",
       "      <td>c</td>\n",
       "      <td>h</td>\n",
       "      <td>8.29</td>\n",
       "      <td>f</td>\n",
       "      <td>f</td>\n",
       "      <td>0</td>\n",
       "      <td>g</td>\n",
       "      <td>0</td>\n",
       "      <td>-</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>690 rows × 14 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "    0      1       2  3  4   5   6     7  8  9   10 12   14 15\n",
       "0    b  30.83   0.000  u  g   w   v  1.25  t  t   1  g    0  +\n",
       "1    a  58.67   4.460  u  g   q   h  3.04  t  t   6  g  560  +\n",
       "2    a  24.50   0.500  u  g   q   h  1.50  t  f   0  g  824  +\n",
       "3    b  27.83   1.540  u  g   w   v  3.75  t  t   5  g    3  +\n",
       "4    b  20.17   5.625  u  g   w   v  1.71  t  f   0  s    0  +\n",
       "..  ..    ...     ... .. ..  ..  ..   ... .. ..  .. ..  ... ..\n",
       "685  b  21.08  10.085  y  p   e   h  1.25  f  f   0  g    0  -\n",
       "686  a  22.67   0.750  u  g   c   v  2.00  f  t   2  g  394  -\n",
       "687  a  25.25  13.500  y  p  ff  ff  2.00  f  t   1  g    1  -\n",
       "688  b  17.92   0.205  u  g  aa   v  0.04  f  f   0  g  750  -\n",
       "689  b  35.00   3.375  u  g   c   h  8.29  f  f   0  g    0  -\n",
       "\n",
       "[690 rows x 14 columns]"
      ]
     },
     "execution_count": 23,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Import train_test_split\n",
    "from sklearn.model_selection import train_test_split\n",
    "\n",
    "# Drop the features 11 and 13\n",
    "cc_apps = cc_apps.drop(cc_apps.columns[[11, 13]], axis = 1)\n",
    "\n",
    "# Split into train and test sets\n",
    "cc_apps_train, cc_apps_test = train_test_split(cc_apps, test_size=0.33, random_state=42)\n",
    "cc_apps"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 4. Handling the missing values (part i)\n",
    "Now we've split our data, we can handle some of the issues we identified when inspecting the DataFrame, including:\n",
    "\n",
    "Our dataset contains both numeric and non-numeric data (specifically data that are of float64, int64 and object types). Specifically, the features 2, 7, 10 and 14 contain numeric values (of types float64, float64, int64 and int64 respectively) and all the other features contain non-numeric values.\n",
    "The dataset also contains values from several ranges. Some features have a value range of 0 - 28, some have a range of 2 - 67, and some have a range of 1017 - 100000. Apart from these, we can get useful statistical information (like mean, max, and min) about the features that have numerical values.\n",
    "Finally, the dataset has missing values, which we'll take care of in this task. The missing values in the dataset are labeled with '?', which can be seen in the last cell's output of the second task.\n",
    "Now, let's temporarily replace these missing value question marks with NaN.\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "      0      1      2    3    4    5    6       7  8  9   10 12    14 15\n",
      "382    a  24.33  2.500    y    p    i   bb   4.500  f  f   0  g   456  -\n",
      "137    b  33.58  2.750    u    g    m    v   4.250  t  t   6  g     0  +\n",
      "346  NaN  32.25  1.500    u    g    c    v   0.250  f  f   0  g   122  -\n",
      "326    b  30.17  1.085    y    p    c    v   0.040  f  f   0  g   179  -\n",
      "33     a  36.75  5.125    u    g    e    v   5.000  t  f   0  g  4000  +\n",
      "..   ...    ...    ...  ...  ...  ...  ...     ... .. ..  .. ..   ... ..\n",
      "71     b  34.83  4.000    u    g    d   bb  12.500  t  f   0  g     0  -\n",
      "106    b  28.75  1.165    u    g    k    v   0.500  t  f   0  s     0  -\n",
      "270    b  37.58  0.000  NaN  NaN  NaN  NaN   0.000  f  f   0  p     0  +\n",
      "435    b  19.00  0.000    y    p   ff   ff   0.000  f  t   4  g     1  -\n",
      "102    b  18.67  5.000    u    g    q    v   0.375  t  t   2  g    38  -\n",
      "\n",
      "[462 rows x 14 columns] \n",
      "\n",
      "\n",
      "        2       7   10    14\n",
      "382  2.500   4.500   0   456\n",
      "137  2.750   4.250   6     0\n",
      "346  1.500   0.250   0   122\n",
      "326  1.085   0.040   0   179\n",
      "33   5.125   5.000   0  4000\n",
      "..     ...     ...  ..   ...\n",
      "71   4.000  12.500   0     0\n",
      "106  1.165   0.500   0     0\n",
      "270  0.000   0.000   0     0\n",
      "435  0.000   0.000   4     1\n",
      "102  5.000   0.375   2    38\n",
      "\n",
      "[462 rows x 4 columns]\n"
     ]
    }
   ],
   "source": [
    "# Import numpy\n",
    "import numpy as np\n",
    "\n",
    "# Replace the '?'s with NaN in the train and test sets\n",
    "cc_apps_train = cc_apps_train.replace(\"?\",\"NaN\")\n",
    "cc_apps_test = cc_apps_test.replace(\"?\",\"NaN\")\n",
    "print(cc_apps_train, \"\\n\\n\")\n",
    "print(cc_apps_train.iloc[:,[2,7,10,12]])"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 5. Handling the missing values (part ii)\n",
    "We replaced all the question marks with NaNs. This is going to help us in the next missing value treatment that we are going to perform.\n",
    "\n",
    "An important question that gets raised here is why are we giving so much importance to missing values? Can't they be just ignored? Ignoring missing values can affect the performance of a machine learning model heavily. While ignoring the missing values our machine learning model may miss out on information about the dataset that may be useful for its training. Then, there are many models which cannot handle missing values implicitly such as Linear Discriminant Analysis (LDA).\n",
    "\n",
    "So, to avoid this problem, we are going to impute the missing values with a strategy called mean imputation."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0     0\n",
      "1     0\n",
      "2     0\n",
      "3     0\n",
      "4     0\n",
      "5     0\n",
      "6     0\n",
      "7     0\n",
      "8     0\n",
      "9     0\n",
      "10    0\n",
      "12    0\n",
      "14    0\n",
      "15    0\n",
      "dtype: int64\n",
      "0     0\n",
      "1     0\n",
      "2     0\n",
      "3     0\n",
      "4     0\n",
      "5     0\n",
      "6     0\n",
      "7     0\n",
      "8     0\n",
      "9     0\n",
      "10    0\n",
      "12    0\n",
      "14    0\n",
      "15    0\n",
      "dtype: int64\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\Users\\hp\\anaconda3\\lib\\site-packages\\pandas\\core\\generic.py:6245: SettingWithCopyWarning: \n",
      "A value is trying to be set on a copy of a slice from a DataFrame\n",
      "\n",
      "See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy\n",
      "  self._update_inplace(new_data)\n"
     ]
    }
   ],
   "source": [
    "# Impute the missing values with mean imputation\n",
    "cc_apps_train.iloc[:,[2,7,10,12]].fillna(cc_apps_train.iloc[:,[2,7,10,12]].mean(), inplace=True)\n",
    "cc_apps_test.iloc[:,[2,7,10,12]].fillna(cc_apps_train.iloc[:,[2,7,10,12]].mean(), inplace=True)\n",
    "\n",
    "# Count the number of NaNs in the datasets and print the counts to verify\n",
    "print(cc_apps_train.isna().sum())\n",
    "print(cc_apps_test.isna().sum())"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 6. Handling the missing values (part iii)\n",
    "We have successfully taken care of the missing values present in the numeric columns. There are still some missing values to be imputed for columns 0, 1, 3, 4, 5, 6 and 13. All of these columns contain non-numeric data and this is why the mean imputation strategy would not work here. This needs a different treatment.\n",
    "\n",
    "We are going to impute these missing values with the most frequent values as present in the respective columns. This is good practice when it comes to imputing missing values for categorical data in general."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0     0\n",
      "1     0\n",
      "2     0\n",
      "3     0\n",
      "4     0\n",
      "5     0\n",
      "6     0\n",
      "7     0\n",
      "8     0\n",
      "9     0\n",
      "10    0\n",
      "12    0\n",
      "14    0\n",
      "15    0\n",
      "dtype: int64 \n",
      "\n",
      "\n",
      "0     0\n",
      "1     0\n",
      "2     0\n",
      "3     0\n",
      "4     0\n",
      "5     0\n",
      "6     0\n",
      "7     0\n",
      "8     0\n",
      "9     0\n",
      "10    0\n",
      "12    0\n",
      "14    0\n",
      "15    0\n",
      "dtype: int64 \n",
      "\n",
      "\n",
      "      0      1      2    3    4    5    6       7  8  9   10 12    14 15\n",
      "382    a  24.33  2.500    y    p    i   bb   4.500  f  f   0  g   456  -\n",
      "137    b  33.58  2.750    u    g    m    v   4.250  t  t   6  g     0  +\n",
      "346  NaN  32.25  1.500    u    g    c    v   0.250  f  f   0  g   122  -\n",
      "326    b  30.17  1.085    y    p    c    v   0.040  f  f   0  g   179  -\n",
      "33     a  36.75  5.125    u    g    e    v   5.000  t  f   0  g  4000  +\n",
      "..   ...    ...    ...  ...  ...  ...  ...     ... .. ..  .. ..   ... ..\n",
      "71     b  34.83  4.000    u    g    d   bb  12.500  t  f   0  g     0  -\n",
      "106    b  28.75  1.165    u    g    k    v   0.500  t  f   0  s     0  -\n",
      "270    b  37.58  0.000  NaN  NaN  NaN  NaN   0.000  f  f   0  p     0  +\n",
      "435    b  19.00  0.000    y    p   ff   ff   0.000  f  t   4  g     1  -\n",
      "102    b  18.67  5.000    u    g    q    v   0.375  t  t   2  g    38  -\n",
      "\n",
      "[462 rows x 14 columns]\n"
     ]
    }
   ],
   "source": [
    "# Iterate over each column of cc_apps_train\n",
    "for col in cc_apps_train.columns:\n",
    "    # Check if the column is of object type\n",
    "    if col == 'object':\n",
    "        # Impute with the most frequent value\n",
    "        cc_apps_train = cc_apps_train.fillna(cc_apps_train.value_counts())\n",
    "        cc_apps_test = cc_apps_test.fillna(cc_apps_test.value_counts())\n",
    "\n",
    "# Count the number of NaNs in the dataset and print the counts to verify\n",
    "print(cc_apps_train.isna().sum(), \"\\n\\n\")\n",
    "print(cc_apps_test.isna().sum(), \"\\n\\n\")\n",
    "print(cc_apps_train)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 7. Preprocessing the data (part i)\n",
    "The missing values are now successfully handled.\n",
    "\n",
    "There is still some minor but essential data preprocessing needed before we proceed towards building our machine learning model. We are going to divide these remaining preprocessing steps into two main tasks:\n",
    "\n",
    "Convert the non-numeric data into numeric.\n",
    "Scale the feature values to a uniform range.\n",
    "First, we will be converting all the non-numeric values into numeric ones. We do this because not only it results in a faster computation but also many machine learning models (like XGBoost) (and especially the ones developed using scikit-learn) require the data to be in a strictly numeric format. We will do this by using the get_dummies() method from pandas."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "         2       7  10    14  0_NaN  0_a  0_b  1_13.75  1_15.83  1_15.92  ...  \\\n",
      "382  2.500   4.500   0   456      0    1    0        0        0        0  ...   \n",
      "137  2.750   4.250   6     0      0    0    1        0        0        0  ...   \n",
      "346  1.500   0.250   0   122      1    0    0        0        0        0  ...   \n",
      "326  1.085   0.040   0   179      0    0    1        0        0        0  ...   \n",
      "33   5.125   5.000   0  4000      0    1    0        0        0        0  ...   \n",
      "..     ...     ...  ..   ...    ...  ...  ...      ...      ...      ...  ...   \n",
      "71   4.000  12.500   0     0      0    0    1        0        0        0  ...   \n",
      "106  1.165   0.500   0     0      0    0    1        0        0        0  ...   \n",
      "270  0.000   0.000   0     0      0    0    1        0        0        0  ...   \n",
      "435  0.000   0.000   4     1      0    0    1        0        0        0  ...   \n",
      "102  5.000   0.375   2    38      0    0    1        0        0        0  ...   \n",
      "\n",
      "     6_z  8_f  8_t  9_f  9_t  12_g  12_p  12_s  15_+  15_-  \n",
      "382    0    1    0    1    0     1     0     0     0     1  \n",
      "137    0    0    1    0    1     1     0     0     1     0  \n",
      "346    0    1    0    1    0     1     0     0     0     1  \n",
      "326    0    1    0    1    0     1     0     0     0     1  \n",
      "33     0    0    1    1    0     1     0     0     1     0  \n",
      "..   ...  ...  ...  ...  ...   ...   ...   ...   ...   ...  \n",
      "71     0    0    1    1    0     1     0     0     0     1  \n",
      "106    0    0    1    1    0     0     0     1     0     1  \n",
      "270    0    1    0    1    0     0     1     0     1     0  \n",
      "435    0    1    0    0    1     1     0     0     0     1  \n",
      "102    0    0    1    0    1     1     0     0     0     1  \n",
      "\n",
      "[462 rows x 335 columns] \n",
      "\n",
      "\n",
      "         2       7  10    14  0_NaN  0_a  0_b  1_13.75  1_15.83  1_15.92  ...  \\\n",
      "286   1.50   0.000   2   105      0    1    0        0        0        0  ...   \n",
      "511   4.00   0.000   0   960      0    1    0        0        0        0  ...   \n",
      "257   0.00   0.500   0     0      0    0    1        0        0        0  ...   \n",
      "336   6.50   1.000   0   228      0    0    1        0        0        0  ...   \n",
      "318   0.00   0.000   0     1      0    0    1        0        0        0  ...   \n",
      "..     ...     ...  ..   ...    ...  ...  ...      ...      ...      ...  ...   \n",
      "375   0.50   1.000   0     0      0    1    0        0        0        0  ...   \n",
      "234  21.00  10.000  13  6700      0    1    0        0        0        0  ...   \n",
      "644   0.42   0.290   0     2      0    0    1        0        0        0  ...   \n",
      "271   2.50   1.250   0     0      0    0    1        0        0        0  ...   \n",
      "311   1.75   2.335   0     6      0    0    1        0        0        0  ...   \n",
      "\n",
      "     6_z  8_f  8_t  9_f  9_t  12_g  12_p  12_s  15_+  15_-  \n",
      "286    0    1    0    0    1     1     0     0     0     1  \n",
      "511    0    0    1    1    0     1     0     0     1     0  \n",
      "257    0    1    0    1    0     1     0     0     0     1  \n",
      "336    0    1    0    1    0     1     0     0     0     1  \n",
      "318    0    1    0    1    0     0     0     1     1     0  \n",
      "..   ...  ...  ...  ...  ...   ...   ...   ...   ...   ...  \n",
      "375    0    1    0    1    0     1     0     0     0     1  \n",
      "234    0    0    1    0    1     1     0     0     1     0  \n",
      "644    0    1    0    1    0     1     0     0     0     1  \n",
      "271    0    1    0    1    0     1     0     0     0     1  \n",
      "311    0    1    0    1    0     1     0     0     0     1  \n",
      "\n",
      "[228 rows x 335 columns] \n",
      "\n",
      "\n"
     ]
    }
   ],
   "source": [
    "# Convert the categorical features in the train and test sets independently\n",
    "cc_apps_train = pd.get_dummies(cc_apps_train)\n",
    "cc_apps_test = pd.get_dummies(cc_apps_test)\n",
    "\n",
    "# Reindex the columns of the test set aligning with the train set\n",
    "cc_apps_test = cc_apps_test.reindex(columns=cc_apps_train.columns, fill_value=0)\n",
    "\n",
    "print(cc_apps_train,\"\\n\\n\")\n",
    "print(cc_apps_test,\"\\n\\n\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 8. Preprocessing the data (part ii)\n",
    "Now, we are only left with one final preprocessing step of scaling before we can fit a machine learning model to the data.\n",
    "\n",
    "Now, let's try to understand what these scaled values mean in the real world. Let's use CreditScore as an example. The credit score of a person is their creditworthiness based on their credit history. The higher this number, the more financially trustworthy a person is considered to be. So, a CreditScore of 1 is the highest since we're rescaling all the values to the range of 0-1."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 44,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[[0.0949307  0.225      0.         ... 0.         0.         0.        ]\n",
      " [0.10442377 0.2125     0.08955224 ... 0.         0.         1.        ]\n",
      " [0.05695842 0.0125     0.         ... 0.         0.         0.        ]\n",
      " ...\n",
      " [0.         0.         0.         ... 1.         0.         1.        ]\n",
      " [0.         0.         0.05970149 ... 0.         0.         0.        ]\n",
      " [0.1898614  0.01875    0.02985075 ... 0.         0.         0.        ]] \n",
      "\n",
      "\n",
      "[[0.05695842 0.         0.02985075 ... 0.         0.         0.        ]\n",
      " [0.15188912 0.         0.         ... 0.         0.         1.        ]\n",
      " [0.         0.025      0.         ... 0.         0.         0.        ]\n",
      " ...\n",
      " [0.01594836 0.0145     0.         ... 0.         0.         0.        ]\n",
      " [0.0949307  0.0625     0.         ... 0.         0.         0.        ]\n",
      " [0.06645149 0.11675    0.         ... 0.         0.         0.        ]] \n",
      "\n",
      "\n",
      "True\n",
      "462\n",
      "462\n"
     ]
    }
   ],
   "source": [
    "# Import MinMaxScaler\n",
    "from sklearn.preprocessing import MinMaxScaler\n",
    "\n",
    "# Segregate features and labels into separate variables\n",
    "X_train, y_train = cc_apps_train.iloc[:,0:-1].values, cc_apps_train.iloc[:,-1].values\n",
    "X_test, y_test = cc_apps_test.iloc[:,0:-1].values, cc_apps_test.iloc[:,-1].values\n",
    "\n",
    "# Instantiate MinMaxScaler and use it to rescale X_train and X_test\n",
    "scaler = MinMaxScaler(feature_range = (0,1))\n",
    "rescaledX_train = scaler.fit_transform(X_train)\n",
    "rescaledX_test = scaler.transform(X_test)\n",
    "\n",
    "print(rescaledX_train,\"\\n\\n\")\n",
    "print(rescaledX_test,\"\\n\\n\")\n",
    "\n",
    "print(len(rescaledX_train) == len(y_train))\n",
    "print(len(rescaledX_train))\n",
    "print(len(y_train))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 9. Fitting a logistic regression model to the train set\n",
    "Essentially, predicting if a credit card application will be approved or not is a classification task. According to UCI, our dataset contains more instances that correspond to \"Denied\" status than instances corresponding to \"Approved\" status. Specifically, out of 690 instances, there are 383 (55.5%) applications that got denied and 307 (44.5%) applications that got approved.\n",
    "\n",
    "This gives us a benchmark. A good machine learning model should be able to accurately predict the status of the applications with respect to these statistics.\n",
    "\n",
    "Which model should we pick? A question to ask is: are the features that affect the credit card approval decision process correlated with each other? Although we can measure correlation, that is outside the scope of this notebook, so we'll rely on our intuition that they indeed are correlated for now. Because of this correlation, we'll take advantage of the fact that generalized linear models perform well in these cases. Let's start our machine learning modeling with a Logistic Regression model (a generalized linear model)."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 42,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "LogisticRegression()"
      ]
     },
     "execution_count": 42,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Import LogisticRegression\n",
    "from sklearn.linear_model import LogisticRegression \n",
    "\n",
    "# Instantiate a LogisticRegression classifier with default parameter values\n",
    "logreg = LogisticRegression()\n",
    "\n",
    "# Fit logreg to the train set\n",
    "logreg.fit(rescaledX_train, y_train)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 10. Making predictions and evaluating performance\n",
    "But how well does our model perform?\n",
    "\n",
    "We will now evaluate our model on the test set with respect to classification accuracy. But we will also take a look the model's confusion matrix. In the case of predicting credit card applications, it is important to see if our machine learning model is equally capable of predicting approved and denied status, in line with the frequency of these labels in our original dataset. If our model is not performing well in this aspect, then it might end up approving the application that should have been approved. The confusion matrix helps us to view our model's performance from these aspects."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 45,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Accuracy of logistic regression classifier:  1.0\n",
      "[[103   0]\n",
      " [  0 125]]\n"
     ]
    }
   ],
   "source": [
    "# Import confusion_matrix\n",
    "from sklearn.metrics import confusion_matrix\n",
    "\n",
    "# Use logreg to predict instances from the test set and store it\n",
    "y_pred = logreg.predict(rescaledX_test)\n",
    "\n",
    "# Get the accuracy score of logreg model and print it\n",
    "print(\"Accuracy of logistic regression classifier: \", logreg.score(rescaledX_test, y_test))\n",
    "\n",
    "# Print the confusion matrix of the logreg model\n",
    "print(confusion_matrix(y_test,y_pred))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 11. Grid searching and making the model perform better\n",
    "Our model was pretty good! In fact it was able to yield an accuracy score of 100%.\n",
    "\n",
    "For the confusion matrix, the first element of the of the first row of the confusion matrix denotes the true negatives meaning the number of negative instances (denied applications) predicted by the model correctly. And the last element of the second row of the confusion matrix denotes the true positives meaning the number of positive instances (approved applications) predicted by the model correctly.\n",
    "\n",
    "But if we hadn't got a perfect score what's to be done?. We can perform a grid search of the model parameters to improve the model's ability to predict credit card approvals.\n",
    "\n",
    "scikit-learn's implementation of logistic regression consists of different hyperparameters but we will grid search over the following two:\n",
    "\n",
    "tol\n",
    "max_iter"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 46,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Import GridSearchCV\n",
    "from sklearn.model_selection import GridSearchCV\n",
    "\n",
    "# Define the grid of values for tol and max_iter\n",
    "tol = [0.01, 0.001, 0.0001]\n",
    "max_iter = [100, 150, 200]\n",
    "\n",
    "# Create a dictionary where tol and max_iter are keys and the lists of their values are corresponding values\n",
    "param_grid = dict(tol = tol, max_iter = max_iter)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 12. Finding the best performing model\n",
    "We have defined the grid of hyperparameter values and converted them into a single dictionary format which GridSearchCV() expects as one of its parameters. Now, we will begin the grid search to see which values perform best.\n",
    "\n",
    "We will instantiate GridSearchCV() with our earlier logreg model with all the data we have. We will also instruct GridSearchCV() to perform a cross-validation of five folds.\n",
    "\n",
    "We'll end the notebook by storing the best-achieved score and the respective best parameters.\n",
    "\n",
    "While building this credit card predictor, we tackled some of the most widely-known preprocessing steps such as scaling, label encoding, and missing value imputation. We finished with some machine learning to predict if a person's application for a credit card would get approved or not given some information about that person."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 47,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Best: 1.000000 using {'max_iter': 100, 'tol': 0.01}\n",
      "Accuracy of logistic regression classifier:  1.0\n"
     ]
    }
   ],
   "source": [
    "# Instantiate GridSearchCV with the required parameters\n",
    "grid_model = GridSearchCV(estimator=logreg, param_grid=param_grid, cv=5)\n",
    "\n",
    "# Fit grid_model to the data\n",
    "grid_model_result = grid_model.fit(rescaledX_train, y_train)\n",
    "\n",
    "# Summarize results\n",
    "best_score, best_params = grid_model_result.best_score_, grid_model_result.best_params_\n",
    "print(\"Best: %f using %s\" % (best_score, best_params))\n",
    "\n",
    "# Extract the best model and evaluate it on the test set\n",
    "best_model = grid_model_result.best_estimator_\n",
    "print(\"Accuracy of logistic regression classifier: \" ,best_model.score(rescaledX_test,y_test))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.3"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}