2.5_implementing-gradient-descent.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Implementing Gradient Descent\n",
    "\n",
    "Gradient Descent for a **binary classification** using **logistic regression**.\n",
    "\n",
    "This is an exercise from Udacity Nanodegree **nd101-ent** lesson 2.5\n",
    "\n",
    "## Data Preparation\n",
    "\n",
    "This requires to have a Pandas version `conda install pandas==0.25.1`"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "import warnings; warnings.simplefilter('ignore')     # Stop Jupyter Notebook from printing warnings\n",
    "import numpy as np\n",
    "import pandas as pd"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>admit</th>\n",
       "      <th>gre</th>\n",
       "      <th>gpa</th>\n",
       "      <th>rank</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>380</td>\n",
       "      <td>3.61</td>\n",
       "      <td>3</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>660</td>\n",
       "      <td>3.67</td>\n",
       "      <td>3</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>2</td>\n",
       "      <td>1</td>\n",
       "      <td>800</td>\n",
       "      <td>4.00</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>3</td>\n",
       "      <td>1</td>\n",
       "      <td>640</td>\n",
       "      <td>3.19</td>\n",
       "      <td>4</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>4</td>\n",
       "      <td>0</td>\n",
       "      <td>520</td>\n",
       "      <td>2.93</td>\n",
       "      <td>4</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>395</td>\n",
       "      <td>0</td>\n",
       "      <td>620</td>\n",
       "      <td>4.00</td>\n",
       "      <td>2</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>396</td>\n",
       "      <td>0</td>\n",
       "      <td>560</td>\n",
       "      <td>3.04</td>\n",
       "      <td>3</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>397</td>\n",
       "      <td>0</td>\n",
       "      <td>460</td>\n",
       "      <td>2.63</td>\n",
       "      <td>2</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>398</td>\n",
       "      <td>0</td>\n",
       "      <td>700</td>\n",
       "      <td>3.65</td>\n",
       "      <td>2</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>399</td>\n",
       "      <td>0</td>\n",
       "      <td>600</td>\n",
       "      <td>3.89</td>\n",
       "      <td>3</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>400 rows × 4 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "     admit  gre   gpa  rank\n",
       "0        0  380  3.61     3\n",
       "1        1  660  3.67     3\n",
       "2        1  800  4.00     1\n",
       "3        1  640  3.19     4\n",
       "4        0  520  2.93     4\n",
       "..     ...  ...   ...   ...\n",
       "395      0  620  4.00     2\n",
       "396      0  560  3.04     3\n",
       "397      0  460  2.63     2\n",
       "398      0  700  3.65     2\n",
       "399      0  600  3.89     3\n",
       "\n",
       "[400 rows x 4 columns]"
      ]
     },
     "execution_count": 10,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "admissions = pd.read_csv('data/binary.csv')\n",
    "admissions"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>admit</th>\n",
       "      <th>gre</th>\n",
       "      <th>gpa</th>\n",
       "      <th>rank</th>\n",
       "      <th>rank_1</th>\n",
       "      <th>rank_2</th>\n",
       "      <th>rank_3</th>\n",
       "      <th>rank_4</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>380</td>\n",
       "      <td>3.61</td>\n",
       "      <td>3</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>660</td>\n",
       "      <td>3.67</td>\n",
       "      <td>3</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>2</td>\n",
       "      <td>1</td>\n",
       "      <td>800</td>\n",
       "      <td>4.00</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>3</td>\n",
       "      <td>1</td>\n",
       "      <td>640</td>\n",
       "      <td>3.19</td>\n",
       "      <td>4</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>4</td>\n",
       "      <td>0</td>\n",
       "      <td>520</td>\n",
       "      <td>2.93</td>\n",
       "      <td>4</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>395</td>\n",
       "      <td>0</td>\n",
       "      <td>620</td>\n",
       "      <td>4.00</td>\n",
       "      <td>2</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>396</td>\n",
       "      <td>0</td>\n",
       "      <td>560</td>\n",
       "      <td>3.04</td>\n",
       "      <td>3</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>397</td>\n",
       "      <td>0</td>\n",
       "      <td>460</td>\n",
       "      <td>2.63</td>\n",
       "      <td>2</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>398</td>\n",
       "      <td>0</td>\n",
       "      <td>700</td>\n",
       "      <td>3.65</td>\n",
       "      <td>2</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>399</td>\n",
       "      <td>0</td>\n",
       "      <td>600</td>\n",
       "      <td>3.89</td>\n",
       "      <td>3</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>400 rows × 8 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "     admit  gre   gpa  rank  rank_1  rank_2  rank_3  rank_4\n",
       "0        0  380  3.61     3       0       0       1       0\n",
       "1        1  660  3.67     3       0       0       1       0\n",
       "2        1  800  4.00     1       1       0       0       0\n",
       "3        1  640  3.19     4       0       0       0       1\n",
       "4        0  520  2.93     4       0       0       0       1\n",
       "..     ...  ...   ...   ...     ...     ...     ...     ...\n",
       "395      0  620  4.00     2       0       1       0       0\n",
       "396      0  560  3.04     3       0       0       1       0\n",
       "397      0  460  2.63     2       0       1       0       0\n",
       "398      0  700  3.65     2       0       1       0       0\n",
       "399      0  600  3.89     3       0       0       1       0\n",
       "\n",
       "[400 rows x 8 columns]"
      ]
     },
     "execution_count": 11,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# make dummy variables for rank\n",
    "data = pd.concat([admissions, pd.get_dummies(admissions['rank'], prefix='rank')], axis=1)\n",
    "data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>admit</th>\n",
       "      <th>gre</th>\n",
       "      <th>gpa</th>\n",
       "      <th>rank_1</th>\n",
       "      <th>rank_2</th>\n",
       "      <th>rank_3</th>\n",
       "      <th>rank_4</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>380</td>\n",
       "      <td>3.61</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>660</td>\n",
       "      <td>3.67</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>2</td>\n",
       "      <td>1</td>\n",
       "      <td>800</td>\n",
       "      <td>4.00</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>3</td>\n",
       "      <td>1</td>\n",
       "      <td>640</td>\n",
       "      <td>3.19</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>4</td>\n",
       "      <td>0</td>\n",
       "      <td>520</td>\n",
       "      <td>2.93</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>395</td>\n",
       "      <td>0</td>\n",
       "      <td>620</td>\n",
       "      <td>4.00</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>396</td>\n",
       "      <td>0</td>\n",
       "      <td>560</td>\n",
       "      <td>3.04</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>397</td>\n",
       "      <td>0</td>\n",
       "      <td>460</td>\n",
       "      <td>2.63</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>398</td>\n",
       "      <td>0</td>\n",
       "      <td>700</td>\n",
       "      <td>3.65</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>399</td>\n",
       "      <td>0</td>\n",
       "      <td>600</td>\n",
       "      <td>3.89</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>400 rows × 7 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "     admit  gre   gpa  rank_1  rank_2  rank_3  rank_4\n",
       "0        0  380  3.61       0       0       1       0\n",
       "1        1  660  3.67       0       0       1       0\n",
       "2        1  800  4.00       1       0       0       0\n",
       "3        1  640  3.19       0       0       0       1\n",
       "4        0  520  2.93       0       0       0       1\n",
       "..     ...  ...   ...     ...     ...     ...     ...\n",
       "395      0  620  4.00       0       1       0       0\n",
       "396      0  560  3.04       0       0       1       0\n",
       "397      0  460  2.63       0       1       0       0\n",
       "398      0  700  3.65       0       1       0       0\n",
       "399      0  600  3.89       0       0       1       0\n",
       "\n",
       "[400 rows x 7 columns]"
      ]
     },
     "execution_count": 12,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "data = data.drop('rank', axis=1)\n",
    "data"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "We **normalize** the GRE and GPA data, which means to scale the values such that they have **zero mean** and a **standard deviation of 1**. \n",
    "\n",
    "This is necessary because the sigmoid function squashes really small and really large inputs. The gradient of really small and large inputs is zero, which means that the gradient descent step will go to zero too. Otherwise we would have to be really careful about how we initialize the weights or the gradient descent steps will die off and the network won't train."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>admit</th>\n",
       "      <th>gre</th>\n",
       "      <th>gpa</th>\n",
       "      <th>rank_1</th>\n",
       "      <th>rank_2</th>\n",
       "      <th>rank_3</th>\n",
       "      <th>rank_4</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>-1.798011</td>\n",
       "      <td>0.578348</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>0.625884</td>\n",
       "      <td>0.736008</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>2</td>\n",
       "      <td>1</td>\n",
       "      <td>1.837832</td>\n",
       "      <td>1.603135</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>3</td>\n",
       "      <td>1</td>\n",
       "      <td>0.452749</td>\n",
       "      <td>-0.525269</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>4</td>\n",
       "      <td>0</td>\n",
       "      <td>-0.586063</td>\n",
       "      <td>-1.208461</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>395</td>\n",
       "      <td>0</td>\n",
       "      <td>0.279614</td>\n",
       "      <td>1.603135</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>396</td>\n",
       "      <td>0</td>\n",
       "      <td>-0.239793</td>\n",
       "      <td>-0.919418</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>397</td>\n",
       "      <td>0</td>\n",
       "      <td>-1.105469</td>\n",
       "      <td>-1.996759</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>398</td>\n",
       "      <td>0</td>\n",
       "      <td>0.972155</td>\n",
       "      <td>0.683454</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>399</td>\n",
       "      <td>0</td>\n",
       "      <td>0.106478</td>\n",
       "      <td>1.314093</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>400 rows × 7 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "     admit       gre       gpa  rank_1  rank_2  rank_3  rank_4\n",
       "0        0 -1.798011  0.578348       0       0       1       0\n",
       "1        1  0.625884  0.736008       0       0       1       0\n",
       "2        1  1.837832  1.603135       1       0       0       0\n",
       "3        1  0.452749 -0.525269       0       0       0       1\n",
       "4        0 -0.586063 -1.208461       0       0       0       1\n",
       "..     ...       ...       ...     ...     ...     ...     ...\n",
       "395      0  0.279614  1.603135       0       1       0       0\n",
       "396      0 -0.239793 -0.919418       0       0       1       0\n",
       "397      0 -1.105469 -1.996759       0       1       0       0\n",
       "398      0  0.972155  0.683454       0       1       0       0\n",
       "399      0  0.106478  1.314093       0       0       1       0\n",
       "\n",
       "[400 rows x 7 columns]"
      ]
     },
     "execution_count": 13,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# normalize features\n",
    "for field in ['gre', 'gpa']:\n",
    "    mean, std = data[field].mean(), data[field].std()\n",
    "    data.loc[:,field] = (data[field] - mean) / std\n",
    "\n",
    "data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Split off random 10% of the data for testing\n",
    "np.random.seed(42)\n",
    "sample = np.random.choice(data.index, size=int(len(data) * 0.9), replace=False)\n",
    "data, test_data = data.ix[sample], data.drop(sample)\n",
    "\n",
    "# Split into features and targets\n",
    "features, targets = data.drop('admit', axis=1), data['admit']\n",
    "features_test, targets_test = test_data.drop('admit', axis=1), test_data['admit']"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Gradient Descent Algorithm\n",
    "\n",
    "We use MSE as Loss Function:\n",
    "\n",
    "$$E = \\frac{1}{2m} \\sum_\\mu^m \\big( y^{\\,\\mu} - \\hat{y}^{\\,\\mu}\\big)^2$$\n",
    "\n",
    "The general algorithm for updating the weights with gradient descent is:\n",
    "\n",
    "* Set the weight step to zero: $\\Delta w_i = 0$\n",
    "* For each record in the training data:\n",
    "  * Make a forward pass through the network, calculating the output $\\hat y = f(\\sum_i w_i x_i)$\n",
    "  * Calculate the error term for the output unit, $\\delta = (y - \\hat y) * f'(\\sum_i w_i x_i)$\n",
    "  * Update the weight step $\\Delta w_i = \\Delta w_i + \\delta x_i$\n",
    "* Update the weights $w_i = w_i + \\eta \\Delta w_i / m$ where $\\eta$ is the learning rate and $m$ is the number of records. Here we're averaging the weight steps to help reduce any large variations in the training data.\n",
    "* Repeat for $e$ epochs."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([ 0.2027827 , -0.05644616,  0.26441774,  0.62177434, -0.09559271,\n",
       "       -0.09558601])"
      ]
     },
     "execution_count": 15,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "import numpy as np\n",
    "np.random.seed(42)\n",
    "\n",
    "def sigmoid(x):\n",
    "    return 1 / (1 + np.exp(-x))\n",
    "\n",
    "n_records, n_features = features.shape\n",
    "last_loss = None\n",
    "\n",
    "# initialize weights\n",
    "weights = np.random.normal(scale=1 / n_features ** .5, size=n_features)\n",
    "weights"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Train loss:  0.2627609384996635\n",
      "Train loss:  0.20928619409324875\n",
      "Train loss:  0.20084292908073426\n",
      "Train loss:  0.19862156475527873\n",
      "Train loss:  0.1977985139668603\n",
      "Train loss:  0.19742577912189863\n",
      "Train loss:  0.1972350774624106\n",
      "Train loss:  0.1971294562509248\n",
      "Train loss:  0.19706766341315082\n",
      "Train loss:  0.19703005801777368\n",
      "Prediction accuracy: 0.725\n"
     ]
    }
   ],
   "source": [
    "# Neural Network Hyperparameters\n",
    "epochs = 1000\n",
    "learnrate = 0.5\n",
    "\n",
    "for e in range(epochs):\n",
    "    del_w = np.zeros(weights.shape)\n",
    "\n",
    "    # loop through all records, x is the input, y is the target\n",
    "    for x, y in zip(features.values, targets):\n",
    "        output = sigmoid(np.dot(x, weights))\n",
    "        error = y - output\n",
    "        error_term = error * output * (1 - output)\n",
    "        del_w += error_term * x\n",
    "\n",
    "    weights += learnrate * del_w / n_records\n",
    "\n",
    "    # printing out the mean square error on the training set\n",
    "    if e % (epochs / 10) == 0:\n",
    "        out = sigmoid(np.dot(features, weights))\n",
    "        loss = np.mean((out - targets) ** 2)\n",
    "        if last_loss and last_loss < loss:\n",
    "            print(\"Train loss: \", loss, \"  WARNING - Loss Increasing\")\n",
    "        else:\n",
    "            print(\"Train loss: \", loss)\n",
    "        last_loss = loss\n",
    "\n",
    "\n",
    "# Calculate accuracy on test data\n",
    "tes_out = sigmoid(np.dot(features_test, weights))\n",
    "predictions = tes_out > 0.5\n",
    "accuracy = np.mean(predictions == targets_test)\n",
    "print(\"Prediction accuracy: {:.3f}\".format(accuracy))"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.7"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}