CNN_Python.ipynb

{
  "cells": [
    {
      "cell_type": "markdown",
      "metadata": {
        "colab_type": "text",
        "id": "view-in-github"
      },
      "source": [
        "<a href=\"https://colab.research.google.com/github/ShoaibSajid/Python_CNN/blob/main/CNN_Python.ipynb\" target=\"_parent\"><img src=\"https://colab.research.google.com/assets/colab-badge.svg\" alt=\"Open In Colab\"/></a>"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 1,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "cw04b-NnPF_e",
        "outputId": "59807758-c12b-41c0-94ac-53f957f7bfbf"
      },
      "outputs": [],
      "source": [
        "%cd ~\n",
        "![ ! -d \"./Python_CNN\" ] && echo \"Github Repo DOES NOT exists.\"\n",
        "![ ! -d \"./Python_CNN\" ] && git clone https://github.com/ShoaibSajid/Python_CNN\n",
        "![ -d \"./Python_CNN\" ] && echo \"Github Repo exist.\"\n",
        "%cd Python_CNN\n",
        "%pip install -r requirements.txt\n",
        "!echo \n",
        "!echo List of files\n",
        "!ls"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 2,
      "metadata": {
        "id": "I-725DWZTpo6"
      },
      "outputs": [],
      "source": [
        "# Importing libraries\n",
        "\n",
        "from logging import raiseExceptions\n",
        "import os\n",
        "import pickle\n",
        "from tqdm import tqdm\n",
        "import mnist\n",
        "import numpy as np\n",
        "from conv import Conv3x3, Conv3x3_n_to_n_padding, Conv3x3_1_to_n_padding\n",
        "from maxpool import MaxPool2\n",
        "from softmax import Softmax\n",
        "from relu import Relu\n",
        "from softmax_test import Softmax_test\n",
        "from fc import FC"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 3,
      "metadata": {
        "id": "oidgGH5PVVWS"
      },
      "outputs": [],
      "source": [
        "# Define settings for run\n",
        "\n",
        "debug=False\n",
        "\n",
        "shuffle_data=False\n",
        "\n",
        "run_train=True\n",
        "run_val=True\n",
        "\n",
        "load_saved_weights=False\n",
        "weight_file='weights/best_99.pkl'\n",
        "\n",
        "total_epoch=100\n",
        "training_acc_internal=1000"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 4,
      "metadata": {
        "id": "iAoYBSKmWKWE"
      },
      "outputs": [],
      "source": [
        "# Initialize the network layers\n",
        "\n",
        "conv0   = Conv3x3_1_to_n_padding( output=8                        )     # 28x28x1   -> 28x28x8  (Convolution with 8 filters)\n",
        "pool0   = MaxPool2              (                                 )     # 28x28x8   -> 14x14x8  (MaxPooling 2x2)\n",
        "conv1   = Conv3x3_n_to_n_padding( output=16     ,   input=8       )     # 14x14x8   -> 14x14x16 (Convolution with 8 filters)\n",
        "pool1   = MaxPool2              (                                 )     # 14x14x16  -> 07x07x16 (MaxPooling 2x2)\n",
        "# conv2   = Conv3x3_n_to_n_padding( output=32     ,   input=16      )\n",
        "# conv3   = Conv3x3_n_to_n_padding( output=64     ,   input=32      )\n",
        "fc0     = FC                    ( 7 * 7 * 16  ,   7 * 7 * 16      )     # 784       -> 784      (FC)\n",
        "fc1     = FC                    ( 7 * 7 * 16  ,   10              )     # 784       -> 10       (FC)\n",
        "softmax = Softmax               (                                 )     # 14x14x8   -> 10       (Softmax)\n",
        "relu    = Relu                  (                                 )     # 14x14x8   -> 10       (Softmax)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 5,
      "metadata": {
        "id": "EwHWdHFrWSao"
      },
      "outputs": [],
      "source": [
        "# Defining the network - Forward propagation\n",
        "def forward(im, label, debug=False):\n",
        "  im        = (im / 255) - 0.5  \n",
        "    \n",
        "  # Conv 0 with Pool\n",
        "  out_conv0 = conv0.forward   ( im            )\n",
        "  out_pool0 = pool0.forward   ( out_conv0     )\n",
        "  \n",
        "  # Conv 1 with Pool\n",
        "  out_conv1 = conv1.forward   ( out_pool0     )\n",
        "  out_pool1 = pool1.forward   ( out_conv1     )\n",
        "  \n",
        "  # Swap axes to realign for flattening\n",
        "  out_pool2 = np.swapaxes(out_pool1,0,2)\n",
        "  out_pool3 = np.swapaxes(out_pool2,1,2)\n",
        "  \n",
        "  # FC0 and Relu\n",
        "  out_fc0   = fc0.forward     ( out_pool3     )\n",
        "  \n",
        "  # FC1 and SoftMax\n",
        "  out_fc1   = fc1.forward     ( out_fc0       )\n",
        "  out_soft  = softmax.forward ( out_fc1       )\n",
        "  \n",
        "  if debug:\n",
        "    print(f\"Input Image: {im[-4]}\\n\")\n",
        "    print(f\"x_conv0 filters : {conv0.filters[0]}\\n\")\n",
        "    print(f\"x_conv0 : {out_conv0[:,:,0][-1]}\\n\")\n",
        "    print(f\"MaxPool0: {out_pool0[:,:,0][-1]}\\n\")\n",
        "    print(f\"x_conv1 filters : {conv1.filters[0,:,:,0]}\\n\")\n",
        "    print(f\"x_conv1 : {out_conv1[:,:,0][-1]}\\n\")\n",
        "    print(f\"MaxPool1: {out_pool1[:,:,0][-1]}\\n\")\n",
        "    print(f\"FC0 Weights: {fc0.weights[:,0][:10]}\\n\")\n",
        "    print(f\"FC0 output: {out_fc0[:10]}\\n\")\n",
        "    print(f\"FC1 Weights: {fc1.weights[:,0][:10]}\\n\")\n",
        "    print(f\"FC1 output: {out_fc1[:10]}\\n\")\n",
        "    print(f\"SoftMax output: {out_soft}\\n\")\n",
        "\n",
        "  return out_soft"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 6,
      "metadata": {
        "id": "IoomUOeIWg1h"
      },
      "outputs": [],
      "source": [
        "# Defining the network - Backward propagation\n",
        "def backward(label, out, loss=1, lr=0.005):\n",
        "  # Calculate initial gradient\n",
        "  gradient = np.zeros(10)\n",
        "  gradient[label] = -1 / out[label]\n",
        "  # gradient[label] = -loss / out[label]\n",
        "\n",
        "  # ------------------------------Backprop-----------------------------------\n",
        "  # SoftMax\n",
        "  gradient_softmax = softmax.backprop ( gradient                              )     \n",
        "\n",
        "  # FC1 and FC0 \n",
        "  gradient_fc1 = fc1.backprop         ( gradient_softmax  ,               lr  )\n",
        "  gradient_fc0 = fc0.backprop         ( gradient_fc1      ,               lr  )\n",
        "\n",
        "  # Swap axes to realign for flattening\n",
        "  gradient_swap0 = np.swapaxes        ( gradient_fc0      ,       1   ,   2   )\n",
        "  gradient_swap1 = np.swapaxes        ( gradient_swap0    ,       0   ,   2   )\n",
        "\n",
        "  # Conv 1 with Pool \n",
        "  gradient_pool1 = pool1.backprop     ( gradient_swap1                        )\n",
        "  gradient_conv1 = conv1.backprop     ( gradient_pool1     ,              lr  )\n",
        "\n",
        "  # Conv 0 with Pool \n",
        "  gradient_pool0 = pool0.backprop     ( gradient_conv1                        ) \n",
        "  gradient_conv0 = conv0.backprop     ( gradient_pool0     ,              lr  )\n",
        "  return None"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 7,
      "metadata": {
        "id": "yhMPrtRNXBXT"
      },
      "outputs": [],
      "source": [
        "# Defining the network - Loss Function (Cross Entropy)\n",
        "# Calculate cross-entropy loss and accuracy. np.log() is the natural log.\n",
        "def cal_loss(out_soft, label):\n",
        "  loss = -np.log(out_soft[label])\n",
        "  acc = 1 if np.argmax(out_soft) == label else 0\n",
        "  return out_soft, loss, acc"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 8,
      "metadata": {
        "id": "jbypb9x2XRej"
      },
      "outputs": [],
      "source": [
        "# Defining the network - Training Function\n",
        "def train(im, label, debug=False, lr=.005):\n",
        "  pred = forward(im, label, debug)\n",
        "  out_soft, loss, acc = cal_loss(pred, label)\n",
        "  backward(label, out_soft, loss=loss, lr=0.005)\n",
        "  return loss, acc"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 9,
      "metadata": {
        "id": "Qlrl0fzUXV_M"
      },
      "outputs": [],
      "source": [
        "# Defining the network - Validation Function\n",
        "def val(im, label):\n",
        "  pred = forward(im, label)\n",
        "  out_soft, loss, acc = cal_loss(pred, label)\n",
        "  return loss, acc"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 10,
      "metadata": {
        "id": "n3132AoGXzc0"
      },
      "outputs": [],
      "source": [
        "# Function definitions to save and load weight files\n",
        "\n",
        "def save_weights(name,lr=0,max_acc=0):\n",
        "  print(f\"\\nSaving new weights ({name}).\")\n",
        "  weights = dict()\n",
        "  weights[\"conv0\"]        = conv0.filters\n",
        "  weights[\"conv1\"]        = conv1.filters\n",
        "  weights[\"fc0_weights\"]  = fc0.weights\n",
        "  weights[\"fc0_biases\" ]  = fc0.biases\n",
        "  weights[\"fc1_weights\"]  = fc1.weights\n",
        "  weights[\"fc1_biases\" ]  = fc1.biases\n",
        "  weights[\"lr\" ]          = lr\n",
        "  weights[\"max_acc\"]      = max_acc\n",
        "  weight_file = open(str(name), \"wb\")\n",
        "  pickle.dump(weights, weight_file)\n",
        "  weight_file.close()\n",
        "  \n",
        "def load_weights(name):\n",
        "  if os.path.isfile(name): \n",
        "    weight_file = open(str(name), \"rb\")\n",
        "    weights = pickle.load(weight_file)\n",
        "    conv0.filters  = weights[\"conv0\"]      \n",
        "    conv1.filters  = weights[\"conv1\"]      \n",
        "    fc0.weights    = weights[\"fc0_weights\"]\n",
        "    fc0.biases     = weights[\"fc0_biases\" ]\n",
        "    fc1.weights    = weights[\"fc1_weights\"]\n",
        "    fc1.biases     = weights[\"fc1_biases\" ]\n",
        "    lr             = weights[\"lr\" ]\n",
        "    max_acc        = weights[\"max_acc\"]\n",
        "    print(f\"\\nLoading weights from {name} file. LR restored to {lr}. Last Accuracy {max_acc}%\")\n",
        "    return lr, max_acc\n",
        "  else:\n",
        "    print(\"Weights file not found.\")\n",
        "    lr=0.005\n",
        "    max_acc=0\n",
        "    return lr, max_acc\n",
        "   "
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 11,
      "metadata": {
        "id": "Y1WIfF2UYstz"
      },
      "outputs": [],
      "source": [
        "# Defining the network - Adjust learning rate\n",
        "def adjust_lr(acc, lr=.005):\n",
        "  if   acc > 98: lr=0.00001\n",
        "  elif acc > 95: lr=0.0005\n",
        "  elif acc > 90: lr=0.001\n",
        "  elif acc > 80: lr=0.002\n",
        "  elif acc > 70: lr=0.003\n",
        "  elif acc > 60: lr=0.004\n",
        "  return lr"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 12,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "1RYPUzZDYHcC",
        "outputId": "772ab62c-0c88-4eff-c48f-b5a676c426ea"
      },
      "outputs": [],
      "source": [
        "# Load Weights\n",
        "if load_saved_weights:\n",
        "    lr, max_acc = load_weights(weight_file)\n",
        "else:\n",
        "    lr, max_acc = 0.005, 0 \n",
        "\n",
        "if debug: save_weights(f'weights/debug.pkl', lr, max_acc)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 13,
      "metadata": {
        "id": "0SG6nSKmXoi7"
      },
      "outputs": [],
      "source": [
        "# Importing the training dataset - MNIST Dataset\n",
        "\n",
        "train_images = mnist.train_images()\n",
        "train_labels = mnist.train_labels()\n",
        "test_images = mnist.test_images()\n",
        "test_labels = mnist.test_labels()"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 14,
      "metadata": {
        "id": "4wNl_3XuYbOB"
      },
      "outputs": [],
      "source": [
        "# Shuffle Data\n",
        "permutation = np.random.permutation(len(train_images))\n",
        "train_images = train_images[permutation]\n",
        "train_labels = train_labels[permutation]"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 15,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "N02tnG1ZVSyk",
        "outputId": "41b37376-5ac3-41f4-a8c5-e10c8533922b"
      },
      "outputs": [
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Training Initialized.\n",
            "\tTotal number of training   images: 60000\n",
            "\tTotal number of validation images: 10000\n",
            "\tTraining will run for 100 epochs.\n",
            "\tResults will be logged after every 1000 images.\n",
            "\n",
            "--- Epoch 1 ---\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "1002it [01:17, 14.43it/s]"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "\n",
            "[Step 1000] : For 1000 iterations, the Avg Loss is 2.3 | Training Acc: 0.108 | LR: 1e-05\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "2000it [02:28, 13.88it/s]"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "\n",
            "[Step 2000] : For 1000 iterations, the Avg Loss is 2.28 | Training Acc: 0.132 | LR: 1e-05\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "3000it [03:39, 14.66it/s]"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "\n",
            "[Step 3000] : For 1000 iterations, the Avg Loss is 1.22 | Training Acc: 0.572 | LR: 1e-05\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "4002it [04:50, 15.03it/s]"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "\n",
            "[Step 4000] : For 1000 iterations, the Avg Loss is 0.68 | Training Acc: 0.773 | LR: 1e-05\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "5002it [06:01, 14.13it/s]"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "\n",
            "[Step 5000] : For 1000 iterations, the Avg Loss is 0.6 | Training Acc: 0.809 | LR: 1e-05\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "6002it [07:11, 14.57it/s]"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "\n",
            "[Step 6000] : For 1000 iterations, the Avg Loss is 0.49 | Training Acc: 0.844 | LR: 1e-05\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "7002it [08:22, 13.52it/s]"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "\n",
            "[Step 7000] : For 1000 iterations, the Avg Loss is 0.51 | Training Acc: 0.831 | LR: 1e-05\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "8002it [09:33, 15.02it/s]"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "\n",
            "[Step 8000] : For 1000 iterations, the Avg Loss is 0.43 | Training Acc: 0.872 | LR: 1e-05\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "9002it [10:44, 13.77it/s]"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "\n",
            "[Step 9000] : For 1000 iterations, the Avg Loss is 0.37 | Training Acc: 0.885 | LR: 1e-05\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "10002it [11:55, 15.08it/s]"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "\n",
            "[Step 10000] : For 1000 iterations, the Avg Loss is 0.3 | Training Acc: 0.92 | LR: 1e-05\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "11000it [13:06, 13.60it/s]"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "\n",
            "[Step 11000] : For 1000 iterations, the Avg Loss is 0.29 | Training Acc: 0.904 | LR: 1e-05\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "12000it [14:17, 14.17it/s]"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "\n",
            "[Step 12000] : For 1000 iterations, the Avg Loss is 0.29 | Training Acc: 0.909 | LR: 1e-05\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "13002it [15:29, 15.06it/s]"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "\n",
            "[Step 13000] : For 1000 iterations, the Avg Loss is 0.27 | Training Acc: 0.927 | LR: 1e-05\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "14000it [16:40, 14.96it/s]"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "\n",
            "[Step 14000] : For 1000 iterations, the Avg Loss is 0.28 | Training Acc: 0.911 | LR: 1e-05\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "15002it [17:50, 14.00it/s]"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "\n",
            "[Step 15000] : For 1000 iterations, the Avg Loss is 0.25 | Training Acc: 0.931 | LR: 1e-05\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "16000it [19:00, 14.87it/s]"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "\n",
            "[Step 16000] : For 1000 iterations, the Avg Loss is 0.17 | Training Acc: 0.947 | LR: 1e-05\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "17002it [20:12, 13.74it/s]"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "\n",
            "[Step 17000] : For 1000 iterations, the Avg Loss is 0.25 | Training Acc: 0.916 | LR: 1e-05\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "18000it [21:22, 11.86it/s]"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "\n",
            "[Step 18000] : For 1000 iterations, the Avg Loss is 0.26 | Training Acc: 0.925 | LR: 1e-05\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "19000it [22:33, 13.54it/s]"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "\n",
            "[Step 19000] : For 1000 iterations, the Avg Loss is 0.18 | Training Acc: 0.941 | LR: 1e-05\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "20002it [23:44, 14.79it/s]"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "\n",
            "[Step 20000] : For 1000 iterations, the Avg Loss is 0.17 | Training Acc: 0.947 | LR: 1e-05\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "21000it [24:55, 13.32it/s]"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "\n",
            "[Step 21000] : For 1000 iterations, the Avg Loss is 0.2 | Training Acc: 0.945 | LR: 1e-05\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "22000it [26:06, 13.95it/s]"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "\n",
            "[Step 22000] : For 1000 iterations, the Avg Loss is 0.17 | Training Acc: 0.955 | LR: 1e-05\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "23002it [27:17, 14.92it/s]"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "\n",
            "[Step 23000] : For 1000 iterations, the Avg Loss is 0.19 | Training Acc: 0.939 | LR: 1e-05\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "24000it [28:28, 12.00it/s]"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "\n",
            "[Step 24000] : For 1000 iterations, the Avg Loss is 0.19 | Training Acc: 0.943 | LR: 1e-05\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "25002it [29:39, 14.00it/s]"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "\n",
            "[Step 25000] : For 1000 iterations, the Avg Loss is 0.18 | Training Acc: 0.948 | LR: 1e-05\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "26002it [30:49, 14.27it/s]"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "\n",
            "[Step 26000] : For 1000 iterations, the Avg Loss is 0.18 | Training Acc: 0.937 | LR: 1e-05\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "27002it [32:00, 15.08it/s]"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "\n",
            "[Step 27000] : For 1000 iterations, the Avg Loss is 0.14 | Training Acc: 0.958 | LR: 1e-05\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "28002it [33:12, 13.99it/s]"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "\n",
            "[Step 28000] : For 1000 iterations, the Avg Loss is 0.16 | Training Acc: 0.949 | LR: 1e-05\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "29002it [34:22, 15.15it/s]"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "\n",
            "[Step 29000] : For 1000 iterations, the Avg Loss is 0.17 | Training Acc: 0.947 | LR: 1e-05\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "30002it [35:32, 14.68it/s]"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "\n",
            "[Step 30000] : For 1000 iterations, the Avg Loss is 0.15 | Training Acc: 0.955 | LR: 1e-05\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "31000it [36:42, 14.69it/s]"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "\n",
            "[Step 31000] : For 1000 iterations, the Avg Loss is 0.17 | Training Acc: 0.942 | LR: 1e-05\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "32002it [37:53, 15.25it/s]"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "\n",
            "[Step 32000] : For 1000 iterations, the Avg Loss is 0.16 | Training Acc: 0.95 | LR: 1e-05\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "33002it [39:04, 15.11it/s]"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "\n",
            "[Step 33000] : For 1000 iterations, the Avg Loss is 0.16 | Training Acc: 0.958 | LR: 1e-05\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "34000it [40:13, 13.61it/s]"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "\n",
            "[Step 34000] : For 1000 iterations, the Avg Loss is 0.19 | Training Acc: 0.944 | LR: 1e-05\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "35002it [41:23, 14.86it/s]"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "\n",
            "[Step 35000] : For 1000 iterations, the Avg Loss is 0.17 | Training Acc: 0.96 | LR: 1e-05\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "36000it [42:34, 14.03it/s]"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "\n",
            "[Step 36000] : For 1000 iterations, the Avg Loss is 0.14 | Training Acc: 0.96 | LR: 1e-05\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "37002it [43:44, 15.00it/s]"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "\n",
            "[Step 37000] : For 1000 iterations, the Avg Loss is 0.17 | Training Acc: 0.938 | LR: 1e-05\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "38002it [44:54, 14.83it/s]"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "\n",
            "[Step 38000] : For 1000 iterations, the Avg Loss is 0.12 | Training Acc: 0.959 | LR: 1e-05\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "39002it [46:05, 14.63it/s]"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "\n",
            "[Step 39000] : For 1000 iterations, the Avg Loss is 0.16 | Training Acc: 0.955 | LR: 1e-05\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "40000it [47:16, 13.09it/s]"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "\n",
            "[Step 40000] : For 1000 iterations, the Avg Loss is 0.15 | Training Acc: 0.955 | LR: 1e-05\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "41002it [48:27, 15.17it/s]"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "\n",
            "[Step 41000] : For 1000 iterations, the Avg Loss is 0.15 | Training Acc: 0.96 | LR: 1e-05\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "42000it [49:39, 14.44it/s]"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "\n",
            "[Step 42000] : For 1000 iterations, the Avg Loss is 0.14 | Training Acc: 0.965 | LR: 1e-05\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "43000it [50:49, 14.81it/s]"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "\n",
            "[Step 43000] : For 1000 iterations, the Avg Loss is 0.14 | Training Acc: 0.956 | LR: 1e-05\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "44002it [51:59, 14.95it/s]"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "\n",
            "[Step 44000] : For 1000 iterations, the Avg Loss is 0.16 | Training Acc: 0.953 | LR: 1e-05\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "45000it [53:10, 14.32it/s]"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "\n",
            "[Step 45000] : For 1000 iterations, the Avg Loss is 0.13 | Training Acc: 0.967 | LR: 1e-05\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "46002it [54:21, 13.78it/s]"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "\n",
            "[Step 46000] : For 1000 iterations, the Avg Loss is 0.14 | Training Acc: 0.959 | LR: 1e-05\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "47000it [55:30, 13.77it/s]"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "\n",
            "[Step 47000] : For 1000 iterations, the Avg Loss is 0.14 | Training Acc: 0.955 | LR: 1e-05\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "48000it [56:40, 12.85it/s]"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "\n",
            "[Step 48000] : For 1000 iterations, the Avg Loss is 0.15 | Training Acc: 0.955 | LR: 1e-05\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "49002it [57:52, 14.04it/s]"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "\n",
            "[Step 49000] : For 1000 iterations, the Avg Loss is 0.18 | Training Acc: 0.953 | LR: 1e-05\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "50000it [59:03, 12.40it/s]"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "\n",
            "[Step 50000] : For 1000 iterations, the Avg Loss is 0.14 | Training Acc: 0.963 | LR: 1e-05\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "51000it [1:00:15, 14.49it/s]"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "\n",
            "[Step 51000] : For 1000 iterations, the Avg Loss is 0.11 | Training Acc: 0.96 | LR: 1e-05\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "52002it [1:01:26, 14.96it/s]"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "\n",
            "[Step 52000] : For 1000 iterations, the Avg Loss is 0.17 | Training Acc: 0.951 | LR: 1e-05\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "53000it [1:02:36, 13.63it/s]"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "\n",
            "[Step 53000] : For 1000 iterations, the Avg Loss is 0.11 | Training Acc: 0.971 | LR: 1e-05\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "54000it [1:03:46, 12.14it/s]"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "\n",
            "[Step 54000] : For 1000 iterations, the Avg Loss is 0.11 | Training Acc: 0.966 | LR: 1e-05\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "55000it [1:04:57, 12.94it/s]"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "\n",
            "[Step 55000] : For 1000 iterations, the Avg Loss is 0.15 | Training Acc: 0.96 | LR: 1e-05\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "56000it [1:06:09, 11.57it/s]"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "\n",
            "[Step 56000] : For 1000 iterations, the Avg Loss is 0.13 | Training Acc: 0.962 | LR: 1e-05\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "57002it [1:07:19, 14.62it/s]"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "\n",
            "[Step 57000] : For 1000 iterations, the Avg Loss is 0.15 | Training Acc: 0.953 | LR: 1e-05\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "58000it [1:08:29, 13.79it/s]"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "\n",
            "[Step 58000] : For 1000 iterations, the Avg Loss is 0.12 | Training Acc: 0.967 | LR: 1e-05\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "59000it [1:09:39, 14.82it/s]"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "\n",
            "[Step 59000] : For 1000 iterations, the Avg Loss is 0.13 | Training Acc: 0.958 | LR: 1e-05\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "60000it [1:10:49, 14.12it/s]\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "\n",
            "[Step 60000] : For 1000 iterations, the Avg Loss is 0.14 | Training Acc: 0.956 | LR: 1e-05\n",
            "End of epoch 1\n",
            "\n",
            "\n",
            "Calculating validation scores at the end of epoch.\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "10000it [05:47, 28.79it/s]\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Test Loss: 0.09158673398928471\n",
            "Test Accuracy: 0.971\n",
            "\n",
            "--- Epoch 2 ---\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "1001it [02:01,  9.36it/s]"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "\n",
            "[Step 1000] : For 1000 iterations, the Avg Loss is 0.13 | Training Acc: 0.957 | LR: 1e-05\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "1566it [15:10,  1.76it/s]"
          ]
        }
      ],
      "source": [
        " # Main function to run the training\n",
        "\n",
        "if run_train:\n",
        "  print(f'Training Initialized.')\n",
        "  print(f\"\\tTotal number of training   images: {len(train_labels)}\")\n",
        "  print(f\"\\tTotal number of validation images: {len(test_labels)}\")\n",
        "  print(f\"\\tTraining will run for {total_epoch} epochs.\")\n",
        "  print(f\"\\tResults will be logged after every {training_acc_internal} images.\")\n",
        "  for epoch in range(total_epoch):\n",
        "    print('\\n--- Epoch %d ---' % (epoch + 1))\n",
        "            \n",
        "    # Initialize Variables\n",
        "    loss, num_correct = 0, 0\n",
        "    for i, (im, label) in tqdm(enumerate(zip(train_images, train_labels))):\n",
        "      \n",
        "      # Logging results\n",
        "      if i % training_acc_internal == training_acc_internal-1:\n",
        "        lr = adjust_lr(num_correct)\n",
        "        print(f'\\n[Step {(i+1)}] : For {training_acc_internal} iterations, the Avg Loss is {np.round((loss / training_acc_internal),2)} | Training Acc: {num_correct/training_acc_internal} | LR: {lr}')\n",
        "        loss, num_correct = 0, 0\n",
        "          \n",
        "      # Train the network\n",
        "      l, acc = train(im, label, debug, lr=lr)\n",
        "      loss += l\n",
        "      num_correct += acc\n",
        "           \n",
        "    print(f\"End of epoch {epoch+1}\")      \n",
        "\n",
        "    print(f\"\\n\\nCalculating validation scores at the end of epoch.\")\n",
        "    loss, num_correct = 0, 0\n",
        "    for im, label in tqdm(zip(test_images, test_labels)):\n",
        "      l, acc = val(im, label)\n",
        "      loss += l\n",
        "      num_correct += acc\n",
        "    num_tests = len(test_images)\n",
        "    test_loss, test_acc =  loss / num_tests , num_correct / num_tests\n",
        "    print('Test Loss:', test_loss)\n",
        "    print('Test Accuracy:', test_acc)\n",
        "\n",
        "    if epoch == 0:\n",
        "      max_acc = test_acc\n",
        "    elif test_acc > max_acc: \n",
        "      max_acc = test_acc\n",
        "      save_weights(f'weights/best_{test_acc}.pkl', lr, max_acc)\n",
        "      save_weights(f'weights/last.pkl', lr, max_acc)\n"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "PonPg6kBZT1J"
      },
      "outputs": [],
      "source": [
        "# Main function to validate the weights\n",
        "\n",
        "if run_val:\n",
        "  print(f'\\n--- Testing the CNN for {len(test_labels)} images---')\n",
        "  loss = 0\n",
        "  num_correct = 0\n",
        "  for im, label in tqdm(zip(test_images, test_labels)):\n",
        "    l, acc = val(im, label)\n",
        "    loss += l\n",
        "    num_correct += acc\n",
        "\n",
        "  num_tests = len(test_images)\n",
        "  print('\\nTest Loss:', loss / num_tests)\n",
        "  print('Test Accuracy:', num_correct / num_tests)"
      ]
    }
  ],
  "metadata": {
    "colab": {
      "collapsed_sections": [],
      "include_colab_link": true,
      "provenance": []
    },
    "kernelspec": {
      "display_name": "Python 3.8.5 ('base')",
      "language": "python",
      "name": "python3"
    },
    "language_info": {
      "codemirror_mode": {
        "name": "ipython",
        "version": 3
      },
      "file_extension": ".py",
      "mimetype": "text/x-python",
      "name": "python",
      "nbconvert_exporter": "python",
      "pygments_lexer": "ipython3",
      "version": "3.8.5"
    },
    "vscode": {
      "interpreter": {
        "hash": "4fbbc6560802a9753d76c1864ab18e70962bbfcb64c614d1799210197ece64ee"
      }
    }
  },
  "nbformat": 4,
  "nbformat_minor": 0
}