pytorch_ipynb/cnn/nin-cifar10.ipynb

{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "accelerator": "GPU",
    "colab": {
      "name": "convnet-vgg16.ipynb",
      "provenance": [],
      "collapsed_sections": [],
      "machine_shape": "hm"
    },
    "kernelspec": {
      "display_name": "Python 3",
      "language": "python",
      "name": "python3"
    },
    "language_info": {
      "codemirror_mode": {
        "name": "ipython",
        "version": 3
      },
      "file_extension": ".py",
      "mimetype": "text/x-python",
      "name": "python",
      "nbconvert_exporter": "python",
      "pygments_lexer": "ipython3",
      "version": "3.7.3"
    },
    "toc": {
      "nav_menu": {},
      "number_sections": true,
      "sideBar": true,
      "skip_h1_title": false,
      "title_cell": "Table of Contents",
      "title_sidebar": "Contents",
      "toc_cell": true,
      "toc_position": {
        "height": "calc(100% - 180px)",
        "left": "10px",
        "top": "150px",
        "width": "371px"
      },
      "toc_section_display": true,
      "toc_window_display": true
    },
    "widgets": {
      "application/vnd.jupyter.widget-state+json": {
        "a58868b7441a4a8a86f6a1fb37bc41bb": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "state": {
            "_view_name": "HBoxView",
            "_dom_classes": [],
            "_model_name": "HBoxModel",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "box_style": "",
            "layout": "IPY_MODEL_b8963c52da9949698fee962c4f3cb9cf",
            "_model_module": "@jupyter-widgets/controls",
            "children": [
              "IPY_MODEL_8b49c987252240dba15a599e8451c15f",
              "IPY_MODEL_f5efa87270da4aedbe59ff0fbec1c7ad"
            ]
          }
        },
        "b8963c52da9949698fee962c4f3cb9cf": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "8b49c987252240dba15a599e8451c15f": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "state": {
            "_view_name": "ProgressView",
            "style": "IPY_MODEL_ea1e831d6ff84984b905e4429d82ec3c",
            "_dom_classes": [],
            "description": "",
            "_model_name": "FloatProgressModel",
            "bar_style": "info",
            "max": 1,
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": 1,
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "orientation": "horizontal",
            "min": 0,
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_8fa99a97204947fb86999d9cbfb48927"
          }
        },
        "f5efa87270da4aedbe59ff0fbec1c7ad": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "state": {
            "_view_name": "HTMLView",
            "style": "IPY_MODEL_47ae73ccc2db4427955e9a600ca1e3d0",
            "_dom_classes": [],
            "description": "",
            "_model_name": "HTMLModel",
            "placeholder": "​",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": " 170500096/? [00:20&lt;00:00, 86268340.37it/s]",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_3c03c4988c2b4e07a71cb1665995eaf9"
          }
        },
        "ea1e831d6ff84984b905e4429d82ec3c": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "ProgressStyleModel",
            "description_width": "initial",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "bar_color": null,
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "8fa99a97204947fb86999d9cbfb48927": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "47ae73ccc2db4427955e9a600ca1e3d0": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "DescriptionStyleModel",
            "description_width": "",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "3c03c4988c2b4e07a71cb1665995eaf9": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        }
      }
    }
  },
  "cells": [
    {
      "cell_type": "markdown",
      "metadata": {
        "colab_type": "text",
        "id": "UEBilEjLj5wY"
      },
      "source": [
        "Deep Learning Models -- A collection of various deep learning architectures, models, and tips for TensorFlow and PyTorch in Jupyter Notebooks.\n",
        "- Author: Sebastian Raschka\n",
        "- GitHub Repository: https://github.com/rasbt/deeplearning-models"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "DwN2cVT9NJG-",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "!pip install -q IPython\n",
        "!pip install -q ipykernel\n",
        "!pip install -q watermark\n",
        "!pip install -q matplotlib\n",
        "!pip install -q sklearn\n",
        "!pip install -q pandas\n",
        "!pip install -q pydot\n",
        "!pip install -q hiddenlayer\n",
        "!pip install -q graphviz"
      ],
      "execution_count": 1,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab_type": "code",
        "id": "GOzuY8Yvj5wb",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 121
        },
        "outputId": "aff1388d-f754-4cd0-aa7c-51ec81aff700"
      },
      "source": [
        "%load_ext watermark\n",
        "%watermark -a 'Sebastian Raschka' -v -p torch"
      ],
      "execution_count": 2,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Sebastian Raschka \n",
            "\n",
            "CPython 3.6.9\n",
            "IPython 5.5.0\n",
            "\n",
            "torch 1.5.1+cu101\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "colab_type": "text",
        "id": "rH4XmErYj5wm"
      },
      "source": [
        "# Network in Network CIFAR-10 Classifier"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "AsJv3tOdNEyJ",
        "colab_type": "text"
      },
      "source": [
        "based on \n",
        "\n",
        "- Lin, Min, Qiang Chen, and Shuicheng Yan. \"Network in network.\" arXiv preprint arXiv:1312.4400 (2013)."
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "colab_type": "text",
        "id": "MkoGLH_Tj5wn"
      },
      "source": [
        "## Imports"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab_type": "code",
        "id": "ORj09gnrj5wp",
        "colab": {}
      },
      "source": [
        "import os\n",
        "import time\n",
        "\n",
        "import numpy as np\n",
        "import pandas as pd\n",
        "\n",
        "import torch\n",
        "import torch.nn as nn\n",
        "import torch.nn.functional as F\n",
        "from torch.utils.data import DataLoader\n",
        "from torch.utils.data import Subset\n",
        "\n",
        "from torchvision import datasets\n",
        "from torchvision import transforms\n",
        "\n",
        "import matplotlib.pyplot as plt\n",
        "from PIL import Image\n",
        "\n",
        "\n",
        "if torch.cuda.is_available():\n",
        "    torch.backends.cudnn.deterministic = True"
      ],
      "execution_count": 3,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "67AFMavUNEyM",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 35
        },
        "outputId": "19ca8fd2-ead8-45fc-9c46-4b1c9b3d2617"
      },
      "source": [
        "torch.tensor([1]).device"
      ],
      "execution_count": 4,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "device(type='cpu')"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 4
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "colab_type": "text",
        "id": "I6hghKPxj5w0"
      },
      "source": [
        "## Model Settings"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab_type": "code",
        "id": "NnT0sZIwj5wu",
        "colab": {}
      },
      "source": [
        "##########################\n",
        "### SETTINGS\n",
        "##########################\n",
        "\n",
        "# Hyperparameters\n",
        "RANDOM_SEED = 1\n",
        "LEARNING_RATE = 0.0005\n",
        "BATCH_SIZE = 256\n",
        "NUM_EPOCHS = 100\n",
        "\n",
        "# Architecture\n",
        "NUM_CLASSES = 10\n",
        "\n",
        "# Other\n",
        "DEVICE = torch.device(\"cuda\" if torch.cuda.is_available() else \"cpu\")\n",
        "GRAYSCALE = False"
      ],
      "execution_count": 5,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "Qmh49l-aNEyS",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 187,
          "referenced_widgets": [
            "a58868b7441a4a8a86f6a1fb37bc41bb",
            "b8963c52da9949698fee962c4f3cb9cf",
            "8b49c987252240dba15a599e8451c15f",
            "f5efa87270da4aedbe59ff0fbec1c7ad",
            "ea1e831d6ff84984b905e4429d82ec3c",
            "8fa99a97204947fb86999d9cbfb48927",
            "47ae73ccc2db4427955e9a600ca1e3d0",
            "3c03c4988c2b4e07a71cb1665995eaf9"
          ]
        },
        "outputId": "a0c8a769-2845-45d0-b9aa-a72b3f0ca34c"
      },
      "source": [
        "##########################\n",
        "### CIFAR-10 Dataset\n",
        "##########################\n",
        "\n",
        "\n",
        "# Note transforms.ToTensor() scales input images\n",
        "# to 0-1 range\n",
        "\n",
        "\n",
        "train_indices = torch.arange(0, 49000)\n",
        "valid_indices = torch.arange(49000, 50000)\n",
        "\n",
        "\n",
        "train_and_valid = datasets.CIFAR10(root='data', \n",
        "                                   train=True, \n",
        "                                   transform=transforms.ToTensor(),\n",
        "                                   download=True)\n",
        "\n",
        "train_dataset = Subset(train_and_valid, train_indices)\n",
        "valid_dataset = Subset(train_and_valid, valid_indices)\n",
        "\n",
        "\n",
        "test_dataset = datasets.CIFAR10(root='data', \n",
        "                                train=False, \n",
        "                                transform=transforms.ToTensor())\n",
        "\n",
        "\n",
        "#####################################################\n",
        "### Data Loaders\n",
        "#####################################################\n",
        "\n",
        "train_loader = DataLoader(dataset=train_dataset, \n",
        "                          batch_size=BATCH_SIZE,\n",
        "                          num_workers=8,\n",
        "                          shuffle=True)\n",
        "\n",
        "valid_loader = DataLoader(dataset=valid_dataset, \n",
        "                          batch_size=BATCH_SIZE,\n",
        "                          num_workers=8,\n",
        "                          shuffle=False)\n",
        "\n",
        "test_loader = DataLoader(dataset=test_dataset, \n",
        "                         batch_size=BATCH_SIZE,\n",
        "                         num_workers=8,\n",
        "                         shuffle=False)\n",
        "\n",
        "#####################################################\n",
        "\n",
        "# Checking the dataset\n",
        "for images, labels in train_loader:  \n",
        "    print('Image batch dimensions:', images.shape)\n",
        "    print('Image label dimensions:', labels.shape)\n",
        "    break\n",
        "\n",
        "for images, labels in test_loader:  \n",
        "    print('Image batch dimensions:', images.shape)\n",
        "    print('Image label dimensions:', labels.shape)\n",
        "    break\n",
        "    \n",
        "for images, labels in valid_loader:  \n",
        "    print('Image batch dimensions:', images.shape)\n",
        "    print('Image label dimensions:', labels.shape)\n",
        "    break"
      ],
      "execution_count": 6,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Downloading https://www.cs.toronto.edu/~kriz/cifar-10-python.tar.gz to data/cifar-10-python.tar.gz\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "display_data",
          "data": {
            "application/vnd.jupyter.widget-view+json": {
              "model_id": "a58868b7441a4a8a86f6a1fb37bc41bb",
              "version_minor": 0,
              "version_major": 2
            },
            "text/plain": [
              "HBox(children=(FloatProgress(value=1.0, bar_style='info', max=1.0), HTML(value='')))"
            ]
          },
          "metadata": {
            "tags": []
          }
        },
        {
          "output_type": "stream",
          "text": [
            "Extracting data/cifar-10-python.tar.gz to data\n",
            "Image batch dimensions: torch.Size([256, 3, 32, 32])\n",
            "Image label dimensions: torch.Size([256])\n",
            "Image batch dimensions: torch.Size([256, 3, 32, 32])\n",
            "Image label dimensions: torch.Size([256])\n",
            "Image batch dimensions: torch.Size([256, 3, 32, 32])\n",
            "Image label dimensions: torch.Size([256])\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "yOqDwyamNEyU",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "##########################\n",
        "### MODEL\n",
        "##########################\n",
        "\n",
        "\n",
        "class NiN(nn.Module):\n",
        "    def __init__(self, num_classes):\n",
        "        super(NiN, self).__init__()\n",
        "        self.num_classes = num_classes\n",
        "        self.classifier = nn.Sequential(\n",
        "                nn.Conv2d(3, 192, kernel_size=5, stride=1, padding=2),\n",
        "                nn.ReLU(inplace=True),\n",
        "                nn.Conv2d(192, 160, kernel_size=1, stride=1, padding=0),\n",
        "                nn.ReLU(inplace=True),\n",
        "                nn.Conv2d(160,  96, kernel_size=1, stride=1, padding=0),\n",
        "                nn.ReLU(inplace=True),\n",
        "                nn.MaxPool2d(kernel_size=3, stride=2, padding=1),\n",
        "                nn.Dropout(0.5),\n",
        "\n",
        "                nn.Conv2d(96, 192, kernel_size=5, stride=1, padding=2),\n",
        "                nn.ReLU(inplace=True),\n",
        "                nn.Conv2d(192, 192, kernel_size=1, stride=1, padding=0),\n",
        "                nn.ReLU(inplace=True),\n",
        "                nn.Conv2d(192, 192, kernel_size=1, stride=1, padding=0),\n",
        "                nn.ReLU(inplace=True),\n",
        "                nn.AvgPool2d(kernel_size=3, stride=2, padding=1),\n",
        "                nn.Dropout(0.5),\n",
        "\n",
        "                nn.Conv2d(192, 192, kernel_size=3, stride=1, padding=1),\n",
        "                nn.ReLU(inplace=True),\n",
        "                nn.Conv2d(192, 192, kernel_size=1, stride=1, padding=0),\n",
        "                nn.ReLU(inplace=True),\n",
        "                nn.Conv2d(192,  10, kernel_size=1, stride=1, padding=0),\n",
        "                nn.ReLU(inplace=True),\n",
        "                nn.AvgPool2d(kernel_size=8, stride=1, padding=0),\n",
        "\n",
        "                )\n",
        "\n",
        "    def forward(self, x):\n",
        "        x = self.classifier(x)\n",
        "        logits = x.view(x.size(0), self.num_classes)\n",
        "        probas = torch.softmax(logits, dim=1)\n",
        "        return logits, probas"
      ],
      "execution_count": 7,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "colab_type": "text",
        "id": "RAodboScj5w6"
      },
      "source": [
        "## Training without Pinned Memory"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab_type": "code",
        "id": "_lza9t_uj5w1",
        "colab": {}
      },
      "source": [
        "torch.manual_seed(RANDOM_SEED)\n",
        "\n",
        "model = NiN(NUM_CLASSES)\n",
        "model.to(DEVICE)\n",
        "\n",
        "optimizer = torch.optim.Adam(model.parameters(), lr=LEARNING_RATE)  "
      ],
      "execution_count": 8,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "iGVo5nelNRme",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 337
        },
        "outputId": "4de97ef9-5b55-40f6-bbb1-c93f9c41ad0a"
      },
      "source": [
        "import hiddenlayer as hl\n",
        "hl.build_graph(model, torch.zeros([25, 3, 32, 32]).to(DEVICE))"
      ],
      "execution_count": 9,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "/usr/local/lib/python3.6/dist-packages/torch/onnx/symbolic_opset9.py:1310: UserWarning: Dropout is a training op and should not be exported in inference mode. Make sure to call eval() on the model, and to export it with param training=False.\n",
            "  warnings.warn(\"Dropout is a training op and should not be exported in inference mode. \"\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "<hiddenlayer.graph.Graph at 0x7f2e2b300898>"
            ],
            "image/svg+xml": "<?xml version=\"1.0\" encoding=\"UTF-8\" standalone=\"no\"?>\n<!DOCTYPE svg PUBLIC \"-//W3C//DTD SVG 1.1//EN\"\n \"http://www.w3.org/Graphics/SVG/1.1/DTD/svg11.dtd\">\n<!-- Generated by graphviz version 2.40.1 (20161225.0304)\n -->\n<!-- Title: %3 Pages: 1 -->\n<svg width=\"2241pt\" height=\"196pt\"\n viewBox=\"0.00 0.00 2241.00 196.00\" xmlns=\"http://www.w3.org/2000/svg\" xmlns:xlink=\"http://www.w3.org/1999/xlink\">\n<g id=\"graph0\" class=\"graph\" transform=\"scale(1 1) rotate(0) translate(72 160)\">\n<title>%3</title>\n<polygon fill=\"#ffffff\" stroke=\"transparent\" points=\"-72,36 -72,-160 2169,-160 2169,36 -72,36\"/>\n<!-- /outputs/25 -->\n<g id=\"node1\" class=\"node\">\n<title>/outputs/25</title>\n<polygon fill=\"#e8e8e8\" stroke=\"#000000\" points=\"248,-40 178,-40 178,-4 248,-4 248,-40\"/>\n<text text-anchor=\"start\" x=\"186\" y=\"-19\" font-family=\"Times\" font-size=\"10.00\" fill=\"#000000\">MaxPool3x3</text>\n</g>\n<!-- /outputs/26/27 -->\n<g id=\"node2\" class=\"node\">\n<title>/outputs/26/27</title>\n<polygon fill=\"#e8e8e8\" stroke=\"#000000\" points=\"396,-40 342,-40 342,-4 396,-4 396,-40\"/>\n<text text-anchor=\"start\" x=\"352\" y=\"-19\" font-family=\"Times\" font-size=\"10.00\" fill=\"#000000\">Dropout</text>\n</g>\n<!-- /outputs/25&#45;&gt;/outputs/26/27 -->\n<g id=\"edge1\" class=\"edge\">\n<title>/outputs/25&#45;&gt;/outputs/26/27</title>\n<path fill=\"none\" stroke=\"#000000\" d=\"M248.0094,-22C272.9589,-22 306.4424,-22 331.8599,-22\"/>\n<polygon fill=\"#000000\" stroke=\"#000000\" points=\"331.8631,-25.5001 341.8631,-22 331.863,-18.5001 331.8631,-25.5001\"/>\n<text text-anchor=\"middle\" x=\"295\" y=\"-25\" font-family=\"Times\" font-size=\"10.00\" fill=\"#000000\">25x96x16x16</text>\n</g>\n<!-- 2334275721088331492 -->\n<g id=\"node18\" class=\"node\">\n<title>2334275721088331492</title>\n<polygon fill=\"#e8e8e8\" stroke=\"#000000\" points=\"574,-44 490,-44 490,0 574,0 574,-44\"/>\n<text text-anchor=\"start\" x=\"498\" y=\"-28\" font-family=\"Times\" font-size=\"10.00\" fill=\"#000000\">Conv5x5 &gt; Relu</text>\n<text text-anchor=\"start\" x=\"559\" y=\"-7\" font-family=\"Times\" font-size=\"10.00\" fill=\"#000000\">x3</text>\n</g>\n<!-- /outputs/26/27&#45;&gt;2334275721088331492 -->\n<g id=\"edge16\" class=\"edge\">\n<title>/outputs/26/27&#45;&gt;2334275721088331492</title>\n<path fill=\"none\" stroke=\"#000000\" d=\"M396.2825,-22C418.9448,-22 451.8428,-22 479.5731,-22\"/>\n<polygon fill=\"#000000\" stroke=\"#000000\" points=\"479.9159,-25.5001 489.9158,-22 479.9158,-18.5001 479.9159,-25.5001\"/>\n<text text-anchor=\"middle\" x=\"443\" y=\"-25\" font-family=\"Times\" font-size=\"10.00\" fill=\"#000000\">25x96x16x16</text>\n</g>\n<!-- /outputs/34 -->\n<g id=\"node3\" class=\"node\">\n<title>/outputs/34</title>\n<polygon fill=\"#e8e8e8\" stroke=\"#000000\" points=\"728,-40 674,-40 674,-4 728,-4 728,-40\"/>\n<text text-anchor=\"start\" x=\"693\" y=\"-19\" font-family=\"Times\" font-size=\"10.00\" fill=\"#000000\">Pad</text>\n</g>\n<!-- /outputs/35 -->\n<g id=\"node4\" class=\"node\">\n<title>/outputs/35</title>\n<polygon fill=\"#e8e8e8\" stroke=\"#000000\" points=\"849,-40 765,-40 765,-4 849,-4 849,-40\"/>\n<text text-anchor=\"start\" x=\"773\" y=\"-19\" font-family=\"Times\" font-size=\"10.00\" fill=\"#000000\">AveragePool3x3</text>\n</g>\n<!-- /outputs/34&#45;&gt;/outputs/35 -->\n<g id=\"edge2\" class=\"edge\">\n<title>/outputs/34&#45;&gt;/outputs/35</title>\n<path fill=\"none\" stroke=\"#000000\" d=\"M728.0225,-22C736.1336,-22 745.323,-22 754.4927,-22\"/>\n<polygon fill=\"#000000\" stroke=\"#000000\" points=\"754.7788,-25.5001 764.7788,-22 754.7788,-18.5001 754.7788,-25.5001\"/>\n</g>\n<!-- /outputs/36/37 -->\n<g id=\"node5\" class=\"node\">\n<title>/outputs/36/37</title>\n<polygon fill=\"#e8e8e8\" stroke=\"#000000\" points=\"992,-40 938,-40 938,-4 992,-4 992,-40\"/>\n<text text-anchor=\"start\" x=\"948\" y=\"-19\" font-family=\"Times\" font-size=\"10.00\" fill=\"#000000\">Dropout</text>\n</g>\n<!-- /outputs/35&#45;&gt;/outputs/36/37 -->\n<g id=\"edge3\" class=\"edge\">\n<title>/outputs/35&#45;&gt;/outputs/36/37</title>\n<path fill=\"none\" stroke=\"#000000\" d=\"M849.3411,-22C873.7559,-22 904.173,-22 927.6857,-22\"/>\n<polygon fill=\"#000000\" stroke=\"#000000\" points=\"927.9567,-25.5001 937.9567,-22 927.9567,-18.5001 927.9567,-25.5001\"/>\n<text text-anchor=\"middle\" x=\"893.5\" y=\"-25\" font-family=\"Times\" font-size=\"10.00\" fill=\"#000000\">25x192x8x8</text>\n</g>\n<!-- 4304698631915573932 -->\n<g id=\"node19\" class=\"node\">\n<title>4304698631915573932</title>\n<polygon fill=\"#e8e8e8\" stroke=\"#000000\" points=\"1165,-44 1081,-44 1081,0 1165,0 1165,-44\"/>\n<text text-anchor=\"start\" x=\"1089\" y=\"-28\" font-family=\"Times\" font-size=\"10.00\" fill=\"#000000\">Conv3x3 &gt; Relu</text>\n<text text-anchor=\"start\" x=\"1150\" y=\"-7\" font-family=\"Times\" font-size=\"10.00\" fill=\"#000000\">x3</text>\n</g>\n<!-- /outputs/36/37&#45;&gt;4304698631915573932 -->\n<g id=\"edge18\" class=\"edge\">\n<title>/outputs/36/37&#45;&gt;4304698631915573932</title>\n<path fill=\"none\" stroke=\"#000000\" d=\"M992.1612,-22C1013.6593,-22 1044.3273,-22 1070.5687,-22\"/>\n<polygon fill=\"#000000\" stroke=\"#000000\" points=\"1070.753,-25.5001 1080.753,-22 1070.7529,-18.5001 1070.753,-25.5001\"/>\n<text text-anchor=\"middle\" x=\"1036.5\" y=\"-25\" font-family=\"Times\" font-size=\"10.00\" fill=\"#000000\">25x192x8x8</text>\n</g>\n<!-- /outputs/44 -->\n<g id=\"node6\" class=\"node\">\n<title>/outputs/44</title>\n<polygon fill=\"#e8e8e8\" stroke=\"#000000\" points=\"1303,-40 1249,-40 1249,-4 1303,-4 1303,-40\"/>\n<text text-anchor=\"start\" x=\"1268\" y=\"-19\" font-family=\"Times\" font-size=\"10.00\" fill=\"#000000\">Pad</text>\n</g>\n<!-- /outputs/45 -->\n<g id=\"node7\" class=\"node\">\n<title>/outputs/45</title>\n<polygon fill=\"#e8e8e8\" stroke=\"#000000\" points=\"1424,-40 1340,-40 1340,-4 1424,-4 1424,-40\"/>\n<text text-anchor=\"start\" x=\"1348\" y=\"-19\" font-family=\"Times\" font-size=\"10.00\" fill=\"#000000\">AveragePool8x8</text>\n</g>\n<!-- /outputs/44&#45;&gt;/outputs/45 -->\n<g id=\"edge4\" class=\"edge\">\n<title>/outputs/44&#45;&gt;/outputs/45</title>\n<path fill=\"none\" stroke=\"#000000\" d=\"M1303.0225,-22C1311.1336,-22 1320.323,-22 1329.4927,-22\"/>\n<polygon fill=\"#000000\" stroke=\"#000000\" points=\"1329.7788,-25.5001 1339.7788,-22 1329.7788,-18.5001 1329.7788,-25.5001\"/>\n</g>\n<!-- /outputs/46 -->\n<g id=\"node8\" class=\"node\">\n<title>/outputs/46</title>\n<polygon fill=\"#e8e8e8\" stroke=\"#000000\" points=\"1562,-70 1508,-70 1508,-34 1562,-34 1562,-70\"/>\n<text text-anchor=\"start\" x=\"1523\" y=\"-49\" font-family=\"Times\" font-size=\"10.00\" fill=\"#000000\">Shape</text>\n</g>\n<!-- /outputs/45&#45;&gt;/outputs/46 -->\n<g id=\"edge5\" class=\"edge\">\n<title>/outputs/45&#45;&gt;/outputs/46</title>\n<path fill=\"none\" stroke=\"#000000\" d=\"M1424.2129,-30.277C1447.2732,-34.7987 1475.5743,-40.3479 1497.7998,-44.7058\"/>\n<polygon fill=\"#000000\" stroke=\"#000000\" points=\"1497.3707,-48.1883 1507.8574,-46.6779 1498.7177,-41.3191 1497.3707,-48.1883\"/>\n<text text-anchor=\"middle\" x=\"1466\" y=\"-46\" font-family=\"Times\" font-size=\"10.00\" fill=\"#000000\">25x10x1x1</text>\n</g>\n<!-- /outputs/53 -->\n<g id=\"node15\" class=\"node\">\n<title>/outputs/53</title>\n<polygon fill=\"#e8e8e8\" stroke=\"#000000\" points=\"1980,-36 1926,-36 1926,0 1980,0 1980,-36\"/>\n<text text-anchor=\"start\" x=\"1936\" y=\"-15\" font-family=\"Times\" font-size=\"10.00\" fill=\"#000000\">Reshape</text>\n</g>\n<!-- /outputs/45&#45;&gt;/outputs/53 -->\n<g id=\"edge6\" class=\"edge\">\n<title>/outputs/45&#45;&gt;/outputs/53</title>\n<path fill=\"none\" stroke=\"#000000\" d=\"M1424.0883,-15.2146C1454.7531,-10.8538 1497.3426,-6 1535,-6 1535,-6 1535,-6 1862,-6 1879.6472,-6 1899.0554,-8.2456 1915.3988,-10.7939\"/>\n<polygon fill=\"#000000\" stroke=\"#000000\" points=\"1915.2092,-14.3101 1925.6468,-12.4917 1916.3534,-7.4043 1915.2092,-14.3101\"/>\n<text text-anchor=\"middle\" x=\"1695\" y=\"-9\" font-family=\"Times\" font-size=\"10.00\" fill=\"#000000\">25x10x1x1</text>\n</g>\n<!-- /outputs/48 -->\n<g id=\"node10\" class=\"node\">\n<title>/outputs/48</title>\n<polygon fill=\"#e8e8e8\" stroke=\"#000000\" points=\"1653,-70 1599,-70 1599,-34 1653,-34 1653,-70\"/>\n<text text-anchor=\"start\" x=\"1612\" y=\"-49\" font-family=\"Times\" font-size=\"10.00\" fill=\"#000000\">Gather</text>\n</g>\n<!-- /outputs/46&#45;&gt;/outputs/48 -->\n<g id=\"edge7\" class=\"edge\">\n<title>/outputs/46&#45;&gt;/outputs/48</title>\n<path fill=\"none\" stroke=\"#000000\" d=\"M1562.303,-52C1570.5813,-52 1579.8217,-52 1588.6405,-52\"/>\n<polygon fill=\"#000000\" stroke=\"#000000\" points=\"1588.7163,-55.5001 1598.7163,-52 1588.7162,-48.5001 1588.7163,-55.5001\"/>\n</g>\n<!-- /outputs/47 -->\n<g id=\"node9\" class=\"node\">\n<title>/outputs/47</title>\n<polygon fill=\"#e8e8e8\" stroke=\"#000000\" points=\"1562,-124 1508,-124 1508,-88 1562,-88 1562,-124\"/>\n<text text-anchor=\"start\" x=\"1517\" y=\"-103\" font-family=\"Times\" font-size=\"10.00\" fill=\"#000000\">Constant</text>\n</g>\n<!-- /outputs/47&#45;&gt;/outputs/48 -->\n<g id=\"edge8\" class=\"edge\">\n<title>/outputs/47&#45;&gt;/outputs/48</title>\n<path fill=\"none\" stroke=\"#000000\" d=\"M1562.303,-89.7982C1571.0312,-84.6189 1580.829,-78.8048 1590.0744,-73.3185\"/>\n<polygon fill=\"#000000\" stroke=\"#000000\" points=\"1591.9025,-76.3035 1598.7163,-68.1903 1588.3303,-70.2836 1591.9025,-76.3035\"/>\n</g>\n<!-- /outputs/50 -->\n<g id=\"node12\" class=\"node\">\n<title>/outputs/50</title>\n<polygon fill=\"#e8e8e8\" stroke=\"#000000\" points=\"1798,-70 1737,-70 1737,-34 1798,-34 1798,-70\"/>\n<text text-anchor=\"start\" x=\"1745.5\" y=\"-49\" font-family=\"Times\" font-size=\"10.00\" fill=\"#000000\">Unsqueeze</text>\n</g>\n<!-- /outputs/48&#45;&gt;/outputs/50 -->\n<g id=\"edge9\" class=\"edge\">\n<title>/outputs/48&#45;&gt;/outputs/50</title>\n<path fill=\"none\" stroke=\"#000000\" d=\"M1653.2826,-52C1674.1297,-52 1703.1976,-52 1726.7915,-52\"/>\n<polygon fill=\"#000000\" stroke=\"#000000\" points=\"1726.862,-55.5001 1736.862,-52 1726.8619,-48.5001 1726.862,-55.5001\"/>\n</g>\n<!-- /outputs/49 -->\n<g id=\"node11\" class=\"node\">\n<title>/outputs/49</title>\n<polygon fill=\"#e8e8e8\" stroke=\"#000000\" points=\"1653,-124 1599,-124 1599,-88 1653,-88 1653,-124\"/>\n<text text-anchor=\"start\" x=\"1608\" y=\"-103\" font-family=\"Times\" font-size=\"10.00\" fill=\"#000000\">Constant</text>\n</g>\n<!-- /outputs/51 -->\n<g id=\"node13\" class=\"node\">\n<title>/outputs/51</title>\n<polygon fill=\"#e8e8e8\" stroke=\"#000000\" points=\"1798,-124 1737,-124 1737,-88 1798,-88 1798,-124\"/>\n<text text-anchor=\"start\" x=\"1745.5\" y=\"-103\" font-family=\"Times\" font-size=\"10.00\" fill=\"#000000\">Unsqueeze</text>\n</g>\n<!-- /outputs/49&#45;&gt;/outputs/51 -->\n<g id=\"edge10\" class=\"edge\">\n<title>/outputs/49&#45;&gt;/outputs/51</title>\n<path fill=\"none\" stroke=\"#000000\" d=\"M1653.2826,-106C1674.1297,-106 1703.1976,-106 1726.7915,-106\"/>\n<polygon fill=\"#000000\" stroke=\"#000000\" points=\"1726.862,-109.5001 1736.862,-106 1726.8619,-102.5001 1726.862,-109.5001\"/>\n</g>\n<!-- /outputs/52 -->\n<g id=\"node14\" class=\"node\">\n<title>/outputs/52</title>\n<polygon fill=\"#e8e8e8\" stroke=\"#000000\" points=\"1889,-70 1835,-70 1835,-34 1889,-34 1889,-70\"/>\n<text text-anchor=\"start\" x=\"1847\" y=\"-49\" font-family=\"Times\" font-size=\"10.00\" fill=\"#000000\">Concat</text>\n</g>\n<!-- /outputs/50&#45;&gt;/outputs/52 -->\n<g id=\"edge11\" class=\"edge\">\n<title>/outputs/50&#45;&gt;/outputs/52</title>\n<path fill=\"none\" stroke=\"#000000\" d=\"M1798.1814,-52C1806.7033,-52 1816.0485,-52 1824.8957,-52\"/>\n<polygon fill=\"#000000\" stroke=\"#000000\" points=\"1824.9735,-55.5001 1834.9735,-52 1824.9734,-48.5001 1824.9735,-55.5001\"/>\n</g>\n<!-- /outputs/51&#45;&gt;/outputs/52 -->\n<g id=\"edge12\" class=\"edge\">\n<title>/outputs/51&#45;&gt;/outputs/52</title>\n<path fill=\"none\" stroke=\"#000000\" d=\"M1798.1814,-88.4677C1807.0738,-83.3864 1816.8627,-77.7927 1826.0467,-72.5447\"/>\n<polygon fill=\"#000000\" stroke=\"#000000\" points=\"1828.0275,-75.444 1834.9735,-67.4437 1824.5545,-69.3663 1828.0275,-75.444\"/>\n</g>\n<!-- /outputs/52&#45;&gt;/outputs/53 -->\n<g id=\"edge13\" class=\"edge\">\n<title>/outputs/52&#45;&gt;/outputs/53</title>\n<path fill=\"none\" stroke=\"#000000\" d=\"M1889.303,-41.7989C1897.7612,-38.6387 1907.224,-35.1031 1916.215,-31.7438\"/>\n<polygon fill=\"#000000\" stroke=\"#000000\" points=\"1917.5738,-34.9726 1925.7163,-28.1939 1915.1237,-28.4153 1917.5738,-34.9726\"/>\n</g>\n<!-- /outputs/54 -->\n<g id=\"node16\" class=\"node\">\n<title>/outputs/54</title>\n<polygon fill=\"#e8e8e8\" stroke=\"#000000\" points=\"2097,-36 2043,-36 2043,0 2097,0 2097,-36\"/>\n<text text-anchor=\"start\" x=\"2053\" y=\"-15\" font-family=\"Times\" font-size=\"10.00\" fill=\"#000000\">Softmax</text>\n</g>\n<!-- /outputs/53&#45;&gt;/outputs/54 -->\n<g id=\"edge14\" class=\"edge\">\n<title>/outputs/53&#45;&gt;/outputs/54</title>\n<path fill=\"none\" stroke=\"#000000\" d=\"M1980.1362,-18C1995.6773,-18 2015.4112,-18 2032.4253,-18\"/>\n<polygon fill=\"#000000\" stroke=\"#000000\" points=\"2032.6057,-21.5001 2042.6056,-18 2032.6056,-14.5001 2032.6057,-21.5001\"/>\n<text text-anchor=\"middle\" x=\"2011.5\" y=\"-21\" font-family=\"Times\" font-size=\"10.00\" fill=\"#000000\">25x10</text>\n</g>\n<!-- 5398377922648145120 -->\n<g id=\"node17\" class=\"node\">\n<title>5398377922648145120</title>\n<polygon fill=\"#e8e8e8\" stroke=\"#000000\" points=\"84,-44 0,-44 0,0 84,0 84,-44\"/>\n<text text-anchor=\"start\" x=\"8\" y=\"-28\" font-family=\"Times\" font-size=\"10.00\" fill=\"#000000\">Conv5x5 &gt; Relu</text>\n<text text-anchor=\"start\" x=\"69\" y=\"-7\" font-family=\"Times\" font-size=\"10.00\" fill=\"#000000\">x3</text>\n</g>\n<!-- 5398377922648145120&#45;&gt;/outputs/25 -->\n<g id=\"edge15\" class=\"edge\">\n<title>5398377922648145120&#45;&gt;/outputs/25</title>\n<path fill=\"none\" stroke=\"#000000\" d=\"M84.2697,-22C109.5651,-22 141.7113,-22 167.5519,-22\"/>\n<polygon fill=\"#000000\" stroke=\"#000000\" points=\"167.8392,-25.5001 177.8391,-22 167.8391,-18.5001 167.8392,-25.5001\"/>\n<text text-anchor=\"middle\" x=\"131\" y=\"-25\" font-family=\"Times\" font-size=\"10.00\" fill=\"#000000\">25x96x32x32</text>\n</g>\n<!-- 2334275721088331492&#45;&gt;/outputs/34 -->\n<g id=\"edge17\" class=\"edge\">\n<title>2334275721088331492&#45;&gt;/outputs/34</title>\n<path fill=\"none\" stroke=\"#000000\" d=\"M574.2099,-22C601.7427,-22 637.3463,-22 663.8125,-22\"/>\n<polygon fill=\"#000000\" stroke=\"#000000\" points=\"663.8289,-25.5001 673.8289,-22 663.8289,-18.5001 663.8289,-25.5001\"/>\n<text text-anchor=\"middle\" x=\"624\" y=\"-25\" font-family=\"Times\" font-size=\"10.00\" fill=\"#000000\">25x192x16x16</text>\n</g>\n<!-- 4304698631915573932&#45;&gt;/outputs/44 -->\n<g id=\"edge19\" class=\"edge\">\n<title>4304698631915573932&#45;&gt;/outputs/44</title>\n<path fill=\"none\" stroke=\"#000000\" d=\"M1165.2129,-22C1188.2732,-22 1216.5743,-22 1238.7998,-22\"/>\n<polygon fill=\"#000000\" stroke=\"#000000\" points=\"1238.8574,-25.5001 1248.8574,-22 1238.8573,-18.5001 1238.8574,-25.5001\"/>\n<text text-anchor=\"middle\" x=\"1207\" y=\"-25\" font-family=\"Times\" font-size=\"10.00\" fill=\"#000000\">25x10x8x8</text>\n</g>\n</g>\n</svg>\n"
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 9
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab_type": "code",
        "id": "Dzh3ROmRj5w7",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 1000
        },
        "outputId": "6f7b9611-7323-4468-974c-f69d7af75e48"
      },
      "source": [
        "def compute_accuracy(model, data_loader, device):\n",
        "    correct_pred, num_examples = 0, 0\n",
        "    for i, (features, targets) in enumerate(data_loader):\n",
        "            \n",
        "        features = features.to(device)\n",
        "        targets = targets.to(device)\n",
        "\n",
        "        logits, probas = model(features)\n",
        "        _, predicted_labels = torch.max(probas, 1)\n",
        "        num_examples += targets.size(0)\n",
        "        correct_pred += (predicted_labels == targets).sum()\n",
        "    return correct_pred.float()/num_examples * 100\n",
        "    \n",
        "\n",
        "start_time = time.time()\n",
        "for epoch in range(NUM_EPOCHS):\n",
        "    \n",
        "    model.train()\n",
        "    \n",
        "    for batch_idx, (features, targets) in enumerate(train_loader):\n",
        "    \n",
        "        ### PREPARE MINIBATCH\n",
        "        features = features.to(DEVICE)\n",
        "        targets = targets.to(DEVICE)\n",
        "            \n",
        "        ### FORWARD AND BACK PROP\n",
        "        logits, probas = model(features)\n",
        "        cost = F.cross_entropy(logits, targets)\n",
        "        optimizer.zero_grad()\n",
        "        \n",
        "        cost.backward()\n",
        "        \n",
        "        ### UPDATE MODEL PARAMETERS\n",
        "        optimizer.step()\n",
        "        \n",
        "        ### LOGGING\n",
        "        if not batch_idx % 120:\n",
        "            print (f'Epoch: {epoch+1:03d}/{NUM_EPOCHS:03d} | '\n",
        "                   f'Batch {batch_idx:03d}/{len(train_loader):03d} |' \n",
        "                   f' Cost: {cost:.4f}')\n",
        "\n",
        "    # no need to build the computation graph for backprop when computing accuracy\n",
        "    with torch.set_grad_enabled(False):\n",
        "        train_acc = compute_accuracy(model, train_loader, device=DEVICE)\n",
        "        valid_acc = compute_accuracy(model, valid_loader, device=DEVICE)\n",
        "        print(f'Epoch: {epoch+1:03d}/{NUM_EPOCHS:03d} Train Acc.: {train_acc:.2f}%'\n",
        "              f' | Validation Acc.: {valid_acc:.2f}%')\n",
        "        \n",
        "    elapsed = (time.time() - start_time)/60\n",
        "    print(f'Time elapsed: {elapsed:.2f} min')\n",
        "  \n",
        "elapsed = (time.time() - start_time)/60\n",
        "print(f'Total Training Time: {elapsed:.2f} min')"
      ],
      "execution_count": 10,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Epoch: 001/100 | Batch 000/192 | Cost: 2.3043\n",
            "Epoch: 001/100 | Batch 120/192 | Cost: 2.0041\n",
            "Epoch: 001/100 Train Acc.: 24.18% | Validation Acc.: 27.00%\n",
            "Time elapsed: 0.35 min\n",
            "Epoch: 002/100 | Batch 000/192 | Cost: 1.9350\n",
            "Epoch: 002/100 | Batch 120/192 | Cost: 1.7446\n",
            "Epoch: 002/100 Train Acc.: 36.08% | Validation Acc.: 36.70%\n",
            "Time elapsed: 0.69 min\n",
            "Epoch: 003/100 | Batch 000/192 | Cost: 1.6324\n",
            "Epoch: 003/100 | Batch 120/192 | Cost: 1.6135\n",
            "Epoch: 003/100 Train Acc.: 41.75% | Validation Acc.: 42.50%\n",
            "Time elapsed: 1.04 min\n",
            "Epoch: 004/100 | Batch 000/192 | Cost: 1.5062\n",
            "Epoch: 004/100 | Batch 120/192 | Cost: 1.4272\n",
            "Epoch: 004/100 Train Acc.: 43.88% | Validation Acc.: 43.90%\n",
            "Time elapsed: 1.38 min\n",
            "Epoch: 005/100 | Batch 000/192 | Cost: 1.4328\n",
            "Epoch: 005/100 | Batch 120/192 | Cost: 1.4579\n",
            "Epoch: 005/100 Train Acc.: 49.44% | Validation Acc.: 48.30%\n",
            "Time elapsed: 1.72 min\n",
            "Epoch: 006/100 | Batch 000/192 | Cost: 1.3332\n",
            "Epoch: 006/100 | Batch 120/192 | Cost: 1.2475\n",
            "Epoch: 006/100 Train Acc.: 53.83% | Validation Acc.: 54.60%\n",
            "Time elapsed: 2.07 min\n",
            "Epoch: 007/100 | Batch 000/192 | Cost: 1.3004\n",
            "Epoch: 007/100 | Batch 120/192 | Cost: 1.1738\n",
            "Epoch: 007/100 Train Acc.: 56.48% | Validation Acc.: 56.30%\n",
            "Time elapsed: 2.41 min\n",
            "Epoch: 008/100 | Batch 000/192 | Cost: 1.1414\n",
            "Epoch: 008/100 | Batch 120/192 | Cost: 1.1363\n",
            "Epoch: 008/100 Train Acc.: 56.78% | Validation Acc.: 56.40%\n",
            "Time elapsed: 2.76 min\n",
            "Epoch: 009/100 | Batch 000/192 | Cost: 1.2452\n",
            "Epoch: 009/100 | Batch 120/192 | Cost: 1.0806\n",
            "Epoch: 009/100 Train Acc.: 59.56% | Validation Acc.: 62.10%\n",
            "Time elapsed: 3.11 min\n",
            "Epoch: 010/100 | Batch 000/192 | Cost: 1.0228\n",
            "Epoch: 010/100 | Batch 120/192 | Cost: 1.1042\n",
            "Epoch: 010/100 Train Acc.: 60.55% | Validation Acc.: 63.40%\n",
            "Time elapsed: 3.45 min\n",
            "Epoch: 011/100 | Batch 000/192 | Cost: 1.0565\n",
            "Epoch: 011/100 | Batch 120/192 | Cost: 1.0033\n",
            "Epoch: 011/100 Train Acc.: 61.82% | Validation Acc.: 61.00%\n",
            "Time elapsed: 3.79 min\n",
            "Epoch: 012/100 | Batch 000/192 | Cost: 1.0023\n",
            "Epoch: 012/100 | Batch 120/192 | Cost: 1.1402\n",
            "Epoch: 012/100 Train Acc.: 63.87% | Validation Acc.: 63.80%\n",
            "Time elapsed: 4.14 min\n",
            "Epoch: 013/100 | Batch 000/192 | Cost: 0.8760\n",
            "Epoch: 013/100 | Batch 120/192 | Cost: 0.9804\n",
            "Epoch: 013/100 Train Acc.: 63.73% | Validation Acc.: 62.80%\n",
            "Time elapsed: 4.48 min\n",
            "Epoch: 014/100 | Batch 000/192 | Cost: 0.9027\n",
            "Epoch: 014/100 | Batch 120/192 | Cost: 1.1282\n",
            "Epoch: 014/100 Train Acc.: 66.42% | Validation Acc.: 66.30%\n",
            "Time elapsed: 4.83 min\n",
            "Epoch: 015/100 | Batch 000/192 | Cost: 0.8892\n",
            "Epoch: 015/100 | Batch 120/192 | Cost: 0.9479\n",
            "Epoch: 015/100 Train Acc.: 65.14% | Validation Acc.: 65.90%\n",
            "Time elapsed: 5.17 min\n",
            "Epoch: 016/100 | Batch 000/192 | Cost: 0.8575\n",
            "Epoch: 016/100 | Batch 120/192 | Cost: 0.9928\n",
            "Epoch: 016/100 Train Acc.: 66.79% | Validation Acc.: 65.50%\n",
            "Time elapsed: 5.52 min\n",
            "Epoch: 017/100 | Batch 000/192 | Cost: 0.8890\n",
            "Epoch: 017/100 | Batch 120/192 | Cost: 0.9584\n",
            "Epoch: 017/100 Train Acc.: 69.32% | Validation Acc.: 68.50%\n",
            "Time elapsed: 5.86 min\n",
            "Epoch: 018/100 | Batch 000/192 | Cost: 0.8710\n",
            "Epoch: 018/100 | Batch 120/192 | Cost: 0.8033\n",
            "Epoch: 018/100 Train Acc.: 69.13% | Validation Acc.: 68.10%\n",
            "Time elapsed: 6.21 min\n",
            "Epoch: 019/100 | Batch 000/192 | Cost: 0.9469\n",
            "Epoch: 019/100 | Batch 120/192 | Cost: 0.9746\n",
            "Epoch: 019/100 Train Acc.: 70.75% | Validation Acc.: 69.20%\n",
            "Time elapsed: 6.55 min\n",
            "Epoch: 020/100 | Batch 000/192 | Cost: 0.8509\n",
            "Epoch: 020/100 | Batch 120/192 | Cost: 0.8083\n",
            "Epoch: 020/100 Train Acc.: 71.13% | Validation Acc.: 71.40%\n",
            "Time elapsed: 6.89 min\n",
            "Epoch: 021/100 | Batch 000/192 | Cost: 0.8532\n",
            "Epoch: 021/100 | Batch 120/192 | Cost: 0.8372\n",
            "Epoch: 021/100 Train Acc.: 71.29% | Validation Acc.: 69.70%\n",
            "Time elapsed: 7.24 min\n",
            "Epoch: 022/100 | Batch 000/192 | Cost: 0.7992\n",
            "Epoch: 022/100 | Batch 120/192 | Cost: 0.6424\n",
            "Epoch: 022/100 Train Acc.: 71.69% | Validation Acc.: 70.30%\n",
            "Time elapsed: 7.58 min\n",
            "Epoch: 023/100 | Batch 000/192 | Cost: 0.8724\n",
            "Epoch: 023/100 | Batch 120/192 | Cost: 0.8339\n",
            "Epoch: 023/100 Train Acc.: 72.01% | Validation Acc.: 69.90%\n",
            "Time elapsed: 7.93 min\n",
            "Epoch: 024/100 | Batch 000/192 | Cost: 0.7950\n",
            "Epoch: 024/100 | Batch 120/192 | Cost: 0.7528\n",
            "Epoch: 024/100 Train Acc.: 73.46% | Validation Acc.: 72.30%\n",
            "Time elapsed: 8.27 min\n",
            "Epoch: 025/100 | Batch 000/192 | Cost: 0.7893\n",
            "Epoch: 025/100 | Batch 120/192 | Cost: 0.6749\n",
            "Epoch: 025/100 Train Acc.: 74.21% | Validation Acc.: 71.80%\n",
            "Time elapsed: 8.62 min\n",
            "Epoch: 026/100 | Batch 000/192 | Cost: 0.6514\n",
            "Epoch: 026/100 | Batch 120/192 | Cost: 0.7255\n",
            "Epoch: 026/100 Train Acc.: 73.04% | Validation Acc.: 72.20%\n",
            "Time elapsed: 8.96 min\n",
            "Epoch: 027/100 | Batch 000/192 | Cost: 0.7122\n",
            "Epoch: 027/100 | Batch 120/192 | Cost: 0.6967\n",
            "Epoch: 027/100 Train Acc.: 75.32% | Validation Acc.: 74.30%\n",
            "Time elapsed: 9.30 min\n",
            "Epoch: 028/100 | Batch 000/192 | Cost: 0.6683\n",
            "Epoch: 028/100 | Batch 120/192 | Cost: 0.6337\n",
            "Epoch: 028/100 Train Acc.: 76.00% | Validation Acc.: 73.60%\n",
            "Time elapsed: 9.65 min\n",
            "Epoch: 029/100 | Batch 000/192 | Cost: 0.7235\n",
            "Epoch: 029/100 | Batch 120/192 | Cost: 0.6535\n",
            "Epoch: 029/100 Train Acc.: 76.30% | Validation Acc.: 74.90%\n",
            "Time elapsed: 9.99 min\n",
            "Epoch: 030/100 | Batch 000/192 | Cost: 0.7498\n",
            "Epoch: 030/100 | Batch 120/192 | Cost: 0.6587\n",
            "Epoch: 030/100 Train Acc.: 76.64% | Validation Acc.: 74.40%\n",
            "Time elapsed: 10.34 min\n",
            "Epoch: 031/100 | Batch 000/192 | Cost: 0.6098\n",
            "Epoch: 031/100 | Batch 120/192 | Cost: 0.5858\n",
            "Epoch: 031/100 Train Acc.: 75.52% | Validation Acc.: 74.20%\n",
            "Time elapsed: 10.68 min\n",
            "Epoch: 032/100 | Batch 000/192 | Cost: 0.6672\n",
            "Epoch: 032/100 | Batch 120/192 | Cost: 0.7482\n",
            "Epoch: 032/100 Train Acc.: 77.17% | Validation Acc.: 73.90%\n",
            "Time elapsed: 11.03 min\n",
            "Epoch: 033/100 | Batch 000/192 | Cost: 0.7271\n",
            "Epoch: 033/100 | Batch 120/192 | Cost: 0.7013\n",
            "Epoch: 033/100 Train Acc.: 77.68% | Validation Acc.: 75.00%\n",
            "Time elapsed: 11.37 min\n",
            "Epoch: 034/100 | Batch 000/192 | Cost: 0.5911\n",
            "Epoch: 034/100 | Batch 120/192 | Cost: 0.6259\n",
            "Epoch: 034/100 Train Acc.: 77.92% | Validation Acc.: 73.70%\n",
            "Time elapsed: 11.72 min\n",
            "Epoch: 035/100 | Batch 000/192 | Cost: 0.6155\n",
            "Epoch: 035/100 | Batch 120/192 | Cost: 0.7030\n",
            "Epoch: 035/100 Train Acc.: 79.51% | Validation Acc.: 75.70%\n",
            "Time elapsed: 12.07 min\n",
            "Epoch: 036/100 | Batch 000/192 | Cost: 0.6942\n",
            "Epoch: 036/100 | Batch 120/192 | Cost: 0.6422\n",
            "Epoch: 036/100 Train Acc.: 79.58% | Validation Acc.: 75.90%\n",
            "Time elapsed: 12.41 min\n",
            "Epoch: 037/100 | Batch 000/192 | Cost: 0.6403\n",
            "Epoch: 037/100 | Batch 120/192 | Cost: 0.6027\n",
            "Epoch: 037/100 Train Acc.: 79.19% | Validation Acc.: 77.70%\n",
            "Time elapsed: 12.76 min\n",
            "Epoch: 038/100 | Batch 000/192 | Cost: 0.5637\n",
            "Epoch: 038/100 | Batch 120/192 | Cost: 0.5646\n",
            "Epoch: 038/100 Train Acc.: 79.34% | Validation Acc.: 76.70%\n",
            "Time elapsed: 13.10 min\n",
            "Epoch: 039/100 | Batch 000/192 | Cost: 0.5208\n",
            "Epoch: 039/100 | Batch 120/192 | Cost: 0.5630\n",
            "Epoch: 039/100 Train Acc.: 79.13% | Validation Acc.: 76.80%\n",
            "Time elapsed: 13.44 min\n",
            "Epoch: 040/100 | Batch 000/192 | Cost: 0.6377\n",
            "Epoch: 040/100 | Batch 120/192 | Cost: 0.5755\n",
            "Epoch: 040/100 Train Acc.: 80.89% | Validation Acc.: 77.00%\n",
            "Time elapsed: 13.79 min\n",
            "Epoch: 041/100 | Batch 000/192 | Cost: 0.5138\n",
            "Epoch: 041/100 | Batch 120/192 | Cost: 0.7050\n",
            "Epoch: 041/100 Train Acc.: 80.64% | Validation Acc.: 75.10%\n",
            "Time elapsed: 14.13 min\n",
            "Epoch: 042/100 | Batch 000/192 | Cost: 0.4783\n",
            "Epoch: 042/100 | Batch 120/192 | Cost: 0.6303\n",
            "Epoch: 042/100 Train Acc.: 81.08% | Validation Acc.: 78.90%\n",
            "Time elapsed: 14.48 min\n",
            "Epoch: 043/100 | Batch 000/192 | Cost: 0.4734\n",
            "Epoch: 043/100 | Batch 120/192 | Cost: 0.4774\n",
            "Epoch: 043/100 Train Acc.: 81.57% | Validation Acc.: 76.50%\n",
            "Time elapsed: 14.83 min\n",
            "Epoch: 044/100 | Batch 000/192 | Cost: 0.5157\n",
            "Epoch: 044/100 | Batch 120/192 | Cost: 0.5578\n",
            "Epoch: 044/100 Train Acc.: 81.04% | Validation Acc.: 78.40%\n",
            "Time elapsed: 15.17 min\n",
            "Epoch: 045/100 | Batch 000/192 | Cost: 0.5093\n",
            "Epoch: 045/100 | Batch 120/192 | Cost: 0.6032\n",
            "Epoch: 045/100 Train Acc.: 82.74% | Validation Acc.: 78.30%\n",
            "Time elapsed: 15.52 min\n",
            "Epoch: 046/100 | Batch 000/192 | Cost: 0.4896\n",
            "Epoch: 046/100 | Batch 120/192 | Cost: 0.4682\n",
            "Epoch: 046/100 Train Acc.: 82.58% | Validation Acc.: 77.40%\n",
            "Time elapsed: 15.86 min\n",
            "Epoch: 047/100 | Batch 000/192 | Cost: 0.4972\n",
            "Epoch: 047/100 | Batch 120/192 | Cost: 0.4998\n",
            "Epoch: 047/100 Train Acc.: 82.84% | Validation Acc.: 78.90%\n",
            "Time elapsed: 16.21 min\n",
            "Epoch: 048/100 | Batch 000/192 | Cost: 0.4225\n",
            "Epoch: 048/100 | Batch 120/192 | Cost: 0.6129\n",
            "Epoch: 048/100 Train Acc.: 82.92% | Validation Acc.: 77.30%\n",
            "Time elapsed: 16.55 min\n",
            "Epoch: 049/100 | Batch 000/192 | Cost: 0.4496\n",
            "Epoch: 049/100 | Batch 120/192 | Cost: 0.4332\n",
            "Epoch: 049/100 Train Acc.: 83.66% | Validation Acc.: 78.40%\n",
            "Time elapsed: 16.90 min\n",
            "Epoch: 050/100 | Batch 000/192 | Cost: 0.5423\n",
            "Epoch: 050/100 | Batch 120/192 | Cost: 0.5665\n",
            "Epoch: 050/100 Train Acc.: 83.84% | Validation Acc.: 79.00%\n",
            "Time elapsed: 17.24 min\n",
            "Epoch: 051/100 | Batch 000/192 | Cost: 0.4372\n",
            "Epoch: 051/100 | Batch 120/192 | Cost: 0.4252\n",
            "Epoch: 051/100 Train Acc.: 83.27% | Validation Acc.: 79.80%\n",
            "Time elapsed: 17.58 min\n",
            "Epoch: 052/100 | Batch 000/192 | Cost: 0.5137\n",
            "Epoch: 052/100 | Batch 120/192 | Cost: 0.5231\n",
            "Epoch: 052/100 Train Acc.: 83.68% | Validation Acc.: 78.10%\n",
            "Time elapsed: 17.93 min\n",
            "Epoch: 053/100 | Batch 000/192 | Cost: 0.5262\n",
            "Epoch: 053/100 | Batch 120/192 | Cost: 0.3709\n",
            "Epoch: 053/100 Train Acc.: 82.26% | Validation Acc.: 77.40%\n",
            "Time elapsed: 18.27 min\n",
            "Epoch: 054/100 | Batch 000/192 | Cost: 0.4764\n",
            "Epoch: 054/100 | Batch 120/192 | Cost: 0.5095\n",
            "Epoch: 054/100 Train Acc.: 83.71% | Validation Acc.: 77.60%\n",
            "Time elapsed: 18.62 min\n",
            "Epoch: 055/100 | Batch 000/192 | Cost: 0.4363\n",
            "Epoch: 055/100 | Batch 120/192 | Cost: 0.4260\n",
            "Epoch: 055/100 Train Acc.: 83.07% | Validation Acc.: 78.10%\n",
            "Time elapsed: 18.96 min\n",
            "Epoch: 056/100 | Batch 000/192 | Cost: 0.5525\n",
            "Epoch: 056/100 | Batch 120/192 | Cost: 0.3717\n",
            "Epoch: 056/100 Train Acc.: 84.69% | Validation Acc.: 78.20%\n",
            "Time elapsed: 19.31 min\n",
            "Epoch: 057/100 | Batch 000/192 | Cost: 0.3486\n",
            "Epoch: 057/100 | Batch 120/192 | Cost: 0.4213\n",
            "Epoch: 057/100 Train Acc.: 84.81% | Validation Acc.: 79.00%\n",
            "Time elapsed: 19.65 min\n",
            "Epoch: 058/100 | Batch 000/192 | Cost: 0.3746\n",
            "Epoch: 058/100 | Batch 120/192 | Cost: 0.3970\n",
            "Epoch: 058/100 Train Acc.: 85.43% | Validation Acc.: 79.30%\n",
            "Time elapsed: 20.00 min\n",
            "Epoch: 059/100 | Batch 000/192 | Cost: 0.4380\n",
            "Epoch: 059/100 | Batch 120/192 | Cost: 0.4323\n",
            "Epoch: 059/100 Train Acc.: 85.17% | Validation Acc.: 79.20%\n",
            "Time elapsed: 20.34 min\n",
            "Epoch: 060/100 | Batch 000/192 | Cost: 0.2895\n",
            "Epoch: 060/100 | Batch 120/192 | Cost: 0.4876\n",
            "Epoch: 060/100 Train Acc.: 86.06% | Validation Acc.: 80.00%\n",
            "Time elapsed: 20.68 min\n",
            "Epoch: 061/100 | Batch 000/192 | Cost: 0.4768\n",
            "Epoch: 061/100 | Batch 120/192 | Cost: 0.3555\n",
            "Epoch: 061/100 Train Acc.: 86.73% | Validation Acc.: 80.40%\n",
            "Time elapsed: 21.03 min\n",
            "Epoch: 062/100 | Batch 000/192 | Cost: 0.4041\n",
            "Epoch: 062/100 | Batch 120/192 | Cost: 0.2696\n",
            "Epoch: 062/100 Train Acc.: 86.88% | Validation Acc.: 80.00%\n",
            "Time elapsed: 21.37 min\n",
            "Epoch: 063/100 | Batch 000/192 | Cost: 0.3945\n",
            "Epoch: 063/100 | Batch 120/192 | Cost: 0.5223\n",
            "Epoch: 063/100 Train Acc.: 86.07% | Validation Acc.: 80.40%\n",
            "Time elapsed: 21.72 min\n",
            "Epoch: 064/100 | Batch 000/192 | Cost: 0.3680\n",
            "Epoch: 064/100 | Batch 120/192 | Cost: 0.3739\n",
            "Epoch: 064/100 Train Acc.: 87.35% | Validation Acc.: 81.20%\n",
            "Time elapsed: 22.06 min\n",
            "Epoch: 065/100 | Batch 000/192 | Cost: 0.3576\n",
            "Epoch: 065/100 | Batch 120/192 | Cost: 0.4235\n",
            "Epoch: 065/100 Train Acc.: 85.52% | Validation Acc.: 77.30%\n",
            "Time elapsed: 22.41 min\n",
            "Epoch: 066/100 | Batch 000/192 | Cost: 0.3803\n",
            "Epoch: 066/100 | Batch 120/192 | Cost: 0.4552\n",
            "Epoch: 066/100 Train Acc.: 87.29% | Validation Acc.: 79.50%\n",
            "Time elapsed: 22.75 min\n",
            "Epoch: 067/100 | Batch 000/192 | Cost: 0.3927\n",
            "Epoch: 067/100 | Batch 120/192 | Cost: 0.3092\n",
            "Epoch: 067/100 Train Acc.: 85.73% | Validation Acc.: 79.00%\n",
            "Time elapsed: 23.10 min\n",
            "Epoch: 068/100 | Batch 000/192 | Cost: 0.4226\n",
            "Epoch: 068/100 | Batch 120/192 | Cost: 0.2751\n",
            "Epoch: 068/100 Train Acc.: 87.38% | Validation Acc.: 79.60%\n",
            "Time elapsed: 23.44 min\n",
            "Epoch: 069/100 | Batch 000/192 | Cost: 0.4338\n",
            "Epoch: 069/100 | Batch 120/192 | Cost: 0.4057\n",
            "Epoch: 069/100 Train Acc.: 87.81% | Validation Acc.: 81.30%\n",
            "Time elapsed: 23.79 min\n",
            "Epoch: 070/100 | Batch 000/192 | Cost: 0.2754\n",
            "Epoch: 070/100 | Batch 120/192 | Cost: 0.3225\n",
            "Epoch: 070/100 Train Acc.: 88.06% | Validation Acc.: 79.30%\n",
            "Time elapsed: 24.13 min\n",
            "Epoch: 071/100 | Batch 000/192 | Cost: 0.3481\n",
            "Epoch: 071/100 | Batch 120/192 | Cost: 0.2608\n",
            "Epoch: 071/100 Train Acc.: 87.96% | Validation Acc.: 80.70%\n",
            "Time elapsed: 24.47 min\n",
            "Epoch: 072/100 | Batch 000/192 | Cost: 0.3075\n",
            "Epoch: 072/100 | Batch 120/192 | Cost: 0.4370\n",
            "Epoch: 072/100 Train Acc.: 88.27% | Validation Acc.: 80.00%\n",
            "Time elapsed: 24.82 min\n",
            "Epoch: 073/100 | Batch 000/192 | Cost: 0.3717\n",
            "Epoch: 073/100 | Batch 120/192 | Cost: 0.4243\n",
            "Epoch: 073/100 Train Acc.: 88.51% | Validation Acc.: 80.40%\n",
            "Time elapsed: 25.16 min\n",
            "Epoch: 074/100 | Batch 000/192 | Cost: 0.3309\n",
            "Epoch: 074/100 | Batch 120/192 | Cost: 0.2892\n",
            "Epoch: 074/100 Train Acc.: 88.29% | Validation Acc.: 79.40%\n",
            "Time elapsed: 25.51 min\n",
            "Epoch: 075/100 | Batch 000/192 | Cost: 0.2358\n",
            "Epoch: 075/100 | Batch 120/192 | Cost: 0.3298\n",
            "Epoch: 075/100 Train Acc.: 88.92% | Validation Acc.: 81.10%\n",
            "Time elapsed: 25.85 min\n",
            "Epoch: 076/100 | Batch 000/192 | Cost: 0.2908\n",
            "Epoch: 076/100 | Batch 120/192 | Cost: 0.2683\n",
            "Epoch: 076/100 Train Acc.: 88.92% | Validation Acc.: 80.20%\n",
            "Time elapsed: 26.20 min\n",
            "Epoch: 077/100 | Batch 000/192 | Cost: 0.2659\n",
            "Epoch: 077/100 | Batch 120/192 | Cost: 0.4185\n",
            "Epoch: 077/100 Train Acc.: 89.90% | Validation Acc.: 81.10%\n",
            "Time elapsed: 26.54 min\n",
            "Epoch: 078/100 | Batch 000/192 | Cost: 0.2309\n",
            "Epoch: 078/100 | Batch 120/192 | Cost: 0.3875\n",
            "Epoch: 078/100 Train Acc.: 89.88% | Validation Acc.: 81.50%\n",
            "Time elapsed: 26.89 min\n",
            "Epoch: 079/100 | Batch 000/192 | Cost: 0.2501\n",
            "Epoch: 079/100 | Batch 120/192 | Cost: 0.2655\n",
            "Epoch: 079/100 Train Acc.: 89.75% | Validation Acc.: 80.10%\n",
            "Time elapsed: 27.23 min\n",
            "Epoch: 080/100 | Batch 000/192 | Cost: 0.2901\n",
            "Epoch: 080/100 | Batch 120/192 | Cost: 0.3772\n",
            "Epoch: 080/100 Train Acc.: 89.28% | Validation Acc.: 80.20%\n",
            "Time elapsed: 27.58 min\n",
            "Epoch: 081/100 | Batch 000/192 | Cost: 0.2804\n",
            "Epoch: 081/100 | Batch 120/192 | Cost: 0.3445\n",
            "Epoch: 081/100 Train Acc.: 88.92% | Validation Acc.: 79.80%\n",
            "Time elapsed: 27.92 min\n",
            "Epoch: 082/100 | Batch 000/192 | Cost: 0.3334\n",
            "Epoch: 082/100 | Batch 120/192 | Cost: 0.2989\n",
            "Epoch: 082/100 Train Acc.: 88.80% | Validation Acc.: 79.10%\n",
            "Time elapsed: 28.27 min\n",
            "Epoch: 083/100 | Batch 000/192 | Cost: 0.2613\n",
            "Epoch: 083/100 | Batch 120/192 | Cost: 0.3674\n",
            "Epoch: 083/100 Train Acc.: 90.38% | Validation Acc.: 81.10%\n",
            "Time elapsed: 28.61 min\n",
            "Epoch: 084/100 | Batch 000/192 | Cost: 0.2914\n",
            "Epoch: 084/100 | Batch 120/192 | Cost: 0.2686\n",
            "Epoch: 084/100 Train Acc.: 88.18% | Validation Acc.: 79.30%\n",
            "Time elapsed: 28.96 min\n",
            "Epoch: 085/100 | Batch 000/192 | Cost: 0.2974\n",
            "Epoch: 085/100 | Batch 120/192 | Cost: 0.3652\n",
            "Epoch: 085/100 Train Acc.: 91.11% | Validation Acc.: 81.10%\n",
            "Time elapsed: 29.30 min\n",
            "Epoch: 086/100 | Batch 000/192 | Cost: 0.2652\n",
            "Epoch: 086/100 | Batch 120/192 | Cost: 0.3336\n",
            "Epoch: 086/100 Train Acc.: 90.83% | Validation Acc.: 80.70%\n",
            "Time elapsed: 29.65 min\n",
            "Epoch: 087/100 | Batch 000/192 | Cost: 0.2700\n",
            "Epoch: 087/100 | Batch 120/192 | Cost: 0.2431\n",
            "Epoch: 087/100 Train Acc.: 90.62% | Validation Acc.: 81.60%\n",
            "Time elapsed: 29.99 min\n",
            "Epoch: 088/100 | Batch 000/192 | Cost: 0.2632\n",
            "Epoch: 088/100 | Batch 120/192 | Cost: 0.2094\n",
            "Epoch: 088/100 Train Acc.: 89.47% | Validation Acc.: 80.60%\n",
            "Time elapsed: 30.34 min\n",
            "Epoch: 089/100 | Batch 000/192 | Cost: 0.3107\n",
            "Epoch: 089/100 | Batch 120/192 | Cost: 0.2491\n",
            "Epoch: 089/100 Train Acc.: 91.47% | Validation Acc.: 80.60%\n",
            "Time elapsed: 30.68 min\n",
            "Epoch: 090/100 | Batch 000/192 | Cost: 0.1940\n",
            "Epoch: 090/100 | Batch 120/192 | Cost: 0.2782\n",
            "Epoch: 090/100 Train Acc.: 90.34% | Validation Acc.: 80.90%\n",
            "Time elapsed: 31.02 min\n",
            "Epoch: 091/100 | Batch 000/192 | Cost: 0.2669\n",
            "Epoch: 091/100 | Batch 120/192 | Cost: 0.3550\n",
            "Epoch: 091/100 Train Acc.: 91.09% | Validation Acc.: 80.00%\n",
            "Time elapsed: 31.37 min\n",
            "Epoch: 092/100 | Batch 000/192 | Cost: 0.2262\n",
            "Epoch: 092/100 | Batch 120/192 | Cost: 0.2175\n",
            "Epoch: 092/100 Train Acc.: 92.32% | Validation Acc.: 81.80%\n",
            "Time elapsed: 31.71 min\n",
            "Epoch: 093/100 | Batch 000/192 | Cost: 0.1634\n",
            "Epoch: 093/100 | Batch 120/192 | Cost: 0.2062\n",
            "Epoch: 093/100 Train Acc.: 91.18% | Validation Acc.: 81.20%\n",
            "Time elapsed: 32.06 min\n",
            "Epoch: 094/100 | Batch 000/192 | Cost: 0.2585\n",
            "Epoch: 094/100 | Batch 120/192 | Cost: 0.3068\n",
            "Epoch: 094/100 Train Acc.: 91.94% | Validation Acc.: 80.60%\n",
            "Time elapsed: 32.40 min\n",
            "Epoch: 095/100 | Batch 000/192 | Cost: 0.2778\n",
            "Epoch: 095/100 | Batch 120/192 | Cost: 0.3047\n",
            "Epoch: 095/100 Train Acc.: 91.51% | Validation Acc.: 80.60%\n",
            "Time elapsed: 32.74 min\n",
            "Epoch: 096/100 | Batch 000/192 | Cost: 0.2717\n",
            "Epoch: 096/100 | Batch 120/192 | Cost: 0.2376\n",
            "Epoch: 096/100 Train Acc.: 91.56% | Validation Acc.: 82.00%\n",
            "Time elapsed: 33.09 min\n",
            "Epoch: 097/100 | Batch 000/192 | Cost: 0.1985\n",
            "Epoch: 097/100 | Batch 120/192 | Cost: 0.2482\n",
            "Epoch: 097/100 Train Acc.: 90.97% | Validation Acc.: 80.00%\n",
            "Time elapsed: 33.44 min\n",
            "Epoch: 098/100 | Batch 000/192 | Cost: 0.2598\n",
            "Epoch: 098/100 | Batch 120/192 | Cost: 0.3135\n",
            "Epoch: 098/100 Train Acc.: 91.84% | Validation Acc.: 80.30%\n",
            "Time elapsed: 33.78 min\n",
            "Epoch: 099/100 | Batch 000/192 | Cost: 0.2027\n",
            "Epoch: 099/100 | Batch 120/192 | Cost: 0.2253\n",
            "Epoch: 099/100 Train Acc.: 91.41% | Validation Acc.: 80.80%\n",
            "Time elapsed: 34.12 min\n",
            "Epoch: 100/100 | Batch 000/192 | Cost: 0.2435\n",
            "Epoch: 100/100 | Batch 120/192 | Cost: 0.2352\n",
            "Epoch: 100/100 Train Acc.: 90.86% | Validation Acc.: 81.00%\n",
            "Time elapsed: 34.47 min\n",
            "Total Training Time: 34.47 min\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "SseIzGCvNEyc",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 104
        },
        "outputId": "58808ec4-dc29-4dc7-a508-4d8da59bf7b2"
      },
      "source": [
        "%watermark -iv"
      ],
      "execution_count": 11,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "numpy     1.18.5\n",
            "PIL.Image 7.0.0\n",
            "torch     1.5.1+cu101\n",
            "pandas    1.0.5\n",
            "\n"
          ],
          "name": "stdout"
        }
      ]
    }
  ]
}