Autoencoders With Optimization Techniques v1.0.ipynb

{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "colab": {
      "provenance": []
    },
    "kernelspec": {
      "name": "python3",
      "display_name": "Python 3"
    },
    "language_info": {
      "name": "python"
    }
  },
  "cells": [
    {
      "cell_type": "markdown",
      "source": [
        "Imports"
      ],
      "metadata": {
        "id": "Qcn2myqUABo5"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "from scipy.sparse import load_npz\n",
        "import csv\n",
        "import os\n",
        "import torch\n",
        "import numpy as np\n",
        "\n",
        "from utils import *\n",
        "from torch.autograd import Variable\n",
        "\n",
        "import torch.nn as nn\n",
        "import torch.nn.functional as F\n",
        "import torch.optim as optim\n",
        "import torch.utils.data\n",
        "import matplotlib.pyplot as plt"
      ],
      "metadata": {
        "id": "8BiykqzGl3t8"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "source": [
        "Autoencoder Initialization"
      ],
      "metadata": {
        "id": "foUCUtMAAEts"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "class AutoEncoder(nn.Module):\n",
        "    torch.manual_seed(42)\n",
        "    def __init__(self, num_question, k = 50, j = 75):\n",
        "        \"\"\" Initialize a class AutoEncoder.\n",
        "\n",
        "        :param num_question: int\n",
        "        :param k: int\n",
        "        \"\"\"\n",
        "        super(AutoEncoder, self).__init__()\n",
        "\n",
        "        # Define linear functions.\n",
        "        self.encoder = nn.Linear(num_question, j)\n",
        "        self.layer1 = nn.Linear(j, k)\n",
        "        self.layer2 = nn.Linear(k, j)\n",
        "        self.dropout = nn.Dropout(p=0.25)\n",
        "        self.decoder = nn.Linear(j, num_question)\n",
        "\n",
        "    def get_weight_norm(self):\n",
        "        \"\"\" Return ||W^1||^2 + ||W^2||^2.\n",
        "\n",
        "        :return: float\n",
        "        \"\"\"\n",
        "        g_w_norm = torch.norm(self.encoder.weight, 2) ** 2\n",
        "        h_w_norm = torch.norm(self.decoder.weight, 2) ** 2\n",
        "        return g_w_norm + h_w_norm\n",
        "\n",
        "    def forward(self, inputs):\n",
        "        \"\"\" Return a forward pass given inputs.\n",
        "\n",
        "        :param inputs: user vector.\n",
        "        :return: user vector.\n",
        "        \"\"\"\n",
        "        a = torch.sigmoid(self.encoder(inputs))\n",
        "        b = F.relu(self.layer1(a))\n",
        "        c = F.relu(self.layer2(b))\n",
        "        d = F.relu(self.dropout(c))\n",
        "        out = torch.sigmoid(self.decoder(d))\n",
        "\n",
        "        return out"
      ],
      "metadata": {
        "id": "1nL-s8FHA_1x"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "source": [
        "Training Function"
      ],
      "metadata": {
        "id": "wtReHjmlA44N"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "def train(model, lr, lamb, train_matrix, zero_train_data, train_data, valid_data, num_epoch):\n",
        "    \"\"\" Train the neural network, where the objective also includes\n",
        "    a regularizer.\n",
        "\n",
        "    :param model: Module\n",
        "    :param lr: float\n",
        "    :param lamb: float\n",
        "    :param train_data: 2D FloatTensor\n",
        "    :param zero_train_data: 2D FloatTensor\n",
        "    :param valid_data: Dict\n",
        "    :param num_epoch: int\n",
        "    :return: None\n",
        "    \"\"\"\n",
        "    norm = model.get_weight_norm()\n",
        "    norm = norm.detach().numpy()\n",
        "\n",
        "    # Tell PyTorch you are training the model.\n",
        "    model.train()\n",
        "\n",
        "    # Define optimizers and loss function.\n",
        "    optimizer = optim.SGD(model.parameters(), lr=lr, momentum=0.1)\n",
        "    num_student = train_matrix.shape[0]\n",
        "\n",
        "    previous_acc = 0\n",
        "    patience = 3\n",
        "    count = 0\n",
        "\n",
        "    eps = []\n",
        "    train_accs = []\n",
        "    val_accs = []\n",
        "    train_losses = []\n",
        "\n",
        "    for epoch in range(0, num_epoch):\n",
        "        train_loss = 0.\n",
        "\n",
        "        for user_id in range(num_student):\n",
        "            inputs = Variable(zero_train_data[user_id]).unsqueeze(0)\n",
        "            target = inputs.clone()\n",
        "\n",
        "            optimizer.zero_grad()\n",
        "            output = model(inputs)\n",
        "\n",
        "            # Mask the target to only compute the gradient of valid entries.\n",
        "            nan_mask = np.isnan(train_matrix[user_id].unsqueeze(0).numpy())\n",
        "            target[0][nan_mask] = output[0][nan_mask]\n",
        "\n",
        "            loss = torch.sum((output - target) ** 2.) + lamb*norm/2\n",
        "            loss.backward()\n",
        "\n",
        "            train_loss += loss.item()\n",
        "            optimizer.step()\n",
        "        train_acc = evaluate(model,zero_train_data, train_data)\n",
        "        valid_acc = evaluate(model, zero_train_data, valid_data)\n",
        "        print(\"Epoch: {} \\tTraining Cost: {:.6f}\\t \"\n",
        "              \"Valid Acc: {}\".format(epoch, train_loss, valid_acc))\n",
        "\n",
        "        eps.append(epoch)\n",
        "        train_losses.append(train_loss)\n",
        "        train_accs.append(train_acc)\n",
        "        val_accs.append(val_acc)\n",
        "\n",
        "        # Early stopping\n",
        "        if valid_acc <= previous_acc:\n",
        "            count += 1\n",
        "            if count >= patience:\n",
        "                break\n",
        "\n",
        "        elif valid_acc > previous_acc:  #to modify patience to 3 consecutive declining iterations\n",
        "            count = 0\n",
        "\n",
        "        previous_acc = valid_acc\n",
        "\n",
        "          #plotting\n",
        "    plt.title(\"Training Loss vs. Epochs\")\n",
        "    plt.plot(eps, train_losses, label=\"Training Curve\")\n",
        "    plt.xlabel(\"Epochs\")\n",
        "    plt.ylabel(\"Training Loss\")\n",
        "    plt.show()\n",
        "\n",
        "    plt.title(\"Accuracy vs. Epochs\")\n",
        "    plt.plot(eps, train_accs, label=\"Training Curve\")\n",
        "    plt.plot(eps, val_accs, label=\"Validation Curve\")\n",
        "    plt.xlabel(\"Epochs\")\n",
        "    plt.ylabel(\"Accuracy\")\n",
        "    plt.legend(loc='best')\n",
        "    plt.show()"
      ],
      "metadata": {
        "id": "ziEUm_QcpB0R"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "source": [
        "Accuracy Evaluation Function"
      ],
      "metadata": {
        "id": "vQSjeTkI7TWC"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "def evaluate(model, train_data, valid_data):\n",
        "    \"\"\" Evaluate the valid_data on the current model.\n",
        "\n",
        "    :param model: Module\n",
        "    :param train_data: 2D FloatTensor\n",
        "    :param valid_data: A dictionary {user_id: list,\n",
        "    question_id: list, is_correct: list}\n",
        "    :return: float\n",
        "    \"\"\"\n",
        "    # Tell PyTorch you are evaluating the model.\n",
        "    model.eval()\n",
        "\n",
        "    total = 0\n",
        "    correct = 0\n",
        "\n",
        "    for i, u in enumerate(valid_data[\"user_id\"]):\n",
        "        inputs = Variable(train_data[u]).unsqueeze(0)\n",
        "        output = model(inputs)\n",
        "\n",
        "        guess = output[0][valid_data[\"question_id\"][i]].item() >= 0.5\n",
        "        if guess == valid_data[\"is_correct\"][i]:\n",
        "            correct += 1\n",
        "        total += 1\n",
        "    return correct / float(total)"
      ],
      "metadata": {
        "id": "nbs7AI7A7Tqw"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "source": [
        "Main Function"
      ],
      "metadata": {
        "id": "Nt6Q_jLt7LMu"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "def main():\n",
        "    zero_train_matrix, train_matrix, train_data, valid_data, test_data = load_data()\n",
        "\n",
        "    # Set model hyperparameters.\n",
        "    k = 50\n",
        "    j = 75\n",
        "    num_questions = zero_train_matrix.shape[1]\n",
        "    model = AutoEncoder(num_question=num_questions, k=k, j=j)\n",
        "\n",
        "    # Set optimization hyperparameters.\n",
        "    lr = 0.01\n",
        "    num_epoch = 40\n",
        "    lamb = 0.001\n",
        "\n",
        "    train(model, lr, lamb, train_matrix, zero_train_matrix, train_data,\n",
        "          valid_data, num_epoch)\n",
        "    # train(model, lr, lamb, train_matrix, zero_train_matrix,\n",
        "    #       test_data, num_epoch)\n",
        "\n",
        "\n",
        "if __name__ == \"__main__\":\n",
        "    main()"
      ],
      "metadata": {
        "id": "lLIcYqq97PDK"
      },
      "execution_count": null,
      "outputs": []
    }
  ]
}