Ensembled Methods v1.0.ipynb

{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "colab": {
      "provenance": []
    },
    "kernelspec": {
      "name": "python3",
      "display_name": "Python 3"
    },
    "language_info": {
      "name": "python"
    }
  },
  "cells": [
    {
      "cell_type": "markdown",
      "source": [
        "Imports"
      ],
      "metadata": {
        "id": "IVbi7xGs_cHp"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "from utils import *\n",
        "from torch.autograd import Variable\n",
        "\n",
        "import torch.nn as nn\n",
        "import torch.nn.functional as F\n",
        "import torch.optim as optim\n",
        "import torch.utils.data\n",
        "\n",
        "import numpy as np\n",
        "import torch\n",
        "\n",
        "import matplotlib.pyplot as plt"
      ],
      "metadata": {
        "id": "hLW-MkuOlyIB"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "from scipy.sparse import load_npz\n",
        "import csv\n",
        "import os"
      ],
      "metadata": {
        "id": "8BiykqzGl3t8"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "source": [
        "Initialize Autoencoders"
      ],
      "metadata": {
        "id": "k4Qw_l6I1tfE"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "class AutoEncoder(nn.Module):\n",
        "    torch.manual_seed(42)\n",
        "    def __init__(self, num_question, k=100):\n",
        "        \"\"\" Initialize a class AutoEncoder.\n",
        "\n",
        "        :param num_question: int\n",
        "        :param k: int\n",
        "        \"\"\"\n",
        "        super(AutoEncoder, self).__init__()\n",
        "\n",
        "        # Define linear functions.\n",
        "        self.encoder = nn.Linear(num_question, k) #self.g\n",
        "        self.decoder = nn.Linear(k, num_question) #self.h\n",
        "\n",
        "    def get_weight_norm(self):\n",
        "        \"\"\" Return ||W^1||^2 + ||W^2||^2.\n",
        "\n",
        "        :return: float\n",
        "        \"\"\"\n",
        "        g_w_norm = torch.norm(self.encoder.weight, 2) ** 2\n",
        "        h_w_norm = torch.norm(self.decoder.weight, 2) ** 2\n",
        "\n",
        "        return g_w_norm + h_w_norm\n",
        "\n",
        "    def forward(self, inputs):\n",
        "        \"\"\" Return a forward pass given inputs.\n",
        "\n",
        "        :param inputs: user vector.\n",
        "        :return: user vector.\n",
        "        \"\"\"\n",
        "\n",
        "        x = torch.sigmoid(self.encoder(inputs))\n",
        "        out = torch.sigmoid(self.decoder(x))\n",
        "\n",
        "        return out"
      ],
      "metadata": {
        "id": "_COOnY7o1wGI"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "source": [
        "Training Function"
      ],
      "metadata": {
        "id": "vwmqW0RY1YBI"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "def train_bagging(model, lr, lamb, train_data, zero_train_data, valid_data, num_epoch):\n",
        "    \"\"\" Train the neural network, where the objective also includes\n",
        "    a regularizer.\n",
        "\n",
        "    :param model: Module\n",
        "    :param lr: float\n",
        "    :param lamb: float\n",
        "    :param train_data: 2D FloatTensor\n",
        "    :param zero_train_data: 2D FloatTensor\n",
        "    :param valid_data: Dict\n",
        "    :param num_epoch: int\n",
        "    :return: None\n",
        "    \"\"\"\n",
        "\n",
        "    # TODO: Add a regularizer to the cost function.\n",
        "    norm = model.get_weight_norm()\n",
        "    norm = norm.detach().numpy()\n",
        "    print(\"norm = \", norm)\n",
        "\n",
        "    # Tell PyTorch you are training the model.\n",
        "    model.train()\n",
        "\n",
        "    # Define optimizers and loss function.\n",
        "    optimizer = optim.SGD(model.parameters(), lr=lr)\n",
        "    num_student = train_data.shape[0]\n",
        "\n",
        "    for epoch in range(0, num_epoch):\n",
        "        train_loss = 0.\n",
        "\n",
        "        for user_id in range(num_student):\n",
        "            inputs = Variable(zero_train_data[user_id]).unsqueeze(0)  #answers to all questions by a student\n",
        "            target = inputs.clone()\n",
        "\n",
        "            optimizer.zero_grad()\n",
        "            output = model(inputs)\n",
        "\n",
        "            # Mask the target to only compute the gradient of valid entries.\n",
        "            nan_mask = np.isnan(train_data[user_id].unsqueeze(0).numpy())\n",
        "            target[0][nan_mask] = output[0][nan_mask]\n",
        "\n",
        "            loss = torch.sum(((output - target) ** 2.)) + lamb*norm/2\n",
        "            loss.backward()\n",
        "\n",
        "            train_loss += loss.item()\n",
        "            optimizer.step()\n",
        "\n",
        "        valid_acc = evaluate(model, zero_train_data, valid_data)\n",
        "        print(\"Epoch: {} \\tTraining Cost: {:.6f}\\t \"\n",
        "              \"Valid Acc: {}\".format(epoch, train_loss, valid_acc))\n"
      ],
      "metadata": {
        "id": "iJAX1o3hrhI7"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "source": [
        "Accuracy Evaluation Function"
      ],
      "metadata": {
        "id": "z7R2QsUS1gaB"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "def evaluate_bagging(model1, model2, model3, train_data, valid_data):\n",
        "    \"\"\" Evaluate the valid_data on the current model.\n",
        "\n",
        "    :param model: Module\n",
        "    :param train_data: 2D FloatTensor\n",
        "    :param valid_data: A dictionary {user_id: list,\n",
        "    question_id: list, is_correct: list}\n",
        "    :return: float\n",
        "    \"\"\"\n",
        "    # Tell PyTorch you are evaluating the model.\n",
        "    model.eval()\n",
        "\n",
        "    total = 0\n",
        "    correct = 0\n",
        "\n",
        "    for i, u in enumerate(valid_data[\"user_id\"]):\n",
        "        inputs = Variable(train_data[u]).unsqueeze(0)\n",
        "        output1 = model1(inputs)\n",
        "        output2 = model2(inputs)\n",
        "        output3 = model3(inputs)\n",
        "\n",
        "        guess = (((output1[0][valid_data[\"question_id\"][i]].item() >= 0.5)+(output2[0][valid_data[\"question_id\"][i]].item() >= 0.5)+(output3[0][valid_data[\"question_id\"][i]].item() >= 0.5))/3) >= 0.5\n",
        "\n",
        "        if guess == valid_data[\"is_correct\"][i]:\n",
        "            correct += 1\n",
        "        total += 1\n",
        "    return correct / float(total)"
      ],
      "metadata": {
        "id": "yYJLMjqctg6g"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "source": [
        "Bootstrapping Training Data"
      ],
      "metadata": {
        "id": "DPS2fWLT1_Bk"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "#create list of randomly sorted iterables (equal to # of students)\n",
        "import random\n",
        "\n",
        "def bootstrap(train_matrix):\n",
        "  '''Input: np train_matrix\n",
        "  Returns: tensor zero_train_matrix2, train_matrix2, zero_train_matrix3,train_matrix3\n",
        "  '''\n",
        "\n",
        "  train_matrix2 = train_matrix.copy()\n",
        "  train_matrix3 = train_matrix.copy()\n",
        "\n",
        "  rand_list1 = list(range(train_matrix.shape[0]))\n",
        "  random.shuffle(rand_list1)\n",
        "  rand_list2 = list(range(train_matrix.shape[0]))\n",
        "  random.shuffle(rand_list2)\n",
        "\n",
        "  for i in range(train_matrix.shape[0]):\n",
        "    j = rand_list1[i]\n",
        "    k = rand_list2[i]\n",
        "\n",
        "    train_matrix2[i] = train_matrix[j]\n",
        "    train_matrix3[i] = train_matrix[k]\n",
        "\n",
        "  zero_train_matrix2 = train_matrix2.copy()\n",
        "  zero_train_matrix3 = train_matrix3.copy()\n",
        "\n",
        "  #Fill in the missing entries to 0.\n",
        "  zero_train_matrix2[np.isnan(train_matrix2)] = 0\n",
        "  zero_train_matrix3[np.isnan(train_matrix3)] = 0\n",
        "\n",
        "  # Change to Float Tensor for PyTorch.\n",
        "  zero_train_matrix2 = torch.FloatTensor(zero_train_matrix2)\n",
        "  train_matrix2 = torch.FloatTensor(train_matrix2)\n",
        "\n",
        "  zero_train_matrix3 = torch.FloatTensor(zero_train_matrix3)\n",
        "  train_matrix3 = torch.FloatTensor(train_matrix3)\n",
        "\n",
        "  return zero_train_matrix2, train_matrix2, zero_train_matrix3, train_matrix3"
      ],
      "metadata": {
        "id": "vdS0EBFTI4Qa"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "source": [
        "Main Functions"
      ],
      "metadata": {
        "id": "jpq5lRhy2DIh"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "if __name__ == \"__main__\":\n",
        "\n",
        "\n",
        "  zero_train_matrix1, train_matrix1, train_data, valid_data, test_data, np_train_matrix = load_data_for_bagging()\n",
        "  zero_train_matrix2, train_matrix2, zero_train_matrix3, train_matrix3 = bootstrap(np_train_matrix)\n",
        "\n",
        "\n",
        "  # Set model hyperparameters.\n",
        "  k = 50\n",
        "  num_questions = zero_train_matrix1.shape[1]\n",
        "\n",
        "  # Set optimization hyperparameters.\n",
        "  lr1 = 0.01; lr2 = 0.02; lr3 = 0.001\n",
        "  num_epoch1 = 40; num_epoch2 = 30; num_epoch3 = 40\n",
        "  lamb1 = 0.001; lamb2 = 0.001; lamb3 = 0.001\n",
        "\n",
        "  #initialize models\n",
        "  modelb1 = AutoEncoder(num_question = num_questions, k = k); modelb2 = AutoEncoder(num_question = num_questions, k = k); modelb3 = AutoEncoder(num_question = num_questions, k = k)\n",
        "\n",
        "  #train the models on the three training datasets\n",
        "  train_bagging(modelb1, lr1, lamb1, train_matrix1, zero_train_matrix1, valid_data, num_epoch1)\n",
        "  train_bagging(modelb2, lr2, lamb2, train_matrix2, zero_train_matrix2, valid_data, num_epoch2)\n",
        "  train_bagging(modelb3, lr3, lamb3, train_matrix3, zero_train_matrix3, valid_data, num_epoch3)\n",
        "\n",
        "  #make ensembled validation predictions and evaluate accuracy (only one epoch -- because we have the fully trained model)\n",
        "  val_acc_b = evaluate_bagging(modelb1, modelb2, modelb3, zero_train_matrix1,valid_data) #choice of train matrix doesn't matter because eval function takes the ztm list of user ids and makes predictions on that specific list\n",
        "\n",
        "  #make ensembled test predictions and evaluate accuracy (only one epoch -- because we have the fully trained model)\n",
        "  test_acc_b = evaluate_bagging(modelb1, modelb2, modelb3, zero_train_matrix1,test_data)\n",
        "\n",
        "  print(\"Validation Accuracy = \", val_acc_b, \"     \", \"Test Accuracy = \", test_acc_b)"
      ],
      "metadata": {
        "id": "6DQLEzUsrhTR"
      },
      "execution_count": null,
      "outputs": []
    }
  ]
}