github_FNNs_CNNs_music_genre.ipynb

{
  "cells": [
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "pzWFLo8HInnp"
      },
      "source": [
        "#**Pattern Recognition - Machine Learning** | Assignment 3\n",
        "\n",
        "\n",
        "\n"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "hKzW8Erhgxop"
      },
      "source": [
        "#**1) Music Track classification**\n",
        "\n",
        "\n",
        "\n"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "ul1mcvP7pPCU"
      },
      "outputs": [],
      "source": [
        "from google.colab import drive\n",
        "\n",
        "# Mount Google Drive\n",
        "drive.mount('/content/gdrive')"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "n2x_bL7lO2cQ"
      },
      "source": [
        "### Question 1: Feedforward Neural Network"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "hHPi9cXhPMZs"
      },
      "source": [
        "1.   Loading Data"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "UknIm9esPN_a"
      },
      "outputs": [],
      "source": [
        "import numpy as np\n",
        "import torch\n",
        "from torch.utils.data import DataLoader, TensorDataset\n",
        "\n",
        "train_data_path = \"/content/gdrive/MyDrive/data/music_genre_data_di/train/mfccs/\"\n",
        "valid_data_path = \"/content/gdrive/MyDrive/data/music_genre_data_di/val/mfccs/\"\n",
        "test_data_path = \"/content/gdrive/MyDrive/data/music_genre_data_di/test/mfccs/\"\n",
        "\n",
        "# Load MFCC data\n",
        "# train data\n",
        "train_data = np.load(train_data_path + 'X.npy')\n",
        "train_labels = np.load(train_data_path + 'labels.npy')\n",
        "\n",
        "# validation data\n",
        "valid_data = np.load(valid_data_path + 'X.npy')\n",
        "valid_labels = np.load(valid_data_path + 'labels.npy')\n",
        "\n",
        "# test data\n",
        "test_data = np.load(test_data_path + 'X.npy')\n",
        "test_labels = np.load(test_data_path + 'labels.npy')\n",
        "\n",
        "# Create label mapping\n",
        "label_mapping = {label: idx for idx, label in enumerate(np.unique(train_labels))}\n",
        "num_classes = len(label_mapping)\n",
        "\n",
        "# Map label strings to integer numbers\n",
        "train_labels = np.array([label_mapping[label] for label in train_labels])   # train\n",
        "valid_labels = np.array([label_mapping[label] for label in valid_labels])   # validation\n",
        "test_labels = np.array([label_mapping[label] for label in test_labels])     # test\n",
        "\n",
        "# Convert data and labels to PyTorch tensors\n",
        "train_data = torch.from_numpy(train_data).float()     # train\n",
        "train_labels = torch.from_numpy(train_labels).long()\n",
        "valid_data = torch.from_numpy(valid_data).float()     # validation\n",
        "valid_labels = torch.from_numpy(valid_labels).long()\n",
        "test_data = torch.from_numpy(test_data).float()       # test\n",
        "test_labels = torch.from_numpy(test_labels).long()\n",
        "\n",
        "# Create TensorDataset\n",
        "train_dataset = TensorDataset(train_data, train_labels) # train\n",
        "valid_dataset = TensorDataset(valid_data, valid_labels) # validation\n",
        "test_dataset = TensorDataset(test_data, test_labels)    # test\n",
        "\n",
        "# Create train, validation, and testing dataloaders\n",
        "batch_size = 16\n",
        "train_loader = DataLoader(train_dataset, batch_size=batch_size, shuffle=True) # train DataLoader\n",
        "valid_loader = DataLoader(valid_dataset, batch_size=batch_size, shuffle=True) # validation DataLoader\n",
        "test_loader = DataLoader(test_dataset, batch_size=batch_size)                 # test DataLoader\n"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "EDrrvYpRUT4B"
      },
      "source": [
        "2.   FCNN (Fully Connected Neural Network) initialization"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "lfu7_MY5UT_n"
      },
      "outputs": [],
      "source": [
        "import torch.nn as nn\n",
        "\n",
        "class FullyConnectedNet(nn.Module):\n",
        "    def __init__(self):\n",
        "        super(FullyConnectedNet, self).__init__()\n",
        "\n",
        "        # Definition of the layers and the neurons\n",
        "        self.fc1 = nn.Linear(26, 128)               # Input layer with 26 neurons and 128 neurons in the first layer\n",
        "        self.fc2 = nn.Linear(128, 32)               # Second hidden layer with 128 neurons and 32 neurons in the third layer\n",
        "        self.fc3 = nn.Linear(32, 4)                 # Third hidden layer with 32 neurons and 4 neurons in the output layer\n",
        "        self.relu = nn.ReLU()                       # ReLU activation function\n",
        "\n",
        "    def forward(self, x):\n",
        "        # Perform the forward pass through the network\n",
        "        x = self.relu(self.fc1(x))  # Apply ReLU activation to the output of the first layer\n",
        "        x = self.relu(self.fc2(x))  # Apply ReLU activation to the output of the second layer\n",
        "        x = self.fc3(x)             # Output layer\n",
        "        return x\n",
        "\n"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "v8iuCKAmaghK"
      },
      "source": [
        "3.   Function to train the FCNN"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "1cQJ6sXgagtq"
      },
      "outputs": [],
      "source": [
        "# function to train the NN\n",
        "def train(epochs, optimizer, dataloader, cost_function, model):\n",
        "\n",
        "    for epoch in range(epochs):\n",
        "        running_loss = 0.0\n",
        "\n",
        "        # Iterate over the batches in the dataloader\n",
        "        for batch_inputs, batch_labels in dataloader:\n",
        "            # Zero the gradients\n",
        "            optimizer.zero_grad()\n",
        "\n",
        "            # Forward pass\n",
        "            batch_outputs = model(batch_inputs)\n",
        "\n",
        "            # Calculate the loss\n",
        "            loss = cost_function(batch_outputs, batch_labels)\n",
        "\n",
        "            # Backward pass and optimization\n",
        "            loss.backward()\n",
        "            optimizer.step()\n",
        "\n",
        "            # Update the running loss\n",
        "            running_loss += loss.item()\n",
        "\n",
        "        # Print the average loss for the epoch\n",
        "        epoch_loss = running_loss / len(dataloader)\n",
        "        print(f\"Epoch [{epoch+1}/{epochs}], Loss: {epoch_loss:.4f}\")\n",
        "\n",
        "    # Return the trained model\n",
        "    return model\n"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "oXkk5w4Lf2CE"
      },
      "source": [
        "4.   Function to evaluate the trained FCNN model"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "EaA3hYDaf2JK"
      },
      "outputs": [],
      "source": [
        "from sklearn.metrics import f1_score, accuracy_score, confusion_matrix\n",
        "import time\n",
        "\n",
        "# function that evaluates the trained model\n",
        "def evaluate(model, dataloader, loss_function):\n",
        "\n",
        "    model.eval()                              # set the model to evaluation mode\n",
        "    device = next(model.parameters()).device  # get the device of the model's parameters\n",
        "    running_loss = 0.0\n",
        "    all_predictions = []\n",
        "    all_labels = []\n",
        "\n",
        "    with torch.no_grad():\n",
        "        for batch_inputs, batch_labels in dataloader:\n",
        "            batch_inputs = batch_inputs.to(device)\n",
        "            batch_labels = batch_labels.to(device)\n",
        "\n",
        "            # forward passing\n",
        "            batch_outputs = model(batch_inputs)\n",
        "\n",
        "            # loss calculation\n",
        "            loss = loss_function(batch_outputs, batch_labels)\n",
        "            running_loss += loss.item()\n",
        "\n",
        "            # convert logits to predicted labels\n",
        "            _, predicted = torch.max(batch_outputs, dim=1)\n",
        "\n",
        "            all_predictions.extend(predicted.cpu().numpy())\n",
        "            all_labels.extend(batch_labels.cpu().numpy())\n",
        "\n",
        "    # calculate metrics\n",
        "    epoch_loss = running_loss / len(dataloader)\n",
        "    f1 = f1_score(all_labels, all_predictions, average='macro')\n",
        "    accuracy = accuracy_score(all_labels, all_predictions)\n",
        "    confusion_mat = confusion_matrix(all_labels, all_predictions)\n",
        "\n",
        "    # printing evaluation metrics\n",
        "    print(f\"Loss: {epoch_loss:.4f}\")\n",
        "    print(f\"F1 Score (macro-averaged): {f1:.4f}\")\n",
        "    print(f\"Accuracy: {accuracy:.4f}\")\n",
        "    print(\"Confusion Matrix:\")\n",
        "    print(confusion_mat)\n",
        "\n",
        "    return epoch_loss, f1, accuracy, confusion_mat\n"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "W71NIiwKh26b"
      },
      "source": [
        "5.   Call train() to train the model. Then, call the evaluate() to evaluate the efficiency of the trained model."
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "YCD3-pF5h3E3"
      },
      "outputs": [],
      "source": [
        "# initialization of all input parameters for the train() function\n",
        "\n",
        "# initialize a FCNN\n",
        "model = FullyConnectedNet()\n",
        "\n",
        "# initialize the optimizer with learning rate 'lr'=0.002\n",
        "optimizer = torch.optim.SGD(model.parameters(), lr=0.002)\n",
        "\n",
        "# cost function\n",
        "cost_function = nn.CrossEntropyLoss()\n",
        "\n",
        "# epochs\n",
        "epochs = 30\n",
        "\n",
        "print(\"\\n-------------------------Model training results-------------------------\\n\")\n",
        "\n",
        "# Measure CPU time\n",
        "start_time = time.time()\n",
        "\n",
        "# acquire the trained model\n",
        "trained_model = train(epochs, optimizer, train_loader, cost_function, model)\n",
        "\n",
        "training_time = time.time() - start_time\n",
        "print(\"\\n> Training time: \", training_time, \"sec\")"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "Q3sj3LYPigOD"
      },
      "outputs": [],
      "source": [
        "# initialize the inut parameters and call the evaluate() function\n",
        "model = trained_model\n",
        "loss_function = nn.CrossEntropyLoss()\n",
        "print(\"\\n-------------------------Model evaluation results-------------------------\\n\")\n",
        "\n",
        "# Measure time\n",
        "start_time = time.time()\n",
        "\n",
        "loss, f1_macro_avg, accuracy, confusion_m = evaluate(model, test_loader, loss_function)\n",
        "\n",
        "evaluation_time = time.time() - start_time\n",
        "print(\"\\n> Evaluation time: \", evaluation_time, \"sec\")"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "AbTsTScx0rPw"
      },
      "source": [
        "6.   Train the FCNN using GPU"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "8O6PTHSiHtIU"
      },
      "source": [
        "For finding the results in this question, we first run the entire code using CPU runtime (selecting \"None\" as the hardware accelerator in Colab settings). The results we obtained for CPU runtime are as follows:"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "OgtfIEnF00r7"
      },
      "outputs": [],
      "source": [
        "# -------------------------Model training results-------------------------\n",
        "\n",
        "# Epoch [1/30], Loss: 1.3829\n",
        "# Epoch [2/30], Loss: 1.3602\n",
        "# Epoch [3/30], Loss: 1.3519\n",
        "# Epoch [4/30], Loss: 1.3405\n",
        "# Epoch [5/30], Loss: 1.3272\n",
        "# Epoch [6/30], Loss: 1.3162\n",
        "# Epoch [7/30], Loss: 1.3012\n",
        "# Epoch [8/30], Loss: 1.2848\n",
        "# Epoch [9/30], Loss: 1.2668\n",
        "# Epoch [10/30], Loss: 1.2470\n",
        "# Epoch [11/30], Loss: 1.2262\n",
        "# Epoch [12/30], Loss: 1.2080\n",
        "# Epoch [13/30], Loss: 1.1876\n",
        "# Epoch [14/30], Loss: 1.1696\n",
        "# Epoch [15/30], Loss: 1.1493\n",
        "# Epoch [16/30], Loss: 1.1335\n",
        "# Epoch [17/30], Loss: 1.1194\n",
        "# Epoch [18/30], Loss: 1.1036\n",
        "# Epoch [19/30], Loss: 1.0866\n",
        "# Epoch [20/30], Loss: 1.0762\n",
        "# Epoch [21/30], Loss: 1.0637\n",
        "# Epoch [22/30], Loss: 1.0532\n",
        "# Epoch [23/30], Loss: 1.0403\n",
        "# Epoch [24/30], Loss: 1.0315\n",
        "# Epoch [25/30], Loss: 1.0216\n",
        "# Epoch [26/30], Loss: 1.0106\n",
        "# Epoch [27/30], Loss: 1.0005\n",
        "# Epoch [28/30], Loss: 0.9882\n",
        "# Epoch [29/30], Loss: 0.9900\n",
        "# Epoch [30/30], Loss: 0.9794\n",
        "\n",
        "# > Training time:  7.532282829284668 sec\n",
        "\n",
        "\n",
        "# -------------------------Model evaluation results-------------------------\n",
        "\n",
        "# Loss: 1.0728\n",
        "# F1 Score (macro-averaged): 0.5322\n",
        "# Accuracy: 0.5392\n",
        "# Confusion Matrix:\n",
        "# [[112  91  52  69]\n",
        "#  [ 19 271   5   2]\n",
        "#  [ 93  54 194  15]\n",
        "#  [112  84  38 165]]\n",
        "\n",
        "# > Evaluation time:  0.06703901290893555 sec"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "qBC5QULfMluq"
      },
      "source": [
        "Next, we execute the code using GPU runtime, selecting \"GPU\" as the hardware accelerator in Colab settings and \"A100\". We utilize the paid version of GColab Pro, as I also use it for professional purposes. The results for GPU A100 runtime are:"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "mwmJTzMkM4Mz"
      },
      "outputs": [],
      "source": [
        "# -------------------------Model training results-------------------------\n",
        "\n",
        "# Epoch [1/30], Loss: 1.3786\n",
        "# Epoch [2/30], Loss: 1.3666\n",
        "# Epoch [3/30], Loss: 1.3587\n",
        "# Epoch [4/30], Loss: 1.3506\n",
        "# Epoch [5/30], Loss: 1.3410\n",
        "# Epoch [6/30], Loss: 1.3303\n",
        "# Epoch [7/30], Loss: 1.3173\n",
        "# Epoch [8/30], Loss: 1.3028\n",
        "# Epoch [9/30], Loss: 1.2871\n",
        "# Epoch [10/30], Loss: 1.2701\n",
        "# Epoch [11/30], Loss: 1.2517\n",
        "# Epoch [12/30], Loss: 1.2317\n",
        "# Epoch [13/30], Loss: 1.2114\n",
        "# Epoch [14/30], Loss: 1.1908\n",
        "# Epoch [15/30], Loss: 1.1711\n",
        "# Epoch [16/30], Loss: 1.1550\n",
        "# Epoch [17/30], Loss: 1.1364\n",
        "# Epoch [18/30], Loss: 1.1215\n",
        "# Epoch [19/30], Loss: 1.1103\n",
        "# Epoch [20/30], Loss: 1.0960\n",
        "# Epoch [21/30], Loss: 1.0869\n",
        "# Epoch [22/30], Loss: 1.0765\n",
        "# Epoch [23/30], Loss: 1.0651\n",
        "# Epoch [24/30], Loss: 1.0560\n",
        "# Epoch [25/30], Loss: 1.0455\n",
        "# Epoch [26/30], Loss: 1.0377\n",
        "# Epoch [27/30], Loss: 1.0326\n",
        "# Epoch [28/30], Loss: 1.0232\n",
        "# Epoch [29/30], Loss: 1.0089\n",
        "# Epoch [30/30], Loss: 1.0032\n",
        "\n",
        "# > Training time:  5.595116376876831 sec\n",
        "\n",
        "\n",
        "# -------------------------Model evaluation results-------------------------\n",
        "\n",
        "# Loss: 1.0313\n",
        "# F1 Score (macro-averaged): 0.5502\n",
        "# Accuracy: 0.6025\n",
        "# Confusion Matrix:\n",
        "# [[ 23  21 168 112]\n",
        "#  [ 29 210  33  25]\n",
        "#  [ 10  12 312  22]\n",
        "#  [ 21  18  76 284]]\n",
        "\n",
        "# > Evaluation time:  0.05098462104797363 sec"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "fa8fEZ7DNyv5"
      },
      "source": [
        "Indeed, we can observe that GPU times are better than CPU times, which is theoretically expected since GPUs operate more efficiently in tasks such as model training, neural network training and evaluation, and parallel computing tasks due to their different hardware architecture in general."
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "3jG_HAWDWB2H"
      },
      "source": [
        "7.   Find the best instance of the FCNN through the epochs"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "mDxpgGBYWKkX"
      },
      "outputs": [],
      "source": [
        "# function to train the NN and find the best model instance based on F1 metric\n",
        "def train_and_find_best_instance(epochs, optimizer, train_loader, valid_loader, cost_function, model):\n",
        "\n",
        "    best_f1 = 0.0\n",
        "    best_epoch = 0\n",
        "    f1_best_instance = None\n",
        "\n",
        "    for epoch in range(epochs):\n",
        "        running_loss = 0.0\n",
        "\n",
        "        # Training phase\n",
        "        model.train()\n",
        "        for batch_inputs, batch_labels in train_loader:\n",
        "            optimizer.zero_grad()\n",
        "            batch_outputs = model(batch_inputs)\n",
        "            loss = cost_function(batch_outputs, batch_labels)\n",
        "            loss.backward()\n",
        "            optimizer.step()\n",
        "            running_loss += loss.item()\n",
        "\n",
        "        # Calculate average training loss\n",
        "        epoch_loss = running_loss / len(train_loader)\n",
        "        print(f\"Epoch [{epoch+1}/{epochs}], Training Loss: {epoch_loss:.4f}\")\n",
        "\n",
        "        # Evaluation phase\n",
        "        model.eval()\n",
        "        with torch.no_grad():\n",
        "            all_predictions = []\n",
        "            all_labels = []\n",
        "\n",
        "            for batch_inputs, batch_labels in valid_loader:\n",
        "                batch_outputs = model(batch_inputs)\n",
        "                _, predicted = torch.max(batch_outputs, dim=1)\n",
        "\n",
        "                all_predictions.extend(predicted.tolist())\n",
        "                all_labels.extend(batch_labels.tolist())\n",
        "\n",
        "            # Calculate evaluation metrics\n",
        "            f1 = f1_score(all_labels, all_predictions, average='macro')\n",
        "\n",
        "            # Print F1 score\n",
        "            print(f\"F1 Score (macro-averaged): {f1:.4f}\\n\")\n",
        "\n",
        "            # Check if current model instance has the best F1 score\n",
        "            if f1 > best_f1:\n",
        "                best_f1 = f1\n",
        "                best_epoch = epoch + 1\n",
        "                f1_best_instance = model.state_dict().copy()\n",
        "\n",
        "    # Load the best model instance\n",
        "    model.load_state_dict(f1_best_instance)\n",
        "\n",
        "    # Print the epoch with the best model instance\n",
        "    print(f\"> The best model instance is from epoch {best_epoch}\\n\")\n",
        "\n",
        "    # Return the best model instance\n",
        "    return model, best_epoch\n"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "gmZukVnTX7Z_"
      },
      "outputs": [],
      "source": [
        "# initialization of all input parameters for the train() function\n",
        "\n",
        "# initialize a FCNN\n",
        "model = FullyConnectedNet()\n",
        "\n",
        "# initialize the optimizer with learning rate 'lr'=0.002\n",
        "optimizer = torch.optim.SGD(model.parameters(), lr=0.002)\n",
        "\n",
        "# cost function\n",
        "cost_function = nn.CrossEntropyLoss()\n",
        "\n",
        "# epochs\n",
        "epochs = 30\n",
        "\n",
        "print(\"\\n-------------------------Model training results and best model instance computation-------------------------\\n\")\n",
        "\n",
        "# Measure CPU time\n",
        "start_time = time.time()\n",
        "\n",
        "# acquire the BEST trained model based on the 'f1' metric\n",
        "best_trained_model, best_epoch = train_and_find_best_instance(epochs, optimizer, train_loader, valid_loader, cost_function, model)\n",
        "\n",
        "training_time = time.time() - start_time\n",
        "print(\"> Training and best instance search time: \", training_time, \"sec\")"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "_h7a3Q2Ia5DR"
      },
      "source": [
        "* Δοκιμή του ανωτέρω μοντέλου στο test dataset και εξαγωγή συμπερασμάτων/μετρικών"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "25JA2BxqbBdi"
      },
      "outputs": [],
      "source": [
        "# initialization of parameters and\n",
        "\n",
        "loss_function = nn.CrossEntropyLoss()\n",
        "print(\"\\n-------------------------Model evaluation results-------------------------\\n\")\n",
        "\n",
        "# Measure time\n",
        "start_time = time.time()\n",
        "\n",
        "loss, f1_macro_avg, accuracy, confusion_m = evaluate(best_trained_model, test_loader, loss_function)\n",
        "\n",
        "evaluation_time = time.time() - start_time\n",
        "print(f\"\\n> Evaluation time of the best model instance (training epoch {best_epoch}): \", evaluation_time, \"sec\")"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "8Wk3kebrdK5S"
      },
      "source": [
        "The model instance that we found to be the best in terms of the 'f1' metric during the training process is expected to be the most efficient in terms of this metric when applied to the test dataset."
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "HLDBbAkheJkW"
      },
      "source": [
        "### Question 2: Convolutional Neural Network"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "idd5M76BeTdk"
      },
      "source": [
        "1.   Data loading for spectrograms"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "YH1tgUzjgyqP"
      },
      "outputs": [],
      "source": [
        "# for executing melgrams code separate from MFCCs\n",
        "\n",
        "from google.colab import drive\n",
        "\n",
        "# Mount Google Drive\n",
        "drive.mount('/content/gdrive')"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "KMdgYA63eYzm"
      },
      "outputs": [],
      "source": [
        "import numpy as np\n",
        "import torch\n",
        "from torch.utils.data import DataLoader, TensorDataset\n",
        "\n",
        "mel_train_data_path = \"/content/gdrive/MyDrive/data/music_genre_data_di/train/melgrams/\"\n",
        "mel_valid_data_path = \"/content/gdrive/MyDrive/data/music_genre_data_di/val/melgrams/\"\n",
        "mel_test_data_path = \"/content/gdrive/MyDrive/data/music_genre_data_di/test/melgrams/\"\n",
        "\n",
        "# Load melgarms data\n",
        "# train data\n",
        "mel_train_data = np.load(mel_train_data_path + 'X.npy')\n",
        "mel_train_labels = np.load(mel_train_data_path + 'labels.npy')\n",
        "\n",
        "# validation data\n",
        "mel_valid_data = np.load(mel_valid_data_path + 'X.npy')\n",
        "mel_valid_labels = np.load(mel_valid_data_path + 'labels.npy')\n",
        "\n",
        "# test data\n",
        "mel_test_data = np.load(mel_test_data_path + 'X.npy')\n",
        "mel_test_labels = np.load(mel_test_data_path + 'labels.npy')\n",
        "\n",
        "# Create label mapping\n",
        "mel_label_mapping = {label: idx for idx, label in enumerate(np.unique(mel_train_labels))}\n",
        "mel_num_classes = len(mel_label_mapping)\n",
        "\n",
        "# Map label strings to integer numbers\n",
        "mel_train_labels = np.array([mel_label_mapping[label] for label in mel_train_labels])   # train\n",
        "mel_valid_labels = np.array([mel_label_mapping[label] for label in mel_valid_labels])   # validation\n",
        "mel_test_labels = np.array([mel_label_mapping[label] for label in mel_test_labels])     # test\n",
        "\n",
        "# Convert data and labels to PyTorch tensors\n",
        "mel_train_data = torch.from_numpy(mel_train_data).float()     # train\n",
        "mel_train_labels = torch.from_numpy(mel_train_labels).long()\n",
        "mel_valid_data = torch.from_numpy(mel_valid_data).float()     # validation\n",
        "mel_valid_labels = torch.from_numpy(mel_valid_labels).long()\n",
        "mel_test_data = torch.from_numpy(mel_test_data).float()       # test\n",
        "mel_test_labels = torch.from_numpy(mel_test_labels).long()\n",
        "\n",
        "# Create TensorDataset\n",
        "mel_train_dataset = TensorDataset(mel_train_data, mel_train_labels) # train\n",
        "mel_valid_dataset = TensorDataset(mel_valid_data, mel_valid_labels) # validation\n",
        "mel_test_dataset = TensorDataset(mel_test_data, mel_test_labels)    # test\n",
        "\n",
        "# Create train, validation, and testing dataloaders\n",
        "batch_size = 16\n",
        "mel_train_loader = DataLoader(mel_train_dataset, batch_size=batch_size, shuffle=True) # train DataLoader\n",
        "mel_valid_loader = DataLoader(mel_valid_dataset, batch_size=batch_size, shuffle=True) # validation DataLoader\n",
        "mel_test_loader = DataLoader(mel_test_dataset, batch_size=batch_size)                 # test DataLoader\n",
        "\n",
        "# melgram visualization\n",
        "import random\n",
        "import matplotlib.pyplot as plt\n",
        "\n",
        "# Function to visualize random melgrams\n",
        "def visualize_random_melgrams(dataloader, label_mapping):\n",
        "    fig, axs = plt.subplots(nrows=len(label_mapping), ncols=1, figsize=(8, 8))\n",
        "    fig.tight_layout(pad=2.0)\n",
        "\n",
        "    # Create a list to keep track of visited labels\n",
        "    visited_labels = []\n",
        "\n",
        "    # Iterate over the dataloader\n",
        "    for batch_inputs, batch_labels in dataloader:\n",
        "        for melgram, label in zip(batch_inputs, batch_labels):\n",
        "            if label.item() not in visited_labels:\n",
        "                class_name = list(label_mapping.keys())[list(label_mapping.values()).index(label.item())]\n",
        "                axs[label].imshow(melgram.squeeze(), cmap='jet')\n",
        "                axs[label].set_title(class_name)\n",
        "                axs[label].set_xlabel('Time')\n",
        "                axs[label].set_ylabel('Frequency')\n",
        "                axs[label].axis('off')\n",
        "                visited_labels.append(label.item())\n",
        "            if len(visited_labels) == len(label_mapping):\n",
        "                break\n",
        "        if len(visited_labels) == len(label_mapping):\n",
        "            break\n",
        "\n",
        "    plt.show()\n",
        "\n",
        "# Visualize random melgrams by calling visualization function\n",
        "print(\"\\n---------------------------------Random class track visualization---------------------------------\\n\")\n",
        "visualize_random_melgrams(mel_train_loader, mel_label_mapping)"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "KShHbrgehaJX"
      },
      "source": [
        "2.   CNN (Convolutional Neural Network) initialization\n",
        "\n"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "eJlX0Wmshidx"
      },
      "outputs": [],
      "source": [
        "import torch.nn as nn\n",
        "\n",
        "class CNN_FCNN(nn.Module):\n",
        "    def __init__(self, in_channels, out_dim):   # we can set 'in_channels'=1 and 'out_dim'=any dimension we want\n",
        "        super(CNN_FCNN, self).__init__()\n",
        "\n",
        "        # Convolutional layers\n",
        "        self.conv1 = nn.Conv2d(in_channels, 16, kernel_size=5)\n",
        "        self.conv2 = nn.Conv2d(16, 32, kernel_size=5)\n",
        "        self.conv3 = nn.Conv2d(32, 64, kernel_size=5)\n",
        "        self.conv4 = nn.Conv2d(64, 128, kernel_size=5)\n",
        "\n",
        "        # Fully connected layers\n",
        "        self.fc1 = nn.Linear(71680, 1024)   # output dimension of the convolutional layers is 71680\n",
        "        self.fc2 = nn.Linear(1024, 256)\n",
        "        self.fc3 = nn.Linear(256, 32)\n",
        "        self.fc4 = nn.Linear(32, out_dim)\n",
        "\n",
        "        self.relu = nn.ReLU()\n",
        "\n",
        "    def forward(self, x):\n",
        "        # Convolutional layers\n",
        "        x = self.relu(self.conv1(x))\n",
        "        x = self.relu(self.conv2(x))\n",
        "        x = self.relu(self.conv3(x))\n",
        "        x = self.relu(self.conv4(x))\n",
        "\n",
        "        # Flatten the tensor for the fully connected layers\n",
        "        x = x.view(x.size(0), -1)\n",
        "\n",
        "        # Fully connected layers\n",
        "        x = self.relu(self.fc1(x))\n",
        "        x = self.relu(self.fc2(x))\n",
        "        x = self.relu(self.fc3(x))\n",
        "        x = self.fc4(x)\n",
        "\n",
        "        return x\n"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "nitjNewIkYQf"
      },
      "source": [
        "3.   Training and evaluation procedure of the CNN"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "yNWPjxJYkYW5"
      },
      "outputs": [],
      "source": [
        "from sklearn.metrics import f1_score\n",
        "\n",
        "# Function to train the CNN and find the best model instance based on F1 metric\n",
        "def train_and_find_best_CNN(epochs, optimizer, train_loader, valid_loader, cost_function, model):\n",
        "\n",
        "    best_f1 = 0.0\n",
        "    best_epoch = 0\n",
        "    device = next(model.parameters()).device  # get the device of the model's parameters\n",
        "    f1_best_instance = None\n",
        "\n",
        "    for epoch in range(epochs):\n",
        "        running_loss = 0.0\n",
        "\n",
        "        # Training phase\n",
        "        model.train()\n",
        "        for batch_inputs, batch_labels in train_loader:\n",
        "            optimizer.zero_grad()\n",
        "            batch_inputs = batch_inputs.unsqueeze(1).to(device)  # Adjust input tensor shape\n",
        "            batch_labels = batch_labels.to(device)\n",
        "            batch_outputs = model(batch_inputs)\n",
        "            loss = cost_function(batch_outputs, batch_labels)\n",
        "            loss.backward()\n",
        "            optimizer.step()\n",
        "            running_loss += loss.item()\n",
        "\n",
        "        # Calculate average training loss\n",
        "        epoch_loss = running_loss / len(train_loader)\n",
        "        print(f\"Epoch [{epoch+1}/{epochs}], Training Loss: {epoch_loss:.4f}\")\n",
        "\n",
        "        # Evaluation phase\n",
        "        model.eval()\n",
        "        with torch.no_grad():\n",
        "            all_predictions = []\n",
        "            all_labels = []\n",
        "\n",
        "            for batch_inputs, batch_labels in valid_loader:\n",
        "                batch_inputs = batch_inputs.unsqueeze(1).to(device)  # Adjust input tensor shape\n",
        "                batch_labels = batch_labels.to(device)\n",
        "                batch_outputs = model(batch_inputs)\n",
        "                _, predicted = torch.max(batch_outputs, dim=1)\n",
        "\n",
        "                all_predictions.extend(predicted.tolist())\n",
        "                all_labels.extend(batch_labels.tolist())\n",
        "\n",
        "            # Calculate evaluation metrics\n",
        "            f1 = f1_score(all_labels, all_predictions, average='macro')\n",
        "\n",
        "            # Print F1 score\n",
        "            print(f\"F1 Score (macro-averaged): {f1:.4f}\\n\")\n",
        "\n",
        "            # Check if current model instance has the best F1 score\n",
        "            if f1 > best_f1:\n",
        "                best_f1 = f1\n",
        "                best_epoch = epoch + 1\n",
        "                f1_best_instance = model.state_dict().copy()\n",
        "\n",
        "    # Load the best model instance\n",
        "    model.load_state_dict(f1_best_instance)\n",
        "\n",
        "    # Print the epoch with the best model instance\n",
        "    print(f\"> The best CNN model instance is from epoch {best_epoch}\\n\")\n",
        "\n",
        "    # Return the best model instance and the best epoch\n",
        "    return model, best_epoch\n"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "akjGpz4nmcuu"
      },
      "outputs": [],
      "source": [
        "from sklearn.metrics import f1_score, accuracy_score, confusion_matrix\n",
        "\n",
        "# Function that evaluates the trained CNN model\n",
        "def evaluate_CNN(model, dataloader, loss_function):\n",
        "\n",
        "    model.eval()                              # Set the model to evaluation mode\n",
        "    device = next(model.parameters()).device  # Get the device of the model's parameters\n",
        "    running_loss = 0.0\n",
        "    all_predictions = []\n",
        "    all_labels = []\n",
        "\n",
        "    with torch.no_grad():\n",
        "        for batch_inputs, batch_labels in dataloader:\n",
        "            batch_inputs = batch_inputs.unsqueeze(1).to(device)\n",
        "            batch_labels = batch_labels.to(device)\n",
        "\n",
        "            # Forward pass\n",
        "            batch_outputs = model(batch_inputs)\n",
        "\n",
        "            # Loss calculation\n",
        "            loss = loss_function(batch_outputs, batch_labels)\n",
        "            running_loss += loss.item()\n",
        "\n",
        "            # Convert logits to predicted labels\n",
        "            _, predicted = torch.max(batch_outputs, dim=1)\n",
        "\n",
        "            all_predictions.extend(predicted.cpu().numpy())\n",
        "            all_labels.extend(batch_labels.cpu().numpy())\n",
        "\n",
        "    # Calculate metrics\n",
        "    epoch_loss = running_loss / len(dataloader)\n",
        "    f1 = f1_score(all_labels, all_predictions, average='macro')\n",
        "    accuracy = accuracy_score(all_labels, all_predictions)\n",
        "    confusion_mat = confusion_matrix(all_labels, all_predictions)\n",
        "\n",
        "    # Print evaluation metrics\n",
        "    print(f\"Loss: {epoch_loss:.4f}\")\n",
        "    print(f\"F1 Score (macro-averaged): {f1:.4f}\")\n",
        "    print(f\"Accuracy: {accuracy:.4f}\")\n",
        "    print(\"Confusion Matrix:\")\n",
        "    print(confusion_mat)\n",
        "\n",
        "    return epoch_loss, f1, accuracy, confusion_mat\n"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "P50LOqispmJM"
      },
      "source": [
        "Code to test run the above functions:"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "RvGG98tXpq0N"
      },
      "outputs": [],
      "source": [
        "# Import the required modules\n",
        "import torch.nn as nn\n",
        "import time\n",
        "\n",
        "# Initialize a CNN model\n",
        "model = CNN_FCNN(in_channels=1, out_dim=16)\n",
        "\n",
        "# Initialize the optimizer with learning rate 'lr'=0.002\n",
        "optimizer = torch.optim.SGD(model.parameters(), lr=0.002)\n",
        "\n",
        "# Cost function\n",
        "cost_function = nn.CrossEntropyLoss()\n",
        "\n",
        "# Epochs\n",
        "epochs = 30\n",
        "\n",
        "print(\"\\n-------------------------CNN training results and best model instance computation-------------------------\\n\")\n",
        "\n",
        "# Measure CPU time\n",
        "start_time = time.time()\n",
        "\n",
        "# Acquire the BEST trained model based on the 'f1' metric\n",
        "best_CNN_trained_model, CNN_best_epoch = train_and_find_best_CNN(epochs, optimizer, mel_train_loader, mel_valid_loader, cost_function, model)\n",
        "\n",
        "training_time = time.time() - start_time\n",
        "print(\"> CNN training and best instance search time: \", training_time, \"sec\")\n"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "XnxAzKmI-rED"
      },
      "source": [
        "We observe variation and unpredictable fluctuations in the values of the training loss and 'f1' metric, despite the fact that as the epochs progress, the parameters should improve. This inconsistency can be attributed to various factors, such as the initialization method of the model, the lack of data normalization, or the small quantity of input data, among others."
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "X_Zu4Svjpq7g"
      },
      "outputs": [],
      "source": [
        "# initialization of parameters to find the best trained CNN instance through the epochs\n",
        "\n",
        "loss_function = nn.CrossEntropyLoss()\n",
        "print(\"\\n-------------------------Best CNN model instance evaluation results on test data-------------------------\\n\")\n",
        "\n",
        "# Measure time\n",
        "start_time = time.time()\n",
        "\n",
        "loss, f1_macro_avg, accuracy, confusion_m = evaluate_CNN(best_CNN_trained_model, mel_test_loader, loss_function)\n",
        "\n",
        "evaluation_time = time.time() - start_time\n",
        "print(f\"\\n> Evaluation time of the best CNN model instance (training epoch {CNN_best_epoch}): \", evaluation_time, \"sec\")"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "SXGM_w4A_k0c"
      },
      "source": [
        "\n",
        "\n",
        "4.   Add pooling and padding in the data\n",
        "\n",
        "\n",
        "\n"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "JZHODLyCAJWh"
      },
      "source": [
        "We create a new initialization method for our CNN to compare the previous execution logic with the current one."
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "8_QjKyAs_rbl"
      },
      "outputs": [],
      "source": [
        "import torch\n",
        "import torch.nn as nn\n",
        "\n",
        "class new_CNN_FCNN(nn.Module):\n",
        "    def __init__(self, in_channels, out_dim):\n",
        "        super(new_CNN_FCNN, self).__init__()\n",
        "\n",
        "        # Convolutional layers\n",
        "        self.conv1 = nn.Sequential(\n",
        "            nn.Conv2d(in_channels, 16, kernel_size=5, padding=2),\n",
        "            nn.ReLU(),\n",
        "            nn.MaxPool2d(kernel_size=2, padding=1)\n",
        "        )\n",
        "        self.conv2 = nn.Sequential(\n",
        "            nn.Conv2d(16, 32, kernel_size=5, padding=2),\n",
        "            nn.ReLU(),\n",
        "            nn.MaxPool2d(kernel_size=2, padding=1)\n",
        "        )\n",
        "        self.conv3 = nn.Sequential(\n",
        "            nn.Conv2d(32, 64, kernel_size=5, padding=2),\n",
        "            nn.ReLU(),\n",
        "            nn.MaxPool2d(kernel_size=2, padding=1)\n",
        "        )\n",
        "        self.conv4 = nn.Sequential(\n",
        "            nn.Conv2d(64, 128, kernel_size=5, padding=2),\n",
        "            nn.ReLU(),\n",
        "            nn.MaxPool2d(kernel_size=2, padding=1)\n",
        "        )\n",
        "\n",
        "        # Fully connected layers\n",
        "        self.fc1 = nn.Linear(3456, 1024)   # Input size is 3456 after failed testings and debugging\n",
        "        self.fc2 = nn.Linear(1024, 256)\n",
        "        self.fc3 = nn.Linear(256, 32)\n",
        "        self.fc4 = nn.Linear(32, out_dim)\n",
        "\n",
        "        self.relu = nn.ReLU()\n",
        "\n",
        "    def forward(self, x):\n",
        "        # Convolutional layers\n",
        "        x = self.conv1(x)\n",
        "        x = self.conv2(x)\n",
        "        x = self.conv3(x)\n",
        "        x = self.conv4(x)\n",
        "\n",
        "        # Flatten the tensor for the fully connected layers\n",
        "        x = x.view(x.size(0), -1)\n",
        "\n",
        "        # Fully connected layers\n",
        "        x = self.relu(self.fc1(x))\n",
        "        x = self.relu(self.fc2(x))\n",
        "        x = self.relu(self.fc3(x))\n",
        "        x = self.fc4(x)\n",
        "\n",
        "        return x\n",
        "\n"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "Ipc749eQCUkS"
      },
      "outputs": [],
      "source": [
        "# Import the required modules\n",
        "import torch.nn as nn\n",
        "import time\n",
        "\n",
        "# Initialize a CNN model\n",
        "model = new_CNN_FCNN(in_channels=1, out_dim=16)\n",
        "\n",
        "# Initialize the optimizer with learning rate 'lr'=0.002\n",
        "optimizer = torch.optim.SGD(model.parameters(), lr=0.002)\n",
        "\n",
        "# Cost function\n",
        "cost_function = nn.CrossEntropyLoss()\n",
        "\n",
        "# Epochs\n",
        "epochs = 30\n",
        "\n",
        "print(\"\\n-------------------------New CNN (+ pooling,padding) training results and best model instance computation-------------------------\\n\")\n",
        "\n",
        "# Measure CPU time\n",
        "start_time = time.time()\n",
        "\n",
        "# Acquire the BEST trained model based on the 'f1' metric\n",
        "best_new_CNN_trained_model, new_CNN_best_epoch = train_and_find_best_CNN(epochs, optimizer, mel_train_loader, mel_valid_loader, cost_function, model)\n",
        "\n",
        "training_time = time.time() - start_time\n",
        "print(\"> CNN training and best instance search time: \", training_time, \"sec\")\n"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "doL6hl1TkOqw"
      },
      "source": [
        "* The new model (after adding padding and max pooling) is trained more efficiently than the previous \"simpler\" version. When comparing the two best-trained models from both cases, we observe better performance in the new CNN model.\n",
        "* Additionally, if we observe the training epochs, we notice that the simple CNN model trains relatively quickly in the first 4-5 epochs, as expected. However, in the subsequent epochs, it exhibits a behavior of stagnation/recycling within a range of values (as if it reaches a \"dead end\"). In contrast, the new CNN, after adding padding and max pooling, gradually trains with increasing performance, which is more expected, without displaying a similar \"stagnation.\"\n",
        "* The padding parameter helps in training the network by creating boundary pixels within the data, which act as \"sealed\" borders with other data regions. This allows the model to process separated data more effectively and produce fewer errors.\n",
        "* Max pooling is a method of dividing the data into non-overlapping data pools (e.g., 2x2 square regions in a 16x16 data matrix). The \"max\" property means that the maximum data value (e.g., numerical or based on another convention of the problem) is extracted from each pool/region. The purpose of pooling is to reduce the dimensions of the problem while retaining the dominant data components. This achieves better computational and time complexity."
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "SnnYiWVpBo5-"
      },
      "outputs": [],
      "source": [
        "# initialization of parameters and evaluation of the best trained CNN instance on the test data\n",
        "\n",
        "loss_function = nn.CrossEntropyLoss()\n",
        "print(\"\\n-------------------------New CNN (+ pooling,padding) Model evaluation results on test data-------------------------\\n\")\n",
        "\n",
        "# Measure time\n",
        "start_time = time.time()\n",
        "\n",
        "loss, f1_macro_avg, accuracy, confusion_m = evaluate_CNN(best_new_CNN_trained_model, mel_test_loader, loss_function)\n",
        "\n",
        "evaluation_time = time.time() - start_time\n",
        "print(f\"\\n> Evaluation time of the best model instance (+ pooling,padding) (training epoch {new_CNN_best_epoch}): \", evaluation_time, \"sec\")"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "RDLIeY44odH1"
      },
      "source": [
        "5.   Optimization algorithms"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "QnReobTIqAeS"
      },
      "source": [
        "The final results table includes the performances of the best trained instance for each optimizer category in terms of 'f1' score on the test data, following the same reasoning as the previous sub-questions."
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "ktj_j5ljo46p"
      },
      "outputs": [],
      "source": [
        "import torch.optim as optim\n",
        "\n",
        "# Try a list of optimizers\n",
        "optimizers = [\n",
        "    optim.SGD,\n",
        "    optim.Adam,\n",
        "    optim.RMSprop,\n",
        "    optim.Adadelta,\n",
        "    optim.Adagrad,\n",
        "    optim.Adamax,\n",
        "    optim.ASGD,\n",
        "    optim.Rprop,\n",
        "]\n",
        "\n",
        "# Create an empty matrix to store the results\n",
        "results_matrix = torch.zeros((2, len(optimizers)))\n",
        "\n",
        "# Iterate over the optimizers\n",
        "for i, optimizer_class in enumerate(optimizers):\n",
        "\n",
        "    # Create an instance of the CNN model\n",
        "    model = model = new_CNN_FCNN(in_channels=1, out_dim=16)\n",
        "\n",
        "    # Define other hyperparameters and settings\n",
        "    learning_rate = 0.002\n",
        "    # less epochs for faster computation\n",
        "    epochs = 20\n",
        "\n",
        "    # Define the optimizer based on the optimizer class\n",
        "    optimizer = optimizer_class(model.parameters(), lr=learning_rate)\n",
        "\n",
        "    # Define the loss function\n",
        "    cost_function = nn.CrossEntropyLoss()\n",
        "\n",
        "    print(f\"\\n> Optimizer {optimizer_class} ------------New CNN (+ pooling,padding) training results and best model instance computation---------------\\n\")\n",
        "\n",
        "    # Measure CPU time\n",
        "    start_time = time.time()\n",
        "\n",
        "    # Acquire the BEST trained model based on the 'f1' metric\n",
        "    best_new_CNN_trained_model, new_CNN_best_epoch = train_and_find_best_CNN(epochs, optimizer, mel_train_loader, mel_valid_loader, cost_function, model)\n",
        "\n",
        "    training_time = time.time() - start_time\n",
        "    print(\"> CNN training and best instance search time: \", training_time, \"sec\")\n",
        "\n",
        "    # initialization of parameters and evaluation of the best trained CNN instance on the test data\n",
        "\n",
        "    loss_function = nn.CrossEntropyLoss()\n",
        "    print(f\"\\n> Optimizer {optimizer_class} ------------New CNN (+ pooling,padding) Model evaluation results on test data----------------\\n\")\n",
        "\n",
        "    # Measure time\n",
        "    start_time = time.time()\n",
        "\n",
        "    loss, f1_macro_avg, accuracy, confusion_m = evaluate_CNN(best_new_CNN_trained_model, mel_test_loader, loss_function)\n",
        "\n",
        "    evaluation_time = time.time() - start_time\n",
        "    print(f\"\\n> Evaluation time of the best model instance (+ pooling,padding) (training epoch {new_CNN_best_epoch}): \", evaluation_time, \"sec\")\n",
        "    # printing to separate sections\n",
        "    print(\"##################################################################################################################################\\n\")\n",
        "    # Store the metrics in the results matrix\n",
        "    results_matrix[0, i] = accuracy\n",
        "    results_matrix[1, i] = f1_macro_avg\n",
        "\n",
        "# Print the results matrix\n",
        "\n",
        "print(\"Results Matrix:\")\n",
        "print(\"          \", end=\"\")\n",
        "for i, optimizer_class in enumerate(optimizers):\n",
        "    print(f\"{optimizer_class}\", end=\" \")\n",
        "print()\n",
        "\n",
        "print(\"accuracy | \", end=\"        \")\n",
        "for i in range(len(optimizers)):\n",
        "    print(f\"{results_matrix[0, i]:.4f}\", end=\"                             \")\n",
        "print()\n",
        "\n",
        "print(\"f1       | \", end=\"        \")\n",
        "for i in range(len(optimizers)):\n",
        "    print(f\"{results_matrix[1, i]:.4f}\", end=\"                             \")\n",
        "print()\n"
      ]
    }
  ],
  "metadata": {
    "accelerator": "GPU",
    "colab": {
      "machine_shape": "hm",
      "provenance": []
    },
    "kernelspec": {
      "display_name": "Python 3",
      "name": "python3"
    },
    "language_info": {
      "name": "python"
    }
  },
  "nbformat": 4,
  "nbformat_minor": 0
}