batch_size/batch_size_comparisons.ipynb

{
  "cells": [
    {
      "cell_type": "code",
      "metadata": {
        "id": "A_XowC2mjrfA",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "!pip install -U tf-nightly==2.4.0.dev20200703\n",
        "OUTPUT_DIR = 'output/'"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "0jOyqZR4D1E2",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "import tensorflow as tf\n",
        "\n",
        "from tensorflow import keras\n",
        "from tensorflow.keras import layers\n",
        "\n",
        "from PHNetworks.PHOptimizer import PortHamiltonianOptimizer as PHOpt\n",
        "\n",
        "from tf_plot_callback import Plotter\n",
        "import matplotlib.pyplot as pyplot"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "vGtPfa-0D1k1",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "# Load MNIST handwritten digits dataset\n",
        "(x_train, y_train), (x_test, y_test) = keras.datasets.mnist.load_data()\n",
        "\n",
        "# Prescale pixel byte brightness to float in range [0, 1]\n",
        "x_train, x_test = x_train / 255.0, x_test / 255.0\n",
        "\n",
        "# Convert integer category to activation of the output neurons\n",
        "y_train = keras.utils.to_categorical(y_train, 10)\n",
        "y_test  = keras.utils.to_categorical(y_test, 10)"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "OOeIEGK6EBEk",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "def make_model():\n",
        "    keras.backend.set_floatx('float64')\n",
        "    model = keras.models.Sequential(\n",
        "        layers=[\n",
        "            # Make 28x28-entry two-dimensional input 784-entry one-dimensional\n",
        "            layers.Flatten(input_shape=(28, 28), name='input_layer'),\n",
        "            # Hidden layers: 250 nodes with Softplus activation function\n",
        "            layers.Dense(250, activation='softplus', name='hidden_layer_1'),\n",
        "            # Hidden layers: 250 nodes with Softplus activation function\n",
        "            layers.Dense(250, activation='softplus', name='hidden_layer_2'),\n",
        "            # Output layer: 10 nodes for 10 possible digits\n",
        "            layers.Dense(10, activation='softmax', name='output_layer')\n",
        "        ],\n",
        "        name='mnist_model')\n",
        "    return model"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "Xw97SUcQENCE",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "def train_test(batch_size, ivp_duration):\n",
        "    # Create Model\n",
        "    model = make_model()\n",
        "    model.compile(loss=keras.losses.CategoricalCrossentropy(), metrics=['accuracy'])\n",
        "\n",
        "    # Create a new optimizer instance\n",
        "    optimizer = PHOpt(ivp_period=ivp_duration, ivp_step_size=0.05, alpha=1.2, beta=0.25, gamma=1.5, resistive=0.65)\n",
        "\n",
        "    # Initialize a plotter instance, jump through some hoops to\n",
        "    # gain access to protected properties of the optimizer\n",
        "    optimizer._check_model_and_state(model)\n",
        "    harry = Plotter(optimizer.ivp_period)\n",
        "\n",
        "    print(f'BEGINNING TRAINING WITH batch_size={batch_size} AND ivp_duration={ivp_duration}')\n",
        "\n",
        "    # Fit the model\n",
        "    optimizer.train(\n",
        "        model, x_train, y_train,\n",
        "        batch_size=batch_size,\n",
        "        metrics=[keras.metrics.CategoricalAccuracy()],\n",
        "        callbacks=[\n",
        "            harry\n",
        "        ],\n",
        "        epochs=3,\n",
        "        verbose=True\n",
        "    )\n",
        "\n",
        "    _, accuracy = model.evaluate(x_test, y_test, verbose=False)\n",
        "\n",
        "    fig = harry.plot()\n",
        "\n",
        "    fig.axes[1].set_title(f'Trainingsverlauf (batch_size = {batch_size}, $t^*$ = {ivp_duration})')\n",
        "\n",
        "    name = f'batch-size-comparison_size={batch_size}_duration={ivp_duration}_accuracy={accuracy}.pdf'\n",
        "    fig.savefig(OUTPUT_DIR + name)\n",
        "\n",
        "    pyplot.close(fig)\n",
        "\n",
        "    return accuracy\n"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "R5sZazq8XYoz",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 278
        },
        "outputId": "032c10fb-85d8-4843-dd2f-33f57dc8345a"
      },
      "source": [
        "BATCH_SIZES = [60000, 10000, 1000, 250, 50, 20, 1]\n",
        "IVP_PERIODS = [0.1, 0.5, 2.0, 5.0, 20.0, 50.0]\n",
        "\n",
        "for bs in BATCH_SIZES:\n",
        "    for ivp_period in IVP_PERIODS:\n",
        "      accuracy = train_test(bs, ivp_period)\n",
        "      print(f'Reached accuracy of {accuracy} with batch_size={bs} and ivp_duration={ivp_period}')"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "mis9l0VAptk_",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "batch_size = 1\n",
        "ivp_duration = 0.05\n",
        "\n",
        "# Create Model\n",
        "model = make_model()\n",
        "model.compile(loss=keras.losses.CategoricalCrossentropy(), metrics=['accuracy'])\n",
        "\n",
        "# Create a new optimizer instance\n",
        "optimizer = PHOpt(ivp_period=ivp_duration, ivp_step_size=0.01, alpha=1.2, beta=0.25, gamma=1.5, resistive=0.65)\n",
        "\n",
        "# Initialize a plotter instance, jump through some hoops to\n",
        "# gain access to protected properties of the optimizer\n",
        "optimizer._check_model_and_state(model)\n",
        "harry = Plotter(optimizer.ivp_period)\n",
        "\n",
        "print(f'BEGINNING TRAINING WITH batch_size={batch_size} AND ivp_duration={ivp_duration}')\n",
        "\n",
        "# Fit the model\n",
        "optimizer.train(\n",
        "    model, x_train, y_train,\n",
        "    batch_size=batch_size,\n",
        "    metrics=[keras.metrics.CategoricalAccuracy()],\n",
        "    callbacks=[\n",
        "        harry\n",
        "    ],\n",
        "    epochs=1,\n",
        "    verbose=True\n",
        ")\n",
        "\n",
        "_, accuracy = model.evaluate(x_test, y_test, verbose=False)\n",
        "\n",
        "fig = harry.plot()\n",
        "\n",
        "fig.axes[1].set_title(f'Trainingsverlauf (batch_size = {batch_size}, $t^*$ = {ivp_duration})')\n",
        "\n",
        "name = f'batch-size-comparison_size={batch_size}_duration={ivp_duration}_accuracy={accuracy}.pdf'\n",
        "fig.savefig(OUTPUT_DIR + name)\n",
        "\n",
        "pyplot.close(fig)\n",
        "\n",
        "print(f'Reached accuracy of {accuracy} with batch_size={batch_size} and ivp_duration={ivp_duration}')"
      ],
      "execution_count": null,
      "outputs": [
        "BEGINNING TRAINING WITH batch_size=1 AND ivp_duration=0.05\n",
        "60000/60000 [==============================] - 3138 s 69ms/step - loss: 14.3119 - energy: 120.7871 - categorical_accuracy: 0.0992",
        "Reached accuracy of 0."
      ]
    }
  ],
  "metadata": {
    "colab": {
      "name": "batch_size_comparisons.ipynb",
      "provenance": [],
      "collapsed_sections": []
    },
    "kernelspec": {
      "name": "python3",
      "display_name": "Python 3"
    },
    "accelerator": "GPU"
  },
  "nbformat": 4,
  "nbformat_minor": 0
}