architecture_evaluation.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 34
    },
    "colab_type": "code",
    "id": "hhvKZvwx4Xpo",
    "outputId": "0e0b4a87-4e29-4361-ed04-6b4476806919"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Num GPUs Available:  1\n"
     ]
    }
   ],
   "source": [
    "import tensorflow as tf\n",
    "\n",
    "print(\"Num GPUs Available: \", len(tf.config.experimental.list_physical_devices('GPU')))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 1000
    },
    "colab_type": "code",
    "id": "GW9NtoXt3mle",
    "outputId": "b13d0f0f-c820-4017-af15-0755d02b3ee0"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "COUN: Training 31 networks\n",
      ">>> run,l_cnt,layers,p_cnt,acc,time\n",
      "DESC: Training model 000_784-10_7850\n",
      "TIME: Training took 10.52s\n",
      "ACCU: Reached accuracy of 90.70%\n",
      ">>> 0,1,784-10,7850,90.70%,10.52s\n",
      "DESC: Training model 001_784-50-10_39760\n",
      "TIME: Training took 10.84s\n",
      "ACCU: Reached accuracy of 89.86%\n",
      ">>> 1,2,784-50-10,39760,89.86%,10.84s\n",
      "DESC: Training model 002_784-50-50-10_42310\n",
      "TIME: Training took 11.38s\n",
      "ACCU: Reached accuracy of 79.92%\n",
      ">>> 2,3,784-50-50-10,42310,79.92%,11.38s\n",
      "DESC: Training model 003_784-250-10_198760\n",
      "TIME: Training took 17.12s\n",
      "ACCU: Reached accuracy of 90.15%\n",
      ">>> 3,2,784-250-10,198760,90.15%,17.12s\n",
      "DESC: Training model 004_784-250-50-10_209310\n",
      "TIME: Training took 18.09s\n",
      "ACCU: Reached accuracy of 85.50%\n",
      ">>> 4,3,784-250-50-10,209310,85.50%,18.09s\n",
      "DESC: Training model 005_784-250-50-50-10_211860\n",
      "TIME: Training took 18.61s\n",
      "ACCU: Reached accuracy of 39.59%\n",
      ">>> 5,4,784-250-50-50-10,211860,39.59%,18.61s\n",
      "DESC: Training model 006_784-500-10_397510\n",
      "TIME: Training took 25.00s\n",
      "ACCU: Reached accuracy of 90.12%\n",
      ">>> 6,2,784-500-10,397510,90.12%,25.00s\n",
      "DESC: Training model 007_784-250-250-10_261510\n",
      "TIME: Training took 21.45s\n",
      "ACCU: Reached accuracy of 88.04%\n",
      ">>> 7,3,784-250-250-10,261510,88.04%,21.45s\n",
      "DESC: Training model 008_784-500-50-10_418060\n",
      "TIME: Training took 26.48s\n",
      "ACCU: Reached accuracy of 87.33%\n",
      ">>> 8,3,784-500-50-10,418060,87.33%,26.48s\n",
      "DESC: Training model 009_784-250-250-50-10_272060\n",
      "TIME: Training took 22.47s\n",
      "ACCU: Reached accuracy of 39.65%\n",
      ">>> 9,4,784-250-250-50-10,272060,39.65%,22.47s\n",
      "DESC: Training model 010_784-500-50-50-10_420610\n",
      "TIME: Training took 26.82s\n",
      "ACCU: Reached accuracy of 53.10%\n",
      ">>> 10,4,784-500-50-50-10,420610,53.10%,26.82s\n",
      "DESC: Training model 011_784-500-250-10_520260\n",
      "TIME: Training took 32.36s\n",
      "ACCU: Reached accuracy of 87.63%\n",
      ">>> 11,3,784-500-250-10,520260,87.63%,32.36s\n",
      "DESC: Training model 012_784-500-250-50-10_530810\n",
      "TIME: Training took 33.12s\n",
      "ACCU: Reached accuracy of 56.21%\n",
      ">>> 12,4,784-500-250-50-10,530810,56.21%,33.12s\n",
      "DESC: Training model 013_784-1000-10_795010\n",
      "TIME: Training took 41.11s\n",
      "ACCU: Reached accuracy of 90.31%\n",
      ">>> 13,2,784-1000-10,795010,90.31%,41.11s\n",
      "DESC: Training model 014_784-500-500-10_648010\n",
      "TIME: Training took 39.48s\n",
      "ACCU: Reached accuracy of 88.20%\n",
      ">>> 14,3,784-500-500-10,648010,88.20%,39.48s\n",
      "DESC: Training model 015_784-500-250-250-10_583010\n",
      "TIME: Training took 36.91s\n",
      "ACCU: Reached accuracy of 44.65%\n",
      ">>> 15,4,784-500-250-250-10,583010,44.65%,36.91s\n",
      "DESC: Training model 016_784-1000-50-10_835560\n",
      "TIME: Training took 43.33s\n",
      "ACCU: Reached accuracy of 86.95%\n",
      ">>> 16,3,784-1000-50-10,835560,86.95%,43.33s\n",
      "DESC: Training model 017_784-500-500-50-10_668560\n",
      "TIME: Training took 41.06s\n",
      "ACCU: Reached accuracy of 63.12%\n",
      ">>> 17,4,784-500-500-50-10,668560,63.12%,41.06s\n",
      "DESC: Training model 018_784-1000-50-50-10_838110\n",
      "TIME: Training took 43.50s\n",
      "ACCU: Reached accuracy of 50.68%\n",
      ">>> 18,4,784-1000-50-50-10,838110,50.68%,43.50s\n",
      "DESC: Training model 019_784-1000-250-10_1037760\n",
      "TIME: Training took 54.04s\n",
      "ACCU: Reached accuracy of 88.65%\n",
      ">>> 19,3,784-1000-250-10,1037760,88.65%,54.04s\n",
      "DESC: Training model 020_784-500-500-250-10_770760\n",
      "TIME: Training took 47.25s\n",
      "ACCU: Reached accuracy of 64.59%\n",
      ">>> 20,4,784-500-500-250-10,770760,64.59%,47.25s\n",
      "DESC: Training model 021_784-1000-250-50-10_1048310\n",
      "TIME: Training took 55.03s\n",
      "ACCU: Reached accuracy of 54.11%\n",
      ">>> 21,4,784-1000-250-50-10,1048310,54.11%,55.03s\n",
      "DESC: Training model 022_784-1000-500-10_1290510\n",
      "TIME: Training took 69.39s\n",
      "ACCU: Reached accuracy of 88.77%\n",
      ">>> 22,3,784-1000-500-10,1290510,88.77%,69.39s\n",
      "DESC: Training model 023_784-1000-250-250-10_1100510\n",
      "TIME: Training took 58.91s\n",
      "ACCU: Reached accuracy of 48.86%\n",
      ">>> 23,4,784-1000-250-250-10,1100510,48.86%,58.91s\n",
      "DESC: Training model 024_784-1000-500-50-10_1311060\n",
      "TIME: Training took 73.23s\n",
      "ACCU: Reached accuracy of 57.55%\n",
      ">>> 24,4,784-1000-500-50-10,1311060,57.55%,73.23s\n",
      "DESC: Training model 025_784-1000-500-250-10_1413260\n",
      "TIME: Training took 79.70s\n",
      "ACCU: Reached accuracy of 66.51%\n",
      ">>> 25,4,784-1000-500-250-10,1413260,66.51%,79.70s\n",
      "DESC: Training model 026_784-1000-1000-10_1796010\n",
      "TIME: Training took 101.69s\n",
      "ACCU: Reached accuracy of 88.81%\n",
      ">>> 26,3,784-1000-1000-10,1796010,88.81%,101.69s\n",
      "DESC: Training model 027_784-1000-500-500-10_1541010\n",
      "TIME: Training took 89.45s\n",
      "ACCU: Reached accuracy of 68.44%\n",
      ">>> 27,4,784-1000-500-500-10,1541010,68.44%,89.45s\n",
      "DESC: Training model 028_784-1000-1000-50-10_1836560\n",
      "TIME: Training took 102.64s\n",
      "ACCU: Reached accuracy of 61.75%\n",
      ">>> 28,4,784-1000-1000-50-10,1836560,61.75%,102.64s\n",
      "DESC: Training model 029_784-1000-1000-250-10_2038760\n",
      "TIME: Training took 112.65s\n",
      "ACCU: Reached accuracy of 71.18%\n",
      ">>> 29,4,784-1000-1000-250-10,2038760,71.18%,112.65s\n",
      "DESC: Training model 030_784-1000-1000-500-10_2291510\n",
      "TIME: Training took 131.04s\n",
      "ACCU: Reached accuracy of 65.56%\n",
      ">>> 30,4,784-1000-1000-500-10,2291510,65.56%,131.04s\n"
     ]
    }
   ],
   "source": [
    "#!/usr/bin/env python3\n",
    "\n",
    "import itertools\n",
    "from timeit import default_timer as timer\n",
    "\n",
    "import tensorflow as tf\n",
    "from tensorflow import keras\n",
    "from tensorflow.keras import layers\n",
    "from keras.utils.layer_utils import count_params\n",
    "\n",
    "from PHNetworks.PHOptimizer import PortHamiltonianOptimizer as PHOpt\n",
    "\n",
    "\n",
    "DEPTHS = [0, 1, 2, 3]\n",
    "LAYER_SIZES = [1000, 500, 250, 50]\n",
    "EPOCHS = 2\n",
    "IVP_PERIOD = 20.0  # Divided by EPOCHS at initialization of optimizer\n",
    "IVP_STEP_SIZE = 0.20\n",
    "BATCH_SIZE = 15000\n",
    "START_WITH_LARGEST = False\n",
    "\n",
    "#########################################\n",
    "# MODEL-CREATION-SPECIFIC FUNCTIONALITY #\n",
    "#########################################\n",
    "def get_layer_sizes(depths):\n",
    "    all_descending_layer_sizes = itertools.chain(*[\n",
    "            itertools.combinations_with_replacement(LAYER_SIZES, r) for r in depths\n",
    "    ])\n",
    "    return all_descending_layer_sizes\n",
    "\n",
    "\n",
    "def make_model(layer_sizes):\n",
    "    keras.backend.set_floatx('float64')\n",
    "    model = keras.models.Sequential(name='mnist_model')\n",
    "    \n",
    "    # Make 28x28-entry two-dimensional input 784-entry one-dimensional\n",
    "    model.add(layers.Flatten(input_shape=(28, 28), name='input_layer'))\n",
    "    \n",
    "    # Add hidden layers with respective layer sizes\n",
    "    for i, size in enumerate(layer_sizes, start=1):\n",
    "        model.add(layers.Dropout(0.25))\n",
    "        model.add(layers.Dense(size, activation='sigmoid', name=f'hidden_layer_{i}'))\n",
    "    \n",
    "    # Output layer: 10 nodes for 10 possible digits\n",
    "    model.add(layers.Dense(10, activation='sigmoid', name='output_layer'))\n",
    "\n",
    "    # Compile the model with additional info\n",
    "    model.compile(loss=keras.losses.CategoricalCrossentropy(), metrics=['accuracy'])\n",
    "    return model\n",
    "\n",
    "\n",
    "def has_repetitions(iter, count=3):\n",
    "    last = None\n",
    "    reps = 0\n",
    "    for item in iter:\n",
    "        if item == last:\n",
    "            reps += 1\n",
    "            if reps >= count:\n",
    "                return True\n",
    "        else:\n",
    "            last = item\n",
    "            reps = 1\n",
    "    return False\n",
    "\n",
    "\n",
    "#################################\n",
    "# MODEL TRAINING AND EVALUATION #\n",
    "#################################\n",
    "# Load MNIST handwritten digits dataset\n",
    "(x_train, y_train), (x_test, y_test) = keras.datasets.mnist.load_data()\n",
    "\n",
    "\n",
    "# Prescale pixel byte brightness to float in range [0, 1]\n",
    "x_train, x_test = x_train / 255.0, x_test / 255.0\n",
    "\n",
    "# Convert integer category to activation of the output neurons\n",
    "y_train = keras.utils.to_categorical(y_train, 10)\n",
    "y_test  = keras.utils.to_categorical(y_test, 10)\n",
    "\n",
    "\n",
    "def train_and_evaluate(sizes, id=None):\n",
    "    model = make_model(sizes)\n",
    "\n",
    "    label = f'{id:03d}_784-{\"\".join([f\"{size}-\" for size in sizes])}10_{count_params(model.trainable_weights)}'\n",
    "    logdir = f'arch_eval_logs/{label}'\n",
    "    \n",
    "    optimizer = PHOpt(alpha=2.0, beta=0.05, gamma=1.65, resistive=0.65, ivp_period=IVP_PERIOD/EPOCHS, ivp_step_size=IVP_STEP_SIZE)\n",
    "    \n",
    "    print(f'DESC: Training model {label}')\n",
    "    start = timer()\n",
    "    \n",
    "    with tf.device('/gpu:0'):\n",
    "        optimizer.train(\n",
    "            model, x_train, y_train,\n",
    "            batch_size=BATCH_SIZE,\n",
    "            metrics=[keras.metrics.CategoricalAccuracy()],\n",
    "            callbacks=[\n",
    "            #    tf.keras.callbacks.TensorBoard(logdir, update_freq=4, profile_batch=0)\n",
    "            ],\n",
    "            epochs=EPOCHS\n",
    "        )\n",
    "    \n",
    "    end = timer()\n",
    "    print(f'TIME: Training took {end - start:.2f}s')\n",
    "    \n",
    "    _, accuracy = model.evaluate(x_test, y_test, verbose=0)\n",
    "    print(f'ACCU: Reached accuracy of {accuracy * 100:02.2f}%')\n",
    "\n",
    "    print(f'>>> {id:d},{len(sizes)+1:d},784-{\"\".join([f\"{size}-\" for size in sizes])}10,{count_params(model.trainable_weights)},{accuracy * 100:02.2f}%,{end - start:.2f}s')\n",
    "    \n",
    "    del model\n",
    "\n",
    "\n",
    "def exec():\n",
    "    layer_sizes = get_layer_sizes(DEPTHS)\n",
    "    layer_sizes = itertools.filterfalse(has_repetitions, layer_sizes)\n",
    "    layer_sizes = list(layer_sizes)\n",
    "    layer_sizes = sorted(layer_sizes, key=sum)\n",
    "    layer_sizes = list(enumerate(layer_sizes))\n",
    "    if START_WITH_LARGEST:\n",
    "        layer_sizes = list(reversed(layer_sizes))\n",
    "    \n",
    "    print(f'COUN: Training {len(layer_sizes)} networks')\n",
    "    print('>>> run,layer count,layers,param count,accuracy,time')\n",
    "    \n",
    "    for id, layer in layer_sizes:\n",
    "        train_and_evaluate(layer, id)\n",
    "\n",
    "\n",
    "if __name__ == '__main__':\n",
    "    exec()\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "1fmcnkrOqrHI"
   },
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "accelerator": "GPU",
  "colab": {
   "collapsed_sections": [],
   "name": "architecture_evaluation.ipynb",
   "provenance": []
  },
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.10"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}