assignment3.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Deep Learning\n",
    "## Assignment 3\n",
    "Previously in assignment2.ipynb, we trained a logistic regression and a neural network model.\n",
    "The goal of this assignment is to explore regularization techniques."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "from __future__ import print_function\n",
    "import numpy as np\n",
    "import tensorflow as tf\n",
    "import matplotlib.pyplot as plt\n",
    "from six.moves import cPickle as pickle\n",
    "\n",
    "%matplotlib inline"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "First reload the data we generated in assignment1.ipynb."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Training set (200000, 28, 28) (200000,)\n",
      "Validation set (10000, 28, 28) (10000,)\n",
      "Test set (10000, 28, 28) (10000,)\n"
     ]
    }
   ],
   "source": [
    "pickle_file = 'notMNIST.pickle'\n",
    "\n",
    "with open(pickle_file, 'rb') as f:\n",
    "    save = pickle.load(f)\n",
    "    train_dataset = save['train_dataset']\n",
    "    train_labels = save['train_labels']\n",
    "    valid_dataset = save['valid_dataset']\n",
    "    valid_labels = save['valid_labels']\n",
    "    test_dataset = save['test_dataset']\n",
    "    test_labels = save['test_labels']\n",
    "    del save  # hint to help gc free up memory\n",
    "    print('Training set', train_dataset.shape, train_labels.shape)\n",
    "    print('Validation set', valid_dataset.shape, valid_labels.shape)\n",
    "    print('Test set', test_dataset.shape, test_labels.shape)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Reformat into a shape that's more adapted to the models we're going to train:\n",
    "data as a flat matrix,\n",
    "labels as float 1-hot encodings."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Training set (200000, 784) (200000, 10)\n",
      "Validation set (10000, 784) (10000, 10)\n",
      "Test set (10000, 784) (10000, 10)\n"
     ]
    }
   ],
   "source": [
    "IMAGE_SIZE = 28\n",
    "IMAGE_PIXELS = IMAGE_SIZE * IMAGE_SIZE\n",
    "NUM_LABELS = 10\n",
    "\n",
    "def reformat(dataset, labels):\n",
    "    dataset = dataset.reshape((-1, IMAGE_PIXELS)).astype(np.float32)\n",
    "    # Map 0 to [1.0, 0.0, 0.0 ...], 1 to [0.0, 1.0, 0.0 ...]\n",
    "    labels = (np.arange(NUM_LABELS) == labels[:,None]).astype(np.float32)\n",
    "    return dataset, labels\n",
    "train_dataset, train_labels = reformat(train_dataset, train_labels)\n",
    "valid_dataset, valid_labels = reformat(valid_dataset, valid_labels)\n",
    "test_dataset, test_labels = reformat(test_dataset, test_labels)\n",
    "print('Training set', train_dataset.shape, train_labels.shape)\n",
    "print('Validation set', valid_dataset.shape, valid_labels.shape)\n",
    "print('Test set', test_dataset.shape, test_labels.shape)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def accuracy(predictions, labels):\n",
    "    return (100.0 * np.sum(np.argmax(predictions, 1) == np.argmax(labels, 1)) / predictions.shape[0])"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Problem 1\n",
    "Let's introduce and tune L2 regularization for both logistic and neural network models. L2 amounts to adding a penalty on the norm of the weights to the loss. In TensorFlow, we can compute the L2 loss for a tensor `t` using `nn.l2_loss(t)`. The right amount of regularization should improve our validation / test accuracy."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# With gradient descent training, even this much data is prohibitive.\n",
    "# Subset the training data for faster turnaround.\n",
    "train_subset = 10000\n",
    "l2_beta = 0.01\n",
    "\n",
    "graph = tf.Graph()\n",
    "with graph.as_default():\n",
    "\n",
    "    # Input data.\n",
    "    # Load the training, validation and test data into constants that are attached to the graph.\n",
    "    tf_train_dataset = tf.constant(train_dataset[:train_subset, :])\n",
    "    tf_train_labels = tf.constant(train_labels[:train_subset])\n",
    "    tf_valid_dataset = tf.constant(valid_dataset)\n",
    "    tf_test_dataset = tf.constant(test_dataset)\n",
    "  \n",
    "    # Variables.\n",
    "    # These are the parameters that we are going to be training. The weight matrix will be initialized \n",
    "    # using random values following a (truncated) normal distribution. The biases get initialized to zero.\n",
    "    weights = tf.Variable(tf.truncated_normal([IMAGE_PIXELS, NUM_LABELS],\n",
    "                                              stddev=1.0/np.sqrt(float(IMAGE_PIXELS))))\n",
    "    biases = tf.Variable(tf.zeros([NUM_LABELS]))\n",
    "  \n",
    "    # Training computation.\n",
    "    # We multiply the inputs with the weight matrix, and add biases. We compute the softmax and cross-entropy\n",
    "    # (it's one operation in TensorFlow, because it's very common, and it can be optimized). We take the average\n",
    "    # of this cross-entropy across all training examples: that's our loss.\n",
    "    logits = tf.matmul(tf_train_dataset, weights) + biases\n",
    "    loss = tf.reduce_mean(tf.nn.softmax_cross_entropy_with_logits(labels=tf_train_labels, logits=logits)) \\\n",
    "            + l2_beta * tf.nn.l2_loss(weights)\n",
    "  \n",
    "    # Optimizer.\n",
    "    # We are going to find the minimum of this loss using gradient descent.\n",
    "    optimizer = tf.train.GradientDescentOptimizer(0.5).minimize(loss)\n",
    "  \n",
    "    # Predictions for the training, validation, and test data.\n",
    "    # These are not part of training, but merely here so that we can report accuracy figures as we train.\n",
    "    train_prediction = tf.nn.softmax(logits)\n",
    "    valid_prediction = tf.nn.softmax(tf.matmul(tf_valid_dataset, weights) + biases)\n",
    "    test_prediction = tf.nn.softmax(tf.matmul(tf_test_dataset, weights) + biases)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Let's run this computation and iterate:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Initialized\n",
      "Loss at step 0: 2.323459\n",
      "Training accuracy: 14.9%\n",
      "Validation accuracy: 65.1%\n",
      "Loss at step 100: 0.708942\n",
      "Training accuracy: 83.8%\n",
      "Validation accuracy: 81.7%\n",
      "Loss at step 200: 0.688805\n",
      "Training accuracy: 84.5%\n",
      "Validation accuracy: 81.9%\n",
      "Loss at step 300: 0.682661\n",
      "Training accuracy: 84.7%\n",
      "Validation accuracy: 82.0%\n",
      "Loss at step 400: 0.680448\n",
      "Training accuracy: 84.7%\n",
      "Validation accuracy: 82.0%\n",
      "Loss at step 500: 0.679551\n",
      "Training accuracy: 84.8%\n",
      "Validation accuracy: 82.1%\n",
      "Loss at step 600: 0.679143\n",
      "Training accuracy: 84.8%\n",
      "Validation accuracy: 82.1%\n",
      "Loss at step 700: 0.678935\n",
      "Training accuracy: 84.9%\n",
      "Validation accuracy: 82.1%\n",
      "Loss at step 800: 0.678816\n",
      "Training accuracy: 84.9%\n",
      "Validation accuracy: 82.1%\n",
      "Test accuracy: 88.8%\n"
     ]
    }
   ],
   "source": [
    "num_steps = 801\n",
    "\n",
    "def accuracy(predictions, labels):\n",
    "    return (100.0 * np.sum(np.argmax(predictions, 1) == np.argmax(labels, 1))\n",
    "          / predictions.shape[0])\n",
    "\n",
    "with tf.Session(graph=graph) as session:\n",
    "    # This is a one-time operation which ensures the parameters get initialized as\n",
    "    # we described in the graph: random weights for the matrix, zeros for the biases. \n",
    "    tf.global_variables_initializer().run()\n",
    "    print('Initialized')\n",
    "    for step in range(num_steps):\n",
    "        # Run the computations. We tell .run() that we want to run the optimizer,\n",
    "        # and get the loss value and the training predictions returned as numpy arrays.\n",
    "        _, l, predictions = session.run([optimizer, loss, train_prediction])\n",
    "        if (step % 100 == 0):\n",
    "            print('Loss at step %d: %f' % (step, l))\n",
    "            print('Training accuracy: %.1f%%' % accuracy(predictions, train_labels[:train_subset, :]))\n",
    "            # Calling .eval() on valid_prediction is basically like calling run(), but\n",
    "            # just to get that one numpy array. Note that it recomputes all its graph\n",
    "            # dependencies.\n",
    "            print('Validation accuracy: %.1f%%' % accuracy(valid_prediction.eval(), valid_labels))\n",
    "    print('Test accuracy: %.1f%%' % accuracy(test_prediction.eval(), test_labels))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Let's now switch to stochastic gradient descent training instead, which is much faster.\n",
    "The graph will be similar, except that instead of holding all the training data into a constant node, we create a Placeholder node which will be fed actual data at every call of `session.run()`."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "BATCH_SIZE = 128\n",
    "HIDDEN_NODES = 1024\n",
    "\n",
    "graph = tf.Graph()\n",
    "with graph.as_default():\n",
    "\n",
    "    # Input data. For the training data, we use a placeholder that will be fed\n",
    "    # at run time with a training minibatch.\n",
    "    tf_train_dataset = tf.placeholder(tf.float32, shape=(BATCH_SIZE, IMAGE_PIXELS))\n",
    "    tf_train_labels = tf.placeholder(tf.float32, shape=(BATCH_SIZE, NUM_LABELS))\n",
    "    tf_valid_dataset = tf.constant(valid_dataset)\n",
    "    tf_test_dataset = tf.constant(test_dataset)\n",
    "    l2_beta = tf.placeholder(tf.float32)  # factor for regularization penalty\n",
    "  \n",
    "    # Variables.\n",
    "    w1 = tf.Variable(tf.truncated_normal([IMAGE_PIXELS, HIDDEN_NODES],\n",
    "                                         stddev=1.0/np.sqrt(float(IMAGE_PIXELS))))\n",
    "    b1 = tf.Variable(tf.zeros([HIDDEN_NODES]))\n",
    "    w2 = tf.Variable(tf.truncated_normal([HIDDEN_NODES, NUM_LABELS],\n",
    "                                         stddev=1.0/np.sqrt(float(IMAGE_PIXELS))))\n",
    "    b2 = tf.Variable(tf.zeros([NUM_LABELS]))\n",
    "  \n",
    "    # Training computation.\n",
    "    hidden1 = tf.nn.relu(tf.matmul(tf_train_dataset, w1) + b1)\n",
    "    logits = tf.matmul(hidden1, w2) + b2\n",
    "    loss = tf.reduce_mean(tf.nn.softmax_cross_entropy_with_logits(labels=tf_train_labels, logits=logits)) \\\n",
    "            + l2_beta * (tf.nn.l2_loss(w1) + tf.nn.l2_loss(w2))\n",
    "  \n",
    "    # Optimizer. \n",
    "    optimizer = tf.train.GradientDescentOptimizer(0.5).minimize(loss)\n",
    "  \n",
    "    # Predictions for the training, validation, and test data.\n",
    "    train_prediction = tf.nn.softmax(logits)\n",
    "    hidden1_valid = tf.nn.relu(tf.matmul(tf_valid_dataset, w1) + b1)\n",
    "    valid_prediction = tf.nn.softmax(tf.matmul(hidden1_valid, w2) + b2)\n",
    "    hidden1_test = tf.nn.relu(tf.matmul(tf_test_dataset, w1) + b1)\n",
    "    test_prediction = tf.nn.softmax(tf.matmul(hidden1_test, w2) + b2)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Let's run it:\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Initialized\n",
      "Minibatch loss at step 0: 4.326218\n",
      "Minibatch accuracy: 7.8%\n",
      "Validation accuracy: 39.6%\n",
      "Minibatch loss at step 500: 0.672074\n",
      "Minibatch accuracy: 89.1%\n",
      "Validation accuracy: 84.3%\n",
      "Minibatch loss at step 1000: 0.736165\n",
      "Minibatch accuracy: 82.8%\n",
      "Validation accuracy: 84.2%\n",
      "Minibatch loss at step 1500: 0.492880\n",
      "Minibatch accuracy: 89.1%\n",
      "Validation accuracy: 85.2%\n",
      "Minibatch loss at step 2000: 0.517733\n",
      "Minibatch accuracy: 92.2%\n",
      "Validation accuracy: 84.9%\n",
      "Minibatch loss at step 2500: 0.627064\n",
      "Minibatch accuracy: 85.9%\n",
      "Validation accuracy: 84.5%\n",
      "Minibatch loss at step 3000: 0.658148\n",
      "Minibatch accuracy: 85.9%\n",
      "Validation accuracy: 85.0%\n",
      "Test accuracy: 91.7%\n"
     ]
    }
   ],
   "source": [
    "num_steps = 3001\n",
    "\n",
    "with tf.Session(graph=graph) as session:\n",
    "    tf.global_variables_initializer().run()\n",
    "    print(\"Initialized\")\n",
    "    for step in range(num_steps):\n",
    "        # Pick an offset within the training data, which has been randomized.\n",
    "        # Note: we could use better randomization across epochs.\n",
    "        offset = (step * BATCH_SIZE) % (train_labels.shape[0] - BATCH_SIZE)\n",
    "        # Generate a minibatch.\n",
    "        batch_data = train_dataset[offset:(offset + BATCH_SIZE), :]\n",
    "        batch_labels = train_labels[offset:(offset + BATCH_SIZE), :]\n",
    "        # Prepare a dictionary telling the session where to feed the minibatch.\n",
    "        # The key of the dictionary is the placeholder node of the graph to be fed,\n",
    "        # and the value is the numpy array to feed to it.\n",
    "        feed_dict = {tf_train_dataset: batch_data, tf_train_labels: batch_labels, l2_beta: 0.005}\n",
    "        _, l, predictions = session.run([optimizer, loss, train_prediction], feed_dict=feed_dict)\n",
    "        if (step % 500 == 0):\n",
    "            print(\"Minibatch loss at step %d: %f\" % (step, l))\n",
    "            print(\"Minibatch accuracy: %.1f%%\" % accuracy(predictions, batch_labels))\n",
    "            print(\"Validation accuracy: %.1f%%\" % accuracy(valid_prediction.eval(), valid_labels))\n",
    "    print(\"Test accuracy: %.1f%%\" % accuracy(test_prediction.eval(), test_labels))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Problem 2\n",
    "Let's demonstrate an extreme case of overfitting by restricting the training data, see how the accuracy on the test reduces. We will run fewer steps of gradient descent only on the first few batches of training data."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Initialized\n",
      "Minibatch loss at step 0: 2.363691\n",
      "Minibatch accuracy: 7.8%\n",
      "Validation accuracy: 33.4%\n",
      "Minibatch loss at step 50: 0.042726\n",
      "Minibatch accuracy: 100.0%\n",
      "Validation accuracy: 78.1%\n",
      "Minibatch loss at step 100: 0.019839\n",
      "Minibatch accuracy: 100.0%\n",
      "Validation accuracy: 78.4%\n",
      "Test accuracy: 85.5%\n"
     ]
    }
   ],
   "source": [
    "num_steps = 101\n",
    "NUM_BATCHES = 3\n",
    "\n",
    "with tf.Session(graph=graph) as session:\n",
    "    tf.global_variables_initializer().run()\n",
    "    print(\"Initialized\")\n",
    "    for step in range(num_steps):\n",
    "        # Pick an offset within the training data, which has been randomized.\n",
    "        # Note: we could use better randomization across epochs.\n",
    "        offset = ((step % NUM_BATCHES) * BATCH_SIZE) % (train_labels.shape[0] - BATCH_SIZE)\n",
    "        # Generate a minibatch.\n",
    "        batch_data = train_dataset[offset:(offset + BATCH_SIZE), :]\n",
    "        batch_labels = train_labels[offset:(offset + BATCH_SIZE), :]\n",
    "        # Prepare a dictionary telling the session where to feed the minibatch.\n",
    "        # The key of the dictionary is the placeholder node of the graph to be fed,\n",
    "        # and the value is the numpy array to feed to it.\n",
    "        feed_dict = {tf_train_dataset: batch_data, tf_train_labels: batch_labels, l2_beta: 0}\n",
    "        _, l, predictions = session.run([optimizer, loss, train_prediction], feed_dict=feed_dict)\n",
    "        if (step % 50 == 0):\n",
    "            print(\"Minibatch loss at step %d: %f\" % (step, l))\n",
    "            print(\"Minibatch accuracy: %.1f%%\" % accuracy(predictions, batch_labels))\n",
    "            print(\"Validation accuracy: %.1f%%\" % accuracy(valid_prediction.eval(), valid_labels))\n",
    "    print(\"Test accuracy: %.1f%%\" % accuracy(test_prediction.eval(), test_labels))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "As expected, the network quickly overfits to the small data and the minibatch accuracy gets to 100%. However, it fails to generalize, leading to a poor test score."
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Problem 3\n",
    "Now let's introduce Dropout on the hidden layer of the neural network. Note that dropout is only introduced during training, not evaluation, otherwise the evaluation results would be stochastic as well. TensorFlow provides `nn.dropout()` for that. "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "BATCH_SIZE = 128\n",
    "HIDDEN_NODES = 1024\n",
    "\n",
    "graph = tf.Graph()\n",
    "with graph.as_default():\n",
    "\n",
    "    # Input data. For the training data, we use a placeholder that will be fed\n",
    "    # at run time with a training minibatch.\n",
    "    tf_train_dataset = tf.placeholder(tf.float32, shape=(BATCH_SIZE, IMAGE_PIXELS))\n",
    "    tf_train_labels = tf.placeholder(tf.float32, shape=(BATCH_SIZE, NUM_LABELS))\n",
    "    tf_valid_dataset = tf.constant(valid_dataset)\n",
    "    tf_test_dataset = tf.constant(test_dataset)\n",
    "    l2_beta = tf.placeholder(tf.float32)  # factor for regularization penalty\n",
    "    do_keep_prob = tf.placeholder(tf.float32)  # dropout keep probability\n",
    "  \n",
    "    # Variables.\n",
    "    w1 = tf.Variable(tf.truncated_normal([IMAGE_PIXELS, HIDDEN_NODES],\n",
    "                                         stddev=1.0/np.sqrt(float(IMAGE_PIXELS))))\n",
    "    b1 = tf.Variable(tf.zeros([HIDDEN_NODES]))\n",
    "    w2 = tf.Variable(tf.truncated_normal([HIDDEN_NODES, NUM_LABELS],\n",
    "                                         stddev=1.0/np.sqrt(float(IMAGE_PIXELS))))\n",
    "    b2 = tf.Variable(tf.zeros([NUM_LABELS]))\n",
    "  \n",
    "    # Training computation.\n",
    "    hidden1 = tf.nn.dropout(tf.nn.relu(tf.matmul(tf_train_dataset, w1) + b1), do_keep_prob)\n",
    "    logits = tf.matmul(hidden1, w2) + b2\n",
    "    loss = tf.reduce_mean(tf.nn.softmax_cross_entropy_with_logits(labels=tf_train_labels, logits=logits)) \\\n",
    "            + l2_beta * (tf.nn.l2_loss(w1) + tf.nn.l2_loss(w2))\n",
    "  \n",
    "    # Optimizer. \n",
    "    optimizer = tf.train.GradientDescentOptimizer(0.5).minimize(loss)\n",
    "  \n",
    "    # Predictions for the training, validation, and test data.\n",
    "    train_prediction = tf.nn.softmax(logits)\n",
    "    hidden1_valid = tf.nn.relu(tf.matmul(tf_valid_dataset, w1) + b1)\n",
    "    valid_prediction = tf.nn.softmax(tf.matmul(hidden1_valid, w2) + b2)\n",
    "    hidden1_test = tf.nn.relu(tf.matmul(tf_test_dataset, w1) + b1)\n",
    "    test_prediction = tf.nn.softmax(tf.matmul(hidden1_test, w2) + b2)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Initialized\n",
      "Minibatch loss at step 0: 2.383565\n",
      "Minibatch accuracy: 4.7%\n",
      "Validation accuracy: 37.8%\n",
      "Minibatch loss at step 50: 0.061826\n",
      "Minibatch accuracy: 100.0%\n",
      "Validation accuracy: 78.2%\n",
      "Minibatch loss at step 100: 0.043888\n",
      "Minibatch accuracy: 99.2%\n",
      "Validation accuracy: 78.6%\n",
      "Test accuracy: 85.5%\n"
     ]
    }
   ],
   "source": [
    "num_steps = 101\n",
    "NUM_BATCHES = 3\n",
    "\n",
    "with tf.Session(graph=graph) as session:\n",
    "    tf.global_variables_initializer().run()\n",
    "    print(\"Initialized\")\n",
    "    for step in range(num_steps):\n",
    "        # Pick an offset within the training data, which has been randomized.\n",
    "        # Note: we could use better randomization across epochs.\n",
    "        offset = ((step % NUM_BATCHES) * BATCH_SIZE) % (train_labels.shape[0] - BATCH_SIZE)\n",
    "        # Generate a minibatch.\n",
    "        batch_data = train_dataset[offset:(offset + BATCH_SIZE), :]\n",
    "        batch_labels = train_labels[offset:(offset + BATCH_SIZE), :]\n",
    "        # Prepare a dictionary telling the session where to feed the minibatch.\n",
    "        # The key of the dictionary is the placeholder node of the graph to be fed,\n",
    "        # and the value is the numpy array to feed to it.\n",
    "        feed_dict = {tf_train_dataset: batch_data, tf_train_labels: batch_labels, l2_beta: 0, do_keep_prob: 0.5}\n",
    "        _, l, predictions = session.run([optimizer, loss, train_prediction], feed_dict=feed_dict)\n",
    "        if (step % 50 == 0):\n",
    "            print(\"Minibatch loss at step %d: %f\" % (step, l))\n",
    "            print(\"Minibatch accuracy: %.1f%%\" % accuracy(predictions, batch_labels))\n",
    "            print(\"Validation accuracy: %.1f%%\" % accuracy(valid_prediction.eval(), valid_labels))\n",
    "    print(\"Test accuracy: %.1f%%\" % accuracy(test_prediction.eval(), test_labels))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Notice how we still are able to get 100% accuracy on the minibatch, but our generalized score is better. This is due to dropout forcing the network to learn multiple representations of the small training data, helping on newer cases."
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Problem 4\n",
    "Now let's try to get the best performance we can using a multi-layer model! According to Yaroslav Bulatov the best reported test accuracy using a deep network is 97.1%.\n",
    "We will explore two paths:\n",
    "* Adding a new fully-connected hidden layer of 256 nodes, effectively making our network 796-1024-256-10 nodes; and\n",
    "* Using learning rate decay as shown on this code snippet:\n",
    "```\n",
    "global_step = tf.Variable(0)  # count the number of steps taken.\n",
    "learning_rate = tf.train.exponential_decay(0.5, global_step, ...)\n",
    "optimizer = tf.train.GradientDescentOptimizer(learning_rate).minimize(loss, global_step=global_step)\n",
    "```"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "BATCH_SIZE = 128\n",
    "HIDDEN_NODES_1 = 1024\n",
    "HIDDEN_NODES_2 = 256\n",
    "\n",
    "graph = tf.Graph()\n",
    "with graph.as_default():\n",
    "\n",
    "    # Input data. For the training data, we use a placeholder that will be fed\n",
    "    # at run time with a training minibatch.\n",
    "    tf_train_dataset = tf.placeholder(tf.float32, shape=(BATCH_SIZE, IMAGE_PIXELS))\n",
    "    tf_train_labels = tf.placeholder(tf.float32, shape=(BATCH_SIZE, NUM_LABELS))\n",
    "    tf_valid_dataset = tf.constant(valid_dataset)\n",
    "    tf_test_dataset = tf.constant(test_dataset)\n",
    "    l2_beta = tf.placeholder(tf.float32)  # factor for regularization penalty\n",
    "    do_keep_prob = tf.placeholder(tf.float32)  # dropout keep probability\n",
    "    global_step = tf.Variable(0)  # count the number of steps taken.\n",
    "  \n",
    "    # Variables.\n",
    "    w1 = tf.Variable(tf.truncated_normal([IMAGE_PIXELS, HIDDEN_NODES_1],\n",
    "                                         stddev=1.0/np.sqrt(float(IMAGE_PIXELS))))\n",
    "    b1 = tf.Variable(tf.zeros([HIDDEN_NODES_1]))\n",
    "    w2 = tf.Variable(tf.truncated_normal([HIDDEN_NODES_1, HIDDEN_NODES_2],\n",
    "                                         stddev=1.0/np.sqrt(float(IMAGE_PIXELS))))\n",
    "    b2 = tf.Variable(tf.zeros([HIDDEN_NODES_2]))\n",
    "    w3 = tf.Variable(tf.truncated_normal([HIDDEN_NODES_2, NUM_LABELS],\n",
    "                                         stddev=1.0/np.sqrt(float(IMAGE_PIXELS))))\n",
    "    b3 = tf.Variable(tf.zeros([NUM_LABELS]))\n",
    "  \n",
    "    # Training computation.\n",
    "    hidden1 = tf.nn.dropout(tf.nn.relu(tf.matmul(tf_train_dataset, w1) + b1), do_keep_prob)\n",
    "    hidden2 = tf.nn.dropout(tf.nn.relu(tf.matmul(hidden1, w2) + b2), do_keep_prob)\n",
    "    logits = tf.matmul(hidden2, w3) + b3\n",
    "    loss = tf.reduce_mean(tf.nn.softmax_cross_entropy_with_logits(labels=tf_train_labels, logits=logits)) \\\n",
    "            + l2_beta * (tf.nn.l2_loss(w1) + tf.nn.l2_loss(w2) + tf.nn.l2_loss(w3))\n",
    "  \n",
    "    # Optimizer. \n",
    "    learning_rate = tf.train.exponential_decay(0.1, global_step, decay_steps=200, decay_rate=0.95)\n",
    "    optimizer = tf.train.GradientDescentOptimizer(learning_rate).minimize(loss, global_step=global_step)\n",
    "  \n",
    "    # Predictions for the training, validation, and test data.\n",
    "    train_prediction = tf.nn.softmax(logits)\n",
    "    hidden1_valid = tf.nn.relu(tf.matmul(tf_valid_dataset, w1) + b1)\n",
    "    hidden2_valid = tf.nn.relu(tf.matmul(hidden1_valid, w2) + b2)\n",
    "    valid_prediction = tf.nn.softmax(tf.matmul(hidden2_valid, w3) + b3)\n",
    "    hidden1_test = tf.nn.relu(tf.matmul(tf_test_dataset, w1) + b1)\n",
    "    hidden2_test = tf.nn.relu(tf.matmul(hidden1_test, w2) + b2)\n",
    "    test_prediction = tf.nn.softmax(tf.matmul(hidden2_test, w3) + b3)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Before expanding the model, let's try to tune our hyperparemeters on a smaller scale..."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def run_session(graph, num_steps, beta, dropout, batch_size=128, num_batches=num_steps, verbose=True):\n",
    "    with tf.Session(graph=graph) as session:\n",
    "        tf.global_variables_initializer().run()\n",
    "        print(\"Initialized\")\n",
    "        for step in range(num_steps):\n",
    "            # Pick an offset within the training data, which has been randomized.\n",
    "            # Note: we could use better randomization across epochs.\n",
    "            offset = ((step % num_batches) * batch_size) % (train_labels.shape[0] - batch_size)\n",
    "            # Generate a minibatch.\n",
    "            batch_data = train_dataset[offset:(offset + batch_size), :]\n",
    "            batch_labels = train_labels[offset:(offset + batch_size), :]\n",
    "            # Prepare a dictionary telling the session where to feed the minibatch.\n",
    "            # The key of the dictionary is the placeholder node of the graph to be fed,\n",
    "            # and the value is the numpy array to feed to it.\n",
    "            feed_dict = {tf_train_dataset: batch_data, tf_train_labels: batch_labels, l2_beta: beta,\n",
    "                         do_keep_prob: dropout}\n",
    "            _, lr, l, predictions = session.run([optimizer, learning_rate, loss, train_prediction], feed_dict=feed_dict)\n",
    "            if (step % (num_steps/10) == 0) and verbose == True:\n",
    "                print(\"Minibatch loss at step %d: %f\" % (step, l))\n",
    "                print(\"Minibatch accuracy: %.1f%%\" % accuracy(predictions, batch_labels))\n",
    "                print(\"Validation accuracy: %.1f%%\" % accuracy(valid_prediction.eval(), valid_labels))\n",
    "                print(\"Learning rate: %.3f\" % lr)\n",
    "        print(\"Test accuracy: %.1f%%\" % accuracy(test_prediction.eval(), test_labels))\n",
    "        return accuracy(test_prediction.eval(), test_labels)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Beta: 0.000\n",
      "Initialized\n",
      "Test accuracy: 91.7%\n",
      "Beta: 0.001\n",
      "Initialized\n",
      "Test accuracy: 91.8%\n",
      "Beta: 0.002\n",
      "Initialized\n",
      "Test accuracy: 91.7%\n",
      "Beta: 0.003\n",
      "Initialized\n",
      "Test accuracy: 91.8%\n",
      "Beta: 0.004\n",
      "Initialized\n",
      "Test accuracy: 91.7%\n",
      "Beta: 0.005\n",
      "Initialized\n",
      "Test accuracy: 91.5%\n",
      "Beta: 0.006\n",
      "Initialized\n",
      "Test accuracy: 91.4%\n",
      "Beta: 0.007\n",
      "Initialized\n",
      "Test accuracy: 91.2%\n",
      "Beta: 0.008\n",
      "Initialized\n",
      "Test accuracy: 91.2%\n",
      "Beta: 0.009\n",
      "Initialized\n",
      "Test accuracy: 91.0%\n"
     ]
    }
   ],
   "source": [
    "param = [i for i in np.arange(0, 1e-2, 1e-3)]\n",
    "results = []\n",
    "\n",
    "for i in param:\n",
    "    print(\"Beta: %0.3f\" % i)\n",
    "    results.append(run_session(graph, 2000, beta=i, dropout=1, verbose=False))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYYAAAEMCAYAAADAqxFbAAAABHNCSVQICAgIfAhkiAAAAAlwSFlz\nAAALEgAACxIB0t1+/AAAIABJREFUeJzt3Xl4VOXd//H3NwkJECAsWYCEPRBALIJRwI2IKG7VQjfb\nap9uUlurVNtauz7P0z72Z1trW7tZKt2oS90XQEELYamCsiprkH1Pwp6EhCz37485oUnMMkkmOTOT\nz+u6ciVzZs59vrPkfObc91nMOYeIiEi1GL8LEBGR8KJgEBGRWhQMIiJSi4JBRERqUTCIiEgtCgYR\nEalFwSBRx8w6m5kzswy/a2kuM1tpZre2Yv4dZjYpxDUlmFmRmfUPZbte24fN7LJQtyuto2DwgfdP\nVv1TZWZnatz+TCvabdVKRSKfc26Yc+6t1rRR93PknCtzznVzzh1sfYWho1BpO3F+F9AROee6Vf9t\nZruBLznn3vCvovZhZnHOuQq/62iNcH0O4VqXRCZtMYQhM4s1sx+Y2U4zKzSzx82sp3dfopk9ZWbH\nzOyEma0ys15m9gvgIuAxb8vjF/W0G2dmz5nZEW/eJWaWVeP+RDN7xMz2mdlJM1tqZnHefTneN8mT\nZrbXzD7tTa/17dLM7jCzN7y/q7t0vmJmO4CN3vQ/mNl+MztlZm+b2cQ6Nf6399xPmdk7ZtbXzOaY\n2QN1ns8iM/tKIy/lR8xst5kVmNkDFtDVa3d4jXYyzKyk+jWus4w7zGyxmf3OzI4D93vTv2xm27z3\nYb6ZpdeY5wYz2+69xr+q+RqZ2YNm9liNx440s3pX6N59ud4yCszsb2bWvcb9h83sm2a2CThVY9pl\n3meo5pZpsfde9DWzFDN71WvzmJm9ZGb9vPk/8DmyOl1zZtbbzJ7w5t9lZveZmdV4vf7lfY5OWKBr\na2oj7xHAJWa21atltpkl1HiO083sXa+t5WY22pv+DJAKLPLqvNua+HxLMzjn9OPjD7AbmFpn2reB\n5UB/oDPwV+Av3n2zgGeBLgS2+C4CEr37VgK3NrKsOOCzQDev3T8AK2vcPwdYBPQFYoHLvd+ZQBHw\nUa+NFGBsfcsE7gDe8P7uDDhgPtAT6OJN/yzQC+gEfA/YB3Ty7vsBsM5bZgwwzpv3CmAXYN7j+gMl\nQO96nmf1chd68w4BdlbXCfwZ+N86r/czDbxmdwAVwO3ea9EF+CSwBRjhPYf/A5Z4j+/nvVY3evfd\nB5TXWPaDwGM12h8JVNS4vbLGY0cCU4B47z1ZCTxY47GHgXe816JLjWmX1fM8Hgbe8J5DGnCz91yS\ngJeAp+qroc7rmeHdfhp4xvscZXrvy2dqvF7l3nscC9wD7G7kM3nYe7/7e5+rd4Dve/dNBA4BF3pt\nzQTygLj6nitNfL7104z1kt8FdPQf6g+GXcClNW4P8VaCBnwVWAqMqaetRoOhnsf3Baq8f6JO3j90\nVj2P+1/gyQbaCCYYLmmkBvOeW5Z3ew8wrYHH7QQu925/E3i+gTarl5tTY9q9wHzv78nA+zXuew+4\nqYG27gDy6kxbUr0i9G5Xv3Zp3sprSY37YoB8WhAM9dRyC/BWjduHgU/XecwHgsFbWb5PPSHq3T8R\nONTIe3ouGIAEoBIYWuP+WcBrNV6vjTXu6+3N27OBZR8GPlfj9gxgk/f3X4Dv1Xn8HmBCQ8+1oc93\nc/8vO/qPupLCjLdJPgBY4G0OnyDwjSoG6EPgW/1S4FmvO+YnZhYbZNtxXtfATjM7BWwlsMLtQ+Cb\nbhywo55ZBzQwPVj76tTxHa8b5iRwnMCKJ9l77un1LcsF/tP/DlR3W90KzG3GcvcQ+FYKsAyINbNJ\nZnYBgef+arD1A4OAR2u8PwUEtioyvGWce7xzrgo40ESd9TKz/mb2jJkd8N6vx4DkJmqr28YE4BfA\nzc65Y9607mb2Zwt0CZ4isJVYt92G9CXwWdxbY9oeAu9btcM1/i7xfnejYQ29T4OA71a/zt5rnVJn\nWec08fmWZlAwhBlvBXgAmOKc61njp7NzrtAF9hD5oXNuJIHulY8T+CYJgW9mjfk8cDVwJYEuhJHe\ndCOwyV4BDKtnvn0NTAcoBrrWuN23vqdV/YeZXQ3cBUwn0M3TGzhDoIuo+rk3tKy/Ax8zswsJhNX8\nBh5XbUCNvwcCB+EDIXMbgW6U8kbaqfu67iPwLbfm+9PFObeGwOt4bjdZM4uh9oosmNer2s+9x49x\nzvUAvkTgvWqstnMssHvpcwR2bthU4677vRov8tq9pk67jX2ODhP4Fj6wxrSBtDD8PPW+TwRe5x/W\neZ27Oueeb6DOxj7f0gwKhvD0KPCgmQ0AMLNUM/uw9/dUMxvtrXBOEViZV3nzHQGGNtJud6AUOAok\nEugbB8BbMf4d+LWZpXmDl5d5WyNzgRu9gcA4b/DyQ96s6wmsrDub2Ujgc008t+4Eul0KCPSd/4jA\nFkO1x4CfmNlQCxhn3qCwc24nsJlAF8M/nXNnm1jWt80sycwGA18D/lnjvr8DnwA+5f3dHI8C368e\n2LTA4P9HvfteBiaY2fUWGLi/l8B4SrX1wJVmlm5mvQiMbzSkO4HxilNmNtBrKyhmFg88D/zROfdS\nPe2WACfMLBn4fp37G/wcOefKgBcIvEeJZjaMQFfSP4KtrR53m1k/r5b7+c/7NBu4y8yyvc9CNzO7\nycyqg7VunQ1+vqV5FAzh6WcEBgoXm9lp4E1gvHdfOoHBwtME9vJZwH/+kX4JfNbMjpvZz+ppdw6B\nFfJhAv3qK+rcfzeBbpx1BP65fkzgm/z7BAYrvwscA1YD59WoNc5rdzZNryBeIdCVs4PAmEGhN2+1\nBwlsCSwmEHyPEujXrvY34Hya7kbCa2eDV+8zNWtzzu0AtgGnnXNvB9HWOc65J4HfAs97XRbrCXxT\nxTl3iEDYPOI9twwCr3VZjZrmEQi4lcCLjSzqh8BlwEkCK+PnmlHmUGACgXCsuXdSKvAQga6jowQ+\nAwvqzNvU5+jL3u89BN6nx4DHm1FbXU8RGLfZTuC1+hmAc+7fBD6TfwROEBh4/jT/2VJ4AHjA62b6\nGk1/viVI1Xt4iEQEM7sG+L1zLjMEbT0BbHbOtdk3S2+r4TDwYdfKA89E2ou2GCRieN0jdxPYMmlt\nW5kEdin9S2vbqqft67wurM7AfxPotlkT6uWItBUFg0QEb++h4wT6kX/XyrZ+RqC77EfOudYMmjak\n+piLfOAqYHoQ4yEiYUNdSSIiUou2GEREpBYFg4iI1BKWZ1dNTk52gwcPbtG8xcXFJCYmhrYgEZEw\nt2bNmkLnXEoo2grLYBg8eDCrV69u0by5ubnk5OSEtiARkTBnZntC1Za6kkREpBYFg4iI1KJgEBGR\nWhQMIiJSi4JBRERqUTCIiEgtYbm7akst3HSYkyVVTT9QREQaFDXBUFpeyV1PruNsRRV/3JLLlJGp\nTBmZRvbgXnSK1YaRiEiwwvIketnZ2a4lB7jtLixm9rw32VvRg1W7jlJe6ejeOY4rhqcwZWQqOVkp\n9OmW0HRDIiIRxszWOOeyQ9FWUFsMZjYLuJ3AtVP/5Jz7lZl9HPgfYBRwsXOu3jW5md1D4Fq1jsBV\nlT7vnCsNQe0fMDg5kWsGdyInZwJFZRWs2F7Ikq35LN6Wz/z3DmEGYzN6elsTqZzXvweB68+LiEi1\nJoPBzMYQCIWLgbPAa2Y2j8BlJWcQuOxeQ/OmE7iwymjn3Bkze5rAhev/2vrSG9ctIY5rx/Tl2jF9\nqapybDp4isVb81m89QgPv57Hw6/nkdYjgSuzAiFxaWYyiQlR07MmItJiwawJRwGrnHMlAGa2FJjh\nnPuZdzuYZXQxs3KgK3Cw5eW2TEyMcX5GEudnJDFr6nAKTpeRuy2fJdvymffuIZ56Zx/xsTFMGNqb\nq7yxiYF9ujbdsIhIFAomGDYSuOB2H+AMcD2Bi6s3yTl3wMweAvZ68y5yzi2q77FmNhOYCZCWlkZu\nbm4wi/iAoqKioOZNAT6RDjP6xZN3PI4NBRVsOHiU5dsL+Z9XNtMv0RibEsvYlDiG94ohLkZdTiLS\nMTQZDM65LWb2U2ARUAysByqDadzMegE3A0OAE8AzZnarc+4f9SxnNt61fLOzs11Lz5DakrOrTq3x\n9+7CYhZvDWxN/GvnUV7bXaEBbBHpUILqVHfOzQHmAJjZT4D9QbY/FdjlnCvw5n0euAT4QDCEi8HJ\niXzhsiF84bIhjQ5gXzUylSs1gC0iUSjYvZJSnXP5ZjaQwIDzxCDb3wtMNLOuBLqSriLIbqhw0OAA\n9rZ8fvF6Hr/QALaIRKFg12LPeWMM5cCdzrkTZjYd+A2B7vr5ZrbeOTfNzPoDjznnrnfOrTKzZ4G1\nQAWwDq+7KNJoAFtEOoqoOsAN/LmC29mKKlbvPnZua2JnQTEAw1ISuWpUGldmpeoIbBFpU+1+gJs0\nLj4uhksyk7kkM5nv3zi61gD2X/69i9nLdmoAO8LtKixm6bZ8EhPiuPFD/ekSH+t3SSJtRsHQBjSA\nHfnOVlTx9q5j5wJ+V2Hxuft+PG8zH70wg1snDmJYSjcfqxRpG+pKakd1B7A37DsBQFqPBKaMTOXK\nLA1g+yn/VClLtuWzeGs+K7YXUny2kvi4GCYO7cOUrBSmjEzj8KlS5q7cw2sbD1Fe6bg0sw+3TRzE\n1FFpxKmrUHwUyq4kBYOPag5gL88r5HRZhQaw21FVlePdAycDWwVb83nvwEkA+vbozJUjq/c060PX\n+A8GdcHpMp5evY/HV+7h4MlS+vbozKcuHsgtFw8grUfn9n4qIgqGxkRSMNSkAez2caq0nOV5hSze\nms/SvHwKi84SYzBuYK9zW22j+nUPumuvssqxeGs+c1fuYVleAXExxjXnpXHrxEFMGtpHXYTSbhQM\njYjUYKir5gD2qp3HOFtZpQHsFnDOsaOgyDuBYj6rdx+nosqR1KUTk0cEXsvJI1LolRjf6mXtLizm\nibf38vTqfZwoKWdYSiK3TRzEjAsz6NG5UwiejUjDFAyNiJZgqKnuAHbB6TINYDeitLySVbuOBV6v\nrfnsPVYCQFZad6aMCnQRjRvQs83GBErLK5n37iHmrtzDhn0n6NIplo+M68+tEwdxXv+kNlmmiIKh\nEdEYDDXVHcB+d/8JnNMA9uGTpee2Cv79fiFnyitJiIvh0szkc+MF6T27tHtd7+4/wT9W7uHlDQcp\nLa9i/MCe3DZpENeN6UfnTtrlVUJHwdCIaA+GujrqAHZllWP9vuNeGBSw5dApANJ7djl3IaZJw/qE\nzcr3ZEk5z67dzz9W7mFXYTG9unbiExcN4NYJgxjQO/reH2l/CoZGdLRgqCnaB7BPlpSzdHsBS7bm\nk7stn+Ml5cTGGBcO7HWui2h4arew7lKrqnK8ueMoc1fu5vXNR3BAzogUbps0iMkjUonV6d2lhRQM\njejIwVBXgwPYI1KYkhX+A9jOOfKOFJ3bnXTN3uNUVjl6de1ETlZgbGXy8BSSukbmwO6hk2d48u19\nPPn2XgpOl5HRqwufnjCQT2YPCOv3RcKTgqERCob6RcoAdml5JW/uKPTCoIADJ84AMLpfj8AYyshU\nLhjQM6q+WZdXVrFo0xHmrtzNyp3HiI+N4frz+3LbpEGMH9jL9/dEIoOCoREKhqaF2wD2gRNnzm0V\nvLmjkNLyKrrGx3JpZvK5evomdYyDxrYfOc3jq/by3Jr9nC6rYFS/Htw2cRA3X9C/w+1QIM2jYGiE\ngqH5GhrAnjjsP6eCCOUAdkVlFWv3njgXBtuOnAZgYO+u5waOJwztTUJceAwc+6G4rIKX1h/k72/t\nZuvh03RPiGPG+HRunTiI4Wnd/S5PwpCCoREKhtZpqwHsY8VnWZoX2INoWV4BJ8+UExdjXDS497ku\nomEpieo2qcM5x9q9x5n71h4WvHeYs5VVTBzam9smDuaa89IidkcCCT0FQyMUDKHV0gFs5xxbDp0+\nd1K6dXuPU+UguVs8Od4V7y4bnqwjgpvhaFEZT6/ez+Or9rD/+BlSuifwqYsG8KkJA+mX1P7HaEh4\nUTA0QsHQdmoOYC/Zlk9+nQHsyVkpHDlVxmJvd9JDJ0sB+FBG0rnLn56fnkRMFA0c+6GyyrE0L5+5\nb+0hN6+AGDOmjkrltomDuWRYH72+HZSCoREKhvZRVeXYfOgU/9pSewAbAtfKvswbOM7JSiFVZxtt\nM/uOlfD4qr388529HC8p5/bLh/C9G0b7XZb4QMHQCAWDPwpOl/HmjkKSuyVw0eDexMep77s9lZZX\n8p3n32PeuwdZ/I0cHU3dAYUyGPTfKyGR0j2Bmy9I59LMZIWCDzp3iuXb144kxoxfvbHd73Ikwuk/\nWCRK9E3qzH9dMpgX1u1nu7cLsEhLKBhEosgdk4fRNT6OXyzK87sUiWAKBpEo0jsxntsvH8prmw6f\nu6a4SHMpGESizBcvH0LvxHgeWrTN71IkQikYRKJMt4Q4vpozjOXbC3lzR6Hf5UgEUjCIRKFbJw6i\nX1Jnfr5wG+G4S7qENwWDSBTq3CmWWVcNZ93eE7yxJd/vciTCKBhEotTHLsxgSHIiDy3cRlWVthok\neAoGkSgVFxvDvVePYNuR07y84aDf5UgEUTCIRLEbzu/H6H49ePj1PMorq/wuRyKEgkEkisXEGN+a\nlsXeYyX88519fpcjEULBIBLlcrJSyB7Ui0f+tZ3S8kq/y5EIoGAQiXJmxn3XjiT/dBl/e3O33+VI\nBFAwiHQAFw/pzeQRKfxh6Q5OlZb7XY6EOQWDSAfxrWlZnCgp57FlO/0uRcKcgkGkgxiTnsQN5/fj\nsRW7KCwq87scCWMKBpEO5N5rRlBaXsnvl+zwuxQJYwoGkQ5kWEo3PnZhBv9YuYcDJ874XY6EqaCC\nwcxmmdlGM9tkZl/3pn3cu11lZg1eZ9TMeprZs2a21cy2mNmkUBUvIs03a+oIAB7RJUClAU0Gg5mN\nAW4HLgbGAjeaWSawEZgBLGuiiV8DrznnRnrzb2lVxSLSKuk9u/CZiQN5du1+dhQU+V2OhKFgthhG\nAauccyXOuQpgKTDDObfFOdfolUDMLAm4ApgD4Jw765zTZaVEfHbnlZkkxMXw8Ou6BKh8UFwQj9kI\nPGBmfYAzwPXA6iDbHwIUAH8xs7HAGmCWc6647gPNbCYwEyAtLY3c3NwgF1FbUVFRi+cV6UiuGhDD\nK+8e4uJu/2JQj1i/y5Ew0mQwOOe2mNlPgUVAMbAeCPa4+jhgPHCXc26Vmf0auB/4QT3LmQ3MBsjO\nznY5OTlBLqK23NxcWjqvSEcyfmI5y366hCVHu/PXmy72uxwJI0ENPjvn5jjnLnTOXQEcB4Ld/twP\n7HfOrfJuP0sgKETEZz06d+IrOcPI3VbA27uO+V2OhJFg90pK9X4PJDDg/EQw8znnDgP7zCzLm3QV\nsLkFdYpIG/ivSYNJ7Z7Azxdu1SVA5Zxgj2N4zsw2A68AdzrnTpjZdDPbD0wC5pvZQgAz629mC2rM\nexfwuJm9C1wA/CSE9YtIK3SJj+Wuq4bzzu7j5OYV+F2OhIlgBp9xzl1ez7QXgBfqmX6QwAB19e31\nQIPHOYiIvz6ZPYA/LdvJz1/bxuThKcTEmN8lic905LNIBxcfF8M9Vw9n86FTLNh4yO9yJAwoGESE\nm8amk5XWnYcX5VGhS4B2eAoGESE2xvjGNSPYWVjMc2v3+12O+EzBICIAXD06jQsG9ORXb+gSoB2d\ngkFEAO8SoNOyOHSylMdX7fW7HPGRgkFEzrkkM5nLMpP53ZL3KSqr8Lsc8YmCQURq+ea0LI4Vn+XP\nK3b5XYr4RMEgIrVcMKAn085L40/LdnK8+Kzf5YgPFAwi8gHfuCaLorMVPLpUlwDtiBQMIvIBI9K6\nM31cOn99czeHT5b6XY60MwWDiNTrnqkjqHKO3yzWJUA7GgWDiNRrQO+u3HLRQP75zj72HP3AtbUk\niikYRKRBd03JJC7W+KUuAdqhKBhEpEGpPTrzuUuG8NKGg2w9fMrvcqSdKBhEpFF3TB5Kt4Q4Hlqo\nrYaOQsEgIo3q2TWeL18xlDe2HGHNnuN+lyPtQMEgIk36/KVDSO4Wr0uAdhAKBhFpUmJCHHdemcnK\nncdY8X6h3+VIG1MwiEhQPj1hIOk9u/Dzhdu01RDlFAwiEpSEuFhmTR3Ou/tPsnDTYb/LkTakYBCR\noM0Yl86wlEQeWpRHZZW2GqKVgkFEghYXG8M3rsni/fwiXlh3wO9ypI0oGESkWa4b05fz05P45et5\nlFXoEqDRSMEgIs1iZnxzWhYHTpzhqbf3+V2OtAEFg4g02xXDk5kwpDe/Wfw+JWd1CdBoo2AQkWYz\nM+67NovCojL+8u/dfpcjIaZgEJEWuXBQb64amcofl+7gZEm53+VICCkYRKTFvnFNFqdKK/jjMl0C\nNJooGESkxUb378FNY/vzl3/vJv+0LgEaLRQMItIq91w9gvLKKj47523ezy/yuxwJAQWDiLTKkORE\n5nzuIvJPl3HTb1fw/Nr9fpckraRgEJFWmzwihQV3X86Y9CTufXoD33pmA2fO6uC3SKVgEJGQ6JvU\nmSe+NIG7p2Ty7Nr93PTbFeQdOe13WdICCgYRCZm42BjuvSaLuV+YwPGScm767QqeXr1Pp+mOMAoG\nEQm5y4Yns2DWZYwf2Iv7nn2Xbzy9geIyHSEdKRQMItImUrt3Zu4XJ3DP1BG8uP4AH/7tCrYcOuV3\nWRIEBYOItJnYGGPW1OE8/qWJFJVW8JHf/ZsnVu1V11KYUzCISJubNKwPC2ZdzsVDevPdF97j7qfW\nc7pUp9EIV0EFg5nNMrONZrbJzL7uTfu4d7vKzLKbmD/WzNaZ2bxQFC0ikSe5WwJ/+/zFfGtaFgve\nO8SHf7OCjQdO+l2W1KPJYDCzMcDtwMXAWOBGM8sENgIzgGVBLGcWsKUVdYpIFIiJMe68MpOnZk6k\ntLyKGb9/k7lv7VbXUpgJZothFLDKOVfinKsAlgIznHNbnHPbmprZzDKAG4DHWleqiESLiwb3ZsGs\ny7k0sw8/eGkTdz6xllPqWgobcUE8ZiPwgJn1Ac4A1wOrm7GMXwH3Ad0be5CZzQRmAqSlpZGbm9uM\nRfxHUVFRi+cVkfZ122BHKvE8u/Ew77x/hK9ckMDQpFi/y+rwmgwG59wWM/spsAgoBtYDQR3rbmY3\nAvnOuTVmltPEcmYDswGys7NdTk6jD29Qbm4uLZ1XRNrfFOATe45z95Pr+H9vl/Kd60bx+UsHY2Z+\nl9ZhBTX47Jyb45y70Dl3BXAcyAuy/UuBm8xsN/AUMMXM/tGiSkUkal04qBfz776MySNS+dG8zcyc\nu4YTJWf9LqvDCnavpFTv90ACA85PBDOfc+47zrkM59xg4BZgsXPu1hbWKiJRrGfXeP702Qv5wY2j\nyd2Wzw2PrGDt3uN+l9UhBXscw3Nmthl4BbjTOXfCzKab2X5gEjDfzBYCmFl/M1vQRvWKSBQzM754\n2RCeveMSYmLgE4++xexlO6iq0l5L7cnCcTex7Oxst3p1c8a3/0NjDCLR4eSZcu5/7l1e3XiYKSNT\neejjY+mdGO93WWHLzNY45xo9pixYOvJZRMJSUpdO/P4z4/nRzeexYnshNzyyXF1L7UTBICJhy8z4\n7KTBPP/VS4iLNb48d43O0toOFAwiEvbGpCfx61vGUXC6jEeX7vC7nKinYBCRiDB+YC9uGtuf2ct2\ncuDEGb/LiWoKBhGJGN++biQAP311q8+VRDcFg4hEjPSeXbj98qG8vOGgBqLbkIJBRCLKV3KGkdI9\ngR/P26yzsrYRBYOIRJTEhDi+NS2LdXtP8PKGg36XE5UUDCIScT42PoPz+vfgp69upbQ8qHN6SjMo\nGEQk4sTEGD+4cTQHT5by2PKdfpcTdRQMIhKRJg7tw7Tz0vh97g6OnCr1u5yoomAQkYj1netGUV5Z\nxUMLm7yYpDSDgkFEItbg5EQ+d8lgnl27n40HTvpdTtRQMIhIRPvalOH06hqv3VdDSMEgIhEtqUsn\n7rl6BKt2HWPhpiN+lxMVFAwiEvE+ddEARqR14ycLtlBWod1XW0vBICIRLy42hu/dMJq9x0r425u7\n/S4n4ikYRCQqTB6RQk5WCr/51/scLSrzu5yIpmAQkajx/RtGUVJeyS/fyPO7lIimYBCRqJGZ2p1b\nJwzkiVV7yTty2u9yIpaCQUSiytenjqBbQpx2X20FBYOIRJVeifHcfdVwlm8vJHdbgd/lRCQFg4hE\nnc9OGsyQ5ET+b/5myiur/C4n4igYRCTqxMfF8J3rRrKjoJgnVu31u5yIo2AQkah09eg0Jg3twy/f\nyONkSbnf5UQUBYOIRCWzwDUbTp4p55HF2/0uJ6IoGEQkao3u34NPZg/gb2/uZmdBkd/lRAwFg4hE\ntXuvGUFCXAw/WbDV71IihoJBRKJaavfOfPXKTN7YcoQ33y/0u5yIoGAQkaj3xcuGkN6zCz+at5nK\nKh301hQFg4hEvc6dYrn/upFsPXyaZ1bv87ucsKdgEJEO4cYP9ePCQb14aFEep0u1+2pjFAwi0iGY\nGT+8cTSFRWX8PneH3+WENQWDiHQYYwf0ZPq4dOas2MW+YyV+lxO2FAwi0qHcd20WMQYPvqbdVxui\nYBCRDqVfUhdmXjGM+e8eYvXuY36XE5YUDCLS4dwxeShpPRL48bzNVGn31Q9QMIhIh9M1Po77po1k\nw/6TvLj+gN/lhJ2ggsHMZpnZRjPbZGZf96Z93LtdZWbZDcw3wMyWmNlm77GzQlm8iEhLTR+Xzocy\nknjw1a2s2F6oq73V0GQwmNkY4HbgYmAscKOZZQIbgRnAskZmrwC+4ZwbDUwE7jSz0a2uWkSklWJi\njJ9MP5/YGOPWOav45OyVrNx51O+ywkIwWwyjgFXOuRLnXAWwFJjhnNvinNvW2IzOuUPOubXe36eB\nLUB6a4sWEQmFMelJLPlmDv9703nsLizmltkr+cxjKzv8oHRcEI/ZCDxgZn2AM8D1wOrmLsjMBgPj\ngFUN3D8TmAmQlpZGbm5ucxcBQFFRUYvnFZGOaRDwfxNjWbIvnvk7j/KxR99iTHIs0zM7MaxnrN/l\ntTsLpl/7DQjYAAAJA0lEQVTNzL4IfBUoBjYBZc656rGGXOCbzrkGw8LMuhHY0njAOfd8U8vLzs52\nq1c3O3sAyM3NJScnp0XzioiUnK1g7lt7eHTpDo6XlDNlZCr3TB3B+RlJfpfWKDNb45yrd7y3uYIa\nfHbOzXHOXeicuwI4DuQFuwAz6wQ8BzweTCiIiPipa3wcX548jOXfnsK3pmWxZs9xPvzbFcz8+2o2\nHzzld3ntIti9klK93wMJDDg/EeR8BswBtjjnHm5pkSIi7a1bQhx3XpnJ8m9fyT1TR/DWzqNc/8hy\nvvr4GvKOnPa7vDYV7HEMz5nZZuAV4E7n3Akzm25m+4FJwHwzWwhgZv3NbIE336XAbcAUM1vv/Vwf\n6ichItJWenTuxKypw1lx3xTunpLJsrxCpv1qGXc/uY4dUXq50KDGGNqbxhhEJFwdLz7L7OU7+dub\nuyktr+Qj49K5e8pwBicn+lpXu48xiIhIQK/EeL597UiW3XclX7p8KAveO8RVDy/lvmc3UHC6zO/y\nQkLBICLSAsndEvju9aNYdt+V/Nekwby47iA/fGmj32WFRDDHMYiISANSu3fmhx8OnNDhHyv3cLKk\nnKSunXyuqnW0xSAiEgIzxqdztrKKee8d9LuUVlMwiIiEwHn9ezAirRvPr438s7UqGEREQsDMmD4u\ngzV7jrPnaLHf5bSKgkFEJEQ+Mq4/ZkT8VoOCQUQkRPoldeGSYX14cf2BiL6+g4JBRCSEpo/LYM/R\nEtbuPe53KS2mYBARCaFrx/SlS6dYnovg7iQFg4hICHVLiGPaeWnM23CQsopKv8tpEQWDiEiIzRif\nwanSChZvyfe7lBZRMIiIhNilmcmkdk/g+XWR2Z2kYBARCbHYGOPmC/qzZGs+x4rP+l1OsykYRETa\nwIzxGVRUOea9G3mnyFAwiIi0gVH9ejCyb/eIPNhNwSAi0kY+Oj6D9ftORNyV3hQMIiJt5OYL+hNj\n8GKEDUIrGERE2khqj85cmpnM82sPUFUVOafIUDCIiLShj47P4MCJM7yz+5jfpQRNwSAi0oauOS+N\nrvGxvBBB3UkKBhGRNtQ1Po7rxvRj/ruHKC2PjFNkKBhERNrYjPHpnC6r4I0tR/wuJSgKBhGRNjZx\naB/6JXWOmGMaFAwiIm0scIqMdJbmFVBYVOZ3OU1SMIiItIMZ49OprHK8siH8T5GhYBARaQcj0roz\nJr1HRHQnKRhERNrJ9HEZvHfgJNuPnPa7lEYpGERE2slNY/sTG2Nhf50GBYOISDtJ6Z7AFcOTeWld\neJ8iQ8EgItKOpo/P4ODJUlbuOup3KQ1SMIiItKNrRqfRPSEurAehFQwiIu2oc6dYrju/L6++d4gz\nZ8PzFBkKBhGRdjZjfAbFZytZtPmw36XUS8EgItLOLh7cm/SeXcK2O0nBICLSzmJijOnj0lm+vYD8\n06V+l/MBCgYRER9MH59OlYOX14ffKTIUDCIiPhiW0o2xGUlh2Z0UVDCY2Swz22hmm8zs6960j3u3\nq8wsu5F5rzWzbWb2vpndH6rCRUQi3YzxGWw+dIqth0/5XUotTQaDmY0BbgcuBsYCN5pZJrARmAEs\na2TeWOB3wHXAaOBTZjY6BHWLiES8D4/tT1yM8UKYbTUEs8UwCljlnCtxzlUAS4EZzrktzrltTcx7\nMfC+c26nc+4s8BRwc+tKFhGJDr0T48nJSuXF9QeoDKNTZMQF8ZiNwANm1gc4A1wPrA6y/XRgX43b\n+4EJ9T3QzGYCMwHS0tLIzc0NchG1FRUVtXheEZH2lpVQwRunyvjD84sZkxzrdzlAEMHgnNtiZj8F\nFgHFwHog5IfrOedmA7MBsrOzXU5OTovayc3NpaXzioi0t4nllczd+gY7K/vwtZwL/C4HCHLw2Tk3\nxzl3oXPuCuA4kBdk+weAATVuZ3jTRESEwCkybvhQf17deJjisgq/ywGC3ysp1fs9kMCA8xNBtv8O\nMNzMhphZPHAL8HJLChURiVYzxqdzpryShZvC4xQZwR7H8JyZbQZeAe50zp0ws+lmth+YBMw3s4UA\nZtbfzBYAeIPVXwMWAluAp51zm0L+LEREIlj2oF4M6B0+p8gIZvAZ59zl9Ux7AXihnukHCQxQV99e\nACxoRY0iIlHNzJg+LoPl2ws4W1FFfJy/xx4HFQwiItK2Zl01nHuvHuF3GYBOiSEiEhZiY8zvEs5R\nMIiISC0KBhERqUXBICIitSgYRESkFgWDiIjUomAQEZFaFAwiIlKLORc+5wCvZmYnge0tnD0ZKAxh\nOdI8ScBJv4toQ+H+/Pysrz2W3RbLCGWbrW2rNfNnOee6t2LZ54Trkc//dM7NbMmMZrbaOdfgpUal\nbZnZ7Ja+d5Eg3J+fn/W1x7LbYhmhbLO1bbVmfjML9jo5TQrXrqRX/C5AWiza37twf35+1tcey26L\nZYSyzda2FRafr7DsSmoNbTGISEcUynVfuG4xtMZsvwsQEfFByNZ9UbfFICIirRONWwwiItIKCgYR\nEalFwSAiIrWE63EMIWdmo4BZBA6A+5dz7g8+lyQi0i7M7CPADUAPYI5zblFjj4+ILQYz+7OZ5ZvZ\nxjrTrzWzbWb2vpnd31gbzrktzrk7gE8Al7ZlvSIioRKi9d+LzrnbgTuATza5zEjYK8nMrgCKgL87\n58Z402KBPOBqYD/wDvApIBb4f3Wa+IJzLt/MbgK+Asx1zj3RXvWLiLRUqNZ/3ny/AB53zq1tdJmR\nEAwAZjYYmFfjhZkE/I9zbpp3+zsAzrm6L0p9bc13zt3QdtWKiIROa9d/ZmbAg8Drzrk3mlpeJI8x\npAP7atzeD0xo6MFmlgPMABKABW1amYhI22rW+g+4C5gKJJlZpnPu0cYaj+RgaBbnXC6Q63MZIiLt\nzjn3CPBIsI+PiMHnBhwABtS4neFNExGJdm26/ovkYHgHGG5mQ8wsHrgFeNnnmkRE2kObrv8iIhjM\n7EngLSDLzPab2RedcxXA14CFwBbgaefcJj/rFBEJNT/WfxGzV5KIiLSPiNhiEBGR9qNgEBGRWhQM\nIiJSi4JBRERqUTCIiEgtCgYREalFwSAiIrUoGEREpBYFg4iI1PL/AXjZ9llsBT1XAAAAAElFTkSu\nQmCC\n",
      "text/plain": [
       "<matplotlib.figure.Figure at 0x11bafb510>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "plt.semilogx(param, results)\n",
    "plt.grid(True)\n",
    "plt.title('Test accuracy by regularization beta')\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Dropout: 1.00\n",
      "Initialized\n",
      "Test accuracy: 91.6%\n",
      "Dropout: 0.95\n",
      "Initialized\n",
      "Test accuracy: 91.7%\n",
      "Dropout: 0.90\n",
      "Initialized\n",
      "Test accuracy: 91.9%\n",
      "Dropout: 0.85\n",
      "Initialized\n",
      "Test accuracy: 91.9%\n",
      "Dropout: 0.80\n",
      "Initialized\n",
      "Test accuracy: 92.0%\n",
      "Dropout: 0.75\n",
      "Initialized\n",
      "Test accuracy: 92.0%\n",
      "Dropout: 0.70\n",
      "Initialized\n",
      "Test accuracy: 92.0%\n",
      "Dropout: 0.65\n",
      "Initialized\n",
      "Test accuracy: 91.8%\n",
      "Dropout: 0.60\n",
      "Initialized\n",
      "Test accuracy: 91.9%\n",
      "Dropout: 0.55\n",
      "Initialized\n",
      "Test accuracy: 91.8%\n",
      "Dropout: 0.50\n",
      "Initialized\n",
      "Test accuracy: 92.0%\n",
      "Dropout: 0.45\n",
      "Initialized\n",
      "Test accuracy: 91.6%\n",
      "Dropout: 0.40\n",
      "Initialized\n",
      "Test accuracy: 91.4%\n",
      "Dropout: 0.35\n",
      "Initialized\n",
      "Test accuracy: 91.2%\n",
      "Dropout: 0.30\n",
      "Initialized\n",
      "Test accuracy: 91.0%\n"
     ]
    }
   ],
   "source": [
    "param = [i for i in np.arange(1, 0.25, -0.05)]\n",
    "results = []\n",
    "\n",
    "for i in param:\n",
    "    print(\"Dropout: %0.2f\" % i)\n",
    "    results.append(run_session(graph, 2000, beta=0, dropout=i, verbose=False))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true,
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "plt.semilogx(param, results)\n",
    "plt.grid(True)\n",
    "plt.title('Test accuracy by dropout keep probability')\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "BATCH_SIZE = 128\n",
    "HIDDEN_NODES_1 = 1024\n",
    "HIDDEN_NODES_2 = 1024\n",
    "\n",
    "graph = tf.Graph()\n",
    "with graph.as_default():\n",
    "\n",
    "    # Input data. For the training data, we use a placeholder that will be fed\n",
    "    # at run time with a training minibatch.\n",
    "    tf_train_dataset = tf.placeholder(tf.float32, shape=(BATCH_SIZE, IMAGE_PIXELS))\n",
    "    tf_train_labels = tf.placeholder(tf.float32, shape=(BATCH_SIZE, NUM_LABELS))\n",
    "    tf_valid_dataset = tf.constant(valid_dataset)\n",
    "    tf_test_dataset = tf.constant(test_dataset)\n",
    "    l2_beta = tf.placeholder(tf.float32)  # factor for regularization penalty\n",
    "    do_keep_prob = tf.placeholder(tf.float32)  # dropout keep probability\n",
    "    global_step = tf.Variable(0)  # count the number of steps taken.\n",
    "  \n",
    "    # Variables.\n",
    "    w1 = tf.Variable(tf.truncated_normal([IMAGE_PIXELS, HIDDEN_NODES_1],\n",
    "                                         stddev=1.0/np.sqrt(float(IMAGE_PIXELS))))\n",
    "    b1 = tf.Variable(tf.zeros([HIDDEN_NODES_1]))\n",
    "    w2 = tf.Variable(tf.truncated_normal([HIDDEN_NODES_1, HIDDEN_NODES_2],\n",
    "                                         stddev=1.0/np.sqrt(float(IMAGE_PIXELS))))\n",
    "    b2 = tf.Variable(tf.zeros([HIDDEN_NODES_2]))\n",
    "    w3 = tf.Variable(tf.truncated_normal([HIDDEN_NODES_2, NUM_LABELS],\n",
    "                                         stddev=1.0/np.sqrt(float(IMAGE_PIXELS))))\n",
    "    b3 = tf.Variable(tf.zeros([NUM_LABELS]))\n",
    "  \n",
    "    # Training computation.\n",
    "    hidden1 = tf.nn.dropout(tf.nn.relu(tf.matmul(tf_train_dataset, w1) + b1), do_keep_prob)\n",
    "    hidden2 = tf.nn.dropout(tf.nn.relu(tf.matmul(hidden1, w2) + b2), do_keep_prob)\n",
    "    logits = tf.matmul(hidden2, w3) + b3\n",
    "    loss = tf.reduce_mean(tf.nn.softmax_cross_entropy_with_logits(labels=tf_train_labels, logits=logits)) \\\n",
    "            + l2_beta * (tf.nn.l2_loss(w1) + tf.nn.l2_loss(w2) + tf.nn.l2_loss(w3))\n",
    "  \n",
    "    # Optimizer. \n",
    "    learning_rate = tf.train.exponential_decay(0.1, global_step, decay_steps=200, decay_rate=0.95)\n",
    "    optimizer = tf.train.GradientDescentOptimizer(learning_rate).minimize(loss, global_step=global_step)\n",
    "  \n",
    "    # Predictions for the training, validation, and test data.\n",
    "    train_prediction = tf.nn.softmax(logits)\n",
    "    hidden1_valid = tf.nn.relu(tf.matmul(tf_valid_dataset, w1) + b1)\n",
    "    hidden2_valid = tf.nn.relu(tf.matmul(hidden1_valid, w2) + b2)\n",
    "    valid_prediction = tf.nn.softmax(tf.matmul(hidden2_valid, w3) + b3)\n",
    "    hidden1_test = tf.nn.relu(tf.matmul(tf_test_dataset, w1) + b1)\n",
    "    hidden2_test = tf.nn.relu(tf.matmul(hidden1_test, w2) + b2)\n",
    "    test_prediction = tf.nn.softmax(tf.matmul(hidden2_test, w3) + b3)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "run_session(graph, 8000, beta=1e-3, dropout=0.75, verbose=True)"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 2",
   "language": "python",
   "name": "python2"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 2
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython2",
   "version": "2.7.13"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}