l08c09_forecasting_with_cnn.ipynb

{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "accelerator": "GPU",
    "colab": {
      "name": "l08c09_forecasting_with_cnn.ipynb",
      "provenance": [],
      "private_outputs": true,
      "collapsed_sections": [
        "vidayERjaO5q"
      ],
      "toc_visible": true
    },
    "kernelspec": {
      "display_name": "Python 3",
      "name": "python3"
    }
  },
  "cells": [
    {
      "cell_type": "markdown",
      "metadata": {
        "colab_type": "text",
        "id": "Za8-Nr5k11fh"
      },
      "source": [
        "##### Copyright 2018 The TensorFlow Authors."
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "cellView": "form",
        "colab_type": "code",
        "id": "Eq10uEbw0E4l",
        "colab": {}
      },
      "source": [
        "#@title Licensed under the Apache License, Version 2.0 (the \"License\");\n",
        "# you may not use this file except in compliance with the License.\n",
        "# You may obtain a copy of the License at\n",
        "#\n",
        "# https://www.apache.org/licenses/LICENSE-2.0\n",
        "#\n",
        "# Unless required by applicable law or agreed to in writing, software\n",
        "# distributed under the License is distributed on an \"AS IS\" BASIS,\n",
        "# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n",
        "# See the License for the specific language governing permissions and\n",
        "# limitations under the License."
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "colab_type": "text",
        "id": "Tt5S6SiPi7ze"
      },
      "source": [
        "# Forecasting with a CNN"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "colab_type": "text",
        "id": "W2iENc3Nh6g7"
      },
      "source": [
        "<table class=\"tfo-notebook-buttons\" align=\"left\">\n",
        "  <td>\n",
        "    <a target=\"_blank\" href=\"https://colab.research.google.com/github/tensorflow/examples/blob/master/courses/udacity_intro_to_tensorflow_for_deep_learning/l08c09_forecasting_with_cnn.ipynb\"><img src=\"https://www.tensorflow.org/images/colab_logo_32px.png\" />Run in Google Colab</a>\n",
        "  </td>\n",
        "  <td>\n",
        "    <a target=\"_blank\" href=\"https://github.com/tensorflow/examples/blob/master/courses/udacity_intro_to_tensorflow_for_deep_learning/l08c09_forecasting_with_cnn.ipynb\"><img src=\"https://www.tensorflow.org/images/GitHub-Mark-32px.png\" />View source on GitHub</a>\n",
        "  </td>\n",
        "</table>"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "colab_type": "text",
        "id": "vidayERjaO5q"
      },
      "source": [
        "## Setup"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab_type": "code",
        "id": "gqWabzlJ63nL",
        "colab": {}
      },
      "source": [
        "import numpy as np\n",
        "import matplotlib.pyplot as plt\n",
        "import tensorflow as tf\n",
        "\n",
        "keras = tf.keras"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab_type": "code",
        "id": "cg1hfKCPldZG",
        "colab": {}
      },
      "source": [
        "def plot_series(time, series, format=\"-\", start=0, end=None, label=None):\n",
        "    plt.plot(time[start:end], series[start:end], format, label=label)\n",
        "    plt.xlabel(\"Time\")\n",
        "    plt.ylabel(\"Value\")\n",
        "    if label:\n",
        "        plt.legend(fontsize=14)\n",
        "    plt.grid(True)\n",
        "\n",
        "\n",
        "def trend(time, slope=0):\n",
        "    return slope * time\n",
        "  \n",
        "  \n",
        "def seasonal_pattern(season_time):\n",
        "    \"\"\"Just an arbitrary pattern, you can change it if you wish\"\"\"\n",
        "    return np.where(season_time < 0.4,\n",
        "                    np.cos(season_time * 2 * np.pi),\n",
        "                    1 / np.exp(3 * season_time))\n",
        "\n",
        "  \n",
        "def seasonality(time, period, amplitude=1, phase=0):\n",
        "    \"\"\"Repeats the same pattern at each period\"\"\"\n",
        "    season_time = ((time + phase) % period) / period\n",
        "    return amplitude * seasonal_pattern(season_time)\n",
        "  \n",
        "  \n",
        "def white_noise(time, noise_level=1, seed=None):\n",
        "    rnd = np.random.RandomState(seed)\n",
        "    return rnd.randn(len(time)) * noise_level\n",
        "  \n",
        "\n",
        "def seq2seq_window_dataset(series, window_size, batch_size=32,\n",
        "                           shuffle_buffer=1000):\n",
        "    series = tf.expand_dims(series, axis=-1)\n",
        "    ds = tf.data.Dataset.from_tensor_slices(series)\n",
        "    ds = ds.window(window_size + 1, shift=1, drop_remainder=True)\n",
        "    ds = ds.flat_map(lambda w: w.batch(window_size + 1))\n",
        "    ds = ds.shuffle(shuffle_buffer)\n",
        "    ds = ds.map(lambda w: (w[:-1], w[1:]))\n",
        "    return ds.batch(batch_size).prefetch(1)\n",
        "  \n",
        "\n",
        "def model_forecast(model, series, window_size):\n",
        "    ds = tf.data.Dataset.from_tensor_slices(series)\n",
        "    ds = ds.window(window_size, shift=1, drop_remainder=True)\n",
        "    ds = ds.flat_map(lambda w: w.batch(window_size))\n",
        "    ds = ds.batch(32).prefetch(1)\n",
        "    forecast = model.predict(ds)\n",
        "    return forecast"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab_type": "code",
        "id": "iL2DDjV3lel6",
        "colab": {}
      },
      "source": [
        "time = np.arange(4 * 365 + 1)\n",
        "\n",
        "slope = 0.05\n",
        "baseline = 10\n",
        "amplitude = 40\n",
        "series = baseline + trend(time, slope) + seasonality(time, period=365, amplitude=amplitude)\n",
        "\n",
        "noise_level = 5\n",
        "noise = white_noise(time, noise_level, seed=42)\n",
        "\n",
        "series += noise\n",
        "\n",
        "plt.figure(figsize=(10, 6))\n",
        "plot_series(time, series)\n",
        "plt.show()"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab_type": "code",
        "id": "Zmp1JXKxk9Vb",
        "colab": {}
      },
      "source": [
        "split_time = 1000\n",
        "time_train = time[:split_time]\n",
        "x_train = series[:split_time]\n",
        "time_valid = time[split_time:]\n",
        "x_valid = series[split_time:]"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "colab_type": "text",
        "id": "DI2GlupZ8OJW"
      },
      "source": [
        "## Preprocessing With 1D-Convolutional Layers"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab_type": "code",
        "id": "6GFE82ci8OJW",
        "colab": {}
      },
      "source": [
        "keras.backend.clear_session()\n",
        "tf.random.set_seed(42)\n",
        "np.random.seed(42)\n",
        "\n",
        "window_size = 30\n",
        "train_set = seq2seq_window_dataset(x_train, window_size,\n",
        "                                   batch_size=128)\n",
        "\n",
        "model = keras.models.Sequential([\n",
        "  keras.layers.Conv1D(filters=32, kernel_size=5,\n",
        "                      strides=1, padding=\"causal\",\n",
        "                      activation=\"relu\",\n",
        "                      input_shape=[None, 1]),\n",
        "  keras.layers.LSTM(32, return_sequences=True),\n",
        "  keras.layers.LSTM(32, return_sequences=True),\n",
        "  keras.layers.Dense(1),\n",
        "  keras.layers.Lambda(lambda x: x * 200)\n",
        "])\n",
        "lr_schedule = keras.callbacks.LearningRateScheduler(\n",
        "    lambda epoch: 1e-8 * 10**(epoch / 20))\n",
        "optimizer = keras.optimizers.SGD(lr=1e-8, momentum=0.9)\n",
        "model.compile(loss=keras.losses.Huber(),\n",
        "              optimizer=optimizer,\n",
        "              metrics=[\"mae\"])\n",
        "history = model.fit(train_set, epochs=100, callbacks=[lr_schedule])"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab_type": "code",
        "id": "Uungw0H58OJX",
        "colab": {}
      },
      "source": [
        "plt.semilogx(history.history[\"lr\"], history.history[\"loss\"])\n",
        "plt.axis([1e-8, 1e-4, 0, 30])"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab_type": "code",
        "id": "UG7cO0yr8OJY",
        "colab": {}
      },
      "source": [
        "keras.backend.clear_session()\n",
        "tf.random.set_seed(42)\n",
        "np.random.seed(42)\n",
        "\n",
        "window_size = 30\n",
        "train_set = seq2seq_window_dataset(x_train, window_size,\n",
        "                                   batch_size=128)\n",
        "valid_set = seq2seq_window_dataset(x_valid, window_size,\n",
        "                                   batch_size=128)\n",
        "\n",
        "model = keras.models.Sequential([\n",
        "  keras.layers.Conv1D(filters=32, kernel_size=5,\n",
        "                      strides=1, padding=\"causal\",\n",
        "                      activation=\"relu\",\n",
        "                      input_shape=[None, 1]),\n",
        "  keras.layers.LSTM(32, return_sequences=True),\n",
        "  keras.layers.LSTM(32, return_sequences=True),\n",
        "  keras.layers.Dense(1),\n",
        "  keras.layers.Lambda(lambda x: x * 200)\n",
        "])\n",
        "optimizer = keras.optimizers.SGD(lr=1e-5, momentum=0.9)\n",
        "model.compile(loss=keras.losses.Huber(),\n",
        "              optimizer=optimizer,\n",
        "              metrics=[\"mae\"])\n",
        "\n",
        "model_checkpoint = keras.callbacks.ModelCheckpoint(\n",
        "    \"my_checkpoint.h5\", save_best_only=True)\n",
        "early_stopping = keras.callbacks.EarlyStopping(patience=50)\n",
        "model.fit(train_set, epochs=500,\n",
        "          validation_data=valid_set,\n",
        "          callbacks=[early_stopping, model_checkpoint])"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab_type": "code",
        "id": "BlqzLwfn8OJa",
        "colab": {}
      },
      "source": [
        "model = keras.models.load_model(\"my_checkpoint.h5\")"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab_type": "code",
        "id": "Pj0rpT-48OJc",
        "colab": {}
      },
      "source": [
        "rnn_forecast = model_forecast(model, series[:,  np.newaxis], window_size)\n",
        "rnn_forecast = rnn_forecast[split_time - window_size:-1, -1, 0]"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab_type": "code",
        "id": "3vnDU8wm8OJd",
        "colab": {}
      },
      "source": [
        "plt.figure(figsize=(10, 6))\n",
        "plot_series(time_valid, x_valid)\n",
        "plot_series(time_valid, rnn_forecast)"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab_type": "code",
        "id": "W6tWOoE88OJe",
        "colab": {}
      },
      "source": [
        "keras.metrics.mean_absolute_error(x_valid, rnn_forecast).numpy()"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "colab_type": "text",
        "id": "kfPTmghd8OJe"
      },
      "source": [
        "## Fully Convolutional Forecasting"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab_type": "code",
        "id": "4-cPF5CX8OJf",
        "colab": {}
      },
      "source": [
        "keras.backend.clear_session()\n",
        "tf.random.set_seed(42)\n",
        "np.random.seed(42)\n",
        "\n",
        "window_size = 64\n",
        "train_set = seq2seq_window_dataset(x_train, window_size,\n",
        "                                   batch_size=128)\n",
        "\n",
        "model = keras.models.Sequential()\n",
        "model.add(keras.layers.InputLayer(input_shape=[None, 1]))\n",
        "for dilation_rate in (1, 2, 4, 8, 16, 32):\n",
        "    model.add(\n",
        "      keras.layers.Conv1D(filters=32,\n",
        "                          kernel_size=2,\n",
        "                          strides=1,\n",
        "                          dilation_rate=dilation_rate,\n",
        "                          padding=\"causal\",\n",
        "                          activation=\"relu\")\n",
        "    )\n",
        "model.add(keras.layers.Conv1D(filters=1, kernel_size=1))\n",
        "lr_schedule = keras.callbacks.LearningRateScheduler(\n",
        "    lambda epoch: 1e-4 * 10**(epoch / 30))\n",
        "optimizer = keras.optimizers.Adam(lr=1e-4)\n",
        "model.compile(loss=keras.losses.Huber(),\n",
        "              optimizer=optimizer,\n",
        "              metrics=[\"mae\"])\n",
        "history = model.fit(train_set, epochs=100, callbacks=[lr_schedule])"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab_type": "code",
        "id": "GfWVZ8k-8OJf",
        "colab": {}
      },
      "source": [
        "plt.semilogx(history.history[\"lr\"], history.history[\"loss\"])\n",
        "plt.axis([1e-4, 1e-1, 0, 30])"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab_type": "code",
        "id": "WVrxlzbk8OJg",
        "colab": {}
      },
      "source": [
        "keras.backend.clear_session()\n",
        "tf.random.set_seed(42)\n",
        "np.random.seed(42)\n",
        "\n",
        "window_size = 64\n",
        "train_set = seq2seq_window_dataset(x_train, window_size,\n",
        "                                   batch_size=128)\n",
        "valid_set = seq2seq_window_dataset(x_valid, window_size,\n",
        "                                   batch_size=128)\n",
        "\n",
        "model = keras.models.Sequential()\n",
        "model.add(keras.layers.InputLayer(input_shape=[None, 1]))\n",
        "for dilation_rate in (1, 2, 4, 8, 16, 32):\n",
        "    model.add(\n",
        "      keras.layers.Conv1D(filters=32,\n",
        "                          kernel_size=2,\n",
        "                          strides=1,\n",
        "                          dilation_rate=dilation_rate,\n",
        "                          padding=\"causal\",\n",
        "                          activation=\"relu\")\n",
        "    )\n",
        "model.add(keras.layers.Conv1D(filters=1, kernel_size=1))\n",
        "optimizer = keras.optimizers.Adam(lr=3e-4)\n",
        "model.compile(loss=keras.losses.Huber(),\n",
        "              optimizer=optimizer,\n",
        "              metrics=[\"mae\"])\n",
        "\n",
        "model_checkpoint = keras.callbacks.ModelCheckpoint(\n",
        "    \"my_checkpoint.h5\", save_best_only=True)\n",
        "early_stopping = keras.callbacks.EarlyStopping(patience=50)\n",
        "history = model.fit(train_set, epochs=500,\n",
        "                    validation_data=valid_set,\n",
        "                    callbacks=[early_stopping, model_checkpoint])"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab_type": "code",
        "id": "eNwWZB0d8OJh",
        "colab": {}
      },
      "source": [
        "model = keras.models.load_model(\"my_checkpoint.h5\")"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab_type": "code",
        "id": "PgYwn9VM8OJi",
        "colab": {}
      },
      "source": [
        "cnn_forecast = model_forecast(model, series[..., np.newaxis], window_size)\n",
        "cnn_forecast = cnn_forecast[split_time - window_size:-1, -1, 0]"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab_type": "code",
        "id": "MCgshNPx8OJi",
        "colab": {}
      },
      "source": [
        "plt.figure(figsize=(10, 6))\n",
        "plot_series(time_valid, x_valid)\n",
        "plot_series(time_valid, cnn_forecast)"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab_type": "code",
        "id": "epK1gFEN8OJj",
        "colab": {}
      },
      "source": [
        "keras.metrics.mean_absolute_error(x_valid, cnn_forecast).numpy()"
      ],
      "execution_count": 0,
      "outputs": []
    }
  ]
}