bigdata_notebook.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Team Members:\n",
    "\n",
    "    Fayme Shahriar, 218203184\n",
    "    Abdur Rahman, 218203240\n",
    "    MD Naiem Siddique, 219203013\n",
    "    Tanvir Ahmed, 217100799\n",
    "    Md. Shariful Islam, 218100924"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 165,
   "metadata": {},
   "outputs": [],
   "source": [
    "from pyspark import SparkContext\n",
    "from pyspark.sql import SQLContext\n",
    "from pyspark.sql import SparkSession\n",
    "from pyspark.sql.functions import *\n",
    "from pyspark.sql.types import *\n",
    "import os\n",
    "from datetime import datetime \n",
    "from functools import reduce\n",
    "from operator import add\n",
    "import pandas as pd\n",
    "from pyspark.ml.regression import LinearRegression\n",
    "from pyspark.ml.feature import VectorAssembler\n",
    "from pyspark.ml.evaluation import RegressionEvaluator\n",
    "import matplotlib\n",
    "import matplotlib.pyplot as plt\n",
    "from sklearn import linear_model\n",
    "%matplotlib inline "
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "**Linear Rigression using Pyspark LinearRegression**"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 257,
   "metadata": {},
   "outputs": [],
   "source": [
    "def predict(df):\n",
    "    t_df =  df.sort('Date_value', ascending=False).limit(20)\n",
    "    t_df = t_df.sort('Date_value')\n",
    "    vectorAssembler = VectorAssembler(inputCols = ['Date_value'], outputCol = 'features')\n",
    "    train_df = vectorAssembler.transform(t_df).limit(18)\n",
    "    train_df = train_df.select(['features', 'Cases'])\n",
    "    \n",
    "    lr = LinearRegression(featuresCol = 'features', labelCol='Cases', maxIter=10, regParam=0.3, elasticNetParam=0.8)\n",
    "\n",
    "    # Fit the model\n",
    "    lrModel = lr.fit(train_df)\n",
    "\n",
    "    # Print the coefficients and intercept for linear regression\n",
    "    print(\"Coefficients: %s\" % str(lrModel.coefficients))\n",
    "    print(\"Intercept: %s\" % str(lrModel.intercept))\n",
    "\n",
    "    # Summarize the model over the training set and print out some metrics\n",
    "    trainingSummary = lrModel.summary\n",
    "    print(\"numIterations: %d\" % trainingSummary.totalIterations)\n",
    "    print(\"objectiveHistory: %s\" % str(trainingSummary.objectiveHistory))\n",
    "    trainingSummary.residuals.show()\n",
    "    print(\"RMSE: %f\" % trainingSummary.rootMeanSquaredError)\n",
    "    print(\"r2: %f\" % trainingSummary.r2)\n",
    "    \n",
    "    test_df = vectorAssembler.transform(t_df.sort('Date_Value', ascending=False)).limit(2)\n",
    "    test_df.show()\n",
    "    lr_predictions = lrModel.transform(test_df.select(col(\"features\")))\n",
    "    lr_predictions.show()\n",
    "    \n",
    "    print(\"Difference in 3/22/20: \" + str(test_df.select(col(\"Cases\")).collect()[1][0] - lr_predictions.select(col(\"prediction\")).collect()[1][0]))\n",
    "    print(\"Difference in 3/23/20: \" + str(test_df.select(col(\"Cases\")).collect()[0][0] - lr_predictions.select(col(\"prediction\")).collect()[0][0]))\n",
    "    print()\n",
    "    print()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "**Filter data to find only country with Australia**"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 166,
   "metadata": {},
   "outputs": [],
   "source": [
    "def filterData(df):\n",
    "      return df.filter(col(\"Country/Region\") == \"Australia\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "**Transpose data**"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 167,
   "metadata": {},
   "outputs": [],
   "source": [
    "def t_data(df):\n",
    "\n",
    "    # Filter dtypes and split into column names and type description\n",
    "    cols, dtypes = zip(*((c, t) for (c, t) in df.dtypes))\n",
    "    # Spark SQL supports only homogeneous columns\n",
    "    assert len(set(dtypes)) == 1, \"All columns have to be of the same type\"\n",
    "\n",
    "    # Create and explode an array of (column_name, column_value) structs\n",
    "    kvs = explode(array([\n",
    "      struct(lit(c).alias(\"Date\"), col(c).alias(\"Cases\")) for c in cols\n",
    "    ])).alias(\"kvs\")\n",
    "\n",
    "    return df.select([kvs]).select([\"kvs.Date\", \"kvs.Cases\"])"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "**Filter column**"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 168,
   "metadata": {},
   "outputs": [],
   "source": [
    "def filter_province(province):\n",
    "    return df.filter(col(\"Province/State\") == province)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "**Relative path**"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 169,
   "metadata": {},
   "outputs": [],
   "source": [
    "dirname = os.path.abspath('')\n",
    "filepath = os.path.join(dirname, 'time_series_19-covid-Confirmed_archived_0325.csv')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "**Read file**"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 170,
   "metadata": {},
   "outputs": [],
   "source": [
    "sc = SparkContext.getOrCreate()\n",
    "sqlContext = SQLContext(sc)\n",
    "df = sqlContext.read.option(\"inferSchema\", \"true\").option(\"header\", \"true\").csv(filepath)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "**Filter data to find only country with Australia**"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 171,
   "metadata": {
    "scrolled": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Number of rows in df -> 9\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Province/State</th>\n",
       "      <th>Country/Region</th>\n",
       "      <th>Lat</th>\n",
       "      <th>Long</th>\n",
       "      <th>1/22/20</th>\n",
       "      <th>1/23/20</th>\n",
       "      <th>1/24/20</th>\n",
       "      <th>1/25/20</th>\n",
       "      <th>1/26/20</th>\n",
       "      <th>1/27/20</th>\n",
       "      <th>...</th>\n",
       "      <th>3/14/20</th>\n",
       "      <th>3/15/20</th>\n",
       "      <th>3/16/20</th>\n",
       "      <th>3/17/20</th>\n",
       "      <th>3/18/20</th>\n",
       "      <th>3/19/20</th>\n",
       "      <th>3/20/20</th>\n",
       "      <th>3/21/20</th>\n",
       "      <th>3/22/20</th>\n",
       "      <th>3/23/20</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <td>0</td>\n",
       "      <td>New South Wales</td>\n",
       "      <td>Australia</td>\n",
       "      <td>-33.8688</td>\n",
       "      <td>151.2093</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>3</td>\n",
       "      <td>4</td>\n",
       "      <td>...</td>\n",
       "      <td>112</td>\n",
       "      <td>134</td>\n",
       "      <td>171</td>\n",
       "      <td>210</td>\n",
       "      <td>267</td>\n",
       "      <td>307</td>\n",
       "      <td>353</td>\n",
       "      <td>436</td>\n",
       "      <td>533</td>\n",
       "      <td>533</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>1</td>\n",
       "      <td>Victoria</td>\n",
       "      <td>Australia</td>\n",
       "      <td>-37.8136</td>\n",
       "      <td>144.9631</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>...</td>\n",
       "      <td>49</td>\n",
       "      <td>57</td>\n",
       "      <td>71</td>\n",
       "      <td>94</td>\n",
       "      <td>121</td>\n",
       "      <td>121</td>\n",
       "      <td>121</td>\n",
       "      <td>229</td>\n",
       "      <td>296</td>\n",
       "      <td>296</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>2</td>\n",
       "      <td>Queensland</td>\n",
       "      <td>Australia</td>\n",
       "      <td>-28.0167</td>\n",
       "      <td>153.4000</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>46</td>\n",
       "      <td>61</td>\n",
       "      <td>68</td>\n",
       "      <td>78</td>\n",
       "      <td>94</td>\n",
       "      <td>144</td>\n",
       "      <td>184</td>\n",
       "      <td>221</td>\n",
       "      <td>221</td>\n",
       "      <td>221</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>3</td>\n",
       "      <td>South Australia</td>\n",
       "      <td>Australia</td>\n",
       "      <td>-34.9285</td>\n",
       "      <td>138.6007</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>19</td>\n",
       "      <td>20</td>\n",
       "      <td>29</td>\n",
       "      <td>29</td>\n",
       "      <td>37</td>\n",
       "      <td>42</td>\n",
       "      <td>50</td>\n",
       "      <td>67</td>\n",
       "      <td>100</td>\n",
       "      <td>100</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>4</td>\n",
       "      <td>From Diamond Princess</td>\n",
       "      <td>Australia</td>\n",
       "      <td>35.4437</td>\n",
       "      <td>139.6380</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>5</td>\n",
       "      <td>Western Australia</td>\n",
       "      <td>Australia</td>\n",
       "      <td>-31.9505</td>\n",
       "      <td>115.8605</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>17</td>\n",
       "      <td>17</td>\n",
       "      <td>28</td>\n",
       "      <td>31</td>\n",
       "      <td>35</td>\n",
       "      <td>52</td>\n",
       "      <td>64</td>\n",
       "      <td>90</td>\n",
       "      <td>120</td>\n",
       "      <td>120</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>6</td>\n",
       "      <td>Tasmania</td>\n",
       "      <td>Australia</td>\n",
       "      <td>-41.4545</td>\n",
       "      <td>145.9707</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>5</td>\n",
       "      <td>6</td>\n",
       "      <td>7</td>\n",
       "      <td>7</td>\n",
       "      <td>10</td>\n",
       "      <td>10</td>\n",
       "      <td>10</td>\n",
       "      <td>16</td>\n",
       "      <td>22</td>\n",
       "      <td>22</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>7</td>\n",
       "      <td>Northern Territory</td>\n",
       "      <td>Australia</td>\n",
       "      <td>-12.4634</td>\n",
       "      <td>130.8456</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>3</td>\n",
       "      <td>3</td>\n",
       "      <td>3</td>\n",
       "      <td>3</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>8</td>\n",
       "      <td>Australian Capital Territory</td>\n",
       "      <td>Australia</td>\n",
       "      <td>-35.4735</td>\n",
       "      <td>149.0124</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>2</td>\n",
       "      <td>2</td>\n",
       "      <td>3</td>\n",
       "      <td>4</td>\n",
       "      <td>6</td>\n",
       "      <td>9</td>\n",
       "      <td>19</td>\n",
       "      <td>19</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>9 rows × 66 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "                 Province/State Country/Region      Lat      Long  1/22/20  \\\n",
       "0               New South Wales      Australia -33.8688  151.2093        0   \n",
       "1                      Victoria      Australia -37.8136  144.9631        0   \n",
       "2                    Queensland      Australia -28.0167  153.4000        0   \n",
       "3               South Australia      Australia -34.9285  138.6007        0   \n",
       "4         From Diamond Princess      Australia  35.4437  139.6380        0   \n",
       "5             Western Australia      Australia -31.9505  115.8605        0   \n",
       "6                      Tasmania      Australia -41.4545  145.9707        0   \n",
       "7            Northern Territory      Australia -12.4634  130.8456        0   \n",
       "8  Australian Capital Territory      Australia -35.4735  149.0124        0   \n",
       "\n",
       "   1/23/20  1/24/20  1/25/20  1/26/20  1/27/20  ...  3/14/20  3/15/20  \\\n",
       "0        0        0        0        3        4  ...      112      134   \n",
       "1        0        0        0        1        1  ...       49       57   \n",
       "2        0        0        0        0        0  ...       46       61   \n",
       "3        0        0        0        0        0  ...       19       20   \n",
       "4        0        0        0        0        0  ...        0        0   \n",
       "5        0        0        0        0        0  ...       17       17   \n",
       "6        0        0        0        0        0  ...        5        6   \n",
       "7        0        0        0        0        0  ...        1        1   \n",
       "8        0        0        0        0        0  ...        1        1   \n",
       "\n",
       "   3/16/20  3/17/20  3/18/20  3/19/20  3/20/20  3/21/20  3/22/20  3/23/20  \n",
       "0      171      210      267      307      353      436      533      533  \n",
       "1       71       94      121      121      121      229      296      296  \n",
       "2       68       78       94      144      184      221      221      221  \n",
       "3       29       29       37       42       50       67      100      100  \n",
       "4        0        0        0        0        0        0        0        0  \n",
       "5       28       31       35       52       64       90      120      120  \n",
       "6        7        7       10       10       10       16       22       22  \n",
       "7        1        1        1        1        3        3        3        3  \n",
       "8        2        2        3        4        6        9       19       19  \n",
       "\n",
       "[9 rows x 66 columns]"
      ]
     },
     "execution_count": 171,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df = filterData(df)\n",
    "df.toPandas()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "**Dropped columns which are outside the time range**"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 172,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Province/State</th>\n",
       "      <th>Country/Region</th>\n",
       "      <th>Lat</th>\n",
       "      <th>Long</th>\n",
       "      <th>2/1/20</th>\n",
       "      <th>2/2/20</th>\n",
       "      <th>2/3/20</th>\n",
       "      <th>2/4/20</th>\n",
       "      <th>2/5/20</th>\n",
       "      <th>2/6/20</th>\n",
       "      <th>...</th>\n",
       "      <th>3/14/20</th>\n",
       "      <th>3/15/20</th>\n",
       "      <th>3/16/20</th>\n",
       "      <th>3/17/20</th>\n",
       "      <th>3/18/20</th>\n",
       "      <th>3/19/20</th>\n",
       "      <th>3/20/20</th>\n",
       "      <th>3/21/20</th>\n",
       "      <th>3/22/20</th>\n",
       "      <th>3/23/20</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <td>0</td>\n",
       "      <td>New South Wales</td>\n",
       "      <td>Australia</td>\n",
       "      <td>-33.8688</td>\n",
       "      <td>151.2093</td>\n",
       "      <td>4</td>\n",
       "      <td>4</td>\n",
       "      <td>4</td>\n",
       "      <td>4</td>\n",
       "      <td>4</td>\n",
       "      <td>4</td>\n",
       "      <td>...</td>\n",
       "      <td>112</td>\n",
       "      <td>134</td>\n",
       "      <td>171</td>\n",
       "      <td>210</td>\n",
       "      <td>267</td>\n",
       "      <td>307</td>\n",
       "      <td>353</td>\n",
       "      <td>436</td>\n",
       "      <td>533</td>\n",
       "      <td>533</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>1</td>\n",
       "      <td>Victoria</td>\n",
       "      <td>Australia</td>\n",
       "      <td>-37.8136</td>\n",
       "      <td>144.9631</td>\n",
       "      <td>4</td>\n",
       "      <td>4</td>\n",
       "      <td>4</td>\n",
       "      <td>4</td>\n",
       "      <td>4</td>\n",
       "      <td>4</td>\n",
       "      <td>...</td>\n",
       "      <td>49</td>\n",
       "      <td>57</td>\n",
       "      <td>71</td>\n",
       "      <td>94</td>\n",
       "      <td>121</td>\n",
       "      <td>121</td>\n",
       "      <td>121</td>\n",
       "      <td>229</td>\n",
       "      <td>296</td>\n",
       "      <td>296</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>2</td>\n",
       "      <td>Queensland</td>\n",
       "      <td>Australia</td>\n",
       "      <td>-28.0167</td>\n",
       "      <td>153.4000</td>\n",
       "      <td>3</td>\n",
       "      <td>2</td>\n",
       "      <td>2</td>\n",
       "      <td>3</td>\n",
       "      <td>3</td>\n",
       "      <td>4</td>\n",
       "      <td>...</td>\n",
       "      <td>46</td>\n",
       "      <td>61</td>\n",
       "      <td>68</td>\n",
       "      <td>78</td>\n",
       "      <td>94</td>\n",
       "      <td>144</td>\n",
       "      <td>184</td>\n",
       "      <td>221</td>\n",
       "      <td>221</td>\n",
       "      <td>221</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>3</td>\n",
       "      <td>South Australia</td>\n",
       "      <td>Australia</td>\n",
       "      <td>-34.9285</td>\n",
       "      <td>138.6007</td>\n",
       "      <td>1</td>\n",
       "      <td>2</td>\n",
       "      <td>2</td>\n",
       "      <td>2</td>\n",
       "      <td>2</td>\n",
       "      <td>2</td>\n",
       "      <td>...</td>\n",
       "      <td>19</td>\n",
       "      <td>20</td>\n",
       "      <td>29</td>\n",
       "      <td>29</td>\n",
       "      <td>37</td>\n",
       "      <td>42</td>\n",
       "      <td>50</td>\n",
       "      <td>67</td>\n",
       "      <td>100</td>\n",
       "      <td>100</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>4</td>\n",
       "      <td>From Diamond Princess</td>\n",
       "      <td>Australia</td>\n",
       "      <td>35.4437</td>\n",
       "      <td>139.6380</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>5</td>\n",
       "      <td>Western Australia</td>\n",
       "      <td>Australia</td>\n",
       "      <td>-31.9505</td>\n",
       "      <td>115.8605</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>17</td>\n",
       "      <td>17</td>\n",
       "      <td>28</td>\n",
       "      <td>31</td>\n",
       "      <td>35</td>\n",
       "      <td>52</td>\n",
       "      <td>64</td>\n",
       "      <td>90</td>\n",
       "      <td>120</td>\n",
       "      <td>120</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>6</td>\n",
       "      <td>Tasmania</td>\n",
       "      <td>Australia</td>\n",
       "      <td>-41.4545</td>\n",
       "      <td>145.9707</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>5</td>\n",
       "      <td>6</td>\n",
       "      <td>7</td>\n",
       "      <td>7</td>\n",
       "      <td>10</td>\n",
       "      <td>10</td>\n",
       "      <td>10</td>\n",
       "      <td>16</td>\n",
       "      <td>22</td>\n",
       "      <td>22</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>7</td>\n",
       "      <td>Northern Territory</td>\n",
       "      <td>Australia</td>\n",
       "      <td>-12.4634</td>\n",
       "      <td>130.8456</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>3</td>\n",
       "      <td>3</td>\n",
       "      <td>3</td>\n",
       "      <td>3</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>8</td>\n",
       "      <td>Australian Capital Territory</td>\n",
       "      <td>Australia</td>\n",
       "      <td>-35.4735</td>\n",
       "      <td>149.0124</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>2</td>\n",
       "      <td>2</td>\n",
       "      <td>3</td>\n",
       "      <td>4</td>\n",
       "      <td>6</td>\n",
       "      <td>9</td>\n",
       "      <td>19</td>\n",
       "      <td>19</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>9 rows × 56 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "                 Province/State Country/Region      Lat      Long  2/1/20  \\\n",
       "0               New South Wales      Australia -33.8688  151.2093       4   \n",
       "1                      Victoria      Australia -37.8136  144.9631       4   \n",
       "2                    Queensland      Australia -28.0167  153.4000       3   \n",
       "3               South Australia      Australia -34.9285  138.6007       1   \n",
       "4         From Diamond Princess      Australia  35.4437  139.6380       0   \n",
       "5             Western Australia      Australia -31.9505  115.8605       0   \n",
       "6                      Tasmania      Australia -41.4545  145.9707       0   \n",
       "7            Northern Territory      Australia -12.4634  130.8456       0   \n",
       "8  Australian Capital Territory      Australia -35.4735  149.0124       0   \n",
       "\n",
       "   2/2/20  2/3/20  2/4/20  2/5/20  2/6/20  ...  3/14/20  3/15/20  3/16/20  \\\n",
       "0       4       4       4       4       4  ...      112      134      171   \n",
       "1       4       4       4       4       4  ...       49       57       71   \n",
       "2       2       2       3       3       4  ...       46       61       68   \n",
       "3       2       2       2       2       2  ...       19       20       29   \n",
       "4       0       0       0       0       0  ...        0        0        0   \n",
       "5       0       0       0       0       0  ...       17       17       28   \n",
       "6       0       0       0       0       0  ...        5        6        7   \n",
       "7       0       0       0       0       0  ...        1        1        1   \n",
       "8       0       0       0       0       0  ...        1        1        2   \n",
       "\n",
       "   3/17/20  3/18/20  3/19/20  3/20/20  3/21/20  3/22/20  3/23/20  \n",
       "0      210      267      307      353      436      533      533  \n",
       "1       94      121      121      121      229      296      296  \n",
       "2       78       94      144      184      221      221      221  \n",
       "3       29       37       42       50       67      100      100  \n",
       "4        0        0        0        0        0        0        0  \n",
       "5       31       35       52       64       90      120      120  \n",
       "6        7       10       10       10       16       22       22  \n",
       "7        1        1        1        3        3        3        3  \n",
       "8        2        3        4        6        9       19       19  \n",
       "\n",
       "[9 rows x 56 columns]"
      ]
     },
     "execution_count": 172,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "names = df.schema.names\n",
    "start_date = datetime.strptime(\"1/31/20\", '%m/%d/%y')\n",
    "end_date = datetime.strptime(\"3/24/20\", '%m/%d/%y')\n",
    "#len(names)\n",
    "for column in names:\n",
    "    if column != \"Province/State\" and column != \"Country/Region\" and column != \"Lat\" and column != \"Long\":\n",
    "        col_name = datetime.strptime(column, '%m/%d/%y')\n",
    "        if not(col_name > start_date and col_name < end_date):\n",
    "            df = df.drop(column)\n",
    "    \n",
    "\n",
    "\n",
    "df.toPandas()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 173,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<matplotlib.axes._subplots.AxesSubplot at 0x7feb1ad74410>"
      ]
     },
     "execution_count": 173,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAA3YAAAEvCAYAAAAJs1ObAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4xLjEsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy8QZhcZAAAgAElEQVR4nO3deZxddX3/8ddn9kz2ZbIDSQw7soawSF1QECwCrVqxqEGxoFVrq7aitT9rbX/F2rq0KspPVFARcAOqUAiIW5ElkAAyScgGZJ1M1sk2k1m+vz/uSRiSSTLJTObcO/N6Ph73cc/5nu8593Pz+ObOfd+zRUoJSZIkSVLpKsu7AEmSJElSzxjsJEmSJKnEGewkSZIkqcQZ7CRJkiSpxBnsJEmSJKnEGewkSZIkqcRV5F3A/owZMyZNmTIl7zIkSZIkKRdPPPHEupRS3YH6FXWwmzJlCnPmzMm7DEmSJEnKRUS80J1+HoopSZIkSSXOYCdJkiRJJc5gJ0mSJEklzmAnSZIkSSXOYCdJkiRJJc5gJ0mSJEklzmAnSZIkSSXOYCdJkiRJJc5gJ0mSJEklriLvAiRJkiSpP1rSuJXHlm3ok9cy2EmSJEnSYfDhW+dSv7qpT17LYCdJkiRJvey5hi3Ur27ib994LG85ffIhb2fC57vXz2AnSZIkSb3szrkrKS8L3n7mEYwZUn3YX8+Lp0iSJElSL+roSNw1bxXnTR/TJ6EODHaSJEmS1KvmvLCRlZt2cPlpE/vsNQ12kiRJktSL7py3kkGV5Vx4wvg+e02DnSRJkiT1kp1tHdzzzGouOGEcg6v77pImBjtJkiRJ6iW/fq6RTdtb+ZPTJvXp6xrsJEmSJKmX3DlvJaMGV3He0WP69HUNdpIkSZLUC7Y0t/JAfQOXnDyByvK+jVoGO0mSJEnqBfc920BLWweXndq3h2GCwU6SJEmSesVd81Zy5KhaTj9yRJ+/tsFOkiRJknpobVMz/7t4HZedOpGI6PPXN9hJkiRJUg/d/dQqOhK5HIYJBjtJkiRJ6rG75q3ipEnDmD52SC6vb7CTJEmSpB5Y0riVZ1Zu5vKc9taBwU6SJEmSeuSuuSuJgDefMjG3Ggx2kiRJknSIUkrcOW8V575iNOOG1eRWxwGDXUR8OyLWRsQfOrV9ISIWRMTTEfGziBjRadknI2JxRCyMiDd2ar8oa1scEdf1/luRJEmSpL41d/kmXtywPdfDMKF7e+y+C1y0R9ts4KSU0snAc8AnASLiBOAK4MRsna9HRHlElANfAy4GTgDekfWVJEmSpJJ119yVVFeUcdFJ43Ot44DBLqX0G2DDHm33p5TastlHgMnZ9GXAbSmllpTSMmAxMDN7LE4pLU0p7QRuy/pKkiRJUklqbe/g50+v5g3Hj2NoTWWutfTGOXbvBe7NpicByzstW5G17atdkiRJkkrS7xavY/22nVx2an4XTdmlR8EuIv4eaAN+sKupi25pP+1dbfOaiJgTEXMaGxt7Up4kSZIkHTZ3zl3J8EGVvPbYsXmXcujBLiJmAZcAV6aUdoW0FcARnbpNBlbtp30vKaUbU0ozUkoz6urqDrU8SZIkSTpstrW0cf+zDbzplROoqsj/ZgOHVEFEXAR8Arg0pbS906K7gSsiojoipgJHA48BjwNHR8TUiKiicIGVu3tWuiRJkiTlY3Z9Azta27m8CA7DBKg4UIeI+CHwWmBMRKwAPkPhKpjVwOyIAHgkpfT+lNKzEXEHUE/hEM0PppTas+18CLgPKAe+nVJ69jC8H0mSJEk67O6ct5KJw2s4c8qovEsBuhHsUkrv6KL5pv30/xfgX7povwe456CqkyRJkqQis25rC79dtI6/+KNplJV1dTmRvpf/waCSJEmSVEJ+8fRq2jsSl59WHIdhgsFOkiRJkg7KnfNWctz4oRw3fljepexmsJMkSZKkbnph/TbmvriJy08rrttyG+wkSZIkqZvumreKCLj0lOI5DBMMdpIkSZLULSkl7py7kplTRjFxxKC8y3kZg50kSZIkdcMzKzezdN22ojsMEwx2kiRJktQtd85dRVV5GW86aULepezFYCdJkiRJB9Dekfjvp1fx2mPrGF5bmXc5ezHYSZIkSdIBPLxkHY1bWoryMEww2EmSJEnSAd05dxVDqys4/7ixeZfSJYOdJEmSJO1Hc2s79z27hotOGk9NZXne5XTJYCdJkiRJ+/HA/Aa2trQV7WGYYLCTJEmSpP26c+4qxg2r5uxpo/MuZZ8MdpIkSZK0Dxu37eRXC9dy6SkTKS+LvMvZJ4OdJEmSJO3DL55ZTVtH4rJTi/cwTDDYSZIkSdI+3TVvJdPHDuHEicPyLmW/DHaSJEmS1IUVG7fz+PMbufzUiUQU72GYYLCTJEmSpC7dNW8VQNEfhgkGO0mSJEnaS0qJu+at5IyjRnLEqNq8yzkgg50kSZIk7WHR2q0817CVy06dmHcp3WKwkyRJkqQ93P/sGgDeeOL4nCvpHoOdJEmSJO1hdn0DpxwxgnHDavIupVsMdpIkSZLUSUNTM0+t2MyFJ4zLu5RuM9hJkiRJUiez6xsAuMBgJ0mSJEmlaXZ9A0eNruXosUPyLqXbDHaSJEmSlNna0sbvl6znguPHFf1NyTsz2EmSJElS5tcLG9nZ3lFSh2GCwU6SJEmSdptdv4aRtZWccdTIvEs5KAY7SZIkSQJa2zv45YK1nH/cOCrKSysqHbDaiPh2RKyNiD90ahsVEbMjYlH2PDJrj4j4z4hYHBFPR8TpndaZlfVfFBGzDs/bkSRJkqRD89iyDTQ1t3HhiaV1GCZ0b4/dd4GL9mi7DngwpXQ08GA2D3AxcHT2uAa4AQpBEPgMcBYwE/jMrjAoSZIkScVgdn0D1RVl/NHRY/Iu5aAdMNillH4DbNij+TLg5mz6ZuDyTu23pIJHgBERMQF4IzA7pbQhpbQRmM3eYVGSJEmScpFSYnZ9A3909BhqqyryLuegHeqBo+NSSqsBsuexWfskYHmnfiuytn21S5IkSVLu6lc3sXLTjpK7GuYuvX1GYFc3ekj7ad97AxHXRMSciJjT2NjYq8VJkiRJUldm1zcQAecfN7CCXUN2iCXZ89qsfQVwRKd+k4FV+2nfS0rpxpTSjJTSjLq6ukMsT5IkSZK6b3Z9A6cfOZK6odV5l3JIDjXY3Q3surLlLOCuTu3vzq6OeTawOTtU8z7gwogYmV005cKsTZIkSZJytXLTDp5d1VSyh2ECHPCswIj4IfBaYExErKBwdcvrgTsi4mrgReBtWfd7gDcBi4HtwHsAUkobIuJzwONZv39KKe15QRZJkiRJ6nMP1DcA9O9gl1J6xz4Wvb6Lvgn44D62823g2wdVnSRJkiQdZrPrG5hWN5hX1A3Ju5RDVlq3U5ckSZKkXrR5RyuPLF1f0nvrwGAnSZIkaQD71cK1tHUkLjTYSZIkSVJpml3fwJghVZx6xMi8S+kRg50kSZKkAWlnWwe/XtjIG44fR3lZV7feLh0GO0mSJEkD0iNL17Olpa3kz68Dg50kSZKkAer++jUMqiznVdPH5F1KjxnsJEmSJA04KSUeqF/Lq48ZQ01led7l9JjBTpIkSdKA88zKzaxpauaCE8bnXUqvMNhJkiRJGnBm1zdQFnD+cWPzLqVXGOwkSZIkDTiz6xuYMWUUowZX5V1KrzDYSZIkSRpQlm/YzoI1W0r+puSdGewkSZIkDSj31zcA9IvbHOxisJMkSZI0oMyuX8Mx44Zw1OjBeZfSawx2kiRJkgaMTdt38vjzG/vV3jow2EmSJEkaQH65YC3tHanf3OZgF4OdJEmSpAFjdn0D44ZVc/Kk4XmX0qsMdpIkSZIGhObWdn79XCNvOH4cZWWRdzm9ymAnSZIkaUD4/ZL1bN/Z3u/OrwODnSRJkqQB4v76BoZUV3DOK0bnXUqvM9hJkiRJ6vc6OhIPzG/gNcfUUV1Rnnc5vc5gJ0mSJKnfm7diE41bWvrlYZhgsJMkSZI0AMyub6C8LHjdsWPzLuWwMNhJkiRJ6vdm1zdw1tRRDK+tzLuUw8JgJ0mSJKlfW7ZuG4vXbu23h2GCwU6SJElSPze7fg2AwU6SJEmSStXs+gaOnzCMySNr8y7lsDHYSZIkSeq31m9t4YkXNvbrvXVgsJMkSZLUjz24YC0dCS402EmSJElSaZpd38CkEYM4ceKwvEs5rHoU7CLibyLi2Yj4Q0T8MCJqImJqRDwaEYsi4vaIqMr6Vmfzi7PlU3rjDUiSJElSV3bsbOe3ixp5w/FjiYi8yzmsDjnYRcQk4K+AGSmlk4By4Arg88CXUkpHAxuBq7NVrgY2ppSmA1/K+kmSJEnSYfG7xetobu3gghPG513KYdfTQzErgEERUQHUAquB84EfZ8tvBi7Ppi/L5smWvz76e2yWJEmSlJvZ9WsYWlPBWdNG5V3KYXfIwS6ltBL4d+BFCoFuM/AEsCml1JZ1WwFMyqYnAcuzdduy/qMP9fUlSZIkaV/aOxIPzl/L644dS2V5/7+0SE8OxRxJYS/cVGAiMBi4uIuuadcq+1nWebvXRMSciJjT2Nh4qOVJkiRJGsDmvriR9dt29vvbHOzSk+j6BmBZSqkxpdQK/BQ4FxiRHZoJMBlYlU2vAI4AyJYPBzbsudGU0o0ppRkppRl1dXU9KE+SJEnSQJRS4qbfLaOqvIzXHDswMkVPgt2LwNkRUZudK/d6oB54CHhr1mcWcFc2fXc2T7b8lymlvfbYSZIkSVJP3PS7Zdz7hzX8zQXHMKymMu9y+kRPzrF7lMJFUJ4Ensm2dSPwCeCjEbGYwjl0N2Wr3ASMzto/ClzXg7olSZIkaS+/X7Kef713ARedOJ73v2Za3uX0mSjmnWYzZsxIc+bMybsMSZIkSSVg9eYdvPm/fsfwQZXc+cFXMbQf7K2LiCdSSjMO1K/iQB0kSZIkqdi1tLXzlz94kh0727ntmrP7Rag7GAY7SZIkSSXvn/67nrkvbuKGK09n+tiheZfT5/r/DR0kSZIk9Ws/mrOcHzz6Ite+ZhoXv3JC3uXkwmAnSZIkqWT9YeVm/v7OP3DuK0bztxcem3c5uTHYSZIkSSpJG7ft5NrvPcGYwVX81ztOo6J84MYbz7GTJEmSVHLaOxJ/ddtcGre08KP3n8PoIdV5l5Qrg50kSZKkkvPF2Qv57aJ1XP+nr+SUI0bkXU7uBu6+SkmSJEkl6f5n1/C1h5ZwxZlHcMXMI/MupygY7CRJkiSVjCWNW/noHU9x8uTh/OOlJ+ZdTtEw2EmSJEkqCdta2nj/956gqqKMG955BjWV5XmXVDQMdpIkSZKKXkqJv/vx0yxp3MpX33Eak0YMyrukomKwkyRJklT0vvXbZfzimdX83UXHce70MXmXU3QMdpIkSZKK2sNL1vGv987n4pPGc+2rp+VdTlEy2EmSJEkqWqs27eDDt85l6pjBfOFtpxAReZdUlAx2kiRJkopSS1s7H/jBk7S0dfDNd81gSLW34d4X/2UkSZIkFaXP/nc9Ty3fxDfeeTrTxw7Ju5yi5h47SZIkSUXnjseXc+ujL/L+17yCi06akHc5Rc9gJ0mSJKmo/GHlZj591x941fTRfPzCY/IupyQY7CRJkiQVjc07WvnLHzzJqNoq/vOK06goN7J0h+fYSZIkSSoKKSU+/qOnWLVpB7dfezajh1TnXVLJMP5KkiRJKgr/77dLmV3fwCffdDxnHDUq73JKisFOkiRJUu4eW7aBz//PQi4+aTzvfdWUvMspOQY7SZIkSblq3NLCh259kiNH1fJvbz3Zm5AfAoOdJEmSpNy0dyQ+cttcNu9o5etXns7Qmsq8SypJXjxFkiRJUm6+/MBzPLxkPV9468kcP2FY3uWULPfYSZIkScrFQwvX8l+/XMzbZxzB22YckXc5Jc1gJ0mSJKnPrdi4nb+5fR7HTxjGZy87Me9ySp7BTpIkSVKfamlr54O3zqW9PfH1K0+nprI875JKnufYSZIkSepT//cX83lq+Sa+8c7TmTpmcN7l9As92mMXESMi4scRsSAi5kfEORExKiJmR8Si7Hlk1jci4j8jYnFEPB0Rp/fOW5AkSZJUKu5+ahU3//4F3nfeVC46aULe5fQbPT0U8yvA/6SUjgNOAeYD1wEPppSOBh7M5gEuBo7OHtcAN/TwtSVJkiSVkMVrt3DdT57mjKNG8omLj8u7nH7lkINdRAwDXg3cBJBS2plS2gRcBtycdbsZuDybvgy4JRU8AoyICCO6JEmSNABs39nGB77/JIMqy/nan59OZbmX++hNPfnXnAY0At+JiLkR8a2IGAyMSymtBsiex2b9JwHLO62/ImuTJEmS1I+llPjUT59hceNWvnLFaYwfXpN3Sf1OT4JdBXA6cENK6TRgGy8ddtmV6KIt7dUp4pqImBMRcxobG3tQniRJkqRicOtjL3LnvFX8zRuO4byjx+RdTr/Uk2C3AliRUno0m/8xhaDXsOsQy+x5baf+ne86OBlYtedGU0o3ppRmpJRm1NXV9aA8SZIkSXl7ZsVmPnt3Pa8+po4PvW563uX0W4cc7FJKa4DlEXFs1vR6oB64G5iVtc0C7sqm7wbenV0d82xg865DNiVJkiT1P5u3t/KXtz7BmCFVfPntp1JW1tVBfOoNPb2P3YeBH0REFbAUeA+FsHhHRFwNvAi8Let7D/AmYDGwPesrSZIkqR/q6Eh87EfzWLO5mduvPYdRg6vyLqlf61GwSynNA2Z0sej1XfRNwAd78nqSJEmSSsONv13KA/PX8o9vPoHTjxyZdzn9ntcYlSRJktSrHl26ni/ct5A/fuUEZp07Je9yBgSDnSRJkqRes35rC39121yOHFXL9W95JRGeV9cXDHaSJEmSekVHR+KjdzzFxu2tfPXPT2NoTWXeJQ0YBjtJkiRJveKbv1nKr59r5B8uOYETJw7Pu5wBxWAnSZIkqcfmPL+Bf7+/cF7dO886Mu9yBhyDnSRJkqQe2bhtJx/+4VwmjRjEv3peXS56eh87SZIkSQNYSomP/+gp1m1t4acfeBXDPK8uF+6xkyRJknTIbvrdMh5csJZPvel4XjnZ8+ryYrCTJEmSdEjmvriR6+9dwBtPHMdV3q8uVwY7SZIkSQdt8/ZWPnTrXMYPr+Hf3nKK59XlzHPsJEmSJB2UlBJ/++OnaGhq5kfvP4fhtZ5Xlzf32EmSJEk6KN99+Hnur2/guouP47QjR+ZdjjDYSZIkSToIT6/YxP+9Zz5vOH4sV583Ne9ylDHYSZIkSeqWpubCeXV1Q6r597d5Xl0x8Rw7SZIkSQeUUuK6nzzNyk07uOPasxlRW5V3SerEPXaSJEmSDuj7j77IPc+s4eMXHssZR43KuxztwWAnSZIkab+eXbWZz/28ntceW8e1r56WdznqgsFOkiRJ0j5tbWnjQ7fOZWRtJf/xtlMoK/O8umLkOXaSJEmSupRS4lM/fYYX1m/jh39xNqOHVOddkvbBPXaSJEmSunTb48u5+6lVfPSCYzhr2ui8y9F+GOwkSZIk7WX+6ib+8e5nOW/6GD7w2ul5l6MDMNhJkiRJepltLW186NYnGTaoki+9/VTKPa+u6HmOnSRJkqTdWtra+fiPnmLpum384OqzqBvqeXWlwGAnSZIkCYDN21u59vtzeGTpBj79x8dz7vQxeZekbjLYSZIkSWL5hu1c9Z3HWL5hB19++6lcftqkvEvSQTDYSZIkSQPcvOWbeN/Nj9Panrjl6pmc7RUwS47BTpIkSRrA7nt2DR+5bS51Q6u57aqZTB87JO+SdAgMdpIkSdIAddPvlvHPv6jnlMkj+NasGYzxBuQly2AnSZIkDTDtHYnP/bye7z78PG88cRxffvtpDKoqz7ss9UCP72MXEeURMTcifp7NT42IRyNiUUTcHhFVWXt1Nr84Wz6lp68tSZIk6eBs39nGtd97gu8+/DzvO28qX7/yDENdP9AbNyj/CDC/0/zngS+llI4GNgJXZ+1XAxtTStOBL2X9JEmSJPWRtVuaueLGR/jlggY+e+mJfPqSE7z5eD/Ro2AXEZOBPwa+lc0HcD7w46zLzcDl2fRl2TzZ8tdn/SVJkiQdZosatvAnX3uYRQ1bufFdM5h17pS8S1Iv6uk5dl8G/g4Yms2PBjallNqy+RXArhtgTAKWA6SU2iJic9Z/XQ9rkCRJkrQfDy9ex7Xff4KaynLuuPYcXjl5eN4lqZcd8h67iLgEWJtSeqJzcxddUzeWdd7uNRExJyLmNDY2Hmp5kiRJkoCfPLGCWd95jAnDa/jZX55rqOunerLH7lXApRHxJqAGGEZhD96IiKjI9tpNBlZl/VcARwArIqICGA5s2HOjKaUbgRsBZsyYsVfwkyRJknRgKSW+8uAivvzAIs59xWhueOcZDB9UmXdZOkwOeY9dSumTKaXJKaUpwBXAL1NKVwIPAW/Nus0C7sqm787myZb/MqVkcJMkSZJ62c62Dj72o6f48gOLeOsZk/nue2Ya6vq5w3Efu08At0XEPwNzgZuy9puA70XEYgp76q44DK8tSZIkDWhNza28/3tP8PCS9Xz0gmP48PnT8ZqF/V+vBLuU0q+AX2XTS4GZXfRpBt7WG68nSZIkaW/rtrYw69uPsXDNFr74Z6fwp6dPzrsk9ZHDscdOkiRJUh9bvmE77/72Y6zZ3My3Zs3gtceOzbsk9SGDnSRJklTiFjVs4V03Pcb2nW18/30zOeOoUXmXpD5msJMkSZJK2Lzlm7jqO49RWV7G7deew/EThuVdknJgsJMkSZJK1O8WreOa781hzJBqvn/1WRw5ujbvkpQTg50kSZJUgu59ZjUfuW0e0+oGc8t7ZzJ2WE3eJSlHBjtJkiSpxNz22It86mfPcNqRI/n2rDMZXus96gY6g50kSZJUQr7x6yVcf+8CXnNMHTe883Rqq/xKL4OdJEmSVBJSSlx/7wK++ZulXHrKRP79badQVVGWd1kqEgY7SZIkqci1dyQ+9dNnuH3Oct519lF89tITKSuLvMtSETHYSZIkSUWspa2dj/xwHv/z7Br+6vzp/M0FxxBhqNPLGewkSZKkIrW1pY1rvzeH/128nv9zyQm897ypeZekImWwkyRJkorQxm07ueo7j/GHVU38x9tO4S1nTM67JBUxg50kSZJUZFZv3sG7bnqMFzds5xvvPIMLThiXd0kqcgY7SZIkqYgsXLOF9373cZp2tHLLe2dy9rTReZekEmCwkyRJkorA2qZmvvTAIu6Ys5wRgyr54TVnc9Kk4XmXpRJhsJMkSZJytKW5lRt/s5Rv/XYZre0dvOvso/jw+dMZPaQ679JUQgx2kiRJUg52tnVw66Mv8F+/XMz6bTu55OQJ/O0bj+Wo0YPzLk0lyGAnSZIk9aGUEr94ZjVfuG8hL6zfzjnTRnPdxcdxyhEj8i5NJcxgJ0mSJPWR3y9Zz/X3zuepFZs5bvxQvvOeM3ntMXXecFw9ZrCTJEmSDrMFa5r4/L0LeGhhIxOG1/CFt57Mn54+mfIyA516h8FOkiRJOkxWbdrBF2c/x0+eXMGQ6gquu/g4rjp3CjWV5XmXpn7GYCdJkiT1ss07WrnhV0v4zv8uIyV433lT+eDrpjOitirv0tRPGewkSZKkXrKzrYNbfv88X31oMZt3tPInp07ioxcew+SRtXmXpn7OYCdJkiT1gl8uaOBzP5/PsnXb+KOjx/DJi4/nhInD8i5LA4TBTpIkSeqBxWu38rmf1/Pr5xqZVjeY77znTF537Ni8y9IAY7CTJEmSDsHmHa185YFF3PL75xlUVc4/XHIC7z7nKCrLy/IuTQOQwU6SJEk6CO0didsef5H/uP85Nm7fyTtmHsnHLjiG0UOq8y5NA5jBTpIkSeqm3y9Zzz/9vJ75q5uYOXUUn3nzCZw4cXjeZUkGO0mSJOlAlm/Yzr/eO597nlnDpBGD+PqVp3PxSeOJ8AbjKg6HHOwi4gjgFmA80AHcmFL6SkSMAm4HpgDPA3+WUtoYhVH/FeBNwHbgqpTSkz0rX5IkSTp8tu9s44ZfLeGbv1lKeQQfu+AY/uLV07zBuIpOT/bYtQEfSyk9GRFDgSciYjZwFfBgSun6iLgOuA74BHAxcHT2OAu4IXuWJEmSikpKibvmreL6exewpqmZy0+dyCcuPo4JwwflXZrUpUMOdiml1cDqbHpLRMwHJgGXAa/Nut0M/IpCsLsMuCWllIBHImJEREzItiNJkiQVhaeWb+Kz//0sT764iZMnD+drV57GGUeNyrssab965Ry7iJgCnAY8CozbFdZSSqsjYtdNPCYByzuttiJrM9hJkiQpdx0diS/Ofo6vPrSYMUOq+cJbT+Ytp0+mrMzz6FT8ehzsImII8BPgr1NKTfs5gbSrBamL7V0DXANw5JFH9rQ8SZIk6YC2tbTx0Tvmcd+zDbx9xhF8+pLjGVpTmXdZUrf1KNhFRCWFUPeDlNJPs+aGXYdYRsQEYG3WvgI4otPqk4FVe24zpXQjcCPAjBkz9gp+kiRJUm9auWkH77t5DgvXNPGZN5/AVedO8WqXKjllh7pidpXLm4D5KaUvdlp0NzArm54F3NWp/d1RcDaw2fPrJEmSlKcnXtjIZV/9X1Zs2M63rzqT97xqqqFOJakne+xeBbwLeCYi5mVtnwKuB+6IiKuBF4G3ZcvuoXCrg8UUbnfwnh68tiRJktQjP5u7gk/8+BkmjKjhtmvOYvrYoXmXJB2ynlwV83d0fd4cwOu76J+ADx7q60mSJEm9oaMj8YX7F3LDr5Zw9rRR3HDlGYwcXJV3WVKP9MpVMSVJkqRSsK2ljb++fR6z6xt4x8wj+afLTqSy/JDPTpKKhsFOkiRJA8KKjdt5381zeK5hixdJUb9jsJMkSVK/98QLG7n2e3NoaevgO++ZyWuOqcu7JKlXGewkSZLUr/30yRVc95NdF0k5k+ljh+RdktTrDHaSJEnqlzo6Ev9230K+8eslnDNtNF+/8nQvkqJ+y2AnSZKkfmdrSxt/fds8HpjfwJ+fdSSfvdSLpKh/M9hJkiSpX+l8kZR/fPMJzPIiKRoADHaSJEkqCmubmnl6xWbaOjoOeRtbmtv4/P8soKWtg+++Zyav9mIxT1AAAA+aSURBVCIpGiAMdpIkSepzKSVWbNzBY8s28Oiy9Ty2bAPPr9/eK9ueMrrWi6RowDHYSZIk6bBLKbGkcRuPLdvAY1mQW7W5GYARtZWcOWUU7zz7KE47cgS1VT37ijp1zGBqKst7o2ypZBjsJEmS1Os6OhIL1mwphLjnN/DYsg2s27oTgLqh1Zw1dRQfmDqKmVNHc/TYIZSVeQ6c1BMGO0mSJPVYU3MrC1ZvYd7yjTy6dAOPP7+BpuY2ACaPHMSrj6njrCzITRld68VMpF5msJMkSVK37To3rn51E/Wrmpi/uon5a5pYvmHH7j7T6gbzxydPYGYW5CaNGJRjxdLAYLCTJElSl5pb23muYQvzd4e4Lcxf08SWbE9cROF8tlMmj+CKM4/khAnDOGnScOqGVudcuTTwGOwkSZIGuJQSDU0tLFiThbfVhT1xSxq30pEKfQZXlXPchGFcfuokjp8wjBMmDuPYcUMZVOVFSqRiYLCTJEkaQDZvb2Vhw5bCY00Tz63ZysKGLWze0bq7z6QRgzh+wlAuPmn87hB3xMhaL3AiFTGDnSRJUj/U3NrOooatuwPcwoatPLdmC2uamnf3GVpTwbHjhnLJyRM4dvxQjhk3lOPHD2N4bWWOlUs6FAY7SZKkEra1pY3n121jSeNWljRuK+yFa9jK8+u3kbLDKKsqyjh67BDOfcXoQoAbP5Rjxw1lwvAar04p9RMGO0mSpCLX1t7B8o07WLZuK0sbt7F03TaWNm5l2bptNDS17O5XFjBl9GCOGz+US0+ZyHFZiDtqVC0V5WU5vgNJh5vBTpIkqQiklFi3defuwFYIb9tYum4rL67fTtuuq5gAI2ormTZmMOdNr2Na3WCmjRnMtLohHDW6lppKL2YiDUQGO0mSpG5qbe9ga3MbW5rbaGpuZUtzG1uaW9na0sbWljZ27Gynpa2D5tZ2mls7aG5rp7m1nZbWQtvuZW3Z8qxfS1uhz872jt2vVVVRxtTRgzlm7FAuOnE8U7PwNm3MYEYOrsrxX0FSMTLYSZKkfiWltDtAvSxktbZnoeql6ZYsfLW0drBtZ9vuoLaluRDUmjrNb2lupbm148AFABVlQU1lOTWVZVRXvPy5prKcYYMqC9MV5VRXvtQ+bmg1U7PwNnHEIMq9CqWkbjLYSZKkktHekVixcfvuwxR3nXP2wvrtbG1p2x3mDlVtVTlDayoYWlPJ0JoKhtVUMHnEoKyt0D6k+qXpYZ36Dq6uYFBVOTUVZZ7PJqnPGewkSVJRSSmxYdtOlq7bxrLGbSxZt5Vl2QVDXly//WWHKw6rqWBa3RBmTh3FsJoKaio77QGrKKc6e67ptFds771ohelBleUGMkkly2AnSZL63LaWNtY0NdOwuZmGLc2s3LjjZVd7bGpu2923qryMo0bXMm3MYN5w/LjsQiGDmTpmMKMGV3m5fknCYCdJknpRa3sHjVtaaGhqpqGpmTWbm2nY0kLD5uZCkGtqpqGpha0tbXutO2F4DdPqBnPpqROZNmYIU+sG84oxQ5g00nPNJOlADHaSJA0AKSXaOxKt7Ymd7R207nq0JXa2732Fxt3znaZb9rwYSXbFx+bWDtZva6GhqYV1W1t23xR7l8ryYOzQGsYNq+bY8UP5o6PrGD+8MD9uWA3jh9UwfngNtVV+LZGkQ+UnqCT1c+0dqcurAbZ3pAOv3M+lBB0p0ZES7R2Ff6uOLAC1p0RHR9rd1pH2WJ5N7xliDroGCjef3tmesqBVCFy759s7aH3ZdAc72/aYb0+719vVf3d4a8vW7+joca3AXld3rMmmxwyp5qSJwxk3rKYQ1oZX754eVVtFmXvcJOmwMthJUo5SKnwB397SzradhXtgbdvZzvaWtsLzzja2tRSet+8s9NnVt6WLPSp77nFpaWuntd0AV4qqKsqoKi+jsjyoLC+jsryMqoo95rPn2qpdy19aVliezVe8NF+x57JseU1FIahVV+y6wMjLw1t1ZRnVFWWezyZJRarPg11EXAR8BSgHvpVSun5ffZuaW5ld39BntXWlY9cvtil1+qWWLtq6+FU369PzGl56vV2/Hre/rK6XDrF5qZ199H2p3o4O9vkeCs/sbpPUu9o70u6w1nYQ/8eqKsoYXFVObVVFpyv8Fb6AjxhUufuLeXXl3ntUOn9Rr64op7LcL+gAZRGUlQXlEZSVQXkE5WUvtZWXRdaHrE+8rE9v7Ih6eVArbNsAJUk6GH0a7CKiHPgacAGwAng8Iu5OKdV31f+F9dv5i1vm9GWJRass2P3lonz3F5Do1JZ9Odm1PPuy8bJ19lofKsvK9t7my77UQFlZEPgFQ+pN5WVQW1XB4OpCSKutKmdwVQW11YXnQXvM11aXU+ul2CVJ0j709R67mcDilNJSgIi4DbgM6DLYTR87hNs/fF4flte1fYWiPcNQWVdhKoLe+NHVX24lSZIk7UtfB7tJwPJO8yuAs/bVeVBlOSdNGn7Yi5IkSZKkUtbXx/R0tdvpZSeXRMQ1ETEnIuY0Njb2UVmSJEmSVLr6OtitAI7oND8ZWNW5Q0rpxpTSjJTSjLq6uj4tTpIkSZJKUV8Hu8eBoyNiakRUAVcAd/dxDZIkSZLUr/TpOXYppbaI+BBwH4XbHXw7pfRsX9YgSZIkSf1Nn9/HLqV0D3BPX7+uJEmSJPVX3hBJkiRJkkqcwU6SJEmSSpzBTpIkSZJKnMFOkiRJkkqcwU6SJEmSSlyklPKuYZ8iYguwMO86pE7GAOvyLkLKOB5VTByPKiaORxWTno7Ho1JKdQfq1Oe3OzhIC1NKM/IuQtolIuY4JlUsHI8qJo5HFRPHo4pJX41HD8WUJEmSpBJnsJMkSZKkElfswe7GvAuQ9uCYVDFxPKqYOB5VTByPKiZ9Mh6L+uIpkiRJkqQDK/Y9dpIkSZKkAzjswS4ijoiIhyJifkQ8GxEf6bTsnIj4fxExOuuzNSK+2sU2PhkRV0bERyOiPiKejogHI+KoTn1mRcSi7DHrcL8vlY6ejsGI+JeIWB4RW7vY9oSIuD8iTo2I32fbfzoi3t6pz9SIeDQbm7dHRNXhfccqZt0cjxdExBMR8Uz2fH6nPo5HHTYRURMRj0XEU9n4+WynZe+IiL/P/h4/nT0ejohT9tjGNyPiVRHxhYhYkPX7WUSM6NTnkxGxOCIWRsQb+/I9qnh1c/wdl32+tUTExzst3+dna7a8O5+vZ2TtiyPiPyMi+uadq1j15DMxlzGZUjqsD2ACcHo2PRR4Djghm/8s8BZgMHAe8H7gq11s4yGgDngdUJu1fQC4PZseBSzNnkdm0yMP93vzURqPno5B4OxsG1u72PZ7gI8BxwBHZ20TgdXAiGz+DuCKbPobwAfy/jfxkd+jm+PxNGBi1nYSsLLT+o5HH4ftAQQwJJuuBB4Fzs7mbwbOAM7d9TcWuBh4dI9tzAPKgQuBiqzt88Dns+kTgKeAamAqsAQoz/u9+8j/0c3xNxY4E/gX4OOd1t3nZ2vW1p3P18eAc7I67gUuzvvfxEe+j558JuYxJg/7HruU0uqU0pPZ9BZgPjApW/x64IGU0raU0u+A5j3Xj4hhQFVKqTGl9FBKaXu26BFgcjb9RmB2SmlDSmkjMBu46PC9K5WSno7BlNIjKaXV+9j8RcC9KaXnUkqLsv6rgLVAXfbLyvnAj7P+NwOX99JbUwnq5nicm40jgGeBmoioztZxPOqwSQW79gZXZo+UjZ1TgSdTSg9nf2vh5X+LiYjjgedSSu0ppftTSm1d9LsMuC2l1JJSWgYsBmYe3nemUtDN8bc2pfQ40LrHuvv7bIUDfL5GxARgWErp96nwjfoW/Hwc8HrymZjHmOzTc+wiYgqFVPpoRIwBWlNKmw+w2huAB7tov5pCcoXCP9LyTstW8PJ/OAk45DG4r22VA8emlOr3aJ8JVFH4FXo0sKnTlxvHpnbr5nh8CzA3pdRygG05HtUrIqI8IuZR+EFgdkrpUQrj9Knsy0Vnnf8WQ+HX6v/pYrPvxb/Z6oaDHH/72saUbJ1Hs/nufL5OojAWd3FcCujxZ+KubUyhD8ZkRTffU49FxBDgJ8Bfp5SaIuLPgfu7sepFwHf22NY7gRnAa3Y1dbGel/vUy/RgDO7LWWT/QTu9xgTge8CslFLHPo6FdmyqW+MxIk6kcAjbhd3YpONRvSKl1A6cmp0T97OIOIlsb3DnfhHxOgpfYs7r1PxGCocEd+7390Ab8INdTV29bO9Ur1LX3fG3L3t+tmbNF3Lgz1fHpbrUw8/EPh2TfbLHLiIqKbyhH6SUfpo17+tXvT3NpHB86a5tvQH4e+DSTr9grwCO6LTOZGAVUqaHY3BfXrZ+dtjwL4BPp5QeyZrXASMiYtePKI5NdWs8RsRk4GfAu1NKS7qxWcejelVKaRPwKwpfYF72JSQiTga+BVyWUlqftdVSOJdzVad+s4BLgCs7/bLt32wd0P7G377s47MVuvf5uoJOhxXjuNQeDvYzMWvv0zHZF1fFDOAmYH5K6Yud2k6mcIL1/tY9EViQJWUi4jTgmxRC3dpOXe8DLoyIkRExksI/9n29/mZUknoyBg/g9WSHCUfhyoI/A25JKf1oV4fsi8xDwFuzplnAXT14TZW47ozH7FfBXwCfTCn9bzc37XhUj0VEXTb+iIhBFE6HWEThIii7AtyRwE+Bd6WUnuu0+usojK9d27oI+ASFv9nbO/W7G7giO4dkKnA0nX7A1cDVnfG3n3X3+mzt1H7Az9fs3OUtEXF2ts678fNxwOvJZ2IeY7IvDsV8FfAu4Jns+FQovPm5nY9LjYjngWFAVURcTiGc7blH5QvAEOBH2RFFL6aULk0pbYiIzwGPZ/3+KaW04TC+J5WWQx6DKaX6iPg34M+B2ohYQeEXma8BzZ12qf8Z8GpgdERclbVdlVKaR+GLzW0R8c/AXAr/yTVwdWc8fgiYDvxDRPxD1nZhSmmt41GH2QTg5uyczTIKV1GtBB7o1Of/UDhf8+vZ3+K2lNIMCn+zf9yp31cpXPlydtbvkZTS+1NKz0bEHUA9hUM0P7jrB1wNeAccfxExHphD4e91R0T8NYUrrZ7M3p+tn6JwXlS3Pl8pXHH9u8AgCofZdevwT/VrPflM7Orv/WEdk9HN81B7VUR8GlicUrrtAP1mU9glua8rwEmHpLtjcD/rvxOYnFK6vncr00DkeFQxi4hvAd/qdEjvvvo9CZyVUmrdXz/pYHR3/O1n/R59vkp7KuYxmUuwkyRJkiT1nj693YEkSZIkqfcZ7CRJkiSpxBnsJEmSJKnEGewkSZIkqcQZ7CRJkiSpxBnsJEmSJKnEGewkSZIkqcT9fzqrnugiypHiAAAAAElFTkSuQmCC\n",
      "text/plain": [
       "<Figure size 1080x360 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "#df.describe().toPandas().transpose()\n",
    "df_plot = df.drop(*[\"Province/State\",\"Country/Region\", \"Lat\", \"Long\"]).toPandas()\n",
    "df_plot.sum().plot(figsize=(15,5))\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "**Prediction for whole Australia**"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 258,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "+-----------+-----+----------+\n",
      "|       Date|Cases|Date_value|\n",
      "+-----------+-----+----------+\n",
      "|sum(2/1/20)|   12|         1|\n",
      "|sum(2/2/20)|   12|         2|\n",
      "|sum(2/3/20)|   12|         3|\n",
      "|sum(2/4/20)|   13|         4|\n",
      "|sum(2/5/20)|   13|         5|\n",
      "+-----------+-----+----------+\n",
      "only showing top 5 rows\n",
      "\n",
      "Coefficients: [50.26843708786918]\n",
      "Intercept: -1783.5290280354602\n",
      "numIterations: 3\n",
      "objectiveHistory: [0.4999999999999999, 0.4113206736317112, 0.09705662939313293]\n",
      "+--------------------+\n",
      "|           residuals|\n",
      "+--------------------+\n",
      "|  176.67060413577724|\n",
      "|  129.40216704790805|\n",
      "|   84.13372996003886|\n",
      "|  36.865292872169675|\n",
      "|-0.40314421569951264|\n",
      "|   -35.6715813035687|\n",
      "|  -69.94001839143789|\n",
      "|  -99.20845547930708|\n",
      "| -149.47689256717604|\n",
      "| -127.74532965504523|\n",
      "|  -128.0137667429144|\n",
      "|  -131.2822038307836|\n",
      "| -101.55064091865279|\n",
      "|  -76.81907800652198|\n",
      "| -11.087515094391165|\n",
      "|   51.64404781773965|\n",
      "|  111.37561072987046|\n",
      "|  341.10717364200127|\n",
      "+--------------------+\n",
      "\n",
      "RMSE: 127.464294\n",
      "r2: 0.807536\n",
      "+------------+-----+----------+--------+\n",
      "|        Date|Cases|Date_value|features|\n",
      "+------------+-----+----------+--------+\n",
      "|sum(3/23/20)| 1314|        52|  [52.0]|\n",
      "|sum(3/22/20)| 1314|        51|  [51.0]|\n",
      "+------------+-----+----------+--------+\n",
      "\n",
      "+--------+-----------------+\n",
      "|features|       prediction|\n",
      "+--------+-----------------+\n",
      "|  [52.0]|830.4297005337371|\n",
      "|  [51.0]|780.1612634458679|\n",
      "+--------+-----------------+\n",
      "\n",
      "Difference in 3/22/20: 533.8387365541321\n",
      "Difference in 3/23/20: 483.5702994662629\n",
      "\n",
      "\n"
     ]
    }
   ],
   "source": [
    "df_australia = df.drop(*[\"Province/State\",\"Country/Region\", \"Lat\", \"Long\"])\n",
    "df_australia_t = t_data(df_australia.groupBy().sum())\n",
    "df_australia_t = df_australia_t.select(\"*\").withColumn(\"Date_value\", monotonically_increasing_id()+1)\n",
    "df_australia_t.show(5)\n",
    "predict(df_australia_t)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "**Prediction for individual Province**"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 259,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Prediction for Province/State: New South Wales\n",
      "Coefficients: [21.88019584605882]\n",
      "Intercept: -771.8614609447745\n",
      "numIterations: 3\n",
      "objectiveHistory: [0.5, 0.4094703210047335, 0.08028668903793099]\n",
      "+-------------------+\n",
      "|          residuals|\n",
      "+-------------------+\n",
      "|  71.81499802483347|\n",
      "|  49.93480217877459|\n",
      "|  32.05460633271582|\n",
      "| 12.174410486656939|\n",
      "|0.29421464059817026|\n",
      "|-11.585981205460712|\n",
      "| -26.46617705151948|\n",
      "| -38.34637289757836|\n",
      "| -60.22656874363713|\n",
      "| -55.10676458969601|\n",
      "| -56.98696043575478|\n",
      "|-56.867156281813664|\n",
      "| -41.74735212787243|\n",
      "|-24.627547973931314|\n",
      "| 10.492256180009917|\n",
      "|  28.61206033395115|\n",
      "|  52.73186448789238|\n",
      "| 113.85166864183338|\n",
      "+-------------------+\n",
      "\n",
      "RMSE: 49.045413\n",
      "r2: 0.843381\n",
      "+-------+-----+----------+--------+\n",
      "|   Date|Cases|Date_value|features|\n",
      "+-------+-----+----------+--------+\n",
      "|3/23/20|  533|        52|  [52.0]|\n",
      "|3/22/20|  533|        51|  [51.0]|\n",
      "+-------+-----+----------+--------+\n",
      "\n",
      "+--------+------------------+\n",
      "|features|        prediction|\n",
      "+--------+------------------+\n",
      "|  [52.0]|365.90872305028415|\n",
      "|  [51.0]| 344.0285272042254|\n",
      "+--------+------------------+\n",
      "\n",
      "Difference in 3/22/20: 188.97147279577462\n",
      "Difference in 3/23/20: 167.09127694971585\n",
      "\n",
      "\n",
      "Prediction for Province/State: Victoria\n",
      "Coefficients: [9.720784780487497]\n",
      "Intercept: -346.4125683902311\n",
      "numIterations: 3\n",
      "objectiveHistory: [0.5, 0.41459814691128793, 0.12235498559356363]\n",
      "+-------------------+\n",
      "|          residuals|\n",
      "+-------------------+\n",
      "|  35.62667063414369|\n",
      "|  25.90588585365623|\n",
      "|  16.18510107316871|\n",
      "| 7.4643162926812465|\n",
      "|-2.2564684878062735|\n",
      "|-7.9772532682937936|\n",
      "|-14.698038048781257|\n",
      "|-21.418822829268777|\n",
      "| -31.13960760975624|\n",
      "| -25.86039239024376|\n",
      "| -22.58117717073128|\n",
      "|-24.301961951218743|\n",
      "|-20.022746731706263|\n",
      "|-6.7435315121937265|\n",
      "| 10.535683707318753|\n",
      "| 0.8148989268312903|\n",
      "|  -8.90588585365623|\n",
      "|  89.37332936585625|\n",
      "+-------------------+\n",
      "\n",
      "RMSE: 28.250331\n",
      "r2: 0.763251\n",
      "+-------+-----+----------+--------+\n",
      "|   Date|Cases|Date_value|features|\n",
      "+-------+-----+----------+--------+\n",
      "|3/23/20|  296|        52|  [52.0]|\n",
      "|3/22/20|  296|        51|  [51.0]|\n",
      "+-------+-----+----------+--------+\n",
      "\n",
      "+--------+------------------+\n",
      "|features|        prediction|\n",
      "+--------+------------------+\n",
      "|  [52.0]|159.06824019511873|\n",
      "|  [51.0]| 149.3474554146312|\n",
      "+--------+------------------+\n",
      "\n",
      "Difference in 3/22/20: 146.6525445853688\n",
      "Difference in 3/23/20: 136.93175980488127\n",
      "\n",
      "\n",
      "Prediction for Province/State: Queensland\n",
      "Coefficients: [10.237919030318382]\n",
      "Intercept: -365.48475086932393\n",
      "numIterations: 3\n",
      "objectiveHistory: [0.49999999999999983, 0.4155468978358045, 0.13033455306720848]\n",
      "+-------------------+\n",
      "|          residuals|\n",
      "+-------------------+\n",
      "|  38.63342286881732|\n",
      "|  30.39550383849894|\n",
      "|  20.15758480818056|\n",
      "|  9.919665777862178|\n",
      "|  1.681746747543798|\n",
      "| -8.556172282774583|\n",
      "|-15.794091313092963|\n",
      "|-24.032010343411343|\n",
      "|-34.269929373729724|\n",
      "|-29.507848404048104|\n",
      "|-28.745767434366485|\n",
      "|-23.983686464684865|\n",
      "|-27.221605495003246|\n",
      "|-27.459524525321626|\n",
      "|-21.697443555640007|\n",
      "| 18.064637414041556|\n",
      "| 47.826718383723176|\n",
      "|   74.5887993534048|\n",
      "+-------------------+\n",
      "\n",
      "RMSE: 31.103676\n",
      "r2: 0.746723\n",
      "+-------+-----+----------+--------+\n",
      "|   Date|Cases|Date_value|features|\n",
      "+-------+-----+----------+--------+\n",
      "|3/23/20|  221|        52|  [52.0]|\n",
      "|3/22/20|  221|        51|  [51.0]|\n",
      "+-------+-----+----------+--------+\n",
      "\n",
      "+--------+------------------+\n",
      "|features|        prediction|\n",
      "+--------+------------------+\n",
      "|  [52.0]|166.88703870723197|\n",
      "|  [51.0]|156.64911967691359|\n",
      "+--------+------------------+\n",
      "\n",
      "Difference in 3/22/20: 64.35088032308641\n",
      "Difference in 3/23/20: 54.112961292768034\n",
      "\n",
      "\n",
      "Prediction for Province/State: South Australia\n",
      "Coefficients: [3.0064558864894595]\n",
      "Intercept: -104.10125262264592\n",
      "numIterations: 3\n",
      "objectiveHistory: [0.4999999999999999, 0.413521172776411, 0.10695905996457997]\n",
      "+--------------------+\n",
      "|           residuals|\n",
      "+--------------------+\n",
      "|    9.88820836849375|\n",
      "|   6.881752482004288|\n",
      "|    5.87529659551484|\n",
      "|  2.8688407090253776|\n",
      "|-0.13761517746408458|\n",
      "| -3.1440710639535467|\n",
      "|  -6.150526950443009|\n",
      "|  -7.156982836932457|\n",
      "| -10.163438723421919|\n",
      "|  -6.169894609911381|\n",
      "|  -6.176350496400829|\n",
      "|  -8.182806382890305|\n",
      "| -2.1892622693797534|\n",
      "|   -5.19571815586923|\n",
      "|-0.20217404235867775|\n",
      "|  1.7913700711518459|\n",
      "|   6.784914184662398|\n",
      "|   20.77845829817295|\n",
      "+--------------------+\n",
      "\n",
      "RMSE: 7.623525\n",
      "r2: 0.812858\n",
      "+-------+-----+----------+--------+\n",
      "|   Date|Cases|Date_value|features|\n",
      "+-------+-----+----------+--------+\n",
      "|3/23/20|  100|        52|  [52.0]|\n",
      "|3/22/20|  100|        51|  [51.0]|\n",
      "+-------+-----+----------+--------+\n",
      "\n",
      "+--------+------------------+\n",
      "|features|        prediction|\n",
      "+--------+------------------+\n",
      "|  [52.0]|52.234453474805974|\n",
      "|  [51.0]|49.227997588316526|\n",
      "+--------+------------------+\n",
      "\n",
      "Difference in 3/22/20: 50.772002411683474\n",
      "Difference in 3/23/20: 47.765546525194026\n",
      "\n",
      "\n",
      "Prediction for Province/State: From Diamond Princess\n",
      "Coefficients: [0.0]\n",
      "Intercept: 0.0\n",
      "numIterations: 1\n",
      "objectiveHistory: [0.0]\n",
      "+---------+\n",
      "|residuals|\n",
      "+---------+\n",
      "|      0.0|\n",
      "|      0.0|\n",
      "|      0.0|\n",
      "|      0.0|\n",
      "|      0.0|\n",
      "|      0.0|\n",
      "|      0.0|\n",
      "|      0.0|\n",
      "|      0.0|\n",
      "|      0.0|\n",
      "|      0.0|\n",
      "|      0.0|\n",
      "|      0.0|\n",
      "|      0.0|\n",
      "|      0.0|\n",
      "|      0.0|\n",
      "|      0.0|\n",
      "|      0.0|\n",
      "+---------+\n",
      "\n",
      "RMSE: 0.000000\n",
      "r2: nan\n",
      "+-------+-----+----------+--------+\n",
      "|   Date|Cases|Date_value|features|\n",
      "+-------+-----+----------+--------+\n",
      "|3/23/20|    0|        52|  [52.0]|\n",
      "|3/22/20|    0|        51|  [51.0]|\n",
      "+-------+-----+----------+--------+\n",
      "\n",
      "+--------+----------+\n",
      "|features|prediction|\n",
      "+--------+----------+\n",
      "|  [52.0]|       0.0|\n",
      "|  [51.0]|       0.0|\n",
      "+--------+----------+\n",
      "\n",
      "Difference in 3/22/20: 0.0\n",
      "Difference in 3/23/20: 0.0\n",
      "\n",
      "\n",
      "Prediction for Province/State: Western Australia\n",
      "Coefficients: [3.9727425447247837]\n",
      "Intercept: -143.20214893941184\n",
      "numIterations: 3\n",
      "objectiveHistory: [0.5, 0.4163826602487993, 0.1331473239833481]\n",
      "+-------------------+\n",
      "|          residuals|\n",
      "+-------------------+\n",
      "| 14.101644963493982|\n",
      "|   11.1289024187692|\n",
      "|   7.15615987404442|\n",
      "| 3.1834173293196386|\n",
      "|-0.7893252154051424|\n",
      "| -3.762067760129952|\n",
      "| -5.734810304854733|\n",
      "| -6.707552849579514|\n",
      "|-10.680295394304295|\n",
      "| -9.653037939029076|\n",
      "|-10.625780483753857|\n",
      "|-14.598523028478638|\n",
      "| -7.571265573203419|\n",
      "|   -8.5440081179282|\n",
      "| -8.516750662652981|\n",
      "| 4.5105067926222375|\n",
      "| 12.537764247897428|\n",
      "|  34.56502170317265|\n",
      "+-------------------+\n",
      "\n",
      "RMSE: 11.984071\n",
      "r2: 0.752572\n",
      "+-------+-----+----------+--------+\n",
      "|   Date|Cases|Date_value|features|\n",
      "+-------+-----+----------+--------+\n",
      "|3/23/20|  120|        52|  [52.0]|\n",
      "|3/22/20|  120|        51|  [51.0]|\n",
      "+-------+-----+----------+--------+\n",
      "\n",
      "+--------+------------------+\n",
      "|features|        prediction|\n",
      "+--------+------------------+\n",
      "|  [52.0]|63.380463386276915|\n",
      "|  [51.0]|59.407720841552134|\n",
      "+--------+------------------+\n",
      "\n",
      "Difference in 3/22/20: 60.592279158447866\n",
      "Difference in 3/23/20: 56.619536613723085\n",
      "\n",
      "\n",
      "Prediction for Province/State: Tasmania\n",
      "Coefficients: [0.680641686887322]\n",
      "Intercept: -23.135518894712757\n",
      "numIterations: 3\n",
      "objectiveHistory: [0.5, 0.4190875411890086, 0.12516340286616062]\n",
      "+--------------------+\n",
      "|           residuals|\n",
      "+--------------------+\n",
      "|  1.6743432274311303|\n",
      "|  0.9937015405438103|\n",
      "|  0.3130598536564868|\n",
      "|-0.36758183323083315|\n",
      "|-0.04822352011815667|\n",
      "| -0.7288652070054766|\n",
      "| -1.4095068938928002|\n",
      "| -1.0901485807801201|\n",
      "| -1.7707902676674436|\n",
      "|-0.45143195455476715|\n",
      "|  -1.132073641442087|\n",
      "| -0.8127153283294106|\n",
      "| -0.4933570152167306|\n",
      "|  -1.173998702104054|\n",
      "|   1.145359611008626|\n",
      "| 0.46471792412129886|\n",
      "| -0.2159237627660211|\n",
      "|   5.103434550346659|\n",
      "+--------------------+\n",
      "\n",
      "RMSE: 1.532163\n",
      "r2: 0.860900\n",
      "+-------+-----+----------+--------+\n",
      "|   Date|Cases|Date_value|features|\n",
      "+-------+-----+----------+--------+\n",
      "|3/23/20|   22|        52|  [52.0]|\n",
      "|3/22/20|   22|        51|  [51.0]|\n",
      "+-------+-----+----------+--------+\n",
      "\n",
      "+--------+------------------+\n",
      "|features|        prediction|\n",
      "+--------+------------------+\n",
      "|  [52.0]|12.257848823427988|\n",
      "|  [51.0]|11.577207136540661|\n",
      "+--------+------------------+\n",
      "\n",
      "Difference in 3/22/20: 10.422792863459339\n",
      "Difference in 3/23/20: 9.742151176572012\n",
      "\n",
      "\n",
      "Prediction for Province/State: Northern Territory\n",
      "Coefficients: [0.05688732876696581]\n",
      "Intercept: -1.360824143829081\n",
      "numIterations: 3\n",
      "objectiveHistory: [0.5000000000000003, 0.4787333694503094, 0.42986908937906887]\n",
      "+--------------------+\n",
      "|           residuals|\n",
      "+--------------------+\n",
      "| 0.48354229451920916|\n",
      "|  0.4266549657522434|\n",
      "| -0.6302323630147224|\n",
      "| -0.6871196917816882|\n",
      "|  -0.744007020548654|\n",
      "| -0.8008943493156198|\n",
      "| 0.14221832191741446|\n",
      "| 0.08533099315044868|\n",
      "|0.028443664383482448|\n",
      "|-0.02844366438348...|\n",
      "|-0.08533099315044912|\n",
      "| -0.1422183219174149|\n",
      "| -0.1991056506843807|\n",
      "|-0.25599297945134647|\n",
      "|-0.31288030821831225|\n",
      "|-0.36976763698527804|\n",
      "|  1.5733450342477562|\n",
      "|  1.5164577054807904|\n",
      "+--------------------+\n",
      "\n",
      "RMSE: 0.651990\n",
      "r2: 0.362363\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "+-------+-----+----------+--------+\n",
      "|   Date|Cases|Date_value|features|\n",
      "+-------+-----+----------+--------+\n",
      "|3/23/20|    3|        52|  [52.0]|\n",
      "|3/22/20|    3|        51|  [51.0]|\n",
      "+-------+-----+----------+--------+\n",
      "\n",
      "+--------+------------------+\n",
      "|features|        prediction|\n",
      "+--------+------------------+\n",
      "|  [52.0]|1.5973169520531412|\n",
      "|  [51.0]|1.5404296232861754|\n",
      "+--------+------------------+\n",
      "\n",
      "Difference in 3/22/20: 1.4595703767138246\n",
      "Difference in 3/23/20: 1.4026830479468588\n",
      "\n",
      "\n",
      "Prediction for Province/State: Australian Capital Territory\n",
      "Coefficients: [0.3264837659778905]\n",
      "Intercept: -11.937965176971346\n",
      "numIterations: 3\n",
      "objectiveHistory: [0.49999999999999994, 0.4384691192416912, 0.2510351188129972]\n",
      "+--------------------+\n",
      "|           residuals|\n",
      "+--------------------+\n",
      "|  1.1640008997009588|\n",
      "|  0.8375171337230682|\n",
      "|  0.5110333677451777|\n",
      "| 0.18454960176728719|\n",
      "|-0.14193416421060334|\n",
      "|-0.46841793018849387|\n",
      "| -0.7949016961663844|\n",
      "|  -1.121385462144275|\n",
      "| -1.4478692281221655|\n",
      "|  -0.774352994100056|\n",
      "| -1.1008367600779465|\n",
      "|  -1.427320526055837|\n",
      "| -0.7538042920337276|\n",
      "|  -1.080288058011618|\n",
      "| -0.4067718239895086|\n",
      "| 0.26674441003260085|\n",
      "|  1.9402606440547103|\n",
      "|    4.61377687807682|\n",
      "+--------------------+\n",
      "\n",
      "RMSE: 1.442392\n",
      "r2: 0.647632\n",
      "+-------+-----+----------+--------+\n",
      "|   Date|Cases|Date_value|features|\n",
      "+-------+-----+----------+--------+\n",
      "|3/23/20|   19|        52|  [52.0]|\n",
      "|3/22/20|   19|        51|  [51.0]|\n",
      "+-------+-----+----------+--------+\n",
      "\n",
      "+--------+-----------------+\n",
      "|features|       prediction|\n",
      "+--------+-----------------+\n",
      "|  [52.0]|5.039190653878961|\n",
      "|  [51.0]|4.712706887901071|\n",
      "+--------+-----------------+\n",
      "\n",
      "Difference in 3/22/20: 14.28729311209893\n",
      "Difference in 3/23/20: 13.960809346121039\n",
      "\n",
      "\n"
     ]
    }
   ],
   "source": [
    "\n",
    "for row in df.collect():\n",
    "    df_state = filter_province(row[\"Province/State\"])\n",
    "    df_state = df_state.drop(*[\"Province/State\",\"Country/Region\", \"Lat\", \"Long\"])\n",
    "    df_state_t = t_data(df_state)\n",
    "    df_state_t = df_state_t.select(\"*\").withColumn(\"Date_value\", monotonically_increasing_id()+1)\n",
    "    print(\"Prediction for Province/State: \" + row[\"Province/State\"])\n",
    "    predict(df_state_t)\n",
    "    \n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.4"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}