classification.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/bit/miniforge3/envs/pace-up/lib/python3.11/site-packages/tqdm/auto.py:21: TqdmWarning: IProgress not found. Please update jupyter and ipywidgets. See https://ipywidgets.readthedocs.io/en/stable/user_install.html\n",
      "  from .autonotebook import tqdm as notebook_tqdm\n",
      "INFO:albumentations.check_version:A new version of Albumentations is available: 1.4.21 (you have 1.4.10). Upgrade using: pip install --upgrade albumentations\n"
     ]
    }
   ],
   "source": [
    "import gc\n",
    "import torch\n",
    "import pandas as pd\n",
    "import utils.constants as const\n",
    "\n",
    "from functools import partial\n",
    "\n",
    "from utils.losses import MosLoss\n",
    "from utils.xception import Mos_Xception\n",
    "from utils.data_loader import get_data_loaders\n",
    "from utils.metrics import accuracy, macro_f1\n",
    "\n",
    "from fastai.vision.all import *\n",
    "from fastai.callback.tracker import SaveModelCallback"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "device = torch.device('cuda:0')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "mom      0.741241\n",
       "alpha    0.712219\n",
       "eps      0.007894\n",
       "wd       0.077542\n",
       "f1       0.824052\n",
       "dtype: float64"
      ]
     },
     "execution_count": 3,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "params_df = pd.read_csv('results/params.csv')\n",
    "avg_params_series = params_df.mean(axis=0)\n",
    "avg_params_df = pd.DataFrame(\n",
    "    data=[avg_params_series.values],\n",
    "    columns=avg_params_series.index,\n",
    ")\n",
    "params_df.mean(axis=0)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>mom</th>\n",
       "      <th>alpha</th>\n",
       "      <th>eps</th>\n",
       "      <th>wd</th>\n",
       "      <th>f1</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>0.741241</td>\n",
       "      <td>0.712219</td>\n",
       "      <td>0.007894</td>\n",
       "      <td>0.077542</td>\n",
       "      <td>0.824052</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "        mom     alpha       eps        wd        f1\n",
       "0  0.741241  0.712219  0.007894  0.077542  0.824052"
      ]
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "avg_params_df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "mom = avg_params_df['mom'].item()\n",
    "alpha = avg_params_df['alpha'].item()\n",
    "eps = avg_params_df['eps'].item()\n",
    "wd = avg_params_df['wd'].item()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Focal Loss with gamma =  0\n"
     ]
    }
   ],
   "source": [
    "opt_func = partial(ranger, mom=mom, alpha=alpha, eps=eps)\n",
    "loss = MosLoss(const.NUM_CLASSES)\n",
    "metrics = [accuracy, macro_f1]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "\n",
       "<style>\n",
       "    /* Turns off some styling */\n",
       "    progress {\n",
       "        /* gets rid of default border in Firefox and Opera. */\n",
       "        border: none;\n",
       "        /* Needs to be in here for Safari polyfill so background images work as expected. */\n",
       "        background-size: auto;\n",
       "    }\n",
       "    progress:not([value]), progress:not([value])::-webkit-progress-bar {\n",
       "        background: repeating-linear-gradient(45deg, #7e7e7e, #7e7e7e 10px, #5c5c5c 10px, #5c5c5c 20px);\n",
       "    }\n",
       "    .progress-bar-interrupted, .progress-bar-interrupted::-webkit-progress-bar {\n",
       "        background: #F44336;\n",
       "    }\n",
       "</style>\n"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: left;\">\n",
       "      <th>epoch</th>\n",
       "      <th>train_loss</th>\n",
       "      <th>valid_loss</th>\n",
       "      <th>accuracy</th>\n",
       "      <th>macro_f1</th>\n",
       "      <th>time</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <td>0</td>\n",
       "      <td>5.218000</td>\n",
       "      <td>4.538208</td>\n",
       "      <td>0.108974</td>\n",
       "      <td>0.107445</td>\n",
       "      <td>01:06</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>1</td>\n",
       "      <td>3.849785</td>\n",
       "      <td>3.210979</td>\n",
       "      <td>0.166667</td>\n",
       "      <td>0.153232</td>\n",
       "      <td>01:00</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>2</td>\n",
       "      <td>3.206007</td>\n",
       "      <td>2.948654</td>\n",
       "      <td>0.471154</td>\n",
       "      <td>0.314615</td>\n",
       "      <td>00:50</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>3</td>\n",
       "      <td>2.733851</td>\n",
       "      <td>2.158173</td>\n",
       "      <td>0.663462</td>\n",
       "      <td>0.511302</td>\n",
       "      <td>00:48</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>4</td>\n",
       "      <td>2.030416</td>\n",
       "      <td>1.390083</td>\n",
       "      <td>0.730769</td>\n",
       "      <td>0.563337</td>\n",
       "      <td>00:49</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>5</td>\n",
       "      <td>1.440167</td>\n",
       "      <td>0.974477</td>\n",
       "      <td>0.858974</td>\n",
       "      <td>0.679199</td>\n",
       "      <td>00:49</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>6</td>\n",
       "      <td>1.051142</td>\n",
       "      <td>0.715534</td>\n",
       "      <td>0.855769</td>\n",
       "      <td>0.710300</td>\n",
       "      <td>00:48</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>7</td>\n",
       "      <td>0.760756</td>\n",
       "      <td>0.611793</td>\n",
       "      <td>0.881410</td>\n",
       "      <td>0.719715</td>\n",
       "      <td>00:46</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>8</td>\n",
       "      <td>0.657553</td>\n",
       "      <td>0.423815</td>\n",
       "      <td>0.929487</td>\n",
       "      <td>0.769274</td>\n",
       "      <td>00:44</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>9</td>\n",
       "      <td>0.576446</td>\n",
       "      <td>1.015526</td>\n",
       "      <td>0.817308</td>\n",
       "      <td>0.636543</td>\n",
       "      <td>00:46</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>10</td>\n",
       "      <td>0.571708</td>\n",
       "      <td>0.585178</td>\n",
       "      <td>0.884615</td>\n",
       "      <td>0.833063</td>\n",
       "      <td>00:44</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>11</td>\n",
       "      <td>0.524341</td>\n",
       "      <td>0.470313</td>\n",
       "      <td>0.907051</td>\n",
       "      <td>0.748463</td>\n",
       "      <td>00:44</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>12</td>\n",
       "      <td>0.516259</td>\n",
       "      <td>0.438939</td>\n",
       "      <td>0.903846</td>\n",
       "      <td>0.775959</td>\n",
       "      <td>00:43</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>13</td>\n",
       "      <td>0.484523</td>\n",
       "      <td>0.383039</td>\n",
       "      <td>0.932692</td>\n",
       "      <td>0.775537</td>\n",
       "      <td>00:45</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>14</td>\n",
       "      <td>0.416653</td>\n",
       "      <td>0.603437</td>\n",
       "      <td>0.865385</td>\n",
       "      <td>0.742026</td>\n",
       "      <td>00:46</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>15</td>\n",
       "      <td>0.371218</td>\n",
       "      <td>0.600065</td>\n",
       "      <td>0.884615</td>\n",
       "      <td>0.798697</td>\n",
       "      <td>00:43</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>16</td>\n",
       "      <td>0.320568</td>\n",
       "      <td>0.529670</td>\n",
       "      <td>0.891026</td>\n",
       "      <td>0.767784</td>\n",
       "      <td>00:44</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>17</td>\n",
       "      <td>0.308908</td>\n",
       "      <td>0.496684</td>\n",
       "      <td>0.903846</td>\n",
       "      <td>0.784062</td>\n",
       "      <td>00:45</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>18</td>\n",
       "      <td>0.321946</td>\n",
       "      <td>0.340696</td>\n",
       "      <td>0.932692</td>\n",
       "      <td>0.814050</td>\n",
       "      <td>00:42</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>19</td>\n",
       "      <td>0.283952</td>\n",
       "      <td>0.149645</td>\n",
       "      <td>0.974359</td>\n",
       "      <td>0.865582</td>\n",
       "      <td>00:42</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>20</td>\n",
       "      <td>0.299263</td>\n",
       "      <td>0.397913</td>\n",
       "      <td>0.932692</td>\n",
       "      <td>0.808598</td>\n",
       "      <td>00:42</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>21</td>\n",
       "      <td>0.295807</td>\n",
       "      <td>0.158253</td>\n",
       "      <td>0.971154</td>\n",
       "      <td>0.919751</td>\n",
       "      <td>00:43</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>22</td>\n",
       "      <td>0.284600</td>\n",
       "      <td>0.348938</td>\n",
       "      <td>0.939103</td>\n",
       "      <td>0.793324</td>\n",
       "      <td>00:43</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>23</td>\n",
       "      <td>0.219807</td>\n",
       "      <td>0.375444</td>\n",
       "      <td>0.935897</td>\n",
       "      <td>0.855782</td>\n",
       "      <td>00:43</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>24</td>\n",
       "      <td>0.181801</td>\n",
       "      <td>0.132781</td>\n",
       "      <td>0.980769</td>\n",
       "      <td>0.885314</td>\n",
       "      <td>00:43</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>25</td>\n",
       "      <td>0.172145</td>\n",
       "      <td>0.190163</td>\n",
       "      <td>0.967949</td>\n",
       "      <td>0.895502</td>\n",
       "      <td>00:42</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>26</td>\n",
       "      <td>0.177705</td>\n",
       "      <td>0.213230</td>\n",
       "      <td>0.971154</td>\n",
       "      <td>0.937111</td>\n",
       "      <td>00:43</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>27</td>\n",
       "      <td>0.156216</td>\n",
       "      <td>0.297422</td>\n",
       "      <td>0.955128</td>\n",
       "      <td>0.872956</td>\n",
       "      <td>00:43</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>28</td>\n",
       "      <td>0.124410</td>\n",
       "      <td>0.119036</td>\n",
       "      <td>0.980769</td>\n",
       "      <td>0.877255</td>\n",
       "      <td>00:42</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>29</td>\n",
       "      <td>0.135330</td>\n",
       "      <td>0.099570</td>\n",
       "      <td>0.980769</td>\n",
       "      <td>0.927114</td>\n",
       "      <td>00:42</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>30</td>\n",
       "      <td>0.105112</td>\n",
       "      <td>0.250122</td>\n",
       "      <td>0.955128</td>\n",
       "      <td>0.791442</td>\n",
       "      <td>00:42</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>31</td>\n",
       "      <td>0.104396</td>\n",
       "      <td>0.080901</td>\n",
       "      <td>0.990385</td>\n",
       "      <td>0.937396</td>\n",
       "      <td>00:44</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>32</td>\n",
       "      <td>0.126727</td>\n",
       "      <td>0.117441</td>\n",
       "      <td>0.977564</td>\n",
       "      <td>0.865719</td>\n",
       "      <td>00:44</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>33</td>\n",
       "      <td>0.141596</td>\n",
       "      <td>0.132596</td>\n",
       "      <td>0.974359</td>\n",
       "      <td>0.857145</td>\n",
       "      <td>00:43</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>34</td>\n",
       "      <td>0.113859</td>\n",
       "      <td>0.178488</td>\n",
       "      <td>0.967949</td>\n",
       "      <td>0.842499</td>\n",
       "      <td>00:44</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>35</td>\n",
       "      <td>0.079967</td>\n",
       "      <td>0.121592</td>\n",
       "      <td>0.980769</td>\n",
       "      <td>0.878382</td>\n",
       "      <td>00:43</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>36</td>\n",
       "      <td>0.056520</td>\n",
       "      <td>0.100617</td>\n",
       "      <td>0.983974</td>\n",
       "      <td>0.905351</td>\n",
       "      <td>00:42</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>37</td>\n",
       "      <td>0.079267</td>\n",
       "      <td>0.117821</td>\n",
       "      <td>0.980769</td>\n",
       "      <td>0.878382</td>\n",
       "      <td>00:43</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>38</td>\n",
       "      <td>0.084336</td>\n",
       "      <td>0.100633</td>\n",
       "      <td>0.980769</td>\n",
       "      <td>0.887667</td>\n",
       "      <td>00:43</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>39</td>\n",
       "      <td>0.063483</td>\n",
       "      <td>0.123505</td>\n",
       "      <td>0.980769</td>\n",
       "      <td>0.903316</td>\n",
       "      <td>00:43</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>40</td>\n",
       "      <td>0.042435</td>\n",
       "      <td>0.128969</td>\n",
       "      <td>0.980769</td>\n",
       "      <td>0.892188</td>\n",
       "      <td>00:43</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>41</td>\n",
       "      <td>0.049852</td>\n",
       "      <td>0.124780</td>\n",
       "      <td>0.980769</td>\n",
       "      <td>0.899845</td>\n",
       "      <td>00:43</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>42</td>\n",
       "      <td>0.053603</td>\n",
       "      <td>0.115291</td>\n",
       "      <td>0.974359</td>\n",
       "      <td>0.869126</td>\n",
       "      <td>00:43</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>43</td>\n",
       "      <td>0.042774</td>\n",
       "      <td>0.114280</td>\n",
       "      <td>0.980769</td>\n",
       "      <td>0.896777</td>\n",
       "      <td>00:43</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>44</td>\n",
       "      <td>0.034310</td>\n",
       "      <td>0.094156</td>\n",
       "      <td>0.983974</td>\n",
       "      <td>0.912283</td>\n",
       "      <td>00:43</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>45</td>\n",
       "      <td>0.030932</td>\n",
       "      <td>0.105305</td>\n",
       "      <td>0.983974</td>\n",
       "      <td>0.913171</td>\n",
       "      <td>00:43</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>46</td>\n",
       "      <td>0.027214</td>\n",
       "      <td>0.068267</td>\n",
       "      <td>0.987179</td>\n",
       "      <td>0.911429</td>\n",
       "      <td>00:43</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>47</td>\n",
       "      <td>0.029476</td>\n",
       "      <td>0.069498</td>\n",
       "      <td>0.987179</td>\n",
       "      <td>0.911429</td>\n",
       "      <td>00:43</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>48</td>\n",
       "      <td>0.036984</td>\n",
       "      <td>0.065256</td>\n",
       "      <td>0.987179</td>\n",
       "      <td>0.911429</td>\n",
       "      <td>00:43</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>49</td>\n",
       "      <td>0.030002</td>\n",
       "      <td>0.088562</td>\n",
       "      <td>0.983974</td>\n",
       "      <td>0.907551</td>\n",
       "      <td>00:43</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>50</td>\n",
       "      <td>0.020191</td>\n",
       "      <td>0.106447</td>\n",
       "      <td>0.977564</td>\n",
       "      <td>0.854615</td>\n",
       "      <td>00:43</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>51</td>\n",
       "      <td>0.035500</td>\n",
       "      <td>0.060897</td>\n",
       "      <td>0.993590</td>\n",
       "      <td>0.955805</td>\n",
       "      <td>00:44</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>52</td>\n",
       "      <td>0.021948</td>\n",
       "      <td>0.058754</td>\n",
       "      <td>0.993590</td>\n",
       "      <td>0.955805</td>\n",
       "      <td>00:43</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>53</td>\n",
       "      <td>0.012916</td>\n",
       "      <td>0.064093</td>\n",
       "      <td>0.993590</td>\n",
       "      <td>0.955805</td>\n",
       "      <td>00:42</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>54</td>\n",
       "      <td>0.023356</td>\n",
       "      <td>0.061283</td>\n",
       "      <td>0.990385</td>\n",
       "      <td>0.929337</td>\n",
       "      <td>00:42</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>55</td>\n",
       "      <td>0.020073</td>\n",
       "      <td>0.064358</td>\n",
       "      <td>0.990385</td>\n",
       "      <td>0.929337</td>\n",
       "      <td>00:42</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>56</td>\n",
       "      <td>0.014721</td>\n",
       "      <td>0.056505</td>\n",
       "      <td>0.993590</td>\n",
       "      <td>0.955805</td>\n",
       "      <td>00:43</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>57</td>\n",
       "      <td>0.010210</td>\n",
       "      <td>0.061411</td>\n",
       "      <td>0.990385</td>\n",
       "      <td>0.929337</td>\n",
       "      <td>00:43</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>58</td>\n",
       "      <td>0.025464</td>\n",
       "      <td>0.057746</td>\n",
       "      <td>0.993590</td>\n",
       "      <td>0.955805</td>\n",
       "      <td>00:42</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>59</td>\n",
       "      <td>0.024893</td>\n",
       "      <td>0.058789</td>\n",
       "      <td>0.993590</td>\n",
       "      <td>0.955805</td>\n",
       "      <td>00:42</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/bit/miniforge3/envs/pace-up/lib/python3.11/site-packages/torch/nn/modules/conv.py:456: UserWarning: Applied workaround for CuDNN issue, install nvrtc.so (Triggered internally at /opt/conda/conda-bld/pytorch_1716905971190/work/aten/src/ATen/native/cudnn/Conv_v8.cpp:84.)\n",
      "  return F.conv2d(input, weight, bias, self.stride,\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Better model found at epoch 0 with macro_f1 value: 0.10744549568078977.\n",
      "Better model found at epoch 1 with macro_f1 value: 0.15323157095404455.\n",
      "Better model found at epoch 2 with macro_f1 value: 0.31461526965732245.\n",
      "Better model found at epoch 3 with macro_f1 value: 0.5113015902517793.\n",
      "Better model found at epoch 4 with macro_f1 value: 0.5633369219390725.\n",
      "Better model found at epoch 5 with macro_f1 value: 0.6791994944626524.\n",
      "Better model found at epoch 6 with macro_f1 value: 0.7103000381261252.\n",
      "Better model found at epoch 7 with macro_f1 value: 0.7197145621185793.\n",
      "Better model found at epoch 8 with macro_f1 value: 0.7692743199237595.\n",
      "Better model found at epoch 10 with macro_f1 value: 0.8330626588386372.\n",
      "Better model found at epoch 19 with macro_f1 value: 0.8655815117904999.\n",
      "Better model found at epoch 21 with macro_f1 value: 0.9197506513745828.\n",
      "Better model found at epoch 26 with macro_f1 value: 0.9371114198700405.\n",
      "Better model found at epoch 31 with macro_f1 value: 0.9373957741754819.\n",
      "Better model found at epoch 51 with macro_f1 value: 0.9558047156613466.\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "\n",
       "<style>\n",
       "    /* Turns off some styling */\n",
       "    progress {\n",
       "        /* gets rid of default border in Firefox and Opera. */\n",
       "        border: none;\n",
       "        /* Needs to be in here for Safari polyfill so background images work as expected. */\n",
       "        background-size: auto;\n",
       "    }\n",
       "    progress:not([value]), progress:not([value])::-webkit-progress-bar {\n",
       "        background: repeating-linear-gradient(45deg, #7e7e7e, #7e7e7e 10px, #5c5c5c 10px, #5c5c5c 20px);\n",
       "    }\n",
       "    .progress-bar-interrupted, .progress-bar-interrupted::-webkit-progress-bar {\n",
       "        background: #F44336;\n",
       "    }\n",
       "</style>\n"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: left;\">\n",
       "      <th>epoch</th>\n",
       "      <th>train_loss</th>\n",
       "      <th>valid_loss</th>\n",
       "      <th>accuracy</th>\n",
       "      <th>macro_f1</th>\n",
       "      <th>time</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <td>0</td>\n",
       "      <td>5.264956</td>\n",
       "      <td>4.554950</td>\n",
       "      <td>0.202572</td>\n",
       "      <td>0.084449</td>\n",
       "      <td>00:44</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>1</td>\n",
       "      <td>3.860215</td>\n",
       "      <td>3.192590</td>\n",
       "      <td>0.347267</td>\n",
       "      <td>0.226875</td>\n",
       "      <td>00:44</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>2</td>\n",
       "      <td>3.183466</td>\n",
       "      <td>2.844607</td>\n",
       "      <td>0.546624</td>\n",
       "      <td>0.429438</td>\n",
       "      <td>00:44</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>3</td>\n",
       "      <td>2.635085</td>\n",
       "      <td>2.038682</td>\n",
       "      <td>0.697749</td>\n",
       "      <td>0.547006</td>\n",
       "      <td>00:44</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>4</td>\n",
       "      <td>1.912460</td>\n",
       "      <td>1.349947</td>\n",
       "      <td>0.790997</td>\n",
       "      <td>0.646482</td>\n",
       "      <td>00:43</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>5</td>\n",
       "      <td>1.390773</td>\n",
       "      <td>0.831477</td>\n",
       "      <td>0.884244</td>\n",
       "      <td>0.719413</td>\n",
       "      <td>00:46</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>6</td>\n",
       "      <td>1.029158</td>\n",
       "      <td>0.581282</td>\n",
       "      <td>0.916399</td>\n",
       "      <td>0.694008</td>\n",
       "      <td>00:45</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>7</td>\n",
       "      <td>0.816825</td>\n",
       "      <td>0.505411</td>\n",
       "      <td>0.926045</td>\n",
       "      <td>0.741604</td>\n",
       "      <td>00:43</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>8</td>\n",
       "      <td>0.684918</td>\n",
       "      <td>0.414225</td>\n",
       "      <td>0.916399</td>\n",
       "      <td>0.707785</td>\n",
       "      <td>00:43</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>9</td>\n",
       "      <td>0.609217</td>\n",
       "      <td>0.725482</td>\n",
       "      <td>0.826367</td>\n",
       "      <td>0.692236</td>\n",
       "      <td>00:44</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>10</td>\n",
       "      <td>0.604559</td>\n",
       "      <td>0.360506</td>\n",
       "      <td>0.932476</td>\n",
       "      <td>0.821679</td>\n",
       "      <td>00:43</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>11</td>\n",
       "      <td>0.499412</td>\n",
       "      <td>0.525324</td>\n",
       "      <td>0.906752</td>\n",
       "      <td>0.802930</td>\n",
       "      <td>00:43</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>12</td>\n",
       "      <td>0.445327</td>\n",
       "      <td>0.658879</td>\n",
       "      <td>0.871383</td>\n",
       "      <td>0.683149</td>\n",
       "      <td>00:44</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>13</td>\n",
       "      <td>0.503216</td>\n",
       "      <td>0.555291</td>\n",
       "      <td>0.903537</td>\n",
       "      <td>0.731714</td>\n",
       "      <td>00:46</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>14</td>\n",
       "      <td>0.466608</td>\n",
       "      <td>0.426531</td>\n",
       "      <td>0.922830</td>\n",
       "      <td>0.771144</td>\n",
       "      <td>00:43</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>15</td>\n",
       "      <td>0.414778</td>\n",
       "      <td>0.695270</td>\n",
       "      <td>0.868167</td>\n",
       "      <td>0.747564</td>\n",
       "      <td>00:43</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>16</td>\n",
       "      <td>0.385786</td>\n",
       "      <td>0.571123</td>\n",
       "      <td>0.881029</td>\n",
       "      <td>0.791624</td>\n",
       "      <td>00:43</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>17</td>\n",
       "      <td>0.353605</td>\n",
       "      <td>0.192931</td>\n",
       "      <td>0.961415</td>\n",
       "      <td>0.877803</td>\n",
       "      <td>00:43</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>18</td>\n",
       "      <td>0.310963</td>\n",
       "      <td>0.194403</td>\n",
       "      <td>0.967846</td>\n",
       "      <td>0.856528</td>\n",
       "      <td>00:43</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>19</td>\n",
       "      <td>0.284718</td>\n",
       "      <td>0.198658</td>\n",
       "      <td>0.958199</td>\n",
       "      <td>0.828062</td>\n",
       "      <td>00:44</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>20</td>\n",
       "      <td>0.243721</td>\n",
       "      <td>0.135663</td>\n",
       "      <td>0.971061</td>\n",
       "      <td>0.858199</td>\n",
       "      <td>00:45</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>21</td>\n",
       "      <td>0.268665</td>\n",
       "      <td>0.293789</td>\n",
       "      <td>0.945338</td>\n",
       "      <td>0.841478</td>\n",
       "      <td>00:45</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>22</td>\n",
       "      <td>0.227747</td>\n",
       "      <td>0.320666</td>\n",
       "      <td>0.932476</td>\n",
       "      <td>0.794637</td>\n",
       "      <td>00:43</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>23</td>\n",
       "      <td>0.236653</td>\n",
       "      <td>0.243137</td>\n",
       "      <td>0.948553</td>\n",
       "      <td>0.803073</td>\n",
       "      <td>00:43</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>24</td>\n",
       "      <td>0.211778</td>\n",
       "      <td>0.333990</td>\n",
       "      <td>0.945338</td>\n",
       "      <td>0.807903</td>\n",
       "      <td>00:45</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>25</td>\n",
       "      <td>0.222561</td>\n",
       "      <td>0.170069</td>\n",
       "      <td>0.961415</td>\n",
       "      <td>0.813819</td>\n",
       "      <td>00:46</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>26</td>\n",
       "      <td>0.182709</td>\n",
       "      <td>0.309911</td>\n",
       "      <td>0.938907</td>\n",
       "      <td>0.856015</td>\n",
       "      <td>00:44</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>27</td>\n",
       "      <td>0.155808</td>\n",
       "      <td>0.444608</td>\n",
       "      <td>0.926045</td>\n",
       "      <td>0.794574</td>\n",
       "      <td>00:42</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>28</td>\n",
       "      <td>0.178122</td>\n",
       "      <td>0.092029</td>\n",
       "      <td>0.990354</td>\n",
       "      <td>0.954979</td>\n",
       "      <td>00:43</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>29</td>\n",
       "      <td>0.139815</td>\n",
       "      <td>0.075078</td>\n",
       "      <td>0.987138</td>\n",
       "      <td>0.911274</td>\n",
       "      <td>00:43</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>30</td>\n",
       "      <td>0.118187</td>\n",
       "      <td>0.111887</td>\n",
       "      <td>0.971061</td>\n",
       "      <td>0.918556</td>\n",
       "      <td>00:43</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>31</td>\n",
       "      <td>0.137827</td>\n",
       "      <td>0.077298</td>\n",
       "      <td>0.983923</td>\n",
       "      <td>0.922526</td>\n",
       "      <td>00:42</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>32</td>\n",
       "      <td>0.112548</td>\n",
       "      <td>0.046724</td>\n",
       "      <td>0.993569</td>\n",
       "      <td>0.967349</td>\n",
       "      <td>00:41</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>33</td>\n",
       "      <td>0.103646</td>\n",
       "      <td>0.091079</td>\n",
       "      <td>0.980707</td>\n",
       "      <td>0.916063</td>\n",
       "      <td>00:42</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>34</td>\n",
       "      <td>0.102979</td>\n",
       "      <td>0.054541</td>\n",
       "      <td>0.987138</td>\n",
       "      <td>0.893787</td>\n",
       "      <td>00:40</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>35</td>\n",
       "      <td>0.100896</td>\n",
       "      <td>0.041451</td>\n",
       "      <td>0.996785</td>\n",
       "      <td>0.973447</td>\n",
       "      <td>00:41</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>36</td>\n",
       "      <td>0.077572</td>\n",
       "      <td>0.043746</td>\n",
       "      <td>0.993569</td>\n",
       "      <td>0.946893</td>\n",
       "      <td>00:41</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>37</td>\n",
       "      <td>0.105091</td>\n",
       "      <td>0.107253</td>\n",
       "      <td>0.980707</td>\n",
       "      <td>0.866431</td>\n",
       "      <td>00:40</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>38</td>\n",
       "      <td>0.082994</td>\n",
       "      <td>0.035005</td>\n",
       "      <td>0.993569</td>\n",
       "      <td>0.946893</td>\n",
       "      <td>00:40</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>39</td>\n",
       "      <td>0.078599</td>\n",
       "      <td>0.031965</td>\n",
       "      <td>0.996785</td>\n",
       "      <td>0.982216</td>\n",
       "      <td>00:40</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>40</td>\n",
       "      <td>0.051370</td>\n",
       "      <td>0.026524</td>\n",
       "      <td>0.993569</td>\n",
       "      <td>0.946893</td>\n",
       "      <td>00:42</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>41</td>\n",
       "      <td>0.047316</td>\n",
       "      <td>0.039656</td>\n",
       "      <td>0.993569</td>\n",
       "      <td>0.946893</td>\n",
       "      <td>00:41</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>42</td>\n",
       "      <td>0.044111</td>\n",
       "      <td>0.030499</td>\n",
       "      <td>0.993569</td>\n",
       "      <td>0.946893</td>\n",
       "      <td>00:41</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>43</td>\n",
       "      <td>0.049875</td>\n",
       "      <td>0.091943</td>\n",
       "      <td>0.987138</td>\n",
       "      <td>0.945062</td>\n",
       "      <td>00:40</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>44</td>\n",
       "      <td>0.035122</td>\n",
       "      <td>0.100008</td>\n",
       "      <td>0.977492</td>\n",
       "      <td>0.874434</td>\n",
       "      <td>00:40</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>45</td>\n",
       "      <td>0.030537</td>\n",
       "      <td>0.030881</td>\n",
       "      <td>0.996785</td>\n",
       "      <td>0.973447</td>\n",
       "      <td>00:40</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>46</td>\n",
       "      <td>0.025155</td>\n",
       "      <td>0.040680</td>\n",
       "      <td>0.993569</td>\n",
       "      <td>0.946893</td>\n",
       "      <td>00:40</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>47</td>\n",
       "      <td>0.026533</td>\n",
       "      <td>0.123623</td>\n",
       "      <td>0.977492</td>\n",
       "      <td>0.904289</td>\n",
       "      <td>00:40</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>48</td>\n",
       "      <td>0.020676</td>\n",
       "      <td>0.059076</td>\n",
       "      <td>0.990354</td>\n",
       "      <td>0.920340</td>\n",
       "      <td>00:45</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>49</td>\n",
       "      <td>0.016498</td>\n",
       "      <td>0.071689</td>\n",
       "      <td>0.980707</td>\n",
       "      <td>0.912922</td>\n",
       "      <td>00:51</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>50</td>\n",
       "      <td>0.018268</td>\n",
       "      <td>0.054871</td>\n",
       "      <td>0.987138</td>\n",
       "      <td>0.917008</td>\n",
       "      <td>00:52</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>51</td>\n",
       "      <td>0.013770</td>\n",
       "      <td>0.048921</td>\n",
       "      <td>0.990354</td>\n",
       "      <td>0.920340</td>\n",
       "      <td>00:55</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>52</td>\n",
       "      <td>0.016570</td>\n",
       "      <td>0.039357</td>\n",
       "      <td>0.990354</td>\n",
       "      <td>0.920340</td>\n",
       "      <td>00:54</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>53</td>\n",
       "      <td>0.014281</td>\n",
       "      <td>0.034719</td>\n",
       "      <td>0.993569</td>\n",
       "      <td>0.946893</td>\n",
       "      <td>00:57</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>54</td>\n",
       "      <td>0.017768</td>\n",
       "      <td>0.032563</td>\n",
       "      <td>0.990354</td>\n",
       "      <td>0.920340</td>\n",
       "      <td>00:55</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>55</td>\n",
       "      <td>0.022821</td>\n",
       "      <td>0.048563</td>\n",
       "      <td>0.990354</td>\n",
       "      <td>0.920340</td>\n",
       "      <td>00:53</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>56</td>\n",
       "      <td>0.017629</td>\n",
       "      <td>0.039805</td>\n",
       "      <td>0.990354</td>\n",
       "      <td>0.920340</td>\n",
       "      <td>01:01</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>57</td>\n",
       "      <td>0.011686</td>\n",
       "      <td>0.039363</td>\n",
       "      <td>0.990354</td>\n",
       "      <td>0.920340</td>\n",
       "      <td>00:55</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>58</td>\n",
       "      <td>0.014437</td>\n",
       "      <td>0.041781</td>\n",
       "      <td>0.990354</td>\n",
       "      <td>0.920340</td>\n",
       "      <td>00:55</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>59</td>\n",
       "      <td>0.022792</td>\n",
       "      <td>0.046153</td>\n",
       "      <td>0.990354</td>\n",
       "      <td>0.920340</td>\n",
       "      <td>00:59</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Better model found at epoch 0 with macro_f1 value: 0.0844492464615216.\n",
      "Better model found at epoch 1 with macro_f1 value: 0.22687469881019692.\n",
      "Better model found at epoch 2 with macro_f1 value: 0.4294382818968678.\n",
      "Better model found at epoch 3 with macro_f1 value: 0.5470056653147153.\n",
      "Better model found at epoch 4 with macro_f1 value: 0.6464817400525825.\n",
      "Better model found at epoch 5 with macro_f1 value: 0.7194127749879228.\n",
      "Better model found at epoch 7 with macro_f1 value: 0.7416041641673085.\n",
      "Better model found at epoch 10 with macro_f1 value: 0.8216792960906641.\n",
      "Better model found at epoch 17 with macro_f1 value: 0.8778031861023219.\n",
      "Better model found at epoch 28 with macro_f1 value: 0.9549786036664992.\n",
      "Better model found at epoch 32 with macro_f1 value: 0.9673493221465422.\n",
      "Better model found at epoch 35 with macro_f1 value: 0.9734467378902604.\n",
      "Better model found at epoch 39 with macro_f1 value: 0.9822158707474893.\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "\n",
       "<style>\n",
       "    /* Turns off some styling */\n",
       "    progress {\n",
       "        /* gets rid of default border in Firefox and Opera. */\n",
       "        border: none;\n",
       "        /* Needs to be in here for Safari polyfill so background images work as expected. */\n",
       "        background-size: auto;\n",
       "    }\n",
       "    progress:not([value]), progress:not([value])::-webkit-progress-bar {\n",
       "        background: repeating-linear-gradient(45deg, #7e7e7e, #7e7e7e 10px, #5c5c5c 10px, #5c5c5c 20px);\n",
       "    }\n",
       "    .progress-bar-interrupted, .progress-bar-interrupted::-webkit-progress-bar {\n",
       "        background: #F44336;\n",
       "    }\n",
       "</style>\n"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: left;\">\n",
       "      <th>epoch</th>\n",
       "      <th>train_loss</th>\n",
       "      <th>valid_loss</th>\n",
       "      <th>accuracy</th>\n",
       "      <th>macro_f1</th>\n",
       "      <th>time</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <td>0</td>\n",
       "      <td>5.330852</td>\n",
       "      <td>4.607382</td>\n",
       "      <td>0.157556</td>\n",
       "      <td>0.056198</td>\n",
       "      <td>00:57</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>1</td>\n",
       "      <td>3.891535</td>\n",
       "      <td>3.214992</td>\n",
       "      <td>0.344051</td>\n",
       "      <td>0.152530</td>\n",
       "      <td>00:57</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>2</td>\n",
       "      <td>3.204437</td>\n",
       "      <td>2.957956</td>\n",
       "      <td>0.508039</td>\n",
       "      <td>0.327165</td>\n",
       "      <td>00:57</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>3</td>\n",
       "      <td>2.703763</td>\n",
       "      <td>2.216852</td>\n",
       "      <td>0.623794</td>\n",
       "      <td>0.414269</td>\n",
       "      <td>00:58</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>4</td>\n",
       "      <td>1.969512</td>\n",
       "      <td>1.418118</td>\n",
       "      <td>0.774920</td>\n",
       "      <td>0.517611</td>\n",
       "      <td>00:53</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>5</td>\n",
       "      <td>1.389579</td>\n",
       "      <td>0.995804</td>\n",
       "      <td>0.768489</td>\n",
       "      <td>0.550069</td>\n",
       "      <td>00:52</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>6</td>\n",
       "      <td>1.005738</td>\n",
       "      <td>0.620149</td>\n",
       "      <td>0.900322</td>\n",
       "      <td>0.682332</td>\n",
       "      <td>00:55</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>7</td>\n",
       "      <td>0.777403</td>\n",
       "      <td>0.838140</td>\n",
       "      <td>0.797428</td>\n",
       "      <td>0.608672</td>\n",
       "      <td>00:55</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>8</td>\n",
       "      <td>0.689747</td>\n",
       "      <td>0.512068</td>\n",
       "      <td>0.913183</td>\n",
       "      <td>0.697290</td>\n",
       "      <td>00:53</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>9</td>\n",
       "      <td>0.624452</td>\n",
       "      <td>0.604392</td>\n",
       "      <td>0.887460</td>\n",
       "      <td>0.706428</td>\n",
       "      <td>00:54</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>10</td>\n",
       "      <td>0.574015</td>\n",
       "      <td>0.454732</td>\n",
       "      <td>0.890675</td>\n",
       "      <td>0.809248</td>\n",
       "      <td>00:54</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>11</td>\n",
       "      <td>0.523870</td>\n",
       "      <td>0.542654</td>\n",
       "      <td>0.887460</td>\n",
       "      <td>0.723460</td>\n",
       "      <td>00:55</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>12</td>\n",
       "      <td>0.490031</td>\n",
       "      <td>0.470899</td>\n",
       "      <td>0.897106</td>\n",
       "      <td>0.718099</td>\n",
       "      <td>00:57</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>13</td>\n",
       "      <td>0.416683</td>\n",
       "      <td>0.623991</td>\n",
       "      <td>0.864952</td>\n",
       "      <td>0.693962</td>\n",
       "      <td>00:54</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>14</td>\n",
       "      <td>0.429902</td>\n",
       "      <td>0.403508</td>\n",
       "      <td>0.929260</td>\n",
       "      <td>0.775877</td>\n",
       "      <td>00:54</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>15</td>\n",
       "      <td>0.395269</td>\n",
       "      <td>0.853001</td>\n",
       "      <td>0.845659</td>\n",
       "      <td>0.765590</td>\n",
       "      <td>00:58</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>16</td>\n",
       "      <td>0.381532</td>\n",
       "      <td>0.546314</td>\n",
       "      <td>0.890675</td>\n",
       "      <td>0.771034</td>\n",
       "      <td>00:56</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>17</td>\n",
       "      <td>0.350636</td>\n",
       "      <td>0.293881</td>\n",
       "      <td>0.938907</td>\n",
       "      <td>0.799798</td>\n",
       "      <td>00:52</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>18</td>\n",
       "      <td>0.323571</td>\n",
       "      <td>0.271713</td>\n",
       "      <td>0.954984</td>\n",
       "      <td>0.839761</td>\n",
       "      <td>00:56</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>19</td>\n",
       "      <td>0.290259</td>\n",
       "      <td>0.155273</td>\n",
       "      <td>0.980707</td>\n",
       "      <td>0.923080</td>\n",
       "      <td>00:53</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>20</td>\n",
       "      <td>0.229002</td>\n",
       "      <td>0.104141</td>\n",
       "      <td>0.987138</td>\n",
       "      <td>0.927135</td>\n",
       "      <td>00:52</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>21</td>\n",
       "      <td>0.252110</td>\n",
       "      <td>0.304666</td>\n",
       "      <td>0.964630</td>\n",
       "      <td>0.846443</td>\n",
       "      <td>00:59</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>22</td>\n",
       "      <td>0.231593</td>\n",
       "      <td>0.489886</td>\n",
       "      <td>0.919614</td>\n",
       "      <td>0.816108</td>\n",
       "      <td>01:07</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>23</td>\n",
       "      <td>0.196874</td>\n",
       "      <td>0.138398</td>\n",
       "      <td>0.967846</td>\n",
       "      <td>0.854483</td>\n",
       "      <td>01:05</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>24</td>\n",
       "      <td>0.160468</td>\n",
       "      <td>0.144057</td>\n",
       "      <td>0.971061</td>\n",
       "      <td>0.878802</td>\n",
       "      <td>00:54</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>25</td>\n",
       "      <td>0.238398</td>\n",
       "      <td>0.271605</td>\n",
       "      <td>0.935691</td>\n",
       "      <td>0.861061</td>\n",
       "      <td>00:56</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>26</td>\n",
       "      <td>0.204749</td>\n",
       "      <td>0.221598</td>\n",
       "      <td>0.958199</td>\n",
       "      <td>0.833427</td>\n",
       "      <td>01:01</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>27</td>\n",
       "      <td>0.147235</td>\n",
       "      <td>0.144931</td>\n",
       "      <td>0.980707</td>\n",
       "      <td>0.923080</td>\n",
       "      <td>00:57</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>28</td>\n",
       "      <td>0.151639</td>\n",
       "      <td>0.162333</td>\n",
       "      <td>0.980707</td>\n",
       "      <td>0.923080</td>\n",
       "      <td>00:56</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>29</td>\n",
       "      <td>0.196923</td>\n",
       "      <td>0.218170</td>\n",
       "      <td>0.964630</td>\n",
       "      <td>0.884219</td>\n",
       "      <td>00:57</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>30</td>\n",
       "      <td>0.128070</td>\n",
       "      <td>0.205626</td>\n",
       "      <td>0.967846</td>\n",
       "      <td>0.832030</td>\n",
       "      <td>00:54</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>31</td>\n",
       "      <td>0.130831</td>\n",
       "      <td>0.302371</td>\n",
       "      <td>0.951769</td>\n",
       "      <td>0.813908</td>\n",
       "      <td>00:56</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>32</td>\n",
       "      <td>0.101407</td>\n",
       "      <td>0.255529</td>\n",
       "      <td>0.958199</td>\n",
       "      <td>0.814964</td>\n",
       "      <td>01:01</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>33</td>\n",
       "      <td>0.097675</td>\n",
       "      <td>0.185864</td>\n",
       "      <td>0.974277</td>\n",
       "      <td>0.881643</td>\n",
       "      <td>01:03</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>34</td>\n",
       "      <td>0.070783</td>\n",
       "      <td>0.143511</td>\n",
       "      <td>0.980707</td>\n",
       "      <td>0.923080</td>\n",
       "      <td>01:03</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>35</td>\n",
       "      <td>0.084878</td>\n",
       "      <td>0.180823</td>\n",
       "      <td>0.971061</td>\n",
       "      <td>0.878311</td>\n",
       "      <td>01:03</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>36</td>\n",
       "      <td>0.075607</td>\n",
       "      <td>0.253770</td>\n",
       "      <td>0.974277</td>\n",
       "      <td>0.900590</td>\n",
       "      <td>01:03</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>37</td>\n",
       "      <td>0.073538</td>\n",
       "      <td>0.126475</td>\n",
       "      <td>0.980707</td>\n",
       "      <td>0.904647</td>\n",
       "      <td>00:55</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>38</td>\n",
       "      <td>0.054921</td>\n",
       "      <td>0.143389</td>\n",
       "      <td>0.980707</td>\n",
       "      <td>0.917385</td>\n",
       "      <td>01:01</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>39</td>\n",
       "      <td>0.070866</td>\n",
       "      <td>0.135826</td>\n",
       "      <td>0.983923</td>\n",
       "      <td>0.935350</td>\n",
       "      <td>01:03</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>40</td>\n",
       "      <td>0.056876</td>\n",
       "      <td>0.155153</td>\n",
       "      <td>0.980707</td>\n",
       "      <td>0.927931</td>\n",
       "      <td>01:11</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>41</td>\n",
       "      <td>0.043569</td>\n",
       "      <td>0.129561</td>\n",
       "      <td>0.987138</td>\n",
       "      <td>0.954949</td>\n",
       "      <td>01:06</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>42</td>\n",
       "      <td>0.035823</td>\n",
       "      <td>0.117893</td>\n",
       "      <td>0.987138</td>\n",
       "      <td>0.940129</td>\n",
       "      <td>01:10</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>43</td>\n",
       "      <td>0.048096</td>\n",
       "      <td>0.134785</td>\n",
       "      <td>0.977492</td>\n",
       "      <td>0.909341</td>\n",
       "      <td>01:11</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>44</td>\n",
       "      <td>0.031144</td>\n",
       "      <td>0.211946</td>\n",
       "      <td>0.974277</td>\n",
       "      <td>0.908945</td>\n",
       "      <td>01:04</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>45</td>\n",
       "      <td>0.032619</td>\n",
       "      <td>0.187131</td>\n",
       "      <td>0.977492</td>\n",
       "      <td>0.926910</td>\n",
       "      <td>01:08</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>46</td>\n",
       "      <td>0.022698</td>\n",
       "      <td>0.160240</td>\n",
       "      <td>0.977492</td>\n",
       "      <td>0.897139</td>\n",
       "      <td>01:05</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>47</td>\n",
       "      <td>0.034829</td>\n",
       "      <td>0.165463</td>\n",
       "      <td>0.977492</td>\n",
       "      <td>0.904093</td>\n",
       "      <td>01:13</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>48</td>\n",
       "      <td>0.028621</td>\n",
       "      <td>0.178398</td>\n",
       "      <td>0.983923</td>\n",
       "      <td>0.954349</td>\n",
       "      <td>01:13</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>49</td>\n",
       "      <td>0.027693</td>\n",
       "      <td>0.194680</td>\n",
       "      <td>0.977492</td>\n",
       "      <td>0.909830</td>\n",
       "      <td>01:11</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>50</td>\n",
       "      <td>0.028264</td>\n",
       "      <td>0.175251</td>\n",
       "      <td>0.980707</td>\n",
       "      <td>0.927796</td>\n",
       "      <td>01:07</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>51</td>\n",
       "      <td>0.029507</td>\n",
       "      <td>0.171225</td>\n",
       "      <td>0.977492</td>\n",
       "      <td>0.904093</td>\n",
       "      <td>01:02</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>52</td>\n",
       "      <td>0.028585</td>\n",
       "      <td>0.161526</td>\n",
       "      <td>0.977492</td>\n",
       "      <td>0.904093</td>\n",
       "      <td>01:01</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>53</td>\n",
       "      <td>0.016002</td>\n",
       "      <td>0.177456</td>\n",
       "      <td>0.977492</td>\n",
       "      <td>0.930046</td>\n",
       "      <td>01:02</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>54</td>\n",
       "      <td>0.013930</td>\n",
       "      <td>0.177235</td>\n",
       "      <td>0.974277</td>\n",
       "      <td>0.886128</td>\n",
       "      <td>00:54</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>55</td>\n",
       "      <td>0.016764</td>\n",
       "      <td>0.167886</td>\n",
       "      <td>0.974277</td>\n",
       "      <td>0.903493</td>\n",
       "      <td>00:52</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>56</td>\n",
       "      <td>0.016455</td>\n",
       "      <td>0.168066</td>\n",
       "      <td>0.977492</td>\n",
       "      <td>0.904093</td>\n",
       "      <td>00:54</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>57</td>\n",
       "      <td>0.018440</td>\n",
       "      <td>0.165155</td>\n",
       "      <td>0.977492</td>\n",
       "      <td>0.904093</td>\n",
       "      <td>00:50</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>58</td>\n",
       "      <td>0.012359</td>\n",
       "      <td>0.164214</td>\n",
       "      <td>0.974277</td>\n",
       "      <td>0.903493</td>\n",
       "      <td>00:52</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>59</td>\n",
       "      <td>0.015531</td>\n",
       "      <td>0.173681</td>\n",
       "      <td>0.974277</td>\n",
       "      <td>0.903493</td>\n",
       "      <td>01:02</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Better model found at epoch 0 with macro_f1 value: 0.05619800401024788.\n",
      "Better model found at epoch 1 with macro_f1 value: 0.15253007819595904.\n",
      "Better model found at epoch 2 with macro_f1 value: 0.32716507939483347.\n",
      "Better model found at epoch 3 with macro_f1 value: 0.4142688228445481.\n",
      "Better model found at epoch 4 with macro_f1 value: 0.5176113246404971.\n",
      "Better model found at epoch 5 with macro_f1 value: 0.5500693032369409.\n",
      "Better model found at epoch 6 with macro_f1 value: 0.6823319894619996.\n",
      "Better model found at epoch 8 with macro_f1 value: 0.6972903469073506.\n",
      "Better model found at epoch 9 with macro_f1 value: 0.7064280018084957.\n",
      "Better model found at epoch 10 with macro_f1 value: 0.8092482902110991.\n",
      "Better model found at epoch 18 with macro_f1 value: 0.8397614105612403.\n",
      "Better model found at epoch 19 with macro_f1 value: 0.9230795432572566.\n",
      "Better model found at epoch 20 with macro_f1 value: 0.9271350645689408.\n",
      "Better model found at epoch 39 with macro_f1 value: 0.9353501533994568.\n",
      "Better model found at epoch 41 with macro_f1 value: 0.9549489897185501.\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "\n",
       "<style>\n",
       "    /* Turns off some styling */\n",
       "    progress {\n",
       "        /* gets rid of default border in Firefox and Opera. */\n",
       "        border: none;\n",
       "        /* Needs to be in here for Safari polyfill so background images work as expected. */\n",
       "        background-size: auto;\n",
       "    }\n",
       "    progress:not([value]), progress:not([value])::-webkit-progress-bar {\n",
       "        background: repeating-linear-gradient(45deg, #7e7e7e, #7e7e7e 10px, #5c5c5c 10px, #5c5c5c 20px);\n",
       "    }\n",
       "    .progress-bar-interrupted, .progress-bar-interrupted::-webkit-progress-bar {\n",
       "        background: #F44336;\n",
       "    }\n",
       "</style>\n"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: left;\">\n",
       "      <th>epoch</th>\n",
       "      <th>train_loss</th>\n",
       "      <th>valid_loss</th>\n",
       "      <th>accuracy</th>\n",
       "      <th>macro_f1</th>\n",
       "      <th>time</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <td>0</td>\n",
       "      <td>5.323751</td>\n",
       "      <td>4.580914</td>\n",
       "      <td>0.170418</td>\n",
       "      <td>0.135106</td>\n",
       "      <td>01:01</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>1</td>\n",
       "      <td>3.895346</td>\n",
       "      <td>3.211316</td>\n",
       "      <td>0.382637</td>\n",
       "      <td>0.240232</td>\n",
       "      <td>00:58</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>2</td>\n",
       "      <td>3.209976</td>\n",
       "      <td>2.915032</td>\n",
       "      <td>0.508039</td>\n",
       "      <td>0.363374</td>\n",
       "      <td>00:53</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>3</td>\n",
       "      <td>2.686677</td>\n",
       "      <td>2.155744</td>\n",
       "      <td>0.652733</td>\n",
       "      <td>0.498690</td>\n",
       "      <td>00:51</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>4</td>\n",
       "      <td>1.949023</td>\n",
       "      <td>1.326573</td>\n",
       "      <td>0.762058</td>\n",
       "      <td>0.616308</td>\n",
       "      <td>00:53</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>5</td>\n",
       "      <td>1.383703</td>\n",
       "      <td>1.073965</td>\n",
       "      <td>0.752412</td>\n",
       "      <td>0.645141</td>\n",
       "      <td>00:59</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>6</td>\n",
       "      <td>1.018063</td>\n",
       "      <td>0.703507</td>\n",
       "      <td>0.845659</td>\n",
       "      <td>0.700180</td>\n",
       "      <td>00:52</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>7</td>\n",
       "      <td>0.837501</td>\n",
       "      <td>0.545440</td>\n",
       "      <td>0.903537</td>\n",
       "      <td>0.732005</td>\n",
       "      <td>00:55</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>8</td>\n",
       "      <td>0.714138</td>\n",
       "      <td>0.590157</td>\n",
       "      <td>0.858521</td>\n",
       "      <td>0.730049</td>\n",
       "      <td>00:58</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>9</td>\n",
       "      <td>0.628869</td>\n",
       "      <td>0.771521</td>\n",
       "      <td>0.832797</td>\n",
       "      <td>0.706548</td>\n",
       "      <td>00:56</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>10</td>\n",
       "      <td>0.568458</td>\n",
       "      <td>0.481241</td>\n",
       "      <td>0.893891</td>\n",
       "      <td>0.760898</td>\n",
       "      <td>00:58</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>11</td>\n",
       "      <td>0.537159</td>\n",
       "      <td>0.541074</td>\n",
       "      <td>0.887460</td>\n",
       "      <td>0.738163</td>\n",
       "      <td>01:00</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>12</td>\n",
       "      <td>0.419597</td>\n",
       "      <td>0.373773</td>\n",
       "      <td>0.922830</td>\n",
       "      <td>0.785890</td>\n",
       "      <td>00:55</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>13</td>\n",
       "      <td>0.419309</td>\n",
       "      <td>1.045443</td>\n",
       "      <td>0.797428</td>\n",
       "      <td>0.679412</td>\n",
       "      <td>00:56</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>14</td>\n",
       "      <td>0.387896</td>\n",
       "      <td>0.231402</td>\n",
       "      <td>0.951769</td>\n",
       "      <td>0.797912</td>\n",
       "      <td>00:57</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>15</td>\n",
       "      <td>0.369151</td>\n",
       "      <td>0.311363</td>\n",
       "      <td>0.945338</td>\n",
       "      <td>0.764746</td>\n",
       "      <td>00:55</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>16</td>\n",
       "      <td>0.373773</td>\n",
       "      <td>0.531422</td>\n",
       "      <td>0.877814</td>\n",
       "      <td>0.764133</td>\n",
       "      <td>00:58</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>17</td>\n",
       "      <td>0.348910</td>\n",
       "      <td>0.419318</td>\n",
       "      <td>0.919614</td>\n",
       "      <td>0.765149</td>\n",
       "      <td>00:56</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>18</td>\n",
       "      <td>0.273223</td>\n",
       "      <td>0.162952</td>\n",
       "      <td>0.974277</td>\n",
       "      <td>0.863141</td>\n",
       "      <td>00:51</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>19</td>\n",
       "      <td>0.282850</td>\n",
       "      <td>0.262862</td>\n",
       "      <td>0.942122</td>\n",
       "      <td>0.827862</td>\n",
       "      <td>00:54</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>20</td>\n",
       "      <td>0.332146</td>\n",
       "      <td>0.336556</td>\n",
       "      <td>0.926045</td>\n",
       "      <td>0.828441</td>\n",
       "      <td>00:55</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>21</td>\n",
       "      <td>0.246233</td>\n",
       "      <td>0.543236</td>\n",
       "      <td>0.897106</td>\n",
       "      <td>0.790920</td>\n",
       "      <td>01:05</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>22</td>\n",
       "      <td>0.223954</td>\n",
       "      <td>0.300488</td>\n",
       "      <td>0.951769</td>\n",
       "      <td>0.850031</td>\n",
       "      <td>01:12</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>23</td>\n",
       "      <td>0.180552</td>\n",
       "      <td>0.315467</td>\n",
       "      <td>0.942122</td>\n",
       "      <td>0.813330</td>\n",
       "      <td>01:12</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>24</td>\n",
       "      <td>0.190190</td>\n",
       "      <td>0.194584</td>\n",
       "      <td>0.977492</td>\n",
       "      <td>0.888687</td>\n",
       "      <td>01:09</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>25</td>\n",
       "      <td>0.147438</td>\n",
       "      <td>0.264209</td>\n",
       "      <td>0.948553</td>\n",
       "      <td>0.859368</td>\n",
       "      <td>01:14</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>26</td>\n",
       "      <td>0.183662</td>\n",
       "      <td>0.143605</td>\n",
       "      <td>0.971061</td>\n",
       "      <td>0.864653</td>\n",
       "      <td>01:13</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>27</td>\n",
       "      <td>0.169326</td>\n",
       "      <td>0.538116</td>\n",
       "      <td>0.906752</td>\n",
       "      <td>0.799386</td>\n",
       "      <td>01:13</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>28</td>\n",
       "      <td>0.111032</td>\n",
       "      <td>0.174445</td>\n",
       "      <td>0.971061</td>\n",
       "      <td>0.879443</td>\n",
       "      <td>01:12</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>29</td>\n",
       "      <td>0.157204</td>\n",
       "      <td>0.247855</td>\n",
       "      <td>0.958199</td>\n",
       "      <td>0.816404</td>\n",
       "      <td>01:22</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>30</td>\n",
       "      <td>0.157111</td>\n",
       "      <td>0.181948</td>\n",
       "      <td>0.971061</td>\n",
       "      <td>0.870764</td>\n",
       "      <td>01:19</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>31</td>\n",
       "      <td>0.125301</td>\n",
       "      <td>0.182045</td>\n",
       "      <td>0.974277</td>\n",
       "      <td>0.883340</td>\n",
       "      <td>01:10</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>32</td>\n",
       "      <td>0.097038</td>\n",
       "      <td>0.178120</td>\n",
       "      <td>0.974277</td>\n",
       "      <td>0.864407</td>\n",
       "      <td>01:05</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>33</td>\n",
       "      <td>0.119069</td>\n",
       "      <td>0.110347</td>\n",
       "      <td>0.977492</td>\n",
       "      <td>0.920637</td>\n",
       "      <td>01:15</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>34</td>\n",
       "      <td>0.076569</td>\n",
       "      <td>0.131419</td>\n",
       "      <td>0.967846</td>\n",
       "      <td>0.873860</td>\n",
       "      <td>01:17</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>35</td>\n",
       "      <td>0.085716</td>\n",
       "      <td>0.134717</td>\n",
       "      <td>0.977492</td>\n",
       "      <td>0.875297</td>\n",
       "      <td>01:04</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>36</td>\n",
       "      <td>0.069521</td>\n",
       "      <td>0.088018</td>\n",
       "      <td>0.983923</td>\n",
       "      <td>0.884899</td>\n",
       "      <td>00:56</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>37</td>\n",
       "      <td>0.079330</td>\n",
       "      <td>0.175142</td>\n",
       "      <td>0.974277</td>\n",
       "      <td>0.888407</td>\n",
       "      <td>00:55</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>38</td>\n",
       "      <td>0.046638</td>\n",
       "      <td>0.082921</td>\n",
       "      <td>0.983923</td>\n",
       "      <td>0.910821</td>\n",
       "      <td>01:00</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>39</td>\n",
       "      <td>0.042054</td>\n",
       "      <td>0.135327</td>\n",
       "      <td>0.987138</td>\n",
       "      <td>0.920040</td>\n",
       "      <td>00:57</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>40</td>\n",
       "      <td>0.051811</td>\n",
       "      <td>0.136885</td>\n",
       "      <td>0.977492</td>\n",
       "      <td>0.872908</td>\n",
       "      <td>01:00</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>41</td>\n",
       "      <td>0.067931</td>\n",
       "      <td>0.090889</td>\n",
       "      <td>0.987138</td>\n",
       "      <td>0.911452</td>\n",
       "      <td>01:07</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>42</td>\n",
       "      <td>0.050709</td>\n",
       "      <td>0.104025</td>\n",
       "      <td>0.987138</td>\n",
       "      <td>0.901632</td>\n",
       "      <td>00:54</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>43</td>\n",
       "      <td>0.036036</td>\n",
       "      <td>0.141551</td>\n",
       "      <td>0.974277</td>\n",
       "      <td>0.883474</td>\n",
       "      <td>00:52</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>44</td>\n",
       "      <td>0.043227</td>\n",
       "      <td>0.094333</td>\n",
       "      <td>0.983923</td>\n",
       "      <td>0.893668</td>\n",
       "      <td>00:52</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>45</td>\n",
       "      <td>0.023723</td>\n",
       "      <td>0.090107</td>\n",
       "      <td>0.987138</td>\n",
       "      <td>0.920040</td>\n",
       "      <td>00:55</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>46</td>\n",
       "      <td>0.023726</td>\n",
       "      <td>0.091209</td>\n",
       "      <td>0.983923</td>\n",
       "      <td>0.893668</td>\n",
       "      <td>00:58</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>47</td>\n",
       "      <td>0.035418</td>\n",
       "      <td>0.117612</td>\n",
       "      <td>0.983923</td>\n",
       "      <td>0.901332</td>\n",
       "      <td>01:14</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>48</td>\n",
       "      <td>0.031644</td>\n",
       "      <td>0.098169</td>\n",
       "      <td>0.990354</td>\n",
       "      <td>0.938006</td>\n",
       "      <td>01:14</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>49</td>\n",
       "      <td>0.032740</td>\n",
       "      <td>0.113937</td>\n",
       "      <td>0.990354</td>\n",
       "      <td>0.938006</td>\n",
       "      <td>01:00</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>50</td>\n",
       "      <td>0.035935</td>\n",
       "      <td>0.096333</td>\n",
       "      <td>0.987138</td>\n",
       "      <td>0.911452</td>\n",
       "      <td>00:51</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>51</td>\n",
       "      <td>0.033551</td>\n",
       "      <td>0.101474</td>\n",
       "      <td>0.987138</td>\n",
       "      <td>0.911452</td>\n",
       "      <td>00:50</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>52</td>\n",
       "      <td>0.019459</td>\n",
       "      <td>0.105395</td>\n",
       "      <td>0.983923</td>\n",
       "      <td>0.893668</td>\n",
       "      <td>00:53</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>53</td>\n",
       "      <td>0.023157</td>\n",
       "      <td>0.103326</td>\n",
       "      <td>0.987138</td>\n",
       "      <td>0.911452</td>\n",
       "      <td>00:48</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>54</td>\n",
       "      <td>0.013641</td>\n",
       "      <td>0.103166</td>\n",
       "      <td>0.987138</td>\n",
       "      <td>0.911452</td>\n",
       "      <td>00:49</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>55</td>\n",
       "      <td>0.014431</td>\n",
       "      <td>0.109146</td>\n",
       "      <td>0.983923</td>\n",
       "      <td>0.893668</td>\n",
       "      <td>00:54</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>56</td>\n",
       "      <td>0.021602</td>\n",
       "      <td>0.104874</td>\n",
       "      <td>0.987138</td>\n",
       "      <td>0.911452</td>\n",
       "      <td>00:49</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>57</td>\n",
       "      <td>0.033418</td>\n",
       "      <td>0.113461</td>\n",
       "      <td>0.983923</td>\n",
       "      <td>0.893668</td>\n",
       "      <td>00:49</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>58</td>\n",
       "      <td>0.020882</td>\n",
       "      <td>0.113799</td>\n",
       "      <td>0.983923</td>\n",
       "      <td>0.893668</td>\n",
       "      <td>00:53</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>59</td>\n",
       "      <td>0.012295</td>\n",
       "      <td>0.113880</td>\n",
       "      <td>0.980707</td>\n",
       "      <td>0.875703</td>\n",
       "      <td>00:47</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Better model found at epoch 0 with macro_f1 value: 0.13510587993580037.\n",
      "Better model found at epoch 1 with macro_f1 value: 0.24023239056506232.\n",
      "Better model found at epoch 2 with macro_f1 value: 0.3633735238902109.\n",
      "Better model found at epoch 3 with macro_f1 value: 0.49868963194783894.\n",
      "Better model found at epoch 4 with macro_f1 value: 0.6163084344865083.\n",
      "Better model found at epoch 5 with macro_f1 value: 0.6451407417233027.\n",
      "Better model found at epoch 6 with macro_f1 value: 0.7001797744912264.\n",
      "Better model found at epoch 7 with macro_f1 value: 0.73200507148967.\n",
      "Better model found at epoch 10 with macro_f1 value: 0.7608982359334044.\n",
      "Better model found at epoch 12 with macro_f1 value: 0.7858896648808382.\n",
      "Better model found at epoch 14 with macro_f1 value: 0.7979118949683601.\n",
      "Better model found at epoch 18 with macro_f1 value: 0.8631406654424293.\n",
      "Better model found at epoch 24 with macro_f1 value: 0.8886867466198795.\n",
      "Better model found at epoch 33 with macro_f1 value: 0.9206367825722666.\n",
      "Better model found at epoch 48 with macro_f1 value: 0.9380055088798994.\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "\n",
       "<style>\n",
       "    /* Turns off some styling */\n",
       "    progress {\n",
       "        /* gets rid of default border in Firefox and Opera. */\n",
       "        border: none;\n",
       "        /* Needs to be in here for Safari polyfill so background images work as expected. */\n",
       "        background-size: auto;\n",
       "    }\n",
       "    progress:not([value]), progress:not([value])::-webkit-progress-bar {\n",
       "        background: repeating-linear-gradient(45deg, #7e7e7e, #7e7e7e 10px, #5c5c5c 10px, #5c5c5c 20px);\n",
       "    }\n",
       "    .progress-bar-interrupted, .progress-bar-interrupted::-webkit-progress-bar {\n",
       "        background: #F44336;\n",
       "    }\n",
       "</style>\n"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: left;\">\n",
       "      <th>epoch</th>\n",
       "      <th>train_loss</th>\n",
       "      <th>valid_loss</th>\n",
       "      <th>accuracy</th>\n",
       "      <th>macro_f1</th>\n",
       "      <th>time</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <td>0</td>\n",
       "      <td>5.409146</td>\n",
       "      <td>4.655257</td>\n",
       "      <td>0.176849</td>\n",
       "      <td>0.078694</td>\n",
       "      <td>00:50</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>1</td>\n",
       "      <td>3.928831</td>\n",
       "      <td>3.218301</td>\n",
       "      <td>0.356913</td>\n",
       "      <td>0.194193</td>\n",
       "      <td>00:51</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>2</td>\n",
       "      <td>3.192704</td>\n",
       "      <td>2.838488</td>\n",
       "      <td>0.530547</td>\n",
       "      <td>0.309762</td>\n",
       "      <td>00:51</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>3</td>\n",
       "      <td>2.634944</td>\n",
       "      <td>2.085931</td>\n",
       "      <td>0.681672</td>\n",
       "      <td>0.461723</td>\n",
       "      <td>00:52</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>4</td>\n",
       "      <td>1.975378</td>\n",
       "      <td>1.387347</td>\n",
       "      <td>0.797428</td>\n",
       "      <td>0.561576</td>\n",
       "      <td>00:49</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>5</td>\n",
       "      <td>1.404483</td>\n",
       "      <td>0.925521</td>\n",
       "      <td>0.836013</td>\n",
       "      <td>0.661933</td>\n",
       "      <td>00:50</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>6</td>\n",
       "      <td>1.001878</td>\n",
       "      <td>0.699980</td>\n",
       "      <td>0.874598</td>\n",
       "      <td>0.709673</td>\n",
       "      <td>00:51</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>7</td>\n",
       "      <td>0.804687</td>\n",
       "      <td>0.615493</td>\n",
       "      <td>0.887460</td>\n",
       "      <td>0.695400</td>\n",
       "      <td>00:49</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>8</td>\n",
       "      <td>0.648958</td>\n",
       "      <td>0.415318</td>\n",
       "      <td>0.929260</td>\n",
       "      <td>0.753629</td>\n",
       "      <td>00:51</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>9</td>\n",
       "      <td>0.626619</td>\n",
       "      <td>0.941507</td>\n",
       "      <td>0.816720</td>\n",
       "      <td>0.662767</td>\n",
       "      <td>00:52</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>10</td>\n",
       "      <td>0.559287</td>\n",
       "      <td>0.374804</td>\n",
       "      <td>0.929260</td>\n",
       "      <td>0.772140</td>\n",
       "      <td>00:49</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>11</td>\n",
       "      <td>0.488998</td>\n",
       "      <td>0.845066</td>\n",
       "      <td>0.836013</td>\n",
       "      <td>0.694088</td>\n",
       "      <td>00:51</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>12</td>\n",
       "      <td>0.495279</td>\n",
       "      <td>0.505008</td>\n",
       "      <td>0.893891</td>\n",
       "      <td>0.767369</td>\n",
       "      <td>00:52</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>13</td>\n",
       "      <td>0.426862</td>\n",
       "      <td>0.487224</td>\n",
       "      <td>0.916399</td>\n",
       "      <td>0.713943</td>\n",
       "      <td>00:49</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>14</td>\n",
       "      <td>0.456391</td>\n",
       "      <td>0.462856</td>\n",
       "      <td>0.900322</td>\n",
       "      <td>0.752213</td>\n",
       "      <td>00:50</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>15</td>\n",
       "      <td>0.376730</td>\n",
       "      <td>0.319784</td>\n",
       "      <td>0.948553</td>\n",
       "      <td>0.855251</td>\n",
       "      <td>00:51</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>16</td>\n",
       "      <td>0.379817</td>\n",
       "      <td>0.491080</td>\n",
       "      <td>0.903537</td>\n",
       "      <td>0.803757</td>\n",
       "      <td>00:49</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>17</td>\n",
       "      <td>0.339677</td>\n",
       "      <td>0.431305</td>\n",
       "      <td>0.913183</td>\n",
       "      <td>0.824646</td>\n",
       "      <td>00:54</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>18</td>\n",
       "      <td>0.324609</td>\n",
       "      <td>0.375893</td>\n",
       "      <td>0.932476</td>\n",
       "      <td>0.804010</td>\n",
       "      <td>00:51</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>19</td>\n",
       "      <td>0.236851</td>\n",
       "      <td>0.559932</td>\n",
       "      <td>0.906752</td>\n",
       "      <td>0.787502</td>\n",
       "      <td>00:50</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>20</td>\n",
       "      <td>0.237683</td>\n",
       "      <td>0.570599</td>\n",
       "      <td>0.893891</td>\n",
       "      <td>0.747161</td>\n",
       "      <td>00:51</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>21</td>\n",
       "      <td>0.239782</td>\n",
       "      <td>0.245879</td>\n",
       "      <td>0.954984</td>\n",
       "      <td>0.834958</td>\n",
       "      <td>00:52</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>22</td>\n",
       "      <td>0.214558</td>\n",
       "      <td>0.163724</td>\n",
       "      <td>0.971061</td>\n",
       "      <td>0.870561</td>\n",
       "      <td>00:51</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>23</td>\n",
       "      <td>0.235050</td>\n",
       "      <td>0.200801</td>\n",
       "      <td>0.971061</td>\n",
       "      <td>0.870510</td>\n",
       "      <td>00:51</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>24</td>\n",
       "      <td>0.197705</td>\n",
       "      <td>0.151393</td>\n",
       "      <td>0.977492</td>\n",
       "      <td>0.905926</td>\n",
       "      <td>00:52</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>25</td>\n",
       "      <td>0.251579</td>\n",
       "      <td>0.447560</td>\n",
       "      <td>0.922830</td>\n",
       "      <td>0.752207</td>\n",
       "      <td>00:49</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>26</td>\n",
       "      <td>0.154326</td>\n",
       "      <td>0.247158</td>\n",
       "      <td>0.951769</td>\n",
       "      <td>0.892799</td>\n",
       "      <td>00:50</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>27</td>\n",
       "      <td>0.137643</td>\n",
       "      <td>0.275604</td>\n",
       "      <td>0.951769</td>\n",
       "      <td>0.857527</td>\n",
       "      <td>00:53</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>28</td>\n",
       "      <td>0.156946</td>\n",
       "      <td>0.214340</td>\n",
       "      <td>0.974277</td>\n",
       "      <td>0.891360</td>\n",
       "      <td>00:49</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>29</td>\n",
       "      <td>0.162380</td>\n",
       "      <td>0.178380</td>\n",
       "      <td>0.980707</td>\n",
       "      <td>0.908810</td>\n",
       "      <td>00:50</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>30</td>\n",
       "      <td>0.190721</td>\n",
       "      <td>0.593797</td>\n",
       "      <td>0.916399</td>\n",
       "      <td>0.815265</td>\n",
       "      <td>00:52</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>31</td>\n",
       "      <td>0.122781</td>\n",
       "      <td>0.137817</td>\n",
       "      <td>0.977492</td>\n",
       "      <td>0.915373</td>\n",
       "      <td>00:49</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>32</td>\n",
       "      <td>0.108274</td>\n",
       "      <td>0.154006</td>\n",
       "      <td>0.974277</td>\n",
       "      <td>0.874173</td>\n",
       "      <td>00:54</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>33</td>\n",
       "      <td>0.084279</td>\n",
       "      <td>0.151013</td>\n",
       "      <td>0.974277</td>\n",
       "      <td>0.903200</td>\n",
       "      <td>00:54</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>34</td>\n",
       "      <td>0.060302</td>\n",
       "      <td>0.160824</td>\n",
       "      <td>0.980707</td>\n",
       "      <td>0.883705</td>\n",
       "      <td>00:50</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>35</td>\n",
       "      <td>0.097096</td>\n",
       "      <td>0.156161</td>\n",
       "      <td>0.971061</td>\n",
       "      <td>0.829796</td>\n",
       "      <td>00:50</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>36</td>\n",
       "      <td>0.101503</td>\n",
       "      <td>0.130838</td>\n",
       "      <td>0.977492</td>\n",
       "      <td>0.855420</td>\n",
       "      <td>00:52</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>37</td>\n",
       "      <td>0.099718</td>\n",
       "      <td>0.219568</td>\n",
       "      <td>0.971061</td>\n",
       "      <td>0.893705</td>\n",
       "      <td>00:50</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>38</td>\n",
       "      <td>0.067880</td>\n",
       "      <td>0.177204</td>\n",
       "      <td>0.964630</td>\n",
       "      <td>0.860729</td>\n",
       "      <td>00:51</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>39</td>\n",
       "      <td>0.060511</td>\n",
       "      <td>0.075175</td>\n",
       "      <td>0.983923</td>\n",
       "      <td>0.919409</td>\n",
       "      <td>00:51</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>40</td>\n",
       "      <td>0.055977</td>\n",
       "      <td>0.075184</td>\n",
       "      <td>0.980707</td>\n",
       "      <td>0.893081</td>\n",
       "      <td>00:49</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>41</td>\n",
       "      <td>0.054042</td>\n",
       "      <td>0.147451</td>\n",
       "      <td>0.967846</td>\n",
       "      <td>0.851703</td>\n",
       "      <td>00:52</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>42</td>\n",
       "      <td>0.058785</td>\n",
       "      <td>0.109593</td>\n",
       "      <td>0.980707</td>\n",
       "      <td>0.918733</td>\n",
       "      <td>00:51</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>43</td>\n",
       "      <td>0.052702</td>\n",
       "      <td>0.141260</td>\n",
       "      <td>0.977492</td>\n",
       "      <td>0.892180</td>\n",
       "      <td>00:50</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>44</td>\n",
       "      <td>0.045131</td>\n",
       "      <td>0.145799</td>\n",
       "      <td>0.971061</td>\n",
       "      <td>0.897351</td>\n",
       "      <td>00:52</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>45</td>\n",
       "      <td>0.041105</td>\n",
       "      <td>0.102513</td>\n",
       "      <td>0.980707</td>\n",
       "      <td>0.918733</td>\n",
       "      <td>00:52</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>46</td>\n",
       "      <td>0.027659</td>\n",
       "      <td>0.141561</td>\n",
       "      <td>0.977492</td>\n",
       "      <td>0.874661</td>\n",
       "      <td>00:50</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>47</td>\n",
       "      <td>0.018782</td>\n",
       "      <td>0.120653</td>\n",
       "      <td>0.977492</td>\n",
       "      <td>0.918030</td>\n",
       "      <td>00:51</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>48</td>\n",
       "      <td>0.014323</td>\n",
       "      <td>0.110692</td>\n",
       "      <td>0.987138</td>\n",
       "      <td>0.937375</td>\n",
       "      <td>00:52</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>49</td>\n",
       "      <td>0.019867</td>\n",
       "      <td>0.112844</td>\n",
       "      <td>0.983923</td>\n",
       "      <td>0.919155</td>\n",
       "      <td>00:49</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>50</td>\n",
       "      <td>0.022227</td>\n",
       "      <td>0.112303</td>\n",
       "      <td>0.980707</td>\n",
       "      <td>0.901371</td>\n",
       "      <td>00:55</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>51</td>\n",
       "      <td>0.028763</td>\n",
       "      <td>0.117743</td>\n",
       "      <td>0.980707</td>\n",
       "      <td>0.901371</td>\n",
       "      <td>00:52</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>52</td>\n",
       "      <td>0.017681</td>\n",
       "      <td>0.115190</td>\n",
       "      <td>0.980707</td>\n",
       "      <td>0.901371</td>\n",
       "      <td>00:49</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>53</td>\n",
       "      <td>0.016773</td>\n",
       "      <td>0.119803</td>\n",
       "      <td>0.980707</td>\n",
       "      <td>0.901371</td>\n",
       "      <td>00:52</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>54</td>\n",
       "      <td>0.014990</td>\n",
       "      <td>0.108874</td>\n",
       "      <td>0.987138</td>\n",
       "      <td>0.937120</td>\n",
       "      <td>00:50</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>55</td>\n",
       "      <td>0.021137</td>\n",
       "      <td>0.110850</td>\n",
       "      <td>0.987138</td>\n",
       "      <td>0.937120</td>\n",
       "      <td>00:49</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>56</td>\n",
       "      <td>0.014044</td>\n",
       "      <td>0.111376</td>\n",
       "      <td>0.983923</td>\n",
       "      <td>0.919336</td>\n",
       "      <td>00:52</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>57</td>\n",
       "      <td>0.007958</td>\n",
       "      <td>0.105863</td>\n",
       "      <td>0.987138</td>\n",
       "      <td>0.937120</td>\n",
       "      <td>00:51</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>58</td>\n",
       "      <td>0.026899</td>\n",
       "      <td>0.114197</td>\n",
       "      <td>0.980707</td>\n",
       "      <td>0.901371</td>\n",
       "      <td>00:49</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>59</td>\n",
       "      <td>0.011902</td>\n",
       "      <td>0.106987</td>\n",
       "      <td>0.983923</td>\n",
       "      <td>0.919336</td>\n",
       "      <td>00:51</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Better model found at epoch 0 with macro_f1 value: 0.07869417847822817.\n",
      "Better model found at epoch 1 with macro_f1 value: 0.19419280109532508.\n",
      "Better model found at epoch 2 with macro_f1 value: 0.3097617502054242.\n",
      "Better model found at epoch 3 with macro_f1 value: 0.4617225478977442.\n",
      "Better model found at epoch 4 with macro_f1 value: 0.5615759963470389.\n",
      "Better model found at epoch 5 with macro_f1 value: 0.66193334914984.\n",
      "Better model found at epoch 6 with macro_f1 value: 0.7096727721558488.\n",
      "Better model found at epoch 8 with macro_f1 value: 0.7536293091319175.\n",
      "Better model found at epoch 10 with macro_f1 value: 0.7721399038635703.\n",
      "Better model found at epoch 15 with macro_f1 value: 0.8552505764395204.\n",
      "Better model found at epoch 22 with macro_f1 value: 0.8705607985565478.\n",
      "Better model found at epoch 24 with macro_f1 value: 0.9059264637524154.\n",
      "Better model found at epoch 29 with macro_f1 value: 0.9088099528484045.\n",
      "Better model found at epoch 31 with macro_f1 value: 0.915372908095095.\n",
      "Better model found at epoch 39 with macro_f1 value: 0.9194091417748286.\n",
      "Better model found at epoch 48 with macro_f1 value: 0.9373747417339977.\n"
     ]
    }
   ],
   "source": [
    "for fold in range(1, 6):\n",
    "    model_dir = f'model_weights/fold_{fold}'\n",
    "    data_csv_path = f'data/splits/data_fold_{fold}.csv'\n",
    "\n",
    "    df = pd.read_csv(data_csv_path)\n",
    "    train_dl, val_dl = get_data_loaders(df)\n",
    "    dls = DataLoaders(train_dl, val_dl)\n",
    "    net = Mos_Xception(const.NUM_CLASSES)\n",
    "\n",
    "    learn = Learner(\n",
    "        dls,\n",
    "        net,\n",
    "        wd=wd,\n",
    "        opt_func=opt_func,\n",
    "        metrics=metrics,\n",
    "        loss_func=loss,\n",
    "        model_dir=model_dir,\n",
    "    )\n",
    "    cb = SaveModelCallback(monitor='macro_f1')\n",
    "    learn.fit_one_cycle(\n",
    "        60,\n",
    "        2e-03,\n",
    "        div=25,\n",
    "        pct_start=0.3,\n",
    "        cbs=[cb],\n",
    "    )\n",
    "\n",
    "    del learn\n",
    "    del net\n",
    "    gc.collect()\n",
    "    torch.cuda.empty_cache()"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "pace-up",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.11.9"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}