fb3-submit-single.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "20c60102",
   "metadata": {
    "_cell_guid": "b1076dfc-b9ad-4769-8c92-a6c4dae69d19",
    "_uuid": "8f2839f25d086af736a60e9eeb907d3b93b6e0e5",
    "execution": {
     "iopub.execute_input": "2022-11-29T01:12:51.893143Z",
     "iopub.status.busy": "2022-11-29T01:12:51.892600Z",
     "iopub.status.idle": "2022-11-29T01:13:24.705359Z",
     "shell.execute_reply": "2022-11-29T01:13:24.704162Z"
    },
    "papermill": {
     "duration": 32.826776,
     "end_time": "2022-11-29T01:13:24.708394",
     "exception": false,
     "start_time": "2022-11-29T01:12:51.881618",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "tokenizers.__version__: 0.12.1\n",
      "transformers.__version__: 4.20.1\n",
      "device: cuda\n"
     ]
    }
   ],
   "source": [
    "# ====================================================\n",
    "# Library\n",
    "# ====================================================\n",
    "import os\n",
    "import gc\n",
    "import re\n",
    "import ast\n",
    "import sys\n",
    "import copy\n",
    "import json\n",
    "import time\n",
    "import math \n",
    "import string\n",
    "import pickle\n",
    "import random\n",
    "import joblib\n",
    "import itertools\n",
    "import warnings\n",
    "warnings.filterwarnings(\"ignore\")\n",
    "from IPython. display import clear_output\n",
    "\n",
    "import scipy as sp\n",
    "import numpy as np\n",
    "import pandas as pd\n",
    "pd.set_option('display.max_rows', 500)\n",
    "pd.set_option('display.max_columns', 500)\n",
    "pd.set_option('display.width', 1000)\n",
    "from tqdm.auto import tqdm\n",
    "from sklearn.metrics import mean_squared_error\n",
    "from sklearn.model_selection import StratifiedKFold, GroupKFold, KFold\n",
    "from nltk.corpus import stopwords\n",
    "from nltk.tokenize import word_tokenize\n",
    "\n",
    "import torch\n",
    "import torch.nn as nn\n",
    "from torch.nn import Parameter\n",
    "import torch.nn.functional as F\n",
    "from torch.optim import Adam, SGD, AdamW\n",
    "from torch.utils.data import DataLoader, Dataset\n",
    "\n",
    "os.system('pip uninstall -y transformers')\n",
    "os.system('pip uninstall -y tokenizers')\n",
    "os.system('python -m pip install --no-index --find-links=../input/fb3-my-pip-wheels transformers')\n",
    "os.system('python -m pip install --no-index --find-links=../input/fb3-my-pip-wheels tokenizers')\n",
    "import tokenizers\n",
    "import transformers\n",
    "from transformers import AutoTokenizer, AutoModel, AutoConfig\n",
    "from transformers import get_linear_schedule_with_warmup, get_cosine_schedule_with_warmup\n",
    "from transformers import DataCollatorWithPadding\n",
    "%env TOKENIZERS_PARALLELISM=false\n",
    "\n",
    "clear_output()\n",
    "device = torch.device('cuda' if torch.cuda.is_available() else 'cpu')\n",
    "print(f\"tokenizers.__version__: {tokenizers.__version__}\")\n",
    "print(f\"transformers.__version__: {transformers.__version__}\")\n",
    "print('device:', device)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "964db140",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2022-11-29T01:13:24.720848Z",
     "iopub.status.busy": "2022-11-29T01:13:24.720289Z",
     "iopub.status.idle": "2022-11-29T01:13:24.726009Z",
     "shell.execute_reply": "2022-11-29T01:13:24.725095Z"
    },
    "papermill": {
     "duration": 0.013844,
     "end_time": "2022-11-29T01:13:24.727951",
     "exception": false,
     "start_time": "2022-11-29T01:13:24.714107",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "BASE_PATH = '/kaggle/input/feedback-prize-english-language-learning'\n",
    "SUBMISSION_PATH = os.path.join(BASE_PATH, 'sample_submission.csv')\n",
    "TRAIN_PATH = os.path.join(BASE_PATH, 'train.csv')\n",
    "TEST_PATH = os.path.join(BASE_PATH, 'test.csv')\n",
    "\n",
    "TARGET_COLS = ['cohesion', 'syntax', 'vocabulary', 'phraseology', 'grammar', 'conventions']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "5073efd4",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2022-11-29T01:13:24.739985Z",
     "iopub.status.busy": "2022-11-29T01:13:24.739722Z",
     "iopub.status.idle": "2022-11-29T01:13:24.824544Z",
     "shell.execute_reply": "2022-11-29T01:13:24.823677Z"
    },
    "papermill": {
     "duration": 0.093561,
     "end_time": "2022-11-29T01:13:24.826672",
     "exception": false,
     "start_time": "2022-11-29T01:13:24.733111",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "class MeanPooling(nn.Module):\n",
    "    def __init__(self):\n",
    "        super(MeanPooling, self).__init__()\n",
    "    \n",
    "    def forward(self, last_hidden_state, attention_mask):\n",
    "        input_mask_expanded = attention_mask.unsqueeze(-1).expand(last_hidden_state.size()).float()\n",
    "        sum_embeddings = torch.sum(last_hidden_state * input_mask_expanded, 1)\n",
    "        sum_mask = torch.clamp(input_mask_expanded.sum(1), min=1e-9)\n",
    "        mean_embeddings = sum_embeddings / sum_mask\n",
    "        return mean_embeddings\n",
    "\n",
    "class AttentionHead(nn.Module):\n",
    "    def __init__(self, in_features, hidden_dim):\n",
    "        super().__init__()\n",
    "        self.in_features = in_features\n",
    "        self.middle_features = hidden_dim\n",
    "        self.W = nn.Linear(in_features, hidden_dim)\n",
    "        self.V = nn.Linear(hidden_dim, 1)\n",
    "        self.out_features = hidden_dim\n",
    "\n",
    "    def forward(self, features, *args):\n",
    "        att = torch.tanh(self.W(features))\n",
    "        score = self.V(att)\n",
    "        attention_weights = torch.softmax(score, dim=1)\n",
    "        context_vector = attention_weights * features\n",
    "        context_vector = torch.sum(context_vector, dim=1)\n",
    "\n",
    "        return context_vector\n",
    "    \n",
    "class MaxPooling(nn.Module):\n",
    "    def __init__(self):\n",
    "        super(MaxPooling, self).__init__()\n",
    "    \n",
    "    def forward(self, last_hidden_state, attention_mask):\n",
    "        input_mask_expanded = attention_mask.unsqueeze(-1).expand(last_hidden_state.size()).float()\n",
    "        last_hidden_state_masked = last_hidden_state\n",
    "        last_hidden_state_masked[input_mask_expanded == 0] = -1e-9 \n",
    "        max_embeddings = torch.max(last_hidden_state_masked, 1)[0]\n",
    "        return max_embeddings\n",
    "    \n",
    "class LSTMPooling(nn.Module):\n",
    "    def __init__(self, num_layers, hidden_size, hiddendim_lstm):\n",
    "        super(LSTMPooling, self).__init__()\n",
    "        self.num_hidden_layers = num_layers\n",
    "        self.hidden_size = hidden_size\n",
    "        self.hiddendim_lstm = hiddendim_lstm\n",
    "        self.lstm = nn.LSTM(self.hidden_size, self.hiddendim_lstm, batch_first=True)\n",
    "        self.dropout = nn.Dropout(0.1)\n",
    "    \n",
    "    def forward(self, all_hidden_states):\n",
    "        # only use first and last\n",
    "        hidden_states = torch.stack([\n",
    "            all_hidden_states[layer_i][:, 0].squeeze()\n",
    "            for layer_i in (-1, 0)],\n",
    "            dim=-1)\n",
    "        hidden_states = hidden_states.view(-1, self.num_hidden_layers, self.hidden_size)\n",
    "        out, _ = self.lstm(hidden_states, None)\n",
    "        #out = self.dropout(out[:, -1, :])\n",
    "        out = self.dropout(out.mean(dim=1))\n",
    "        return out\n",
    "\n",
    "class WeightedLayerPooling(nn.Module):\n",
    "    def __init__(self, num_hidden_layers, layer_start: int = 9, layer_weights = None):\n",
    "        super(WeightedLayerPooling, self).__init__()\n",
    "        self.layer_start = layer_start\n",
    "        self.num_hidden_layers = num_hidden_layers\n",
    "        self.layer_weights = layer_weights if layer_weights is not None \\\n",
    "            else nn.Parameter(\n",
    "                torch.tensor([1] * (num_hidden_layers+1 - layer_start), dtype=torch.float)\n",
    "            )\n",
    "\n",
    "    def forward(self, all_hidden_states):\n",
    "        all_layer_embedding = all_hidden_states[self.layer_start:, :, :, :]\n",
    "        weight_factor = self.layer_weights.unsqueeze(-1).unsqueeze(-1).unsqueeze(-1).expand(all_layer_embedding.size())\n",
    "        weighted_average = (weight_factor*all_layer_embedding).sum(dim=0) / self.layer_weights.sum()\n",
    "        return weighted_average\n",
    "\n",
    "class AttentionPooling(nn.Module):\n",
    "    def __init__(self, in_dim):\n",
    "        super().__init__()\n",
    "        self.attention = nn.Sequential(\n",
    "        nn.Linear(in_dim, in_dim),\n",
    "        nn.LayerNorm(in_dim),\n",
    "        nn.GELU(),\n",
    "        nn.Linear(in_dim, 1),\n",
    "        )\n",
    "\n",
    "    def forward(self, last_hidden_state, attention_mask):\n",
    "        w = self.attention(last_hidden_state).float()\n",
    "        w[attention_mask==0]=float('-inf')\n",
    "        w = torch.softmax(w,1)\n",
    "        attention_embeddings = torch.sum(w * last_hidden_state, dim=1)\n",
    "        return attention_embeddings\n",
    "\n",
    "class MultiSampleDropout(nn.Module):\n",
    "    def __init__(self, fc, num_dropout, prob_dropout):\n",
    "        super(MultiSampleDropout, self).__init__()\n",
    "        self.dropout = nn.Dropout\n",
    "        self.num_dropout = num_dropout\n",
    "        self.prob_dropout = prob_dropout\n",
    "        self.classifier = fc\n",
    "    def forward(self, out):\n",
    "        if not type(self.prob_dropout) in [float, int]:            \n",
    "            fcs = [self.classifier(self.dropout(p)(out)) for p in self.prob_dropout]\n",
    "        else:\n",
    "            fcs = [self.classifier(self.dropout(self.prob_dropout)(out)) for _ in range(self.num_dropout)]\n",
    "        \n",
    "        return torch.mean(torch.stack(fcs, dim=0), dim=0)\n",
    "\n",
    "# ====================================================\n",
    "# Model classes\n",
    "# ====================================================\n",
    "class FB3Model(nn.Module):\n",
    "    def __init__(self, cfg, config_path=None, pretrained=False):\n",
    "        super().__init__()\n",
    "        self.cfg = cfg \n",
    "        if config_path is None:\n",
    "            self.config = AutoConfig.from_pretrained(cfg.model, output_hidden_states=True)\n",
    "            # Turn off dropouts.\n",
    "            self.config.hidden_dropout = 0.\n",
    "            self.config.hidden_dropout_prob = 0.\n",
    "            self.config.attention_dropout = 0.\n",
    "            self.config.attention_probs_dropout_prob = 0.\n",
    "            #LOGGER.info(self.config)\n",
    "        else:\n",
    "            self.config = torch.load(config_path)\n",
    "        \n",
    "        if pretrained:\n",
    "            self.deberta_v3 = AutoModel.from_pretrained(cfg.model, config=self.config)\n",
    "        else:\n",
    "            self.deberta_v3 = AutoModel.from_config(self.config)\n",
    "\n",
    "        #if self.cfg.reinit_last_layer:\n",
    "        #    # Re-init last layer of deberta.\n",
    "        #    for module in self.deberta_v3.encoder.layer[-1:].modules():\n",
    "        #        self._init_weights(module)\n",
    "        # self.deberta_v3.gradient_checkpointing_enable()\n",
    "\n",
    "    def _init_weights(self, module):\n",
    "        if isinstance(module, nn.Linear):\n",
    "            module.weight.data.normal_(mean=0.0, std=self.config.initializer_range)\n",
    "            #nn.init.xavier_uniform_(module.weight.data, gain=1.0)\n",
    "            if module.bias is not None:\n",
    "                module.bias.data.zero_()\n",
    "        elif isinstance(module, nn.Embedding):\n",
    "            module.weight.data.normal_(mean=0.0, std=self.config.initializer_range)\n",
    "            if module.padding_idx is not None:\n",
    "                module.weight.data[module.padding_idx].zero_()\n",
    "        elif isinstance(module, nn.LayerNorm):\n",
    "            module.bias.data.zero_()\n",
    "            module.weight.data.fill_(1.0)\n",
    "\n",
    "class WMPoolModel(FB3Model):\n",
    "    def __init__(self, cfg, config_path=None, pretrained=False):\n",
    "        super().__init__(cfg, config_path=config_path, pretrained=pretrained)\n",
    "\n",
    "        # Poolings.\n",
    "        self.mean_head = MeanPooling()\n",
    "        self.wpool_head = WeightedLayerPooling(self.config.num_hidden_layers, layer_start=12)\n",
    "\n",
    "        self.fc_out = nn.Linear(self.config.hidden_size, cfg.num_target)\n",
    "        self._init_weights(self.fc_out)\n",
    "        \n",
    "        self.layer_norm = nn.LayerNorm(self.config.hidden_size)\n",
    "        self.qa_output = torch.nn.Linear(self.config.hidden_size, 2)\n",
    "        self.attention_head = AttentionHead(self.config.hidden_size*4, self.config.hidden_size)\n",
    "        \n",
    "    def forward(self, x):\n",
    "        pt_out = self.deberta_v3(**x)\n",
    "        all_hidden_states = torch.stack(pt_out.hidden_states)\n",
    "        # Weighted pooling of last n layers.\n",
    "        logits = self.wpool_head(all_hidden_states)[:, 0] # Bx768\n",
    "        y_hat = self.fc_out(logits)\n",
    "        return y_hat\n",
    "\n",
    "class MultiPoolModel(FB3Model):\n",
    "    def __init__(self, cfg, config_path=None, pretrained=False, pool='mean'):\n",
    "        super().__init__(cfg, config_path=config_path, pretrained=pretrained)\n",
    "        \n",
    "        # Define model layers.\n",
    "        self.pool_name = cfg.pool_head\n",
    "        self.fc = nn.Linear(self.config.hidden_size, self.cfg.num_target)\n",
    "        if cfg.pool_head in ['mean', 'attention', 'weighted']:\n",
    "            self.pool = self._pool_layer(cfg.pool_head)\n",
    "        elif '-' in cfg.pool_head:\n",
    "            pools = cfg.pool_head.split('-')\n",
    "            self.pool = nn.ModuleList([])\n",
    "            for pool_ in pools:\n",
    "                self.pool.append(self._pool_layer(pool_))\n",
    "            self.fc = nn.Linear(self.config.hidden_size * len(self.pool), self.cfg.num_target)\n",
    "        self._init_weights(self.fc)\n",
    "        \n",
    "        # Multi-sample dropout.\n",
    "        self.multi_dropout = MultiSampleDropout(self.fc, self.cfg.num_dropout, self.cfg.prob_dropout)\n",
    "    \n",
    "    def _pool_layer(self, pool_name):\n",
    "        assert pool_name in ['mean', 'attention', 'weighted']\n",
    "        if pool_name == 'mean':\n",
    "            pool = MeanPooling()\n",
    "        elif pool_name == 'attention':\n",
    "            pool = AttentionHead(self.config.hidden_size, self.config.hidden_size)\n",
    "        elif pool_name == 'weighted':\n",
    "            pool = WeightedLayerPooling(\n",
    "                self.config.num_hidden_layers, \n",
    "                layer_start=9,\n",
    "                layer_weights=None)\n",
    "        return pool\n",
    "    \n",
    "    def _pool_feature(self, pool, pool_name, pt_outputs, attention_mask):\n",
    "        assert pool_name in ['mean', 'attention', 'weighted']\n",
    "        last_hidden_state = pt_outputs.last_hidden_state #batch_size x max_len x hidden_size\n",
    "        all_hidden_states = torch.stack(pt_outputs.hidden_states) #num_layer x batch_size x max_len x hidden_size\n",
    "        \n",
    "        if pool_name == 'mean':\n",
    "            pool_feature = pool(last_hidden_state, attention_mask)\n",
    "        elif pool_name == 'attention':\n",
    "            pool_feature = pool(last_hidden_state)\n",
    "        elif pool_name == 'weighted':\n",
    "            # Take the CLS token only.\n",
    "            pool_feature = pool(all_hidden_states)[:, 0]\n",
    "        return pool_feature\n",
    "\n",
    "    def feature(self, x):\n",
    "        pt_outputs = self.deberta_v3(**x)\n",
    "        \n",
    "        # Pooling feat.\n",
    "        if type(self.pool) == nn.ModuleList:\n",
    "            pool_features = []\n",
    "            pool_names = self.pool_name.split('-')\n",
    "            \n",
    "            for pool_name, pool in zip(pool_names, self.pool):\n",
    "                pool_features.append(self._pool_feature(pool, pool_name, pt_outputs, x['attention_mask']))\n",
    "            pool_features = torch.cat(pool_features, dim=1)\n",
    "        else:\n",
    "            pool_features = self._pool_feature(self.pool, self.pool_name, pt_outputs, x['attention_mask'])\n",
    "        return pool_features\n",
    "    \n",
    "    def forward(self, x):\n",
    "        feature = self.feature(x)\n",
    "        if self.cfg.use_dropout and self.training:\n",
    "            y_hat = self.multi_dropout(feature)\n",
    "        else:\n",
    "            y_hat = self.fc(feature)\n",
    "        return y_hat\n",
    "\n",
    "\n",
    "class Attention4Model(FB3Model):\n",
    "    def __init__(self, cfg, config_path=None, pretrained=False):\n",
    "        super().__init__(cfg, config_path=config_path, pretrained=pretrained)\n",
    "        \n",
    "        self.head = AttentionHead(self.config.hidden_size*4, self.config.hidden_size)\n",
    "        self.fc_out = nn.Linear(self.config.hidden_size*4*2, self.cfg.num_target)\n",
    "        self._init_weights(self.fc_out)\n",
    "\n",
    "    def forward(self, x):\n",
    "        pt_out = self.deberta_v3(**x)\n",
    "        \n",
    "        all_hidden_states = torch.stack(pt_out.hidden_states)\n",
    "        cat_over_last_layers = torch.cat(\n",
    "            (all_hidden_states[-1], all_hidden_states[-2], all_hidden_states[-3], all_hidden_states[-4]),\n",
    "            -1)\n",
    "        # [CLS] embedding.\n",
    "        cls_pooling = cat_over_last_layers[:, 0]   \n",
    "        # Concat of 4 last layers.\n",
    "        head_logits = self.head(cat_over_last_layers)\n",
    "\n",
    "        if self.cfg.use_dropout and self.training:\n",
    "            y_hat = self.multi_dropout(torch.cat([head_logits, cls_pooling], -1))\n",
    "        else:\n",
    "            y_hat = self.fc_out(torch.cat([head_logits, cls_pooling], -1))\n",
    "\n",
    "        return y_hat\n",
    "    \n",
    "class AttentionModel(FB3Model):\n",
    "    def __init__(self, cfg, config_path=None, pretrained=False):\n",
    "        super().__init__(cfg, config_path=config_path, pretrained=pretrained)\n",
    "\n",
    "        # Poolings.\n",
    "        self.att = nn.Sequential(\n",
    "            nn.Linear(self.config.hidden_size, self.cfg.ap_hidden_size),\n",
    "            nn.Tanh(),\n",
    "            nn.Linear(self.cfg.ap_hidden_size, 1),\n",
    "            nn.Softmax(dim=1),\n",
    "        )\n",
    "        self._init_weights(self.att)\n",
    "        \n",
    "        self.fc_out = nn.Linear(self.config.hidden_size, cfg.num_target)\n",
    "        self._init_weights(self.fc_out)\n",
    "\n",
    "    def forward(self, x):\n",
    "        pt_out = self.deberta_v3(**x)\n",
    "        last_hidden_states = pt_out.last_hidden_state\n",
    "        att_weights = self.att(last_hidden_states)\n",
    "        logits =  torch.sum(att_weights * last_hidden_states, dim=1)\n",
    "        \n",
    "        y_hat = self.fc_out(logits)\n",
    "        return y_hat\n",
    "    \n",
    "class MeanModel(FB3Model):\n",
    "    def __init__(self, cfg, config_path=None, pretrained=False):\n",
    "        super().__init__(cfg, config_path=config_path, pretrained=pretrained)\n",
    "\n",
    "        # Poolings.\n",
    "        self.mean_head = MeanPooling()\n",
    "\n",
    "        # Head.\n",
    "        self.fc_out = nn.Linear(self.config.hidden_size, cfg.num_target)\n",
    "        self._init_weights(self.fc_out)\n",
    "    \n",
    "    def forward(self, x):\n",
    "        pt_out = self.deberta_v3(**x)\n",
    "        # Mean pooling.\n",
    "        logits = self.mean_head(pt_out.last_hidden_state, x['attention_mask'])\n",
    "        y_hat = self.fc_out(logits)\n",
    "        return y_hat\n",
    "\n",
    "######################################\n",
    "\n",
    "class FB3Model(nn.Module):\n",
    "    def __init__(self, cfg, config_path=None, pretrained=False):\n",
    "        super().__init__()\n",
    "        self.cfg = cfg \n",
    "        if config_path is None:\n",
    "            self.config = AutoConfig.from_pretrained(cfg.model, output_hidden_states=True)\n",
    "            # Turn off dropouts.\n",
    "            self.config.hidden_dropout = 0.\n",
    "            self.config.hidden_dropout_prob = 0.\n",
    "            self.config.attention_dropout = 0.\n",
    "            self.config.attention_probs_dropout_prob = 0.\n",
    "            #LOGGER.info(self.config)\n",
    "        else:\n",
    "            self.config = torch.load(config_path)\n",
    "        \n",
    "        if pretrained:\n",
    "            self.model = AutoModel.from_pretrained(cfg.model, config=self.config)\n",
    "        else:\n",
    "            self.model = AutoModel.from_config(self.config)\n",
    "\n",
    "        #if self.cfg.reinit_last_layer:\n",
    "        #    # Re-init last layer of deberta.\n",
    "        #    for module in self.model.encoder.layer[-1:].modules():\n",
    "        #        self._init_weights(module)\n",
    "        #self.model.gradient_checkpointing_enable()\n",
    "\n",
    "    def _init_weights(self, module):\n",
    "        if isinstance(module, nn.Linear):\n",
    "            module.weight.data.normal_(mean=0.0, std=self.config.initializer_range)\n",
    "            if module.bias is not None:\n",
    "                module.bias.data.zero_()\n",
    "        elif isinstance(module, nn.Embedding):\n",
    "            module.weight.data.normal_(mean=0.0, std=self.config.initializer_range)\n",
    "            if module.padding_idx is not None:\n",
    "                module.weight.data[module.padding_idx].zero_()\n",
    "        elif isinstance(module, nn.LayerNorm):\n",
    "            module.bias.data.zero_()\n",
    "            module.weight.data.fill_(1.0)\n",
    "\n",
    "class WeightedAttentionModel(FB3Model):\n",
    "    def __init__(self, cfg, config_path=None, pretrained=False):\n",
    "        super().__init__(cfg, config_path=config_path, pretrained=pretrained)\n",
    "\n",
    "        self.weighted_pool = WeightedLayerPooling(\n",
    "            self.config.num_hidden_layers, layer_start=9, layer_weights=None)\n",
    "        self.att_pool = AttentionPooling(self.config.hidden_size)\n",
    "\n",
    "        self.fc_out = nn.Linear(self.config.hidden_size*2, cfg.num_target)\n",
    "        self._init_weights(self.fc_out)\n",
    "\n",
    "    def forward(self, x):\n",
    "        pt_out = self.model(**x)\n",
    "        hidden_states = pt_out.hidden_states\n",
    "        last_hidden_state = pt_out.last_hidden_state\n",
    "\n",
    "        x1 = self.weighted_pool(torch.stack(hidden_states))[:, 0]\n",
    "        x2 = self.att_pool(last_hidden_state, x['attention_mask'])\n",
    "\n",
    "        y_hat = self.fc_out(torch.cat([x1, x2], dim=1))\n",
    "        return y_hat\n",
    "    \n",
    "########################\n",
    "\n",
    "class WeightedLayerPooling_(nn.Module):\n",
    "    def __init__(self, num_hidden_layers, layer_start: int = 4, layers = None, layer_weights = None):\n",
    "        super(WeightedLayerPooling_, self).__init__()\n",
    "        self.layer_start = layer_start\n",
    "        self.num_hidden_layers = num_hidden_layers\n",
    "        \n",
    "        if layers:\n",
    "            self.layer_weights = layer_weights if layer_weights is not None \\\n",
    "                else nn.Parameter(\n",
    "                    torch.tensor([1] * len(layers), dtype=torch.float)\n",
    "                )\n",
    "            self.layers = layers\n",
    "        else:\n",
    "            self.layer_weights = layer_weights if layer_weights is not None \\\n",
    "                else nn.Parameter(\n",
    "                   torch.tensor([1] * (num_hidden_layers+1 - layer_start), dtype=torch.float)\n",
    "                )\n",
    "            self.layers = list(range(layer_start, num_hidden_layers+1))\n",
    "            \n",
    "\n",
    "    def forward(self, ft_all_layers):\n",
    "        all_layer_embedding = torch.stack(ft_all_layers)\n",
    "        all_layer_embedding = all_layer_embedding[self.layers, :, :, :]\n",
    "        \n",
    "        weight_factor = self.layer_weights.unsqueeze(-1).unsqueeze(-1).unsqueeze(-1).expand(all_layer_embedding.size())\n",
    "        weighted_average = (weight_factor*all_layer_embedding).sum(dim=0) / self.layer_weights.sum()\n",
    "\n",
    "        return weighted_average\n",
    "    \n",
    "class CustomModel(nn.Module):\n",
    "\n",
    "    def __init__(self, CFG, config_path = None, pretrained = False):\n",
    "        super().__init__()\n",
    "        self.CFG = CFG\n",
    "        self.config = AutoConfig.from_pretrained(config_path, output_hidden_states=True)\n",
    "        self.model = AutoModel.from_config(self.config)\n",
    "        self.pretrained = pretrained\n",
    "                        \n",
    "        fc_hidden_size = self.config.hidden_size\n",
    "        if CFG.pooling == 'mean':\n",
    "            self.pool = MeanPooling()\n",
    "        elif CFG.pooling == 'max':\n",
    "            self.pool = MaxPooling()\n",
    "        elif CFG.pooling == 'min':\n",
    "            self.pool = MinPooling()\n",
    "        elif CFG.pooling == 'attention':\n",
    "            self.pool = AttentionPooling(self.config.hidden_size)\n",
    "        elif CFG.pooling == 'weightedlayer':\n",
    "            self.pool = WeightedLayerPooling_(self.config.num_hidden_layers, layer_start = CFG.layer_start, layer_weights = None)\n",
    "        elif CFG.pooling == 'weightedlayer-mean':\n",
    "            self.pool = WeightedLayerPooling_(self.config.num_hidden_layers, layer_start = CFG.layer_start, layer_weights = None)\n",
    "            self.mean_pool = MeanPooling()\n",
    "        elif self.CFG.pooling == 'attention4':\n",
    "            self.pool = AttentionHead(fc_hidden_size*4, 512)\n",
    "            fc_hidden_size = fc_hidden_size*8\n",
    "        self.fc = nn.Linear(fc_hidden_size, 6)\n",
    "                        \n",
    "    def feature(self, inputs):\n",
    "        outputs = self.model(**inputs)\n",
    "        if self.CFG.pooling == 'attention4':\n",
    "            all_layer_embeddings = torch.stack(outputs.hidden_states)\n",
    "            cat_over_last_layers = torch.cat((all_layer_embeddings[-1], all_layer_embeddings[-2], all_layer_embeddings[-3], all_layer_embeddings[-4]), -1)\n",
    "            cls_pooling = cat_over_last_layers[:, 0]\n",
    "            head_logits = self.pool(cat_over_last_layers)\n",
    "            feature = torch.cat([head_logits, cls_pooling], -1)\n",
    "        elif self.CFG.pooling == 'weightedlayer':\n",
    "            all_layer_embeddings = outputs[1]\n",
    "            feature = self.pool(all_layer_embeddings)[:, 0]\n",
    "        elif self.CFG.pooling == 'weightedlayer-mean':\n",
    "            all_layer_embeddings = outputs[1]\n",
    "            feature = self.pool(all_layer_embeddings)\n",
    "            feature = self.mean_pool(feature, inputs['attention_mask'])\n",
    "        else:\n",
    "            last_hidden_states = outputs[0]\n",
    "            feature = self.pool(last_hidden_states, inputs['attention_mask'])    \n",
    "        return feature\n",
    "\n",
    "    def forward(self, inputs):\n",
    "        feature = self.feature(inputs)\n",
    "        output = self.fc(feature)\n",
    "        return output\n",
    "    \n",
    "##############################\n",
    "\n",
    "class MeanAttentionModel(nn.Module):\n",
    "    def __init__(self, cfg, config_path=None, pretrained=False):\n",
    "        super().__init__()\n",
    "        self.cfg = cfg \n",
    "        if config_path is None:\n",
    "            self.config = AutoConfig.from_pretrained(cfg.model, output_hidden_states=True)\n",
    "            # Turn off dropouts.\n",
    "            self.config.hidden_dropout = 0.\n",
    "            self.config.hidden_dropout_prob = 0.\n",
    "            self.config.attention_dropout = 0.\n",
    "            self.config.attention_probs_dropout_prob = 0.\n",
    "            #LOGGER.info(self.config)\n",
    "        else:\n",
    "            self.config = torch.load(config_path)\n",
    "        \n",
    "        if pretrained:\n",
    "            self.deberta_v3 = AutoModel.from_pretrained(cfg.model, config=self.config)\n",
    "            # Expand embedding dim for new tokens.\n",
    "            self.deberta_v3.resize_token_embeddings(len(cfg.tokenizer))\n",
    "        else:\n",
    "            self.deberta_v3 = AutoModel.from_config(self.config)\n",
    "            \n",
    "        self.deberta_v3.gradient_checkpointing_enable()\n",
    "        \n",
    "        # Define model layers.\n",
    "        self.fc = nn.Linear(self.config.hidden_size, 6)\n",
    "\n",
    "        if cfg.pool == 'mean':\n",
    "            self.pool = MeanPooling()\n",
    "        elif cfg.pool == 'attention':\n",
    "            self.pool = AttentionHead(self.config.hidden_size, self.config.hidden_size)\n",
    "        elif cfg.pool == 'mean-attention':\n",
    "            self.pool = nn.ModuleList([\n",
    "                MeanPooling(),\n",
    "                AttentionHead(self.config.hidden_size, self.config.hidden_size)\n",
    "            ])\n",
    "            self.fc = nn.Linear(self.config.hidden_size * len(self.pool), 6)\n",
    "        elif cfg.pool == 'mean-attention-with-mask':\n",
    "            self.pool = nn.ModuleList([\n",
    "                MeanPooling(),\n",
    "                AttentionPooling(self.config.hidden_size)\n",
    "            ])\n",
    "            self.fc = nn.Linear(self.config.hidden_size * len(self.pool), 6)\n",
    "        # Re-init weights.\n",
    "        self._init_weights(self.fc)\n",
    "        \n",
    "        # Multi-sample dropout.\n",
    "        self.multi_dropout = MultiSampleDropout(self.fc, cfg.num_dropout, cfg.prob_dropout)\n",
    "        \n",
    "    def global_avg_pool(x):\n",
    "        return torch.mean(x.view(x.size(0), x.size(1), -1), dim=-1)\n",
    "        \n",
    "    def _init_weights(self, module):\n",
    "        if isinstance(module, nn.Linear):\n",
    "            module.weight.data.normal_(mean=0.0, std=self.config.initializer_range)\n",
    "            if module.bias is not None:\n",
    "                module.bias.data.zero_()\n",
    "        elif isinstance(module, nn.Embedding):\n",
    "            module.weight.data.normal_(mean=0.0, std=self.config.initializer_range)\n",
    "            if module.padding_idx is not None:\n",
    "                module.weight.data[module.padding_idx].zero_()\n",
    "        elif isinstance(module, nn.LayerNorm):\n",
    "            module.bias.data.zero_()\n",
    "            module.weight.data.fill_(1.0)\n",
    "    \n",
    "    def feature(self, x):\n",
    "        pt_outputs = self.deberta_v3(**x)\n",
    "        last_hidden_states = pt_outputs[0] # N x max_len x 768\n",
    "        # Pooling feat.\n",
    "        if type(self.pool) == nn.ModuleList:\n",
    "            pool_feature = [pool(last_hidden_states, x['attention_mask']) for pool in self.pool]\n",
    "            pool_feature = torch.cat(pool_feature, dim=1)\n",
    "        else:\n",
    "            pool_feature = self.pool(last_hidden_states, x['attention_mask']) # N x 768\n",
    "        return pool_feature\n",
    "    \n",
    "    def forward(self, x, y=None, loss_fn=None):\n",
    "        feature = self.feature(x)\n",
    "        # if self.training:\n",
    "        #    out = self.multi_dropout(feature)\n",
    "        # else:\n",
    "        #    out = self.fc(feature)\n",
    "        out = self.fc(feature)\n",
    "        return out"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "410164da",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2022-11-29T01:13:24.839499Z",
     "iopub.status.busy": "2022-11-29T01:13:24.838062Z",
     "iopub.status.idle": "2022-11-29T01:13:24.844890Z",
     "shell.execute_reply": "2022-11-29T01:13:24.844057Z"
    },
    "papermill": {
     "duration": 0.014693,
     "end_time": "2022-11-29T01:13:24.846819",
     "exception": false,
     "start_time": "2022-11-29T01:13:24.832126",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "class Config(dict):\n",
    "    __getattr__ = dict.__getitem__\n",
    "    __setattr__ = dict.__setitem__\n",
    "    __delattr__ = dict.__delitem__\n",
    "    \n",
    "    def init(self, kwargs):\n",
    "        super().init(kwargs)\n",
    "\n",
    "        for k, v in kwargs.items():\n",
    "            setattr(self, k, v)\n",
    "\n",
    "    def set(self, key, val):\n",
    "        self[key] = val\n",
    "        setattr(self, key, val)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "id": "0f3ba606",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2022-11-29T01:13:24.859214Z",
     "iopub.status.busy": "2022-11-29T01:13:24.857823Z",
     "iopub.status.idle": "2022-11-29T01:13:24.869427Z",
     "shell.execute_reply": "2022-11-29T01:13:24.868561Z"
    },
    "papermill": {
     "duration": 0.019573,
     "end_time": "2022-11-29T01:13:24.871412",
     "exception": false,
     "start_time": "2022-11-29T01:13:24.851839",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "def seed_everything(seed=42):\n",
    "    '''\n",
    "    Sets the seed of the entire notebook so results are the same every time we run.\n",
    "    This is for REPRODUCIBILITY.\n",
    "    '''\n",
    "    random.seed(seed)\n",
    "    # Set a fixed value for the hash seed\n",
    "    os.environ['PYTHONHASHSEED'] = str(seed)\n",
    "    np.random.seed(seed)\n",
    "    torch.manual_seed(seed)\n",
    "    \n",
    "    if torch.cuda.is_available():\n",
    "        torch.cuda.manual_seed(seed)\n",
    "        torch.cuda.manual_seed_all(seed)\n",
    "        # When running on the CuDNN backend, two further options must be set\n",
    "        torch.backends.cudnn.deterministic = True\n",
    "        torch.backends.cudnn.benchmark = False\n",
    "seed_everything(seed=42)\n",
    "\n",
    "def mc_rmse(y_true, y_pred):\n",
    "    scores = []\n",
    "    ncols = y_true.shape[1]\n",
    "    \n",
    "    for n in range(ncols):\n",
    "        yn_true = y_true[:, n]\n",
    "        yn_pred = y_pred[:, n]\n",
    "        rmse_ = mean_squared_error(yn_true, yn_pred, squared=False)\n",
    "        scores.append(rmse_)\n",
    "    score = np.mean(scores) \n",
    "    return score, scores\n",
    "\n",
    "def get_result(cfg, oof_df):\n",
    "    labels = oof_df[TARGET_COLS].values\n",
    "    preds = oof_df[[f\"pred_{c}\" for c in TARGET_COLS]].values\n",
    "    score, scores = mc_rmse(labels, preds)\n",
    "    print(f'score: {score:<.6f}  scores: {scores}')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "id": "5d03a8c8",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2022-11-29T01:13:24.882557Z",
     "iopub.status.busy": "2022-11-29T01:13:24.882287Z",
     "iopub.status.idle": "2022-11-29T01:13:24.898180Z",
     "shell.execute_reply": "2022-11-29T01:13:24.897342Z"
    },
    "papermill": {
     "duration": 0.023824,
     "end_time": "2022-11-29T01:13:24.900234",
     "exception": false,
     "start_time": "2022-11-29T01:13:24.876410",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "def encode_text(cfg, text):\n",
    "    if cfg.pretrained:\n",
    "        inputs = cfg.tokenizer(\n",
    "            text,\n",
    "            None,\n",
    "            add_special_tokens=True,\n",
    "            padding='max_length',\n",
    "            truncation=True,\n",
    "            max_length=cfg.max_len,\n",
    "            return_token_type_ids=True,\n",
    "            return_tensors='pt'\n",
    "        )\n",
    "        inputs = {k:v.squeeze(0) for k,v in inputs.items()}\n",
    "    else:\n",
    "        if 'roberta' in cfg.model:\n",
    "            inputs = cfg.tokenizer.encode_plus(\n",
    "                text,\n",
    "                return_tensors = None,\n",
    "                add_special_tokens = True,\n",
    "                max_length = cfg.max_len,\n",
    "                pad_to_max_length = True,\n",
    "                truncation = True)\n",
    "        elif '512' in cfg.name or '768' in cfg.name or '1024' in cfg.name:\n",
    "            inputs = cfg.tokenizer.encode_plus(\n",
    "                text,\n",
    "                return_tensors = None,\n",
    "                add_special_tokens = True,\n",
    "                max_length = cfg.max_len,\n",
    "                truncation = True)\n",
    "        else:\n",
    "            inputs = cfg.tokenizer.encode_plus(\n",
    "                text, \n",
    "                return_tensors=None, \n",
    "                add_special_tokens=True)\n",
    "        for k, v in inputs.items():\n",
    "            inputs[k] = torch.tensor(v, dtype=torch.long)\n",
    "    return inputs \n",
    "\n",
    "def preprocess(texts):\n",
    "    texts = (\n",
    "        texts\n",
    "        .str.replace(r'\\r\\n', '<newline>', regex=True)\n",
    "        .str.replace(r'\\n', '<newline>', regex=True)\n",
    "        .str.replace('<newline><newline>', '<newline>', regex=False)\n",
    "        .values \n",
    "    )\n",
    "    return texts\n",
    "\n",
    "class TestDataset(Dataset):\n",
    "    def __init__(self, cfg, df):\n",
    "        self.cfg = cfg\n",
    "        if not cfg.pretrained and cfg.version in ['1', 'mean-attention']:\n",
    "        #if not pretrained:\n",
    "            print('preprocess')\n",
    "            self.texts = preprocess(df['full_text'])\n",
    "        else:\n",
    "            self.texts = df['full_text'].values\n",
    "\n",
    "    def __len__(self):\n",
    "        return len(self.texts)\n",
    "\n",
    "    def __getitem__(self, item):\n",
    "        inputs = encode_text(self.cfg, self.texts[item])\n",
    "        return inputs\n",
    "\n",
    "class TrainDataset(Dataset):\n",
    "    def __init__(self, cfg, data):\n",
    "        self.cfg = cfg\n",
    "        self.xs = data['full_text']\n",
    "        self.ys = data[TARGET_COLS].values \n",
    "        \n",
    "    def __len__(self):\n",
    "        return len(self.xs)\n",
    "    \n",
    "    def __getitem__(self, idx):\n",
    "        x = encode_text(self.cfg, self.xs[idx])\n",
    "        y = torch.tensor(self.ys[idx], dtype=torch.float32)\n",
    "        return x, y\n",
    "\n",
    "def create_data_loaders(cfg, folds, fold):\n",
    "    train_folds = folds[folds['fold'] != fold].reset_index(drop=True)\n",
    "    valid_folds = folds[folds['fold'] == fold].reset_index(drop=True)\n",
    "    train_dataset = TrainDataset(cfg, train_folds)\n",
    "    valid_dataset = TrainDataset(cfg, valid_folds)\n",
    "    \n",
    "    train_loader = DataLoader(\n",
    "        train_dataset,\n",
    "        batch_size=cfg.batch_size,\n",
    "        shuffle=True,\n",
    "        num_workers=cfg.num_workers,\n",
    "        pin_memory=True,\n",
    "        drop_last=True\n",
    "    )\n",
    "    valid_loader = DataLoader(\n",
    "        valid_dataset,\n",
    "        batch_size=cfg.batch_size,\n",
    "        shuffle=False,\n",
    "        num_workers=0,\n",
    "        pin_memory=True,\n",
    "        drop_last=False\n",
    "    )\n",
    "    return train_loader, valid_loader"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "id": "98e2edae",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2022-11-29T01:13:24.912386Z",
     "iopub.status.busy": "2022-11-29T01:13:24.911038Z",
     "iopub.status.idle": "2022-11-29T01:13:24.922231Z",
     "shell.execute_reply": "2022-11-29T01:13:24.921393Z"
    },
    "papermill": {
     "duration": 0.018927,
     "end_time": "2022-11-29T01:13:24.924167",
     "exception": false,
     "start_time": "2022-11-29T01:13:24.905240",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "def load_config(input_path, inference_weight=1):\n",
    "    # Load CFG class.\n",
    "    cfg = Config(**json.load(open(os.path.join(input_path, 'CFG.json'), 'r')))\n",
    "    cfg.path = input_path\n",
    "    cfg.config_path = os.path.join(cfg.path, 'config.pth')\n",
    "    # Load tokenizer.\n",
    "    tokenizer = AutoTokenizer.from_pretrained(os.path.join(cfg.path, 'tokenizer'))\n",
    "    cfg.tokenizer = tokenizer\n",
    "    \n",
    "    cfg.inference_weight = inference_weight\n",
    "    return cfg\n",
    "\n",
    "def load_model(cfg, fold, version='1', **model_kwargs):\n",
    "    # Load torch model.\n",
    "    if version == '1':\n",
    "        model = MultiPoolModel(cfg, config_path=cfg.config_path, pretrained=False, **model_kwargs)\n",
    "    elif version == '2':\n",
    "        model = Attention4Model(cfg, config_path=cfg.config_path, pretrained=False, **model_kwargs)\n",
    "    elif version == '21':\n",
    "        model = WMPoolModel(cfg, config_path=cfg.config_path, pretrained=False, **model_kwargs)\n",
    "    elif version == 'mean':\n",
    "        model = MeanModel(cfg, config_path=cfg.config_path, pretrained=False, **model_kwargs)\n",
    "    elif version == 'attention':\n",
    "        model = AttentionModel(cfg, config_path=cfg.config_path, pretrained=False, **model_kwargs)\n",
    "    elif version == 'mix':\n",
    "        model = MixModel(cfg, config_path=cfg.config_path, pretrained=False, **model_kwargs)\n",
    "    elif version == 'weighted-attention':\n",
    "        model = WeightedAttentionModel(cfg, config_path=cfg.config_path, pretrained=False, **model_kwargs)\n",
    "    elif version == 'custom':\n",
    "        model = CustomModel(cfg, config_path=cfg.config_path, pretrained=False, **model_kwargs)\n",
    "    elif version == 'mean-attention':\n",
    "        model = MeanAttentionModel(cfg, config_path=cfg.config_path, pretrained=False)\n",
    "    state = torch.load(\n",
    "        os.path.join(cfg.path, f\"{cfg.model.replace('/', '-')}_fold{fold}_best.pth\"),\n",
    "        map_location=torch.device('cpu'))\n",
    "    model.load_state_dict(state['model'])\n",
    "    return model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "id": "711da5a4",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2022-11-29T01:13:24.935838Z",
     "iopub.status.busy": "2022-11-29T01:13:24.934988Z",
     "iopub.status.idle": "2022-11-29T01:13:24.947473Z",
     "shell.execute_reply": "2022-11-29T01:13:24.946675Z"
    },
    "papermill": {
     "duration": 0.020245,
     "end_time": "2022-11-29T01:13:24.949413",
     "exception": false,
     "start_time": "2022-11-29T01:13:24.929168",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "def mean_pooling(model_output, attention_mask):\n",
    "    token_embeddings = model_output.last_hidden_state.detach().cpu()\n",
    "    input_mask_expanded = (\n",
    "        attention_mask.unsqueeze(-1).expand(token_embeddings.size()).float()\n",
    "    )\n",
    "    return torch.sum(token_embeddings * input_mask_expanded, 1) / torch.clamp(\n",
    "        input_mask_expanded.sum(1), min=1e-9\n",
    "    )\n",
    "\n",
    "def inference_fn(test_loader, model, device):\n",
    "    preds = []\n",
    "    model.eval()\n",
    "    model.to(device)\n",
    "    #tk0 = tqdm(test_loader, total=len(test_loader))\n",
    "    for inputs in test_loader:\n",
    "        for k, v in inputs.items():\n",
    "            inputs[k] = v.to(device)\n",
    "        with torch.no_grad():\n",
    "            y_preds = model(inputs)\n",
    "        preds.append(y_preds.to('cpu').numpy())\n",
    "    predictions = np.concatenate(preds)\n",
    "    return predictions\n",
    "\n",
    "class Inferencer:\n",
    "    def __init__(self, input_path=None, cfg=None, inference_weight=1):\n",
    "        if cfg == None:\n",
    "            self.cfg = load_config(input_path, inference_weight)\n",
    "        else:\n",
    "            self.cfg = cfg\n",
    "    \n",
    "    def predict(self, test_loader, device, stat_fn=np.mean):\n",
    "        preds = []\n",
    "        \n",
    "        for fold in self.cfg.trn_fold:\n",
    "            start = time.time()\n",
    "            print('#'*10, cfg.path, '#'*10)\n",
    "            \n",
    "            print(f'Predicting fold {fold}...')\n",
    "            model = load_model(self.cfg, fold, version=self.cfg.version)\n",
    "            pred = inference_fn(test_loader, model, device)\n",
    "            preds.append(pred)\n",
    "            del model, pred; gc.collect()\n",
    "            torch.cuda.empty_cache()\n",
    "            \n",
    "            end = time.time() - start\n",
    "            print('#'*10, f'ETA: {end:.2f}s', '#'*10, '\\n')\n",
    "        \n",
    "        \n",
    "        self.preds = stat_fn(preds, axis=0) \n",
    "        self.preds = np.clip(self.preds, 1, 5)\n",
    "        return self.preds\n",
    "    \n",
    "    def get_oof_result(self, file_type='pkl'):\n",
    "        return get_result(self.cfg, self.get_oof_df(file_type))\n",
    "    \n",
    "    def get_oof_df(self, file_type='pkl'):\n",
    "        if file_type == 'pkl':\n",
    "            return pd.read_pickle(os.path.join(cfg.path, 'oof_df.pkl'))\n",
    "        return pd.read_csv(os.path.join(cfg.path, 'oof_df.csv'))"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "42230f61",
   "metadata": {
    "papermill": {
     "duration": 0.004705,
     "end_time": "2022-11-29T01:13:24.960492",
     "exception": false,
     "start_time": "2022-11-29T01:13:24.955787",
     "status": "completed"
    },
    "tags": []
   },
   "source": [
    "# Fine-tuned models"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "id": "9e7b1130",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2022-11-29T01:13:24.972494Z",
     "iopub.status.busy": "2022-11-29T01:13:24.971126Z",
     "iopub.status.idle": "2022-11-29T01:13:26.161044Z",
     "shell.execute_reply": "2022-11-29T01:13:26.160043Z"
    },
    "papermill": {
     "duration": 1.198058,
     "end_time": "2022-11-29T01:13:26.163459",
     "exception": false,
     "start_time": "2022-11-29T01:13:24.965401",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "v114_CFG = Config(\n",
    "    model=\"microsoft/deberta-v3-base\",\n",
    "    version='1',\n",
    "    num_target = 6,\n",
    "    reinit_last_layer=True,\n",
    "    reinit_fc=True,\n",
    "    weight_decay=0.01,\n",
    "    learning_rate=2e-5,\n",
    "    layerwise_learning_rate_decay=1.5,\n",
    "    use_dropout=False,\n",
    "    prob_dropout=[0.06, 0.08, 0.1, 0.12, 0.14],\n",
    "    num_dropout=5,\n",
    "    pool_head='mean-attention',\n",
    "    seed=42,\n",
    "    n_fold=4,\n",
    "    trn_fold=[0,1,2,3],\n",
    "    path='../input/fb3models/v114/',\n",
    "    config_path='../input/fb3models/v114/config.pth',\n",
    "    tokenizer=AutoTokenizer.from_pretrained('../input/fb3models/v114/tokenizer')\n",
    ")\n",
    "\n",
    "weightedpool_CFG = Config(\n",
    "    model='microsoft/deberta-v3-base',\n",
    "    name='weighted_pool',\n",
    "    version='1',\n",
    "    num_target=6,\n",
    "    reinit_last_layer=True,\n",
    "    reinit_fc=True,\n",
    "    weight_decay=0.01,\n",
    "    learning_rate=1.5e-5,\n",
    "    layerwise_learning_rate_decay=1.5,\n",
    "    use_dropout=False,\n",
    "    prob_dropout=[0.06, 0.08, 0.1, 0.12, 0.14],\n",
    "    num_dropout=5,\n",
    "    pool_head='weighted',\n",
    "    seed=42,\n",
    "    n_fold=4,\n",
    "    trn_fold=[0,1,2,3],\n",
    "    train=True,\n",
    "    path='../input/fb3-train/',\n",
    "    config_path='../input/fb3-train/config.pth',\n",
    "    tokenizer=AutoTokenizer.from_pretrained('../input/fb3-train/tokenizer'),\n",
    "    inference_weight=0.1)\n",
    "\n",
    "v116_CFG = load_config('../input/fb3-colab-models/v116', inference_weight=0.1)\n",
    "v116_CFG.path = '../input/fb3models/v116'\n",
    "v116_CFG.config_path = '../input/fb3models/v116/config.pth'\n",
    "v116_CFG.version = '1'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "id": "13bfb74c",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2022-11-29T01:13:26.176185Z",
     "iopub.status.busy": "2022-11-29T01:13:26.175270Z",
     "iopub.status.idle": "2022-11-29T01:13:31.329544Z",
     "shell.execute_reply": "2022-11-29T01:13:31.328213Z"
    },
    "papermill": {
     "duration": 5.163185,
     "end_time": "2022-11-29T01:13:31.332056",
     "exception": false,
     "start_time": "2022-11-29T01:13:26.168871",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "v112_CFG = Config(\n",
    "    num_workers=1,\n",
    "    batch_size=3,\n",
    "    max_len=512,\n",
    "    model=\"microsoft/deberta-v3-base\",\n",
    "    name='v112',\n",
    "    version='1',\n",
    "    num_target = 6,\n",
    "    reinit_last_layer=True,\n",
    "    reinit_fc=True,\n",
    "    weight_decay=0.01,\n",
    "    learning_rate=2e-5,\n",
    "    layerwise_learning_rate_decay=1.5,\n",
    "    use_dropout=False,\n",
    "    prob_dropout=[0.06, 0.08, 0.1, 0.12, 0.14],\n",
    "    num_dropout=5,\n",
    "    pool_head='attention',\n",
    "    seed=42,\n",
    "    n_fold=4,\n",
    "    trn_fold=[0,1,2,3],\n",
    "    train=True,\n",
    "    path='../input/fb3models/v112/',\n",
    "    config_path='../input/fb3models/v112/config.pth',\n",
    "    inference_weight=1.0,\n",
    "    tokenizer=AutoTokenizer.from_pretrained('../input/fb3models/v112/tokenizer')\n",
    ")\n",
    "\n",
    "#####\n",
    "v2_CFG = load_config('../input/fb3models/v2/', inference_weight=1.0)\n",
    "v2_CFG.name = 'v2'\n",
    "v2_CFG.version = '2'\n",
    "v2_CFG.trn_fold = [0,1,2,3]\n",
    "\n",
    "#####\n",
    "v21_CFG = load_config('../input/fb3models/v21/', inference_weight=1)\n",
    "v21_CFG.name = 'v21'\n",
    "v21_CFG.version = '21'\n",
    "\n",
    "#####\n",
    "attention_fgm_CFG = load_config('../input/fb3models/20221114-192943-deberta-v3-base/', inference_weight=1.0)\n",
    "attention_fgm_CFG.name = 'attention_fgm'\n",
    "attention_fgm_CFG.version = 'custom'\n",
    "attention_fgm_CFG.config_path = '../input/fb3models/20221114-192943-deberta-v3-base/config/config.json'\n",
    "\n",
    "weighted_fgm_CFG = Config(\n",
    "    pretrained=False,\n",
    "    path='../input/fb3models/20221115-061243-deberta-v3-base',\n",
    "    config_path='../input/fb3models/20221115-061243-deberta-v3-base/config/config.json',\n",
    "    tokenizer=AutoTokenizer.from_pretrained('../input/fb3models/20221115-061243-deberta-v3-base/tokenizer'),\n",
    "    name='weighted_fgm',\n",
    "    version='custom',\n",
    "    train = True,\n",
    "    debug = False,\n",
    "    offline = False,\n",
    "    models_path = 'FB3-models',\n",
    "    epochs = 5,\n",
    "    save_all_models = False,\n",
    "    competition = 'FB3',\n",
    "    apex = True,\n",
    "    print_freq = 20,\n",
    "    num_workers = 4,\n",
    "    model = 'microsoft/deberta-v3-base', #If you want to train on the kaggle platform, v3-base is realistic. v3-large will time out.\n",
    "    loss_func = 'SmoothL1', # 'SmoothL1', 'RMSE'\n",
    "    gradient_checkpointing = True,\n",
    "    scheduler = 'cosine',\n",
    "    batch_scheduler = True,\n",
    "    num_cycles = 0.5,\n",
    "    num_warmup_steps = 0,\n",
    "    encoder_lr = 2e-5,\n",
    "    decoder_lr = 2e-5,\n",
    "    min_lr = 1e-6,\n",
    "    #Layer-Wise Learning Rate Decay\n",
    "    llrd = True,\n",
    "    layerwise_lr = 5e-5,\n",
    "    layerwise_lr_decay = 0.9,\n",
    "    layerwise_weight_decay = 0.01,\n",
    "    layerwise_adam_epsilon = 1e-6,\n",
    "    layerwise_use_bertadam = False,\n",
    "    #pooling\n",
    "    pooling = 'weightedlayer', # mean, max, min, attention, weightedlayer\n",
    "    layer_start = 11,\n",
    "    layers=None,\n",
    "    #init_weight\n",
    "    init_weight = 'normal', # normal, xavier_uniform, xavier_normal, kaiming_uniform, kaiming_normal, orthogonal\n",
    "    #re-init\n",
    "    reinit = True,\n",
    "    reinit_n = 1,\n",
    "    #adversarial\n",
    "    fgm = True,\n",
    "    awp = False,\n",
    "    adv_lr = 1,\n",
    "    adv_eps = 0.2,\n",
    "    unscale = False,\n",
    "    eps = 1e-6,\n",
    "    betas = (0.9, 0.999),\n",
    "    max_len = 512,\n",
    "    weight_decay = 0.01,\n",
    "    gradient_accumulation_steps = 1,\n",
    "    max_grad_norm = 1000,\n",
    "    target_cols = ['cohesion', 'syntax', 'vocabulary', 'phraseology', 'grammar', 'conventions'],\n",
    "    seed = 42,\n",
    "    cv_seed = 42,\n",
    "    n_fold = 4,\n",
    "    trn_fold = [0,1,2,3],\n",
    "    batch_size = 8,\n",
    "    n_targets = 6,\n",
    "    gpu_id = 0) \n",
    "\n",
    "weighted_attention_CFG = load_config('../input/fb3models/weighted_attention_v3', inference_weight=1.0)\n",
    "weighted_attention_CFG.name = 'weighted_attention'\n",
    "weighted_attention_CFG.version = 'weighted-attention'\n",
    "\n",
    "mean_attention_no_fgm_CFG = load_config('../input/fb3models/20221117-183420-deberta-v3-base-mean-attention-with-mask', inference_weight=1.0)\n",
    "mean_attention_no_fgm_CFG.name = 'mean_attention_no_fgm'\n",
    "mean_attention_no_fgm_CFG.version = 'mean-attention'\n",
    "\n",
    "attention_large_fgm_CFG = load_config('../input/fb3models/20221118-164148-deberta-v3-large-attention_fgm')\n",
    "attention_large_fgm_CFG.name = 'attention_large_fgm'\n",
    "attention_large_fgm_CFG.version = 'custom'\n",
    "attention_large_fgm_CFG.config_path = '../input/fb3models/20221118-164148-deberta-v3-large-attention_fgm/config/config.json'\n",
    "\n",
    "attention_fgm_512_CFG = load_config('../input/fb3models/20221121-143655-deberta-v3-base-attention_fgm_512')\n",
    "attention_fgm_512_CFG.name = 'attention_fgm_512'\n",
    "attention_fgm_512_CFG.version = 'custom'\n",
    "attention_fgm_512_CFG.config_path = '../input/fb3models/20221121-143655-deberta-v3-base-attention_fgm_512/config/config.json'\n",
    "\n",
    "attention_fgm_768_CFG = load_config('../input/fb3models/20221120-072218-deberta-v3-base-attention_fgm')\n",
    "attention_fgm_768_CFG.name = 'attention_fgm_768'\n",
    "attention_fgm_768_CFG.version = 'custom'\n",
    "attention_fgm_768_CFG.config_path = '../input/fb3models/20221120-072218-deberta-v3-base-attention_fgm/config/config.json'\n",
    "\n",
    "weightedmean2last_fgm_1024_CFG = load_config('/kaggle/input/fb3models/20221127-065251-deberta-v3-base-weightedmean2last_fgm_1024')\n",
    "weightedmean2last_fgm_1024_CFG.name = 'weightedmean2last_fgm_1024'\n",
    "weightedmean2last_fgm_1024_CFG.version = 'custom'\n",
    "weightedmean2last_fgm_1024_CFG.config_path = '/kaggle/input/fb3models/20221127-065251-deberta-v3-base-weightedmean2last_fgm_1024/config/config.json'\n",
    "\n",
    "weighted2last_fgm_512_CFG = load_config('../input/fb3models/20221124-060246-deberta-v3-base-weighted2last_fgm')\n",
    "weighted2last_fgm_512_CFG.name = 'weighted2last_fgm_512'\n",
    "weighted2last_fgm_512_CFG.version = 'custom'\n",
    "weighted2last_fgm_512_CFG.config_path = '../input/fb3models/20221124-060246-deberta-v3-base-weighted2last_fgm/config/config.json'\n",
    "\n",
    "weightedmean2last_fgm_512_CFG = load_config('../input/fb3models/20221124-160318-deberta-v3-base-weightedmean2last_fgm')\n",
    "weightedmean2last_fgm_512_CFG.name = 'weightedmean2last_fgm_512'\n",
    "weightedmean2last_fgm_512_CFG.version = 'custom'\n",
    "weightedmean2last_fgm_512_CFG.config_path = '../input/fb3models/20221124-160318-deberta-v3-base-weightedmean2last_fgm/config/config.json'\n",
    "\n",
    "roberta_attention_fgm_CFG = load_config('../input/fb3models/20221121-173739-roberta-base')\n",
    "roberta_attention_fgm_CFG.name = 'roberta_attention_large_fgm'\n",
    "roberta_attention_fgm_CFG.version = 'custom'\n",
    "roberta_attention_fgm_CFG.config_path = '../input/fb3models/20221121-173739-roberta-base/config/config.json'\n",
    "\n",
    "##########\n",
    "v112_CFG.pretrained = False\n",
    "v114_CFG.pretrained = False\n",
    "v116_CFG.pretrained = False\n",
    "v21_CFG.pretrained = False\n",
    "v2_CFG.pretrained = False\n",
    "attention_fgm_CFG.pretrained = False\n",
    "weighted_attention_CFG.pretrained = False\n",
    "weightedpool_CFG.pretrained = False\n",
    "mean_attention_no_fgm_CFG.pretrained=False\n",
    "attention_large_fgm_CFG.pretrained=False\n",
    "attention_fgm_512_CFG.pretrained = False\n",
    "attention_fgm_768_CFG.pretrained = False\n",
    "weighted2last_fgm_512_CFG.pretrained = False\n",
    "weightedmean2last_fgm_512_CFG.pretrained = False\n",
    "roberta_attention_fgm_CFG.pretrained = False\n",
    "weightedmean2last_fgm_1024_CFG.pretrained = False\n",
    "\n",
    "\n",
    "weighted_fgm_CFG.inference_weight = 1.0\n",
    "v114_CFG.inference_weight = 1.0 \n",
    "v2_CFG.inference_weight = 1.0 \n",
    "v21_CFG.inference_weight = 1.0\n",
    "attention_fgm_CFG.inference_weight = 1.0\n",
    "weighted_attention_CFG.inference_weight = 1.0\n",
    "attention_large_fgm_CFG.inference_weight = 1.0\n",
    "attention_fgm_512_CFG.inference_weight = 1.0\n",
    "attention_fgm_768_CFG.inference_weight = 1.0\n",
    "weighted2last_fgm_512_CFG.inference_weight = 1.0\n",
    "weightedmean2last_fgm_512_CFG.inference_weight = 1.0\n",
    "roberta_attention_fgm_CFG.inference_weight = 1.0\n",
    "weightedmean2last_fgm_1024_CFG.inference_weight = 1.0"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "id": "8599ab0b",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2022-11-29T01:13:31.345288Z",
     "iopub.status.busy": "2022-11-29T01:13:31.343592Z",
     "iopub.status.idle": "2022-11-29T01:13:31.349711Z",
     "shell.execute_reply": "2022-11-29T01:13:31.348868Z"
    },
    "papermill": {
     "duration": 0.013935,
     "end_time": "2022-11-29T01:13:31.351564",
     "exception": false,
     "start_time": "2022-11-29T01:13:31.337629",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "fine_tuned_models_cfg = [\n",
    "    attention_large_fgm_CFG, weighted2last_fgm_512_CFG, weightedmean2last_fgm_512_CFG, attention_fgm_512_CFG, attention_fgm_768_CFG, \n",
    "    v21_CFG, v112_CFG, mean_attention_no_fgm_CFG, v2_CFG, weightedpool_CFG, attention_fgm_CFG]\n",
    "\n",
    "# for i,cfg in enumerate(fine_tuned_models_cfg):\n",
    "#     fine_tuned_models_cfg[i].inference_weight = optimal_weights[i]\n",
    "#     print(fine_tuned_models_cfg[i].name, fine_tuned_models_cfg[i].inference_weight)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "id": "bda77428",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2022-11-29T01:13:31.364259Z",
     "iopub.status.busy": "2022-11-29T01:13:31.362707Z",
     "iopub.status.idle": "2022-11-29T01:13:31.369616Z",
     "shell.execute_reply": "2022-11-29T01:13:31.368549Z"
    },
    "papermill": {
     "duration": 0.015266,
     "end_time": "2022-11-29T01:13:31.372192",
     "exception": false,
     "start_time": "2022-11-29T01:13:31.356926",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Total number of models: 11\n"
     ]
    }
   ],
   "source": [
    "print('Total number of models:', len(fine_tuned_models_cfg))\n",
    "stacking = False"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "id": "f421ef7f",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2022-11-29T01:13:31.383792Z",
     "iopub.status.busy": "2022-11-29T01:13:31.383366Z",
     "iopub.status.idle": "2022-11-29T01:13:31.397390Z",
     "shell.execute_reply": "2022-11-29T01:13:31.396520Z"
    },
    "papermill": {
     "duration": 0.021791,
     "end_time": "2022-11-29T01:13:31.399468",
     "exception": false,
     "start_time": "2022-11-29T01:13:31.377677",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "test = pd.read_csv(TEST_PATH)\n",
    "# test['tokenize_length'] = [len(cfg.tokenizer(text)['input_ids']) for text in test['full_text'].values]\n",
    "# test = test.sort_values('tokenize_length', ascending=True).reset_index(drop=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "id": "a874fb5c",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2022-11-29T01:13:31.411050Z",
     "iopub.status.busy": "2022-11-29T01:13:31.410775Z",
     "iopub.status.idle": "2022-11-29T01:24:25.529004Z",
     "shell.execute_reply": "2022-11-29T01:24:25.527953Z"
    },
    "papermill": {
     "duration": 654.12816,
     "end_time": "2022-11-29T01:24:25.532711",
     "exception": false,
     "start_time": "2022-11-29T01:13:31.404551",
     "status": "completed"
    },
    "scrolled": true,
    "tags": []
   },
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "ca2ae9b00c26451fa9393c8fef61412b",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "  0%|          | 0/11 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "score: 0.452063  scores: [0.48157374223152627, 0.4452090627651757, 0.4154085184288849, 0.4536476489584571, 0.47240178087223106, 0.44413673294015477]\n",
      "########## ../input/fb3models/20221118-164148-deberta-v3-large-attention_fgm ##########\n",
      "Predicting fold 0...\n",
      "########## ETA: 36.16s ########## \n",
      "\n",
      "########## ../input/fb3models/20221118-164148-deberta-v3-large-attention_fgm ##########\n",
      "Predicting fold 1...\n",
      "########## ETA: 30.33s ########## \n",
      "\n",
      "########## ../input/fb3models/20221118-164148-deberta-v3-large-attention_fgm ##########\n",
      "Predicting fold 2...\n",
      "########## ETA: 31.22s ########## \n",
      "\n",
      "########## ../input/fb3models/20221118-164148-deberta-v3-large-attention_fgm ##########\n",
      "Predicting fold 3...\n",
      "########## ETA: 30.90s ########## \n",
      "\n",
      "score: 0.453033  scores: [0.48389889660631685, 0.44678944048963176, 0.41465519997691047, 0.45360604538329913, 0.4717536101123449, 0.4474973036716195]\n",
      "max_len=512\n",
      "########## ../input/fb3models/20221124-060246-deberta-v3-base-weighted2last_fgm ##########\n",
      "Predicting fold 0...\n",
      "########## ETA: 13.80s ########## \n",
      "\n",
      "########## ../input/fb3models/20221124-060246-deberta-v3-base-weighted2last_fgm ##########\n",
      "Predicting fold 1...\n",
      "########## ETA: 11.28s ########## \n",
      "\n",
      "########## ../input/fb3models/20221124-060246-deberta-v3-base-weighted2last_fgm ##########\n",
      "Predicting fold 2...\n",
      "########## ETA: 12.38s ########## \n",
      "\n",
      "########## ../input/fb3models/20221124-060246-deberta-v3-base-weighted2last_fgm ##########\n",
      "Predicting fold 3...\n",
      "########## ETA: 12.48s ########## \n",
      "\n",
      "score: 0.452766  scores: [0.48130576936482516, 0.4463892153474189, 0.4154102670831803, 0.4533288332620337, 0.471398800994229, 0.4487615040883342]\n",
      "max_len=512\n",
      "########## ../input/fb3models/20221124-160318-deberta-v3-base-weightedmean2last_fgm ##########\n",
      "Predicting fold 0...\n",
      "########## ETA: 12.43s ########## \n",
      "\n",
      "########## ../input/fb3models/20221124-160318-deberta-v3-base-weightedmean2last_fgm ##########\n",
      "Predicting fold 1...\n",
      "########## ETA: 11.35s ########## \n",
      "\n",
      "########## ../input/fb3models/20221124-160318-deberta-v3-base-weightedmean2last_fgm ##########\n",
      "Predicting fold 2...\n",
      "########## ETA: 12.64s ########## \n",
      "\n",
      "########## ../input/fb3models/20221124-160318-deberta-v3-base-weightedmean2last_fgm ##########\n",
      "Predicting fold 3...\n",
      "########## ETA: 12.61s ########## \n",
      "\n",
      "score: 0.454154  scores: [0.48439478507550315, 0.4476619277192037, 0.41633279856096794, 0.4540781761924119, 0.4753848249763217, 0.4470715017077387]\n",
      "max_len=512\n",
      "########## ../input/fb3models/20221121-143655-deberta-v3-base-attention_fgm_512 ##########\n",
      "Predicting fold 0...\n",
      "########## ETA: 14.02s ########## \n",
      "\n",
      "########## ../input/fb3models/20221121-143655-deberta-v3-base-attention_fgm_512 ##########\n",
      "Predicting fold 1...\n",
      "########## ETA: 13.21s ########## \n",
      "\n",
      "########## ../input/fb3models/20221121-143655-deberta-v3-base-attention_fgm_512 ##########\n",
      "Predicting fold 2...\n",
      "########## ETA: 12.64s ########## \n",
      "\n",
      "########## ../input/fb3models/20221121-143655-deberta-v3-base-attention_fgm_512 ##########\n",
      "Predicting fold 3...\n",
      "########## ETA: 13.51s ########## \n",
      "\n",
      "score: 0.452899  scores: [0.4811456435689267, 0.4475128373931811, 0.4152303639507731, 0.4542057256276823, 0.47347977633523913, 0.4458225341021222]\n",
      "max_len=768\n",
      "########## ../input/fb3models/20221120-072218-deberta-v3-base-attention_fgm ##########\n",
      "Predicting fold 0...\n",
      "########## ETA: 12.84s ########## \n",
      "\n",
      "########## ../input/fb3models/20221120-072218-deberta-v3-base-attention_fgm ##########\n",
      "Predicting fold 1...\n",
      "########## ETA: 12.47s ########## \n",
      "\n",
      "########## ../input/fb3models/20221120-072218-deberta-v3-base-attention_fgm ##########\n",
      "Predicting fold 2...\n",
      "########## ETA: 12.28s ########## \n",
      "\n",
      "########## ../input/fb3models/20221120-072218-deberta-v3-base-attention_fgm ##########\n",
      "Predicting fold 3...\n",
      "########## ETA: 13.33s ########## \n",
      "\n",
      "score: 0.457102  scores: [0.48798883116384423, 0.44872254666199257, 0.42012263776409287, 0.4555113322262075, 0.4788518700393329, 0.45141752961770576]\n",
      "########## ../input/fb3models/v21/ ##########\n",
      "Predicting fold 0...\n",
      "########## ETA: 12.65s ########## \n",
      "\n",
      "########## ../input/fb3models/v21/ ##########\n",
      "Predicting fold 1...\n",
      "########## ETA: 13.06s ########## \n",
      "\n",
      "########## ../input/fb3models/v21/ ##########\n",
      "Predicting fold 2...\n",
      "########## ETA: 11.36s ########## \n",
      "\n",
      "########## ../input/fb3models/v21/ ##########\n",
      "Predicting fold 3...\n",
      "########## ETA: 12.51s ########## \n",
      "\n",
      "score: 0.453517  scores: [0.4855633786238422, 0.44703209067484795, 0.41299155171034585, 0.4534606468878697, 0.4717306751532209, 0.4503226293834107]\n",
      "preprocess\n",
      "########## ../input/fb3models/v112/ ##########\n",
      "Predicting fold 0...\n",
      "########## ETA: 13.67s ########## \n",
      "\n",
      "########## ../input/fb3models/v112/ ##########\n",
      "Predicting fold 1...\n",
      "########## ETA: 13.34s ########## \n",
      "\n",
      "########## ../input/fb3models/v112/ ##########\n",
      "Predicting fold 2...\n",
      "########## ETA: 13.45s ########## \n",
      "\n",
      "########## ../input/fb3models/v112/ ##########\n",
      "Predicting fold 3...\n",
      "########## ETA: 13.90s ########## \n",
      "\n",
      "score: 0.452924  scores: [0.48674365143324116, 0.44712902084312334, 0.41187184815036604, 0.4543429818169591, 0.4707566701582764, 0.44670122183285366]\n",
      "preprocess\n",
      "########## ../input/fb3models/20221117-183420-deberta-v3-base-mean-attention-with-mask ##########\n",
      "Predicting fold 0...\n",
      "########## ETA: 12.81s ########## \n",
      "\n",
      "########## ../input/fb3models/20221117-183420-deberta-v3-base-mean-attention-with-mask ##########\n",
      "Predicting fold 1...\n",
      "########## ETA: 13.70s ########## \n",
      "\n",
      "########## ../input/fb3models/20221117-183420-deberta-v3-base-mean-attention-with-mask ##########\n",
      "Predicting fold 2...\n",
      "########## ETA: 13.12s ########## \n",
      "\n",
      "########## ../input/fb3models/20221117-183420-deberta-v3-base-mean-attention-with-mask ##########\n",
      "Predicting fold 3...\n",
      "########## ETA: 15.70s ########## \n",
      "\n",
      "score: 0.456659  scores: [0.4862241326420343, 0.45220519968314205, 0.41968210267694495, 0.45746617426033787, 0.4755639270504457, 0.44881410849161313]\n",
      "########## ../input/fb3models/v2/ ##########\n",
      "Predicting fold 0...\n",
      "########## ETA: 12.99s ########## \n",
      "\n",
      "########## ../input/fb3models/v2/ ##########\n",
      "Predicting fold 1...\n",
      "########## ETA: 12.84s ########## \n",
      "\n",
      "########## ../input/fb3models/v2/ ##########\n",
      "Predicting fold 2...\n",
      "########## ETA: 13.24s ########## \n",
      "\n",
      "########## ../input/fb3models/v2/ ##########\n",
      "Predicting fold 3...\n",
      "########## ETA: 13.81s ########## \n",
      "\n",
      "score: 0.465777  scores: [0.4974350833689818, 0.4560072906674801, 0.42235798291514465, 0.46613323072377133, 0.4908856845317672, 0.46184042368634104]\n",
      "preprocess\n",
      "########## ../input/fb3-train/ ##########\n",
      "Predicting fold 0...\n",
      "########## ETA: 14.19s ########## \n",
      "\n",
      "########## ../input/fb3-train/ ##########\n",
      "Predicting fold 1...\n",
      "########## ETA: 14.22s ########## \n",
      "\n",
      "########## ../input/fb3-train/ ##########\n",
      "Predicting fold 2...\n",
      "########## ETA: 14.04s ########## \n",
      "\n",
      "########## ../input/fb3-train/ ##########\n",
      "Predicting fold 3...\n",
      "########## ETA: 13.64s ########## \n",
      "\n",
      "score: 0.454263  scores: [0.4842567673971252, 0.44804979963987146, 0.4164547697759723, 0.45478343500954116, 0.4747050515151748, 0.4473257540335437]\n",
      "########## ../input/fb3models/20221114-192943-deberta-v3-base/ ##########\n",
      "Predicting fold 0...\n",
      "########## ETA: 13.43s ########## \n",
      "\n",
      "########## ../input/fb3models/20221114-192943-deberta-v3-base/ ##########\n",
      "Predicting fold 1...\n",
      "########## ETA: 12.90s ########## \n",
      "\n",
      "########## ../input/fb3models/20221114-192943-deberta-v3-base/ ##########\n",
      "Predicting fold 2...\n",
      "########## ETA: 12.54s ########## \n",
      "\n",
      "########## ../input/fb3models/20221114-192943-deberta-v3-base/ ##########\n",
      "Predicting fold 3...\n",
      "########## ETA: 12.76s ########## \n",
      "\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "array([[2.9428473, 2.8288896, 3.1413603, 3.016755 , 2.7340455, 2.7014422],\n",
       "       [2.6816514, 2.4797902, 2.7202091, 2.3856783, 2.155445 , 2.627569 ],\n",
       "       [3.577352 , 3.3825758, 3.56533  , 3.5409017, 3.3748686, 3.2661853]],\n",
       "      dtype=float32)"
      ]
     },
     "execution_count": 14,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "fine_tuned_predictions = []\n",
    "total_weight = 0\n",
    "for cfg in tqdm(fine_tuned_models_cfg):\n",
    "    infer_ = Inferencer(cfg=cfg, inference_weight=cfg.inference_weight)\n",
    "    if cfg.path in [\n",
    "        attention_large_fgm_CFG.path,\n",
    "        attention_fgm_CFG.path, attention_fgm_768_CFG.path, roberta_attention_fgm_CFG.path, attention_fgm_512_CFG.path, \n",
    "        weighted2last_fgm_512_CFG.path, weightedmean2last_fgm_512_CFG.path, weightedmean2last_fgm_1024_CFG.path\n",
    "    ]:\n",
    "        file_type = 'csv'\n",
    "    else:\n",
    "        file_type = 'pkl'\n",
    "\n",
    "    infer_.get_oof_result(file_type)\n",
    "    \n",
    "    test_dataset = TestDataset(cfg, test)\n",
    "    if cfg.path == roberta_attention_fgm_CFG.path:\n",
    "        collate_fn = DataCollatorWithPadding(tokenizer=cfg.tokenizer, max_length=cfg.max_len, padding='max_length')\n",
    "        print('roberta=512')\n",
    "    elif '512' in cfg.name:\n",
    "        collate_fn = DataCollatorWithPadding(tokenizer=cfg.tokenizer, padding='max_length', max_length=512)\n",
    "        print('max_len=512') \n",
    "    elif '768' in cfg.name:\n",
    "        collate_fn = DataCollatorWithPadding(tokenizer=cfg.tokenizer, padding='max_length', max_length=768)\n",
    "        print('max_len=768') \n",
    "    elif '1024' in cfg.name:\n",
    "        collate_fn = DataCollatorWithPadding(tokenizer=cfg.tokenizer, padding='max_length', max_length=1024)\n",
    "        print('max_len=1024')\n",
    "    else:\n",
    "        collate_fn = DataCollatorWithPadding(tokenizer=cfg.tokenizer, padding='longest')\n",
    "    test_loader = DataLoader(\n",
    "        test_dataset,\n",
    "        batch_size=12,\n",
    "        shuffle=False,\n",
    "        collate_fn=collate_fn,\n",
    "        #num_workers=1,\n",
    "        num_workers=0,\n",
    "        pin_memory=True, \n",
    "        drop_last=False)\n",
    "    prediction = infer_.predict(test_loader, device) * cfg.inference_weight\n",
    "    \n",
    "    fine_tuned_predictions.append(prediction)\n",
    "    total_weight += cfg.inference_weight\n",
    "    \n",
    "    del infer_, test_dataset, test_loader, prediction; gc.collect; torch.cuda.empty_cache();\n",
    "    \n",
    "final_fine_tuned_predictions = np.sum(fine_tuned_predictions, axis=0)/total_weight\n",
    "final_fine_tuned_predictions"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "id": "c1363130",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2022-11-29T01:24:25.549902Z",
     "iopub.status.busy": "2022-11-29T01:24:25.549573Z",
     "iopub.status.idle": "2022-11-29T01:24:25.555737Z",
     "shell.execute_reply": "2022-11-29T01:24:25.554693Z"
    },
    "papermill": {
     "duration": 0.017866,
     "end_time": "2022-11-29T01:24:25.558676",
     "exception": false,
     "start_time": "2022-11-29T01:24:25.540810",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "10.1"
      ]
     },
     "execution_count": 15,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "total_weight"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "id": "3334071c",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2022-11-29T01:24:25.575765Z",
     "iopub.status.busy": "2022-11-29T01:24:25.575478Z",
     "iopub.status.idle": "2022-11-29T01:24:25.584384Z",
     "shell.execute_reply": "2022-11-29T01:24:25.583371Z"
    },
    "papermill": {
     "duration": 0.019556,
     "end_time": "2022-11-29T01:24:25.586401",
     "exception": false,
     "start_time": "2022-11-29T01:24:25.566845",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "final_fine_tuned_predictions = np.clip(final_fine_tuned_predictions, 1, 5)\n",
    "test[TARGET_COLS] = final_fine_tuned_predictions"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "8a88b34e",
   "metadata": {
    "papermill": {
     "duration": 0.007634,
     "end_time": "2022-11-29T01:24:25.601763",
     "exception": false,
     "start_time": "2022-11-29T01:24:25.594129",
     "status": "completed"
    },
    "tags": []
   },
   "source": [
    "# submit"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "id": "ca290d24",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2022-11-29T01:24:25.619533Z",
     "iopub.status.busy": "2022-11-29T01:24:25.618134Z",
     "iopub.status.idle": "2022-11-29T01:24:25.659949Z",
     "shell.execute_reply": "2022-11-29T01:24:25.659064Z"
    },
    "papermill": {
     "duration": 0.052575,
     "end_time": "2022-11-29T01:24:25.662089",
     "exception": false,
     "start_time": "2022-11-29T01:24:25.609514",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>text_id</th>\n",
       "      <th>cohesion</th>\n",
       "      <th>syntax</th>\n",
       "      <th>vocabulary</th>\n",
       "      <th>phraseology</th>\n",
       "      <th>grammar</th>\n",
       "      <th>conventions</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>0000C359D63E</td>\n",
       "      <td>2.942847</td>\n",
       "      <td>2.828890</td>\n",
       "      <td>3.141360</td>\n",
       "      <td>3.016755</td>\n",
       "      <td>2.734046</td>\n",
       "      <td>2.701442</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>000BAD50D026</td>\n",
       "      <td>2.681651</td>\n",
       "      <td>2.479790</td>\n",
       "      <td>2.720209</td>\n",
       "      <td>2.385678</td>\n",
       "      <td>2.155445</td>\n",
       "      <td>2.627569</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>00367BB2546B</td>\n",
       "      <td>3.577352</td>\n",
       "      <td>3.382576</td>\n",
       "      <td>3.565330</td>\n",
       "      <td>3.540902</td>\n",
       "      <td>3.374869</td>\n",
       "      <td>3.266185</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "        text_id  cohesion    syntax  vocabulary  phraseology   grammar  conventions\n",
       "0  0000C359D63E  2.942847  2.828890    3.141360     3.016755  2.734046     2.701442\n",
       "1  000BAD50D026  2.681651  2.479790    2.720209     2.385678  2.155445     2.627569\n",
       "2  00367BB2546B  3.577352  3.382576    3.565330     3.540902  3.374869     3.266185"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "submission = pd.read_csv(SUBMISSION_PATH)\n",
    "submission = submission.drop(columns=TARGET_COLS).merge(test[['text_id'] + TARGET_COLS], on='text_id', how='left')\n",
    "display(submission.head())\n",
    "submission[['text_id'] + TARGET_COLS].to_csv('submission.csv', index=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "510e67be",
   "metadata": {
    "papermill": {
     "duration": 0.007713,
     "end_time": "2022-11-29T01:24:25.678120",
     "exception": false,
     "start_time": "2022-11-29T01:24:25.670407",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.12"
  },
  "papermill": {
   "default_parameters": {},
   "duration": 704.354533,
   "end_time": "2022-11-29T01:24:28.505908",
   "environment_variables": {},
   "exception": null,
   "input_path": "__notebook__.ipynb",
   "output_path": "__notebook__.ipynb",
   "parameters": {},
   "start_time": "2022-11-29T01:12:44.151375",
   "version": "2.3.4"
  },
  "widgets": {
   "application/vnd.jupyter.widget-state+json": {
    "state": {
     "05f4f5c62e854e9b867f9f1360baac38": {
      "model_module": "@jupyter-widgets/controls",
      "model_module_version": "1.5.0",
      "model_name": "DescriptionStyleModel",
      "state": {
       "_model_module": "@jupyter-widgets/controls",
       "_model_module_version": "1.5.0",
       "_model_name": "DescriptionStyleModel",
       "_view_count": null,
       "_view_module": "@jupyter-widgets/base",
       "_view_module_version": "1.2.0",
       "_view_name": "StyleView",
       "description_width": ""
      }
     },
     "36ad1bf24d6c435a880fe3304243d849": {
      "model_module": "@jupyter-widgets/controls",
      "model_module_version": "1.5.0",
      "model_name": "ProgressStyleModel",
      "state": {
       "_model_module": "@jupyter-widgets/controls",
       "_model_module_version": "1.5.0",
       "_model_name": "ProgressStyleModel",
       "_view_count": null,
       "_view_module": "@jupyter-widgets/base",
       "_view_module_version": "1.2.0",
       "_view_name": "StyleView",
       "bar_color": null,
       "description_width": ""
      }
     },
     "3a2509d36e4043da9d69c0d8542ca1d2": {
      "model_module": "@jupyter-widgets/base",
      "model_module_version": "1.2.0",
      "model_name": "LayoutModel",
      "state": {
       "_model_module": "@jupyter-widgets/base",
       "_model_module_version": "1.2.0",
       "_model_name": "LayoutModel",
       "_view_count": null,
       "_view_module": "@jupyter-widgets/base",
       "_view_module_version": "1.2.0",
       "_view_name": "LayoutView",
       "align_content": null,
       "align_items": null,
       "align_self": null,
       "border": null,
       "bottom": null,
       "display": null,
       "flex": null,
       "flex_flow": null,
       "grid_area": null,
       "grid_auto_columns": null,
       "grid_auto_flow": null,
       "grid_auto_rows": null,
       "grid_column": null,
       "grid_gap": null,
       "grid_row": null,
       "grid_template_areas": null,
       "grid_template_columns": null,
       "grid_template_rows": null,
       "height": null,
       "justify_content": null,
       "justify_items": null,
       "left": null,
       "margin": null,
       "max_height": null,
       "max_width": null,
       "min_height": null,
       "min_width": null,
       "object_fit": null,
       "object_position": null,
       "order": null,
       "overflow": null,
       "overflow_x": null,
       "overflow_y": null,
       "padding": null,
       "right": null,
       "top": null,
       "visibility": null,
       "width": null
      }
     },
     "436fa3a0fabf4db68c09c5f012b465ea": {
      "model_module": "@jupyter-widgets/controls",
      "model_module_version": "1.5.0",
      "model_name": "HTMLModel",
      "state": {
       "_dom_classes": [],
       "_model_module": "@jupyter-widgets/controls",
       "_model_module_version": "1.5.0",
       "_model_name": "HTMLModel",
       "_view_count": null,
       "_view_module": "@jupyter-widgets/controls",
       "_view_module_version": "1.5.0",
       "_view_name": "HTMLView",
       "description": "",
       "description_tooltip": null,
       "layout": "IPY_MODEL_eaf0a16839254877ac0f8f63f99faec8",
       "placeholder": "​",
       "style": "IPY_MODEL_05f4f5c62e854e9b867f9f1360baac38",
       "value": "100%"
      }
     },
     "52616eb34b8c4a11bf5f47c899e87136": {
      "model_module": "@jupyter-widgets/controls",
      "model_module_version": "1.5.0",
      "model_name": "HTMLModel",
      "state": {
       "_dom_classes": [],
       "_model_module": "@jupyter-widgets/controls",
       "_model_module_version": "1.5.0",
       "_model_name": "HTMLModel",
       "_view_count": null,
       "_view_module": "@jupyter-widgets/controls",
       "_view_module_version": "1.5.0",
       "_view_name": "HTMLView",
       "description": "",
       "description_tooltip": null,
       "layout": "IPY_MODEL_dcfa961439864534b9994ebbf7e62886",
       "placeholder": "​",
       "style": "IPY_MODEL_b15d1d56b38342a985984a36ca4e733a",
       "value": " 11/11 [10:54&lt;00:00, 54.07s/it]"
      }
     },
     "7f1574d185fe410ca0daf8b073610d8a": {
      "model_module": "@jupyter-widgets/base",
      "model_module_version": "1.2.0",
      "model_name": "LayoutModel",
      "state": {
       "_model_module": "@jupyter-widgets/base",
       "_model_module_version": "1.2.0",
       "_model_name": "LayoutModel",
       "_view_count": null,
       "_view_module": "@jupyter-widgets/base",
       "_view_module_version": "1.2.0",
       "_view_name": "LayoutView",
       "align_content": null,
       "align_items": null,
       "align_self": null,
       "border": null,
       "bottom": null,
       "display": null,
       "flex": null,
       "flex_flow": null,
       "grid_area": null,
       "grid_auto_columns": null,
       "grid_auto_flow": null,
       "grid_auto_rows": null,
       "grid_column": null,
       "grid_gap": null,
       "grid_row": null,
       "grid_template_areas": null,
       "grid_template_columns": null,
       "grid_template_rows": null,
       "height": null,
       "justify_content": null,
       "justify_items": null,
       "left": null,
       "margin": null,
       "max_height": null,
       "max_width": null,
       "min_height": null,
       "min_width": null,
       "object_fit": null,
       "object_position": null,
       "order": null,
       "overflow": null,
       "overflow_x": null,
       "overflow_y": null,
       "padding": null,
       "right": null,
       "top": null,
       "visibility": null,
       "width": null
      }
     },
     "b15d1d56b38342a985984a36ca4e733a": {
      "model_module": "@jupyter-widgets/controls",
      "model_module_version": "1.5.0",
      "model_name": "DescriptionStyleModel",
      "state": {
       "_model_module": "@jupyter-widgets/controls",
       "_model_module_version": "1.5.0",
       "_model_name": "DescriptionStyleModel",
       "_view_count": null,
       "_view_module": "@jupyter-widgets/base",
       "_view_module_version": "1.2.0",
       "_view_name": "StyleView",
       "description_width": ""
      }
     },
     "ca2ae9b00c26451fa9393c8fef61412b": {
      "model_module": "@jupyter-widgets/controls",
      "model_module_version": "1.5.0",
      "model_name": "HBoxModel",
      "state": {
       "_dom_classes": [],
       "_model_module": "@jupyter-widgets/controls",
       "_model_module_version": "1.5.0",
       "_model_name": "HBoxModel",
       "_view_count": null,
       "_view_module": "@jupyter-widgets/controls",
       "_view_module_version": "1.5.0",
       "_view_name": "HBoxView",
       "box_style": "",
       "children": [
        "IPY_MODEL_436fa3a0fabf4db68c09c5f012b465ea",
        "IPY_MODEL_ed8f6f4163764af0ae53c13aa00df88a",
        "IPY_MODEL_52616eb34b8c4a11bf5f47c899e87136"
       ],
       "layout": "IPY_MODEL_3a2509d36e4043da9d69c0d8542ca1d2"
      }
     },
     "dcfa961439864534b9994ebbf7e62886": {
      "model_module": "@jupyter-widgets/base",
      "model_module_version": "1.2.0",
      "model_name": "LayoutModel",
      "state": {
       "_model_module": "@jupyter-widgets/base",
       "_model_module_version": "1.2.0",
       "_model_name": "LayoutModel",
       "_view_count": null,
       "_view_module": "@jupyter-widgets/base",
       "_view_module_version": "1.2.0",
       "_view_name": "LayoutView",
       "align_content": null,
       "align_items": null,
       "align_self": null,
       "border": null,
       "bottom": null,
       "display": null,
       "flex": null,
       "flex_flow": null,
       "grid_area": null,
       "grid_auto_columns": null,
       "grid_auto_flow": null,
       "grid_auto_rows": null,
       "grid_column": null,
       "grid_gap": null,
       "grid_row": null,
       "grid_template_areas": null,
       "grid_template_columns": null,
       "grid_template_rows": null,
       "height": null,
       "justify_content": null,
       "justify_items": null,
       "left": null,
       "margin": null,
       "max_height": null,
       "max_width": null,
       "min_height": null,
       "min_width": null,
       "object_fit": null,
       "object_position": null,
       "order": null,
       "overflow": null,
       "overflow_x": null,
       "overflow_y": null,
       "padding": null,
       "right": null,
       "top": null,
       "visibility": null,
       "width": null
      }
     },
     "eaf0a16839254877ac0f8f63f99faec8": {
      "model_module": "@jupyter-widgets/base",
      "model_module_version": "1.2.0",
      "model_name": "LayoutModel",
      "state": {
       "_model_module": "@jupyter-widgets/base",
       "_model_module_version": "1.2.0",
       "_model_name": "LayoutModel",
       "_view_count": null,
       "_view_module": "@jupyter-widgets/base",
       "_view_module_version": "1.2.0",
       "_view_name": "LayoutView",
       "align_content": null,
       "align_items": null,
       "align_self": null,
       "border": null,
       "bottom": null,
       "display": null,
       "flex": null,
       "flex_flow": null,
       "grid_area": null,
       "grid_auto_columns": null,
       "grid_auto_flow": null,
       "grid_auto_rows": null,
       "grid_column": null,
       "grid_gap": null,
       "grid_row": null,
       "grid_template_areas": null,
       "grid_template_columns": null,
       "grid_template_rows": null,
       "height": null,
       "justify_content": null,
       "justify_items": null,
       "left": null,
       "margin": null,
       "max_height": null,
       "max_width": null,
       "min_height": null,
       "min_width": null,
       "object_fit": null,
       "object_position": null,
       "order": null,
       "overflow": null,
       "overflow_x": null,
       "overflow_y": null,
       "padding": null,
       "right": null,
       "top": null,
       "visibility": null,
       "width": null
      }
     },
     "ed8f6f4163764af0ae53c13aa00df88a": {
      "model_module": "@jupyter-widgets/controls",
      "model_module_version": "1.5.0",
      "model_name": "FloatProgressModel",
      "state": {
       "_dom_classes": [],
       "_model_module": "@jupyter-widgets/controls",
       "_model_module_version": "1.5.0",
       "_model_name": "FloatProgressModel",
       "_view_count": null,
       "_view_module": "@jupyter-widgets/controls",
       "_view_module_version": "1.5.0",
       "_view_name": "ProgressView",
       "bar_style": "success",
       "description": "",
       "description_tooltip": null,
       "layout": "IPY_MODEL_7f1574d185fe410ca0daf8b073610d8a",
       "max": 11.0,
       "min": 0.0,
       "orientation": "horizontal",
       "style": "IPY_MODEL_36ad1bf24d6c435a880fe3304243d849",
       "value": 11.0
      }
     }
    },
    "version_major": 2,
    "version_minor": 0
   }
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}