fb3-submit-ensemble.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "ac16b8ff",
   "metadata": {
    "_cell_guid": "b1076dfc-b9ad-4769-8c92-a6c4dae69d19",
    "_uuid": "8f2839f25d086af736a60e9eeb907d3b93b6e0e5",
    "execution": {
     "iopub.execute_input": "2022-11-29T18:17:33.821031Z",
     "iopub.status.busy": "2022-11-29T18:17:33.820521Z",
     "iopub.status.idle": "2022-11-29T18:18:06.484980Z",
     "shell.execute_reply": "2022-11-29T18:18:06.483757Z"
    },
    "papermill": {
     "duration": 32.678563,
     "end_time": "2022-11-29T18:18:06.487405",
     "exception": false,
     "start_time": "2022-11-29T18:17:33.808842",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "tokenizers.__version__: 0.12.1\n",
      "transformers.__version__: 4.20.1\n",
      "device: cuda\n"
     ]
    }
   ],
   "source": [
    "# ====================================================\n",
    "# Library\n",
    "# ====================================================\n",
    "import os\n",
    "import gc\n",
    "import re\n",
    "import ast\n",
    "import sys\n",
    "import copy\n",
    "import json\n",
    "import time\n",
    "import math \n",
    "import string\n",
    "import pickle\n",
    "import random\n",
    "import joblib\n",
    "import itertools\n",
    "from IPython. display import clear_output\n",
    "\n",
    "import scipy as sp\n",
    "import numpy as np\n",
    "import pandas as pd\n",
    "pd.set_option('display.max_rows', 500)\n",
    "pd.set_option('display.max_columns', 500)\n",
    "pd.set_option('display.width', 1000)\n",
    "from tqdm.auto import tqdm\n",
    "from sklearn.metrics import mean_squared_error\n",
    "from sklearn.model_selection import StratifiedKFold, GroupKFold, KFold\n",
    "from nltk.corpus import stopwords\n",
    "from nltk.tokenize import word_tokenize\n",
    "\n",
    "import torch\n",
    "import torch.nn as nn\n",
    "from torch.nn import Parameter\n",
    "import torch.nn.functional as F\n",
    "from torch.optim import Adam, SGD, AdamW\n",
    "from torch.utils.data import DataLoader, Dataset\n",
    "\n",
    "#os.system('pip install iterative-stratification==0.1.7')\n",
    "#from iterstrat.ml_stratifiers import MultilabelStratifiedKFold\n",
    "sys.path.append('../input/iterativestratification')\n",
    "from iterstrat.ml_stratifiers import MultilabelStratifiedKFold\n",
    "\n",
    "os.system('pip uninstall -y transformers')\n",
    "os.system('pip uninstall -y tokenizers')\n",
    "os.system('python -m pip install --no-index --find-links=../input/fb3-my-pip-wheels transformers')\n",
    "os.system('python -m pip install --no-index --find-links=../input/fb3-my-pip-wheels tokenizers')\n",
    "import tokenizers\n",
    "import transformers\n",
    "from transformers import AutoTokenizer, AutoModel, AutoConfig\n",
    "from transformers import get_linear_schedule_with_warmup, get_cosine_schedule_with_warmup\n",
    "from transformers import DataCollatorWithPadding\n",
    "%env TOKENIZERS_PARALLELISM=false\n",
    "\n",
    "clear_output()\n",
    "device = torch.device('cuda' if torch.cuda.is_available() else 'cpu')\n",
    "print(f\"tokenizers.__version__: {tokenizers.__version__}\")\n",
    "print(f\"transformers.__version__: {transformers.__version__}\")\n",
    "print('device:', device)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "40fd8c93",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2022-11-29T18:18:06.506770Z",
     "iopub.status.busy": "2022-11-29T18:18:06.505519Z",
     "iopub.status.idle": "2022-11-29T18:18:06.511164Z",
     "shell.execute_reply": "2022-11-29T18:18:06.510323Z"
    },
    "papermill": {
     "duration": 0.016956,
     "end_time": "2022-11-29T18:18:06.513119",
     "exception": false,
     "start_time": "2022-11-29T18:18:06.496163",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "BASE_PATH = '/kaggle/input/feedback-prize-english-language-learning'\n",
    "SUBMISSION_PATH = os.path.join(BASE_PATH, 'sample_submission.csv')\n",
    "TRAIN_PATH = os.path.join(BASE_PATH, 'train.csv')\n",
    "TEST_PATH = os.path.join(BASE_PATH, 'test.csv')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "93615e7d",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2022-11-29T18:18:06.531175Z",
     "iopub.status.busy": "2022-11-29T18:18:06.530881Z",
     "iopub.status.idle": "2022-11-29T18:18:06.551771Z",
     "shell.execute_reply": "2022-11-29T18:18:06.550929Z"
    },
    "papermill": {
     "duration": 0.032208,
     "end_time": "2022-11-29T18:18:06.553774",
     "exception": false,
     "start_time": "2022-11-29T18:18:06.521566",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "class MeanPooling(nn.Module):\n",
    "    def __init__(self):\n",
    "        super(MeanPooling, self).__init__()\n",
    "    \n",
    "    def forward(self, last_hidden_state, attention_mask):\n",
    "        input_mask_expanded = attention_mask.unsqueeze(-1).expand(last_hidden_state.size()).float()\n",
    "        sum_embeddings = torch.sum(last_hidden_state * input_mask_expanded, 1)\n",
    "        sum_mask = torch.clamp(input_mask_expanded.sum(1), min=1e-9)\n",
    "        mean_embeddings = sum_embeddings / sum_mask\n",
    "        return mean_embeddings\n",
    "\n",
    "class WeightedLayerPooling(nn.Module):\n",
    "    def __init__(self, num_hidden_layers, layer_start: int = 9, layer_weights = None):\n",
    "        super(WeightedLayerPooling, self).__init__()\n",
    "        self.layer_start = layer_start\n",
    "        self.num_hidden_layers = num_hidden_layers\n",
    "        self.layer_weights = layer_weights if layer_weights is not None \\\n",
    "            else nn.Parameter(\n",
    "                torch.tensor([1] * (num_hidden_layers+1 - layer_start), dtype=torch.float))\n",
    "        \n",
    "    def forward(self, all_hidden_states):\n",
    "        all_layer_embedding = all_hidden_states[self.layer_start:, :, :, :]\n",
    "        weight_factor = self.layer_weights.unsqueeze(-1).unsqueeze(-1).unsqueeze(-1).expand(all_layer_embedding.size())\n",
    "        weighted_average = (weight_factor*all_layer_embedding).sum(dim=0) / self.layer_weights.sum()\n",
    "        return weighted_average\n",
    "\n",
    "class AttentionHead(nn.Module):\n",
    "    def __init__(self, in_features, hidden_dim):\n",
    "        super().__init__()\n",
    "        self.in_features = in_features\n",
    "        self.middle_features = hidden_dim\n",
    "        self.W = nn.Linear(in_features, hidden_dim)\n",
    "        self.V = nn.Linear(hidden_dim, 1)\n",
    "        self.out_features = hidden_dim\n",
    "\n",
    "    def forward(self, features, *args):\n",
    "        att = torch.tanh(self.W(features))\n",
    "        score = self.V(att)\n",
    "        attention_weights = torch.softmax(score, dim=1)\n",
    "        context_vector = attention_weights * features\n",
    "        context_vector = torch.sum(context_vector, dim=1)\n",
    "\n",
    "        return context_vector\n",
    "\n",
    "class AttentionPooling(nn.Module):\n",
    "    def __init__(self, in_dim):\n",
    "        super().__init__()\n",
    "        self.attention = nn.Sequential(\n",
    "        nn.Linear(in_dim, in_dim),\n",
    "        nn.LayerNorm(in_dim),\n",
    "        nn.GELU(),\n",
    "        nn.Linear(in_dim, 1),\n",
    "        )\n",
    "\n",
    "    def forward(self, last_hidden_state, attention_mask):\n",
    "        w = self.attention(last_hidden_state).float()\n",
    "        w[attention_mask==0]=float('-inf')\n",
    "        w = torch.softmax(w,1)\n",
    "        attention_embeddings = torch.sum(w * last_hidden_state, dim=1)\n",
    "        return attention_embeddings\n",
    "    \n",
    "def GlobalAveragePool1d(x):\n",
    "    return F.avg_pool1d(x, x.size()[-1]).squeeze(-1)\n",
    "\n",
    "def GlobalMaxPool1d(x):\n",
    "    return F.max_pool1d(x, x.size()[-1]).squeeze(-1)\n",
    "\n",
    "def Conv1dReg(x, in_channels, out_channels, kernel_size, device):\n",
    "    out = nn.Conv1d(in_channels, out_channels, kernel_size, padding='same', stride=1, device=device)(x)\n",
    "    out = nn.BatchNorm1d(out_channels, device=device)(out)\n",
    "    out = F.relu(out)\n",
    "    return out\n",
    "\n",
    "class MultiSampleDropout(nn.Module):\n",
    "    def __init__(self, fc, num_dropout, prob_dropout):\n",
    "        super(MultiSampleDropout, self).__init__()\n",
    "        self.dropout = nn.Dropout\n",
    "        self.num_dropout = num_dropout\n",
    "        self.prob_dropout = prob_dropout\n",
    "        self.classifier = fc\n",
    "    def forward(self, out):\n",
    "        if not type(self.prob_dropout) in [float, int]:            \n",
    "            fcs = [self.classifier(self.dropout(p)(out)) for p in self.prob_dropout]\n",
    "        else:\n",
    "            fcs = [self.classifier(self.dropout(self.prob_dropout)(out)) for _ in range(self.num_dropout)]\n",
    "        \n",
    "        return torch.mean(torch.stack(fcs, dim=0), dim=0)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "1ed3912e",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2022-11-29T18:18:06.572149Z",
     "iopub.status.busy": "2022-11-29T18:18:06.571865Z",
     "iopub.status.idle": "2022-11-29T18:18:06.634425Z",
     "shell.execute_reply": "2022-11-29T18:18:06.633540Z"
    },
    "papermill": {
     "duration": 0.074401,
     "end_time": "2022-11-29T18:18:06.636531",
     "exception": false,
     "start_time": "2022-11-29T18:18:06.562130",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "# ====================================================\n",
    "# Model class\n",
    "# ====================================================\n",
    "class FB3Model(nn.Module):\n",
    "    def __init__(self, cfg, config_path=None, pretrained=False):\n",
    "        super().__init__()\n",
    "        self.cfg = cfg \n",
    "        if config_path is None:\n",
    "            self.config = AutoConfig.from_pretrained(cfg.model, output_hidden_states=True)\n",
    "            # Turn off dropouts.\n",
    "            self.config.hidden_dropout = 0.\n",
    "            self.config.hidden_dropout_prob = 0.\n",
    "            self.config.attention_dropout = 0.\n",
    "            self.config.attention_probs_dropout_prob = 0.\n",
    "            #LOGGER.info(self.config)\n",
    "        else:\n",
    "            self.config = torch.load(config_path)\n",
    "        \n",
    "        if pretrained:\n",
    "            self.deberta_v3 = AutoModel.from_pretrained(cfg.model, config=self.config)\n",
    "        else:\n",
    "            self.deberta_v3 = AutoModel.from_config(self.config)\n",
    "\n",
    "        if self.cfg.reinit_last_layer:\n",
    "            # Re-init last layer of deberta.\n",
    "            for module in self.deberta_v3.encoder.layer[-1:].modules():\n",
    "                self._init_weights(module)\n",
    "        self.deberta_v3.gradient_checkpointing_enable()\n",
    "\n",
    "    def _init_weights(self, module):\n",
    "        if isinstance(module, nn.Linear):\n",
    "            module.weight.data.normal_(mean=0.0, std=self.config.initializer_range)\n",
    "            #nn.init.xavier_uniform_(module.weight.data, gain=1.0)\n",
    "            if module.bias is not None:\n",
    "                module.bias.data.zero_()\n",
    "        elif isinstance(module, nn.Embedding):\n",
    "            module.weight.data.normal_(mean=0.0, std=self.config.initializer_range)\n",
    "            if module.padding_idx is not None:\n",
    "                module.weight.data[module.padding_idx].zero_()\n",
    "        elif isinstance(module, nn.LayerNorm):\n",
    "            module.bias.data.zero_()\n",
    "            module.weight.data.fill_(1.0)\n",
    "\n",
    "class WMPoolModel(FB3Model):\n",
    "    def __init__(self, cfg, config_path=None, pretrained=False):\n",
    "        super().__init__(cfg, config_path=config_path, pretrained=pretrained)\n",
    "\n",
    "        # Poolings.\n",
    "        self.mean_head = MeanPooling()\n",
    "        self.wpool_head = WeightedLayerPooling(self.config.num_hidden_layers, layer_start=12)\n",
    "\n",
    "        self.fc_out = nn.Linear(self.config.hidden_size, cfg.num_target)\n",
    "        self._init_weights(self.fc_out)\n",
    "        \n",
    "        self.layer_norm = nn.LayerNorm(self.config.hidden_size)\n",
    "        self.qa_output = torch.nn.Linear(self.config.hidden_size, 2)\n",
    "        self.attention_head = AttentionHead(self.config.hidden_size*4, self.config.hidden_size)\n",
    "        \n",
    "    def forward(self, x):\n",
    "        pt_out = self.deberta_v3(**x)\n",
    "        all_hidden_states = torch.stack(pt_out.hidden_states)\n",
    "        # Weighted pooling of last n layers.\n",
    "        logits = self.wpool_head(all_hidden_states)[:, 0] # Bx768\n",
    "        y_hat = self.fc_out(logits)\n",
    "        return y_hat\n",
    "\n",
    "class MultiPoolModel(FB3Model):\n",
    "    def __init__(self, cfg, config_path=None, pretrained=False, pool='mean'):\n",
    "        super().__init__(cfg, config_path=config_path, pretrained=pretrained)\n",
    "        \n",
    "        # Define model layers.\n",
    "        self.pool_name = cfg.pool_head\n",
    "        self.fc = nn.Linear(self.config.hidden_size, self.cfg.num_target)\n",
    "        if cfg.pool_head in ['mean', 'attention', 'weighted']:\n",
    "            self.pool = self._pool_layer(cfg.pool_head)\n",
    "        elif '-' in cfg.pool_head:\n",
    "            pools = cfg.pool_head.split('-')\n",
    "            self.pool = nn.ModuleList([])\n",
    "            for pool_ in pools:\n",
    "                self.pool.append(self._pool_layer(pool_))\n",
    "            self.fc = nn.Linear(self.config.hidden_size * len(self.pool), self.cfg.num_target)\n",
    "        self._init_weights(self.fc)\n",
    "        \n",
    "        # Multi-sample dropout.\n",
    "        self.multi_dropout = MultiSampleDropout(self.fc, self.cfg.num_dropout, self.cfg.prob_dropout)\n",
    "    \n",
    "    def _pool_layer(self, pool_name):\n",
    "        assert pool_name in ['mean', 'attention', 'weighted']\n",
    "        if pool_name == 'mean':\n",
    "            pool = MeanPooling()\n",
    "        elif pool_name == 'attention':\n",
    "            pool = AttentionHead(self.config.hidden_size, self.config.hidden_size)\n",
    "        elif pool_name == 'weighted':\n",
    "            pool = WeightedLayerPooling(\n",
    "                self.config.num_hidden_layers, \n",
    "                layer_start=9,\n",
    "                layer_weights=None)\n",
    "        return pool\n",
    "    \n",
    "    def _pool_feature(self, pool, pool_name, pt_outputs, attention_mask):\n",
    "        assert pool_name in ['mean', 'attention', 'weighted']\n",
    "        last_hidden_state = pt_outputs.last_hidden_state #batch_size x max_len x hidden_size\n",
    "        all_hidden_states = torch.stack(pt_outputs.hidden_states) #num_layer x batch_size x max_len x hidden_size\n",
    "        \n",
    "        if pool_name == 'mean':\n",
    "            pool_feature = pool(last_hidden_state, attention_mask)\n",
    "        elif pool_name == 'attention':\n",
    "            pool_feature = pool(last_hidden_state)\n",
    "        elif pool_name == 'weighted':\n",
    "            # Take the CLS token only.\n",
    "            pool_feature = pool(all_hidden_states)[:, 0]\n",
    "        return pool_feature\n",
    "\n",
    "    def feature(self, x):\n",
    "        pt_outputs = self.deberta_v3(**x)\n",
    "        \n",
    "        # Pooling feat.\n",
    "        if type(self.pool) == nn.ModuleList:\n",
    "            pool_features = []\n",
    "            pool_names = self.pool_name.split('-')\n",
    "            \n",
    "            for pool_name, pool in zip(pool_names, self.pool):\n",
    "                pool_features.append(self._pool_feature(pool, pool_name, pt_outputs, x['attention_mask']))\n",
    "            pool_features = torch.cat(pool_features, dim=1)\n",
    "        else:\n",
    "            pool_features = self._pool_feature(self.pool, self.pool_name, pt_outputs, x['attention_mask'])\n",
    "        return pool_features\n",
    "    \n",
    "    def forward(self, x):\n",
    "        feature = self.feature(x)\n",
    "        if self.cfg.use_dropout and self.training:\n",
    "            y_hat = self.multi_dropout(feature)\n",
    "        else:\n",
    "            y_hat = self.fc(feature)\n",
    "        return y_hat\n",
    "\n",
    "\n",
    "class Attention4Model(FB3Model):\n",
    "    def __init__(self, cfg, config_path=None, pretrained=False):\n",
    "        super().__init__(cfg, config_path=config_path, pretrained=pretrained)\n",
    "        \n",
    "        self.head = AttentionHead(self.config.hidden_size*4, self.config.hidden_size)\n",
    "        self.fc_out = nn.Linear(self.config.hidden_size*4*2, self.cfg.num_target)\n",
    "        self._init_weights(self.fc_out)\n",
    "\n",
    "    def forward(self, x):\n",
    "        pt_out = self.deberta_v3(**x)\n",
    "        \n",
    "        all_hidden_states = torch.stack(pt_out.hidden_states)\n",
    "        cat_over_last_layers = torch.cat(\n",
    "            (all_hidden_states[-1], all_hidden_states[-2], all_hidden_states[-3], all_hidden_states[-4]),\n",
    "            -1)\n",
    "        # [CLS] embedding.\n",
    "        cls_pooling = cat_over_last_layers[:, 0]   \n",
    "        # Concat of 4 last layers.\n",
    "        head_logits = self.head(cat_over_last_layers)\n",
    "\n",
    "        if self.cfg.use_dropout and self.training:\n",
    "            y_hat = self.multi_dropout(torch.cat([head_logits, cls_pooling], -1))\n",
    "        else:\n",
    "            y_hat = self.fc_out(torch.cat([head_logits, cls_pooling], -1))\n",
    "\n",
    "        return y_hat\n",
    "\n",
    "######################################\n",
    "\n",
    "class FB3Model1(nn.Module):\n",
    "    def __init__(self, cfg, config_path=None, pretrained=False):\n",
    "        super().__init__()\n",
    "        self.cfg = cfg \n",
    "        if config_path is None:\n",
    "            self.config = AutoConfig.from_pretrained(cfg.model, output_hidden_states=True)\n",
    "            # Turn off dropouts.\n",
    "            self.config.hidden_dropout = 0.\n",
    "            self.config.hidden_dropout_prob = 0.\n",
    "            self.config.attention_dropout = 0.\n",
    "            self.config.attention_probs_dropout_prob = 0.\n",
    "            #LOGGER.info(self.config)\n",
    "        else:\n",
    "            self.config = torch.load(config_path)\n",
    "        \n",
    "        if pretrained:\n",
    "            self.model = AutoModel.from_pretrained(cfg.model, config=self.config)\n",
    "        else:\n",
    "            self.model = AutoModel.from_config(self.config)\n",
    "\n",
    "        if self.cfg.reinit_last_layer:\n",
    "            # Re-init last layer of deberta.\n",
    "            for module in self.model.encoder.layer[-1:].modules():\n",
    "                self._init_weights(module)\n",
    "        self.model.gradient_checkpointing_enable()\n",
    "\n",
    "    def _init_weights(self, module):\n",
    "        if isinstance(module, nn.Linear):\n",
    "            module.weight.data.normal_(mean=0.0, std=self.config.initializer_range)\n",
    "            if module.bias is not None:\n",
    "                module.bias.data.zero_()\n",
    "        elif isinstance(module, nn.Embedding):\n",
    "            module.weight.data.normal_(mean=0.0, std=self.config.initializer_range)\n",
    "            if module.padding_idx is not None:\n",
    "                module.weight.data[module.padding_idx].zero_()\n",
    "        elif isinstance(module, nn.LayerNorm):\n",
    "            module.bias.data.zero_()\n",
    "            module.weight.data.fill_(1.0)\n",
    "            \n",
    "class WeightedAttentionModel(FB3Model1):\n",
    "    def __init__(self, cfg, config_path=None, pretrained=False):\n",
    "        super().__init__(cfg, config_path=config_path, pretrained=pretrained)\n",
    "\n",
    "        self.weighted_pool = WeightedLayerPooling(\n",
    "            self.config.num_hidden_layers, layer_start=9, layer_weights=None)\n",
    "        self.att_pool = AttentionPooling(self.config.hidden_size)\n",
    "\n",
    "        self.fc_out = nn.Linear(self.config.hidden_size*2, cfg.num_target)\n",
    "        self._init_weights(self.fc_out)\n",
    "\n",
    "    def forward(self, x):\n",
    "        pt_out = self.model(**x)\n",
    "        hidden_states = pt_out.hidden_states\n",
    "        last_hidden_state = pt_out.last_hidden_state\n",
    "\n",
    "        x1 = self.weighted_pool(torch.stack(hidden_states))[:, 0]\n",
    "        x2 = self.att_pool(last_hidden_state, x['attention_mask'])\n",
    "\n",
    "        y_hat = self.fc_out(torch.cat([x1, x2], dim=1))\n",
    "        return y_hat\n",
    "    \n",
    "##########\n",
    "class WeightedLayerPooling_(nn.Module):\n",
    "    def __init__(self, num_hidden_layers, layer_start: int = 4, layers = None, layer_weights = None):\n",
    "        super(WeightedLayerPooling_, self).__init__()\n",
    "        self.layer_start = layer_start\n",
    "        self.num_hidden_layers = num_hidden_layers\n",
    "        \n",
    "        if layers:\n",
    "            self.layer_weights = layer_weights if layer_weights is not None \\\n",
    "                else nn.Parameter(\n",
    "                    torch.tensor([1] * len(layers), dtype=torch.float)\n",
    "                )\n",
    "            self.layers = layers\n",
    "        else:\n",
    "            self.layer_weights = layer_weights if layer_weights is not None \\\n",
    "                else nn.Parameter(\n",
    "                   torch.tensor([1] * (num_hidden_layers+1 - layer_start), dtype=torch.float)\n",
    "                )\n",
    "            self.layers = list(range(layer_start, num_hidden_layers+1))\n",
    "            \n",
    "\n",
    "    def forward(self, ft_all_layers):\n",
    "        all_layer_embedding = torch.stack(ft_all_layers)\n",
    "        all_layer_embedding = all_layer_embedding[self.layers, :, :, :]\n",
    "        \n",
    "        weight_factor = self.layer_weights.unsqueeze(-1).unsqueeze(-1).unsqueeze(-1).expand(all_layer_embedding.size())\n",
    "        weighted_average = (weight_factor*all_layer_embedding).sum(dim=0) / self.layer_weights.sum()\n",
    "\n",
    "        return weighted_average\n",
    "    \n",
    "class CustomModel(nn.Module):\n",
    "\n",
    "    def __init__(self, CFG, config_path = None, pretrained = False):\n",
    "        super().__init__()\n",
    "        self.CFG = CFG\n",
    "        self.config = AutoConfig.from_pretrained(config_path, output_hidden_states=True)\n",
    "        self.model = AutoModel.from_config(self.config)\n",
    "        self.pretrained = pretrained\n",
    "                        \n",
    "        fc_hidden_size = self.config.hidden_size\n",
    "        if CFG.pooling == 'mean':\n",
    "            self.pool = MeanPooling()\n",
    "        elif CFG.pooling == 'max':\n",
    "            self.pool = MaxPooling()\n",
    "        elif CFG.pooling == 'min':\n",
    "            self.pool = MinPooling()\n",
    "        elif CFG.pooling == 'attention':\n",
    "            self.pool = AttentionPooling(self.config.hidden_size)\n",
    "        elif CFG.pooling == 'weightedlayer':\n",
    "            self.pool = WeightedLayerPooling_(self.config.num_hidden_layers, layer_start = CFG.layer_start, layer_weights = None)\n",
    "        elif CFG.pooling == 'weightedlayer-mean':\n",
    "            self.pool = WeightedLayerPooling_(self.config.num_hidden_layers, layer_start = CFG.layer_start, layer_weights = None)\n",
    "            self.mean_pool = MeanPooling()\n",
    "        elif self.CFG.pooling == 'attention4':\n",
    "            self.pool = AttentionHead(fc_hidden_size*4, 512)\n",
    "            fc_hidden_size = fc_hidden_size*8\n",
    "        self.fc = nn.Linear(fc_hidden_size, 6)\n",
    "                        \n",
    "    def feature(self, inputs):\n",
    "        outputs = self.model(**inputs)\n",
    "        if self.CFG.pooling == 'attention4':\n",
    "            all_layer_embeddings = torch.stack(outputs.hidden_states)\n",
    "            cat_over_last_layers = torch.cat((all_layer_embeddings[-1], all_layer_embeddings[-2], all_layer_embeddings[-3], all_layer_embeddings[-4]), -1)\n",
    "            cls_pooling = cat_over_last_layers[:, 0]\n",
    "            head_logits = self.pool(cat_over_last_layers)\n",
    "            feature = torch.cat([head_logits, cls_pooling], -1)\n",
    "        elif self.CFG.pooling == 'weightedlayer':\n",
    "            all_layer_embeddings = outputs[1]\n",
    "            feature = self.pool(all_layer_embeddings)[:, 0]\n",
    "        elif self.CFG.pooling == 'weightedlayer-mean':\n",
    "            all_layer_embeddings = outputs[1]\n",
    "            feature = self.pool(all_layer_embeddings)\n",
    "            feature = self.mean_pool(feature, inputs['attention_mask'])\n",
    "        else:\n",
    "            last_hidden_states = outputs[0]\n",
    "            feature = self.pool(last_hidden_states, inputs['attention_mask'])    \n",
    "        return feature\n",
    "\n",
    "    def forward(self, inputs):\n",
    "        feature = self.feature(inputs)\n",
    "        output = self.fc(feature)\n",
    "        return output\n",
    "    \n",
    "class MeanAttentionModel(nn.Module):\n",
    "    def __init__(self, cfg, config_path=None, pretrained=False):\n",
    "        super().__init__()\n",
    "        self.cfg = cfg \n",
    "        if config_path is None:\n",
    "            self.config = AutoConfig.from_pretrained(cfg.model, output_hidden_states=True)\n",
    "            # Turn off dropouts.\n",
    "            self.config.hidden_dropout = 0.\n",
    "            self.config.hidden_dropout_prob = 0.\n",
    "            self.config.attention_dropout = 0.\n",
    "            self.config.attention_probs_dropout_prob = 0.\n",
    "            #LOGGER.info(self.config)\n",
    "        else:\n",
    "            self.config = torch.load(config_path)\n",
    "        \n",
    "        if pretrained:\n",
    "            self.deberta_v3 = AutoModel.from_pretrained(cfg.model, config=self.config)\n",
    "            # Expand embedding dim for new tokens.\n",
    "            self.deberta_v3.resize_token_embeddings(len(cfg.tokenizer))\n",
    "        else:\n",
    "            self.deberta_v3 = AutoModel.from_config(self.config)\n",
    "            \n",
    "        self.deberta_v3.gradient_checkpointing_enable()\n",
    "        \n",
    "        # Define model layers.\n",
    "        self.fc = nn.Linear(self.config.hidden_size, 6)\n",
    "\n",
    "        if cfg.pool == 'mean':\n",
    "            self.pool = MeanPooling()\n",
    "        elif cfg.pool == 'attention':\n",
    "            self.pool = AttentionHead(self.config.hidden_size, self.config.hidden_size)\n",
    "        elif cfg.pool == 'mean-attention':\n",
    "            self.pool = nn.ModuleList([\n",
    "                MeanPooling(),\n",
    "                AttentionHead(self.config.hidden_size, self.config.hidden_size)\n",
    "            ])\n",
    "            self.fc = nn.Linear(self.config.hidden_size * len(self.pool), 6)\n",
    "        elif cfg.pool == 'mean-attention-with-mask':\n",
    "            self.pool = nn.ModuleList([\n",
    "                MeanPooling(),\n",
    "                AttentionPooling(self.config.hidden_size)\n",
    "            ])\n",
    "            self.fc = nn.Linear(self.config.hidden_size * len(self.pool), 6)\n",
    "        # Re-init weights.\n",
    "        self._init_weights(self.fc)\n",
    "        \n",
    "        # Multi-sample dropout.\n",
    "        self.multi_dropout = MultiSampleDropout(self.fc, cfg.num_dropout, cfg.prob_dropout)\n",
    "        \n",
    "    def global_avg_pool(x):\n",
    "        return torch.mean(x.view(x.size(0), x.size(1), -1), dim=-1)\n",
    "        \n",
    "    def _init_weights(self, module):\n",
    "        if isinstance(module, nn.Linear):\n",
    "            module.weight.data.normal_(mean=0.0, std=self.config.initializer_range)\n",
    "            if module.bias is not None:\n",
    "                module.bias.data.zero_()\n",
    "        elif isinstance(module, nn.Embedding):\n",
    "            module.weight.data.normal_(mean=0.0, std=self.config.initializer_range)\n",
    "            if module.padding_idx is not None:\n",
    "                module.weight.data[module.padding_idx].zero_()\n",
    "        elif isinstance(module, nn.LayerNorm):\n",
    "            module.bias.data.zero_()\n",
    "            module.weight.data.fill_(1.0)\n",
    "    \n",
    "    def feature(self, x):\n",
    "        pt_outputs = self.deberta_v3(**x)\n",
    "        last_hidden_states = pt_outputs[0] # N x max_len x 768\n",
    "        # Pooling feat.\n",
    "        if type(self.pool) == nn.ModuleList:\n",
    "            pool_feature = [pool(last_hidden_states, x['attention_mask']) for pool in self.pool]\n",
    "            pool_feature = torch.cat(pool_feature, dim=1)\n",
    "        else:\n",
    "            pool_feature = self.pool(last_hidden_states, x['attention_mask']) # N x 768\n",
    "        return pool_feature\n",
    "    \n",
    "    def forward(self, x, y=None, loss_fn=None):\n",
    "        feature = self.feature(x)\n",
    "        out = self.fc(feature)\n",
    "        return out"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "id": "0fc33120",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2022-11-29T18:18:06.656388Z",
     "iopub.status.busy": "2022-11-29T18:18:06.655826Z",
     "iopub.status.idle": "2022-11-29T18:18:06.661723Z",
     "shell.execute_reply": "2022-11-29T18:18:06.660830Z"
    },
    "papermill": {
     "duration": 0.017263,
     "end_time": "2022-11-29T18:18:06.663752",
     "exception": false,
     "start_time": "2022-11-29T18:18:06.646489",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "class Config(dict):\n",
    "    __getattr__ = dict.__getitem__\n",
    "    __setattr__ = dict.__setitem__\n",
    "    __delattr__ = dict.__delitem__\n",
    "    \n",
    "    def init(self, kwargs):\n",
    "        super().init(kwargs)\n",
    "\n",
    "        for k, v in kwargs.items():\n",
    "            setattr(self, k, v)\n",
    "\n",
    "    def set(self, key, val):\n",
    "        self[key] = val\n",
    "        setattr(self, key, val)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "id": "407c62c7",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2022-11-29T18:18:06.681983Z",
     "iopub.status.busy": "2022-11-29T18:18:06.681186Z",
     "iopub.status.idle": "2022-11-29T18:18:06.692964Z",
     "shell.execute_reply": "2022-11-29T18:18:06.692154Z"
    },
    "papermill": {
     "duration": 0.022863,
     "end_time": "2022-11-29T18:18:06.694903",
     "exception": false,
     "start_time": "2022-11-29T18:18:06.672040",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "def seed_everything(seed=42):\n",
    "    '''\n",
    "    Sets the seed of the entire notebook so results are the same every time we run.\n",
    "    This is for REPRODUCIBILITY.\n",
    "    '''\n",
    "    random.seed(seed)\n",
    "    # Set a fixed value for the hash seed\n",
    "    os.environ['PYTHONHASHSEED'] = str(seed)\n",
    "    np.random.seed(seed)\n",
    "    torch.manual_seed(seed)\n",
    "    \n",
    "    if torch.cuda.is_available():\n",
    "        torch.cuda.manual_seed(seed)\n",
    "        torch.cuda.manual_seed_all(seed)\n",
    "        # When running on the CuDNN backend, two further options must be set\n",
    "        torch.backends.cudnn.deterministic = True\n",
    "        torch.backends.cudnn.benchmark = False\n",
    "seed_everything(seed=42)\n",
    "\n",
    "def mc_rmse(y_true, y_pred):\n",
    "    scores = []\n",
    "    ncols = y_true.shape[1]\n",
    "    \n",
    "    for n in range(ncols):\n",
    "        yn_true = y_true[:, n]\n",
    "        yn_pred = y_pred[:, n]\n",
    "        rmse_ = mean_squared_error(yn_true, yn_pred, squared=False)\n",
    "        scores.append(rmse_)\n",
    "    score = np.mean(scores) \n",
    "    return score, scores\n",
    "\n",
    "target_cols = ['cohesion', 'syntax', 'vocabulary', 'phraseology', 'grammar', 'conventions']\n",
    "def get_result(cfg, oof_df):\n",
    "    labels = oof_df[target_cols].values\n",
    "    preds = oof_df[[f\"pred_{c}\" for c in target_cols]].values\n",
    "    score, scores = mc_rmse(labels, preds)\n",
    "    print(f'score: {score:<.6f}  scores: {scores}')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "id": "1d49ac43",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2022-11-29T18:18:06.712912Z",
     "iopub.status.busy": "2022-11-29T18:18:06.712137Z",
     "iopub.status.idle": "2022-11-29T18:18:06.723028Z",
     "shell.execute_reply": "2022-11-29T18:18:06.722240Z"
    },
    "papermill": {
     "duration": 0.02185,
     "end_time": "2022-11-29T18:18:06.724929",
     "exception": false,
     "start_time": "2022-11-29T18:18:06.703079",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "def encode_text(cfg, text):\n",
    "    if cfg.pretrained:\n",
    "        inputs = cfg.tokenizer(\n",
    "            text,\n",
    "            None,\n",
    "            add_special_tokens=True,\n",
    "            padding='max_length',\n",
    "            truncation=True,\n",
    "            max_length=cfg.max_len,\n",
    "            return_token_type_ids=True,\n",
    "            return_tensors='pt'\n",
    "        )\n",
    "        inputs = {k:v.squeeze(0) for k,v in inputs.items()}\n",
    "    else:\n",
    "        if 'roberta' in cfg.model:\n",
    "            inputs = cfg.tokenizer.encode_plus(\n",
    "                text,\n",
    "                return_tensors = None,\n",
    "                add_special_tokens = True,\n",
    "                max_length = cfg.max_len,\n",
    "                pad_to_max_length = True,\n",
    "                truncation = True)\n",
    "        elif '512' in cfg.name or '768' in cfg.name:\n",
    "            inputs = cfg.tokenizer.encode_plus(\n",
    "                text,\n",
    "                return_tensors = None,\n",
    "                add_special_tokens = True,\n",
    "                max_length = cfg.max_len,\n",
    "                truncation = True)\n",
    "        else:\n",
    "            inputs = cfg.tokenizer.encode_plus(\n",
    "                text, \n",
    "                return_tensors=None, \n",
    "                add_special_tokens=True)\n",
    "        for k, v in inputs.items():\n",
    "            inputs[k] = torch.tensor(v, dtype=torch.long)\n",
    "    return inputs \n",
    "\n",
    "def preprocess(texts):\n",
    "    texts = (\n",
    "        texts\n",
    "        .str.replace(r'\\r\\n', '<newline>', regex=True)\n",
    "        .str.replace(r'\\n', '<newline>', regex=True)\n",
    "        .str.replace('<newline><newline>', '<newline>', regex=False)\n",
    "        .values \n",
    "    )\n",
    "    return texts\n",
    "\n",
    "class TestDataset(Dataset):\n",
    "    def __init__(self, cfg, df):\n",
    "        self.cfg = cfg\n",
    "        #if not cfg.pretrained and cfg.version in ['1', 'mean-attention']:\n",
    "        if not cfg.pretrained:\n",
    "            print('preprocess')\n",
    "            self.texts = preprocess(df['full_text'])\n",
    "        else:\n",
    "            self.texts = df['full_text'].values\n",
    "\n",
    "    def __len__(self):\n",
    "        return len(self.texts)\n",
    "\n",
    "    def __getitem__(self, item):\n",
    "        inputs = encode_text(self.cfg, self.texts[item])\n",
    "        return inputs"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "id": "00250c77",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2022-11-29T18:18:06.745323Z",
     "iopub.status.busy": "2022-11-29T18:18:06.744693Z",
     "iopub.status.idle": "2022-11-29T18:18:06.753579Z",
     "shell.execute_reply": "2022-11-29T18:18:06.752709Z"
    },
    "papermill": {
     "duration": 0.019855,
     "end_time": "2022-11-29T18:18:06.755531",
     "exception": false,
     "start_time": "2022-11-29T18:18:06.735676",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "def load_config(input_path, inference_weight=1):\n",
    "    # Load CFG class.\n",
    "    cfg = Config(**json.load(open(os.path.join(input_path, 'CFG.json'), 'r')))\n",
    "    cfg.path = input_path\n",
    "    cfg.config_path = os.path.join(cfg.path, 'config.pth')\n",
    "    # Load tokenizer.\n",
    "    tokenizer = AutoTokenizer.from_pretrained(os.path.join(cfg.path, 'tokenizer'))\n",
    "    cfg.tokenizer = tokenizer\n",
    "    \n",
    "    cfg.inference_weight = inference_weight\n",
    "    return cfg\n",
    "\n",
    "def load_model(cfg, fold, version='1', **model_kwargs):\n",
    "    # Load torch model.\n",
    "    if version == '1':\n",
    "        model = MultiPoolModel(cfg, config_path=cfg.config_path, pretrained=False, **model_kwargs)\n",
    "    elif version == '2':\n",
    "        model = Attention4Model(cfg, config_path=cfg.config_path, pretrained=False, **model_kwargs)\n",
    "    elif version == '21':\n",
    "        model = WMPoolModel(cfg, config_path=cfg.config_path, pretrained=False, **model_kwargs)\n",
    "    elif version == 'weighted-attention':\n",
    "        model = WeightedAttentionModel(cfg, config_path=cfg.config_path, pretrained=False, **model_kwargs)\n",
    "    elif version == 'custom':\n",
    "        model = CustomModel(cfg, config_path=cfg.config_path, pretrained=False, **model_kwargs)\n",
    "    elif version == 'mean-attention':\n",
    "        model = MeanAttentionModel(cfg, config_path=cfg.config_path, pretrained=False)\n",
    "    state = torch.load(\n",
    "        os.path.join(cfg.path, f\"{cfg.model.replace('/', '-')}_fold{fold}_best.pth\"),\n",
    "        map_location=torch.device('cpu'))\n",
    "    model.load_state_dict(state['model'])\n",
    "    return model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "id": "eff8a9f5",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2022-11-29T18:18:06.773292Z",
     "iopub.status.busy": "2022-11-29T18:18:06.773015Z",
     "iopub.status.idle": "2022-11-29T18:18:06.790630Z",
     "shell.execute_reply": "2022-11-29T18:18:06.789786Z"
    },
    "papermill": {
     "duration": 0.028586,
     "end_time": "2022-11-29T18:18:06.792526",
     "exception": false,
     "start_time": "2022-11-29T18:18:06.763940",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "def mean_pooling(model_output, attention_mask):\n",
    "    token_embeddings = model_output.last_hidden_state.detach().cpu()\n",
    "    input_mask_expanded = (\n",
    "        attention_mask.unsqueeze(-1).expand(token_embeddings.size()).float()\n",
    "    )\n",
    "    return torch.sum(token_embeddings * input_mask_expanded, 1) / torch.clamp(\n",
    "        input_mask_expanded.sum(1), min=1e-9\n",
    "    )\n",
    "\n",
    "def inference_fn(test_loader, model, device):\n",
    "    preds = []\n",
    "    model.eval()\n",
    "    model.to(device)\n",
    "    #tk0 = tqdm(test_loader, total=len(test_loader))\n",
    "    for inputs in test_loader:\n",
    "        for k, v in inputs.items():\n",
    "            inputs[k] = v.to(device)\n",
    "        with torch.no_grad():\n",
    "            y_preds = model(inputs)\n",
    "        preds.append(y_preds.to('cpu').numpy())\n",
    "    predictions = np.concatenate(preds)\n",
    "    return predictions\n",
    "\n",
    "class Inferencer:\n",
    "    def __init__(self, input_path=None, cfg=None, inference_weight=1):\n",
    "        if cfg == None:\n",
    "            self.cfg = load_config(input_path, inference_weight)\n",
    "        else:\n",
    "            self.cfg = cfg\n",
    "    \n",
    "    def predict(self, test_loader, device, stat_fn=np.mean):\n",
    "        preds = []\n",
    "        \n",
    "        for fold in self.cfg.trn_fold:\n",
    "            start = time.time()\n",
    "            print('#'*10, cfg.path, '#'*10)\n",
    "            \n",
    "            print(f'Predicting fold {fold}...')\n",
    "            model = load_model(self.cfg, fold, version=self.cfg.version)\n",
    "            pred = inference_fn(test_loader, model, device)\n",
    "            preds.append(pred)\n",
    "            del model, pred; gc.collect()\n",
    "            torch.cuda.empty_cache()\n",
    "            \n",
    "            end = time.time() - start\n",
    "            print('#'*10, f'ETA: {end:.2f}s', '#'*10, '\\n')\n",
    "        \n",
    "        \n",
    "        self.preds = stat_fn(preds, axis=0) \n",
    "        self.preds = np.clip(self.preds, 1, 5)\n",
    "        return self.preds\n",
    "    \n",
    "    def get_oof_result(self, file_type='pkl'):\n",
    "        return get_result(self.cfg, self.get_oof_df(file_type))\n",
    "    \n",
    "    def get_oof_df(self, file_type='pkl'):\n",
    "        if file_type == 'pkl':\n",
    "            return pd.read_pickle(os.path.join(cfg.path, 'oof_df.pkl'))\n",
    "        return pd.read_csv(os.path.join(cfg.path, 'oof_df.csv'))\n",
    "    \n",
    "    def get_text_embedding(self, data_loader, device, fold=None): \n",
    "        # pretrained=True: not fine-tuned models.\n",
    "        if not self.cfg.pretrained:\n",
    "            model = load_model(self.cfg, fold, pool=self.cfg.pool_head)            \n",
    "        else:\n",
    "            model = AutoModel.from_pretrained(self.cfg.model)\n",
    "        model.to(device)\n",
    "        model.eval()\n",
    "            \n",
    "        fold_emb = []\n",
    "        for inputs in data_loader:\n",
    "            for k, v in inputs.items():\n",
    "                inputs[k] = v.to(device)\n",
    "            if not self.cfg.pretrained:\n",
    "                with torch.no_grad():\n",
    "                    emb = model.feature(**inputs)\n",
    "            else:\n",
    "                input_ids = inputs['input_ids'].to(device)\n",
    "                attention_mask = inputs['attention_mask'].to(device)\n",
    "                token_type_ids = inputs['token_type_ids'].to(device)\n",
    "                \n",
    "                with torch.no_grad():\n",
    "                    try:\n",
    "                        output = model(input_ids=input_ids, attention_mask=attention_mask, token_type_ids=token_type_ids)\n",
    "                    except:\n",
    "                        output = model(input_ids=input_ids, attention_mask=attention_mask)\n",
    "                emb = mean_pooling(output, attention_mask.detach().cpu())\n",
    "                emb = F.normalize(emb, p=2, dim=1)\n",
    "                emb = emb.squeeze(0)\n",
    "            fold_emb.extend(emb.detach().cpu().numpy())\n",
    "            del emb; gc.collect(); torch.cuda.empty_cache();\n",
    "            #print(torch.cuda.memory_allocated() /1024/1024)\n",
    "            \n",
    "        fold_emb = np.array(fold_emb)\n",
    "        return fold_emb\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "id": "e90b6f39",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2022-11-29T18:18:06.810386Z",
     "iopub.status.busy": "2022-11-29T18:18:06.809651Z",
     "iopub.status.idle": "2022-11-29T18:18:07.240860Z",
     "shell.execute_reply": "2022-11-29T18:18:07.239810Z"
    },
    "papermill": {
     "duration": 0.44302,
     "end_time": "2022-11-29T18:18:07.243728",
     "exception": false,
     "start_time": "2022-11-29T18:18:06.800708",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "cfg = load_config('../input/fb3models/v21/', inference_weight=1)\n",
    "cfg.name = 'v21'\n",
    "cfg.version = '21'\n",
    "\n",
    "test = pd.read_csv(TEST_PATH)\n",
    "test['tokenize_length'] = [len(cfg.tokenizer(text)['input_ids']) for text in test['full_text'].values]\n",
    "test = test.sort_values('tokenize_length', ascending=True).reset_index(drop=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "id": "5a2a5415",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2022-11-29T18:18:07.264352Z",
     "iopub.status.busy": "2022-11-29T18:18:07.263991Z",
     "iopub.status.idle": "2022-11-29T18:18:07.278868Z",
     "shell.execute_reply": "2022-11-29T18:18:07.277769Z"
    },
    "papermill": {
     "duration": 0.027778,
     "end_time": "2022-11-29T18:18:07.280999",
     "exception": false,
     "start_time": "2022-11-29T18:18:07.253221",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "##################################################\n",
    "deberta_base = Config(\n",
    "    model='../input/huggingface-deberta-variants/deberta-base/deberta-base',\n",
    "    file_name='microsoft_deberta_base_768',\n",
    "    pretrained=True, inference_weight=1, max_len=640) #\n",
    "deberta_large = Config(\n",
    "    model='../input/huggingface-deberta-variants/deberta-large/deberta-large', \n",
    "    file_name='microsoft_deberta_large_1024',\n",
    "    pretrained=True, inference_weight=1, max_len=640) #\n",
    "deberta_xlarge = Config(\n",
    "    model='../input/huggingface-deberta-variants/deberta-xlarge/deberta-xlarge', \n",
    "    file_name='microsoft_deberta_xlarge_1024',\n",
    "    pretrained=True, inference_weight=1, max_len=640)\n",
    "deberta_v2_xlarge = Config(\n",
    "    model='../input/bert-shopping-mall/deberta-v2-xlarge', \n",
    "    file_name='microsoft_deberta_v2_xlarge_1536',\n",
    "    pretrained=True, inference_weight=1, max_len=640)\n",
    "deberta_v2_xxlarge = Config(\n",
    "    model='../input/bert-shopping-mall/deberta-v2-xxlarge', \n",
    "    file_name='microsoft_deberta_v2_xxlarge_1536',\n",
    "    pretrained=True, inference_weight=1, max_len=640)\n",
    "\n",
    "deberta_v3_base = Config(\n",
    "    model='../input/bert-shopping-mall/deberta-v3-base',\n",
    "    file_name='microsoft_deberta_v3_base_768',\n",
    "    pretrained=True, inference_weight=1, max_len=640) #\n",
    "deberta_v3_large = Config(\n",
    "    model='../input/bert-shopping-mall/deberta-v3-large', \n",
    "    file_name='microsoft_deberta_v3_large_1024',\n",
    "    pretrained=True, inference_weight=1, max_len=640) # \n",
    "\n",
    "deberta_large_mnli = Config(\n",
    "    model='../input/huggingface-deberta-variants/deberta-large-mnli/deberta-large-mnli',\n",
    "    file_name='microsoft_deberta_large_mnli_1024',\n",
    "    pretrained=True, inference_weight=1, max_len=640) # \n",
    "\n",
    "gpt2 = Config(\n",
    "    model='../input/hugging-face-gpt2/gpt2',\n",
    "    file_name='gpt2_768',\n",
    "    pretrained=True, inference_weight=1, max_len=512) #\n",
    "\n",
    "roberta_base = Config(\n",
    "    model='../input/transformers/roberta-base', \n",
    "    file_name='roberta_base_768',\n",
    "    pretrained=True, inference_weight=1, max_len=512) #\n",
    "roberta_large = Config(\n",
    "    model='../input/transformers/roberta-large',\n",
    "    file_name='roberta_large_1024',\n",
    "    pretrained=True, inference_weight=1, max_len=512) # \n",
    "\n",
    "xlnet_base = Config(\n",
    "    model='../input/transformers/xlnet-base-cased',\n",
    "    file_name='xlnet_base_cased_768',\n",
    "    pretrained=True, inference_weight=1, max_len=640) #\n",
    "xlnet_large = Config(\n",
    "    model='../input/transformers/xlnet-large-cased', \n",
    "    file_name='xlnet_large_cased_1024',\n",
    "    pretrained=True, inference_weight=1, max_len=640) #\n",
    "\n",
    "bart_base = Config(\n",
    "    model='../input/transformers/facebook-bart-base',\n",
    "    file_name='facebook_bart_base_768',\n",
    "    pretrained=True, inference_weight=1, max_len=640)\n",
    "bart_large = Config(\n",
    "    model='../input/transformers/facebook-bart-large',\n",
    "    file_name='facebook_bart_large_1024',\n",
    "    pretrained=True, inference_weight=1, max_len=640)\n",
    "bart_lage_mnli = Config(\n",
    "    model='../input/facebook-bart-large-mnli',\n",
    "    file_name='facebook_bart_large_mnli_1024',\n",
    "    pretrained=True, inference_weight=1, max_len=640)\n",
    "\n",
    "bert_base_uncased = Config(\n",
    "    model='../input/transformers/bert-base-uncased/',\n",
    "    file_name='bert_base_uncased_768',\n",
    "    pretrained=True, inference_weight=1, max_len=512)\n",
    "bert_large_uncased = Config(\n",
    "    model='../input/transformers/bert-large-uncased',\n",
    "    file_name='bert_large_uncased_1024',\n",
    "    pretrained=True, inference_weight=1, max_len=512)\n",
    "\n",
    "muppet_roberta_large = Config(\n",
    "    model='../input/muppet-roberta-large',\n",
    "    file_name='facebook_muppet_roberta_large_1024',\n",
    "    pretrained=True, inference_weight=1, max_len=512)\n",
    "# muppet_roberta_base = Config(model='facebook/muppet-roberta-base', pretrained=True, inference_weight=1, max_len=512)\n",
    "\n",
    "funnel_small = Config(\n",
    "    model='../input/transformers/funnel-transformer-small',\n",
    "    file_name='funnel_transformer_small_768',\n",
    "    pretrained=True, inference_weight=1, max_len=640)\n",
    "funnel_large = Config(\n",
    "    model='../input/transformers/funnel-transformer-large',\n",
    "    file_name='funnel_transformer_large_1024',\n",
    "    pretrained=True, inference_weight=1, max_len=640)\n",
    "\n",
    "##################################################\n",
    "\n",
    "target_cols = ['cohesion', 'syntax', 'vocabulary', 'phraseology', 'grammar', 'conventions']"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "9b458b1b",
   "metadata": {
    "papermill": {
     "duration": 0.00873,
     "end_time": "2022-11-29T18:18:07.298755",
     "exception": false,
     "start_time": "2022-11-29T18:18:07.290025",
     "status": "completed"
    },
    "tags": []
   },
   "source": [
    "# SVR from pretrained models"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "id": "95d55bde",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2022-11-29T18:18:07.318127Z",
     "iopub.status.busy": "2022-11-29T18:18:07.317806Z",
     "iopub.status.idle": "2022-11-29T18:18:07.326343Z",
     "shell.execute_reply": "2022-11-29T18:18:07.325266Z"
    },
    "papermill": {
     "duration": 0.020971,
     "end_time": "2022-11-29T18:18:07.328452",
     "exception": false,
     "start_time": "2022-11-29T18:18:07.307481",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "def get_text_embedding(cfg, dfs):\n",
    "    cfg.tokenizer = AutoTokenizer.from_pretrained(cfg.model)\n",
    "    infer_ = Inferencer(cfg=cfg, inference_weight=cfg.inference_weight)\n",
    "    if 'gpt2' in cfg.model:\n",
    "        cfg.tokenizer.pad_token = cfg.tokenizer.eos_token\n",
    "    text_embs = []\n",
    "    for df in dfs:\n",
    "        dataset = TestDataset(cfg, df)\n",
    "        loader = DataLoader(\n",
    "            dataset,\n",
    "            batch_size=4,\n",
    "            shuffle=False)\n",
    "\n",
    "        # Text embedding for SVM\n",
    "        test_text_emb = []\n",
    "        if not cfg.pretrained:\n",
    "            for fold in infer_.cfg.trn_fold:\n",
    "                test_text_emb.append(infer_.get_text_embedding(loader, device, fold))\n",
    "            text_emb = np.mean(text_emb, axis=0)\n",
    "        else:\n",
    "            text_emb = infer_.get_text_embedding(loader, device)\n",
    "        text_embs.append(text_emb)\n",
    "        del dataset, loader; gc.collect(); torch.cuda.empty_cache();\n",
    "    del infer_; gc.collect(); torch.cuda.empty_cache();\n",
    "    return text_embs"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "id": "ae7ae3c3",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2022-11-29T18:18:07.348221Z",
     "iopub.status.busy": "2022-11-29T18:18:07.347410Z",
     "iopub.status.idle": "2022-11-29T18:18:07.357057Z",
     "shell.execute_reply": "2022-11-29T18:18:07.356037Z"
    },
    "papermill": {
     "duration": 0.021749,
     "end_time": "2022-11-29T18:18:07.359247",
     "exception": false,
     "start_time": "2022-11-29T18:18:07.337498",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "14"
      ]
     },
     "execution_count": 13,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "pretrained_models_cfg = [\n",
    "    deberta_large_mnli,\n",
    "    #gpt2,\n",
    "    roberta_base,\n",
    "    roberta_large,\n",
    "    #xlnet_base, \n",
    "    #xlnet_large,\n",
    "    deberta_base, \n",
    "    deberta_large, \n",
    "    deberta_xlarge,\n",
    "    deberta_v2_xlarge, \n",
    "    deberta_v2_xxlarge,\n",
    "    deberta_v3_base, \n",
    "    deberta_v3_large,\n",
    "    \n",
    "    #bart_base,\n",
    "    bart_large,\n",
    "    #bart_lage_mnli,\n",
    "    #bert_base_uncased,\n",
    "    bert_large_uncased,\n",
    "    #muppet_roberta_large,\n",
    "    funnel_small,\n",
    "    funnel_large\n",
    "]\n",
    "len(pretrained_models_cfg)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "id": "9f9fbba0",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2022-11-29T18:18:07.379345Z",
     "iopub.status.busy": "2022-11-29T18:18:07.378426Z",
     "iopub.status.idle": "2022-11-29T18:22:38.203339Z",
     "shell.execute_reply": "2022-11-29T18:22:38.202236Z"
    },
    "papermill": {
     "duration": 270.851153,
     "end_time": "2022-11-29T18:22:38.219587",
     "exception": false,
     "start_time": "2022-11-29T18:18:07.368434",
     "status": "completed"
    },
    "scrolled": true,
    "tags": []
   },
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "db2f144862df4e6196309c4ba129d05a",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "  0%|          | 0/14 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Some weights of the model checkpoint at ../input/huggingface-deberta-variants/deberta-large-mnli/deberta-large-mnli were not used when initializing DebertaModel: ['config', 'pooler.dense.weight', 'classifier.weight', 'pooler.dense.bias', 'classifier.bias']\n",
      "- This IS expected if you are initializing DebertaModel from the checkpoint of a model trained on another task or with another architecture (e.g. initializing a BertForSequenceClassification model from a BertForPreTraining model).\n",
      "- This IS NOT expected if you are initializing DebertaModel from the checkpoint of a model that you expect to be exactly identical (initializing a BertForSequenceClassification model from a BertForSequenceClassification model).\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "../input/huggingface-deberta-variants/deberta-large-mnli/deberta-large-mnli loaded.\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Some weights of the model checkpoint at ../input/transformers/roberta-base were not used when initializing RobertaModel: ['lm_head.dense.weight', 'lm_head.dense.bias', 'lm_head.bias', 'lm_head.decoder.weight', 'lm_head.layer_norm.bias', 'lm_head.layer_norm.weight']\n",
      "- This IS expected if you are initializing RobertaModel from the checkpoint of a model trained on another task or with another architecture (e.g. initializing a BertForSequenceClassification model from a BertForPreTraining model).\n",
      "- This IS NOT expected if you are initializing RobertaModel from the checkpoint of a model that you expect to be exactly identical (initializing a BertForSequenceClassification model from a BertForSequenceClassification model).\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "../input/transformers/roberta-base loaded.\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Some weights of the model checkpoint at ../input/transformers/roberta-large were not used when initializing RobertaModel: ['lm_head.dense.weight', 'lm_head.dense.bias', 'lm_head.bias', 'lm_head.decoder.weight', 'lm_head.layer_norm.bias', 'lm_head.layer_norm.weight']\n",
      "- This IS expected if you are initializing RobertaModel from the checkpoint of a model trained on another task or with another architecture (e.g. initializing a BertForSequenceClassification model from a BertForPreTraining model).\n",
      "- This IS NOT expected if you are initializing RobertaModel from the checkpoint of a model that you expect to be exactly identical (initializing a BertForSequenceClassification model from a BertForSequenceClassification model).\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "../input/transformers/roberta-large loaded.\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Some weights of the model checkpoint at ../input/huggingface-deberta-variants/deberta-base/deberta-base were not used when initializing DebertaModel: ['lm_predictions.lm_head.bias', 'config', 'lm_predictions.lm_head.dense.weight', 'lm_predictions.lm_head.dense.bias', 'lm_predictions.lm_head.LayerNorm.weight', 'lm_predictions.lm_head.LayerNorm.bias']\n",
      "- This IS expected if you are initializing DebertaModel from the checkpoint of a model trained on another task or with another architecture (e.g. initializing a BertForSequenceClassification model from a BertForPreTraining model).\n",
      "- This IS NOT expected if you are initializing DebertaModel from the checkpoint of a model that you expect to be exactly identical (initializing a BertForSequenceClassification model from a BertForSequenceClassification model).\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "../input/huggingface-deberta-variants/deberta-base/deberta-base loaded.\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Some weights of the model checkpoint at ../input/huggingface-deberta-variants/deberta-large/deberta-large were not used when initializing DebertaModel: ['lm_predictions.lm_head.bias', 'config', 'lm_predictions.lm_head.dense.weight', 'lm_predictions.lm_head.dense.bias', 'lm_predictions.lm_head.LayerNorm.weight', 'lm_predictions.lm_head.LayerNorm.bias']\n",
      "- This IS expected if you are initializing DebertaModel from the checkpoint of a model trained on another task or with another architecture (e.g. initializing a BertForSequenceClassification model from a BertForPreTraining model).\n",
      "- This IS NOT expected if you are initializing DebertaModel from the checkpoint of a model that you expect to be exactly identical (initializing a BertForSequenceClassification model from a BertForSequenceClassification model).\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "../input/huggingface-deberta-variants/deberta-large/deberta-large loaded.\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Some weights of the model checkpoint at ../input/huggingface-deberta-variants/deberta-xlarge/deberta-xlarge were not used when initializing DebertaModel: ['lm_predictions.lm_head.bias', 'lm_predictions.lm_head.dense.weight', 'lm_predictions.lm_head.dense.bias', 'lm_predictions.lm_head.LayerNorm.weight', 'lm_predictions.lm_head.LayerNorm.bias']\n",
      "- This IS expected if you are initializing DebertaModel from the checkpoint of a model trained on another task or with another architecture (e.g. initializing a BertForSequenceClassification model from a BertForPreTraining model).\n",
      "- This IS NOT expected if you are initializing DebertaModel from the checkpoint of a model that you expect to be exactly identical (initializing a BertForSequenceClassification model from a BertForSequenceClassification model).\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "../input/huggingface-deberta-variants/deberta-xlarge/deberta-xlarge loaded.\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Special tokens have been added in the vocabulary, make sure the associated word embeddings are fine-tuned or trained.\n",
      "Special tokens have been added in the vocabulary, make sure the associated word embeddings are fine-tuned or trained.\n",
      "Some weights of the model checkpoint at ../input/bert-shopping-mall/deberta-v2-xlarge were not used when initializing DebertaV2Model: ['lm_predictions.lm_head.bias', 'lm_predictions.lm_head.dense.weight', 'lm_predictions.lm_head.dense.bias', 'lm_predictions.lm_head.LayerNorm.weight', 'lm_predictions.lm_head.LayerNorm.bias']\n",
      "- This IS expected if you are initializing DebertaV2Model from the checkpoint of a model trained on another task or with another architecture (e.g. initializing a BertForSequenceClassification model from a BertForPreTraining model).\n",
      "- This IS NOT expected if you are initializing DebertaV2Model from the checkpoint of a model that you expect to be exactly identical (initializing a BertForSequenceClassification model from a BertForSequenceClassification model).\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "../input/bert-shopping-mall/deberta-v2-xlarge loaded.\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Special tokens have been added in the vocabulary, make sure the associated word embeddings are fine-tuned or trained.\n",
      "Special tokens have been added in the vocabulary, make sure the associated word embeddings are fine-tuned or trained.\n",
      "Some weights of the model checkpoint at ../input/bert-shopping-mall/deberta-v2-xxlarge were not used when initializing DebertaV2Model: ['lm_predictions.lm_head.bias', 'lm_predictions.lm_head.dense.weight', 'lm_predictions.lm_head.dense.bias', 'lm_predictions.lm_head.LayerNorm.weight', 'lm_predictions.lm_head.LayerNorm.bias']\n",
      "- This IS expected if you are initializing DebertaV2Model from the checkpoint of a model trained on another task or with another architecture (e.g. initializing a BertForSequenceClassification model from a BertForPreTraining model).\n",
      "- This IS NOT expected if you are initializing DebertaV2Model from the checkpoint of a model that you expect to be exactly identical (initializing a BertForSequenceClassification model from a BertForSequenceClassification model).\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "../input/bert-shopping-mall/deberta-v2-xxlarge loaded.\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Special tokens have been added in the vocabulary, make sure the associated word embeddings are fine-tuned or trained.\n",
      "/opt/conda/lib/python3.7/site-packages/transformers/convert_slow_tokenizer.py:435: UserWarning: The sentencepiece tokenizer that you are converting to a fast tokenizer uses the byte fallback option which is not implemented in the fast tokenizers. In practice this means that the fast version of the tokenizer can produce unknown tokens whereas the sentencepiece version would have converted these unknown tokens into a sequence of byte tokens matching the original piece of text.\n",
      "  \"The sentencepiece tokenizer that you are converting to a fast tokenizer uses the byte fallback option\"\n",
      "Special tokens have been added in the vocabulary, make sure the associated word embeddings are fine-tuned or trained.\n",
      "Some weights of the model checkpoint at ../input/bert-shopping-mall/deberta-v3-base were not used when initializing DebertaV2Model: ['lm_predictions.lm_head.bias', 'mask_predictions.LayerNorm.bias', 'mask_predictions.dense.bias', 'mask_predictions.classifier.bias', 'mask_predictions.dense.weight', 'lm_predictions.lm_head.dense.weight', 'lm_predictions.lm_head.dense.bias', 'mask_predictions.classifier.weight', 'mask_predictions.LayerNorm.weight', 'lm_predictions.lm_head.LayerNorm.weight', 'lm_predictions.lm_head.LayerNorm.bias']\n",
      "- This IS expected if you are initializing DebertaV2Model from the checkpoint of a model trained on another task or with another architecture (e.g. initializing a BertForSequenceClassification model from a BertForPreTraining model).\n",
      "- This IS NOT expected if you are initializing DebertaV2Model from the checkpoint of a model that you expect to be exactly identical (initializing a BertForSequenceClassification model from a BertForSequenceClassification model).\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "../input/bert-shopping-mall/deberta-v3-base loaded.\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Special tokens have been added in the vocabulary, make sure the associated word embeddings are fine-tuned or trained.\n",
      "Special tokens have been added in the vocabulary, make sure the associated word embeddings are fine-tuned or trained.\n",
      "Some weights of the model checkpoint at ../input/bert-shopping-mall/deberta-v3-large were not used when initializing DebertaV2Model: ['lm_predictions.lm_head.bias', 'mask_predictions.dense.bias', 'mask_predictions.LayerNorm.bias', 'mask_predictions.classifier.bias', 'mask_predictions.dense.weight', 'mask_predictions.classifier.weight', 'lm_predictions.lm_head.dense.bias', 'lm_predictions.lm_head.dense.weight', 'mask_predictions.LayerNorm.weight', 'lm_predictions.lm_head.LayerNorm.weight', 'lm_predictions.lm_head.LayerNorm.bias']\n",
      "- This IS expected if you are initializing DebertaV2Model from the checkpoint of a model trained on another task or with another architecture (e.g. initializing a BertForSequenceClassification model from a BertForPreTraining model).\n",
      "- This IS NOT expected if you are initializing DebertaV2Model from the checkpoint of a model that you expect to be exactly identical (initializing a BertForSequenceClassification model from a BertForSequenceClassification model).\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "../input/bert-shopping-mall/deberta-v3-large loaded.\n",
      "../input/transformers/facebook-bart-large loaded.\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Some weights of the model checkpoint at ../input/transformers/bert-large-uncased were not used when initializing BertModel: ['cls.predictions.decoder.weight', 'cls.predictions.transform.LayerNorm.weight', 'cls.predictions.transform.dense.weight', 'cls.seq_relationship.weight', 'cls.predictions.transform.LayerNorm.bias', 'cls.predictions.bias', 'cls.seq_relationship.bias', 'cls.predictions.transform.dense.bias']\n",
      "- This IS expected if you are initializing BertModel from the checkpoint of a model trained on another task or with another architecture (e.g. initializing a BertForSequenceClassification model from a BertForPreTraining model).\n",
      "- This IS NOT expected if you are initializing BertModel from the checkpoint of a model that you expect to be exactly identical (initializing a BertForSequenceClassification model from a BertForSequenceClassification model).\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "../input/transformers/bert-large-uncased loaded.\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/opt/conda/lib/python3.7/site-packages/torch/nn/functional.py:780: UserWarning: Note that order of the arguments: ceil_mode and return_indices will changeto match the args list in nn.MaxPool2d in a future release.\n",
      "  warnings.warn(\"Note that order of the arguments: ceil_mode and return_indices will change\"\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "../input/transformers/funnel-transformer-small loaded.\n",
      "../input/transformers/funnel-transformer-large loaded.\n"
     ]
    }
   ],
   "source": [
    "all_test_text_emb = []\n",
    "for cfg in tqdm(pretrained_models_cfg):\n",
    "    test_text_emb = get_text_embedding(cfg, [test])[0]\n",
    "    all_test_text_emb.append(test_text_emb)\n",
    "    \n",
    "    del test_text_emb; gc.collect(); torch.cuda.empty_cache();\n",
    "    print(f'{cfg.model} loaded.')\n",
    "    \n",
    "gc.collect(); torch.cuda.empty_cache();\n",
    "\n",
    "final_test_text_emb = np.concatenate(all_test_text_emb, axis=1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "id": "f3bc0968",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2022-11-29T18:22:38.254556Z",
     "iopub.status.busy": "2022-11-29T18:22:38.254025Z",
     "iopub.status.idle": "2022-11-29T18:22:38.260876Z",
     "shell.execute_reply": "2022-11-29T18:22:38.259958Z"
    },
    "papermill": {
     "duration": 0.030183,
     "end_time": "2022-11-29T18:22:38.266579",
     "exception": false,
     "start_time": "2022-11-29T18:22:38.236396",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(3, 14336)"
      ]
     },
     "execution_count": 15,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# del all_test_text_emb; gc.collect()\n",
    "final_test_text_emb.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "id": "9d2be3a9",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2022-11-29T18:22:38.301537Z",
     "iopub.status.busy": "2022-11-29T18:22:38.301024Z",
     "iopub.status.idle": "2022-11-29T18:25:28.319463Z",
     "shell.execute_reply": "2022-11-29T18:25:28.318476Z"
    },
    "papermill": {
     "duration": 170.039294,
     "end_time": "2022-11-29T18:25:28.322560",
     "exception": false,
     "start_time": "2022-11-29T18:22:38.283266",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "c3a5a1ac0ff840708129e31bc35fd6f6",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "  0%|          | 0/15 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/plain": [
       "array([[2.6882355, 2.475212 , 2.6849074, 2.2693486, 1.9479848, 2.6718163],\n",
       "       [3.6548452, 3.4533987, 3.5731485, 3.6607795, 3.4216313, 3.3330445],\n",
       "       [2.9247746, 2.7894423, 3.095936 , 2.9856098, 2.6844265, 2.609502 ]],\n",
       "      dtype=float32)"
      ]
     },
     "execution_count": 16,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "import glob\n",
    "from joblib import dump, load\n",
    "from cuml.svm import SVR\n",
    "import cuml\n",
    "\n",
    "def svr_inference_fn(model_path, te_text_feats):\n",
    "    model = load(model_path)\n",
    "    preds = model.predict(te_text_feats)\n",
    "    return preds\n",
    "\n",
    "predictions = []\n",
    "svr_model_paths = glob.glob('../input/fb3-svr-no-train/*.model')\n",
    "for model_path in tqdm(svr_model_paths):\n",
    "    #model_path = os.path.join('../input/fb3-svr-train/', model_path)\n",
    "    preds = svr_inference_fn(model_path, final_test_text_emb)\n",
    "    predictions.append(preds)\n",
    "svr_predictions = np.mean(predictions, axis=0)\n",
    "svr_predictions"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "74aadff5",
   "metadata": {
    "papermill": {
     "duration": 0.010695,
     "end_time": "2022-11-29T18:25:28.344220",
     "exception": false,
     "start_time": "2022-11-29T18:25:28.333525",
     "status": "completed"
    },
    "tags": []
   },
   "source": [
    "# Fine-tuned models"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "id": "e3752eea",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2022-11-29T18:25:28.368231Z",
     "iopub.status.busy": "2022-11-29T18:25:28.366701Z",
     "iopub.status.idle": "2022-11-29T18:25:29.330764Z",
     "shell.execute_reply": "2022-11-29T18:25:29.329791Z"
    },
    "papermill": {
     "duration": 0.978489,
     "end_time": "2022-11-29T18:25:29.333416",
     "exception": false,
     "start_time": "2022-11-29T18:25:28.354927",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "v114_CFG = Config(\n",
    "    model=\"microsoft/deberta-v3-base\",\n",
    "    version='1',\n",
    "    num_target = 6,\n",
    "    reinit_last_layer=True,\n",
    "    reinit_fc=True,\n",
    "    weight_decay=0.01,\n",
    "    learning_rate=2e-5,\n",
    "    layerwise_learning_rate_decay=1.5,\n",
    "    use_dropout=False,\n",
    "    prob_dropout=[0.06, 0.08, 0.1, 0.12, 0.14],\n",
    "    num_dropout=5,\n",
    "    pool_head='mean-attention',\n",
    "    seed=42,\n",
    "    n_fold=4,\n",
    "    trn_fold=[0,1,2,3],\n",
    "    path='../input/fb3models/v114/',\n",
    "    config_path='../input/fb3models/v114/config.pth',\n",
    "    tokenizer=AutoTokenizer.from_pretrained('../input/fb3models/v114/tokenizer')\n",
    ")\n",
    "\n",
    "weightedpool_CFG = Config(\n",
    "    model='microsoft/deberta-v3-base',\n",
    "    name='weightedpool',\n",
    "    version='1',\n",
    "    num_target=6,\n",
    "    reinit_last_layer=True,\n",
    "    reinit_fc=True,\n",
    "    weight_decay=0.01,\n",
    "    learning_rate=1.5e-5,\n",
    "    layerwise_learning_rate_decay=1.5,\n",
    "    use_dropout=False,\n",
    "    prob_dropout=[0.06, 0.08, 0.1, 0.12, 0.14],\n",
    "    num_dropout=5,\n",
    "    pool_head='weighted',\n",
    "    seed=42,\n",
    "    n_fold=4,\n",
    "    trn_fold=[0,1,2,3],\n",
    "    train=True,\n",
    "    path='../input/fb3-train/',\n",
    "    config_path='../input/fb3-train/config.pth',\n",
    "    tokenizer=AutoTokenizer.from_pretrained('../input/fb3-train/tokenizer'),\n",
    "    inference_weight=0.1)\n",
    "\n",
    "v116_CFG = load_config('../input/fb3-colab-models/v116', inference_weight=0.1)\n",
    "v116_CFG.path = '../input/fb3models/v116'\n",
    "v116_CFG.config_path = '../input/fb3models/v116/config.pth'\n",
    "v116_CFG.version = '1'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "id": "e3ea1298",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2022-11-29T18:25:29.358861Z",
     "iopub.status.busy": "2022-11-29T18:25:29.358562Z",
     "iopub.status.idle": "2022-11-29T18:25:33.399042Z",
     "shell.execute_reply": "2022-11-29T18:25:33.398095Z"
    },
    "papermill": {
     "duration": 4.057196,
     "end_time": "2022-11-29T18:25:33.401743",
     "exception": false,
     "start_time": "2022-11-29T18:25:29.344547",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "v112_CFG = Config(\n",
    "    num_workers=1,\n",
    "    batch_size=3,\n",
    "    max_len=512,\n",
    "    model=\"microsoft/deberta-v3-base\",\n",
    "    name='v112',\n",
    "    version='1',\n",
    "    num_target = 6,\n",
    "    reinit_last_layer=True,\n",
    "    reinit_fc=True,\n",
    "    weight_decay=0.01,\n",
    "    learning_rate=2e-5,\n",
    "    layerwise_learning_rate_decay=1.5,\n",
    "    use_dropout=False,\n",
    "    prob_dropout=[0.06, 0.08, 0.1, 0.12, 0.14],\n",
    "    num_dropout=5,\n",
    "    pool_head='attention',\n",
    "    seed=42,\n",
    "    n_fold=4,\n",
    "    trn_fold=[0,1,2,3],\n",
    "    train=True,\n",
    "    path='../input/fb3models/v112/',\n",
    "    config_path='../input/fb3models/v112/config.pth',\n",
    "    inference_weight=1.0,\n",
    "    tokenizer=AutoTokenizer.from_pretrained('../input/fb3models/v112/tokenizer')\n",
    ")\n",
    "\n",
    "#####\n",
    "v2_CFG = load_config('../input/fb3models/v2/', inference_weight=1.0)\n",
    "v2_CFG.name = 'v2'\n",
    "v2_CFG.version = '2'\n",
    "v2_CFG.trn_fold = [0,1,2,3]\n",
    "\n",
    "#####\n",
    "v21_CFG = load_config('../input/fb3models/v21/', inference_weight=1)\n",
    "v21_CFG.name = 'v21'\n",
    "v21_CFG.version = '21'\n",
    "\n",
    "#####\n",
    "attention_fgm_CFG = load_config('../input/fb3models/20221114-192943-deberta-v3-base/', inference_weight=1.0)\n",
    "attention_fgm_CFG.name = 'attention_fgm'\n",
    "attention_fgm_CFG.version = 'custom'\n",
    "attention_fgm_CFG.config_path = '../input/fb3models/20221114-192943-deberta-v3-base/config/config.json'\n",
    "\n",
    "weighted_fgm_CFG = Config(\n",
    "    pretrained=False,\n",
    "    path='../input/fb3models/20221115-061243-deberta-v3-base',\n",
    "    config_path='../input/fb3models/20221115-061243-deberta-v3-base/config/config.json',\n",
    "    tokenizer=AutoTokenizer.from_pretrained('../input/fb3models/20221115-061243-deberta-v3-base/tokenizer'),\n",
    "    name='weighted_fgm',\n",
    "    version='custom',\n",
    "    train = True,\n",
    "    debug = False,\n",
    "    offline = False,\n",
    "    models_path = 'FB3-models',\n",
    "    epochs = 5,\n",
    "    save_all_models = False,\n",
    "    competition = 'FB3',\n",
    "    apex = True,\n",
    "    print_freq = 20,\n",
    "    num_workers = 4,\n",
    "    model = 'microsoft/deberta-v3-base', #If you want to train on the kaggle platform, v3-base is realistic. v3-large will time out.\n",
    "    loss_func = 'SmoothL1', # 'SmoothL1', 'RMSE'\n",
    "    gradient_checkpointing = True,\n",
    "    scheduler = 'cosine',\n",
    "    batch_scheduler = True,\n",
    "    num_cycles = 0.5,\n",
    "    num_warmup_steps = 0,\n",
    "    encoder_lr = 2e-5,\n",
    "    decoder_lr = 2e-5,\n",
    "    min_lr = 1e-6,\n",
    "    #Layer-Wise Learning Rate Decay\n",
    "    llrd = True,\n",
    "    layerwise_lr = 5e-5,\n",
    "    layerwise_lr_decay = 0.9,\n",
    "    layerwise_weight_decay = 0.01,\n",
    "    layerwise_adam_epsilon = 1e-6,\n",
    "    layerwise_use_bertadam = False,\n",
    "    #pooling\n",
    "    pooling = 'weightedlayer', # mean, max, min, attention, weightedlayer\n",
    "    layer_start = 11,\n",
    "    layers=None,\n",
    "    #init_weight\n",
    "    init_weight = 'normal', # normal, xavier_uniform, xavier_normal, kaiming_uniform, kaiming_normal, orthogonal\n",
    "    #re-init\n",
    "    reinit = True,\n",
    "    reinit_n = 1,\n",
    "    #adversarial\n",
    "    fgm = True,\n",
    "    awp = False,\n",
    "    adv_lr = 1,\n",
    "    adv_eps = 0.2,\n",
    "    unscale = False,\n",
    "    eps = 1e-6,\n",
    "    betas = (0.9, 0.999),\n",
    "    max_len = 512,\n",
    "    weight_decay = 0.01,\n",
    "    gradient_accumulation_steps = 1,\n",
    "    max_grad_norm = 1000,\n",
    "    target_cols = ['cohesion', 'syntax', 'vocabulary', 'phraseology', 'grammar', 'conventions'],\n",
    "    seed = 42,\n",
    "    cv_seed = 42,\n",
    "    n_fold = 4,\n",
    "    trn_fold = [0,1,2,3],\n",
    "    batch_size = 8,\n",
    "    n_targets = 6,\n",
    "    gpu_id = 0) \n",
    "\n",
    "weighted_attention_CFG = load_config('../input/fb3models/weighted_attention_v3', inference_weight=1.0)\n",
    "weighted_attention_CFG.name = 'weighted_attention'\n",
    "weighted_attention_CFG.version = 'weighted-attention'\n",
    "\n",
    "mean_attention_no_fgm_CFG = load_config('../input/fb3models/20221117-183420-deberta-v3-base-mean-attention-with-mask', inference_weight=1.0)\n",
    "mean_attention_no_fgm_CFG.name = 'mean_attention_no_fgm'\n",
    "mean_attention_no_fgm_CFG.version = 'mean-attention'\n",
    "\n",
    "attention_large_fgm_CFG = load_config('../input/fb3models/20221118-164148-deberta-v3-large-attention_fgm')\n",
    "attention_large_fgm_CFG.name = 'attention_large_fgm'\n",
    "attention_large_fgm_CFG.version = 'custom'\n",
    "attention_large_fgm_CFG.config_path = '../input/fb3models/20221118-164148-deberta-v3-large-attention_fgm/config/config.json'\n",
    "\n",
    "attention_fgm_512_CFG = load_config('../input/fb3models/20221121-143655-deberta-v3-base-attention_fgm_512')\n",
    "attention_fgm_512_CFG.name = 'attention_fgm_512'\n",
    "attention_fgm_512_CFG.version = 'custom'\n",
    "attention_fgm_512_CFG.config_path = '../input/fb3models/20221121-143655-deberta-v3-base-attention_fgm_512/config/config.json'\n",
    "\n",
    "attention_fgm_768_CFG = load_config('../input/fb3models/20221120-072218-deberta-v3-base-attention_fgm')\n",
    "attention_fgm_768_CFG.name = 'attention_fgm_768'\n",
    "attention_fgm_768_CFG.version = 'custom'\n",
    "attention_fgm_768_CFG.config_path = '../input/fb3models/20221120-072218-deberta-v3-base-attention_fgm/config/config.json'\n",
    "\n",
    "weighted2last_fgm_512_CFG = load_config('../input/fb3models/20221124-060246-deberta-v3-base-weighted2last_fgm')\n",
    "weighted2last_fgm_512_CFG.name = 'weighted2last_fgm_512'\n",
    "weighted2last_fgm_512_CFG.version = 'custom'\n",
    "weighted2last_fgm_512_CFG.config_path = '../input/fb3models/20221124-060246-deberta-v3-base-weighted2last_fgm/config/config.json'\n",
    "\n",
    "weightedmean2last_fgm_512_CFG = load_config('../input/fb3models/20221124-160318-deberta-v3-base-weightedmean2last_fgm')\n",
    "weightedmean2last_fgm_512_CFG.name = 'weightedmean2last_fgm_512'\n",
    "weightedmean2last_fgm_512_CFG.version = 'custom'\n",
    "weightedmean2last_fgm_512_CFG.config_path = '../input/fb3models/20221124-160318-deberta-v3-base-weightedmean2last_fgm/config/config.json'\n",
    "\n",
    "roberta_attention_fgm_CFG = load_config('../input/fb3models/20221121-173739-roberta-base')\n",
    "roberta_attention_fgm_CFG.name = 'roberta_attention_large_fgm'\n",
    "roberta_attention_fgm_CFG.version = 'custom'\n",
    "roberta_attention_fgm_CFG.config_path = '../input/fb3models/20221121-173739-roberta-base/config/config.json'\n",
    "\n",
    "##########\n",
    "v112_CFG.pretrained = False\n",
    "v114_CFG.pretrained = False\n",
    "v116_CFG.pretrained = False\n",
    "v21_CFG.pretrained = False\n",
    "v2_CFG.pretrained = False\n",
    "attention_fgm_CFG.pretrained = False\n",
    "weighted_attention_CFG.pretrained = False\n",
    "weightedpool_CFG.pretrained = False\n",
    "mean_attention_no_fgm_CFG.pretrained=False\n",
    "attention_large_fgm_CFG.pretrained=False\n",
    "attention_fgm_512_CFG.pretrained = False\n",
    "attention_fgm_768_CFG.pretrained = False\n",
    "weighted2last_fgm_512_CFG.pretrained = False\n",
    "weightedmean2last_fgm_512_CFG.pretrained = False\n",
    "roberta_attention_fgm_CFG.pretrained = False\n",
    "\n",
    "\n",
    "weighted_fgm_CFG.inference_weight = 1.0\n",
    "v114_CFG.inference_weight = 1.0 \n",
    "v2_CFG.inference_weight = 1.0 \n",
    "v21_CFG.inference_weight = 1.0\n",
    "attention_fgm_CFG.inference_weight = 1.0\n",
    "weighted_attention_CFG.inference_weight = 1.0\n",
    "attention_large_fgm_CFG.inference_weight = 1.0\n",
    "attention_fgm_512_CFG.inference_weight = 1.0\n",
    "attention_fgm_768_CFG.inference_weight = 1.0\n",
    "weighted2last_fgm_512_CFG.inference_weight = 1.0\n",
    "weightedmean2last_fgm_512_CFG.inference_weight = 1.0\n",
    "roberta_attention_fgm_CFG.inference_weight = 1.0"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "id": "7545368a",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2022-11-29T18:25:33.427013Z",
     "iopub.status.busy": "2022-11-29T18:25:33.426702Z",
     "iopub.status.idle": "2022-11-29T18:25:33.433928Z",
     "shell.execute_reply": "2022-11-29T18:25:33.432992Z"
    },
    "papermill": {
     "duration": 0.022372,
     "end_time": "2022-11-29T18:25:33.436217",
     "exception": false,
     "start_time": "2022-11-29T18:25:33.413845",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "attention_large_fgm 0.02473498233215548\n",
      "weighted2last_fgm_512 0.03886925795053004\n",
      "weightedmean2last_fgm_512 0.049469964664310966\n",
      "attention_fgm_512 0.05653710247349824\n",
      "attention_fgm_768 0.06007067137809188\n",
      "v21 0.06713780918727916\n",
      "v112 0.08127208480565372\n",
      "mean_attention_no_fgm 0.10247349823321557\n",
      "v2 0.1660777385159011\n",
      "weightedpool 0.353356890459364\n",
      "Total number of models: 10\n"
     ]
    }
   ],
   "source": [
    "fine_tuned_models_cfg = [\n",
    "    attention_large_fgm_CFG, weighted2last_fgm_512_CFG, weightedmean2last_fgm_512_CFG, attention_fgm_512_CFG, attention_fgm_768_CFG, \n",
    "    v21_CFG, v112_CFG, mean_attention_no_fgm_CFG, v2_CFG, weightedpool_CFG]\n",
    "optimal_weights = sorted([\n",
    "    0.353356890459364,\n",
    "    0.1660777385159011,\n",
    "    0.10247349823321557,\n",
    "    0.05653710247349824,\n",
    "    0.06007067137809188,\n",
    "    0.049469964664310966,\n",
    "    0.08127208480565372,\n",
    "    0.06713780918727916,\n",
    "    0.03886925795053004,\n",
    "    0.02473498233215548\n",
    "])\n",
    "\n",
    "for i,cfg in enumerate(fine_tuned_models_cfg):\n",
    "    fine_tuned_models_cfg[i].inference_weight = optimal_weights[i]\n",
    "    print(fine_tuned_models_cfg[i].name, fine_tuned_models_cfg[i].inference_weight)\n",
    "    \n",
    "print('Total number of models:', len(fine_tuned_models_cfg))\n",
    "stacking = False"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "id": "f1ffde93",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2022-11-29T18:25:33.459707Z",
     "iopub.status.busy": "2022-11-29T18:25:33.459368Z",
     "iopub.status.idle": "2022-11-29T18:34:32.397858Z",
     "shell.execute_reply": "2022-11-29T18:34:32.396547Z"
    },
    "papermill": {
     "duration": 538.966366,
     "end_time": "2022-11-29T18:34:32.413423",
     "exception": false,
     "start_time": "2022-11-29T18:25:33.447057",
     "status": "completed"
    },
    "scrolled": true,
    "tags": []
   },
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "b28086d9fb754daaafb5b1e87bf9112c",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "  0%|          | 0/10 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "score: 0.452063  scores: [0.48157374223152627, 0.4452090627651757, 0.4154085184288849, 0.4536476489584571, 0.47240178087223106, 0.44413673294015477]\n",
      "preprocess\n",
      "########## ../input/fb3models/20221118-164148-deberta-v3-large-attention_fgm ##########\n",
      "Predicting fold 0...\n",
      "########## ETA: 27.44s ########## \n",
      "\n",
      "########## ../input/fb3models/20221118-164148-deberta-v3-large-attention_fgm ##########\n",
      "Predicting fold 1...\n",
      "########## ETA: 26.37s ########## \n",
      "\n",
      "########## ../input/fb3models/20221118-164148-deberta-v3-large-attention_fgm ##########\n",
      "Predicting fold 2...\n",
      "########## ETA: 27.87s ########## \n",
      "\n",
      "########## ../input/fb3models/20221118-164148-deberta-v3-large-attention_fgm ##########\n",
      "Predicting fold 3...\n",
      "########## ETA: 26.70s ########## \n",
      "\n",
      "score: 0.453033  scores: [0.48389889660631685, 0.44678944048963176, 0.41465519997691047, 0.45360604538329913, 0.4717536101123449, 0.4474973036716195]\n",
      "max_len=512\n",
      "preprocess\n",
      "########## ../input/fb3models/20221124-060246-deberta-v3-base-weighted2last_fgm ##########\n",
      "Predicting fold 0...\n",
      "########## ETA: 11.94s ########## \n",
      "\n",
      "########## ../input/fb3models/20221124-060246-deberta-v3-base-weighted2last_fgm ##########\n",
      "Predicting fold 1...\n",
      "########## ETA: 10.60s ########## \n",
      "\n",
      "########## ../input/fb3models/20221124-060246-deberta-v3-base-weighted2last_fgm ##########\n",
      "Predicting fold 2...\n",
      "########## ETA: 11.34s ########## \n",
      "\n",
      "########## ../input/fb3models/20221124-060246-deberta-v3-base-weighted2last_fgm ##########\n",
      "Predicting fold 3...\n",
      "########## ETA: 11.37s ########## \n",
      "\n",
      "score: 0.452766  scores: [0.48130576936482516, 0.4463892153474189, 0.4154102670831803, 0.4533288332620337, 0.471398800994229, 0.4487615040883342]\n",
      "max_len=512\n",
      "preprocess\n",
      "########## ../input/fb3models/20221124-160318-deberta-v3-base-weightedmean2last_fgm ##########\n",
      "Predicting fold 0...\n",
      "########## ETA: 12.04s ########## \n",
      "\n",
      "########## ../input/fb3models/20221124-160318-deberta-v3-base-weightedmean2last_fgm ##########\n",
      "Predicting fold 1...\n",
      "########## ETA: 11.34s ########## \n",
      "\n",
      "########## ../input/fb3models/20221124-160318-deberta-v3-base-weightedmean2last_fgm ##########\n",
      "Predicting fold 2...\n",
      "########## ETA: 12.20s ########## \n",
      "\n",
      "########## ../input/fb3models/20221124-160318-deberta-v3-base-weightedmean2last_fgm ##########\n",
      "Predicting fold 3...\n",
      "########## ETA: 11.57s ########## \n",
      "\n",
      "score: 0.454154  scores: [0.48439478507550315, 0.4476619277192037, 0.41633279856096794, 0.4540781761924119, 0.4753848249763217, 0.4470715017077387]\n",
      "max_len=512\n",
      "preprocess\n",
      "########## ../input/fb3models/20221121-143655-deberta-v3-base-attention_fgm_512 ##########\n",
      "Predicting fold 0...\n",
      "########## ETA: 11.63s ########## \n",
      "\n",
      "########## ../input/fb3models/20221121-143655-deberta-v3-base-attention_fgm_512 ##########\n",
      "Predicting fold 1...\n",
      "########## ETA: 11.61s ########## \n",
      "\n",
      "########## ../input/fb3models/20221121-143655-deberta-v3-base-attention_fgm_512 ##########\n",
      "Predicting fold 2...\n",
      "########## ETA: 11.23s ########## \n",
      "\n",
      "########## ../input/fb3models/20221121-143655-deberta-v3-base-attention_fgm_512 ##########\n",
      "Predicting fold 3...\n",
      "########## ETA: 11.47s ########## \n",
      "\n",
      "score: 0.452899  scores: [0.4811456435689267, 0.4475128373931811, 0.4152303639507731, 0.4542057256276823, 0.47347977633523913, 0.4458225341021222]\n",
      "max_len=768\n",
      "preprocess\n",
      "########## ../input/fb3models/20221120-072218-deberta-v3-base-attention_fgm ##########\n",
      "Predicting fold 0...\n",
      "########## ETA: 12.17s ########## \n",
      "\n",
      "########## ../input/fb3models/20221120-072218-deberta-v3-base-attention_fgm ##########\n",
      "Predicting fold 1...\n",
      "########## ETA: 12.42s ########## \n",
      "\n",
      "########## ../input/fb3models/20221120-072218-deberta-v3-base-attention_fgm ##########\n",
      "Predicting fold 2...\n",
      "########## ETA: 12.37s ########## \n",
      "\n",
      "########## ../input/fb3models/20221120-072218-deberta-v3-base-attention_fgm ##########\n",
      "Predicting fold 3...\n",
      "########## ETA: 10.42s ########## \n",
      "\n",
      "score: 0.457102  scores: [0.48798883116384423, 0.44872254666199257, 0.42012263776409287, 0.4555113322262075, 0.4788518700393329, 0.45141752961770576]\n",
      "preprocess\n",
      "########## ../input/fb3models/v21/ ##########\n",
      "Predicting fold 0...\n",
      "########## ETA: 11.86s ########## \n",
      "\n",
      "########## ../input/fb3models/v21/ ##########\n",
      "Predicting fold 1...\n",
      "########## ETA: 12.24s ########## \n",
      "\n",
      "########## ../input/fb3models/v21/ ##########\n",
      "Predicting fold 2...\n",
      "########## ETA: 12.17s ########## \n",
      "\n",
      "########## ../input/fb3models/v21/ ##########\n",
      "Predicting fold 3...\n",
      "########## ETA: 12.41s ########## \n",
      "\n",
      "score: 0.453517  scores: [0.4855633786238422, 0.44703209067484795, 0.41299155171034585, 0.4534606468878697, 0.4717306751532209, 0.4503226293834107]\n",
      "preprocess\n",
      "########## ../input/fb3models/v112/ ##########\n",
      "Predicting fold 0...\n",
      "########## ETA: 11.80s ########## \n",
      "\n",
      "########## ../input/fb3models/v112/ ##########\n",
      "Predicting fold 1...\n",
      "########## ETA: 13.62s ########## \n",
      "\n",
      "########## ../input/fb3models/v112/ ##########\n",
      "Predicting fold 2...\n",
      "########## ETA: 11.53s ########## \n",
      "\n",
      "########## ../input/fb3models/v112/ ##########\n",
      "Predicting fold 3...\n",
      "########## ETA: 12.29s ########## \n",
      "\n",
      "score: 0.452924  scores: [0.48674365143324116, 0.44712902084312334, 0.41187184815036604, 0.4543429818169591, 0.4707566701582764, 0.44670122183285366]\n",
      "preprocess\n",
      "########## ../input/fb3models/20221117-183420-deberta-v3-base-mean-attention-with-mask ##########\n",
      "Predicting fold 0...\n",
      "########## ETA: 11.55s ########## \n",
      "\n",
      "########## ../input/fb3models/20221117-183420-deberta-v3-base-mean-attention-with-mask ##########\n",
      "Predicting fold 1...\n",
      "########## ETA: 12.12s ########## \n",
      "\n",
      "########## ../input/fb3models/20221117-183420-deberta-v3-base-mean-attention-with-mask ##########\n",
      "Predicting fold 2...\n",
      "########## ETA: 12.06s ########## \n",
      "\n",
      "########## ../input/fb3models/20221117-183420-deberta-v3-base-mean-attention-with-mask ##########\n",
      "Predicting fold 3...\n",
      "########## ETA: 11.46s ########## \n",
      "\n",
      "score: 0.456659  scores: [0.4862241326420343, 0.45220519968314205, 0.41968210267694495, 0.45746617426033787, 0.4755639270504457, 0.44881410849161313]\n",
      "preprocess\n",
      "########## ../input/fb3models/v2/ ##########\n",
      "Predicting fold 0...\n",
      "########## ETA: 12.12s ########## \n",
      "\n",
      "########## ../input/fb3models/v2/ ##########\n",
      "Predicting fold 1...\n",
      "########## ETA: 12.16s ########## \n",
      "\n",
      "########## ../input/fb3models/v2/ ##########\n",
      "Predicting fold 2...\n",
      "########## ETA: 11.67s ########## \n",
      "\n",
      "########## ../input/fb3models/v2/ ##########\n",
      "Predicting fold 3...\n",
      "########## ETA: 12.60s ########## \n",
      "\n",
      "score: 0.465777  scores: [0.4974350833689818, 0.4560072906674801, 0.42235798291514465, 0.46613323072377133, 0.4908856845317672, 0.46184042368634104]\n",
      "preprocess\n",
      "########## ../input/fb3-train/ ##########\n",
      "Predicting fold 0...\n",
      "########## ETA: 12.06s ########## \n",
      "\n",
      "########## ../input/fb3-train/ ##########\n",
      "Predicting fold 1...\n",
      "########## ETA: 12.33s ########## \n",
      "\n",
      "########## ../input/fb3-train/ ##########\n",
      "Predicting fold 2...\n",
      "########## ETA: 12.52s ########## \n",
      "\n",
      "########## ../input/fb3-train/ ##########\n",
      "Predicting fold 3...\n",
      "########## ETA: 12.22s ########## \n",
      "\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "array([[2.6278195, 2.4231508, 2.6908426, 2.359161 , 2.1601028, 2.5797715],\n",
       "       [3.481747 , 3.3418107, 3.51337  , 3.4832504, 3.3819103, 3.305533 ],\n",
       "       [2.9401727, 2.8120873, 3.152896 , 3.0040162, 2.7296119, 2.702186 ]],\n",
       "      dtype=float32)"
      ]
     },
     "execution_count": 20,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "fine_tuned_predictions = []\n",
    "total_weight = 0\n",
    "for cfg in tqdm(fine_tuned_models_cfg):\n",
    "    # infer_ = Inferencer(setup['path'], setup['inference_weight'])\n",
    "    infer_ = Inferencer(cfg=cfg, inference_weight=cfg.inference_weight)\n",
    "    if cfg.path in [\n",
    "        attention_large_fgm_CFG.path,\n",
    "        attention_fgm_CFG.path, attention_fgm_768_CFG.path, roberta_attention_fgm_CFG.path, attention_fgm_512_CFG.path, \n",
    "        weighted2last_fgm_512_CFG.path, weightedmean2last_fgm_512_CFG.path, \n",
    "    ]:\n",
    "        file_type = 'csv'\n",
    "    else:\n",
    "        file_type = 'pkl'\n",
    "    \n",
    "    infer_.get_oof_result(file_type)\n",
    "    \n",
    "    if cfg.path == roberta_attention_fgm_CFG.path:\n",
    "        collate_fn = DataCollatorWithPadding(tokenizer=cfg.tokenizer, max_length=cfg.max_len, padding='max_length')\n",
    "    elif '512' in cfg.name:\n",
    "        collate_fn = DataCollatorWithPadding(tokenizer=cfg.tokenizer, padding='max_length', max_length=512)\n",
    "        print('max_len=512')\n",
    "    elif '768' in cfg.name:\n",
    "        collate_fn = DataCollatorWithPadding(tokenizer=cfg.tokenizer, padding='max_length', max_length=768)\n",
    "        print('max_len=768')\n",
    "    else:\n",
    "        collate_fn = DataCollatorWithPadding(tokenizer=cfg.tokenizer, padding='longest')\n",
    "    test_dataset = TestDataset(cfg, test)\n",
    "    test_loader = DataLoader(\n",
    "        test_dataset,\n",
    "        batch_size=12,\n",
    "        shuffle=False,\n",
    "        collate_fn=collate_fn,\n",
    "        num_workers=2, \n",
    "        pin_memory=True, \n",
    "        drop_last=False)\n",
    "    \n",
    "    if stacking:\n",
    "        prediction = infer_.predict(test_loader, device)\n",
    "    else:\n",
    "        prediction = infer_.predict(test_loader, device) * cfg.inference_weight\n",
    "    \n",
    "    fine_tuned_predictions.append(prediction)\n",
    "    total_weight += cfg.inference_weight\n",
    "    \n",
    "    del infer_, test_dataset, test_loader, prediction; gc.collect; torch.cuda.empty_cache();\n",
    "    \n",
    "if stacking:\n",
    "    from glob import glob\n",
    "    from sklearn.linear_model import Ridge, Lasso, BayesianRidge\n",
    "    from joblib import dump, load\n",
    "    if str(device) == 'cpu':\n",
    "        from sklearn.svm import SVR\n",
    "    else:\n",
    "        from cuml.svm import SVR\n",
    "\n",
    "    final_fine_tuned_predictions = np.stack(fine_tuned_predictions, axis=1).reshape(-1, len(fine_tuned_models_cfg)*6)\n",
    "    final_fine_tuned_predictions_ = []\n",
    "    for meta_model_path in sorted(glob('../input/fb3-stacking/*.model')):\n",
    "        meta_model = load(meta_model_path)\n",
    "        meta_model_preds = meta_model.predict(final_fine_tuned_predictions)\n",
    "        final_fine_tuned_predictions_.append(meta_model_preds)\n",
    "    final_fine_tuned_predictions = np.mean(final_fine_tuned_predictions_, axis=0)\n",
    "else:\n",
    "    final_fine_tuned_predictions = np.sum(fine_tuned_predictions, axis=0)/total_weight\n",
    "    \n",
    "final_fine_tuned_predictions"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "id": "69eabc5e",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2022-11-29T18:34:32.445416Z",
     "iopub.status.busy": "2022-11-29T18:34:32.445019Z",
     "iopub.status.idle": "2022-11-29T18:34:32.451960Z",
     "shell.execute_reply": "2022-11-29T18:34:32.450809Z"
    },
    "papermill": {
     "duration": 0.025291,
     "end_time": "2022-11-29T18:34:32.454193",
     "exception": false,
     "start_time": "2022-11-29T18:34:32.428902",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "1.0000000000000002"
      ]
     },
     "execution_count": 21,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "total_weight"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "948390c6",
   "metadata": {
    "papermill": {
     "duration": 0.014855,
     "end_time": "2022-11-29T18:34:32.483926",
     "exception": false,
     "start_time": "2022-11-29T18:34:32.469071",
     "status": "completed"
    },
    "tags": []
   },
   "source": [
    "# Stackings"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "id": "3d8fedb3",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2022-11-29T18:34:32.515290Z",
     "iopub.status.busy": "2022-11-29T18:34:32.514924Z",
     "iopub.status.idle": "2022-11-29T18:34:32.520882Z",
     "shell.execute_reply": "2022-11-29T18:34:32.519875Z"
    },
    "papermill": {
     "duration": 0.024382,
     "end_time": "2022-11-29T18:34:32.523143",
     "exception": false,
     "start_time": "2022-11-29T18:34:32.498761",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "# from glob import glob\n",
    "# from sklearn.linear_model import Ridge, Lasso, BayesianRidge\n",
    "# from joblib import dump, load\n",
    "# if str(device) == 'cpu':\n",
    "#     from sklearn.svm import SVR\n",
    "# else:\n",
    "#     from cuml.svm import SVR"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "547da112",
   "metadata": {
    "papermill": {
     "duration": 0.014598,
     "end_time": "2022-11-29T18:34:32.552511",
     "exception": false,
     "start_time": "2022-11-29T18:34:32.537913",
     "status": "completed"
    },
    "tags": []
   },
   "source": [
    "## bad models"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "id": "6eaf98fe",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2022-11-29T18:34:32.583225Z",
     "iopub.status.busy": "2022-11-29T18:34:32.582488Z",
     "iopub.status.idle": "2022-11-29T18:34:32.588972Z",
     "shell.execute_reply": "2022-11-29T18:34:32.588013Z"
    },
    "papermill": {
     "duration": 0.023846,
     "end_time": "2022-11-29T18:34:32.591043",
     "exception": false,
     "start_time": "2022-11-29T18:34:32.567197",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "# use_weights = True\n",
    "\n",
    "# stacking_models_cfg = [\n",
    "#     mean_attention_no_fgm_CFG,\n",
    "#     v2_CFG, \n",
    "#     attention_large_fgm_CFG,\n",
    "#     v116_CFG, \n",
    "#     weightedpool_CFG, \n",
    "#     weighted_fgm_CFG,\n",
    "#     roberta_attention_fgm_CFG,\n",
    "#     attention_fgm_512_CFG,\n",
    "# ]\n",
    "# optimal_weights = sorted([0.10381541,0.02391661,-0.01459959,0.00525827,0.03253774,0.66132155,-0.1136627,0.30141271], reverse=True)\n",
    "\n",
    "# for i, ftm_cfg in enumerate(stacking_models_cfg):\n",
    "#     if use_weights:\n",
    "#         stacking_models_cfg[i].inference_weight = optimal_weights[i]\n",
    "#     else:\n",
    "#         stacking_models_cfg[i].inference_weight = 1.0\n",
    "    \n",
    "# weak_stack_predictions = []\n",
    "# for cfg in tqdm(stacking_models_cfg):\n",
    "#     infer_ = Inferencer(cfg=cfg, inference_weight=cfg.inference_weight)\n",
    "#     if cfg.path in [attention_fgm_CFG.path, attention_fgm_768_CFG.path, roberta_attention_fgm_CFG.path, attention_fgm_512_CFG.path, attention_large_fgm_CFG.path]:\n",
    "#         file_type = 'csv'\n",
    "#     else:\n",
    "#         file_type = 'pkl'\n",
    "#     infer_.get_oof_result(file_type)\n",
    "    \n",
    "#     test_dataset = TestDataset(cfg, test)\n",
    "#     test_loader = DataLoader(\n",
    "#         test_dataset,\n",
    "#         batch_size=12,\n",
    "#         shuffle=False,\n",
    "#         collate_fn=DataCollatorWithPadding(tokenizer=cfg.tokenizer, padding='longest'),\n",
    "#         num_workers=2, \n",
    "#         pin_memory=True, \n",
    "#         drop_last=False)\n",
    "    \n",
    "#     prediction = infer_.predict(test_loader, device)\n",
    "#     weak_stack_predictions.append(prediction)\n",
    "    \n",
    "#     del infer_, test_dataset, test_loader, prediction; gc.collect; torch.cuda.empty_cache();\n",
    "\n",
    "# if use_weights:\n",
    "#     ensemble_predictions = np.zeros((len(test), 6))\n",
    "#     for i, cfg in enumerate(stacking_models_cfg):\n",
    "#         ensemble_predictions += weak_stack_predictions[i] * cfg.inference_weight\n",
    "#     weak_stack_predictions.append(ensemble_predictions)\n",
    "    \n",
    "# final_weak_stack_predictions = np.concatenate(weak_stack_predictions, axis=1)\n",
    "# final_weak_stack_predictions_ = []\n",
    "# for meta_model_path in sorted(glob('../input/fb3-stacking/*.model')):\n",
    "#     if '_weak_' in meta_model_path:\n",
    "#         meta_model = load(meta_model_path)\n",
    "#         meta_model_preds = meta_model.predict(final_weak_stack_predictions)\n",
    "#         final_weak_stack_predictions_.append(meta_model_preds)\n",
    "# final_weak_stack_predictions = np.mean(final_weak_stack_predictions_, axis=0)\n",
    "    \n",
    "# final_weak_stack_predictions"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "id": "9214ddf3",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2022-11-29T18:34:32.620507Z",
     "iopub.status.busy": "2022-11-29T18:34:32.620210Z",
     "iopub.status.idle": "2022-11-29T18:34:32.624589Z",
     "shell.execute_reply": "2022-11-29T18:34:32.623509Z"
    },
    "papermill": {
     "duration": 0.021777,
     "end_time": "2022-11-29T18:34:32.626945",
     "exception": false,
     "start_time": "2022-11-29T18:34:32.605168",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "# final_weak_stack_predictions"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "60d3cb71",
   "metadata": {
    "papermill": {
     "duration": 0.013671,
     "end_time": "2022-11-29T18:34:32.654803",
     "exception": false,
     "start_time": "2022-11-29T18:34:32.641132",
     "status": "completed"
    },
    "tags": []
   },
   "source": [
    "## good models"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "id": "22770afc",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2022-11-29T18:34:32.683439Z",
     "iopub.status.busy": "2022-11-29T18:34:32.683172Z",
     "iopub.status.idle": "2022-11-29T18:34:32.688394Z",
     "shell.execute_reply": "2022-11-29T18:34:32.687547Z"
    },
    "papermill": {
     "duration": 0.02159,
     "end_time": "2022-11-29T18:34:32.690449",
     "exception": false,
     "start_time": "2022-11-29T18:34:32.668859",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "# use_weights = True"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "id": "4fefbc60",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2022-11-29T18:34:32.718408Z",
     "iopub.status.busy": "2022-11-29T18:34:32.718090Z",
     "iopub.status.idle": "2022-11-29T18:34:32.723509Z",
     "shell.execute_reply": "2022-11-29T18:34:32.722507Z"
    },
    "papermill": {
     "duration": 0.021959,
     "end_time": "2022-11-29T18:34:32.725866",
     "exception": false,
     "start_time": "2022-11-29T18:34:32.703907",
     "status": "completed"
    },
    "scrolled": true,
    "tags": []
   },
   "outputs": [],
   "source": [
    "# optimal_weights = [0.29754274, 0.44465927, 0.25779799, 1]\n",
    "# stacking_models_cfg = [v112_CFG, v21_CFG, weighted_attention_CFG, attention_fgm_CFG]\n",
    "\n",
    "# if use_weights:\n",
    "#     for i, ftm_cfg in enumerate(stacking_models_cfg):\n",
    "#         stacking_models_cfg[i].inference_weight = optimal_weights[i]\n",
    "    \n",
    "# strong_stack_predictions = []\n",
    "# for cfg in tqdm(stacking_models_cfg):\n",
    "#     if not use_weights:\n",
    "#         cfg.inference_weight = 1.0\n",
    "#     infer_ = Inferencer(cfg=cfg, inference_weight=cfg.inference_weight)\n",
    "#     if cfg.path == attention_fgm_CFG.path:\n",
    "#         file_type = 'csv'\n",
    "#     else:\n",
    "#         file_type = 'pkl'\n",
    "#     infer_.get_oof_result(file_type)\n",
    "    \n",
    "#     test_dataset = TestDataset(cfg, test)\n",
    "#     test_loader = DataLoader(\n",
    "#         test_dataset,\n",
    "#         batch_size=12,\n",
    "#         shuffle=False,\n",
    "#         collate_fn=DataCollatorWithPadding(tokenizer=cfg.tokenizer, padding='longest'),\n",
    "#         num_workers=2, \n",
    "#         pin_memory=True, \n",
    "#         drop_last=False)\n",
    "    \n",
    "#     prediction = infer_.predict(test_loader, device)\n",
    "#     strong_stack_predictions.append(prediction)\n",
    "    \n",
    "#     del infer_, test_dataset, test_loader, prediction; gc.collect; torch.cuda.empty_cache();\n",
    "\n",
    "# if use_weights:\n",
    "#     ensemble_predictions = np.zeros((len(test), 6))\n",
    "#     for i, cfg in enumerate(stacking_models_cfg):\n",
    "#         ensemble_predictions += strong_stack_predictions[i] * cfg.inference_weight\n",
    "#     strong_stack_predictions.append(ensemble_predictions)\n",
    "    \n",
    "# # final_strong_stack_predictions = np.stack(strong_stack_predictions, axis=1).reshape(-1, len(stacking_models_cfg)*6)\n",
    "# final_strong_stack_predictions = np.concatenate(strong_stack_predictions, axis=1)\n",
    "# final_strong_stack_predictions_ = []\n",
    "# for meta_model_path in sorted(glob('../input/fb3-stacking/*.model')):\n",
    "#     if '_strong_' in meta_model_path:\n",
    "#         meta_model = load(meta_model_path)\n",
    "#         meta_model_preds = meta_model.predict(final_strong_stack_predictions)\n",
    "#         final_strong_stack_predictions_.append(meta_model_preds)\n",
    "# final_strong_stack_predictions = np.mean(final_strong_stack_predictions_, axis=0)\n",
    "\n",
    "# final_strong_stack_predictions"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "17d50b90",
   "metadata": {
    "papermill": {
     "duration": 0.012757,
     "end_time": "2022-11-29T18:34:32.751894",
     "exception": false,
     "start_time": "2022-11-29T18:34:32.739137",
     "status": "completed"
    },
    "tags": []
   },
   "source": [
    "# Combine predictions"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "id": "13ebad92",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2022-11-29T18:34:32.779806Z",
     "iopub.status.busy": "2022-11-29T18:34:32.779535Z",
     "iopub.status.idle": "2022-11-29T18:34:32.788260Z",
     "shell.execute_reply": "2022-11-29T18:34:32.787326Z"
    },
    "papermill": {
     "duration": 0.025253,
     "end_time": "2022-11-29T18:34:32.790442",
     "exception": false,
     "start_time": "2022-11-29T18:34:32.765189",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "final_predictions = (0.345*svr_predictions + 0.655*final_fine_tuned_predictions)/1\n",
    "final_predictions = np.clip(final_predictions, 1, 5)\n",
    "test[target_cols] = final_predictions"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "727af8b6",
   "metadata": {
    "papermill": {
     "duration": 0.013296,
     "end_time": "2022-11-29T18:34:32.817759",
     "exception": false,
     "start_time": "2022-11-29T18:34:32.804463",
     "status": "completed"
    },
    "tags": []
   },
   "source": [
    "# Postprocess"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "id": "5a1486a4",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2022-11-29T18:34:32.845533Z",
     "iopub.status.busy": "2022-11-29T18:34:32.845275Z",
     "iopub.status.idle": "2022-11-29T18:34:32.849435Z",
     "shell.execute_reply": "2022-11-29T18:34:32.848355Z"
    },
    "papermill": {
     "duration": 0.020709,
     "end_time": "2022-11-29T18:34:32.851828",
     "exception": false,
     "start_time": "2022-11-29T18:34:32.831119",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "# !mkdir -p /tmp/pip/cache/\n",
    "# !cp ../input/hdbscan0828whl/hdbscan-0.8.28-cp37-cp37m-linux_x86_64.whl /tmp/pip/cache/\n",
    "# !ls /tmp/pip/cache/\n",
    "# !pip install --no-index --find-links /tmp/pip/cache/ hdbscan\n",
    "\n",
    "# sys.path.append('../input/sentence-transformers/sentence-transformers-master/')\n",
    "# sys.path.append('../input/bertopic/BERTopic/')\n",
    "# from bertopic import BERTopic"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "id": "3376adb8",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2022-11-29T18:34:32.879495Z",
     "iopub.status.busy": "2022-11-29T18:34:32.879224Z",
     "iopub.status.idle": "2022-11-29T18:34:32.883458Z",
     "shell.execute_reply": "2022-11-29T18:34:32.882433Z"
    },
    "papermill": {
     "duration": 0.020771,
     "end_time": "2022-11-29T18:34:32.885742",
     "exception": false,
     "start_time": "2022-11-29T18:34:32.864971",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "# stopwords_ = stopwords.words('english') + [\"n't\",  \"'s\", \"'ve\"]\n",
    "# test_docs = []\n",
    "# for fl in tqdm(test['full_text']):\n",
    "#     word_tokens = word_tokenize(fl)\n",
    "#     txt = \" \".join([w for w in word_tokens if not w.lower() in stopwords_])\n",
    "#     test_docs.append(txt)\n",
    "    \n",
    "# bertopic_model = BERTopic.load('../input/fb3-bertopic/bertopic_train.model')\n",
    "# topics, probs = bertopic_model.transform(test_docs)\n",
    "# test['topic_idx'] = topics"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "id": "5857c659",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2022-11-29T18:34:32.913086Z",
     "iopub.status.busy": "2022-11-29T18:34:32.912822Z",
     "iopub.status.idle": "2022-11-29T18:34:32.917049Z",
     "shell.execute_reply": "2022-11-29T18:34:32.916016Z"
    },
    "papermill": {
     "duration": 0.02032,
     "end_time": "2022-11-29T18:34:32.919345",
     "exception": false,
     "start_time": "2022-11-29T18:34:32.899025",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "# train_topics = pd.read_csv('../input/fb3-bertopic/train_topics.csv')\n",
    "# target_cols = ['cohesion', 'syntax', 'vocabulary', 'phraseology', 'grammar', 'conventions']\n",
    "# df_ = train_topics.set_index('topic_name')[target_cols].stack() .reset_index()\n",
    "# df_.columns = ['topic', 'score', 'value']\n",
    "# topic_stats = df_.groupby(['topic', 'score'])['value'].agg(['min', 'mean', 'median', 'max']).reset_index()\n",
    "# topic_stats['topic_idx'] = topic_stats['topic'].str.split('_', expand=True)[0].astype(int)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "id": "4f572b90",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2022-11-29T18:34:32.946450Z",
     "iopub.status.busy": "2022-11-29T18:34:32.946185Z",
     "iopub.status.idle": "2022-11-29T18:34:32.951380Z",
     "shell.execute_reply": "2022-11-29T18:34:32.950552Z"
    },
    "papermill": {
     "duration": 0.021149,
     "end_time": "2022-11-29T18:34:32.953465",
     "exception": false,
     "start_time": "2022-11-29T18:34:32.932316",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "# for target in target_cols:\n",
    "#     for topic in sorted(test['topic_idx'].unique()):\n",
    "#         if topic == -1:\n",
    "#             continue\n",
    "#         topic_max = topic_stats[(topic_stats['topic_idx'] == topic) & (topic_stats['score'] == target)]['max'].values[0]\n",
    "#         topic_min = topic_stats[(topic_stats['topic_idx'] == topic) & (topic_stats['score'] == target)]['min'].values[0]\n",
    "#         print(f'topic={topic}    target={target}    min,max={topic_min,topic_max}')\n",
    "#         test.loc[test['topic_idx'] == topic, target] = test[target].apply(lambda x: np.clip(x, topic_min, topic_max))\n",
    "#     print('='*50)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "48a3d564",
   "metadata": {
    "papermill": {
     "duration": 0.012944,
     "end_time": "2022-11-29T18:34:32.979763",
     "exception": false,
     "start_time": "2022-11-29T18:34:32.966819",
     "status": "completed"
    },
    "tags": []
   },
   "source": [
    "# submit"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "id": "98ea7034",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2022-11-29T18:34:33.007870Z",
     "iopub.status.busy": "2022-11-29T18:34:33.007542Z",
     "iopub.status.idle": "2022-11-29T18:34:33.067295Z",
     "shell.execute_reply": "2022-11-29T18:34:33.066338Z"
    },
    "papermill": {
     "duration": 0.076356,
     "end_time": "2022-11-29T18:34:33.069600",
     "exception": false,
     "start_time": "2022-11-29T18:34:32.993244",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>text_id</th>\n",
       "      <th>cohesion</th>\n",
       "      <th>syntax</th>\n",
       "      <th>vocabulary</th>\n",
       "      <th>phraseology</th>\n",
       "      <th>grammar</th>\n",
       "      <th>conventions</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>0000C359D63E</td>\n",
       "      <td>2.934860</td>\n",
       "      <td>2.804275</td>\n",
       "      <td>3.133245</td>\n",
       "      <td>2.997666</td>\n",
       "      <td>2.714023</td>\n",
       "      <td>2.670210</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>000BAD50D026</td>\n",
       "      <td>2.648663</td>\n",
       "      <td>2.441112</td>\n",
       "      <td>2.688795</td>\n",
       "      <td>2.328176</td>\n",
       "      <td>2.086922</td>\n",
       "      <td>2.611527</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>00367BB2546B</td>\n",
       "      <td>3.541466</td>\n",
       "      <td>3.380309</td>\n",
       "      <td>3.533994</td>\n",
       "      <td>3.544498</td>\n",
       "      <td>3.395614</td>\n",
       "      <td>3.315024</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "        text_id  cohesion    syntax  vocabulary  phraseology   grammar  conventions\n",
       "0  0000C359D63E  2.934860  2.804275    3.133245     2.997666  2.714023     2.670210\n",
       "1  000BAD50D026  2.648663  2.441112    2.688795     2.328176  2.086922     2.611527\n",
       "2  00367BB2546B  3.541466  3.380309    3.533994     3.544498  3.395614     3.315024"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "submission = pd.read_csv(SUBMISSION_PATH)\n",
    "submission = submission.drop(columns=target_cols).merge(test[['text_id'] + target_cols], on='text_id', how='left')\n",
    "display(submission.head())\n",
    "submission[['text_id'] + target_cols].to_csv('submission.csv', index=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "bae3af5d",
   "metadata": {
    "papermill": {
     "duration": 0.013329,
     "end_time": "2022-11-29T18:34:33.096700",
     "exception": false,
     "start_time": "2022-11-29T18:34:33.083371",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.12"
  },
  "papermill": {
   "default_parameters": {},
   "duration": 1030.419894,
   "end_time": "2022-11-29T18:34:36.031886",
   "environment_variables": {},
   "exception": null,
   "input_path": "__notebook__.ipynb",
   "output_path": "__notebook__.ipynb",
   "parameters": {},
   "start_time": "2022-11-29T18:17:25.611992",
   "version": "2.3.4"
  },
  "widgets": {
   "application/vnd.jupyter.widget-state+json": {
    "state": {
     "06f431efdde54401a88feba8ea383d05": {
      "model_module": "@jupyter-widgets/controls",
      "model_module_version": "1.5.0",
      "model_name": "FloatProgressModel",
      "state": {
       "_dom_classes": [],
       "_model_module": "@jupyter-widgets/controls",
       "_model_module_version": "1.5.0",
       "_model_name": "FloatProgressModel",
       "_view_count": null,
       "_view_module": "@jupyter-widgets/controls",
       "_view_module_version": "1.5.0",
       "_view_name": "ProgressView",
       "bar_style": "success",
       "description": "",
       "description_tooltip": null,
       "layout": "IPY_MODEL_9283401e969146a39088b2e77ae22090",
       "max": 15.0,
       "min": 0.0,
       "orientation": "horizontal",
       "style": "IPY_MODEL_37b28ba15e714db79c5ed55eff01946e",
       "value": 15.0
      }
     },
     "0a055fa1eeb346c3a746794056ed8022": {
      "model_module": "@jupyter-widgets/controls",
      "model_module_version": "1.5.0",
      "model_name": "HTMLModel",
      "state": {
       "_dom_classes": [],
       "_model_module": "@jupyter-widgets/controls",
       "_model_module_version": "1.5.0",
       "_model_name": "HTMLModel",
       "_view_count": null,
       "_view_module": "@jupyter-widgets/controls",
       "_view_module_version": "1.5.0",
       "_view_name": "HTMLView",
       "description": "",
       "description_tooltip": null,
       "layout": "IPY_MODEL_805e68c3b16844b78abf6ad7a7929fa0",
       "placeholder": "​",
       "style": "IPY_MODEL_5fa5b1e252454f15a807e2faf8bb55b6",
       "value": "100%"
      }
     },
     "106055fdb24947f98c35319450705206": {
      "model_module": "@jupyter-widgets/base",
      "model_module_version": "1.2.0",
      "model_name": "LayoutModel",
      "state": {
       "_model_module": "@jupyter-widgets/base",
       "_model_module_version": "1.2.0",
       "_model_name": "LayoutModel",
       "_view_count": null,
       "_view_module": "@jupyter-widgets/base",
       "_view_module_version": "1.2.0",
       "_view_name": "LayoutView",
       "align_content": null,
       "align_items": null,
       "align_self": null,
       "border": null,
       "bottom": null,
       "display": null,
       "flex": null,
       "flex_flow": null,
       "grid_area": null,
       "grid_auto_columns": null,
       "grid_auto_flow": null,
       "grid_auto_rows": null,
       "grid_column": null,
       "grid_gap": null,
       "grid_row": null,
       "grid_template_areas": null,
       "grid_template_columns": null,
       "grid_template_rows": null,
       "height": null,
       "justify_content": null,
       "justify_items": null,
       "left": null,
       "margin": null,
       "max_height": null,
       "max_width": null,
       "min_height": null,
       "min_width": null,
       "object_fit": null,
       "object_position": null,
       "order": null,
       "overflow": null,
       "overflow_x": null,
       "overflow_y": null,
       "padding": null,
       "right": null,
       "top": null,
       "visibility": null,
       "width": null
      }
     },
     "189b794761ec4e118d9eeed3d855e769": {
      "model_module": "@jupyter-widgets/controls",
      "model_module_version": "1.5.0",
      "model_name": "FloatProgressModel",
      "state": {
       "_dom_classes": [],
       "_model_module": "@jupyter-widgets/controls",
       "_model_module_version": "1.5.0",
       "_model_name": "FloatProgressModel",
       "_view_count": null,
       "_view_module": "@jupyter-widgets/controls",
       "_view_module_version": "1.5.0",
       "_view_name": "ProgressView",
       "bar_style": "success",
       "description": "",
       "description_tooltip": null,
       "layout": "IPY_MODEL_b58c098d6a9b4f8697ff30c82421f3f2",
       "max": 14.0,
       "min": 0.0,
       "orientation": "horizontal",
       "style": "IPY_MODEL_99cf6ca0035a4fbaada0520c17cb12be",
       "value": 14.0
      }
     },
     "233d45e7a181463cbe02dacb56da4ff1": {
      "model_module": "@jupyter-widgets/base",
      "model_module_version": "1.2.0",
      "model_name": "LayoutModel",
      "state": {
       "_model_module": "@jupyter-widgets/base",
       "_model_module_version": "1.2.0",
       "_model_name": "LayoutModel",
       "_view_count": null,
       "_view_module": "@jupyter-widgets/base",
       "_view_module_version": "1.2.0",
       "_view_name": "LayoutView",
       "align_content": null,
       "align_items": null,
       "align_self": null,
       "border": null,
       "bottom": null,
       "display": null,
       "flex": null,
       "flex_flow": null,
       "grid_area": null,
       "grid_auto_columns": null,
       "grid_auto_flow": null,
       "grid_auto_rows": null,
       "grid_column": null,
       "grid_gap": null,
       "grid_row": null,
       "grid_template_areas": null,
       "grid_template_columns": null,
       "grid_template_rows": null,
       "height": null,
       "justify_content": null,
       "justify_items": null,
       "left": null,
       "margin": null,
       "max_height": null,
       "max_width": null,
       "min_height": null,
       "min_width": null,
       "object_fit": null,
       "object_position": null,
       "order": null,
       "overflow": null,
       "overflow_x": null,
       "overflow_y": null,
       "padding": null,
       "right": null,
       "top": null,
       "visibility": null,
       "width": null
      }
     },
     "23e7b74423ad4cfeb2c80ced3d256a13": {
      "model_module": "@jupyter-widgets/base",
      "model_module_version": "1.2.0",
      "model_name": "LayoutModel",
      "state": {
       "_model_module": "@jupyter-widgets/base",
       "_model_module_version": "1.2.0",
       "_model_name": "LayoutModel",
       "_view_count": null,
       "_view_module": "@jupyter-widgets/base",
       "_view_module_version": "1.2.0",
       "_view_name": "LayoutView",
       "align_content": null,
       "align_items": null,
       "align_self": null,
       "border": null,
       "bottom": null,
       "display": null,
       "flex": null,
       "flex_flow": null,
       "grid_area": null,
       "grid_auto_columns": null,
       "grid_auto_flow": null,
       "grid_auto_rows": null,
       "grid_column": null,
       "grid_gap": null,
       "grid_row": null,
       "grid_template_areas": null,
       "grid_template_columns": null,
       "grid_template_rows": null,
       "height": null,
       "justify_content": null,
       "justify_items": null,
       "left": null,
       "margin": null,
       "max_height": null,
       "max_width": null,
       "min_height": null,
       "min_width": null,
       "object_fit": null,
       "object_position": null,
       "order": null,
       "overflow": null,
       "overflow_x": null,
       "overflow_y": null,
       "padding": null,
       "right": null,
       "top": null,
       "visibility": null,
       "width": null
      }
     },
     "347711f8419248b6a111a665c2813134": {
      "model_module": "@jupyter-widgets/controls",
      "model_module_version": "1.5.0",
      "model_name": "DescriptionStyleModel",
      "state": {
       "_model_module": "@jupyter-widgets/controls",
       "_model_module_version": "1.5.0",
       "_model_name": "DescriptionStyleModel",
       "_view_count": null,
       "_view_module": "@jupyter-widgets/base",
       "_view_module_version": "1.2.0",
       "_view_name": "StyleView",
       "description_width": ""
      }
     },
     "35eca863db854202a11306ecc2651d5a": {
      "model_module": "@jupyter-widgets/base",
      "model_module_version": "1.2.0",
      "model_name": "LayoutModel",
      "state": {
       "_model_module": "@jupyter-widgets/base",
       "_model_module_version": "1.2.0",
       "_model_name": "LayoutModel",
       "_view_count": null,
       "_view_module": "@jupyter-widgets/base",
       "_view_module_version": "1.2.0",
       "_view_name": "LayoutView",
       "align_content": null,
       "align_items": null,
       "align_self": null,
       "border": null,
       "bottom": null,
       "display": null,
       "flex": null,
       "flex_flow": null,
       "grid_area": null,
       "grid_auto_columns": null,
       "grid_auto_flow": null,
       "grid_auto_rows": null,
       "grid_column": null,
       "grid_gap": null,
       "grid_row": null,
       "grid_template_areas": null,
       "grid_template_columns": null,
       "grid_template_rows": null,
       "height": null,
       "justify_content": null,
       "justify_items": null,
       "left": null,
       "margin": null,
       "max_height": null,
       "max_width": null,
       "min_height": null,
       "min_width": null,
       "object_fit": null,
       "object_position": null,
       "order": null,
       "overflow": null,
       "overflow_x": null,
       "overflow_y": null,
       "padding": null,
       "right": null,
       "top": null,
       "visibility": null,
       "width": null
      }
     },
     "37b28ba15e714db79c5ed55eff01946e": {
      "model_module": "@jupyter-widgets/controls",
      "model_module_version": "1.5.0",
      "model_name": "ProgressStyleModel",
      "state": {
       "_model_module": "@jupyter-widgets/controls",
       "_model_module_version": "1.5.0",
       "_model_name": "ProgressStyleModel",
       "_view_count": null,
       "_view_module": "@jupyter-widgets/base",
       "_view_module_version": "1.2.0",
       "_view_name": "StyleView",
       "bar_color": null,
       "description_width": ""
      }
     },
     "4c104b4b66a84a3ca090e373bff513c4": {
      "model_module": "@jupyter-widgets/controls",
      "model_module_version": "1.5.0",
      "model_name": "HTMLModel",
      "state": {
       "_dom_classes": [],
       "_model_module": "@jupyter-widgets/controls",
       "_model_module_version": "1.5.0",
       "_model_name": "HTMLModel",
       "_view_count": null,
       "_view_module": "@jupyter-widgets/controls",
       "_view_module_version": "1.5.0",
       "_view_name": "HTMLView",
       "description": "",
       "description_tooltip": null,
       "layout": "IPY_MODEL_9fe1ff5be7324753914abcfee6a8cc9a",
       "placeholder": "​",
       "style": "IPY_MODEL_5c2318d717d0425483d4f1dfeb278629",
       "value": "100%"
      }
     },
     "5b518247cd634fa8bbbe3514f63fc8d1": {
      "model_module": "@jupyter-widgets/controls",
      "model_module_version": "1.5.0",
      "model_name": "HTMLModel",
      "state": {
       "_dom_classes": [],
       "_model_module": "@jupyter-widgets/controls",
       "_model_module_version": "1.5.0",
       "_model_name": "HTMLModel",
       "_view_count": null,
       "_view_module": "@jupyter-widgets/controls",
       "_view_module_version": "1.5.0",
       "_view_name": "HTMLView",
       "description": "",
       "description_tooltip": null,
       "layout": "IPY_MODEL_9b8ff7f96e144578af5bbb71029dd747",
       "placeholder": "​",
       "style": "IPY_MODEL_61cabb526ecb44c4b591c814cd806e06",
       "value": " 10/10 [08:58&lt;00:00, 49.25s/it]"
      }
     },
     "5c2318d717d0425483d4f1dfeb278629": {
      "model_module": "@jupyter-widgets/controls",
      "model_module_version": "1.5.0",
      "model_name": "DescriptionStyleModel",
      "state": {
       "_model_module": "@jupyter-widgets/controls",
       "_model_module_version": "1.5.0",
       "_model_name": "DescriptionStyleModel",
       "_view_count": null,
       "_view_module": "@jupyter-widgets/base",
       "_view_module_version": "1.2.0",
       "_view_name": "StyleView",
       "description_width": ""
      }
     },
     "5fa5b1e252454f15a807e2faf8bb55b6": {
      "model_module": "@jupyter-widgets/controls",
      "model_module_version": "1.5.0",
      "model_name": "DescriptionStyleModel",
      "state": {
       "_model_module": "@jupyter-widgets/controls",
       "_model_module_version": "1.5.0",
       "_model_name": "DescriptionStyleModel",
       "_view_count": null,
       "_view_module": "@jupyter-widgets/base",
       "_view_module_version": "1.2.0",
       "_view_name": "StyleView",
       "description_width": ""
      }
     },
     "61cabb526ecb44c4b591c814cd806e06": {
      "model_module": "@jupyter-widgets/controls",
      "model_module_version": "1.5.0",
      "model_name": "DescriptionStyleModel",
      "state": {
       "_model_module": "@jupyter-widgets/controls",
       "_model_module_version": "1.5.0",
       "_model_name": "DescriptionStyleModel",
       "_view_count": null,
       "_view_module": "@jupyter-widgets/base",
       "_view_module_version": "1.2.0",
       "_view_name": "StyleView",
       "description_width": ""
      }
     },
     "6a91791f7d934e2ca322c0592bb10c55": {
      "model_module": "@jupyter-widgets/controls",
      "model_module_version": "1.5.0",
      "model_name": "HTMLModel",
      "state": {
       "_dom_classes": [],
       "_model_module": "@jupyter-widgets/controls",
       "_model_module_version": "1.5.0",
       "_model_name": "HTMLModel",
       "_view_count": null,
       "_view_module": "@jupyter-widgets/controls",
       "_view_module_version": "1.5.0",
       "_view_name": "HTMLView",
       "description": "",
       "description_tooltip": null,
       "layout": "IPY_MODEL_83f05b99a7c04ed29a6ecb75bc5706fa",
       "placeholder": "​",
       "style": "IPY_MODEL_86718f4c50864a97bc33918329d0d17d",
       "value": " 15/15 [02:47&lt;00:00, 10.79s/it]"
      }
     },
     "70741e3a448f415089d47bacb71bbe09": {
      "model_module": "@jupyter-widgets/controls",
      "model_module_version": "1.5.0",
      "model_name": "DescriptionStyleModel",
      "state": {
       "_model_module": "@jupyter-widgets/controls",
       "_model_module_version": "1.5.0",
       "_model_name": "DescriptionStyleModel",
       "_view_count": null,
       "_view_module": "@jupyter-widgets/base",
       "_view_module_version": "1.2.0",
       "_view_name": "StyleView",
       "description_width": ""
      }
     },
     "805e68c3b16844b78abf6ad7a7929fa0": {
      "model_module": "@jupyter-widgets/base",
      "model_module_version": "1.2.0",
      "model_name": "LayoutModel",
      "state": {
       "_model_module": "@jupyter-widgets/base",
       "_model_module_version": "1.2.0",
       "_model_name": "LayoutModel",
       "_view_count": null,
       "_view_module": "@jupyter-widgets/base",
       "_view_module_version": "1.2.0",
       "_view_name": "LayoutView",
       "align_content": null,
       "align_items": null,
       "align_self": null,
       "border": null,
       "bottom": null,
       "display": null,
       "flex": null,
       "flex_flow": null,
       "grid_area": null,
       "grid_auto_columns": null,
       "grid_auto_flow": null,
       "grid_auto_rows": null,
       "grid_column": null,
       "grid_gap": null,
       "grid_row": null,
       "grid_template_areas": null,
       "grid_template_columns": null,
       "grid_template_rows": null,
       "height": null,
       "justify_content": null,
       "justify_items": null,
       "left": null,
       "margin": null,
       "max_height": null,
       "max_width": null,
       "min_height": null,
       "min_width": null,
       "object_fit": null,
       "object_position": null,
       "order": null,
       "overflow": null,
       "overflow_x": null,
       "overflow_y": null,
       "padding": null,
       "right": null,
       "top": null,
       "visibility": null,
       "width": null
      }
     },
     "82c1dcd75b4346d3990e554cdc0d9587": {
      "model_module": "@jupyter-widgets/base",
      "model_module_version": "1.2.0",
      "model_name": "LayoutModel",
      "state": {
       "_model_module": "@jupyter-widgets/base",
       "_model_module_version": "1.2.0",
       "_model_name": "LayoutModel",
       "_view_count": null,
       "_view_module": "@jupyter-widgets/base",
       "_view_module_version": "1.2.0",
       "_view_name": "LayoutView",
       "align_content": null,
       "align_items": null,
       "align_self": null,
       "border": null,
       "bottom": null,
       "display": null,
       "flex": null,
       "flex_flow": null,
       "grid_area": null,
       "grid_auto_columns": null,
       "grid_auto_flow": null,
       "grid_auto_rows": null,
       "grid_column": null,
       "grid_gap": null,
       "grid_row": null,
       "grid_template_areas": null,
       "grid_template_columns": null,
       "grid_template_rows": null,
       "height": null,
       "justify_content": null,
       "justify_items": null,
       "left": null,
       "margin": null,
       "max_height": null,
       "max_width": null,
       "min_height": null,
       "min_width": null,
       "object_fit": null,
       "object_position": null,
       "order": null,
       "overflow": null,
       "overflow_x": null,
       "overflow_y": null,
       "padding": null,
       "right": null,
       "top": null,
       "visibility": null,
       "width": null
      }
     },
     "83f05b99a7c04ed29a6ecb75bc5706fa": {
      "model_module": "@jupyter-widgets/base",
      "model_module_version": "1.2.0",
      "model_name": "LayoutModel",
      "state": {
       "_model_module": "@jupyter-widgets/base",
       "_model_module_version": "1.2.0",
       "_model_name": "LayoutModel",
       "_view_count": null,
       "_view_module": "@jupyter-widgets/base",
       "_view_module_version": "1.2.0",
       "_view_name": "LayoutView",
       "align_content": null,
       "align_items": null,
       "align_self": null,
       "border": null,
       "bottom": null,
       "display": null,
       "flex": null,
       "flex_flow": null,
       "grid_area": null,
       "grid_auto_columns": null,
       "grid_auto_flow": null,
       "grid_auto_rows": null,
       "grid_column": null,
       "grid_gap": null,
       "grid_row": null,
       "grid_template_areas": null,
       "grid_template_columns": null,
       "grid_template_rows": null,
       "height": null,
       "justify_content": null,
       "justify_items": null,
       "left": null,
       "margin": null,
       "max_height": null,
       "max_width": null,
       "min_height": null,
       "min_width": null,
       "object_fit": null,
       "object_position": null,
       "order": null,
       "overflow": null,
       "overflow_x": null,
       "overflow_y": null,
       "padding": null,
       "right": null,
       "top": null,
       "visibility": null,
       "width": null
      }
     },
     "86718f4c50864a97bc33918329d0d17d": {
      "model_module": "@jupyter-widgets/controls",
      "model_module_version": "1.5.0",
      "model_name": "DescriptionStyleModel",
      "state": {
       "_model_module": "@jupyter-widgets/controls",
       "_model_module_version": "1.5.0",
       "_model_name": "DescriptionStyleModel",
       "_view_count": null,
       "_view_module": "@jupyter-widgets/base",
       "_view_module_version": "1.2.0",
       "_view_name": "StyleView",
       "description_width": ""
      }
     },
     "869540960af14622915870d33ccde413": {
      "model_module": "@jupyter-widgets/controls",
      "model_module_version": "1.5.0",
      "model_name": "FloatProgressModel",
      "state": {
       "_dom_classes": [],
       "_model_module": "@jupyter-widgets/controls",
       "_model_module_version": "1.5.0",
       "_model_name": "FloatProgressModel",
       "_view_count": null,
       "_view_module": "@jupyter-widgets/controls",
       "_view_module_version": "1.5.0",
       "_view_name": "ProgressView",
       "bar_style": "success",
       "description": "",
       "description_tooltip": null,
       "layout": "IPY_MODEL_23e7b74423ad4cfeb2c80ced3d256a13",
       "max": 10.0,
       "min": 0.0,
       "orientation": "horizontal",
       "style": "IPY_MODEL_d6f5c15dc5c3469680ba28a0b8cccb31",
       "value": 10.0
      }
     },
     "9283401e969146a39088b2e77ae22090": {
      "model_module": "@jupyter-widgets/base",
      "model_module_version": "1.2.0",
      "model_name": "LayoutModel",
      "state": {
       "_model_module": "@jupyter-widgets/base",
       "_model_module_version": "1.2.0",
       "_model_name": "LayoutModel",
       "_view_count": null,
       "_view_module": "@jupyter-widgets/base",
       "_view_module_version": "1.2.0",
       "_view_name": "LayoutView",
       "align_content": null,
       "align_items": null,
       "align_self": null,
       "border": null,
       "bottom": null,
       "display": null,
       "flex": null,
       "flex_flow": null,
       "grid_area": null,
       "grid_auto_columns": null,
       "grid_auto_flow": null,
       "grid_auto_rows": null,
       "grid_column": null,
       "grid_gap": null,
       "grid_row": null,
       "grid_template_areas": null,
       "grid_template_columns": null,
       "grid_template_rows": null,
       "height": null,
       "justify_content": null,
       "justify_items": null,
       "left": null,
       "margin": null,
       "max_height": null,
       "max_width": null,
       "min_height": null,
       "min_width": null,
       "object_fit": null,
       "object_position": null,
       "order": null,
       "overflow": null,
       "overflow_x": null,
       "overflow_y": null,
       "padding": null,
       "right": null,
       "top": null,
       "visibility": null,
       "width": null
      }
     },
     "99cf6ca0035a4fbaada0520c17cb12be": {
      "model_module": "@jupyter-widgets/controls",
      "model_module_version": "1.5.0",
      "model_name": "ProgressStyleModel",
      "state": {
       "_model_module": "@jupyter-widgets/controls",
       "_model_module_version": "1.5.0",
       "_model_name": "ProgressStyleModel",
       "_view_count": null,
       "_view_module": "@jupyter-widgets/base",
       "_view_module_version": "1.2.0",
       "_view_name": "StyleView",
       "bar_color": null,
       "description_width": ""
      }
     },
     "9b8ff7f96e144578af5bbb71029dd747": {
      "model_module": "@jupyter-widgets/base",
      "model_module_version": "1.2.0",
      "model_name": "LayoutModel",
      "state": {
       "_model_module": "@jupyter-widgets/base",
       "_model_module_version": "1.2.0",
       "_model_name": "LayoutModel",
       "_view_count": null,
       "_view_module": "@jupyter-widgets/base",
       "_view_module_version": "1.2.0",
       "_view_name": "LayoutView",
       "align_content": null,
       "align_items": null,
       "align_self": null,
       "border": null,
       "bottom": null,
       "display": null,
       "flex": null,
       "flex_flow": null,
       "grid_area": null,
       "grid_auto_columns": null,
       "grid_auto_flow": null,
       "grid_auto_rows": null,
       "grid_column": null,
       "grid_gap": null,
       "grid_row": null,
       "grid_template_areas": null,
       "grid_template_columns": null,
       "grid_template_rows": null,
       "height": null,
       "justify_content": null,
       "justify_items": null,
       "left": null,
       "margin": null,
       "max_height": null,
       "max_width": null,
       "min_height": null,
       "min_width": null,
       "object_fit": null,
       "object_position": null,
       "order": null,
       "overflow": null,
       "overflow_x": null,
       "overflow_y": null,
       "padding": null,
       "right": null,
       "top": null,
       "visibility": null,
       "width": null
      }
     },
     "9d8177ec76e547659e8556d2485035f6": {
      "model_module": "@jupyter-widgets/base",
      "model_module_version": "1.2.0",
      "model_name": "LayoutModel",
      "state": {
       "_model_module": "@jupyter-widgets/base",
       "_model_module_version": "1.2.0",
       "_model_name": "LayoutModel",
       "_view_count": null,
       "_view_module": "@jupyter-widgets/base",
       "_view_module_version": "1.2.0",
       "_view_name": "LayoutView",
       "align_content": null,
       "align_items": null,
       "align_self": null,
       "border": null,
       "bottom": null,
       "display": null,
       "flex": null,
       "flex_flow": null,
       "grid_area": null,
       "grid_auto_columns": null,
       "grid_auto_flow": null,
       "grid_auto_rows": null,
       "grid_column": null,
       "grid_gap": null,
       "grid_row": null,
       "grid_template_areas": null,
       "grid_template_columns": null,
       "grid_template_rows": null,
       "height": null,
       "justify_content": null,
       "justify_items": null,
       "left": null,
       "margin": null,
       "max_height": null,
       "max_width": null,
       "min_height": null,
       "min_width": null,
       "object_fit": null,
       "object_position": null,
       "order": null,
       "overflow": null,
       "overflow_x": null,
       "overflow_y": null,
       "padding": null,
       "right": null,
       "top": null,
       "visibility": null,
       "width": null
      }
     },
     "9fe1ff5be7324753914abcfee6a8cc9a": {
      "model_module": "@jupyter-widgets/base",
      "model_module_version": "1.2.0",
      "model_name": "LayoutModel",
      "state": {
       "_model_module": "@jupyter-widgets/base",
       "_model_module_version": "1.2.0",
       "_model_name": "LayoutModel",
       "_view_count": null,
       "_view_module": "@jupyter-widgets/base",
       "_view_module_version": "1.2.0",
       "_view_name": "LayoutView",
       "align_content": null,
       "align_items": null,
       "align_self": null,
       "border": null,
       "bottom": null,
       "display": null,
       "flex": null,
       "flex_flow": null,
       "grid_area": null,
       "grid_auto_columns": null,
       "grid_auto_flow": null,
       "grid_auto_rows": null,
       "grid_column": null,
       "grid_gap": null,
       "grid_row": null,
       "grid_template_areas": null,
       "grid_template_columns": null,
       "grid_template_rows": null,
       "height": null,
       "justify_content": null,
       "justify_items": null,
       "left": null,
       "margin": null,
       "max_height": null,
       "max_width": null,
       "min_height": null,
       "min_width": null,
       "object_fit": null,
       "object_position": null,
       "order": null,
       "overflow": null,
       "overflow_x": null,
       "overflow_y": null,
       "padding": null,
       "right": null,
       "top": null,
       "visibility": null,
       "width": null
      }
     },
     "a10154d509ca4d7792786f67399dee13": {
      "model_module": "@jupyter-widgets/controls",
      "model_module_version": "1.5.0",
      "model_name": "HTMLModel",
      "state": {
       "_dom_classes": [],
       "_model_module": "@jupyter-widgets/controls",
       "_model_module_version": "1.5.0",
       "_model_name": "HTMLModel",
       "_view_count": null,
       "_view_module": "@jupyter-widgets/controls",
       "_view_module_version": "1.5.0",
       "_view_name": "HTMLView",
       "description": "",
       "description_tooltip": null,
       "layout": "IPY_MODEL_35eca863db854202a11306ecc2651d5a",
       "placeholder": "​",
       "style": "IPY_MODEL_347711f8419248b6a111a665c2813134",
       "value": " 14/14 [04:30&lt;00:00, 16.71s/it]"
      }
     },
     "b28086d9fb754daaafb5b1e87bf9112c": {
      "model_module": "@jupyter-widgets/controls",
      "model_module_version": "1.5.0",
      "model_name": "HBoxModel",
      "state": {
       "_dom_classes": [],
       "_model_module": "@jupyter-widgets/controls",
       "_model_module_version": "1.5.0",
       "_model_name": "HBoxModel",
       "_view_count": null,
       "_view_module": "@jupyter-widgets/controls",
       "_view_module_version": "1.5.0",
       "_view_name": "HBoxView",
       "box_style": "",
       "children": [
        "IPY_MODEL_0a055fa1eeb346c3a746794056ed8022",
        "IPY_MODEL_869540960af14622915870d33ccde413",
        "IPY_MODEL_5b518247cd634fa8bbbe3514f63fc8d1"
       ],
       "layout": "IPY_MODEL_82c1dcd75b4346d3990e554cdc0d9587"
      }
     },
     "b58c098d6a9b4f8697ff30c82421f3f2": {
      "model_module": "@jupyter-widgets/base",
      "model_module_version": "1.2.0",
      "model_name": "LayoutModel",
      "state": {
       "_model_module": "@jupyter-widgets/base",
       "_model_module_version": "1.2.0",
       "_model_name": "LayoutModel",
       "_view_count": null,
       "_view_module": "@jupyter-widgets/base",
       "_view_module_version": "1.2.0",
       "_view_name": "LayoutView",
       "align_content": null,
       "align_items": null,
       "align_self": null,
       "border": null,
       "bottom": null,
       "display": null,
       "flex": null,
       "flex_flow": null,
       "grid_area": null,
       "grid_auto_columns": null,
       "grid_auto_flow": null,
       "grid_auto_rows": null,
       "grid_column": null,
       "grid_gap": null,
       "grid_row": null,
       "grid_template_areas": null,
       "grid_template_columns": null,
       "grid_template_rows": null,
       "height": null,
       "justify_content": null,
       "justify_items": null,
       "left": null,
       "margin": null,
       "max_height": null,
       "max_width": null,
       "min_height": null,
       "min_width": null,
       "object_fit": null,
       "object_position": null,
       "order": null,
       "overflow": null,
       "overflow_x": null,
       "overflow_y": null,
       "padding": null,
       "right": null,
       "top": null,
       "visibility": null,
       "width": null
      }
     },
     "c3a5a1ac0ff840708129e31bc35fd6f6": {
      "model_module": "@jupyter-widgets/controls",
      "model_module_version": "1.5.0",
      "model_name": "HBoxModel",
      "state": {
       "_dom_classes": [],
       "_model_module": "@jupyter-widgets/controls",
       "_model_module_version": "1.5.0",
       "_model_name": "HBoxModel",
       "_view_count": null,
       "_view_module": "@jupyter-widgets/controls",
       "_view_module_version": "1.5.0",
       "_view_name": "HBoxView",
       "box_style": "",
       "children": [
        "IPY_MODEL_4c104b4b66a84a3ca090e373bff513c4",
        "IPY_MODEL_06f431efdde54401a88feba8ea383d05",
        "IPY_MODEL_6a91791f7d934e2ca322c0592bb10c55"
       ],
       "layout": "IPY_MODEL_233d45e7a181463cbe02dacb56da4ff1"
      }
     },
     "d6f5c15dc5c3469680ba28a0b8cccb31": {
      "model_module": "@jupyter-widgets/controls",
      "model_module_version": "1.5.0",
      "model_name": "ProgressStyleModel",
      "state": {
       "_model_module": "@jupyter-widgets/controls",
       "_model_module_version": "1.5.0",
       "_model_name": "ProgressStyleModel",
       "_view_count": null,
       "_view_module": "@jupyter-widgets/base",
       "_view_module_version": "1.2.0",
       "_view_name": "StyleView",
       "bar_color": null,
       "description_width": ""
      }
     },
     "db2f144862df4e6196309c4ba129d05a": {
      "model_module": "@jupyter-widgets/controls",
      "model_module_version": "1.5.0",
      "model_name": "HBoxModel",
      "state": {
       "_dom_classes": [],
       "_model_module": "@jupyter-widgets/controls",
       "_model_module_version": "1.5.0",
       "_model_name": "HBoxModel",
       "_view_count": null,
       "_view_module": "@jupyter-widgets/controls",
       "_view_module_version": "1.5.0",
       "_view_name": "HBoxView",
       "box_style": "",
       "children": [
        "IPY_MODEL_f6c0c3201cee49fd86f426353f47ecb0",
        "IPY_MODEL_189b794761ec4e118d9eeed3d855e769",
        "IPY_MODEL_a10154d509ca4d7792786f67399dee13"
       ],
       "layout": "IPY_MODEL_9d8177ec76e547659e8556d2485035f6"
      }
     },
     "f6c0c3201cee49fd86f426353f47ecb0": {
      "model_module": "@jupyter-widgets/controls",
      "model_module_version": "1.5.0",
      "model_name": "HTMLModel",
      "state": {
       "_dom_classes": [],
       "_model_module": "@jupyter-widgets/controls",
       "_model_module_version": "1.5.0",
       "_model_name": "HTMLModel",
       "_view_count": null,
       "_view_module": "@jupyter-widgets/controls",
       "_view_module_version": "1.5.0",
       "_view_name": "HTMLView",
       "description": "",
       "description_tooltip": null,
       "layout": "IPY_MODEL_106055fdb24947f98c35319450705206",
       "placeholder": "​",
       "style": "IPY_MODEL_70741e3a448f415089d47bacb71bbe09",
       "value": "100%"
      }
     }
    },
    "version_major": 2,
    "version_minor": 0
   }
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}