fb3-svr-train.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "49d4c467",
   "metadata": {
    "_cell_guid": "b1076dfc-b9ad-4769-8c92-a6c4dae69d19",
    "_uuid": "8f2839f25d086af736a60e9eeb907d3b93b6e0e5",
    "execution": {
     "iopub.execute_input": "2022-10-23T20:06:09.766288Z",
     "iopub.status.busy": "2022-10-23T20:06:09.765513Z",
     "iopub.status.idle": "2022-10-23T20:06:55.604800Z",
     "shell.execute_reply": "2022-10-23T20:06:55.603260Z"
    },
    "papermill": {
     "duration": 45.849559,
     "end_time": "2022-10-23T20:06:55.607922",
     "exception": false,
     "start_time": "2022-10-23T20:06:09.758363",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "tokenizers.__version__: 0.12.1\n",
      "transformers.__version__: 4.21.2\n",
      "device: cuda\n"
     ]
    }
   ],
   "source": [
    "# ====================================================\n",
    "# Library\n",
    "# ====================================================\n",
    "import os\n",
    "import gc\n",
    "import re\n",
    "import ast\n",
    "import sys\n",
    "import copy\n",
    "import json\n",
    "import time\n",
    "import math \n",
    "import string\n",
    "import pickle\n",
    "import random\n",
    "import joblib\n",
    "import itertools\n",
    "import warnings\n",
    "warnings.filterwarnings(\"ignore\")\n",
    "from IPython. display import clear_output\n",
    "\n",
    "import scipy as sp\n",
    "import numpy as np\n",
    "import pandas as pd\n",
    "pd.set_option('display.max_rows', 500)\n",
    "pd.set_option('display.max_columns', 500)\n",
    "pd.set_option('display.width', 1000)\n",
    "from tqdm.auto import tqdm\n",
    "from sklearn.metrics import mean_squared_error\n",
    "from sklearn.model_selection import StratifiedKFold, GroupKFold, KFold\n",
    "from nltk.corpus import stopwords\n",
    "from nltk.tokenize import word_tokenize\n",
    "\n",
    "import torch\n",
    "import torch.nn as nn\n",
    "from torch.nn import Parameter\n",
    "import torch.nn.functional as F\n",
    "from torch.optim import Adam, SGD, AdamW\n",
    "from torch.utils.data import DataLoader, Dataset\n",
    "\n",
    "#os.system('pip install iterative-stratification==0.1.7')\n",
    "#from iterstrat.ml_stratifiers import MultilabelStratifiedKFold\n",
    "\n",
    "os.system('pip uninstall -y transformers')\n",
    "os.system('pip uninstall -y tokenizers')\n",
    "os.system('python -m pip install --no-index --find-links=../input/fb3-my-pip-wheels transformers')\n",
    "os.system('python -m pip install --no-index --find-links=../input/fb3-my-pip-wheels tokenizers')\n",
    "import tokenizers\n",
    "import transformers\n",
    "from transformers import AutoTokenizer, AutoModel, AutoConfig\n",
    "from transformers import get_linear_schedule_with_warmup, get_cosine_schedule_with_warmup\n",
    "from transformers import DataCollatorWithPadding\n",
    "%env TOKENIZERS_PARALLELISM=false\n",
    "\n",
    "clear_output()\n",
    "device = torch.device('cuda' if torch.cuda.is_available() else 'cpu')\n",
    "print(f\"tokenizers.__version__: {tokenizers.__version__}\")\n",
    "print(f\"transformers.__version__: {transformers.__version__}\")\n",
    "print('device:', device)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "aed00faf",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2022-10-23T20:06:55.618841Z",
     "iopub.status.busy": "2022-10-23T20:06:55.618293Z",
     "iopub.status.idle": "2022-10-23T20:06:55.623868Z",
     "shell.execute_reply": "2022-10-23T20:06:55.622745Z"
    },
    "papermill": {
     "duration": 0.013112,
     "end_time": "2022-10-23T20:06:55.626028",
     "exception": false,
     "start_time": "2022-10-23T20:06:55.612916",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "BASE_PATH = '/kaggle/input/feedback-prize-english-language-learning'\n",
    "SUBMISSION_PATH = os.path.join(BASE_PATH, 'sample_submission.csv')\n",
    "TRAIN_PATH = os.path.join(BASE_PATH, 'train.csv')\n",
    "TEST_PATH = os.path.join(BASE_PATH, 'test.csv')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "3aa72bcb",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2022-10-23T20:06:55.636409Z",
     "iopub.status.busy": "2022-10-23T20:06:55.636113Z",
     "iopub.status.idle": "2022-10-23T20:06:55.674403Z",
     "shell.execute_reply": "2022-10-23T20:06:55.673257Z"
    },
    "papermill": {
     "duration": 0.046514,
     "end_time": "2022-10-23T20:06:55.676651",
     "exception": false,
     "start_time": "2022-10-23T20:06:55.630137",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "class FB3Dataset(Dataset):\n",
    "    def __init__(self, cfg, data):\n",
    "        self.cfg = cfg\n",
    "        self.xs = preprocess(data['full_text'])\n",
    "        self.ys = data[cfg.target_cols].values \n",
    "        \n",
    "    def __len__(self):\n",
    "        return len(self.xs)\n",
    "    \n",
    "    def __getitem__(self, idx):\n",
    "        x = encode_text(self.cfg, self.xs[idx])\n",
    "        y = torch.tensor(self.ys[idx], dtype=torch.float)\n",
    "        return x, y\n",
    "    \n",
    "def collate(inputs):\n",
    "    # Trimming input.\n",
    "    mask_len = int(inputs['attention_mask'].sum(axis=1).max())\n",
    "    for k, v in inputs.items():\n",
    "        inputs[k] = inputs[k][:,:mask_len]\n",
    "    #print(type(inputs), inputs['attention_mask'].size())\n",
    "    return inputs\n",
    "\n",
    "##################################################################################\n",
    "\n",
    "class MeanPooling(nn.Module):\n",
    "    def __init__(self):\n",
    "        super(MeanPooling, self).__init__()\n",
    "    \n",
    "    def forward(self, last_hidden_state, attention_mask):\n",
    "        input_mask_expanded = attention_mask.unsqueeze(-1).expand(last_hidden_state.size()).float()\n",
    "        sum_embeddings = torch.sum(last_hidden_state * input_mask_expanded, 1)\n",
    "        sum_mask = torch.clamp(input_mask_expanded.sum(1), min=1e-9)\n",
    "        mean_embeddings = sum_embeddings / sum_mask\n",
    "        return mean_embeddings\n",
    "\n",
    "class WeightedLayerPooling(nn.Module):\n",
    "    def __init__(self, num_hidden_layers, layer_start: int = 9, layer_weights = None):\n",
    "        super(WeightedLayerPooling, self).__init__()\n",
    "        self.layer_start = layer_start\n",
    "        self.num_hidden_layers = num_hidden_layers\n",
    "        self.layer_weights = layer_weights if layer_weights is not None \\\n",
    "            else nn.Parameter(\n",
    "                torch.tensor([1] * (num_hidden_layers+1 - layer_start), dtype=torch.float))\n",
    "        \n",
    "    def forward(self, all_hidden_states):\n",
    "        all_layer_embedding = all_hidden_states[self.layer_start:, :, :, :]\n",
    "        weight_factor = self.layer_weights.unsqueeze(-1).unsqueeze(-1).unsqueeze(-1).expand(all_layer_embedding.size())\n",
    "        weighted_average = (weight_factor*all_layer_embedding).sum(dim=0) / self.layer_weights.sum()\n",
    "        return weighted_average\n",
    "\n",
    "class AttentionHead(nn.Module):\n",
    "    def __init__(self, in_features, hidden_dim):\n",
    "        super().__init__()\n",
    "        self.in_features = in_features\n",
    "        self.middle_features = hidden_dim\n",
    "        self.W = nn.Linear(in_features, hidden_dim)\n",
    "        self.V = nn.Linear(hidden_dim, 1)\n",
    "        self.out_features = hidden_dim\n",
    "\n",
    "    def forward(self, features, *args):\n",
    "        att = torch.tanh(self.W(features))\n",
    "        score = self.V(att)\n",
    "        attention_weights = torch.softmax(score, dim=1)\n",
    "        context_vector = attention_weights * features\n",
    "        context_vector = torch.sum(context_vector, dim=1)\n",
    "\n",
    "        return context_vector\n",
    "\n",
    "def GlobalAveragePool1d(x):\n",
    "    return F.avg_pool1d(x, x.size()[-1]).squeeze(-1)\n",
    "\n",
    "def GlobalMaxPool1d(x):\n",
    "    return F.max_pool1d(x, x.size()[-1]).squeeze(-1)\n",
    "\n",
    "def Conv1dReg(x, in_channels, out_channels, kernel_size, device):\n",
    "    out = nn.Conv1d(in_channels, out_channels, kernel_size, padding='same', stride=1, device=device)(x)\n",
    "    out = nn.BatchNorm1d(out_channels, device=device)(out)\n",
    "    out = F.relu(out)\n",
    "    return out\n",
    "\n",
    "class MultiSampleDropout(nn.Module):\n",
    "    def __init__(self, fc, num_dropout, prob_dropout):\n",
    "        super(MultiSampleDropout, self).__init__()\n",
    "        self.dropout = nn.Dropout\n",
    "        self.num_dropout = num_dropout\n",
    "        self.prob_dropout = prob_dropout\n",
    "        self.classifier = fc\n",
    "    def forward(self, out):\n",
    "        if not type(self.prob_dropout) in [float, int]:            \n",
    "            fcs = [self.classifier(self.dropout(p)(out)) for p in self.prob_dropout]\n",
    "        else:\n",
    "            fcs = [self.classifier(self.dropout(self.prob_dropout)(out)) for _ in range(self.num_dropout)]\n",
    "        \n",
    "        return torch.mean(torch.stack(fcs, dim=0), dim=0)\n",
    "\n",
    "# ====================================================\n",
    "# Model class\n",
    "# ====================================================\n",
    "class FB3Model(nn.Module):\n",
    "    def __init__(self, cfg, config_path=None, pretrained=False, pool='mean'):\n",
    "        super().__init__()\n",
    "        self.cfg = cfg \n",
    "        if config_path is None:\n",
    "            self.config = AutoConfig.from_pretrained(cfg.model, output_hidden_states=True)\n",
    "            # Turn off dropouts.\n",
    "            self.config.hidden_dropout = 0.\n",
    "            self.config.hidden_dropout_prob = 0.\n",
    "            self.config.attention_dropout = 0.\n",
    "            self.config.attention_probs_dropout_prob = 0.\n",
    "            #LOGGER.info(self.config)\n",
    "        else:\n",
    "            self.config = torch.load(config_path)\n",
    "        \n",
    "        if pretrained:\n",
    "            self.deberta_v3 = AutoModel.from_pretrained(cfg.model, config=self.config)\n",
    "            # Expand embedding dim for new tokens.\n",
    "            self.deberta_v3.resize_token_embeddings(len(cfg.tokenizer))\n",
    "        else:\n",
    "            self.deberta_v3 = AutoModel.from_config(self.config)\n",
    "        \n",
    "        if self.cfg.reinit_last_layer:\n",
    "            # Re-init last layer of deberta.\n",
    "            for module in self.deberta_v3.encoder.layer[-1].modules():\n",
    "                self._init_weights(module)\n",
    "        self.deberta_v3.gradient_checkpointing_enable()\n",
    "        \n",
    "        # Define model layers.\n",
    "        self.pool_name = pool\n",
    "        self.fc = nn.Linear(self.config.hidden_size, cfg.num_target)\n",
    "        if pool in ['mean', 'attention', 'weighted']:\n",
    "            self.pool = self._pool_layer(pool)\n",
    "        elif '-' in pool:\n",
    "            pools = pool.split('-')\n",
    "            self.pool = nn.ModuleList([])\n",
    "            for pool_ in pools:\n",
    "                self.pool.append(self._pool_layer(pool_))\n",
    "            self.fc = nn.Linear(self.config.hidden_size * len(self.pool), cfg.num_target)\n",
    "    \n",
    "        if cfg.reinit_fc:\n",
    "            self._init_weights(self.fc)\n",
    "        \n",
    "        # Multi-sample dropout.\n",
    "        self.multi_dropout = MultiSampleDropout(self.fc, cfg.num_dropout, cfg.prob_dropout)\n",
    "    \n",
    "    def _pool_layer(self, pool_name):\n",
    "        assert pool_name in ['mean', 'attention', 'weighted']\n",
    "        if pool_name == 'mean':\n",
    "            pool = MeanPooling()\n",
    "        elif pool_name == 'attention':\n",
    "            pool = AttentionHead(self.config.hidden_size, self.config.hidden_size)\n",
    "        elif pool_name == 'weighted':\n",
    "            pool = WeightedLayerPooling(\n",
    "                self.config.num_hidden_layers, \n",
    "                layer_start=9,\n",
    "                layer_weights=None)\n",
    "        return pool\n",
    "    \n",
    "    def _pool_feature(self, pool, pool_name, pt_outputs, attention_mask):\n",
    "        assert pool_name in ['mean', 'attention', 'weighted']\n",
    "        last_hidden_state = pt_outputs.last_hidden_state #batch_size x max_len x hidden_size\n",
    "        all_hidden_states = torch.stack(pt_outputs.hidden_states) #num_layer x batch_size x max_len x hidden_size\n",
    "        \n",
    "        if pool_name == 'mean':\n",
    "            pool_feature = pool(last_hidden_state, attention_mask)\n",
    "        elif pool_name == 'attention':\n",
    "            pool_feature = pool(last_hidden_state)\n",
    "        elif pool_name == 'weighted':\n",
    "            # Take the CLS token only.\n",
    "            pool_feature = pool(all_hidden_states)[:, 0]\n",
    "        return pool_feature\n",
    "\n",
    "    def _init_weights(self, module):\n",
    "        if isinstance(module, nn.Linear):\n",
    "            module.weight.data.normal_(mean=0.0, std=self.config.initializer_range)\n",
    "            if module.bias is not None:\n",
    "                module.bias.data.zero_()\n",
    "        elif isinstance(module, nn.Embedding):\n",
    "            module.weight.data.normal_(mean=0.0, std=self.config.initializer_range)\n",
    "            if module.padding_idx is not None:\n",
    "                module.weight.data[module.padding_idx].zero_()\n",
    "        elif isinstance(module, nn.LayerNorm):\n",
    "            module.bias.data.zero_()\n",
    "            module.weight.data.fill_(1.0)\n",
    "    \n",
    "    def feature(self, x):\n",
    "        pt_outputs = self.deberta_v3(**x)\n",
    "        \n",
    "        # Pooling feat.\n",
    "        if type(self.pool) == nn.ModuleList:\n",
    "            pool_features = []\n",
    "            pool_names = self.pool_name.split('-')\n",
    "            \n",
    "            for pool_name, pool in zip(pool_names, self.pool):\n",
    "                pool_features.append(self._pool_feature(pool, pool_name, pt_outputs, x['attention_mask']))\n",
    "            pool_features = torch.cat(pool_features, dim=1)\n",
    "        else:\n",
    "            pool_features = self._pool_feature(self.pool, self.pool_name, pt_outputs, x['attention_mask'])\n",
    "        return pool_features\n",
    "    \n",
    "    def forward(self, x, y=None, loss_fn=None):\n",
    "        feature = self.feature(x)\n",
    "        if self.cfg.use_dropout and self.training:\n",
    "            out = self.multi_dropout(feature)\n",
    "        else:\n",
    "            out = self.fc(feature)\n",
    "        return out"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "a06535ce",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2022-10-23T20:06:55.686600Z",
     "iopub.status.busy": "2022-10-23T20:06:55.685993Z",
     "iopub.status.idle": "2022-10-23T20:06:55.692968Z",
     "shell.execute_reply": "2022-10-23T20:06:55.692085Z"
    },
    "papermill": {
     "duration": 0.014271,
     "end_time": "2022-10-23T20:06:55.695027",
     "exception": false,
     "start_time": "2022-10-23T20:06:55.680756",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "class Config(dict):\n",
    "    __getattr__ = dict.__getitem__\n",
    "    __setattr__ = dict.__setitem__\n",
    "    __delattr__ = dict.__delitem__\n",
    "    \n",
    "    def init(self, kwargs):\n",
    "        super().init(kwargs)\n",
    "\n",
    "        for k, v in kwargs.items():\n",
    "            setattr(self, k, v)\n",
    "\n",
    "    def set(self, key, val):\n",
    "        self[key] = val\n",
    "        setattr(self, key, val)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "id": "0e96631f",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2022-10-23T20:06:55.705172Z",
     "iopub.status.busy": "2022-10-23T20:06:55.704299Z",
     "iopub.status.idle": "2022-10-23T20:06:55.711797Z",
     "shell.execute_reply": "2022-10-23T20:06:55.710956Z"
    },
    "papermill": {
     "duration": 0.014808,
     "end_time": "2022-10-23T20:06:55.714017",
     "exception": false,
     "start_time": "2022-10-23T20:06:55.699209",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "def load_config(input_path, inference_weight=1):\n",
    "    # Load CFG class.\n",
    "    cfg = Config(**json.load(open(os.path.join(input_path, 'CFG.json'), 'r')))\n",
    "    cfg.path = '../input/fb3-colab-models/v114' #TODO: change to input_path\n",
    "    cfg.config_path = os.path.join(cfg.path, 'config.pth')\n",
    "    # Load tokenizer.\n",
    "    tokenizer = AutoTokenizer.from_pretrained(os.path.join(cfg.path, 'tokenizer'))\n",
    "    cfg.tokenizer = tokenizer\n",
    "    \n",
    "    cfg.inference_weight = inference_weight\n",
    "    return cfg\n",
    "\n",
    "def load_model(cfg, fold, **model_kwargs):\n",
    "    # Load torch model.\n",
    "    model = FB3Model(cfg, config_path=cfg.config_path, pretrained=False, **model_kwargs)\n",
    "    state = torch.load(\n",
    "        os.path.join(cfg.path, f\"{cfg.model.replace('/', '-')}_fold{fold}_best.pth\"),\n",
    "        map_location=torch.device('cpu'))\n",
    "    model.load_state_dict(state['model'])\n",
    "    \n",
    "    return model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "id": "21944e37",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2022-10-23T20:06:55.724027Z",
     "iopub.status.busy": "2022-10-23T20:06:55.723213Z",
     "iopub.status.idle": "2022-10-23T20:06:55.736920Z",
     "shell.execute_reply": "2022-10-23T20:06:55.736011Z"
    },
    "papermill": {
     "duration": 0.020738,
     "end_time": "2022-10-23T20:06:55.738982",
     "exception": false,
     "start_time": "2022-10-23T20:06:55.718244",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "def seed_everything(seed=42):\n",
    "    '''\n",
    "    Sets the seed of the entire notebook so results are the same every time we run.\n",
    "    This is for REPRODUCIBILITY.\n",
    "    '''\n",
    "    random.seed(seed)\n",
    "    # Set a fixed value for the hash seed\n",
    "    os.environ['PYTHONHASHSEED'] = str(seed)\n",
    "    np.random.seed(seed)\n",
    "    torch.manual_seed(seed)\n",
    "    \n",
    "    if torch.cuda.is_available():\n",
    "        torch.cuda.manual_seed(seed)\n",
    "        torch.cuda.manual_seed_all(seed)\n",
    "        # When running on the CuDNN backend, two further options must be set\n",
    "        torch.backends.cudnn.deterministic = True\n",
    "        torch.backends.cudnn.benchmark = False\n",
    "seed_everything(seed=42)\n",
    "\n",
    "def mc_rmse(y_true, y_pred):\n",
    "    scores = []\n",
    "    ncols = y_true.shape[1]\n",
    "    \n",
    "    for n in range(ncols):\n",
    "        yn_true = y_true[:, n]\n",
    "        yn_pred = y_pred[:, n]\n",
    "        rmse_ = mean_squared_error(yn_true, yn_pred, squared=False)\n",
    "        scores.append(rmse_)\n",
    "    score = np.mean(scores) \n",
    "    return score, scores\n",
    "\n",
    "def get_result(cfg, oof_df):\n",
    "    labels = oof_df[cfg.target_cols].values\n",
    "    preds = oof_df[[f\"pred_{c}\" for c in cfg.target_cols]].values\n",
    "    score, scores = mc_rmse(labels, preds)\n",
    "    print(f'score: {score:<.6f}  scores: {scores}')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "id": "b7e78e6f",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2022-10-23T20:06:55.748856Z",
     "iopub.status.busy": "2022-10-23T20:06:55.748550Z",
     "iopub.status.idle": "2022-10-23T20:06:55.756779Z",
     "shell.execute_reply": "2022-10-23T20:06:55.755753Z"
    },
    "papermill": {
     "duration": 0.015862,
     "end_time": "2022-10-23T20:06:55.759124",
     "exception": false,
     "start_time": "2022-10-23T20:06:55.743262",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "# ====================================================\n",
    "# Dataset\n",
    "# ====================================================\n",
    "def encode_text(cfg, text):\n",
    "    inputs = cfg.tokenizer.encode_plus(\n",
    "        text, \n",
    "        return_tensors=None, \n",
    "        add_special_tokens=True, \n",
    "        #max_length=CFG.max_len,\n",
    "        #pad_to_max_length=True,\n",
    "        #truncation=True\n",
    "    )\n",
    "    for k, v in inputs.items():\n",
    "        inputs[k] = torch.tensor(v, dtype=torch.long)\n",
    "    return inputs \n",
    "\n",
    "def preprocess(texts):\n",
    "    texts = (\n",
    "        texts\n",
    "        .str.replace(r'\\r\\n', '<newline>', regex=True)\n",
    "        .str.replace(r'\\n', '<newline>', regex=True)\n",
    "        .str.replace('<newline><newline>', '<newline>', regex=False)\n",
    "        .values \n",
    "    )\n",
    "    return texts\n",
    "\n",
    "class TestDataset(Dataset):\n",
    "    def __init__(self, cfg, df):\n",
    "        self.cfg = cfg\n",
    "        self.texts = preprocess(df['full_text'])\n",
    "\n",
    "    def __len__(self):\n",
    "        return len(self.texts)\n",
    "\n",
    "    def __getitem__(self, item):\n",
    "        inputs = encode_text(self.cfg, self.texts[item])\n",
    "        return inputs"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "id": "f1dd58d7",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2022-10-23T20:06:55.769130Z",
     "iopub.status.busy": "2022-10-23T20:06:55.768647Z",
     "iopub.status.idle": "2022-10-23T20:06:55.782228Z",
     "shell.execute_reply": "2022-10-23T20:06:55.781155Z"
    },
    "papermill": {
     "duration": 0.021243,
     "end_time": "2022-10-23T20:06:55.784752",
     "exception": false,
     "start_time": "2022-10-23T20:06:55.763509",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "def inference_fn(test_loader, model, device):\n",
    "    preds = []\n",
    "    model.eval()\n",
    "    model.to(device)\n",
    "    #tk0 = tqdm(test_loader, total=len(test_loader))\n",
    "    for inputs in test_loader:\n",
    "        for k, v in inputs.items():\n",
    "            inputs[k] = v.to(device)\n",
    "        with torch.no_grad():\n",
    "            y_preds = model(inputs)\n",
    "        preds.append(y_preds.to('cpu').numpy())\n",
    "    predictions = np.concatenate(preds)\n",
    "    return predictions\n",
    "\n",
    "class Inferencer:\n",
    "    def __init__(self, input_path=None, cfg=None, inference_weight=1):\n",
    "        if cfg == None:\n",
    "            self.cfg = load_config(input_path, inference_weight)\n",
    "        else:\n",
    "            self.cfg = cfg\n",
    "        self.pool = cfg.pool_head\n",
    "    \n",
    "    def predict(self, test_loader, device, stat_fn=np.mean):\n",
    "        preds = []\n",
    "        start = time.time()\n",
    "        print('#'*10, cfg.path, '#'*10)\n",
    "        for fold in self.cfg.trn_fold:\n",
    "            print(f'Predicting fold {fold}...')\n",
    "            model = load_model(self.cfg, fold, pool=self.pool)\n",
    "            pred = inference_fn(test_loader, model, device)\n",
    "            preds.append(pred)\n",
    "            del model; gc.collect()\n",
    "            torch.cuda.empty_cache()\n",
    "        end = time.time() - start\n",
    "        print('#'*10, f'ETA: {end:.2f}s', '#'*10, '\\n')\n",
    "        \n",
    "        self.preds = stat_fn(preds, axis=0) \n",
    "        self.preds = np.clip(self.preds, 1, 5)\n",
    "        return self.preds\n",
    "    \n",
    "    def get_oof_result(self):\n",
    "        return get_result(pd.read_pickle(os.path.join(cfg.path, 'oof_df.pkl')))\n",
    "    \n",
    "    def get_text_embedding(self, data_loader, device, fold): \n",
    "        model = load_model(self.cfg, fold, pool=self.pool)\n",
    "        model.eval()\n",
    "        model.to(device)\n",
    "\n",
    "        fold_emb = []\n",
    "        for inputs in data_loader:\n",
    "            for k, v in inputs.items():\n",
    "                inputs[k] = v.to(device)\n",
    "            with torch.no_grad():\n",
    "                emb = model.feature(inputs)\n",
    "            fold_emb.extend(emb.to('cpu').numpy())\n",
    "        fold_emb = np.array(fold_emb)\n",
    "        return fold_emb"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "id": "4745d1a3",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2022-10-23T20:06:55.795422Z",
     "iopub.status.busy": "2022-10-23T20:06:55.794620Z",
     "iopub.status.idle": "2022-10-23T20:06:56.452956Z",
     "shell.execute_reply": "2022-10-23T20:06:56.451879Z"
    },
    "papermill": {
     "duration": 0.666341,
     "end_time": "2022-10-23T20:06:56.455582",
     "exception": false,
     "start_time": "2022-10-23T20:06:55.789241",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "v112_CFG = Config(\n",
    "    competition='FB3',\n",
    "    debug=False,\n",
    "\n",
    "    ####################\n",
    "\n",
    "    apex=True,\n",
    "    print_freq=20,\n",
    "    num_workers=1,\n",
    "    model='microsoft/deberta-v3-base',\n",
    "    model_type='deberta_v3',\n",
    "    gradient_checkpointing=True,\n",
    "    scheduler='cosine', # ['linear', 'cosine']\n",
    "    batch_scheduler=True,\n",
    "    num_cycles=0.5,\n",
    "    num_warmup_steps=0.6,\n",
    "    \n",
    "    ####################\n",
    "\n",
    "    epochs=6,\n",
    "    val_step=60,\n",
    "    encoder_lr=2e-5,\n",
    "    decoder_lr=2e-5,\n",
    "    min_lr=1e-6,\n",
    "    eps=1e-6,\n",
    "    betas=(0.9, 0.999),\n",
    "    batch_size=8,\n",
    "    max_len=512,\n",
    "\n",
    "    ####################\n",
    "\n",
    "    reinit_last_layer=True,\n",
    "    reinit_fc=True,\n",
    "\n",
    "    weight_decay=0.01,\n",
    "    learning_rate=2e-5,\n",
    "    layerwise_learning_rate_decay=1.5,\n",
    "\n",
    "    use_dropout=False,\n",
    "    prob_dropout=[0.06, 0.08, 0.1, 0.12, 0.14],\n",
    "    num_dropout=5,\n",
    "\n",
    "    pool_head='attention',\n",
    "    \n",
    "    ####################\n",
    "    \n",
    "    gradient_accumulation_steps=1,\n",
    "    max_grad_norm=10,\n",
    "    target_cols=['cohesion', 'syntax', 'vocabulary', 'phraseology', 'grammar', 'conventions'],\n",
    "    num_target=6,\n",
    "    seed=42,\n",
    "    n_fold=4,\n",
    "    trn_fold=[0,1,2,3],\n",
    "    train=True,\n",
    "    \n",
    "    path='../input/fb3-colab-models/v112/',\n",
    "    config_path='../input/fb3-colab-models/v112/config.pth',\n",
    "    inference_weight=0.7\n",
    ")\n",
    "v114_CFG = Config(\n",
    "    competition='FB3',\n",
    "    debug=False,\n",
    "\n",
    "    ####################\n",
    "\n",
    "    apex=True,\n",
    "    print_freq=20,\n",
    "    num_workers=1,\n",
    "    model='microsoft/deberta-v3-base',\n",
    "    model_type='deberta_v3',\n",
    "    gradient_checkpointing=True,\n",
    "    scheduler='cosine', # ['linear', 'cosine']\n",
    "    batch_scheduler=True,\n",
    "    num_cycles=0.5,\n",
    "    num_warmup_steps=0.6,\n",
    "    \n",
    "    ####################\n",
    "\n",
    "    epochs=6,\n",
    "    val_step=60,\n",
    "    encoder_lr=2e-5,\n",
    "    decoder_lr=2e-5,\n",
    "    min_lr=1e-6,\n",
    "    eps=1e-6,\n",
    "    betas=(0.9, 0.999),\n",
    "    batch_size=8,\n",
    "    max_len=512,\n",
    "\n",
    "    ####################\n",
    "\n",
    "    reinit_last_layer=True,\n",
    "    reinit_fc=True,\n",
    "\n",
    "    weight_decay=0.01,\n",
    "    learning_rate=2e-5,\n",
    "    layerwise_learning_rate_decay=1.5,\n",
    "\n",
    "    use_dropout=False,\n",
    "    prob_dropout=[0.06, 0.08, 0.1, 0.12, 0.14],\n",
    "    num_dropout=5,\n",
    "\n",
    "    pool_head='mean-attention',\n",
    "    \n",
    "    ####################\n",
    "    \n",
    "    gradient_accumulation_steps=1,\n",
    "    max_grad_norm=10,\n",
    "    target_cols=['cohesion', 'syntax', 'vocabulary', 'phraseology', 'grammar', 'conventions'],\n",
    "    num_target=6,\n",
    "    seed=42,\n",
    "    n_fold=4,\n",
    "    trn_fold=[0,1,2,3],\n",
    "    train=True,\n",
    "    \n",
    "    path='../input/fb3-colab-models/v114/',\n",
    "    config_path='../input/fb3-colab-models/v114/config.pth',\n",
    "    inference_weight=0.3\n",
    ")\n",
    "\n",
    "tokenizer = AutoTokenizer.from_pretrained(os.path.join(v112_CFG.path, 'tokenizer'))\n",
    "v112_CFG.tokenizer = tokenizer\n",
    "\n",
    "tokenizer = AutoTokenizer.from_pretrained(os.path.join(v114_CFG.path, 'tokenizer'))\n",
    "v114_CFG.tokenizer = tokenizer\n",
    "\n",
    "\n",
    "setups = [\n",
    "    v112_CFG, v114_CFG\n",
    "]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "id": "8e91b127",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2022-10-23T20:06:56.466309Z",
     "iopub.status.busy": "2022-10-23T20:06:56.465958Z",
     "iopub.status.idle": "2022-10-23T21:09:25.432270Z",
     "shell.execute_reply": "2022-10-23T21:09:25.430994Z"
    },
    "papermill": {
     "duration": 3748.975008,
     "end_time": "2022-10-23T21:09:25.435189",
     "exception": false,
     "start_time": "2022-10-23T20:06:56.460181",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "30afe16ea9d54769b9a33adce33b8819",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "  0%|          | 0/2 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0\n",
      "1\n",
      "2\n",
      "3\n",
      "0\n",
      "1\n",
      "2\n",
      "3\n"
     ]
    }
   ],
   "source": [
    "train = pd.read_csv(TRAIN_PATH)\n",
    "\n",
    "train_dataset = TestDataset(v112_CFG, train)\n",
    "train_loader = DataLoader(\n",
    "    train_dataset,\n",
    "    batch_size=8,\n",
    "    shuffle=False,\n",
    "    collate_fn=DataCollatorWithPadding(tokenizer=v112_CFG.tokenizer, padding='longest'),\n",
    "    num_workers=6, \n",
    "    pin_memory=True, \n",
    "    drop_last=False)\n",
    "\n",
    "all_train_text_emb = []\n",
    "for setup in tqdm(setups):\n",
    "    # infer_ = Inferencer(setup['path'], setup['inference_weight'])\n",
    "    infer_ = Inferencer(cfg=setup, inference_weight=setup.inference_weight)\n",
    "    \n",
    "    # Text embedding for SVM\n",
    "    train_text_emb = []\n",
    "    for fold in infer_.cfg.trn_fold:\n",
    "        print(fold)\n",
    "        train_text_emb.append(infer_.get_text_embedding(train_loader, device, fold))\n",
    "    all_train_text_emb.append(np.mean(train_text_emb, axis=0))\n",
    "final_train_text_emb = np.concatenate(all_train_text_emb, axis=1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "id": "d5a98fbc",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2022-10-23T21:09:25.446962Z",
     "iopub.status.busy": "2022-10-23T21:09:25.445877Z",
     "iopub.status.idle": "2022-10-23T21:09:25.673023Z",
     "shell.execute_reply": "2022-10-23T21:09:25.672037Z"
    },
    "papermill": {
     "duration": 0.235701,
     "end_time": "2022-10-23T21:09:25.675756",
     "exception": false,
     "start_time": "2022-10-23T21:09:25.440055",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "import sys\n",
    "sys.path.append('../input/iterativestratification')\n",
    "from iterstrat.ml_stratifiers import MultilabelStratifiedKFold\n",
    "\n",
    "svr_folds = 4\n",
    "target_cols = ['cohesion', 'syntax', 'vocabulary', 'phraseology', 'grammar', 'conventions']\n",
    "    \n",
    "skf = MultilabelStratifiedKFold(n_splits=svr_folds, shuffle=True, random_state=42)\n",
    "for i,(train_index, val_index) in enumerate(skf.split(train,train[target_cols])):\n",
    "    train.loc[val_index,'fold'] = i"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "id": "d7ddff5e",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2022-10-23T21:09:25.688063Z",
     "iopub.status.busy": "2022-10-23T21:09:25.687779Z",
     "iopub.status.idle": "2022-10-23T21:09:44.114799Z",
     "shell.execute_reply": "2022-10-23T21:09:44.113813Z"
    },
    "papermill": {
     "duration": 18.436151,
     "end_time": "2022-10-23T21:09:44.117619",
     "exception": false,
     "start_time": "2022-10-23T21:09:25.681468",
     "status": "completed"
    },
    "scrolled": true,
    "tags": []
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "#########################\n",
      "### Fold 1\n",
      "#########################\n",
      "Fold : 0  score: 0.41851912303372973\n",
      "#########################\n",
      "### Fold 2\n",
      "#########################\n",
      "Fold : 1  score: 0.4097098036753621\n",
      "#########################\n",
      "### Fold 3\n",
      "#########################\n",
      "Fold : 2  score: 0.42114867082485113\n",
      "#########################\n",
      "### Fold 4\n",
      "#########################\n",
      "Fold : 3  score: 0.40808792034056657\n",
      "#########################\n",
      "Overall CV = 0.41436637946862737\n"
     ]
    }
   ],
   "source": [
    "from joblib import dump, load\n",
    "from sklearn.multioutput import MultiOutputRegressor\n",
    "from cuml.svm import SVR\n",
    "import cuml\n",
    "\n",
    "from sklearn.metrics import mean_squared_error\n",
    "\n",
    "scores = []\n",
    "def mc_rmse(y_true, y_pred):\n",
    "    scores = []\n",
    "    ncols = y_true.shape[1]\n",
    "    \n",
    "    for n in range(ncols):\n",
    "        yn_true = y_true[:, n]\n",
    "        yn_pred = y_pred[:, n]\n",
    "        rmse_ = mean_squared_error(yn_true, yn_pred, squared=False)\n",
    "        scores.append(rmse_)\n",
    "    score = np.mean(scores) \n",
    "    return score, scores\n",
    "\n",
    "\n",
    "for fold in range(svr_folds):\n",
    "    print('#'*25)\n",
    "    print('### Fold',fold+1)\n",
    "    print('#'*25)\n",
    "    \n",
    "    train_ = train[train['fold']!=fold]\n",
    "    eval_ = train[train['fold']==fold]\n",
    "    \n",
    "    tr_text_feats = final_train_text_emb[list(train_.index),:]\n",
    "    ev_text_feats = final_train_text_emb[list(eval_.index),:]\n",
    "    \n",
    "    clf = MultiOutputRegressor(SVR(C=1))\n",
    "    clf.fit(tr_text_feats, train_[target_cols].values)\n",
    "    \n",
    "    ev_preds = clf.predict(ev_text_feats)\n",
    "    score, _ = mc_rmse(eval_[target_cols].values, ev_preds)\n",
    "    scores.append(score)\n",
    "    \n",
    "    print(\"Fold : {}  score: {}\".format(fold,score))\n",
    "    dump(clf, f'svr_{fold}.model')\n",
    "    \n",
    "print('#'*25)\n",
    "print('Overall CV =', np.mean(scores))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "id": "1623e887",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2022-10-23T21:09:44.135717Z",
     "iopub.status.busy": "2022-10-23T21:09:44.135367Z",
     "iopub.status.idle": "2022-10-23T21:11:11.215723Z",
     "shell.execute_reply": "2022-10-23T21:11:11.214482Z"
    },
    "papermill": {
     "duration": 87.092283,
     "end_time": "2022-10-23T21:11:11.218459",
     "exception": false,
     "start_time": "2022-10-23T21:09:44.126176",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "4ac88fe9da5d4ae1b177e6b2adb848cd",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "  0%|          | 0/2 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0\n",
      "1\n",
      "2\n",
      "3\n",
      "0\n",
      "1\n",
      "2\n",
      "3\n"
     ]
    }
   ],
   "source": [
    "test = pd.read_csv(TEST_PATH)\n",
    "\n",
    "test_dataset = TestDataset(v112_CFG, test)\n",
    "test_loader = DataLoader(\n",
    "    test_dataset,\n",
    "    batch_size=8,\n",
    "    shuffle=False,\n",
    "    collate_fn=DataCollatorWithPadding(tokenizer=v112_CFG.tokenizer, padding='longest'),\n",
    "    num_workers=6, \n",
    "    pin_memory=True, \n",
    "    drop_last=False)\n",
    "\n",
    "all_test_text_emb = []\n",
    "for setup in tqdm(setups):\n",
    "    # infer_ = Inferencer(setup['path'], setup['inference_weight'])\n",
    "    infer_ = Inferencer(cfg=setup, inference_weight=setup.inference_weight)\n",
    "    \n",
    "    # Text embedding for SVM\n",
    "    test_text_emb = []\n",
    "    for fold in infer_.cfg.trn_fold:\n",
    "        print(fold)\n",
    "        test_text_emb.append(infer_.get_text_embedding(test_loader, device, fold))\n",
    "    all_test_text_emb.append(np.mean(test_text_emb, axis=0))\n",
    "final_test_text_emb = np.concatenate(all_test_text_emb, axis=1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "id": "5b67d1fd",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2022-10-23T21:11:11.233032Z",
     "iopub.status.busy": "2022-10-23T21:11:11.232087Z",
     "iopub.status.idle": "2022-10-23T21:11:11.962646Z",
     "shell.execute_reply": "2022-10-23T21:11:11.961476Z"
    },
    "papermill": {
     "duration": 0.741629,
     "end_time": "2022-10-23T21:11:11.966361",
     "exception": false,
     "start_time": "2022-10-23T21:11:11.224732",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "def svr_inference_fn(model_path, te_text_feats):\n",
    "    model = load(model_path)\n",
    "    preds = model.predict(te_text_feats)\n",
    "    return preds\n",
    "\n",
    "predictions = []\n",
    "for fold in range(svr_folds):\n",
    "    model_path = f'svr_{fold}.model'\n",
    "    preds = svr_inference_fn(model_path, final_test_text_emb)\n",
    "    predictions.append(preds)\n",
    "predictions = np.mean(predictions, axis=0)\n",
    "predictions = np.clip(predictions, 1, 5)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "id": "ecefeeaf",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2022-10-23T21:11:11.988260Z",
     "iopub.status.busy": "2022-10-23T21:11:11.987729Z",
     "iopub.status.idle": "2022-10-23T21:11:12.076795Z",
     "shell.execute_reply": "2022-10-23T21:11:12.075860Z"
    },
    "papermill": {
     "duration": 0.101586,
     "end_time": "2022-10-23T21:11:12.079407",
     "exception": false,
     "start_time": "2022-10-23T21:11:11.977821",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>text_id</th>\n",
       "      <th>cohesion</th>\n",
       "      <th>syntax</th>\n",
       "      <th>vocabulary</th>\n",
       "      <th>phraseology</th>\n",
       "      <th>grammar</th>\n",
       "      <th>conventions</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>0000C359D63E</td>\n",
       "      <td>3.068707</td>\n",
       "      <td>2.900846</td>\n",
       "      <td>3.222927</td>\n",
       "      <td>3.118434</td>\n",
       "      <td>2.752864</td>\n",
       "      <td>2.719249</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>000BAD50D026</td>\n",
       "      <td>2.779561</td>\n",
       "      <td>2.510198</td>\n",
       "      <td>2.840374</td>\n",
       "      <td>2.479988</td>\n",
       "      <td>2.201199</td>\n",
       "      <td>2.673922</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>00367BB2546B</td>\n",
       "      <td>3.731355</td>\n",
       "      <td>3.398375</td>\n",
       "      <td>3.550659</td>\n",
       "      <td>3.671269</td>\n",
       "      <td>3.419664</td>\n",
       "      <td>3.253832</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "        text_id  cohesion    syntax  vocabulary  phraseology   grammar  conventions\n",
       "0  0000C359D63E  3.068707  2.900846    3.222927     3.118434  2.752864     2.719249\n",
       "1  000BAD50D026  2.779561  2.510198    2.840374     2.479988  2.201199     2.673922\n",
       "2  00367BB2546B  3.731355  3.398375    3.550659     3.671269  3.419664     3.253832"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "submission = pd.read_csv(SUBMISSION_PATH)\n",
    "test[target_cols] = predictions\n",
    "submission = submission.drop(columns=target_cols).merge(test[['text_id'] + target_cols], on='text_id', how='left')\n",
    "display(submission.head())\n",
    "submission[['text_id'] + target_cols].to_csv('submission.csv', index=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "1bdb5b61",
   "metadata": {
    "papermill": {
     "duration": 0.008813,
     "end_time": "2022-10-23T21:11:12.097593",
     "exception": false,
     "start_time": "2022-10-23T21:11:12.088780",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.12"
  },
  "papermill": {
   "default_parameters": {},
   "duration": 3916.947604,
   "end_time": "2022-10-23T21:11:15.398076",
   "environment_variables": {},
   "exception": null,
   "input_path": "__notebook__.ipynb",
   "output_path": "__notebook__.ipynb",
   "parameters": {},
   "start_time": "2022-10-23T20:05:58.450472",
   "version": "2.3.4"
  },
  "widgets": {
   "application/vnd.jupyter.widget-state+json": {
    "state": {
     "076a12ae9be84ec9a0e0e4a8eec7bd62": {
      "model_module": "@jupyter-widgets/base",
      "model_module_version": "1.2.0",
      "model_name": "LayoutModel",
      "state": {
       "_model_module": "@jupyter-widgets/base",
       "_model_module_version": "1.2.0",
       "_model_name": "LayoutModel",
       "_view_count": null,
       "_view_module": "@jupyter-widgets/base",
       "_view_module_version": "1.2.0",
       "_view_name": "LayoutView",
       "align_content": null,
       "align_items": null,
       "align_self": null,
       "border": null,
       "bottom": null,
       "display": null,
       "flex": null,
       "flex_flow": null,
       "grid_area": null,
       "grid_auto_columns": null,
       "grid_auto_flow": null,
       "grid_auto_rows": null,
       "grid_column": null,
       "grid_gap": null,
       "grid_row": null,
       "grid_template_areas": null,
       "grid_template_columns": null,
       "grid_template_rows": null,
       "height": null,
       "justify_content": null,
       "justify_items": null,
       "left": null,
       "margin": null,
       "max_height": null,
       "max_width": null,
       "min_height": null,
       "min_width": null,
       "object_fit": null,
       "object_position": null,
       "order": null,
       "overflow": null,
       "overflow_x": null,
       "overflow_y": null,
       "padding": null,
       "right": null,
       "top": null,
       "visibility": null,
       "width": null
      }
     },
     "191891783afe4c9b9f006f4089d0b140": {
      "model_module": "@jupyter-widgets/base",
      "model_module_version": "1.2.0",
      "model_name": "LayoutModel",
      "state": {
       "_model_module": "@jupyter-widgets/base",
       "_model_module_version": "1.2.0",
       "_model_name": "LayoutModel",
       "_view_count": null,
       "_view_module": "@jupyter-widgets/base",
       "_view_module_version": "1.2.0",
       "_view_name": "LayoutView",
       "align_content": null,
       "align_items": null,
       "align_self": null,
       "border": null,
       "bottom": null,
       "display": null,
       "flex": null,
       "flex_flow": null,
       "grid_area": null,
       "grid_auto_columns": null,
       "grid_auto_flow": null,
       "grid_auto_rows": null,
       "grid_column": null,
       "grid_gap": null,
       "grid_row": null,
       "grid_template_areas": null,
       "grid_template_columns": null,
       "grid_template_rows": null,
       "height": null,
       "justify_content": null,
       "justify_items": null,
       "left": null,
       "margin": null,
       "max_height": null,
       "max_width": null,
       "min_height": null,
       "min_width": null,
       "object_fit": null,
       "object_position": null,
       "order": null,
       "overflow": null,
       "overflow_x": null,
       "overflow_y": null,
       "padding": null,
       "right": null,
       "top": null,
       "visibility": null,
       "width": null
      }
     },
     "20f51b30c1a84093ad473e0572a0ddb9": {
      "model_module": "@jupyter-widgets/controls",
      "model_module_version": "1.5.0",
      "model_name": "DescriptionStyleModel",
      "state": {
       "_model_module": "@jupyter-widgets/controls",
       "_model_module_version": "1.5.0",
       "_model_name": "DescriptionStyleModel",
       "_view_count": null,
       "_view_module": "@jupyter-widgets/base",
       "_view_module_version": "1.2.0",
       "_view_name": "StyleView",
       "description_width": ""
      }
     },
     "2f30d2e43093452490687e3aeac18fcb": {
      "model_module": "@jupyter-widgets/base",
      "model_module_version": "1.2.0",
      "model_name": "LayoutModel",
      "state": {
       "_model_module": "@jupyter-widgets/base",
       "_model_module_version": "1.2.0",
       "_model_name": "LayoutModel",
       "_view_count": null,
       "_view_module": "@jupyter-widgets/base",
       "_view_module_version": "1.2.0",
       "_view_name": "LayoutView",
       "align_content": null,
       "align_items": null,
       "align_self": null,
       "border": null,
       "bottom": null,
       "display": null,
       "flex": null,
       "flex_flow": null,
       "grid_area": null,
       "grid_auto_columns": null,
       "grid_auto_flow": null,
       "grid_auto_rows": null,
       "grid_column": null,
       "grid_gap": null,
       "grid_row": null,
       "grid_template_areas": null,
       "grid_template_columns": null,
       "grid_template_rows": null,
       "height": null,
       "justify_content": null,
       "justify_items": null,
       "left": null,
       "margin": null,
       "max_height": null,
       "max_width": null,
       "min_height": null,
       "min_width": null,
       "object_fit": null,
       "object_position": null,
       "order": null,
       "overflow": null,
       "overflow_x": null,
       "overflow_y": null,
       "padding": null,
       "right": null,
       "top": null,
       "visibility": null,
       "width": null
      }
     },
     "30afe16ea9d54769b9a33adce33b8819": {
      "model_module": "@jupyter-widgets/controls",
      "model_module_version": "1.5.0",
      "model_name": "HBoxModel",
      "state": {
       "_dom_classes": [],
       "_model_module": "@jupyter-widgets/controls",
       "_model_module_version": "1.5.0",
       "_model_name": "HBoxModel",
       "_view_count": null,
       "_view_module": "@jupyter-widgets/controls",
       "_view_module_version": "1.5.0",
       "_view_name": "HBoxView",
       "box_style": "",
       "children": [
        "IPY_MODEL_54e516e0cb72477398961ff7ca3ddc01",
        "IPY_MODEL_f4b44d77020b43058fcdeca67f254129",
        "IPY_MODEL_3190bcb50b074ce387ee46c0b86c4a25"
       ],
       "layout": "IPY_MODEL_2f30d2e43093452490687e3aeac18fcb"
      }
     },
     "3190bcb50b074ce387ee46c0b86c4a25": {
      "model_module": "@jupyter-widgets/controls",
      "model_module_version": "1.5.0",
      "model_name": "HTMLModel",
      "state": {
       "_dom_classes": [],
       "_model_module": "@jupyter-widgets/controls",
       "_model_module_version": "1.5.0",
       "_model_name": "HTMLModel",
       "_view_count": null,
       "_view_module": "@jupyter-widgets/controls",
       "_view_module_version": "1.5.0",
       "_view_name": "HTMLView",
       "description": "",
       "description_tooltip": null,
       "layout": "IPY_MODEL_076a12ae9be84ec9a0e0e4a8eec7bd62",
       "placeholder": "​",
       "style": "IPY_MODEL_63ddac27fc034bf28323839ea6769887",
       "value": " 2/2 [1:02:28&lt;00:00, 1874.34s/it]"
      }
     },
     "393f2fb4be324e9480ee6428cc06e6bd": {
      "model_module": "@jupyter-widgets/controls",
      "model_module_version": "1.5.0",
      "model_name": "DescriptionStyleModel",
      "state": {
       "_model_module": "@jupyter-widgets/controls",
       "_model_module_version": "1.5.0",
       "_model_name": "DescriptionStyleModel",
       "_view_count": null,
       "_view_module": "@jupyter-widgets/base",
       "_view_module_version": "1.2.0",
       "_view_name": "StyleView",
       "description_width": ""
      }
     },
     "4ac88fe9da5d4ae1b177e6b2adb848cd": {
      "model_module": "@jupyter-widgets/controls",
      "model_module_version": "1.5.0",
      "model_name": "HBoxModel",
      "state": {
       "_dom_classes": [],
       "_model_module": "@jupyter-widgets/controls",
       "_model_module_version": "1.5.0",
       "_model_name": "HBoxModel",
       "_view_count": null,
       "_view_module": "@jupyter-widgets/controls",
       "_view_module_version": "1.5.0",
       "_view_name": "HBoxView",
       "box_style": "",
       "children": [
        "IPY_MODEL_a4a0c7d3bcdd44b6badb5021ee022260",
        "IPY_MODEL_908bd482f9a54fc4a3dce47942062ea0",
        "IPY_MODEL_64284f557d39489c9a7b6279d25553e8"
       ],
       "layout": "IPY_MODEL_191891783afe4c9b9f006f4089d0b140"
      }
     },
     "54e516e0cb72477398961ff7ca3ddc01": {
      "model_module": "@jupyter-widgets/controls",
      "model_module_version": "1.5.0",
      "model_name": "HTMLModel",
      "state": {
       "_dom_classes": [],
       "_model_module": "@jupyter-widgets/controls",
       "_model_module_version": "1.5.0",
       "_model_name": "HTMLModel",
       "_view_count": null,
       "_view_module": "@jupyter-widgets/controls",
       "_view_module_version": "1.5.0",
       "_view_name": "HTMLView",
       "description": "",
       "description_tooltip": null,
       "layout": "IPY_MODEL_7b60ec0a6d3a4e2bb6ac19583d162278",
       "placeholder": "​",
       "style": "IPY_MODEL_806459a922704ad0ac4d9932222e37ec",
       "value": "100%"
      }
     },
     "56ecc6d275564e3da4b8c41f525f5e1d": {
      "model_module": "@jupyter-widgets/base",
      "model_module_version": "1.2.0",
      "model_name": "LayoutModel",
      "state": {
       "_model_module": "@jupyter-widgets/base",
       "_model_module_version": "1.2.0",
       "_model_name": "LayoutModel",
       "_view_count": null,
       "_view_module": "@jupyter-widgets/base",
       "_view_module_version": "1.2.0",
       "_view_name": "LayoutView",
       "align_content": null,
       "align_items": null,
       "align_self": null,
       "border": null,
       "bottom": null,
       "display": null,
       "flex": null,
       "flex_flow": null,
       "grid_area": null,
       "grid_auto_columns": null,
       "grid_auto_flow": null,
       "grid_auto_rows": null,
       "grid_column": null,
       "grid_gap": null,
       "grid_row": null,
       "grid_template_areas": null,
       "grid_template_columns": null,
       "grid_template_rows": null,
       "height": null,
       "justify_content": null,
       "justify_items": null,
       "left": null,
       "margin": null,
       "max_height": null,
       "max_width": null,
       "min_height": null,
       "min_width": null,
       "object_fit": null,
       "object_position": null,
       "order": null,
       "overflow": null,
       "overflow_x": null,
       "overflow_y": null,
       "padding": null,
       "right": null,
       "top": null,
       "visibility": null,
       "width": null
      }
     },
     "63ddac27fc034bf28323839ea6769887": {
      "model_module": "@jupyter-widgets/controls",
      "model_module_version": "1.5.0",
      "model_name": "DescriptionStyleModel",
      "state": {
       "_model_module": "@jupyter-widgets/controls",
       "_model_module_version": "1.5.0",
       "_model_name": "DescriptionStyleModel",
       "_view_count": null,
       "_view_module": "@jupyter-widgets/base",
       "_view_module_version": "1.2.0",
       "_view_name": "StyleView",
       "description_width": ""
      }
     },
     "64284f557d39489c9a7b6279d25553e8": {
      "model_module": "@jupyter-widgets/controls",
      "model_module_version": "1.5.0",
      "model_name": "HTMLModel",
      "state": {
       "_dom_classes": [],
       "_model_module": "@jupyter-widgets/controls",
       "_model_module_version": "1.5.0",
       "_model_name": "HTMLModel",
       "_view_count": null,
       "_view_module": "@jupyter-widgets/controls",
       "_view_module_version": "1.5.0",
       "_view_name": "HTMLView",
       "description": "",
       "description_tooltip": null,
       "layout": "IPY_MODEL_76178161efcd4b15824091a107693992",
       "placeholder": "​",
       "style": "IPY_MODEL_393f2fb4be324e9480ee6428cc06e6bd",
       "value": " 2/2 [01:27&lt;00:00, 43.48s/it]"
      }
     },
     "6de9f56829cc40d5b3a6912ee14fce7d": {
      "model_module": "@jupyter-widgets/controls",
      "model_module_version": "1.5.0",
      "model_name": "ProgressStyleModel",
      "state": {
       "_model_module": "@jupyter-widgets/controls",
       "_model_module_version": "1.5.0",
       "_model_name": "ProgressStyleModel",
       "_view_count": null,
       "_view_module": "@jupyter-widgets/base",
       "_view_module_version": "1.2.0",
       "_view_name": "StyleView",
       "bar_color": null,
       "description_width": ""
      }
     },
     "76178161efcd4b15824091a107693992": {
      "model_module": "@jupyter-widgets/base",
      "model_module_version": "1.2.0",
      "model_name": "LayoutModel",
      "state": {
       "_model_module": "@jupyter-widgets/base",
       "_model_module_version": "1.2.0",
       "_model_name": "LayoutModel",
       "_view_count": null,
       "_view_module": "@jupyter-widgets/base",
       "_view_module_version": "1.2.0",
       "_view_name": "LayoutView",
       "align_content": null,
       "align_items": null,
       "align_self": null,
       "border": null,
       "bottom": null,
       "display": null,
       "flex": null,
       "flex_flow": null,
       "grid_area": null,
       "grid_auto_columns": null,
       "grid_auto_flow": null,
       "grid_auto_rows": null,
       "grid_column": null,
       "grid_gap": null,
       "grid_row": null,
       "grid_template_areas": null,
       "grid_template_columns": null,
       "grid_template_rows": null,
       "height": null,
       "justify_content": null,
       "justify_items": null,
       "left": null,
       "margin": null,
       "max_height": null,
       "max_width": null,
       "min_height": null,
       "min_width": null,
       "object_fit": null,
       "object_position": null,
       "order": null,
       "overflow": null,
       "overflow_x": null,
       "overflow_y": null,
       "padding": null,
       "right": null,
       "top": null,
       "visibility": null,
       "width": null
      }
     },
     "7b60ec0a6d3a4e2bb6ac19583d162278": {
      "model_module": "@jupyter-widgets/base",
      "model_module_version": "1.2.0",
      "model_name": "LayoutModel",
      "state": {
       "_model_module": "@jupyter-widgets/base",
       "_model_module_version": "1.2.0",
       "_model_name": "LayoutModel",
       "_view_count": null,
       "_view_module": "@jupyter-widgets/base",
       "_view_module_version": "1.2.0",
       "_view_name": "LayoutView",
       "align_content": null,
       "align_items": null,
       "align_self": null,
       "border": null,
       "bottom": null,
       "display": null,
       "flex": null,
       "flex_flow": null,
       "grid_area": null,
       "grid_auto_columns": null,
       "grid_auto_flow": null,
       "grid_auto_rows": null,
       "grid_column": null,
       "grid_gap": null,
       "grid_row": null,
       "grid_template_areas": null,
       "grid_template_columns": null,
       "grid_template_rows": null,
       "height": null,
       "justify_content": null,
       "justify_items": null,
       "left": null,
       "margin": null,
       "max_height": null,
       "max_width": null,
       "min_height": null,
       "min_width": null,
       "object_fit": null,
       "object_position": null,
       "order": null,
       "overflow": null,
       "overflow_x": null,
       "overflow_y": null,
       "padding": null,
       "right": null,
       "top": null,
       "visibility": null,
       "width": null
      }
     },
     "806459a922704ad0ac4d9932222e37ec": {
      "model_module": "@jupyter-widgets/controls",
      "model_module_version": "1.5.0",
      "model_name": "DescriptionStyleModel",
      "state": {
       "_model_module": "@jupyter-widgets/controls",
       "_model_module_version": "1.5.0",
       "_model_name": "DescriptionStyleModel",
       "_view_count": null,
       "_view_module": "@jupyter-widgets/base",
       "_view_module_version": "1.2.0",
       "_view_name": "StyleView",
       "description_width": ""
      }
     },
     "908bd482f9a54fc4a3dce47942062ea0": {
      "model_module": "@jupyter-widgets/controls",
      "model_module_version": "1.5.0",
      "model_name": "FloatProgressModel",
      "state": {
       "_dom_classes": [],
       "_model_module": "@jupyter-widgets/controls",
       "_model_module_version": "1.5.0",
       "_model_name": "FloatProgressModel",
       "_view_count": null,
       "_view_module": "@jupyter-widgets/controls",
       "_view_module_version": "1.5.0",
       "_view_name": "ProgressView",
       "bar_style": "success",
       "description": "",
       "description_tooltip": null,
       "layout": "IPY_MODEL_56ecc6d275564e3da4b8c41f525f5e1d",
       "max": 2.0,
       "min": 0.0,
       "orientation": "horizontal",
       "style": "IPY_MODEL_6de9f56829cc40d5b3a6912ee14fce7d",
       "value": 2.0
      }
     },
     "a4a0c7d3bcdd44b6badb5021ee022260": {
      "model_module": "@jupyter-widgets/controls",
      "model_module_version": "1.5.0",
      "model_name": "HTMLModel",
      "state": {
       "_dom_classes": [],
       "_model_module": "@jupyter-widgets/controls",
       "_model_module_version": "1.5.0",
       "_model_name": "HTMLModel",
       "_view_count": null,
       "_view_module": "@jupyter-widgets/controls",
       "_view_module_version": "1.5.0",
       "_view_name": "HTMLView",
       "description": "",
       "description_tooltip": null,
       "layout": "IPY_MODEL_a9e97bd6367d4b2889fc1dfb44b151e6",
       "placeholder": "​",
       "style": "IPY_MODEL_20f51b30c1a84093ad473e0572a0ddb9",
       "value": "100%"
      }
     },
     "a9e97bd6367d4b2889fc1dfb44b151e6": {
      "model_module": "@jupyter-widgets/base",
      "model_module_version": "1.2.0",
      "model_name": "LayoutModel",
      "state": {
       "_model_module": "@jupyter-widgets/base",
       "_model_module_version": "1.2.0",
       "_model_name": "LayoutModel",
       "_view_count": null,
       "_view_module": "@jupyter-widgets/base",
       "_view_module_version": "1.2.0",
       "_view_name": "LayoutView",
       "align_content": null,
       "align_items": null,
       "align_self": null,
       "border": null,
       "bottom": null,
       "display": null,
       "flex": null,
       "flex_flow": null,
       "grid_area": null,
       "grid_auto_columns": null,
       "grid_auto_flow": null,
       "grid_auto_rows": null,
       "grid_column": null,
       "grid_gap": null,
       "grid_row": null,
       "grid_template_areas": null,
       "grid_template_columns": null,
       "grid_template_rows": null,
       "height": null,
       "justify_content": null,
       "justify_items": null,
       "left": null,
       "margin": null,
       "max_height": null,
       "max_width": null,
       "min_height": null,
       "min_width": null,
       "object_fit": null,
       "object_position": null,
       "order": null,
       "overflow": null,
       "overflow_x": null,
       "overflow_y": null,
       "padding": null,
       "right": null,
       "top": null,
       "visibility": null,
       "width": null
      }
     },
     "bf1285ba4f1c467d9c51bc99f630bee3": {
      "model_module": "@jupyter-widgets/controls",
      "model_module_version": "1.5.0",
      "model_name": "ProgressStyleModel",
      "state": {
       "_model_module": "@jupyter-widgets/controls",
       "_model_module_version": "1.5.0",
       "_model_name": "ProgressStyleModel",
       "_view_count": null,
       "_view_module": "@jupyter-widgets/base",
       "_view_module_version": "1.2.0",
       "_view_name": "StyleView",
       "bar_color": null,
       "description_width": ""
      }
     },
     "e504dc180cb845aa8a38c4f69e7ad2e9": {
      "model_module": "@jupyter-widgets/base",
      "model_module_version": "1.2.0",
      "model_name": "LayoutModel",
      "state": {
       "_model_module": "@jupyter-widgets/base",
       "_model_module_version": "1.2.0",
       "_model_name": "LayoutModel",
       "_view_count": null,
       "_view_module": "@jupyter-widgets/base",
       "_view_module_version": "1.2.0",
       "_view_name": "LayoutView",
       "align_content": null,
       "align_items": null,
       "align_self": null,
       "border": null,
       "bottom": null,
       "display": null,
       "flex": null,
       "flex_flow": null,
       "grid_area": null,
       "grid_auto_columns": null,
       "grid_auto_flow": null,
       "grid_auto_rows": null,
       "grid_column": null,
       "grid_gap": null,
       "grid_row": null,
       "grid_template_areas": null,
       "grid_template_columns": null,
       "grid_template_rows": null,
       "height": null,
       "justify_content": null,
       "justify_items": null,
       "left": null,
       "margin": null,
       "max_height": null,
       "max_width": null,
       "min_height": null,
       "min_width": null,
       "object_fit": null,
       "object_position": null,
       "order": null,
       "overflow": null,
       "overflow_x": null,
       "overflow_y": null,
       "padding": null,
       "right": null,
       "top": null,
       "visibility": null,
       "width": null
      }
     },
     "f4b44d77020b43058fcdeca67f254129": {
      "model_module": "@jupyter-widgets/controls",
      "model_module_version": "1.5.0",
      "model_name": "FloatProgressModel",
      "state": {
       "_dom_classes": [],
       "_model_module": "@jupyter-widgets/controls",
       "_model_module_version": "1.5.0",
       "_model_name": "FloatProgressModel",
       "_view_count": null,
       "_view_module": "@jupyter-widgets/controls",
       "_view_module_version": "1.5.0",
       "_view_name": "ProgressView",
       "bar_style": "success",
       "description": "",
       "description_tooltip": null,
       "layout": "IPY_MODEL_e504dc180cb845aa8a38c4f69e7ad2e9",
       "max": 2.0,
       "min": 0.0,
       "orientation": "horizontal",
       "style": "IPY_MODEL_bf1285ba4f1c467d9c51bc99f630bee3",
       "value": 2.0
      }
     }
    },
    "version_major": 2,
    "version_minor": 0
   }
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}