fb3-cv.ipynb

{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.7.12","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"# ====================================================\n# Library\n# ====================================================\nimport os\nimport gc\nimport re\nimport ast\nimport sys\nimport copy\nimport json\nimport time\nimport math \nimport string\nimport pickle\nimport random\nimport joblib\nimport itertools\nimport warnings\nwarnings.filterwarnings(\"ignore\")\nfrom IPython. display import clear_output\n\nimport scipy as sp\nimport numpy as np\nimport pandas as pd\npd.set_option('display.max_rows', 500)\npd.set_option('display.max_columns', 500)\npd.set_option('display.width', 1000)\nfrom tqdm.auto import tqdm\nfrom sklearn.metrics import mean_squared_error\nfrom sklearn.model_selection import StratifiedKFold, GroupKFold, KFold\nfrom nltk.corpus import stopwords\nfrom nltk.tokenize import word_tokenize\n\nimport torch\nimport torch.nn as nn\nfrom torch.nn import Parameter\nimport torch.nn.functional as F\nfrom torch.optim import Adam, SGD, AdamW\nfrom torch.utils.data import DataLoader, Dataset\n\n#os.system('pip install iterative-stratification==0.1.7')\n#from iterstrat.ml_stratifiers import MultilabelStratifiedKFold\nsys.path.append('../input/iterativestratification')\nfrom iterstrat.ml_stratifiers import MultilabelStratifiedKFold\n\nos.system('pip uninstall -y transformers')\nos.system('pip uninstall -y tokenizers')\nos.system('python -m pip install --no-index --find-links=../input/fb3-my-pip-wheels transformers')\nos.system('python -m pip install --no-index --find-links=../input/fb3-my-pip-wheels tokenizers')\nimport tokenizers\nimport transformers\nfrom transformers import AutoTokenizer, AutoModel, AutoConfig\nfrom transformers import get_linear_schedule_with_warmup, get_cosine_schedule_with_warmup\nfrom transformers import DataCollatorWithPadding\n%env TOKENIZERS_PARALLELISM=false\n\nclear_output()\ndevice = torch.device('cuda' if torch.cuda.is_available() else 'cpu')\nprint(f\"tokenizers.__version__: {tokenizers.__version__}\")\nprint(f\"transformers.__version__: {transformers.__version__}\")\nprint('device:', device)","metadata":{"_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","papermill":{"duration":33.332943,"end_time":"2022-11-18T16:51:59.066334","exception":false,"start_time":"2022-11-18T16:51:25.733391","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2022-11-29T03:45:36.552978Z","iopub.execute_input":"2022-11-29T03:45:36.553512Z","iopub.status.idle":"2022-11-29T03:46:26.420784Z","shell.execute_reply.started":"2022-11-29T03:45:36.553407Z","shell.execute_reply":"2022-11-29T03:46:26.419080Z"},"trusted":true},"execution_count":1,"outputs":[{"name":"stdout","text":"tokenizers.__version__: 0.12.1\ntransformers.__version__: 4.20.1\ndevice: cuda\n","output_type":"stream"}]},{"cell_type":"code","source":"BASE_PATH = '/kaggle/input/feedback-prize-english-language-learning'\nSUBMISSION_PATH = os.path.join(BASE_PATH, 'sample_submission.csv')\nTRAIN_PATH = os.path.join(BASE_PATH, 'train.csv')\nTEST_PATH = os.path.join(BASE_PATH, 'test.csv')","metadata":{"papermill":{"duration":0.017496,"end_time":"2022-11-18T16:51:59.092905","exception":false,"start_time":"2022-11-18T16:51:59.075409","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2022-11-29T03:46:26.423552Z","iopub.execute_input":"2022-11-29T03:46:26.425337Z","iopub.status.idle":"2022-11-29T03:46:26.433799Z","shell.execute_reply.started":"2022-11-29T03:46:26.425274Z","shell.execute_reply":"2022-11-29T03:46:26.431997Z"},"trusted":true},"execution_count":2,"outputs":[]},{"cell_type":"code","source":"class FB3Dataset(Dataset):\n    def __init__(self, cfg, data):\n        self.cfg = cfg\n        self.xs = preprocess(data['full_text'])\n        self.ys = data[cfg.target_cols].values \n        \n    def __len__(self):\n        return len(self.xs)\n    \n    def __getitem__(self, idx):\n        x = encode_text(self.cfg, self.xs[idx])\n        y = torch.tensor(self.ys[idx], dtype=torch.float)\n        return x, y\n    \ndef collate(inputs):\n    # Trimming input.\n    mask_len = int(inputs['attention_mask'].sum(axis=1).max())\n    for k, v in inputs.items():\n        inputs[k] = inputs[k][:,:mask_len]\n    #print(type(inputs), inputs['attention_mask'].size())\n    return inputs\n\n##################################################################################\n\nclass MeanPooling(nn.Module):\n    def __init__(self):\n        super(MeanPooling, self).__init__()\n    \n    def forward(self, last_hidden_state, attention_mask):\n        input_mask_expanded = attention_mask.unsqueeze(-1).expand(last_hidden_state.size()).float()\n        sum_embeddings = torch.sum(last_hidden_state * input_mask_expanded, 1)\n        sum_mask = torch.clamp(input_mask_expanded.sum(1), min=1e-9)\n        mean_embeddings = sum_embeddings / sum_mask\n        return mean_embeddings\n\nclass WeightedLayerPooling(nn.Module):\n    def __init__(self, num_hidden_layers, layer_start: int = 9, layer_weights = None):\n        super(WeightedLayerPooling, self).__init__()\n        self.layer_start = layer_start\n        self.num_hidden_layers = num_hidden_layers\n        self.layer_weights = layer_weights if layer_weights is not None \\\n            else nn.Parameter(\n                torch.tensor([1] * (num_hidden_layers+1 - layer_start), dtype=torch.float))\n        \n    def forward(self, all_hidden_states):\n        all_layer_embedding = all_hidden_states[self.layer_start:, :, :, :]\n        weight_factor = self.layer_weights.unsqueeze(-1).unsqueeze(-1).unsqueeze(-1).expand(all_layer_embedding.size())\n        weighted_average = (weight_factor*all_layer_embedding).sum(dim=0) / self.layer_weights.sum()\n        return weighted_average\n\nclass AttentionHead(nn.Module):\n    def __init__(self, in_features, hidden_dim):\n        super().__init__()\n        self.in_features = in_features\n        self.middle_features = hidden_dim\n        self.W = nn.Linear(in_features, hidden_dim)\n        self.V = nn.Linear(hidden_dim, 1)\n        self.out_features = hidden_dim\n\n    def forward(self, features, *args):\n        att = torch.tanh(self.W(features))\n        score = self.V(att)\n        attention_weights = torch.softmax(score, dim=1)\n        context_vector = attention_weights * features\n        context_vector = torch.sum(context_vector, dim=1)\n\n        return context_vector\n\nclass AttentionPooling(nn.Module):\n    def __init__(self, in_dim):\n        super().__init__()\n        self.attention = nn.Sequential(\n        nn.Linear(in_dim, in_dim),\n        nn.LayerNorm(in_dim),\n        nn.GELU(),\n        nn.Linear(in_dim, 1),\n        )\n\n    def forward(self, last_hidden_state, attention_mask):\n        w = self.attention(last_hidden_state).float()\n        w[attention_mask==0]=float('-inf')\n        w = torch.softmax(w,1)\n        attention_embeddings = torch.sum(w * last_hidden_state, dim=1)\n        return attention_embeddings\n    \ndef GlobalAveragePool1d(x):\n    return F.avg_pool1d(x, x.size()[-1]).squeeze(-1)\n\ndef GlobalMaxPool1d(x):\n    return F.max_pool1d(x, x.size()[-1]).squeeze(-1)\n\ndef Conv1dReg(x, in_channels, out_channels, kernel_size, device):\n    out = nn.Conv1d(in_channels, out_channels, kernel_size, padding='same', stride=1, device=device)(x)\n    out = nn.BatchNorm1d(out_channels, device=device)(out)\n    out = F.relu(out)\n    return out\n\nclass MultiSampleDropout(nn.Module):\n    def __init__(self, fc, num_dropout, prob_dropout):\n        super(MultiSampleDropout, self).__init__()\n        self.dropout = nn.Dropout\n        self.num_dropout = num_dropout\n        self.prob_dropout = prob_dropout\n        self.classifier = fc\n    def forward(self, out):\n        if not type(self.prob_dropout) in [float, int]:            \n            fcs = [self.classifier(self.dropout(p)(out)) for p in self.prob_dropout]\n        else:\n            fcs = [self.classifier(self.dropout(self.prob_dropout)(out)) for _ in range(self.num_dropout)]\n        \n        return torch.mean(torch.stack(fcs, dim=0), dim=0)","metadata":{"papermill":{"duration":0.034462,"end_time":"2022-11-18T16:51:59.135581","exception":false,"start_time":"2022-11-18T16:51:59.101119","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2022-11-29T03:46:26.436732Z","iopub.execute_input":"2022-11-29T03:46:26.437629Z","iopub.status.idle":"2022-11-29T03:46:26.474634Z","shell.execute_reply.started":"2022-11-29T03:46:26.437325Z","shell.execute_reply":"2022-11-29T03:46:26.472979Z"},"trusted":true},"execution_count":3,"outputs":[]},{"cell_type":"code","source":"# ====================================================\n# Model class\n# ====================================================\nclass FB3Model(nn.Module):\n    def __init__(self, cfg, config_path=None, pretrained=False):\n        super().__init__()\n        self.cfg = cfg \n        if config_path is None:\n            self.config = AutoConfig.from_pretrained(cfg.model, output_hidden_states=True)\n            # Turn off dropouts.\n            self.config.hidden_dropout = 0.\n            self.config.hidden_dropout_prob = 0.\n            self.config.attention_dropout = 0.\n            self.config.attention_probs_dropout_prob = 0.\n            #LOGGER.info(self.config)\n        else:\n            self.config = torch.load(config_path)\n        \n        if pretrained:\n            self.deberta_v3 = AutoModel.from_pretrained(cfg.model, config=self.config)\n        else:\n            self.deberta_v3 = AutoModel.from_config(self.config)\n\n        if self.cfg.reinit_last_layer:\n            # Re-init last layer of deberta.\n            for module in self.deberta_v3.encoder.layer[-1:].modules():\n                self._init_weights(module)\n        self.deberta_v3.gradient_checkpointing_enable()\n\n    def _init_weights(self, module):\n        if isinstance(module, nn.Linear):\n            module.weight.data.normal_(mean=0.0, std=self.config.initializer_range)\n            #nn.init.xavier_uniform_(module.weight.data, gain=1.0)\n            if module.bias is not None:\n                module.bias.data.zero_()\n        elif isinstance(module, nn.Embedding):\n            module.weight.data.normal_(mean=0.0, std=self.config.initializer_range)\n            if module.padding_idx is not None:\n                module.weight.data[module.padding_idx].zero_()\n        elif isinstance(module, nn.LayerNorm):\n            module.bias.data.zero_()\n            module.weight.data.fill_(1.0)\n\nclass WMPoolModel(FB3Model):\n    def __init__(self, cfg, config_path=None, pretrained=False):\n        super().__init__(cfg, config_path=config_path, pretrained=pretrained)\n\n        # Poolings.\n        self.mean_head = MeanPooling()\n        self.wpool_head = WeightedLayerPooling(self.config.num_hidden_layers, layer_start=12)\n\n        self.fc_out = nn.Linear(self.config.hidden_size, cfg.num_target)\n        self._init_weights(self.fc_out)\n        \n        self.layer_norm = nn.LayerNorm(self.config.hidden_size)\n        self.qa_output = torch.nn.Linear(self.config.hidden_size, 2)\n        self.attention_head = AttentionHead(self.config.hidden_size*4, self.config.hidden_size)\n        \n    def forward(self, x):\n        pt_out = self.deberta_v3(**x)\n        all_hidden_states = torch.stack(pt_out.hidden_states)\n        # Weighted pooling of last n layers.\n        logits = self.wpool_head(all_hidden_states)[:, 0] # Bx768\n        y_hat = self.fc_out(logits)\n        return y_hat\n\nclass MultiPoolModel(FB3Model):\n    def __init__(self, cfg, config_path=None, pretrained=False, pool='mean'):\n        super().__init__(cfg, config_path=config_path, pretrained=pretrained)\n        \n        # Define model layers.\n        self.pool_name = cfg.pool_head\n        self.fc = nn.Linear(self.config.hidden_size, self.cfg.num_target)\n        if cfg.pool_head in ['mean', 'attention', 'weighted']:\n            self.pool = self._pool_layer(cfg.pool_head)\n        elif '-' in cfg.pool_head:\n            pools = cfg.pool_head.split('-')\n            self.pool = nn.ModuleList([])\n            for pool_ in pools:\n                self.pool.append(self._pool_layer(pool_))\n            self.fc = nn.Linear(self.config.hidden_size * len(self.pool), self.cfg.num_target)\n        self._init_weights(self.fc)\n        \n        # Multi-sample dropout.\n        self.multi_dropout = MultiSampleDropout(self.fc, self.cfg.num_dropout, self.cfg.prob_dropout)\n    \n    def _pool_layer(self, pool_name):\n        assert pool_name in ['mean', 'attention', 'weighted']\n        if pool_name == 'mean':\n            pool = MeanPooling()\n        elif pool_name == 'attention':\n            pool = AttentionHead(self.config.hidden_size, self.config.hidden_size)\n        elif pool_name == 'weighted':\n            pool = WeightedLayerPooling(\n                self.config.num_hidden_layers, \n                layer_start=9,\n                layer_weights=None)\n        return pool\n    \n    def _pool_feature(self, pool, pool_name, pt_outputs, attention_mask):\n        assert pool_name in ['mean', 'attention', 'weighted']\n        last_hidden_state = pt_outputs.last_hidden_state #batch_size x max_len x hidden_size\n        all_hidden_states = torch.stack(pt_outputs.hidden_states) #num_layer x batch_size x max_len x hidden_size\n        \n        if pool_name == 'mean':\n            pool_feature = pool(last_hidden_state, attention_mask)\n        elif pool_name == 'attention':\n            pool_feature = pool(last_hidden_state)\n        elif pool_name == 'weighted':\n            # Take the CLS token only.\n            pool_feature = pool(all_hidden_states)[:, 0]\n        return pool_feature\n\n    def feature(self, x):\n        pt_outputs = self.deberta_v3(**x)\n        \n        # Pooling feat.\n        if type(self.pool) == nn.ModuleList:\n            pool_features = []\n            pool_names = self.pool_name.split('-')\n            \n            for pool_name, pool in zip(pool_names, self.pool):\n                pool_features.append(self._pool_feature(pool, pool_name, pt_outputs, x['attention_mask']))\n            pool_features = torch.cat(pool_features, dim=1)\n        else:\n            pool_features = self._pool_feature(self.pool, self.pool_name, pt_outputs, x['attention_mask'])\n        return pool_features\n    \n    def forward(self, x):\n        feature = self.feature(x)\n        if self.cfg.use_dropout and self.training:\n            y_hat = self.multi_dropout(feature)\n        else:\n            y_hat = self.fc(feature)\n        return y_hat\n\n\nclass Attention4Model(FB3Model):\n    def __init__(self, cfg, config_path=None, pretrained=False):\n        super().__init__(cfg, config_path=config_path, pretrained=pretrained)\n        \n        self.head = AttentionHead(self.config.hidden_size*4, self.config.hidden_size)\n        self.fc_out = nn.Linear(self.config.hidden_size*4*2, self.cfg.num_target)\n        self._init_weights(self.fc_out)\n\n    def forward(self, x):\n        pt_out = self.deberta_v3(**x)\n        \n        all_hidden_states = torch.stack(pt_out.hidden_states)\n        cat_over_last_layers = torch.cat(\n            (all_hidden_states[-1], all_hidden_states[-2], all_hidden_states[-3], all_hidden_states[-4]),\n            -1)\n        # [CLS] embedding.\n        cls_pooling = cat_over_last_layers[:, 0]   \n        # Concat of 4 last layers.\n        head_logits = self.head(cat_over_last_layers)\n\n        if self.cfg.use_dropout and self.training:\n            y_hat = self.multi_dropout(torch.cat([head_logits, cls_pooling], -1))\n        else:\n            y_hat = self.fc_out(torch.cat([head_logits, cls_pooling], -1))\n\n        return y_hat\n\n######################################\n\nclass FB3Model1(nn.Module):\n    def __init__(self, cfg, config_path=None, pretrained=False):\n        super().__init__()\n        self.cfg = cfg \n        if config_path is None:\n            self.config = AutoConfig.from_pretrained(cfg.model, output_hidden_states=True)\n            # Turn off dropouts.\n            self.config.hidden_dropout = 0.\n            self.config.hidden_dropout_prob = 0.\n            self.config.attention_dropout = 0.\n            self.config.attention_probs_dropout_prob = 0.\n            #LOGGER.info(self.config)\n        else:\n            self.config = torch.load(config_path)\n        \n        if pretrained:\n            self.model = AutoModel.from_pretrained(cfg.model, config=self.config)\n        else:\n            self.model = AutoModel.from_config(self.config)\n\n        if self.cfg.reinit_last_layer:\n            # Re-init last layer of deberta.\n            for module in self.model.encoder.layer[-1:].modules():\n                self._init_weights(module)\n        self.model.gradient_checkpointing_enable()\n\n    def _init_weights(self, module):\n        if isinstance(module, nn.Linear):\n            module.weight.data.normal_(mean=0.0, std=self.config.initializer_range)\n            if module.bias is not None:\n                module.bias.data.zero_()\n        elif isinstance(module, nn.Embedding):\n            module.weight.data.normal_(mean=0.0, std=self.config.initializer_range)\n            if module.padding_idx is not None:\n                module.weight.data[module.padding_idx].zero_()\n        elif isinstance(module, nn.LayerNorm):\n            module.bias.data.zero_()\n            module.weight.data.fill_(1.0)\n            \nclass WeightedAttentionModel(FB3Model1):\n    def __init__(self, cfg, config_path=None, pretrained=False):\n        super().__init__(cfg, config_path=config_path, pretrained=pretrained)\n\n        self.weighted_pool = WeightedLayerPooling(\n            self.config.num_hidden_layers, layer_start=9, layer_weights=None)\n        self.att_pool = AttentionPooling(self.config.hidden_size)\n\n        self.fc_out = nn.Linear(self.config.hidden_size*2, cfg.num_target)\n        self._init_weights(self.fc_out)\n\n    def forward(self, x):\n        pt_out = self.model(**x)\n        hidden_states = pt_out.hidden_states\n        last_hidden_state = pt_out.last_hidden_state\n\n        x1 = self.weighted_pool(torch.stack(hidden_states))[:, 0]\n        x2 = self.att_pool(last_hidden_state, x['attention_mask'])\n\n        y_hat = self.fc_out(torch.cat([x1, x2], dim=1))\n        return y_hat\n    \nclass MeanAttentionModel(nn.Module):\n    def __init__(self, cfg, config_path=None, pretrained=False):\n        super().__init__()\n        self.cfg = cfg \n        if config_path is None:\n            self.config = AutoConfig.from_pretrained(cfg.model, output_hidden_states=True)\n            # Turn off dropouts.\n            self.config.hidden_dropout = 0.\n            self.config.hidden_dropout_prob = 0.\n            self.config.attention_dropout = 0.\n            self.config.attention_probs_dropout_prob = 0.\n            #LOGGER.info(self.config)\n        else:\n            self.config = torch.load(config_path)\n        \n        if pretrained:\n            self.deberta_v3 = AutoModel.from_pretrained(cfg.model, config=self.config)\n            # Expand embedding dim for new tokens.\n            self.deberta_v3.resize_token_embeddings(len(cfg.tokenizer))\n        else:\n            self.deberta_v3 = AutoModel.from_config(self.config)\n            \n        self.deberta_v3.gradient_checkpointing_enable()\n        \n        # Define model layers.\n        self.fc = nn.Linear(self.config.hidden_size, 6)\n\n        if cfg.pool == 'mean':\n            self.pool = MeanPooling()\n        elif cfg.pool == 'attention':\n            self.pool = AttentionHead(self.config.hidden_size, self.config.hidden_size)\n        elif cfg.pool == 'mean-attention':\n            self.pool = nn.ModuleList([\n                MeanPooling(),\n                AttentionHead(self.config.hidden_size, self.config.hidden_size)\n            ])\n            self.fc = nn.Linear(self.config.hidden_size * len(self.pool), 6)\n        elif cfg.pool == 'mean-attention-with-mask':\n            self.pool = nn.ModuleList([\n                MeanPooling(),\n                AttentionPooling(self.config.hidden_size)\n            ])\n            self.fc = nn.Linear(self.config.hidden_size * len(self.pool), 6)\n        # Re-init weights.\n        self._init_weights(self.fc)\n        \n        # Multi-sample dropout.\n        self.multi_dropout = MultiSampleDropout(self.fc, cfg.num_dropout, cfg.prob_dropout)\n        \n    def global_avg_pool(x):\n        return torch.mean(x.view(x.size(0), x.size(1), -1), dim=-1)\n        \n    def _init_weights(self, module):\n        if isinstance(module, nn.Linear):\n            module.weight.data.normal_(mean=0.0, std=self.config.initializer_range)\n            if module.bias is not None:\n                module.bias.data.zero_()\n        elif isinstance(module, nn.Embedding):\n            module.weight.data.normal_(mean=0.0, std=self.config.initializer_range)\n            if module.padding_idx is not None:\n                module.weight.data[module.padding_idx].zero_()\n        elif isinstance(module, nn.LayerNorm):\n            module.bias.data.zero_()\n            module.weight.data.fill_(1.0)\n    \n    def feature(self, x):\n        pt_outputs = self.deberta_v3(**x)\n        last_hidden_states = pt_outputs[0] # N x max_len x 768\n        # Pooling feat.\n        if type(self.pool) == nn.ModuleList:\n            pool_feature = [pool(last_hidden_states, x['attention_mask']) for pool in self.pool]\n            pool_feature = torch.cat(pool_feature, dim=1)\n        else:\n            pool_feature = self.pool(last_hidden_states, x['attention_mask']) # N x 768\n        return pool_feature\n    \n    def forward(self, x, y=None, loss_fn=None):\n        feature = self.feature(x)\n        # if self.training:\n        #    out = self.multi_dropout(feature)\n        # else:\n        #    out = self.fc(feature)\n        out = self.fc(feature)\n        return out\n    \n##########\nclass WeightedLayerPooling_(nn.Module):\n    def __init__(self, num_hidden_layers, layer_start: int = 4, layers = None, layer_weights = None):\n        super(WeightedLayerPooling_, self).__init__()\n        self.layer_start = layer_start\n        self.num_hidden_layers = num_hidden_layers\n        \n        if layers:\n            self.layer_weights = layer_weights if layer_weights is not None \\\n                else nn.Parameter(\n                    torch.tensor([1] * len(layers), dtype=torch.float)\n                )\n            self.layers = layers\n        else:\n            self.layer_weights = layer_weights if layer_weights is not None \\\n                else nn.Parameter(\n                   torch.tensor([1] * (num_hidden_layers+1 - layer_start), dtype=torch.float)\n                )\n            self.layers = list(range(layer_start, num_hidden_layers+1))\n            \n\n    def forward(self, ft_all_layers):\n        all_layer_embedding = torch.stack(ft_all_layers)\n        all_layer_embedding = all_layer_embedding[self.layers, :, :, :]\n        \n        weight_factor = self.layer_weights.unsqueeze(-1).unsqueeze(-1).unsqueeze(-1).expand(all_layer_embedding.size())\n        weighted_average = (weight_factor*all_layer_embedding).sum(dim=0) / self.layer_weights.sum()\n\n        return weighted_average\n    \nclass CustomModel(nn.Module):\n\n    def __init__(self, CFG, config_path = None, pretrained = False):\n        super().__init__()\n        self.CFG = CFG\n        self.config = AutoConfig.from_pretrained(config_path, output_hidden_states=True)\n        self.model = AutoModel.from_config(self.config)\n        self.pretrained = pretrained\n                        \n        fc_hidden_size = self.config.hidden_size\n        if CFG.pooling == 'mean':\n            self.pool = MeanPooling()\n        elif CFG.pooling == 'max':\n            self.pool = MaxPooling()\n        elif CFG.pooling == 'min':\n            self.pool = MinPooling()\n        elif CFG.pooling == 'attention':\n            self.pool = AttentionPooling(self.config.hidden_size)\n        elif CFG.pooling == 'weightedlayer':\n            self.pool = WeightedLayerPooling_(self.config.num_hidden_layers, layer_start = CFG.layer_start, layer_weights = None)\n        elif CFG.pooling == 'weightedlayer-mean':\n            self.pool = WeightedLayerPooling_(self.config.num_hidden_layers, layer_start = CFG.layer_start, layer_weights = None)\n            self.mean_pool = MeanPooling()\n        elif self.CFG.pooling == 'attention4':\n            self.pool = AttentionHead(fc_hidden_size*4, 512)\n            fc_hidden_size = fc_hidden_size*8\n        self.fc = nn.Linear(fc_hidden_size, 6)\n                        \n    def feature(self, inputs):\n        outputs = self.model(**inputs)\n        if self.CFG.pooling == 'attention4':\n            all_layer_embeddings = torch.stack(outputs.hidden_states)\n            cat_over_last_layers = torch.cat((all_layer_embeddings[-1], all_layer_embeddings[-2], all_layer_embeddings[-3], all_layer_embeddings[-4]), -1)\n            cls_pooling = cat_over_last_layers[:, 0]\n            head_logits = self.pool(cat_over_last_layers)\n            feature = torch.cat([head_logits, cls_pooling], -1)\n        elif self.CFG.pooling == 'weightedlayer':\n            all_layer_embeddings = outputs[1]\n            feature = self.pool(all_layer_embeddings)[:, 0]\n        elif self.CFG.pooling == 'weightedlayer-mean':\n            all_layer_embeddings = outputs[1]\n            feature = self.pool(all_layer_embeddings)\n            feature = self.mean_pool(feature, inputs['attention_mask'])\n        else:\n            last_hidden_states = outputs[0]\n            feature = self.pool(last_hidden_states, inputs['attention_mask'])    \n        return feature\n\n    def forward(self, inputs):\n        feature = self.feature(inputs)\n        output = self.fc(feature)\n        return output","metadata":{"papermill":{"duration":0.06014,"end_time":"2022-11-18T16:51:59.203954","exception":false,"start_time":"2022-11-18T16:51:59.143814","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2022-11-29T03:46:26.479151Z","iopub.execute_input":"2022-11-29T03:46:26.479813Z","iopub.status.idle":"2022-11-29T03:46:26.568882Z","shell.execute_reply.started":"2022-11-29T03:46:26.479756Z","shell.execute_reply":"2022-11-29T03:46:26.567487Z"},"trusted":true},"execution_count":4,"outputs":[]},{"cell_type":"code","source":"class Config(dict):\n    __getattr__ = dict.__getitem__\n    __setattr__ = dict.__setitem__\n    __delattr__ = dict.__delitem__\n    \n    def init(self, kwargs):\n        super().init(kwargs)\n\n        for k, v in kwargs.items():\n            setattr(self, k, v)\n\n    def set(self, key, val):\n        self[key] = val\n        setattr(self, key, val)","metadata":{"papermill":{"duration":0.016698,"end_time":"2022-11-18T16:51:59.228755","exception":false,"start_time":"2022-11-18T16:51:59.212057","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2022-11-29T03:46:26.571546Z","iopub.execute_input":"2022-11-29T03:46:26.572112Z","iopub.status.idle":"2022-11-29T03:46:26.584006Z","shell.execute_reply.started":"2022-11-29T03:46:26.572048Z","shell.execute_reply":"2022-11-29T03:46:26.582536Z"},"trusted":true},"execution_count":5,"outputs":[]},{"cell_type":"code","source":"loggers = {}\ndef get_logger(filename='inference'):\n    from logging import getLogger, INFO, StreamHandler, FileHandler, Formatter\n    logger = getLogger(__name__)\n    logger.propagate = False\n    logger.setLevel(INFO)\n    handler1 = StreamHandler()\n    handler1.setFormatter(Formatter(\"%(message)s\"))\n    handler2 = FileHandler(filename=f\"{filename}.log\")\n    handler2.setFormatter(Formatter(\"%(message)s\"))\n    logger.addHandler(handler1)\n    logger.addHandler(handler2)\n    return logger\n\nlogger = get_logger()","metadata":{"execution":{"iopub.status.busy":"2022-11-29T03:46:26.586746Z","iopub.execute_input":"2022-11-29T03:46:26.587347Z","iopub.status.idle":"2022-11-29T03:46:26.598479Z","shell.execute_reply.started":"2022-11-29T03:46:26.587297Z","shell.execute_reply":"2022-11-29T03:46:26.596989Z"},"trusted":true},"execution_count":6,"outputs":[]},{"cell_type":"code","source":"def seed_everything(seed=42):\n    '''\n    Sets the seed of the entire notebook so results are the same every time we run.\n    This is for REPRODUCIBILITY.\n    '''\n    random.seed(seed)\n    # Set a fixed value for the hash seed\n    os.environ['PYTHONHASHSEED'] = str(seed)\n    np.random.seed(seed)\n    torch.manual_seed(seed)\n    \n    if torch.cuda.is_available():\n        torch.cuda.manual_seed(seed)\n        torch.cuda.manual_seed_all(seed)\n        # When running on the CuDNN backend, two further options must be set\n        torch.backends.cudnn.deterministic = True\n        torch.backends.cudnn.benchmark = False\nseed_everything(seed=42)\n\ndef mc_rmse(y_true, y_pred):\n    scores = []\n    ncols = y_true.shape[1]\n    \n    for n in range(ncols):\n        yn_true = y_true[:, n]\n        yn_pred = y_pred[:, n]\n        rmse_ = mean_squared_error(yn_true, yn_pred, squared=False)\n        scores.append(rmse_)\n    score = np.mean(scores) \n    return score, scores\n\ntarget_cols = ['cohesion', 'syntax', 'vocabulary', 'phraseology', 'grammar', 'conventions']\npred_cols = [f'pred_{col}' for col in target_cols]\ndef get_result(cfg, oof_df):\n    labels = oof_df[target_cols].values\n    preds = oof_df[[f\"pred_{c}\" for c in target_cols]].values\n    score, scores = mc_rmse(labels, preds)\n    print(f'score: {score:<.6f}  scores: {scores}')","metadata":{"papermill":{"duration":0.022951,"end_time":"2022-11-18T16:51:59.259473","exception":false,"start_time":"2022-11-18T16:51:59.236522","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2022-11-29T03:46:26.600423Z","iopub.execute_input":"2022-11-29T03:46:26.602342Z","iopub.status.idle":"2022-11-29T03:46:26.622645Z","shell.execute_reply.started":"2022-11-29T03:46:26.602278Z","shell.execute_reply":"2022-11-29T03:46:26.620956Z"},"trusted":true},"execution_count":7,"outputs":[]},{"cell_type":"code","source":"# ====================================================\n# Dataset\n# ====================================================\ndef encode_text(cfg, text):\n    if cfg.pretrained:\n        inputs = cfg.tokenizer(\n            text,\n            None,\n            add_special_tokens=True,\n            padding='max_length',\n            truncation=True,\n            max_length=cfg.max_len,\n            return_token_type_ids=True,\n            return_tensors='pt'\n        )\n        inputs = {k:v.squeeze(0) for k,v in inputs.items()}\n    else:\n        if 'roberta' in cfg.model:\n            inputs = cfg.tokenizer.encode_plus(\n                text,\n                return_tensors = None,\n                add_special_tokens = True,\n                max_length = cfg.max_len,\n                pad_to_max_length = True,\n                truncation = True)\n        elif '512' in cfg.name or '768' in cfg.name:\n            inputs = cfg.tokenizer.encode_plus(\n                text,\n                return_tensors = None,\n                add_special_tokens = True,\n                max_length = cfg.max_len,\n                truncation = True)\n        else:\n            inputs = cfg.tokenizer.encode_plus(\n                text, \n                return_tensors=None, \n                add_special_tokens=True)\n        for k, v in inputs.items():\n            inputs[k] = torch.tensor(v, dtype=torch.long)\n    return inputs \n\ndef preprocess(texts):\n    texts = (\n        texts\n        .str.replace(r'\\r\\n', '<newline>', regex=True)\n        .str.replace(r'\\n', '<newline>', regex=True)\n        .str.replace('<newline><newline>', '<newline>', regex=False)\n        .values \n    )\n    return texts\n\nclass TestDataset(Dataset):\n    def __init__(self, cfg, df):\n        self.cfg = cfg\n        if cfg.pretrained:\n            self.texts = df['full_text'].values\n        else:\n            self.texts = preprocess(df['full_text'])\n\n    def __len__(self):\n        return len(self.texts)\n\n    def __getitem__(self, item):\n        inputs = encode_text(self.cfg, self.texts[item])\n        return inputs","metadata":{"papermill":{"duration":0.020607,"end_time":"2022-11-18T16:51:59.288893","exception":false,"start_time":"2022-11-18T16:51:59.268286","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2022-11-29T03:46:26.624945Z","iopub.execute_input":"2022-11-29T03:46:26.625507Z","iopub.status.idle":"2022-11-29T03:46:26.644815Z","shell.execute_reply.started":"2022-11-29T03:46:26.625454Z","shell.execute_reply":"2022-11-29T03:46:26.643090Z"},"trusted":true},"execution_count":8,"outputs":[]},{"cell_type":"code","source":"def load_config(input_path, inference_weight=1):\n    # Load CFG class.\n    cfg = Config(**json.load(open(os.path.join(input_path, 'CFG.json'), 'r')))\n    cfg.path = input_path\n    cfg.config_path = os.path.join(cfg.path, 'config.pth')\n    # Load tokenizer.\n    tokenizer = AutoTokenizer.from_pretrained(os.path.join(cfg.path, 'tokenizer'))\n    cfg.tokenizer = tokenizer\n    \n    cfg.inference_weight = inference_weight\n    return cfg\n\ndef load_model(cfg, fold, version='1', **model_kwargs):\n    # Load torch model.\n    if version == '1':\n        model = MultiPoolModel(cfg, config_path=cfg.config_path, pretrained=False, **model_kwargs)\n    elif version == '2':\n        model = Attention4Model(cfg, config_path=cfg.config_path, pretrained=False, **model_kwargs)\n    elif version == '21':\n        model = WMPoolModel(cfg, config_path=cfg.config_path, pretrained=False, **model_kwargs)\n    elif version == 'weighted-attention':\n        model = WeightedAttentionModel(cfg, config_path=cfg.config_path, pretrained=False, **model_kwargs)\n    elif version == 'custom':\n        model = CustomModel(cfg, config_path=cfg.config_path, pretrained=False, **model_kwargs)\n    elif version == 'mean-attention':\n        model = MeanAttentionModel(cfg, config_path=cfg.config_path, pretrained=False)\n    state = torch.load(\n        os.path.join(cfg.path, f\"{cfg.model.replace('/', '-')}_fold{fold}_best.pth\"),\n        map_location=torch.device('cpu'))\n    model.load_state_dict(state['model'])\n    return model","metadata":{"papermill":{"duration":0.019417,"end_time":"2022-11-18T16:51:59.316114","exception":false,"start_time":"2022-11-18T16:51:59.296697","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2022-11-29T03:46:26.647009Z","iopub.execute_input":"2022-11-29T03:46:26.647976Z","iopub.status.idle":"2022-11-29T03:46:26.664436Z","shell.execute_reply.started":"2022-11-29T03:46:26.647918Z","shell.execute_reply":"2022-11-29T03:46:26.662718Z"},"trusted":true},"execution_count":9,"outputs":[]},{"cell_type":"code","source":"def mean_pooling(model_output, attention_mask):\n    token_embeddings = model_output.last_hidden_state.detach().cpu()\n    input_mask_expanded = (\n        attention_mask.unsqueeze(-1).expand(token_embeddings.size()).float()\n    )\n    return torch.sum(token_embeddings * input_mask_expanded, 1) / torch.clamp(\n        input_mask_expanded.sum(1), min=1e-9\n    )\n\ndef inference_fn(test_loader, model, device):\n    preds = []\n    model.eval()\n    model.to(device)\n    #tk0 = tqdm(test_loader, total=len(test_loader))\n    for inputs in test_loader:\n        for k, v in inputs.items():\n            inputs[k] = v.to(device)\n        with torch.no_grad():\n            y_preds = model(inputs)\n        preds.append(y_preds.to('cpu').numpy())\n    predictions = np.concatenate(preds)\n    return predictions\n\nclass Inferencer:\n    def __init__(self, input_path=None, cfg=None, inference_weight=1):\n        if cfg == None:\n            self.cfg = load_config(input_path, inference_weight)\n        else:\n            self.cfg = cfg\n    \n    def predict(self, test_loader, device, stat_fn=np.mean):\n        preds = []\n        \n        for fold in self.cfg.trn_fold:\n            start = time.time()\n            print('#'*10, cfg.path, '#'*10)\n            \n            print(f'Predicting fold {fold}...')\n            model = load_model(self.cfg, fold, version=self.cfg.version)\n            pred = inference_fn(test_loader, model, device)\n            preds.append(pred)\n            del model, pred; gc.collect()\n            torch.cuda.empty_cache()\n            \n            end = time.time() - start\n            print('#'*10, f'ETA: {end:.2f}s', '#'*10, '\\n')\n        \n        \n        self.preds = stat_fn(preds, axis=0) \n        self.preds = np.clip(self.preds, 1, 5)\n        return self.preds\n    \n    def get_oof_result(self, file_type='pkl'):\n        return get_result(self.cfg, self.get_oof_df(file_type))\n    \n    def get_oof_df(self, file_type='pkl'):\n        if file_type == 'pkl':\n            return pd.read_pickle(os.path.join(cfg.path, 'oof_df.pkl'))\n        return pd.read_csv(os.path.join(cfg.path, 'oof_df.csv'))\n    \n    def get_text_embedding(self, data_loader, device, fold=None): \n        # pretrained=True: not fine-tuned models.\n        if not self.cfg.pretrained:\n            model = load_model(self.cfg, fold, pool=self.cfg.pool_head)            \n        else:\n            model = AutoModel.from_pretrained(self.cfg.model)\n        model.to(device)\n        model.eval()\n            \n        fold_emb = []\n        for inputs in data_loader:\n            for k, v in inputs.items():\n                inputs[k] = v.to(device)\n            if not self.cfg.pretrained:\n                with torch.no_grad():\n                    emb = model.feature(**inputs)\n            else:\n                input_ids = inputs['input_ids'].to(device)\n                attention_mask = inputs['attention_mask'].to(device)\n                token_type_ids = inputs['token_type_ids'].to(device)\n                \n                with torch.no_grad():\n                    try:\n                        output = model(input_ids=input_ids, attention_mask=attention_mask, token_type_ids=token_type_ids)\n                    except:\n                        output = model(input_ids=input_ids, attention_mask=attention_mask)\n                emb = mean_pooling(output, attention_mask.detach().cpu())\n                emb = F.normalize(emb, p=2, dim=1)\n                emb = emb.squeeze(0)\n            fold_emb.extend(emb.detach().cpu().numpy())\n            del emb; gc.collect(); torch.cuda.empty_cache();\n            #print(torch.cuda.memory_allocated() /1024/1024)\n            \n        fold_emb = np.array(fold_emb)\n        return fold_emb\n","metadata":{"papermill":{"duration":0.029249,"end_time":"2022-11-18T16:51:59.353145","exception":false,"start_time":"2022-11-18T16:51:59.323896","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2022-11-29T03:46:26.673356Z","iopub.execute_input":"2022-11-29T03:46:26.674024Z","iopub.status.idle":"2022-11-29T03:46:26.701869Z","shell.execute_reply.started":"2022-11-29T03:46:26.673968Z","shell.execute_reply":"2022-11-29T03:46:26.700141Z"},"trusted":true},"execution_count":10,"outputs":[]},{"cell_type":"code","source":"def get_text_embedding(cfg, dfs):\n    cfg.tokenizer = AutoTokenizer.from_pretrained(cfg.model)\n    infer_ = Inferencer(cfg=cfg, inference_weight=cfg.inference_weight)\n    if 'gpt2' in cfg.model:\n        cfg.tokenizer.pad_token = cfg.tokenizer.eos_token\n    text_embs = []\n    for df in dfs:\n        dataset = TestDataset(cfg, df)\n        loader = DataLoader(\n            dataset,\n            batch_size=4,\n            shuffle=False)\n\n        # Text embedding for SVM\n        test_text_emb = []\n        if not cfg.pretrained:\n            for fold in infer_.cfg.trn_fold:\n                test_text_emb.append(infer_.get_text_embedding(loader, device, fold))\n            text_emb = np.mean(text_emb, axis=0)\n        else:\n            text_emb = infer_.get_text_embedding(loader, device)\n        text_embs.append(text_emb)\n        del dataset, loader; gc.collect(); torch.cuda.empty_cache();\n    del infer_; gc.collect(); torch.cuda.empty_cache();\n    return text_embs\n\n##################################################\ndeberta_base = Config(\n    model='../input/huggingface-deberta-variants/deberta-base/deberta-base',\n    file_name='microsoft_deberta_base_768',\n    pretrained=True, inference_weight=1, max_len=640) #\ndeberta_large = Config(\n    model='../input/huggingface-deberta-variants/deberta-large/deberta-large', \n    file_name='microsoft_deberta_large_1024',\n    pretrained=True, inference_weight=1, max_len=640) #\ndeberta_xlarge = Config(\n    model='../input/huggingface-deberta-variants/deberta-xlarge/deberta-xlarge', \n    file_name='microsoft_deberta_xlarge_1024',\n    pretrained=True, inference_weight=1, max_len=640)\ndeberta_v2_xlarge = Config(\n    model='../input/bert-shopping-mall/deberta-v2-xlarge', \n    file_name='microsoft_deberta_v2_xlarge_1536',\n    pretrained=True, inference_weight=1, max_len=640)\ndeberta_v2_xxlarge = Config(\n    model='../input/bert-shopping-mall/deberta-v2-xxlarge', \n    file_name='microsoft_deberta_v2_xxlarge_1536',\n    pretrained=True, inference_weight=1, max_len=640)\n\ndeberta_v3_base = Config(\n    model='../input/bert-shopping-mall/deberta-v3-base',\n    file_name='microsoft_deberta_v3_base_768',\n    pretrained=True, inference_weight=1, max_len=640) #\ndeberta_v3_large = Config(\n    model='../input/bert-shopping-mall/deberta-v3-large', \n    file_name='microsoft_deberta_v3_large_1024',\n    pretrained=True, inference_weight=1, max_len=640) # \n\ndeberta_large_mnli = Config(\n    model='../input/huggingface-deberta-variants/deberta-large-mnli/deberta-large-mnli',\n    file_name='microsoft_deberta_large_mnli_1024',\n    pretrained=True, inference_weight=1, max_len=640) # \n\ngpt2 = Config(\n    model='../input/hugging-face-gpt2/gpt2',\n    file_name='gpt2_768',\n    pretrained=True, inference_weight=1, max_len=512) #\n\nroberta_base = Config(\n    model='../input/transformers/roberta-base', \n    file_name='roberta_base_768',\n    pretrained=True, inference_weight=1, max_len=512) #\nroberta_large = Config(\n    model='../input/transformers/roberta-large',\n    file_name='roberta_large_1024',\n    pretrained=True, inference_weight=1, max_len=512) # \n\nxlnet_base = Config(\n    model='../input/transformers/xlnet-base-cased',\n    file_name='xlnet_base_cased_768',\n    pretrained=True, inference_weight=1, max_len=640) #\nxlnet_large = Config(\n    model='../input/transformers/xlnet-large-cased', \n    file_name='xlnet_large_cased_1024',\n    pretrained=True, inference_weight=1, max_len=640) #\n\nbart_base = Config(\n    model='../input/transformers/facebook-bart-base',\n    file_name='facebook_bart_base_768',\n    pretrained=True, inference_weight=1, max_len=640)\nbart_large = Config(\n    model='../input/transformers/facebook-bart-large',\n    file_name='facebook_bart_large_1024',\n    pretrained=True, inference_weight=1, max_len=640)\nbart_lage_mnli = Config(\n    model='../input/facebook-bart-large-mnli',\n    file_name='facebook_bart_large_mnli_1024',\n    pretrained=True, inference_weight=1, max_len=640)\n\nbert_base_uncased = Config(\n    model='../input/transformers/bert-base-uncased/',\n    file_name='bert_base_uncased_768',\n    pretrained=True, inference_weight=1, max_len=512)\nbert_large_uncased = Config(\n    model='../input/transformers/bert-large-uncased',\n    file_name='bert_large_uncased_1024',\n    pretrained=True, inference_weight=1, max_len=512)\n\nmuppet_roberta_large = Config(\n    model='../input/muppet-roberta-large',\n    file_name='facebook_muppet_roberta_large_1024',\n    pretrained=True, inference_weight=1, max_len=512)\n# muppet_roberta_base = Config(model='facebook/muppet-roberta-base', pretrained=True, inference_weight=1, max_len=512)\n\nfunnel_small = Config(\n    model='../input/transformers/funnel-transformer-small',\n    file_name='funnel_transformer_small_768',\n    pretrained=True, inference_weight=1, max_len=640)\nfunnel_large = Config(\n    model='../input/transformers/funnel-transformer-large',\n    file_name='funnel_transformer_large_1024',\n    pretrained=True, inference_weight=1, max_len=640)\n","metadata":{"execution":{"iopub.status.busy":"2022-11-29T03:46:26.707209Z","iopub.execute_input":"2022-11-29T03:46:26.707943Z","iopub.status.idle":"2022-11-29T03:46:26.736262Z","shell.execute_reply.started":"2022-11-29T03:46:26.707883Z","shell.execute_reply":"2022-11-29T03:46:26.734181Z"},"trusted":true},"execution_count":11,"outputs":[]},{"cell_type":"code","source":"def build_preds(fine_tuned_models_cfg):\n    pretrained_oof_results = {\n        0: [],\n        1: [],\n        2: [],\n        3: []}\n\n    total_weight = 0\n\n    for cfg in tqdm(fine_tuned_models_cfg):\n        # infer_ = Inferencer(setup['path'], setup['inference_weight'])\n        infer_ = Inferencer(cfg=cfg, inference_weight=cfg.inference_weight)\n        oof = test.copy()\n        for fold in range(CFG.n_folds):\n            if 'oof_results' in cfg and fold in cfg['oof_results']:\n                predictions = cfg['oof_results'][fold]\n            else:\n                try:\n                    predictions = torch.load(\n                        os.path.join(cfg.path, f\"{cfg.model.replace('/', '-')}_fold{fold}_best.pth\"), \n                        map_location = torch.device('cpu'))['predictions']\n                except KeyError:\n                    predictions = torch.load(\n                        os.path.join(cfg.path, f\"{cfg.model.replace('/', '-')}_fold{fold}_best.pth\"), \n                        map_location = torch.device('cpu'))['valid_preds']\n                if 'oof_results' not in cfg:\n                    cfg['oof_results'] = {}\n                cfg['oof_results'][fold] = predictions\n            oof.loc[oof['fold'] == fold, [f'pred_{target_col}' for target_col in target_cols]] = predictions\n            predictions = predictions * cfg.inference_weight\n            pretrained_oof_results[fold].append(predictions)\n\n            total_weight += cfg.inference_weight\n            cfg['oof'] = oof.copy()\n    oof_preds = [cfg['oof'][[f'pred_{target_col}' for target_col in target_cols]].values for cfg in fine_tuned_models_cfg]\n    return oof_preds","metadata":{"execution":{"iopub.status.busy":"2022-11-29T04:47:41.581419Z","iopub.execute_input":"2022-11-29T04:47:41.582053Z","iopub.status.idle":"2022-11-29T04:47:41.602192Z","shell.execute_reply.started":"2022-11-29T04:47:41.581998Z","shell.execute_reply":"2022-11-29T04:47:41.600564Z"},"trusted":true},"execution_count":43,"outputs":[]},{"cell_type":"code","source":"from scipy import optimize\ndef find_optimal_weights(oof_preds, labels, bounds=(0.0, 1.0), method='SLSQP'):\n    weights = [1] * len(oof_preds)\n    def loss(weights):\n        return mc_rmse(labels, np.clip(np.average(oof_preds, weights=weights, axis=0), 1, 5))[0]\n\n    opt_weights = optimize.minimize(\n        loss,\n        [1/len(oof_preds)] * len(oof_preds),\n        constraints=({'type': 'eq','fun': lambda w: 1-sum(w)}),\n        method= method, #'Nelder-Mead',\n        bounds=[bounds] * len(oof_preds),\n        options = {'ftol':1e-10},\n    )['x']\n\n    opt_weights = np.array(opt_weights) / sum(opt_weights)\n    #print('\\n')\n    #print(\"score:\", loss(opt_weights))\n    #print(', '.join([str(i) for i in opt_weights]))\n    return opt_weights, loss(opt_weights)","metadata":{"execution":{"iopub.status.busy":"2022-11-29T04:47:43.879702Z","iopub.execute_input":"2022-11-29T04:47:43.880165Z","iopub.status.idle":"2022-11-29T04:47:43.891948Z","shell.execute_reply.started":"2022-11-29T04:47:43.880127Z","shell.execute_reply":"2022-11-29T04:47:43.890362Z"},"trusted":true},"execution_count":44,"outputs":[]},{"cell_type":"code","source":"CFG = Config(\n    n_folds=4,\n    random_state=42\n)","metadata":{"execution":{"iopub.status.busy":"2022-11-29T03:46:26.755218Z","iopub.execute_input":"2022-11-29T03:46:26.755859Z","iopub.status.idle":"2022-11-29T03:46:26.765541Z","shell.execute_reply.started":"2022-11-29T03:46:26.755809Z","shell.execute_reply":"2022-11-29T03:46:26.764029Z"},"trusted":true},"execution_count":13,"outputs":[]},{"cell_type":"markdown","source":"# Dataset","metadata":{}},{"cell_type":"code","source":"test = pd.read_csv(TRAIN_PATH)\nmskfold = MultilabelStratifiedKFold(n_splits = CFG.n_folds, shuffle = True, random_state = CFG.random_state)\nfor n, (train_index, val_index) in enumerate(mskfold.split(test, test[target_cols])):\n    test.loc[val_index, 'fold'] = int(n)","metadata":{"execution":{"iopub.status.busy":"2022-11-29T03:46:26.767540Z","iopub.execute_input":"2022-11-29T03:46:26.768841Z","iopub.status.idle":"2022-11-29T03:46:27.269708Z","shell.execute_reply.started":"2022-11-29T03:46:26.768785Z","shell.execute_reply":"2022-11-29T03:46:27.267967Z"},"trusted":true},"execution_count":14,"outputs":[]},{"cell_type":"markdown","source":"# Fine-tuned models","metadata":{"papermill":{"duration":0.009934,"end_time":"2022-11-18T16:58:00.946968","exception":false,"start_time":"2022-11-18T16:58:00.937034","status":"completed"},"tags":[]}},{"cell_type":"code","source":"v114_CFG = Config(\n    model=\"microsoft/deberta-v3-base\",\n    version='1',\n    num_target = 6,\n    reinit_last_layer=True,\n    reinit_fc=True,\n    weight_decay=0.01,\n    learning_rate=2e-5,\n    layerwise_learning_rate_decay=1.5,\n    use_dropout=False,\n    prob_dropout=[0.06, 0.08, 0.1, 0.12, 0.14],\n    num_dropout=5,\n    pool_head='mean-attention',\n    seed=42,\n    n_fold=4,\n    trn_fold=[0,1,2,3],\n    path='../input/fb3models/v114/',\n    config_path='../input/fb3models/v114/config.pth',\n    tokenizer=AutoTokenizer.from_pretrained('../input/fb3models/v114/tokenizer')\n)\n\nweightedpool_CFG = Config(\n    model='microsoft/deberta-v3-base',\n    name='weightedpool',\n    version='1',\n    num_target=6,\n    reinit_last_layer=True,\n    reinit_fc=True,\n    weight_decay=0.01,\n    learning_rate=1.5e-5,\n    layerwise_learning_rate_decay=1.5,\n    use_dropout=False,\n    prob_dropout=[0.06, 0.08, 0.1, 0.12, 0.14],\n    num_dropout=5,\n    pool_head='weighted',\n    seed=42,\n    n_fold=4,\n    trn_fold=[0,1,2,3],\n    train=True,\n    path='../input/fb3-train/',\n    config_path='../input/fb3-train/config.pth',\n    tokenizer=AutoTokenizer.from_pretrained('../input/fb3-train/tokenizer'),\n    inference_weight=1.0)\n\nv116_CFG = load_config('../input/fb3-colab-models/v116', inference_weight=1.0)\nv116_CFG.path = '../input/fb3models/v116'\nv116_CFG.config_path = '../input/fb3models/v116/config.pth'\nv116_CFG.version = '1'\nv116_CFG.name = 'v116'\n\nv112_CFG = Config(\n    num_workers=1,\n    batch_size=3,\n    max_len=512,\n    model=\"microsoft/deberta-v3-base\",\n    name='v112',\n    version='1',\n    num_target = 6,\n    reinit_last_layer=True,\n    reinit_fc=True,\n    weight_decay=0.01,\n    learning_rate=2e-5,\n    layerwise_learning_rate_decay=1.5,\n    use_dropout=False,\n    prob_dropout=[0.06, 0.08, 0.1, 0.12, 0.14],\n    num_dropout=5,\n    pool_head='attention',\n    seed=42,\n    n_fold=4,\n    trn_fold=[0,1,2,3],\n    train=True,\n    path='../input/fb3models/v112/',\n    config_path='../input/fb3models/v112/config.pth',\n    inference_weight=1.0,\n    tokenizer=AutoTokenizer.from_pretrained('../input/fb3models/v112/tokenizer')\n)\n\n#####\nv2_CFG = load_config('../input/fb3models/v2/', inference_weight=1.0)\nv2_CFG.name = 'v2'\nv2_CFG.version = '2'\nv2_CFG.trn_fold = [0,1,2,3]\n\n#####\nv21_CFG = load_config('../input/fb3models/v21/', inference_weight=1)\nv21_CFG.name = 'v21'\nv21_CFG.version = '21'\n\n#####\nattention_fgm_CFG = load_config('../input/fb3models/20221114-192943-deberta-v3-base/', inference_weight=1.0)\nattention_fgm_CFG.name = 'attention_fgm'\nattention_fgm_CFG.version = 'custom'\nattention_fgm_CFG.config_path = '../input/fb3models/20221114-192943-deberta-v3-base/config/config.json'\n\nweighted_fgm_CFG = Config(\n    pretrained=False,\n    path='../input/fb3models/20221115-061243-deberta-v3-base',\n    config_path='../input/fb3models/20221115-061243-deberta-v3-base/config/config.json',\n    tokenizer=AutoTokenizer.from_pretrained('../input/fb3models/20221115-061243-deberta-v3-base/tokenizer'),\n    name='weighted_fgm',\n    version='custom',\n    train = True,\n    debug = False,\n    offline = False,\n    models_path = 'FB3-models',\n    epochs = 5,\n    save_all_models = False,\n    competition = 'FB3',\n    apex = True,\n    print_freq = 20,\n    num_workers = 4,\n    model = 'microsoft/deberta-v3-base', #If you want to train on the kaggle platform, v3-base is realistic. v3-large will time out.\n    loss_func = 'SmoothL1', # 'SmoothL1', 'RMSE'\n    gradient_checkpointing = True,\n    scheduler = 'cosine',\n    batch_scheduler = True,\n    num_cycles = 0.5,\n    num_warmup_steps = 0,\n    encoder_lr = 2e-5,\n    decoder_lr = 2e-5,\n    min_lr = 1e-6,\n    #Layer-Wise Learning Rate Decay\n    llrd = True,\n    layerwise_lr = 5e-5,\n    layerwise_lr_decay = 0.9,\n    layerwise_weight_decay = 0.01,\n    layerwise_adam_epsilon = 1e-6,\n    layerwise_use_bertadam = False,\n    #pooling\n    pooling = 'weightedlayer', # mean, max, min, attention, weightedlayer\n    layer_start = 11,\n    layers=None,\n    #init_weight\n    init_weight = 'normal', # normal, xavier_uniform, xavier_normal, kaiming_uniform, kaiming_normal, orthogonal\n    #re-init\n    reinit = True,\n    reinit_n = 1,\n    #adversarial\n    fgm = True,\n    awp = False,\n    adv_lr = 1,\n    adv_eps = 0.2,\n    unscale = False,\n    eps = 1e-6,\n    betas = (0.9, 0.999),\n    max_len = 512,\n    weight_decay = 0.01,\n    gradient_accumulation_steps = 1,\n    max_grad_norm = 1000,\n    target_cols = ['cohesion', 'syntax', 'vocabulary', 'phraseology', 'grammar', 'conventions'],\n    seed = 42,\n    cv_seed = 42,\n    n_fold = 4,\n    trn_fold = [0,1,2,3],\n    batch_size = 8,\n    n_targets = 6,\n    gpu_id = 0) \n\nweighted_attention_CFG = load_config('../input/fb3models/weighted_attention_v3', inference_weight=1.0)\nweighted_attention_CFG.name = 'weighted_attention'\nweighted_attention_CFG.version = 'weighted-attention'\n\nmean_attention_no_fgm_CFG = load_config('../input/fb3models/20221117-183420-deberta-v3-base-mean-attention-with-mask', inference_weight=1.0)\nmean_attention_no_fgm_CFG.name = 'mean_attention_no_fgm'\nmean_attention_no_fgm_CFG.version = 'mean-attention'\n\nattention_large_fgm_CFG = load_config('../input/fb3models/20221118-164148-deberta-v3-large-attention_fgm')\nattention_large_fgm_CFG.name = 'attention_large_fgm'\nattention_large_fgm_CFG.version = 'custom'\nattention_large_fgm_CFG.config_path = '../input/fb3models/20221118-164148-deberta-v3-large-attention_fgm/config/config.json'\n\nattention_fgm_512_CFG = load_config('../input/fb3models/20221121-143655-deberta-v3-base-attention_fgm_512')\nattention_fgm_512_CFG.name = 'attention_fgm_512'\nattention_fgm_512_CFG.version = 'custom'\nattention_fgm_512_CFG.config_path = '../input/fb3models/20221121-143655-deberta-v3-base-attention_fgm_512/config/config.json'\n\nattention_fgm_768_CFG = load_config('../input/fb3models/20221120-072218-deberta-v3-base-attention_fgm')\nattention_fgm_768_CFG.name = 'attention_fgm_768'\nattention_fgm_768_CFG.version = 'custom'\nattention_fgm_768_CFG.config_path = '../input/fb3models/20221120-072218-deberta-v3-base-attention_fgm/config/config.json'\n\nweighted2last_fgm_512_CFG = load_config('../input/fb3models/20221124-060246-deberta-v3-base-weighted2last_fgm')\nweighted2last_fgm_512_CFG.name = 'weighted2last_fgm_512'\nweighted2last_fgm_512_CFG.version = 'custom'\nweighted2last_fgm_512_CFG.config_path = '../input/fb3models/20221124-060246-deberta-v3-base-weighted2last_fgm/config/config.json'\n\nweightedmean2last_fgm_512_CFG = load_config('../input/fb3models/20221124-160318-deberta-v3-base-weightedmean2last_fgm')\nweightedmean2last_fgm_512_CFG.name = 'weightedmean2last_fgm_512'\nweightedmean2last_fgm_512_CFG.version = 'custom'\nweightedmean2last_fgm_512_CFG.config_path = '../input/fb3models/20221124-160318-deberta-v3-base-weightedmean2last_fgm/config/config.json'\n\nroberta_attention_fgm_CFG = load_config('../input/fb3models/20221121-173739-roberta-base')\nroberta_attention_fgm_CFG.name = 'roberta_attention_large_fgm'\nroberta_attention_fgm_CFG.version = 'custom'\nroberta_attention_fgm_CFG.config_path = '../input/fb3models/20221121-173739-roberta-base/config/config.json'\n\n##########\nv112_CFG.pretrained = False\nv114_CFG.pretrained = False\nv116_CFG.pretrained = False\nv21_CFG.pretrained = False\nv2_CFG.pretrained = False\nattention_fgm_CFG.pretrained = False\nweighted_attention_CFG.pretrained = False\nweightedpool_CFG.pretrained = False\nmean_attention_no_fgm_CFG.pretrained=False\nattention_large_fgm_CFG.pretrained=False\nattention_fgm_512_CFG.pretrained = False\nattention_fgm_768_CFG.pretrained = False\nweighted2last_fgm_512_CFG.pretrained = False\nweightedmean2last_fgm_512_CFG.pretrained = False\nroberta_attention_fgm_CFG.pretrained = False\n\nweighted_fgm_CFG.inference_weight = 1.0\nv114_CFG.inference_weight = 1.0 \nv116_CFG.inference_weight = 1.0\nv2_CFG.inference_weight = 1.0 \nv21_CFG.inference_weight = 1.0\nattention_fgm_CFG.inference_weight = 1.0\nweighted_attention_CFG.inference_weight = 1.0\nattention_large_fgm_CFG.inference_weight = 1.0\nattention_fgm_512_CFG.inference_weight = 1.0\nattention_fgm_768_CFG.inference_weight = 1.0\nweighted2last_fgm_512_CFG.inference_weight = 1.0\nweightedmean2last_fgm_512_CFG.inference_weight = 1.0\nroberta_attention_fgm_CFG.inference_weight = 1.0","metadata":{"papermill":{"duration":2.771044,"end_time":"2022-11-18T16:58:03.727915","exception":false,"start_time":"2022-11-18T16:58:00.956871","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2022-11-29T03:46:27.272564Z","iopub.execute_input":"2022-11-29T03:46:27.273191Z","iopub.status.idle":"2022-11-29T03:46:35.418111Z","shell.execute_reply.started":"2022-11-29T03:46:27.273133Z","shell.execute_reply":"2022-11-29T03:46:35.416824Z"},"trusted":true},"execution_count":15,"outputs":[]},{"cell_type":"code","source":"def get_ensemble_score(fine_tuned_models_cfg):\n    pretrained_oof_results = {\n        0: [],\n        1: [],\n        2: [],\n        3: []}\n\n    total_weight = 0\n    \n    for cfg in tqdm(fine_tuned_models_cfg):\n        # infer_ = Inferencer(setup['path'], setup['inference_weight'])\n        infer_ = Inferencer(cfg=cfg, inference_weight=cfg.inference_weight)\n        oof = test.copy()\n        for fold in range(CFG.n_folds):\n            if 'oof_results' in cfg and fold in cfg['oof_results']:\n                predictions = cfg['oof_results'][fold]\n            else:\n                try:\n                    predictions = torch.load(\n                        os.path.join(cfg.path, f\"{cfg.model.replace('/', '-')}_fold{fold}_best.pth\"), \n                        map_location = torch.device('cpu'))['predictions']\n                except KeyError:\n                    predictions = torch.load(\n                        os.path.join(cfg.path, f\"{cfg.model.replace('/', '-')}_fold{fold}_best.pth\"), \n                        map_location = torch.device('cpu'))['valid_preds']\n                if 'oof_results' not in cfg:\n                    cfg['oof_results'] = {}\n                cfg['oof_results'][fold] = predictions\n            oof.loc[oof['fold'] == fold, [f'pred_{target_col}' for target_col in target_cols]] = predictions\n            predictions = predictions * cfg.inference_weight\n            pretrained_oof_results[fold].append(predictions)\n            \n        total_weight += cfg.inference_weight\n\n        cfg['oof'] = oof.copy()\n    \n    weighted_mean_oof = test.copy()\n    for fold in range(CFG.n_folds):\n        weighted_mean_oof.loc[\n            weighted_mean_oof['fold']==fold, \n            [f\"pred_{c}\" for c in target_cols]\n        ] = np.array(pretrained_oof_results[fold]).sum(axis=0)/total_weight\n        get_result(CFG, weighted_mean_oof[weighted_mean_oof['fold']==fold])\n    print('\\n')\n    get_result(CFG, weighted_mean_oof)\n    oof_preds = [cfg['oof'][[f'pred_{target_col}' for target_col in target_cols]].values for cfg in fine_tuned_models_cfg]\n    \n    return oof_preds, weighted_mean_oof","metadata":{"execution":{"iopub.status.busy":"2022-11-29T03:46:35.421466Z","iopub.execute_input":"2022-11-29T03:46:35.422411Z","iopub.status.idle":"2022-11-29T03:46:35.440497Z","shell.execute_reply.started":"2022-11-29T03:46:35.422356Z","shell.execute_reply":"2022-11-29T03:46:35.437948Z"},"trusted":true},"execution_count":16,"outputs":[]},{"cell_type":"markdown","source":"## all models","metadata":{}},{"cell_type":"code","source":"fine_tuned_models_cfg = [\n    weighted2last_fgm_512_CFG, weightedmean2last_fgm_512_CFG, attention_fgm_512_CFG, attention_fgm_768_CFG, attention_fgm_CFG,\n    \n    v21_CFG, v112_CFG, mean_attention_no_fgm_CFG, weighted_attention_CFG, attention_large_fgm_CFG, roberta_attention_fgm_CFG,\n    \n    #v116_CFG, v2_CFG, weightedpool_CFG, weighted_fgm_CFG\n]","metadata":{"execution":{"iopub.status.busy":"2022-11-29T03:46:35.442908Z","iopub.execute_input":"2022-11-29T03:46:35.443549Z","iopub.status.idle":"2022-11-29T03:46:35.455576Z","shell.execute_reply.started":"2022-11-29T03:46:35.443497Z","shell.execute_reply":"2022-11-29T03:46:35.454148Z"},"trusted":true},"execution_count":17,"outputs":[]},{"cell_type":"code","source":"oof_preds, weighted_mean_oof = get_ensemble_score(fine_tuned_models_cfg)","metadata":{"execution":{"iopub.status.busy":"2022-11-29T03:46:35.457932Z","iopub.execute_input":"2022-11-29T03:46:35.458644Z","iopub.status.idle":"2022-11-29T03:52:04.251014Z","shell.execute_reply.started":"2022-11-29T03:46:35.458592Z","shell.execute_reply":"2022-11-29T03:52:04.244780Z"},"trusted":true},"execution_count":18,"outputs":[{"output_type":"display_data","data":{"text/plain":"  0%|          | 0/11 [00:00<?, ?it/s]","application/vnd.jupyter.widget-view+json":{"version_major":2,"version_minor":0,"model_id":"fdb1a19867404378a0a621153a08a0be"}},"metadata":{}},{"name":"stdout","text":"score: 0.442586  scores: [0.4720707712448348, 0.4355797749846355, 0.40576698432082664, 0.44899382866400633, 0.4593160512276641, 0.43378811829965147]\nscore: 0.450717  scores: [0.4841311235160003, 0.44270950380457397, 0.4140075444874199, 0.4489088277434105, 0.4679909010945405, 0.44655664110850923]\nscore: 0.454329  scores: [0.47638724159007645, 0.4469377533509186, 0.41335473912549847, 0.4621074164389505, 0.4770209752763032, 0.4501638648142493]\nscore: 0.439753  scores: [0.47382198477708376, 0.43959541318438017, 0.4047618580088698, 0.4320182282758933, 0.45902760084537103, 0.4292924064407926]\n\n\nscore: 0.446901  scores: [0.4766231371417863, 0.4412248819958981, 0.4094934656205698, 0.4481341588050277, 0.46589700260883876, 0.44003367127242504]\n","output_type":"stream"}]},{"cell_type":"code","source":"find_optimal_weights(oof_preds, test[target_cols].values, bounds=(-1.0, 1.0))","metadata":{"execution":{"iopub.status.busy":"2022-11-29T03:52:04.260498Z","iopub.execute_input":"2022-11-29T03:52:04.263693Z","iopub.status.idle":"2022-11-29T03:52:05.777362Z","shell.execute_reply.started":"2022-11-29T03:52:04.263608Z","shell.execute_reply":"2022-11-29T03:52:05.775571Z"},"trusted":true},"execution_count":19,"outputs":[{"execution_count":19,"output_type":"execute_result","data":{"text/plain":"(array([ 0.03053147,  0.11300867,  0.13574549,  0.15840433, -0.23651732,\n         0.09266294,  0.16754533,  0.16681941, -0.00694779,  0.32735722,\n         0.05139025]),\n 0.44550548104196364)"},"metadata":{}}]},{"cell_type":"code","source":"find_optimal_weights(oof_preds, test[target_cols].values, bounds=(0.01, 1.0))","metadata":{"execution":{"iopub.status.busy":"2022-11-29T03:52:05.779900Z","iopub.execute_input":"2022-11-29T03:52:05.781669Z","iopub.status.idle":"2022-11-29T03:52:07.596369Z","shell.execute_reply.started":"2022-11-29T03:52:05.781590Z","shell.execute_reply":"2022-11-29T03:52:07.594717Z"},"trusted":true},"execution_count":20,"outputs":[{"execution_count":20,"output_type":"execute_result","data":{"text/plain":"(array([0.0409746 , 0.11473658, 0.02306713, 0.06851962, 0.01      ,\n        0.06476458, 0.15137027, 0.15345884, 0.01      , 0.30815366,\n        0.05495472]),\n 0.4457266903344244)"},"metadata":{}}]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## stacking (optinal)","metadata":{"papermill":{"duration":0.010883,"end_time":"2022-11-18T17:00:59.736287","exception":false,"start_time":"2022-11-18T17:00:59.725404","status":"completed"},"tags":[]}},{"cell_type":"code","source":"from glob import glob\nfrom sklearn.linear_model import Ridge, Lasso, BayesianRidge\nfrom joblib import dump, load\nif str(device) == 'cpu':\n    from sklearn.svm import SVR\nelse:\n    from cuml.svm import SVR","metadata":{"papermill":{"duration":0.019894,"end_time":"2022-11-18T17:00:59.767232","exception":false,"start_time":"2022-11-18T17:00:59.747338","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2022-11-29T03:52:07.598759Z","iopub.execute_input":"2022-11-29T03:52:07.599366Z","iopub.status.idle":"2022-11-29T03:52:11.839883Z","shell.execute_reply.started":"2022-11-29T03:52:07.599311Z","shell.execute_reply":"2022-11-29T03:52:11.838250Z"},"trusted":true},"execution_count":21,"outputs":[]},{"cell_type":"markdown","source":"### strong models","metadata":{}},{"cell_type":"code","source":"strong_models = [\n    weighted2last_fgm_512_CFG, weightedmean2last_fgm_512_CFG, attention_fgm_512_CFG, attention_fgm_768_CFG, attention_fgm_CFG\n]\noof_preds, weighted_mean_oof = get_ensemble_score(strong_models)\n\noptimal_weights, optimal_score = find_optimal_weights(oof_preds, test[target_cols].values, bounds=(-1.0, 1.0))\nprint('\\n')\nprint('Optimal score:', optimal_score)\nprint('Optimal weights:', optimal_weights)","metadata":{"execution":{"iopub.status.busy":"2022-11-29T03:52:11.842001Z","iopub.execute_input":"2022-11-29T03:52:11.842596Z","iopub.status.idle":"2022-11-29T03:52:12.311132Z","shell.execute_reply.started":"2022-11-29T03:52:11.842541Z","shell.execute_reply":"2022-11-29T03:52:12.309464Z"},"trusted":true},"execution_count":22,"outputs":[{"output_type":"display_data","data":{"text/plain":"  0%|          | 0/5 [00:00<?, ?it/s]","application/vnd.jupyter.widget-view+json":{"version_major":2,"version_minor":0,"model_id":"5912e92d3ceb4447b1c66df66cb4c474"}},"metadata":{}},{"name":"stdout","text":"score: 0.446933  scores: [0.4753812173163872, 0.43871284543114714, 0.40946724912529026, 0.45404290019775057, 0.4645873941594094, 0.43940815673455463]\nscore: 0.453408  scores: [0.48725456086384644, 0.44524752907446424, 0.4149177826972703, 0.4517977913330938, 0.4700491976532688, 0.45117936972871886]\nscore: 0.456213  scores: [0.4781926486156212, 0.4493594682500296, 0.4162138916638546, 0.4625873626903548, 0.47868859488293003, 0.4522335181234556]\nscore: 0.442238  scores: [0.47657228960055403, 0.44098628673290685, 0.4086124353200152, 0.4336169815683645, 0.46216288146169143, 0.4314777676885187]\n\n\nscore: 0.449748  scores: [0.4793709089272753, 0.44359487646221163, 0.41231544939967507, 0.45063451956939254, 0.46891467924967745, 0.4436563105215204]\n\n\nOptimal score: 0.44956837283536816\nOptimal weights: [0.23303419 0.28660832 0.12286597 0.26682334 0.09066818]\n","output_type":"stream"}]},{"cell_type":"markdown","source":"### weak models","metadata":{"papermill":{"duration":0.011038,"end_time":"2022-11-18T17:00:59.872425","exception":false,"start_time":"2022-11-18T17:00:59.861387","status":"completed"},"tags":[]}},{"cell_type":"code","source":"weak_models = [\n    v21_CFG, v112_CFG, mean_attention_no_fgm_CFG, weighted_attention_CFG, attention_large_fgm_CFG, roberta_attention_fgm_CFG\n]\noof_preds, weighted_mean_oof = get_ensemble_score(weak_models)\n\noptimal_weights, optimal_score = find_optimal_weights(oof_preds, test[target_cols].values, bounds=(-1.0, 1.0))\nprint('\\n')\nprint('Optimal score:', optimal_score)\nprint('Optimal weights:', optimal_weights)","metadata":{"execution":{"iopub.status.busy":"2022-11-29T03:52:12.313434Z","iopub.execute_input":"2022-11-29T03:52:12.314348Z","iopub.status.idle":"2022-11-29T03:52:12.943100Z","shell.execute_reply.started":"2022-11-29T03:52:12.314286Z","shell.execute_reply":"2022-11-29T03:52:12.941415Z"},"trusted":true},"execution_count":23,"outputs":[{"output_type":"display_data","data":{"text/plain":"  0%|          | 0/6 [00:00<?, ?it/s]","application/vnd.jupyter.widget-view+json":{"version_major":2,"version_minor":0,"model_id":"ccdec289016c4bbdaa299751236ba468"}},"metadata":{}},{"name":"stdout","text":"score: 0.441358  scores: [0.4723432817516506, 0.43475533526581256, 0.40438267956673907, 0.4466873529750857, 0.4579991563759526, 0.4319815080751596]\nscore: 0.450743  scores: [0.4847768527770989, 0.44236350308832534, 0.41502939442167147, 0.44874839009192147, 0.46852998482643593, 0.4450126024812419]\nscore: 0.454960  scores: [0.4778479759679727, 0.4467714726012902, 0.4129731049101703, 0.46378947095062967, 0.47811288763594223, 0.4502621289769793]\nscore: 0.440245  scores: [0.47500970833239853, 0.44059740556898846, 0.40356991996169567, 0.433061497825408, 0.45910059517169005, 0.43013179660209655]\n\n\nscore: 0.446886  scores: [0.4775150622743201, 0.4411426611784146, 0.40901868872780806, 0.4482039557681362, 0.466006022477172, 0.4394281900204986]\n\n\nOptimal score: 0.44609322341329655\nOptimal weights: [0.13595877 0.20074556 0.19400446 0.0293235  0.35083864 0.08912907]\n","output_type":"stream"}]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## ew models","metadata":{}},{"cell_type":"code","source":"eww_models = [v116_CFG, v2_CFG, weightedpool_CFG, weighted_fgm_CFG]\n\n_, eww_oof = get_ensemble_score(eww_models)\n\noptimal_weights, optimal_score = find_optimal_weights(oof_preds, test[target_cols].values, bounds=(-1.0, 1.0))\nprint('\\n')\nprint('Optimal score:', optimal_score)\nprint('Optimal weights:', optimal_weights)","metadata":{"execution":{"iopub.status.busy":"2022-11-29T03:52:12.945313Z","iopub.execute_input":"2022-11-29T03:52:12.946174Z","iopub.status.idle":"2022-11-29T03:54:22.861425Z","shell.execute_reply.started":"2022-11-29T03:52:12.946115Z","shell.execute_reply":"2022-11-29T03:54:22.859849Z"},"trusted":true},"execution_count":24,"outputs":[{"output_type":"display_data","data":{"text/plain":"  0%|          | 0/4 [00:00<?, ?it/s]","application/vnd.jupyter.widget-view+json":{"version_major":2,"version_minor":0,"model_id":"221b1a4c891c4ab78c7633806a5592a0"}},"metadata":{}},{"name":"stdout","text":"score: 0.446501  scores: [0.47471472794293434, 0.44000923123030955, 0.40905796462951055, 0.4526002092572088, 0.46565641016142073, 0.43697033835690124]\nscore: 0.453990  scores: [0.4852156761732504, 0.4456777427146603, 0.4154352683337913, 0.45350188889397264, 0.4736715588387051, 0.4504360085755037]\nscore: 0.457225  scores: [0.48330686353579266, 0.4489836500837208, 0.4168993660186848, 0.46182656052028287, 0.47762095142810335, 0.45471306383244536]\nscore: 0.444855  scores: [0.4777196906444851, 0.4397781698779775, 0.41114604184823744, 0.43849825411669596, 0.4654621202481103, 0.4365287082374503]\n\n\nscore: 0.450680  scores: [0.48025645669970124, 0.4436288052924128, 0.41314618791695784, 0.45168399378503193, 0.4706310818703639, 0.4447335546336968]\n\n\nOptimal score: 0.44609322341329655\nOptimal weights: [0.13595877 0.20074556 0.19400446 0.0293235  0.35083864 0.08912907]\n","output_type":"stream"}]},{"cell_type":"code","source":"eww_oof_preds = eww_oof[pred_cols].values","metadata":{"execution":{"iopub.status.busy":"2022-11-29T03:54:22.864857Z","iopub.execute_input":"2022-11-29T03:54:22.865984Z","iopub.status.idle":"2022-11-29T03:54:22.874108Z","shell.execute_reply.started":"2022-11-29T03:54:22.865926Z","shell.execute_reply":"2022-11-29T03:54:22.872348Z"},"trusted":true},"execution_count":25,"outputs":[]},{"cell_type":"markdown","source":"## forward selection","metadata":{}},{"cell_type":"code","source":" # START ENSEMBLE USING MODEL WITH LARGEST CV\n#   Repeat until CV does not increase by 0.0003+ :\n#     # TRY ADDING EVERY MODEL ONE AT A TIME AND REMEMBER \n#     # HOW MUCH EACH INCREASES THE ENSEMBLE CV SCORE\n#     for k in range( len(models) ):\n#         for w in [0.01, 0.02, ..., 0.98, 0.99]:\n#             # TRY ADDING MODEL k WITH WEIGHT w TO ENSEMBLE\n#             trial = w * model[k,] + (1-w) * ensemble\n#             auc_trial = roc_auc_score(true, trial)\n#     # ADD ONE NEW MODEL TO ENSEMBLE THAT INCREASED CV THE MOST\n#     # CHECK NEW CV SCORE. IF IT INCREASED REPEAT LOOP","metadata":{"execution":{"iopub.status.busy":"2022-11-29T03:54:22.876535Z","iopub.execute_input":"2022-11-29T03:54:22.877407Z","iopub.status.idle":"2022-11-29T03:54:22.890333Z","shell.execute_reply.started":"2022-11-29T03:54:22.877346Z","shell.execute_reply":"2022-11-29T03:54:22.888592Z"},"trusted":true},"execution_count":26,"outputs":[]},{"cell_type":"code","source":"labels = test[target_cols].values","metadata":{"execution":{"iopub.status.busy":"2022-11-29T03:54:22.893984Z","iopub.execute_input":"2022-11-29T03:54:22.895399Z","iopub.status.idle":"2022-11-29T03:54:22.905311Z","shell.execute_reply.started":"2022-11-29T03:54:22.895336Z","shell.execute_reply":"2022-11-29T03:54:22.903806Z"},"trusted":true},"execution_count":27,"outputs":[]},{"cell_type":"code","source":"def get_optimal_score(features, labels):\n    return find_optimal_weights(build_preds(features), labels)[1], features\n\ndef mc_rmse_(y_true, y_pred):\n    scores = []\n    ncols = y_true.shape[1]\n    \n    for n in range(ncols):\n        yn_true = y_true[:, n]\n        yn_pred = y_pred[:, n]\n        rmse_ = mean_squared_error(yn_true, yn_pred, squared=False)\n        scores.append(rmse_)\n    score = np.mean(scores) \n    return score\n\ndef median_avg(predictions,beta=0.5):\n    '''\n    predictions should be of a vector shape n_models\n    beta: if beta is 0.5, then the middle 50% will be averaged\n    '''\n    sorted_predictions=np.sort(predictions)\n    n_model=len(sorted_predictions)\n    mid_point=n_model//2+1\n    n_avg=int(n_model*beta)\n    debug = True\n    if debug:\n        print('sorted_list',sorted_predictions.shape)\n        print('after_cut',sorted_predictions[mid_point-n_avg//2-1:mid_point+n_avg//2].shape)\n    to_avg=sorted_predictions[mid_point-n_avg//2-1:mid_point+n_avg//2]\n    return np.mean(to_avg)","metadata":{"execution":{"iopub.status.busy":"2022-11-29T03:54:22.916211Z","iopub.execute_input":"2022-11-29T03:54:22.917073Z","iopub.status.idle":"2022-11-29T03:54:22.929288Z","shell.execute_reply.started":"2022-11-29T03:54:22.917029Z","shell.execute_reply":"2022-11-29T03:54:22.927591Z"},"trusted":true},"execution_count":28,"outputs":[]},{"cell_type":"code","source":"# all_models = [\n#     v21_CFG, v112_CFG, mean_attention_no_fgm_CFG, weighted_attention_CFG,\n#     weighted2last_fgm_512_CFG, weightedmean2last_fgm_512_CFG, attention_fgm_512_CFG, attention_fgm_768_CFG, roberta_attention_fgm_CFG\n# ]\nall_models = [\n    weighted2last_fgm_512_CFG, weightedmean2last_fgm_512_CFG, attention_fgm_512_CFG, attention_fgm_768_CFG, attention_fgm_CFG,\n    \n    v21_CFG, v112_CFG, mean_attention_no_fgm_CFG, weighted_attention_CFG, roberta_attention_fgm_CFG,\n    \n    v2_CFG, weightedpool_CFG,\n    #v116_CFG, v2_CFG, weightedpool_CFG, weighted_fgm_CFG\n]\nPATIENCE = 10\nTOL = -0.003","metadata":{"execution":{"iopub.status.busy":"2022-11-29T03:54:22.932647Z","iopub.execute_input":"2022-11-29T03:54:22.933335Z","iopub.status.idle":"2022-11-29T03:54:22.948145Z","shell.execute_reply.started":"2022-11-29T03:54:22.933273Z","shell.execute_reply":"2022-11-29T03:54:22.946404Z"},"trusted":true},"execution_count":29,"outputs":[]},{"cell_type":"code","source":"weights_range = np.arange(0.01, 1.01, 0.01)\nscores = [mc_rmse_(labels, w * build_preds([attention_large_fgm_CFG])[0]) for w in weights_range]","metadata":{"execution":{"iopub.status.busy":"2022-11-29T04:47:57.103107Z","iopub.execute_input":"2022-11-29T04:47:57.104532Z","iopub.status.idle":"2022-11-29T04:48:06.864302Z","shell.execute_reply.started":"2022-11-29T04:47:57.104470Z","shell.execute_reply":"2022-11-29T04:48:06.862789Z"},"scrolled":true,"trusted":true},"execution_count":45,"outputs":[{"output_type":"display_data","data":{"text/plain":"  0%|          | 0/1 [00:00<?, ?it/s]","application/vnd.jupyter.widget-view+json":{"version_major":2,"version_minor":0,"model_id":"86315650445f4b999a04d46934ab1af6"}},"metadata":{}},{"output_type":"display_data","data":{"text/plain":"  0%|          | 0/1 [00:00<?, ?it/s]","application/vnd.jupyter.widget-view+json":{"version_major":2,"version_minor":0,"model_id":"77d7e416a5dd4464810820a3c4835dce"}},"metadata":{}},{"output_type":"display_data","data":{"text/plain":"  0%|          | 0/1 [00:00<?, ?it/s]","application/vnd.jupyter.widget-view+json":{"version_major":2,"version_minor":0,"model_id":"86989c5f336c40de94508cf12439a9f8"}},"metadata":{}},{"output_type":"display_data","data":{"text/plain":"  0%|          | 0/1 [00:00<?, ?it/s]","application/vnd.jupyter.widget-view+json":{"version_major":2,"version_minor":0,"model_id":"40e44913f10d499a9e2ccf6d4cd87f29"}},"metadata":{}},{"output_type":"display_data","data":{"text/plain":"  0%|          | 0/1 [00:00<?, ?it/s]","application/vnd.jupyter.widget-view+json":{"version_major":2,"version_minor":0,"model_id":"8a6857259a9f49108ca6ad772cd25b0a"}},"metadata":{}},{"output_type":"display_data","data":{"text/plain":"  0%|          | 0/1 [00:00<?, ?it/s]","application/vnd.jupyter.widget-view+json":{"version_major":2,"version_minor":0,"model_id":"ca6c1ad04ab64a31803dc7efdb59e4b7"}},"metadata":{}},{"output_type":"display_data","data":{"text/plain":"  0%|          | 0/1 [00:00<?, ?it/s]","application/vnd.jupyter.widget-view+json":{"version_major":2,"version_minor":0,"model_id":"4d6d5e77cca04860aa992fc8471045fb"}},"metadata":{}},{"output_type":"display_data","data":{"text/plain":"  0%|          | 0/1 [00:00<?, ?it/s]","application/vnd.jupyter.widget-view+json":{"version_major":2,"version_minor":0,"model_id":"fc45df6af8c2457ea89158b9edfc462c"}},"metadata":{}},{"output_type":"display_data","data":{"text/plain":"  0%|          | 0/1 [00:00<?, ?it/s]","application/vnd.jupyter.widget-view+json":{"version_major":2,"version_minor":0,"model_id":"a8fe54b0e37c4a899a75e0dcfb2c4de3"}},"metadata":{}},{"output_type":"display_data","data":{"text/plain":"  0%|          | 0/1 [00:00<?, ?it/s]","application/vnd.jupyter.widget-view+json":{"version_major":2,"version_minor":0,"model_id":"f56d822106c042329f5e2f264b4af257"}},"metadata":{}},{"output_type":"display_data","data":{"text/plain":"  0%|          | 0/1 [00:00<?, ?it/s]","application/vnd.jupyter.widget-view+json":{"version_major":2,"version_minor":0,"model_id":"1b68e2e468de4eeaa5371bd5694688d5"}},"metadata":{}},{"output_type":"display_data","data":{"text/plain":"  0%|          | 0/1 [00:00<?, ?it/s]","application/vnd.jupyter.widget-view+json":{"version_major":2,"version_minor":0,"model_id":"354c9673818d48a5aaa271dc012e978e"}},"metadata":{}},{"output_type":"display_data","data":{"text/plain":"  0%|          | 0/1 [00:00<?, ?it/s]","application/vnd.jupyter.widget-view+json":{"version_major":2,"version_minor":0,"model_id":"a883892ecd274c84995e62e6b1f7eeda"}},"metadata":{}},{"output_type":"display_data","data":{"text/plain":"  0%|          | 0/1 [00:00<?, ?it/s]","application/vnd.jupyter.widget-view+json":{"version_major":2,"version_minor":0,"model_id":"63bc1aefdc0046409929a07a8306bce3"}},"metadata":{}},{"output_type":"display_data","data":{"text/plain":"  0%|          | 0/1 [00:00<?, ?it/s]","application/vnd.jupyter.widget-view+json":{"version_major":2,"version_minor":0,"model_id":"d9577f0742584066b2a7da56b51f006c"}},"metadata":{}},{"output_type":"display_data","data":{"text/plain":"  0%|          | 0/1 [00:00<?, ?it/s]","application/vnd.jupyter.widget-view+json":{"version_major":2,"version_minor":0,"model_id":"a867d66358a84693a79ba1f2bc1f6923"}},"metadata":{}},{"output_type":"display_data","data":{"text/plain":"  0%|          | 0/1 [00:00<?, ?it/s]","application/vnd.jupyter.widget-view+json":{"version_major":2,"version_minor":0,"model_id":"4ae76c0c1e1545fb8442d518b8faa43e"}},"metadata":{}},{"output_type":"display_data","data":{"text/plain":"  0%|          | 0/1 [00:00<?, ?it/s]","application/vnd.jupyter.widget-view+json":{"version_major":2,"version_minor":0,"model_id":"1b9c7a8b7a35418987d825e09da94e87"}},"metadata":{}},{"output_type":"display_data","data":{"text/plain":"  0%|          | 0/1 [00:00<?, ?it/s]","application/vnd.jupyter.widget-view+json":{"version_major":2,"version_minor":0,"model_id":"9065054459ec4be9b00c3c162b4f3fb8"}},"metadata":{}},{"output_type":"display_data","data":{"text/plain":"  0%|          | 0/1 [00:00<?, ?it/s]","application/vnd.jupyter.widget-view+json":{"version_major":2,"version_minor":0,"model_id":"535d5d40cf8e4083b2399959cc7608da"}},"metadata":{}},{"output_type":"display_data","data":{"text/plain":"  0%|          | 0/1 [00:00<?, ?it/s]","application/vnd.jupyter.widget-view+json":{"version_major":2,"version_minor":0,"model_id":"354b77b1353b42e3b52fba6d8422fab4"}},"metadata":{}},{"output_type":"display_data","data":{"text/plain":"  0%|          | 0/1 [00:00<?, ?it/s]","application/vnd.jupyter.widget-view+json":{"version_major":2,"version_minor":0,"model_id":"d99baba50edb45a581a500d31fd51bb2"}},"metadata":{}},{"output_type":"display_data","data":{"text/plain":"  0%|          | 0/1 [00:00<?, ?it/s]","application/vnd.jupyter.widget-view+json":{"version_major":2,"version_minor":0,"model_id":"9ce38b0f1c8a42e1adce8ddfb2b7f3c8"}},"metadata":{}},{"output_type":"display_data","data":{"text/plain":"  0%|          | 0/1 [00:00<?, ?it/s]","application/vnd.jupyter.widget-view+json":{"version_major":2,"version_minor":0,"model_id":"1d396d949f01465abcec690769d2c42a"}},"metadata":{}},{"output_type":"display_data","data":{"text/plain":"  0%|          | 0/1 [00:00<?, ?it/s]","application/vnd.jupyter.widget-view+json":{"version_major":2,"version_minor":0,"model_id":"08facf9de3d24628b7eba75809030041"}},"metadata":{}},{"output_type":"display_data","data":{"text/plain":"  0%|          | 0/1 [00:00<?, ?it/s]","application/vnd.jupyter.widget-view+json":{"version_major":2,"version_minor":0,"model_id":"fb765f38141c4b9d81cf320d185a5a0e"}},"metadata":{}},{"output_type":"display_data","data":{"text/plain":"  0%|          | 0/1 [00:00<?, ?it/s]","application/vnd.jupyter.widget-view+json":{"version_major":2,"version_minor":0,"model_id":"b71492344a55436db3895f6e20bab415"}},"metadata":{}},{"output_type":"display_data","data":{"text/plain":"  0%|          | 0/1 [00:00<?, ?it/s]","application/vnd.jupyter.widget-view+json":{"version_major":2,"version_minor":0,"model_id":"541e45cebeb1457095dcb777f7266c93"}},"metadata":{}},{"output_type":"display_data","data":{"text/plain":"  0%|          | 0/1 [00:00<?, ?it/s]","application/vnd.jupyter.widget-view+json":{"version_major":2,"version_minor":0,"model_id":"8ebad61f7a174fc4bd30d6043d4dcf1b"}},"metadata":{}},{"output_type":"display_data","data":{"text/plain":"  0%|          | 0/1 [00:00<?, ?it/s]","application/vnd.jupyter.widget-view+json":{"version_major":2,"version_minor":0,"model_id":"d642430e7b3b4046b2d3a6f805a88263"}},"metadata":{}},{"output_type":"display_data","data":{"text/plain":"  0%|          | 0/1 [00:00<?, ?it/s]","application/vnd.jupyter.widget-view+json":{"version_major":2,"version_minor":0,"model_id":"ce5f2423b21b41ca858a68e6600e81ec"}},"metadata":{}},{"output_type":"display_data","data":{"text/plain":"  0%|          | 0/1 [00:00<?, ?it/s]","application/vnd.jupyter.widget-view+json":{"version_major":2,"version_minor":0,"model_id":"ba1be1a2a643461085578fd0e55ad0ed"}},"metadata":{}},{"output_type":"display_data","data":{"text/plain":"  0%|          | 0/1 [00:00<?, ?it/s]","application/vnd.jupyter.widget-view+json":{"version_major":2,"version_minor":0,"model_id":"41666e6293a54fbeb2c26af57b75f2d6"}},"metadata":{}},{"output_type":"display_data","data":{"text/plain":"  0%|          | 0/1 [00:00<?, ?it/s]","application/vnd.jupyter.widget-view+json":{"version_major":2,"version_minor":0,"model_id":"9dd506dcf81040de85e18ccbecdbe9e2"}},"metadata":{}},{"output_type":"display_data","data":{"text/plain":"  0%|          | 0/1 [00:00<?, ?it/s]","application/vnd.jupyter.widget-view+json":{"version_major":2,"version_minor":0,"model_id":"2932f632dd1b47aaafd164b68be32682"}},"metadata":{}},{"output_type":"display_data","data":{"text/plain":"  0%|          | 0/1 [00:00<?, ?it/s]","application/vnd.jupyter.widget-view+json":{"version_major":2,"version_minor":0,"model_id":"95e4daff64724d1cbd62b98226dbb1a2"}},"metadata":{}},{"output_type":"display_data","data":{"text/plain":"  0%|          | 0/1 [00:00<?, ?it/s]","application/vnd.jupyter.widget-view+json":{"version_major":2,"version_minor":0,"model_id":"70e661913baf461aa56e1082898b94e6"}},"metadata":{}},{"output_type":"display_data","data":{"text/plain":"  0%|          | 0/1 [00:00<?, ?it/s]","application/vnd.jupyter.widget-view+json":{"version_major":2,"version_minor":0,"model_id":"fb7cc72a09144ae2ac5bbc4c70875dd0"}},"metadata":{}},{"output_type":"display_data","data":{"text/plain":"  0%|          | 0/1 [00:00<?, ?it/s]","application/vnd.jupyter.widget-view+json":{"version_major":2,"version_minor":0,"model_id":"14a572ef0eaf46a6829aeaac409a0599"}},"metadata":{}},{"output_type":"display_data","data":{"text/plain":"  0%|          | 0/1 [00:00<?, ?it/s]","application/vnd.jupyter.widget-view+json":{"version_major":2,"version_minor":0,"model_id":"294fbdb3cec649cdb97f29c5ef174682"}},"metadata":{}},{"output_type":"display_data","data":{"text/plain":"  0%|          | 0/1 [00:00<?, ?it/s]","application/vnd.jupyter.widget-view+json":{"version_major":2,"version_minor":0,"model_id":"53b0ef9b43dd46d3963cbe3112b82ae8"}},"metadata":{}},{"output_type":"display_data","data":{"text/plain":"  0%|          | 0/1 [00:00<?, ?it/s]","application/vnd.jupyter.widget-view+json":{"version_major":2,"version_minor":0,"model_id":"8ef3726be648418492128d2aa388d07b"}},"metadata":{}},{"output_type":"display_data","data":{"text/plain":"  0%|          | 0/1 [00:00<?, ?it/s]","application/vnd.jupyter.widget-view+json":{"version_major":2,"version_minor":0,"model_id":"1009bad304bd473f8aefc8b28bc885d9"}},"metadata":{}},{"output_type":"display_data","data":{"text/plain":"  0%|          | 0/1 [00:00<?, ?it/s]","application/vnd.jupyter.widget-view+json":{"version_major":2,"version_minor":0,"model_id":"f198f5c3f9e44ed7910419a2775ffa83"}},"metadata":{}},{"output_type":"display_data","data":{"text/plain":"  0%|          | 0/1 [00:00<?, ?it/s]","application/vnd.jupyter.widget-view+json":{"version_major":2,"version_minor":0,"model_id":"d04eb73d35584b0499bc30b00e196c52"}},"metadata":{}},{"output_type":"display_data","data":{"text/plain":"  0%|          | 0/1 [00:00<?, ?it/s]","application/vnd.jupyter.widget-view+json":{"version_major":2,"version_minor":0,"model_id":"59eeb104b0db4e0ca97c63b5cfe45cfb"}},"metadata":{}},{"output_type":"display_data","data":{"text/plain":"  0%|          | 0/1 [00:00<?, ?it/s]","application/vnd.jupyter.widget-view+json":{"version_major":2,"version_minor":0,"model_id":"c181522733be42a99cec44ab044ae846"}},"metadata":{}},{"output_type":"display_data","data":{"text/plain":"  0%|          | 0/1 [00:00<?, ?it/s]","application/vnd.jupyter.widget-view+json":{"version_major":2,"version_minor":0,"model_id":"923f217b69f24a988aa54f7900c1ab8c"}},"metadata":{}},{"output_type":"display_data","data":{"text/plain":"  0%|          | 0/1 [00:00<?, ?it/s]","application/vnd.jupyter.widget-view+json":{"version_major":2,"version_minor":0,"model_id":"280931623b524d80a80452d92668da44"}},"metadata":{}},{"output_type":"display_data","data":{"text/plain":"  0%|          | 0/1 [00:00<?, ?it/s]","application/vnd.jupyter.widget-view+json":{"version_major":2,"version_minor":0,"model_id":"e15e02bc8a1a4d1b8260bbf5c3b4d234"}},"metadata":{}},{"output_type":"display_data","data":{"text/plain":"  0%|          | 0/1 [00:00<?, ?it/s]","application/vnd.jupyter.widget-view+json":{"version_major":2,"version_minor":0,"model_id":"fbd4eb2809624006a87cd075f857af4e"}},"metadata":{}},{"output_type":"display_data","data":{"text/plain":"  0%|          | 0/1 [00:00<?, ?it/s]","application/vnd.jupyter.widget-view+json":{"version_major":2,"version_minor":0,"model_id":"4dd897a6a9af43a3b7a5f7220e58c81b"}},"metadata":{}},{"output_type":"display_data","data":{"text/plain":"  0%|          | 0/1 [00:00<?, ?it/s]","application/vnd.jupyter.widget-view+json":{"version_major":2,"version_minor":0,"model_id":"fe0370f776494343a0a67b5d01b6758e"}},"metadata":{}},{"output_type":"display_data","data":{"text/plain":"  0%|          | 0/1 [00:00<?, ?it/s]","application/vnd.jupyter.widget-view+json":{"version_major":2,"version_minor":0,"model_id":"2a93119287f1496dab40b1757306d202"}},"metadata":{}},{"output_type":"display_data","data":{"text/plain":"  0%|          | 0/1 [00:00<?, ?it/s]","application/vnd.jupyter.widget-view+json":{"version_major":2,"version_minor":0,"model_id":"22db58161edd4b92ab7706cd9f6ad21d"}},"metadata":{}},{"output_type":"display_data","data":{"text/plain":"  0%|          | 0/1 [00:00<?, ?it/s]","application/vnd.jupyter.widget-view+json":{"version_major":2,"version_minor":0,"model_id":"2da21fe7defe45f5b05ec07140395440"}},"metadata":{}},{"output_type":"display_data","data":{"text/plain":"  0%|          | 0/1 [00:00<?, ?it/s]","application/vnd.jupyter.widget-view+json":{"version_major":2,"version_minor":0,"model_id":"b1dc2c914dbc43e3a266be69d8deaea8"}},"metadata":{}},{"output_type":"display_data","data":{"text/plain":"  0%|          | 0/1 [00:00<?, ?it/s]","application/vnd.jupyter.widget-view+json":{"version_major":2,"version_minor":0,"model_id":"bdbd15461e6b44508e0b8fbb911e58b8"}},"metadata":{}},{"output_type":"display_data","data":{"text/plain":"  0%|          | 0/1 [00:00<?, ?it/s]","application/vnd.jupyter.widget-view+json":{"version_major":2,"version_minor":0,"model_id":"c6e4453d94284d0391571a584cf93054"}},"metadata":{}},{"output_type":"display_data","data":{"text/plain":"  0%|          | 0/1 [00:00<?, ?it/s]","application/vnd.jupyter.widget-view+json":{"version_major":2,"version_minor":0,"model_id":"63d3e7dbf1b641a8a8b8ca06f7b1010a"}},"metadata":{}},{"output_type":"display_data","data":{"text/plain":"  0%|          | 0/1 [00:00<?, ?it/s]","application/vnd.jupyter.widget-view+json":{"version_major":2,"version_minor":0,"model_id":"3a6bca6d53eb463792bd17936301f0a2"}},"metadata":{}},{"output_type":"display_data","data":{"text/plain":"  0%|          | 0/1 [00:00<?, ?it/s]","application/vnd.jupyter.widget-view+json":{"version_major":2,"version_minor":0,"model_id":"151db99ed040438e99a5f3107f60714d"}},"metadata":{}},{"output_type":"display_data","data":{"text/plain":"  0%|          | 0/1 [00:00<?, ?it/s]","application/vnd.jupyter.widget-view+json":{"version_major":2,"version_minor":0,"model_id":"8b8f442324454babae674fafa2c225f4"}},"metadata":{}},{"output_type":"display_data","data":{"text/plain":"  0%|          | 0/1 [00:00<?, ?it/s]","application/vnd.jupyter.widget-view+json":{"version_major":2,"version_minor":0,"model_id":"3aa1de67453a457989db537b57db7a01"}},"metadata":{}},{"output_type":"display_data","data":{"text/plain":"  0%|          | 0/1 [00:00<?, ?it/s]","application/vnd.jupyter.widget-view+json":{"version_major":2,"version_minor":0,"model_id":"83c108f55c2b4e13b93d717afe97eb60"}},"metadata":{}},{"output_type":"display_data","data":{"text/plain":"  0%|          | 0/1 [00:00<?, ?it/s]","application/vnd.jupyter.widget-view+json":{"version_major":2,"version_minor":0,"model_id":"d8813617b2c14d7fbabb669959895d3d"}},"metadata":{}},{"output_type":"display_data","data":{"text/plain":"  0%|          | 0/1 [00:00<?, ?it/s]","application/vnd.jupyter.widget-view+json":{"version_major":2,"version_minor":0,"model_id":"531a51322f3643f3a2cd23b7656b7dfd"}},"metadata":{}},{"output_type":"display_data","data":{"text/plain":"  0%|          | 0/1 [00:00<?, ?it/s]","application/vnd.jupyter.widget-view+json":{"version_major":2,"version_minor":0,"model_id":"5316838b5f1a4c72ae84282670a0bf8d"}},"metadata":{}},{"output_type":"display_data","data":{"text/plain":"  0%|          | 0/1 [00:00<?, ?it/s]","application/vnd.jupyter.widget-view+json":{"version_major":2,"version_minor":0,"model_id":"a005b0207d6d4279ae4bfee42ca16c7a"}},"metadata":{}},{"output_type":"display_data","data":{"text/plain":"  0%|          | 0/1 [00:00<?, ?it/s]","application/vnd.jupyter.widget-view+json":{"version_major":2,"version_minor":0,"model_id":"815482b1e91c4695be169632f5d114dd"}},"metadata":{}},{"output_type":"display_data","data":{"text/plain":"  0%|          | 0/1 [00:00<?, ?it/s]","application/vnd.jupyter.widget-view+json":{"version_major":2,"version_minor":0,"model_id":"d2d9d6c37f2348e9ba90c9b009ca8754"}},"metadata":{}},{"output_type":"display_data","data":{"text/plain":"  0%|          | 0/1 [00:00<?, ?it/s]","application/vnd.jupyter.widget-view+json":{"version_major":2,"version_minor":0,"model_id":"6c0e33ebb3e747e8bbcb74e3c7ca0d66"}},"metadata":{}},{"output_type":"display_data","data":{"text/plain":"  0%|          | 0/1 [00:00<?, ?it/s]","application/vnd.jupyter.widget-view+json":{"version_major":2,"version_minor":0,"model_id":"1845a4a3bc35406cb529e8df1452d118"}},"metadata":{}},{"output_type":"display_data","data":{"text/plain":"  0%|          | 0/1 [00:00<?, ?it/s]","application/vnd.jupyter.widget-view+json":{"version_major":2,"version_minor":0,"model_id":"a28ebc638a304c22aa5385def5da2bc1"}},"metadata":{}},{"output_type":"display_data","data":{"text/plain":"  0%|          | 0/1 [00:00<?, ?it/s]","application/vnd.jupyter.widget-view+json":{"version_major":2,"version_minor":0,"model_id":"53ffd8d97c9541539857f88682687857"}},"metadata":{}},{"output_type":"display_data","data":{"text/plain":"  0%|          | 0/1 [00:00<?, ?it/s]","application/vnd.jupyter.widget-view+json":{"version_major":2,"version_minor":0,"model_id":"91091e2c375d45f68151d02b59115152"}},"metadata":{}},{"output_type":"display_data","data":{"text/plain":"  0%|          | 0/1 [00:00<?, ?it/s]","application/vnd.jupyter.widget-view+json":{"version_major":2,"version_minor":0,"model_id":"0332d360549a4647b8e30be9b23a4168"}},"metadata":{}},{"output_type":"display_data","data":{"text/plain":"  0%|          | 0/1 [00:00<?, ?it/s]","application/vnd.jupyter.widget-view+json":{"version_major":2,"version_minor":0,"model_id":"207b8ec8c6fc49cb8e3e5e7435c480f8"}},"metadata":{}},{"output_type":"display_data","data":{"text/plain":"  0%|          | 0/1 [00:00<?, ?it/s]","application/vnd.jupyter.widget-view+json":{"version_major":2,"version_minor":0,"model_id":"e3b12ca8d0804572a9abec477927d83c"}},"metadata":{}},{"output_type":"display_data","data":{"text/plain":"  0%|          | 0/1 [00:00<?, ?it/s]","application/vnd.jupyter.widget-view+json":{"version_major":2,"version_minor":0,"model_id":"9a430a8650ad476486a40eb55c4eaf0c"}},"metadata":{}},{"output_type":"display_data","data":{"text/plain":"  0%|          | 0/1 [00:00<?, ?it/s]","application/vnd.jupyter.widget-view+json":{"version_major":2,"version_minor":0,"model_id":"77b1c145faa2439fbe010a6d39835f74"}},"metadata":{}},{"output_type":"display_data","data":{"text/plain":"  0%|          | 0/1 [00:00<?, ?it/s]","application/vnd.jupyter.widget-view+json":{"version_major":2,"version_minor":0,"model_id":"c6b4cc161646441997124811681a5d44"}},"metadata":{}},{"output_type":"display_data","data":{"text/plain":"  0%|          | 0/1 [00:00<?, ?it/s]","application/vnd.jupyter.widget-view+json":{"version_major":2,"version_minor":0,"model_id":"d243f9b7be054dc989ff3894783f9d9a"}},"metadata":{}},{"output_type":"display_data","data":{"text/plain":"  0%|          | 0/1 [00:00<?, ?it/s]","application/vnd.jupyter.widget-view+json":{"version_major":2,"version_minor":0,"model_id":"c91dd60c72ee427fa29c3746b56a9146"}},"metadata":{}},{"output_type":"display_data","data":{"text/plain":"  0%|          | 0/1 [00:00<?, ?it/s]","application/vnd.jupyter.widget-view+json":{"version_major":2,"version_minor":0,"model_id":"744d81d73ac94f7cbf0e34e9d351877a"}},"metadata":{}},{"output_type":"display_data","data":{"text/plain":"  0%|          | 0/1 [00:00<?, ?it/s]","application/vnd.jupyter.widget-view+json":{"version_major":2,"version_minor":0,"model_id":"0a217f5004024d25bb93b90437134d5b"}},"metadata":{}},{"output_type":"display_data","data":{"text/plain":"  0%|          | 0/1 [00:00<?, ?it/s]","application/vnd.jupyter.widget-view+json":{"version_major":2,"version_minor":0,"model_id":"3e3e50e1f07b4abaa40944e0e08bf53b"}},"metadata":{}},{"output_type":"display_data","data":{"text/plain":"  0%|          | 0/1 [00:00<?, ?it/s]","application/vnd.jupyter.widget-view+json":{"version_major":2,"version_minor":0,"model_id":"c7a85e4415ac4586a3eacf28640c3171"}},"metadata":{}},{"output_type":"display_data","data":{"text/plain":"  0%|          | 0/1 [00:00<?, ?it/s]","application/vnd.jupyter.widget-view+json":{"version_major":2,"version_minor":0,"model_id":"c41e9724caf34495b1ee0a4036be3983"}},"metadata":{}},{"output_type":"display_data","data":{"text/plain":"  0%|          | 0/1 [00:00<?, ?it/s]","application/vnd.jupyter.widget-view+json":{"version_major":2,"version_minor":0,"model_id":"a57895ea8e9f4d4b85d7d500833fded0"}},"metadata":{}},{"output_type":"display_data","data":{"text/plain":"  0%|          | 0/1 [00:00<?, ?it/s]","application/vnd.jupyter.widget-view+json":{"version_major":2,"version_minor":0,"model_id":"25a5a5e67baf4aa4a3aa35b4f4731204"}},"metadata":{}},{"output_type":"display_data","data":{"text/plain":"  0%|          | 0/1 [00:00<?, ?it/s]","application/vnd.jupyter.widget-view+json":{"version_major":2,"version_minor":0,"model_id":"89400bc5add24147a140e936aad635bf"}},"metadata":{}},{"output_type":"display_data","data":{"text/plain":"  0%|          | 0/1 [00:00<?, ?it/s]","application/vnd.jupyter.widget-view+json":{"version_major":2,"version_minor":0,"model_id":"4126e6249f8b48ebb74ac08cc590506e"}},"metadata":{}},{"output_type":"display_data","data":{"text/plain":"  0%|          | 0/1 [00:00<?, ?it/s]","application/vnd.jupyter.widget-view+json":{"version_major":2,"version_minor":0,"model_id":"9d2a2d2b3efc4a8a9f071f98fe098e44"}},"metadata":{}},{"output_type":"display_data","data":{"text/plain":"  0%|          | 0/1 [00:00<?, ?it/s]","application/vnd.jupyter.widget-view+json":{"version_major":2,"version_minor":0,"model_id":"c7598fb1c3be4befada35a7c5c02fe0e"}},"metadata":{}},{"output_type":"display_data","data":{"text/plain":"  0%|          | 0/1 [00:00<?, ?it/s]","application/vnd.jupyter.widget-view+json":{"version_major":2,"version_minor":0,"model_id":"5611ed8e25244bab80020168b6bf98d9"}},"metadata":{}},{"output_type":"display_data","data":{"text/plain":"  0%|          | 0/1 [00:00<?, ?it/s]","application/vnd.jupyter.widget-view+json":{"version_major":2,"version_minor":0,"model_id":"87ebb3fcd0444ac5a6ea5d69047053ed"}},"metadata":{}},{"output_type":"display_data","data":{"text/plain":"  0%|          | 0/1 [00:00<?, ?it/s]","application/vnd.jupyter.widget-view+json":{"version_major":2,"version_minor":0,"model_id":"9bd183eb0fd04b0da2c5e9d4d8607e50"}},"metadata":{}},{"output_type":"display_data","data":{"text/plain":"  0%|          | 0/1 [00:00<?, ?it/s]","application/vnd.jupyter.widget-view+json":{"version_major":2,"version_minor":0,"model_id":"3b37a75efbca4b33b5ad9158618f7974"}},"metadata":{}},{"output_type":"display_data","data":{"text/plain":"  0%|          | 0/1 [00:00<?, ?it/s]","application/vnd.jupyter.widget-view+json":{"version_major":2,"version_minor":0,"model_id":"8fd1b0e6107748a08ec16a5917886502"}},"metadata":{}}]},{"cell_type":"code","source":"all_preds = build_preds(all_models)\nall_preds_df = pd.DataFrame()\n\nfor pred in all_preds:\n    pred_df = pd.DataFrame(pred, columns=pred_cols)\n    all_preds_df = all_preds_df.append(pred_df)\n\nprint('Median score:', mc_rmse_(labels, all_preds_df.reset_index().groupby('index').median().values))\nprint('Mean score:', mc_rmse_(labels, all_preds_df.reset_index().groupby('index').mean().values))","metadata":{"execution":{"iopub.status.busy":"2022-11-29T04:48:13.402321Z","iopub.execute_input":"2022-11-29T04:48:13.403546Z","iopub.status.idle":"2022-11-29T04:48:13.872561Z","shell.execute_reply.started":"2022-11-29T04:48:13.403498Z","shell.execute_reply":"2022-11-29T04:48:13.871092Z"},"trusted":true},"execution_count":46,"outputs":[{"output_type":"display_data","data":{"text/plain":"  0%|          | 0/12 [00:00<?, ?it/s]","application/vnd.jupyter.widget-view+json":{"version_major":2,"version_minor":0,"model_id":"a6cb338debba4419b47ce3c4dd86ab88"}},"metadata":{}},{"name":"stdout","text":"Median score: 0.4479615501235969\nMean score: 0.44736027232053655\n","output_type":"stream"}]},{"cell_type":"code","source":"weights = [1]\nselected_models = [attention_large_fgm_CFG]\nselected_preds =  np.mean([weights[i] * build_preds([m]) for i,m in enumerate(selected_models)], axis=0)\nselected_scores = [mc_rmse_(labels, selected_pred) for selected_pred in selected_preds]\nens_best_score = np.min(selected_scores)\n\nold_score = ens_best_score\nfor i,model in enumerate(all_models):\n    best_score = np.inf \n    best_w = None\n    ct = 0\n    for w in np.arange(0.01, 1, 0.01):\n        trial = w * build_preds([model])[0] + (1-w) * np.average(build_preds(selected_models), axis=0, weights=weights)\n        score = mc_rmse_(labels, trial)\n        \n        if score < best_score:\n            best_score = score\n            best_w = w\n        else:\n            ct += 1\n        if ct > PATIENCE: break\n    if best_score < ens_best_score:\n        ens_best_score = best_score \n        ens_best_w = best_w\n        ens_best_m = model\n\n    inc = old_score-best_score\n    if inc <= TOL:\n        print(f'Decrease by {inc:.4f}. Stopping.')\n        break\n\n    # DISPLAY RESULTS\n    print(); #print(kk,mx,mx_k,mx_w,'%.5f'%inc)\n    print('Ensemble score = %.4f after adding model %s with weight %.3f. Increase of %.4f'%(best_score, model.name, best_w, inc))\n    print()\n    \n    if ens_best_score == best_score:\n        old_score = ens_best_score\n        selected_models.append(ens_best_m)\n        weights.append(ens_best_w)","metadata":{"scrolled":true,"execution":{"iopub.status.busy":"2022-11-29T04:48:15.183589Z","iopub.execute_input":"2022-11-29T04:48:15.187011Z","iopub.status.idle":"2022-11-29T04:49:50.817823Z","shell.execute_reply.started":"2022-11-29T04:48:15.186956Z","shell.execute_reply":"2022-11-29T04:49:50.816283Z"},"trusted":true},"execution_count":47,"outputs":[{"output_type":"display_data","data":{"text/plain":"  0%|          | 0/1 [00:00<?, ?it/s]","application/vnd.jupyter.widget-view+json":{"version_major":2,"version_minor":0,"model_id":"c1d969f4937c42c5b17d2c98660cb424"}},"metadata":{}},{"output_type":"display_data","data":{"text/plain":"  0%|          | 0/1 [00:00<?, ?it/s]","application/vnd.jupyter.widget-view+json":{"version_major":2,"version_minor":0,"model_id":"9b3829d036914c2f9fa82c251a952308"}},"metadata":{}},{"output_type":"display_data","data":{"text/plain":"  0%|          | 0/1 [00:00<?, ?it/s]","application/vnd.jupyter.widget-view+json":{"version_major":2,"version_minor":0,"model_id":"a46b1bf3bdd6460694b96b3602768fc4"}},"metadata":{}},{"output_type":"display_data","data":{"text/plain":"  0%|          | 0/1 [00:00<?, ?it/s]","application/vnd.jupyter.widget-view+json":{"version_major":2,"version_minor":0,"model_id":"d697a91517ae475293399782b660db50"}},"metadata":{}},{"output_type":"display_data","data":{"text/plain":"  0%|          | 0/1 [00:00<?, ?it/s]","application/vnd.jupyter.widget-view+json":{"version_major":2,"version_minor":0,"model_id":"ae9f7c523a8b454cbc798184d3b06d48"}},"metadata":{}},{"output_type":"display_data","data":{"text/plain":"  0%|          | 0/1 [00:00<?, ?it/s]","application/vnd.jupyter.widget-view+json":{"version_major":2,"version_minor":0,"model_id":"b5988e46481742afb41bb2435be5d1bc"}},"metadata":{}},{"output_type":"display_data","data":{"text/plain":"  0%|          | 0/1 [00:00<?, ?it/s]","application/vnd.jupyter.widget-view+json":{"version_major":2,"version_minor":0,"model_id":"1cb9a396393743e9b5bf5f694a4cbd01"}},"metadata":{}},{"output_type":"display_data","data":{"text/plain":"  0%|          | 0/1 [00:00<?, ?it/s]","application/vnd.jupyter.widget-view+json":{"version_major":2,"version_minor":0,"model_id":"66054ac33c3f4a42bb8de655351b84a3"}},"metadata":{}},{"output_type":"display_data","data":{"text/plain":"  0%|          | 0/1 [00:00<?, ?it/s]","application/vnd.jupyter.widget-view+json":{"version_major":2,"version_minor":0,"model_id":"f226c33f01fc47efa391e605c0c026b0"}},"metadata":{}},{"output_type":"display_data","data":{"text/plain":"  0%|          | 0/1 [00:00<?, ?it/s]","application/vnd.jupyter.widget-view+json":{"version_major":2,"version_minor":0,"model_id":"72cddbaadb3840e1a6ad8643b0c7f99a"}},"metadata":{}},{"output_type":"display_data","data":{"text/plain":"  0%|          | 0/1 [00:00<?, ?it/s]","application/vnd.jupyter.widget-view+json":{"version_major":2,"version_minor":0,"model_id":"441de8c7f0c1419bb7b5979d7d60c6d9"}},"metadata":{}},{"output_type":"display_data","data":{"text/plain":"  0%|          | 0/1 [00:00<?, ?it/s]","application/vnd.jupyter.widget-view+json":{"version_major":2,"version_minor":0,"model_id":"cdd7fd5469d041579746b7f655c2b7f5"}},"metadata":{}},{"output_type":"display_data","data":{"text/plain":"  0%|          | 0/1 [00:00<?, ?it/s]","application/vnd.jupyter.widget-view+json":{"version_major":2,"version_minor":0,"model_id":"4e8a0f9d7f104e0a85138d9c225f6d6c"}},"metadata":{}},{"output_type":"display_data","data":{"text/plain":"  0%|          | 0/1 [00:00<?, ?it/s]","application/vnd.jupyter.widget-view+json":{"version_major":2,"version_minor":0,"model_id":"f559409023a7413e87d3e74146bf175e"}},"metadata":{}},{"output_type":"display_data","data":{"text/plain":"  0%|          | 0/1 [00:00<?, ?it/s]","application/vnd.jupyter.widget-view+json":{"version_major":2,"version_minor":0,"model_id":"081235007b1f4526a525c188e368d1c7"}},"metadata":{}},{"output_type":"display_data","data":{"text/plain":"  0%|          | 0/1 [00:00<?, ?it/s]","application/vnd.jupyter.widget-view+json":{"version_major":2,"version_minor":0,"model_id":"c9348032cddd493791265157ef21b2a6"}},"metadata":{}},{"output_type":"display_data","data":{"text/plain":"  0%|          | 0/1 [00:00<?, ?it/s]","application/vnd.jupyter.widget-view+json":{"version_major":2,"version_minor":0,"model_id":"eea57397235245fca89ecc51758c48bc"}},"metadata":{}},{"output_type":"display_data","data":{"text/plain":"  0%|          | 0/1 [00:00<?, ?it/s]","application/vnd.jupyter.widget-view+json":{"version_major":2,"version_minor":0,"model_id":"d61352cd404148f997622f152d417994"}},"metadata":{}},{"output_type":"display_data","data":{"text/plain":"  0%|          | 0/1 [00:00<?, ?it/s]","application/vnd.jupyter.widget-view+json":{"version_major":2,"version_minor":0,"model_id":"83d5915348f2414f8d1e830182a9bba3"}},"metadata":{}},{"output_type":"display_data","data":{"text/plain":"  0%|          | 0/1 [00:00<?, ?it/s]","application/vnd.jupyter.widget-view+json":{"version_major":2,"version_minor":0,"model_id":"660588deced14b1b8aaa06495d37e9a6"}},"metadata":{}},{"output_type":"display_data","data":{"text/plain":"  0%|          | 0/1 [00:00<?, ?it/s]","application/vnd.jupyter.widget-view+json":{"version_major":2,"version_minor":0,"model_id":"83c6574b38df48f68f13719e0db28594"}},"metadata":{}},{"output_type":"display_data","data":{"text/plain":"  0%|          | 0/1 [00:00<?, ?it/s]","application/vnd.jupyter.widget-view+json":{"version_major":2,"version_minor":0,"model_id":"dc56e8232a5649359d64bf6c854a6019"}},"metadata":{}},{"output_type":"display_data","data":{"text/plain":"  0%|          | 0/1 [00:00<?, ?it/s]","application/vnd.jupyter.widget-view+json":{"version_major":2,"version_minor":0,"model_id":"e747f236761f44c4b5158212be77dd9c"}},"metadata":{}},{"output_type":"display_data","data":{"text/plain":"  0%|          | 0/1 [00:00<?, ?it/s]","application/vnd.jupyter.widget-view+json":{"version_major":2,"version_minor":0,"model_id":"0e92535a6918423a829752a53d015d9e"}},"metadata":{}},{"output_type":"display_data","data":{"text/plain":"  0%|          | 0/1 [00:00<?, ?it/s]","application/vnd.jupyter.widget-view+json":{"version_major":2,"version_minor":0,"model_id":"4d4407f85fd841e593413322acd0015e"}},"metadata":{}},{"output_type":"display_data","data":{"text/plain":"  0%|          | 0/1 [00:00<?, ?it/s]","application/vnd.jupyter.widget-view+json":{"version_major":2,"version_minor":0,"model_id":"c5c9879cf6a04e64948b69e9af8b7a64"}},"metadata":{}},{"output_type":"display_data","data":{"text/plain":"  0%|          | 0/1 [00:00<?, ?it/s]","application/vnd.jupyter.widget-view+json":{"version_major":2,"version_minor":0,"model_id":"29dacc6d2de64a02a1f0ecd3fad19f24"}},"metadata":{}},{"output_type":"display_data","data":{"text/plain":"  0%|          | 0/1 [00:00<?, ?it/s]","application/vnd.jupyter.widget-view+json":{"version_major":2,"version_minor":0,"model_id":"4fd4fb0dbe6040ec9199f1cc64fd2d12"}},"metadata":{}},{"output_type":"display_data","data":{"text/plain":"  0%|          | 0/1 [00:00<?, ?it/s]","application/vnd.jupyter.widget-view+json":{"version_major":2,"version_minor":0,"model_id":"e01ff30b5f6348a8ab86872fef279bdc"}},"metadata":{}},{"output_type":"display_data","data":{"text/plain":"  0%|          | 0/1 [00:00<?, ?it/s]","application/vnd.jupyter.widget-view+json":{"version_major":2,"version_minor":0,"model_id":"14996e382d834548b6825e1cdd10f3e9"}},"metadata":{}},{"output_type":"display_data","data":{"text/plain":"  0%|          | 0/1 [00:00<?, ?it/s]","application/vnd.jupyter.widget-view+json":{"version_major":2,"version_minor":0,"model_id":"5c011268500a48fd90b332ed9e0871c3"}},"metadata":{}},{"output_type":"display_data","data":{"text/plain":"  0%|          | 0/1 [00:00<?, ?it/s]","application/vnd.jupyter.widget-view+json":{"version_major":2,"version_minor":0,"model_id":"f9a1d22b3c5440c8b2596ab7f486801e"}},"metadata":{}},{"output_type":"display_data","data":{"text/plain":"  0%|          | 0/1 [00:00<?, ?it/s]","application/vnd.jupyter.widget-view+json":{"version_major":2,"version_minor":0,"model_id":"6be106fd548a486fa9df4c4e27807ecb"}},"metadata":{}},{"output_type":"display_data","data":{"text/plain":"  0%|          | 0/1 [00:00<?, ?it/s]","application/vnd.jupyter.widget-view+json":{"version_major":2,"version_minor":0,"model_id":"bbf0524babcb4b3e851cea42b8a3dff8"}},"metadata":{}},{"output_type":"display_data","data":{"text/plain":"  0%|          | 0/1 [00:00<?, ?it/s]","application/vnd.jupyter.widget-view+json":{"version_major":2,"version_minor":0,"model_id":"7aeba71d26db494bbe11dc7fe36e5ba6"}},"metadata":{}},{"output_type":"display_data","data":{"text/plain":"  0%|          | 0/1 [00:00<?, ?it/s]","application/vnd.jupyter.widget-view+json":{"version_major":2,"version_minor":0,"model_id":"77ebecc9373641aab3960f6a8b876477"}},"metadata":{}},{"output_type":"display_data","data":{"text/plain":"  0%|          | 0/1 [00:00<?, ?it/s]","application/vnd.jupyter.widget-view+json":{"version_major":2,"version_minor":0,"model_id":"7410b98dcf0e45adae27469003fb48ee"}},"metadata":{}},{"output_type":"display_data","data":{"text/plain":"  0%|          | 0/1 [00:00<?, ?it/s]","application/vnd.jupyter.widget-view+json":{"version_major":2,"version_minor":0,"model_id":"6de90f64a7794205856e34407f2417c1"}},"metadata":{}},{"output_type":"display_data","data":{"text/plain":"  0%|          | 0/1 [00:00<?, ?it/s]","application/vnd.jupyter.widget-view+json":{"version_major":2,"version_minor":0,"model_id":"2fb971437ee246538b8a84ee5b94da7c"}},"metadata":{}},{"output_type":"display_data","data":{"text/plain":"  0%|          | 0/1 [00:00<?, ?it/s]","application/vnd.jupyter.widget-view+json":{"version_major":2,"version_minor":0,"model_id":"f652dbf47ded4a22baf52f333d6c1cc1"}},"metadata":{}},{"output_type":"display_data","data":{"text/plain":"  0%|          | 0/1 [00:00<?, ?it/s]","application/vnd.jupyter.widget-view+json":{"version_major":2,"version_minor":0,"model_id":"de45876bb3c74baea6d6582e302419be"}},"metadata":{}},{"output_type":"display_data","data":{"text/plain":"  0%|          | 0/1 [00:00<?, ?it/s]","application/vnd.jupyter.widget-view+json":{"version_major":2,"version_minor":0,"model_id":"f55a725df2af4eba85155174a54ff142"}},"metadata":{}},{"output_type":"display_data","data":{"text/plain":"  0%|          | 0/1 [00:00<?, ?it/s]","application/vnd.jupyter.widget-view+json":{"version_major":2,"version_minor":0,"model_id":"8c97c10e7e2b4e9ca7f11551813c0221"}},"metadata":{}},{"output_type":"display_data","data":{"text/plain":"  0%|          | 0/1 [00:00<?, ?it/s]","application/vnd.jupyter.widget-view+json":{"version_major":2,"version_minor":0,"model_id":"79ecb02b829543e18288f56073208f50"}},"metadata":{}},{"output_type":"display_data","data":{"text/plain":"  0%|          | 0/1 [00:00<?, ?it/s]","application/vnd.jupyter.widget-view+json":{"version_major":2,"version_minor":0,"model_id":"0ff8987a2f774a7f8ad7ac7f387ae8dd"}},"metadata":{}},{"output_type":"display_data","data":{"text/plain":"  0%|          | 0/1 [00:00<?, ?it/s]","application/vnd.jupyter.widget-view+json":{"version_major":2,"version_minor":0,"model_id":"e8572c8240d941aea6c893974aa4fbe4"}},"metadata":{}},{"output_type":"display_data","data":{"text/plain":"  0%|          | 0/1 [00:00<?, ?it/s]","application/vnd.jupyter.widget-view+json":{"version_major":2,"version_minor":0,"model_id":"70e62930a84c45afb8f922af33c2183c"}},"metadata":{}},{"output_type":"display_data","data":{"text/plain":"  0%|          | 0/1 [00:00<?, ?it/s]","application/vnd.jupyter.widget-view+json":{"version_major":2,"version_minor":0,"model_id":"1e05075e7f2040eaa1baa57c23d0c588"}},"metadata":{}},{"output_type":"display_data","data":{"text/plain":"  0%|          | 0/1 [00:00<?, ?it/s]","application/vnd.jupyter.widget-view+json":{"version_major":2,"version_minor":0,"model_id":"5ff4601937c646ccb1ddb64420452412"}},"metadata":{}},{"output_type":"display_data","data":{"text/plain":"  0%|          | 0/1 [00:00<?, ?it/s]","application/vnd.jupyter.widget-view+json":{"version_major":2,"version_minor":0,"model_id":"aa46fec431504b84b865e8ff37c45e00"}},"metadata":{}},{"output_type":"display_data","data":{"text/plain":"  0%|          | 0/1 [00:00<?, ?it/s]","application/vnd.jupyter.widget-view+json":{"version_major":2,"version_minor":0,"model_id":"8aea827627b5435d97e913827ac1ded5"}},"metadata":{}},{"output_type":"display_data","data":{"text/plain":"  0%|          | 0/1 [00:00<?, ?it/s]","application/vnd.jupyter.widget-view+json":{"version_major":2,"version_minor":0,"model_id":"3994a9bb64754f36b557c3d602d8d665"}},"metadata":{}},{"output_type":"display_data","data":{"text/plain":"  0%|          | 0/1 [00:00<?, ?it/s]","application/vnd.jupyter.widget-view+json":{"version_major":2,"version_minor":0,"model_id":"f41d3a160ce040c0bf1cea45fff828cd"}},"metadata":{}},{"output_type":"display_data","data":{"text/plain":"  0%|          | 0/1 [00:00<?, ?it/s]","application/vnd.jupyter.widget-view+json":{"version_major":2,"version_minor":0,"model_id":"a0ec166899d243e399e8f9db033554e5"}},"metadata":{}},{"output_type":"display_data","data":{"text/plain":"  0%|          | 0/1 [00:00<?, ?it/s]","application/vnd.jupyter.widget-view+json":{"version_major":2,"version_minor":0,"model_id":"3bf61fbf15184694807d5127ef061e99"}},"metadata":{}},{"output_type":"display_data","data":{"text/plain":"  0%|          | 0/1 [00:00<?, ?it/s]","application/vnd.jupyter.widget-view+json":{"version_major":2,"version_minor":0,"model_id":"a72c172a49ce4c639c8eb690ed55e0e7"}},"metadata":{}},{"output_type":"display_data","data":{"text/plain":"  0%|          | 0/1 [00:00<?, ?it/s]","application/vnd.jupyter.widget-view+json":{"version_major":2,"version_minor":0,"model_id":"229897dc09384817b9bdc8ca5c8e857b"}},"metadata":{}},{"output_type":"display_data","data":{"text/plain":"  0%|          | 0/1 [00:00<?, ?it/s]","application/vnd.jupyter.widget-view+json":{"version_major":2,"version_minor":0,"model_id":"43cf9daf4c1b4b99af2612e07a48bfbf"}},"metadata":{}},{"output_type":"display_data","data":{"text/plain":"  0%|          | 0/1 [00:00<?, ?it/s]","application/vnd.jupyter.widget-view+json":{"version_major":2,"version_minor":0,"model_id":"ad46d19a3e2d449498287a9b80bbe422"}},"metadata":{}},{"output_type":"display_data","data":{"text/plain":"  0%|          | 0/1 [00:00<?, ?it/s]","application/vnd.jupyter.widget-view+json":{"version_major":2,"version_minor":0,"model_id":"92d6568ff15c401e8f97fa51a48775e1"}},"metadata":{}},{"output_type":"display_data","data":{"text/plain":"  0%|          | 0/1 [00:00<?, ?it/s]","application/vnd.jupyter.widget-view+json":{"version_major":2,"version_minor":0,"model_id":"0313e13c178a42b7902d0e9d89de5029"}},"metadata":{}},{"output_type":"display_data","data":{"text/plain":"  0%|          | 0/1 [00:00<?, ?it/s]","application/vnd.jupyter.widget-view+json":{"version_major":2,"version_minor":0,"model_id":"dbc6265527cb49beb145619d132de32a"}},"metadata":{}},{"output_type":"display_data","data":{"text/plain":"  0%|          | 0/1 [00:00<?, ?it/s]","application/vnd.jupyter.widget-view+json":{"version_major":2,"version_minor":0,"model_id":"d6771624d5ef4ef48ac6ea5a17335a68"}},"metadata":{}},{"output_type":"display_data","data":{"text/plain":"  0%|          | 0/1 [00:00<?, ?it/s]","application/vnd.jupyter.widget-view+json":{"version_major":2,"version_minor":0,"model_id":"5c807349ebb04bd6934559ec0f0d1d33"}},"metadata":{}},{"output_type":"display_data","data":{"text/plain":"  0%|          | 0/1 [00:00<?, ?it/s]","application/vnd.jupyter.widget-view+json":{"version_major":2,"version_minor":0,"model_id":"ad97ec13fd38452aa24e083cea03256c"}},"metadata":{}},{"output_type":"display_data","data":{"text/plain":"  0%|          | 0/1 [00:00<?, ?it/s]","application/vnd.jupyter.widget-view+json":{"version_major":2,"version_minor":0,"model_id":"cf34d934d4dd448993f4a9154541987c"}},"metadata":{}},{"output_type":"display_data","data":{"text/plain":"  0%|          | 0/1 [00:00<?, ?it/s]","application/vnd.jupyter.widget-view+json":{"version_major":2,"version_minor":0,"model_id":"7c8b850adc1d4a479aaca86d311940ab"}},"metadata":{}},{"output_type":"display_data","data":{"text/plain":"  0%|          | 0/1 [00:00<?, ?it/s]","application/vnd.jupyter.widget-view+json":{"version_major":2,"version_minor":0,"model_id":"4adb3cfb74de4063b4f9b4ab4bd85686"}},"metadata":{}},{"output_type":"display_data","data":{"text/plain":"  0%|          | 0/1 [00:00<?, ?it/s]","application/vnd.jupyter.widget-view+json":{"version_major":2,"version_minor":0,"model_id":"108595a9d2ee42539a881a9d733da047"}},"metadata":{}},{"output_type":"display_data","data":{"text/plain":"  0%|          | 0/1 [00:00<?, ?it/s]","application/vnd.jupyter.widget-view+json":{"version_major":2,"version_minor":0,"model_id":"e02faa4805cb45bd9f5610945397377f"}},"metadata":{}},{"output_type":"display_data","data":{"text/plain":"  0%|          | 0/1 [00:00<?, ?it/s]","application/vnd.jupyter.widget-view+json":{"version_major":2,"version_minor":0,"model_id":"3f30c99a5f4843f79138090788304f8f"}},"metadata":{}},{"output_type":"display_data","data":{"text/plain":"  0%|          | 0/1 [00:00<?, ?it/s]","application/vnd.jupyter.widget-view+json":{"version_major":2,"version_minor":0,"model_id":"9dbae5c047654fcfb89190fdd3d5351a"}},"metadata":{}},{"output_type":"display_data","data":{"text/plain":"  0%|          | 0/1 [00:00<?, ?it/s]","application/vnd.jupyter.widget-view+json":{"version_major":2,"version_minor":0,"model_id":"73b3b3eb8a174e8e994483e1878c9b33"}},"metadata":{}},{"output_type":"display_data","data":{"text/plain":"  0%|          | 0/1 [00:00<?, ?it/s]","application/vnd.jupyter.widget-view+json":{"version_major":2,"version_minor":0,"model_id":"3baea3e6fe8c4241b6e82af57194367d"}},"metadata":{}},{"output_type":"display_data","data":{"text/plain":"  0%|          | 0/1 [00:00<?, ?it/s]","application/vnd.jupyter.widget-view+json":{"version_major":2,"version_minor":0,"model_id":"2688caa4aae34fb497ea23d95e14d85d"}},"metadata":{}},{"output_type":"display_data","data":{"text/plain":"  0%|          | 0/1 [00:00<?, ?it/s]","application/vnd.jupyter.widget-view+json":{"version_major":2,"version_minor":0,"model_id":"a00c8495c6b948c0b8efa88e4c527301"}},"metadata":{}},{"output_type":"display_data","data":{"text/plain":"  0%|          | 0/1 [00:00<?, ?it/s]","application/vnd.jupyter.widget-view+json":{"version_major":2,"version_minor":0,"model_id":"5637883136d842a790207d2782735e98"}},"metadata":{}},{"output_type":"display_data","data":{"text/plain":"  0%|          | 0/1 [00:00<?, ?it/s]","application/vnd.jupyter.widget-view+json":{"version_major":2,"version_minor":0,"model_id":"1f467ffe41e349dcb5e348e7be71d894"}},"metadata":{}},{"output_type":"display_data","data":{"text/plain":"  0%|          | 0/1 [00:00<?, ?it/s]","application/vnd.jupyter.widget-view+json":{"version_major":2,"version_minor":0,"model_id":"58ed94c4f1a74a9eaf664554eb723dbf"}},"metadata":{}},{"output_type":"display_data","data":{"text/plain":"  0%|          | 0/1 [00:00<?, ?it/s]","application/vnd.jupyter.widget-view+json":{"version_major":2,"version_minor":0,"model_id":"883acf7dcce74062a09d6058b206b11b"}},"metadata":{}},{"output_type":"display_data","data":{"text/plain":"  0%|          | 0/1 [00:00<?, ?it/s]","application/vnd.jupyter.widget-view+json":{"version_major":2,"version_minor":0,"model_id":"80c6f39b91fa4c6a942a5e58c1b1f5be"}},"metadata":{}},{"output_type":"display_data","data":{"text/plain":"  0%|          | 0/1 [00:00<?, ?it/s]","application/vnd.jupyter.widget-view+json":{"version_major":2,"version_minor":0,"model_id":"8c16a0bdfc38457ab9686792a4836a25"}},"metadata":{}},{"output_type":"display_data","data":{"text/plain":"  0%|          | 0/1 [00:00<?, ?it/s]","application/vnd.jupyter.widget-view+json":{"version_major":2,"version_minor":0,"model_id":"0bbbe05a021b4be6b75df6737ced82ae"}},"metadata":{}},{"output_type":"display_data","data":{"text/plain":"  0%|          | 0/1 [00:00<?, ?it/s]","application/vnd.jupyter.widget-view+json":{"version_major":2,"version_minor":0,"model_id":"eb0f801e2634476e849e2fda490eb05f"}},"metadata":{}},{"output_type":"display_data","data":{"text/plain":"  0%|          | 0/1 [00:00<?, ?it/s]","application/vnd.jupyter.widget-view+json":{"version_major":2,"version_minor":0,"model_id":"0042ffdf74c846638d9070db5b90835f"}},"metadata":{}},{"output_type":"display_data","data":{"text/plain":"  0%|          | 0/1 [00:00<?, ?it/s]","application/vnd.jupyter.widget-view+json":{"version_major":2,"version_minor":0,"model_id":"43711035d9f8489f817b5d589910bab8"}},"metadata":{}},{"output_type":"display_data","data":{"text/plain":"  0%|          | 0/1 [00:00<?, ?it/s]","application/vnd.jupyter.widget-view+json":{"version_major":2,"version_minor":0,"model_id":"2a364903fe9e4161b4117d867619734e"}},"metadata":{}},{"output_type":"display_data","data":{"text/plain":"  0%|          | 0/1 [00:00<?, ?it/s]","application/vnd.jupyter.widget-view+json":{"version_major":2,"version_minor":0,"model_id":"a3e448771fab40d2acb5b1b8b5fbb7e1"}},"metadata":{}},{"output_type":"display_data","data":{"text/plain":"  0%|          | 0/1 [00:00<?, ?it/s]","application/vnd.jupyter.widget-view+json":{"version_major":2,"version_minor":0,"model_id":"0cb28c2371aa4a14a592cb7af903d087"}},"metadata":{}},{"output_type":"display_data","data":{"text/plain":"  0%|          | 0/1 [00:00<?, ?it/s]","application/vnd.jupyter.widget-view+json":{"version_major":2,"version_minor":0,"model_id":"457d9896ca0d4cab95f3290b50b78643"}},"metadata":{}},{"output_type":"display_data","data":{"text/plain":"  0%|          | 0/1 [00:00<?, ?it/s]","application/vnd.jupyter.widget-view+json":{"version_major":2,"version_minor":0,"model_id":"273aa3d1d9674b4d9b7b91d218c5e2b0"}},"metadata":{}},{"output_type":"display_data","data":{"text/plain":"  0%|          | 0/1 [00:00<?, ?it/s]","application/vnd.jupyter.widget-view+json":{"version_major":2,"version_minor":0,"model_id":"7430aa2bc5324873b34fc20e3a83321f"}},"metadata":{}},{"output_type":"display_data","data":{"text/plain":"  0%|          | 0/1 [00:00<?, ?it/s]","application/vnd.jupyter.widget-view+json":{"version_major":2,"version_minor":0,"model_id":"000a71bb8fc2440380fa40f2fbe91c97"}},"metadata":{}},{"output_type":"display_data","data":{"text/plain":"  0%|          | 0/1 [00:00<?, ?it/s]","application/vnd.jupyter.widget-view+json":{"version_major":2,"version_minor":0,"model_id":"c59ce2a5eee1479baad2cf691d98225a"}},"metadata":{}},{"output_type":"display_data","data":{"text/plain":"  0%|          | 0/1 [00:00<?, ?it/s]","application/vnd.jupyter.widget-view+json":{"version_major":2,"version_minor":0,"model_id":"d5653571986646358751385c1f5560ba"}},"metadata":{}},{"output_type":"display_data","data":{"text/plain":"  0%|          | 0/1 [00:00<?, ?it/s]","application/vnd.jupyter.widget-view+json":{"version_major":2,"version_minor":0,"model_id":"c1e51832453847d882042158ffb69f5d"}},"metadata":{}},{"output_type":"display_data","data":{"text/plain":"  0%|          | 0/1 [00:00<?, ?it/s]","application/vnd.jupyter.widget-view+json":{"version_major":2,"version_minor":0,"model_id":"6daec1c71c2a424d992c5e86759f3a2a"}},"metadata":{}},{"output_type":"display_data","data":{"text/plain":"  0%|          | 0/1 [00:00<?, ?it/s]","application/vnd.jupyter.widget-view+json":{"version_major":2,"version_minor":0,"model_id":"14cc48f48e4a4322ad06b706beb18f32"}},"metadata":{}},{"output_type":"display_data","data":{"text/plain":"  0%|          | 0/1 [00:00<?, ?it/s]","application/vnd.jupyter.widget-view+json":{"version_major":2,"version_minor":0,"model_id":"a076b56b84ce4145908f0b8983d885f7"}},"metadata":{}},{"output_type":"display_data","data":{"text/plain":"  0%|          | 0/1 [00:00<?, ?it/s]","application/vnd.jupyter.widget-view+json":{"version_major":2,"version_minor":0,"model_id":"834edcba256548018b0d79589c6278d2"}},"metadata":{}},{"output_type":"display_data","data":{"text/plain":"  0%|          | 0/1 [00:00<?, ?it/s]","application/vnd.jupyter.widget-view+json":{"version_major":2,"version_minor":0,"model_id":"5c8090cecfbe4834a167bcf28a71fa0c"}},"metadata":{}},{"output_type":"display_data","data":{"text/plain":"  0%|          | 0/1 [00:00<?, ?it/s]","application/vnd.jupyter.widget-view+json":{"version_major":2,"version_minor":0,"model_id":"fd0b6e5520804b4f9ed3d8b21aa8f9c3"}},"metadata":{}},{"output_type":"display_data","data":{"text/plain":"  0%|          | 0/1 [00:00<?, ?it/s]","application/vnd.jupyter.widget-view+json":{"version_major":2,"version_minor":0,"model_id":"a667fa3bc31b4be695ef6015f784dd00"}},"metadata":{}},{"output_type":"display_data","data":{"text/plain":"  0%|          | 0/1 [00:00<?, ?it/s]","application/vnd.jupyter.widget-view+json":{"version_major":2,"version_minor":0,"model_id":"f2ffdbcb56614623a0d8d7b2ea4bdf54"}},"metadata":{}},{"output_type":"display_data","data":{"text/plain":"  0%|          | 0/1 [00:00<?, ?it/s]","application/vnd.jupyter.widget-view+json":{"version_major":2,"version_minor":0,"model_id":"ff457b04338347b99e7d883e824b1df9"}},"metadata":{}},{"output_type":"display_data","data":{"text/plain":"  0%|          | 0/1 [00:00<?, ?it/s]","application/vnd.jupyter.widget-view+json":{"version_major":2,"version_minor":0,"model_id":"b172810438f547fbbfb3ab977054b08f"}},"metadata":{}},{"output_type":"display_data","data":{"text/plain":"  0%|          | 0/1 [00:00<?, ?it/s]","application/vnd.jupyter.widget-view+json":{"version_major":2,"version_minor":0,"model_id":"aeeafa431ce249be89d70a5fa9db583c"}},"metadata":{}},{"output_type":"display_data","data":{"text/plain":"  0%|          | 0/1 [00:00<?, ?it/s]","application/vnd.jupyter.widget-view+json":{"version_major":2,"version_minor":0,"model_id":"2f5605515438463a9ef7c0782ddb6369"}},"metadata":{}},{"output_type":"display_data","data":{"text/plain":"  0%|          | 0/1 [00:00<?, ?it/s]","application/vnd.jupyter.widget-view+json":{"version_major":2,"version_minor":0,"model_id":"d1b8017b85e94ce8b9517203f83b2cd3"}},"metadata":{}},{"output_type":"display_data","data":{"text/plain":"  0%|          | 0/1 [00:00<?, ?it/s]","application/vnd.jupyter.widget-view+json":{"version_major":2,"version_minor":0,"model_id":"3b8ee8d5e4af41feb9f68ac783d36f47"}},"metadata":{}},{"output_type":"display_data","data":{"text/plain":"  0%|          | 0/1 [00:00<?, ?it/s]","application/vnd.jupyter.widget-view+json":{"version_major":2,"version_minor":0,"model_id":"4c83362601714ce1999193bae7d6a2e3"}},"metadata":{}},{"output_type":"display_data","data":{"text/plain":"  0%|          | 0/1 [00:00<?, ?it/s]","application/vnd.jupyter.widget-view+json":{"version_major":2,"version_minor":0,"model_id":"1879146682cb4a408876e12a626bdb70"}},"metadata":{}},{"output_type":"display_data","data":{"text/plain":"  0%|          | 0/1 [00:00<?, ?it/s]","application/vnd.jupyter.widget-view+json":{"version_major":2,"version_minor":0,"model_id":"b63a09da8e3549cd861c49a319d8bac8"}},"metadata":{}},{"output_type":"display_data","data":{"text/plain":"  0%|          | 0/1 [00:00<?, ?it/s]","application/vnd.jupyter.widget-view+json":{"version_major":2,"version_minor":0,"model_id":"1203aea47a0e44e7a043fcb525b140f4"}},"metadata":{}},{"output_type":"display_data","data":{"text/plain":"  0%|          | 0/1 [00:00<?, ?it/s]","application/vnd.jupyter.widget-view+json":{"version_major":2,"version_minor":0,"model_id":"77c61a1c20e140ec8d8816c8f91a7e4a"}},"metadata":{}},{"output_type":"display_data","data":{"text/plain":"  0%|          | 0/1 [00:00<?, ?it/s]","application/vnd.jupyter.widget-view+json":{"version_major":2,"version_minor":0,"model_id":"631e04c02b184ebba17e3b486a858ab1"}},"metadata":{}},{"output_type":"display_data","data":{"text/plain":"  0%|          | 0/1 [00:00<?, ?it/s]","application/vnd.jupyter.widget-view+json":{"version_major":2,"version_minor":0,"model_id":"61587bc7d3a64d0f93b2b4069903d491"}},"metadata":{}},{"name":"stdout","text":"\nEnsemble score = 0.4477 after adding model weighted2last_fgm_512 with weight 0.470. Increase of 0.0044\n\n","output_type":"stream"},{"output_type":"display_data","data":{"text/plain":"  0%|          | 0/1 [00:00<?, ?it/s]","application/vnd.jupyter.widget-view+json":{"version_major":2,"version_minor":0,"model_id":"50f2f833d4794c62912a03fa8eab7924"}},"metadata":{}},{"output_type":"display_data","data":{"text/plain":"  0%|          | 0/2 [00:00<?, ?it/s]","application/vnd.jupyter.widget-view+json":{"version_major":2,"version_minor":0,"model_id":"5f3fb5a69d52410cac8b0549e9c682fe"}},"metadata":{}},{"output_type":"display_data","data":{"text/plain":"  0%|          | 0/1 [00:00<?, ?it/s]","application/vnd.jupyter.widget-view+json":{"version_major":2,"version_minor":0,"model_id":"d79b22aa07934e708e464da3fd2f9df0"}},"metadata":{}},{"output_type":"display_data","data":{"text/plain":"  0%|          | 0/2 [00:00<?, ?it/s]","application/vnd.jupyter.widget-view+json":{"version_major":2,"version_minor":0,"model_id":"6b1424a71fe54c5888977ebc93280c06"}},"metadata":{}},{"output_type":"display_data","data":{"text/plain":"  0%|          | 0/1 [00:00<?, ?it/s]","application/vnd.jupyter.widget-view+json":{"version_major":2,"version_minor":0,"model_id":"50264d0bc2474b5198be6d612845bef7"}},"metadata":{}},{"output_type":"display_data","data":{"text/plain":"  0%|          | 0/2 [00:00<?, ?it/s]","application/vnd.jupyter.widget-view+json":{"version_major":2,"version_minor":0,"model_id":"f15745dd122b42a08a065faaf2f6c57d"}},"metadata":{}},{"output_type":"display_data","data":{"text/plain":"  0%|          | 0/1 [00:00<?, ?it/s]","application/vnd.jupyter.widget-view+json":{"version_major":2,"version_minor":0,"model_id":"2b8e21532ead4147bca1420285963272"}},"metadata":{}},{"output_type":"display_data","data":{"text/plain":"  0%|          | 0/2 [00:00<?, ?it/s]","application/vnd.jupyter.widget-view+json":{"version_major":2,"version_minor":0,"model_id":"f546b89328024e388a065380800e0997"}},"metadata":{}},{"output_type":"display_data","data":{"text/plain":"  0%|          | 0/1 [00:00<?, ?it/s]","application/vnd.jupyter.widget-view+json":{"version_major":2,"version_minor":0,"model_id":"861bfb0d21134bbfae73963049e9d9fa"}},"metadata":{}},{"output_type":"display_data","data":{"text/plain":"  0%|          | 0/2 [00:00<?, ?it/s]","application/vnd.jupyter.widget-view+json":{"version_major":2,"version_minor":0,"model_id":"8e94b3c732ba47a78b269757458ccc8b"}},"metadata":{}},{"output_type":"display_data","data":{"text/plain":"  0%|          | 0/1 [00:00<?, ?it/s]","application/vnd.jupyter.widget-view+json":{"version_major":2,"version_minor":0,"model_id":"fd86ad17e62f4b94bdf32d0b4a4ef4ff"}},"metadata":{}},{"output_type":"display_data","data":{"text/plain":"  0%|          | 0/2 [00:00<?, ?it/s]","application/vnd.jupyter.widget-view+json":{"version_major":2,"version_minor":0,"model_id":"558d1f6dcd94433f80fcb939c1a401a8"}},"metadata":{}},{"output_type":"display_data","data":{"text/plain":"  0%|          | 0/1 [00:00<?, ?it/s]","application/vnd.jupyter.widget-view+json":{"version_major":2,"version_minor":0,"model_id":"17e371f2314e46bca6dd90b2669faa4f"}},"metadata":{}},{"output_type":"display_data","data":{"text/plain":"  0%|          | 0/2 [00:00<?, ?it/s]","application/vnd.jupyter.widget-view+json":{"version_major":2,"version_minor":0,"model_id":"bf09c18b44cf439c92d11d7b1bf55611"}},"metadata":{}},{"output_type":"display_data","data":{"text/plain":"  0%|          | 0/1 [00:00<?, ?it/s]","application/vnd.jupyter.widget-view+json":{"version_major":2,"version_minor":0,"model_id":"aec54a8928614a11a6f1f58545d524cc"}},"metadata":{}},{"output_type":"display_data","data":{"text/plain":"  0%|          | 0/2 [00:00<?, ?it/s]","application/vnd.jupyter.widget-view+json":{"version_major":2,"version_minor":0,"model_id":"021c10102ff142908c03c5fa71494831"}},"metadata":{}},{"output_type":"display_data","data":{"text/plain":"  0%|          | 0/1 [00:00<?, ?it/s]","application/vnd.jupyter.widget-view+json":{"version_major":2,"version_minor":0,"model_id":"62c398e2d8a54fba8fc88dce9ad86832"}},"metadata":{}},{"output_type":"display_data","data":{"text/plain":"  0%|          | 0/2 [00:00<?, ?it/s]","application/vnd.jupyter.widget-view+json":{"version_major":2,"version_minor":0,"model_id":"5204bae161384ed8b8bea5f850f0c3a2"}},"metadata":{}},{"output_type":"display_data","data":{"text/plain":"  0%|          | 0/1 [00:00<?, ?it/s]","application/vnd.jupyter.widget-view+json":{"version_major":2,"version_minor":0,"model_id":"de4fc6a32a284d2282c41235cb3df3f5"}},"metadata":{}},{"output_type":"display_data","data":{"text/plain":"  0%|          | 0/2 [00:00<?, ?it/s]","application/vnd.jupyter.widget-view+json":{"version_major":2,"version_minor":0,"model_id":"1e602ad3e4d24416b5cab28e618a1d24"}},"metadata":{}},{"output_type":"display_data","data":{"text/plain":"  0%|          | 0/1 [00:00<?, ?it/s]","application/vnd.jupyter.widget-view+json":{"version_major":2,"version_minor":0,"model_id":"e343643467a34921aaddd2f5030c50e9"}},"metadata":{}},{"output_type":"display_data","data":{"text/plain":"  0%|          | 0/2 [00:00<?, ?it/s]","application/vnd.jupyter.widget-view+json":{"version_major":2,"version_minor":0,"model_id":"8604eadfcd6d4e788f76dd6c23537d2f"}},"metadata":{}},{"output_type":"display_data","data":{"text/plain":"  0%|          | 0/1 [00:00<?, ?it/s]","application/vnd.jupyter.widget-view+json":{"version_major":2,"version_minor":0,"model_id":"8d9dbe94eba5418584f869e0f6c4d7d1"}},"metadata":{}},{"output_type":"display_data","data":{"text/plain":"  0%|          | 0/2 [00:00<?, ?it/s]","application/vnd.jupyter.widget-view+json":{"version_major":2,"version_minor":0,"model_id":"e9c13f2ce9c2484f85e37dfb960338d9"}},"metadata":{}},{"output_type":"display_data","data":{"text/plain":"  0%|          | 0/1 [00:00<?, ?it/s]","application/vnd.jupyter.widget-view+json":{"version_major":2,"version_minor":0,"model_id":"8dd4cc47740b4fb0a415ad14ea8fdc9f"}},"metadata":{}},{"output_type":"display_data","data":{"text/plain":"  0%|          | 0/2 [00:00<?, ?it/s]","application/vnd.jupyter.widget-view+json":{"version_major":2,"version_minor":0,"model_id":"138a1194727241b2ae045947989f153e"}},"metadata":{}},{"output_type":"display_data","data":{"text/plain":"  0%|          | 0/1 [00:00<?, ?it/s]","application/vnd.jupyter.widget-view+json":{"version_major":2,"version_minor":0,"model_id":"2f0dfbb6c6cb42c1ae035690452ea20f"}},"metadata":{}},{"output_type":"display_data","data":{"text/plain":"  0%|          | 0/2 [00:00<?, ?it/s]","application/vnd.jupyter.widget-view+json":{"version_major":2,"version_minor":0,"model_id":"afe210623b0b417eb45bb1e621248533"}},"metadata":{}},{"output_type":"display_data","data":{"text/plain":"  0%|          | 0/1 [00:00<?, ?it/s]","application/vnd.jupyter.widget-view+json":{"version_major":2,"version_minor":0,"model_id":"afaf147e50264bd9b196b9b7dcd4df78"}},"metadata":{}},{"output_type":"display_data","data":{"text/plain":"  0%|          | 0/2 [00:00<?, ?it/s]","application/vnd.jupyter.widget-view+json":{"version_major":2,"version_minor":0,"model_id":"9e0c3b41f43b4ed091dab0a33f9a0050"}},"metadata":{}},{"output_type":"display_data","data":{"text/plain":"  0%|          | 0/1 [00:00<?, ?it/s]","application/vnd.jupyter.widget-view+json":{"version_major":2,"version_minor":0,"model_id":"741dac1149c14410aa78d258b1cafeeb"}},"metadata":{}},{"output_type":"display_data","data":{"text/plain":"  0%|          | 0/2 [00:00<?, ?it/s]","application/vnd.jupyter.widget-view+json":{"version_major":2,"version_minor":0,"model_id":"73e42751ab6e400db442d49a7562a93d"}},"metadata":{}},{"output_type":"display_data","data":{"text/plain":"  0%|          | 0/1 [00:00<?, ?it/s]","application/vnd.jupyter.widget-view+json":{"version_major":2,"version_minor":0,"model_id":"68ae4b271a3e4533b2cdfe4820478ea4"}},"metadata":{}},{"output_type":"display_data","data":{"text/plain":"  0%|          | 0/2 [00:00<?, ?it/s]","application/vnd.jupyter.widget-view+json":{"version_major":2,"version_minor":0,"model_id":"9e977b32afab4ad0af9b7c537fd92360"}},"metadata":{}},{"output_type":"display_data","data":{"text/plain":"  0%|          | 0/1 [00:00<?, ?it/s]","application/vnd.jupyter.widget-view+json":{"version_major":2,"version_minor":0,"model_id":"11b5ed1c453848b49714c3efb9db7249"}},"metadata":{}},{"output_type":"display_data","data":{"text/plain":"  0%|          | 0/2 [00:00<?, ?it/s]","application/vnd.jupyter.widget-view+json":{"version_major":2,"version_minor":0,"model_id":"0dfac9ea998641a684fee94d01e6a487"}},"metadata":{}},{"output_type":"display_data","data":{"text/plain":"  0%|          | 0/1 [00:00<?, ?it/s]","application/vnd.jupyter.widget-view+json":{"version_major":2,"version_minor":0,"model_id":"0523c87ff305486fa051fa3508f47b18"}},"metadata":{}},{"output_type":"display_data","data":{"text/plain":"  0%|          | 0/2 [00:00<?, ?it/s]","application/vnd.jupyter.widget-view+json":{"version_major":2,"version_minor":0,"model_id":"f37f0dced85941ffac0788313fd4a762"}},"metadata":{}},{"output_type":"display_data","data":{"text/plain":"  0%|          | 0/1 [00:00<?, ?it/s]","application/vnd.jupyter.widget-view+json":{"version_major":2,"version_minor":0,"model_id":"1bb7778ae1c14d79b27c2ad632e0e1e5"}},"metadata":{}},{"output_type":"display_data","data":{"text/plain":"  0%|          | 0/2 [00:00<?, ?it/s]","application/vnd.jupyter.widget-view+json":{"version_major":2,"version_minor":0,"model_id":"6752940e1d6d4fd1928a9977061557f7"}},"metadata":{}},{"output_type":"display_data","data":{"text/plain":"  0%|          | 0/1 [00:00<?, ?it/s]","application/vnd.jupyter.widget-view+json":{"version_major":2,"version_minor":0,"model_id":"2f9b0f82f3484fcb8451e7270440356f"}},"metadata":{}},{"output_type":"display_data","data":{"text/plain":"  0%|          | 0/2 [00:00<?, ?it/s]","application/vnd.jupyter.widget-view+json":{"version_major":2,"version_minor":0,"model_id":"728933e142814af6842fc7c6e11ac76c"}},"metadata":{}},{"output_type":"display_data","data":{"text/plain":"  0%|          | 0/1 [00:00<?, ?it/s]","application/vnd.jupyter.widget-view+json":{"version_major":2,"version_minor":0,"model_id":"fc03b0f5a85444759cb987499460982f"}},"metadata":{}},{"output_type":"display_data","data":{"text/plain":"  0%|          | 0/2 [00:00<?, ?it/s]","application/vnd.jupyter.widget-view+json":{"version_major":2,"version_minor":0,"model_id":"7e7f7f3c1085407c9b141acb020711a3"}},"metadata":{}},{"output_type":"display_data","data":{"text/plain":"  0%|          | 0/1 [00:00<?, ?it/s]","application/vnd.jupyter.widget-view+json":{"version_major":2,"version_minor":0,"model_id":"7aa167591c8347b0ad5a8aa285179368"}},"metadata":{}},{"output_type":"display_data","data":{"text/plain":"  0%|          | 0/2 [00:00<?, ?it/s]","application/vnd.jupyter.widget-view+json":{"version_major":2,"version_minor":0,"model_id":"af846581de9f4e3b97d7a9dfff2e4bfa"}},"metadata":{}},{"output_type":"display_data","data":{"text/plain":"  0%|          | 0/1 [00:00<?, ?it/s]","application/vnd.jupyter.widget-view+json":{"version_major":2,"version_minor":0,"model_id":"9371e6e587ae43d6a1941ae1381e4f46"}},"metadata":{}},{"output_type":"display_data","data":{"text/plain":"  0%|          | 0/2 [00:00<?, ?it/s]","application/vnd.jupyter.widget-view+json":{"version_major":2,"version_minor":0,"model_id":"cab72c777c474798917a51aa004b6645"}},"metadata":{}},{"output_type":"display_data","data":{"text/plain":"  0%|          | 0/1 [00:00<?, ?it/s]","application/vnd.jupyter.widget-view+json":{"version_major":2,"version_minor":0,"model_id":"a4e65a4b3f2e4be08254c9b654b264ab"}},"metadata":{}},{"output_type":"display_data","data":{"text/plain":"  0%|          | 0/2 [00:00<?, ?it/s]","application/vnd.jupyter.widget-view+json":{"version_major":2,"version_minor":0,"model_id":"79775004800e4ccea98bcc3fecc41ca5"}},"metadata":{}},{"output_type":"display_data","data":{"text/plain":"  0%|          | 0/1 [00:00<?, ?it/s]","application/vnd.jupyter.widget-view+json":{"version_major":2,"version_minor":0,"model_id":"984c130a2ac94b34b74358a907ce4d4a"}},"metadata":{}},{"output_type":"display_data","data":{"text/plain":"  0%|          | 0/2 [00:00<?, ?it/s]","application/vnd.jupyter.widget-view+json":{"version_major":2,"version_minor":0,"model_id":"c639bdce45204fce86733350a2dcb64d"}},"metadata":{}},{"output_type":"display_data","data":{"text/plain":"  0%|          | 0/1 [00:00<?, ?it/s]","application/vnd.jupyter.widget-view+json":{"version_major":2,"version_minor":0,"model_id":"0510becb9b3e4429a1a02151d9a344e1"}},"metadata":{}},{"output_type":"display_data","data":{"text/plain":"  0%|          | 0/2 [00:00<?, ?it/s]","application/vnd.jupyter.widget-view+json":{"version_major":2,"version_minor":0,"model_id":"efadf288e48948368b293c83f2e006b6"}},"metadata":{}},{"output_type":"display_data","data":{"text/plain":"  0%|          | 0/1 [00:00<?, ?it/s]","application/vnd.jupyter.widget-view+json":{"version_major":2,"version_minor":0,"model_id":"396244bddb504a3183ab7ebf44d9f7a2"}},"metadata":{}},{"output_type":"display_data","data":{"text/plain":"  0%|          | 0/2 [00:00<?, ?it/s]","application/vnd.jupyter.widget-view+json":{"version_major":2,"version_minor":0,"model_id":"d6dac5bbb8d94f5abf9d37661fbd2da7"}},"metadata":{}},{"output_type":"display_data","data":{"text/plain":"  0%|          | 0/1 [00:00<?, ?it/s]","application/vnd.jupyter.widget-view+json":{"version_major":2,"version_minor":0,"model_id":"11f065f144c54f7da35eaf52a430dc1c"}},"metadata":{}},{"output_type":"display_data","data":{"text/plain":"  0%|          | 0/2 [00:00<?, ?it/s]","application/vnd.jupyter.widget-view+json":{"version_major":2,"version_minor":0,"model_id":"99e231055bfa4c2a9746891d4cd71816"}},"metadata":{}},{"output_type":"display_data","data":{"text/plain":"  0%|          | 0/1 [00:00<?, ?it/s]","application/vnd.jupyter.widget-view+json":{"version_major":2,"version_minor":0,"model_id":"9c8bd1402bbe40ffaa287e82f3bfa869"}},"metadata":{}},{"output_type":"display_data","data":{"text/plain":"  0%|          | 0/2 [00:00<?, ?it/s]","application/vnd.jupyter.widget-view+json":{"version_major":2,"version_minor":0,"model_id":"7de5959fe8614776a4b3e26fb5a2d604"}},"metadata":{}},{"output_type":"display_data","data":{"text/plain":"  0%|          | 0/1 [00:00<?, ?it/s]","application/vnd.jupyter.widget-view+json":{"version_major":2,"version_minor":0,"model_id":"64c41e593ba340c59cfbb887ee4d146b"}},"metadata":{}},{"output_type":"display_data","data":{"text/plain":"  0%|          | 0/2 [00:00<?, ?it/s]","application/vnd.jupyter.widget-view+json":{"version_major":2,"version_minor":0,"model_id":"21ca21a6b1794a91921da03d04990ce1"}},"metadata":{}},{"output_type":"display_data","data":{"text/plain":"  0%|          | 0/1 [00:00<?, ?it/s]","application/vnd.jupyter.widget-view+json":{"version_major":2,"version_minor":0,"model_id":"9cf1b0b60b4b438691a011e31f21d413"}},"metadata":{}},{"output_type":"display_data","data":{"text/plain":"  0%|          | 0/2 [00:00<?, ?it/s]","application/vnd.jupyter.widget-view+json":{"version_major":2,"version_minor":0,"model_id":"930513125b9947499a5ad2b3dff8e93b"}},"metadata":{}},{"output_type":"display_data","data":{"text/plain":"  0%|          | 0/1 [00:00<?, ?it/s]","application/vnd.jupyter.widget-view+json":{"version_major":2,"version_minor":0,"model_id":"a6bb353853b34342a91248b00877c0f6"}},"metadata":{}},{"output_type":"display_data","data":{"text/plain":"  0%|          | 0/2 [00:00<?, ?it/s]","application/vnd.jupyter.widget-view+json":{"version_major":2,"version_minor":0,"model_id":"d08137a249924d0d83976eb4397e86f6"}},"metadata":{}},{"output_type":"display_data","data":{"text/plain":"  0%|          | 0/1 [00:00<?, ?it/s]","application/vnd.jupyter.widget-view+json":{"version_major":2,"version_minor":0,"model_id":"f9d9d2476b084a1b826d9892a53a878e"}},"metadata":{}},{"output_type":"display_data","data":{"text/plain":"  0%|          | 0/2 [00:00<?, ?it/s]","application/vnd.jupyter.widget-view+json":{"version_major":2,"version_minor":0,"model_id":"51597589814c4939b06c4e603799baf7"}},"metadata":{}},{"output_type":"display_data","data":{"text/plain":"  0%|          | 0/1 [00:00<?, ?it/s]","application/vnd.jupyter.widget-view+json":{"version_major":2,"version_minor":0,"model_id":"a08f17c41f634334a7a8096ea81dd67a"}},"metadata":{}},{"output_type":"display_data","data":{"text/plain":"  0%|          | 0/2 [00:00<?, ?it/s]","application/vnd.jupyter.widget-view+json":{"version_major":2,"version_minor":0,"model_id":"5a8c0b0027b842eabba6bad2085294da"}},"metadata":{}},{"output_type":"display_data","data":{"text/plain":"  0%|          | 0/1 [00:00<?, ?it/s]","application/vnd.jupyter.widget-view+json":{"version_major":2,"version_minor":0,"model_id":"bbb21de69c454fb0a29f87e86c14377f"}},"metadata":{}},{"output_type":"display_data","data":{"text/plain":"  0%|          | 0/2 [00:00<?, ?it/s]","application/vnd.jupyter.widget-view+json":{"version_major":2,"version_minor":0,"model_id":"608e3da37e504f938febc4fd5f2ed87c"}},"metadata":{}},{"output_type":"display_data","data":{"text/plain":"  0%|          | 0/1 [00:00<?, ?it/s]","application/vnd.jupyter.widget-view+json":{"version_major":2,"version_minor":0,"model_id":"e3aa6bd7acfa4e7e8ee4b01b543e1d12"}},"metadata":{}},{"output_type":"display_data","data":{"text/plain":"  0%|          | 0/2 [00:00<?, ?it/s]","application/vnd.jupyter.widget-view+json":{"version_major":2,"version_minor":0,"model_id":"c3543aadb10f4265a1fffda1d5c3438a"}},"metadata":{}},{"output_type":"display_data","data":{"text/plain":"  0%|          | 0/1 [00:00<?, ?it/s]","application/vnd.jupyter.widget-view+json":{"version_major":2,"version_minor":0,"model_id":"e651b7702261408ca19368746c28bf4c"}},"metadata":{}},{"output_type":"display_data","data":{"text/plain":"  0%|          | 0/2 [00:00<?, ?it/s]","application/vnd.jupyter.widget-view+json":{"version_major":2,"version_minor":0,"model_id":"722dca8c95bd4b188de1a21518178340"}},"metadata":{}},{"output_type":"display_data","data":{"text/plain":"  0%|          | 0/1 [00:00<?, ?it/s]","application/vnd.jupyter.widget-view+json":{"version_major":2,"version_minor":0,"model_id":"1117a761f56c4e04b9c7755349ab5020"}},"metadata":{}},{"output_type":"display_data","data":{"text/plain":"  0%|          | 0/2 [00:00<?, ?it/s]","application/vnd.jupyter.widget-view+json":{"version_major":2,"version_minor":0,"model_id":"a60b964677004b208d2e7a8612682bd4"}},"metadata":{}},{"output_type":"display_data","data":{"text/plain":"  0%|          | 0/1 [00:00<?, ?it/s]","application/vnd.jupyter.widget-view+json":{"version_major":2,"version_minor":0,"model_id":"f4a375313f9040f89cd97e73f2017beb"}},"metadata":{}},{"output_type":"display_data","data":{"text/plain":"  0%|          | 0/2 [00:00<?, ?it/s]","application/vnd.jupyter.widget-view+json":{"version_major":2,"version_minor":0,"model_id":"be413e962b8744c79968f6d8a99099b8"}},"metadata":{}},{"name":"stdout","text":"\nEnsemble score = 0.4472 after adding model weightedmean2last_fgm_512 with weight 0.290. Increase of 0.0005\n\n","output_type":"stream"},{"output_type":"display_data","data":{"text/plain":"  0%|          | 0/1 [00:00<?, ?it/s]","application/vnd.jupyter.widget-view+json":{"version_major":2,"version_minor":0,"model_id":"c53ef28a00df4c1ba71a48151bba89ab"}},"metadata":{}},{"output_type":"display_data","data":{"text/plain":"  0%|          | 0/3 [00:00<?, ?it/s]","application/vnd.jupyter.widget-view+json":{"version_major":2,"version_minor":0,"model_id":"cfb2197fecf44302840d7917a5913bac"}},"metadata":{}},{"output_type":"display_data","data":{"text/plain":"  0%|          | 0/1 [00:00<?, ?it/s]","application/vnd.jupyter.widget-view+json":{"version_major":2,"version_minor":0,"model_id":"0c5a7099ade64aecaee972172418c245"}},"metadata":{}},{"output_type":"display_data","data":{"text/plain":"  0%|          | 0/3 [00:00<?, ?it/s]","application/vnd.jupyter.widget-view+json":{"version_major":2,"version_minor":0,"model_id":"9378ecb08b5e4b508d7581cee1fbc52e"}},"metadata":{}},{"output_type":"display_data","data":{"text/plain":"  0%|          | 0/1 [00:00<?, ?it/s]","application/vnd.jupyter.widget-view+json":{"version_major":2,"version_minor":0,"model_id":"2b8e9f479c2c42d296550144fe269b8b"}},"metadata":{}},{"output_type":"display_data","data":{"text/plain":"  0%|          | 0/3 [00:00<?, ?it/s]","application/vnd.jupyter.widget-view+json":{"version_major":2,"version_minor":0,"model_id":"92ebcb2ec4914fc4a75bafa9951525b8"}},"metadata":{}},{"output_type":"display_data","data":{"text/plain":"  0%|          | 0/1 [00:00<?, ?it/s]","application/vnd.jupyter.widget-view+json":{"version_major":2,"version_minor":0,"model_id":"e586682276924e68b241196e1bc86e16"}},"metadata":{}},{"output_type":"display_data","data":{"text/plain":"  0%|          | 0/3 [00:00<?, ?it/s]","application/vnd.jupyter.widget-view+json":{"version_major":2,"version_minor":0,"model_id":"74b35beb444e4f4d9966061c9a2332cf"}},"metadata":{}},{"output_type":"display_data","data":{"text/plain":"  0%|          | 0/1 [00:00<?, ?it/s]","application/vnd.jupyter.widget-view+json":{"version_major":2,"version_minor":0,"model_id":"0cdd547669604262b1028b21fac76d52"}},"metadata":{}},{"output_type":"display_data","data":{"text/plain":"  0%|          | 0/3 [00:00<?, ?it/s]","application/vnd.jupyter.widget-view+json":{"version_major":2,"version_minor":0,"model_id":"1485214e5dce4629a48083e06ede61d2"}},"metadata":{}},{"output_type":"display_data","data":{"text/plain":"  0%|          | 0/1 [00:00<?, ?it/s]","application/vnd.jupyter.widget-view+json":{"version_major":2,"version_minor":0,"model_id":"c4c1b30e3cf048d99788eede9d704ba5"}},"metadata":{}},{"output_type":"display_data","data":{"text/plain":"  0%|          | 0/3 [00:00<?, ?it/s]","application/vnd.jupyter.widget-view+json":{"version_major":2,"version_minor":0,"model_id":"92f7d53c6a544bdbb088c6011fb06b04"}},"metadata":{}},{"output_type":"display_data","data":{"text/plain":"  0%|          | 0/1 [00:00<?, ?it/s]","application/vnd.jupyter.widget-view+json":{"version_major":2,"version_minor":0,"model_id":"568c91bd93aa4c51bd63a9ed41827b56"}},"metadata":{}},{"output_type":"display_data","data":{"text/plain":"  0%|          | 0/3 [00:00<?, ?it/s]","application/vnd.jupyter.widget-view+json":{"version_major":2,"version_minor":0,"model_id":"c33340c55eb94b689d7d322985d76ee8"}},"metadata":{}},{"output_type":"display_data","data":{"text/plain":"  0%|          | 0/1 [00:00<?, ?it/s]","application/vnd.jupyter.widget-view+json":{"version_major":2,"version_minor":0,"model_id":"4b63bcc9ff1c40da8e5668a473fe7df4"}},"metadata":{}},{"output_type":"display_data","data":{"text/plain":"  0%|          | 0/3 [00:00<?, ?it/s]","application/vnd.jupyter.widget-view+json":{"version_major":2,"version_minor":0,"model_id":"6afb28ecf0ff48ad9b3fd207993fa277"}},"metadata":{}},{"output_type":"display_data","data":{"text/plain":"  0%|          | 0/1 [00:00<?, ?it/s]","application/vnd.jupyter.widget-view+json":{"version_major":2,"version_minor":0,"model_id":"c745e158e446497f94051d8b7c613b9b"}},"metadata":{}},{"output_type":"display_data","data":{"text/plain":"  0%|          | 0/3 [00:00<?, ?it/s]","application/vnd.jupyter.widget-view+json":{"version_major":2,"version_minor":0,"model_id":"2c3becad167d49ada23a92c2388c06f9"}},"metadata":{}},{"output_type":"display_data","data":{"text/plain":"  0%|          | 0/1 [00:00<?, ?it/s]","application/vnd.jupyter.widget-view+json":{"version_major":2,"version_minor":0,"model_id":"ab901d46f32f434899804977e2151cb0"}},"metadata":{}},{"output_type":"display_data","data":{"text/plain":"  0%|          | 0/3 [00:00<?, ?it/s]","application/vnd.jupyter.widget-view+json":{"version_major":2,"version_minor":0,"model_id":"f554a5db1c474c71acd71ab4c4a4dbe0"}},"metadata":{}},{"output_type":"display_data","data":{"text/plain":"  0%|          | 0/1 [00:00<?, ?it/s]","application/vnd.jupyter.widget-view+json":{"version_major":2,"version_minor":0,"model_id":"8f1dfe514c1943a79bfc336a56861c4e"}},"metadata":{}},{"output_type":"display_data","data":{"text/plain":"  0%|          | 0/3 [00:00<?, ?it/s]","application/vnd.jupyter.widget-view+json":{"version_major":2,"version_minor":0,"model_id":"86b55cd41d0f4d73989a047fa2c40172"}},"metadata":{}},{"output_type":"display_data","data":{"text/plain":"  0%|          | 0/1 [00:00<?, ?it/s]","application/vnd.jupyter.widget-view+json":{"version_major":2,"version_minor":0,"model_id":"26088ec95a3243a59cbdd474753a7f9c"}},"metadata":{}},{"output_type":"display_data","data":{"text/plain":"  0%|          | 0/3 [00:00<?, ?it/s]","application/vnd.jupyter.widget-view+json":{"version_major":2,"version_minor":0,"model_id":"f1816aeb35ef45adab3bccd157ca9f10"}},"metadata":{}},{"output_type":"display_data","data":{"text/plain":"  0%|          | 0/1 [00:00<?, ?it/s]","application/vnd.jupyter.widget-view+json":{"version_major":2,"version_minor":0,"model_id":"375866231b804083a34db59d3833a343"}},"metadata":{}},{"output_type":"display_data","data":{"text/plain":"  0%|          | 0/3 [00:00<?, ?it/s]","application/vnd.jupyter.widget-view+json":{"version_major":2,"version_minor":0,"model_id":"6d68b08772fa479084cc4a5ac9d865cd"}},"metadata":{}},{"output_type":"display_data","data":{"text/plain":"  0%|          | 0/1 [00:00<?, ?it/s]","application/vnd.jupyter.widget-view+json":{"version_major":2,"version_minor":0,"model_id":"0a50db24eb5649bbb5eca36b9ea4bec4"}},"metadata":{}},{"output_type":"display_data","data":{"text/plain":"  0%|          | 0/3 [00:00<?, ?it/s]","application/vnd.jupyter.widget-view+json":{"version_major":2,"version_minor":0,"model_id":"e00e79e0bfd84f42b030bc36296c123c"}},"metadata":{}},{"output_type":"display_data","data":{"text/plain":"  0%|          | 0/1 [00:00<?, ?it/s]","application/vnd.jupyter.widget-view+json":{"version_major":2,"version_minor":0,"model_id":"3a2356602eaf4741ad20da7ef2450016"}},"metadata":{}},{"output_type":"display_data","data":{"text/plain":"  0%|          | 0/3 [00:00<?, ?it/s]","application/vnd.jupyter.widget-view+json":{"version_major":2,"version_minor":0,"model_id":"7615728309d84f7985c8d7ac8c5e5ad5"}},"metadata":{}},{"output_type":"display_data","data":{"text/plain":"  0%|          | 0/1 [00:00<?, ?it/s]","application/vnd.jupyter.widget-view+json":{"version_major":2,"version_minor":0,"model_id":"928db4e2fee24560b068b4cef5e553af"}},"metadata":{}},{"output_type":"display_data","data":{"text/plain":"  0%|          | 0/3 [00:00<?, ?it/s]","application/vnd.jupyter.widget-view+json":{"version_major":2,"version_minor":0,"model_id":"6bd66c25f3844c1f840a7853a41483f4"}},"metadata":{}},{"output_type":"display_data","data":{"text/plain":"  0%|          | 0/1 [00:00<?, ?it/s]","application/vnd.jupyter.widget-view+json":{"version_major":2,"version_minor":0,"model_id":"52c55006d9294f499c864072ad0548eb"}},"metadata":{}},{"output_type":"display_data","data":{"text/plain":"  0%|          | 0/3 [00:00<?, ?it/s]","application/vnd.jupyter.widget-view+json":{"version_major":2,"version_minor":0,"model_id":"67b42c6487064fc5ab055a6f6025b7d1"}},"metadata":{}},{"output_type":"display_data","data":{"text/plain":"  0%|          | 0/1 [00:00<?, ?it/s]","application/vnd.jupyter.widget-view+json":{"version_major":2,"version_minor":0,"model_id":"50c327117b07450ba23e552dda4ea290"}},"metadata":{}},{"output_type":"display_data","data":{"text/plain":"  0%|          | 0/3 [00:00<?, ?it/s]","application/vnd.jupyter.widget-view+json":{"version_major":2,"version_minor":0,"model_id":"456001e35a3b444a833469cfe84d21de"}},"metadata":{}},{"output_type":"display_data","data":{"text/plain":"  0%|          | 0/1 [00:00<?, ?it/s]","application/vnd.jupyter.widget-view+json":{"version_major":2,"version_minor":0,"model_id":"0e6c83ea847a4d19a6e2f9412f8133ba"}},"metadata":{}},{"output_type":"display_data","data":{"text/plain":"  0%|          | 0/3 [00:00<?, ?it/s]","application/vnd.jupyter.widget-view+json":{"version_major":2,"version_minor":0,"model_id":"c403120d70b84d75871da21ef5118b14"}},"metadata":{}},{"output_type":"display_data","data":{"text/plain":"  0%|          | 0/1 [00:00<?, ?it/s]","application/vnd.jupyter.widget-view+json":{"version_major":2,"version_minor":0,"model_id":"ff12436e42f14b19b2fb7c9a69467d36"}},"metadata":{}},{"output_type":"display_data","data":{"text/plain":"  0%|          | 0/3 [00:00<?, ?it/s]","application/vnd.jupyter.widget-view+json":{"version_major":2,"version_minor":0,"model_id":"b3d312e3d19043f2b95179ced86c861c"}},"metadata":{}},{"output_type":"display_data","data":{"text/plain":"  0%|          | 0/1 [00:00<?, ?it/s]","application/vnd.jupyter.widget-view+json":{"version_major":2,"version_minor":0,"model_id":"5dae5397298141f0a79c39ea2f7818a0"}},"metadata":{}},{"output_type":"display_data","data":{"text/plain":"  0%|          | 0/3 [00:00<?, ?it/s]","application/vnd.jupyter.widget-view+json":{"version_major":2,"version_minor":0,"model_id":"c258f54eb4514e11876cff7c0f813daf"}},"metadata":{}},{"output_type":"display_data","data":{"text/plain":"  0%|          | 0/1 [00:00<?, ?it/s]","application/vnd.jupyter.widget-view+json":{"version_major":2,"version_minor":0,"model_id":"81f844f735d242ceb2cda532426ff008"}},"metadata":{}},{"output_type":"display_data","data":{"text/plain":"  0%|          | 0/3 [00:00<?, ?it/s]","application/vnd.jupyter.widget-view+json":{"version_major":2,"version_minor":0,"model_id":"22fd4bc4485e48b0ba8f3b9beae8469e"}},"metadata":{}},{"output_type":"display_data","data":{"text/plain":"  0%|          | 0/1 [00:00<?, ?it/s]","application/vnd.jupyter.widget-view+json":{"version_major":2,"version_minor":0,"model_id":"fb58a000122843a1a8379ce387a89647"}},"metadata":{}},{"output_type":"display_data","data":{"text/plain":"  0%|          | 0/3 [00:00<?, ?it/s]","application/vnd.jupyter.widget-view+json":{"version_major":2,"version_minor":0,"model_id":"bc90c24360284b1a8149fd903aae732d"}},"metadata":{}},{"output_type":"display_data","data":{"text/plain":"  0%|          | 0/1 [00:00<?, ?it/s]","application/vnd.jupyter.widget-view+json":{"version_major":2,"version_minor":0,"model_id":"b5089925f9f44190959fb9eab506f456"}},"metadata":{}},{"output_type":"display_data","data":{"text/plain":"  0%|          | 0/3 [00:00<?, ?it/s]","application/vnd.jupyter.widget-view+json":{"version_major":2,"version_minor":0,"model_id":"e31e1888af824da0b5448fc00e1fb53c"}},"metadata":{}},{"output_type":"display_data","data":{"text/plain":"  0%|          | 0/1 [00:00<?, ?it/s]","application/vnd.jupyter.widget-view+json":{"version_major":2,"version_minor":0,"model_id":"bcc7ed9ca3694480b9ab82a5881b96ff"}},"metadata":{}},{"output_type":"display_data","data":{"text/plain":"  0%|          | 0/3 [00:00<?, ?it/s]","application/vnd.jupyter.widget-view+json":{"version_major":2,"version_minor":0,"model_id":"86fe873fbcb44c5db1a26aa955aa031a"}},"metadata":{}},{"output_type":"display_data","data":{"text/plain":"  0%|          | 0/1 [00:00<?, ?it/s]","application/vnd.jupyter.widget-view+json":{"version_major":2,"version_minor":0,"model_id":"df10989dd6b74ac7b9744dba5a26a9e1"}},"metadata":{}},{"output_type":"display_data","data":{"text/plain":"  0%|          | 0/3 [00:00<?, ?it/s]","application/vnd.jupyter.widget-view+json":{"version_major":2,"version_minor":0,"model_id":"c88e5dd627ec4331a90c1220b9e311fa"}},"metadata":{}},{"output_type":"display_data","data":{"text/plain":"  0%|          | 0/1 [00:00<?, ?it/s]","application/vnd.jupyter.widget-view+json":{"version_major":2,"version_minor":0,"model_id":"eec7570b60fa47fc9031cf7d14fb3207"}},"metadata":{}},{"output_type":"display_data","data":{"text/plain":"  0%|          | 0/3 [00:00<?, ?it/s]","application/vnd.jupyter.widget-view+json":{"version_major":2,"version_minor":0,"model_id":"e3da493bc4404932b05b03c4f0d11ee8"}},"metadata":{}},{"name":"stdout","text":"\nEnsemble score = 0.4471 after adding model attention_fgm_512 with weight 0.160. Increase of 0.0001\n\n","output_type":"stream"},{"output_type":"display_data","data":{"text/plain":"  0%|          | 0/1 [00:00<?, ?it/s]","application/vnd.jupyter.widget-view+json":{"version_major":2,"version_minor":0,"model_id":"a1a0663711854d7890e192d28842ab60"}},"metadata":{}},{"output_type":"display_data","data":{"text/plain":"  0%|          | 0/4 [00:00<?, ?it/s]","application/vnd.jupyter.widget-view+json":{"version_major":2,"version_minor":0,"model_id":"633c0a24fb3c4383ad36759118343f4e"}},"metadata":{}},{"output_type":"display_data","data":{"text/plain":"  0%|          | 0/1 [00:00<?, ?it/s]","application/vnd.jupyter.widget-view+json":{"version_major":2,"version_minor":0,"model_id":"e6de69bc828c4885a1e5fc2882c59f6c"}},"metadata":{}},{"output_type":"display_data","data":{"text/plain":"  0%|          | 0/4 [00:00<?, ?it/s]","application/vnd.jupyter.widget-view+json":{"version_major":2,"version_minor":0,"model_id":"667eaa23ac524c1ebe99ab36ee6b1c87"}},"metadata":{}},{"output_type":"display_data","data":{"text/plain":"  0%|          | 0/1 [00:00<?, ?it/s]","application/vnd.jupyter.widget-view+json":{"version_major":2,"version_minor":0,"model_id":"d80a9ae83ce04566bbaf76f67a8c27d4"}},"metadata":{}},{"output_type":"display_data","data":{"text/plain":"  0%|          | 0/4 [00:00<?, ?it/s]","application/vnd.jupyter.widget-view+json":{"version_major":2,"version_minor":0,"model_id":"e986092d62f5421394da1a2802bdeba5"}},"metadata":{}},{"output_type":"display_data","data":{"text/plain":"  0%|          | 0/1 [00:00<?, ?it/s]","application/vnd.jupyter.widget-view+json":{"version_major":2,"version_minor":0,"model_id":"6da0ae804b1b4809a60ffda1ac145500"}},"metadata":{}},{"output_type":"display_data","data":{"text/plain":"  0%|          | 0/4 [00:00<?, ?it/s]","application/vnd.jupyter.widget-view+json":{"version_major":2,"version_minor":0,"model_id":"2e7d7a1aeff240aa946d8f8d2fcf1af8"}},"metadata":{}},{"output_type":"display_data","data":{"text/plain":"  0%|          | 0/1 [00:00<?, ?it/s]","application/vnd.jupyter.widget-view+json":{"version_major":2,"version_minor":0,"model_id":"ea41f3c4115c4636b0251086aa19f54a"}},"metadata":{}},{"output_type":"display_data","data":{"text/plain":"  0%|          | 0/4 [00:00<?, ?it/s]","application/vnd.jupyter.widget-view+json":{"version_major":2,"version_minor":0,"model_id":"1bf8e2388cb84029be4f573e9d26ae9a"}},"metadata":{}},{"output_type":"display_data","data":{"text/plain":"  0%|          | 0/1 [00:00<?, ?it/s]","application/vnd.jupyter.widget-view+json":{"version_major":2,"version_minor":0,"model_id":"f0b0058b5c4d414bae2879d9952cd414"}},"metadata":{}},{"output_type":"display_data","data":{"text/plain":"  0%|          | 0/4 [00:00<?, ?it/s]","application/vnd.jupyter.widget-view+json":{"version_major":2,"version_minor":0,"model_id":"5a19f8e0582443078237e030cdbbfa34"}},"metadata":{}},{"output_type":"display_data","data":{"text/plain":"  0%|          | 0/1 [00:00<?, ?it/s]","application/vnd.jupyter.widget-view+json":{"version_major":2,"version_minor":0,"model_id":"9e662cbe2d5843af9440089ff733c69a"}},"metadata":{}},{"output_type":"display_data","data":{"text/plain":"  0%|          | 0/4 [00:00<?, ?it/s]","application/vnd.jupyter.widget-view+json":{"version_major":2,"version_minor":0,"model_id":"025627e6a4654e46aff6aa7f0c89fd1b"}},"metadata":{}},{"output_type":"display_data","data":{"text/plain":"  0%|          | 0/1 [00:00<?, ?it/s]","application/vnd.jupyter.widget-view+json":{"version_major":2,"version_minor":0,"model_id":"62fa8107ceb74ae2a6183ee252053778"}},"metadata":{}},{"output_type":"display_data","data":{"text/plain":"  0%|          | 0/4 [00:00<?, ?it/s]","application/vnd.jupyter.widget-view+json":{"version_major":2,"version_minor":0,"model_id":"2efa9aeed71947379e80736d0cf43280"}},"metadata":{}},{"output_type":"display_data","data":{"text/plain":"  0%|          | 0/1 [00:00<?, ?it/s]","application/vnd.jupyter.widget-view+json":{"version_major":2,"version_minor":0,"model_id":"a405fe407a7c4d5cad1eb3af9bc9c2a2"}},"metadata":{}},{"output_type":"display_data","data":{"text/plain":"  0%|          | 0/4 [00:00<?, ?it/s]","application/vnd.jupyter.widget-view+json":{"version_major":2,"version_minor":0,"model_id":"c88405228d95452aa891952ecd4a1245"}},"metadata":{}},{"output_type":"display_data","data":{"text/plain":"  0%|          | 0/1 [00:00<?, ?it/s]","application/vnd.jupyter.widget-view+json":{"version_major":2,"version_minor":0,"model_id":"0076c10e448b4778b7a80181947d8966"}},"metadata":{}},{"output_type":"display_data","data":{"text/plain":"  0%|          | 0/4 [00:00<?, ?it/s]","application/vnd.jupyter.widget-view+json":{"version_major":2,"version_minor":0,"model_id":"f8724673dfcf47478ebb56484676d588"}},"metadata":{}},{"output_type":"display_data","data":{"text/plain":"  0%|          | 0/1 [00:00<?, ?it/s]","application/vnd.jupyter.widget-view+json":{"version_major":2,"version_minor":0,"model_id":"8a00028933f84dfba086a041f0718889"}},"metadata":{}},{"output_type":"display_data","data":{"text/plain":"  0%|          | 0/4 [00:00<?, ?it/s]","application/vnd.jupyter.widget-view+json":{"version_major":2,"version_minor":0,"model_id":"20dabcea11704fe29363f48f85b42fb1"}},"metadata":{}},{"output_type":"display_data","data":{"text/plain":"  0%|          | 0/1 [00:00<?, ?it/s]","application/vnd.jupyter.widget-view+json":{"version_major":2,"version_minor":0,"model_id":"1c771d156d9f43c3a22f8f6d55a4d4ab"}},"metadata":{}},{"output_type":"display_data","data":{"text/plain":"  0%|          | 0/4 [00:00<?, ?it/s]","application/vnd.jupyter.widget-view+json":{"version_major":2,"version_minor":0,"model_id":"3b0561d53c45497db8005384fc16e78e"}},"metadata":{}},{"output_type":"display_data","data":{"text/plain":"  0%|          | 0/1 [00:00<?, ?it/s]","application/vnd.jupyter.widget-view+json":{"version_major":2,"version_minor":0,"model_id":"3ca5076c1bdb40749ba40c063d6b748d"}},"metadata":{}},{"output_type":"display_data","data":{"text/plain":"  0%|          | 0/4 [00:00<?, ?it/s]","application/vnd.jupyter.widget-view+json":{"version_major":2,"version_minor":0,"model_id":"e9383ad4869f4dba9a92bb203175abaf"}},"metadata":{}},{"output_type":"display_data","data":{"text/plain":"  0%|          | 0/1 [00:00<?, ?it/s]","application/vnd.jupyter.widget-view+json":{"version_major":2,"version_minor":0,"model_id":"cc8241b52f674ae6981ca574c65e1a4d"}},"metadata":{}},{"output_type":"display_data","data":{"text/plain":"  0%|          | 0/4 [00:00<?, ?it/s]","application/vnd.jupyter.widget-view+json":{"version_major":2,"version_minor":0,"model_id":"3bd1604ec9b9462d8b5483a47d4050d1"}},"metadata":{}},{"output_type":"display_data","data":{"text/plain":"  0%|          | 0/1 [00:00<?, ?it/s]","application/vnd.jupyter.widget-view+json":{"version_major":2,"version_minor":0,"model_id":"af1d7def4123492698b117d657a0dbcf"}},"metadata":{}},{"output_type":"display_data","data":{"text/plain":"  0%|          | 0/4 [00:00<?, ?it/s]","application/vnd.jupyter.widget-view+json":{"version_major":2,"version_minor":0,"model_id":"1824cd17cbd94b9aba9948caa322af15"}},"metadata":{}},{"output_type":"display_data","data":{"text/plain":"  0%|          | 0/1 [00:00<?, ?it/s]","application/vnd.jupyter.widget-view+json":{"version_major":2,"version_minor":0,"model_id":"f29589339e70406987b326017232fddc"}},"metadata":{}},{"output_type":"display_data","data":{"text/plain":"  0%|          | 0/4 [00:00<?, ?it/s]","application/vnd.jupyter.widget-view+json":{"version_major":2,"version_minor":0,"model_id":"fe83a10a53914e0aade1ae10ceb24853"}},"metadata":{}},{"output_type":"display_data","data":{"text/plain":"  0%|          | 0/1 [00:00<?, ?it/s]","application/vnd.jupyter.widget-view+json":{"version_major":2,"version_minor":0,"model_id":"3952d81dce1b40018dc28e2d4bdfa759"}},"metadata":{}},{"output_type":"display_data","data":{"text/plain":"  0%|          | 0/4 [00:00<?, ?it/s]","application/vnd.jupyter.widget-view+json":{"version_major":2,"version_minor":0,"model_id":"63f0b98c5c3d4a7c87353a2db3700f62"}},"metadata":{}},{"output_type":"display_data","data":{"text/plain":"  0%|          | 0/1 [00:00<?, ?it/s]","application/vnd.jupyter.widget-view+json":{"version_major":2,"version_minor":0,"model_id":"a8965b319d1a4ada83ec0874f919ef2d"}},"metadata":{}},{"output_type":"display_data","data":{"text/plain":"  0%|          | 0/4 [00:00<?, ?it/s]","application/vnd.jupyter.widget-view+json":{"version_major":2,"version_minor":0,"model_id":"5b89d61775fa498f86ba3aec91014e6b"}},"metadata":{}},{"output_type":"display_data","data":{"text/plain":"  0%|          | 0/1 [00:00<?, ?it/s]","application/vnd.jupyter.widget-view+json":{"version_major":2,"version_minor":0,"model_id":"ccce27c147604154aaa8bbe0b29e4359"}},"metadata":{}},{"output_type":"display_data","data":{"text/plain":"  0%|          | 0/4 [00:00<?, ?it/s]","application/vnd.jupyter.widget-view+json":{"version_major":2,"version_minor":0,"model_id":"5c15ae7b18a5459d8638125de5db5e15"}},"metadata":{}},{"output_type":"display_data","data":{"text/plain":"  0%|          | 0/1 [00:00<?, ?it/s]","application/vnd.jupyter.widget-view+json":{"version_major":2,"version_minor":0,"model_id":"af23f8a379c049098f7ed80e8f676921"}},"metadata":{}},{"output_type":"display_data","data":{"text/plain":"  0%|          | 0/4 [00:00<?, ?it/s]","application/vnd.jupyter.widget-view+json":{"version_major":2,"version_minor":0,"model_id":"033d58fa50244e0fb02677e2e1ef74ad"}},"metadata":{}},{"output_type":"display_data","data":{"text/plain":"  0%|          | 0/1 [00:00<?, ?it/s]","application/vnd.jupyter.widget-view+json":{"version_major":2,"version_minor":0,"model_id":"33d79af63690485a81b797f3d06e178a"}},"metadata":{}},{"output_type":"display_data","data":{"text/plain":"  0%|          | 0/4 [00:00<?, ?it/s]","application/vnd.jupyter.widget-view+json":{"version_major":2,"version_minor":0,"model_id":"414fe401be154477938815c9f74edae1"}},"metadata":{}},{"output_type":"display_data","data":{"text/plain":"  0%|          | 0/1 [00:00<?, ?it/s]","application/vnd.jupyter.widget-view+json":{"version_major":2,"version_minor":0,"model_id":"94c59d0da3244e1983fba08268e4a519"}},"metadata":{}},{"output_type":"display_data","data":{"text/plain":"  0%|          | 0/4 [00:00<?, ?it/s]","application/vnd.jupyter.widget-view+json":{"version_major":2,"version_minor":0,"model_id":"c12e9ada65be44fcb67c1b4628e110ba"}},"metadata":{}},{"output_type":"display_data","data":{"text/plain":"  0%|          | 0/1 [00:00<?, ?it/s]","application/vnd.jupyter.widget-view+json":{"version_major":2,"version_minor":0,"model_id":"d9ab815fa9214858bc153649c3717067"}},"metadata":{}},{"output_type":"display_data","data":{"text/plain":"  0%|          | 0/4 [00:00<?, ?it/s]","application/vnd.jupyter.widget-view+json":{"version_major":2,"version_minor":0,"model_id":"ef29bf703a9c4ca99461243ba5eec668"}},"metadata":{}},{"output_type":"display_data","data":{"text/plain":"  0%|          | 0/1 [00:00<?, ?it/s]","application/vnd.jupyter.widget-view+json":{"version_major":2,"version_minor":0,"model_id":"4d77a58ac717433ab99bafe671b3f93b"}},"metadata":{}},{"output_type":"display_data","data":{"text/plain":"  0%|          | 0/4 [00:00<?, ?it/s]","application/vnd.jupyter.widget-view+json":{"version_major":2,"version_minor":0,"model_id":"d914574da8fd41469cc6bad414e9f82f"}},"metadata":{}},{"output_type":"display_data","data":{"text/plain":"  0%|          | 0/1 [00:00<?, ?it/s]","application/vnd.jupyter.widget-view+json":{"version_major":2,"version_minor":0,"model_id":"f45b9e65d6304e0381607b4381e765f7"}},"metadata":{}},{"output_type":"display_data","data":{"text/plain":"  0%|          | 0/4 [00:00<?, ?it/s]","application/vnd.jupyter.widget-view+json":{"version_major":2,"version_minor":0,"model_id":"bb91f87c617243818a4c08f4281abc67"}},"metadata":{}},{"output_type":"display_data","data":{"text/plain":"  0%|          | 0/1 [00:00<?, ?it/s]","application/vnd.jupyter.widget-view+json":{"version_major":2,"version_minor":0,"model_id":"bcff3d5189d24e0192519f77fcf4b5f4"}},"metadata":{}},{"output_type":"display_data","data":{"text/plain":"  0%|          | 0/4 [00:00<?, ?it/s]","application/vnd.jupyter.widget-view+json":{"version_major":2,"version_minor":0,"model_id":"01515b3961f14d0386f07aa8d98b5f01"}},"metadata":{}},{"output_type":"display_data","data":{"text/plain":"  0%|          | 0/1 [00:00<?, ?it/s]","application/vnd.jupyter.widget-view+json":{"version_major":2,"version_minor":0,"model_id":"840d3b18cde04009b1e15e9cd943cd9c"}},"metadata":{}},{"output_type":"display_data","data":{"text/plain":"  0%|          | 0/4 [00:00<?, ?it/s]","application/vnd.jupyter.widget-view+json":{"version_major":2,"version_minor":0,"model_id":"a9b09ae2d7f64762b206b31c449da5f2"}},"metadata":{}},{"output_type":"display_data","data":{"text/plain":"  0%|          | 0/1 [00:00<?, ?it/s]","application/vnd.jupyter.widget-view+json":{"version_major":2,"version_minor":0,"model_id":"0f13b838d62a4a84af39b27055e8c586"}},"metadata":{}},{"output_type":"display_data","data":{"text/plain":"  0%|          | 0/4 [00:00<?, ?it/s]","application/vnd.jupyter.widget-view+json":{"version_major":2,"version_minor":0,"model_id":"7917759d67f24afab125a6b80cec3747"}},"metadata":{}},{"name":"stdout","text":"\nEnsemble score = 0.4468 after adding model attention_fgm_768 with weight 0.170. Increase of 0.0003\n\n","output_type":"stream"},{"output_type":"display_data","data":{"text/plain":"  0%|          | 0/1 [00:00<?, ?it/s]","application/vnd.jupyter.widget-view+json":{"version_major":2,"version_minor":0,"model_id":"9191caddbe0e4423956507d6d531b835"}},"metadata":{}},{"output_type":"display_data","data":{"text/plain":"  0%|          | 0/5 [00:00<?, ?it/s]","application/vnd.jupyter.widget-view+json":{"version_major":2,"version_minor":0,"model_id":"9d2d15c03ae8422195ae97e8df6e4528"}},"metadata":{}},{"output_type":"display_data","data":{"text/plain":"  0%|          | 0/1 [00:00<?, ?it/s]","application/vnd.jupyter.widget-view+json":{"version_major":2,"version_minor":0,"model_id":"cdafc2960c234ceab62ee8e5f4ccd686"}},"metadata":{}},{"output_type":"display_data","data":{"text/plain":"  0%|          | 0/5 [00:00<?, ?it/s]","application/vnd.jupyter.widget-view+json":{"version_major":2,"version_minor":0,"model_id":"808aed1ce8a0454ba83765bdb0cb602d"}},"metadata":{}},{"output_type":"display_data","data":{"text/plain":"  0%|          | 0/1 [00:00<?, ?it/s]","application/vnd.jupyter.widget-view+json":{"version_major":2,"version_minor":0,"model_id":"f001bb489f104fec8e5a5da917c73551"}},"metadata":{}},{"output_type":"display_data","data":{"text/plain":"  0%|          | 0/5 [00:00<?, ?it/s]","application/vnd.jupyter.widget-view+json":{"version_major":2,"version_minor":0,"model_id":"0c5190b72b5c44b69787e0f4aabac8f2"}},"metadata":{}},{"output_type":"display_data","data":{"text/plain":"  0%|          | 0/1 [00:00<?, ?it/s]","application/vnd.jupyter.widget-view+json":{"version_major":2,"version_minor":0,"model_id":"c1f739f7053546529ed3a8f1360853fe"}},"metadata":{}},{"output_type":"display_data","data":{"text/plain":"  0%|          | 0/5 [00:00<?, ?it/s]","application/vnd.jupyter.widget-view+json":{"version_major":2,"version_minor":0,"model_id":"8fb5a1548c714eb883bc19c9f03451f0"}},"metadata":{}},{"output_type":"display_data","data":{"text/plain":"  0%|          | 0/1 [00:00<?, ?it/s]","application/vnd.jupyter.widget-view+json":{"version_major":2,"version_minor":0,"model_id":"0a375fbb2cf34f0db53b0a0a81cb2e8c"}},"metadata":{}},{"output_type":"display_data","data":{"text/plain":"  0%|          | 0/5 [00:00<?, ?it/s]","application/vnd.jupyter.widget-view+json":{"version_major":2,"version_minor":0,"model_id":"a4783d8ce1d04c2da0d40561e57fcbad"}},"metadata":{}},{"output_type":"display_data","data":{"text/plain":"  0%|          | 0/1 [00:00<?, ?it/s]","application/vnd.jupyter.widget-view+json":{"version_major":2,"version_minor":0,"model_id":"fdff340070194c79adfc00a46e05107f"}},"metadata":{}},{"output_type":"display_data","data":{"text/plain":"  0%|          | 0/5 [00:00<?, ?it/s]","application/vnd.jupyter.widget-view+json":{"version_major":2,"version_minor":0,"model_id":"86203f37110f4ffd928a87e458319369"}},"metadata":{}},{"output_type":"display_data","data":{"text/plain":"  0%|          | 0/1 [00:00<?, ?it/s]","application/vnd.jupyter.widget-view+json":{"version_major":2,"version_minor":0,"model_id":"99aebd44a53340c49b5f044164625e1a"}},"metadata":{}},{"output_type":"display_data","data":{"text/plain":"  0%|          | 0/5 [00:00<?, ?it/s]","application/vnd.jupyter.widget-view+json":{"version_major":2,"version_minor":0,"model_id":"1c0db390f53e4677aab78a0c5d1b9969"}},"metadata":{}},{"output_type":"display_data","data":{"text/plain":"  0%|          | 0/1 [00:00<?, ?it/s]","application/vnd.jupyter.widget-view+json":{"version_major":2,"version_minor":0,"model_id":"45d86a5e4427436393847421dcd71204"}},"metadata":{}},{"output_type":"display_data","data":{"text/plain":"  0%|          | 0/5 [00:00<?, ?it/s]","application/vnd.jupyter.widget-view+json":{"version_major":2,"version_minor":0,"model_id":"2eea84d0a0a94595ba8a2560e6ae71d2"}},"metadata":{}},{"output_type":"display_data","data":{"text/plain":"  0%|          | 0/1 [00:00<?, ?it/s]","application/vnd.jupyter.widget-view+json":{"version_major":2,"version_minor":0,"model_id":"9f6616c16bfb48a0899330068f6379d3"}},"metadata":{}},{"output_type":"display_data","data":{"text/plain":"  0%|          | 0/5 [00:00<?, ?it/s]","application/vnd.jupyter.widget-view+json":{"version_major":2,"version_minor":0,"model_id":"b351a253f1b74c8b86e26ee8678f839c"}},"metadata":{}},{"output_type":"display_data","data":{"text/plain":"  0%|          | 0/1 [00:00<?, ?it/s]","application/vnd.jupyter.widget-view+json":{"version_major":2,"version_minor":0,"model_id":"7d1106a8e4bb4b8e9d4b034923c07a93"}},"metadata":{}},{"output_type":"display_data","data":{"text/plain":"  0%|          | 0/5 [00:00<?, ?it/s]","application/vnd.jupyter.widget-view+json":{"version_major":2,"version_minor":0,"model_id":"1e0cc1571e954ed19df8cbd5fcb59819"}},"metadata":{}},{"output_type":"display_data","data":{"text/plain":"  0%|          | 0/1 [00:00<?, ?it/s]","application/vnd.jupyter.widget-view+json":{"version_major":2,"version_minor":0,"model_id":"1b2f19ac55244ea3b0c37271571cf0d6"}},"metadata":{}},{"output_type":"display_data","data":{"text/plain":"  0%|          | 0/5 [00:00<?, ?it/s]","application/vnd.jupyter.widget-view+json":{"version_major":2,"version_minor":0,"model_id":"53c2623abbf34504940f0a34c40bd7e6"}},"metadata":{}},{"output_type":"display_data","data":{"text/plain":"  0%|          | 0/1 [00:00<?, ?it/s]","application/vnd.jupyter.widget-view+json":{"version_major":2,"version_minor":0,"model_id":"c0ffcfb5333a47b2b103488a759b997a"}},"metadata":{}},{"output_type":"display_data","data":{"text/plain":"  0%|          | 0/5 [00:00<?, ?it/s]","application/vnd.jupyter.widget-view+json":{"version_major":2,"version_minor":0,"model_id":"3bd1fac4d7064140bdb8ba43646bb146"}},"metadata":{}},{"name":"stdout","text":"\nEnsemble score = 0.4469 after adding model attention_fgm with weight 0.010. Increase of -0.0001\n\n","output_type":"stream"},{"output_type":"display_data","data":{"text/plain":"  0%|          | 0/1 [00:00<?, ?it/s]","application/vnd.jupyter.widget-view+json":{"version_major":2,"version_minor":0,"model_id":"a6a388fa504c4b139182da3649c093b6"}},"metadata":{}},{"output_type":"display_data","data":{"text/plain":"  0%|          | 0/5 [00:00<?, ?it/s]","application/vnd.jupyter.widget-view+json":{"version_major":2,"version_minor":0,"model_id":"e5e88f0afa50427c8a9846ee8dc8da75"}},"metadata":{}},{"output_type":"display_data","data":{"text/plain":"  0%|          | 0/1 [00:00<?, ?it/s]","application/vnd.jupyter.widget-view+json":{"version_major":2,"version_minor":0,"model_id":"6ec1e782931a43a99ed58953f4b7d03e"}},"metadata":{}},{"output_type":"display_data","data":{"text/plain":"  0%|          | 0/5 [00:00<?, ?it/s]","application/vnd.jupyter.widget-view+json":{"version_major":2,"version_minor":0,"model_id":"c22d3aafbe644bfea485c9e382d73f4c"}},"metadata":{}},{"output_type":"display_data","data":{"text/plain":"  0%|          | 0/1 [00:00<?, ?it/s]","application/vnd.jupyter.widget-view+json":{"version_major":2,"version_minor":0,"model_id":"fa41a84ac915487d957843df899399c2"}},"metadata":{}},{"output_type":"display_data","data":{"text/plain":"  0%|          | 0/5 [00:00<?, ?it/s]","application/vnd.jupyter.widget-view+json":{"version_major":2,"version_minor":0,"model_id":"284fc4dfc5424432940fc637b086b9a0"}},"metadata":{}},{"output_type":"display_data","data":{"text/plain":"  0%|          | 0/1 [00:00<?, ?it/s]","application/vnd.jupyter.widget-view+json":{"version_major":2,"version_minor":0,"model_id":"5125681b2d624a7e86ceb3943d7a82ad"}},"metadata":{}},{"output_type":"display_data","data":{"text/plain":"  0%|          | 0/5 [00:00<?, ?it/s]","application/vnd.jupyter.widget-view+json":{"version_major":2,"version_minor":0,"model_id":"015d325909564be3876992f68c514547"}},"metadata":{}},{"output_type":"display_data","data":{"text/plain":"  0%|          | 0/1 [00:00<?, ?it/s]","application/vnd.jupyter.widget-view+json":{"version_major":2,"version_minor":0,"model_id":"56c8bd9cef794b7db7b2cb00830ee591"}},"metadata":{}},{"output_type":"display_data","data":{"text/plain":"  0%|          | 0/5 [00:00<?, ?it/s]","application/vnd.jupyter.widget-view+json":{"version_major":2,"version_minor":0,"model_id":"5f296d81dad14e3e914b78228f14bf7f"}},"metadata":{}},{"output_type":"display_data","data":{"text/plain":"  0%|          | 0/1 [00:00<?, ?it/s]","application/vnd.jupyter.widget-view+json":{"version_major":2,"version_minor":0,"model_id":"768112bc2ac44b5ab6dc489baa1820e0"}},"metadata":{}},{"output_type":"display_data","data":{"text/plain":"  0%|          | 0/5 [00:00<?, ?it/s]","application/vnd.jupyter.widget-view+json":{"version_major":2,"version_minor":0,"model_id":"8bfe05679c4b4e8baa6ae3677a226cd0"}},"metadata":{}},{"output_type":"display_data","data":{"text/plain":"  0%|          | 0/1 [00:00<?, ?it/s]","application/vnd.jupyter.widget-view+json":{"version_major":2,"version_minor":0,"model_id":"4a1afc4c55494acf9cadc178e18896e1"}},"metadata":{}},{"output_type":"display_data","data":{"text/plain":"  0%|          | 0/5 [00:00<?, ?it/s]","application/vnd.jupyter.widget-view+json":{"version_major":2,"version_minor":0,"model_id":"fcf3552a51d04e19ae35dada5526dddc"}},"metadata":{}},{"output_type":"display_data","data":{"text/plain":"  0%|          | 0/1 [00:00<?, ?it/s]","application/vnd.jupyter.widget-view+json":{"version_major":2,"version_minor":0,"model_id":"169b232a22ab401aa8483212b5dfd6a7"}},"metadata":{}},{"output_type":"display_data","data":{"text/plain":"  0%|          | 0/5 [00:00<?, ?it/s]","application/vnd.jupyter.widget-view+json":{"version_major":2,"version_minor":0,"model_id":"19a003f80747405eb60b865296bc135e"}},"metadata":{}},{"output_type":"display_data","data":{"text/plain":"  0%|          | 0/1 [00:00<?, ?it/s]","application/vnd.jupyter.widget-view+json":{"version_major":2,"version_minor":0,"model_id":"a91fa611d93249ffbd578d077181311c"}},"metadata":{}},{"output_type":"display_data","data":{"text/plain":"  0%|          | 0/5 [00:00<?, ?it/s]","application/vnd.jupyter.widget-view+json":{"version_major":2,"version_minor":0,"model_id":"4676509c5a4742bb823b3c34c1a9bce5"}},"metadata":{}},{"output_type":"display_data","data":{"text/plain":"  0%|          | 0/1 [00:00<?, ?it/s]","application/vnd.jupyter.widget-view+json":{"version_major":2,"version_minor":0,"model_id":"11833e0c28604d4bad73ac2bdbd99fd3"}},"metadata":{}},{"output_type":"display_data","data":{"text/plain":"  0%|          | 0/5 [00:00<?, ?it/s]","application/vnd.jupyter.widget-view+json":{"version_major":2,"version_minor":0,"model_id":"92857a616e6f4768bb018758e8a88fe1"}},"metadata":{}},{"output_type":"display_data","data":{"text/plain":"  0%|          | 0/1 [00:00<?, ?it/s]","application/vnd.jupyter.widget-view+json":{"version_major":2,"version_minor":0,"model_id":"2de493e979e54be093dd132ab94a0148"}},"metadata":{}},{"output_type":"display_data","data":{"text/plain":"  0%|          | 0/5 [00:00<?, ?it/s]","application/vnd.jupyter.widget-view+json":{"version_major":2,"version_minor":0,"model_id":"16287d300dce4f5ca4fe86eeb05fddb3"}},"metadata":{}},{"output_type":"display_data","data":{"text/plain":"  0%|          | 0/1 [00:00<?, ?it/s]","application/vnd.jupyter.widget-view+json":{"version_major":2,"version_minor":0,"model_id":"d9eee1856554431d97eb82fd16f3d045"}},"metadata":{}},{"output_type":"display_data","data":{"text/plain":"  0%|          | 0/5 [00:00<?, ?it/s]","application/vnd.jupyter.widget-view+json":{"version_major":2,"version_minor":0,"model_id":"e47d59b4cf3f45f6ba8540aa52cf236f"}},"metadata":{}},{"output_type":"display_data","data":{"text/plain":"  0%|          | 0/1 [00:00<?, ?it/s]","application/vnd.jupyter.widget-view+json":{"version_major":2,"version_minor":0,"model_id":"fe2ac1646bcd4622a86af0a1b600749f"}},"metadata":{}},{"output_type":"display_data","data":{"text/plain":"  0%|          | 0/5 [00:00<?, ?it/s]","application/vnd.jupyter.widget-view+json":{"version_major":2,"version_minor":0,"model_id":"7565b23b17664ed79ea82e0fed0acd54"}},"metadata":{}},{"output_type":"display_data","data":{"text/plain":"  0%|          | 0/1 [00:00<?, ?it/s]","application/vnd.jupyter.widget-view+json":{"version_major":2,"version_minor":0,"model_id":"4d64c5c22693462db92311e32cc0a777"}},"metadata":{}},{"output_type":"display_data","data":{"text/plain":"  0%|          | 0/5 [00:00<?, ?it/s]","application/vnd.jupyter.widget-view+json":{"version_major":2,"version_minor":0,"model_id":"d09348633d9a4be08f3985be732bf8b7"}},"metadata":{}},{"output_type":"display_data","data":{"text/plain":"  0%|          | 0/1 [00:00<?, ?it/s]","application/vnd.jupyter.widget-view+json":{"version_major":2,"version_minor":0,"model_id":"80a17ffb694747f4b317a94ff8fd2c1f"}},"metadata":{}},{"output_type":"display_data","data":{"text/plain":"  0%|          | 0/5 [00:00<?, ?it/s]","application/vnd.jupyter.widget-view+json":{"version_major":2,"version_minor":0,"model_id":"1a3ffe5b1ff84711b36440672c7d76bd"}},"metadata":{}},{"output_type":"display_data","data":{"text/plain":"  0%|          | 0/1 [00:00<?, ?it/s]","application/vnd.jupyter.widget-view+json":{"version_major":2,"version_minor":0,"model_id":"1c5ae804a2a74d6db45df2aae1645aac"}},"metadata":{}},{"output_type":"display_data","data":{"text/plain":"  0%|          | 0/5 [00:00<?, ?it/s]","application/vnd.jupyter.widget-view+json":{"version_major":2,"version_minor":0,"model_id":"a9a0171a32bc476f90b76c7d54aba490"}},"metadata":{}},{"output_type":"display_data","data":{"text/plain":"  0%|          | 0/1 [00:00<?, ?it/s]","application/vnd.jupyter.widget-view+json":{"version_major":2,"version_minor":0,"model_id":"ca25e31ccef94109854e6634ee102feb"}},"metadata":{}},{"output_type":"display_data","data":{"text/plain":"  0%|          | 0/5 [00:00<?, ?it/s]","application/vnd.jupyter.widget-view+json":{"version_major":2,"version_minor":0,"model_id":"559bd40a97b144b6b3e0055fd91d4f86"}},"metadata":{}},{"output_type":"display_data","data":{"text/plain":"  0%|          | 0/1 [00:00<?, ?it/s]","application/vnd.jupyter.widget-view+json":{"version_major":2,"version_minor":0,"model_id":"2019a27cfa2642adbbce7071fcf9cce6"}},"metadata":{}},{"output_type":"display_data","data":{"text/plain":"  0%|          | 0/5 [00:00<?, ?it/s]","application/vnd.jupyter.widget-view+json":{"version_major":2,"version_minor":0,"model_id":"8e7fbf5633f5499b9e2abf54fc650996"}},"metadata":{}},{"output_type":"display_data","data":{"text/plain":"  0%|          | 0/1 [00:00<?, ?it/s]","application/vnd.jupyter.widget-view+json":{"version_major":2,"version_minor":0,"model_id":"e13373feef47477c9530411ed086a8b2"}},"metadata":{}},{"output_type":"display_data","data":{"text/plain":"  0%|          | 0/5 [00:00<?, ?it/s]","application/vnd.jupyter.widget-view+json":{"version_major":2,"version_minor":0,"model_id":"06e5522e322a4080a8a080c89e6e9a2c"}},"metadata":{}},{"output_type":"display_data","data":{"text/plain":"  0%|          | 0/1 [00:00<?, ?it/s]","application/vnd.jupyter.widget-view+json":{"version_major":2,"version_minor":0,"model_id":"55e291ed53a74c74a56a4e29edc73eeb"}},"metadata":{}},{"output_type":"display_data","data":{"text/plain":"  0%|          | 0/5 [00:00<?, ?it/s]","application/vnd.jupyter.widget-view+json":{"version_major":2,"version_minor":0,"model_id":"6da7916f717244e89cad6b0d59dd7b07"}},"metadata":{}},{"output_type":"display_data","data":{"text/plain":"  0%|          | 0/1 [00:00<?, ?it/s]","application/vnd.jupyter.widget-view+json":{"version_major":2,"version_minor":0,"model_id":"2f3bc4b26ff1456c953440a8042e0c0e"}},"metadata":{}},{"output_type":"display_data","data":{"text/plain":"  0%|          | 0/5 [00:00<?, ?it/s]","application/vnd.jupyter.widget-view+json":{"version_major":2,"version_minor":0,"model_id":"adfe54b8266442d2b37e1e5d39f6a7bb"}},"metadata":{}},{"output_type":"display_data","data":{"text/plain":"  0%|          | 0/1 [00:00<?, ?it/s]","application/vnd.jupyter.widget-view+json":{"version_major":2,"version_minor":0,"model_id":"e0d66b5f05ac46c783d3bf40cf1772ea"}},"metadata":{}},{"output_type":"display_data","data":{"text/plain":"  0%|          | 0/5 [00:00<?, ?it/s]","application/vnd.jupyter.widget-view+json":{"version_major":2,"version_minor":0,"model_id":"8c863e9453d94a7eabf1e985ea618003"}},"metadata":{}},{"output_type":"display_data","data":{"text/plain":"  0%|          | 0/1 [00:00<?, ?it/s]","application/vnd.jupyter.widget-view+json":{"version_major":2,"version_minor":0,"model_id":"73b4c00651c340e499f8d579ad009db0"}},"metadata":{}},{"output_type":"display_data","data":{"text/plain":"  0%|          | 0/5 [00:00<?, ?it/s]","application/vnd.jupyter.widget-view+json":{"version_major":2,"version_minor":0,"model_id":"acce082bc89a4c53b8d95a702ca1bd2a"}},"metadata":{}},{"output_type":"display_data","data":{"text/plain":"  0%|          | 0/1 [00:00<?, ?it/s]","application/vnd.jupyter.widget-view+json":{"version_major":2,"version_minor":0,"model_id":"55e2946bd6234f64ae092d6ca9fd476e"}},"metadata":{}},{"output_type":"display_data","data":{"text/plain":"  0%|          | 0/5 [00:00<?, ?it/s]","application/vnd.jupyter.widget-view+json":{"version_major":2,"version_minor":0,"model_id":"c9cb6740dc384d49a5a186930fbb86fc"}},"metadata":{}},{"output_type":"display_data","data":{"text/plain":"  0%|          | 0/1 [00:00<?, ?it/s]","application/vnd.jupyter.widget-view+json":{"version_major":2,"version_minor":0,"model_id":"d4f7603cefa447a6896108ffc6e159ec"}},"metadata":{}},{"output_type":"display_data","data":{"text/plain":"  0%|          | 0/5 [00:00<?, ?it/s]","application/vnd.jupyter.widget-view+json":{"version_major":2,"version_minor":0,"model_id":"b8836cc359d543a2a1d64ee8e0050ddb"}},"metadata":{}},{"name":"stdout","text":"\nEnsemble score = 0.4466 after adding model v21 with weight 0.140. Increase of 0.0002\n\n","output_type":"stream"},{"output_type":"display_data","data":{"text/plain":"  0%|          | 0/1 [00:00<?, ?it/s]","application/vnd.jupyter.widget-view+json":{"version_major":2,"version_minor":0,"model_id":"bfaef1346b36418f8148705606c13375"}},"metadata":{}},{"output_type":"display_data","data":{"text/plain":"  0%|          | 0/6 [00:00<?, ?it/s]","application/vnd.jupyter.widget-view+json":{"version_major":2,"version_minor":0,"model_id":"ca4d04cd02774642acc08057f8d6bef8"}},"metadata":{}},{"output_type":"display_data","data":{"text/plain":"  0%|          | 0/1 [00:00<?, ?it/s]","application/vnd.jupyter.widget-view+json":{"version_major":2,"version_minor":0,"model_id":"1051fbf21b464b23adbc612095210b25"}},"metadata":{}},{"output_type":"display_data","data":{"text/plain":"  0%|          | 0/6 [00:00<?, ?it/s]","application/vnd.jupyter.widget-view+json":{"version_major":2,"version_minor":0,"model_id":"2aa7e4eda9344b9699a14511046a4def"}},"metadata":{}},{"output_type":"display_data","data":{"text/plain":"  0%|          | 0/1 [00:00<?, ?it/s]","application/vnd.jupyter.widget-view+json":{"version_major":2,"version_minor":0,"model_id":"ed9a706987f34cff8b2be850f8c1d120"}},"metadata":{}},{"output_type":"display_data","data":{"text/plain":"  0%|          | 0/6 [00:00<?, ?it/s]","application/vnd.jupyter.widget-view+json":{"version_major":2,"version_minor":0,"model_id":"c0cd134c9e9a4daa82a6efef00303817"}},"metadata":{}},{"output_type":"display_data","data":{"text/plain":"  0%|          | 0/1 [00:00<?, ?it/s]","application/vnd.jupyter.widget-view+json":{"version_major":2,"version_minor":0,"model_id":"022fb1c209d34e4fbd136ce488378623"}},"metadata":{}},{"output_type":"display_data","data":{"text/plain":"  0%|          | 0/6 [00:00<?, ?it/s]","application/vnd.jupyter.widget-view+json":{"version_major":2,"version_minor":0,"model_id":"e95cd93a81494fc29375566a12ec00dc"}},"metadata":{}},{"output_type":"display_data","data":{"text/plain":"  0%|          | 0/1 [00:00<?, ?it/s]","application/vnd.jupyter.widget-view+json":{"version_major":2,"version_minor":0,"model_id":"b16b49149e1343aab64083a77be94b51"}},"metadata":{}},{"output_type":"display_data","data":{"text/plain":"  0%|          | 0/6 [00:00<?, ?it/s]","application/vnd.jupyter.widget-view+json":{"version_major":2,"version_minor":0,"model_id":"9580fd0cf956440b89a9787f7a63176e"}},"metadata":{}},{"output_type":"display_data","data":{"text/plain":"  0%|          | 0/1 [00:00<?, ?it/s]","application/vnd.jupyter.widget-view+json":{"version_major":2,"version_minor":0,"model_id":"1414be13b7244bbf870214fe439a2f86"}},"metadata":{}},{"output_type":"display_data","data":{"text/plain":"  0%|          | 0/6 [00:00<?, ?it/s]","application/vnd.jupyter.widget-view+json":{"version_major":2,"version_minor":0,"model_id":"678f90158e9d4559a83c67e36509bae2"}},"metadata":{}},{"output_type":"display_data","data":{"text/plain":"  0%|          | 0/1 [00:00<?, ?it/s]","application/vnd.jupyter.widget-view+json":{"version_major":2,"version_minor":0,"model_id":"247179a002a14a7c9d02beab7a3f8ef4"}},"metadata":{}},{"output_type":"display_data","data":{"text/plain":"  0%|          | 0/6 [00:00<?, ?it/s]","application/vnd.jupyter.widget-view+json":{"version_major":2,"version_minor":0,"model_id":"5f0185eb469e4d1ba342f2948fbe654a"}},"metadata":{}},{"output_type":"display_data","data":{"text/plain":"  0%|          | 0/1 [00:00<?, ?it/s]","application/vnd.jupyter.widget-view+json":{"version_major":2,"version_minor":0,"model_id":"6c609b6306d643cdb09277f1b5d7a752"}},"metadata":{}},{"output_type":"display_data","data":{"text/plain":"  0%|          | 0/6 [00:00<?, ?it/s]","application/vnd.jupyter.widget-view+json":{"version_major":2,"version_minor":0,"model_id":"aff86b2190f0431494c3d6005c95ac33"}},"metadata":{}},{"output_type":"display_data","data":{"text/plain":"  0%|          | 0/1 [00:00<?, ?it/s]","application/vnd.jupyter.widget-view+json":{"version_major":2,"version_minor":0,"model_id":"a504cc7236144217aa444bd2e169b2e0"}},"metadata":{}},{"output_type":"display_data","data":{"text/plain":"  0%|          | 0/6 [00:00<?, ?it/s]","application/vnd.jupyter.widget-view+json":{"version_major":2,"version_minor":0,"model_id":"ed1cc8578cd346ba90fb9aaf3ed53c02"}},"metadata":{}},{"output_type":"display_data","data":{"text/plain":"  0%|          | 0/1 [00:00<?, ?it/s]","application/vnd.jupyter.widget-view+json":{"version_major":2,"version_minor":0,"model_id":"04a9914b938b46d6ba00b42998ba92a1"}},"metadata":{}},{"output_type":"display_data","data":{"text/plain":"  0%|          | 0/6 [00:00<?, ?it/s]","application/vnd.jupyter.widget-view+json":{"version_major":2,"version_minor":0,"model_id":"e7645d53f83f4d3e8cfad855ce97deaf"}},"metadata":{}},{"output_type":"display_data","data":{"text/plain":"  0%|          | 0/1 [00:00<?, ?it/s]","application/vnd.jupyter.widget-view+json":{"version_major":2,"version_minor":0,"model_id":"6714a857b4b94845a0e13e2ef60cd6ae"}},"metadata":{}},{"output_type":"display_data","data":{"text/plain":"  0%|          | 0/6 [00:00<?, ?it/s]","application/vnd.jupyter.widget-view+json":{"version_major":2,"version_minor":0,"model_id":"d757d709679d4ace88b8e9519898fc5a"}},"metadata":{}},{"output_type":"display_data","data":{"text/plain":"  0%|          | 0/1 [00:00<?, ?it/s]","application/vnd.jupyter.widget-view+json":{"version_major":2,"version_minor":0,"model_id":"8dc4ae223b8142f890017fb63a439360"}},"metadata":{}},{"output_type":"display_data","data":{"text/plain":"  0%|          | 0/6 [00:00<?, ?it/s]","application/vnd.jupyter.widget-view+json":{"version_major":2,"version_minor":0,"model_id":"29b5611f35ff4bfbb3e8c427e8903e05"}},"metadata":{}},{"output_type":"display_data","data":{"text/plain":"  0%|          | 0/1 [00:00<?, ?it/s]","application/vnd.jupyter.widget-view+json":{"version_major":2,"version_minor":0,"model_id":"a02afad5d6d647c091dfb741dd89239f"}},"metadata":{}},{"output_type":"display_data","data":{"text/plain":"  0%|          | 0/6 [00:00<?, ?it/s]","application/vnd.jupyter.widget-view+json":{"version_major":2,"version_minor":0,"model_id":"b978677537e1495ebab0191da2bb233f"}},"metadata":{}},{"output_type":"display_data","data":{"text/plain":"  0%|          | 0/1 [00:00<?, ?it/s]","application/vnd.jupyter.widget-view+json":{"version_major":2,"version_minor":0,"model_id":"4c5f13b9d3f9410f96ff117048b01ef4"}},"metadata":{}},{"output_type":"display_data","data":{"text/plain":"  0%|          | 0/6 [00:00<?, ?it/s]","application/vnd.jupyter.widget-view+json":{"version_major":2,"version_minor":0,"model_id":"114a9d0912124bd18d86fbfb8df2d44d"}},"metadata":{}},{"output_type":"display_data","data":{"text/plain":"  0%|          | 0/1 [00:00<?, ?it/s]","application/vnd.jupyter.widget-view+json":{"version_major":2,"version_minor":0,"model_id":"b28a46e159694d919c65329d5c135e30"}},"metadata":{}},{"output_type":"display_data","data":{"text/plain":"  0%|          | 0/6 [00:00<?, ?it/s]","application/vnd.jupyter.widget-view+json":{"version_major":2,"version_minor":0,"model_id":"91cb70ab82f743a1bcb6d9575bcab285"}},"metadata":{}},{"output_type":"display_data","data":{"text/plain":"  0%|          | 0/1 [00:00<?, ?it/s]","application/vnd.jupyter.widget-view+json":{"version_major":2,"version_minor":0,"model_id":"e994ad83df9244f19989064f05437865"}},"metadata":{}},{"output_type":"display_data","data":{"text/plain":"  0%|          | 0/6 [00:00<?, ?it/s]","application/vnd.jupyter.widget-view+json":{"version_major":2,"version_minor":0,"model_id":"a75cebddfe8640c189b7c4578ccd7555"}},"metadata":{}},{"output_type":"display_data","data":{"text/plain":"  0%|          | 0/1 [00:00<?, ?it/s]","application/vnd.jupyter.widget-view+json":{"version_major":2,"version_minor":0,"model_id":"eb1fe5ec326b4021b3e996db2eb4747d"}},"metadata":{}},{"output_type":"display_data","data":{"text/plain":"  0%|          | 0/6 [00:00<?, ?it/s]","application/vnd.jupyter.widget-view+json":{"version_major":2,"version_minor":0,"model_id":"bce2319699f7451c8933f090f3f9f896"}},"metadata":{}},{"output_type":"display_data","data":{"text/plain":"  0%|          | 0/1 [00:00<?, ?it/s]","application/vnd.jupyter.widget-view+json":{"version_major":2,"version_minor":0,"model_id":"44cde045d22e4f5c811d12a1ce53c27e"}},"metadata":{}},{"output_type":"display_data","data":{"text/plain":"  0%|          | 0/6 [00:00<?, ?it/s]","application/vnd.jupyter.widget-view+json":{"version_major":2,"version_minor":0,"model_id":"8a01796ad25d460ebb9fd61f3574a211"}},"metadata":{}},{"output_type":"display_data","data":{"text/plain":"  0%|          | 0/1 [00:00<?, ?it/s]","application/vnd.jupyter.widget-view+json":{"version_major":2,"version_minor":0,"model_id":"9ef2996892ec455cab50214622120b3e"}},"metadata":{}},{"output_type":"display_data","data":{"text/plain":"  0%|          | 0/6 [00:00<?, ?it/s]","application/vnd.jupyter.widget-view+json":{"version_major":2,"version_minor":0,"model_id":"386659b8b1a84d7b93beb886b9606b41"}},"metadata":{}},{"output_type":"display_data","data":{"text/plain":"  0%|          | 0/1 [00:00<?, ?it/s]","application/vnd.jupyter.widget-view+json":{"version_major":2,"version_minor":0,"model_id":"d15b8382585d43a1a47312ee2f31f882"}},"metadata":{}},{"output_type":"display_data","data":{"text/plain":"  0%|          | 0/6 [00:00<?, ?it/s]","application/vnd.jupyter.widget-view+json":{"version_major":2,"version_minor":0,"model_id":"7c70a4e3de31478a92b638ce47e5ac45"}},"metadata":{}},{"output_type":"display_data","data":{"text/plain":"  0%|          | 0/1 [00:00<?, ?it/s]","application/vnd.jupyter.widget-view+json":{"version_major":2,"version_minor":0,"model_id":"69516dbf967e4192ae506d5242b7607c"}},"metadata":{}},{"output_type":"display_data","data":{"text/plain":"  0%|          | 0/6 [00:00<?, ?it/s]","application/vnd.jupyter.widget-view+json":{"version_major":2,"version_minor":0,"model_id":"8c24936412e14373a668c885e29caa59"}},"metadata":{}},{"output_type":"display_data","data":{"text/plain":"  0%|          | 0/1 [00:00<?, ?it/s]","application/vnd.jupyter.widget-view+json":{"version_major":2,"version_minor":0,"model_id":"f3562021f1144f4a90f37471101b6cee"}},"metadata":{}},{"output_type":"display_data","data":{"text/plain":"  0%|          | 0/6 [00:00<?, ?it/s]","application/vnd.jupyter.widget-view+json":{"version_major":2,"version_minor":0,"model_id":"d5941a70b77f4ef6bbf4566841c097c5"}},"metadata":{}},{"output_type":"display_data","data":{"text/plain":"  0%|          | 0/1 [00:00<?, ?it/s]","application/vnd.jupyter.widget-view+json":{"version_major":2,"version_minor":0,"model_id":"b11f9ce43feb4df69c4d7bfbcfaee7d8"}},"metadata":{}},{"output_type":"display_data","data":{"text/plain":"  0%|          | 0/6 [00:00<?, ?it/s]","application/vnd.jupyter.widget-view+json":{"version_major":2,"version_minor":0,"model_id":"139b50d2897b4d9fb5cd4858e03a74be"}},"metadata":{}},{"output_type":"display_data","data":{"text/plain":"  0%|          | 0/1 [00:00<?, ?it/s]","application/vnd.jupyter.widget-view+json":{"version_major":2,"version_minor":0,"model_id":"926f7e3d2fb946a0bd63775a73e0d82f"}},"metadata":{}},{"output_type":"display_data","data":{"text/plain":"  0%|          | 0/6 [00:00<?, ?it/s]","application/vnd.jupyter.widget-view+json":{"version_major":2,"version_minor":0,"model_id":"82bde38672c54eae8d33de2006a3554b"}},"metadata":{}},{"output_type":"display_data","data":{"text/plain":"  0%|          | 0/1 [00:00<?, ?it/s]","application/vnd.jupyter.widget-view+json":{"version_major":2,"version_minor":0,"model_id":"d2fee3d72f584f9487cf43da74a84133"}},"metadata":{}},{"output_type":"display_data","data":{"text/plain":"  0%|          | 0/6 [00:00<?, ?it/s]","application/vnd.jupyter.widget-view+json":{"version_major":2,"version_minor":0,"model_id":"1bcd060dfe8948f89978b08cfe3dbb95"}},"metadata":{}},{"output_type":"display_data","data":{"text/plain":"  0%|          | 0/1 [00:00<?, ?it/s]","application/vnd.jupyter.widget-view+json":{"version_major":2,"version_minor":0,"model_id":"ad1119ceac3f41c4adee8837f7726681"}},"metadata":{}},{"output_type":"display_data","data":{"text/plain":"  0%|          | 0/6 [00:00<?, ?it/s]","application/vnd.jupyter.widget-view+json":{"version_major":2,"version_minor":0,"model_id":"73c825a73c214cdb827e375b9bacc91b"}},"metadata":{}},{"output_type":"display_data","data":{"text/plain":"  0%|          | 0/1 [00:00<?, ?it/s]","application/vnd.jupyter.widget-view+json":{"version_major":2,"version_minor":0,"model_id":"e3ddacdfc50e42ceba25699d0019afa9"}},"metadata":{}},{"output_type":"display_data","data":{"text/plain":"  0%|          | 0/6 [00:00<?, ?it/s]","application/vnd.jupyter.widget-view+json":{"version_major":2,"version_minor":0,"model_id":"4b5c85ffffbe4e2f874ef75bd1351ab7"}},"metadata":{}},{"output_type":"display_data","data":{"text/plain":"  0%|          | 0/1 [00:00<?, ?it/s]","application/vnd.jupyter.widget-view+json":{"version_major":2,"version_minor":0,"model_id":"d86fc7857ec344b284a2f1d81a9835dc"}},"metadata":{}},{"output_type":"display_data","data":{"text/plain":"  0%|          | 0/6 [00:00<?, ?it/s]","application/vnd.jupyter.widget-view+json":{"version_major":2,"version_minor":0,"model_id":"2d8679fde5864df2bf0f27418f6e24b2"}},"metadata":{}},{"output_type":"display_data","data":{"text/plain":"  0%|          | 0/1 [00:00<?, ?it/s]","application/vnd.jupyter.widget-view+json":{"version_major":2,"version_minor":0,"model_id":"11f909e03bfa45f0aa900a1f3002f936"}},"metadata":{}},{"output_type":"display_data","data":{"text/plain":"  0%|          | 0/6 [00:00<?, ?it/s]","application/vnd.jupyter.widget-view+json":{"version_major":2,"version_minor":0,"model_id":"eef31dddc9da40f890c5c2502913328e"}},"metadata":{}},{"output_type":"display_data","data":{"text/plain":"  0%|          | 0/1 [00:00<?, ?it/s]","application/vnd.jupyter.widget-view+json":{"version_major":2,"version_minor":0,"model_id":"7a4b009caf5847afb6a6c291e149371e"}},"metadata":{}},{"output_type":"display_data","data":{"text/plain":"  0%|          | 0/6 [00:00<?, ?it/s]","application/vnd.jupyter.widget-view+json":{"version_major":2,"version_minor":0,"model_id":"8afdfbc1b5be4b9599ace34bce9c3e75"}},"metadata":{}},{"output_type":"display_data","data":{"text/plain":"  0%|          | 0/1 [00:00<?, ?it/s]","application/vnd.jupyter.widget-view+json":{"version_major":2,"version_minor":0,"model_id":"39331f8acb084cc0a3807ce1d993486d"}},"metadata":{}},{"output_type":"display_data","data":{"text/plain":"  0%|          | 0/6 [00:00<?, ?it/s]","application/vnd.jupyter.widget-view+json":{"version_major":2,"version_minor":0,"model_id":"b709b4b466844798b54be52a5130ddef"}},"metadata":{}},{"output_type":"display_data","data":{"text/plain":"  0%|          | 0/1 [00:00<?, ?it/s]","application/vnd.jupyter.widget-view+json":{"version_major":2,"version_minor":0,"model_id":"c7683539f8f547b48bafd7b03bb2ff83"}},"metadata":{}},{"output_type":"display_data","data":{"text/plain":"  0%|          | 0/6 [00:00<?, ?it/s]","application/vnd.jupyter.widget-view+json":{"version_major":2,"version_minor":0,"model_id":"70bd4a8fc5144458b4799eec88ede301"}},"metadata":{}},{"output_type":"display_data","data":{"text/plain":"  0%|          | 0/1 [00:00<?, ?it/s]","application/vnd.jupyter.widget-view+json":{"version_major":2,"version_minor":0,"model_id":"cb3d2f99c18e45439095a45d01d1c22a"}},"metadata":{}},{"output_type":"display_data","data":{"text/plain":"  0%|          | 0/6 [00:00<?, ?it/s]","application/vnd.jupyter.widget-view+json":{"version_major":2,"version_minor":0,"model_id":"e423106a111545e697172adb36872299"}},"metadata":{}},{"output_type":"display_data","data":{"text/plain":"  0%|          | 0/1 [00:00<?, ?it/s]","application/vnd.jupyter.widget-view+json":{"version_major":2,"version_minor":0,"model_id":"435a45a8db7a46a9a28e818caac9b1b8"}},"metadata":{}},{"output_type":"display_data","data":{"text/plain":"  0%|          | 0/6 [00:00<?, ?it/s]","application/vnd.jupyter.widget-view+json":{"version_major":2,"version_minor":0,"model_id":"e6bd5459223b4518a401cbff10cf0a57"}},"metadata":{}},{"name":"stdout","text":"\nEnsemble score = 0.4461 after adding model v112 with weight 0.230. Increase of 0.0006\n\n","output_type":"stream"},{"output_type":"display_data","data":{"text/plain":"  0%|          | 0/1 [00:00<?, ?it/s]","application/vnd.jupyter.widget-view+json":{"version_major":2,"version_minor":0,"model_id":"e441edcf585c406bac34da2f1844231f"}},"metadata":{}},{"output_type":"display_data","data":{"text/plain":"  0%|          | 0/7 [00:00<?, ?it/s]","application/vnd.jupyter.widget-view+json":{"version_major":2,"version_minor":0,"model_id":"6e017c86a8dc46c5a52a6f3a438cc180"}},"metadata":{}},{"output_type":"display_data","data":{"text/plain":"  0%|          | 0/1 [00:00<?, ?it/s]","application/vnd.jupyter.widget-view+json":{"version_major":2,"version_minor":0,"model_id":"7149a13289264dbd949a5eb1aeb95ac2"}},"metadata":{}},{"output_type":"display_data","data":{"text/plain":"  0%|          | 0/7 [00:00<?, ?it/s]","application/vnd.jupyter.widget-view+json":{"version_major":2,"version_minor":0,"model_id":"b32a8c9d7abd4aa38b6d740f3adc9b78"}},"metadata":{}},{"output_type":"display_data","data":{"text/plain":"  0%|          | 0/1 [00:00<?, ?it/s]","application/vnd.jupyter.widget-view+json":{"version_major":2,"version_minor":0,"model_id":"0cf55d0273b94f03897784e781355752"}},"metadata":{}},{"output_type":"display_data","data":{"text/plain":"  0%|          | 0/7 [00:00<?, ?it/s]","application/vnd.jupyter.widget-view+json":{"version_major":2,"version_minor":0,"model_id":"198a905c4d0241539606ba50bbf116be"}},"metadata":{}},{"output_type":"display_data","data":{"text/plain":"  0%|          | 0/1 [00:00<?, ?it/s]","application/vnd.jupyter.widget-view+json":{"version_major":2,"version_minor":0,"model_id":"40c28404cd58434e82f9591234393990"}},"metadata":{}},{"output_type":"display_data","data":{"text/plain":"  0%|          | 0/7 [00:00<?, ?it/s]","application/vnd.jupyter.widget-view+json":{"version_major":2,"version_minor":0,"model_id":"b2ef35ab36fa4d509e6e940e6a4e95fc"}},"metadata":{}},{"output_type":"display_data","data":{"text/plain":"  0%|          | 0/1 [00:00<?, ?it/s]","application/vnd.jupyter.widget-view+json":{"version_major":2,"version_minor":0,"model_id":"688d1effd7b84eb28f85645ad7549429"}},"metadata":{}},{"output_type":"display_data","data":{"text/plain":"  0%|          | 0/7 [00:00<?, ?it/s]","application/vnd.jupyter.widget-view+json":{"version_major":2,"version_minor":0,"model_id":"bd988b0ee7724fce98d981f3d5445165"}},"metadata":{}},{"output_type":"display_data","data":{"text/plain":"  0%|          | 0/1 [00:00<?, ?it/s]","application/vnd.jupyter.widget-view+json":{"version_major":2,"version_minor":0,"model_id":"f259134afc20406b811911f4b40187e2"}},"metadata":{}},{"output_type":"display_data","data":{"text/plain":"  0%|          | 0/7 [00:00<?, ?it/s]","application/vnd.jupyter.widget-view+json":{"version_major":2,"version_minor":0,"model_id":"dfe18abedd1d4384bf6842b60a3f86f7"}},"metadata":{}},{"output_type":"display_data","data":{"text/plain":"  0%|          | 0/1 [00:00<?, ?it/s]","application/vnd.jupyter.widget-view+json":{"version_major":2,"version_minor":0,"model_id":"ec34304af20e464da6f1ac9316810f78"}},"metadata":{}},{"output_type":"display_data","data":{"text/plain":"  0%|          | 0/7 [00:00<?, ?it/s]","application/vnd.jupyter.widget-view+json":{"version_major":2,"version_minor":0,"model_id":"401d66e7f1ef43b099f6db07e3d8c02c"}},"metadata":{}},{"output_type":"display_data","data":{"text/plain":"  0%|          | 0/1 [00:00<?, ?it/s]","application/vnd.jupyter.widget-view+json":{"version_major":2,"version_minor":0,"model_id":"0b37c6a482cf4f6c8f7424171e4b4768"}},"metadata":{}},{"output_type":"display_data","data":{"text/plain":"  0%|          | 0/7 [00:00<?, ?it/s]","application/vnd.jupyter.widget-view+json":{"version_major":2,"version_minor":0,"model_id":"7ef15fef620442d2adea348fdf5020a3"}},"metadata":{}},{"output_type":"display_data","data":{"text/plain":"  0%|          | 0/1 [00:00<?, ?it/s]","application/vnd.jupyter.widget-view+json":{"version_major":2,"version_minor":0,"model_id":"5ec8a10cfcd048bc99b6e515c9bfdcbc"}},"metadata":{}},{"output_type":"display_data","data":{"text/plain":"  0%|          | 0/7 [00:00<?, ?it/s]","application/vnd.jupyter.widget-view+json":{"version_major":2,"version_minor":0,"model_id":"97b94ec2d02849aab378f6cace5a7442"}},"metadata":{}},{"output_type":"display_data","data":{"text/plain":"  0%|          | 0/1 [00:00<?, ?it/s]","application/vnd.jupyter.widget-view+json":{"version_major":2,"version_minor":0,"model_id":"167ae996521243408f783c3302050f4a"}},"metadata":{}},{"output_type":"display_data","data":{"text/plain":"  0%|          | 0/7 [00:00<?, ?it/s]","application/vnd.jupyter.widget-view+json":{"version_major":2,"version_minor":0,"model_id":"8201e8343e8c438caafbf902a1331347"}},"metadata":{}},{"output_type":"display_data","data":{"text/plain":"  0%|          | 0/1 [00:00<?, ?it/s]","application/vnd.jupyter.widget-view+json":{"version_major":2,"version_minor":0,"model_id":"a69be5b5afe344c892537e308c969468"}},"metadata":{}},{"output_type":"display_data","data":{"text/plain":"  0%|          | 0/7 [00:00<?, ?it/s]","application/vnd.jupyter.widget-view+json":{"version_major":2,"version_minor":0,"model_id":"3941ee5041ac495b9c90eec1eab15c2f"}},"metadata":{}},{"output_type":"display_data","data":{"text/plain":"  0%|          | 0/1 [00:00<?, ?it/s]","application/vnd.jupyter.widget-view+json":{"version_major":2,"version_minor":0,"model_id":"1487d6a7bef146afb00568d2d276feef"}},"metadata":{}},{"output_type":"display_data","data":{"text/plain":"  0%|          | 0/7 [00:00<?, ?it/s]","application/vnd.jupyter.widget-view+json":{"version_major":2,"version_minor":0,"model_id":"e62efebf1e1f45a4ba25f3ae44ea3dea"}},"metadata":{}},{"output_type":"display_data","data":{"text/plain":"  0%|          | 0/1 [00:00<?, ?it/s]","application/vnd.jupyter.widget-view+json":{"version_major":2,"version_minor":0,"model_id":"d28f3495b0dd4d18a4d51b66d7837632"}},"metadata":{}},{"output_type":"display_data","data":{"text/plain":"  0%|          | 0/7 [00:00<?, ?it/s]","application/vnd.jupyter.widget-view+json":{"version_major":2,"version_minor":0,"model_id":"6a609b8f067f4a7abd9cf2b6818d4db9"}},"metadata":{}},{"output_type":"display_data","data":{"text/plain":"  0%|          | 0/1 [00:00<?, ?it/s]","application/vnd.jupyter.widget-view+json":{"version_major":2,"version_minor":0,"model_id":"7712d57d1c0a4814ba78c49446543c89"}},"metadata":{}},{"output_type":"display_data","data":{"text/plain":"  0%|          | 0/7 [00:00<?, ?it/s]","application/vnd.jupyter.widget-view+json":{"version_major":2,"version_minor":0,"model_id":"0bef1d2998c64a7989253e26f083ec42"}},"metadata":{}},{"output_type":"display_data","data":{"text/plain":"  0%|          | 0/1 [00:00<?, ?it/s]","application/vnd.jupyter.widget-view+json":{"version_major":2,"version_minor":0,"model_id":"568f2080b63c4c7cbec8a8be5692b027"}},"metadata":{}},{"output_type":"display_data","data":{"text/plain":"  0%|          | 0/7 [00:00<?, ?it/s]","application/vnd.jupyter.widget-view+json":{"version_major":2,"version_minor":0,"model_id":"36bf119fb6c047d8b11a13a31020236c"}},"metadata":{}},{"output_type":"display_data","data":{"text/plain":"  0%|          | 0/1 [00:00<?, ?it/s]","application/vnd.jupyter.widget-view+json":{"version_major":2,"version_minor":0,"model_id":"a8366d9c202e470e95a394f4581dc488"}},"metadata":{}},{"output_type":"display_data","data":{"text/plain":"  0%|          | 0/7 [00:00<?, ?it/s]","application/vnd.jupyter.widget-view+json":{"version_major":2,"version_minor":0,"model_id":"60bbe0bb42544ad19bc7d64f9ecd0522"}},"metadata":{}},{"output_type":"display_data","data":{"text/plain":"  0%|          | 0/1 [00:00<?, ?it/s]","application/vnd.jupyter.widget-view+json":{"version_major":2,"version_minor":0,"model_id":"9e4da5d4fbb543a1b28e3a6a73d54e0a"}},"metadata":{}},{"output_type":"display_data","data":{"text/plain":"  0%|          | 0/7 [00:00<?, ?it/s]","application/vnd.jupyter.widget-view+json":{"version_major":2,"version_minor":0,"model_id":"ea5b9bcdfb1b46c1910c89f69458c9c0"}},"metadata":{}},{"output_type":"display_data","data":{"text/plain":"  0%|          | 0/1 [00:00<?, ?it/s]","application/vnd.jupyter.widget-view+json":{"version_major":2,"version_minor":0,"model_id":"5e710971f5d7407784268ec3f0bcb59c"}},"metadata":{}},{"output_type":"display_data","data":{"text/plain":"  0%|          | 0/7 [00:00<?, ?it/s]","application/vnd.jupyter.widget-view+json":{"version_major":2,"version_minor":0,"model_id":"fd0f1edcdb93416892aff6576b66121f"}},"metadata":{}},{"output_type":"display_data","data":{"text/plain":"  0%|          | 0/1 [00:00<?, ?it/s]","application/vnd.jupyter.widget-view+json":{"version_major":2,"version_minor":0,"model_id":"d42fc1e3c78e4ceab9f49b576c696317"}},"metadata":{}},{"output_type":"display_data","data":{"text/plain":"  0%|          | 0/7 [00:00<?, ?it/s]","application/vnd.jupyter.widget-view+json":{"version_major":2,"version_minor":0,"model_id":"b46b2b01bb114d5094c6141d1215650d"}},"metadata":{}},{"output_type":"display_data","data":{"text/plain":"  0%|          | 0/1 [00:00<?, ?it/s]","application/vnd.jupyter.widget-view+json":{"version_major":2,"version_minor":0,"model_id":"10f77c42e36b4d41bc4674797de1ce77"}},"metadata":{}},{"output_type":"display_data","data":{"text/plain":"  0%|          | 0/7 [00:00<?, ?it/s]","application/vnd.jupyter.widget-view+json":{"version_major":2,"version_minor":0,"model_id":"3d527db4ed9b47568e6b26e43191e504"}},"metadata":{}},{"output_type":"display_data","data":{"text/plain":"  0%|          | 0/1 [00:00<?, ?it/s]","application/vnd.jupyter.widget-view+json":{"version_major":2,"version_minor":0,"model_id":"d9089db679ff44f3b7b73479ad2a4975"}},"metadata":{}},{"output_type":"display_data","data":{"text/plain":"  0%|          | 0/7 [00:00<?, ?it/s]","application/vnd.jupyter.widget-view+json":{"version_major":2,"version_minor":0,"model_id":"52d0d01fed8f4322acfe2ab590f8af25"}},"metadata":{}},{"output_type":"display_data","data":{"text/plain":"  0%|          | 0/1 [00:00<?, ?it/s]","application/vnd.jupyter.widget-view+json":{"version_major":2,"version_minor":0,"model_id":"086aa8a11f684317a904b0a01e37bd5e"}},"metadata":{}},{"output_type":"display_data","data":{"text/plain":"  0%|          | 0/7 [00:00<?, ?it/s]","application/vnd.jupyter.widget-view+json":{"version_major":2,"version_minor":0,"model_id":"9851e99671e9481c95056611554d4110"}},"metadata":{}},{"output_type":"display_data","data":{"text/plain":"  0%|          | 0/1 [00:00<?, ?it/s]","application/vnd.jupyter.widget-view+json":{"version_major":2,"version_minor":0,"model_id":"1630462e99024cfda6e32aa3ae7e5d10"}},"metadata":{}},{"output_type":"display_data","data":{"text/plain":"  0%|          | 0/7 [00:00<?, ?it/s]","application/vnd.jupyter.widget-view+json":{"version_major":2,"version_minor":0,"model_id":"b026d029e143438099c128b9e4f1aa04"}},"metadata":{}},{"output_type":"display_data","data":{"text/plain":"  0%|          | 0/1 [00:00<?, ?it/s]","application/vnd.jupyter.widget-view+json":{"version_major":2,"version_minor":0,"model_id":"7aff638d1b194b678b8948cfc738b623"}},"metadata":{}},{"output_type":"display_data","data":{"text/plain":"  0%|          | 0/7 [00:00<?, ?it/s]","application/vnd.jupyter.widget-view+json":{"version_major":2,"version_minor":0,"model_id":"dd14a023e88041a89cae1d1f11c76f8e"}},"metadata":{}},{"output_type":"display_data","data":{"text/plain":"  0%|          | 0/1 [00:00<?, ?it/s]","application/vnd.jupyter.widget-view+json":{"version_major":2,"version_minor":0,"model_id":"a8cbca284522499a98aef64c65ea111d"}},"metadata":{}},{"output_type":"display_data","data":{"text/plain":"  0%|          | 0/7 [00:00<?, ?it/s]","application/vnd.jupyter.widget-view+json":{"version_major":2,"version_minor":0,"model_id":"8682118a3b83452b9ed0bfa15c00f23f"}},"metadata":{}},{"output_type":"display_data","data":{"text/plain":"  0%|          | 0/1 [00:00<?, ?it/s]","application/vnd.jupyter.widget-view+json":{"version_major":2,"version_minor":0,"model_id":"139f1a09289c4d0a9fcb3c899c623e0d"}},"metadata":{}},{"output_type":"display_data","data":{"text/plain":"  0%|          | 0/7 [00:00<?, ?it/s]","application/vnd.jupyter.widget-view+json":{"version_major":2,"version_minor":0,"model_id":"d62b46abbd3d4ae3ba7a8f278ad03a70"}},"metadata":{}},{"output_type":"display_data","data":{"text/plain":"  0%|          | 0/1 [00:00<?, ?it/s]","application/vnd.jupyter.widget-view+json":{"version_major":2,"version_minor":0,"model_id":"49abf6a78d9e4b7c8b8113f3e28716c8"}},"metadata":{}},{"output_type":"display_data","data":{"text/plain":"  0%|          | 0/7 [00:00<?, ?it/s]","application/vnd.jupyter.widget-view+json":{"version_major":2,"version_minor":0,"model_id":"c38a7b03d04a419fb0773afaac0cf636"}},"metadata":{}},{"output_type":"display_data","data":{"text/plain":"  0%|          | 0/1 [00:00<?, ?it/s]","application/vnd.jupyter.widget-view+json":{"version_major":2,"version_minor":0,"model_id":"8ea06736c7c84f74bb6fd8153ba34734"}},"metadata":{}},{"output_type":"display_data","data":{"text/plain":"  0%|          | 0/7 [00:00<?, ?it/s]","application/vnd.jupyter.widget-view+json":{"version_major":2,"version_minor":0,"model_id":"7a725064291f4e0687819ffcd41688dd"}},"metadata":{}},{"output_type":"display_data","data":{"text/plain":"  0%|          | 0/1 [00:00<?, ?it/s]","application/vnd.jupyter.widget-view+json":{"version_major":2,"version_minor":0,"model_id":"563fa2b1ce10492199ce2efa31e20bdb"}},"metadata":{}},{"output_type":"display_data","data":{"text/plain":"  0%|          | 0/7 [00:00<?, ?it/s]","application/vnd.jupyter.widget-view+json":{"version_major":2,"version_minor":0,"model_id":"fbde680f204748d7ba0c27dc429dae5d"}},"metadata":{}},{"output_type":"display_data","data":{"text/plain":"  0%|          | 0/1 [00:00<?, ?it/s]","application/vnd.jupyter.widget-view+json":{"version_major":2,"version_minor":0,"model_id":"5fb862f5b5c04f79b5551d51eb569363"}},"metadata":{}},{"output_type":"display_data","data":{"text/plain":"  0%|          | 0/7 [00:00<?, ?it/s]","application/vnd.jupyter.widget-view+json":{"version_major":2,"version_minor":0,"model_id":"fb06330058c24111a8dffb229e55e0de"}},"metadata":{}},{"name":"stdout","text":"\nEnsemble score = 0.4459 after adding model mean_attention_no_fgm with weight 0.190. Increase of 0.0002\n\n","output_type":"stream"},{"output_type":"display_data","data":{"text/plain":"  0%|          | 0/1 [00:00<?, ?it/s]","application/vnd.jupyter.widget-view+json":{"version_major":2,"version_minor":0,"model_id":"37843e6e1c50441d8147b8f90510eb15"}},"metadata":{}},{"output_type":"display_data","data":{"text/plain":"  0%|          | 0/8 [00:00<?, ?it/s]","application/vnd.jupyter.widget-view+json":{"version_major":2,"version_minor":0,"model_id":"0cd4c07ab6c744f980bd3a1b894108c8"}},"metadata":{}},{"output_type":"display_data","data":{"text/plain":"  0%|          | 0/1 [00:00<?, ?it/s]","application/vnd.jupyter.widget-view+json":{"version_major":2,"version_minor":0,"model_id":"ca01a92c759347cdb3b216279db3155d"}},"metadata":{}},{"output_type":"display_data","data":{"text/plain":"  0%|          | 0/8 [00:00<?, ?it/s]","application/vnd.jupyter.widget-view+json":{"version_major":2,"version_minor":0,"model_id":"e092218a242a41e2b94276cb2fa9c01f"}},"metadata":{}},{"output_type":"display_data","data":{"text/plain":"  0%|          | 0/1 [00:00<?, ?it/s]","application/vnd.jupyter.widget-view+json":{"version_major":2,"version_minor":0,"model_id":"3df7ec8147734ff8a6fb34b4fef8684b"}},"metadata":{}},{"output_type":"display_data","data":{"text/plain":"  0%|          | 0/8 [00:00<?, ?it/s]","application/vnd.jupyter.widget-view+json":{"version_major":2,"version_minor":0,"model_id":"c85d1f535366439583811cb370c9982a"}},"metadata":{}},{"output_type":"display_data","data":{"text/plain":"  0%|          | 0/1 [00:00<?, ?it/s]","application/vnd.jupyter.widget-view+json":{"version_major":2,"version_minor":0,"model_id":"d40836e347df44d9b58d2dbf787c3755"}},"metadata":{}},{"output_type":"display_data","data":{"text/plain":"  0%|          | 0/8 [00:00<?, ?it/s]","application/vnd.jupyter.widget-view+json":{"version_major":2,"version_minor":0,"model_id":"f7e8c30b619b4707afa1aa214c98b6a4"}},"metadata":{}},{"output_type":"display_data","data":{"text/plain":"  0%|          | 0/1 [00:00<?, ?it/s]","application/vnd.jupyter.widget-view+json":{"version_major":2,"version_minor":0,"model_id":"e6c083f82d8f471e9a468e945734e2c6"}},"metadata":{}},{"output_type":"display_data","data":{"text/plain":"  0%|          | 0/8 [00:00<?, ?it/s]","application/vnd.jupyter.widget-view+json":{"version_major":2,"version_minor":0,"model_id":"c72e870d381b477489ea597e7ae4b88e"}},"metadata":{}},{"output_type":"display_data","data":{"text/plain":"  0%|          | 0/1 [00:00<?, ?it/s]","application/vnd.jupyter.widget-view+json":{"version_major":2,"version_minor":0,"model_id":"54ec18fa20514016bd592d62ee981e23"}},"metadata":{}},{"output_type":"display_data","data":{"text/plain":"  0%|          | 0/8 [00:00<?, ?it/s]","application/vnd.jupyter.widget-view+json":{"version_major":2,"version_minor":0,"model_id":"fc4709b3e3aa40a5acb36d3d926cf36b"}},"metadata":{}},{"output_type":"display_data","data":{"text/plain":"  0%|          | 0/1 [00:00<?, ?it/s]","application/vnd.jupyter.widget-view+json":{"version_major":2,"version_minor":0,"model_id":"6b0dc144089e46bc964d54f2443239f9"}},"metadata":{}},{"output_type":"display_data","data":{"text/plain":"  0%|          | 0/8 [00:00<?, ?it/s]","application/vnd.jupyter.widget-view+json":{"version_major":2,"version_minor":0,"model_id":"7b1d92eb1b754f54b45c860756630637"}},"metadata":{}},{"output_type":"display_data","data":{"text/plain":"  0%|          | 0/1 [00:00<?, ?it/s]","application/vnd.jupyter.widget-view+json":{"version_major":2,"version_minor":0,"model_id":"19e24dfac5d643b599498126d513c7db"}},"metadata":{}},{"output_type":"display_data","data":{"text/plain":"  0%|          | 0/8 [00:00<?, ?it/s]","application/vnd.jupyter.widget-view+json":{"version_major":2,"version_minor":0,"model_id":"ab58bfe3f82c4451ab7974572ea32ef1"}},"metadata":{}},{"output_type":"display_data","data":{"text/plain":"  0%|          | 0/1 [00:00<?, ?it/s]","application/vnd.jupyter.widget-view+json":{"version_major":2,"version_minor":0,"model_id":"df943193cd844645a827c75c3e58b17b"}},"metadata":{}},{"output_type":"display_data","data":{"text/plain":"  0%|          | 0/8 [00:00<?, ?it/s]","application/vnd.jupyter.widget-view+json":{"version_major":2,"version_minor":0,"model_id":"f8ce4922d4544518bbbf54e8245ced0c"}},"metadata":{}},{"output_type":"display_data","data":{"text/plain":"  0%|          | 0/1 [00:00<?, ?it/s]","application/vnd.jupyter.widget-view+json":{"version_major":2,"version_minor":0,"model_id":"31618b2dacff4fb2b1a59c3863f1c2bd"}},"metadata":{}},{"output_type":"display_data","data":{"text/plain":"  0%|          | 0/8 [00:00<?, ?it/s]","application/vnd.jupyter.widget-view+json":{"version_major":2,"version_minor":0,"model_id":"0ba506f70d4f458fab4fbd91bd2302ae"}},"metadata":{}},{"output_type":"display_data","data":{"text/plain":"  0%|          | 0/1 [00:00<?, ?it/s]","application/vnd.jupyter.widget-view+json":{"version_major":2,"version_minor":0,"model_id":"7d0d970fa7ad4cedb616d40d66aeb960"}},"metadata":{}},{"output_type":"display_data","data":{"text/plain":"  0%|          | 0/8 [00:00<?, ?it/s]","application/vnd.jupyter.widget-view+json":{"version_major":2,"version_minor":0,"model_id":"45cbfb55f377413ebedee372ff34dfad"}},"metadata":{}},{"output_type":"display_data","data":{"text/plain":"  0%|          | 0/1 [00:00<?, ?it/s]","application/vnd.jupyter.widget-view+json":{"version_major":2,"version_minor":0,"model_id":"ded439c2aba647348d9d99711c67121e"}},"metadata":{}},{"output_type":"display_data","data":{"text/plain":"  0%|          | 0/8 [00:00<?, ?it/s]","application/vnd.jupyter.widget-view+json":{"version_major":2,"version_minor":0,"model_id":"2a5a167ebf804c9daa05451a2baab397"}},"metadata":{}},{"name":"stdout","text":"\nEnsemble score = 0.4461 after adding model weighted_attention with weight 0.010. Increase of -0.0002\n\n","output_type":"stream"},{"output_type":"display_data","data":{"text/plain":"  0%|          | 0/1 [00:00<?, ?it/s]","application/vnd.jupyter.widget-view+json":{"version_major":2,"version_minor":0,"model_id":"ab479922cd0b49aeac6aa8d7666ac7fb"}},"metadata":{}},{"output_type":"display_data","data":{"text/plain":"  0%|          | 0/8 [00:00<?, ?it/s]","application/vnd.jupyter.widget-view+json":{"version_major":2,"version_minor":0,"model_id":"cf1a7ec5daee42b0bd6d134e10692c63"}},"metadata":{}},{"output_type":"display_data","data":{"text/plain":"  0%|          | 0/1 [00:00<?, ?it/s]","application/vnd.jupyter.widget-view+json":{"version_major":2,"version_minor":0,"model_id":"b1a43c4a5c9c4761a17b07c9a8e586a3"}},"metadata":{}},{"output_type":"display_data","data":{"text/plain":"  0%|          | 0/8 [00:00<?, ?it/s]","application/vnd.jupyter.widget-view+json":{"version_major":2,"version_minor":0,"model_id":"1ac32a915f144bae80cb67dbe3eb891f"}},"metadata":{}},{"output_type":"display_data","data":{"text/plain":"  0%|          | 0/1 [00:00<?, ?it/s]","application/vnd.jupyter.widget-view+json":{"version_major":2,"version_minor":0,"model_id":"53970ad3c6ef46acade95bd86f02c21f"}},"metadata":{}},{"output_type":"display_data","data":{"text/plain":"  0%|          | 0/8 [00:00<?, ?it/s]","application/vnd.jupyter.widget-view+json":{"version_major":2,"version_minor":0,"model_id":"a26331ef34f247959b4a141c121fd9f7"}},"metadata":{}},{"output_type":"display_data","data":{"text/plain":"  0%|          | 0/1 [00:00<?, ?it/s]","application/vnd.jupyter.widget-view+json":{"version_major":2,"version_minor":0,"model_id":"a98f3d457319421d80d7125503986de3"}},"metadata":{}},{"output_type":"display_data","data":{"text/plain":"  0%|          | 0/8 [00:00<?, ?it/s]","application/vnd.jupyter.widget-view+json":{"version_major":2,"version_minor":0,"model_id":"838b06016f8740f2a9e5fd95dc33c5c6"}},"metadata":{}},{"output_type":"display_data","data":{"text/plain":"  0%|          | 0/1 [00:00<?, ?it/s]","application/vnd.jupyter.widget-view+json":{"version_major":2,"version_minor":0,"model_id":"b0eaf7b6cf4e47a591d41d9ca7795b90"}},"metadata":{}},{"output_type":"display_data","data":{"text/plain":"  0%|          | 0/8 [00:00<?, ?it/s]","application/vnd.jupyter.widget-view+json":{"version_major":2,"version_minor":0,"model_id":"daef87fd8b8b442898e1b98e4d2facb5"}},"metadata":{}},{"output_type":"display_data","data":{"text/plain":"  0%|          | 0/1 [00:00<?, ?it/s]","application/vnd.jupyter.widget-view+json":{"version_major":2,"version_minor":0,"model_id":"47e82f39c3324e42b5c9f43a60201748"}},"metadata":{}},{"output_type":"display_data","data":{"text/plain":"  0%|          | 0/8 [00:00<?, ?it/s]","application/vnd.jupyter.widget-view+json":{"version_major":2,"version_minor":0,"model_id":"e772f4e810eb423e95a9804fc17c8de2"}},"metadata":{}},{"output_type":"display_data","data":{"text/plain":"  0%|          | 0/1 [00:00<?, ?it/s]","application/vnd.jupyter.widget-view+json":{"version_major":2,"version_minor":0,"model_id":"980a6b1aed2e4f968a681b380673292b"}},"metadata":{}},{"output_type":"display_data","data":{"text/plain":"  0%|          | 0/8 [00:00<?, ?it/s]","application/vnd.jupyter.widget-view+json":{"version_major":2,"version_minor":0,"model_id":"39439b2c351c4b7c98dc2655c0624e05"}},"metadata":{}},{"output_type":"display_data","data":{"text/plain":"  0%|          | 0/1 [00:00<?, ?it/s]","application/vnd.jupyter.widget-view+json":{"version_major":2,"version_minor":0,"model_id":"80f842f905aa4e07ac6f0b3d43e8937f"}},"metadata":{}},{"output_type":"display_data","data":{"text/plain":"  0%|          | 0/8 [00:00<?, ?it/s]","application/vnd.jupyter.widget-view+json":{"version_major":2,"version_minor":0,"model_id":"1d2b82f8cb314d2e8150f0fac722ac8b"}},"metadata":{}},{"output_type":"display_data","data":{"text/plain":"  0%|          | 0/1 [00:00<?, ?it/s]","application/vnd.jupyter.widget-view+json":{"version_major":2,"version_minor":0,"model_id":"31ea75d07eba4741bbfb3eb078a5e82d"}},"metadata":{}},{"output_type":"display_data","data":{"text/plain":"  0%|          | 0/8 [00:00<?, ?it/s]","application/vnd.jupyter.widget-view+json":{"version_major":2,"version_minor":0,"model_id":"a52e89d0973248e8a4e34d925c8fbadc"}},"metadata":{}},{"output_type":"display_data","data":{"text/plain":"  0%|          | 0/1 [00:00<?, ?it/s]","application/vnd.jupyter.widget-view+json":{"version_major":2,"version_minor":0,"model_id":"757a11ff96084900bc791d1d907c2b79"}},"metadata":{}},{"output_type":"display_data","data":{"text/plain":"  0%|          | 0/8 [00:00<?, ?it/s]","application/vnd.jupyter.widget-view+json":{"version_major":2,"version_minor":0,"model_id":"9d1aaba8437743c395311cad6a415db6"}},"metadata":{}},{"output_type":"display_data","data":{"text/plain":"  0%|          | 0/1 [00:00<?, ?it/s]","application/vnd.jupyter.widget-view+json":{"version_major":2,"version_minor":0,"model_id":"4bf697fa7ba8444087c187ccf2f10022"}},"metadata":{}},{"output_type":"display_data","data":{"text/plain":"  0%|          | 0/8 [00:00<?, ?it/s]","application/vnd.jupyter.widget-view+json":{"version_major":2,"version_minor":0,"model_id":"336f876889a642778a9063753f805baa"}},"metadata":{}},{"output_type":"display_data","data":{"text/plain":"  0%|          | 0/1 [00:00<?, ?it/s]","application/vnd.jupyter.widget-view+json":{"version_major":2,"version_minor":0,"model_id":"c20750b90722414690ccd656b31aa755"}},"metadata":{}},{"output_type":"display_data","data":{"text/plain":"  0%|          | 0/8 [00:00<?, ?it/s]","application/vnd.jupyter.widget-view+json":{"version_major":2,"version_minor":0,"model_id":"8b561d4e90cc4ad78fe3d82c910ee3e2"}},"metadata":{}},{"output_type":"display_data","data":{"text/plain":"  0%|          | 0/1 [00:00<?, ?it/s]","application/vnd.jupyter.widget-view+json":{"version_major":2,"version_minor":0,"model_id":"7ebfadafac564e69a166e5a4f4e03425"}},"metadata":{}},{"output_type":"display_data","data":{"text/plain":"  0%|          | 0/8 [00:00<?, ?it/s]","application/vnd.jupyter.widget-view+json":{"version_major":2,"version_minor":0,"model_id":"9129ae69b55d4a5497d10073ed753400"}},"metadata":{}},{"output_type":"display_data","data":{"text/plain":"  0%|          | 0/1 [00:00<?, ?it/s]","application/vnd.jupyter.widget-view+json":{"version_major":2,"version_minor":0,"model_id":"b31e4ce2b4974121b18840d2e6161991"}},"metadata":{}},{"output_type":"display_data","data":{"text/plain":"  0%|          | 0/8 [00:00<?, ?it/s]","application/vnd.jupyter.widget-view+json":{"version_major":2,"version_minor":0,"model_id":"28b61df4f2ef4bd9891fcda425a08535"}},"metadata":{}},{"output_type":"display_data","data":{"text/plain":"  0%|          | 0/1 [00:00<?, ?it/s]","application/vnd.jupyter.widget-view+json":{"version_major":2,"version_minor":0,"model_id":"3ce7bc1177124f13b2c027700d8b579e"}},"metadata":{}},{"output_type":"display_data","data":{"text/plain":"  0%|          | 0/8 [00:00<?, ?it/s]","application/vnd.jupyter.widget-view+json":{"version_major":2,"version_minor":0,"model_id":"8a90070ee94241439f8de7eb79047939"}},"metadata":{}},{"output_type":"display_data","data":{"text/plain":"  0%|          | 0/1 [00:00<?, ?it/s]","application/vnd.jupyter.widget-view+json":{"version_major":2,"version_minor":0,"model_id":"83e609055ff6481e89dc5c919a23123b"}},"metadata":{}},{"output_type":"display_data","data":{"text/plain":"  0%|          | 0/8 [00:00<?, ?it/s]","application/vnd.jupyter.widget-view+json":{"version_major":2,"version_minor":0,"model_id":"cc81f766d3804e81a1d8c272eac9d585"}},"metadata":{}},{"output_type":"display_data","data":{"text/plain":"  0%|          | 0/1 [00:00<?, ?it/s]","application/vnd.jupyter.widget-view+json":{"version_major":2,"version_minor":0,"model_id":"90a6cb957cfd4ad88357dc3377433d56"}},"metadata":{}},{"output_type":"display_data","data":{"text/plain":"  0%|          | 0/8 [00:00<?, ?it/s]","application/vnd.jupyter.widget-view+json":{"version_major":2,"version_minor":0,"model_id":"29f2773f27e94232980d745fbc6542d4"}},"metadata":{}},{"output_type":"display_data","data":{"text/plain":"  0%|          | 0/1 [00:00<?, ?it/s]","application/vnd.jupyter.widget-view+json":{"version_major":2,"version_minor":0,"model_id":"3fc7d575b0cf4bb69d677b1e9b77b0f5"}},"metadata":{}},{"output_type":"display_data","data":{"text/plain":"  0%|          | 0/8 [00:00<?, ?it/s]","application/vnd.jupyter.widget-view+json":{"version_major":2,"version_minor":0,"model_id":"89e67e23b6f040b9919f1fd3ec998a11"}},"metadata":{}},{"name":"stdout","text":"\nEnsemble score = 0.4460 after adding model roberta_attention_large_fgm with weight 0.070. Increase of -0.0001\n\n","output_type":"stream"},{"output_type":"display_data","data":{"text/plain":"  0%|          | 0/1 [00:00<?, ?it/s]","application/vnd.jupyter.widget-view+json":{"version_major":2,"version_minor":0,"model_id":"41b3be52a78a4e75a49abccde0c0a3bc"}},"metadata":{}},{"output_type":"display_data","data":{"text/plain":"  0%|          | 0/8 [00:00<?, ?it/s]","application/vnd.jupyter.widget-view+json":{"version_major":2,"version_minor":0,"model_id":"9d00d5dfb37f41379c944d635e5c14cd"}},"metadata":{}},{"output_type":"display_data","data":{"text/plain":"  0%|          | 0/1 [00:00<?, ?it/s]","application/vnd.jupyter.widget-view+json":{"version_major":2,"version_minor":0,"model_id":"b71267606a4e4b9a9fd16f2a69b50f16"}},"metadata":{}},{"output_type":"display_data","data":{"text/plain":"  0%|          | 0/8 [00:00<?, ?it/s]","application/vnd.jupyter.widget-view+json":{"version_major":2,"version_minor":0,"model_id":"ff254d0056424d078329d5b7f5eae446"}},"metadata":{}},{"output_type":"display_data","data":{"text/plain":"  0%|          | 0/1 [00:00<?, ?it/s]","application/vnd.jupyter.widget-view+json":{"version_major":2,"version_minor":0,"model_id":"bf0679ae2fc44233bc2d778164576c54"}},"metadata":{}},{"output_type":"display_data","data":{"text/plain":"  0%|          | 0/8 [00:00<?, ?it/s]","application/vnd.jupyter.widget-view+json":{"version_major":2,"version_minor":0,"model_id":"e42bff11366e447eaeb2267ace57365d"}},"metadata":{}},{"output_type":"display_data","data":{"text/plain":"  0%|          | 0/1 [00:00<?, ?it/s]","application/vnd.jupyter.widget-view+json":{"version_major":2,"version_minor":0,"model_id":"2dbafa6e767241f2a1a84680b59f65a6"}},"metadata":{}},{"output_type":"display_data","data":{"text/plain":"  0%|          | 0/8 [00:00<?, ?it/s]","application/vnd.jupyter.widget-view+json":{"version_major":2,"version_minor":0,"model_id":"b731fcadc2c244d28bb3d0e52dc4619e"}},"metadata":{}},{"output_type":"display_data","data":{"text/plain":"  0%|          | 0/1 [00:00<?, ?it/s]","application/vnd.jupyter.widget-view+json":{"version_major":2,"version_minor":0,"model_id":"981a20f6c05a41078091414e8d9686b6"}},"metadata":{}},{"output_type":"display_data","data":{"text/plain":"  0%|          | 0/8 [00:00<?, ?it/s]","application/vnd.jupyter.widget-view+json":{"version_major":2,"version_minor":0,"model_id":"2993c90480f64588a304a42d146212da"}},"metadata":{}},{"output_type":"display_data","data":{"text/plain":"  0%|          | 0/1 [00:00<?, ?it/s]","application/vnd.jupyter.widget-view+json":{"version_major":2,"version_minor":0,"model_id":"538c8a803f3a4e17892c444e1e2feb3a"}},"metadata":{}},{"output_type":"display_data","data":{"text/plain":"  0%|          | 0/8 [00:00<?, ?it/s]","application/vnd.jupyter.widget-view+json":{"version_major":2,"version_minor":0,"model_id":"65eba162178a43678d6db0e397670cf9"}},"metadata":{}},{"output_type":"display_data","data":{"text/plain":"  0%|          | 0/1 [00:00<?, ?it/s]","application/vnd.jupyter.widget-view+json":{"version_major":2,"version_minor":0,"model_id":"9c696f6564774d998578d2ddb1405e81"}},"metadata":{}},{"output_type":"display_data","data":{"text/plain":"  0%|          | 0/8 [00:00<?, ?it/s]","application/vnd.jupyter.widget-view+json":{"version_major":2,"version_minor":0,"model_id":"f30d133c682244009d4c8c38d24e1242"}},"metadata":{}},{"output_type":"display_data","data":{"text/plain":"  0%|          | 0/1 [00:00<?, ?it/s]","application/vnd.jupyter.widget-view+json":{"version_major":2,"version_minor":0,"model_id":"66f990cf079544f1bd62b1dce236a1a9"}},"metadata":{}},{"output_type":"display_data","data":{"text/plain":"  0%|          | 0/8 [00:00<?, ?it/s]","application/vnd.jupyter.widget-view+json":{"version_major":2,"version_minor":0,"model_id":"324fddd8ed49412785257fe9de52a630"}},"metadata":{}},{"output_type":"display_data","data":{"text/plain":"  0%|          | 0/1 [00:00<?, ?it/s]","application/vnd.jupyter.widget-view+json":{"version_major":2,"version_minor":0,"model_id":"3b5ff09a12c24ce6938cf8c879694f05"}},"metadata":{}},{"output_type":"display_data","data":{"text/plain":"  0%|          | 0/8 [00:00<?, ?it/s]","application/vnd.jupyter.widget-view+json":{"version_major":2,"version_minor":0,"model_id":"7565caf4a4c64fdeb8dc073e17d19181"}},"metadata":{}},{"output_type":"display_data","data":{"text/plain":"  0%|          | 0/1 [00:00<?, ?it/s]","application/vnd.jupyter.widget-view+json":{"version_major":2,"version_minor":0,"model_id":"c02260fac40c498b859e241b243a8ae6"}},"metadata":{}},{"output_type":"display_data","data":{"text/plain":"  0%|          | 0/8 [00:00<?, ?it/s]","application/vnd.jupyter.widget-view+json":{"version_major":2,"version_minor":0,"model_id":"951bb8cd82da43beb7b5ed4bce231869"}},"metadata":{}},{"output_type":"display_data","data":{"text/plain":"  0%|          | 0/1 [00:00<?, ?it/s]","application/vnd.jupyter.widget-view+json":{"version_major":2,"version_minor":0,"model_id":"80cc4bf045f04415b9f5b0931b744b63"}},"metadata":{}},{"output_type":"display_data","data":{"text/plain":"  0%|          | 0/8 [00:00<?, ?it/s]","application/vnd.jupyter.widget-view+json":{"version_major":2,"version_minor":0,"model_id":"1a7adcd9313944539ae82bce242d5a23"}},"metadata":{}},{"output_type":"display_data","data":{"text/plain":"  0%|          | 0/1 [00:00<?, ?it/s]","application/vnd.jupyter.widget-view+json":{"version_major":2,"version_minor":0,"model_id":"2f0c920657304034855f1d67991b5e7a"}},"metadata":{}},{"output_type":"display_data","data":{"text/plain":"  0%|          | 0/8 [00:00<?, ?it/s]","application/vnd.jupyter.widget-view+json":{"version_major":2,"version_minor":0,"model_id":"7851e836143e4fd0ad8a080c8dd4cde5"}},"metadata":{}},{"output_type":"display_data","data":{"text/plain":"  0%|          | 0/1 [00:00<?, ?it/s]","application/vnd.jupyter.widget-view+json":{"version_major":2,"version_minor":0,"model_id":"51b79725f5f34feebc45fbc10f16e84d"}},"metadata":{}},{"output_type":"display_data","data":{"text/plain":"  0%|          | 0/8 [00:00<?, ?it/s]","application/vnd.jupyter.widget-view+json":{"version_major":2,"version_minor":0,"model_id":"5a0146e89c874fbd8c61a33ec89b05b3"}},"metadata":{}},{"output_type":"display_data","data":{"text/plain":"  0%|          | 0/1 [00:00<?, ?it/s]","application/vnd.jupyter.widget-view+json":{"version_major":2,"version_minor":0,"model_id":"6c16e8c94b644d0dabdd0b1dad6231b4"}},"metadata":{}},{"output_type":"display_data","data":{"text/plain":"  0%|          | 0/8 [00:00<?, ?it/s]","application/vnd.jupyter.widget-view+json":{"version_major":2,"version_minor":0,"model_id":"9be725569d2848ffa4ba77118e70bafe"}},"metadata":{}},{"output_type":"display_data","data":{"text/plain":"  0%|          | 0/1 [00:00<?, ?it/s]","application/vnd.jupyter.widget-view+json":{"version_major":2,"version_minor":0,"model_id":"c3783635b532407aaeadb1081d866d69"}},"metadata":{}},{"output_type":"display_data","data":{"text/plain":"  0%|          | 0/8 [00:00<?, ?it/s]","application/vnd.jupyter.widget-view+json":{"version_major":2,"version_minor":0,"model_id":"e53bc33bfc87497691a700ba72b392ba"}},"metadata":{}},{"output_type":"display_data","data":{"text/plain":"  0%|          | 0/1 [00:00<?, ?it/s]","application/vnd.jupyter.widget-view+json":{"version_major":2,"version_minor":0,"model_id":"15614cb74b4c4276ac88dc90f85c3975"}},"metadata":{}},{"output_type":"display_data","data":{"text/plain":"  0%|          | 0/8 [00:00<?, ?it/s]","application/vnd.jupyter.widget-view+json":{"version_major":2,"version_minor":0,"model_id":"85f4238f42f74712bfa16f68959eb9b5"}},"metadata":{}},{"output_type":"display_data","data":{"text/plain":"  0%|          | 0/1 [00:00<?, ?it/s]","application/vnd.jupyter.widget-view+json":{"version_major":2,"version_minor":0,"model_id":"49f6191998b24d0b9a2e2e2712d0838b"}},"metadata":{}},{"output_type":"display_data","data":{"text/plain":"  0%|          | 0/8 [00:00<?, ?it/s]","application/vnd.jupyter.widget-view+json":{"version_major":2,"version_minor":0,"model_id":"207c5ebcecc94bff9b3502d7de5f2939"}},"metadata":{}},{"output_type":"display_data","data":{"text/plain":"  0%|          | 0/1 [00:00<?, ?it/s]","application/vnd.jupyter.widget-view+json":{"version_major":2,"version_minor":0,"model_id":"5dc08d2b7d0d4f0b92a82a1a51054126"}},"metadata":{}},{"output_type":"display_data","data":{"text/plain":"  0%|          | 0/8 [00:00<?, ?it/s]","application/vnd.jupyter.widget-view+json":{"version_major":2,"version_minor":0,"model_id":"8e5df1f1b00e4ea69a9e988df75cff51"}},"metadata":{}},{"output_type":"display_data","data":{"text/plain":"  0%|          | 0/1 [00:00<?, ?it/s]","application/vnd.jupyter.widget-view+json":{"version_major":2,"version_minor":0,"model_id":"4703bd8f525b4454bddd93453a82fb07"}},"metadata":{}},{"output_type":"display_data","data":{"text/plain":"  0%|          | 0/8 [00:00<?, ?it/s]","application/vnd.jupyter.widget-view+json":{"version_major":2,"version_minor":0,"model_id":"b2a7bc2fa02a4fdf8290ab8a4cc44d9b"}},"metadata":{}},{"output_type":"display_data","data":{"text/plain":"  0%|          | 0/1 [00:00<?, ?it/s]","application/vnd.jupyter.widget-view+json":{"version_major":2,"version_minor":0,"model_id":"c4059371fde44de68d386be4ee25a144"}},"metadata":{}},{"output_type":"display_data","data":{"text/plain":"  0%|          | 0/8 [00:00<?, ?it/s]","application/vnd.jupyter.widget-view+json":{"version_major":2,"version_minor":0,"model_id":"562ee8ee39db45bb81b32c4a19238433"}},"metadata":{}},{"output_type":"display_data","data":{"text/plain":"  0%|          | 0/1 [00:00<?, ?it/s]","application/vnd.jupyter.widget-view+json":{"version_major":2,"version_minor":0,"model_id":"2905675c03ef4993a7901abc294f913d"}},"metadata":{}},{"output_type":"display_data","data":{"text/plain":"  0%|          | 0/8 [00:00<?, ?it/s]","application/vnd.jupyter.widget-view+json":{"version_major":2,"version_minor":0,"model_id":"422b13ed3345400993405ca72d8ac587"}},"metadata":{}},{"output_type":"display_data","data":{"text/plain":"  0%|          | 0/1 [00:00<?, ?it/s]","application/vnd.jupyter.widget-view+json":{"version_major":2,"version_minor":0,"model_id":"239b803e3f2d4c63b1a670b8d8cc01dd"}},"metadata":{}},{"output_type":"display_data","data":{"text/plain":"  0%|          | 0/8 [00:00<?, ?it/s]","application/vnd.jupyter.widget-view+json":{"version_major":2,"version_minor":0,"model_id":"7388e2c6f1ee4f1bbbb875b2e2fd80cb"}},"metadata":{}},{"name":"stdout","text":"\nEnsemble score = 0.4459 after adding model v2 with weight 0.110. Increase of 0.0000\n\n","output_type":"stream"},{"output_type":"display_data","data":{"text/plain":"  0%|          | 0/1 [00:00<?, ?it/s]","application/vnd.jupyter.widget-view+json":{"version_major":2,"version_minor":0,"model_id":"994d598bbdc14a7b9dfbf665eddd22ca"}},"metadata":{}},{"output_type":"display_data","data":{"text/plain":"  0%|          | 0/9 [00:00<?, ?it/s]","application/vnd.jupyter.widget-view+json":{"version_major":2,"version_minor":0,"model_id":"49a9cb4b6b164f918723f38a12d356cb"}},"metadata":{}},{"output_type":"display_data","data":{"text/plain":"  0%|          | 0/1 [00:00<?, ?it/s]","application/vnd.jupyter.widget-view+json":{"version_major":2,"version_minor":0,"model_id":"28d351fbec474ea48e4126853f4f8018"}},"metadata":{}},{"output_type":"display_data","data":{"text/plain":"  0%|          | 0/9 [00:00<?, ?it/s]","application/vnd.jupyter.widget-view+json":{"version_major":2,"version_minor":0,"model_id":"1942898fc4e64086acd1153323de09f6"}},"metadata":{}},{"output_type":"display_data","data":{"text/plain":"  0%|          | 0/1 [00:00<?, ?it/s]","application/vnd.jupyter.widget-view+json":{"version_major":2,"version_minor":0,"model_id":"4a97ac4734eb45d7b7567ac83795e166"}},"metadata":{}},{"output_type":"display_data","data":{"text/plain":"  0%|          | 0/9 [00:00<?, ?it/s]","application/vnd.jupyter.widget-view+json":{"version_major":2,"version_minor":0,"model_id":"3c3bb03aa5ce421faf4edb7f2c5bbcc7"}},"metadata":{}},{"output_type":"display_data","data":{"text/plain":"  0%|          | 0/1 [00:00<?, ?it/s]","application/vnd.jupyter.widget-view+json":{"version_major":2,"version_minor":0,"model_id":"a327558fa0e44184993f8aab195f6bea"}},"metadata":{}},{"output_type":"display_data","data":{"text/plain":"  0%|          | 0/9 [00:00<?, ?it/s]","application/vnd.jupyter.widget-view+json":{"version_major":2,"version_minor":0,"model_id":"613f787142a94d30a8d397814d20af4e"}},"metadata":{}},{"output_type":"display_data","data":{"text/plain":"  0%|          | 0/1 [00:00<?, ?it/s]","application/vnd.jupyter.widget-view+json":{"version_major":2,"version_minor":0,"model_id":"de1e03bf2da34ab880462a49d03d955a"}},"metadata":{}},{"output_type":"display_data","data":{"text/plain":"  0%|          | 0/9 [00:00<?, ?it/s]","application/vnd.jupyter.widget-view+json":{"version_major":2,"version_minor":0,"model_id":"d301b0f8147f4c2884c5ae71de0f970e"}},"metadata":{}},{"output_type":"display_data","data":{"text/plain":"  0%|          | 0/1 [00:00<?, ?it/s]","application/vnd.jupyter.widget-view+json":{"version_major":2,"version_minor":0,"model_id":"1829bef9e26848b58b4060558bbc702e"}},"metadata":{}},{"output_type":"display_data","data":{"text/plain":"  0%|          | 0/9 [00:00<?, ?it/s]","application/vnd.jupyter.widget-view+json":{"version_major":2,"version_minor":0,"model_id":"f32d741966bb4542a803a52099673f7c"}},"metadata":{}},{"output_type":"display_data","data":{"text/plain":"  0%|          | 0/1 [00:00<?, ?it/s]","application/vnd.jupyter.widget-view+json":{"version_major":2,"version_minor":0,"model_id":"3df54d6b22714cf9a0326a16b238ae33"}},"metadata":{}},{"output_type":"display_data","data":{"text/plain":"  0%|          | 0/9 [00:00<?, ?it/s]","application/vnd.jupyter.widget-view+json":{"version_major":2,"version_minor":0,"model_id":"c70c745601514577b7b8e9cce298c473"}},"metadata":{}},{"output_type":"display_data","data":{"text/plain":"  0%|          | 0/1 [00:00<?, ?it/s]","application/vnd.jupyter.widget-view+json":{"version_major":2,"version_minor":0,"model_id":"4d32d28189be414786b5e54612ab1401"}},"metadata":{}},{"output_type":"display_data","data":{"text/plain":"  0%|          | 0/9 [00:00<?, ?it/s]","application/vnd.jupyter.widget-view+json":{"version_major":2,"version_minor":0,"model_id":"cb7e07f5d5ac4e0fb4b09a64bb844c49"}},"metadata":{}},{"output_type":"display_data","data":{"text/plain":"  0%|          | 0/1 [00:00<?, ?it/s]","application/vnd.jupyter.widget-view+json":{"version_major":2,"version_minor":0,"model_id":"629c8c8a079e4f9cb1700cd45ac88f69"}},"metadata":{}},{"output_type":"display_data","data":{"text/plain":"  0%|          | 0/9 [00:00<?, ?it/s]","application/vnd.jupyter.widget-view+json":{"version_major":2,"version_minor":0,"model_id":"b8492664140144b9b70eed814611fd89"}},"metadata":{}},{"output_type":"display_data","data":{"text/plain":"  0%|          | 0/1 [00:00<?, ?it/s]","application/vnd.jupyter.widget-view+json":{"version_major":2,"version_minor":0,"model_id":"52d8868f7e8a46fbb530edd24322c32b"}},"metadata":{}},{"output_type":"display_data","data":{"text/plain":"  0%|          | 0/9 [00:00<?, ?it/s]","application/vnd.jupyter.widget-view+json":{"version_major":2,"version_minor":0,"model_id":"a8ad3331cff146f5b636759e3f4546f2"}},"metadata":{}},{"output_type":"display_data","data":{"text/plain":"  0%|          | 0/1 [00:00<?, ?it/s]","application/vnd.jupyter.widget-view+json":{"version_major":2,"version_minor":0,"model_id":"ece382be9a41463a853ea8a2c7c70838"}},"metadata":{}},{"output_type":"display_data","data":{"text/plain":"  0%|          | 0/9 [00:00<?, ?it/s]","application/vnd.jupyter.widget-view+json":{"version_major":2,"version_minor":0,"model_id":"78213c0dd6fc4a21af02b77b5234c0f9"}},"metadata":{}},{"output_type":"display_data","data":{"text/plain":"  0%|          | 0/1 [00:00<?, ?it/s]","application/vnd.jupyter.widget-view+json":{"version_major":2,"version_minor":0,"model_id":"8abec2ae17f94419847fd70b8457123d"}},"metadata":{}},{"output_type":"display_data","data":{"text/plain":"  0%|          | 0/9 [00:00<?, ?it/s]","application/vnd.jupyter.widget-view+json":{"version_major":2,"version_minor":0,"model_id":"a8a0f4cbdfd04231b636db5c88a55d01"}},"metadata":{}},{"output_type":"display_data","data":{"text/plain":"  0%|          | 0/1 [00:00<?, ?it/s]","application/vnd.jupyter.widget-view+json":{"version_major":2,"version_minor":0,"model_id":"8610281741d74d22a1dd84676ad02b3f"}},"metadata":{}},{"output_type":"display_data","data":{"text/plain":"  0%|          | 0/9 [00:00<?, ?it/s]","application/vnd.jupyter.widget-view+json":{"version_major":2,"version_minor":0,"model_id":"d5853902f8fc44a3b40461cf22f5efa3"}},"metadata":{}},{"output_type":"display_data","data":{"text/plain":"  0%|          | 0/1 [00:00<?, ?it/s]","application/vnd.jupyter.widget-view+json":{"version_major":2,"version_minor":0,"model_id":"ebe08b7f78964b3d96e6ac3a27832612"}},"metadata":{}},{"output_type":"display_data","data":{"text/plain":"  0%|          | 0/9 [00:00<?, ?it/s]","application/vnd.jupyter.widget-view+json":{"version_major":2,"version_minor":0,"model_id":"e76016ea566d49698bc381b41f89a289"}},"metadata":{}},{"output_type":"display_data","data":{"text/plain":"  0%|          | 0/1 [00:00<?, ?it/s]","application/vnd.jupyter.widget-view+json":{"version_major":2,"version_minor":0,"model_id":"148f35d54ec44022b7ba8638173fde21"}},"metadata":{}},{"output_type":"display_data","data":{"text/plain":"  0%|          | 0/9 [00:00<?, ?it/s]","application/vnd.jupyter.widget-view+json":{"version_major":2,"version_minor":0,"model_id":"6362337789264386bcfcc75598fb5dcf"}},"metadata":{}},{"output_type":"display_data","data":{"text/plain":"  0%|          | 0/1 [00:00<?, ?it/s]","application/vnd.jupyter.widget-view+json":{"version_major":2,"version_minor":0,"model_id":"b5169cbdac0a4937a5fcf81f296f9065"}},"metadata":{}},{"output_type":"display_data","data":{"text/plain":"  0%|          | 0/9 [00:00<?, ?it/s]","application/vnd.jupyter.widget-view+json":{"version_major":2,"version_minor":0,"model_id":"b860a6c987ec48dfb4140a41140d647a"}},"metadata":{}},{"output_type":"display_data","data":{"text/plain":"  0%|          | 0/1 [00:00<?, ?it/s]","application/vnd.jupyter.widget-view+json":{"version_major":2,"version_minor":0,"model_id":"37515949f50247d6b18abcbf24f37878"}},"metadata":{}},{"output_type":"display_data","data":{"text/plain":"  0%|          | 0/9 [00:00<?, ?it/s]","application/vnd.jupyter.widget-view+json":{"version_major":2,"version_minor":0,"model_id":"315db6cb2fdc4468aeb6dfe52d65c131"}},"metadata":{}},{"output_type":"display_data","data":{"text/plain":"  0%|          | 0/1 [00:00<?, ?it/s]","application/vnd.jupyter.widget-view+json":{"version_major":2,"version_minor":0,"model_id":"d685062682a34cee8150d15413254050"}},"metadata":{}},{"output_type":"display_data","data":{"text/plain":"  0%|          | 0/9 [00:00<?, ?it/s]","application/vnd.jupyter.widget-view+json":{"version_major":2,"version_minor":0,"model_id":"3ac107cde47142fd8bcb3d3d90f435e1"}},"metadata":{}},{"name":"stdout","text":"\nEnsemble score = 0.4458 after adding model weightedpool with weight 0.070. Increase of 0.0000\n\n","output_type":"stream"}]},{"cell_type":"code","source":"ens_best_score","metadata":{"execution":{"iopub.status.busy":"2022-11-29T04:51:19.334359Z","iopub.execute_input":"2022-11-29T04:51:19.335514Z","iopub.status.idle":"2022-11-29T04:51:19.345128Z","shell.execute_reply.started":"2022-11-29T04:51:19.335466Z","shell.execute_reply":"2022-11-29T04:51:19.343492Z"},"trusted":true},"execution_count":48,"outputs":[{"execution_count":48,"output_type":"execute_result","data":{"text/plain":"0.44584549142087876"},"metadata":{}}]},{"cell_type":"code","source":"print('Total models:', len(selected_models))\nprint('Selected models:', [selected_model.name for selected_model in selected_models])\nprint('Optimal weights:', weights)","metadata":{"execution":{"iopub.status.busy":"2022-11-29T04:51:20.016744Z","iopub.execute_input":"2022-11-29T04:51:20.017197Z","iopub.status.idle":"2022-11-29T04:51:20.025525Z","shell.execute_reply.started":"2022-11-29T04:51:20.017162Z","shell.execute_reply":"2022-11-29T04:51:20.023825Z"},"trusted":true},"execution_count":49,"outputs":[{"name":"stdout","text":"Total models: 10\nSelected models: ['attention_large_fgm', 'weighted2last_fgm_512', 'weightedmean2last_fgm_512', 'attention_fgm_512', 'attention_fgm_768', 'v21', 'v112', 'mean_attention_no_fgm', 'v2', 'weightedpool']\nOptimal weights: [1, 0.47000000000000003, 0.29000000000000004, 0.16, 0.17, 0.14, 0.23, 0.19, 0.11, 0.06999999999999999]\n","output_type":"stream"}]},{"cell_type":"code","source":"weights = [w/sum(weights) for w in weights]","metadata":{"execution":{"iopub.status.busy":"2022-11-29T04:51:24.323202Z","iopub.execute_input":"2022-11-29T04:51:24.324789Z","iopub.status.idle":"2022-11-29T04:51:24.332554Z","shell.execute_reply.started":"2022-11-29T04:51:24.324733Z","shell.execute_reply":"2022-11-29T04:51:24.330636Z"},"trusted":true},"execution_count":50,"outputs":[]},{"cell_type":"code","source":"weights","metadata":{"execution":{"iopub.status.busy":"2022-11-29T04:51:24.794907Z","iopub.execute_input":"2022-11-29T04:51:24.796071Z","iopub.status.idle":"2022-11-29T04:51:24.807124Z","shell.execute_reply.started":"2022-11-29T04:51:24.796025Z","shell.execute_reply":"2022-11-29T04:51:24.804441Z"},"trusted":true},"execution_count":51,"outputs":[{"execution_count":51,"output_type":"execute_result","data":{"text/plain":"[0.353356890459364,\n 0.1660777385159011,\n 0.10247349823321557,\n 0.05653710247349824,\n 0.06007067137809188,\n 0.049469964664310966,\n 0.08127208480565372,\n 0.06713780918727916,\n 0.03886925795053004,\n 0.02473498233215548]"},"metadata":{}}]},{"cell_type":"code","source":"opt_fine_tuned_models = selected_models\nopt_fine_tuned_weights = weights\n\nfor i,m in enumerate(opt_fine_tuned_models):\n    m.inference_weight = opt_fine_tuned_weights[i]\n    \n_, fine_tuned_oof = get_ensemble_score(opt_fine_tuned_models)","metadata":{"execution":{"iopub.status.busy":"2022-11-29T04:51:27.702805Z","iopub.execute_input":"2022-11-29T04:51:27.703307Z","iopub.status.idle":"2022-11-29T04:51:28.081099Z","shell.execute_reply.started":"2022-11-29T04:51:27.703264Z","shell.execute_reply":"2022-11-29T04:51:28.079699Z"},"trusted":true},"execution_count":52,"outputs":[{"output_type":"display_data","data":{"text/plain":"  0%|          | 0/10 [00:00<?, ?it/s]","application/vnd.jupyter.widget-view+json":{"version_major":2,"version_minor":0,"model_id":"720e0d0222334314ba84c5588f3a074d"}},"metadata":{}},{"name":"stdout","text":"score: 0.441634  scores: [0.4708088008965855, 0.43557707950892754, 0.40570654844137066, 0.4470577364807026, 0.45828980767092026, 0.4323633351609973]\nscore: 0.449143  scores: [0.4811032367301944, 0.43957839434782076, 0.4144354121560431, 0.44892059911831245, 0.46513693521184996, 0.445684491347229]\nscore: 0.453617  scores: [0.47536046744193905, 0.4470521856306097, 0.4118435024534858, 0.4624788094030018, 0.47635847088231065, 0.44860831099062715]\nscore: 0.438954  scores: [0.4735643022618435, 0.43917074599876654, 0.40330047239598266, 0.43096512797882347, 0.457883147075871, 0.4288374526945873]\n\n\nscore: 0.445893  scores: [0.475222638218856, 0.44036458337173895, 0.4088447595466631, 0.44749486252320375, 0.46447703339811036, 0.43895261923000717]\n","output_type":"stream"}]},{"cell_type":"code","source":"fine_tuned_oof_preds = fine_tuned_oof[[f'pred_{col}' for col in target_cols]].values","metadata":{"execution":{"iopub.status.busy":"2022-11-29T04:51:34.548942Z","iopub.execute_input":"2022-11-29T04:51:34.549605Z","iopub.status.idle":"2022-11-29T04:51:34.559719Z","shell.execute_reply.started":"2022-11-29T04:51:34.549533Z","shell.execute_reply":"2022-11-29T04:51:34.558024Z"},"trusted":true},"execution_count":53,"outputs":[]},{"cell_type":"markdown","source":"# SVR","metadata":{}},{"cell_type":"code","source":"sys.path.append('../input/iterativestratification')\nfrom iterstrat.ml_stratifiers import MultilabelStratifiedKFold\nsvr_folds = 15\n\ntrain = pd.read_csv(TRAIN_PATH)\nskf = MultilabelStratifiedKFold(n_splits=svr_folds, shuffle=True, random_state=42)\nfor i,(train_index, val_index) in enumerate(skf.split(train,train[target_cols])):\n    train.loc[val_index,'fold'] = i","metadata":{"execution":{"iopub.status.busy":"2022-11-29T04:51:41.783051Z","iopub.execute_input":"2022-11-29T04:51:41.784452Z","iopub.status.idle":"2022-11-29T04:51:42.269362Z","shell.execute_reply.started":"2022-11-29T04:51:41.784398Z","shell.execute_reply":"2022-11-29T04:51:42.267765Z"},"trusted":true},"execution_count":54,"outputs":[]},{"cell_type":"code","source":"from glob import glob \n\ndef get_text_embedding(cfg, dfs):\n    cfg.tokenizer = AutoTokenizer.from_pretrained(cfg.model)\n    infer_ = Inferencer(cfg=cfg, inference_weight=cfg.inference_weight)\n    if cfg.model == 'gpt2':\n        cfg.tokenizer.pad_token = cfg.tokenizer.eos_token\n    text_embs = []\n    for df in dfs:\n        dataset = TestDataset(cfg, df)\n        loader = DataLoader(\n            dataset,\n            batch_size=4,\n            shuffle=False)\n\n        # Text embedding for SVM\n        test_text_emb = []\n        if not cfg.pretrained:\n            for fold in infer_.cfg.trn_fold:\n                test_text_emb.append(infer_.get_text_embedding(loader, device, fold))\n            text_emb = np.mean(text_emb, axis=0)\n        else:\n            text_emb = infer_.get_text_embedding(loader, device)\n        text_embs.append(text_emb)\n        del dataset, loader; gc.collect(); torch.cuda.empty_cache();\n    del infer_; gc.collect(); torch.cuda.empty_cache();\n    return text_embs\n\n\ndef get_learner_score(models_cfg, folds=4, manual_features=None, save=False, verbose=False):\n    for i, model_cfg in enumerate(models_cfg):\n        model_name = model_cfg.model.split('/')[-1].replace('-', '_')\n        models_cfg[i].model_name = model_name\n        model_file = f'../input/fb3embeddings/train_text_emb_{model_cfg.file_name}.npy'\n        if 'embedding' in model_cfg:\n            continue\n        with open(model_file, 'rb') as f:\n            models_cfg[i].embedding = np.load(f)   \n    embeddings = np.concatenate(\n        [model_cfg.embedding for model_cfg in models_cfg],\n        axis=1)\n    if type(manual_features) != type(None):\n        embeddings = np.concatenate(\n            [embeddings, manual_features],\n            axis=1)\n    #print(embeddings.shape)\n    def svr_inference_fn(model_path, te_text_feats):\n        model = load(model_path)\n        preds = model.predict(te_text_feats)\n        return preds\n\n    oof = train.copy()\n    for fold in range(folds):\n        val_idx = oof[oof['fold'] == fold].index\n        preds = svr_inference_fn(f'../input/fb3-train-svr/svr_{fold}.model', embeddings[val_idx, :])\n        oof.loc[oof['fold']==fold, [f'pred_{col}' for col in target_cols]] = preds\n    labels = oof[target_cols].values\n    oof_preds = oof[[f'pred_{col}' for col in target_cols]].values\n    print(mc_rmse(labels, oof_preds))\n    return oof","metadata":{"execution":{"iopub.status.busy":"2022-11-29T04:51:43.302604Z","iopub.execute_input":"2022-11-29T04:51:43.303914Z","iopub.status.idle":"2022-11-29T04:51:43.331834Z","shell.execute_reply.started":"2022-11-29T04:51:43.303855Z","shell.execute_reply":"2022-11-29T04:51:43.329398Z"},"trusted":true},"execution_count":55,"outputs":[]},{"cell_type":"code","source":"pretrained_models_cfg = [\n    deberta_large_mnli,\n    #gpt2,\n    roberta_base,\n    roberta_large,\n    #xlnet_base, \n    #xlnet_large,\n    deberta_base, \n    deberta_large, \n    deberta_xlarge,\n    deberta_v2_xlarge, \n    deberta_v2_xxlarge,\n    deberta_v3_base, \n    deberta_v3_large,\n    \n    #bart_base,\n    bart_large,\n    #bart_lage_mnli,\n    #bert_base_uncased,\n    bert_large_uncased,\n    #muppet_roberta_large,\n    funnel_small,\n    funnel_large\n]\nlen(pretrained_models_cfg)","metadata":{"execution":{"iopub.status.busy":"2022-11-29T04:53:34.835546Z","iopub.execute_input":"2022-11-29T04:53:34.835996Z","iopub.status.idle":"2022-11-29T04:53:34.847820Z","shell.execute_reply.started":"2022-11-29T04:53:34.835959Z","shell.execute_reply":"2022-11-29T04:53:34.846272Z"},"trusted":true},"execution_count":59,"outputs":[{"execution_count":59,"output_type":"execute_result","data":{"text/plain":"14"},"metadata":{}}]},{"cell_type":"code","source":"import glob\nfrom joblib import dump, load\nfrom cuml.svm import SVR\nimport cuml\n\nsvr_oof = get_learner_score(pretrained_models_cfg, folds=15, manual_features=None, save=False, verbose=False)\nsvr_preds = svr_oof[[f'pred_{col}' for col in target_cols]].values","metadata":{"execution":{"iopub.status.busy":"2022-11-29T04:53:35.778492Z","iopub.execute_input":"2022-11-29T04:53:35.779218Z","iopub.status.idle":"2022-11-29T04:56:22.824767Z","shell.execute_reply.started":"2022-11-29T04:53:35.779163Z","shell.execute_reply":"2022-11-29T04:56:22.823343Z"},"trusted":true},"execution_count":60,"outputs":[{"name":"stdout","text":"(0.4497834390128143, [0.4790045080354355, 0.445047470578311, 0.40857737854453746, 0.45267682596283126, 0.4698106269184335, 0.4435838240373371])\n","output_type":"stream"}]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# all together","metadata":{}},{"cell_type":"code","source":"final_weights, _ = find_optimal_weights([fine_tuned_oof_preds, svr_preds], labels)","metadata":{"execution":{"iopub.status.busy":"2022-11-29T04:57:24.185781Z","iopub.execute_input":"2022-11-29T04:57:24.186300Z","iopub.status.idle":"2022-11-29T04:57:24.247736Z","shell.execute_reply.started":"2022-11-29T04:57:24.186260Z","shell.execute_reply":"2022-11-29T04:57:24.246316Z"},"trusted":true},"execution_count":70,"outputs":[]},{"cell_type":"code","source":"final_weights","metadata":{"execution":{"iopub.status.busy":"2022-11-29T04:57:25.424714Z","iopub.execute_input":"2022-11-29T04:57:25.425161Z","iopub.status.idle":"2022-11-29T04:57:25.434834Z","shell.execute_reply.started":"2022-11-29T04:57:25.425126Z","shell.execute_reply":"2022-11-29T04:57:25.433292Z"},"trusted":true},"execution_count":71,"outputs":[{"execution_count":71,"output_type":"execute_result","data":{"text/plain":"array([0.65498877, 0.34501123])"},"metadata":{}}]},{"cell_type":"code","source":"mc_rmse(labels, np.average([fine_tuned_oof_preds, svr_preds], axis=0, weights=final_weights))","metadata":{"scrolled":true,"execution":{"iopub.status.busy":"2022-11-29T04:57:28.069934Z","iopub.execute_input":"2022-11-29T04:57:28.070721Z","iopub.status.idle":"2022-11-29T04:57:28.087392Z","shell.execute_reply.started":"2022-11-29T04:57:28.070648Z","shell.execute_reply":"2022-11-29T04:57:28.085464Z"},"trusted":true},"execution_count":72,"outputs":[{"execution_count":72,"output_type":"execute_result","data":{"text/plain":"(0.4443898364196513,\n [0.47357347953631784,\n  0.43932546148809903,\n  0.40603249313580864,\n  0.4465081264094765,\n  0.463396342743972,\n  0.437503115204234])"},"metadata":{}}]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]}]}