lb-0-43-simple-ensemble-deberta-base-svr.ipynb

{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.7.12","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"# Introduction","metadata":{}},{"cell_type":"markdown","source":"* **TL;DR**: I ensemble a fine-tuned deberta-v3-base model (CV: 0.4571, LB: 0.44) with a cuml.SVR trained on embeddings extracted on 12 transformer models. This shows that we can easily achieve 0.43 without much training. \n* Dataset of embeddings can be found here: https://www.kaggle.com/datasets/quangphm/fb3embeddings\n* Reference: https://www.kaggle.com/code/cdeotte/rapids-svr-cv-0-450-lb-0-44x","metadata":{"execution":{"iopub.status.busy":"2022-11-05T15:28:45.533409Z","iopub.execute_input":"2022-11-05T15:28:45.533856Z","iopub.status.idle":"2022-11-05T15:28:45.560896Z","shell.execute_reply.started":"2022-11-05T15:28:45.533771Z","shell.execute_reply":"2022-11-05T15:28:45.559434Z"}}},{"cell_type":"code","source":"# ====================================================\n# Library\n# ====================================================\nimport os\nimport gc\nimport re\nimport ast\nimport sys\nimport copy\nimport json\nimport time\nimport math \nimport string\nimport pickle\nimport random\nimport joblib\nimport itertools\nimport warnings\nwarnings.filterwarnings(\"ignore\")\nfrom IPython. display import clear_output\n\nimport scipy as sp\nimport numpy as np\nimport pandas as pd\npd.set_option('display.max_rows', 500)\npd.set_option('display.max_columns', 500)\npd.set_option('display.width', 1000)\nfrom tqdm.auto import tqdm\nfrom sklearn.metrics import mean_squared_error\nfrom sklearn.model_selection import StratifiedKFold, GroupKFold, KFold\nfrom nltk.corpus import stopwords\nfrom nltk.tokenize import word_tokenize\n\nimport torch\nimport torch.nn as nn\nfrom torch.nn import Parameter\nimport torch.nn.functional as F\nfrom torch.optim import Adam, SGD, AdamW\nfrom torch.utils.data import DataLoader, Dataset\n\n#os.system('pip install iterative-stratification==0.1.7')\n#from iterstrat.ml_stratifiers import MultilabelStratifiedKFold\nsys.path.append('../input/iterativestratification')\nfrom iterstrat.ml_stratifiers import MultilabelStratifiedKFold\n\nos.system('pip uninstall -y transformers')\nos.system('pip uninstall -y tokenizers')\nos.system('python -m pip install --no-index --find-links=../input/fb3-my-pip-wheels transformers')\nos.system('python -m pip install --no-index --find-links=../input/fb3-my-pip-wheels tokenizers')\nimport tokenizers\nimport transformers\nfrom transformers import AutoTokenizer, AutoModel, AutoConfig\nfrom transformers import get_linear_schedule_with_warmup, get_cosine_schedule_with_warmup\nfrom transformers import DataCollatorWithPadding\n%env TOKENIZERS_PARALLELISM=false\n\nclear_output()\ndevice = torch.device('cuda' if torch.cuda.is_available() else 'cpu')\nprint(f\"tokenizers.__version__: {tokenizers.__version__}\")\nprint(f\"transformers.__version__: {transformers.__version__}\")\nprint('device:', device)","metadata":{"scrolled":true,"execution":{"iopub.status.busy":"2022-11-05T06:23:38.748549Z","iopub.execute_input":"2022-11-05T06:23:38.749485Z","iopub.status.idle":"2022-11-05T06:24:08.883457Z","shell.execute_reply.started":"2022-11-05T06:23:38.749369Z","shell.execute_reply":"2022-11-05T06:24:08.882313Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"BASE_PATH = '/kaggle/input/feedback-prize-english-language-learning'\nSUBMISSION_PATH = os.path.join(BASE_PATH, 'sample_submission.csv')\nTRAIN_PATH = os.path.join(BASE_PATH, 'train.csv')\nTEST_PATH = os.path.join(BASE_PATH, 'test.csv')","metadata":{"execution":{"iopub.status.busy":"2022-11-05T06:24:08.889408Z","iopub.execute_input":"2022-11-05T06:24:08.889756Z","iopub.status.idle":"2022-11-05T06:24:08.895986Z","shell.execute_reply.started":"2022-11-05T06:24:08.889724Z","shell.execute_reply":"2022-11-05T06:24:08.894542Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"class Config(dict):\n    __getattr__ = dict.__getitem__\n    __setattr__ = dict.__setitem__\n    __delattr__ = dict.__delitem__\n    \n    def init(self, kwargs):\n        super().init(kwargs)\n\n        for k, v in kwargs.items():\n            setattr(self, k, v)\n\n    def set(self, key, val):\n        self[key] = val\n        setattr(self, key, val)\n        \ndef get_logger(filename='inference'):\n    from logging import getLogger, INFO, StreamHandler, FileHandler, Formatter\n    logger = getLogger(__name__)\n    logger.propagate = False\n    logger.setLevel(INFO)\n    handler1 = StreamHandler()\n    handler1.setFormatter(Formatter(\"%(message)s\"))\n    handler2 = FileHandler(filename=f\"{filename}.log\")\n    handler2.setFormatter(Formatter(\"%(message)s\"))\n    logger.addHandler(handler1)\n    logger.addHandler(handler2)\n    return logger\n\nLOGGER = get_logger()","metadata":{"execution":{"iopub.status.busy":"2022-11-05T06:24:08.898067Z","iopub.execute_input":"2022-11-05T06:24:08.898675Z","iopub.status.idle":"2022-11-05T06:24:08.916006Z","shell.execute_reply.started":"2022-11-05T06:24:08.898485Z","shell.execute_reply":"2022-11-05T06:24:08.914647Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def seed_everything(seed=42):\n    '''\n    Sets the seed of the entire notebook so results are the same every time we run.\n    This is for REPRODUCIBILITY.\n    '''\n    random.seed(seed)\n    # Set a fixed value for the hash seed\n    os.environ['PYTHONHASHSEED'] = str(seed)\n    np.random.seed(seed)\n    torch.manual_seed(seed)\n    \n    if torch.cuda.is_available():\n        torch.cuda.manual_seed(seed)\n        torch.cuda.manual_seed_all(seed)\n        # When running on the CuDNN backend, two further options must be set\n        torch.backends.cudnn.deterministic = True\n        torch.backends.cudnn.benchmark = False\nseed_everything(seed=42)\n\ndef mc_rmse(y_true, y_pred):\n    scores = []\n    ncols = y_true.shape[1]\n    \n    for n in range(ncols):\n        yn_true = y_true[:, n]\n        yn_pred = y_pred[:, n]\n        rmse_ = mean_squared_error(yn_true, yn_pred, squared=False)\n        scores.append(rmse_)\n    score = np.mean(scores) \n    return score, scores\n\ndef get_result(cfg, oof_df):\n    labels = oof_df[cfg.target_cols].values\n    preds = oof_df[[f\"pred_{c}\" for c in cfg.target_cols]].values\n    score, scores = mc_rmse(labels, preds)\n    print(f'score: {score:<.6f}  scores: {scores}')","metadata":{"execution":{"iopub.status.busy":"2022-11-05T06:24:08.919159Z","iopub.execute_input":"2022-11-05T06:24:08.919547Z","iopub.status.idle":"2022-11-05T06:24:08.931610Z","shell.execute_reply.started":"2022-11-05T06:24:08.919507Z","shell.execute_reply":"2022-11-05T06:24:08.930451Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# ====================================================\n# Dataset\n# ====================================================\ndef encode_text(cfg, text):\n    if cfg.pretrained:\n        inputs = cfg.tokenizer(\n            text,\n            None,\n            add_special_tokens=True,\n            padding='max_length',\n            truncation=True,\n            max_length=cfg.max_len,\n            return_token_type_ids=True,\n            return_tensors='pt'\n        )\n        inputs = {k:v.squeeze(0) for k,v in inputs.items()}\n    else:\n        inputs = cfg.tokenizer.encode_plus(\n            text, \n            return_tensors=None, \n            add_special_tokens=True, \n            #max_length=CFG.max_len,\n            #pad_to_max_length=True,\n            #truncation=True\n        )\n        for k, v in inputs.items():\n            inputs[k] = torch.tensor(v, dtype=torch.long)\n    return inputs \n\ndef preprocess(texts):\n    texts = (\n        texts\n        .str.replace(r'\\r\\n', '<newline>', regex=True)\n        .str.replace(r'\\n', '<newline>', regex=True)\n        .str.replace('<newline><newline>', '<newline>', regex=False)\n        .values \n    )\n    return texts\n\nclass TestDataset(Dataset):\n    def __init__(self, cfg, df):\n        self.cfg = cfg\n        if cfg.pretrained:\n            self.texts = df['full_text'].values\n        else:\n            self.texts = preprocess(df['full_text'])\n\n    def __len__(self):\n        return len(self.texts)\n\n    def __getitem__(self, item):\n        inputs = encode_text(self.cfg, self.texts[item])\n        return inputs","metadata":{"execution":{"iopub.status.busy":"2022-11-05T06:43:23.958136Z","iopub.execute_input":"2022-11-05T06:43:23.959183Z","iopub.status.idle":"2022-11-05T06:43:23.971583Z","shell.execute_reply.started":"2022-11-05T06:43:23.959143Z","shell.execute_reply":"2022-11-05T06:43:23.970358Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"class AttentionHead(nn.Module):\n    def __init__(self, in_features, hidden_dim):\n        super().__init__()\n        self.in_features = in_features\n        self.middle_features = hidden_dim\n        self.W = nn.Linear(in_features, hidden_dim)\n        self.V = nn.Linear(hidden_dim, 1)\n        self.out_features = hidden_dim\n\n    def forward(self, features, *args):\n        att = torch.tanh(self.W(features))\n        score = self.V(att)\n        attention_weights = torch.softmax(score, dim=1)\n        context_vector = attention_weights * features\n        context_vector = torch.sum(context_vector, dim=1)\n\n        return context_vector\n    \nclass WeightedLayerPooling(nn.Module):\n    def __init__(self, num_hidden_layers, layer_start: int = 9, layer_weights = None):\n        super(WeightedLayerPooling, self).__init__()\n        self.layer_start = layer_start\n        self.num_hidden_layers = num_hidden_layers\n        self.layer_weights = layer_weights if layer_weights is not None \\\n            else nn.Parameter(\n                torch.tensor([1] * (num_hidden_layers+1 - layer_start), dtype=torch.float))\n        \n    def forward(self, all_hidden_states):\n        all_layer_embedding = all_hidden_states[self.layer_start:, :, :, :]\n        weight_factor = self.layer_weights.unsqueeze(-1).unsqueeze(-1).unsqueeze(-1).expand(all_layer_embedding.size())\n        weighted_average = (weight_factor*all_layer_embedding).sum(dim=0) / self.layer_weights.sum()\n        return weighted_average\n    \nclass FB3Model(nn.Module):\n    def __init__(self, cfg, config_path=None, pretrained=False):\n        super().__init__()\n        self.cfg = cfg \n        if config_path is None:\n            self.config = AutoConfig.from_pretrained(cfg.model, output_hidden_states=True)\n            # Turn off dropouts.\n            self.config.hidden_dropout = 0.\n            self.config.hidden_dropout_prob = 0.\n            self.config.attention_dropout = 0.\n            self.config.attention_probs_dropout_prob = 0.\n            #LOGGER.info(self.config)\n        else:\n            self.config = torch.load(config_path)\n        \n        if pretrained:\n            self.deberta_v3 = AutoModel.from_pretrained(cfg.model, config=self.config)\n        else:\n            self.deberta_v3 = AutoModel.from_config(self.config)\n\n        if self.cfg.reinit_last_layer:\n            # Re-init last layer of deberta.\n            for module in self.deberta_v3.encoder.layer[-1:].modules():\n                self._init_weights(module)\n        self.deberta_v3.gradient_checkpointing_enable()\n\n    def _init_weights(self, module):\n        if isinstance(module, nn.Linear):\n            module.weight.data.normal_(mean=0.0, std=self.config.initializer_range)\n            #nn.init.xavier_uniform_(module.weight.data, gain=1.0)\n            if module.bias is not None:\n                module.bias.data.zero_()\n        elif isinstance(module, nn.Embedding):\n            module.weight.data.normal_(mean=0.0, std=self.config.initializer_range)\n            if module.padding_idx is not None:\n                module.weight.data[module.padding_idx].zero_()\n        elif isinstance(module, nn.LayerNorm):\n            module.bias.data.zero_()\n            module.weight.data.fill_(1.0)\n\nclass WMPoolModel(FB3Model):\n    def __init__(self, cfg, config_path=None, pretrained=False):\n        super().__init__(cfg, config_path=config_path, pretrained=pretrained)\n\n        # Poolings.\n        self.wpool_head = WeightedLayerPooling(self.config.num_hidden_layers, layer_start=12)\n\n        self.fc_out = nn.Linear(self.config.hidden_size, cfg.num_target)\n        self._init_weights(self.fc_out)\n        \n        self.layer_norm = nn.LayerNorm(self.config.hidden_size)\n        self.qa_output = torch.nn.Linear(self.config.hidden_size, 2)\n        self.attention_head = AttentionHead(self.config.hidden_size*4, self.config.hidden_size)\n        \n    def forward(self, x):\n        pt_out = self.deberta_v3(**x)\n        all_hidden_states = torch.stack(pt_out.hidden_states)\n        # Weighted pooling of last n layers.\n        logits = self.wpool_head(all_hidden_states)[:, 0] # Bx768\n        y_hat = self.fc_out(logits)\n        return y_hat\n    \n","metadata":{"execution":{"iopub.status.busy":"2022-11-05T06:43:25.964975Z","iopub.execute_input":"2022-11-05T06:43:25.965819Z","iopub.status.idle":"2022-11-05T06:43:25.988501Z","shell.execute_reply.started":"2022-11-05T06:43:25.965778Z","shell.execute_reply":"2022-11-05T06:43:25.987679Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def load_config(input_path, inference_weight=1):\n    # Load CFG class.\n    cfg = Config(**json.load(open(os.path.join(input_path, 'CFG.json'), 'r')))\n    cfg.path = input_path\n    cfg.config_path = os.path.join(cfg.path, 'config.pth')\n    # Load tokenizer.\n    tokenizer = AutoTokenizer.from_pretrained(os.path.join(cfg.path, 'tokenizer'))\n    cfg.tokenizer = tokenizer\n    \n    cfg.inference_weight = inference_weight\n    return cfg\n\n\ndef load_model(cfg, fold, **model_kwargs):\n    # Load torch model.\n    model = WMPoolModel(cfg, config_path=cfg.config_path, pretrained=False, **model_kwargs)\n    state = torch.load(\n        os.path.join(cfg.path, f\"{cfg.model.replace('/', '-')}_fold{fold}_best.pth\"),\n        map_location=torch.device('cpu'))\n    model.load_state_dict(state['model'])\n    return model","metadata":{"execution":{"iopub.status.busy":"2022-11-05T06:43:27.875509Z","iopub.execute_input":"2022-11-05T06:43:27.876385Z","iopub.status.idle":"2022-11-05T06:43:27.885078Z","shell.execute_reply.started":"2022-11-05T06:43:27.876347Z","shell.execute_reply":"2022-11-05T06:43:27.883798Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def mean_pooling(model_output, attention_mask):\n    token_embeddings = model_output.last_hidden_state.detach().cpu()\n    input_mask_expanded = (\n        attention_mask.unsqueeze(-1).expand(token_embeddings.size()).float()\n    )\n    return torch.sum(token_embeddings * input_mask_expanded, 1) / torch.clamp(\n        input_mask_expanded.sum(1), min=1e-9\n    )\n\ndef inference_fn(test_loader, model, device):\n    preds = []\n    model.eval()\n    model.to(device)\n    #tk0 = tqdm(test_loader, total=len(test_loader))\n    for inputs in test_loader:\n        for k, v in inputs.items():\n            inputs[k] = v.to(device)\n        with torch.no_grad():\n            y_preds = model(inputs)\n        preds.append(y_preds.to('cpu').numpy())\n    predictions = np.concatenate(preds)\n    return predictions\n\nclass Inferencer:\n    def __init__(self, input_path=None, cfg=None, inference_weight=1):\n        if cfg == None:\n            self.cfg = load_config(input_path, inference_weight)\n        else:\n            self.cfg = cfg\n    \n    def predict(self, test_loader, device, stat_fn=np.mean):\n        preds = []\n        start = time.time()\n        print('#'*10, cfg.path, '#'*10)\n        for fold in self.cfg.trn_fold:\n            print(f'Predicting fold {fold}...')\n            model = load_model(self.cfg, fold)\n            pred = inference_fn(test_loader, model, device)\n            preds.append(pred)\n            del model, pred; gc.collect()\n            torch.cuda.empty_cache()\n        end = time.time() - start\n        print('#'*10, f'ETA: {end:.2f}s', '#'*10, '\\n')\n        \n        self.preds = stat_fn(preds, axis=0) \n        self.preds = np.clip(self.preds, 1, 5)\n        return self.preds\n    \n    def get_oof_result(self):\n        return get_result(pd.read_pickle(os.path.join(cfg.path, 'oof_df.pkl')))\n    \n    def get_text_embedding(self, data_loader, device, fold=None): \n        # pretrained=True: not fine-tuned models.\n        if not self.cfg.pretrained:\n            model = load_model(self.cfg, fold, pool=self.cfg.pool_head)            \n        else:\n            model = AutoModel.from_pretrained(self.cfg.model)\n        model.to(device)\n        model.eval()\n            \n        fold_emb = []\n        for inputs in data_loader:\n            for k, v in inputs.items():\n                inputs[k] = v.to(device)\n            if not self.cfg.pretrained:\n                with torch.no_grad():\n                    emb = model.feature(**inputs)\n            else:\n                input_ids = inputs['input_ids'].to(device)\n                attention_mask = inputs['attention_mask'].to(device)\n                token_type_ids = inputs['token_type_ids'].to(device)\n                \n                with torch.no_grad():\n                    output = model(input_ids=input_ids, attention_mask=attention_mask, token_type_ids=token_type_ids)\n                emb = mean_pooling(output, attention_mask.detach().cpu())\n                emb = F.normalize(emb, p=2, dim=1)\n                emb = emb.squeeze(0)\n            fold_emb.extend(emb.detach().cpu().numpy())\n            del emb; gc.collect(); torch.cuda.empty_cache();\n            #print(torch.cuda.memory_allocated() /1024/1024)\n            \n        fold_emb = np.array(fold_emb)\n        return fold_emb","metadata":{"execution":{"iopub.status.busy":"2022-11-05T06:43:29.284336Z","iopub.execute_input":"2022-11-05T06:43:29.284726Z","iopub.status.idle":"2022-11-05T06:43:29.305875Z","shell.execute_reply.started":"2022-11-05T06:43:29.284695Z","shell.execute_reply":"2022-11-05T06:43:29.304641Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# svr + pretrained embeddings\n* Listed below are more than 20 transformers but I only use 12 for this submission. \n* Using the method `get_text_embedding` from `Inferencer` you can create your own embeddings.","metadata":{}},{"cell_type":"code","source":"##################################################\ndeberta_base = Config(\n    model='../input/huggingface-deberta-variants/deberta-base/deberta-base',\n    file_name='microsoft_deberta_base_768',\n    pretrained=True, inference_weight=1, max_len=640) #\ndeberta_large = Config(\n    model='../input/huggingface-deberta-variants/deberta-large/deberta-large', \n    file_name='microsoft_deberta_large_1024',\n    pretrained=True, inference_weight=1, max_len=640) #\ndeberta_xlarge = Config(\n    model='../input/huggingface-deberta-variants/deberta-xlarge/deberta-xlarge', \n    file_name='microsoft_deberta_xlarge_1024',\n    pretrained=True, inference_weight=1, max_len=640)\ndeberta_v2_xlarge = Config(\n    model='../input/bert-shopping-mall/deberta-v2-xlarge', \n    file_name='microsoft_deberta_v2_xlarge_1536',\n    pretrained=True, inference_weight=1, max_len=640)\ndeberta_v2_xxlarge = Config(\n    model='../input/bert-shopping-mall/deberta-v2-xxlarge', \n    file_name='microsoft_deberta_v2_xxlarge_1536',\n    pretrained=True, inference_weight=1, max_len=640)\n\ndeberta_v3_base = Config(\n    model='../input/bert-shopping-mall/deberta-v3-base',\n    file_name='microsoft_deberta_v3_base_768',\n    pretrained=True, inference_weight=1, max_len=640) #\ndeberta_v3_large = Config(\n    model='../input/bert-shopping-mall/deberta-v3-large', \n    file_name='microsoft_deberta_v3_large_1024',\n    pretrained=True, inference_weight=1, max_len=640) # \n\ndeberta_large_mnli = Config(\n    model='../input/huggingface-deberta-variants/deberta-large-mnli/deberta-large-mnli',\n    file_name='microsoft_deberta_large_mnli_1024',\n    pretrained=True, inference_weight=1, max_len=640) # \n\ngpt2 = Config(\n    model='../input/hugging-face-gpt2/gpt2',\n    file_name='gpt2_768',\n    pretrained=True, inference_weight=1, max_len=512) #\n\nroberta_base = Config(\n    model='../input/transformers/roberta-base', \n    file_name='roberta_base_768',\n    pretrained=True, inference_weight=1, max_len=512) #\nroberta_large = Config(\n    model='../input/transformers/roberta-large',\n    file_name='roberta_large_1024',\n    pretrained=True, inference_weight=1, max_len=512) # \n\nxlnet_base = Config(\n    model='../input/transformers/xlnet-base-cased',\n    file_name='xlnet_base_cased_768',\n    pretrained=True, inference_weight=1, max_len=640) #\nxlnet_large = Config(\n    model='../input/transformers/xlnet-large-cased', \n    file_name='xlnet_large_cased_1024',\n    pretrained=True, inference_weight=1, max_len=640) #\n\nbart_base = Config(\n    model='../input/transformers/facebook-bart-base',\n    file_name='facebook_bart_base_768',\n    pretrained=True, inference_weight=1, max_len=640)\nbart_large = Config(\n    model='../input/transformers/facebook-bart-large',\n    file_name='facebook_bart_large_1024',\n    pretrained=True, inference_weight=1, max_len=640)\nbart_lage_mnli = Config(\n    model='../input/facebook-bart-large-mnli',\n    file_name='facebook_bart_large_mnli_1024',\n    pretrained=True, inference_weight=1, max_len=640)\n\nbert_base_uncased = Config(\n    model='../input/transformers/bert-base-uncased/',\n    file_name='bert_base_uncased_768',\n    pretrained=True, inference_weight=1, max_len=512)\nbert_large_uncased = Config(\n    model='../input/transformers/bert-large-uncased',\n    file_name='bert_large_uncased_1024',\n    pretrained=True, inference_weight=1, max_len=512)\n\nmuppet_roberta_large = Config(\n    model='../input/muppet-roberta-large',\n    file_name='facebook_muppet_roberta_large_1024',\n    pretrained=True, inference_weight=1, max_len=512)\n\nfunnel_small = Config(\n    model='../input/transformers/funnel-transformer-small',\n    file_name='funnel_transformer_small_768',\n    pretrained=True, inference_weight=1, max_len=640)\nfunnel_large = Config(\n    model='../input/transformers/funnel-transformer-large',\n    file_name='funnel_transformer_large_1024',\n    pretrained=True, inference_weight=1, max_len=640)\n\n##################################################\n\ntarget_cols = ['cohesion', 'syntax', 'vocabulary', 'phraseology', 'grammar', 'conventions']","metadata":{"execution":{"iopub.status.busy":"2022-11-05T06:43:31.605033Z","iopub.execute_input":"2022-11-05T06:43:31.605414Z","iopub.status.idle":"2022-11-05T06:43:31.623031Z","shell.execute_reply.started":"2022-11-05T06:43:31.605365Z","shell.execute_reply":"2022-11-05T06:43:31.621839Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## load embeddings","metadata":{}},{"cell_type":"code","source":"from sklearn.metrics import make_scorer\nfrom joblib import dump, load\nfrom sklearn.gaussian_process import GaussianProcessRegressor\nfrom sklearn.gaussian_process.kernels import RBF\nfrom sklearn.ensemble import StackingRegressor\nfrom sklearn.multioutput import MultiOutputRegressor\nfrom sklearn.linear_model import RidgeCV, Ridge, Lasso\nfrom sklearn.metrics import mean_squared_error\nfrom lightgbm import LGBMRegressor\n\nif str(device) == 'cpu':\n    from sklearn.svm import SVR\nelse:\n    from cuml.svm import SVR\n    import cuml\ndevice","metadata":{"execution":{"iopub.status.busy":"2022-11-05T06:24:09.035714Z","iopub.execute_input":"2022-11-05T06:24:09.036242Z","iopub.status.idle":"2022-11-05T06:24:09.566017Z","shell.execute_reply.started":"2022-11-05T06:24:09.036206Z","shell.execute_reply":"2022-11-05T06:24:09.564862Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train = pd.read_csv(TRAIN_PATH)\ntest = pd.read_csv(TEST_PATH)","metadata":{"execution":{"iopub.status.busy":"2022-11-05T06:24:09.567572Z","iopub.execute_input":"2022-11-05T06:24:09.567918Z","iopub.status.idle":"2022-11-05T06:24:09.673799Z","shell.execute_reply.started":"2022-11-05T06:24:09.567887Z","shell.execute_reply":"2022-11-05T06:24:09.672639Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import sys\nsys.path.append('../input/iterativestratification')\nfrom iterstrat.ml_stratifiers import MultilabelStratifiedKFold\nsvr_folds = 15\n\nskf = MultilabelStratifiedKFold(n_splits=svr_folds, shuffle=True, random_state=42)\nfor i,(train_index, val_index) in enumerate(skf.split(train,train[target_cols])):\n    train.loc[val_index,'fold'] = i","metadata":{"scrolled":true,"execution":{"iopub.status.busy":"2022-11-05T06:24:09.675556Z","iopub.execute_input":"2022-11-05T06:24:09.676059Z","iopub.status.idle":"2022-11-05T06:24:09.837955Z","shell.execute_reply.started":"2022-11-05T06:24:09.676015Z","shell.execute_reply":"2022-11-05T06:24:09.836647Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from glob import glob \n\ndef get_text_embedding(cfg, dfs):\n    # Simply load saved embeddings for training.\n    cfg.tokenizer = AutoTokenizer.from_pretrained(cfg.model)\n    infer_ = Inferencer(cfg=cfg, inference_weight=cfg.inference_weight)\n    if cfg.model == 'gpt2':\n        cfg.tokenizer.pad_token = cfg.tokenizer.eos_token\n    text_embs = []\n    for df in dfs:\n        dataset = TestDataset(cfg, df)\n        loader = DataLoader(\n            dataset,\n            batch_size=4,\n            shuffle=False)\n\n        # Text embedding for SVM\n        test_text_emb = []\n        if not cfg.pretrained:\n            for fold in infer_.cfg.trn_fold:\n                test_text_emb.append(infer_.get_text_embedding(loader, device, fold))\n            text_emb = np.mean(text_emb, axis=0)\n        else:\n            text_emb = infer_.get_text_embedding(loader, device)\n        text_embs.append(text_emb)\n        del dataset, loader; gc.collect(); torch.cuda.empty_cache();\n    del infer_; gc.collect(); torch.cuda.empty_cache();\n    return text_embs\n\ndef learner_cv(features, learner, folds=15, save=False, verbose=False):\n    scores = []\n    for fold in range(folds):\n        dftr_ = train[train['fold']!=fold]\n        dfev_ = train[train['fold']==fold]\n\n        tr_text_feats = features[list(dftr_.index),:]\n        ev_text_feats = features[list(dfev_.index),:]\n\n        # clf = MultiOutputRegressor(SVR(C=2.0))\n        clf = MultiOutputRegressor(learner)\n        clf.fit(tr_text_feats, dftr_[target_cols].values)\n        ev_preds = clf.predict(ev_text_feats)\n\n        score,_ = mc_rmse(dfev_[target_cols].values, ev_preds)\n        scores.append(score)\n\n        if verbose:\n            print('#'*25)\n            print('### Fold',fold+1)\n            print(\"Score: {}\".format(score))\n        if save:\n            dump(clf, f'svr_{fold}.model')\n\n    # print('#'*25)\n    # print('Overall CV =', np.mean(scores))\n    return np.mean(scores)\n\ndef get_learner_score(models_cfg, learner, folds=5, save=False, verbose=False):\n    for i, model_cfg in enumerate(models_cfg):\n        model_name = model_cfg.model.split('/')[-1].replace('-', '_')\n        models_cfg[i].model_name = model_name\n        model_file = f'../input/fb3embeddings/train_text_emb_{model_cfg.file_name}.npy'\n        if 'embedding' in model_cfg:\n            continue\n        with open(model_file, 'rb') as f:\n            models_cfg[i].embedding = np.load(f)   \n    embeddings = np.concatenate([model_cfg.embedding for model_cfg in models_cfg], axis=1)\n    svr_score = learner_cv(embeddings, learner, folds=folds, save=save, verbose=verbose)\n    print('\\n')\n    print(f'model_set={[m.model_name for m in models_cfg]};   score={svr_score}')\n    return svr_score, models_cfg","metadata":{"execution":{"iopub.status.busy":"2022-11-05T06:24:09.842514Z","iopub.execute_input":"2022-11-05T06:24:09.842922Z","iopub.status.idle":"2022-11-05T06:24:09.861410Z","shell.execute_reply.started":"2022-11-05T06:24:09.842884Z","shell.execute_reply":"2022-11-05T06:24:09.860061Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## models selection\n* Simple hill climbing algorithm to find the (somewhat) best set of models to train SVR on.","metadata":{}},{"cell_type":"code","source":"# model_selection = False","metadata":{"execution":{"iopub.status.busy":"2022-11-05T06:24:09.863540Z","iopub.execute_input":"2022-11-05T06:24:09.863995Z","iopub.status.idle":"2022-11-05T06:24:09.872617Z","shell.execute_reply.started":"2022-11-05T06:24:09.863949Z","shell.execute_reply":"2022-11-05T06:24:09.871569Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# if model_selection:\n#     for i, first_model in enumerate(tqdm(pretrained_models_cfg[1:])):\n#         features = [first_model]\n#         prev_score,_ = get_learner_score(features)\n#         cur_score = 0\n        \n#         while True:\n#             models = [feat.model for feat in features]\n#             if len(models) == len(pretrained_models_cfg):\n#                 break\n                \n#             scores_and_cfgs = [get_learner_score(features + [feat], folds=15, save=False) for feat in pretrained_models_cfg if feat.model not in models]\n#             scores = [s for s,c in scores_and_cfgs]\n#             cur_features = [c for s,c in scores_and_cfgs]\n            \n#             cur_score = np.min(scores)\n#             cur_best_feature = cur_features[np.argmin(scores)][-1]\n#             features.append(cur_best_feature)\n            \n#             if prev_score < cur_score:\n#                 break\n#             prev_score = cur_score\n\n#             del scores_and_cfgs, scores, cur_best_feature, cur_features; gc.collect(); torch.cuda.empty_cache();\n        \n#         LOGGER.info(f'Interation {i+1}:')\n#         LOGGER.info(f'model_set={[c.model_name for c in features]} \\nbest_score={cur_score}')\n#         LOGGER.info('#'*50)\n#         LOGGER.info('\\n')","metadata":{"scrolled":true,"execution":{"iopub.status.busy":"2022-11-05T06:24:09.874196Z","iopub.execute_input":"2022-11-05T06:24:09.874729Z","iopub.status.idle":"2022-11-05T06:24:09.883496Z","shell.execute_reply.started":"2022-11-05T06:24:09.874683Z","shell.execute_reply":"2022-11-05T06:24:09.882544Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## train","metadata":{}},{"cell_type":"code","source":"pretrained_models_cfg = [\n    deberta_large_mnli,\n    #gpt2,\n    roberta_base,\n    roberta_large,\n    xlnet_base, \n    xlnet_large,\n    deberta_base, \n    deberta_large, \n    deberta_xlarge,\n    deberta_v2_xlarge, \n    deberta_v2_xxlarge,\n    deberta_v3_base, \n    deberta_v3_large,\n]","metadata":{"execution":{"iopub.status.busy":"2022-11-05T06:24:09.884656Z","iopub.execute_input":"2022-11-05T06:24:09.885097Z","iopub.status.idle":"2022-11-05T06:24:09.899768Z","shell.execute_reply.started":"2022-11-05T06:24:09.885050Z","shell.execute_reply":"2022-11-05T06:24:09.898324Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# learner = Ridge(alpha=2.0)\nlearner = SVR(C=2.0)\nsvr_score, models_cfg = get_learner_score(pretrained_models_cfg, learner, folds=svr_folds, save=True, verbose=True)","metadata":{"scrolled":true,"execution":{"iopub.status.busy":"2022-11-05T06:24:09.902574Z","iopub.execute_input":"2022-11-05T06:24:09.903485Z","iopub.status.idle":"2022-11-05T06:27:15.951367Z","shell.execute_reply.started":"2022-11-05T06:24:09.903418Z","shell.execute_reply":"2022-11-05T06:27:15.949741Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## predict","metadata":{}},{"cell_type":"code","source":"all_test_text_emb = []\nfor cfg in tqdm(pretrained_models_cfg):\n    test_text_emb = get_text_embedding(cfg, [test])[0]\n    all_test_text_emb.append(test_text_emb)\n    \n    del test_text_emb; gc.collect(); torch.cuda.empty_cache();\n    print(f'{cfg.model} loaded.')\n    \ngc.collect(); torch.cuda.empty_cache();\n\nfinal_test_text_emb = np.concatenate(all_test_text_emb, axis=1)","metadata":{"execution":{"iopub.status.busy":"2022-11-05T06:43:46.305242Z","iopub.execute_input":"2022-11-05T06:43:46.305677Z","iopub.status.idle":"2022-11-05T06:52:13.066724Z","shell.execute_reply.started":"2022-11-05T06:43:46.305639Z","shell.execute_reply":"2022-11-05T06:52:13.065515Z"},"scrolled":true,"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import glob\ndef svr_inference_fn(model_path, te_text_feats):\n    model = load(model_path)\n    preds = model.predict(te_text_feats)\n    return preds\n\npredictions = []\nsvr_model_paths = glob.glob('./*.model')\nfor model_path in tqdm(svr_model_paths):\n    #model_path = os.path.join('../input/fb3-svr-train/', model_path)\n    preds = svr_inference_fn(model_path, final_test_text_emb)\n    predictions.append(preds)\nsvr_predictions = np.mean(predictions, axis=0)\nsvr_predictions\n","metadata":{"execution":{"iopub.status.busy":"2022-11-05T06:52:55.394653Z","iopub.execute_input":"2022-11-05T06:52:55.395258Z","iopub.status.idle":"2022-11-05T06:52:55.620195Z","shell.execute_reply.started":"2022-11-05T06:52:55.395215Z","shell.execute_reply":"2022-11-05T06:52:55.617996Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"---","metadata":{}},{"cell_type":"markdown","source":"# fine-tuned models","metadata":{"execution":{"iopub.status.busy":"2022-11-05T06:16:01.717240Z","iopub.execute_input":"2022-11-05T06:16:01.717783Z","iopub.status.idle":"2022-11-05T06:16:01.737870Z","shell.execute_reply.started":"2022-11-05T06:16:01.717669Z","shell.execute_reply":"2022-11-05T06:16:01.736820Z"}}},{"cell_type":"code","source":"v21_CFG = load_config('../input/fb3models/v21/', inference_weight=1)\nv21_CFG.version = '21'\n\ntokenizer = AutoTokenizer.from_pretrained(os.path.join(v21_CFG.path, 'tokenizer'))\nv21_CFG.tokenizer = tokenizer\nv21_CFG.pretrained = False","metadata":{"execution":{"iopub.status.busy":"2022-11-05T06:27:15.953857Z","iopub.execute_input":"2022-11-05T06:27:15.954882Z","iopub.status.idle":"2022-11-05T06:27:16.795159Z","shell.execute_reply.started":"2022-11-05T06:27:15.954817Z","shell.execute_reply":"2022-11-05T06:27:16.793860Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test = pd.read_csv(TEST_PATH)","metadata":{"execution":{"iopub.status.busy":"2022-11-05T06:27:16.796710Z","iopub.execute_input":"2022-11-05T06:27:16.797222Z","iopub.status.idle":"2022-11-05T06:27:16.805895Z","shell.execute_reply.started":"2022-11-05T06:27:16.797150Z","shell.execute_reply":"2022-11-05T06:27:16.804675Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"fine_tuned_models_cfg = [v21_CFG]\n\nfine_tuned_predictions = []\ntotal_weight = 0\nfor cfg in tqdm(fine_tuned_models_cfg):\n    # infer_ = Inferencer(setup['path'], setup['inference_weight'])\n    infer_ = Inferencer(cfg=cfg, inference_weight=cfg.inference_weight)\n    \n    test_dataset = TestDataset(cfg, test)\n    test_loader = DataLoader(\n        test_dataset,\n        batch_size=3,\n        shuffle=False,\n        collate_fn=DataCollatorWithPadding(tokenizer=cfg.tokenizer, padding='longest'),\n        num_workers=1, \n        pin_memory=True, \n        drop_last=False)\n    prediction = infer_.predict(test_loader, device) * cfg.inference_weight\n    \n    fine_tuned_predictions.append(prediction)\n    total_weight += cfg.inference_weight\n    \n    del infer_, test_dataset, test_loader, prediction; gc.collect; torch.cuda.empty_cache();\n    \nfinal_fine_tuned_predictions = np.sum(fine_tuned_predictions, axis=0)/total_weight    \nfinal_fine_tuned_predictions","metadata":{"execution":{"iopub.status.busy":"2022-11-05T06:27:16.807315Z","iopub.execute_input":"2022-11-05T06:27:16.807647Z","iopub.status.idle":"2022-11-05T06:28:36.406924Z","shell.execute_reply.started":"2022-11-05T06:27:16.807617Z","shell.execute_reply":"2022-11-05T06:28:36.405415Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# combine & submit","metadata":{}},{"cell_type":"code","source":"# Simply average two predictions.\nfinal_predictions = (svr_predictions + final_fine_tuned_predictions)/2\nfinal_predictions = np.clip(final_predictions, 1, 5)\ntest[target_cols] = final_predictions","metadata":{"execution":{"iopub.status.busy":"2022-11-05T06:53:00.025627Z","iopub.execute_input":"2022-11-05T06:53:00.026356Z","iopub.status.idle":"2022-11-05T06:53:00.037864Z","shell.execute_reply.started":"2022-11-05T06:53:00.026310Z","shell.execute_reply":"2022-11-05T06:53:00.036618Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"submission = pd.read_csv(SUBMISSION_PATH)\nsubmission = submission.drop(columns=target_cols).merge(test[['text_id'] + target_cols], on='text_id', how='left')\ndisplay(submission.head())\nsubmission[['text_id'] + target_cols].to_csv('submission.csv', index=False)","metadata":{"execution":{"iopub.status.busy":"2022-11-05T06:53:00.195918Z","iopub.execute_input":"2022-11-05T06:53:00.197293Z","iopub.status.idle":"2022-11-05T06:53:00.248347Z","shell.execute_reply.started":"2022-11-05T06:53:00.197239Z","shell.execute_reply":"2022-11-05T06:53:00.247335Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]}]}