finetune.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "a47ff888",
   "metadata": {
    "_cell_guid": "b1076dfc-b9ad-4769-8c92-a6c4dae69d19",
    "_uuid": "8f2839f25d086af736a60e9eeb907d3b93b6e0e5",
    "execution": {
     "iopub.execute_input": "2023-11-22T05:35:44.669686Z",
     "iopub.status.busy": "2023-11-22T05:35:44.669305Z",
     "iopub.status.idle": "2023-11-22T05:38:56.841111Z",
     "shell.execute_reply": "2023-11-22T05:38:56.839973Z"
    },
    "papermill": {
     "duration": 192.178389,
     "end_time": "2023-11-22T05:38:56.843458",
     "exception": false,
     "start_time": "2023-11-22T05:35:44.665069",
     "status": "completed"
    },
    "scrolled": true,
    "tags": []
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Found existing installation: torch 2.0.0\r\n",
      "Uninstalling torch-2.0.0:\r\n",
      "  Successfully uninstalled torch-2.0.0\r\n",
      "Looking in indexes: https://download.pytorch.org/whl/cu118\r\n",
      "Collecting torch==2.0.1\r\n",
      "  Downloading https://download.pytorch.org/whl/cu118/torch-2.0.1%2Bcu118-cp310-cp310-linux_x86_64.whl (2267.3 MB)\r\n",
      "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m2.3/2.3 GB\u001b[0m \u001b[31m487.6 kB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\r\n",
      "\u001b[?25hRequirement already satisfied: filelock in /opt/conda/lib/python3.10/site-packages (from torch==2.0.1) (3.12.2)\r\n",
      "Requirement already satisfied: typing-extensions in /opt/conda/lib/python3.10/site-packages (from torch==2.0.1) (4.5.0)\r\n",
      "Requirement already satisfied: sympy in /opt/conda/lib/python3.10/site-packages (from torch==2.0.1) (1.12)\r\n",
      "Requirement already satisfied: networkx in /opt/conda/lib/python3.10/site-packages (from torch==2.0.1) (3.1)\r\n",
      "Requirement already satisfied: jinja2 in /opt/conda/lib/python3.10/site-packages (from torch==2.0.1) (3.1.2)\r\n",
      "Collecting triton==2.0.0 (from torch==2.0.1)\r\n",
      "  Downloading https://download.pytorch.org/whl/triton-2.0.0-1-cp310-cp310-manylinux2014_x86_64.manylinux_2_17_x86_64.whl (63.3 MB)\r\n",
      "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m63.3/63.3 MB\u001b[0m \u001b[31m19.4 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\r\n",
      "\u001b[?25hCollecting cmake (from triton==2.0.0->torch==2.0.1)\r\n",
      "  Downloading https://download.pytorch.org/whl/cmake-3.25.0-py2.py3-none-manylinux_2_17_x86_64.manylinux2014_x86_64.whl (23.7 MB)\r\n",
      "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m23.7/23.7 MB\u001b[0m \u001b[31m29.3 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\r\n",
      "\u001b[?25hCollecting lit (from triton==2.0.0->torch==2.0.1)\r\n",
      "  Downloading https://download.pytorch.org/whl/lit-15.0.7.tar.gz (132 kB)\r\n",
      "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m132.3/132.3 kB\u001b[0m \u001b[31m13.8 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\r\n",
      "\u001b[?25h  Preparing metadata (setup.py) ... \u001b[?25l-\b \bdone\r\n",
      "\u001b[?25hRequirement already satisfied: MarkupSafe>=2.0 in /opt/conda/lib/python3.10/site-packages (from jinja2->torch==2.0.1) (2.1.3)\r\n",
      "Requirement already satisfied: mpmath>=0.19 in /opt/conda/lib/python3.10/site-packages (from sympy->torch==2.0.1) (1.3.0)\r\n",
      "Building wheels for collected packages: lit\r\n",
      "  Building wheel for lit (setup.py) ... \u001b[?25l-\b \b\\\b \bdone\r\n",
      "\u001b[?25h  Created wheel for lit: filename=lit-15.0.7-py3-none-any.whl size=89989 sha256=f4fed555f3d28fb078a851586b5ec2656826378ea804e7f8988d9af72aefa948\r\n",
      "  Stored in directory: /root/.cache/pip/wheels/27/2c/b6/3ed2983b1b44fe0dea1bb35234b09f2c22fb8ebb308679c922\r\n",
      "Successfully built lit\r\n",
      "Installing collected packages: lit, cmake, triton, torch\r\n",
      "\u001b[31mERROR: pip's dependency resolver does not currently take into account all the packages that are installed. This behaviour is the source of the following dependency conflicts.\r\n",
      "torchdata 0.6.0 requires torch==2.0.0, but you have torch 2.0.1+cu118 which is incompatible.\u001b[0m\u001b[31m\r\n",
      "\u001b[0mSuccessfully installed cmake-3.25.0 lit-15.0.7 torch-2.0.1+cu118 triton-2.0.0\r\n",
      "Collecting bitsandbytes\r\n",
      "  Obtaining dependency information for bitsandbytes from https://files.pythonhosted.org/packages/c2/49/557f8f4aa9cfc1e9d7875fd850a44a6d3d881a42c483bc8cf56a6b597dfe/bitsandbytes-0.41.2.post2-py3-none-any.whl.metadata\r\n",
      "  Downloading bitsandbytes-0.41.2.post2-py3-none-any.whl.metadata (9.8 kB)\r\n",
      "Requirement already satisfied: accelerate in /opt/conda/lib/python3.10/site-packages (0.24.1)\r\n",
      "Collecting evaluate\r\n",
      "  Obtaining dependency information for evaluate from https://files.pythonhosted.org/packages/70/63/7644a1eb7b0297e585a6adec98ed9e575309bb973c33b394dae66bc35c69/evaluate-0.4.1-py3-none-any.whl.metadata\r\n",
      "  Downloading evaluate-0.4.1-py3-none-any.whl.metadata (9.4 kB)\r\n",
      "Collecting rouge_score\r\n",
      "  Downloading rouge_score-0.1.2.tar.gz (17 kB)\r\n",
      "  Preparing metadata (setup.py) ... \u001b[?25l-\b \bdone\r\n",
      "\u001b[?25hRequirement already satisfied: numpy>=1.17 in /opt/conda/lib/python3.10/site-packages (from accelerate) (1.24.3)\r\n",
      "Requirement already satisfied: packaging>=20.0 in /opt/conda/lib/python3.10/site-packages (from accelerate) (21.3)\r\n",
      "Requirement already satisfied: psutil in /opt/conda/lib/python3.10/site-packages (from accelerate) (5.9.3)\r\n",
      "Requirement already satisfied: pyyaml in /opt/conda/lib/python3.10/site-packages (from accelerate) (6.0.1)\r\n",
      "Requirement already satisfied: torch>=1.10.0 in /opt/conda/lib/python3.10/site-packages (from accelerate) (2.0.1+cu118)\r\n",
      "Requirement already satisfied: huggingface-hub in /opt/conda/lib/python3.10/site-packages (from accelerate) (0.17.3)\r\n",
      "Requirement already satisfied: datasets>=2.0.0 in /opt/conda/lib/python3.10/site-packages (from evaluate) (2.1.0)\r\n",
      "Requirement already satisfied: dill in /opt/conda/lib/python3.10/site-packages (from evaluate) (0.3.7)\r\n",
      "Requirement already satisfied: pandas in /opt/conda/lib/python3.10/site-packages (from evaluate) (2.0.3)\r\n",
      "Requirement already satisfied: requests>=2.19.0 in /opt/conda/lib/python3.10/site-packages (from evaluate) (2.31.0)\r\n",
      "Requirement already satisfied: tqdm>=4.62.1 in /opt/conda/lib/python3.10/site-packages (from evaluate) (4.66.1)\r\n",
      "Requirement already satisfied: xxhash in /opt/conda/lib/python3.10/site-packages (from evaluate) (3.4.1)\r\n",
      "Requirement already satisfied: multiprocess in /opt/conda/lib/python3.10/site-packages (from evaluate) (0.70.15)\r\n",
      "Requirement already satisfied: fsspec[http]>=2021.05.0 in /opt/conda/lib/python3.10/site-packages (from evaluate) (2023.10.0)\r\n",
      "Requirement already satisfied: responses<0.19 in /opt/conda/lib/python3.10/site-packages (from evaluate) (0.18.0)\r\n",
      "Requirement already satisfied: absl-py in /opt/conda/lib/python3.10/site-packages (from rouge_score) (1.4.0)\r\n",
      "Requirement already satisfied: nltk in /opt/conda/lib/python3.10/site-packages (from rouge_score) (3.2.4)\r\n",
      "Requirement already satisfied: six>=1.14.0 in /opt/conda/lib/python3.10/site-packages (from rouge_score) (1.16.0)\r\n",
      "Requirement already satisfied: pyarrow>=5.0.0 in /opt/conda/lib/python3.10/site-packages (from datasets>=2.0.0->evaluate) (11.0.0)\r\n",
      "Requirement already satisfied: aiohttp in /opt/conda/lib/python3.10/site-packages (from datasets>=2.0.0->evaluate) (3.8.5)\r\n",
      "Requirement already satisfied: filelock in /opt/conda/lib/python3.10/site-packages (from huggingface-hub->accelerate) (3.12.2)\r\n",
      "Requirement already satisfied: typing-extensions>=3.7.4.3 in /opt/conda/lib/python3.10/site-packages (from huggingface-hub->accelerate) (4.5.0)\r\n",
      "Requirement already satisfied: pyparsing!=3.0.5,>=2.0.2 in /opt/conda/lib/python3.10/site-packages (from packaging>=20.0->accelerate) (3.0.9)\r\n",
      "Requirement already satisfied: charset-normalizer<4,>=2 in /opt/conda/lib/python3.10/site-packages (from requests>=2.19.0->evaluate) (3.2.0)\r\n",
      "Requirement already satisfied: idna<4,>=2.5 in /opt/conda/lib/python3.10/site-packages (from requests>=2.19.0->evaluate) (3.4)\r\n",
      "Requirement already satisfied: urllib3<3,>=1.21.1 in /opt/conda/lib/python3.10/site-packages (from requests>=2.19.0->evaluate) (1.26.15)\r\n",
      "Requirement already satisfied: certifi>=2017.4.17 in /opt/conda/lib/python3.10/site-packages (from requests>=2.19.0->evaluate) (2023.7.22)\r\n",
      "Requirement already satisfied: sympy in /opt/conda/lib/python3.10/site-packages (from torch>=1.10.0->accelerate) (1.12)\r\n",
      "Requirement already satisfied: networkx in /opt/conda/lib/python3.10/site-packages (from torch>=1.10.0->accelerate) (3.1)\r\n",
      "Requirement already satisfied: jinja2 in /opt/conda/lib/python3.10/site-packages (from torch>=1.10.0->accelerate) (3.1.2)\r\n",
      "Requirement already satisfied: triton==2.0.0 in /opt/conda/lib/python3.10/site-packages (from torch>=1.10.0->accelerate) (2.0.0)\r\n",
      "Requirement already satisfied: cmake in /opt/conda/lib/python3.10/site-packages (from triton==2.0.0->torch>=1.10.0->accelerate) (3.25.0)\r\n",
      "Requirement already satisfied: lit in /opt/conda/lib/python3.10/site-packages (from triton==2.0.0->torch>=1.10.0->accelerate) (15.0.7)\r\n",
      "Requirement already satisfied: python-dateutil>=2.8.2 in /opt/conda/lib/python3.10/site-packages (from pandas->evaluate) (2.8.2)\r\n",
      "Requirement already satisfied: pytz>=2020.1 in /opt/conda/lib/python3.10/site-packages (from pandas->evaluate) (2023.3)\r\n",
      "Requirement already satisfied: tzdata>=2022.1 in /opt/conda/lib/python3.10/site-packages (from pandas->evaluate) (2023.3)\r\n",
      "Requirement already satisfied: attrs>=17.3.0 in /opt/conda/lib/python3.10/site-packages (from aiohttp->datasets>=2.0.0->evaluate) (23.1.0)\r\n",
      "Requirement already satisfied: multidict<7.0,>=4.5 in /opt/conda/lib/python3.10/site-packages (from aiohttp->datasets>=2.0.0->evaluate) (6.0.4)\r\n",
      "Requirement already satisfied: async-timeout<5.0,>=4.0.0a3 in /opt/conda/lib/python3.10/site-packages (from aiohttp->datasets>=2.0.0->evaluate) (4.0.3)\r\n",
      "Requirement already satisfied: yarl<2.0,>=1.0 in /opt/conda/lib/python3.10/site-packages (from aiohttp->datasets>=2.0.0->evaluate) (1.9.2)\r\n",
      "Requirement already satisfied: frozenlist>=1.1.1 in /opt/conda/lib/python3.10/site-packages (from aiohttp->datasets>=2.0.0->evaluate) (1.4.0)\r\n",
      "Requirement already satisfied: aiosignal>=1.1.2 in /opt/conda/lib/python3.10/site-packages (from aiohttp->datasets>=2.0.0->evaluate) (1.3.1)\r\n",
      "Requirement already satisfied: MarkupSafe>=2.0 in /opt/conda/lib/python3.10/site-packages (from jinja2->torch>=1.10.0->accelerate) (2.1.3)\r\n",
      "Requirement already satisfied: mpmath>=0.19 in /opt/conda/lib/python3.10/site-packages (from sympy->torch>=1.10.0->accelerate) (1.3.0)\r\n",
      "Downloading bitsandbytes-0.41.2.post2-py3-none-any.whl (92.6 MB)\r\n",
      "\u001b[2K   \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m92.6/92.6 MB\u001b[0m \u001b[31m14.2 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\r\n",
      "\u001b[?25hDownloading evaluate-0.4.1-py3-none-any.whl (84 kB)\r\n",
      "\u001b[2K   \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m84.1/84.1 kB\u001b[0m \u001b[31m10.0 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\r\n",
      "\u001b[?25hBuilding wheels for collected packages: rouge_score\r\n",
      "  Building wheel for rouge_score (setup.py) ... \u001b[?25l-\b \b\\\b \bdone\r\n",
      "\u001b[?25h  Created wheel for rouge_score: filename=rouge_score-0.1.2-py3-none-any.whl size=24932 sha256=2cbe1d2ffc90ae65a080ba7d09ac39b0a5b4adaf029e0e3e1815ad7a0aac65d1\r\n",
      "  Stored in directory: /root/.cache/pip/wheels/5f/dd/89/461065a73be61a532ff8599a28e9beef17985c9e9c31e541b4\r\n",
      "Successfully built rouge_score\r\n",
      "Installing collected packages: bitsandbytes, rouge_score, evaluate\r\n",
      "Successfully installed bitsandbytes-0.41.2.post2 evaluate-0.4.1 rouge_score-0.1.2\r\n",
      "Token will not been saved to git credential helper. Pass `add_to_git_credential=True` if you want to set the git credential as well.\r\n",
      "Token is valid (permission: read).\r\n",
      "Your token has been saved to /root/.cache/huggingface/token\r\n",
      "Login successful\r\n"
     ]
    }
   ],
   "source": [
    "!pip uninstall torch --y\n",
    "!pip install --upgrade torch==2.0.1 --index-url https://download.pytorch.org/whl/cu118\n",
    "!pip install -q -U trl transformers==4.35.0 git+https://github.com/huggingface/peft.git\n",
    "# !pip install -q -U trl git+https://github.com/huggingface/peft.git\n",
    "!pip install bitsandbytes accelerate evaluate rouge_score\n",
    "# !pip install -U flash-attn\n",
    "!huggingface-cli login --token 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"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "2af6c34e",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2023-11-22T05:38:57.043456Z",
     "iopub.status.busy": "2023-11-22T05:38:57.042944Z",
     "iopub.status.idle": "2023-11-22T05:38:57.056127Z",
     "shell.execute_reply": "2023-11-22T05:38:57.055313Z"
    },
    "papermill": {
     "duration": 0.11443,
     "end_time": "2023-11-22T05:38:57.058259",
     "exception": false,
     "start_time": "2023-11-22T05:38:56.943829",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Writing train.py\n"
     ]
    }
   ],
   "source": [
    "%%writefile train.py\n",
    "\n",
    "import torch\n",
    "from torch import nn\n",
    "import numpy as np\n",
    "import pandas as pd\n",
    "import json\n",
    "from pathlib import Path\n",
    "from sklearn.model_selection import train_test_split\n",
    "from datasets import Dataset\n",
    "from transformers import (\n",
    "    BloomForSequenceClassification,\n",
    "    AutoConfig,\n",
    "    AutoModelForCausalLM,\n",
    "    AutoTokenizer,\n",
    "    GenerationConfig,\n",
    "    BitsAndBytesConfig,\n",
    "    DataCollatorWithPadding\n",
    ")\n",
    "from peft import (\n",
    "    LoraConfig,\n",
    "    get_peft_model,\n",
    "    prepare_model_for_kbit_training,\n",
    "    PeftModel\n",
    ")\n",
    "from transformers import PreTrainedModel, TrainingArguments, Trainer\n",
    "from transformers.modeling_outputs import SequenceClassifierOutput\n",
    "\n",
    "if __name__ == '__main__':\n",
    "    device = 'cuda' if torch.cuda.is_available() else 'cpu'\n",
    "    print(torch.cuda.current_device())\n",
    "    \n",
    "    model_path = \"bigscience/bloomz-7b1-mt\"\n",
    "\n",
    "    def get_context(row):\n",
    "        row_id = str(row['index'])\n",
    "        context = contexts[row_id]\n",
    "        context = ' '.join([' '.join(c) for c in context])\n",
    "        if context[-1] != '.':\n",
    "            context += '.'\n",
    "        return context\n",
    "\n",
    "    contexts = json.load(open('/kaggle/input/72hour/ft/context_dict.json', 'r'))\n",
    "\n",
    "    import datasets\n",
    "    accuracy = datasets.load_metric('accuracy')\n",
    "\n",
    "    def compute_metrics(eval_pred):\n",
    "        predictions, labels = eval_pred\n",
    "        predictions = np.argmax(predictions, axis=1)\n",
    "        return {'accuracy': accuracy.compute(predictions=predictions, references=labels)}\n",
    "\n",
    "    from datetime import datetime\n",
    "    import wandb\n",
    "    wandb.login(key=\"5d1dc32c97d754cc8ce0b6666d9f85d9f7618ccd\", force=True)\n",
    "\n",
    "    import gc\n",
    "    with torch.no_grad():\n",
    "        model = None\n",
    "        torch.cuda.empty_cache()\n",
    "        gc.collect()\n",
    "\n",
    "    class BloomClassifier(PreTrainedModel):\n",
    "        def __init__(self, backbone):\n",
    "            super(BloomClassifier, self).__init__(backbone.config)\n",
    "            self.backbone = backbone\n",
    "\n",
    "            self.in_features = self.backbone.lm_head.out_features\n",
    "            self.linear = nn.Linear(in_features=self.in_features, out_features=1)\n",
    "\n",
    "        def forward(self,\n",
    "                input_ids=None,\n",
    "                attention_mask=None,\n",
    "                labels=None,\n",
    "                return_dict=None,\n",
    "                **kwargs):\n",
    "            \n",
    "            # input_ids = input_ids.to(device)\n",
    "            # attention_mask = attention_mask.to(device)\n",
    "            # self.backbone.to(input_ids.device)\n",
    "            out = self.backbone(\n",
    "                input_ids=input_ids.squeeze(1), \n",
    "                attention_mask=attention_mask.squeeze(1)).logits\n",
    "            # out = out[:, -1, :].float()\n",
    "\n",
    "            pooled = [] \n",
    "            for jjj in range(len(out)):\n",
    "                #att_idx = attention_mask.sum(dim=1)[jjj] - 1\n",
    "                pooled.append(out[jjj, -1, :].float().unsqueeze(0)) # (bs, 1, in_features)\n",
    "            del out; gc.collect(); torch.cuda.empty_cache()\n",
    "            pooled = torch.cat(pooled) # (bs, in_features)\n",
    "            # self.linear.to(pooled.device)\n",
    "            logits = self.linear(pooled) # (bs, 1)\n",
    "\n",
    "            loss = None\n",
    "            if labels is not None:\n",
    "                loss_fn = nn.BCEWithLogitsLoss()\n",
    "                loss = loss_fn(logits, labels.unsqueeze(1).to(logits.device))\n",
    "\n",
    "            # preds = torch.sigmoid(logits).squeeze(-1).detach().cpu().numpy()\n",
    "            return SequenceClassifierOutput(loss=loss, logits=logits)\n",
    "\n",
    "    # out = torch.rand(4, 2048, 250880)\n",
    "    # pooled = [] \n",
    "    # for jjj in range(len(out)):\n",
    "    #     att_idx = 2048-1\n",
    "    #     pooled.append(out[jjj, att_idx, :].float().unsqueeze(0))\n",
    "    # del out; gc.collect(); torch.cuda.empty_cache()\n",
    "    # pooled = torch.cat(pooled)\n",
    "\n",
    "    # linear = nn.Linear(in_features=backbone.lm_head.out_features, out_features=1)\n",
    "    # logits = linear(pooled)\n",
    "    # labels = torch.rand(4, dtype=float)\n",
    "    # loss_fn = nn.BCEWithLogitsLoss()\n",
    "    # loss_fn(logits, labels.unsqueeze(1))\n",
    "\n",
    "    # Tokenizer.\n",
    "    tokenizer = AutoTokenizer.from_pretrained(model_path, trust_remote_code=True)\n",
    "    tokenizer.pad_token_id = 0\n",
    "    tokenizer.padding_side = \"left\"\n",
    "\n",
    "    lora_config = LoraConfig(\n",
    "        r= 8,\n",
    "        lora_alpha= 32,\n",
    "        bias = \"none\",\n",
    "        task_type = \"SEQ_CLS\",\n",
    "        lora_dropout = 0.05,\n",
    "    )\n",
    "    # Model.\n",
    "    quantization_config = BitsAndBytesConfig(\n",
    "        load_in_4bit=True, \n",
    "        bnb_4bit_compute_dtype='float16', \n",
    "        bnb_4bit_quant_type='fp4', \n",
    "        bnb_4bit_use_double_quant=True)\n",
    "    backbone = BloomForSequenceClassification.from_pretrained(  \n",
    "        model_path, \n",
    "        torch_dtype=torch.float16,\n",
    "        device_map='auto',\n",
    "        quantization_config=quantization_config,\n",
    "        problem_type='single_label_classification'\n",
    "    )\n",
    "    backbone = prepare_model_for_kbit_training(backbone)\n",
    "    backbone = get_peft_model(backbone, lora_config)\n",
    "    # backbone.to(device)\n",
    "    backbone.print_trainable_parameters()\n",
    "\n",
    "    def preprocess(example):\n",
    "        inputs = tokenizer(\n",
    "            example['prompt'],\n",
    "            return_tensors=None,\n",
    "            add_special_tokens=False,\n",
    "            truncation=True,\n",
    "            padding=\"max_length\",\n",
    "            max_length=2048\n",
    "        )\n",
    "        #for k,v in inputs.items():\n",
    "        #    inputs[k] = v[0] #.squeeze(0)\n",
    "        # inputs['labels'] = torch.tensor(example['labels'], dtype=torch.float).unsqueeze(0)\n",
    "        inputs['labels'] = example['labels']\n",
    "        return inputs\n",
    "\n",
    "    instructions = pd.read_json('/kaggle/input/72hour/dev.json')\n",
    "    instructions['context'] = instructions.apply(get_context, axis=1)\n",
    "    instructions['labels'] = instructions['grade'].apply(lambda x: x == 'Đúng').astype(int) #.astype(str)\n",
    "    instructions['prompt'] = (\n",
    "        'Bối cảnh: ' + instructions['context'] + '\\n\\n'\n",
    "        \n",
    "        \"Câu hỏi: Bệnh nào khó chữa nhất?\\n\"\n",
    "        \"Câu trả lời đề xuất: Không có câu trả lời.\\n\"\n",
    "        \"Điểm:1\\n\\n\"\n",
    "\n",
    "        \"Câu hỏi: Cách nào để phòng ngừa bệnh tim mạch?\\n\"\n",
    "        \"Câu trả lời đề xuất: Tập thể dục đều đặn.\\n\"\n",
    "        \"Điểm:1\\n\\n\"\n",
    "        \n",
    "        'Câu hỏi: ' + instructions['question'] + '\\n'\n",
    "        'Câu trả lời đề xuất: ' + instructions['answer'] + '.\\n'\n",
    "        'Điểm:'\n",
    "    )\n",
    "    train_df, valid_df = train_test_split(instructions, test_size=0.2, random_state=42)\n",
    "\n",
    "    remove_columns = ['index', 'question', 'answer', 'grade', 'context', 'prompt', '__index_level_0__',]\n",
    "    train_dataset = Dataset.from_pandas(train_df).map(preprocess, remove_columns=remove_columns)\n",
    "    valid_dataset = Dataset.from_pandas(valid_df).map(preprocess, remove_columns=remove_columns)\n",
    "    data_collator = DataCollatorWithPadding(tokenizer=tokenizer, padding=\"longest\")\n",
    "    \n",
    "    class MyDataset(torch.utils.data.Dataset):\n",
    "        def __init__(self, df, tokenizer):\n",
    "            self.prompt = df['prompt'].values\n",
    "            self.labels = df['labels'].values\n",
    "            self.tokenizer = tokenizer\n",
    "\n",
    "        def __getitem__(self, idx):\n",
    "            example = self.prompt[idx]\n",
    "            inputs = tokenizer(\n",
    "                example,\n",
    "                return_tensors=\"pt\",\n",
    "                add_special_tokens=False,\n",
    "                truncation=True,\n",
    "                padding=\"max_length\",\n",
    "                max_length=2048\n",
    "            )\n",
    "            #for k, v in inputs.items():\n",
    "            #    inputs[k] = torch.tensor(v, dtype = torch.long).to(device)\n",
    "            inputs['labels'] = torch.tensor(self.labels[idx], dtype = torch.float)\n",
    "            return inputs\n",
    "\n",
    "        def __len__(self):\n",
    "            return len(self.labels)\n",
    "\n",
    "    #train_dataset = MyDataset(train_df, tokenizer)\n",
    "    #valid_dataset = MyDataset(valid_df, tokenizer)\n",
    "\n",
    "    training_args = TrainingArguments(\n",
    "        # label_names=['labels'],\n",
    "        warmup_ratio=0.1, \n",
    "        learning_rate=2e-5,\n",
    "        per_device_train_batch_size=1,\n",
    "        per_device_eval_batch_size=1,\n",
    "        num_train_epochs=1,\n",
    "        output_dir = f'/kaggle/working/',\n",
    "        overwrite_output_dir=True,\n",
    "        fp16=True,\n",
    "        optim=\"adamw_torch\",\n",
    "        gradient_accumulation_steps=8,\n",
    "        logging_steps=10,\n",
    "        evaluation_strategy='steps',\n",
    "        eval_steps=50,\n",
    "        save_strategy=\"steps\",\n",
    "        save_steps=50,\n",
    "        load_best_model_at_end=False,\n",
    "        metric_for_best_model='accuracy',\n",
    "        lr_scheduler_type='cosine',\n",
    "        weight_decay=0.01,\n",
    "        report_to=\"wandb\",\n",
    "        run_name=f\"bloomz-{datetime.now().strftime('%Y-%m-%d-%H-%M')}\",\n",
    "        remove_unused_columns=False\n",
    "    )\n",
    "\n",
    "    import gc\n",
    "    with torch.no_grad():\n",
    "        model = None\n",
    "        torch.cuda.empty_cache()\n",
    "    gc.collect()\n",
    "\n",
    "    # model = BloomClassifier(backbone)\n",
    "    trainer = Trainer(\n",
    "        model=backbone,\n",
    "        args=training_args,\n",
    "        train_dataset=train_dataset,\n",
    "        eval_dataset=valid_dataset,\n",
    "        tokenizer=tokenizer,\n",
    "        compute_metrics=compute_metrics,\n",
    "        data_collator=data_collator,\n",
    "        # preprocess_logits_for_metrics=lambda logits, labels: torch.sigmoid(logits).squeeze(-1).detach().cpu().numpy()\n",
    "    )\n",
    "    # for i in model.named_parameters():\n",
    "    #    print(f\"{i[0]} -> {i[1].device}\")\n",
    "\n",
    "    trainer.train()\n",
    "    trainer.save_model('bloomz_binary')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "8165f4e2",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2023-11-22T05:38:57.256597Z",
     "iopub.status.busy": "2023-11-22T05:38:57.255918Z",
     "iopub.status.idle": "2023-11-22T09:53:38.940910Z",
     "shell.execute_reply": "2023-11-22T09:53:38.939877Z"
    },
    "papermill": {
     "duration": 15281.786663,
     "end_time": "2023-11-22T09:53:38.944248",
     "exception": false,
     "start_time": "2023-11-22T05:38:57.157585",
     "status": "completed"
    },
    "scrolled": true,
    "tags": []
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "/opt/conda/lib/python3.10/site-packages/scipy/__init__.py:146: UserWarning: A NumPy version >=1.16.5 and <1.23.0 is required for this version of SciPy (detected version 1.24.3\r\n",
      "  warnings.warn(f\"A NumPy version >={np_minversion} and <{np_maxversion}\"\r\n",
      "0\r\n",
      "Downloading builder script: 3.19kB [00:00, 2.63MB/s]                            \r\n",
      "\u001b[34m\u001b[1mwandb\u001b[0m: W&B API key is configured. Use \u001b[1m`wandb login --relogin`\u001b[0m to force relogin\r\n",
      "\u001b[34m\u001b[1mwandb\u001b[0m: \u001b[33mWARNING\u001b[0m If you're specifying your api key in code, ensure this code is not shared publicly.\r\n",
      "\u001b[34m\u001b[1mwandb\u001b[0m: \u001b[33mWARNING\u001b[0m Consider setting the WANDB_API_KEY environment variable, or running `wandb login` from the command line.\r\n",
      "\u001b[34m\u001b[1mwandb\u001b[0m: Appending key for api.wandb.ai to your netrc file: /root/.netrc\r\n",
      "Downloading tokenizer_config.json: 100%|███████| 199/199 [00:00<00:00, 1.17MB/s]\r\n",
      "Downloading tokenizer.json: 100%|██████████| 14.5M/14.5M [00:01<00:00, 13.8MB/s]\r\n",
      "Downloading (…)cial_tokens_map.json: 100%|████| 85.0/85.0 [00:00<00:00, 393kB/s]\r\n",
      "Downloading config.json: 100%|█████████████████| 747/747 [00:00<00:00, 4.37MB/s]\r\n",
      "Downloading pytorch_model.bin: 100%|███████| 14.1G/14.1G [05:57<00:00, 39.6MB/s]\r\n",
      "Some weights of BloomForSequenceClassification were not initialized from the model checkpoint at bigscience/bloomz-7b1-mt and are newly initialized: ['score.weight']\r\n",
      "You should probably TRAIN this model on a down-stream task to be able to use it for predictions and inference.\r\n",
      "trainable params: 3,940,352 || all params: 7,072,964,608 || trainable%: 0.05571004830906684\r\n",
      "/opt/conda/lib/python3.10/site-packages/pyarrow/pandas_compat.py:373: FutureWarning: is_sparse is deprecated and will be removed in a future version. Check `isinstance(dtype, pd.SparseDtype)` instead.\r\n",
      "  if _pandas_api.is_sparse(col):\r\n",
      "100%|██████████████████████████████████████| 1624/1624 [00:05<00:00, 314.79ex/s]\r\n",
      "100%|████████████████████████████████████████| 407/407 [00:01<00:00, 374.54ex/s]\r\n",
      "\u001b[34m\u001b[1mwandb\u001b[0m: Currently logged in as: \u001b[33mquangphamm1902\u001b[0m. Use \u001b[1m`wandb login --relogin`\u001b[0m to force relogin\r\n",
      "\u001b[34m\u001b[1mwandb\u001b[0m: Tracking run with wandb version 0.16.0\r\n",
      "\u001b[34m\u001b[1mwandb\u001b[0m: Run data is saved locally in \u001b[35m\u001b[1m/kaggle/working/wandb/run-20231122_054724-7ewrxor3\u001b[0m\r\n",
      "\u001b[34m\u001b[1mwandb\u001b[0m: Run \u001b[1m`wandb offline`\u001b[0m to turn off syncing.\r\n",
      "\u001b[34m\u001b[1mwandb\u001b[0m: Syncing run \u001b[33mbloomz-2023-11-22-05-47\u001b[0m\r\n",
      "\u001b[34m\u001b[1mwandb\u001b[0m: ⭐️ View project at \u001b[34m\u001b[4mhttps://wandb.ai/quangphamm1902/huggingface\u001b[0m\r\n",
      "\u001b[34m\u001b[1mwandb\u001b[0m: 🚀 View run at \u001b[34m\u001b[4mhttps://wandb.ai/quangphamm1902/huggingface/runs/7ewrxor3\u001b[0m\r\n",
      "  0%|                                                   | 0/203 [00:00<?, ?it/s]You're using a BloomTokenizerFast tokenizer. Please note that with a fast tokenizer, using the `__call__` method is faster than using a method to encode the text followed by a call to the `pad` method to get a padded encoding.\r\n",
      "`use_cache=True` is incompatible with gradient checkpointing. Setting `use_cache=False`...\r\n",
      "{'loss': 11.4244, 'learning_rate': 3.80952380952381e-06, 'epoch': 0.05}\r\n",
      "{'loss': 10.7292, 'learning_rate': 1.3333333333333333e-05, 'epoch': 0.1}\r\n",
      "{'loss': 6.4584, 'learning_rate': 1.9986594826804563e-05, 'epoch': 0.15}\r\n",
      "{'loss': 1.4893, 'learning_rate': 1.9749279121818235e-05, 'epoch': 0.2}\r\n",
      "{'loss': 0.8793, 'learning_rate': 1.9222195128618108e-05, 'epoch': 0.25}\r\n",
      " 25%|█████████                            | 50/203 [1:31:59<4:41:28, 110.38s/it]\r\n",
      "  0%|                                                   | 0/407 [00:00<?, ?it/s]\u001b[A\r\n",
      "  0%|▏                                          | 2/407 [00:04<15:20,  2.27s/it]\u001b[A\r\n",
      "  1%|▎                                          | 3/407 [00:09<21:41,  3.22s/it]\u001b[A\r\n",
      "  1%|▍                                          | 4/407 [00:13<24:57,  3.72s/it]\u001b[A\r\n",
      "  1%|▌                                          | 5/407 [00:18<26:49,  4.00s/it]\u001b[A\r\n",
      "  1%|▋                                          | 6/407 [00:22<27:57,  4.18s/it]\u001b[A\r\n",
      "  2%|▋                                          | 7/407 [00:27<28:38,  4.30s/it]\u001b[A\r\n",
      "  2%|▊                                          | 8/407 [00:31<29:05,  4.37s/it]\u001b[A\r\n",
      "  2%|▉                                          | 9/407 [00:36<29:21,  4.42s/it]\u001b[A\r\n",
      "  2%|█                                         | 10/407 [00:40<29:31,  4.46s/it]\u001b[A\r\n",
      "  3%|█▏                                        | 11/407 [00:45<29:36,  4.49s/it]\u001b[A\r\n",
      "  3%|█▏                                        | 12/407 [00:49<29:38,  4.50s/it]\u001b[A\r\n",
      "  3%|█▎                                        | 13/407 [00:54<29:38,  4.51s/it]\u001b[A\r\n",
      "  3%|█▍                                        | 14/407 [00:59<29:36,  4.52s/it]\u001b[A\r\n",
      "  4%|█▌                                        | 15/407 [01:03<29:33,  4.52s/it]\u001b[A\r\n",
      "  4%|█▋                                        | 16/407 [01:08<29:30,  4.53s/it]\u001b[A\r\n",
      "  4%|█▊                                        | 17/407 [01:12<29:26,  4.53s/it]\u001b[A\r\n",
      "  4%|█▊                                        | 18/407 [01:17<29:23,  4.53s/it]\u001b[A\r\n",
      "  5%|█▉                                        | 19/407 [01:21<29:19,  4.53s/it]\u001b[A\r\n",
      "  5%|██                                        | 20/407 [01:26<29:15,  4.54s/it]\u001b[A\r\n",
      "  5%|██▏                                       | 21/407 [01:30<29:10,  4.54s/it]\u001b[A\r\n",
      "  5%|██▎                                       | 22/407 [01:35<29:06,  4.54s/it]\u001b[A\r\n",
      "  6%|██▎                                       | 23/407 [01:39<29:02,  4.54s/it]\u001b[A\r\n",
      "  6%|██▍                                       | 24/407 [01:44<28:57,  4.54s/it]\u001b[A\r\n",
      "  6%|██▌                                       | 25/407 [01:48<28:52,  4.54s/it]\u001b[A\r\n",
      "  6%|██▋                                       | 26/407 [01:53<28:48,  4.54s/it]\u001b[A\r\n",
      "  7%|██▊                                       | 27/407 [01:58<28:44,  4.54s/it]\u001b[A\r\n",
      "  7%|██▉                                       | 28/407 [02:02<28:40,  4.54s/it]\u001b[A\r\n",
      "  7%|██▉                                       | 29/407 [02:07<28:35,  4.54s/it]\u001b[A\r\n",
      "  7%|███                                       | 30/407 [02:11<28:31,  4.54s/it]\u001b[A\r\n",
      "  8%|███▏                                      | 31/407 [02:16<28:26,  4.54s/it]\u001b[A\r\n",
      "  8%|███▎                                      | 32/407 [02:20<28:22,  4.54s/it]\u001b[A\r\n",
      "  8%|███▍                                      | 33/407 [02:25<28:18,  4.54s/it]\u001b[A\r\n",
      "  8%|███▌                                      | 34/407 [02:29<28:13,  4.54s/it]\u001b[A\r\n",
      "  9%|███▌                                      | 35/407 [02:34<28:09,  4.54s/it]\u001b[A\r\n",
      "  9%|███▋                                      | 36/407 [02:38<28:04,  4.54s/it]\u001b[A\r\n",
      "  9%|███▊                                      | 37/407 [02:43<28:00,  4.54s/it]\u001b[A\r\n",
      "  9%|███▉                                      | 38/407 [02:47<27:55,  4.54s/it]\u001b[A\r\n",
      " 10%|████                                      | 39/407 [02:52<27:50,  4.54s/it]\u001b[A\r\n",
      " 10%|████▏                                     | 40/407 [02:57<27:45,  4.54s/it]\u001b[A\r\n",
      " 10%|████▏                                     | 41/407 [03:01<27:41,  4.54s/it]\u001b[A\r\n",
      " 10%|████▎                                     | 42/407 [03:06<27:36,  4.54s/it]\u001b[A\r\n",
      " 11%|████▍                                     | 43/407 [03:10<27:32,  4.54s/it]\u001b[A\r\n",
      " 11%|████▌                                     | 44/407 [03:15<27:27,  4.54s/it]\u001b[A\r\n",
      " 11%|████▋                                     | 45/407 [03:19<27:22,  4.54s/it]\u001b[A\r\n",
      " 11%|████▋                                     | 46/407 [03:24<27:17,  4.54s/it]\u001b[A\r\n",
      " 12%|████▊                                     | 47/407 [03:28<27:13,  4.54s/it]\u001b[A\r\n",
      " 12%|████▉                                     | 48/407 [03:33<27:08,  4.54s/it]\u001b[A\r\n",
      " 12%|█████                                     | 49/407 [03:37<27:03,  4.54s/it]\u001b[A\r\n",
      " 12%|█████▏                                    | 50/407 [03:42<26:59,  4.54s/it]\u001b[A\r\n",
      " 13%|█████▎                                    | 51/407 [03:46<26:54,  4.54s/it]\u001b[A\r\n",
      " 13%|█████▎                                    | 52/407 [03:51<26:49,  4.54s/it]\u001b[A\r\n",
      " 13%|█████▍                                    | 53/407 [03:56<26:45,  4.54s/it]\u001b[A\r\n",
      " 13%|█████▌                                    | 54/407 [04:00<26:40,  4.54s/it]\u001b[A\r\n",
      " 14%|█████▋                                    | 55/407 [04:05<26:36,  4.53s/it]\u001b[A\r\n",
      " 14%|█████▊                                    | 56/407 [04:09<26:32,  4.54s/it]\u001b[A\r\n",
      " 14%|█████▉                                    | 57/407 [04:14<26:27,  4.54s/it]\u001b[A\r\n",
      " 14%|█████▉                                    | 58/407 [04:18<26:23,  4.54s/it]\u001b[A\r\n",
      " 14%|██████                                    | 59/407 [04:23<26:18,  4.54s/it]\u001b[A\r\n",
      " 15%|██████▏                                   | 60/407 [04:27<26:14,  4.54s/it]\u001b[A\r\n",
      " 15%|██████▎                                   | 61/407 [04:32<26:10,  4.54s/it]\u001b[A\r\n",
      " 15%|██████▍                                   | 62/407 [04:36<26:05,  4.54s/it]\u001b[A\r\n",
      " 15%|██████▌                                   | 63/407 [04:41<26:00,  4.54s/it]\u001b[A\r\n",
      " 16%|██████▌                                   | 64/407 [04:45<25:56,  4.54s/it]\u001b[A\r\n",
      " 16%|██████▋                                   | 65/407 [04:50<25:52,  4.54s/it]\u001b[A\r\n",
      " 16%|██████▊                                   | 66/407 [04:55<25:47,  4.54s/it]\u001b[A\r\n",
      " 16%|██████▉                                   | 67/407 [04:59<25:42,  4.54s/it]\u001b[A\r\n",
      " 17%|███████                                   | 68/407 [05:04<25:38,  4.54s/it]\u001b[A\r\n",
      " 17%|███████                                   | 69/407 [05:08<25:33,  4.54s/it]\u001b[A\r\n",
      " 17%|███████▏                                  | 70/407 [05:13<25:29,  4.54s/it]\u001b[A\r\n",
      " 17%|███████▎                                  | 71/407 [05:17<25:23,  4.53s/it]\u001b[A\r\n",
      " 18%|███████▍                                  | 72/407 [05:22<25:19,  4.54s/it]\u001b[A\r\n",
      " 18%|███████▌                                  | 73/407 [05:26<25:15,  4.54s/it]\u001b[A\r\n",
      " 18%|███████▋                                  | 74/407 [05:31<25:10,  4.54s/it]\u001b[A\r\n",
      " 18%|███████▋                                  | 75/407 [05:35<25:05,  4.53s/it]\u001b[A\r\n",
      " 19%|███████▊                                  | 76/407 [05:40<25:01,  4.54s/it]\u001b[A\r\n",
      " 19%|███████▉                                  | 77/407 [05:44<24:57,  4.54s/it]\u001b[A\r\n",
      " 19%|████████                                  | 78/407 [05:49<24:52,  4.54s/it]\u001b[A\r\n",
      " 19%|████████▏                                 | 79/407 [05:53<24:47,  4.54s/it]\u001b[A\r\n",
      " 20%|████████▎                                 | 80/407 [05:58<24:43,  4.54s/it]\u001b[A\r\n",
      " 20%|████████▎                                 | 81/407 [06:03<24:39,  4.54s/it]\u001b[A\r\n",
      " 20%|████████▍                                 | 82/407 [06:07<24:34,  4.54s/it]\u001b[A\r\n",
      " 20%|████████▌                                 | 83/407 [06:12<24:29,  4.54s/it]\u001b[A\r\n",
      " 21%|████████▋                                 | 84/407 [06:16<24:25,  4.54s/it]\u001b[A\r\n",
      " 21%|████████▊                                 | 85/407 [06:21<24:21,  4.54s/it]\u001b[A\r\n",
      " 21%|████████▊                                 | 86/407 [06:25<24:16,  4.54s/it]\u001b[A\r\n",
      " 21%|████████▉                                 | 87/407 [06:30<24:11,  4.54s/it]\u001b[A\r\n",
      " 22%|█████████                                 | 88/407 [06:34<24:07,  4.54s/it]\u001b[A\r\n",
      " 22%|█████████▏                                | 89/407 [06:39<24:02,  4.54s/it]\u001b[A\r\n",
      " 22%|█████████▎                                | 90/407 [06:43<23:58,  4.54s/it]\u001b[A\r\n",
      " 22%|█████████▍                                | 91/407 [06:48<23:53,  4.54s/it]\u001b[A\r\n",
      " 23%|█████████▍                                | 92/407 [06:52<23:49,  4.54s/it]\u001b[A\r\n",
      " 23%|█████████▌                                | 93/407 [06:57<23:44,  4.54s/it]\u001b[A\r\n",
      " 23%|█████████▋                                | 94/407 [07:02<23:39,  4.53s/it]\u001b[A\r\n",
      " 23%|█████████▊                                | 95/407 [07:06<23:35,  4.54s/it]\u001b[A\r\n",
      " 24%|█████████▉                                | 96/407 [07:11<23:31,  4.54s/it]\u001b[A\r\n",
      " 24%|██████████                                | 97/407 [07:15<23:26,  4.54s/it]\u001b[A\r\n",
      " 24%|██████████                                | 98/407 [07:20<23:21,  4.54s/it]\u001b[A\r\n",
      " 24%|██████████▏                               | 99/407 [07:24<23:17,  4.54s/it]\u001b[A\r\n",
      " 25%|██████████                               | 100/407 [07:29<23:12,  4.54s/it]\u001b[A\r\n",
      " 25%|██████████▏                              | 101/407 [07:33<23:08,  4.54s/it]\u001b[A\r\n",
      " 25%|██████████▎                              | 102/407 [07:38<23:03,  4.54s/it]\u001b[A\r\n",
      " 25%|██████████▍                              | 103/407 [07:42<22:59,  4.54s/it]\u001b[A\r\n",
      " 26%|██████████▍                              | 104/407 [07:47<22:54,  4.54s/it]\u001b[A\r\n",
      " 26%|██████████▌                              | 105/407 [07:51<22:50,  4.54s/it]\u001b[A\r\n",
      " 26%|██████████▋                              | 106/407 [07:56<22:45,  4.54s/it]\u001b[A\r\n",
      " 26%|██████████▊                              | 107/407 [08:01<22:40,  4.54s/it]\u001b[A\r\n",
      " 27%|██████████▉                              | 108/407 [08:05<22:36,  4.54s/it]\u001b[A\r\n",
      " 27%|██████████▉                              | 109/407 [08:10<22:32,  4.54s/it]\u001b[A\r\n",
      " 27%|███████████                              | 110/407 [08:14<22:27,  4.54s/it]\u001b[A\r\n",
      " 27%|███████████▏                             | 111/407 [08:19<22:22,  4.54s/it]\u001b[A\r\n",
      " 28%|███████████▎                             | 112/407 [08:23<22:18,  4.54s/it]\u001b[A\r\n",
      " 28%|███████████▍                             | 113/407 [08:28<22:14,  4.54s/it]\u001b[A\r\n",
      " 28%|███████████▍                             | 114/407 [08:32<22:09,  4.54s/it]\u001b[A\r\n",
      " 28%|███████████▌                             | 115/407 [08:37<22:05,  4.54s/it]\u001b[A\r\n",
      " 29%|███████████▋                             | 116/407 [08:41<22:00,  4.54s/it]\u001b[A\r\n",
      " 29%|███████████▊                             | 117/407 [08:46<21:56,  4.54s/it]\u001b[A\r\n",
      " 29%|███████████▉                             | 118/407 [08:50<21:52,  4.54s/it]\u001b[A\r\n",
      " 29%|███████████▉                             | 119/407 [08:55<21:47,  4.54s/it]\u001b[A\r\n",
      " 29%|████████████                             | 120/407 [09:00<21:42,  4.54s/it]\u001b[A\r\n",
      " 30%|████████████▏                            | 121/407 [09:04<21:37,  4.54s/it]\u001b[A\r\n",
      " 30%|████████████▎                            | 122/407 [09:09<21:33,  4.54s/it]\u001b[A\r\n",
      " 30%|████████████▍                            | 123/407 [09:13<21:28,  4.54s/it]\u001b[A\r\n",
      " 30%|████████████▍                            | 124/407 [09:18<21:24,  4.54s/it]\u001b[A\r\n",
      " 31%|████████████▌                            | 125/407 [09:22<21:19,  4.54s/it]\u001b[A\r\n",
      " 31%|████████████▋                            | 126/407 [09:27<21:14,  4.54s/it]\u001b[A\r\n",
      " 31%|████████████▊                            | 127/407 [09:31<21:10,  4.54s/it]\u001b[A\r\n",
      " 31%|████████████▉                            | 128/407 [09:36<21:05,  4.54s/it]\u001b[A\r\n",
      " 32%|████████████▉                            | 129/407 [09:40<21:00,  4.54s/it]\u001b[A\r\n",
      " 32%|█████████████                            | 130/407 [09:45<20:56,  4.54s/it]\u001b[A\r\n",
      " 32%|█████████████▏                           | 131/407 [09:49<20:51,  4.54s/it]\u001b[A\r\n",
      " 32%|█████████████▎                           | 132/407 [09:54<20:47,  4.54s/it]\u001b[A\r\n",
      " 33%|█████████████▍                           | 133/407 [09:58<20:42,  4.54s/it]\u001b[A\r\n",
      " 33%|█████████████▍                           | 134/407 [10:03<20:38,  4.54s/it]\u001b[A\r\n",
      " 33%|█████████████▌                           | 135/407 [10:08<20:33,  4.54s/it]\u001b[A\r\n",
      " 33%|█████████████▋                           | 136/407 [10:12<20:29,  4.54s/it]\u001b[A\r\n",
      " 34%|█████████████▊                           | 137/407 [10:17<20:24,  4.54s/it]\u001b[A\r\n",
      " 34%|█████████████▉                           | 138/407 [10:21<20:20,  4.54s/it]\u001b[A\r\n",
      " 34%|██████████████                           | 139/407 [10:26<20:15,  4.54s/it]\u001b[A\r\n",
      " 34%|██████████████                           | 140/407 [10:30<20:11,  4.54s/it]\u001b[A\r\n",
      " 35%|██████████████▏                          | 141/407 [10:35<20:06,  4.54s/it]\u001b[A\r\n",
      " 35%|██████████████▎                          | 142/407 [10:39<20:02,  4.54s/it]\u001b[A\r\n",
      " 35%|██████████████▍                          | 143/407 [10:44<19:57,  4.54s/it]\u001b[A\r\n",
      " 35%|██████████████▌                          | 144/407 [10:48<19:53,  4.54s/it]\u001b[A\r\n",
      " 36%|██████████████▌                          | 145/407 [10:53<19:48,  4.54s/it]\u001b[A\r\n",
      " 36%|██████████████▋                          | 146/407 [10:57<19:43,  4.54s/it]\u001b[A\r\n",
      " 36%|██████████████▊                          | 147/407 [11:02<19:39,  4.54s/it]\u001b[A\r\n",
      " 36%|██████████████▉                          | 148/407 [11:07<19:35,  4.54s/it]\u001b[A\r\n",
      " 37%|███████████████                          | 149/407 [11:11<19:30,  4.54s/it]\u001b[A\r\n",
      " 37%|███████████████                          | 150/407 [11:16<19:26,  4.54s/it]\u001b[A\r\n",
      " 37%|███████████████▏                         | 151/407 [11:20<19:21,  4.54s/it]\u001b[A\r\n",
      " 37%|███████████████▎                         | 152/407 [11:25<19:17,  4.54s/it]\u001b[A\r\n",
      " 38%|███████████████▍                         | 153/407 [11:29<19:12,  4.54s/it]\u001b[A\r\n",
      " 38%|███████████████▌                         | 154/407 [11:34<19:07,  4.54s/it]\u001b[A\r\n",
      " 38%|███████████████▌                         | 155/407 [11:38<19:03,  4.54s/it]\u001b[A\r\n",
      " 38%|███████████████▋                         | 156/407 [11:43<18:59,  4.54s/it]\u001b[A\r\n",
      " 39%|███████████████▊                         | 157/407 [11:47<18:54,  4.54s/it]\u001b[A\r\n",
      " 39%|███████████████▉                         | 158/407 [11:52<18:50,  4.54s/it]\u001b[A\r\n",
      " 39%|████████████████                         | 159/407 [11:56<18:45,  4.54s/it]\u001b[A\r\n",
      " 39%|████████████████                         | 160/407 [12:01<18:41,  4.54s/it]\u001b[A\r\n",
      " 40%|████████████████▏                        | 161/407 [12:06<18:36,  4.54s/it]\u001b[A\r\n",
      " 40%|████████████████▎                        | 162/407 [12:10<18:31,  4.54s/it]\u001b[A\r\n",
      " 40%|████████████████▍                        | 163/407 [12:15<18:27,  4.54s/it]\u001b[A\r\n",
      " 40%|████████████████▌                        | 164/407 [12:19<18:22,  4.54s/it]\u001b[A\r\n",
      " 41%|████████████████▌                        | 165/407 [12:24<18:18,  4.54s/it]\u001b[A\r\n",
      " 41%|████████████████▋                        | 166/407 [12:28<18:13,  4.54s/it]\u001b[A\r\n",
      " 41%|████████████████▊                        | 167/407 [12:33<18:08,  4.54s/it]\u001b[A\r\n",
      " 41%|████████████████▉                        | 168/407 [12:37<18:04,  4.54s/it]\u001b[A\r\n",
      " 42%|█████████████████                        | 169/407 [12:42<17:59,  4.54s/it]\u001b[A\r\n",
      " 42%|█████████████████▏                       | 170/407 [12:46<17:55,  4.54s/it]\u001b[A\r\n",
      " 42%|█████████████████▏                       | 171/407 [12:51<17:50,  4.54s/it]\u001b[A\r\n",
      " 42%|█████████████████▎                       | 172/407 [12:55<17:45,  4.54s/it]\u001b[A\r\n",
      " 43%|█████████████████▍                       | 173/407 [13:00<17:41,  4.54s/it]\u001b[A\r\n",
      " 43%|█████████████████▌                       | 174/407 [13:04<17:36,  4.53s/it]\u001b[A\r\n",
      " 43%|█████████████████▋                       | 175/407 [13:09<17:32,  4.54s/it]\u001b[A\r\n",
      " 43%|█████████████████▋                       | 176/407 [13:14<17:27,  4.53s/it]\u001b[A\r\n",
      " 43%|█████████████████▊                       | 177/407 [13:18<17:22,  4.53s/it]\u001b[A\r\n",
      " 44%|█████████████████▉                       | 178/407 [13:23<17:19,  4.54s/it]\u001b[A\r\n",
      " 44%|██████████████████                       | 179/407 [13:27<17:14,  4.54s/it]\u001b[A\r\n",
      " 44%|██████████████████▏                      | 180/407 [13:32<17:09,  4.53s/it]\u001b[A\r\n",
      " 44%|██████████████████▏                      | 181/407 [13:36<17:04,  4.53s/it]\u001b[A\r\n",
      " 45%|██████████████████▎                      | 182/407 [13:41<16:59,  4.53s/it]\u001b[A\r\n",
      " 45%|██████████████████▍                      | 183/407 [13:45<16:55,  4.53s/it]\u001b[A\r\n",
      " 45%|██████████████████▌                      | 184/407 [13:50<16:50,  4.53s/it]\u001b[A\r\n",
      " 45%|██████████████████▋                      | 185/407 [13:54<16:46,  4.53s/it]\u001b[A\r\n",
      " 46%|██████████████████▋                      | 186/407 [13:59<16:42,  4.54s/it]\u001b[A\r\n",
      " 46%|██████████████████▊                      | 187/407 [14:03<16:38,  4.54s/it]\u001b[A\r\n",
      " 46%|██████████████████▉                      | 188/407 [14:08<16:33,  4.54s/it]\u001b[A\r\n",
      " 46%|███████████████████                      | 189/407 [14:13<16:28,  4.54s/it]\u001b[A\r\n",
      " 47%|███████████████████▏                     | 190/407 [14:17<16:24,  4.54s/it]\u001b[A\r\n",
      " 47%|███████████████████▏                     | 191/407 [14:22<16:20,  4.54s/it]\u001b[A\r\n",
      " 47%|███████████████████▎                     | 192/407 [14:26<16:15,  4.54s/it]\u001b[A\r\n",
      " 47%|███████████████████▍                     | 193/407 [14:31<16:10,  4.54s/it]\u001b[A\r\n",
      " 48%|███████████████████▌                     | 194/407 [14:35<16:06,  4.54s/it]\u001b[A\r\n",
      " 48%|███████████████████▋                     | 195/407 [14:40<16:01,  4.53s/it]\u001b[A\r\n",
      " 48%|███████████████████▋                     | 196/407 [14:44<15:56,  4.53s/it]\u001b[A\r\n",
      " 48%|███████████████████▊                     | 197/407 [14:49<15:52,  4.54s/it]\u001b[A\r\n",
      " 49%|███████████████████▉                     | 198/407 [14:53<15:47,  4.54s/it]\u001b[A\r\n",
      " 49%|████████████████████                     | 199/407 [14:58<15:43,  4.54s/it]\u001b[A\r\n",
      " 49%|████████████████████▏                    | 200/407 [15:02<15:38,  4.54s/it]\u001b[A\r\n",
      " 49%|████████████████████▏                    | 201/407 [15:07<15:34,  4.54s/it]\u001b[A\r\n",
      " 50%|████████████████████▎                    | 202/407 [15:11<15:30,  4.54s/it]\u001b[A\r\n",
      " 50%|████████████████████▍                    | 203/407 [15:16<15:25,  4.54s/it]\u001b[A\r\n",
      " 50%|████████████████████▌                    | 204/407 [15:21<15:20,  4.54s/it]\u001b[A\r\n",
      " 50%|████████████████████▋                    | 205/407 [15:25<15:16,  4.54s/it]\u001b[A\r\n",
      " 51%|████████████████████▊                    | 206/407 [15:30<15:12,  4.54s/it]\u001b[A\r\n",
      " 51%|████████████████████▊                    | 207/407 [15:34<15:07,  4.54s/it]\u001b[A\r\n",
      " 51%|████████████████████▉                    | 208/407 [15:39<15:03,  4.54s/it]\u001b[A\r\n",
      " 51%|█████████████████████                    | 209/407 [15:43<14:58,  4.54s/it]\u001b[A\r\n",
      " 52%|█████████████████████▏                   | 210/407 [15:48<14:53,  4.54s/it]\u001b[A\r\n",
      " 52%|█████████████████████▎                   | 211/407 [15:52<14:49,  4.54s/it]\u001b[A\r\n",
      " 52%|█████████████████████▎                   | 212/407 [15:57<14:44,  4.54s/it]\u001b[A\r\n",
      " 52%|█████████████████████▍                   | 213/407 [16:01<14:40,  4.54s/it]\u001b[A\r\n",
      " 53%|█████████████████████▌                   | 214/407 [16:06<14:35,  4.54s/it]\u001b[A\r\n",
      " 53%|█████████████████████▋                   | 215/407 [16:10<14:31,  4.54s/it]\u001b[A\r\n",
      " 53%|█████████████████████▊                   | 216/407 [16:15<14:26,  4.54s/it]\u001b[A\r\n",
      " 53%|█████████████████████▊                   | 217/407 [16:20<14:21,  4.54s/it]\u001b[A\r\n",
      " 54%|█████████████████████▉                   | 218/407 [16:24<14:16,  4.53s/it]\u001b[A\r\n",
      " 54%|██████████████████████                   | 219/407 [16:29<14:12,  4.53s/it]\u001b[A\r\n",
      " 54%|██████████████████████▏                  | 220/407 [16:33<14:08,  4.53s/it]\u001b[A\r\n",
      " 54%|██████████████████████▎                  | 221/407 [16:38<14:03,  4.53s/it]\u001b[A\r\n",
      " 55%|██████████████████████▎                  | 222/407 [16:42<13:58,  4.54s/it]\u001b[A\r\n",
      " 55%|██████████████████████▍                  | 223/407 [16:47<13:54,  4.54s/it]\u001b[A\r\n",
      " 55%|██████████████████████▌                  | 224/407 [16:51<13:50,  4.54s/it]\u001b[A\r\n",
      " 55%|██████████████████████▋                  | 225/407 [16:56<13:45,  4.54s/it]\u001b[A\r\n",
      " 56%|██████████████████████▊                  | 226/407 [17:00<13:40,  4.54s/it]\u001b[A\r\n",
      " 56%|██████████████████████▊                  | 227/407 [17:05<13:36,  4.54s/it]\u001b[A\r\n",
      " 56%|██████████████████████▉                  | 228/407 [17:09<13:31,  4.53s/it]\u001b[A\r\n",
      " 56%|███████████████████████                  | 229/407 [17:14<13:27,  4.53s/it]\u001b[A\r\n",
      " 57%|███████████████████████▏                 | 230/407 [17:18<13:22,  4.53s/it]\u001b[A\r\n",
      " 57%|███████████████████████▎                 | 231/407 [17:23<13:17,  4.53s/it]\u001b[A\r\n",
      " 57%|███████████████████████▎                 | 232/407 [17:28<13:13,  4.53s/it]\u001b[A\r\n",
      " 57%|███████████████████████▍                 | 233/407 [17:32<13:09,  4.53s/it]\u001b[A\r\n",
      " 57%|███████████████████████▌                 | 234/407 [17:37<13:04,  4.54s/it]\u001b[A\r\n",
      " 58%|███████████████████████▋                 | 235/407 [17:41<13:00,  4.54s/it]\u001b[A\r\n",
      " 58%|███████████████████████▊                 | 236/407 [17:46<12:55,  4.54s/it]\u001b[A\r\n",
      " 58%|███████████████████████▊                 | 237/407 [17:50<12:51,  4.54s/it]\u001b[A\r\n",
      " 58%|███████████████████████▉                 | 238/407 [17:55<12:46,  4.54s/it]\u001b[A\r\n",
      " 59%|████████████████████████                 | 239/407 [17:59<12:41,  4.53s/it]\u001b[A\r\n",
      " 59%|████████████████████████▏                | 240/407 [18:04<12:37,  4.54s/it]\u001b[A\r\n",
      " 59%|████████████████████████▎                | 241/407 [18:08<12:32,  4.53s/it]\u001b[A\r\n",
      " 59%|████████████████████████▍                | 242/407 [18:13<12:28,  4.53s/it]\u001b[A\r\n",
      " 60%|████████████████████████▍                | 243/407 [18:17<12:23,  4.53s/it]\u001b[A\r\n",
      " 60%|████████████████████████▌                | 244/407 [18:22<12:19,  4.53s/it]\u001b[A\r\n",
      " 60%|████████████████████████▋                | 245/407 [18:27<12:14,  4.53s/it]\u001b[A\r\n",
      " 60%|████████████████████████▊                | 246/407 [18:31<12:09,  4.53s/it]\u001b[A\r\n",
      " 61%|████████████████████████▉                | 247/407 [18:36<12:05,  4.53s/it]\u001b[A\r\n",
      " 61%|████████████████████████▉                | 248/407 [18:40<12:00,  4.53s/it]\u001b[A\r\n",
      " 61%|█████████████████████████                | 249/407 [18:45<11:56,  4.53s/it]\u001b[A\r\n",
      " 61%|█████████████████████████▏               | 250/407 [18:49<11:51,  4.53s/it]\u001b[A\r\n",
      " 62%|█████████████████████████▎               | 251/407 [18:54<11:47,  4.53s/it]\u001b[A\r\n",
      " 62%|█████████████████████████▍               | 252/407 [18:58<11:43,  4.54s/it]\u001b[A\r\n",
      " 62%|█████████████████████████▍               | 253/407 [19:03<11:38,  4.54s/it]\u001b[A\r\n",
      " 62%|█████████████████████████▌               | 254/407 [19:07<11:34,  4.54s/it]\u001b[A\r\n",
      " 63%|█████████████████████████▋               | 255/407 [19:12<11:30,  4.54s/it]\u001b[A\r\n",
      " 63%|█████████████████████████▊               | 256/407 [19:16<11:25,  4.54s/it]\u001b[A\r\n",
      " 63%|█████████████████████████▉               | 257/407 [19:21<11:20,  4.54s/it]\u001b[A\r\n",
      " 63%|█████████████████████████▉               | 258/407 [19:25<11:16,  4.54s/it]\u001b[A\r\n",
      " 64%|██████████████████████████               | 259/407 [19:30<11:11,  4.54s/it]\u001b[A\r\n",
      " 64%|██████████████████████████▏              | 260/407 [19:35<11:07,  4.54s/it]\u001b[A\r\n",
      " 64%|██████████████████████████▎              | 261/407 [19:39<11:02,  4.54s/it]\u001b[A\r\n",
      " 64%|██████████████████████████▍              | 262/407 [19:44<10:57,  4.54s/it]\u001b[A\r\n",
      " 65%|██████████████████████████▍              | 263/407 [19:48<10:53,  4.54s/it]\u001b[A\r\n",
      " 65%|██████████████████████████▌              | 264/407 [19:53<10:49,  4.54s/it]\u001b[A\r\n",
      " 65%|██████████████████████████▋              | 265/407 [19:57<10:44,  4.54s/it]\u001b[A\r\n",
      " 65%|██████████████████████████▊              | 266/407 [20:02<10:39,  4.54s/it]\u001b[A\r\n",
      " 66%|██████████████████████████▉              | 267/407 [20:06<10:35,  4.54s/it]\u001b[A\r\n",
      " 66%|██████████████████████████▉              | 268/407 [20:11<10:30,  4.54s/it]\u001b[A\r\n",
      " 66%|███████████████████████████              | 269/407 [20:15<10:26,  4.54s/it]\u001b[A\r\n",
      " 66%|███████████████████████████▏             | 270/407 [20:20<10:21,  4.54s/it]\u001b[A\r\n",
      " 67%|███████████████████████████▎             | 271/407 [20:24<10:17,  4.54s/it]\u001b[A\r\n",
      " 67%|███████████████████████████▍             | 272/407 [20:29<10:12,  4.54s/it]\u001b[A\r\n",
      " 67%|███████████████████████████▌             | 273/407 [20:34<10:07,  4.54s/it]\u001b[A\r\n",
      " 67%|███████████████████████████▌             | 274/407 [20:38<10:03,  4.54s/it]\u001b[A\r\n",
      " 68%|███████████████████████████▋             | 275/407 [20:43<09:58,  4.54s/it]\u001b[A\r\n",
      " 68%|███████████████████████████▊             | 276/407 [20:47<09:54,  4.54s/it]\u001b[A\r\n",
      " 68%|███████████████████████████▉             | 277/407 [20:52<09:49,  4.54s/it]\u001b[A\r\n",
      " 68%|████████████████████████████             | 278/407 [20:56<09:45,  4.54s/it]\u001b[A\r\n",
      " 69%|████████████████████████████             | 279/407 [21:01<09:40,  4.54s/it]\u001b[A\r\n",
      " 69%|████████████████████████████▏            | 280/407 [21:05<09:36,  4.54s/it]\u001b[A\r\n",
      " 69%|████████████████████████████▎            | 281/407 [21:10<09:31,  4.54s/it]\u001b[A\r\n",
      " 69%|████████████████████████████▍            | 282/407 [21:14<09:27,  4.54s/it]\u001b[A\r\n",
      " 70%|████████████████████████████▌            | 283/407 [21:19<09:22,  4.54s/it]\u001b[A\r\n",
      " 70%|████████████████████████████▌            | 284/407 [21:23<09:17,  4.54s/it]\u001b[A\r\n",
      " 70%|████████████████████████████▋            | 285/407 [21:28<09:13,  4.54s/it]\u001b[A\r\n",
      " 70%|████████████████████████████▊            | 286/407 [21:33<09:08,  4.54s/it]\u001b[A\r\n",
      " 71%|████████████████████████████▉            | 287/407 [21:37<09:04,  4.54s/it]\u001b[A\r\n",
      " 71%|█████████████████████████████            | 288/407 [21:42<08:59,  4.54s/it]\u001b[A\r\n",
      " 71%|█████████████████████████████            | 289/407 [21:46<08:55,  4.54s/it]\u001b[A\r\n",
      " 71%|█████████████████████████████▏           | 290/407 [21:51<08:50,  4.54s/it]\u001b[A\r\n",
      " 71%|█████████████████████████████▎           | 291/407 [21:55<08:46,  4.54s/it]\u001b[A\r\n",
      " 72%|█████████████████████████████▍           | 292/407 [22:00<08:41,  4.54s/it]\u001b[A\r\n",
      " 72%|█████████████████████████████▌           | 293/407 [22:04<08:37,  4.54s/it]\u001b[A\r\n",
      " 72%|█████████████████████████████▌           | 294/407 [22:09<08:32,  4.54s/it]\u001b[A\r\n",
      " 72%|█████████████████████████████▋           | 295/407 [22:13<08:28,  4.54s/it]\u001b[A\r\n",
      " 73%|█████████████████████████████▊           | 296/407 [22:18<08:23,  4.54s/it]\u001b[A\r\n",
      " 73%|█████████████████████████████▉           | 297/407 [22:22<08:18,  4.53s/it]\u001b[A\r\n",
      " 73%|██████████████████████████████           | 298/407 [22:27<08:14,  4.53s/it]\u001b[A\r\n",
      " 73%|██████████████████████████████           | 299/407 [22:31<08:09,  4.53s/it]\u001b[A\r\n",
      " 74%|██████████████████████████████▏          | 300/407 [22:36<08:05,  4.54s/it]\u001b[A\r\n",
      " 74%|██████████████████████████████▎          | 301/407 [22:41<08:00,  4.54s/it]\u001b[A\r\n",
      " 74%|██████████████████████████████▍          | 302/407 [22:45<07:56,  4.54s/it]\u001b[A\r\n",
      " 74%|██████████████████████████████▌          | 303/407 [22:50<07:51,  4.54s/it]\u001b[A\r\n",
      " 75%|██████████████████████████████▌          | 304/407 [22:54<07:47,  4.54s/it]\u001b[A\r\n",
      " 75%|██████████████████████████████▋          | 305/407 [22:59<07:42,  4.54s/it]\u001b[A\r\n",
      " 75%|██████████████████████████████▊          | 306/407 [23:03<07:38,  4.54s/it]\u001b[A\r\n",
      " 75%|██████████████████████████████▉          | 307/407 [23:08<07:33,  4.54s/it]\u001b[A\r\n",
      " 76%|███████████████████████████████          | 308/407 [23:12<07:29,  4.54s/it]\u001b[A\r\n",
      " 76%|███████████████████████████████▏         | 309/407 [23:17<07:24,  4.54s/it]\u001b[A\r\n",
      " 76%|███████████████████████████████▏         | 310/407 [23:21<07:19,  4.54s/it]\u001b[A\r\n",
      " 76%|███████████████████████████████▎         | 311/407 [23:26<07:15,  4.53s/it]\u001b[A\r\n",
      " 77%|███████████████████████████████▍         | 312/407 [23:30<07:10,  4.53s/it]\u001b[A\r\n",
      " 77%|███████████████████████████████▌         | 313/407 [23:35<07:06,  4.54s/it]\u001b[A\r\n",
      " 77%|███████████████████████████████▋         | 314/407 [23:40<07:01,  4.54s/it]\u001b[A\r\n",
      " 77%|███████████████████████████████▋         | 315/407 [23:44<06:57,  4.54s/it]\u001b[A\r\n",
      " 78%|███████████████████████████████▊         | 316/407 [23:49<06:52,  4.54s/it]\u001b[A\r\n",
      " 78%|███████████████████████████████▉         | 317/407 [23:53<06:48,  4.54s/it]\u001b[A\r\n",
      " 78%|████████████████████████████████         | 318/407 [23:58<06:43,  4.54s/it]\u001b[A\r\n",
      " 78%|████████████████████████████████▏        | 319/407 [24:02<06:39,  4.54s/it]\u001b[A\r\n",
      " 79%|████████████████████████████████▏        | 320/407 [24:07<06:34,  4.54s/it]\u001b[A\r\n",
      " 79%|████████████████████████████████▎        | 321/407 [24:11<06:30,  4.54s/it]\u001b[A\r\n",
      " 79%|████████████████████████████████▍        | 322/407 [24:16<06:25,  4.53s/it]\u001b[A\r\n",
      " 79%|████████████████████████████████▌        | 323/407 [24:20<06:20,  4.54s/it]\u001b[A\r\n",
      " 80%|████████████████████████████████▋        | 324/407 [24:25<06:16,  4.53s/it]\u001b[A\r\n",
      " 80%|████████████████████████████████▋        | 325/407 [24:29<06:11,  4.53s/it]\u001b[A\r\n",
      " 80%|████████████████████████████████▊        | 326/407 [24:34<06:07,  4.54s/it]\u001b[A\r\n",
      " 80%|████████████████████████████████▉        | 327/407 [24:39<06:02,  4.53s/it]\u001b[A\r\n",
      " 81%|█████████████████████████████████        | 328/407 [24:43<05:58,  4.53s/it]\u001b[A\r\n",
      " 81%|█████████████████████████████████▏       | 329/407 [24:48<05:53,  4.53s/it]\u001b[A\r\n",
      " 81%|█████████████████████████████████▏       | 330/407 [24:52<05:49,  4.54s/it]\u001b[A\r\n",
      " 81%|█████████████████████████████████▎       | 331/407 [24:57<05:44,  4.54s/it]\u001b[A\r\n",
      " 82%|█████████████████████████████████▍       | 332/407 [25:01<05:40,  4.54s/it]\u001b[A\r\n",
      " 82%|█████████████████████████████████▌       | 333/407 [25:06<05:35,  4.54s/it]\u001b[A\r\n",
      " 82%|█████████████████████████████████▋       | 334/407 [25:10<05:31,  4.54s/it]\u001b[A\r\n",
      " 82%|█████████████████████████████████▋       | 335/407 [25:15<05:26,  4.54s/it]\u001b[A\r\n",
      " 83%|█████████████████████████████████▊       | 336/407 [25:19<05:22,  4.54s/it]\u001b[A\r\n",
      " 83%|█████████████████████████████████▉       | 337/407 [25:24<05:17,  4.54s/it]\u001b[A\r\n",
      " 83%|██████████████████████████████████       | 338/407 [25:28<05:12,  4.54s/it]\u001b[A\r\n",
      " 83%|██████████████████████████████████▏      | 339/407 [25:33<05:08,  4.54s/it]\u001b[A\r\n",
      " 84%|██████████████████████████████████▎      | 340/407 [25:37<05:04,  4.54s/it]\u001b[A\r\n",
      " 84%|██████████████████████████████████▎      | 341/407 [25:42<04:59,  4.54s/it]\u001b[A\r\n",
      " 84%|██████████████████████████████████▍      | 342/407 [25:47<04:54,  4.54s/it]\u001b[A\r\n",
      " 84%|██████████████████████████████████▌      | 343/407 [25:51<04:50,  4.54s/it]\u001b[A\r\n",
      " 85%|██████████████████████████████████▋      | 344/407 [25:56<04:45,  4.54s/it]\u001b[A\r\n",
      " 85%|██████████████████████████████████▊      | 345/407 [26:00<04:41,  4.54s/it]\u001b[A\r\n",
      " 85%|██████████████████████████████████▊      | 346/407 [26:05<04:36,  4.54s/it]\u001b[A\r\n",
      " 85%|██████████████████████████████████▉      | 347/407 [26:09<04:32,  4.54s/it]\u001b[A\r\n",
      " 86%|███████████████████████████████████      | 348/407 [26:14<04:27,  4.54s/it]\u001b[A\r\n",
      " 86%|███████████████████████████████████▏     | 349/407 [26:18<04:23,  4.54s/it]\u001b[A\r\n",
      " 86%|███████████████████████████████████▎     | 350/407 [26:23<04:18,  4.54s/it]\u001b[A\r\n",
      " 86%|███████████████████████████████████▎     | 351/407 [26:27<04:14,  4.54s/it]\u001b[A\r\n",
      " 86%|███████████████████████████████████▍     | 352/407 [26:32<04:09,  4.53s/it]\u001b[A\r\n",
      " 87%|███████████████████████████████████▌     | 353/407 [26:36<04:04,  4.54s/it]\u001b[A\r\n",
      " 87%|███████████████████████████████████▋     | 354/407 [26:41<04:00,  4.54s/it]\u001b[A\r\n",
      " 87%|███████████████████████████████████▊     | 355/407 [26:46<03:55,  4.54s/it]\u001b[A\r\n",
      " 87%|███████████████████████████████████▊     | 356/407 [26:50<03:51,  4.54s/it]\u001b[A\r\n",
      " 88%|███████████████████████████████████▉     | 357/407 [26:55<03:46,  4.54s/it]\u001b[A\r\n",
      " 88%|████████████████████████████████████     | 358/407 [26:59<03:42,  4.54s/it]\u001b[A\r\n",
      " 88%|████████████████████████████████████▏    | 359/407 [27:04<03:37,  4.54s/it]\u001b[A\r\n",
      " 88%|████████████████████████████████████▎    | 360/407 [27:08<03:33,  4.54s/it]\u001b[A\r\n",
      " 89%|████████████████████████████████████▎    | 361/407 [27:13<03:28,  4.54s/it]\u001b[A\r\n",
      " 89%|████████████████████████████████████▍    | 362/407 [27:17<03:24,  4.54s/it]\u001b[A\r\n",
      " 89%|████████████████████████████████████▌    | 363/407 [27:22<03:19,  4.53s/it]\u001b[A\r\n",
      " 89%|████████████████████████████████████▋    | 364/407 [27:26<03:15,  4.54s/it]\u001b[A\r\n",
      " 90%|████████████████████████████████████▊    | 365/407 [27:31<03:10,  4.54s/it]\u001b[A\r\n",
      " 90%|████████████████████████████████████▊    | 366/407 [27:35<03:05,  4.54s/it]\u001b[A\r\n",
      " 90%|████████████████████████████████████▉    | 367/407 [27:40<03:01,  4.54s/it]\u001b[A\r\n",
      " 90%|█████████████████████████████████████    | 368/407 [27:45<02:56,  4.54s/it]\u001b[A\r\n",
      " 91%|█████████████████████████████████████▏   | 369/407 [27:49<02:52,  4.54s/it]\u001b[A\r\n",
      " 91%|█████████████████████████████████████▎   | 370/407 [27:54<02:47,  4.54s/it]\u001b[A\r\n",
      " 91%|█████████████████████████████████████▎   | 371/407 [27:58<02:43,  4.54s/it]\u001b[A\r\n",
      " 91%|█████████████████████████████████████▍   | 372/407 [28:03<02:38,  4.53s/it]\u001b[A\r\n",
      " 92%|█████████████████████████████████████▌   | 373/407 [28:07<02:34,  4.54s/it]\u001b[A\r\n",
      " 92%|█████████████████████████████████████▋   | 374/407 [28:12<02:29,  4.54s/it]\u001b[A\r\n",
      " 92%|█████████████████████████████████████▊   | 375/407 [28:16<02:25,  4.53s/it]\u001b[A\r\n",
      " 92%|█████████████████████████████████████▉   | 376/407 [28:21<02:20,  4.54s/it]\u001b[A\r\n",
      " 93%|█████████████████████████████████████▉   | 377/407 [28:25<02:16,  4.54s/it]\u001b[A\r\n",
      " 93%|██████████████████████████████████████   | 378/407 [28:30<02:11,  4.54s/it]\u001b[A\r\n",
      " 93%|██████████████████████████████████████▏  | 379/407 [28:34<02:07,  4.54s/it]\u001b[A\r\n",
      " 93%|██████████████████████████████████████▎  | 380/407 [28:39<02:02,  4.54s/it]\u001b[A\r\n",
      " 94%|██████████████████████████████████████▍  | 381/407 [28:43<01:57,  4.53s/it]\u001b[A\r\n",
      " 94%|██████████████████████████████████████▍  | 382/407 [28:48<01:53,  4.54s/it]\u001b[A\r\n",
      " 94%|██████████████████████████████████████▌  | 383/407 [28:53<01:48,  4.53s/it]\u001b[A\r\n",
      " 94%|██████████████████████████████████████▋  | 384/407 [28:57<01:44,  4.53s/it]\u001b[A\r\n",
      " 95%|██████████████████████████████████████▊  | 385/407 [29:02<01:39,  4.54s/it]\u001b[A\r\n",
      " 95%|██████████████████████████████████████▉  | 386/407 [29:06<01:35,  4.54s/it]\u001b[A\r\n",
      " 95%|██████████████████████████████████████▉  | 387/407 [29:11<01:30,  4.54s/it]\u001b[A\r\n",
      " 95%|███████████████████████████████████████  | 388/407 [29:15<01:26,  4.54s/it]\u001b[A\r\n",
      " 96%|███████████████████████████████████████▏ | 389/407 [29:20<01:21,  4.54s/it]\u001b[A\r\n",
      " 96%|███████████████████████████████████████▎ | 390/407 [29:24<01:17,  4.54s/it]\u001b[A\r\n",
      " 96%|███████████████████████████████████████▍ | 391/407 [29:29<01:12,  4.54s/it]\u001b[A\r\n",
      " 96%|███████████████████████████████████████▍ | 392/407 [29:33<01:08,  4.54s/it]\u001b[A\r\n",
      " 97%|███████████████████████████████████████▌ | 393/407 [29:38<01:03,  4.53s/it]\u001b[A\r\n",
      " 97%|███████████████████████████████████████▋ | 394/407 [29:42<00:58,  4.54s/it]\u001b[A\r\n",
      " 97%|███████████████████████████████████████▊ | 395/407 [29:47<00:54,  4.54s/it]\u001b[A\r\n",
      " 97%|███████████████████████████████████████▉ | 396/407 [29:52<00:49,  4.54s/it]\u001b[A\r\n",
      " 98%|███████████████████████████████████████▉ | 397/407 [29:56<00:45,  4.54s/it]\u001b[A\r\n",
      " 98%|████████████████████████████████████████ | 398/407 [30:01<00:40,  4.54s/it]\u001b[A\r\n",
      " 98%|████████████████████████████████████████▏| 399/407 [30:05<00:36,  4.53s/it]\u001b[A\r\n",
      " 98%|████████████████████████████████████████▎| 400/407 [30:10<00:31,  4.54s/it]\u001b[A\r\n",
      " 99%|████████████████████████████████████████▍| 401/407 [30:14<00:27,  4.54s/it]\u001b[A\r\n",
      " 99%|████████████████████████████████████████▍| 402/407 [30:19<00:22,  4.53s/it]\u001b[A\r\n",
      " 99%|████████████████████████████████████████▌| 403/407 [30:23<00:18,  4.53s/it]\u001b[A\r\n",
      " 99%|████████████████████████████████████████▋| 404/407 [30:28<00:13,  4.53s/it]\u001b[A\r\n",
      "100%|████████████████████████████████████████▊| 405/407 [30:32<00:09,  4.53s/it]\u001b[A\r\n",
      "100%|████████████████████████████████████████▉| 406/407 [30:37<00:04,  4.53s/it]\u001b[A\r\n",
      "\r\n",
      "\u001b[A{'eval_loss': 1.02688729763031, 'eval_accuracy': {'accuracy': 0.5651105651105651}, 'eval_runtime': 1846.4532, 'eval_samples_per_second': 0.22, 'eval_steps_per_second': 0.22, 'epoch': 0.25}\r\n",
      " 25%|█████████                            | 50/203 [2:02:46<4:41:28, 110.38s/it]\r\n",
      "100%|█████████████████████████████████████████| 407/407 [30:41<00:00,  4.53s/it]\u001b[A\r\n",
      "{'loss': 0.8905, 'learning_rate': 1.842100884922812e-05, 'epoch': 0.3}\r\n",
      "{'loss': 0.9078, 'learning_rate': 1.7369533159843368e-05, 'epoch': 0.34}\r\n",
      "{'loss': 0.845, 'learning_rate': 1.609902004400073e-05, 'epoch': 0.39}\r\n",
      "{'loss': 0.7573, 'learning_rate': 1.4647231720437687e-05, 'epoch': 0.44}\r\n",
      "{'loss': 0.7376, 'learning_rate': 1.3057318273597531e-05, 'epoch': 0.49}\r\n",
      " 49%|█████████████████▋                  | 100/203 [3:34:45<3:09:29, 110.39s/it]\r\n",
      "  0%|                                                   | 0/407 [00:00<?, ?it/s]\u001b[A\r\n",
      "  0%|▏                                          | 2/407 [00:04<15:20,  2.27s/it]\u001b[A\r\n",
      "  1%|▎                                          | 3/407 [00:09<21:40,  3.22s/it]\u001b[A\r\n",
      "  1%|▍                                          | 4/407 [00:13<24:56,  3.71s/it]\u001b[A\r\n",
      "  1%|▌                                          | 5/407 [00:18<26:47,  4.00s/it]\u001b[A\r\n",
      "  1%|▋                                          | 6/407 [00:22<27:55,  4.18s/it]\u001b[A\r\n",
      "  2%|▋                                          | 7/407 [00:27<28:37,  4.29s/it]\u001b[A\r\n",
      "  2%|▊                                          | 8/407 [00:31<29:03,  4.37s/it]\u001b[A\r\n",
      "  2%|▉                                          | 9/407 [00:36<29:20,  4.42s/it]\u001b[A\r\n",
      "  2%|█                                         | 10/407 [00:40<29:29,  4.46s/it]\u001b[A\r\n",
      "  3%|█▏                                        | 11/407 [00:45<29:34,  4.48s/it]\u001b[A\r\n",
      "  3%|█▏                                        | 12/407 [00:49<29:37,  4.50s/it]\u001b[A\r\n",
      "  3%|█▎                                        | 13/407 [00:54<29:37,  4.51s/it]\u001b[A\r\n",
      "  3%|█▍                                        | 14/407 [00:59<29:35,  4.52s/it]\u001b[A\r\n",
      "  4%|█▌                                        | 15/407 [01:03<29:33,  4.52s/it]\u001b[A\r\n",
      "  4%|█▋                                        | 16/407 [01:08<29:29,  4.53s/it]\u001b[A\r\n",
      "  4%|█▊                                        | 17/407 [01:12<29:25,  4.53s/it]\u001b[A\r\n",
      "  4%|█▊                                        | 18/407 [01:17<29:22,  4.53s/it]\u001b[A\r\n",
      "  5%|█▉                                        | 19/407 [01:21<29:17,  4.53s/it]\u001b[A\r\n",
      "  5%|██                                        | 20/407 [01:26<29:14,  4.53s/it]\u001b[A\r\n",
      "  5%|██▏                                       | 21/407 [01:30<29:10,  4.53s/it]\u001b[A\r\n",
      "  5%|██▎                                       | 22/407 [01:35<29:06,  4.54s/it]\u001b[A\r\n",
      "  6%|██▎                                       | 23/407 [01:39<29:02,  4.54s/it]\u001b[A\r\n",
      "  6%|██▍                                       | 24/407 [01:44<28:58,  4.54s/it]\u001b[A\r\n",
      "  6%|██▌                                       | 25/407 [01:48<28:53,  4.54s/it]\u001b[A\r\n",
      "  6%|██▋                                       | 26/407 [01:53<28:48,  4.54s/it]\u001b[A\r\n",
      "  7%|██▊                                       | 27/407 [01:57<28:43,  4.54s/it]\u001b[A\r\n",
      "  7%|██▉                                       | 28/407 [02:02<28:39,  4.54s/it]\u001b[A\r\n",
      "  7%|██▉                                       | 29/407 [02:07<28:34,  4.53s/it]\u001b[A\r\n",
      "  7%|███                                       | 30/407 [02:11<28:29,  4.53s/it]\u001b[A\r\n",
      "  8%|███▏                                      | 31/407 [02:16<28:24,  4.53s/it]\u001b[A\r\n",
      "  8%|███▎                                      | 32/407 [02:20<28:19,  4.53s/it]\u001b[A\r\n",
      "  8%|███▍                                      | 33/407 [02:25<28:15,  4.53s/it]\u001b[A\r\n",
      "  8%|███▌                                      | 34/407 [02:29<28:10,  4.53s/it]\u001b[A\r\n",
      "  9%|███▌                                      | 35/407 [02:34<28:06,  4.53s/it]\u001b[A\r\n",
      "  9%|███▋                                      | 36/407 [02:38<28:02,  4.53s/it]\u001b[A\r\n",
      "  9%|███▊                                      | 37/407 [02:43<27:57,  4.53s/it]\u001b[A\r\n",
      "  9%|███▉                                      | 38/407 [02:47<27:53,  4.53s/it]\u001b[A\r\n",
      " 10%|████                                      | 39/407 [02:52<27:48,  4.54s/it]\u001b[A\r\n",
      " 10%|████▏                                     | 40/407 [02:56<27:44,  4.54s/it]\u001b[A\r\n",
      " 10%|████▏                                     | 41/407 [03:01<27:40,  4.54s/it]\u001b[A\r\n",
      " 10%|████▎                                     | 42/407 [03:05<27:35,  4.54s/it]\u001b[A\r\n",
      " 11%|████▍                                     | 43/407 [03:10<27:30,  4.53s/it]\u001b[A\r\n",
      " 11%|████▌                                     | 44/407 [03:15<27:25,  4.53s/it]\u001b[A\r\n",
      " 11%|████▋                                     | 45/407 [03:19<27:21,  4.54s/it]\u001b[A\r\n",
      " 11%|████▋                                     | 46/407 [03:24<27:18,  4.54s/it]\u001b[A\r\n",
      " 12%|████▊                                     | 47/407 [03:28<27:14,  4.54s/it]\u001b[A\r\n",
      " 12%|████▉                                     | 48/407 [03:33<27:08,  4.54s/it]\u001b[A\r\n",
      " 12%|█████                                     | 49/407 [03:37<27:03,  4.53s/it]\u001b[A\r\n",
      " 12%|█████▏                                    | 50/407 [03:42<26:59,  4.54s/it]\u001b[A\r\n",
      " 13%|█████▎                                    | 51/407 [03:46<26:54,  4.54s/it]\u001b[A\r\n",
      " 13%|█████▎                                    | 52/407 [03:51<26:49,  4.53s/it]\u001b[A\r\n",
      " 13%|█████▍                                    | 53/407 [03:55<26:45,  4.53s/it]\u001b[A\r\n",
      " 13%|█████▌                                    | 54/407 [04:00<26:40,  4.53s/it]\u001b[A\r\n",
      " 14%|█████▋                                    | 55/407 [04:04<26:35,  4.53s/it]\u001b[A\r\n",
      " 14%|█████▊                                    | 56/407 [04:09<26:31,  4.53s/it]\u001b[A\r\n",
      " 14%|█████▉                                    | 57/407 [04:14<26:27,  4.53s/it]\u001b[A\r\n",
      " 14%|█████▉                                    | 58/407 [04:18<26:23,  4.54s/it]\u001b[A\r\n",
      " 14%|██████                                    | 59/407 [04:23<26:18,  4.54s/it]\u001b[A\r\n",
      " 15%|██████▏                                   | 60/407 [04:27<26:13,  4.53s/it]\u001b[A\r\n",
      " 15%|██████▎                                   | 61/407 [04:32<26:09,  4.54s/it]\u001b[A\r\n",
      " 15%|██████▍                                   | 62/407 [04:36<26:04,  4.53s/it]\u001b[A\r\n",
      " 15%|██████▌                                   | 63/407 [04:41<26:00,  4.54s/it]\u001b[A\r\n",
      " 16%|██████▌                                   | 64/407 [04:45<25:56,  4.54s/it]\u001b[A\r\n",
      " 16%|██████▋                                   | 65/407 [04:50<25:51,  4.54s/it]\u001b[A\r\n",
      " 16%|██████▊                                   | 66/407 [04:54<25:47,  4.54s/it]\u001b[A\r\n",
      " 16%|██████▉                                   | 67/407 [04:59<25:42,  4.54s/it]\u001b[A\r\n",
      " 17%|███████                                   | 68/407 [05:03<25:38,  4.54s/it]\u001b[A\r\n",
      " 17%|███████                                   | 69/407 [05:08<25:33,  4.54s/it]\u001b[A\r\n",
      " 17%|███████▏                                  | 70/407 [05:13<25:29,  4.54s/it]\u001b[A\r\n",
      " 17%|███████▎                                  | 71/407 [05:17<25:24,  4.54s/it]\u001b[A\r\n",
      " 18%|███████▍                                  | 72/407 [05:22<25:19,  4.54s/it]\u001b[A\r\n",
      " 18%|███████▌                                  | 73/407 [05:26<25:14,  4.54s/it]\u001b[A\r\n",
      " 18%|███████▋                                  | 74/407 [05:31<25:10,  4.54s/it]\u001b[A\r\n",
      " 18%|███████▋                                  | 75/407 [05:35<25:05,  4.53s/it]\u001b[A\r\n",
      " 19%|███████▊                                  | 76/407 [05:40<25:01,  4.53s/it]\u001b[A\r\n",
      " 19%|███████▉                                  | 77/407 [05:44<24:55,  4.53s/it]\u001b[A\r\n",
      " 19%|████████                                  | 78/407 [05:49<24:51,  4.53s/it]\u001b[A\r\n",
      " 19%|████████▏                                 | 79/407 [05:53<24:46,  4.53s/it]\u001b[A\r\n",
      " 20%|████████▎                                 | 80/407 [05:58<24:42,  4.53s/it]\u001b[A\r\n",
      " 20%|████████▎                                 | 81/407 [06:02<24:38,  4.53s/it]\u001b[A\r\n",
      " 20%|████████▍                                 | 82/407 [06:07<24:33,  4.53s/it]\u001b[A\r\n",
      " 20%|████████▌                                 | 83/407 [06:11<24:28,  4.53s/it]\u001b[A\r\n",
      " 21%|████████▋                                 | 84/407 [06:16<24:24,  4.53s/it]\u001b[A\r\n",
      " 21%|████████▊                                 | 85/407 [06:21<24:19,  4.53s/it]\u001b[A\r\n",
      " 21%|████████▊                                 | 86/407 [06:25<24:15,  4.54s/it]\u001b[A\r\n",
      " 21%|████████▉                                 | 87/407 [06:30<24:11,  4.54s/it]\u001b[A\r\n",
      " 22%|█████████                                 | 88/407 [06:34<24:06,  4.54s/it]\u001b[A\r\n",
      " 22%|█████████▏                                | 89/407 [06:39<24:01,  4.53s/it]\u001b[A\r\n",
      " 22%|█████████▎                                | 90/407 [06:43<23:57,  4.54s/it]\u001b[A\r\n",
      " 22%|█████████▍                                | 91/407 [06:48<23:53,  4.54s/it]\u001b[A\r\n",
      " 23%|█████████▍                                | 92/407 [06:52<23:48,  4.53s/it]\u001b[A\r\n",
      " 23%|█████████▌                                | 93/407 [06:57<23:43,  4.53s/it]\u001b[A\r\n",
      " 23%|█████████▋                                | 94/407 [07:01<23:39,  4.54s/it]\u001b[A\r\n",
      " 23%|█████████▊                                | 95/407 [07:06<23:34,  4.53s/it]\u001b[A\r\n",
      " 24%|█████████▉                                | 96/407 [07:10<23:29,  4.53s/it]\u001b[A\r\n",
      " 24%|██████████                                | 97/407 [07:15<23:24,  4.53s/it]\u001b[A\r\n",
      " 24%|██████████                                | 98/407 [07:19<23:20,  4.53s/it]\u001b[A\r\n",
      " 24%|██████████▏                               | 99/407 [07:24<23:15,  4.53s/it]\u001b[A\r\n",
      " 25%|██████████                               | 100/407 [07:29<23:11,  4.53s/it]\u001b[A\r\n",
      " 25%|██████████▏                              | 101/407 [07:33<23:07,  4.53s/it]\u001b[A\r\n",
      " 25%|██████████▎                              | 102/407 [07:38<23:02,  4.53s/it]\u001b[A\r\n",
      " 25%|██████████▍                              | 103/407 [07:42<22:58,  4.54s/it]\u001b[A\r\n",
      " 26%|██████████▍                              | 104/407 [07:47<22:53,  4.53s/it]\u001b[A\r\n",
      " 26%|██████████▌                              | 105/407 [07:51<22:49,  4.53s/it]\u001b[A\r\n",
      " 26%|██████████▋                              | 106/407 [07:56<22:44,  4.53s/it]\u001b[A\r\n",
      " 26%|██████████▊                              | 107/407 [08:00<22:39,  4.53s/it]\u001b[A\r\n",
      " 27%|██████████▉                              | 108/407 [08:05<22:35,  4.53s/it]\u001b[A\r\n",
      " 27%|██████████▉                              | 109/407 [08:09<22:30,  4.53s/it]\u001b[A\r\n",
      " 27%|███████████                              | 110/407 [08:14<22:26,  4.53s/it]\u001b[A\r\n",
      " 27%|███████████▏                             | 111/407 [08:18<22:21,  4.53s/it]\u001b[A\r\n",
      " 28%|███████████▎                             | 112/407 [08:23<22:17,  4.53s/it]\u001b[A\r\n",
      " 28%|███████████▍                             | 113/407 [08:27<22:13,  4.54s/it]\u001b[A\r\n",
      " 28%|███████████▍                             | 114/407 [08:32<22:09,  4.54s/it]\u001b[A\r\n",
      " 28%|███████████▌                             | 115/407 [08:37<22:04,  4.54s/it]\u001b[A\r\n",
      " 29%|███████████▋                             | 116/407 [08:41<21:59,  4.54s/it]\u001b[A\r\n",
      " 29%|███████████▊                             | 117/407 [08:46<21:55,  4.54s/it]\u001b[A\r\n",
      " 29%|███████████▉                             | 118/407 [08:50<21:50,  4.54s/it]\u001b[A\r\n",
      " 29%|███████████▉                             | 119/407 [08:55<21:46,  4.54s/it]\u001b[A\r\n",
      " 29%|████████████                             | 120/407 [08:59<21:42,  4.54s/it]\u001b[A\r\n",
      " 30%|████████████▏                            | 121/407 [09:04<21:37,  4.54s/it]\u001b[A\r\n",
      " 30%|████████████▎                            | 122/407 [09:08<21:32,  4.54s/it]\u001b[A\r\n",
      " 30%|████████████▍                            | 123/407 [09:13<21:28,  4.54s/it]\u001b[A\r\n",
      " 30%|████████████▍                            | 124/407 [09:17<21:24,  4.54s/it]\u001b[A\r\n",
      " 31%|████████████▌                            | 125/407 [09:22<21:19,  4.54s/it]\u001b[A\r\n",
      " 31%|████████████▋                            | 126/407 [09:26<21:14,  4.54s/it]\u001b[A\r\n",
      " 31%|████████████▊                            | 127/407 [09:31<21:09,  4.53s/it]\u001b[A\r\n",
      " 31%|████████████▉                            | 128/407 [09:36<21:05,  4.53s/it]\u001b[A\r\n",
      " 32%|████████████▉                            | 129/407 [09:40<21:00,  4.53s/it]\u001b[A\r\n",
      " 32%|█████████████                            | 130/407 [09:45<20:56,  4.54s/it]\u001b[A\r\n",
      " 32%|█████████████▏                           | 131/407 [09:49<20:51,  4.54s/it]\u001b[A\r\n",
      " 32%|█████████████▎                           | 132/407 [09:54<20:46,  4.53s/it]\u001b[A\r\n",
      " 33%|█████████████▍                           | 133/407 [09:58<20:42,  4.53s/it]\u001b[A\r\n",
      " 33%|█████████████▍                           | 134/407 [10:03<20:38,  4.54s/it]\u001b[A\r\n",
      " 33%|█████████████▌                           | 135/407 [10:07<20:34,  4.54s/it]\u001b[A\r\n",
      " 33%|█████████████▋                           | 136/407 [10:12<20:29,  4.54s/it]\u001b[A\r\n",
      " 34%|█████████████▊                           | 137/407 [10:16<20:24,  4.53s/it]\u001b[A\r\n",
      " 34%|█████████████▉                           | 138/407 [10:21<20:19,  4.54s/it]\u001b[A\r\n",
      " 34%|██████████████                           | 139/407 [10:25<20:15,  4.53s/it]\u001b[A\r\n",
      " 34%|██████████████                           | 140/407 [10:30<20:10,  4.54s/it]\u001b[A\r\n",
      " 35%|██████████████▏                          | 141/407 [10:34<20:06,  4.54s/it]\u001b[A\r\n",
      " 35%|██████████████▎                          | 142/407 [10:39<20:01,  4.54s/it]\u001b[A\r\n",
      " 35%|██████████████▍                          | 143/407 [10:44<19:57,  4.53s/it]\u001b[A\r\n",
      " 35%|██████████████▌                          | 144/407 [10:48<19:52,  4.53s/it]\u001b[A\r\n",
      " 36%|██████████████▌                          | 145/407 [10:53<19:47,  4.53s/it]\u001b[A\r\n",
      " 36%|██████████████▋                          | 146/407 [10:57<19:42,  4.53s/it]\u001b[A\r\n",
      " 36%|██████████████▊                          | 147/407 [11:02<19:38,  4.53s/it]\u001b[A\r\n",
      " 36%|██████████████▉                          | 148/407 [11:06<19:34,  4.53s/it]\u001b[A\r\n",
      " 37%|███████████████                          | 149/407 [11:11<19:29,  4.53s/it]\u001b[A\r\n",
      " 37%|███████████████                          | 150/407 [11:15<19:25,  4.54s/it]\u001b[A\r\n",
      " 37%|███████████████▏                         | 151/407 [11:20<19:21,  4.54s/it]\u001b[A\r\n",
      " 37%|███████████████▎                         | 152/407 [11:24<19:16,  4.54s/it]\u001b[A\r\n",
      " 38%|███████████████▍                         | 153/407 [11:29<19:12,  4.54s/it]\u001b[A\r\n",
      " 38%|███████████████▌                         | 154/407 [11:33<19:08,  4.54s/it]\u001b[A\r\n",
      " 38%|███████████████▌                         | 155/407 [11:38<19:03,  4.54s/it]\u001b[A\r\n",
      " 38%|███████████████▋                         | 156/407 [11:43<18:59,  4.54s/it]\u001b[A\r\n",
      " 39%|███████████████▊                         | 157/407 [11:47<18:54,  4.54s/it]\u001b[A\r\n",
      " 39%|███████████████▉                         | 158/407 [11:52<18:49,  4.54s/it]\u001b[A\r\n",
      " 39%|████████████████                         | 159/407 [11:56<18:44,  4.54s/it]\u001b[A\r\n",
      " 39%|████████████████                         | 160/407 [12:01<18:39,  4.53s/it]\u001b[A\r\n",
      " 40%|████████████████▏                        | 161/407 [12:05<18:35,  4.53s/it]\u001b[A\r\n",
      " 40%|████████████████▎                        | 162/407 [12:10<18:30,  4.53s/it]\u001b[A\r\n",
      " 40%|████████████████▍                        | 163/407 [12:14<18:26,  4.53s/it]\u001b[A\r\n",
      " 40%|████████████████▌                        | 164/407 [12:19<18:21,  4.53s/it]\u001b[A\r\n",
      " 41%|████████████████▌                        | 165/407 [12:23<18:16,  4.53s/it]\u001b[A\r\n",
      " 41%|████████████████▋                        | 166/407 [12:28<18:12,  4.53s/it]\u001b[A\r\n",
      " 41%|████████████████▊                        | 167/407 [12:32<18:08,  4.53s/it]\u001b[A\r\n",
      " 41%|████████████████▉                        | 168/407 [12:37<18:03,  4.53s/it]\u001b[A\r\n",
      " 42%|█████████████████                        | 169/407 [12:41<17:59,  4.53s/it]\u001b[A\r\n",
      " 42%|█████████████████▏                       | 170/407 [12:46<17:54,  4.53s/it]\u001b[A\r\n",
      " 42%|█████████████████▏                       | 171/407 [12:51<17:50,  4.53s/it]\u001b[A\r\n",
      " 42%|█████████████████▎                       | 172/407 [12:55<17:45,  4.53s/it]\u001b[A\r\n",
      " 43%|█████████████████▍                       | 173/407 [13:00<17:41,  4.53s/it]\u001b[A\r\n",
      " 43%|█████████████████▌                       | 174/407 [13:04<17:36,  4.53s/it]\u001b[A\r\n",
      " 43%|█████████████████▋                       | 175/407 [13:09<17:31,  4.53s/it]\u001b[A\r\n",
      " 43%|█████████████████▋                       | 176/407 [13:13<17:27,  4.53s/it]\u001b[A\r\n",
      " 43%|█████████████████▊                       | 177/407 [13:18<17:22,  4.53s/it]\u001b[A\r\n",
      " 44%|█████████████████▉                       | 178/407 [13:22<17:18,  4.54s/it]\u001b[A\r\n",
      " 44%|██████████████████                       | 179/407 [13:27<17:13,  4.53s/it]\u001b[A\r\n",
      " 44%|██████████████████▏                      | 180/407 [13:31<17:09,  4.53s/it]\u001b[A\r\n",
      " 44%|██████████████████▏                      | 181/407 [13:36<17:05,  4.54s/it]\u001b[A\r\n",
      " 45%|██████████████████▎                      | 182/407 [13:40<17:01,  4.54s/it]\u001b[A\r\n",
      " 45%|██████████████████▍                      | 183/407 [13:45<16:56,  4.54s/it]\u001b[A\r\n",
      " 45%|██████████████████▌                      | 184/407 [13:49<16:51,  4.54s/it]\u001b[A\r\n",
      " 45%|██████████████████▋                      | 185/407 [13:54<16:46,  4.54s/it]\u001b[A\r\n",
      " 46%|██████████████████▋                      | 186/407 [13:59<16:42,  4.54s/it]\u001b[A\r\n",
      " 46%|██████████████████▊                      | 187/407 [14:03<16:37,  4.53s/it]\u001b[A\r\n",
      " 46%|██████████████████▉                      | 188/407 [14:08<16:33,  4.53s/it]\u001b[A\r\n",
      " 46%|███████████████████                      | 189/407 [14:12<16:28,  4.53s/it]\u001b[A\r\n",
      " 47%|███████████████████▏                     | 190/407 [14:17<16:23,  4.53s/it]\u001b[A\r\n",
      " 47%|███████████████████▏                     | 191/407 [14:21<16:19,  4.53s/it]\u001b[A\r\n",
      " 47%|███████████████████▎                     | 192/407 [14:26<16:15,  4.54s/it]\u001b[A\r\n",
      " 47%|███████████████████▍                     | 193/407 [14:30<16:10,  4.54s/it]\u001b[A\r\n",
      " 48%|███████████████████▌                     | 194/407 [14:35<16:06,  4.54s/it]\u001b[A\r\n",
      " 48%|███████████████████▋                     | 195/407 [14:39<16:01,  4.54s/it]\u001b[A\r\n",
      " 48%|███████████████████▋                     | 196/407 [14:44<15:56,  4.53s/it]\u001b[A\r\n",
      " 48%|███████████████████▊                     | 197/407 [14:48<15:52,  4.54s/it]\u001b[A\r\n",
      " 49%|███████████████████▉                     | 198/407 [14:53<15:47,  4.54s/it]\u001b[A\r\n",
      " 49%|████████████████████                     | 199/407 [14:57<15:43,  4.54s/it]\u001b[A\r\n",
      " 49%|████████████████████▏                    | 200/407 [15:02<15:39,  4.54s/it]\u001b[A\r\n",
      " 49%|████████████████████▏                    | 201/407 [15:07<15:34,  4.54s/it]\u001b[A\r\n",
      " 50%|████████████████████▎                    | 202/407 [15:11<15:29,  4.54s/it]\u001b[A\r\n",
      " 50%|████████████████████▍                    | 203/407 [15:16<15:25,  4.54s/it]\u001b[A\r\n",
      " 50%|████████████████████▌                    | 204/407 [15:20<15:21,  4.54s/it]\u001b[A\r\n",
      " 50%|████████████████████▋                    | 205/407 [15:25<15:16,  4.54s/it]\u001b[A\r\n",
      " 51%|████████████████████▊                    | 206/407 [15:29<15:11,  4.54s/it]\u001b[A\r\n",
      " 51%|████████████████████▊                    | 207/407 [15:34<15:07,  4.54s/it]\u001b[A\r\n",
      " 51%|████████████████████▉                    | 208/407 [15:38<15:03,  4.54s/it]\u001b[A\r\n",
      " 51%|█████████████████████                    | 209/407 [15:43<14:58,  4.54s/it]\u001b[A\r\n",
      " 52%|█████████████████████▏                   | 210/407 [15:47<14:53,  4.54s/it]\u001b[A\r\n",
      " 52%|█████████████████████▎                   | 211/407 [15:52<14:49,  4.54s/it]\u001b[A\r\n",
      " 52%|█████████████████████▎                   | 212/407 [15:56<14:44,  4.54s/it]\u001b[A\r\n",
      " 52%|█████████████████████▍                   | 213/407 [16:01<14:39,  4.54s/it]\u001b[A\r\n",
      " 53%|█████████████████████▌                   | 214/407 [16:06<14:35,  4.54s/it]\u001b[A\r\n",
      " 53%|█████████████████████▋                   | 215/407 [16:10<14:31,  4.54s/it]\u001b[A\r\n",
      " 53%|█████████████████████▊                   | 216/407 [16:15<14:26,  4.54s/it]\u001b[A\r\n",
      " 53%|█████████████████████▊                   | 217/407 [16:19<14:21,  4.54s/it]\u001b[A\r\n",
      " 54%|█████████████████████▉                   | 218/407 [16:24<14:17,  4.54s/it]\u001b[A\r\n",
      " 54%|██████████████████████                   | 219/407 [16:28<14:12,  4.54s/it]\u001b[A\r\n",
      " 54%|██████████████████████▏                  | 220/407 [16:33<14:08,  4.54s/it]\u001b[A\r\n",
      " 54%|██████████████████████▎                  | 221/407 [16:37<14:03,  4.54s/it]\u001b[A\r\n",
      " 55%|██████████████████████▎                  | 222/407 [16:42<13:59,  4.54s/it]\u001b[A\r\n",
      " 55%|██████████████████████▍                  | 223/407 [16:46<13:54,  4.53s/it]\u001b[A\r\n",
      " 55%|██████████████████████▌                  | 224/407 [16:51<13:49,  4.53s/it]\u001b[A\r\n",
      " 55%|██████████████████████▋                  | 225/407 [16:55<13:45,  4.54s/it]\u001b[A\r\n",
      " 56%|██████████████████████▊                  | 226/407 [17:00<13:40,  4.54s/it]\u001b[A\r\n",
      " 56%|██████████████████████▊                  | 227/407 [17:05<13:36,  4.54s/it]\u001b[A\r\n",
      " 56%|██████████████████████▉                  | 228/407 [17:09<13:31,  4.54s/it]\u001b[A\r\n",
      " 56%|███████████████████████                  | 229/407 [17:14<13:27,  4.54s/it]\u001b[A\r\n",
      " 57%|███████████████████████▏                 | 230/407 [17:18<13:22,  4.54s/it]\u001b[A\r\n",
      " 57%|███████████████████████▎                 | 231/407 [17:23<13:17,  4.53s/it]\u001b[A\r\n",
      " 57%|███████████████████████▎                 | 232/407 [17:27<13:13,  4.53s/it]\u001b[A\r\n",
      " 57%|███████████████████████▍                 | 233/407 [17:32<13:09,  4.53s/it]\u001b[A\r\n",
      " 57%|███████████████████████▌                 | 234/407 [17:36<13:04,  4.53s/it]\u001b[A\r\n",
      " 58%|███████████████████████▋                 | 235/407 [17:41<12:59,  4.53s/it]\u001b[A\r\n",
      " 58%|███████████████████████▊                 | 236/407 [17:45<12:55,  4.53s/it]\u001b[A\r\n",
      " 58%|███████████████████████▊                 | 237/407 [17:50<12:50,  4.53s/it]\u001b[A\r\n",
      " 58%|███████████████████████▉                 | 238/407 [17:54<12:45,  4.53s/it]\u001b[A\r\n",
      " 59%|████████████████████████                 | 239/407 [17:59<12:41,  4.53s/it]\u001b[A\r\n",
      " 59%|████████████████████████▏                | 240/407 [18:03<12:36,  4.53s/it]\u001b[A\r\n",
      " 59%|████████████████████████▎                | 241/407 [18:08<12:32,  4.53s/it]\u001b[A\r\n",
      " 59%|████████████████████████▍                | 242/407 [18:13<12:28,  4.53s/it]\u001b[A\r\n",
      " 60%|████████████████████████▍                | 243/407 [18:17<12:23,  4.53s/it]\u001b[A\r\n",
      " 60%|████████████████████████▌                | 244/407 [18:22<12:19,  4.53s/it]\u001b[A\r\n",
      " 60%|████████████████████████▋                | 245/407 [18:26<12:14,  4.53s/it]\u001b[A\r\n",
      " 60%|████████████████████████▊                | 246/407 [18:31<12:09,  4.53s/it]\u001b[A\r\n",
      " 61%|████████████████████████▉                | 247/407 [18:35<12:05,  4.53s/it]\u001b[A\r\n",
      " 61%|████████████████████████▉                | 248/407 [18:40<12:00,  4.53s/it]\u001b[A\r\n",
      " 61%|█████████████████████████                | 249/407 [18:44<11:56,  4.53s/it]\u001b[A\r\n",
      " 61%|█████████████████████████▏               | 250/407 [18:49<11:51,  4.53s/it]\u001b[A\r\n",
      " 62%|█████████████████████████▎               | 251/407 [18:53<11:47,  4.53s/it]\u001b[A\r\n",
      " 62%|█████████████████████████▍               | 252/407 [18:58<11:42,  4.54s/it]\u001b[A\r\n",
      " 62%|█████████████████████████▍               | 253/407 [19:02<11:38,  4.54s/it]\u001b[A\r\n",
      " 62%|█████████████████████████▌               | 254/407 [19:07<11:34,  4.54s/it]\u001b[A\r\n",
      " 63%|█████████████████████████▋               | 255/407 [19:11<11:29,  4.54s/it]\u001b[A\r\n",
      " 63%|█████████████████████████▊               | 256/407 [19:16<11:24,  4.54s/it]\u001b[A\r\n",
      " 63%|█████████████████████████▉               | 257/407 [19:21<11:20,  4.54s/it]\u001b[A\r\n",
      " 63%|█████████████████████████▉               | 258/407 [19:25<11:15,  4.53s/it]\u001b[A\r\n",
      " 64%|██████████████████████████               | 259/407 [19:30<11:11,  4.53s/it]\u001b[A\r\n",
      " 64%|██████████████████████████▏              | 260/407 [19:34<11:06,  4.53s/it]\u001b[A\r\n",
      " 64%|██████████████████████████▎              | 261/407 [19:39<11:01,  4.53s/it]\u001b[A\r\n",
      " 64%|██████████████████████████▍              | 262/407 [19:43<10:57,  4.53s/it]\u001b[A\r\n",
      " 65%|██████████████████████████▍              | 263/407 [19:48<10:52,  4.53s/it]\u001b[A\r\n",
      " 65%|██████████████████████████▌              | 264/407 [19:52<10:48,  4.53s/it]\u001b[A\r\n",
      " 65%|██████████████████████████▋              | 265/407 [19:57<10:43,  4.53s/it]\u001b[A\r\n",
      " 65%|██████████████████████████▊              | 266/407 [20:01<10:39,  4.53s/it]\u001b[A\r\n",
      " 66%|██████████████████████████▉              | 267/407 [20:06<10:34,  4.53s/it]\u001b[A\r\n",
      " 66%|██████████████████████████▉              | 268/407 [20:10<10:30,  4.54s/it]\u001b[A\r\n",
      " 66%|███████████████████████████              | 269/407 [20:15<10:25,  4.54s/it]\u001b[A\r\n",
      " 66%|███████████████████████████▏             | 270/407 [20:19<10:21,  4.54s/it]\u001b[A\r\n",
      " 67%|███████████████████████████▎             | 271/407 [20:24<10:16,  4.54s/it]\u001b[A\r\n",
      " 67%|███████████████████████████▍             | 272/407 [20:29<10:12,  4.54s/it]\u001b[A\r\n",
      " 67%|███████████████████████████▌             | 273/407 [20:33<10:07,  4.54s/it]\u001b[A\r\n",
      " 67%|███████████████████████████▌             | 274/407 [20:38<10:03,  4.54s/it]\u001b[A\r\n",
      " 68%|███████████████████████████▋             | 275/407 [20:42<09:58,  4.54s/it]\u001b[A\r\n",
      " 68%|███████████████████████████▊             | 276/407 [20:47<09:54,  4.54s/it]\u001b[A\r\n",
      " 68%|███████████████████████████▉             | 277/407 [20:51<09:49,  4.53s/it]\u001b[A\r\n",
      " 68%|████████████████████████████             | 278/407 [20:56<09:45,  4.54s/it]\u001b[A\r\n",
      " 69%|████████████████████████████             | 279/407 [21:00<09:40,  4.54s/it]\u001b[A\r\n",
      " 69%|████████████████████████████▏            | 280/407 [21:05<09:35,  4.53s/it]\u001b[A\r\n",
      " 69%|████████████████████████████▎            | 281/407 [21:09<09:31,  4.54s/it]\u001b[A\r\n",
      " 69%|████████████████████████████▍            | 282/407 [21:14<09:26,  4.54s/it]\u001b[A\r\n",
      " 70%|████████████████████████████▌            | 283/407 [21:18<09:22,  4.54s/it]\u001b[A\r\n",
      " 70%|████████████████████████████▌            | 284/407 [21:23<09:17,  4.54s/it]\u001b[A\r\n",
      " 70%|████████████████████████████▋            | 285/407 [21:28<09:13,  4.54s/it]\u001b[A\r\n",
      " 70%|████████████████████████████▊            | 286/407 [21:32<09:08,  4.54s/it]\u001b[A\r\n",
      " 71%|████████████████████████████▉            | 287/407 [21:37<09:04,  4.54s/it]\u001b[A\r\n",
      " 71%|█████████████████████████████            | 288/407 [21:41<08:59,  4.54s/it]\u001b[A\r\n",
      " 71%|█████████████████████████████            | 289/407 [21:46<08:55,  4.53s/it]\u001b[A\r\n",
      " 71%|█████████████████████████████▏           | 290/407 [21:50<08:50,  4.54s/it]\u001b[A\r\n",
      " 71%|█████████████████████████████▎           | 291/407 [21:55<08:46,  4.54s/it]\u001b[A\r\n",
      " 72%|█████████████████████████████▍           | 292/407 [21:59<08:41,  4.54s/it]\u001b[A\r\n",
      " 72%|█████████████████████████████▌           | 293/407 [22:04<08:36,  4.53s/it]\u001b[A\r\n",
      " 72%|█████████████████████████████▌           | 294/407 [22:08<08:32,  4.54s/it]\u001b[A\r\n",
      " 72%|█████████████████████████████▋           | 295/407 [22:13<08:28,  4.54s/it]\u001b[A\r\n",
      " 73%|█████████████████████████████▊           | 296/407 [22:17<08:23,  4.54s/it]\u001b[A\r\n",
      " 73%|█████████████████████████████▉           | 297/407 [22:22<08:18,  4.54s/it]\u001b[A\r\n",
      " 73%|██████████████████████████████           | 298/407 [22:26<08:14,  4.53s/it]\u001b[A\r\n",
      " 73%|██████████████████████████████           | 299/407 [22:31<08:09,  4.54s/it]\u001b[A\r\n",
      " 74%|██████████████████████████████▏          | 300/407 [22:36<08:05,  4.54s/it]\u001b[A\r\n",
      " 74%|██████████████████████████████▎          | 301/407 [22:40<08:00,  4.53s/it]\u001b[A\r\n",
      " 74%|██████████████████████████████▍          | 302/407 [22:45<07:56,  4.54s/it]\u001b[A\r\n",
      " 74%|██████████████████████████████▌          | 303/407 [22:49<07:51,  4.54s/it]\u001b[A\r\n",
      " 75%|██████████████████████████████▌          | 304/407 [22:54<07:47,  4.54s/it]\u001b[A\r\n",
      " 75%|██████████████████████████████▋          | 305/407 [22:58<07:43,  4.54s/it]\u001b[A\r\n",
      " 75%|██████████████████████████████▊          | 306/407 [23:03<07:38,  4.54s/it]\u001b[A\r\n",
      " 75%|██████████████████████████████▉          | 307/407 [23:07<07:33,  4.54s/it]\u001b[A\r\n",
      " 76%|███████████████████████████████          | 308/407 [23:12<07:29,  4.54s/it]\u001b[A\r\n",
      " 76%|███████████████████████████████▏         | 309/407 [23:16<07:24,  4.53s/it]\u001b[A\r\n",
      " 76%|███████████████████████████████▏         | 310/407 [23:21<07:19,  4.54s/it]\u001b[A\r\n",
      " 76%|███████████████████████████████▎         | 311/407 [23:25<07:15,  4.53s/it]\u001b[A\r\n",
      " 77%|███████████████████████████████▍         | 312/407 [23:30<07:10,  4.53s/it]\u001b[A\r\n",
      " 77%|███████████████████████████████▌         | 313/407 [23:35<07:06,  4.53s/it]\u001b[A\r\n",
      " 77%|███████████████████████████████▋         | 314/407 [23:39<07:02,  4.54s/it]\u001b[A\r\n",
      " 77%|███████████████████████████████▋         | 315/407 [23:44<06:57,  4.54s/it]\u001b[A\r\n",
      " 78%|███████████████████████████████▊         | 316/407 [23:48<06:52,  4.54s/it]\u001b[A\r\n",
      " 78%|███████████████████████████████▉         | 317/407 [23:53<06:48,  4.53s/it]\u001b[A\r\n",
      " 78%|████████████████████████████████         | 318/407 [23:57<06:43,  4.53s/it]\u001b[A\r\n",
      " 78%|████████████████████████████████▏        | 319/407 [24:02<06:39,  4.54s/it]\u001b[A\r\n",
      " 79%|████████████████████████████████▏        | 320/407 [24:06<06:34,  4.53s/it]\u001b[A\r\n",
      " 79%|████████████████████████████████▎        | 321/407 [24:11<06:29,  4.53s/it]\u001b[A\r\n",
      " 79%|████████████████████████████████▍        | 322/407 [24:15<06:25,  4.53s/it]\u001b[A\r\n",
      " 79%|████████████████████████████████▌        | 323/407 [24:20<06:21,  4.54s/it]\u001b[A\r\n",
      " 80%|████████████████████████████████▋        | 324/407 [24:24<06:16,  4.54s/it]\u001b[A\r\n",
      " 80%|████████████████████████████████▋        | 325/407 [24:29<06:12,  4.54s/it]\u001b[A\r\n",
      " 80%|████████████████████████████████▊        | 326/407 [24:33<06:07,  4.54s/it]\u001b[A\r\n",
      " 80%|████████████████████████████████▉        | 327/407 [24:38<06:03,  4.54s/it]\u001b[A\r\n",
      " 81%|█████████████████████████████████        | 328/407 [24:43<05:58,  4.54s/it]\u001b[A\r\n",
      " 81%|█████████████████████████████████▏       | 329/407 [24:47<05:53,  4.54s/it]\u001b[A\r\n",
      " 81%|█████████████████████████████████▏       | 330/407 [24:52<05:49,  4.54s/it]\u001b[A\r\n",
      " 81%|█████████████████████████████████▎       | 331/407 [24:56<05:44,  4.54s/it]\u001b[A\r\n",
      " 82%|█████████████████████████████████▍       | 332/407 [25:01<05:40,  4.54s/it]\u001b[A\r\n",
      " 82%|█████████████████████████████████▌       | 333/407 [25:05<05:35,  4.54s/it]\u001b[A\r\n",
      " 82%|█████████████████████████████████▋       | 334/407 [25:10<05:31,  4.54s/it]\u001b[A\r\n",
      " 82%|█████████████████████████████████▋       | 335/407 [25:14<05:26,  4.54s/it]\u001b[A\r\n",
      " 83%|█████████████████████████████████▊       | 336/407 [25:19<05:22,  4.54s/it]\u001b[A\r\n",
      " 83%|█████████████████████████████████▉       | 337/407 [25:23<05:17,  4.54s/it]\u001b[A\r\n",
      " 83%|██████████████████████████████████       | 338/407 [25:28<05:13,  4.54s/it]\u001b[A\r\n",
      " 83%|██████████████████████████████████▏      | 339/407 [25:32<05:08,  4.54s/it]\u001b[A\r\n",
      " 84%|██████████████████████████████████▎      | 340/407 [25:37<05:03,  4.54s/it]\u001b[A\r\n",
      " 84%|██████████████████████████████████▎      | 341/407 [25:42<04:59,  4.54s/it]\u001b[A\r\n",
      " 84%|██████████████████████████████████▍      | 342/407 [25:46<04:54,  4.54s/it]\u001b[A\r\n",
      " 84%|██████████████████████████████████▌      | 343/407 [25:51<04:50,  4.54s/it]\u001b[A\r\n",
      " 85%|██████████████████████████████████▋      | 344/407 [25:55<04:45,  4.54s/it]\u001b[A\r\n",
      " 85%|██████████████████████████████████▊      | 345/407 [26:00<04:41,  4.54s/it]\u001b[A\r\n",
      " 85%|██████████████████████████████████▊      | 346/407 [26:04<04:36,  4.54s/it]\u001b[A\r\n",
      " 85%|██████████████████████████████████▉      | 347/407 [26:09<04:32,  4.53s/it]\u001b[A\r\n",
      " 86%|███████████████████████████████████      | 348/407 [26:13<04:27,  4.53s/it]\u001b[A\r\n",
      " 86%|███████████████████████████████████▏     | 349/407 [26:18<04:22,  4.53s/it]\u001b[A\r\n",
      " 86%|███████████████████████████████████▎     | 350/407 [26:22<04:18,  4.53s/it]\u001b[A\r\n",
      " 86%|███████████████████████████████████▎     | 351/407 [26:27<04:13,  4.53s/it]\u001b[A\r\n",
      " 86%|███████████████████████████████████▍     | 352/407 [26:31<04:09,  4.53s/it]\u001b[A\r\n",
      " 87%|███████████████████████████████████▌     | 353/407 [26:36<04:04,  4.53s/it]\u001b[A\r\n",
      " 87%|███████████████████████████████████▋     | 354/407 [26:41<04:00,  4.53s/it]\u001b[A\r\n",
      " 87%|███████████████████████████████████▊     | 355/407 [26:45<03:55,  4.54s/it]\u001b[A\r\n",
      " 87%|███████████████████████████████████▊     | 356/407 [26:50<03:51,  4.54s/it]\u001b[A\r\n",
      " 88%|███████████████████████████████████▉     | 357/407 [26:54<03:46,  4.53s/it]\u001b[A\r\n",
      " 88%|████████████████████████████████████     | 358/407 [26:59<03:42,  4.54s/it]\u001b[A\r\n",
      " 88%|████████████████████████████████████▏    | 359/407 [27:03<03:37,  4.53s/it]\u001b[A\r\n",
      " 88%|████████████████████████████████████▎    | 360/407 [27:08<03:33,  4.53s/it]\u001b[A\r\n",
      " 89%|████████████████████████████████████▎    | 361/407 [27:12<03:28,  4.53s/it]\u001b[A\r\n",
      " 89%|████████████████████████████████████▍    | 362/407 [27:17<03:24,  4.53s/it]\u001b[A\r\n",
      " 89%|████████████████████████████████████▌    | 363/407 [27:21<03:19,  4.53s/it]\u001b[A\r\n",
      " 89%|████████████████████████████████████▋    | 364/407 [27:26<03:14,  4.53s/it]\u001b[A\r\n",
      " 90%|████████████████████████████████████▊    | 365/407 [27:30<03:10,  4.53s/it]\u001b[A\r\n",
      " 90%|████████████████████████████████████▊    | 366/407 [27:35<03:05,  4.53s/it]\u001b[A\r\n",
      " 90%|████████████████████████████████████▉    | 367/407 [27:39<03:01,  4.54s/it]\u001b[A\r\n",
      " 90%|█████████████████████████████████████    | 368/407 [27:44<02:56,  4.54s/it]\u001b[A\r\n",
      " 91%|█████████████████████████████████████▏   | 369/407 [27:49<02:52,  4.54s/it]\u001b[A\r\n",
      " 91%|█████████████████████████████████████▎   | 370/407 [27:53<02:47,  4.54s/it]\u001b[A\r\n",
      " 91%|█████████████████████████████████████▎   | 371/407 [27:58<02:43,  4.54s/it]\u001b[A\r\n",
      " 91%|█████████████████████████████████████▍   | 372/407 [28:02<02:38,  4.54s/it]\u001b[A\r\n",
      " 92%|█████████████████████████████████████▌   | 373/407 [28:07<02:34,  4.54s/it]\u001b[A\r\n",
      " 92%|█████████████████████████████████████▋   | 374/407 [28:11<02:29,  4.54s/it]\u001b[A\r\n",
      " 92%|█████████████████████████████████████▊   | 375/407 [28:16<02:25,  4.53s/it]\u001b[A\r\n",
      " 92%|█████████████████████████████████████▉   | 376/407 [28:20<02:20,  4.54s/it]\u001b[A\r\n",
      " 93%|█████████████████████████████████████▉   | 377/407 [28:25<02:16,  4.54s/it]\u001b[A\r\n",
      " 93%|██████████████████████████████████████   | 378/407 [28:29<02:11,  4.53s/it]\u001b[A\r\n",
      " 93%|██████████████████████████████████████▏  | 379/407 [28:34<02:06,  4.53s/it]\u001b[A\r\n",
      " 93%|██████████████████████████████████████▎  | 380/407 [28:38<02:02,  4.53s/it]\u001b[A\r\n",
      " 94%|██████████████████████████████████████▍  | 381/407 [28:43<01:57,  4.53s/it]\u001b[A\r\n",
      " 94%|██████████████████████████████████████▍  | 382/407 [28:47<01:53,  4.53s/it]\u001b[A\r\n",
      " 94%|██████████████████████████████████████▌  | 383/407 [28:52<01:48,  4.53s/it]\u001b[A\r\n",
      " 94%|██████████████████████████████████████▋  | 384/407 [28:57<01:44,  4.53s/it]\u001b[A\r\n",
      " 95%|██████████████████████████████████████▊  | 385/407 [29:01<01:39,  4.53s/it]\u001b[A\r\n",
      " 95%|██████████████████████████████████████▉  | 386/407 [29:06<01:35,  4.53s/it]\u001b[A\r\n",
      " 95%|██████████████████████████████████████▉  | 387/407 [29:10<01:30,  4.53s/it]\u001b[A\r\n",
      " 95%|███████████████████████████████████████  | 388/407 [29:15<01:26,  4.53s/it]\u001b[A\r\n",
      " 96%|███████████████████████████████████████▏ | 389/407 [29:19<01:21,  4.53s/it]\u001b[A\r\n",
      " 96%|███████████████████████████████████████▎ | 390/407 [29:24<01:17,  4.53s/it]\u001b[A\r\n",
      " 96%|███████████████████████████████████████▍ | 391/407 [29:28<01:12,  4.53s/it]\u001b[A\r\n",
      " 96%|███████████████████████████████████████▍ | 392/407 [29:33<01:08,  4.53s/it]\u001b[A\r\n",
      " 97%|███████████████████████████████████████▌ | 393/407 [29:37<01:03,  4.53s/it]\u001b[A\r\n",
      " 97%|███████████████████████████████████████▋ | 394/407 [29:42<00:58,  4.53s/it]\u001b[A\r\n",
      " 97%|███████████████████████████████████████▊ | 395/407 [29:46<00:54,  4.53s/it]\u001b[A\r\n",
      " 97%|███████████████████████████████████████▉ | 396/407 [29:51<00:49,  4.53s/it]\u001b[A\r\n",
      " 98%|███████████████████████████████████████▉ | 397/407 [29:55<00:45,  4.53s/it]\u001b[A\r\n",
      " 98%|████████████████████████████████████████ | 398/407 [30:00<00:40,  4.53s/it]\u001b[A\r\n",
      " 98%|████████████████████████████████████████▏| 399/407 [30:05<00:36,  4.53s/it]\u001b[A\r\n",
      " 98%|████████████████████████████████████████▎| 400/407 [30:09<00:31,  4.53s/it]\u001b[A\r\n",
      " 99%|████████████████████████████████████████▍| 401/407 [30:14<00:27,  4.54s/it]\u001b[A\r\n",
      " 99%|████████████████████████████████████████▍| 402/407 [30:18<00:22,  4.54s/it]\u001b[A\r\n",
      " 99%|████████████████████████████████████████▌| 403/407 [30:23<00:18,  4.54s/it]\u001b[A\r\n",
      " 99%|████████████████████████████████████████▋| 404/407 [30:27<00:13,  4.54s/it]\u001b[A\r\n",
      "100%|████████████████████████████████████████▊| 405/407 [30:32<00:09,  4.54s/it]\u001b[A\r\n",
      "100%|████████████████████████████████████████▉| 406/407 [30:36<00:04,  4.54s/it]\u001b[A\r\n",
      "\r\n",
      "\u001b[A{'eval_loss': 0.7884924411773682, 'eval_accuracy': {'accuracy': 0.6363636363636364}, 'eval_runtime': 1845.8904, 'eval_samples_per_second': 0.22, 'eval_steps_per_second': 0.22, 'epoch': 0.49}\r\n",
      " 49%|█████████████████▋                  | 100/203 [4:05:31<3:09:29, 110.39s/it]\r\n",
      "100%|█████████████████████████████████████████| 407/407 [30:41<00:00,  4.53s/it]\u001b[A\r\n",
      "                                                                                \u001b[ATraceback (most recent call last):\r\n",
      "  File \"/kaggle/working/train.py\", line 259, in <module>\r\n",
      "    trainer.train()\r\n",
      "  File \"/opt/conda/lib/python3.10/site-packages/transformers/trainer.py\", line 1555, in train\r\n",
      "    return inner_training_loop(\r\n",
      "  File \"/opt/conda/lib/python3.10/site-packages/transformers/trainer.py\", line 1922, in _inner_training_loop\r\n",
      "    self._maybe_log_save_evaluate(tr_loss, model, trial, epoch, ignore_keys_for_eval)\r\n",
      "  File \"/opt/conda/lib/python3.10/site-packages/transformers/trainer.py\", line 2282, in _maybe_log_save_evaluate\r\n",
      "    self._save_checkpoint(model, trial, metrics=metrics)\r\n",
      "  File \"/opt/conda/lib/python3.10/site-packages/transformers/trainer.py\", line 2413, in _save_checkpoint\r\n",
      "    or operator(metric_value, self.state.best_metric)\r\n",
      "TypeError: '>' not supported between instances of 'dict' and 'dict'\r\n",
      "\u001b[34m\u001b[1mwandb\u001b[0m: \r\n",
      "\u001b[34m\u001b[1mwandb\u001b[0m: Run history:\r\n",
      "\u001b[34m\u001b[1mwandb\u001b[0m:               eval/loss █▁\r\n",
      "\u001b[34m\u001b[1mwandb\u001b[0m:            eval/runtime █▁\r\n",
      "\u001b[34m\u001b[1mwandb\u001b[0m: eval/samples_per_second ▁▁\r\n",
      "\u001b[34m\u001b[1mwandb\u001b[0m:   eval/steps_per_second ▁▁\r\n",
      "\u001b[34m\u001b[1mwandb\u001b[0m:             train/epoch ▁▂▃▃▄▄▅▆▆▇██\r\n",
      "\u001b[34m\u001b[1mwandb\u001b[0m:       train/global_step ▁▂▃▃▄▄▅▆▆▇██\r\n",
      "\u001b[34m\u001b[1mwandb\u001b[0m:     train/learning_rate ▁▅███▇▇▆▆▅\r\n",
      "\u001b[34m\u001b[1mwandb\u001b[0m:              train/loss ██▅▁▁▁▁▁▁▁\r\n",
      "\u001b[34m\u001b[1mwandb\u001b[0m: \r\n",
      "\u001b[34m\u001b[1mwandb\u001b[0m: Run summary:\r\n",
      "\u001b[34m\u001b[1mwandb\u001b[0m:               eval/loss 0.78849\r\n",
      "\u001b[34m\u001b[1mwandb\u001b[0m:            eval/runtime 1845.8904\r\n",
      "\u001b[34m\u001b[1mwandb\u001b[0m: eval/samples_per_second 0.22\r\n",
      "\u001b[34m\u001b[1mwandb\u001b[0m:   eval/steps_per_second 0.22\r\n",
      "\u001b[34m\u001b[1mwandb\u001b[0m:             train/epoch 0.49\r\n",
      "\u001b[34m\u001b[1mwandb\u001b[0m:       train/global_step 100\r\n",
      "\u001b[34m\u001b[1mwandb\u001b[0m:     train/learning_rate 1e-05\r\n",
      "\u001b[34m\u001b[1mwandb\u001b[0m:              train/loss 0.7376\r\n",
      "\u001b[34m\u001b[1mwandb\u001b[0m: \r\n",
      "\u001b[34m\u001b[1mwandb\u001b[0m: 🚀 View run \u001b[33mbloomz-2023-11-22-05-47\u001b[0m at: \u001b[34m\u001b[4mhttps://wandb.ai/quangphamm1902/huggingface/runs/7ewrxor3\u001b[0m\r\n",
      "\u001b[34m\u001b[1mwandb\u001b[0m: Synced 6 W&B file(s), 0 media file(s), 0 artifact file(s) and 0 other file(s)\r\n",
      "\u001b[34m\u001b[1mwandb\u001b[0m: Find logs at: \u001b[35m\u001b[1m./wandb/run-20231122_054724-7ewrxor3/logs\u001b[0m\r\n"
     ]
    }
   ],
   "source": [
    "!python train.py"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "0196ef9a",
   "metadata": {
    "papermill": {
     "duration": 0.288507,
     "end_time": "2023-11-22T09:53:39.521367",
     "exception": false,
     "start_time": "2023-11-22T09:53:39.232860",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kaggle": {
   "accelerator": "gpu",
   "dataSources": [
    {
     "datasetId": 4019609,
     "sourceId": 7012200,
     "sourceType": "datasetVersion"
    }
   ],
   "dockerImageVersionId": 30588,
   "isGpuEnabled": true,
   "isInternetEnabled": true,
   "language": "python",
   "sourceType": "notebook"
  },
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.10.12"
  },
  "papermill": {
   "default_parameters": {},
   "duration": 15478.719489,
   "end_time": "2023-11-22T09:53:40.160724",
   "environment_variables": {},
   "exception": null,
   "input_path": "__notebook__.ipynb",
   "output_path": "__notebook__.ipynb",
   "parameters": {},
   "start_time": "2023-11-22T05:35:41.441235",
   "version": "2.4.0"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}