sample_size_bert_classification_rob.ipynb

{"cells":[{"cell_type":"markdown","metadata":{"id":"KRDJE4FLLCcO"},"source":["# Config"]},{"cell_type":"code","execution_count":1,"metadata":{"executionInfo":{"elapsed":8,"status":"ok","timestamp":1712601221963,"user":{"displayName":"Diana Shamsutdinova","userId":"09141664087319544367"},"user_tz":-60},"id":"yrlfKY3VLBiw"},"outputs":[],"source":["ff_rob_using = True\n","max_sentence_size = 512"]},{"cell_type":"markdown","metadata":{"id":"qJPJNXTJuVsS"},"source":["# 1. Initial imports"]},{"cell_type":"code","execution_count":null,"metadata":{"colab":{"background_save":true,"base_uri":"https://localhost:8080/"},"id":"HOv3NesKJhB4"},"outputs":[{"name":"stdout","output_type":"stream","text":["Collecting transformers==4.5.0\n","  Downloading transformers-4.5.0-py3-none-any.whl (2.1 MB)\n","\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m2.1/2.1 MB\u001b[0m \u001b[31m8.1 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n","\u001b[?25hRequirement already satisfied: filelock in /usr/local/lib/python3.10/dist-packages (from transformers==4.5.0) (3.13.3)\n","Requirement already satisfied: numpy\u003e=1.17 in /usr/local/lib/python3.10/dist-packages (from transformers==4.5.0) (1.25.2)\n","Requirement already satisfied: packaging in /usr/local/lib/python3.10/dist-packages (from transformers==4.5.0) (24.0)\n","Requirement already satisfied: regex!=2019.12.17 in /usr/local/lib/python3.10/dist-packages (from transformers==4.5.0) (2023.12.25)\n","Requirement already satisfied: requests in /usr/local/lib/python3.10/dist-packages (from transformers==4.5.0) (2.31.0)\n","Collecting sacremoses (from transformers==4.5.0)\n","  Downloading sacremoses-0.1.1-py3-none-any.whl (897 kB)\n","\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m897.5/897.5 kB\u001b[0m \u001b[31m42.0 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n","\u001b[?25hCollecting tokenizers\u003c0.11,\u003e=0.10.1 (from transformers==4.5.0)\n","  Downloading tokenizers-0.10.3.tar.gz (212 kB)\n","\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m212.7/212.7 kB\u001b[0m \u001b[31m15.7 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n","\u001b[?25h  Installing build dependencies ... \u001b[?25l\u001b[?25hdone\n","  Getting requirements to build wheel ... \u001b[?25l\u001b[?25hdone\n","  Preparing metadata (pyproject.toml) ... \u001b[?25l\u001b[?25hdone\n","Requirement already satisfied: tqdm\u003e=4.27 in /usr/local/lib/python3.10/dist-packages (from transformers==4.5.0) (4.66.2)\n","Requirement already satisfied: charset-normalizer\u003c4,\u003e=2 in /usr/local/lib/python3.10/dist-packages (from requests-\u003etransformers==4.5.0) (3.3.2)\n","Requirement already satisfied: idna\u003c4,\u003e=2.5 in /usr/local/lib/python3.10/dist-packages (from requests-\u003etransformers==4.5.0) (3.6)\n","Requirement already satisfied: urllib3\u003c3,\u003e=1.21.1 in /usr/local/lib/python3.10/dist-packages (from requests-\u003etransformers==4.5.0) (2.0.7)\n","Requirement already satisfied: certifi\u003e=2017.4.17 in /usr/local/lib/python3.10/dist-packages (from requests-\u003etransformers==4.5.0) (2024.2.2)\n","Requirement already satisfied: click in /usr/local/lib/python3.10/dist-packages (from sacremoses-\u003etransformers==4.5.0) (8.1.7)\n","Requirement already satisfied: joblib in /usr/local/lib/python3.10/dist-packages (from sacremoses-\u003etransformers==4.5.0) (1.3.2)\n","Building wheels for collected packages: tokenizers\n","  \u001b[1;31merror\u001b[0m: \u001b[1msubprocess-exited-with-error\u001b[0m\n","  \n","  \u001b[31m×\u001b[0m \u001b[32mBuilding wheel for tokenizers \u001b[0m\u001b[1;32m(\u001b[0m\u001b[32mpyproject.toml\u001b[0m\u001b[1;32m)\u001b[0m did not run successfully.\n","  \u001b[31m│\u001b[0m exit code: \u001b[1;36m1\u001b[0m\n","  \u001b[31m╰─\u003e\u001b[0m See above for output.\n","  \n","  \u001b[1;35mnote\u001b[0m: This error originates from a subprocess, and is likely not a problem with pip.\n","  Building wheel for tokenizers (pyproject.toml) ... \u001b[?25l\u001b[?25herror\n","\u001b[31m  ERROR: Failed building wheel for tokenizers\u001b[0m\u001b[31m\n","\u001b[0mFailed to build tokenizers\n","\u001b[31mERROR: Could not build wheels for tokenizers, which is required to install pyproject.toml-based projects\u001b[0m\u001b[31m\n","\u001b[0mCollecting torchmetrics\n","  Downloading torchmetrics-1.3.2-py3-none-any.whl (841 kB)\n","\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m841.5/841.5 kB\u001b[0m \u001b[31m4.4 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n","\u001b[?25hRequirement already satisfied: numpy\u003e1.20.0 in /usr/local/lib/python3.10/dist-packages (from torchmetrics) (1.25.2)\n","Requirement already satisfied: packaging\u003e17.1 in /usr/local/lib/python3.10/dist-packages (from torchmetrics) (24.0)\n","Requirement already satisfied: torch\u003e=1.10.0 in /usr/local/lib/python3.10/dist-packages (from torchmetrics) (2.2.1+cu121)\n","Collecting lightning-utilities\u003e=0.8.0 (from torchmetrics)\n","  Downloading lightning_utilities-0.11.2-py3-none-any.whl (26 kB)\n","Requirement already satisfied: setuptools in /usr/local/lib/python3.10/dist-packages (from lightning-utilities\u003e=0.8.0-\u003etorchmetrics) (67.7.2)\n","Requirement already satisfied: typing-extensions in /usr/local/lib/python3.10/dist-packages (from lightning-utilities\u003e=0.8.0-\u003etorchmetrics) (4.10.0)\n","Requirement already satisfied: filelock in /usr/local/lib/python3.10/dist-packages (from torch\u003e=1.10.0-\u003etorchmetrics) (3.13.3)\n","Requirement already satisfied: sympy in /usr/local/lib/python3.10/dist-packages (from torch\u003e=1.10.0-\u003etorchmetrics) (1.12)\n","Requirement already satisfied: networkx in /usr/local/lib/python3.10/dist-packages (from torch\u003e=1.10.0-\u003etorchmetrics) (3.2.1)\n","Requirement already satisfied: jinja2 in /usr/local/lib/python3.10/dist-packages (from torch\u003e=1.10.0-\u003etorchmetrics) (3.1.3)\n","Requirement already satisfied: fsspec in /usr/local/lib/python3.10/dist-packages (from torch\u003e=1.10.0-\u003etorchmetrics) (2023.6.0)\n","Collecting nvidia-cuda-nvrtc-cu12==12.1.105 (from torch\u003e=1.10.0-\u003etorchmetrics)\n","  Downloading nvidia_cuda_nvrtc_cu12-12.1.105-py3-none-manylinux1_x86_64.whl (23.7 MB)\n","\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m23.7/23.7 MB\u001b[0m \u001b[31m33.6 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n","\u001b[?25hCollecting nvidia-cuda-runtime-cu12==12.1.105 (from torch\u003e=1.10.0-\u003etorchmetrics)\n","  Downloading nvidia_cuda_runtime_cu12-12.1.105-py3-none-manylinux1_x86_64.whl (823 kB)\n","\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m823.6/823.6 kB\u001b[0m \u001b[31m36.8 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n","\u001b[?25hCollecting nvidia-cuda-cupti-cu12==12.1.105 (from torch\u003e=1.10.0-\u003etorchmetrics)\n","  Downloading nvidia_cuda_cupti_cu12-12.1.105-py3-none-manylinux1_x86_64.whl (14.1 MB)\n","\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m14.1/14.1 MB\u001b[0m \u001b[31m45.4 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n","\u001b[?25hCollecting nvidia-cudnn-cu12==8.9.2.26 (from torch\u003e=1.10.0-\u003etorchmetrics)\n","  Downloading nvidia_cudnn_cu12-8.9.2.26-py3-none-manylinux1_x86_64.whl (731.7 MB)\n","\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m731.7/731.7 MB\u001b[0m \u001b[31m788.5 kB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n","\u001b[?25hCollecting nvidia-cublas-cu12==12.1.3.1 (from torch\u003e=1.10.0-\u003etorchmetrics)\n","  Downloading nvidia_cublas_cu12-12.1.3.1-py3-none-manylinux1_x86_64.whl (410.6 MB)\n","\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m410.6/410.6 MB\u001b[0m \u001b[31m1.5 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n","\u001b[?25hCollecting nvidia-cufft-cu12==11.0.2.54 (from torch\u003e=1.10.0-\u003etorchmetrics)\n","  Downloading nvidia_cufft_cu12-11.0.2.54-py3-none-manylinux1_x86_64.whl (121.6 MB)\n","\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m121.6/121.6 MB\u001b[0m \u001b[31m1.0 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n","\u001b[?25hCollecting nvidia-curand-cu12==10.3.2.106 (from torch\u003e=1.10.0-\u003etorchmetrics)\n","  Downloading nvidia_curand_cu12-10.3.2.106-py3-none-manylinux1_x86_64.whl (56.5 MB)\n","\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m56.5/56.5 MB\u001b[0m \u001b[31m1.3 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n","\u001b[?25hCollecting nvidia-cusolver-cu12==11.4.5.107 (from torch\u003e=1.10.0-\u003etorchmetrics)\n","  Downloading nvidia_cusolver_cu12-11.4.5.107-py3-none-manylinux1_x86_64.whl (124.2 MB)\n","\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m124.2/124.2 MB\u001b[0m \u001b[31m1.9 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n","\u001b[?25hCollecting nvidia-cusparse-cu12==12.1.0.106 (from torch\u003e=1.10.0-\u003etorchmetrics)\n","  Downloading nvidia_cusparse_cu12-12.1.0.106-py3-none-manylinux1_x86_64.whl (196.0 MB)\n","\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m196.0/196.0 MB\u001b[0m \u001b[31m1.9 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n","\u001b[?25h\u001b[33mWARNING: Retrying (Retry(total=4, connect=None, read=None, redirect=None, status=None)) after connection broken by 'ProtocolError('Connection aborted.', RemoteDisconnected('Remote end closed connection without response'))': /simple/nvidia-nccl-cu12/\u001b[0m\u001b[33m\n","\u001b[0mCollecting nvidia-nccl-cu12==2.19.3 (from torch\u003e=1.10.0-\u003etorchmetrics)\n","  Downloading nvidia_nccl_cu12-2.19.3-py3-none-manylinux1_x86_64.whl (166.0 MB)\n","\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m166.0/166.0 MB\u001b[0m \u001b[31m1.6 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n","\u001b[?25hCollecting nvidia-nvtx-cu12==12.1.105 (from torch\u003e=1.10.0-\u003etorchmetrics)\n","  Downloading nvidia_nvtx_cu12-12.1.105-py3-none-manylinux1_x86_64.whl (99 kB)\n","\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m99.1/99.1 kB\u001b[0m \u001b[31m2.9 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n","\u001b[?25hRequirement already satisfied: triton==2.2.0 in /usr/local/lib/python3.10/dist-packages (from torch\u003e=1.10.0-\u003etorchmetrics) (2.2.0)\n","Collecting nvidia-nvjitlink-cu12 (from nvidia-cusolver-cu12==11.4.5.107-\u003etorch\u003e=1.10.0-\u003etorchmetrics)\n","  Downloading nvidia_nvjitlink_cu12-12.4.127-py3-none-manylinux2014_x86_64.whl (21.1 MB)\n","\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m21.1/21.1 MB\u001b[0m \u001b[31m3.7 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n","\u001b[?25hRequirement already satisfied: MarkupSafe\u003e=2.0 in /usr/local/lib/python3.10/dist-packages (from jinja2-\u003etorch\u003e=1.10.0-\u003etorchmetrics) (2.1.5)\n","Requirement already satisfied: mpmath\u003e=0.19 in /usr/local/lib/python3.10/dist-packages (from sympy-\u003etorch\u003e=1.10.0-\u003etorchmetrics) (1.3.0)\n","Installing collected packages: nvidia-nvtx-cu12, nvidia-nvjitlink-cu12, nvidia-nccl-cu12, nvidia-curand-cu12, nvidia-cufft-cu12, nvidia-cuda-runtime-cu12, nvidia-cuda-nvrtc-cu12, nvidia-cuda-cupti-cu12, nvidia-cublas-cu12, lightning-utilities, nvidia-cusparse-cu12, nvidia-cudnn-cu12, nvidia-cusolver-cu12, torchmetrics\n","Successfully installed lightning-utilities-0.11.2 nvidia-cublas-cu12-12.1.3.1 nvidia-cuda-cupti-cu12-12.1.105 nvidia-cuda-nvrtc-cu12-12.1.105 nvidia-cuda-runtime-cu12-12.1.105 nvidia-cudnn-cu12-8.9.2.26 nvidia-cufft-cu12-11.0.2.54 nvidia-curand-cu12-10.3.2.106 nvidia-cusolver-cu12-11.4.5.107 nvidia-cusparse-cu12-12.1.0.106 nvidia-nccl-cu12-2.19.3 nvidia-nvjitlink-cu12-12.4.127 nvidia-nvtx-cu12-12.1.105 torchmetrics-1.3.2\n"]}],"source":["!pip install transformers==4.5.0\n","!pip install torchmetrics"]},{"cell_type":"code","execution_count":null,"metadata":{"id":"Ztlz1_Hl4YM5"},"outputs":[],"source":["# to import files from googledrive\n","from google.colab import drive\n","if not ff_rob_using:\n","  drive.mount('/content/drive', force_remount=True)\n","else:\n","  drive.mount('/content/drive/', force_remount=True)\n","  sample_size_dir = 'drive/MyDrive/Sample size for NLP'\n"]},{"cell_type":"code","execution_count":null,"metadata":{"id":"xfZffjko4dHn"},"outputs":[],"source":["import os\n","\n","import pandas as pd\n","import numpy as np\n","\n","import seaborn as sns\n","import matplotlib.pyplot as plt\n","\n","#for text pre-processing\n","import re, string\n","import nltk\n","from nltk.tokenize import word_tokenize\n","from nltk.corpus import stopwords\n","from nltk.tokenize import word_tokenize\n","from nltk.stem import SnowballStemmer\n","from nltk.corpus import wordnet\n","from nltk.stem import WordNetLemmatizer\n","\n","nltk.download('punkt')\n","nltk.download('averaged_perceptron_tagger')\n","nltk.download('wordnet')\n","nltk.download('stopwords')\n","nltk.download('omw-1.4')\n","\n","#for model-building\n","from sklearn.model_selection import train_test_split\n","from sklearn.linear_model import LogisticRegression\n","from sklearn.linear_model import SGDClassifier\n","from sklearn.naive_bayes import MultinomialNB\n","from sklearn.svm import LinearSVC #JC\n","from sklearn.ensemble import RandomForestClassifier #JC\n","from sklearn.neighbors import KNeighborsClassifier #JC\n","from sklearn.naive_bayes import GaussianNB #JC\n","from sklearn.tree import DecisionTreeClassifier #JC\n","from sklearn.svm import SVC #JC\n","from sklearn.metrics import classification_report, f1_score, accuracy_score, confusion_matrix, precision_score, recall_score, precision_recall_fscore_support\n","from sklearn.metrics import roc_curve, auc, roc_auc_score\n","\n","# bag of words\n","from sklearn.feature_extraction.text import TfidfVectorizer\n","from sklearn.feature_extraction.text import CountVectorizer\n","from sklearn.feature_extraction.text import TfidfTransformer #JC\n","\n","#for word embedding\n","import gensim\n","from gensim.models import Word2Vec #Word2Vec is mostly used for huge datasets\n","import time\n","import logging"]},{"cell_type":"code","execution_count":null,"metadata":{"id":"Ryu8A6Y_Rid-"},"outputs":[],"source":["def time_code():\n","    def decorate(func):\n","        def call(*args, **kwargs):\n","            print('Entered function %s' % func.__name__)\n","            start = time.time()\n","            result = func(*args, **kwargs)\n","            end = time.time()\n","            func_time = end - start\n","            print('Exiting function %s - took %.2f seconds' % (func.__name__, func_time))\n","            return result\n","        return call\n","    return decorate"]},{"cell_type":"markdown","metadata":{"id":"SSGW-nh5-GfE"},"source":["# 2. Data for Classification"]},{"cell_type":"markdown","metadata":{"id":"-1qBDOWMRUGm"},"source":["Load the data for classification"]},{"cell_type":"code","execution_count":null,"metadata":{"id":"OAWygjHuhqzT"},"outputs":[],"source":["@time_code()\n","def get_data_for_classifier():\n","  if not ff_rob_using:\n","    path = '/content/drive/My Drive/Sample size for NLP/Sample_Size_Saniya'\n","  else:\n","    path = '%s/Sample_Size_Saniya' % sample_size_dir\n","  return pd.read_csv(path+'/12k_data_for_classifier.csv')#\n","\n"]},{"cell_type":"markdown","metadata":{"id":"ZkuwRxmcjQhr"},"source":["# 3. Split data into different sample sizes and proportions\n","\n"]},{"cell_type":"code","execution_count":null,"metadata":{"id":"V75JRblYH7w0"},"outputs":[],"source":["@time_code()\n","def get_train_test_split(full_data_for_classfier, num_train_samples, portion_of_positive, num_test_samples):\n","\n","  num_samples = num_train_samples + num_test_samples\n","  assert(0 \u003c num_samples \u003c= 12000)\n","  assert(0 \u003c portion_of_positive \u003c 1)\n","\n","  portion_of_negative = 1 - portion_of_positive\n","  ss0 = int(num_samples*portion_of_negative)\n","  ss1 = int(num_samples*portion_of_positive)\n","\n","  s0 = full_data_for_classfier.label[full_data_for_classfier.label.eq(0)].sample(ss0).index\n","  s1 = full_data_for_classfier.label[full_data_for_classfier.label.eq(1)].sample(ss1).index\n","\n","  data_for_classifier = full_data_for_classfier.loc[s0.union(s1)]\n","\n","  test_size = num_test_samples / num_samples\n","  training_data, testing_data = train_test_split(data_for_classifier, test_size=test_size, random_state=1)\n","  return training_data, testing_data\n","\n"]},{"cell_type":"code","execution_count":null,"metadata":{"id":"3QlaH790w7HZ"},"outputs":[],"source":["@time_code()\n","def get_balanced_cross_fold_train_test_split(\n","    full_data_for_classifier: pd.DataFrame,\n","    cross_folds: int,\n","    ):\n","  assert(1 \u003c cross_folds)\n","\n","  positive_samples = int(len(full_data_for_classifier) / 2)\n","  negative_samples = int(len(full_data_for_classifier) / 2)\n","\n","  all_positive = full_data_for_classifier.loc[full_data_for_classifier.label[full_data_for_classifier.label.eq(0)].sample(positive_samples).index]\n","  all_negative = full_data_for_classifier.loc[full_data_for_classifier.label[full_data_for_classifier.label.eq(1)].sample(negative_samples).index]\n","\n","  positive_buckets = np.array_split(all_positive, cross_folds)\n","  negative_buckets = np.array_split(all_negative, cross_folds)\n","\n","  for test_bucket_index in range(cross_folds):\n","\n","    test_positive = positive_buckets[test_bucket_index]\n","    test_negative = negative_buckets[test_bucket_index]\n","\n","    train_positive = pd.concat([pb for pd_index,pb in enumerate(positive_buckets) if pd_index is not test_bucket_index])\n","    train_negative = pd.concat([pb for pd_index,pb in enumerate(negative_buckets) if pd_index is not test_bucket_index])\n","\n","    train = pd.concat([train_positive, train_negative])\n","    test = pd.concat([test_positive, test_negative])\n","\n","    yield train, test\n","\n","\n","###############################################################################\n","###################### REST OF THIS CELL IS JUST TESTING ######################\n","###############################################################################\n","# Do some testing on the cross folds and make sure that we aren't putting\n","# the same samples into different train test datasets\n","cross_fold_indexes = {}\n","for cross_fold_index, (train, test) in enumerate(get_balanced_cross_fold_train_test_split(get_data_for_classifier(), 5)):\n","  cross_fold_indexes[cross_fold_index] = {\n","      'train': list(train.index),\n","      'test': list(test.index),\n","  }\n","\n","\n","for cross_fold_index in range(5):\n","  # Make sure the train and test set when unions is equal to the total number of samples\n","  length = len(set(cross_fold_indexes[cross_fold_index]['train']).union(set(cross_fold_indexes[cross_fold_index]['test'])))\n","  assert(length == 12000)\n","\n","  if cross_fold_index == 0:\n","    common_train = set(cross_fold_indexes[cross_fold_index]['train'])\n","    all_test = set(cross_fold_indexes[cross_fold_index]['test'])\n","  else:\n","    common_train = common_train - set(cross_fold_indexes[cross_fold_index]['train'])\n","    all_test = all_test.union(cross_fold_indexes[cross_fold_index]['test'])\n","\n","assert(len(common_train) == 0)\n","assert(len(all_test) == 12000)"]},{"cell_type":"markdown","metadata":{"id":"klq3iD_y-K43"},"source":["# 4. BERT Classification"]},{"cell_type":"markdown","metadata":{"id":"FSW39FULGy2P"},"source":["# BERT imports"]},{"cell_type":"code","execution_count":null,"metadata":{"id":"MnivD2i-wlkd"},"outputs":[],"source":["import torch\n","\n","# If there's a GPU available...\n","if torch.cuda.is_available():\n","\n","    # Tell PyTorch to use the GPU.\n","    device = torch.device(\"cuda\")\n","\n","    print('There are %d GPU(s) available.' % torch.cuda.device_count())\n","\n","    print('We will use the GPU:', torch.cuda.get_device_name(0))\n","\n","# If not...\n","else:\n","    print('No GPU available, using the CPU instead.')\n","    device = torch.device(\"cpu\")"]},{"cell_type":"code","execution_count":null,"metadata":{"id":"8DpmjFihGplE"},"outputs":[],"source":["from collections import Counter\n","import numpy as np\n","import pandas as pd\n","import torch\n","from torch.utils.data import TensorDataset, DataLoader, RandomSampler, SequentialSampler\n","from sklearn.metrics import accuracy_score, f1_score, precision_score, recall_score, classification_report\n","from sklearn.model_selection import StratifiedKFold\n","from torch.nn.utils.rnn import pad_sequence\n","from torch.utils.data import TensorDataset\n","from tqdm import trange\n","from transformers import BertForSequenceClassification, BertTokenizerFast, BertTokenizer, AdamW, AutoModel, AutoModelForSequenceClassification, AutoTokenizer"]},{"cell_type":"code","execution_count":null,"metadata":{"id":"yoVA7aYpXx49"},"outputs":[],"source":["from transformers import BertTokenizer, AutoTokenizer\n","@time_code()\n","def get_tokenizer(pretrained_string: str, padding_side :str, truncate_side :str, do_lower_case: bool):\n","# Load the BERT tokenizer.\n","  print('Loading BERT tokenizer...')\n","  tokenizer = BertTokenizer.from_pretrained(pretrained_string,\n","                                            padding_side=padding_side,\n","                                            do_lower_case=do_lower_case,\n","                                            truncation_side=truncate_side) #bert-base-uncased\n","  return tokenizer"]},{"cell_type":"markdown","metadata":{"id":"rK8VKDBPXoUu"},"source":["# Data Prep"]},{"cell_type":"code","execution_count":null,"metadata":{"id":"qc2am7q5x0QC"},"outputs":[],"source":["@time_code()\n","def get_loader(data):\n","  # Tokenize all of the sentences and map the tokens to thier word IDs.\n","  sentences = data.text.values\n","  labels = data.label.values\n","\n","  input_ids = []\n","  attention_masks = []\n","\n","  # For every sentence...\n","  for sent in sentences:\n","      # `encode_plus` will:\n","      #   (1) Tokenize the sentence.\n","      #   (2) Prepend the `[CLS]` token to the start.\n","      #   (3) Append the `[SEP]` token to the end.\n","      #   (4) Map tokens to their IDs.\n","      #   (5) Pad or truncate the sentence to `max_length`\n","      #   (6) Create attention masks for [PAD] tokens.\n","      encoded_dict = tokenizer.encode_plus(\n","                          sent,                      # Sentence to encode.\n","                          add_special_tokens = True, # Add '[CLS]' and '[SEP]'\n","                          max_length = max_sentence_size,           # Pad \u0026 truncate all sentences.\n","                          pad_to_max_length = True,\n","                          return_attention_mask = True,   # Construct attn. masks.\n","                          return_tensors = 'pt',     # Return pytorch tensors.\n","                    )\n","\n","      # Add the encoded sentence to the list.\n","      input_ids.append(encoded_dict['input_ids'])\n","\n","      # And its attention mask (simply differentiates padding from non-padding).\n","      attention_masks.append(encoded_dict['attention_mask'])\n","\n","  # Convert the lists into tensors.\n","  input_ids = torch.cat(input_ids, dim=0)\n","  attention_masks = torch.cat(attention_masks, dim=0)\n","  labels = torch.tensor(labels)\n","\n","  # Print sentence 0, now as a list of IDs.\n","  print('Original: ', sentences[0])\n","  print('Token IDs:', input_ids[0])\n","\n","  from torch.utils.data import TensorDataset, random_split\n","\n","  # Combine the training inputs into a TensorDataset.\n","  dataset = TensorDataset(input_ids, attention_masks, labels)\n","\n","  # Create a 90-10 train-validation split.\n","\n","  # Calculate the number of samples to include in each set.\n","  train_size = int(0.9 * len(dataset))\n","  val_size = len(dataset) - train_size\n","\n","  # Divide the dataset by randomly selecting samples.\n","  train_dataset, val_dataset = random_split(dataset, [train_size, val_size])\n","\n","  print('{:\u003e5,} training samples'.format(train_size))\n","  print('{:\u003e5,} validation samples'.format(val_size))\n","  return train_dataset, val_dataset\n","\n"]},{"cell_type":"code","execution_count":null,"metadata":{"id":"37mNhXqbjprJ"},"outputs":[],"source":["import pickle, hashlib, json\n","\n","DISK_CACHE_LOCATION = os.path.join(sample_size_dir, 'embedding_cache.pickle')\n","\n","if not os.path.exists(DISK_CACHE_LOCATION):\n","  with open(DISK_CACHE_LOCATION, 'wb') as f:\n","    pickle.dump({}, f, protocol=pickle.HIGHEST_PROTOCOL)\n","\n","with open(DISK_CACHE_LOCATION, 'rb') as f:\n","  CACHE = pickle.load(f)\n","\n","@time_code()\n","def get_loader_cache(tokenizer, tokenizer_str, data):\n","  # Tokenize all of the sentences and map the tokens to thier word IDs.\n","  sentences = data.text.values\n","  labels = data.label.values\n","\n","  input_ids = []\n","  attention_masks = []\n","\n","  encoder_params = {\n","    'add_special_tokens' : True, # Add '[CLS]' and '[SEP]'\n","    'max_length' : max_sentence_size,           # Pad \u0026 truncate all sentences.\n","    'pad_to_max_length' : True,\n","    'return_attention_mask' : True,   # Construct attn. masks.\n","    'return_tensors' :'pt',     # Return pytorch tensors.\n","  }\n","\n","  encoder_params_json_str = json.dumps(encoder_params, sort_keys=True) #TODO generate hash\n","\n","  m = hashlib.sha256()\n","  m.update(tokenizer_str.encode('utf-8'))\n","  m.update(encoder_params_json_str.encode('utf-8'))\n","  encoder_params_str = m.hexdigest()\n","\n","  if encoder_params_str not in CACHE:\n","    CACHE[encoder_params_str] = {'encoder_params': encoder_params, 'sentences': {}}\n","\n","  # For every sentence...\n","  for sent in sentences:\n","      # `encode_plus` will:\n","      #   (1) Tokenize the sentence.\n","      #   (2) Prepend the `[CLS]` token to the start.\n","      #   (3) Append the `[SEP]` token to the end.\n","      #   (4) Map tokens to their IDs.\n","      #   (5) Pad or truncate the sentence to `max_length`\n","      #   (6) Create attention masks for [PAD] tokens.\n","      if sent not in CACHE:\n","        CACHE[encoder_params_str]['sentences'][sent] = tokenizer.encode_plus(\n","                          sent,                      # Sentence to encode.\n","                  **encoder_params\n","                    )\n","      encoded_dict = CACHE[encoder_params_str]['sentences'][sent]\n","\n","      # Add the encoded sentence to the list.\n","      input_ids.append(encoded_dict['input_ids'])\n","\n","      # And its attention mask (simply differentiates padding from non-padding).\n","      attention_masks.append(encoded_dict['attention_mask'])\n","\n","  # Convert the lists into tensors.\n","  input_ids = torch.cat(input_ids, dim=0)\n","  attention_masks = torch.cat(attention_masks, dim=0)\n","  labels = torch.tensor(labels)\n","\n","  # Print sentence 0, now as a list of IDs.\n","  print('Original: ', sentences[0])\n","  print('Token IDs:', input_ids[0])\n","\n","  from torch.utils.data import TensorDataset, random_split\n","\n","  # Combine the training inputs into a TensorDataset.\n","  dataset = TensorDataset(input_ids, attention_masks, labels)\n","\n","  # Create a 90-10 train-validation split.\n","\n","  # Calculate the number of samples to include in each set.\n","  train_size = int(0.9 * len(dataset))\n","  val_size = len(dataset) - train_size\n","\n","  # Divide the dataset by randomly selecting samples.\n","  train_dataset, val_dataset = random_split(dataset, [train_size, val_size])\n","\n","  print('{:\u003e5,} training samples'.format(train_size))\n","  print('{:\u003e5,} validation samples'.format(val_size))\n","\n","\n","  with open(DISK_CACHE_LOCATION, 'wb') as f:\n","    pickle.dump(CACHE, f, protocol=pickle.HIGHEST_PROTOCOL)\n","\n","  return train_dataset, val_dataset\n","\n"]},{"cell_type":"markdown","metadata":{"id":"2P7R1QDFX6r_"},"source":["# Train"]},{"cell_type":"code","execution_count":null,"metadata":{"id":"vwTafW97yOG-"},"outputs":[],"source":["from torch.utils.data import DataLoader, RandomSampler, SequentialSampler\n","\n","@time_code()\n","def define_dataloaders(train_dataset, validation_dataset):\n","  # The DataLoader needs to know our batch size for training, so we specify it\n","  # here. For fine-tuning BERT on a specific task, the authors recommend a batch\n","  # size of 16 or 32.\n","  batch_size = 16\n","\n","  # Create the DataLoaders for our training and validation sets.\n","  # We'll take training samples in random order.\n","  train_dataloader = DataLoader(\n","              train_dataset,  # The training samples.\n","              sampler = RandomSampler(train_dataset), # Select batches randomly\n","              batch_size = batch_size # Trains with this batch size.\n","          )\n","\n","  # For validation the order doesn't matter, so we'll just read them sequentially.\n","  validation_dataloader = DataLoader(\n","              validation_dataset, # The validation samples.\n","              sampler = SequentialSampler(validation_dataset), # Pull out batches sequentially.\n","              batch_size = batch_size # Evaluate with this batch size.\n","          )\n","  return train_dataloader, validation_dataloader\n"]},{"cell_type":"code","execution_count":null,"metadata":{"id":"X1mpf2dOyN8d"},"outputs":[],"source":["from transformers import BertForSequenceClassification, AdamW, BertConfig, AutoModelForSequenceClassification\n","\n","\n","class PRE_EXISTING_MODELS:\n","\n","  BIO_BERT = \"emilyalsentzer/Bio_ClinicalBERT\"\n","  GATATRON = \"UFNLP/gatortron-base\"\n","  SAP_BERT = \"cambridgeltl/SapBERT-from-PubMedBERT-fulltext\"\n","\n","\n","@time_code()\n","def define_model(base_model_str):\n","  # Load BertForSequenceClassification, the pretrained BERT model with a single\n","  # linear classification layer on top.\n","  model = BertForSequenceClassification.from_pretrained(\n","      base_model_str,\n","      num_labels = 2, # The number of output labels--2 for binary classification.\n","                      # You can increase this for multi-class tasks.\n","      output_attentions = False, # Whether the model returns attentions weights.\n","      output_hidden_states = False, # Whether the model returns all hidden-states.\n","  )\n","\n","  # Tell pytorch to run this model on the GPU.\n","  model.to(device)\n","  return model"]},{"cell_type":"code","execution_count":null,"metadata":{"id":"bTk50as5yNwC"},"outputs":[],"source":["def print_model(model):\n","  # Get all of the model's parameters as a list of tuples.\n","  params = list(model.named_parameters())\n","\n","  print('The BERT model has {:} different named parameters.\\n'.format(len(params)))\n","\n","  print('==== Embedding Layer ====\\n')\n","\n","  for p in params[0:5]:\n","      print(\"{:\u003c55} {:\u003e12}\".format(p[0], str(tuple(p[1].size()))))\n","\n","  print('\\n==== First Transformer ====\\n')\n","\n","  for p in params[5:21]:\n","      print(\"{:\u003c55} {:\u003e12}\".format(p[0], str(tuple(p[1].size()))))\n","\n","  print('\\n==== Output Layer ====\\n')\n","\n","  for p in params[-4:]:\n","      print(\"{:\u003c55} {:\u003e12}\".format(p[0], str(tuple(p[1].size()))))"]},{"cell_type":"code","execution_count":null,"metadata":{"id":"b42b77O_yg5i"},"outputs":[],"source":["from transformers import get_linear_schedule_with_warmup\n","\n","def define_optimiser_and_scheduler(model):\n","  # Note: AdamW is a class from the huggingface library (as opposed to pytorch)\n","  # I believe the 'W' stands for 'Weight Decay fix\"\n","  optimizer = AdamW(model.parameters(),\n","                    lr = 2e-5, # args.learning_rate - default is 5e-5, our notebook had 2e-5\n","                    eps = 1e-8 # args.adam_epsilon  - default is 1e-8.\n","  )\n","  # Create the learning rate scheduler.\n","  scheduler = get_linear_schedule_with_warmup(optimizer,\n","                                            num_warmup_steps = 0, # Default value in run_glue.py\n","                                            num_training_steps = total_steps)\n","  return optimizer, scheduler"]},{"cell_type":"code","execution_count":null,"metadata":{"id":"AscRxQJub0st"},"outputs":[],"source":["# Function to calculate the accuracy of our predictions vs labels\n","def flat_accuracy(preds, labels):\n","    pred_flat = np.argmax(preds, axis=1).flatten()\n","    labels_flat = labels.flatten()\n","    return np.sum(pred_flat == labels_flat) / len(labels_flat)"]},{"cell_type":"code","execution_count":null,"metadata":{"id":"Dl-mFq6OytW9"},"outputs":[],"source":["import random\n","import numpy as np\n","\n","# This training code is based on the `run_glue.py` script here:\n","# https://github.com/huggingface/transformers/blob/5bfcd0485ece086ebcbed2d008813037968a9e58/examples/run_glue.py#L128\n","\n","@time_code()\n","def train(model, train_loader, validation_loader, optimizer, scheduler):\n","  # Number of training epochs. The BERT authors recommend between 2 and 4.\n","  # We chose to run for 4, but we'll see later that this may be over-fitting the\n","  # training data.\n","  epochs = 4\n","\n","  # Total number of training steps is [number of batches] x [number of epochs].\n","  # (Note that this is not the same as the number of training samples).\n","  total_steps = len(train_dataloader) * epochs\n","\n","  # Set the seed value all over the place to make this reproducible.\n","  seed_val = 42\n","\n","  random.seed(seed_val)\n","  np.random.seed(seed_val)\n","  torch.manual_seed(seed_val)\n","  torch.cuda.manual_seed_all(seed_val)\n","\n","  # We'll store a number of quantities such as training and validation loss,\n","  # validation accuracy, and timings.\n","  training_stats = []\n","\n","  # Measure the total training time for the whole run.\n","  total_t0 = time.time()\n","\n","  # For each epoch...\n","  for epoch_i in range(0, epochs):\n","\n","      # ========================================\n","      #               Training\n","      # ========================================\n","\n","      # Perform one full pass over the training set.\n","\n","      print(\"\")\n","      print('======== Epoch {:} / {:} ========'.format(epoch_i + 1, epochs))\n","      print('Training...')\n","\n","      # Measure how long the training epoch takes.\n","      t0 = time.time()\n","\n","      # Reset the total loss for this epoch.\n","      total_train_loss = 0\n","\n","      # Put the model into training mode. Don't be mislead--the call to\n","      # `train` just changes the *mode*, it doesn't *perform* the training.\n","      # `dropout` and `batchnorm` layers behave differently during training\n","      # vs. test (source: https://stackoverflow.com/questions/51433378/what-does-model-train-do-in-pytorch)\n","      model.train()\n","\n","      # For each batch of training data...\n","      for step, batch in enumerate(train_dataloader):\n","\n","          # Progress update every 40 batches.\n","          if step % 40 == 0 and not step == 0:\n","              # Calculate elapsed time in minutes.\n","              elapsed = time.time() - t0\n","\n","              # Report progress.\n","              print('  Batch {:\u003e5,}  of  {:\u003e5,}.    Elapsed: {:}.'.format(step, len(train_dataloader), elapsed))\n","\n","          # Unpack this training batch from our dataloader.\n","          #\n","          # As we unpack the batch, we'll also copy each tensor to the GPU using the\n","          # `to` method.\n","          #\n","          # `batch` contains three pytorch tensors:\n","          #   [0]: input ids\n","          #   [1]: attention masks\n","          #   [2]: labels\n","          b_input_ids = batch[0].to(device)\n","          b_input_mask = batch[1].to(device)\n","          b_labels = batch[2].to(device)\n","\n","          # Always clear any previously calculated gradients before performing a\n","          # backward pass. PyTorch doesn't do this automatically because\n","          # accumulating the gradients is \"convenient while training RNNs\".\n","          # (source: https://stackoverflow.com/questions/48001598/why-do-we-need-to-call-zero-grad-in-pytorch)\n","          model.zero_grad()\n","\n","          # Perform a forward pass (evaluate the model on this training batch).\n","          # In PyTorch, calling `model` will in turn call the model's `forward`\n","          # function and pass down the arguments. The `forward` function is\n","          # documented here:\n","          # https://huggingface.co/transformers/model_doc/bert.html#bertforsequenceclassification\n","          # The results are returned in a results object, documented here:\n","          # https://huggingface.co/transformers/main_classes/output.html#transformers.modeling_outputs.SequenceClassifierOutput\n","          # Specifically, we'll get the loss (because we provided labels) and the\n","          # \"logits\"--the model outputs prior to activation.\n","          result = model(b_input_ids,\n","                        token_type_ids=None,\n","                        attention_mask=b_input_mask,\n","                        labels=b_labels,\n","                        return_dict=True)\n","\n","          loss = result.loss\n","          logits = result.logits\n","\n","          # Accumulate the training loss over all of the batches so that we can\n","          # calculate the average loss at the end. `loss` is a Tensor containing a\n","          # single value; the `.item()` function just returns the Python value\n","          # from the tensor.\n","          total_train_loss += loss.item()\n","\n","          # Perform a backward pass to calculate the gradients.\n","          loss.backward()\n","\n","          # Clip the norm of the gradients to 1.0.\n","          # This is to help prevent the \"exploding gradients\" problem.\n","          torch.nn.utils.clip_grad_norm_(model.parameters(), 1.0)\n","\n","          # Update parameters and take a step using the computed gradient.\n","          # The optimizer dictates the \"update rule\"--how the parameters are\n","          # modified based on their gradients, the learning rate, etc.\n","          optimizer.step()\n","\n","          # Update the learning rate.\n","          scheduler.step()\n","\n","      # Calculate the average loss over all of the batches.\n","      avg_train_loss = total_train_loss / len(train_dataloader)\n","\n","      # Measure how long this epoch took.\n","      training_time = time.time() - t0\n","\n","      print(\"\")\n","      print(\"  Average training loss: {0:.2f}\".format(avg_train_loss))\n","      print(\"  Training epcoh took: {:}\".format(training_time))\n","\n","      # ========================================\n","      #               Validation\n","      # ========================================\n","      # After the completion of each training epoch, measure our performance on\n","      # our validation set.\n","\n","      print(\"\")\n","      print(\"Running Validation...\")\n","\n","      t0 = time.time()\n","\n","      # Put the model in evaluation mode--the dropout layers behave differently\n","      # during evaluation.\n","      model.eval()\n","\n","      # Tracking variables\n","      total_eval_accuracy = 0\n","      total_eval_loss = 0\n","      nb_eval_steps = 0\n","\n","      # Evaluate data for one epoch\n","      for batch in validation_dataloader:\n","\n","          # Unpack this training batch from our dataloader.\n","          #\n","          # As we unpack the batch, we'll also copy each tensor to the GPU using\n","          # the `to` method.\n","          #\n","          # `batch` contains three pytorch tensors:\n","          #   [0]: input ids\n","          #   [1]: attention masks\n","          #   [2]: labels\n","          b_input_ids = batch[0].to(device)\n","          b_input_mask = batch[1].to(device)\n","          b_labels = batch[2].to(device)\n","\n","          # Tell pytorch not to bother with constructing the compute graph during\n","          # the forward pass, since this is only needed for backprop (training).\n","          with torch.no_grad():\n","\n","              # Forward pass, calculate logit predictions.\n","              # token_type_ids is the same as the \"segment ids\", which\n","              # differentiates sentence 1 and 2 in 2-sentence tasks.\n","              result = model(b_input_ids,\n","                            token_type_ids=None,\n","                            attention_mask=b_input_mask,\n","                            labels=b_labels,\n","                            return_dict=True)\n","\n","          # Get the loss and \"logits\" output by the model. The \"logits\" are the\n","          # output values prior to applying an activation function like the\n","          # softmax.\n","          loss = result.loss\n","          logits = result.logits\n","\n","          # Accumulate the validation loss.\n","          total_eval_loss += loss.item()\n","\n","          # Move logits and labels to CPU\n","          logits = logits.detach().cpu().numpy()\n","          label_ids = b_labels.to('cpu').numpy()\n","\n","          # Calculate the accuracy for this batch of test sentences, and\n","          # accumulate it over all batches.\n","          total_eval_accuracy += flat_accuracy(logits, label_ids)\n","\n","\n","      # Report the final accuracy for this validation run.\n","      avg_val_accuracy = total_eval_accuracy / len(validation_dataloader)\n","      print(\"  Accuracy: {0:.2f}\".format(avg_val_accuracy))\n","\n","      # Calculate the average loss over all of the batches.\n","      avg_val_loss = total_eval_loss / len(validation_dataloader)\n","\n","      # Measure how long the validation run took.\n","      validation_time = time.time() - t0\n","\n","      print(\"  Validation Loss: {0:.2f}\".format(avg_val_loss))\n","      print(\"  Validation took: {:}\".format(validation_time))\n","\n","      # Record all statistics from this epoch.\n","      training_stats.append(\n","          {\n","              'epoch': epoch_i + 1,\n","              'Training Loss': avg_train_loss,\n","              'Valid. Loss': avg_val_loss,\n","              'Valid. Accur.': avg_val_accuracy,\n","              'Training Time': training_time,\n","              'Validation Time': validation_time\n","          }\n","      )\n","\n","  print(\"\")\n","  print(\"Training complete!\")\n","\n","  print(\"Total training took {:} (h:mm:ss)\".format(time.time()-total_t0))"]},{"cell_type":"markdown","metadata":{"id":"lLkXZWX08z8S"},"source":["# Evaluate"]},{"cell_type":"code","execution_count":null,"metadata":{"id":"kdFRCAaJytSC"},"outputs":[],"source":["#performance on test set\n","@time_code()\n","def get_test_dataset(data):\n","  # Report the number of sentences.\n","  print('Number of test sentences: {:,}\\n'.format(data.shape[0]))\n","\n","  # Create sentence and label lists\n","  sentences = data.text.values\n","  labels = data.label.values\n","\n","  # Tokenize all of the sentences and map the tokens to thier word IDs.\n","  input_ids = []\n","  attention_masks = []\n","\n","  # For every sentence...\n","  for sent in sentences:\n","      # `encode_plus` will:\n","      #   (1) Tokenize the sentence.\n","      #   (2) Prepend the `[CLS]` token to the start.\n","      #   (3) Append the `[SEP]` token to the end.\n","      #   (4) Map tokens to their IDs.\n","      #   (5) Pad or truncate the sentence to `max_length`\n","      #   (6) Create attention masks for [PAD] tokens.\n","      encoded_dict = tokenizer.encode_plus(\n","                          sent,                      # Sentence to encode.\n","                          add_special_tokens = True, # Add '[CLS]' and '[SEP]'\n","                          max_length = max_sentence_size,           # Pad \u0026 truncate all sentences.\n","                          pad_to_max_length = True,\n","                          return_attention_mask = True,   # Construct attn. masks.\n","                          return_tensors = 'pt',     # Return pytorch tensors.\n","                    )\n","\n","      # Add the encoded sentence to the list.\n","      input_ids.append(encoded_dict['input_ids'])\n","\n","      # And its attention mask (simply differentiates padding from non-padding).\n","      attention_masks.append(encoded_dict['attention_mask'])\n","\n","  # Convert the lists into tensors.\n","  input_ids = torch.cat(input_ids, dim=0)\n","  attention_masks = torch.cat(attention_masks, dim=0)\n","  labels = torch.tensor(labels)\n","\n","  # Set the batch size.\n","  batch_size = 16\n","\n","  # Create the DataLoader.\n","  prediction_data = TensorDataset(input_ids, attention_masks, labels)\n","  prediction_sampler = SequentialSampler(prediction_data)\n","  prediction_dataloader = DataLoader(prediction_data, sampler=prediction_sampler, batch_size=batch_size)\n","  return prediction_dataloader"]},{"cell_type":"code","execution_count":null,"metadata":{"id":"rO1VJgHJ834v"},"outputs":[],"source":["#Evaluate on test set\n","import torchmetrics\n","# Prediction on test set\n","\n","@time_code()\n","def evaluate(model, dataloader):\n","  # Put model in evaluation mode\n","  model.eval()\n","\n","  # Tracking variables\n","  predictions , true_labels = [], []\n","\n","  accuracy_function = torchmetrics.classification.BinaryAccuracy().to('cuda:0')\n","  f1_function = torchmetrics.classification.BinaryF1Score().to('cuda:0')\n","  tnr_function = torchmetrics.classification.BinarySpecificity().to('cuda:0')\n","  tpr_function = torchmetrics.classification.BinaryRecall().to('cuda:0')\n","\n","  mcc_function = torchmetrics.classification.BinaryMatthewsCorrCoef().to('cuda:0')\n","\n","\n","  # Predict\n","  for batch in dataloader:\n","    # Add batch to GPU\n","    batch = tuple(t.to(device) for t in batch)\n","\n","    # Unpack the inputs from our dataloader\n","    b_input_ids, b_input_mask, b_labels = batch\n","\n","    # Telling the model not to compute or store gradients, saving memory and\n","    # speeding up prediction\n","    with torch.no_grad():\n","        # Forward pass, calculate logit predictions.\n","        result = model(b_input_ids,\n","                      token_type_ids=None,\n","                      attention_mask=b_input_mask,\n","                      return_dict=True)\n","\n","    logits = result.logits\n","\n","    # Move logits and labels to CPU\n","\n","    class_preds = torch.argmax(logits, axis=1)\n","    accuracy_function(class_preds, b_labels)\n","    f1_function(class_preds, b_labels)\n","    tnr_function(class_preds, b_labels)\n","    tpr_function(class_preds, b_labels)\n","    mcc_function(class_preds, b_labels)\n","    logits = logits.detach().cpu().numpy()\n","    label_ids = b_labels.to('cpu').numpy()\n","\n","    # Store predictions and true labels\n","    predictions.append(logits)\n","    true_labels.append(label_ids)\n","\n","  acc = accuracy_function.compute().item()\n","  f1 =  f1_function.compute().item()\n","  tnr =  tnr_function.compute().item()\n","  tpr =  tpr_function.compute().item()\n","  mcc =  mcc_function.compute().item()\n","  return {\n","      'accuracy': acc,\n","      'f1': f1,\n","      'tnr': tnr,\n","      'tpr': tpr,\n","      'mcc': mcc,\n","  }"]},{"cell_type":"markdown","metadata":{"id":"IBkwfX0hOfih"},"source":["# Manifests\n","\n","\n"]},{"cell_type":"code","execution_count":null,"metadata":{"colab":{"base_uri":"https://localhost:8080/"},"executionInfo":{"elapsed":28278,"status":"ok","timestamp":1711317018612,"user":{"displayName":"Robert Cobb","userId":"08992261826785514419"},"user_tz":0},"id":"5hhDOyvnPgWB","outputId":"1fbad741-1cb0-48a8-a3e2-e998e1ba8109"},"outputs":[{"name":"stdout","output_type":"stream","text":["Number of test sentences: 2,000\n","\n"]},{"name":"stderr","output_type":"stream","text":["/usr/local/lib/python3.10/dist-packages/transformers/tokenization_utils_base.py:2645: FutureWarning: The `pad_to_max_length` argument is deprecated and will be removed in a future version, use `padding=True` or `padding='longest'` to pad to the longest sequence in the batch, or use `padding='max_length'` to pad to a max length. In this case, you can give a specific length with `max_length` (e.g. `max_length=45`) or leave max_length to None to pad to the maximal input size of the model (e.g. 512 for Bert).\n","  warnings.warn(\n"]},{"name":"stdout","output_type":"stream","text":["Accuracy: 0.79\n","F1: 0.79\n","TNR: 0.82\n","TPR: 0.76\n","MCC: 0.58\n","DONE.\n"]}],"source":["# Example run throught\n","\n","train_dataset, test_dataset = get_train_test_split(full_data_for_classfier, 10000, 0.5)\n","train_dataset, validation_dataset = get_loader(train_dataset)\n","train_dataloader, validation_dataloader = define_dataloaders(train_dataset, validation_dataset)\n","epochs = 8\n","total_steps = len(train_dataloader) * epochs\n","model = define_model()\n","optimiser, scheduler = define_optimiser_and_scheduler(model)\n","train(model, train_dataloader, validation_dataloader, optimiser, scheduler)\n","test_loader = get_test_dataset(test_dataset)\n","evaluate(model, test_loader)"]},{"cell_type":"code","execution_count":null,"metadata":{"id":"6ypc9vI6QFF0"},"outputs":[],"source":["full_data_for_classfier = get_data_for_classifier()\n","\n","\n","results_file = os.path.join(sample_size_dir, 'results_6d923a21-11f3-430d-bb06-95ad90a7aef2.json')\n","\n","if not os.path.exists(results_file):\n","  with open(results_file, 'w') as f:\n","    f.write('{}')\n","\n","with open(results_file, 'r') as f:\n","  results = json.load(f)\n","\n","for padding_side, truncate_side in [('left', 'left'), ('left', 'right'), ('right', 'left'), ('right', 'right')]:\n","  if (padding_side, truncate_side) not in results:\n","    results[str(padding_side) + '_' + str(truncate_side)] = {}\n","  for model_name, model_str in [('BioBert', PRE_EXISTING_MODELS.BIO_BERT)]: #, ('gatatron', PRE_EXISTING_MODELS.GATATRON), ('sap-bert', PRE_EXISTING_MODELS.SAP_BERT)]:\n","\n","    tokenizer_dict = {\n","        'pretrained_string': model_str,\n","        'padding_side': padding_side,\n","        'truncate_side': truncate_side,\n","        'do_lower_case':True\n","    }\n","    tokenizer = get_tokenizer(**tokenizer_dict)\n","    tokenizer_str = json.dumps(tokenizer_dict, sort_keys=True)\n","\n","    if model_name not in results[str(padding_side) + '_' + str(truncate_side)]:\n","      results[str(padding_side) + '_' + str(truncate_side)][model_name] = []\n","    for cross_fold_index, (train_val_df, test_df) in enumerate(get_balanced_cross_fold_train_test_split(\n","        full_data_for_classfier, 5)):\n","\n","      if len(results[str(padding_side) + '_' + str(truncate_side)][model_name]) \u003e cross_fold_index: # we already have results\n","        print(\"Skipping padding=%s model=%s %s cross fold index %d because we have results\" % (str((padding_side, truncate_side)).\n","                                                                                               model_name, cross_fold_index))\n","        continue\n","\n","      train_dataset, validation_dataset = get_loader_cache(tokenizer, tokenizer_str, train_val_df)\n","      train_dataloader, validation_dataloader = define_dataloaders(train_dataset, validation_dataset)\n","\n","      epochs = 4\n","      total_steps = len(train_dataloader) * epochs\n","      model = define_model(model_str)\n","      optimiser, scheduler = define_optimiser_and_scheduler(model)\n","      train(model, train_dataloader, validation_dataloader, optimiser, scheduler)\n","      test_loader = get_test_dataset(test_df)\n","      results[str(padding_side) + '_' + str(truncate_side)][model_name].append(evaluate(model, test_loader))\n","      with open(results_file, 'w') as f:\n","        json.dump(results, f)\n","\n","      print(model_name)\n","      print(results[str(padding_side) + '_' + str(truncate_side)][model_name])\n","  print(results)\n","\n"]},{"cell_type":"code","execution_count":null,"metadata":{"colab":{"base_uri":"https://localhost:8080/"},"executionInfo":{"elapsed":2021399,"status":"ok","timestamp":1711388271546,"user":{"displayName":"Robert Cobb","userId":"08992261826785514419"},"user_tz":0},"id":"rwTVXqFEegxM","outputId":"20266ab4-53e7-40f9-a685-2fd7bb2749cd"},"outputs":[{"name":"stderr","output_type":"stream","text":["/usr/local/lib/python3.10/dist-packages/transformers/tokenization_utils_base.py:2645: FutureWarning: The `pad_to_max_length` argument is deprecated and will be removed in a future version, use `padding=True` or `padding='longest'` to pad to the longest sequence in the batch, or use `padding='max_length'` to pad to a max length. In this case, you can give a specific length with `max_length` (e.g. `max_length=45`) or leave max_length to None to pad to the maximal input size of the model (e.g. 512 for Bert).\n","  warnings.warn(\n"]},{"name":"stdout","output_type":"stream","text":["Original:  CONDITION UPDATE\n","D: PLEASE SEE CAREVUE FOR SPECIFICS. PT ADMITTED TO SICU FROM OR AROUND 1830- S/P SPINAL FUSION. INTUBATED ON PROPOFOL GTT FOR SEDATION AND NEO GTT TO MAINTAIN SBP\u003e100.\n","NEURO: PT LIGHTLY SEDATED ON PROPOFOL AT 50- 30 MCG/KG/MIN. ABLE TO OPEN EYES TO NAME, FOLLOWING COMMANDS AS SHIFT PROGRESSED. MAE ON BED ALTHOUGH LEFT LEG DOES APPEAR WEAKER. FENTANYL GTT STARTED FOR PAIN MANAGEMENT.\n","CV: T MAX 99.5. HR 85-105 NSR-ST. NEO GTT WEANED FROM 1.1MCG TO 0.5. UNABLE TO WEAN OFF SECONDARY TO LOWER BP WHEN SLEEPING. WHEN AWAKE, SBP \u003e100. MULTIPLE ATTEMPTS MADE BUT NEO ALWAYS RESTARTED AFTER 10-15 MINUTES.\n","RESP: BS CLEAR BUT DIMINISHED IN BASES. SX FOR SCANT AMT CLEAR FLUID. SATS\u003e100. REMAINS ON CMV WITH RATE OF 12, FIO2 50%\n","GI: NPO, ABD SOFT WITH + BS. ORAL-GASTRIC TUBE PATENT WITH BILIOUS DRAINAGE.\n","GU: LOW URINE X1 HR- TX'D WITH FLUID BOLUS, PRESENTLY HUO 60-80\n","ENDO: SLIDING SCALE INSULIN STARTED. BS 154-204\n","SKIN: PT NOTED TO HAVE ABRASION NEAR RIGHT EYE FROM TAPE USED DURING OR. ELASTOPLAST DRESSING ON BACK INTACT. LEFT FLANK DRESSING INTACT. HEMOVAC DRAINING 60-120CC BLOODY DRAINAGE Q2HRS.\n","HEME: HCT STABLE AT 32.6-33.4\n","A: HEMODYNAMICS AND RESP PARAMETERS MONITORED, SERIAL HCTS DONE, FENT GTT STARTED FOR PAIN MANAGEMENT, ATTEMPT TO WEAN NEO MAINTAINING SBP\u003e100\n","R: STABLE POST-OP, ? WEAN TO EXTUBATE THIS AM, NEO GTT SHOULD BE ABLE TO BE WEANED OFF AFTER EXTUBATION, CONT. TO MONITOR HCT.\n","\n","Token IDs: tensor([  101,  3879, 11984,   173,   131,  4268,  1267,  1920, 19038,  1111,\n","         2747,  1116,   119,   185,  1204,  4120,  1106, 16554,  1358,  1121,\n","         1137,  1213,  9200,   118,   188,   120,   185, 19245, 11970,   119,\n","         1107, 25098,  2913,  1113, 21146, 10008,  4063,   176,  3069,  1111,\n","        14516, 13759,  1105, 15242,   176,  3069,  1106,  4731,   188,  1830,\n","         1643,   135,  1620,   119, 24928, 11955,   131,   185,  1204,  7863,\n","        14516, 14459,  1113, 21146, 10008,  4063,  1120,  1851,   118,  1476,\n","          182,  1665,  1403,   120,  4023,   120, 11241,   119,  1682,  1106,\n","         1501,  1257,  1106,  1271,   117,  1378, 11443,  1112,  5212, 12687,\n","          119, 12477,  1162,  1113,  1908,  1780,  1286,  3420,  1674,  2845,\n","        16990,   119,   175,  3452, 18266,  1233,   176,  3069,  1408,  1111,\n","         2489,  2635,   119,   172,  1964,   131,   189, 12477,  1775,  4850,\n","          119,   126,   119,   177,  1197,  4859,   118,  8359,   183,  1116,\n","         1197,   118,   188,  1204,   119, 15242,   176,  3069,  1195,  6354,\n","         1181,  1121,   122,   119,   122,  1306,  1665,  1403,  1106,   121,\n","          119,   126,   119,  3372,  1106,  1195,  1389,  1228,  3718,  1106,\n","         2211,   171,  1643,  1165,  5575,   119,  1165,  8264,   117,   188,\n","         1830,  1643,   135,  1620,   119,  2967,  4021,  1189,  1133, 15242,\n","         1579, 27777,  1174,  1170,  1275,   118,  1405,  1904,   119,  1231,\n","        20080,   131,   171,  1116,  2330,  1133, 17017,  1107,  7616,   119,\n","          188,  1775,  1111, 14884,  1204,  1821,  1204,  2330,  8240,   119,\n","         2068,  1116,   135,  1620,   119,  2606,  1113,  3975,  1964,  1114,\n","         2603,  1104,  1367,   117, 20497,  1186,  1477,  1851,   110,   176,\n","         1182,   131,   183,  5674,   117,   170,  1830,  1181,  2991,  1114,\n","          116,   171,  1116,   119,  9619,   118,  3245, 11048,  7159,  8581,\n","         1114, 16516,  9436,  1361, 12779,   119,   176,  1358,   131,  1822,\n","        19968,   193,  1475,   177,  1197,   118,   189,  1775,   112,   173,\n","         1114,  8240,   171,  4063,  1361,   117, 16269,   177, 11848,  2539,\n","          118,  2908,  1322,  1186,   131,  7989,  3418, 26825,  1408,   119,\n","          171,  1116, 17733,   118, 21355,  2241,   131,   185,  1204,  2382,\n","         1106,  1138,   170,  6766,  8427,  1485,  1268,  2552,  1121,  6649,\n","         1215,  1219,  1137,   119,  8468, 12788,  4184, 19268, 11597,  1113,\n","         1171,  9964,   119,  1286, 12509, 11597,  9964,   119, 23123,  8625,\n","         1665, 21892,  2539,   118,  5356, 19515,  7201, 12779,   186,  1477,\n","         8167,  1116,   119, 23123,  1162,   131,   177,  5822,  6111,  1120,\n","         2724,   119,   127,   118,  3081,   119,   125,   170,   131, 23123,\n","        22320, 12881,  4724,  1105,  1231, 20080, 11934, 19232,   117,  7838,\n","          177, 15585,  1694,   117,   175,  3452,   176,  3069,  1408,  1111,\n","         2489,  2635,   117,  2661,  1106,  1195,  1389, 15242,  8338,   188,\n","         1830,  1643,   135,  1620,   187,   131,  6111,  2112,   118, 11769,\n","          117,   136,  1195,  1389,  1106,  4252, 25098,  2193,  1142,  1821,\n","          117, 15242,   176,  3069,  1431,  1129,  1682,  1106,  1129,  1195,\n","         6354,  1181,  1228,  1170,  4252, 25098,  1891,   117, 14255,  1204,\n","          119,  1106,  8804,   177,  5822,   119,   102,     0,     0,     0,\n","            0,     0,     0,     0,     0,     0,     0,     0,     0,     0,\n","            0,     0,     0,     0,     0,     0,     0,     0,     0,     0,\n","            0,     0,     0,     0,     0,     0,     0,     0,     0,     0,\n","            0,     0,     0,     0,     0,     0,     0,     0,     0,     0,\n","            0,     0,     0,     0,     0,     0,     0,     0,     0,     0,\n","            0,     0,     0,     0,     0,     0,     0,     0,     0,     0,\n","            0,     0,     0,     0,     0,     0,     0,     0,     0,     0,\n","            0,     0])\n","  450 training samples\n","   50 validation samples\n"]},{"name":"stderr","output_type":"stream","text":["Some weights of BertForSequenceClassification were not initialized from the model checkpoint at emilyalsentzer/Bio_ClinicalBERT and are newly initialized: ['classifier.bias', 'classifier.weight']\n","You should probably TRAIN this model on a down-stream task to be able to use it for predictions and inference.\n","/usr/local/lib/python3.10/dist-packages/transformers/optimization.py:429: FutureWarning: This implementation of AdamW is deprecated and will be removed in a future version. Use the PyTorch implementation torch.optim.AdamW instead, or set `no_deprecation_warning=True` to disable this warning\n","  warnings.warn(\n"]},{"name":"stdout","output_type":"stream","text":["\n","======== Epoch 1 / 8 ========\n","Training...\n","\n","  Average training loss: 0.67\n","  Training epcoh took: 12.578038930892944\n","\n","Running Validation...\n","  Accuracy: 0.42\n","  Validation Loss: 0.85\n","  Validation took: 0.4594876766204834\n","\n","======== Epoch 2 / 8 ========\n","Training...\n","\n","  Average training loss: 0.61\n","  Training epcoh took: 12.578543424606323\n","\n","Running Validation...\n","  Accuracy: 0.50\n","  Validation Loss: 0.90\n","  Validation took: 0.4643223285675049\n","\n","======== Epoch 3 / 8 ========\n","Training...\n","\n","  Average training loss: 0.53\n","  Training epcoh took: 12.604345321655273\n","\n","Running Validation...\n","  Accuracy: 0.52\n","  Validation Loss: 0.93\n","  Validation took: 0.4687681198120117\n","\n","======== Epoch 4 / 8 ========\n","Training...\n","\n","  Average training loss: 0.45\n","  Training epcoh took: 12.592608213424683\n","\n","Running Validation...\n","  Accuracy: 0.47\n","  Validation Loss: 0.98\n","  Validation took: 0.467909574508667\n","\n","======== Epoch 5 / 8 ========\n","Training...\n","\n","  Average training loss: 0.35\n","  Training epcoh took: 12.597765922546387\n","\n","Running Validation...\n","  Accuracy: 0.50\n","  Validation Loss: 1.12\n","  Validation took: 0.4643125534057617\n","\n","======== Epoch 6 / 8 ========\n","Training...\n","\n","  Average training loss: 0.26\n","  Training epcoh took: 12.603060483932495\n","\n","Running Validation...\n","  Accuracy: 0.34\n","  Validation Loss: 1.25\n","  Validation took: 0.46254611015319824\n","\n","======== Epoch 7 / 8 ========\n","Training...\n","\n","  Average training loss: 0.20\n","  Training epcoh took: 12.680386781692505\n","\n","Running Validation...\n","  Accuracy: 0.45\n","  Validation Loss: 1.31\n","  Validation took: 0.4660611152648926\n","\n","======== Epoch 8 / 8 ========\n","Training...\n","\n","  Average training loss: 0.15\n","  Training epcoh took: 12.7128746509552\n","\n","Running Validation...\n","  Accuracy: 0.34\n","  Validation Loss: 1.39\n","  Validation took: 0.5490977764129639\n","\n","Training complete!\n","Total training took 104.78169560432434 (h:mm:ss)\n","Number of test sentences: 6,000\n","\n","500\n","{'accuracy': 0.6190000176429749, 'f1': 0.620517909526825, 'tnr': 0.6239432096481323, 'tpr': 0.6141965389251709, 'mcc': 0.23812001943588257}\n","Original:  [**2168-2-5**] 8:23 PM\n"," CT ABDOMEN W/CONTRAST; CT PELVIS W/CONTRAST                     Clip # [**Clip Number (Radiology) 35244**]\n"," CT 150CC NONIONIC CONTRAST\n"," Reason: 54 male s/p unrestrained driver MVA\n"," Field of view: 36 Contrast: OPTIRAY Amt: 150\n"," ______________________________________________________________________________\n"," [**Hospital 4**] MEDICAL CONDITION:\n","  53 year old man with\n"," REASON FOR THIS EXAMINATION:\n","  54 male s/p unrestrained driver MVA\n"," ______________________________________________________________________________\n","                                 FINAL REPORT\n"," INDICATION: 54yr old male s/p MVA\n","\n"," COMPARISON: None\n","\n"," TECHNIQUE: Contiguous axial images of the abdomen and pelvis were obtained\n"," with intravenous contrast.\n","\n"," CONTRAST: No oral contrast was used.  150cc of nonionic intravenous contrast\n"," were administered.\n","\n"," ABDOMINAL CT WITH INTRAVENOUS CONTRAST: The lung bases show minimal dependent\n"," atelectasis, with no pleural effusions or pneumothoraces.  The liver contains\n"," a low attenuation area in its posterior right lobe, which is nonspecific in\n"," appearance on this single contrast phase study. Otherwise, the liver has a\n"," homogeneous enhancement pattern, with no evidence of traumatic injury.\n","\n"," The gallbladder, pancreas, spleen, stomach, small and large bowel, and\n"," intraabdominal great vessels are unremarkable.  Bilateral kidneys show\n"," symmetrical excretion of contrast, with no renal injuries, masses or\n"," hydronephrosis.  Bilateral adrenal glands and ureters are unremarkable.  There\n"," is no free fluid, and no free air in the abdomen. There is no mesenteric or\n"," retroperitoneal lymphadenopathy.\n","\n"," PELVIC CT WITH INTRAVENOUS CONTRAST: The rectum, pelvic loops of bowel,\n"," prostate, seminal vesicals and distal ureters are unremarkable.  The urinary\n"," bladder contains the balloon of a Foley catheter.  There is no free fluid, and\n"," no pelvic or inguinal lymphadenopathy.\n","\n"," Osseous structures show no fractures.\n","\n"," IMPRESSION: Low attenuation area in the liver is too small to fully\n"," characterize on this limited study, but could represent a hemangioma or cyst.\n"," Otherwise, there is a normal CT appearance of the abdomen and pelvis, with no\n"," intrabdominal injuries.\n","\n","\n","\n","Token IDs: tensor([  101,   164,   115,   115, 22148,  1604,   118,   123,   118,   126,\n","          115,   115,   166,   129,   131,  1695,  9852,   172,  1204, 14701,\n","          192,   120,  5014,   132,   172,  1204,   185,  1883,  9356,   192,\n","          120,  5014, 13500,   108,   164,   115,   115, 13500,  1295,   113,\n","         2070,  6360,   114,  2588, 19598,  1527,   115,   115,   166,   172,\n","         1204,  4214, 19515,  1664,  1988,  1596,  5014,  2255,   131,  4335,\n","         2581,   188,   120,   185, 18366, 11098,  1174,  3445,   182,  2497,\n","         1768,  1104,  2458,   131,  3164,  5014,   131, 11769,  3121,  6447,\n","         1821,  1204,   131,  4214,   168,   168,   168,   168,   168,   168,\n","          168,   168,   168,   168,   168,   168,   168,   168,   168,   168,\n","          168,   168,   168,   168,   168,   168,   168,   168,   168,   168,\n","          168,   168,   168,   168,   168,   168,   168,   168,   168,   168,\n","          168,   168,   168,   168,   168,   168,   168,   168,   168,   168,\n","          168,   168,   168,   168,   168,   168,   168,   168,   168,   168,\n","          168,   168,   168,   168,   168,   168,   168,   168,   168,   168,\n","          168,   168,   168,   168,   168,   168,   168,   168,   168,   168,\n","          168,   168,   164,   115,   115,  2704,   125,   115,   115,   166,\n","         2657,  3879,   131,  4389,  1214,  1385,  1299,  1114,  2255,  1111,\n","         1142,  8179,   131,  4335,  2581,   188,   120,   185, 18366, 11098,\n","         1174,  3445,   182,  2497,   168,   168,   168,   168,   168,   168,\n","          168,   168,   168,   168,   168,   168,   168,   168,   168,   168,\n","          168,   168,   168,   168,   168,   168,   168,   168,   168,   168,\n","          168,   168,   168,   168,   168,   168,   168,   168,   168,   168,\n","          168,   168,   168,   168,   168,   168,   168,   168,   168,   168,\n","          168,   168,   168,   168,   168,   168,   168,   168,   168,   168,\n","          168,   168,   168,   168,   168,   168,   168,   168,   168,   168,\n","          168,   168,   168,   168,   168,   168,   168,   168,   168,   168,\n","          168,   168,  1509,  2592, 12754,   131,  4335, 12577,  1385,  2581,\n","          188,   120,   185,   182,  2497,  7577,   131,  3839,  5531,   131,\n","        14255,  3121, 22928,   170, 27361,  4351,  1104,  1103, 14701,  1105,\n","          185,  1883,  9356,  1127,  3836,  1114,  1107,  4487,  7912,  2285,\n","         5014,   119,  5014,   131,  1185,  9619,  5014,  1108,  1215,   119,\n","         4214, 19515,  1104,  1664,  1988,  1596,  1107,  4487,  7912,  2285,\n","         5014,  1127,  8318,   119, 24716,   172,  1204,  1114,  1107,  4487,\n","         7912,  2285,  5014,   131,  1103, 13093,  7616,  1437, 10298,  7449,\n","         8756, 18465, 14229,   117,  1114,  1185,   185,  1513, 12602,   174,\n","         3101, 27262,  1137,   185,  1673,  1818, 12858,  6533,  7723,   119,\n","         1103, 11911,  2515,   170,  1822,  1120,  5208, 10255,  1298,  1107,\n","         1157, 16530,  1268, 25163,   117,  1134,  1110,  1664, 20080, 10294,\n","        19814,  1107,  2468,  1113,  1142,  1423,  5014,  4065,  2025,   119,\n","         4303,   117,  1103, 11911,  1144,   170, 16358, 28008, 11778,  1880,\n","         4844,   117,  1114,  1185,  2554,  1104, 23057,  3773,   119,  1103,\n","        20003,  1233, 18075, 19541,   117, 13316, 13782,  2225,   117,   188,\n","         7136,  1424,   117,  3472,   117,  1353,  1105,  1415,  7125,  1883,\n","          117,  1105,  1107,  4487,  6639,  9277, 14196,  1632,  5956,  1132,\n","         8362, 16996, 23822,  1895,   119, 20557, 16042,  1116,  1437, 26795,\n","         4252, 13782,  2116,  1104,  5014,   117,  1114,  1185,  1231,  7050,\n","         5917,   117, 12980,  1137,   177, 19694,  1673,  7880, 21556,   119,\n","        20557,  8050, 23503,  1233, 26310,  1105,   190,  8127,  1468,  1132,\n","         8362, 16996, 23822,  1895,   119,  1175,  1110,  1185,  1714,  8240,\n","          117,  1105,  1185,  1714,  1586,  1107,  1103, 14701,   119,  1175,\n","         1110,   102])\n","  900 training samples\n","  100 validation samples\n"]},{"name":"stderr","output_type":"stream","text":["Some weights of BertForSequenceClassification were not initialized from the model checkpoint at emilyalsentzer/Bio_ClinicalBERT and are newly initialized: ['classifier.bias', 'classifier.weight']\n","You should probably TRAIN this model on a down-stream task to be able to use it for predictions and inference.\n"]},{"name":"stdout","output_type":"stream","text":["\n","======== Epoch 1 / 8 ========\n","Training...\n","  Batch    40  of     57.    Elapsed: 17.841606855392456.\n","\n","  Average training loss: 0.67\n","  Training epcoh took: 25.116600275039673\n","\n","Running Validation...\n","  Accuracy: 0.59\n","  Validation Loss: 0.67\n","  Validation took: 0.9202558994293213\n","\n","======== Epoch 2 / 8 ========\n","Training...\n","  Batch    40  of     57.    Elapsed: 17.861527919769287.\n","\n","  Average training loss: 0.60\n","  Training epcoh took: 25.154041051864624\n","\n","Running Validation...\n","  Accuracy: 0.71\n","  Validation Loss: 0.58\n","  Validation took: 0.922656774520874\n","\n","======== Epoch 3 / 8 ========\n","Training...\n","  Batch    40  of     57.    Elapsed: 17.882230758666992.\n","\n","  Average training loss: 0.50\n","  Training epcoh took: 25.184977531433105\n","\n","Running Validation...\n","  Accuracy: 0.66\n","  Validation Loss: 0.66\n","  Validation took: 0.9199802875518799\n","\n","======== Epoch 4 / 8 ========\n","Training...\n","  Batch    40  of     57.    Elapsed: 17.909728288650513.\n","\n","  Average training loss: 0.41\n","  Training epcoh took: 25.206593990325928\n","\n","Running Validation...\n","  Accuracy: 0.68\n","  Validation Loss: 0.61\n","  Validation took: 0.9265925884246826\n","\n","======== Epoch 5 / 8 ========\n","Training...\n","  Batch    40  of     57.    Elapsed: 17.9020357131958.\n","\n","  Average training loss: 0.32\n","  Training epcoh took: 25.210262060165405\n","\n","Running Validation...\n","  Accuracy: 0.69\n","  Validation Loss: 0.62\n","  Validation took: 0.9241621494293213\n","\n","======== Epoch 6 / 8 ========\n","Training...\n","  Batch    40  of     57.    Elapsed: 17.909459114074707.\n","\n","  Average training loss: 0.24\n","  Training epcoh took: 25.206395864486694\n","\n","Running Validation...\n","  Accuracy: 0.63\n","  Validation Loss: 0.74\n","  Validation took: 0.9237234592437744\n","\n","======== Epoch 7 / 8 ========\n","Training...\n","  Batch    40  of     57.    Elapsed: 17.911067485809326.\n","\n","  Average training loss: 0.17\n","  Training epcoh took: 25.221853256225586\n","\n","Running Validation...\n","  Accuracy: 0.68\n","  Validation Loss: 0.75\n","  Validation took: 0.9264824390411377\n","\n","======== Epoch 8 / 8 ========\n","Training...\n","  Batch    40  of     57.    Elapsed: 17.908443212509155.\n","\n","  Average training loss: 0.14\n","  Training epcoh took: 25.20970582962036\n","\n","Running Validation...\n","  Accuracy: 0.63\n","  Validation Loss: 0.79\n","  Validation took: 0.9323179721832275\n","\n","Training complete!\n","Total training took 208.92550230026245 (h:mm:ss)\n","Number of test sentences: 6,000\n","\n","1000\n","{'accuracy': 0.6626666784286499, 'f1': 0.6615384817123413, 'tnr': 0.6756848096847534, 'tpr': 0.6500164270401001, 'mcc': 0.32573962211608887}\n","Original:  Normal sinus rhythm\n","Elevated voltage suggestive of left ventricular hypertrophy\n","No previous report available for comparison\n","\n","\n","Token IDs: tensor([  101,  2999, 11850,  1361,  6795,  8208, 10323,  5996,  2109,  1104,\n","         1286, 21828,  4907,  5552,   177, 24312,  8005, 22192,  1185,  2166,\n","         2592,  1907,  1111,  7577,   102,     0,     0,     0,     0,     0,\n","            0,     0,     0,     0,     0,     0,     0,     0,     0,     0,\n","            0,     0,     0,     0,     0,     0,     0,     0,     0,     0,\n","            0,     0,     0,     0,     0,     0,     0,     0,     0,     0,\n","            0,     0,     0,     0,     0,     0,     0,     0,     0,     0,\n","            0,     0,     0,     0,     0,     0,     0,     0,     0,     0,\n","            0,     0,     0,     0,     0,     0,     0,     0,     0,     0,\n","            0,     0,     0,     0,     0,     0,     0,     0,     0,     0,\n","            0,     0,     0,     0,     0,     0,     0,     0,     0,     0,\n","            0,     0,     0,     0,     0,     0,     0,     0,     0,     0,\n","            0,     0,     0,     0,     0,     0,     0,     0,     0,     0,\n","            0,     0,     0,     0,     0,     0,     0,     0,     0,     0,\n","            0,     0,     0,     0,     0,     0,     0,     0,     0,     0,\n","            0,     0,     0,     0,     0,     0,     0,     0,     0,     0,\n","            0,     0,     0,     0,     0,     0,     0,     0,     0,     0,\n","            0,     0,     0,     0,     0,     0,     0,     0,     0,     0,\n","            0,     0,     0,     0,     0,     0,     0,     0,     0,     0,\n","            0,     0,     0,     0,     0,     0,     0,     0,     0,     0,\n","            0,     0,     0,     0,     0,     0,     0,     0,     0,     0,\n","            0,     0,     0,     0,     0,     0,     0,     0,     0,     0,\n","            0,     0,     0,     0,     0,     0,     0,     0,     0,     0,\n","            0,     0,     0,     0,     0,     0,     0,     0,     0,     0,\n","            0,     0,     0,     0,     0,     0,     0,     0,     0,     0,\n","            0,     0,     0,     0,     0,     0,     0,     0,     0,     0,\n","            0,     0,     0,     0,     0,     0,     0,     0,     0,     0,\n","            0,     0,     0,     0,     0,     0,     0,     0,     0,     0,\n","            0,     0,     0,     0,     0,     0,     0,     0,     0,     0,\n","            0,     0,     0,     0,     0,     0,     0,     0,     0,     0,\n","            0,     0,     0,     0,     0,     0,     0,     0,     0,     0,\n","            0,     0,     0,     0,     0,     0,     0,     0,     0,     0,\n","            0,     0,     0,     0,     0,     0,     0,     0,     0,     0,\n","            0,     0,     0,     0,     0,     0,     0,     0,     0,     0,\n","            0,     0,     0,     0,     0,     0,     0,     0,     0,     0,\n","            0,     0,     0,     0,     0,     0,     0,     0,     0,     0,\n","            0,     0,     0,     0,     0,     0,     0,     0,     0,     0,\n","            0,     0,     0,     0,     0,     0,     0,     0,     0,     0,\n","            0,     0,     0,     0,     0,     0,     0,     0,     0,     0,\n","            0,     0,     0,     0,     0,     0,     0,     0,     0,     0,\n","            0,     0,     0,     0,     0,     0,     0,     0,     0,     0,\n","            0,     0,     0,     0,     0,     0,     0,     0,     0,     0,\n","            0,     0,     0,     0,     0,     0,     0,     0,     0,     0,\n","            0,     0,     0,     0,     0,     0,     0,     0,     0,     0,\n","            0,     0,     0,     0,     0,     0,     0,     0,     0,     0,\n","            0,     0,     0,     0,     0,     0,     0,     0,     0,     0,\n","            0,     0,     0,     0,     0,     0,     0,     0,     0,     0,\n","            0,     0,     0,     0,     0,     0,     0,     0,     0,     0,\n","            0,     0,     0,     0,     0,     0,     0,     0,     0,     0,\n","            0,     0,     0,     0,     0,     0,     0,     0,     0,     0,\n","            0,     0,     0,     0,     0,     0,     0,     0,     0,     0,\n","            0,     0])\n","1,350 training samples\n","  150 validation samples\n"]},{"name":"stderr","output_type":"stream","text":["Some weights of BertForSequenceClassification were not initialized from the model checkpoint at emilyalsentzer/Bio_ClinicalBERT and are newly initialized: ['classifier.bias', 'classifier.weight']\n","You should probably TRAIN this model on a down-stream task to be able to use it for predictions and inference.\n"]},{"name":"stdout","output_type":"stream","text":["\n","======== Epoch 1 / 8 ========\n","Training...\n","  Batch    40  of     85.    Elapsed: 17.893152236938477.\n","  Batch    80  of     85.    Elapsed: 35.7610228061676.\n","\n","  Average training loss: 0.64\n","  Training epcoh took: 37.73134517669678\n","\n","Running Validation...\n","  Accuracy: 0.67\n","  Validation Loss: 0.61\n","  Validation took: 1.3801689147949219\n","\n","======== Epoch 2 / 8 ========\n","Training...\n","  Batch    40  of     85.    Elapsed: 17.929374933242798.\n","  Batch    80  of     85.    Elapsed: 35.84034466743469.\n","\n","  Average training loss: 0.57\n","  Training epcoh took: 37.81386733055115\n","\n","Running Validation...\n","  Accuracy: 0.65\n","  Validation Loss: 0.62\n","  Validation took: 1.3884186744689941\n","\n","======== Epoch 3 / 8 ========\n","Training...\n","  Batch    40  of     85.    Elapsed: 17.950350522994995.\n","  Batch    80  of     85.    Elapsed: 35.88019371032715.\n","\n","  Average training loss: 0.48\n","  Training epcoh took: 37.85993218421936\n","\n","Running Validation...\n","  Accuracy: 0.67\n","  Validation Loss: 0.61\n","  Validation took: 1.3896889686584473\n","\n","======== Epoch 4 / 8 ========\n","Training...\n","  Batch    40  of     85.    Elapsed: 17.92700695991516.\n","  Batch    80  of     85.    Elapsed: 35.866058349609375.\n","\n","  Average training loss: 0.35\n","  Training epcoh took: 37.83577513694763\n","\n","Running Validation...\n","  Accuracy: 0.59\n","  Validation Loss: 0.83\n","  Validation took: 1.3816940784454346\n","\n","======== Epoch 5 / 8 ========\n","Training...\n","  Batch    40  of     85.    Elapsed: 17.92715096473694.\n","  Batch    80  of     85.    Elapsed: 35.887277364730835.\n","\n","  Average training loss: 0.26\n","  Training epcoh took: 37.870410442352295\n","\n","Running Validation...\n","  Accuracy: 0.64\n","  Validation Loss: 0.76\n","  Validation took: 1.383131742477417\n","\n","======== Epoch 6 / 8 ========\n","Training...\n","  Batch    40  of     85.    Elapsed: 18.068864107131958.\n","  Batch    80  of     85.    Elapsed: 36.019309282302856.\n","\n","  Average training loss: 0.17\n","  Training epcoh took: 38.001758337020874\n","\n","Running Validation...\n","  Accuracy: 0.65\n","  Validation Loss: 0.92\n","  Validation took: 1.3874199390411377\n","\n","======== Epoch 7 / 8 ========\n","Training...\n","  Batch    40  of     85.    Elapsed: 17.932987928390503.\n","  Batch    80  of     85.    Elapsed: 35.86488890647888.\n","\n","  Average training loss: 0.12\n","  Training epcoh took: 37.858888149261475\n","\n","Running Validation...\n","  Accuracy: 0.64\n","  Validation Loss: 0.95\n","  Validation took: 1.3957412242889404\n","\n","======== Epoch 8 / 8 ========\n","Training...\n","  Batch    40  of     85.    Elapsed: 17.94513201713562.\n","  Batch    80  of     85.    Elapsed: 35.86774206161499.\n","\n","  Average training loss: 0.09\n","  Training epcoh took: 37.848642349243164\n","\n","Running Validation...\n","  Accuracy: 0.64\n","  Validation Loss: 1.01\n","  Validation took: 1.3896572589874268\n","\n","Training complete!\n","Total training took 313.93902015686035 (h:mm:ss)\n","Number of test sentences: 6,000\n","\n","1500\n","{'accuracy': 0.6610000133514404, 'f1': 0.6551373600959778, 'tnr': 0.687859296798706, 'tpr': 0.6348997950553894, 'mcc': 0.32310354709625244}\n","Original:  CONDITION UPDATE\n","D: PLEASE SEE CAREVUE FOR SPECIFICS. PT ADMITTED TO SICU FROM OR AROUND 1830- S/P SPINAL FUSION. INTUBATED ON PROPOFOL GTT FOR SEDATION AND NEO GTT TO MAINTAIN SBP\u003e100.\n","NEURO: PT LIGHTLY SEDATED ON PROPOFOL AT 50- 30 MCG/KG/MIN. ABLE TO OPEN EYES TO NAME, FOLLOWING COMMANDS AS SHIFT PROGRESSED. MAE ON BED ALTHOUGH LEFT LEG DOES APPEAR WEAKER. FENTANYL GTT STARTED FOR PAIN MANAGEMENT.\n","CV: T MAX 99.5. HR 85-105 NSR-ST. NEO GTT WEANED FROM 1.1MCG TO 0.5. UNABLE TO WEAN OFF SECONDARY TO LOWER BP WHEN SLEEPING. WHEN AWAKE, SBP \u003e100. MULTIPLE ATTEMPTS MADE BUT NEO ALWAYS RESTARTED AFTER 10-15 MINUTES.\n","RESP: BS CLEAR BUT DIMINISHED IN BASES. SX FOR SCANT AMT CLEAR FLUID. SATS\u003e100. REMAINS ON CMV WITH RATE OF 12, FIO2 50%\n","GI: NPO, ABD SOFT WITH + BS. ORAL-GASTRIC TUBE PATENT WITH BILIOUS DRAINAGE.\n","GU: LOW URINE X1 HR- TX'D WITH FLUID BOLUS, PRESENTLY HUO 60-80\n","ENDO: SLIDING SCALE INSULIN STARTED. BS 154-204\n","SKIN: PT NOTED TO HAVE ABRASION NEAR RIGHT EYE FROM TAPE USED DURING OR. ELASTOPLAST DRESSING ON BACK INTACT. LEFT FLANK DRESSING INTACT. HEMOVAC DRAINING 60-120CC BLOODY DRAINAGE Q2HRS.\n","HEME: HCT STABLE AT 32.6-33.4\n","A: HEMODYNAMICS AND RESP PARAMETERS MONITORED, SERIAL HCTS DONE, FENT GTT STARTED FOR PAIN MANAGEMENT, ATTEMPT TO WEAN NEO MAINTAINING SBP\u003e100\n","R: STABLE POST-OP, ? WEAN TO EXTUBATE THIS AM, NEO GTT SHOULD BE ABLE TO BE WEANED OFF AFTER EXTUBATION, CONT. TO MONITOR HCT.\n","\n","Token IDs: tensor([  101,  3879, 11984,   173,   131,  4268,  1267,  1920, 19038,  1111,\n","         2747,  1116,   119,   185,  1204,  4120,  1106, 16554,  1358,  1121,\n","         1137,  1213,  9200,   118,   188,   120,   185, 19245, 11970,   119,\n","         1107, 25098,  2913,  1113, 21146, 10008,  4063,   176,  3069,  1111,\n","        14516, 13759,  1105, 15242,   176,  3069,  1106,  4731,   188,  1830,\n","         1643,   135,  1620,   119, 24928, 11955,   131,   185,  1204,  7863,\n","        14516, 14459,  1113, 21146, 10008,  4063,  1120,  1851,   118,  1476,\n","          182,  1665,  1403,   120,  4023,   120, 11241,   119,  1682,  1106,\n","         1501,  1257,  1106,  1271,   117,  1378, 11443,  1112,  5212, 12687,\n","          119, 12477,  1162,  1113,  1908,  1780,  1286,  3420,  1674,  2845,\n","        16990,   119,   175,  3452, 18266,  1233,   176,  3069,  1408,  1111,\n","         2489,  2635,   119,   172,  1964,   131,   189, 12477,  1775,  4850,\n","          119,   126,   119,   177,  1197,  4859,   118,  8359,   183,  1116,\n","         1197,   118,   188,  1204,   119, 15242,   176,  3069,  1195,  6354,\n","         1181,  1121,   122,   119,   122,  1306,  1665,  1403,  1106,   121,\n","          119,   126,   119,  3372,  1106,  1195,  1389,  1228,  3718,  1106,\n","         2211,   171,  1643,  1165,  5575,   119,  1165,  8264,   117,   188,\n","         1830,  1643,   135,  1620,   119,  2967,  4021,  1189,  1133, 15242,\n","         1579, 27777,  1174,  1170,  1275,   118,  1405,  1904,   119,  1231,\n","        20080,   131,   171,  1116,  2330,  1133, 17017,  1107,  7616,   119,\n","          188,  1775,  1111, 14884,  1204,  1821,  1204,  2330,  8240,   119,\n","         2068,  1116,   135,  1620,   119,  2606,  1113,  3975,  1964,  1114,\n","         2603,  1104,  1367,   117, 20497,  1186,  1477,  1851,   110,   176,\n","         1182,   131,   183,  5674,   117,   170,  1830,  1181,  2991,  1114,\n","          116,   171,  1116,   119,  9619,   118,  3245, 11048,  7159,  8581,\n","         1114, 16516,  9436,  1361, 12779,   119,   176,  1358,   131,  1822,\n","        19968,   193,  1475,   177,  1197,   118,   189,  1775,   112,   173,\n","         1114,  8240,   171,  4063,  1361,   117, 16269,   177, 11848,  2539,\n","          118,  2908,  1322,  1186,   131,  7989,  3418, 26825,  1408,   119,\n","          171,  1116, 17733,   118, 21355,  2241,   131,   185,  1204,  2382,\n","         1106,  1138,   170,  6766,  8427,  1485,  1268,  2552,  1121,  6649,\n","         1215,  1219,  1137,   119,  8468, 12788,  4184, 19268, 11597,  1113,\n","         1171,  9964,   119,  1286, 12509, 11597,  9964,   119, 23123,  8625,\n","         1665, 21892,  2539,   118,  5356, 19515,  7201, 12779,   186,  1477,\n","         8167,  1116,   119, 23123,  1162,   131,   177,  5822,  6111,  1120,\n","         2724,   119,   127,   118,  3081,   119,   125,   170,   131, 23123,\n","        22320, 12881,  4724,  1105,  1231, 20080, 11934, 19232,   117,  7838,\n","          177, 15585,  1694,   117,   175,  3452,   176,  3069,  1408,  1111,\n","         2489,  2635,   117,  2661,  1106,  1195,  1389, 15242,  8338,   188,\n","         1830,  1643,   135,  1620,   187,   131,  6111,  2112,   118, 11769,\n","          117,   136,  1195,  1389,  1106,  4252, 25098,  2193,  1142,  1821,\n","          117, 15242,   176,  3069,  1431,  1129,  1682,  1106,  1129,  1195,\n","         6354,  1181,  1228,  1170,  4252, 25098,  1891,   117, 14255,  1204,\n","          119,  1106,  8804,   177,  5822,   119,   102,     0,     0,     0,\n","            0,     0,     0,     0,     0,     0,     0,     0,     0,     0,\n","            0,     0,     0,     0,     0,     0,     0,     0,     0,     0,\n","            0,     0,     0,     0,     0,     0,     0,     0,     0,     0,\n","            0,     0,     0,     0,     0,     0,     0,     0,     0,     0,\n","            0,     0,     0,     0,     0,     0,     0,     0,     0,     0,\n","            0,     0,     0,     0,     0,     0,     0,     0,     0,     0,\n","            0,     0,     0,     0,     0,     0,     0,     0,     0,     0,\n","            0,     0])\n","1,800 training samples\n","  200 validation samples\n"]},{"name":"stderr","output_type":"stream","text":["Some weights of BertForSequenceClassification were not initialized from the model checkpoint at emilyalsentzer/Bio_ClinicalBERT and are newly initialized: ['classifier.bias', 'classifier.weight']\n","You should probably TRAIN this model on a down-stream task to be able to use it for predictions and inference.\n"]},{"name":"stdout","output_type":"stream","text":["\n","======== Epoch 1 / 8 ========\n","Training...\n","  Batch    40  of    113.    Elapsed: 17.901743412017822.\n","  Batch    80  of    113.    Elapsed: 35.79721283912659.\n","\n","  Average training loss: 0.65\n","  Training epcoh took: 50.36432075500488\n","\n","Running Validation...\n","  Accuracy: 0.63\n","  Validation Loss: 0.63\n","  Validation took: 1.851879596710205\n","\n","======== Epoch 2 / 8 ========\n","Training...\n","  Batch    40  of    113.    Elapsed: 17.927807331085205.\n","  Batch    80  of    113.    Elapsed: 35.85129427909851.\n","\n","  Average training loss: 0.56\n","  Training epcoh took: 50.43520784378052\n","\n","Running Validation...\n","  Accuracy: 0.63\n","  Validation Loss: 0.60\n","  Validation took: 1.8454434871673584\n","\n","======== Epoch 3 / 8 ========\n","Training...\n","  Batch    40  of    113.    Elapsed: 17.946040153503418.\n","  Batch    80  of    113.    Elapsed: 35.84984302520752.\n","\n","  Average training loss: 0.47\n","  Training epcoh took: 50.42843508720398\n","\n","Running Validation...\n","  Accuracy: 0.63\n","  Validation Loss: 0.65\n","  Validation took: 1.845888614654541\n","\n","======== Epoch 4 / 8 ========\n","Training...\n","  Batch    40  of    113.    Elapsed: 17.94264268875122.\n","  Batch    80  of    113.    Elapsed: 35.8665075302124.\n","\n","  Average training loss: 0.36\n","  Training epcoh took: 50.443639039993286\n","\n","Running Validation...\n","  Accuracy: 0.67\n","  Validation Loss: 0.69\n","  Validation took: 1.841637372970581\n","\n","======== Epoch 5 / 8 ========\n","Training...\n","  Batch    40  of    113.    Elapsed: 17.92932891845703.\n","  Batch    80  of    113.    Elapsed: 35.86311149597168.\n","\n","  Average training loss: 0.27\n","  Training epcoh took: 50.440468549728394\n","\n","Running Validation...\n","  Accuracy: 0.63\n","  Validation Loss: 0.76\n","  Validation took: 1.8438811302185059\n","\n","======== Epoch 6 / 8 ========\n","Training...\n","  Batch    40  of    113.    Elapsed: 17.92884087562561.\n","  Batch    80  of    113.    Elapsed: 35.875534772872925.\n","\n","  Average training loss: 0.19\n","  Training epcoh took: 50.46758580207825\n","\n","Running Validation...\n","  Accuracy: 0.64\n","  Validation Loss: 0.83\n","  Validation took: 1.8405897617340088\n","\n","======== Epoch 7 / 8 ========\n","Training...\n","  Batch    40  of    113.    Elapsed: 17.916980028152466.\n","  Batch    80  of    113.    Elapsed: 35.8482460975647.\n","\n","  Average training loss: 0.14\n","  Training epcoh took: 50.43452858924866\n","\n","Running Validation...\n","  Accuracy: 0.63\n","  Validation Loss: 0.92\n","  Validation took: 1.8476943969726562\n","\n","======== Epoch 8 / 8 ========\n","Training...\n","  Batch    40  of    113.    Elapsed: 17.948973894119263.\n","  Batch    80  of    113.    Elapsed: 35.88763356208801.\n","\n","  Average training loss: 0.12\n","  Training epcoh took: 50.48288130760193\n","\n","Running Validation...\n","  Accuracy: 0.66\n","  Validation Loss: 0.93\n","  Validation took: 1.8539729118347168\n","\n","Training complete!\n","Total training took 418.2964942455292 (h:mm:ss)\n","Number of test sentences: 6,000\n","\n","2000\n","{'accuracy': 0.6980000138282776, 'f1': 0.7095223069190979, 'tnr': 0.667906641960144, 'tpr': 0.7272428274154663, 'mcc': 0.3959462344646454}\n","Original:  CONDITION UPDATE\n","D: PLEASE SEE CAREVUE FOR SPECIFICS. PT ADMITTED TO SICU FROM OR AROUND 1830- S/P SPINAL FUSION. INTUBATED ON PROPOFOL GTT FOR SEDATION AND NEO GTT TO MAINTAIN SBP\u003e100.\n","NEURO: PT LIGHTLY SEDATED ON PROPOFOL AT 50- 30 MCG/KG/MIN. ABLE TO OPEN EYES TO NAME, FOLLOWING COMMANDS AS SHIFT PROGRESSED. MAE ON BED ALTHOUGH LEFT LEG DOES APPEAR WEAKER. FENTANYL GTT STARTED FOR PAIN MANAGEMENT.\n","CV: T MAX 99.5. HR 85-105 NSR-ST. NEO GTT WEANED FROM 1.1MCG TO 0.5. UNABLE TO WEAN OFF SECONDARY TO LOWER BP WHEN SLEEPING. WHEN AWAKE, SBP \u003e100. MULTIPLE ATTEMPTS MADE BUT NEO ALWAYS RESTARTED AFTER 10-15 MINUTES.\n","RESP: BS CLEAR BUT DIMINISHED IN BASES. SX FOR SCANT AMT CLEAR FLUID. SATS\u003e100. REMAINS ON CMV WITH RATE OF 12, FIO2 50%\n","GI: NPO, ABD SOFT WITH + BS. ORAL-GASTRIC TUBE PATENT WITH BILIOUS DRAINAGE.\n","GU: LOW URINE X1 HR- TX'D WITH FLUID BOLUS, PRESENTLY HUO 60-80\n","ENDO: SLIDING SCALE INSULIN STARTED. BS 154-204\n","SKIN: PT NOTED TO HAVE ABRASION NEAR RIGHT EYE FROM TAPE USED DURING OR. ELASTOPLAST DRESSING ON BACK INTACT. LEFT FLANK DRESSING INTACT. HEMOVAC DRAINING 60-120CC BLOODY DRAINAGE Q2HRS.\n","HEME: HCT STABLE AT 32.6-33.4\n","A: HEMODYNAMICS AND RESP PARAMETERS MONITORED, SERIAL HCTS DONE, FENT GTT STARTED FOR PAIN MANAGEMENT, ATTEMPT TO WEAN NEO MAINTAINING SBP\u003e100\n","R: STABLE POST-OP, ? WEAN TO EXTUBATE THIS AM, NEO GTT SHOULD BE ABLE TO BE WEANED OFF AFTER EXTUBATION, CONT. TO MONITOR HCT.\n","\n","Token IDs: tensor([  101,  3879, 11984,   173,   131,  4268,  1267,  1920, 19038,  1111,\n","         2747,  1116,   119,   185,  1204,  4120,  1106, 16554,  1358,  1121,\n","         1137,  1213,  9200,   118,   188,   120,   185, 19245, 11970,   119,\n","         1107, 25098,  2913,  1113, 21146, 10008,  4063,   176,  3069,  1111,\n","        14516, 13759,  1105, 15242,   176,  3069,  1106,  4731,   188,  1830,\n","         1643,   135,  1620,   119, 24928, 11955,   131,   185,  1204,  7863,\n","        14516, 14459,  1113, 21146, 10008,  4063,  1120,  1851,   118,  1476,\n","          182,  1665,  1403,   120,  4023,   120, 11241,   119,  1682,  1106,\n","         1501,  1257,  1106,  1271,   117,  1378, 11443,  1112,  5212, 12687,\n","          119, 12477,  1162,  1113,  1908,  1780,  1286,  3420,  1674,  2845,\n","        16990,   119,   175,  3452, 18266,  1233,   176,  3069,  1408,  1111,\n","         2489,  2635,   119,   172,  1964,   131,   189, 12477,  1775,  4850,\n","          119,   126,   119,   177,  1197,  4859,   118,  8359,   183,  1116,\n","         1197,   118,   188,  1204,   119, 15242,   176,  3069,  1195,  6354,\n","         1181,  1121,   122,   119,   122,  1306,  1665,  1403,  1106,   121,\n","          119,   126,   119,  3372,  1106,  1195,  1389,  1228,  3718,  1106,\n","         2211,   171,  1643,  1165,  5575,   119,  1165,  8264,   117,   188,\n","         1830,  1643,   135,  1620,   119,  2967,  4021,  1189,  1133, 15242,\n","         1579, 27777,  1174,  1170,  1275,   118,  1405,  1904,   119,  1231,\n","        20080,   131,   171,  1116,  2330,  1133, 17017,  1107,  7616,   119,\n","          188,  1775,  1111, 14884,  1204,  1821,  1204,  2330,  8240,   119,\n","         2068,  1116,   135,  1620,   119,  2606,  1113,  3975,  1964,  1114,\n","         2603,  1104,  1367,   117, 20497,  1186,  1477,  1851,   110,   176,\n","         1182,   131,   183,  5674,   117,   170,  1830,  1181,  2991,  1114,\n","          116,   171,  1116,   119,  9619,   118,  3245, 11048,  7159,  8581,\n","         1114, 16516,  9436,  1361, 12779,   119,   176,  1358,   131,  1822,\n","        19968,   193,  1475,   177,  1197,   118,   189,  1775,   112,   173,\n","         1114,  8240,   171,  4063,  1361,   117, 16269,   177, 11848,  2539,\n","          118,  2908,  1322,  1186,   131,  7989,  3418, 26825,  1408,   119,\n","          171,  1116, 17733,   118, 21355,  2241,   131,   185,  1204,  2382,\n","         1106,  1138,   170,  6766,  8427,  1485,  1268,  2552,  1121,  6649,\n","         1215,  1219,  1137,   119,  8468, 12788,  4184, 19268, 11597,  1113,\n","         1171,  9964,   119,  1286, 12509, 11597,  9964,   119, 23123,  8625,\n","         1665, 21892,  2539,   118,  5356, 19515,  7201, 12779,   186,  1477,\n","         8167,  1116,   119, 23123,  1162,   131,   177,  5822,  6111,  1120,\n","         2724,   119,   127,   118,  3081,   119,   125,   170,   131, 23123,\n","        22320, 12881,  4724,  1105,  1231, 20080, 11934, 19232,   117,  7838,\n","          177, 15585,  1694,   117,   175,  3452,   176,  3069,  1408,  1111,\n","         2489,  2635,   117,  2661,  1106,  1195,  1389, 15242,  8338,   188,\n","         1830,  1643,   135,  1620,   187,   131,  6111,  2112,   118, 11769,\n","          117,   136,  1195,  1389,  1106,  4252, 25098,  2193,  1142,  1821,\n","          117, 15242,   176,  3069,  1431,  1129,  1682,  1106,  1129,  1195,\n","         6354,  1181,  1228,  1170,  4252, 25098,  1891,   117, 14255,  1204,\n","          119,  1106,  8804,   177,  5822,   119,   102,     0,     0,     0,\n","            0,     0,     0,     0,     0,     0,     0,     0,     0,     0,\n","            0,     0,     0,     0,     0,     0,     0,     0,     0,     0,\n","            0,     0,     0,     0,     0,     0,     0,     0,     0,     0,\n","            0,     0,     0,     0,     0,     0,     0,     0,     0,     0,\n","            0,     0,     0,     0,     0,     0,     0,     0,     0,     0,\n","            0,     0,     0,     0,     0,     0,     0,     0,     0,     0,\n","            0,     0,     0,     0,     0,     0,     0,     0,     0,     0,\n","            0,     0])\n","2,250 training samples\n","  250 validation samples\n"]},{"name":"stderr","output_type":"stream","text":["Some weights of BertForSequenceClassification were not initialized from the model checkpoint at emilyalsentzer/Bio_ClinicalBERT and are newly initialized: ['classifier.bias', 'classifier.weight']\n","You should probably TRAIN this model on a down-stream task to be able to use it for predictions and inference.\n"]},{"name":"stdout","output_type":"stream","text":["\n","======== Epoch 1 / 8 ========\n","Training...\n","  Batch    40  of    141.    Elapsed: 17.8810715675354.\n","  Batch    80  of    141.    Elapsed: 35.78798961639404.\n","  Batch   120  of    141.    Elapsed: 53.68695020675659.\n","\n","  Average training loss: 0.64\n","  Training epcoh took: 62.936495304107666\n","\n","Running Validation...\n","  Accuracy: 0.66\n","  Validation Loss: 0.60\n","  Validation took: 2.2988529205322266\n","\n","======== Epoch 2 / 8 ========\n","Training...\n","  Batch    40  of    141.    Elapsed: 17.94283413887024.\n","  Batch    80  of    141.    Elapsed: 35.88967204093933.\n","  Batch   120  of    141.    Elapsed: 53.82304072380066.\n","\n","  Average training loss: 0.56\n","  Training epcoh took: 63.10110902786255\n","\n","Running Validation...\n","  Accuracy: 0.67\n","  Validation Loss: 0.60\n","  Validation took: 2.3029868602752686\n","\n","======== Epoch 3 / 8 ========\n","Training...\n","  Batch    40  of    141.    Elapsed: 17.956645250320435.\n","  Batch    80  of    141.    Elapsed: 35.90085816383362.\n","  Batch   120  of    141.    Elapsed: 53.82576131820679.\n","\n","  Average training loss: 0.51\n","  Training epcoh took: 63.102394819259644\n","\n","Running Validation...\n","  Accuracy: 0.67\n","  Validation Loss: 0.59\n","  Validation took: 2.3037331104278564\n","\n","======== Epoch 4 / 8 ========\n","Training...\n","  Batch    40  of    141.    Elapsed: 17.931554317474365.\n","  Batch    80  of    141.    Elapsed: 35.8652400970459.\n","  Batch   120  of    141.    Elapsed: 53.81813430786133.\n","\n","  Average training loss: 0.40\n","  Training epcoh took: 63.079991579055786\n","\n","Running Validation...\n","  Accuracy: 0.70\n","  Validation Loss: 0.64\n","  Validation took: 2.3207268714904785\n","\n","======== Epoch 5 / 8 ========\n","Training...\n","  Batch    40  of    141.    Elapsed: 17.947903394699097.\n","  Batch    80  of    141.    Elapsed: 35.87936067581177.\n","  Batch   120  of    141.    Elapsed: 53.844502687454224.\n","\n","  Average training loss: 0.28\n","  Training epcoh took: 63.1017165184021\n","\n","Running Validation...\n","  Accuracy: 0.72\n","  Validation Loss: 0.74\n","  Validation took: 2.3104958534240723\n","\n","======== Epoch 6 / 8 ========\n","Training...\n","  Batch    40  of    141.    Elapsed: 17.953156232833862.\n","  Batch    80  of    141.    Elapsed: 35.898334980010986.\n","  Batch   120  of    141.    Elapsed: 53.823688983917236.\n","\n","  Average training loss: 0.20\n","  Training epcoh took: 63.08604454994202\n","\n","Running Validation...\n","  Accuracy: 0.67\n","  Validation Loss: 0.85\n","  Validation took: 2.3025665283203125\n","\n","======== Epoch 7 / 8 ========\n","Training...\n","  Batch    40  of    141.    Elapsed: 17.950241804122925.\n","  Batch    80  of    141.    Elapsed: 35.9086754322052.\n","  Batch   120  of    141.    Elapsed: 53.84015107154846.\n","\n","  Average training loss: 0.14\n","  Training epcoh took: 63.10017251968384\n","\n","Running Validation...\n","  Accuracy: 0.71\n","  Validation Loss: 0.93\n","  Validation took: 2.3096468448638916\n","\n","======== Epoch 8 / 8 ========\n","Training...\n","  Batch    40  of    141.    Elapsed: 17.954049587249756.\n","  Batch    80  of    141.    Elapsed: 35.882001876831055.\n","  Batch   120  of    141.    Elapsed: 53.79910707473755.\n","\n","  Average training loss: 0.10\n","  Training epcoh took: 63.04677629470825\n","\n","Running Validation...\n","  Accuracy: 0.70\n","  Validation Loss: 0.96\n","  Validation took: 2.2976415157318115\n","\n","Training complete!\n","Total training took 523.0190873146057 (h:mm:ss)\n","Number of test sentences: 6,000\n","\n","2500\n","{'accuracy': 0.6961666941642761, 'f1': 0.696722686290741, 'tnr': 0.7044301629066467, 'tpr': 0.6881366968154907, 'mcc': 0.39254888892173767}\n","Original:  CONDITION UPDATE\n","D: PLEASE SEE CAREVUE FOR SPECIFICS. PT ADMITTED TO SICU FROM OR AROUND 1830- S/P SPINAL FUSION. INTUBATED ON PROPOFOL GTT FOR SEDATION AND NEO GTT TO MAINTAIN SBP\u003e100.\n","NEURO: PT LIGHTLY SEDATED ON PROPOFOL AT 50- 30 MCG/KG/MIN. ABLE TO OPEN EYES TO NAME, FOLLOWING COMMANDS AS SHIFT PROGRESSED. MAE ON BED ALTHOUGH LEFT LEG DOES APPEAR WEAKER. FENTANYL GTT STARTED FOR PAIN MANAGEMENT.\n","CV: T MAX 99.5. HR 85-105 NSR-ST. NEO GTT WEANED FROM 1.1MCG TO 0.5. UNABLE TO WEAN OFF SECONDARY TO LOWER BP WHEN SLEEPING. WHEN AWAKE, SBP \u003e100. MULTIPLE ATTEMPTS MADE BUT NEO ALWAYS RESTARTED AFTER 10-15 MINUTES.\n","RESP: BS CLEAR BUT DIMINISHED IN BASES. SX FOR SCANT AMT CLEAR FLUID. SATS\u003e100. REMAINS ON CMV WITH RATE OF 12, FIO2 50%\n","GI: NPO, ABD SOFT WITH + BS. ORAL-GASTRIC TUBE PATENT WITH BILIOUS DRAINAGE.\n","GU: LOW URINE X1 HR- TX'D WITH FLUID BOLUS, PRESENTLY HUO 60-80\n","ENDO: SLIDING SCALE INSULIN STARTED. BS 154-204\n","SKIN: PT NOTED TO HAVE ABRASION NEAR RIGHT EYE FROM TAPE USED DURING OR. ELASTOPLAST DRESSING ON BACK INTACT. LEFT FLANK DRESSING INTACT. HEMOVAC DRAINING 60-120CC BLOODY DRAINAGE Q2HRS.\n","HEME: HCT STABLE AT 32.6-33.4\n","A: HEMODYNAMICS AND RESP PARAMETERS MONITORED, SERIAL HCTS DONE, FENT GTT STARTED FOR PAIN MANAGEMENT, ATTEMPT TO WEAN NEO MAINTAINING SBP\u003e100\n","R: STABLE POST-OP, ? WEAN TO EXTUBATE THIS AM, NEO GTT SHOULD BE ABLE TO BE WEANED OFF AFTER EXTUBATION, CONT. TO MONITOR HCT.\n","\n","Token IDs: tensor([  101,  3879, 11984,   173,   131,  4268,  1267,  1920, 19038,  1111,\n","         2747,  1116,   119,   185,  1204,  4120,  1106, 16554,  1358,  1121,\n","         1137,  1213,  9200,   118,   188,   120,   185, 19245, 11970,   119,\n","         1107, 25098,  2913,  1113, 21146, 10008,  4063,   176,  3069,  1111,\n","        14516, 13759,  1105, 15242,   176,  3069,  1106,  4731,   188,  1830,\n","         1643,   135,  1620,   119, 24928, 11955,   131,   185,  1204,  7863,\n","        14516, 14459,  1113, 21146, 10008,  4063,  1120,  1851,   118,  1476,\n","          182,  1665,  1403,   120,  4023,   120, 11241,   119,  1682,  1106,\n","         1501,  1257,  1106,  1271,   117,  1378, 11443,  1112,  5212, 12687,\n","          119, 12477,  1162,  1113,  1908,  1780,  1286,  3420,  1674,  2845,\n","        16990,   119,   175,  3452, 18266,  1233,   176,  3069,  1408,  1111,\n","         2489,  2635,   119,   172,  1964,   131,   189, 12477,  1775,  4850,\n","          119,   126,   119,   177,  1197,  4859,   118,  8359,   183,  1116,\n","         1197,   118,   188,  1204,   119, 15242,   176,  3069,  1195,  6354,\n","         1181,  1121,   122,   119,   122,  1306,  1665,  1403,  1106,   121,\n","          119,   126,   119,  3372,  1106,  1195,  1389,  1228,  3718,  1106,\n","         2211,   171,  1643,  1165,  5575,   119,  1165,  8264,   117,   188,\n","         1830,  1643,   135,  1620,   119,  2967,  4021,  1189,  1133, 15242,\n","         1579, 27777,  1174,  1170,  1275,   118,  1405,  1904,   119,  1231,\n","        20080,   131,   171,  1116,  2330,  1133, 17017,  1107,  7616,   119,\n","          188,  1775,  1111, 14884,  1204,  1821,  1204,  2330,  8240,   119,\n","         2068,  1116,   135,  1620,   119,  2606,  1113,  3975,  1964,  1114,\n","         2603,  1104,  1367,   117, 20497,  1186,  1477,  1851,   110,   176,\n","         1182,   131,   183,  5674,   117,   170,  1830,  1181,  2991,  1114,\n","          116,   171,  1116,   119,  9619,   118,  3245, 11048,  7159,  8581,\n","         1114, 16516,  9436,  1361, 12779,   119,   176,  1358,   131,  1822,\n","        19968,   193,  1475,   177,  1197,   118,   189,  1775,   112,   173,\n","         1114,  8240,   171,  4063,  1361,   117, 16269,   177, 11848,  2539,\n","          118,  2908,  1322,  1186,   131,  7989,  3418, 26825,  1408,   119,\n","          171,  1116, 17733,   118, 21355,  2241,   131,   185,  1204,  2382,\n","         1106,  1138,   170,  6766,  8427,  1485,  1268,  2552,  1121,  6649,\n","         1215,  1219,  1137,   119,  8468, 12788,  4184, 19268, 11597,  1113,\n","         1171,  9964,   119,  1286, 12509, 11597,  9964,   119, 23123,  8625,\n","         1665, 21892,  2539,   118,  5356, 19515,  7201, 12779,   186,  1477,\n","         8167,  1116,   119, 23123,  1162,   131,   177,  5822,  6111,  1120,\n","         2724,   119,   127,   118,  3081,   119,   125,   170,   131, 23123,\n","        22320, 12881,  4724,  1105,  1231, 20080, 11934, 19232,   117,  7838,\n","          177, 15585,  1694,   117,   175,  3452,   176,  3069,  1408,  1111,\n","         2489,  2635,   117,  2661,  1106,  1195,  1389, 15242,  8338,   188,\n","         1830,  1643,   135,  1620,   187,   131,  6111,  2112,   118, 11769,\n","          117,   136,  1195,  1389,  1106,  4252, 25098,  2193,  1142,  1821,\n","          117, 15242,   176,  3069,  1431,  1129,  1682,  1106,  1129,  1195,\n","         6354,  1181,  1228,  1170,  4252, 25098,  1891,   117, 14255,  1204,\n","          119,  1106,  8804,   177,  5822,   119,   102,     0,     0,     0,\n","            0,     0,     0,     0,     0,     0,     0,     0,     0,     0,\n","            0,     0,     0,     0,     0,     0,     0,     0,     0,     0,\n","            0,     0,     0,     0,     0,     0,     0,     0,     0,     0,\n","            0,     0,     0,     0,     0,     0,     0,     0,     0,     0,\n","            0,     0,     0,     0,     0,     0,     0,     0,     0,     0,\n","            0,     0,     0,     0,     0,     0,     0,     0,     0,     0,\n","            0,     0,     0,     0,     0,     0,     0,     0,     0,     0,\n","            0,     0])\n","2,700 training samples\n","  300 validation samples\n"]},{"name":"stderr","output_type":"stream","text":["Some weights of BertForSequenceClassification were not initialized from the model checkpoint at emilyalsentzer/Bio_ClinicalBERT and are newly initialized: ['classifier.bias', 'classifier.weight']\n","You should probably TRAIN this model on a down-stream task to be able to use it for predictions and inference.\n"]},{"name":"stdout","output_type":"stream","text":["\n","======== Epoch 1 / 8 ========\n","Training...\n","  Batch    40  of    169.    Elapsed: 17.843272924423218.\n","  Batch    80  of    169.    Elapsed: 35.705580949783325.\n","  Batch   120  of    169.    Elapsed: 53.617334842681885.\n","  Batch   160  of    169.    Elapsed: 71.52874183654785.\n","\n","  Average training loss: 0.63\n","  Training epcoh took: 75.44956398010254\n","\n","Running Validation...\n","  Accuracy: 0.64\n","  Validation Loss: 0.62\n","  Validation took: 2.7653045654296875\n","\n","======== Epoch 2 / 8 ========\n","Training...\n","  Batch    40  of    169.    Elapsed: 17.91304659843445.\n","  Batch    80  of    169.    Elapsed: 35.84799337387085.\n","  Batch   120  of    169.    Elapsed: 53.769665241241455.\n","  Batch   160  of    169.    Elapsed: 71.67843890190125.\n","\n","  Average training loss: 0.54\n","  Training epcoh took: 75.59719014167786\n","\n","Running Validation...\n","  Accuracy: 0.67\n","  Validation Loss: 0.60\n","  Validation took: 2.766613721847534\n","\n","======== Epoch 3 / 8 ========\n","Training...\n","  Batch    40  of    169.    Elapsed: 17.92220687866211.\n","  Batch    80  of    169.    Elapsed: 35.82296180725098.\n","  Batch   120  of    169.    Elapsed: 53.718284130096436.\n","  Batch   160  of    169.    Elapsed: 71.64319682121277.\n","\n","  Average training loss: 0.45\n","  Training epcoh took: 75.55915117263794\n","\n","Running Validation...\n","  Accuracy: 0.68\n","  Validation Loss: 0.64\n","  Validation took: 2.750345230102539\n","\n","======== Epoch 4 / 8 ========\n","Training...\n","  Batch    40  of    169.    Elapsed: 17.93099570274353.\n","  Batch    80  of    169.    Elapsed: 35.84912085533142.\n","  Batch   120  of    169.    Elapsed: 53.74903392791748.\n","  Batch   160  of    169.    Elapsed: 71.69171929359436.\n","\n","  Average training loss: 0.34\n","  Training epcoh took: 75.60755681991577\n","\n","Running Validation...\n","  Accuracy: 0.69\n","  Validation Loss: 0.69\n","  Validation took: 2.757765293121338\n","\n","======== Epoch 5 / 8 ========\n","Training...\n","  Batch    40  of    169.    Elapsed: 17.91913938522339.\n","  Batch    80  of    169.    Elapsed: 35.83490324020386.\n","  Batch   120  of    169.    Elapsed: 53.74093556404114.\n","  Batch   160  of    169.    Elapsed: 71.6723837852478.\n","\n","  Average training loss: 0.23\n","  Training epcoh took: 75.59293794631958\n","\n","Running Validation...\n","  Accuracy: 0.68\n","  Validation Loss: 0.86\n","  Validation took: 2.75492787361145\n","\n","======== Epoch 6 / 8 ========\n","Training...\n","  Batch    40  of    169.    Elapsed: 17.918448209762573.\n","  Batch    80  of    169.    Elapsed: 35.83048748970032.\n","  Batch   120  of    169.    Elapsed: 53.72812223434448.\n","  Batch   160  of    169.    Elapsed: 71.63718962669373.\n","\n","  Average training loss: 0.17\n","  Training epcoh took: 75.55568861961365\n","\n","Running Validation...\n","  Accuracy: 0.70\n","  Validation Loss: 0.93\n","  Validation took: 2.764150857925415\n","\n","======== Epoch 7 / 8 ========\n","Training...\n","  Batch    40  of    169.    Elapsed: 17.92408013343811.\n","  Batch    80  of    169.    Elapsed: 35.83772611618042.\n","  Batch   120  of    169.    Elapsed: 53.74845814704895.\n","  Batch   160  of    169.    Elapsed: 71.67084527015686.\n","\n","  Average training loss: 0.13\n","  Training epcoh took: 75.60475420951843\n","\n","Running Validation...\n","  Accuracy: 0.71\n","  Validation Loss: 1.03\n","  Validation took: 2.75577712059021\n","\n","======== Epoch 8 / 8 ========\n","Training...\n","  Batch    40  of    169.    Elapsed: 17.89767813682556.\n","  Batch    80  of    169.    Elapsed: 35.83111047744751.\n","  Batch   120  of    169.    Elapsed: 53.76196527481079.\n","  Batch   160  of    169.    Elapsed: 71.67331314086914.\n","\n","  Average training loss: 0.10\n","  Training epcoh took: 75.61148047447205\n","\n","Running Validation...\n","  Accuracy: 0.72\n","  Validation Loss: 1.02\n","  Validation took: 2.7620151042938232\n","\n","Training complete!\n","Total training took 626.6650745868683 (h:mm:ss)\n","Number of test sentences: 6,000\n","\n","3000\n","{'accuracy': 0.7138333320617676, 'f1': 0.7084394693374634, 'tnr': 0.7429827451705933, 'tpr': 0.6855077147483826, 'mcc': 0.4290120601654053}\n","Original:  CONDITION UPDATE\n","D: PLEASE SEE CAREVUE FOR SPECIFICS. PT ADMITTED TO SICU FROM OR AROUND 1830- S/P SPINAL FUSION. INTUBATED ON PROPOFOL GTT FOR SEDATION AND NEO GTT TO MAINTAIN SBP\u003e100.\n","NEURO: PT LIGHTLY SEDATED ON PROPOFOL AT 50- 30 MCG/KG/MIN. ABLE TO OPEN EYES TO NAME, FOLLOWING COMMANDS AS SHIFT PROGRESSED. MAE ON BED ALTHOUGH LEFT LEG DOES APPEAR WEAKER. FENTANYL GTT STARTED FOR PAIN MANAGEMENT.\n","CV: T MAX 99.5. HR 85-105 NSR-ST. NEO GTT WEANED FROM 1.1MCG TO 0.5. UNABLE TO WEAN OFF SECONDARY TO LOWER BP WHEN SLEEPING. WHEN AWAKE, SBP \u003e100. MULTIPLE ATTEMPTS MADE BUT NEO ALWAYS RESTARTED AFTER 10-15 MINUTES.\n","RESP: BS CLEAR BUT DIMINISHED IN BASES. SX FOR SCANT AMT CLEAR FLUID. SATS\u003e100. REMAINS ON CMV WITH RATE OF 12, FIO2 50%\n","GI: NPO, ABD SOFT WITH + BS. ORAL-GASTRIC TUBE PATENT WITH BILIOUS DRAINAGE.\n","GU: LOW URINE X1 HR- TX'D WITH FLUID BOLUS, PRESENTLY HUO 60-80\n","ENDO: SLIDING SCALE INSULIN STARTED. BS 154-204\n","SKIN: PT NOTED TO HAVE ABRASION NEAR RIGHT EYE FROM TAPE USED DURING OR. ELASTOPLAST DRESSING ON BACK INTACT. LEFT FLANK DRESSING INTACT. HEMOVAC DRAINING 60-120CC BLOODY DRAINAGE Q2HRS.\n","HEME: HCT STABLE AT 32.6-33.4\n","A: HEMODYNAMICS AND RESP PARAMETERS MONITORED, SERIAL HCTS DONE, FENT GTT STARTED FOR PAIN MANAGEMENT, ATTEMPT TO WEAN NEO MAINTAINING SBP\u003e100\n","R: STABLE POST-OP, ? WEAN TO EXTUBATE THIS AM, NEO GTT SHOULD BE ABLE TO BE WEANED OFF AFTER EXTUBATION, CONT. TO MONITOR HCT.\n","\n","Token IDs: tensor([  101,  3879, 11984,   173,   131,  4268,  1267,  1920, 19038,  1111,\n","         2747,  1116,   119,   185,  1204,  4120,  1106, 16554,  1358,  1121,\n","         1137,  1213,  9200,   118,   188,   120,   185, 19245, 11970,   119,\n","         1107, 25098,  2913,  1113, 21146, 10008,  4063,   176,  3069,  1111,\n","        14516, 13759,  1105, 15242,   176,  3069,  1106,  4731,   188,  1830,\n","         1643,   135,  1620,   119, 24928, 11955,   131,   185,  1204,  7863,\n","        14516, 14459,  1113, 21146, 10008,  4063,  1120,  1851,   118,  1476,\n","          182,  1665,  1403,   120,  4023,   120, 11241,   119,  1682,  1106,\n","         1501,  1257,  1106,  1271,   117,  1378, 11443,  1112,  5212, 12687,\n","          119, 12477,  1162,  1113,  1908,  1780,  1286,  3420,  1674,  2845,\n","        16990,   119,   175,  3452, 18266,  1233,   176,  3069,  1408,  1111,\n","         2489,  2635,   119,   172,  1964,   131,   189, 12477,  1775,  4850,\n","          119,   126,   119,   177,  1197,  4859,   118,  8359,   183,  1116,\n","         1197,   118,   188,  1204,   119, 15242,   176,  3069,  1195,  6354,\n","         1181,  1121,   122,   119,   122,  1306,  1665,  1403,  1106,   121,\n","          119,   126,   119,  3372,  1106,  1195,  1389,  1228,  3718,  1106,\n","         2211,   171,  1643,  1165,  5575,   119,  1165,  8264,   117,   188,\n","         1830,  1643,   135,  1620,   119,  2967,  4021,  1189,  1133, 15242,\n","         1579, 27777,  1174,  1170,  1275,   118,  1405,  1904,   119,  1231,\n","        20080,   131,   171,  1116,  2330,  1133, 17017,  1107,  7616,   119,\n","          188,  1775,  1111, 14884,  1204,  1821,  1204,  2330,  8240,   119,\n","         2068,  1116,   135,  1620,   119,  2606,  1113,  3975,  1964,  1114,\n","         2603,  1104,  1367,   117, 20497,  1186,  1477,  1851,   110,   176,\n","         1182,   131,   183,  5674,   117,   170,  1830,  1181,  2991,  1114,\n","          116,   171,  1116,   119,  9619,   118,  3245, 11048,  7159,  8581,\n","         1114, 16516,  9436,  1361, 12779,   119,   176,  1358,   131,  1822,\n","        19968,   193,  1475,   177,  1197,   118,   189,  1775,   112,   173,\n","         1114,  8240,   171,  4063,  1361,   117, 16269,   177, 11848,  2539,\n","          118,  2908,  1322,  1186,   131,  7989,  3418, 26825,  1408,   119,\n","          171,  1116, 17733,   118, 21355,  2241,   131,   185,  1204,  2382,\n","         1106,  1138,   170,  6766,  8427,  1485,  1268,  2552,  1121,  6649,\n","         1215,  1219,  1137,   119,  8468, 12788,  4184, 19268, 11597,  1113,\n","         1171,  9964,   119,  1286, 12509, 11597,  9964,   119, 23123,  8625,\n","         1665, 21892,  2539,   118,  5356, 19515,  7201, 12779,   186,  1477,\n","         8167,  1116,   119, 23123,  1162,   131,   177,  5822,  6111,  1120,\n","         2724,   119,   127,   118,  3081,   119,   125,   170,   131, 23123,\n","        22320, 12881,  4724,  1105,  1231, 20080, 11934, 19232,   117,  7838,\n","          177, 15585,  1694,   117,   175,  3452,   176,  3069,  1408,  1111,\n","         2489,  2635,   117,  2661,  1106,  1195,  1389, 15242,  8338,   188,\n","         1830,  1643,   135,  1620,   187,   131,  6111,  2112,   118, 11769,\n","          117,   136,  1195,  1389,  1106,  4252, 25098,  2193,  1142,  1821,\n","          117, 15242,   176,  3069,  1431,  1129,  1682,  1106,  1129,  1195,\n","         6354,  1181,  1228,  1170,  4252, 25098,  1891,   117, 14255,  1204,\n","          119,  1106,  8804,   177,  5822,   119,   102,     0,     0,     0,\n","            0,     0,     0,     0,     0,     0,     0,     0,     0,     0,\n","            0,     0,     0,     0,     0,     0,     0,     0,     0,     0,\n","            0,     0,     0,     0,     0,     0,     0,     0,     0,     0,\n","            0,     0,     0,     0,     0,     0,     0,     0,     0,     0,\n","            0,     0,     0,     0,     0,     0,     0,     0,     0,     0,\n","            0,     0,     0,     0,     0,     0,     0,     0,     0,     0,\n","            0,     0,     0,     0,     0,     0,     0,     0,     0,     0,\n","            0,     0])\n","3,150 training samples\n","  350 validation samples\n"]},{"name":"stderr","output_type":"stream","text":["Some weights of BertForSequenceClassification were not initialized from the model checkpoint at emilyalsentzer/Bio_ClinicalBERT and are newly initialized: ['classifier.bias', 'classifier.weight']\n","You should probably TRAIN this model on a down-stream task to be able to use it for predictions and inference.\n"]},{"name":"stdout","output_type":"stream","text":["\n","======== Epoch 1 / 8 ========\n","Training...\n","  Batch    40  of    197.    Elapsed: 17.850001096725464.\n","  Batch    80  of    197.    Elapsed: 35.7263867855072.\n","  Batch   120  of    197.    Elapsed: 53.63400936126709.\n","  Batch   160  of    197.    Elapsed: 71.53366208076477.\n","\n","  Average training loss: 0.63\n","  Training epcoh took: 88.06856203079224\n","\n","Running Validation...\n","  Accuracy: 0.65\n","  Validation Loss: 0.57\n","  Validation took: 3.228684186935425\n","\n","======== Epoch 2 / 8 ========\n","Training...\n","  Batch    40  of    197.    Elapsed: 17.919413328170776.\n","  Batch    80  of    197.    Elapsed: 35.83256983757019.\n","  Batch   120  of    197.    Elapsed: 53.75912952423096.\n","  Batch   160  of    197.    Elapsed: 71.6845633983612.\n","\n","  Average training loss: 0.53\n","  Training epcoh took: 88.20947790145874\n","\n","Running Validation...\n","  Accuracy: 0.70\n","  Validation Loss: 0.57\n","  Validation took: 3.2358689308166504\n","\n","======== Epoch 3 / 8 ========\n","Training...\n","  Batch    40  of    197.    Elapsed: 17.922062158584595.\n","  Batch    80  of    197.    Elapsed: 35.83626914024353.\n","  Batch   120  of    197.    Elapsed: 53.7693772315979.\n","  Batch   160  of    197.    Elapsed: 71.6973192691803.\n","\n","  Average training loss: 0.42\n","  Training epcoh took: 88.23400282859802\n","\n","Running Validation...\n","  Accuracy: 0.70\n","  Validation Loss: 0.63\n","  Validation took: 3.2236976623535156\n","\n","======== Epoch 4 / 8 ========\n","Training...\n","  Batch    40  of    197.    Elapsed: 17.925848722457886.\n","  Batch    80  of    197.    Elapsed: 35.835580348968506.\n","  Batch   120  of    197.    Elapsed: 53.763099908828735.\n","  Batch   160  of    197.    Elapsed: 71.68767929077148.\n","\n","  Average training loss: 0.31\n","  Training epcoh took: 88.23240613937378\n","\n","Running Validation...\n","  Accuracy: 0.72\n","  Validation Loss: 0.64\n","  Validation took: 3.2154502868652344\n","\n","======== Epoch 5 / 8 ========\n","Training...\n","  Batch    40  of    197.    Elapsed: 17.938688278198242.\n","  Batch    80  of    197.    Elapsed: 35.85379910469055.\n","  Batch   120  of    197.    Elapsed: 53.78308033943176.\n","  Batch   160  of    197.    Elapsed: 71.714035987854.\n","\n","  Average training loss: 0.24\n","  Training epcoh took: 88.24235320091248\n","\n","Running Validation...\n","  Accuracy: 0.71\n","  Validation Loss: 0.77\n","  Validation took: 3.214099407196045\n","\n","======== Epoch 6 / 8 ========\n","Training...\n","  Batch    40  of    197.    Elapsed: 17.95296835899353.\n","  Batch    80  of    197.    Elapsed: 35.87787079811096.\n","  Batch   120  of    197.    Elapsed: 53.79141163825989.\n","  Batch   160  of    197.    Elapsed: 71.7202467918396.\n","\n","  Average training loss: 0.17\n","  Training epcoh took: 88.2644772529602\n","\n","Running Validation...\n","  Accuracy: 0.72\n","  Validation Loss: 0.84\n","  Validation took: 3.2119338512420654\n","\n","======== Epoch 7 / 8 ========\n","Training...\n","  Batch    40  of    197.    Elapsed: 17.921760320663452.\n","  Batch    80  of    197.    Elapsed: 35.84771943092346.\n","  Batch   120  of    197.    Elapsed: 53.765453815460205.\n","  Batch   160  of    197.    Elapsed: 71.68511843681335.\n","\n","  Average training loss: 0.13\n","  Training epcoh took: 88.23137474060059\n","\n","Running Validation...\n","  Accuracy: 0.74\n","  Validation Loss: 0.94\n","  Validation took: 3.2142999172210693\n","\n","======== Epoch 8 / 8 ========\n","Training...\n","  Batch    40  of    197.    Elapsed: 17.919933319091797.\n","  Batch    80  of    197.    Elapsed: 35.84565281867981.\n","  Batch   120  of    197.    Elapsed: 53.75477933883667.\n","  Batch   160  of    197.    Elapsed: 71.68038725852966.\n","\n","  Average training loss: 0.10\n","  Training epcoh took: 88.22414469718933\n","\n","Running Validation...\n","  Accuracy: 0.72\n","  Validation Loss: 0.98\n","  Validation took: 3.218043088912964\n","\n","Training complete!\n","Total training took 731.4847383499146 (h:mm:ss)\n","Number of test sentences: 6,000\n","\n","3500\n","{'accuracy': 0.7319999933242798, 'f1': 0.7293840646743774, 'tnr': 0.7524518370628357, 'tpr': 0.7121261954307556, 'mcc': 0.46479377150535583}\n","Original:  CONDITION UPDATE\n","D: PLEASE SEE CAREVUE FOR SPECIFICS. PT ADMITTED TO SICU FROM OR AROUND 1830- S/P SPINAL FUSION. INTUBATED ON PROPOFOL GTT FOR SEDATION AND NEO GTT TO MAINTAIN SBP\u003e100.\n","NEURO: PT LIGHTLY SEDATED ON PROPOFOL AT 50- 30 MCG/KG/MIN. ABLE TO OPEN EYES TO NAME, FOLLOWING COMMANDS AS SHIFT PROGRESSED. MAE ON BED ALTHOUGH LEFT LEG DOES APPEAR WEAKER. FENTANYL GTT STARTED FOR PAIN MANAGEMENT.\n","CV: T MAX 99.5. HR 85-105 NSR-ST. NEO GTT WEANED FROM 1.1MCG TO 0.5. UNABLE TO WEAN OFF SECONDARY TO LOWER BP WHEN SLEEPING. WHEN AWAKE, SBP \u003e100. MULTIPLE ATTEMPTS MADE BUT NEO ALWAYS RESTARTED AFTER 10-15 MINUTES.\n","RESP: BS CLEAR BUT DIMINISHED IN BASES. SX FOR SCANT AMT CLEAR FLUID. SATS\u003e100. REMAINS ON CMV WITH RATE OF 12, FIO2 50%\n","GI: NPO, ABD SOFT WITH + BS. ORAL-GASTRIC TUBE PATENT WITH BILIOUS DRAINAGE.\n","GU: LOW URINE X1 HR- TX'D WITH FLUID BOLUS, PRESENTLY HUO 60-80\n","ENDO: SLIDING SCALE INSULIN STARTED. BS 154-204\n","SKIN: PT NOTED TO HAVE ABRASION NEAR RIGHT EYE FROM TAPE USED DURING OR. ELASTOPLAST DRESSING ON BACK INTACT. LEFT FLANK DRESSING INTACT. HEMOVAC DRAINING 60-120CC BLOODY DRAINAGE Q2HRS.\n","HEME: HCT STABLE AT 32.6-33.4\n","A: HEMODYNAMICS AND RESP PARAMETERS MONITORED, SERIAL HCTS DONE, FENT GTT STARTED FOR PAIN MANAGEMENT, ATTEMPT TO WEAN NEO MAINTAINING SBP\u003e100\n","R: STABLE POST-OP, ? WEAN TO EXTUBATE THIS AM, NEO GTT SHOULD BE ABLE TO BE WEANED OFF AFTER EXTUBATION, CONT. TO MONITOR HCT.\n","\n","Token IDs: tensor([  101,  3879, 11984,   173,   131,  4268,  1267,  1920, 19038,  1111,\n","         2747,  1116,   119,   185,  1204,  4120,  1106, 16554,  1358,  1121,\n","         1137,  1213,  9200,   118,   188,   120,   185, 19245, 11970,   119,\n","         1107, 25098,  2913,  1113, 21146, 10008,  4063,   176,  3069,  1111,\n","        14516, 13759,  1105, 15242,   176,  3069,  1106,  4731,   188,  1830,\n","         1643,   135,  1620,   119, 24928, 11955,   131,   185,  1204,  7863,\n","        14516, 14459,  1113, 21146, 10008,  4063,  1120,  1851,   118,  1476,\n","          182,  1665,  1403,   120,  4023,   120, 11241,   119,  1682,  1106,\n","         1501,  1257,  1106,  1271,   117,  1378, 11443,  1112,  5212, 12687,\n","          119, 12477,  1162,  1113,  1908,  1780,  1286,  3420,  1674,  2845,\n","        16990,   119,   175,  3452, 18266,  1233,   176,  3069,  1408,  1111,\n","         2489,  2635,   119,   172,  1964,   131,   189, 12477,  1775,  4850,\n","          119,   126,   119,   177,  1197,  4859,   118,  8359,   183,  1116,\n","         1197,   118,   188,  1204,   119, 15242,   176,  3069,  1195,  6354,\n","         1181,  1121,   122,   119,   122,  1306,  1665,  1403,  1106,   121,\n","          119,   126,   119,  3372,  1106,  1195,  1389,  1228,  3718,  1106,\n","         2211,   171,  1643,  1165,  5575,   119,  1165,  8264,   117,   188,\n","         1830,  1643,   135,  1620,   119,  2967,  4021,  1189,  1133, 15242,\n","         1579, 27777,  1174,  1170,  1275,   118,  1405,  1904,   119,  1231,\n","        20080,   131,   171,  1116,  2330,  1133, 17017,  1107,  7616,   119,\n","          188,  1775,  1111, 14884,  1204,  1821,  1204,  2330,  8240,   119,\n","         2068,  1116,   135,  1620,   119,  2606,  1113,  3975,  1964,  1114,\n","         2603,  1104,  1367,   117, 20497,  1186,  1477,  1851,   110,   176,\n","         1182,   131,   183,  5674,   117,   170,  1830,  1181,  2991,  1114,\n","          116,   171,  1116,   119,  9619,   118,  3245, 11048,  7159,  8581,\n","         1114, 16516,  9436,  1361, 12779,   119,   176,  1358,   131,  1822,\n","        19968,   193,  1475,   177,  1197,   118,   189,  1775,   112,   173,\n","         1114,  8240,   171,  4063,  1361,   117, 16269,   177, 11848,  2539,\n","          118,  2908,  1322,  1186,   131,  7989,  3418, 26825,  1408,   119,\n","          171,  1116, 17733,   118, 21355,  2241,   131,   185,  1204,  2382,\n","         1106,  1138,   170,  6766,  8427,  1485,  1268,  2552,  1121,  6649,\n","         1215,  1219,  1137,   119,  8468, 12788,  4184, 19268, 11597,  1113,\n","         1171,  9964,   119,  1286, 12509, 11597,  9964,   119, 23123,  8625,\n","         1665, 21892,  2539,   118,  5356, 19515,  7201, 12779,   186,  1477,\n","         8167,  1116,   119, 23123,  1162,   131,   177,  5822,  6111,  1120,\n","         2724,   119,   127,   118,  3081,   119,   125,   170,   131, 23123,\n","        22320, 12881,  4724,  1105,  1231, 20080, 11934, 19232,   117,  7838,\n","          177, 15585,  1694,   117,   175,  3452,   176,  3069,  1408,  1111,\n","         2489,  2635,   117,  2661,  1106,  1195,  1389, 15242,  8338,   188,\n","         1830,  1643,   135,  1620,   187,   131,  6111,  2112,   118, 11769,\n","          117,   136,  1195,  1389,  1106,  4252, 25098,  2193,  1142,  1821,\n","          117, 15242,   176,  3069,  1431,  1129,  1682,  1106,  1129,  1195,\n","         6354,  1181,  1228,  1170,  4252, 25098,  1891,   117, 14255,  1204,\n","          119,  1106,  8804,   177,  5822,   119,   102,     0,     0,     0,\n","            0,     0,     0,     0,     0,     0,     0,     0,     0,     0,\n","            0,     0,     0,     0,     0,     0,     0,     0,     0,     0,\n","            0,     0,     0,     0,     0,     0,     0,     0,     0,     0,\n","            0,     0,     0,     0,     0,     0,     0,     0,     0,     0,\n","            0,     0,     0,     0,     0,     0,     0,     0,     0,     0,\n","            0,     0,     0,     0,     0,     0,     0,     0,     0,     0,\n","            0,     0,     0,     0,     0,     0,     0,     0,     0,     0,\n","            0,     0])\n","3,600 training samples\n","  400 validation samples\n"]},{"name":"stderr","output_type":"stream","text":["Some weights of BertForSequenceClassification were not initialized from the model checkpoint at emilyalsentzer/Bio_ClinicalBERT and are newly initialized: ['classifier.bias', 'classifier.weight']\n","You should probably TRAIN this model on a down-stream task to be able to use it for predictions and inference.\n"]},{"name":"stdout","output_type":"stream","text":["\n","======== Epoch 1 / 8 ========\n","Training...\n","  Batch    40  of    225.    Elapsed: 17.84437847137451.\n","  Batch    80  of    225.    Elapsed: 35.71501088142395.\n","  Batch   120  of    225.    Elapsed: 53.59526324272156.\n","  Batch   160  of    225.    Elapsed: 71.4765727519989.\n","  Batch   200  of    225.    Elapsed: 89.3811264038086.\n","\n","  Average training loss: 0.61\n","  Training epcoh took: 100.5807032585144\n","\n","Running Validation...\n","  Accuracy: 0.65\n","  Validation Loss: 0.58\n","  Validation took: 3.6657867431640625\n","\n","======== Epoch 2 / 8 ========\n","Training...\n","  Batch    40  of    225.    Elapsed: 17.91173243522644.\n","  Batch    80  of    225.    Elapsed: 35.83786129951477.\n","  Batch   120  of    225.    Elapsed: 53.76331090927124.\n","  Batch   160  of    225.    Elapsed: 71.66580486297607.\n","  Batch   200  of    225.    Elapsed: 89.5940957069397.\n","\n","  Average training loss: 0.51\n","  Training epcoh took: 100.77735829353333\n","\n","Running Validation...\n","  Accuracy: 0.68\n","  Validation Loss: 0.57\n","  Validation took: 3.681011915206909\n","\n","======== Epoch 3 / 8 ========\n","Training...\n","  Batch    40  of    225.    Elapsed: 17.922699689865112.\n","  Batch    80  of    225.    Elapsed: 35.84104800224304.\n","  Batch   120  of    225.    Elapsed: 53.76120376586914.\n","  Batch   160  of    225.    Elapsed: 71.67421531677246.\n","  Batch   200  of    225.    Elapsed: 89.61153221130371.\n","\n","  Average training loss: 0.38\n","  Training epcoh took: 100.79951786994934\n","\n","Running Validation...\n","  Accuracy: 0.71\n","  Validation Loss: 0.58\n","  Validation took: 3.6839563846588135\n","\n","======== Epoch 4 / 8 ========\n","Training...\n","  Batch    40  of    225.    Elapsed: 17.913058042526245.\n","  Batch    80  of    225.    Elapsed: 35.829389810562134.\n","  Batch   120  of    225.    Elapsed: 53.73755884170532.\n","  Batch   160  of    225.    Elapsed: 71.66571879386902.\n","  Batch   200  of    225.    Elapsed: 89.57620072364807.\n","\n","  Average training loss: 0.26\n","  Training epcoh took: 100.77422213554382\n","\n","Running Validation...\n","  Accuracy: 0.71\n","  Validation Loss: 0.67\n","  Validation took: 3.676137685775757\n","\n","======== Epoch 5 / 8 ========\n","Training...\n","  Batch    40  of    225.    Elapsed: 17.92280125617981.\n","  Batch    80  of    225.    Elapsed: 35.82319760322571.\n","  Batch   120  of    225.    Elapsed: 53.735010623931885.\n","  Batch   160  of    225.    Elapsed: 71.66904830932617.\n","  Batch   200  of    225.    Elapsed: 89.57052850723267.\n","\n","  Average training loss: 0.19\n","  Training epcoh took: 100.76737308502197\n","\n","Running Validation...\n","  Accuracy: 0.70\n","  Validation Loss: 0.77\n","  Validation took: 3.659294843673706\n","\n","======== Epoch 6 / 8 ========\n","Training...\n","  Batch    40  of    225.    Elapsed: 17.927822589874268.\n","  Batch    80  of    225.    Elapsed: 35.82535433769226.\n","  Batch   120  of    225.    Elapsed: 53.75291681289673.\n","  Batch   160  of    225.    Elapsed: 71.6577079296112.\n","  Batch   200  of    225.    Elapsed: 89.55660486221313.\n","\n","  Average training loss: 0.14\n","  Training epcoh took: 100.76317763328552\n","\n","Running Validation...\n","  Accuracy: 0.71\n","  Validation Loss: 0.92\n","  Validation took: 3.666844129562378\n","\n","======== Epoch 7 / 8 ========\n","Training...\n","  Batch    40  of    225.    Elapsed: 17.940383434295654.\n","  Batch    80  of    225.    Elapsed: 35.83347535133362.\n","  Batch   120  of    225.    Elapsed: 53.73234677314758.\n","  Batch   160  of    225.    Elapsed: 71.65819025039673.\n","  Batch   200  of    225.    Elapsed: 89.56972026824951.\n","\n","  Average training loss: 0.11\n","  Training epcoh took: 100.7588198184967\n","\n","Running Validation...\n","  Accuracy: 0.70\n","  Validation Loss: 1.04\n","  Validation took: 3.6717958450317383\n","\n","======== Epoch 8 / 8 ========\n","Training...\n","  Batch    40  of    225.    Elapsed: 17.930241346359253.\n","  Batch    80  of    225.    Elapsed: 35.86806559562683.\n","  Batch   120  of    225.    Elapsed: 53.77494263648987.\n","  Batch   160  of    225.    Elapsed: 71.70980834960938.\n","  Batch   200  of    225.    Elapsed: 89.62893223762512.\n","\n","  Average training loss: 0.09\n","  Training epcoh took: 100.84927296638489\n","\n","Running Validation...\n","  Accuracy: 0.69\n","  Validation Loss: 1.08\n","  Validation took: 3.669421434402466\n","\n","Training complete!\n","Total training took 835.4649188518524 (h:mm:ss)\n","Number of test sentences: 6,000\n","\n","4000\n","{'accuracy': 0.7319999933242798, 'f1': 0.7320892810821533, 'tnr': 0.7423064112663269, 'tpr': 0.72198486328125, 'mcc': 0.4642869532108307}\n","Original:  CONDITION UPDATE\n","D: PLEASE SEE CAREVUE FOR SPECIFICS. PT ADMITTED TO SICU FROM OR AROUND 1830- S/P SPINAL FUSION. INTUBATED ON PROPOFOL GTT FOR SEDATION AND NEO GTT TO MAINTAIN SBP\u003e100.\n","NEURO: PT LIGHTLY SEDATED ON PROPOFOL AT 50- 30 MCG/KG/MIN. ABLE TO OPEN EYES TO NAME, FOLLOWING COMMANDS AS SHIFT PROGRESSED. MAE ON BED ALTHOUGH LEFT LEG DOES APPEAR WEAKER. FENTANYL GTT STARTED FOR PAIN MANAGEMENT.\n","CV: T MAX 99.5. HR 85-105 NSR-ST. NEO GTT WEANED FROM 1.1MCG TO 0.5. UNABLE TO WEAN OFF SECONDARY TO LOWER BP WHEN SLEEPING. WHEN AWAKE, SBP \u003e100. MULTIPLE ATTEMPTS MADE BUT NEO ALWAYS RESTARTED AFTER 10-15 MINUTES.\n","RESP: BS CLEAR BUT DIMINISHED IN BASES. SX FOR SCANT AMT CLEAR FLUID. SATS\u003e100. REMAINS ON CMV WITH RATE OF 12, FIO2 50%\n","GI: NPO, ABD SOFT WITH + BS. ORAL-GASTRIC TUBE PATENT WITH BILIOUS DRAINAGE.\n","GU: LOW URINE X1 HR- TX'D WITH FLUID BOLUS, PRESENTLY HUO 60-80\n","ENDO: SLIDING SCALE INSULIN STARTED. BS 154-204\n","SKIN: PT NOTED TO HAVE ABRASION NEAR RIGHT EYE FROM TAPE USED DURING OR. ELASTOPLAST DRESSING ON BACK INTACT. LEFT FLANK DRESSING INTACT. HEMOVAC DRAINING 60-120CC BLOODY DRAINAGE Q2HRS.\n","HEME: HCT STABLE AT 32.6-33.4\n","A: HEMODYNAMICS AND RESP PARAMETERS MONITORED, SERIAL HCTS DONE, FENT GTT STARTED FOR PAIN MANAGEMENT, ATTEMPT TO WEAN NEO MAINTAINING SBP\u003e100\n","R: STABLE POST-OP, ? WEAN TO EXTUBATE THIS AM, NEO GTT SHOULD BE ABLE TO BE WEANED OFF AFTER EXTUBATION, CONT. TO MONITOR HCT.\n","\n","Token IDs: tensor([  101,  3879, 11984,   173,   131,  4268,  1267,  1920, 19038,  1111,\n","         2747,  1116,   119,   185,  1204,  4120,  1106, 16554,  1358,  1121,\n","         1137,  1213,  9200,   118,   188,   120,   185, 19245, 11970,   119,\n","         1107, 25098,  2913,  1113, 21146, 10008,  4063,   176,  3069,  1111,\n","        14516, 13759,  1105, 15242,   176,  3069,  1106,  4731,   188,  1830,\n","         1643,   135,  1620,   119, 24928, 11955,   131,   185,  1204,  7863,\n","        14516, 14459,  1113, 21146, 10008,  4063,  1120,  1851,   118,  1476,\n","          182,  1665,  1403,   120,  4023,   120, 11241,   119,  1682,  1106,\n","         1501,  1257,  1106,  1271,   117,  1378, 11443,  1112,  5212, 12687,\n","          119, 12477,  1162,  1113,  1908,  1780,  1286,  3420,  1674,  2845,\n","        16990,   119,   175,  3452, 18266,  1233,   176,  3069,  1408,  1111,\n","         2489,  2635,   119,   172,  1964,   131,   189, 12477,  1775,  4850,\n","          119,   126,   119,   177,  1197,  4859,   118,  8359,   183,  1116,\n","         1197,   118,   188,  1204,   119, 15242,   176,  3069,  1195,  6354,\n","         1181,  1121,   122,   119,   122,  1306,  1665,  1403,  1106,   121,\n","          119,   126,   119,  3372,  1106,  1195,  1389,  1228,  3718,  1106,\n","         2211,   171,  1643,  1165,  5575,   119,  1165,  8264,   117,   188,\n","         1830,  1643,   135,  1620,   119,  2967,  4021,  1189,  1133, 15242,\n","         1579, 27777,  1174,  1170,  1275,   118,  1405,  1904,   119,  1231,\n","        20080,   131,   171,  1116,  2330,  1133, 17017,  1107,  7616,   119,\n","          188,  1775,  1111, 14884,  1204,  1821,  1204,  2330,  8240,   119,\n","         2068,  1116,   135,  1620,   119,  2606,  1113,  3975,  1964,  1114,\n","         2603,  1104,  1367,   117, 20497,  1186,  1477,  1851,   110,   176,\n","         1182,   131,   183,  5674,   117,   170,  1830,  1181,  2991,  1114,\n","          116,   171,  1116,   119,  9619,   118,  3245, 11048,  7159,  8581,\n","         1114, 16516,  9436,  1361, 12779,   119,   176,  1358,   131,  1822,\n","        19968,   193,  1475,   177,  1197,   118,   189,  1775,   112,   173,\n","         1114,  8240,   171,  4063,  1361,   117, 16269,   177, 11848,  2539,\n","          118,  2908,  1322,  1186,   131,  7989,  3418, 26825,  1408,   119,\n","          171,  1116, 17733,   118, 21355,  2241,   131,   185,  1204,  2382,\n","         1106,  1138,   170,  6766,  8427,  1485,  1268,  2552,  1121,  6649,\n","         1215,  1219,  1137,   119,  8468, 12788,  4184, 19268, 11597,  1113,\n","         1171,  9964,   119,  1286, 12509, 11597,  9964,   119, 23123,  8625,\n","         1665, 21892,  2539,   118,  5356, 19515,  7201, 12779,   186,  1477,\n","         8167,  1116,   119, 23123,  1162,   131,   177,  5822,  6111,  1120,\n","         2724,   119,   127,   118,  3081,   119,   125,   170,   131, 23123,\n","        22320, 12881,  4724,  1105,  1231, 20080, 11934, 19232,   117,  7838,\n","          177, 15585,  1694,   117,   175,  3452,   176,  3069,  1408,  1111,\n","         2489,  2635,   117,  2661,  1106,  1195,  1389, 15242,  8338,   188,\n","         1830,  1643,   135,  1620,   187,   131,  6111,  2112,   118, 11769,\n","          117,   136,  1195,  1389,  1106,  4252, 25098,  2193,  1142,  1821,\n","          117, 15242,   176,  3069,  1431,  1129,  1682,  1106,  1129,  1195,\n","         6354,  1181,  1228,  1170,  4252, 25098,  1891,   117, 14255,  1204,\n","          119,  1106,  8804,   177,  5822,   119,   102,     0,     0,     0,\n","            0,     0,     0,     0,     0,     0,     0,     0,     0,     0,\n","            0,     0,     0,     0,     0,     0,     0,     0,     0,     0,\n","            0,     0,     0,     0,     0,     0,     0,     0,     0,     0,\n","            0,     0,     0,     0,     0,     0,     0,     0,     0,     0,\n","            0,     0,     0,     0,     0,     0,     0,     0,     0,     0,\n","            0,     0,     0,     0,     0,     0,     0,     0,     0,     0,\n","            0,     0,     0,     0,     0,     0,     0,     0,     0,     0,\n","            0,     0])\n","4,050 training samples\n","  450 validation samples\n"]},{"name":"stderr","output_type":"stream","text":["Some weights of BertForSequenceClassification were not initialized from the model checkpoint at emilyalsentzer/Bio_ClinicalBERT and are newly initialized: ['classifier.bias', 'classifier.weight']\n","You should probably TRAIN this model on a down-stream task to be able to use it for predictions and inference.\n"]},{"name":"stdout","output_type":"stream","text":["\n","======== Epoch 1 / 8 ========\n","Training...\n","  Batch    40  of    254.    Elapsed: 17.842668771743774.\n","  Batch    80  of    254.    Elapsed: 35.68932294845581.\n","  Batch   120  of    254.    Elapsed: 53.59790110588074.\n","  Batch   160  of    254.    Elapsed: 71.49430131912231.\n","  Batch   200  of    254.    Elapsed: 89.39577555656433.\n","  Batch   240  of    254.    Elapsed: 107.30784869194031.\n","\n","  Average training loss: 0.62\n","  Training epcoh took: 113.20138096809387\n","\n","Running Validation...\n","  Accuracy: 0.66\n","  Validation Loss: 0.54\n","  Validation took: 4.13847017288208\n","\n","======== Epoch 2 / 8 ========\n","Training...\n","  Batch    40  of    254.    Elapsed: 17.922443389892578.\n","  Batch    80  of    254.    Elapsed: 35.83810472488403.\n","  Batch   120  of    254.    Elapsed: 53.76442241668701.\n","  Batch   160  of    254.    Elapsed: 71.68226790428162.\n","  Batch   200  of    254.    Elapsed: 89.59295153617859.\n","  Batch   240  of    254.    Elapsed: 107.51692247390747.\n","\n","  Average training loss: 0.54\n","  Training epcoh took: 113.42868852615356\n","\n","Running Validation...\n","  Accuracy: 0.69\n","  Validation Loss: 0.52\n","  Validation took: 4.129247665405273\n","\n","======== Epoch 3 / 8 ========\n","Training...\n","  Batch    40  of    254.    Elapsed: 17.92693328857422.\n","  Batch    80  of    254.    Elapsed: 35.84775447845459.\n","  Batch   120  of    254.    Elapsed: 53.75209999084473.\n","  Batch   160  of    254.    Elapsed: 71.67715382575989.\n","  Batch   200  of    254.    Elapsed: 89.5837242603302.\n","  Batch   240  of    254.    Elapsed: 107.50148129463196.\n","\n","  Average training loss: 0.44\n","  Training epcoh took: 113.39608860015869\n","\n","Running Validation...\n","  Accuracy: 0.73\n","  Validation Loss: 0.51\n","  Validation took: 4.13535475730896\n","\n","======== Epoch 4 / 8 ========\n","Training...\n","  Batch    40  of    254.    Elapsed: 17.921372890472412.\n","  Batch    80  of    254.    Elapsed: 35.83552312850952.\n","  Batch   120  of    254.    Elapsed: 53.75808763504028.\n","  Batch   160  of    254.    Elapsed: 71.67001581192017.\n","  Batch   200  of    254.    Elapsed: 89.57795333862305.\n","  Batch   240  of    254.    Elapsed: 107.49984335899353.\n","\n","  Average training loss: 0.31\n","  Training epcoh took: 113.3953628540039\n","\n","Running Validation...\n","  Accuracy: 0.72\n","  Validation Loss: 0.62\n","  Validation took: 4.142174482345581\n","\n","======== Epoch 5 / 8 ========\n","Training...\n","  Batch    40  of    254.    Elapsed: 17.9224271774292.\n","  Batch    80  of    254.    Elapsed: 35.825763463974.\n","  Batch   120  of    254.    Elapsed: 53.749144077301025.\n","  Batch   160  of    254.    Elapsed: 71.64539885520935.\n","  Batch   200  of    254.    Elapsed: 89.55954360961914.\n","  Batch   240  of    254.    Elapsed: 107.46938037872314.\n","\n","  Average training loss: 0.22\n","  Training epcoh took: 113.36339092254639\n","\n","Running Validation...\n","  Accuracy: 0.74\n","  Validation Loss: 0.67\n","  Validation took: 4.145428895950317\n","\n","======== Epoch 6 / 8 ========\n","Training...\n","  Batch    40  of    254.    Elapsed: 17.910439252853394.\n","  Batch    80  of    254.    Elapsed: 35.8066611289978.\n","  Batch   120  of    254.    Elapsed: 53.71711039543152.\n","  Batch   160  of    254.    Elapsed: 71.6343719959259.\n","  Batch   200  of    254.    Elapsed: 89.54579710960388.\n","  Batch   240  of    254.    Elapsed: 107.44468688964844.\n","\n","  Average training loss: 0.18\n","  Training epcoh took: 113.34765601158142\n","\n","Running Validation...\n","  Accuracy: 0.76\n","  Validation Loss: 0.76\n","  Validation took: 4.146454572677612\n","\n","======== Epoch 7 / 8 ========\n","Training...\n","  Batch    40  of    254.    Elapsed: 17.90307903289795.\n","  Batch    80  of    254.    Elapsed: 35.81454658508301.\n","  Batch   120  of    254.    Elapsed: 53.723215103149414.\n","  Batch   160  of    254.    Elapsed: 71.62600922584534.\n","  Batch   200  of    254.    Elapsed: 89.55377411842346.\n","  Batch   240  of    254.    Elapsed: 107.47182774543762.\n","\n","  Average training loss: 0.14\n","  Training epcoh took: 113.38194942474365\n","\n","Running Validation...\n","  Accuracy: 0.72\n","  Validation Loss: 0.90\n","  Validation took: 4.132686138153076\n","\n","======== Epoch 8 / 8 ========\n","Training...\n","  Batch    40  of    254.    Elapsed: 17.906069040298462.\n","  Batch    80  of    254.    Elapsed: 35.84265398979187.\n","  Batch   120  of    254.    Elapsed: 53.75463533401489.\n","  Batch   160  of    254.    Elapsed: 71.65494966506958.\n","  Batch   200  of    254.    Elapsed: 89.57756352424622.\n","  Batch   240  of    254.    Elapsed: 107.47814559936523.\n","\n","  Average training loss: 0.12\n","  Training epcoh took: 113.38209986686707\n","\n","Running Validation...\n","  Accuracy: 0.72\n","  Validation Loss: 0.93\n","  Validation took: 4.125367641448975\n","\n","Training complete!\n","Total training took 940.009557723999 (h:mm:ss)\n","Number of test sentences: 6,000\n","\n","4500\n","{'accuracy': 0.7293333411216736, 'f1': 0.7319247126579285, 'tnr': 0.7301318645477295, 'tpr': 0.7285573482513428, 'mcc': 0.45864784717559814}\n","Original:  CONDITION UPDATE\n","D: PLEASE SEE CAREVUE FOR SPECIFICS. PT ADMITTED TO SICU FROM OR AROUND 1830- S/P SPINAL FUSION. INTUBATED ON PROPOFOL GTT FOR SEDATION AND NEO GTT TO MAINTAIN SBP\u003e100.\n","NEURO: PT LIGHTLY SEDATED ON PROPOFOL AT 50- 30 MCG/KG/MIN. ABLE TO OPEN EYES TO NAME, FOLLOWING COMMANDS AS SHIFT PROGRESSED. MAE ON BED ALTHOUGH LEFT LEG DOES APPEAR WEAKER. FENTANYL GTT STARTED FOR PAIN MANAGEMENT.\n","CV: T MAX 99.5. HR 85-105 NSR-ST. NEO GTT WEANED FROM 1.1MCG TO 0.5. UNABLE TO WEAN OFF SECONDARY TO LOWER BP WHEN SLEEPING. WHEN AWAKE, SBP \u003e100. MULTIPLE ATTEMPTS MADE BUT NEO ALWAYS RESTARTED AFTER 10-15 MINUTES.\n","RESP: BS CLEAR BUT DIMINISHED IN BASES. SX FOR SCANT AMT CLEAR FLUID. SATS\u003e100. REMAINS ON CMV WITH RATE OF 12, FIO2 50%\n","GI: NPO, ABD SOFT WITH + BS. ORAL-GASTRIC TUBE PATENT WITH BILIOUS DRAINAGE.\n","GU: LOW URINE X1 HR- TX'D WITH FLUID BOLUS, PRESENTLY HUO 60-80\n","ENDO: SLIDING SCALE INSULIN STARTED. BS 154-204\n","SKIN: PT NOTED TO HAVE ABRASION NEAR RIGHT EYE FROM TAPE USED DURING OR. ELASTOPLAST DRESSING ON BACK INTACT. LEFT FLANK DRESSING INTACT. HEMOVAC DRAINING 60-120CC BLOODY DRAINAGE Q2HRS.\n","HEME: HCT STABLE AT 32.6-33.4\n","A: HEMODYNAMICS AND RESP PARAMETERS MONITORED, SERIAL HCTS DONE, FENT GTT STARTED FOR PAIN MANAGEMENT, ATTEMPT TO WEAN NEO MAINTAINING SBP\u003e100\n","R: STABLE POST-OP, ? WEAN TO EXTUBATE THIS AM, NEO GTT SHOULD BE ABLE TO BE WEANED OFF AFTER EXTUBATION, CONT. TO MONITOR HCT.\n","\n","Token IDs: tensor([  101,  3879, 11984,   173,   131,  4268,  1267,  1920, 19038,  1111,\n","         2747,  1116,   119,   185,  1204,  4120,  1106, 16554,  1358,  1121,\n","         1137,  1213,  9200,   118,   188,   120,   185, 19245, 11970,   119,\n","         1107, 25098,  2913,  1113, 21146, 10008,  4063,   176,  3069,  1111,\n","        14516, 13759,  1105, 15242,   176,  3069,  1106,  4731,   188,  1830,\n","         1643,   135,  1620,   119, 24928, 11955,   131,   185,  1204,  7863,\n","        14516, 14459,  1113, 21146, 10008,  4063,  1120,  1851,   118,  1476,\n","          182,  1665,  1403,   120,  4023,   120, 11241,   119,  1682,  1106,\n","         1501,  1257,  1106,  1271,   117,  1378, 11443,  1112,  5212, 12687,\n","          119, 12477,  1162,  1113,  1908,  1780,  1286,  3420,  1674,  2845,\n","        16990,   119,   175,  3452, 18266,  1233,   176,  3069,  1408,  1111,\n","         2489,  2635,   119,   172,  1964,   131,   189, 12477,  1775,  4850,\n","          119,   126,   119,   177,  1197,  4859,   118,  8359,   183,  1116,\n","         1197,   118,   188,  1204,   119, 15242,   176,  3069,  1195,  6354,\n","         1181,  1121,   122,   119,   122,  1306,  1665,  1403,  1106,   121,\n","          119,   126,   119,  3372,  1106,  1195,  1389,  1228,  3718,  1106,\n","         2211,   171,  1643,  1165,  5575,   119,  1165,  8264,   117,   188,\n","         1830,  1643,   135,  1620,   119,  2967,  4021,  1189,  1133, 15242,\n","         1579, 27777,  1174,  1170,  1275,   118,  1405,  1904,   119,  1231,\n","        20080,   131,   171,  1116,  2330,  1133, 17017,  1107,  7616,   119,\n","          188,  1775,  1111, 14884,  1204,  1821,  1204,  2330,  8240,   119,\n","         2068,  1116,   135,  1620,   119,  2606,  1113,  3975,  1964,  1114,\n","         2603,  1104,  1367,   117, 20497,  1186,  1477,  1851,   110,   176,\n","         1182,   131,   183,  5674,   117,   170,  1830,  1181,  2991,  1114,\n","          116,   171,  1116,   119,  9619,   118,  3245, 11048,  7159,  8581,\n","         1114, 16516,  9436,  1361, 12779,   119,   176,  1358,   131,  1822,\n","        19968,   193,  1475,   177,  1197,   118,   189,  1775,   112,   173,\n","         1114,  8240,   171,  4063,  1361,   117, 16269,   177, 11848,  2539,\n","          118,  2908,  1322,  1186,   131,  7989,  3418, 26825,  1408,   119,\n","          171,  1116, 17733,   118, 21355,  2241,   131,   185,  1204,  2382,\n","         1106,  1138,   170,  6766,  8427,  1485,  1268,  2552,  1121,  6649,\n","         1215,  1219,  1137,   119,  8468, 12788,  4184, 19268, 11597,  1113,\n","         1171,  9964,   119,  1286, 12509, 11597,  9964,   119, 23123,  8625,\n","         1665, 21892,  2539,   118,  5356, 19515,  7201, 12779,   186,  1477,\n","         8167,  1116,   119, 23123,  1162,   131,   177,  5822,  6111,  1120,\n","         2724,   119,   127,   118,  3081,   119,   125,   170,   131, 23123,\n","        22320, 12881,  4724,  1105,  1231, 20080, 11934, 19232,   117,  7838,\n","          177, 15585,  1694,   117,   175,  3452,   176,  3069,  1408,  1111,\n","         2489,  2635,   117,  2661,  1106,  1195,  1389, 15242,  8338,   188,\n","         1830,  1643,   135,  1620,   187,   131,  6111,  2112,   118, 11769,\n","          117,   136,  1195,  1389,  1106,  4252, 25098,  2193,  1142,  1821,\n","          117, 15242,   176,  3069,  1431,  1129,  1682,  1106,  1129,  1195,\n","         6354,  1181,  1228,  1170,  4252, 25098,  1891,   117, 14255,  1204,\n","          119,  1106,  8804,   177,  5822,   119,   102,     0,     0,     0,\n","            0,     0,     0,     0,     0,     0,     0,     0,     0,     0,\n","            0,     0,     0,     0,     0,     0,     0,     0,     0,     0,\n","            0,     0,     0,     0,     0,     0,     0,     0,     0,     0,\n","            0,     0,     0,     0,     0,     0,     0,     0,     0,     0,\n","            0,     0,     0,     0,     0,     0,     0,     0,     0,     0,\n","            0,     0,     0,     0,     0,     0,     0,     0,     0,     0,\n","            0,     0,     0,     0,     0,     0,     0,     0,     0,     0,\n","            0,     0])\n","4,500 training samples\n","  500 validation samples\n"]},{"name":"stderr","output_type":"stream","text":["Some weights of BertForSequenceClassification were not initialized from the model checkpoint at emilyalsentzer/Bio_ClinicalBERT and are newly initialized: ['classifier.bias', 'classifier.weight']\n","You should probably TRAIN this model on a down-stream task to be able to use it for predictions and inference.\n"]},{"name":"stdout","output_type":"stream","text":["\n","======== Epoch 1 / 8 ========\n","Training...\n","  Batch    40  of    282.    Elapsed: 17.8406982421875.\n","  Batch    80  of    282.    Elapsed: 35.6999077796936.\n","  Batch   120  of    282.    Elapsed: 53.57638478279114.\n","  Batch   160  of    282.    Elapsed: 71.46147608757019.\n","  Batch   200  of    282.    Elapsed: 89.36913585662842.\n","  Batch   240  of    282.    Elapsed: 107.27196907997131.\n","  Batch   280  of    282.    Elapsed: 125.18387937545776.\n","\n","  Average training loss: 0.61\n","  Training epcoh took: 125.76727509498596\n","\n","Running Validation...\n","  Accuracy: 0.69\n","  Validation Loss: 0.55\n","  Validation took: 4.586478233337402\n","\n","======== Epoch 2 / 8 ========\n","Training...\n","  Batch    40  of    282.    Elapsed: 17.934300899505615.\n","  Batch    80  of    282.    Elapsed: 35.83176779747009.\n","  Batch   120  of    282.    Elapsed: 53.746246099472046.\n","  Batch   160  of    282.    Elapsed: 71.65199828147888.\n","  Batch   200  of    282.    Elapsed: 89.57873034477234.\n","  Batch   240  of    282.    Elapsed: 107.50732827186584.\n","  Batch   280  of    282.    Elapsed: 125.43367719650269.\n","\n","  Average training loss: 0.53\n","  Training epcoh took: 126.01561141014099\n","\n","Running Validation...\n","  Accuracy: 0.71\n","  Validation Loss: 0.53\n","  Validation took: 4.60325813293457\n","\n","======== Epoch 3 / 8 ========\n","Training...\n","  Batch    40  of    282.    Elapsed: 17.919343948364258.\n","  Batch    80  of    282.    Elapsed: 35.832234144210815.\n","  Batch   120  of    282.    Elapsed: 53.74245023727417.\n","  Batch   160  of    282.    Elapsed: 71.66810345649719.\n","  Batch   200  of    282.    Elapsed: 89.60449528694153.\n","  Batch   240  of    282.    Elapsed: 107.52378416061401.\n","  Batch   280  of    282.    Elapsed: 125.44638586044312.\n","\n","  Average training loss: 0.40\n","  Training epcoh took: 126.02158188819885\n","\n","Running Validation...\n","  Accuracy: 0.75\n","  Validation Loss: 0.52\n","  Validation took: 4.607351064682007\n","\n","======== Epoch 4 / 8 ========\n","Training...\n","  Batch    40  of    282.    Elapsed: 17.942703247070312.\n","  Batch    80  of    282.    Elapsed: 35.88989567756653.\n","  Batch   120  of    282.    Elapsed: 53.84439992904663.\n","  Batch   160  of    282.    Elapsed: 71.77442765235901.\n","  Batch   200  of    282.    Elapsed: 89.72969198226929.\n","  Batch   240  of    282.    Elapsed: 107.66090488433838.\n","  Batch   280  of    282.    Elapsed: 125.58230018615723.\n","\n","  Average training loss: 0.29\n","  Training epcoh took: 126.1630597114563\n","\n","Running Validation...\n","  Accuracy: 0.76\n","  Validation Loss: 0.60\n","  Validation took: 4.614640235900879\n","\n","======== Epoch 5 / 8 ========\n","Training...\n","  Batch    40  of    282.    Elapsed: 17.95231556892395.\n","  Batch    80  of    282.    Elapsed: 35.8771116733551.\n","  Batch   120  of    282.    Elapsed: 53.82708263397217.\n","  Batch   160  of    282.    Elapsed: 71.78353810310364.\n","  Batch   200  of    282.    Elapsed: 89.7225615978241.\n","  Batch   240  of    282.    Elapsed: 107.67736148834229.\n","  Batch   280  of    282.    Elapsed: 125.6178367137909.\n","\n","  Average training loss: 0.21\n","  Training epcoh took: 126.19673681259155\n","\n","Running Validation...\n","  Accuracy: 0.79\n","  Validation Loss: 0.60\n","  Validation took: 4.593183279037476\n","\n","======== Epoch 6 / 8 ========\n","Training...\n","  Batch    40  of    282.    Elapsed: 17.94056987762451.\n","  Batch    80  of    282.    Elapsed: 35.86871314048767.\n","  Batch   120  of    282.    Elapsed: 53.81399416923523.\n","  Batch   160  of    282.    Elapsed: 71.76266431808472.\n","  Batch   200  of    282.    Elapsed: 89.68757963180542.\n","  Batch   240  of    282.    Elapsed: 107.63839054107666.\n","  Batch   280  of    282.    Elapsed: 125.55075883865356.\n","\n","  Average training loss: 0.16\n","  Training epcoh took: 126.1278567314148\n","\n","Running Validation...\n","  Accuracy: 0.78\n","  Validation Loss: 0.70\n","  Validation took: 4.594923496246338\n","\n","======== Epoch 7 / 8 ========\n","Training...\n","  Batch    40  of    282.    Elapsed: 17.937119245529175.\n","  Batch    80  of    282.    Elapsed: 35.85748815536499.\n","  Batch   120  of    282.    Elapsed: 53.790250062942505.\n","  Batch   160  of    282.    Elapsed: 71.71471881866455.\n","  Batch   200  of    282.    Elapsed: 89.6305844783783.\n","  Batch   240  of    282.    Elapsed: 107.56853699684143.\n","  Batch   280  of    282.    Elapsed: 125.47715616226196.\n","\n","  Average training loss: 0.12\n","  Training epcoh took: 126.05522608757019\n","\n","Running Validation...\n","  Accuracy: 0.77\n","  Validation Loss: 0.79\n","  Validation took: 4.583522081375122\n","\n","======== Epoch 8 / 8 ========\n","Training...\n","  Batch    40  of    282.    Elapsed: 17.933202981948853.\n","  Batch    80  of    282.    Elapsed: 35.835347175598145.\n","  Batch   120  of    282.    Elapsed: 53.75153112411499.\n","  Batch   160  of    282.    Elapsed: 71.6764965057373.\n","  Batch   200  of    282.    Elapsed: 89.59679436683655.\n","  Batch   240  of    282.    Elapsed: 107.53259587287903.\n","  Batch   280  of    282.    Elapsed: 125.45949625968933.\n","\n","  Average training loss: 0.10\n","  Training epcoh took: 126.03600406646729\n","\n","Running Validation...\n","  Accuracy: 0.77\n","  Validation Loss: 0.83\n","  Validation took: 4.589627027511597\n","\n","Training complete!\n","Total training took 1045.1773793697357 (h:mm:ss)\n","Number of test sentences: 6,000\n","\n","5000\n","{'accuracy': 0.7483333349227905, 'f1': 0.7524589896202087, 'tnr': 0.7423064112663269, 'tpr': 0.7541899681091309, 'mcc': 0.49653497338294983}\n","Original:  CONDITION UPDATE\n","D: PLEASE SEE CAREVUE FOR SPECIFICS. PT ADMITTED TO SICU FROM OR AROUND 1830- S/P SPINAL FUSION. INTUBATED ON PROPOFOL GTT FOR SEDATION AND NEO GTT TO MAINTAIN SBP\u003e100.\n","NEURO: PT LIGHTLY SEDATED ON PROPOFOL AT 50- 30 MCG/KG/MIN. ABLE TO OPEN EYES TO NAME, FOLLOWING COMMANDS AS SHIFT PROGRESSED. MAE ON BED ALTHOUGH LEFT LEG DOES APPEAR WEAKER. FENTANYL GTT STARTED FOR PAIN MANAGEMENT.\n","CV: T MAX 99.5. HR 85-105 NSR-ST. NEO GTT WEANED FROM 1.1MCG TO 0.5. UNABLE TO WEAN OFF SECONDARY TO LOWER BP WHEN SLEEPING. WHEN AWAKE, SBP \u003e100. MULTIPLE ATTEMPTS MADE BUT NEO ALWAYS RESTARTED AFTER 10-15 MINUTES.\n","RESP: BS CLEAR BUT DIMINISHED IN BASES. SX FOR SCANT AMT CLEAR FLUID. SATS\u003e100. REMAINS ON CMV WITH RATE OF 12, FIO2 50%\n","GI: NPO, ABD SOFT WITH + BS. ORAL-GASTRIC TUBE PATENT WITH BILIOUS DRAINAGE.\n","GU: LOW URINE X1 HR- TX'D WITH FLUID BOLUS, PRESENTLY HUO 60-80\n","ENDO: SLIDING SCALE INSULIN STARTED. BS 154-204\n","SKIN: PT NOTED TO HAVE ABRASION NEAR RIGHT EYE FROM TAPE USED DURING OR. ELASTOPLAST DRESSING ON BACK INTACT. LEFT FLANK DRESSING INTACT. HEMOVAC DRAINING 60-120CC BLOODY DRAINAGE Q2HRS.\n","HEME: HCT STABLE AT 32.6-33.4\n","A: HEMODYNAMICS AND RESP PARAMETERS MONITORED, SERIAL HCTS DONE, FENT GTT STARTED FOR PAIN MANAGEMENT, ATTEMPT TO WEAN NEO MAINTAINING SBP\u003e100\n","R: STABLE POST-OP, ? WEAN TO EXTUBATE THIS AM, NEO GTT SHOULD BE ABLE TO BE WEANED OFF AFTER EXTUBATION, CONT. TO MONITOR HCT.\n","\n","Token IDs: tensor([  101,  3879, 11984,   173,   131,  4268,  1267,  1920, 19038,  1111,\n","         2747,  1116,   119,   185,  1204,  4120,  1106, 16554,  1358,  1121,\n","         1137,  1213,  9200,   118,   188,   120,   185, 19245, 11970,   119,\n","         1107, 25098,  2913,  1113, 21146, 10008,  4063,   176,  3069,  1111,\n","        14516, 13759,  1105, 15242,   176,  3069,  1106,  4731,   188,  1830,\n","         1643,   135,  1620,   119, 24928, 11955,   131,   185,  1204,  7863,\n","        14516, 14459,  1113, 21146, 10008,  4063,  1120,  1851,   118,  1476,\n","          182,  1665,  1403,   120,  4023,   120, 11241,   119,  1682,  1106,\n","         1501,  1257,  1106,  1271,   117,  1378, 11443,  1112,  5212, 12687,\n","          119, 12477,  1162,  1113,  1908,  1780,  1286,  3420,  1674,  2845,\n","        16990,   119,   175,  3452, 18266,  1233,   176,  3069,  1408,  1111,\n","         2489,  2635,   119,   172,  1964,   131,   189, 12477,  1775,  4850,\n","          119,   126,   119,   177,  1197,  4859,   118,  8359,   183,  1116,\n","         1197,   118,   188,  1204,   119, 15242,   176,  3069,  1195,  6354,\n","         1181,  1121,   122,   119,   122,  1306,  1665,  1403,  1106,   121,\n","          119,   126,   119,  3372,  1106,  1195,  1389,  1228,  3718,  1106,\n","         2211,   171,  1643,  1165,  5575,   119,  1165,  8264,   117,   188,\n","         1830,  1643,   135,  1620,   119,  2967,  4021,  1189,  1133, 15242,\n","         1579, 27777,  1174,  1170,  1275,   118,  1405,  1904,   119,  1231,\n","        20080,   131,   171,  1116,  2330,  1133, 17017,  1107,  7616,   119,\n","          188,  1775,  1111, 14884,  1204,  1821,  1204,  2330,  8240,   119,\n","         2068,  1116,   135,  1620,   119,  2606,  1113,  3975,  1964,  1114,\n","         2603,  1104,  1367,   117, 20497,  1186,  1477,  1851,   110,   176,\n","         1182,   131,   183,  5674,   117,   170,  1830,  1181,  2991,  1114,\n","          116,   171,  1116,   119,  9619,   118,  3245, 11048,  7159,  8581,\n","         1114, 16516,  9436,  1361, 12779,   119,   176,  1358,   131,  1822,\n","        19968,   193,  1475,   177,  1197,   118,   189,  1775,   112,   173,\n","         1114,  8240,   171,  4063,  1361,   117, 16269,   177, 11848,  2539,\n","          118,  2908,  1322,  1186,   131,  7989,  3418, 26825,  1408,   119,\n","          171,  1116, 17733,   118, 21355,  2241,   131,   185,  1204,  2382,\n","         1106,  1138,   170,  6766,  8427,  1485,  1268,  2552,  1121,  6649,\n","         1215,  1219,  1137,   119,  8468, 12788,  4184, 19268, 11597,  1113,\n","         1171,  9964,   119,  1286, 12509, 11597,  9964,   119, 23123,  8625,\n","         1665, 21892,  2539,   118,  5356, 19515,  7201, 12779,   186,  1477,\n","         8167,  1116,   119, 23123,  1162,   131,   177,  5822,  6111,  1120,\n","         2724,   119,   127,   118,  3081,   119,   125,   170,   131, 23123,\n","        22320, 12881,  4724,  1105,  1231, 20080, 11934, 19232,   117,  7838,\n","          177, 15585,  1694,   117,   175,  3452,   176,  3069,  1408,  1111,\n","         2489,  2635,   117,  2661,  1106,  1195,  1389, 15242,  8338,   188,\n","         1830,  1643,   135,  1620,   187,   131,  6111,  2112,   118, 11769,\n","          117,   136,  1195,  1389,  1106,  4252, 25098,  2193,  1142,  1821,\n","          117, 15242,   176,  3069,  1431,  1129,  1682,  1106,  1129,  1195,\n","         6354,  1181,  1228,  1170,  4252, 25098,  1891,   117, 14255,  1204,\n","          119,  1106,  8804,   177,  5822,   119,   102,     0,     0,     0,\n","            0,     0,     0,     0,     0,     0,     0,     0,     0,     0,\n","            0,     0,     0,     0,     0,     0,     0,     0,     0,     0,\n","            0,     0,     0,     0,     0,     0,     0,     0,     0,     0,\n","            0,     0,     0,     0,     0,     0,     0,     0,     0,     0,\n","            0,     0,     0,     0,     0,     0,     0,     0,     0,     0,\n","            0,     0,     0,     0,     0,     0,     0,     0,     0,     0,\n","            0,     0,     0,     0,     0,     0,     0,     0,     0,     0,\n","            0,     0])\n","4,950 training samples\n","  550 validation samples\n"]},{"name":"stderr","output_type":"stream","text":["Some weights of BertForSequenceClassification were not initialized from the model checkpoint at emilyalsentzer/Bio_ClinicalBERT and are newly initialized: ['classifier.bias', 'classifier.weight']\n","You should probably TRAIN this model on a down-stream task to be able to use it for predictions and inference.\n"]},{"name":"stdout","output_type":"stream","text":["\n","======== Epoch 1 / 8 ========\n","Training...\n","  Batch    40  of    310.    Elapsed: 17.83797311782837.\n","  Batch    80  of    310.    Elapsed: 35.70461058616638.\n","  Batch   120  of    310.    Elapsed: 53.61085867881775.\n","  Batch   160  of    310.    Elapsed: 71.51185894012451.\n","  Batch   200  of    310.    Elapsed: 89.4260082244873.\n","  Batch   240  of    310.    Elapsed: 107.34605431556702.\n","  Batch   280  of    310.    Elapsed: 125.25139331817627.\n","\n","  Average training loss: 0.62\n","  Training epcoh took: 138.43448877334595\n","\n","Running Validation...\n","  Accuracy: 0.67\n","  Validation Loss: 0.57\n","  Validation took: 5.055525779724121\n","\n","======== Epoch 2 / 8 ========\n","Training...\n","  Batch    40  of    310.    Elapsed: 17.916588068008423.\n","  Batch    80  of    310.    Elapsed: 35.8347008228302.\n","  Batch   120  of    310.    Elapsed: 53.75418782234192.\n","  Batch   160  of    310.    Elapsed: 71.67059302330017.\n","  Batch   200  of    310.    Elapsed: 89.6104485988617.\n","  Batch   240  of    310.    Elapsed: 107.51643109321594.\n","  Batch   280  of    310.    Elapsed: 125.44417715072632.\n","\n","  Average training loss: 0.52\n","  Training epcoh took: 138.6181080341339\n","\n","Running Validation...\n","  Accuracy: 0.70\n","  Validation Loss: 0.53\n","  Validation took: 5.07448935508728\n","\n","======== Epoch 3 / 8 ========\n","Training...\n","  Batch    40  of    310.    Elapsed: 17.917967557907104.\n","  Batch    80  of    310.    Elapsed: 35.838956117630005.\n","  Batch   120  of    310.    Elapsed: 53.7589750289917.\n","  Batch   160  of    310.    Elapsed: 71.66150975227356.\n","  Batch   200  of    310.    Elapsed: 89.60091972351074.\n","  Batch   240  of    310.    Elapsed: 107.51589155197144.\n","  Batch   280  of    310.    Elapsed: 125.42640948295593.\n","\n","  Average training loss: 0.41\n","  Training epcoh took: 138.61940240859985\n","\n","Running Validation...\n","  Accuracy: 0.74\n","  Validation Loss: 0.51\n","  Validation took: 5.0503716468811035\n","\n","======== Epoch 4 / 8 ========\n","Training...\n","  Batch    40  of    310.    Elapsed: 17.92344856262207.\n","  Batch    80  of    310.    Elapsed: 35.85010886192322.\n","  Batch   120  of    310.    Elapsed: 53.76546764373779.\n","  Batch   160  of    310.    Elapsed: 71.6939377784729.\n","  Batch   200  of    310.    Elapsed: 89.60720181465149.\n","  Batch   240  of    310.    Elapsed: 107.5216703414917.\n","  Batch   280  of    310.    Elapsed: 125.45432591438293.\n","\n","  Average training loss: 0.30\n","  Training epcoh took: 138.63160490989685\n","\n","Running Validation...\n","  Accuracy: 0.77\n","  Validation Loss: 0.54\n","  Validation took: 5.042448282241821\n","\n","======== Epoch 5 / 8 ========\n","Training...\n","  Batch    40  of    310.    Elapsed: 17.902765035629272.\n","  Batch    80  of    310.    Elapsed: 35.82972979545593.\n","  Batch   120  of    310.    Elapsed: 53.7483344078064.\n","  Batch   160  of    310.    Elapsed: 71.67026853561401.\n","  Batch   200  of    310.    Elapsed: 89.58525729179382.\n","  Batch   240  of    310.    Elapsed: 107.49327611923218.\n","  Batch   280  of    310.    Elapsed: 125.41374826431274.\n","\n","  Average training loss: 0.21\n","  Training epcoh took: 138.59822297096252\n","\n","Running Validation...\n","  Accuracy: 0.76\n","  Validation Loss: 0.73\n","  Validation took: 5.037475109100342\n","\n","======== Epoch 6 / 8 ========\n","Training...\n","  Batch    40  of    310.    Elapsed: 17.92308807373047.\n","  Batch    80  of    310.    Elapsed: 35.84154963493347.\n","  Batch   120  of    310.    Elapsed: 53.757211208343506.\n","  Batch   160  of    310.    Elapsed: 71.680424451828.\n","  Batch   200  of    310.    Elapsed: 89.59940958023071.\n","  Batch   240  of    310.    Elapsed: 107.50811171531677.\n","  Batch   280  of    310.    Elapsed: 125.4230546951294.\n","\n","  Average training loss: 0.16\n","  Training epcoh took: 138.6040050983429\n","\n","Running Validation...\n","  Accuracy: 0.76\n","  Validation Loss: 0.85\n","  Validation took: 5.041019439697266\n","\n","======== Epoch 7 / 8 ========\n","Training...\n","  Batch    40  of    310.    Elapsed: 17.916806936264038.\n","  Batch    80  of    310.    Elapsed: 35.84102201461792.\n","  Batch   120  of    310.    Elapsed: 53.7673134803772.\n","  Batch   160  of    310.    Elapsed: 71.7117567062378.\n","  Batch   200  of    310.    Elapsed: 89.62991261482239.\n","  Batch   240  of    310.    Elapsed: 107.55218124389648.\n","  Batch   280  of    310.    Elapsed: 125.47538113594055.\n","\n","  Average training loss: 0.12\n","  Training epcoh took: 138.6624939441681\n","\n","Running Validation...\n","  Accuracy: 0.78\n","  Validation Loss: 0.89\n","  Validation took: 5.0390496253967285\n","\n","======== Epoch 8 / 8 ========\n","Training...\n","  Batch    40  of    310.    Elapsed: 17.926751375198364.\n","  Batch    80  of    310.    Elapsed: 35.848008155822754.\n","  Batch   120  of    310.    Elapsed: 53.76270508766174.\n","  Batch   160  of    310.    Elapsed: 71.67381405830383.\n","  Batch   200  of    310.    Elapsed: 89.59521079063416.\n","  Batch   240  of    310.    Elapsed: 107.52601885795593.\n","  Batch   280  of    310.    Elapsed: 125.44491577148438.\n","\n","  Average training loss: 0.09\n","  Training epcoh took: 138.61303687095642\n","\n","Running Validation...\n","  Accuracy: 0.78\n","  Validation Loss: 0.92\n","  Validation took: 5.052229881286621\n","\n","Training complete!\n","Total training took 1149.1925280094147 (h:mm:ss)\n","Number of test sentences: 6,000\n","\n","5500\n","{'accuracy': 0.7716666460037231, 'f1': 0.7723495960235596, 'tnr': 0.7798444628715515, 'tpr': 0.7637200355529785, 'mcc': 0.5435274839401245}\n","Original:  [**2122-11-10**] 1:45 PM\n"," LUMBAR SP,SINGLE FILM IN O.R.                                   Clip # [**Clip Number (Radiology) 49470**]\n"," Reason: FUSION ANTERIOR THORACOLUMBAR T-10 L-5\n"," ______________________________________________________________________________\n","                                 FINAL REPORT\n","\n"," LUMBAR SPINE FILMS WHICH WERE MADE IN THE OPERATION ROOM.\n","\n"," INDICATION:  62 year old female which underwent an operation for fusion of\n"," anterior thoracolumbar.\n","\n"," FINDINGS:  Three wet films are available, named by number one, two and three.\n"," The films are suboptimal.\n","\n"," Film number one.  A suction tube is projecting over the lumbar spine.  A\n"," foreign body is projecting over D11.  Multiple surgical clips are projecting\n"," left to the lumbar spine.\n","\n"," There is a scoliosis convex to the right of the lumbar spine.\n","\n"," A spinal needle is projecting over the level of L4-5 in the left part.\n","\n"," Film number two.  The status post placement of disc artificial spacers between\n"," L2 and L3, L3 and L4, and L4 and L5.  Multiple surgical clips are projecting\n"," on the spine bilaterally, especially on the left.\n","\n"," An artificial ring is observed projecting over the pelvis.\n","\n"," Film number three:\n","\n"," Very suboptimal film.  Three disc spacers are observed as previously described\n"," and also there are multiple surgical clips as previously described.\n","\n"," IMPRESSION:  Status post spine surgery with disc spacers in three levels, as\n"," described above.\n","\n","\n","\n","\n","Token IDs: tensor([  101,   164,   115,   115, 19538,  1477,   118,  1429,   118,  1275,\n","          115,   115,   166,   122,   131,  2532,  9852,   181, 25509,  1197,\n","          188,  1643,   117,  1423,  1273,  1107,   184,   119,   187,   119,\n","        13500,   108,   164,   115,   115, 13500,  1295,   113,  2070,  6360,\n","          114,  3927, 24766,  1568,   115,   115,   166,  2255,   131, 11970,\n","        16557, 24438,  6533,  2528,  7776,  6824,   189,   118,  1275,   181,\n","          118,   126,   168,   168,   168,   168,   168,   168,   168,   168,\n","          168,   168,   168,   168,   168,   168,   168,   168,   168,   168,\n","          168,   168,   168,   168,   168,   168,   168,   168,   168,   168,\n","          168,   168,   168,   168,   168,   168,   168,   168,   168,   168,\n","          168,   168,   168,   168,   168,   168,   168,   168,   168,   168,\n","          168,   168,   168,   168,   168,   168,   168,   168,   168,   168,\n","          168,   168,   168,   168,   168,   168,   168,   168,   168,   168,\n","          168,   168,   168,   168,   168,   168,   168,   168,   168,   168,\n","         1509,  2592,   181, 25509,  1197,  8340,  2441,  1134,  1127,  1189,\n","         1107,  1103,  2805,  1395,   119, 12754,   131,  5073,  1214,  1385,\n","         2130,  1134,  9315,  1126,  2805,  1111, 11970,  1104, 16557, 24438,\n","         6533,  2528,  7776,  6824,   119,  9505,   131,  1210,  4375,  2441,\n","         1132,  1907,   117,  1417,  1118,  1295,  1141,   117,  1160,  1105,\n","         1210,   119,  1103,  2441,  1132,  4841,  4184,  3121,  7435,   119,\n","         1273,  1295,  1141,   119,   170, 28117,  5796,  7159,  1110, 20266,\n","         1166,  1103,   181, 25509,  1197,  8340,   119,   170,  2880,  1404,\n","         1110, 20266,  1166,   173, 14541,   119,  2967, 13467, 16973,  1132,\n","        20266,  1286,  1106,  1103,   181, 25509,  1197,  8340,   119,  1175,\n","         1110,   170,   188,  2528,  9436,  4863, 20137,  1106,  1103,  1268,\n","         1104,  1103,   181, 25509,  1197,  8340,   119,   170, 19245, 13864,\n","         1110, 20266,  1166,  1103,  1634,  1104,   181,  1527,   118,   126,\n","         1107,  1103,  1286,  1226,   119,  1273,  1295,  1160,   119,  1103,\n","         2781,  2112, 12693,  1104,  6187,  8246,  2000,  1733,  1206,   181,\n","         1477,  1105,   181,  1495,   117,   181,  1495,  1105,   181,  1527,\n","          117,  1105,   181,  1527,  1105,   181,  1571,   119,  2967, 13467,\n","        16973,  1132, 20266,  1113,  1103,  8340, 20557,  1193,   117,  2108,\n","         1113,  1103,  1286,   119,  1126,  8246,  3170,  1110,  4379, 20266,\n","         1166,  1103,   185,  1883,  9356,   119,  1273,  1295,  1210,   131,\n","         1304,  4841,  4184,  3121,  7435,  1273,   119,  1210,  6187,  2000,\n","         1733,  1132,  4379,  1112,  2331,  1758,  1105,  1145,  1175,  1132,\n","         2967, 13467, 16973,  1112,  2331,  1758,   119,  8351,   131,  2781,\n","         2112,  8340,  6059,  1114,  6187,  2000,  1733,  1107,  1210,  3001,\n","          117,  1112,  1758,  1807,   119,   102,     0,     0,     0,     0,\n","            0,     0,     0,     0,     0,     0,     0,     0,     0,     0,\n","            0,     0,     0,     0,     0,     0,     0,     0,     0,     0,\n","            0,     0,     0,     0,     0,     0,     0,     0,     0,     0,\n","            0,     0,     0,     0,     0,     0,     0,     0,     0,     0,\n","            0,     0,     0,     0,     0,     0,     0,     0,     0,     0,\n","            0,     0,     0,     0,     0,     0,     0,     0,     0,     0,\n","            0,     0,     0,     0,     0,     0,     0,     0,     0,     0,\n","            0,     0,     0,     0,     0,     0,     0,     0,     0,     0,\n","            0,     0,     0,     0,     0,     0,     0,     0,     0,     0,\n","            0,     0,     0,     0,     0,     0,     0,     0,     0,     0,\n","            0,     0,     0,     0,     0,     0,     0,     0,     0,     0,\n","            0,     0,     0,     0,     0,     0,     0,     0,     0,     0,\n","            0,     0])\n","5,400 training samples\n","  600 validation samples\n"]},{"name":"stderr","output_type":"stream","text":["Some weights of BertForSequenceClassification were not initialized from the model checkpoint at emilyalsentzer/Bio_ClinicalBERT and are newly initialized: ['classifier.bias', 'classifier.weight']\n","You should probably TRAIN this model on a down-stream task to be able to use it for predictions and inference.\n"]},{"name":"stdout","output_type":"stream","text":["\n","======== Epoch 1 / 8 ========\n","Training...\n","  Batch    40  of    338.    Elapsed: 17.825945615768433.\n","  Batch    80  of    338.    Elapsed: 35.69278860092163.\n","  Batch   120  of    338.    Elapsed: 53.56540250778198.\n","  Batch   160  of    338.    Elapsed: 71.45543003082275.\n","  Batch   200  of    338.    Elapsed: 89.34551286697388.\n","  Batch   240  of    338.    Elapsed: 107.24720001220703.\n","  Batch   280  of    338.    Elapsed: 125.15258431434631.\n","  Batch   320  of    338.    Elapsed: 143.0763759613037.\n","\n","  Average training loss: 0.60\n","  Training epcoh took: 150.93037295341492\n","\n","Running Validation...\n","  Accuracy: 0.70\n","  Validation Loss: 0.54\n","  Validation took: 5.499626159667969\n","\n","======== Epoch 2 / 8 ========\n","Training...\n","  Batch    40  of    338.    Elapsed: 17.918309450149536.\n","  Batch    80  of    338.    Elapsed: 35.84137725830078.\n","  Batch   120  of    338.    Elapsed: 53.76819467544556.\n","  Batch   160  of    338.    Elapsed: 71.69459915161133.\n","  Batch   200  of    338.    Elapsed: 89.61180377006531.\n","  Batch   240  of    338.    Elapsed: 107.52180242538452.\n","  Batch   280  of    338.    Elapsed: 125.43860197067261.\n","  Batch   320  of    338.    Elapsed: 143.36281752586365.\n","\n","  Average training loss: 0.51\n","  Training epcoh took: 151.22394323349\n","\n","Running Validation...\n","  Accuracy: 0.76\n","  Validation Loss: 0.51\n","  Validation took: 5.496549844741821\n","\n","======== Epoch 3 / 8 ========\n","Training...\n","  Batch    40  of    338.    Elapsed: 17.907958984375.\n","  Batch    80  of    338.    Elapsed: 35.824809551239014.\n","  Batch   120  of    338.    Elapsed: 53.73082160949707.\n","  Batch   160  of    338.    Elapsed: 71.660391330719.\n","  Batch   200  of    338.    Elapsed: 89.57711791992188.\n","  Batch   240  of    338.    Elapsed: 107.49563598632812.\n","  Batch   280  of    338.    Elapsed: 125.43528056144714.\n","  Batch   320  of    338.    Elapsed: 143.33216905593872.\n","\n","  Average training loss: 0.40\n","  Training epcoh took: 151.18334698677063\n","\n","Running Validation...\n","  Accuracy: 0.77\n","  Validation Loss: 0.49\n","  Validation took: 5.506589412689209\n","\n","======== Epoch 4 / 8 ========\n","Training...\n","  Batch    40  of    338.    Elapsed: 17.936084270477295.\n","  Batch    80  of    338.    Elapsed: 35.8566632270813.\n","  Batch   120  of    338.    Elapsed: 53.767420053482056.\n","  Batch   160  of    338.    Elapsed: 71.70037055015564.\n","  Batch   200  of    338.    Elapsed: 89.6014974117279.\n","  Batch   240  of    338.    Elapsed: 107.52747106552124.\n","  Batch   280  of    338.    Elapsed: 125.4485194683075.\n","  Batch   320  of    338.    Elapsed: 143.3479926586151.\n","\n","  Average training loss: 0.28\n","  Training epcoh took: 151.20132207870483\n","\n","Running Validation...\n","  Accuracy: 0.79\n","  Validation Loss: 0.52\n","  Validation took: 5.50291109085083\n","\n","======== Epoch 5 / 8 ========\n","Training...\n","  Batch    40  of    338.    Elapsed: 17.924365997314453.\n","  Batch    80  of    338.    Elapsed: 35.8314094543457.\n","  Batch   120  of    338.    Elapsed: 53.75015616416931.\n","  Batch   160  of    338.    Elapsed: 71.66294145584106.\n","  Batch   200  of    338.    Elapsed: 89.5792121887207.\n","  Batch   240  of    338.    Elapsed: 107.50506353378296.\n","  Batch   280  of    338.    Elapsed: 125.4325749874115.\n","  Batch   320  of    338.    Elapsed: 143.34298467636108.\n","\n","  Average training loss: 0.21\n","  Training epcoh took: 151.20662236213684\n","\n","Running Validation...\n","  Accuracy: 0.79\n","  Validation Loss: 0.59\n","  Validation took: 5.502627611160278\n","\n","======== Epoch 6 / 8 ========\n","Training...\n","  Batch    40  of    338.    Elapsed: 17.92379093170166.\n","  Batch    80  of    338.    Elapsed: 35.84952688217163.\n","  Batch   120  of    338.    Elapsed: 53.76419496536255.\n","  Batch   160  of    338.    Elapsed: 71.68161058425903.\n","  Batch   200  of    338.    Elapsed: 89.59333348274231.\n","  Batch   240  of    338.    Elapsed: 107.5235104560852.\n","  Batch   280  of    338.    Elapsed: 125.43983125686646.\n","  Batch   320  of    338.    Elapsed: 143.3632504940033.\n","\n","  Average training loss: 0.17\n","  Training epcoh took: 151.21818923950195\n","\n","Running Validation...\n","  Accuracy: 0.80\n","  Validation Loss: 0.68\n","  Validation took: 5.509702920913696\n","\n","======== Epoch 7 / 8 ========\n","Training...\n","  Batch    40  of    338.    Elapsed: 17.928868770599365.\n","  Batch    80  of    338.    Elapsed: 35.832595348358154.\n","  Batch   120  of    338.    Elapsed: 53.746058225631714.\n","  Batch   160  of    338.    Elapsed: 71.64592337608337.\n","  Batch   200  of    338.    Elapsed: 89.56091928482056.\n","  Batch   240  of    338.    Elapsed: 107.49210381507874.\n","  Batch   280  of    338.    Elapsed: 125.40117120742798.\n","  Batch   320  of    338.    Elapsed: 143.30805087089539.\n","\n","  Average training loss: 0.13\n","  Training epcoh took: 151.155588388443\n","\n","Running Validation...\n","  Accuracy: 0.79\n","  Validation Loss: 0.77\n","  Validation took: 5.507224798202515\n","\n","======== Epoch 8 / 8 ========\n","Training...\n","  Batch    40  of    338.    Elapsed: 17.921666145324707.\n","  Batch    80  of    338.    Elapsed: 35.80952310562134.\n","  Batch   120  of    338.    Elapsed: 53.73573040962219.\n","  Batch   160  of    338.    Elapsed: 71.63852715492249.\n","  Batch   200  of    338.    Elapsed: 89.5679144859314.\n","  Batch   240  of    338.    Elapsed: 107.47667717933655.\n","  Batch   280  of    338.    Elapsed: 125.37474632263184.\n","  Batch   320  of    338.    Elapsed: 143.28741979599.\n","\n","  Average training loss: 0.11\n","  Training epcoh took: 151.1346082687378\n","\n","Running Validation...\n","  Accuracy: 0.79\n","  Validation Loss: 0.80\n","  Validation took: 5.525268316268921\n","\n","Training complete!\n","Total training took 1253.323186159134 (h:mm:ss)\n","Number of test sentences: 6,000\n","\n","6000\n","{'accuracy': 0.7615000009536743, 'f1': 0.7623318433761597, 'tnr': 0.7690226435661316, 'tpr': 0.7541899681091309, 'mcc': 0.523172914981842}\n","{500: {'accuracy': 0.6190000176429749, 'f1': 0.620517909526825, 'tnr': 0.6239432096481323, 'tpr': 0.6141965389251709, 'mcc': 0.23812001943588257}, 1000: {'accuracy': 0.6626666784286499, 'f1': 0.6615384817123413, 'tnr': 0.6756848096847534, 'tpr': 0.6500164270401001, 'mcc': 0.32573962211608887}, 1500: {'accuracy': 0.6610000133514404, 'f1': 0.6551373600959778, 'tnr': 0.687859296798706, 'tpr': 0.6348997950553894, 'mcc': 0.32310354709625244}, 2000: {'accuracy': 0.6980000138282776, 'f1': 0.7095223069190979, 'tnr': 0.667906641960144, 'tpr': 0.7272428274154663, 'mcc': 0.3959462344646454}, 2500: {'accuracy': 0.6961666941642761, 'f1': 0.696722686290741, 'tnr': 0.7044301629066467, 'tpr': 0.6881366968154907, 'mcc': 0.39254888892173767}, 3000: {'accuracy': 0.7138333320617676, 'f1': 0.7084394693374634, 'tnr': 0.7429827451705933, 'tpr': 0.6855077147483826, 'mcc': 0.4290120601654053}, 3500: {'accuracy': 0.7319999933242798, 'f1': 0.7293840646743774, 'tnr': 0.7524518370628357, 'tpr': 0.7121261954307556, 'mcc': 0.46479377150535583}, 4000: {'accuracy': 0.7319999933242798, 'f1': 0.7320892810821533, 'tnr': 0.7423064112663269, 'tpr': 0.72198486328125, 'mcc': 0.4642869532108307}, 4500: {'accuracy': 0.7293333411216736, 'f1': 0.7319247126579285, 'tnr': 0.7301318645477295, 'tpr': 0.7285573482513428, 'mcc': 0.45864784717559814}, 5000: {'accuracy': 0.7483333349227905, 'f1': 0.7524589896202087, 'tnr': 0.7423064112663269, 'tpr': 0.7541899681091309, 'mcc': 0.49653497338294983}, 5500: {'accuracy': 0.7716666460037231, 'f1': 0.7723495960235596, 'tnr': 0.7798444628715515, 'tpr': 0.7637200355529785, 'mcc': 0.5435274839401245}, 6000: {'accuracy': 0.7615000009536743, 'f1': 0.7623318433761597, 'tnr': 0.7690226435661316, 'tpr': 0.7541899681091309, 'mcc': 0.523172914981842}}\n"]}],"source":["\n","def sample(train, num_samples, portion_of_positive):\n","  train = train.copy(deep=True)\n","  postion_of_negative = 1 - portion_of_positive\n","  ss0 = int(num_samples*postion_of_negative)\n","  ss1 = int(num_samples*portion_of_positive)\n","\n","  s0 = train.label[train.label.eq(0)].sample(ss0).index\n","  s1 = train.label[train.label.eq(1)].sample(ss1).index\n","\n","  new_train = train.loc[s0.union(s1)]\n","  return new_train\n","\n","full_data_for_classfier = get_data_for_classifier()\n","train_df, test_df = get_train_test_split(full_data_for_classfier, 6000, 0.5, 6000)\n","\n","\n","epochs = 4\n","\n","\n","results = {}\n","for number_of_train_samples in [500, 1000, 1500, 2000, 2500, 3000, 3500, 4000, 4500, 5000, 5500, 6000]:\n","  if number_of_train_samples == len(train_df):\n","    new_train = train_df.copy(deep=True)\n","  else:\n","    new_train = sample(train_df, number_of_train_samples, 0.5)\n","\n","  train_dataset, validation_dataset = get_loader(new_train)\n","  train_dataloader, validation_dataloader = define_dataloaders(train_dataset, validation_dataset)\n","\n","  total_steps = len(train_dataloader) * epochs\n","  model = define_model()\n","  optimiser, scheduler = define_optimiser_and_scheduler(model)\n","  train(model, train_dataloader, validation_dataloader, optimiser, scheduler)\n","  test_loader = get_test_dataset(test_df)\n","  results[number_of_train_samples] = evaluate(model, test_loader)\n","  print(number_of_train_samples)\n","  print(results[number_of_train_samples])\n","print(results)"]},{"cell_type":"code","execution_count":null,"metadata":{"colab":{"base_uri":"https://localhost:8080/","height":310},"executionInfo":{"elapsed":2039,"status":"ok","timestamp":1711388288985,"user":{"displayName":"Robert Cobb","userId":"08992261826785514419"},"user_tz":0},"id":"sf4d1mRdjVkm","outputId":"42ef156d-88f5-4edd-bad6-d2d4bc1c4826"},"outputs":[{"name":"stdout","output_type":"stream","text":["{500: {'accuracy': 0.6190000176429749, 'f1': 0.620517909526825, 'tnr': 0.6239432096481323, 'tpr': 0.6141965389251709, 'mcc': 0.23812001943588257}, 1000: {'accuracy': 0.6626666784286499, 'f1': 0.6615384817123413, 'tnr': 0.6756848096847534, 'tpr': 0.6500164270401001, 'mcc': 0.32573962211608887}, 1500: {'accuracy': 0.6610000133514404, 'f1': 0.6551373600959778, 'tnr': 0.687859296798706, 'tpr': 0.6348997950553894, 'mcc': 0.32310354709625244}, 2000: {'accuracy': 0.6980000138282776, 'f1': 0.7095223069190979, 'tnr': 0.667906641960144, 'tpr': 0.7272428274154663, 'mcc': 0.3959462344646454}, 2500: {'accuracy': 0.6961666941642761, 'f1': 0.696722686290741, 'tnr': 0.7044301629066467, 'tpr': 0.6881366968154907, 'mcc': 0.39254888892173767}, 3000: {'accuracy': 0.7138333320617676, 'f1': 0.7084394693374634, 'tnr': 0.7429827451705933, 'tpr': 0.6855077147483826, 'mcc': 0.4290120601654053}, 3500: {'accuracy': 0.7319999933242798, 'f1': 0.7293840646743774, 'tnr': 0.7524518370628357, 'tpr': 0.7121261954307556, 'mcc': 0.46479377150535583}, 4000: {'accuracy': 0.7319999933242798, 'f1': 0.7320892810821533, 'tnr': 0.7423064112663269, 'tpr': 0.72198486328125, 'mcc': 0.4642869532108307}, 4500: {'accuracy': 0.7293333411216736, 'f1': 0.7319247126579285, 'tnr': 0.7301318645477295, 'tpr': 0.7285573482513428, 'mcc': 0.45864784717559814}, 5000: {'accuracy': 0.7483333349227905, 'f1': 0.7524589896202087, 'tnr': 0.7423064112663269, 'tpr': 0.7541899681091309, 'mcc': 0.49653497338294983}, 5500: {'accuracy': 0.7716666460037231, 'f1': 0.7723495960235596, 'tnr': 0.7798444628715515, 'tpr': 0.7637200355529785, 'mcc': 0.5435274839401245}, 6000: {'accuracy': 0.7615000009536743, 'f1': 0.7623318433761597, 'tnr': 0.7690226435661316, 'tpr': 0.7541899681091309, 'mcc': 0.523172914981842}}\n"]},{"data":{"image/png":"iVBORw0KGgoAAAANSUhEUgAABNoAAADvCAYAAAAkc7lHAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjcuMSwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/bCgiHAAAACXBIWXMAAA9hAAAPYQGoP6dpAABvAElEQVR4nO3deViUVf8G8HtmgBnWAVkGBBQV3BUU1HCvKFp+pmWmtqBU9paSGuX2VtoqWr1mmeXbW2aLqZW2Wprhkia5oLik4oossu/rwMyc3x8DoyOLgAPDwP25rrmcOXOeZ84zcsPw5TzPkQghBIiIiIiIiIiIiOimSM09ACIiIiIiIiIiovaAhTYiIiIiIiIiIiITYKGNiIiIiIiIiIjIBFhoIyIiIiIiIiIiMgEW2oiIiIiIiIiIiEyAhTYiIiIiIiIiIiITYKGNiIiIiIiIiIjIBFhoIyIiIiIiIiIiMgEW2oiIiIiIiIiIiEyAhTYiImqUQ4cOYfjw4bC3t4dEIkFCQoK5h0RERERERNSmsNBGREQ3VFVVhUmTJiEvLw/vvvsuvvzyS6hUKixcuBC33norHB0dIZFIsHv3bnMPlYiIyOLt378fr7zyCgoKCsw9FCJqIuaXWGgjIqIbunDhAi5fvowXXngBTz31FB599FEkJiZi+fLlSEtLw4ABA8w9RCIionZj//79ePXVV/mLOpEFYn6JhTZq80pLS809BKIOLysrCwDg7OxsaAsODkZubi7Onj2L6OhoM42MiIiIGoOfqYksF/NrWVho64AuX76MmTNnolevXrC1tYWrqysmTZqEpKSkWn0LCgrw3HPPwc/PD3K5HD4+PoiIiEBOTo6hT0VFBV555RX07NkTCoUCXl5eeOCBB3DhwgUAwO7du+s8pSwpKQkSiQTr1q0ztE2fPh0ODg64cOEC7rnnHjg6OuKRRx4BAOzduxeTJk1Cly5dIJfL4evri+eeew7l5eW1xn3mzBk89NBDcHd3h62tLXr16oUXX3wRALBr1y5IJBJ8//33tbb7+uuvIZFIEBcX19S3lajdmj59OsaMGQMAmDRpEiQSCcaOHQtHR0d06tTJzKMjoldeeQUSiQRnz57Fo48+CqVSCXd3d7z88ssQQiAlJQXjx4+Hk5MTPD098Z///Mdo+xv9HAcAnU6H9957DwMGDIBCoYC7uzvuuusuHD58uLUPl6jde+WVVzBv3jwAQLdu3SCRSCCRSAyfnaOiovDDDz+gf//+kMvl6NevH7Zt21ZrHxKJBKdOncLDDz8MFxcXjBw50hyHQ9ShNCa/69evR69evaBQKBAcHIw///yz1j6YX8tmZe4BUOs7dOgQ9u/fjylTpsDHxwdJSUn46KOPMHbsWJw6dQp2dnYAgJKSEowaNQqnT5/G448/jsGDByMnJwc//fQTUlNT4ebmBq1Wi//7v/9DbGwspkyZgjlz5qC4uBg7duzAyZMn0aNHjyaPT6PRIDw8HCNHjsQ777xjGM+3336LsrIyPPPMM3B1dcXBgwexatUqpKam4ttvvzVsf/z4cYwaNQrW1tZ46qmn4OfnhwsXLuDnn3/Gm2++ibFjx8LX1xfr16/H/fffb/Ta69evR48ePRAaGnoT7zBR+/Kvf/0L3t7eWLp0KWbPno0hQ4ZApVKZe1hEdJ3JkyejT58+WLZsGbZu3Yo33ngDnTp1wn//+1/cdtttWL58OdavX48XXngBQ4YMwejRoxv9c/yJJ57AunXrcPfdd+PJJ5+ERqPB3r178ffffyMkJMTMR07UvjzwwAM4e/YsNmzYgHfffRdubm4AAHd3dwDAvn37sGXLFsycOROOjo54//33MXHiRCQnJ8PV1dVoX5MmTUJAQACWLl0KIUSrHwtRR3Oj/O7ZswebNm3C7NmzIZfL8eGHH+Kuu+7CwYMH0b9/f6N9Mb8WTFCHU1ZWVqstLi5OABBffPGFoW3x4sUCgNiyZUut/jqdTgghxNq1awUAsWLFinr77Nq1SwAQu3btMnr+0qVLAoD47LPPDG3Tpk0TAMTChQsbNe6YmBghkUjE5cuXDW2jR48Wjo6ORm3XjkcIIRYtWiTkcrkoKCgwtGVlZQkrKyuxZMmSWq9D1NHV5Pjbb7+t8/lvv/22zpwTUctbsmSJACCeeuopQ5tGoxE+Pj5CIpGIZcuWGdrz8/OFra2tmDZtmhCicT/Hd+7cKQCI2bNn19uHiEzr7bffFgDEpUuXjNoBCBsbG3H+/HlD27FjxwQAsWrVKkNbzfeFqVOnttaQiahaQ/kFIA4fPmxou3z5slAoFOL+++83tDG/lo+njnZAtra2hvtVVVXIzc2Fv78/nJ2dceTIEcNzmzdvRmBgYK1ZXwAgkUgMfdzc3PDss8/W26c5nnnmmQbHXVpaipycHAwfPhxCCBw9ehQAkJ2djT///BOPP/44unTpUu94IiIioFar8d133xnaNm3aBI1Gg0cffbTZ4yYiIjKXJ5980nBfJpMhJCQEQgg88cQThnZnZ2f06tULFy9eBNC4n+ObN2+GRCLBkiVL6u1DRK0nLCzM6KyRgQMHwsnJyZDraz399NOtOTQiuoHQ0FAEBwcbHnfp0gXjx4/H9u3bodVqjfoyv5aLhbYOqLy8HIsXL4avry/kcjnc3Nzg7u6OgoICFBYWGvpduHCh1vTV6124cAG9evWClZXpzkK2srKCj49Prfbk5GRMnz4dnTp1goODA9zd3Q3XjaoZd80HjBuNu3fv3hgyZAjWr19vaFu/fj1uueUW+Pv7m+pQiIiIWs31f2BSKpVQKBSG01aubc/PzwfQuJ/jFy5cQOfOnXlNRqI24vqsA4CLi4sh19fq1q1bawyJiBopICCgVlvPnj1RVlaG7Oxso3bm13LxGm0d0LPPPovPPvsMc+fORWhoKJRKJSQSCaZMmQKdTmfy16vvr93XV+xryOVySKXSWn3vuOMO5OXlYcGCBejduzfs7e2RlpaG6dOnN2vcERERmDNnDlJTU6FWq/H333/jgw8+aPJ+iIiI2gKZTNaoNgC81guRBWtKrq89I4SILAvza7lYaOuAvvvuO0ybNs1o1bGKigoUFBQY9evRowdOnjzZ4L569OiBAwcOoKqqCtbW1nX2cXFxAYBa+798+XKjx3zixAmcPXsWn3/+OSIiIgztO3bsMOrXvXt3ALjhuAFgypQpiI6OxoYNG1BeXg5ra2tMnjy50WMiIiKydI35Od6jRw9s374deXl5nNVG1Ep4WjaR5Woov+fOnavVdvbsWdjZ2RkWTCDLx1NHOyCZTFbrL16rVq2qNcNs4sSJOHbsGL7//vta+6jZfuLEicjJyalzJlhNn65du0Imk9VatvjDDz9s0piv3WfN/ffee8+on7u7O0aPHo21a9ciOTm5zvHUcHNzw913342vvvoK69evx1133VXr9BoiIqL2rDE/xydOnAghBF599dV6+xCRadnb2wOo/YdqImr7GspvXFyc0XXRU1JS8OOPP+LOO++sd7YqWR7OaOuA/u///g9ffvkllEol+vbti7i4OPzxxx+1lgOfN28evvvuO0yaNAmPP/44goODkZeXh59++glr1qxBYGAgIiIi8MUXXyA6OhoHDx7EqFGjUFpaij/++AMzZ87E+PHjoVQqMWnSJKxatQoSiQQ9evTAL7/8gqysrEaPuXfv3ujRowdeeOEFpKWlwcnJCZs3b67zWhTvv/8+Ro4cicGDB+Opp55Ct27dkJSUhK1btyIhIcGob0REBB588EEAwOuvv970N5Oog3vjjTcAAP/88w8A4Msvv8S+ffsAAC+99JLZxkVEjdOYn+O33norHnvsMbz//vs4d+4c7rrrLuh0Ouzduxe33noroqKizH0YRO1OzcXSX3zxRUyZMgXW1tYYN26cmUdFRI3RUH779++P8PBwzJ49G3K53DD5pK4/ZpHlYqGtA3rvvfcgk8mwfv16VFRUYMSIEfjjjz8QHh5u1M/BwQF79+7FkiVL8P333+Pzzz+Hh4cHbr/9dsNiBTKZDL/++ivefPNNfP3119i8eTNcXV0xcuRIDBgwwLCvVatWoaqqCmvWrIFcLsdDDz2Et99++4aLFtSwtrbGzz//jNmzZyMmJgYKhQL3338/oqKiEBgYaNQ3MDAQf//9N15++WV89NFHqKioQNeuXfHQQw/V2u+4cePg4uICnU6H++67r6lvJVGH9/LLLxs9Xrt2reE+C21EbV9jf45/9tlnGDhwID799FPMmzcPSqUSISEhGD58uBlHT9R+DRkyBK+//jrWrFmDbdu2QafT4dKlS+YeFhE1QkP5HTNmDEJDQ/Hqq68iOTkZffv2xbp16zBw4EAzj5pMSSI45586MI1Gg86dO2PcuHH49NNPzT0cIiIiIiIiaockEglmzZrFBfg6AF6jjTq0H374AdnZ2UYLLBARERERERERNQdPHaUO6cCBAzh+/Dhef/11DBo0CGPGjDH3kIiIiIiIiIjIwnFGG3VIH330EZ555hl4eHjgiy++MPdwiIiIiIiIiKgdMGuh7c8//8S4cePQuXNnSCQS/PDDDzfcZvfu3Rg8eDDkcjn8/f2xbt26Fh8ntT/r1q2DRqPB4cOHG70gAxEREREREVFzCCF4fbYOwqyFttLSUgQGBmL16tWN6n/p0iXce++9uPXWW5GQkIC5c+fiySefxPbt21t4pERERERERERERA1rM6uOSiQSfP/995gwYUK9fRYsWICtW7fi5MmThrYpU6agoKAA27Zta4VREhERERERERER1c2iFkOIi4tDWFiYUVt4eDjmzp1b7zZqtRpqtdrwWKfTIS8vD66urpBIJC01VCKLJYRAcXExOnfuDKnU/JdxZIaJGo/5JbJszDCR5WJ+iSybKTNsUYW2jIwMqFQqozaVSoWioiKUl5fD1ta21jYxMTF49dVXW2uIRO1GSkoKfHx8zD0MZpioGZhfIsvGDBNZLuaXyLKZIsMWdepoz549ERkZiUWLFhnafv31V9x7770oKyurs9B2fSW/sLAQXbp0QUpKCpycnEx6DETtQVFREXx9fVFQUAClUmnu4TDDRE3A/BJZNmaYyHIxv0SWzZQZtqgZbZ6ensjMzDRqy8zMhJOTU51FNgCQy+WQy+W12p2cnPgNhqgBbWVKOTNM1HTML5FlY4aJLBfzS2TZTJFh85883gShoaGIjY01atuxYwdCQ0PNNCIiIiIiIiIiIiI9sxbaSkpKkJCQgISEBADApUuXkJCQgOTkZADAokWLEBERYej/9NNP4+LFi5g/fz7OnDmDDz/8EN988w2ee+45cwyfiIiIiIiIiIjIwKyFtsOHD2PQoEEYNGgQACA6OhqDBg3C4sWLAQDp6emGohsAdOvWDVu3bsWOHTsQGBiI//znP/jkk08QHh5ulvETERERERERERHVMOs12saOHYuG1mJYt25dndscPXq0BUdFRERERERERETUdBZ1jTYiIiIiIiIiIqK2ioU2IiIiIiIiIiIiE2ChjYiIiIiIiIiIyARYaCMiIiIiIiIiIjIBFtqIiIiIiIiIiIhMgIU2IiIiIiIiIiIiEzB7oW316tXw8/ODQqHAsGHDcPDgwXr7VlVV4bXXXkOPHj2gUCgQGBiIbdu2teJoiYiIiIiIiIiI6mbWQtumTZsQHR2NJUuW4MiRIwgMDER4eDiysrLq7P/SSy/hv//9L1atWoVTp07h6aefxv3334+jR4+28siJiIiIiIiIiIiMmbXQtmLFCsyYMQORkZHo27cv1qxZAzs7O6xdu7bO/l9++SX+/e9/45577kH37t3xzDPP4J577sF//vOfVh45ERERERERERGRMStzvXBlZSXi4+OxaNEiQ5tUKkVYWBji4uLq3EatVkOhUBi12draYt++ffW+jlqthlqtNjwuKiq6yZETUWtihoksF/NLZNmYYSLLxfwSmY/ZZrTl5ORAq9VCpVIZtatUKmRkZNS5TXh4OFasWIFz585Bp9Nhx44d2LJlC9LT0+t9nZiYGCiVSsPN19fXpMdBRC2LGSayXMwvkWVjhoksF/NLZD5mXwyhKd577z0EBASgd+/esLGxQVRUFCIjIyGV1n8YixYtQmFhoeGWkpLSiiMmopvFDBNZLuaXyLIxw0SWi/klMh+znTrq5uYGmUyGzMxMo/bMzEx4enrWuY27uzt++OEHVFRUIDc3F507d8bChQvRvXv3el9HLpdDLpebdOxE1HqYYSLLxfwSWTZmmMhyMb9E5mO2GW02NjYIDg5GbGysoU2n0yE2NhahoaENbqtQKODt7Q2NRoPNmzdj/PjxLT1cIiIiIiIiIiKiBpltRhsAREdHY9q0aQgJCcHQoUOxcuVKlJaWIjIyEgAQEREBb29vxMTEAAAOHDiAtLQ0BAUFIS0tDa+88gp0Oh3mz59vzsMgIiIiIiIiIiIyb6Ft8uTJyM7OxuLFi5GRkYGgoCBs27bNsEBCcnKy0fXXKioq8NJLL+HixYtwcHDAPffcgy+//BLOzs5mOgIiIiIiIiIiIiI9sxbaACAqKgpRUVF1Prd7926jx2PGjMGpU6daYVRERERERERERERNY1GrjhIREREREREREbVVLLQRERERERERERGZAAttREREREREREREJsBCGxERERERERERkQmw0EZERERERERERGQCLLQRERERERERERGZgNkLbatXr4afnx8UCgWGDRuGgwcPNth/5cqV6NWrF2xtbeHr64vnnnsOFRUVrTRaIiIiIiIiIiKiupm10LZp0yZER0djyZIlOHLkCAIDAxEeHo6srKw6+3/99ddYuHAhlixZgtOnT+PTTz/Fpk2b8O9//7uVR05ERERERERERGTMrIW2FStWYMaMGYiMjETfvn2xZs0a2NnZYe3atXX2379/P0aMGIGHH34Yfn5+uPPOOzF16tQbzoIjIiIiIiIiIiJqaVbmeuHKykrEx8dj0aJFhjapVIqwsDDExcXVuc3w4cPx1Vdf4eDBgxg6dCguXryIX3/9FY899li9r6NWq6FWqw2Pi4qKTHcQRNTimGEiy8X8Elk2ZpjIcjG/ROZjthltOTk50Gq1UKlURu0qlQoZGRl1bvPwww/jtddew8iRI2FtbY0ePXpg7NixDZ46GhMTA6VSabj5+vqa9DiIqGUxw0SWi/klsmzMMJHlYn6JzMfsiyE0xe7du7F06VJ8+OGHOHLkCLZs2YKtW7fi9ddfr3ebRYsWobCw0HBLSUlpxRET0c1ihoksF/NLZNmYYSLLxfwSmY/ZTh11c3ODTCZDZmamUXtmZiY8PT3r3Obll1/GY489hieffBIAMGDAAJSWluKpp57Ciy++CKm0dt1QLpdDLpeb/gCIqFUww0SWi/klsmzMMFm6Kq0Ou85kYVdiFuxsrOClVMBLaQsvZwW8lAp4OCogk0rMPcwWwfwSmY/ZCm02NjYIDg5GbGwsJkyYAADQ6XSIjY1FVFRUnduUlZXVKqbJZDIAgBCiRcdLZE5CCFzKKUWJWgOVkwJuDvJ2+6GAqL3RaHU4n12C81klcFJYQ+WkgMpJDqWtNSQS5piIiMjULmaX4JvDqdh8JBXZxep6+8mkEqgc5fBUKuDlbAsvp+p/lfpCXGdnW37uJqImM1uhDQCio6Mxbdo0hISEYOjQoVi5ciVKS0sRGRkJAIiIiIC3tzdiYmIAAOPGjcOKFSswaNAgDBs2DOfPn8fLL7+McePGGQpuRO2BVidwJqMIBy/l4eClPBxKykNOSaXheakEcHeUw9NJAZWTAp7K6n8N9+VQOSngqLA241EQdTw6nUBSbilOpBXiWEohjqcW4J8rRSiv0tbqK7eSGnLr4XQ1zyqlwvChX+WkgMKaP9+IiIhupLxSi19PpGPToRQcTMoztLs52GBcYGdYy6S4UlCOjMIKpBdWIKOoAlqdwJXCClwprACSC+rcr5VUYvi8/fr4/ujb2amVjoiILJVZC22TJ09GdnY2Fi9ejIyMDAQFBWHbtm2GBRKSk5ONZrC99NJLkEgkeOmll5CWlgZ3d3eMGzcOb775prkOgcgkKjU6nEgrwMFL+Th4KReHk/JRrNYY9bGxksLFzho5JZXQ6gQyi9TILFIDKKx3v/Y2MqiU1QW46l/gA32ccVf/uk/PJqLGE0IgraAcx1MLq28FOJFWiOIKTa2+DnIr9FQ5oKxSi8yiCuSXVUGt0SE5rwzJeWUNvo7S1tpQPFc5KTDn9gD4drJrqcMiIiKyGEIInEgrxMZDKfg54Yrh87NUAozp6Y7JQ7rg9j4esJbVvsSQVieQU6I2FN+uFFYgo7AcVworkF7dllmshkan/3mfVlAOKxlnthGZWolag6yiCmQVq/W3mvvXtGm0Ong4KuDuJIeHoxwejgp4OOo/H3tUt7Wls0UkooOdc1lUVASlUonCwkI4OfGvEWQeZZUaHE0uwIFLeTh0KQ9HU/JRUaUz6uMgt0JwVxcM7dYJw7p1wgAfJeRWMmh1ArklamQUVeg/ABRVVN9XI7Po6uO6ftkHgPsCO+P9qYPqHVtbz0hbHx+1X1nFFTieUojjadVFtdRC5JZW1uont5KiX2cnDPRxxkAfJQb6OKO7mz2k15x2UlGlRXaxPseZ1VnOKlYjo/ov7FnVOb7++wIA7Hx+DLq7O9Q5xraej7Y+PiJza+sZaevjo46joKwSPxxNw8ZDKTiTUWxo79LJDg+F+GBisA+8lLY3/ToarQ45JZW4UqgvvN3aywO2NnXPNG/r+Wjr46P2qVStwfHUQqQXllcXz9TIKjYupJVV1j7zozlsrKRwd5DDw0kOlePVAlxNgc7XxQ7+HnV/hgZMmxGzzmgj6iiyi9U4llKAQ0l5OHApDyfTCqHRGde4O9nbYIifC4Z2c8Wwbp3Q29MRVnX89U0mlcDDSQEPJwUG+tT/mqVqjaHolnlNIW6At9LUh0fU7pSoNTiRWohjqQU4llKAhJQCpBdW1OpnJZWgt5cjBng7I7C6qBagcqjzL+fXUljL4NvJrsGZaUIIFFVorhbQq4txpvjFgYiILFtNASiruAJZRWpkVv+bVaxGtuGXWDVK1Rq4O8qvXqLgujMdPJ0UcHeU3/DnVlug0wnsv5CLTYdTsP2fDFRq9H+MsrGS4u7+npgc4otbursa/WHrZlnJpPBU6k8bJaIbq9LqcDy1APvO5eKv8zk4mpKPKu2N53bZ28igqv5+5OFUM1vt6sw1K5nU6PtddvX3u5qiXUFZFSo1OsPs07qEdnfFhqduMfUh14mFNiITEkIgo6gCJ9OKcDKtUH+7Ulh9iqcxL6UCQ7t1MsxY6+HuYNKprvZyK3R3d6h35gsR6Wm0OiRmFiMhRV9UO5ZSiHNZxbiuFg6JBAjwcNAX1Xz1RbXeno4tdg01iUQCpa01lLbW6KlybJHXICKitqlKq8P+C7lIyik1/HJZcwpVdnEFcksr0djzkorVGlzMKa33eYkEcLWXw1N5zfV/r7luqLezAt7OdvXO5GpJpWoNUvLL8Ps/mfjmcApS86/+At3XywmTh/hiQpA3lHa8LjGROQghcC6rBPvO5eCv8zk4cCkPJdddAsnb2RZ+bnaGopl7zSmf1xTV7OU3V5qqOVvk2j82ZBYZf+8MULXe78UstBE1kxACqfnlhmJaTXGtrlPJJBKgu5s9Qrp2MhTXfFxs28w55EQdRU1ua4pqCSkFOHmlsM5TNDsrFQjq4oxAH2cE+jpjgLfypj8EEBERNSQlrwwbDyXjm8MNr5YJ6M9ycHOwMfzy6uEkh3vN/epfYB3kMmQXV15zqZGrlxnJKtL/IqqpvlZZTokaJ9OK6n09Nwc5fFxsq2928O2k/9fHxRbezrbN+sNTeaUWqfllSM0vv+bfcqRU38+77nO1o8IK44M6Y8qQLujPszSIzCK9sBz7zuVg/4Vc7DufU+t7lbOdNUb0cMMIfzeM9HdDF9eWv7ZwY84WaU38jYHarMLyKpxOL0JiRnGdK/Y1hQSAtUwKGyv9TW4lhc01j2vuy61kV5+/pt1aJkVaQbnRLLWTaUUoLK+q9VoyqQQBHg7o11mJ/t5O6O+tRF8vJ/6CTh1WlVYHtUaHymtvWi0qqnSo1Bq3q6ufM36sq/Ov9jV1agkkdbQZPy5Ra3EyrRDHUgrqLIY7KqyqC2pKBPm6INBHCQ8nniZCREQtr0qrwx+nMvH1wWTsPZdjaHdzkCOkq0ut6wzV3O9kbwNZI06T9Peo/zmdTiC3tNJwiYLM4gpkVl8vNKNIjczCCqQVlKNErTEU4xJSCurcl7ujHL4uV4tvNcU4V3s5soorjApoqfnlSMsvQ05J7Z/J11PaWqNfZydMCvHBXf28zDKzjqgjKyyvQtyFXOy/kIN953NwMdt4hqzCWoohfp0w0l9fXOvr5WTSU7gtEX/zJ7MTQr+s9qkrRfpbeiH+uVJkNDW8rbKWSdDL0xH9OyvRz1uJ/p2d0MfLqcVOJSNq6yqqtPjnShESUgpwNDkfCSkFbS7L1jIJ+ng5Icj36my16xcrICIiamlJOaXYeCgF38WnIqdEPyNEIgFGBbjj4aG+uL2PqsWvnSaVSuBefSpXfTPEhBAoKtdUF8muFspq7qfklaG0Un/aVnaxGkeSC5o0Bke5FXw66YtzvoYinb5Q5+1iC6UtTwulpvnhaBr+uVKICYO80a8zZz5eq6JKi8LyKsOtoOzq/cKyyqvt1zx/ObfU6JIqUgkw0MfZUFgb3NUZciv+/nstFtqoVVVpdTifVVJdUCsy/FvXzDBAfz53Hy8nON/kdRd0QqBKK1CpqZ4po9VBfd1smpqZM1dn3OigveY7itxKij5eTvpZap2V6O+tRIDKgd9UqMMSQuBybplRUe1UelGDFzy1kkqMZ40aZo5enU1a14zTmr/Y18xsExBGj/VtqKNN/8BaKkVvL0cE+TqzGE5ERGZTqdHh91MZ2HAwGX+dzzW0uzvKMTnEF5OH+LaZU59qSCQSKO2sobRT1lmME0KgoKzKuPh2zemgOSWV8HCUXzPTzRa+na7OemMhjUxp9a7zeHt7IgDgf3svIcjXGQ8P64JxAzu329mQxRVVyCzSr+KZWVyBzKKr1yfLLlajoLzSUFBTa2pfLqUxerjbY6S/G4b7u+GW7q7M7Q20iULb6tWr8fbbbyMjIwOBgYFYtWoVhg4dWmffsWPHYs+ePbXa77nnHmzdurWlh0pNUFhWhdMZRTh9TUHtXGYJKrW1w20llSBA5Yi+Xk7o29lJ/6+Xk9kvbKrVCUPhzV4uq3MVUKKOorC8CsdSCnA0uQAJKfrCWn5Z7SK5q70NBnVxRpCvM4J8XdDHyxH2civYyKScNUZERB3SpZxSbDyYjO/iUw2XMJBIgDE93TF1aBfc1tvDIlb+rItEIoGLvQ1c7G0wwIezh8g8hBB45/dErN51AQAwxM8FCdXX401IKcDrv5zCxME+eHhYF4tZZEqt0SKzUI2M6hXoM4uuXuQ/85rrLJZWNu0yS1IJDAtuKe1sDPeda/61s4bTNW1dXe258m4TNavQtmvXLtx6660mGcCmTZsQHR2NNWvWYNiwYVi5ciXCw8ORmJgID4/aFxTYsmULKiuvnsufm5uLwMBATJo0ySTjoabT6gQu55bidHoxTqfrC2tnMorrXVbXUW6FPjXFtOp/2+rMMJlUAlsbWbv96we1XWczi/HNoRTY2egv7Nmlkx26uNpB5aho8WJViVqDtPxypBWUISWvHMdTC3E0Jb/W9RgAwEYmRT9v/WmYg7q4YJCvMxf6ICIigv6X5O3/ZGLDgWTEXbw6e03lpJ+99tAQX/i4tK3Za0SWSAiBV38+hXX7kwAA/76nN54a3QPZxWp8G5+CDQeTkZJXjnX7k7BufxKG+Lng4WFdcHd/rzZ1loMQAqfTi7H3XDb2nsvBwaQ8VDZyBpqj3AoeTvrVPFVOCng46VcRdneUw+WaYprSzhoONlb843cLa1ah7a677oKPjw8iIyMxbdo0+Pr6NnsAK1aswIwZMxAZGQkAWLNmDbZu3Yq1a9di4cKFtfp36tTJ6PHGjRthZ2fHQlsrKaqowpn0YpypmamWXoyzDSxWUHPqZ01BrV9nJ/4STtSAnBI13t1xFhsOJhtdC6GGjUwKn076a5h0qS7A+Rr+tYWjouFZoEII5JVWIq2gvLqYVn1B4mse13cqNwB0dbXTF9V8nRHURT9brS0WyYmIiMzlTEYRvj2cii1HUg0zv6USYGwvD0wd2gW39nLnWRJEJqLVCfx7ywlsOpwCAHh9Qn88dktXAPpTsmeO9cfTo3tg3/kcrD9wGX+czsKhpHwcSsrHqz+fwoPVs9y6uzuYZfzZxWrsO5+NvWdz8Oe5HMP1GmsorKXwdFLAo7qApnKUGwpphqKao5wL77UxzfrfSEtLw5dffonPP/8cr776Km677TY88cQTmDBhAmxsbBq9n8rKSsTHx2PRokWGNqlUirCwMMTFxTVqH59++immTJkCe3v7Op9Xq9VQq69+sRYV1b9kdXuSml+GLUfScCmnFDKpBNYyCayk0qv3ZVJYS/X/WskksL7uOSupxDB9/WJ2CU5n6Ger1XdRc4W1FL08ndDH0xF9vPQLAvTydOS523TTOkqGK6q0WPvXJXy46wJK1BoAQFgfFTyc5EjJK0NyXhnS8stRqdXhYnZpnbPLAMDFztqo+GZnI0NaQUV1Ia0MaQXlqKi68V/GlLbW8Ha2hbeLLfp4OiKoi37hAFcHuUmPm9q3jpJfovaKGW68/NJK/HTsCr6NT8HJtKvvk6eTApOH6GeveTvbmnGE1NF0hPxWaXV4/ptj+OnYFUglwFsPBuLBYJ9a/aRSCUb3dMfonu7ILKrApkMp2HgwGVcKK/DJvkv4ZN8lhHZ3xSO3dMGdfT1hY9VyhXC1Rov4pHzsOacvrp1KN/5/sbWWIbSHK0YFuGF0T3d0d7PnJBUL1KxCm5ubG5577jk899xzOHLkCD777DPMnDkTM2fOxMMPP4wnnngCgYGBN9xPTk4OtFotVCqVUbtKpcKZM2duuP3Bgwdx8uRJfPrpp/X2iYmJwauvvnrjg2oHKqq02P5PBr49nIq/LuQYXRDclDorFYZiWm8vfWHNz9W+UcuLEzVVe8+wEAI/H0/H8t/OGE63HuijxEv39sXQbsYzeDVaHTKKKpCcV2YoviXnlRse55VWIr+sCvllhTiWWtjg63o4yuHtYmsopvlU/+vtrF/hy4F/FSMTaO/5JWrvmOGGabQ6/HkuG9/Fp+KPU1mG6xBbyyQI66PCg8E+GNOTs9fIPNp7fiuqtHh2w1HsOJUJK6kE700ZhHsHet1wO5WTArNvD8CsW/2xOzELXx9Ixq7ELMRdzEXcxVy4OdhgUogvpg7pgi6uN39qtxACF7JL8OfZHPx5LhsHLubVOhusX2cnjO7pjlEBbgju6sKzRdoBiRA3X465cuUKPv74YyxbtgxWVlaoqKhAaGgo1qxZg379+jW4nbe3N/bv34/Q0FBD+/z587Fnzx4cOHCgwdf917/+hbi4OBw/frzePnVV8n19fVFYWAgnJ6cmHGXbJITAibRCfHM4BT8mXEFxhcbw3Ah/V4wKcIcQ+g8CVToBjVYHjU5AoxXQ6HSo0l5tq9LqjNq1Ov3Nt5OtobDWx9P8CxRQyyoqKoJSqWwzGWnPGY6/nI83tp7C0eQCAICXUoH5d/XC+EDvZl03oUStMRTgav4tq9Sis/O1hTRbeDkr+AO8nWJ+iSwbM9x4Qghkl6j1K1zm1axwqV/lMi2/HBIJMMBbiYE+zgj0VaKvl9Jk19w9n1WsPzX0aBqyi6++P/06O2FSsA/uC/JGJ/vGn+VD7QPz23rKK7V46svD2HsuBzZWUqx5dDBu66268Yb1SCsox6aDydh4KAVZ12TaUWEFuZUUNjIpbKyu3qxlV9vkNe3X9pHJYGMlRV6pGvvO5eBKYYXR67k7yjEqwA1jerpjhL8b3HjGSJtgygw3e8pCVVUVfvzxR6xduxY7duxASEgIPvjgA0ydOhXZ2dl46aWXMGnSJJw6darefbi5uUEmkyEzM9OoPTMzE56eng2+fmlpKTZu3IjXXnutwX5yuRxyefv7ws0tUeP7o2n4Lj4VZzKKDe3ezraYFOKDiYN92tzS4ETN0R4znJJXhmXbzmDr8XQAgJ2NDM+M6YEnR3W/qV8CHORWhqI4UVvQHvNL1JGYM8NCCOSWVl5XSNP/m1JdTFPf4CLhF7JL8UPCFQD6Ba56qhwx0FuJgb5KBPo4o6fKsdGniBWWV+HnY1fwbXwqjqUUGNo72dtgQpA3Hgz2Qd/O/PlLbUd7/RlcXFGFJ9YdxsGkPNjZyPBJRAiG+7vd1D69nW0RfWcvPHt7AGJPZ+Hrg8nYey4bxRUaFN948xuysZJiWLdOGBXghlEB7ujt6cjTQdu5ZhXann32WWzYsAFCCDz22GN466230L9/f8Pz9vb2eOedd9C5c+cG92NjY4Pg4GDExsZiwoQJAACdTofY2FhERUU1uO23334LtVqNRx99tDmHYJFqpqd/cygVsWcyUaXVT0a0sZLi7v6eeCjEF6HdXbmCCFEbVVRRhdU7z+Ozv5JQqdVBIgEmh/gi+o6e8HDiktlERNSxqTVarP87Gd8cTkFSbukNrykqleivgebTyQ4+Lrbwcan51xZqjQ7HUwpxIq0Ax1ILkV2sxul0/WJeNRdNt7GSoo+XEwJ99DPfBvoo0cPdwXA5FK1OYN/5HHwXn4rt/2QYVv+TSSW4tZcHHgz2wW29PVr0ek5EdFVBWSWmrT2IY6mFcFRYYV3kEAR37XTjDRvJWibFXf09cVd/T+SVVqKgrBKVWh0qNVdv6useGz2v1UF9zXNyaylu6e6KoX6dTDajlixDswptp06dwqpVq/DAAw/UWyV3c3PDrl27briv6OhoTJs2DSEhIRg6dChWrlyJ0tJSwyqkERER8Pb2RkxMjNF2n376KSZMmABXV9fmHIJFuZhdgm/jU7E5PtVoKmugjxIPhvjivoGdeTonURum0eqw4WAy3v3jHPJKKwHoT+1+8Z6+/Os3ERF1eFqdwI8JaVix46zRwluSmkLaNUU0X5erRTVPpaLBItetvTwA6GfHZRRV4Fh14e14aiGOpxaisLwKx1IKqmeoXQYA2NvI0M9bie5u9thzNhvp15zy1UvliEkhPhgf5A13x/Y3U4ioLcsuVuOxTw/gTEYxXOys8eUTw9DfW9lir9fJ3oangFOzNavQFhsbe+MdW1lhzJgxN+w3efJkZGdnY/HixcjIyEBQUBC2bdtmWCAhOTkZUqnxD9DExETs27cPv//+e3OG3+JyS9TYdDgFafnlsJJKIJNKYS2TQCa9uqJnzQqfRs9J9SuDWlWvEJpfVokfjqbh8OV8w7472dvg/kHemBTig96e/AWdqCWkF5Zj37kcyK1lUNpaG92cFFaNvqixEAK7E7Px5q+ncT6rBADQw90eL97bB7f28uCUcSIi6tCEENh5Jgtvb080XArFw1GO2bcHYKS/m8muKSqRSOCltIWX0hZ39fc0vPbl3DIcSy3AierC28krhSit1OLgpTwcvJQHQL8K9/igzngw2AcDvJX82U1kBlcKyvHoJwdwMacUHo5yfPXkMPRUOZp7WET1alahLSYmBiqVCo8//rhR+9q1a5GdnY0FCxY0aX9RUVH1niq6e/fuWm29evWCCdZwMLkL2SX4dN8lbI5PveE1I5pCKgHG9vLAQyE+uK23itPTiVqAEAIHL+Xhi7jL2PZPBrS6+r/HOMit9EU3W2sobavvK64pyNlZw0Fuhe+PpmHvuRwA+iL5c2EBmDK0C6y5+hgREXVw8ZfzsOy3MziUpP+DsqPCCs+M7YHI4d1a5RQriUQCPzd7+LnZY3yQNwD9zLrzWSU4nlqA89klGOCtRFgfFRTWPOWLyFwu55bi4f8dQFpBObydbbH+yWHwc7M397CIGtSsQtt///tffP3117Xa+/XrhylTpjS50GbJan45/9/eS/jj9NVFHQJ9lBjbywM6IaCpXr2zSqur/ldAq7u6+ue1z2l0+lU/NVoBiQQY3dMdEwf7QMXrNxG1iLJKDX5MuILP9ycZLSwS6OsMW2spCss1KCqvQmF5FUrU+lV9S9QalKg1SCsor2+3BjYyKSJH+GHmrf5Q2vIUbyIi6tjOZhbjrW2Jhs/Ncisppo/wwzNjesDZzrynacmkEvTydEQvT86UIWqOSo0Oe89lI7+sCt3d7dHD3eGmPv+ezyrGI58cQGaRGt3c7PHVk8Pg7WxrwhETtYxmFdoyMjLg5eVVq93d3R3p6ek3PShLoNHq8NvJDPxv70UcTy0EoL+OxO29VXhqdHcM8XPh1HKiNiw5twxf/p2ETYdSUFShL6AprKW4f5A3IkL96ly5U6PVoahCg8Lqwtu1t6Lq27Vt3s62ePa2AHRx5QrARETUsaXml+HdHeew5WgqhNCfsfFQiC/mhAXAS8lfnIkslRACJ9IKsTk+FT8du4L8siqj590c5Ojhbo8eHg7o4e6gv+/uAG9n2wYX8fvnSiEe+/Qg8kor0UvliC+fHAoPR04+IcvQrEKbr68v/vrrL3Tr1s2o/a+//rrhSqOWrkStwaZDKVi775JhNovcSoqJwT54YmQ39HB3MPMIiag+Op3A3vM5+GJ/EnYmZqHmDHTfTraIuMUPD4X4NriwiJVMygujEhERNUFeaSVW7zqPL+Muo1Krv7TKXf088UJ4L/h78HMzkaXKKKzADwlp2ByfinPV1yIGAHdHOfzdHXAppxQZRRXIKVEjp0SNA9XXPayhsJaim9vVwpu+EGeP7m4OOJ1RhOlrD6KoQoMB3kp88fhQuPDzN1mQZhXaZsyYgblz56Kqqgq33XYbAP0CCfPnz8fzzz9v0gG2FRmFFfhs/yV8fSAZxdWzX1ztbfBYaFc8dktXuDpw5SGitqq4ogrfxafiy7jLuJhTamgf3dMd04d3xZieHpA18Bc1IiIiappStQZr913Cx39eRHH1pRdCu7tiwd29EeTrbN7BEVGzlFdq8fupDHwXn4q/zueg5pLGcisp7uzniYmDvTHS382wcFhxRRUu5ZTiQnYJLmRV/5tdgqScMlRU6XA6vQin04tqvY6VVAKNTiCkqwvWRg6Bk4KXXyHL0qxC27x585Cbm4uZM2eisrISAKBQKLBgwQIsWrTIpAM0t1NXivDJ3ov46dgVaKq/k3R3t8eTI7vjgcHevDgqURt2PqsYn++/jC1HUlFaqQWgX8jgwWAfRIR2RXfOQCUiIjKpSo0OGw8l4/3Y88gpUQMA+no5YcHdvTE6wI2XViGyMDqdwKGkPGw+kopfT2QYrlkMAEP9OuGBwd64Z6BXncUwR4U1Bvo4Y6CPs1G7RqtDan65ofBWU4Q7n12CgrIqaHQCI/3d8HFEMOxsmlWyIDKrZn3VSiQSLF++HC+//DJOnz4NW1tbBAQEQC5vH7O6hBDYczYbn+y9hH3ncwztw7p1wlOju+PWXh4Nnk9OROaj1QnEns7E53FJ+Ot8rqHd38MB00K74v7BPnCQ8wc2ERGRqZ1MK8TM9UeQnFcGAOjSyQ7P39kT4wZ25mdnIgtzObcUm4+k4fujqUjJu7oAmG8nWzwwyAcPDPZGV9fmrf5pJZMaVv29vY/K6Lm80krklarRw92BhXmyWDf126aDgwOGDBlyUwNYvXo13n77bWRkZCAwMBCrVq3C0KFD6+1fUFCAF198EVu2bEFeXh66du2KlStX4p577rmpcVzrkU8OYP8F/S/oMqkE9wzwwoxR3WpV4omo7dl7LhtPfRkPQH+h5bA+Kkwb7ofhPVz5w5qIiKgFeTvbIr+0Em4ONph9ewCmDOkCGyupuYdFRI1UVFGFrcfTsTk+FYcv5xvaHeRWuHeAFyYG+yCkq0uLFs55PWRqD5pdaDt8+DC++eYbJCcnG04frbFly5ZG7WPTpk2Ijo7GmjVrMGzYMKxcuRLh4eFITEyEh4dHrf6VlZW444474OHhge+++w7e3t64fPkynJ2dm3sYdRrWzRXHUgowZWgXRI7wg48LVwwkshSjAtwR6OuM0O6ueGRYF/h2Yn6JiIhag4u9DdZGDkFfLyfYc/Y4kcWZtf4I9p7Tn9EllQAjA9wxcbA37uzrCVsbXjKJqLGa9RNw48aNiIiIQHh4OH7//XfceeedOHv2LDIzM3H//fc3ej8rVqzAjBkzEBkZCQBYs2YNtm7dirVr12LhwoW1+q9duxZ5eXnYv38/rK3154D7+fk15xAa9PhIP0wf4QelLS+6SGRpZFIJfpg5nLPXiIiIzGCIXydzD4GImum+wM7ILKrAxME+mDDIGyonhbmHRGSRmlVoW7p0Kd59913MmjULjo6OeO+999CtWzf861//gpeXV6P2UVlZifj4eKPFE6RSKcLCwhAXF1fnNj/99BNCQ0Mxa9Ys/Pjjj3B3d8fDDz+MBQsWQCaru8KuVquhVqsNj4uKaq9qcj1HrmpC1GY0J8MsshGZxuzZs+Hv74/Zs2cbtX/wwQc4f/48Vq5c2eD2zckvEbUdzDCR5WpOfh8Y7IMHg334WZroJjXrogkXLlzAvffeCwCwsbFBaWkpJBIJnnvuOXz88ceN2kdOTg60Wi1UKuOLH6pUKmRkZNS5zcWLF/Hdd99Bq9Xi119/xcsvv4z//Oc/eOONN+p9nZiYGCiVSsPN19e3kUdJRG0BM0xkPps3b8aIESNqtQ8fPhzffffdDbdnfoksGzNM1PqEEEhOTkZFRcVN7ac5+ZVJJSyyEZlAswptLi4uKC4uBgB4e3vj5MmTAPQLFZSVlZludNfR6XTw8PDAxx9/jODgYEyePBkvvvgi1qxZU+82ixYtQmFhoeGWkpLSYuMjItNjhonMJzc3F0qlsla7k5MTcnJy6tjCGPNLZB5VVVWwsrIyfEZvLmaYqPUJIeDv73/TeWN+icynWaeOjh49Gjt27MCAAQMwadIkzJkzBzt37sSOHTtw++23N2ofbm5ukMlkyMzMNGrPzMyEp6dnndt4eXnB2tra6DTRPn36ICMjA5WVlbCxqb06iVwuh1wub8LREVFbwgwTmY+/vz+2bduGqKgoo/bffvsN3bt3v+H2zC+ReVhbW6NLly7QarU3tR9mmKj1SaVSBAQEIDc3FwEBAc3eD/NLZD7NmtH2wQcfYMqUKQCAF198EdHR0cjMzMTEiRPx6aefNmofNjY2CA4ORmxsrKFNp9MhNjYWoaGhdW4zYsQInD9/HjqdztB29uxZeHl51VlkIyIiouaLjo7G/PnzsWTJEuzZswd79uzB4sWLsXDhQjz33HPmHh4RNeDFF1/Ev//9b+Tl5Zl7KETURMuWLcO8efNuelYqEZlHk2e0aTQa/PLLLwgPDwegr7jXtUJoY0RHR2PatGkICQnB0KFDsXLlSpSWlhpWIY2IiIC3tzdiYmIAAM888ww++OADzJkzB88++yzOnTuHpUuX1rpIMxEREd28xx9/HGq1Gm+++SZef/11APrVvj/66CNERESYeXRE1JCaRUs6d+6Mrl27wt7e3uj5I0eOmGlkRHQjERERKCsrQ2BgIGxsbGBra2v0PAvoRG1bkwttVlZWePrpp3H69OmbfvHJkycjOzsbixcvRkZGBoKCgrBt2zbDAgnJycmQSq9OuvP19cX27dvx3HPPYeDAgfD29sacOXOwYMGCmx4LEbWelJQULFmyBGvXrjX3UIjoBp555hk888wzyM7Ohq2tLRwcHMw9JCJqhAkTJph7CETUTDda1ZuI2jaJEEI0daOxY8fiueeew/jx41tiTC2qqKgISqUShYWFcHJyMvdwiNqc1sjIsWPHMHjw4GZdO4YZJqqfqfNx6dIlaDSaWteIOXfuHKytreHn52fW8RG1N209I219fETm1Nbz0dbHR2RupsxIsxZDmDlzJqKjo5GSkoLg4OBaU9EHDhx4U4MiIsv2008/Nfj8xYsXW2kkRHQzpk+fjscff7xWoe3AgQP45JNPsHv3bvMMjIgarbKyEllZWUbXOAaALl26mGlERNQYWq0W33//veFMsr59+2L8+PGwsmrWr/BE1IqaldKahRCuvTaaRCKBEAISieSmVzgiIss2YcIEw/eE+kgkklYcERE1x9GjRzFixIha7bfcckutlUiJ2rPC8ioUlFXC18UOUqll/Pw6e/YsnnjiCezfv9+onZ/Xidq+f/75B/fddx8yMjLQq1cvAMDy5cvh7u6On3/+Gf379zfzCImoIc0qtF26dMnU4yCidsTLywsffvhhvaeXJyQkIDg4uJVHRURNJZFIUFxcXKu9sLCQv6RTh5BeWI7/7rmIDQeTodbo4KSwQqCvMwJ9nKv/VcLDSWHuYdYpMjISVlZW+OWXX+Dl5cU/cBFZkCeffBL9+vXD4cOH4eLiAgDIz8/H9OnT8dRTT9UqoBNR29KsQlvXrl1NPQ4iakeCg4MRHx9fb6HtRrPdiKhtGD16NGJiYrBhwwbIZDIA+lNZYmJiMHLkSDOPjqjlJOeW4aM9F/BdfAqqtPqfV1ZSCYoqNNh7Lgd7z+UY+nZWKjCwpvDmq8QAbyUcFdbmGrpBQkIC4uPj0bt3b3MPhYiaKCEhwajIBgAuLi548803MWTIEDOOjIgao1mFti+++KLB5yMiIpo1GCKyfMePH8e8efNQWlpabx9/f3/s2rWrFUdFRM2xfPlyjB49Gr169cKoUaMAAHv37kVhYSEzTO3ShewSrN51Hj8mXIFWpy+w3dK9E569LQBDu3VCYkYxjqUW4FhKAY6lFOJsVjGuFFbgSmEGtv2TAQCQSAB/dwfDjLdAX2f09nSCjZW0VY+lb9++yMnJuXFHImpzevbsiczMTPTr18+oPSsrC/7+/mYaFRE1VrNWHb22sg4AVVVVKCsrg42NDezs7JCXl2eyAZoaV1shatjNZkQmkyE9PR0eHh7o3r07Dh06BFdX1zYzPqL2rCXykZaWhtWrV+PYsWOwtbXFwIEDERUVhU6dOrWJ8RGZwpmMInyw8zy2nkhHzSfjMT3dEXWbP4b41f+1XqLW4GRaIY6lFOB4aiESUgqQVlBeq5+NTIq+nZ0Q1scDUbcF1LEnPVNmZOfOnXjppZewdOlSDBgwANbWxrPsmrN/ZpiofqbMx6+//or58+fjlVdewS233AIA+Pvvv/Haa69h2bJlRrPKG/tazC9Rw8y+6mh+fn6ttnPnzuGZZ57BvHnzmry/1atX4+2330ZGRgYCAwOxatUqDB06tM6+69atQ2RkpFGbXC5HRUVFk1+XiEzP2dkZly5dgoeHB5KSkmqtckZElsXV1RUTJkzA8OHDDXnet28fAOC+++4z59CIbtrx1AKs2nkeO05lGtru6KtC1K3+CPR1vuH2DnIr3NLdFbd0v/oHpexiNY5Xz3pLSC3E8dQCFJRVISGlAD4uti1xGHUKCwsDANx+++1G7VwMgajt+7//+z8AwEMPPWS4vmLN/Jhx48YZHjPLRG2TydYGDggIwLJly/Doo4/izJkzjd5u06ZNiI6Oxpo1azBs2DCsXLkS4eHhSExMhIeHR53bODk5ITEx0fCYF3clajsmTpyIMWPGGC68HBISYri20/UuXrzYyqMjoqbYtm0bIiIikJubW+u6ivxwT5bscFIeVu08jz1nswHoT/e8Z4AXom71Rx+vm/srtrujHLf3UeH2PioA+l+Gk/PKkJBSAA/H1ls44bPPPoOvr2+tn8E6nQ7JycmtNg4iarob5dfPz888AyOiRjFZoQ0ArKyscOXKlSZts2LFCsyYMcMwS23NmjXYunUr1q5di4ULF9a5jUQigaen502Pl4hM7+OPP8YDDzyA8+fPY/bs2ZgxYwYcHR3NPSwiaoZnn30WkyZNwuLFi6FSqcw9HKKbIoTA/gu5WLXzHP6+qL/MiUwqwfigzpg51h/+Hg4t8roSiQRdXe3R1dW+RfZfn8cff9xwKYdr5ebmIiwsDNOmTWvV8RBR490ov/xDF1Hb1qxC208//WT0WAiB9PR0fPDBBxgxYkSj91NZWYn4+HgsWrTI0CaVShEWFoa4uLh6tyspKUHXrl2h0+kwePBgLF26tNaFImuo1Wqo1WrD46KiokaPj4ia56677gIAxMfHY86cOTdVaGOGicwnMzMT0dHRzS6yMb/UFgghsDsxG6t2nsOR5AIAgLVMggeDffDMGH90cbUz7wBbSM1pZdcrKSmBQtG4mXXMMJF5ML9Elq1ZhbYJEyYYPZZIJHB3d8dtt92G//znP43eT05ODrRaba0P8CqVqt7TT3v16oW1a9di4MCBKCwsxDvvvIPhw4fjn3/+gY+PT63+MTExePXVVxs9JiIync8+++ym98EME5nPgw8+iN27d6NHjx7N2p75JXPKKqrAd0dS8c2hFCTllgEA5FZSTB3aBU+N7o7Ozq13vbTWFB0dDUD/+fzll1+Gnd3VQqJWq8WBAwcQFBTUqH0xw0Sti/klah+ateqoqVy5cgXe3t7Yv38/QkNDDe3z58/Hnj17cODAgRvuo6qqCn369MHUqVPx+uuv13q+rkq+r68vV1shqkdbW5GIGSZqPFPnt6ysDJMmTYK7u3udqxbOnj27we2ZX2ptGq0Of57LxoaDKdh5Jgtanf5jroPcCg8P64InR3Vr1eukNZUpMnzrrbcCAPbs2YPQ0FDY2NgYnrOxsYGfnx9eeOEFBATUv/ppDWaYqPGYXyLLZvZVR03Fzc0NMpkMmZmZRu2ZmZmNvgabtbU1Bg0ahPPnz9f5vFwuh1wuv+mxEpF5MMNE5rNhwwb8/vvvUCgU2L17t9FpLBKJ5IaFNuaXWktKXhm+OZyCbw+nIqPo6kr0wV1dMHmIL+4d4AV7uVk/9raaXbt2AQAiIyPx3nvv3dQvC8wwUetifonah2Z94pg4cSKGDh2KBQsWGLW/9dZbOHToEL799ttG7cfGxgbBwcGIjY01nI6q0+kQGxuLqKioRu1Dq9XixIkTuOeee5p0DERERNSwF198Ea+++ioWLlwIqVRq7uEQGVFrtNhxKhObDqVg3/kc1Jyj4WJnjQcG+2DKEF8EqDruYjymuHwDEZkH80tk2ZpVaPvzzz/xyiuv1Gq/++67m3SNNkB/Hvq0adMQEhKCoUOHYuXKlSgtLTWsQhoREQFvb2/ExMQAAF577TXccsst8Pf3R0FBAd5++21cvnwZTz75ZHMOhYiIiOpRWVmJyZMns8hGbcq5zGJsPJSCLUdSkV9WZWgfFeCGyUN8cUdfFeRWMjOOkIiIiDqyZhXaSkpKjM4Xr2Ftbd3k1UwmT56M7OxsLF68GBkZGQgKCsK2bdsMCyQkJycbfcDPz8/HjBkzkJGRARcXFwQHB2P//v3o27dvcw6FiIiI6jFt2jRs2rQJ//73v809FGplao0WsaezsPV4OvLLKiGRANLqU4clEgmkEkByzX1AUt0HkBju6ztZSSWwl1vBUWEFJ4U1HBX6+47ymvvWhufs5TJYyWoXdssqNfjleDo2HUpB/OV8Q7vKSY6HQnzxUIgvfDu1z9VDiYiIyLI0q9A2YMAAbNq0CYsXLzZq37hxY7MKXlFRUfWeKrp7926jx++++y7efffdJr8GERERNY1Wq8Vbb72F7du3Y+DAgbUWQ1ixYoWZRkYtQQiBoykF2Byfil+Op6OwvOrGG7UAOxuZUQHO3sYKCSkFKFFrAAAyqQS39/bAlKG+GB3gXmdhjoiIiMhcmlVoe/nll/HAAw/gwoULuO222wAAsbGx2LBhQ6Ovz0ZERERt24kTJzBo0CAAwMmTJ42eu3ZhBLJsaQXl+P5IKrYcScPFnFJDu5dSgQmDvNHbU3+dMyEAnRAQAhDQ34cABAR0AtXt+vsQQt9HJ6DRCZSoNSiu0KC4oqr636v3i6rvqzU6AEBZpRZllVpkFqmNxtnV1Q6Th/jiwcE+8HBquyuHEhERUcfWrELbuHHj8MMPP2Dp0qX47rvvYGtri4EDB+KPP/7AmDFjTD1GIiIiMoOa1c+o/SlVa/DbyQxsjk/F35dyDQsJ2FrLcHd/Tzww2AehPVwhk7ZeQbVSozMuxKmv3vd1scUQv06QtuJ4iIiIiJqj2euc33vvvbj33ntNORYiIiIiaiE6nUDcxVxsjk/FbyczUF6lNTwX2t0VDwz2xt0DvOAgb/bHw5tiYyWFq4Mcrg5ys7w+ERERkSk065PUoUOHoNPpMGzYMKP2AwcOQCaTISQkxCSDIyIiIqKbcyG7BJvjU/HD0TRcKawwtHdzs8cDg7xx/2Bv+LhwIQEiIiIiU2hWoW3WrFmYP39+rUJbWloali9fjgMHDphkcERERETUdIVlVfjp+BVsjk9FQkqBod1JYYX/C+yMiYN9MLiLM6+1R0RERGRizSq0nTp1CoMHD67VPmjQIJw6deqmB0VERERETaPVCew9l43v4lPx+6lMVFYvLiCTSjCmpzsmDvbB7X08oLCWmXmkRERERO1Xs9ZDl8vlyMzMrNWenp4OK6um1+5Wr14NPz8/KBQKDBs2DAcPHmzUdhs3boREIsGECROa/JpERERE7cGF7BIs33YGw5fFYvpnh/DL8XRUanTo7emIl+7tg78X3Y6104fg3oFeLLIRERERtbBmzWi78847sWjRIvz4449QKpUAgIKCAvz73//GHXfc0aR9bdq0CdHR0VizZg2GDRuGlStXIjw8HImJifDw8Kh3u6SkJLzwwgsYNWpUcw6BiIiIyGIVVVThl2Pp+C4+BUeSCwztznbWmBDkjQeDfdCvsxNPDSUiIiJqZc0qtL3zzjsYPXo0unbtikGDBgEAEhISoFKp8OWXXzZpXytWrMCMGTMQGRkJAFizZg22bt2KtWvXYuHChXVuo9Vq8cgjj+DVV1/F3r17UVBQ0JzDICIiIrIYOp3A/gu5+DY+BdtOZkB9zamhY3u648FgH9zWxwNyK85aIyIiIjKXZhXavL29cfz4caxfvx7Hjh2Dra0tIiMjMXXqVFhbWzd6P5WVlYiPj8eiRYsMbVKpFGFhYYiLi6t3u9deew0eHh544oknsHfv3gZfQ61WQ61WGx4XFRU1enxEZH7MMJHlYn5NIymnFJuPpGJzfKrRqqEBHg6YFOKDCUHe8HBSmHGE1F4xw0SWi/klMp9mFdoAwN7eHiNHjkSXLl1QWVkJAPjtt98AAPfdd1+j9pGTkwOtVguVSmXUrlKpcObMmTq32bdvHz799FMkJCQ06jViYmLw6quvNqovEbU9zDCR5eqo+c0sqsCiLSdw8FIe5FZSKKxlkFtLobCSQWGtf6y/6dvk1te0X9NHoxPYdjIdh5LyDft2UlhhfPWpoQN9lDw1lFpUR80wUXvA/BKZj0QIIZq60cWLF3H//ffjxIkTkEgkEEIYfdDTarWN2s+VK1fg7e2N/fv3IzQ01NA+f/587NmzBwcOHDDqX1xcjIEDB+LDDz/E3XffDQCYPn06CgoK8MMPP9T5GnVV8n19fVFYWAgnJ6fGHjJRh1FUVASlUtlmMsIMEzUe82t+e89lY+7GBOSWVppsn1IJMLr61NCwPiouaNCOMcNElov5JbJspsxws2a0zZkzB926dUNsbCy6deuGAwcOIC8vD88//zzeeeedRu/Hzc0NMpms1gqmmZmZ8PT0rNX/woULSEpKwrhx4wxtOp3++iRWVlZITExEjx49jLaRy+WQy+VNOTwiakOYYSLL1ZHyq9UJvPfHWazadR5CAL09HbH0gQGwt7FCRZVWf9PoDPfVVTpUaKrbq3RX/9Vcfb5Sq8OgLs54YJAPPJU8NZRaX0fKMFF7w/wSmU+zCm1xcXHYuXMn3NzcIJVKIZPJMHLkSMTExGD27Nk4evRoo/ZjY2OD4OBgxMbGYsKECQD0hbPY2FhERUXV6t+7d2+cOHHCqO2ll15CcXEx3nvvPfj6+jbncIiIiIiaLau4AnM2JCDuYi4AYOpQXywZ148zz4iIiIg6oGYV2rRaLRwdHQHoZ6VduXIFvXr1QteuXZGYmNikfUVHR2PatGkICQnB0KFDsXLlSpSWlhpWIY2IiIC3tzdiYmKgUCjQv39/o+2dnZ0BoFY7ERERUUvbfz4HszcmIKdEDTsbGZbePwATBnmbe1hEREREZCbNKrT1798fx44dQ7du3TBs2DC89dZbsLGxwccff4zu3bs3aV+TJ09GdnY2Fi9ejIyMDAQFBWHbtm2GBRKSk5MhlUqbM0wiIiKiFqHVCazaeQ7vxZ6DEEAvlSNWPzIY/h4O5h4aEREREZlRswptL730EkpLSwEAr732Gv7v//4Po0aNgqurKzZt2tTk/UVFRdV5qigA7N69u8Ft161b1+TXIyIiImqu7GI15m46ir/O608VnRzii1fu6wdbG54qSkRERNTRNavQFh4ebrjv7++PM2fOIC8vDy4uLlxmnoiIiNqtuAu5mL3xKLKL1bC1luGNCf0xMdjH3MMiIiIiojaiWYW2unTq1MlUuyIiIiJqU3Q6gdW7zuPdP85CJ4AADwd8+MhgBKgczT00IiIiImpDTFZoIyIiImqPckrUeG5TAvaeywEAPBjsg9fG94OdDT9GEREREZExfkIkIiIiqseBi/pTRTOL1FBYS/H6+P6YFOJr7mERERERURvFQhsRERHRdXQ6gY/2XMB/fk+ETgD+1aeK9uSpokRERETUABbaiIiIqN3IKqrAc98k4HhqIeRWUtjIpLCxuuZmeCyDjUyq71NHv+OpBYZVRR8Y5I3XJ/SHvZwfm4iIiIioYfzESERERO3CpZxSPPbpAaTmlwMAim9yf3KrmlNFfbiqOhERERE1SpsotK1evRpvv/02MjIyEBgYiFWrVmHo0KF19t2yZQuWLl2K8+fPo6qqCgEBAXj++efx2GOPtfKoiYiIqK04nlqAyM8OIbe0En6udlgxOQi21jJUanSo1Or0/2p0UF/3uFKjNX6++r5MIsFDQ3x5qigRERERNYnZC22bNm1CdHQ01qxZg2HDhmHlypUIDw9HYmIiPDw8avXv1KkTXnzxRfTu3Rs2Njb45ZdfEBkZCQ8PD4SHh5vhCIiIiMic/jybjae/ikdZpRYDvJX4LHII3Bzk5h4WEREREXVAZi+0rVixAjNmzEBkZCQAYM2aNdi6dSvWrl2LhQsX1uo/duxYo8dz5szB559/jn379tVZaFOr1VCr1YbHRUVFpj0AImpRzDCR5WqN/P6YkIbnvzkGjU5gVIAbPno0GA68lhqRSfBnMJHlYn6JzEdqzhevrKxEfHw8wsLCDG1SqRRhYWGIi4u74fZCCMTGxiIxMRGjR4+us09MTAyUSqXh5uvra7LxE1HLY4aJLFdL5/eTvRcxZ2MCNDqB+wI749NpQ1hkIzIh/gwmslzML5H5mLXQlpOTA61WC5VKZdSuUqmQkZFR73aFhYVwcHCAjY0N7r33XqxatQp33HFHnX0XLVqEwsJCwy0lJcWkx0BELYsZJrJcLZVfIQRifjuNN7aeBgA8PqIbVk4Ogo2VWT/WELU7/BlMZLmYXyLzscg/+zo6OiIhIQElJSWIjY1FdHQ0unfvXuu0UgCQy+WQy3mdFiJLxQwTWa6WyG+VVoeFm09g85FUAMDCu3vjX6O7c1VQohbAn8FElov5JTIfsxba3NzcIJPJkJmZadSemZkJT0/PereTSqXw9/cHAAQFBeH06dOIiYmps9BGRERE7UNZpQaz1h/BrsRsyKQSLJ84EA8G+5h7WEREREREBmY9x8LGxgbBwcGIjY01tOl0OsTGxiI0NLTR+9HpdEYXeiQiIqL2Jb+0Eg//7wB2JWZDYS3F/yKCWWQjIiIiojbH7KeORkdHY9q0aQgJCcHQoUOxcuVKlJaWGlYhjYiIgLe3N2JiYgDoL+oYEhKCHj16QK1W49dff8WXX36Jjz76yJyHQURERC0kraAcEZ8ewIXsUjjbWePTaUMQ3NXF3MMiIiIiIqrF7IW2yZMnIzs7G4sXL0ZGRgaCgoKwbds2wwIJycnJkEqvTrwrLS3FzJkzkZqaCltbW/Tu3RtfffUVJk+ebK5DICIiohaSmFGMiLUHkFmkRmelAl88MRT+Ho7mHhYRERERUZ3MXmgDgKioKERFRdX53O7du40ev/HGG3jjjTdaYVRERERkTgcv5eHJzw+hqEKDnioHfP74UHgpbc09LCIiIiKierWJQhsRERHRtX7/JwPPbjgKtUaHkK4u+HTaECjtrM09LCIiIiKiBrHQRkRERG3KxoPJ+Pf3J6ATQFgfFT54eBAU1jJzD4uIiIiI6IZYaCMiIqI2QQiBD3aex392nAUATBniizcm9IeVzKyLpBMRERERNRoLbURERNRmpOaXAwCevc0f0Xf0hEQiMfOIiIiIiIgaj4U2IiIiahMkEgnevL8/7uirQlhflbmHQ0RERETUZDwXg4iIiNoMK5mURTYiIiIislhtotC2evVq+Pn5QaFQYNiwYTh48GC9ff/3v/9h1KhRcHFxgYuLC8LCwhrsT0RERERERERE1BrMXmjbtGkToqOjsWTJEhw5cgSBgYEIDw9HVlZWnf13796NqVOnYteuXYiLi4Ovry/uvPNOpKWltfLIiYiIiIiIiIiIrjJ7oW3FihWYMWMGIiMj0bdvX6xZswZ2dnZYu3Ztnf3Xr1+PmTNnIigoCL1798Ynn3wCnU6H2NjYVh45ERERERERERHRVWZdDKGyshLx8fFYtGiRoU0qlSIsLAxxcXGN2kdZWRmqqqrQqVOnOp9Xq9VQq9WGx0VFRTc3aCJqVcwwkeVifoksGzNMZLmYXyLzMeuMtpycHGi1WqhUxhc9VqlUyMjIaNQ+FixYgM6dOyMsLKzO52NiYqBUKg03X1/fmx43EbUeZpjIcjG/RJaNGSayXMwvkfmY/dTRm7Fs2TJs3LgR33//PRQKRZ19Fi1ahMLCQsMtJSWllUdJRDeDGSayXMwvkWVjhoksF/NLZD5mPXXUzc0NMpkMmZmZRu2ZmZnw9PRscNt33nkHy5Ytwx9//IGBAwfW208ul0Mul5tkvETU+phhIsvF/BJZNmaYyHIxv0TmY9YZbTY2NggODjZayKBmYYPQ0NB6t3vrrbfw+uuvY9u2bQgJCWmNoRIRERERERERETXIrDPaACA6OhrTpk1DSEgIhg4dipUrV6K0tBSRkZEAgIiICHh7eyMmJgYAsHz5cixevBhff/01/Pz8DNdyc3BwgIODg9mOg4iIiIiIiIiIOjazF9omT56M7OxsLF68GBkZGQgKCsK2bdsMCyQkJydDKr068e6jjz5CZWUlHnzwQaP9LFmyBK+88kprDp2IiIiIiIiIiMjA7IU2AIiKikJUVFSdz+3evdvocVJSUssPiIiIiIiIiIiIqIksetVRIiIiIiIiIiKitoKFNiIiIiIiIiIiIhNgoY2IiIiIiIiIiMgEWGgjIiIiIiIiIiIyARbaiIiIiIiIiIiITICFNiIiIiIiIiIiIhMwe6Ft9erV8PPzg0KhwLBhw3Dw4MF6+/7zzz+YOHEi/Pz8IJFIsHLlytYbKBERERERERERUQPMWmjbtGkToqOjsWTJEhw5cgSBgYEIDw9HVlZWnf3LysrQvXt3LFu2DJ6enq08WiIiIiIiIiIiovqZtdC2YsUKzJgxA5GRkejbty/WrFkDOzs7rF27ts7+Q4YMwdtvv40pU6ZALpe38miJiIiIiIiIiIjqZ2WuF66srER8fDwWLVpkaJNKpQgLC0NcXJzJXketVkOtVhseFxYWAgCKiopM9hpE7UlNNoQQZh6JHjNM1HjML5FlY4aJLBfzS2TZTJlhsxXacnJyoNVqoVKpjNpVKhXOnDljsteJiYnBq6++Wqvd19fXZK9B1B4VFxdDqVSaexjMMFEzML9Elo0ZJrJczC+RZTNFhiXCTCX3K1euwNvbG/v370doaKihff78+dizZw8OHDjQ4PZ+fn6YO3cu5s6d22C/6yv5Op0OeXl5cHV1hUQiualjaOuKiorg6+uLlJQUODk5mXs4ZsP34arGvBdCCBQXF6Nz586QSs2+XgozzK9dvg/VmF/Lwq9bPb4PVzHDloVfu3p8H/SYX8vCr1s9vg9XtXaGzTajzc3NDTKZDJmZmUbtmZmZJl3oQC6X17qem7Ozs8n2bwmcnJw6fLAAvg/XutF70Rb+CleDGebXbg2+D3rMr2Xh160e34ermGHLwq9dPb4PesyvZeHXrR7fh6taK8NmK7Xb2NggODgYsbGxhjadTofY2FijGW5ERERERERERESWwGwz2gAgOjoa06ZNQ0hICIYOHYqVK1eitLQUkZGRAICIiAh4e3sjJiYGgH4BhVOnThnup6WlISEhAQ4ODvD39zfbcRAREREREREREZm10DZ58mRkZ2dj8eLFyMjIQFBQELZt22ZYICE5Odno3NgrV65g0KBBhsfvvPMO3nnnHYwZMwa7d+9u7eG3eXK5HEuWLKk1Zbij4ftwFd8Ly8L/Lz2+D3p8HywL/7/0+D5cxffCsvD/S4/vgx7fB8vC/y89vg9XtfZ7YbbFEIiIiIiIiIiIiNoT8y+HQkRERERERERE1A6w0EZERERERERERGQCLLQRERERERERERGZAAttREREREREREREJsBCWxsXExODIUOGwNHRER4eHpgwYQISExON+lRUVGDWrFlwdXWFg4MDJk6ciMzMTKM+ycnJuPfee2FnZwcPDw/MmzcPGo3GqM/u3bsxePBgyOVy+Pv7Y926dS19eM22bNkySCQSzJ0719DWUd6HtLQ0PProo3B1dYWtrS0GDBiAw4cPG54XQmDx4sXw8vKCra0twsLCcO7cOaN95OXl4ZFHHoGTkxOcnZ3xxBNPoKSkxKjP8ePHMWrUKCgUCvj6+uKtt95qleNrT5jfunXk/ALMsCVhhuvWkTPM/FoO5rduHTm/ADNsSZjhunXkDFtUfgW1aeHh4eKzzz4TJ0+eFAkJCeKee+4RXbp0ESUlJYY+Tz/9tPD19RWxsbHi8OHD4pZbbhHDhw83PK/RaET//v1FWFiYOHr0qPj111+Fm5ubWLRokaHPxYsXhZ2dnYiOjhanTp0Sq1atEjKZTGzbtq1Vj7cxDh48KPz8/MTAgQPFnDlzDO0d4X3Iy8sTXbt2FdOnTxcHDhwQFy9eFNu3bxfnz5839Fm2bJlQKpXihx9+EMeOHRP33Xef6NatmygvLzf0ueuuu0RgYKD4+++/xd69e4W/v7+YOnWq4fnCwkKhUqnEI488Ik6ePCk2bNggbG1txX//+99WPV5Lx/zW1pHzKwQzbGmY4do6coaZX8vC/NbWkfMrBDNsaZjh2jpyhi0tvyy0WZisrCwBQOzZs0cIIURBQYGwtrYW3377raHP6dOnBQARFxcnhBDi119/FVKpVGRkZBj6fPTRR8LJyUmo1WohhBDz588X/fr1M3qtyZMni/Dw8JY+pCYpLi4WAQEBYseOHWLMmDGGbzAd5X1YsGCBGDlyZL3P63Q64enpKd5++21DW0FBgZDL5WLDhg1CCCFOnTolAIhDhw4Z+vz2229CIpGItLQ0IYQQH374oXBxcTG8LzWv3atXL1MfUofC/Hbs/ArBDFs6ZrhjZ5j5tWzMb8fOrxDMsKVjhjt2hi0tvzx11MIUFhYCADp16gQAiI+PR1VVFcLCwgx9evfujS5duiAuLg4AEBcXhwEDBkClUhn6hIeHo6ioCP/884+hz7X7qOlTs4+2YtasWbj33ntrjbWjvA8//fQTQkJCMGnSJHh4eGDQoEH43//+Z3j+0qVLyMjIMDoGpVKJYcOGGb0Pzs7OCAkJMfQJCwuDVCrFgQMHDH1Gjx4NGxsbQ5/w8HAkJiYiPz+/pQ+z3WJ+O3Z+AWbY0jHDHTvDzK9lY347dn4BZtjSMcMdO8OWll8W2iyITqfD3LlzMWLECPTv3x8AkJGRARsbGzg7Oxv1ValUyMjIMPS5NlQ1z9c811CfoqIilJeXt8ThNNnGjRtx5MgRxMTE1Hquo7wPFy9exEcffYSAgABs374dzzzzDGbPno3PP/8cwNXjqOsYrj1GDw8Po+etrKzQqVOnJr1X1DTML/MLMMOWjBlmhplfy8X8Mr8AM2zJmGFm2NLya9WEYyMzmzVrFk6ePIl9+/aZeyitLiUlBXPmzMGOHTugUCjMPRyz0el0CAkJwdKlSwEAgwYNwsmTJ7FmzRpMmzbNzKOjhjC/zC/ADFsyZpgZZn4tF/PL/ALMsCVjhplhS8svZ7RZiKioKPzyyy/YtWsXfHx8DO2enp6orKxEQUGBUf/MzEx4enoa+ly/6kjN4xv1cXJygq2trakPp8ni4+ORlZWFwYMHw8rKClZWVtizZw/ef/99WFlZQaVSdYj3wcvLC3379jVq69OnD5KTkwFcPY66juHaY8zKyjJ6XqPRIC8vr0nvFTUe88v81mCGLRMzzAwDzK+lYn6Z3xrMsGVihplhwPLyy0JbGyeEQFRUFL7//nvs3LkT3bp1M3o+ODgY1tbWiI2NNbQlJiYiOTkZoaGhAIDQ0FCcOHHC6Itqx44dcHJyMnyxhoaGGu2jpk/NPszt9ttvx4kTJ5CQkGC4hYSE4JFHHjHc7wjvw4gRI2ota3327Fl07doVANCtWzd4enoaHUNRUREOHDhg9D4UFBQgPj7e0Gfnzp3Q6XQYNmyYoc+ff/6JqqoqQ58dO3agV69ecHFxabHja2+YXz3m9ypm2LIww3rMsB7za1mYXz3m9ypm2LIww3rMsJ7F5bdJSydQq3vmmWeEUqkUu3fvFunp6YZbWVmZoc/TTz8tunTpInbu3CkOHz4sQkNDRWhoqOH5muV877zzTpGQkCC2bdsm3N3d61zOd968eeL06dNi9erVbWo537pcu9qKEB3jfTh48KCwsrISb775pjh37pxYv369sLOzE1999ZWhz7Jly4Szs7P48ccfxfHjx8X48ePrXNZ40KBB4sCBA2Lfvn0iICDAaFnjgoICoVKpxGOPPSZOnjwpNm7cKOzs7LgseRMxv/XriPkVghm2NMxw/Tpihplfy8L81q8j5lcIZtjSMMP164gZtrT8stDWxgGo8/bZZ58Z+pSXl4uZM2cKFxcXYWdnJ+6//36Rnp5utJ+kpCRx9913C1tbW+Hm5iaef/55UVVVZdRn165dIigoSNjY2Iju3bsbvUZbdP03mI7yPvz888+if//+Qi6Xi969e4uPP/7Y6HmdTidefvlloVKphFwuF7fffrtITEw06pObmyumTp0qHBwchJOTk4iMjBTFxcVGfY4dOyZGjhwp5HK58Pb2FsuWLWvxY2tvmN/6ddT8CsEMWxJmuH4dNcPMr+VgfuvXUfMrBDNsSZjh+nXUDFtSfiVCCNH4+W9ERERERERERERUF16jjYiIiIiIiIiIyARYaCMiIiIiIiIiIjIBFtqIiIiIiIiIiIhMgIU2IiIiIiIiIiIiE2ChjYiIiIiIiIiIyARYaCMiIiIiIiIiIjIBFtqIiIiIiIiIiIhMgIU2IiIiIiIiIiIiE2ChrQ1ISkqCRCJBQkKCuYdicObMGdxyyy1QKBQICgpqldf08/PDypUrG91/9+7dkEgkKCgoaLExtWUd/fjbEmZYjxlumo5+/G0JM6zHDDdNRz/+toL51WN+m6ajH39bwgzrMcNN09aPn4U2ANOnT4dEIsGyZcuM2n/44QdIJBIzjcq8lixZAnt7eyQmJiI2NrbOPmPHjsXcuXNN9pqHDh3CU0891ej+w4cPR3p6OpRKpcnGQJaJGa6NGSZLwgzXxgyTpWB+a2N+yZIww7Uxw3SzWGirplAosHz5cuTn55t7KCZTWVnZ7G0vXLiAkSNHomvXrnB1dW32foQQ0Gg0jerr7u4OOzu7Ru/bxsYGnp6eHfYHABljho0xw2RpmGFjzDBZEubXGPNLloYZNsYM081ioa1aWFgYPD09ERMTU2+fV155pdbU0ZUrV8LPz8/wePr06ZgwYQKWLl0KlUoFZ2dnvPbaa9BoNJg3bx46deoEHx8ffPbZZ7X2f+bMGQwfPhwKhQL9+/fHnj17jJ4/efIk7r77bjg4OEClUuGxxx5DTk6O4fmxY8ciKioKc+fOhZubG8LDw+s8Dp1Oh9deew0+Pj6Qy+UICgrCtm3bDM9LJBLEx8fjtddeg0QiwSuvvFJrH9OnT8eePXvw3nvvQSKRQCKRICkpyTCF87fffkNwcDDkcjn27duHCxcuYPz48VCpVHBwcMCQIUPwxx9/GO3z+umyEokEn3zyCe6//37Y2dkhICAAP/30k+H566eLrlu3Ds7Ozti+fTv69OkDBwcH3HXXXUhPTzdso9FoMHv2bDg7O8PV1RULFizAtGnTMGHChDrfKwC4fPkyxo0bBxcXF9jb26Nfv3749ddfAQBarRZPPPEEunXrBltbW/Tq1QvvvfderfeqqV8TNVOoN27c2ODXxPX27duHUaNGwdbWFr6+vpg9ezZKS0sNz3/44YcICAiAQqGASqXCgw8+2OD+LAkzzAzXhxm2DMwwM1wfZrjtY36Z3/owv5aBGWaG68MMN5MgMW3aNDF+/HixZcsWoVAoREpKihBCiO+//15c+xYtWbJEBAYGGm377rvviq5duxrty9HRUcyaNUucOXNGfPrppwKACA8PF2+++aY4e/aseP3114W1tbXhdS5duiQACB8fH/Hdd9+JU6dOiSeffFI4OjqKnJwcIYQQ+fn5wt3dXSxatEicPn1aHDlyRNxxxx3i1ltvNbz2mDFjhIODg5g3b544c+aMOHPmTJ3Hu2LFCuHk5CQ2bNggzpw5I+bPny+sra3F2bNnhRBCpKeni379+onnn39epKeni+Li4lr7KCgoEKGhoWLGjBkiPT1dpKenC41GI3bt2iUAiIEDB4rff/9dnD9/XuTm5oqEhASxZs0aceLECXH27Fnx0ksvCYVCIS5fvmzYZ9euXcW7775reFzznnz99dfi3LlzYvbs2cLBwUHk5uYKIYThtfLz84UQQnz22WfC2tpahIWFiUOHDon4+HjRp08f8fDDDxv2+cYbb4hOnTqJLVu2iNOnT4unn35aODk5ifHjx9f5XgkhxL333ivuuOMOcfz4cXHhwgXx888/iz179gghhKisrBSLFy8Whw4dEhcvXhRfffWVsLOzE5s2bWrxr4nrj//8+fPC3t5evPvuu+Ls2bPir7/+EoMGDRLTp08XQghx6NAhIZPJxNdffy2SkpLEkSNHxHvvvVfvcVsSZpgZZoYtGzPMDDPDlov5ZX6ZX8vGDDPDzLDpsdAmrn5zEUKIW265RTz++ONCiOZ/c+natavQarWGtl69eolRo0YZHms0GmFvby82bNgghLj6hbRs2TJDn6qqKuHj4yOWL18uhBDi9ddfF3feeafRa6ekpAgAIjExUQih/+YyaNCgGx5v586dxZtvvmnUNmTIEDFz5kzD48DAQLFkyZIG9zNmzBgxZ84co7aaL/gffvjhhuPo16+fWLVqleFxXd9cXnrpJcPjkpISAUD89ttvRq917TcXAOL8+fOGbVavXi1UKpXhsUqlEm+//bbhsUajEV26dGnwm8uAAQPEK6+8csPjqTFr1iwxceJEw+OW+pq4/vifeOIJ8dRTTxmNZe/evUIqlYry8nKxefNm4eTkJIqKihp9LJaCGWaGmWHLxgwzw8yw5WJ+mV/m17Ixw8wwM2x6PHX0OsuXL8fnn3+O06dPN3sf/fr1g1R69a1VqVQYMGCA4bFMJoOrqyuysrKMtgsNDTXct7KyQkhIiGEcx44dw65du+Dg4GC49e7dG4D+HPIawcHBDY6tqKgIV65cwYgRI4zaR4wYcVPHfL2QkBCjxyUlJXjhhRfQp08fODs7w8HBAadPn0ZycnKD+xk4cKDhvr29PZycnGq9b9eys7NDjx49DI+9vLwM/QsLC5GZmYmhQ4canpfJZDd8z2bPno033ngDI0aMwJIlS3D8+HGj51evXo3g4GC4u7vDwcEBH3/8ca3jaomviesdO3YM69atM/oaCQ8Ph06nw6VLl3DHHXega9eu6N69Ox577DGsX78eZWVlDR67JWKGTYMZZobNhRk2DWaYGTYH5tc0mF/m11yYYdNghplhFtquM3r0aISHh2PRokW1npNKpRBCGLVVVVXV6mdtbW30WCKR1Nmm0+kaPa6SkhKMGzcOCQkJRrdz585h9OjRhn729vaN3mdLun4cL7zwAr7//nssXboUe/fuRUJCAgYMGHDDi1Q29X2rq//1/2dN9eSTT+LixYt47LHHcOLECYSEhGDVqlUAgI0bN+KFF17AE088gd9//x0JCQmIjIysdVwt8TVxvZKSEvzrX/8y+vo4duwYzp07hx49esDR0RFHjhzBhg0b4OXlhcWLFyMwMLDNLoncXMywaTDDzLC5MMOmwQwzw+bA/JoG88v8mgszbBrMMDPMQlsdli1bhp9//hlxcXFG7e7u7sjIyDD6Yk1ISDDZ6/7999+G+xqNBvHx8ejTpw8AYPDgwfjnn3/g5+cHf39/o1tTvqE4OTmhc+fO+Ouvv4za//rrL/Tt27dJ47WxsYFWq21U37/++gvTp0/H/fffjwEDBsDT0xNJSUlNer2bpVQqoVKpcOjQIUObVqvFkSNHbritr68vnn76aWzZsgXPP/88/ve//wHQH9fw4cMxc+ZMDBo0CP7+/kZ/VblZDX1NXG/w4ME4depUra8Pf39/2NjYAND/JSAsLAxvvfUWjh8/jqSkJOzcudNk420rmOHGYYaZ4baKGW4cZpgZbouY38ZhfpnftooZbhxmmBluCAttdRgwYAAeeeQRvP/++0btY8eORXZ2Nt566y1cuHABq1evxm+//Way1129ejW+//57nDlzBrNmzUJ+fj4ef/xxAMCsWbOQl5eHqVOn4tChQ7hw4QK2b9+OyMjIRge8xrx587B8+XJs2rQJiYmJWLhwIRISEjBnzpwm7cfPzw8HDhxAUlIScnJyGqxABwQEYMuWLYbK8sMPP3xTFevmevbZZxETE4Mff/wRiYmJmDNnDvLz8xtcFnnu3LnYvn07Ll26hCNHjmDXrl2GgAcEBODw4cPYvn07zp49i5dfftnom9fNauhr4noLFizA/v37ERUVZfgLz48//oioqCgAwC+//IL3338fCQkJuHz5Mr744gvodDr06tXLZONtK5jhxmGGmeG2ihluHGaYGW6LmN/GYX6Z37aKGW4cZpgZbggLbfV47bXXan3x9+nTBx9++CFWr16NwMBAHDx4EC+88ILJXnPZsmVYtmwZAgMDsW/fPvz0009wc3MDAEPlXavV4s4778SAAQMwd+5cODs7G53v3BizZ89GdHQ0nn/+eQwYMADbtm3DTz/9hICAgCbt54UXXoBMJkPfvn3h7u7e4DnmK1asgIuLC4YPH45x48YhPDwcgwcPbtLrmcKCBQswdepUREREIDQ01HDutkKhqHcbrVaLWbNmoU+fPrjrrrvQs2dPfPjhhwCAf/3rX3jggQcwefJkDBs2DLm5uZg5c6bJxtvQ18T1Bg4ciD179uDs2bMYNWoUBg0ahMWLF6Nz584AAGdnZ2zZsgW33XYb+vTpgzVr1mDDhg3o16+fycbbljDDN8YMM8NtGTN8Y8wwM9xWMb83xvwyv20ZM3xjzDAz3BCJuNmTdoksmE6nQ58+ffDQQw/h9ddfN/dwDJKSktCtWzccPXoUQUFB5h4OUZvFDBNZNmaYyHIxv0SWjRluOVbmHgBRa7p8+TJ+//13jBkzBmq1Gh988AEuXbqEhx9+2NxDI6JGYIaJLBszTGS5mF8iy8YMtx6eOkodilQqxbp16zBkyBCMGDECJ06cwB9//FHvRRWJqG1hhoksGzNMZLmYXyLLxgy3Hp46SkREREREREREZAKc0UZERERERERERGQCLLQRERERERERERGZAAttREREREREREREJsBCGxERERERERERkQmw0EZERERERERERGQCLLQRERERERERERGZAAttREREREREREREJsBCGxERERERERERkQn8P5EMSnGY9I4HAAAAAElFTkSuQmCC\n","text/plain":["\u003cFigure size 1500x200 with 5 Axes\u003e"]},"metadata":{},"output_type":"display_data"},{"name":"stdout","output_type":"stream","text":["{500: {'accuracy': 0.6190000176429749, 'f1': 0.620517909526825, 'tnr': 0.6239432096481323, 'tpr': 0.6141965389251709, 'mcc': 0.23812001943588257}, 1000: {'accuracy': 0.6626666784286499, 'f1': 0.6615384817123413, 'tnr': 0.6756848096847534, 'tpr': 0.6500164270401001, 'mcc': 0.32573962211608887}, 1500: {'accuracy': 0.6610000133514404, 'f1': 0.6551373600959778, 'tnr': 0.687859296798706, 'tpr': 0.6348997950553894, 'mcc': 0.32310354709625244}, 2000: {'accuracy': 0.6980000138282776, 'f1': 0.7095223069190979, 'tnr': 0.667906641960144, 'tpr': 0.7272428274154663, 'mcc': 0.3959462344646454}, 2500: {'accuracy': 0.6961666941642761, 'f1': 0.696722686290741, 'tnr': 0.7044301629066467, 'tpr': 0.6881366968154907, 'mcc': 0.39254888892173767}, 3000: {'accuracy': 0.7138333320617676, 'f1': 0.7084394693374634, 'tnr': 0.7429827451705933, 'tpr': 0.6855077147483826, 'mcc': 0.4290120601654053}, 3500: {'accuracy': 0.7319999933242798, 'f1': 0.7293840646743774, 'tnr': 0.7524518370628357, 'tpr': 0.7121261954307556, 'mcc': 0.46479377150535583}, 4000: {'accuracy': 0.7319999933242798, 'f1': 0.7320892810821533, 'tnr': 0.7423064112663269, 'tpr': 0.72198486328125, 'mcc': 0.4642869532108307}, 4500: {'accuracy': 0.7293333411216736, 'f1': 0.7319247126579285, 'tnr': 0.7301318645477295, 'tpr': 0.7285573482513428, 'mcc': 0.45864784717559814}, 5000: {'accuracy': 0.7483333349227905, 'f1': 0.7524589896202087, 'tnr': 0.7423064112663269, 'tpr': 0.7541899681091309, 'mcc': 0.49653497338294983}, 5500: {'accuracy': 0.7716666460037231, 'f1': 0.7723495960235596, 'tnr': 0.7798444628715515, 'tpr': 0.7637200355529785, 'mcc': 0.5435274839401245}, 6000: {'accuracy': 0.7615000009536743, 'f1': 0.7623318433761597, 'tnr': 0.7690226435661316, 'tpr': 0.7541899681091309, 'mcc': 0.523172914981842}}\n"]}],"source":["print(results)\n","\n","metrics = sorted(list(results[1000].keys()))\n","fig, subplots = plt.subplots(ncols=len(metrics), sharey=True, figsize=(15, 2))\n","\n","for metric_index, metric in enumerate(metrics):\n","  to_plot_x, to_plot_y = zip(*[(num, results[num][metric]) for num in sorted(results.keys())])\n","\n","  subplots[metric_index].plot(to_plot_x, to_plot_y)\n","  subplots[metric_index].set_title(metric)\n","  subplots[metric_index].set_ylabel(metric)\n","  subplots[metric_index].set_xlabel('Number of training samples')\n","  subplots[metric_index].set_yticks([0.1, 0.2, 0.3, 0.4, 0.5, 0.6, 0.7, 0.8, 0.9, 1.0])\n","\n","plt.show()\n","print(results)"]},{"cell_type":"code","execution_count":null,"metadata":{"colab":{"base_uri":"https://localhost:8080/","height":1000},"executionInfo":{"elapsed":2634102,"status":"error","timestamp":1712536178604,"user":{"displayName":"Robert Cobb","userId":"08992261826785514419"},"user_tz":-60},"id":"6rKtqblLTWVc","outputId":"782299eb-858f-4840-fe05-f656d401e9cd"},"outputs":[{"name":"stdout","output_type":"stream","text":["Entered function get_data_for_classifier\n","Exiting function get_data_for_classifier - took 0.20 seconds\n","Entered function get_tokenizer\n","Loading BERT tokenizer...\n","Exiting function get_tokenizer - took 0.92 seconds\n","Entered function get_balanced_cross_fold_train_test_split\n","Exiting function get_balanced_cross_fold_train_test_split - took 0.00 seconds\n","Skipping BioBert cross fold index 0 because we have results\n","Skipping BioBert cross fold index 1 because we have results\n","Skipping BioBert cross fold index 2 because we have results\n","Skipping BioBert cross fold index 3 because we have results\n","Skipping BioBert cross fold index 4 because we have results\n","{'BioBert': [{'accuracy': 0.7799999713897705, 'f1': 0.7788944840431213, 'tnr': 0.7850000262260437, 'tpr': 0.7749999761581421, 'mcc': 0.5600280165672302}, {'accuracy': 0.7741666436195374, 'f1': 0.7794955372810364, 'tnr': 0.75, 'tpr': 0.7983333468437195, 'mcc': 0.5489749312400818}, {'accuracy': 0.7808333039283752, 'f1': 0.777307391166687, 'tnr': 0.79666668176651, 'tpr': 0.7649999856948853, 'mcc': 0.5619485378265381}, {'accuracy': 0.784166693687439, 'f1': 0.7819865345954895, 'tnr': 0.7941666841506958, 'tpr': 0.7741666436195374, 'mcc': 0.5684469938278198}, {'accuracy': 0.7724999785423279, 'f1': 0.7682512998580933, 'tnr': 0.7908333539962769, 'tpr': 0.7541666626930237, 'mcc': 0.5453667640686035}], 'sap-bert': [{'accuracy': 0.7699999809265137, 'f1': 0.7591623067855835, 'tnr': 0.8149999976158142, 'tpr': 0.7250000238418579, 'mcc': 0.5422003865242004}, {'accuracy': 0.7708333134651184, 'f1': 0.7581354379653931, 'tnr': 0.8233333230018616, 'tpr': 0.7183333039283752, 'mcc': 0.5446774959564209}, {'accuracy': 0.7962499856948853, 'f1': 0.7902187705039978, 'tnr': 0.824999988079071, 'tpr': 0.7674999833106995, 'mcc': 0.5934818983078003}, {'accuracy': 0.7699999809265137, 'f1': 0.7602084875106812, 'tnr': 0.8108333349227905, 'tpr': 0.7291666865348816, 'mcc': 0.5418097972869873}, {'accuracy': 0.7795833349227905, 'f1': 0.7763213515281677, 'tnr': 0.7941666841506958, 'tpr': 0.7649999856948853, 'mcc': 0.5594046711921692}]}\n","Entered function get_tokenizer\n","Loading BERT tokenizer...\n","Exiting function get_tokenizer - took 0.31 seconds\n","Entered function get_balanced_cross_fold_train_test_split\n","Exiting function get_balanced_cross_fold_train_test_split - took 0.00 seconds\n","Skipping sap-bert cross fold index 0 because we have results\n","Skipping sap-bert cross fold index 1 because we have results\n","Skipping sap-bert cross fold index 2 because we have results\n","Skipping sap-bert cross fold index 3 because we have results\n","Skipping sap-bert cross fold index 4 because we have results\n","{'BioBert': [{'accuracy': 0.7799999713897705, 'f1': 0.7788944840431213, 'tnr': 0.7850000262260437, 'tpr': 0.7749999761581421, 'mcc': 0.5600280165672302}, {'accuracy': 0.7741666436195374, 'f1': 0.7794955372810364, 'tnr': 0.75, 'tpr': 0.7983333468437195, 'mcc': 0.5489749312400818}, {'accuracy': 0.7808333039283752, 'f1': 0.777307391166687, 'tnr': 0.79666668176651, 'tpr': 0.7649999856948853, 'mcc': 0.5619485378265381}, {'accuracy': 0.784166693687439, 'f1': 0.7819865345954895, 'tnr': 0.7941666841506958, 'tpr': 0.7741666436195374, 'mcc': 0.5684469938278198}, {'accuracy': 0.7724999785423279, 'f1': 0.7682512998580933, 'tnr': 0.7908333539962769, 'tpr': 0.7541666626930237, 'mcc': 0.5453667640686035}], 'sap-bert': [{'accuracy': 0.7699999809265137, 'f1': 0.7591623067855835, 'tnr': 0.8149999976158142, 'tpr': 0.7250000238418579, 'mcc': 0.5422003865242004}, {'accuracy': 0.7708333134651184, 'f1': 0.7581354379653931, 'tnr': 0.8233333230018616, 'tpr': 0.7183333039283752, 'mcc': 0.5446774959564209}, {'accuracy': 0.7962499856948853, 'f1': 0.7902187705039978, 'tnr': 0.824999988079071, 'tpr': 0.7674999833106995, 'mcc': 0.5934818983078003}, {'accuracy': 0.7699999809265137, 'f1': 0.7602084875106812, 'tnr': 0.8108333349227905, 'tpr': 0.7291666865348816, 'mcc': 0.5418097972869873}, {'accuracy': 0.7795833349227905, 'f1': 0.7763213515281677, 'tnr': 0.7941666841506958, 'tpr': 0.7649999856948853, 'mcc': 0.5594046711921692}]}\n","Entered function get_tokenizer\n","Loading BERT tokenizer...\n"]},{"name":"stderr","output_type":"stream","text":["Truncation was not explicitly activated but `max_length` is provided a specific value, please use `truncation=True` to explicitly truncate examples to max length. Defaulting to 'longest_first' truncation strategy. If you encode pairs of sequences (GLUE-style) with the tokenizer you can select this strategy more precisely by providing a specific strategy to `truncation`.\n"]},{"name":"stdout","output_type":"stream","text":["Exiting function get_tokenizer - took 0.59 seconds\n","Entered function get_balanced_cross_fold_train_test_split\n","Exiting function get_balanced_cross_fold_train_test_split - took 0.00 seconds\n","Entered function get_loader_cache\n"]},{"name":"stderr","output_type":"stream","text":["/usr/local/lib/python3.10/dist-packages/transformers/tokenization_utils_base.py:2645: FutureWarning: The `pad_to_max_length` argument is deprecated and will be removed in a future version, use `padding=True` or `padding='longest'` to pad to the longest sequence in the batch, or use `padding='max_length'` to pad to a max length. In this case, you can give a specific length with `max_length` (e.g. `max_length=45`) or leave max_length to None to pad to the maximal input size of the model (e.g. 512 for Bert).\n","  warnings.warn(\n"]},{"name":"stdout","output_type":"stream","text":["Original:  ADMISSION\n","Pt adm to MICU from RIH, on [**2127-1-25**] at [**2133**] for trachial stent placement in am. Pt had several weeksPt oriented to room. A\u0026O x3, follows all commands. CM pattern s/r s/t with hr 90-120, Resident and intern in with pt to answer questions.Pt npo. IV, d5 [**12-19**] infusing at 150 hr.\n","\n","Token IDs: tensor([    0,     0,     0,     0,     0,     0,     0,     0,     0,     0,\n","            0,     0,     0,     0,     0,     0,     0,     0,     0,     0,\n","            0,     0,     0,     0,     0,     0,     0,     0,     0,     0,\n","            0,     0,     0,     0,     0,     0,     0,     0,     0,     0,\n","            0,     0,     0,     0,     0,     0,     0,     0,     0,     0,\n","            0,     0,     0,     0,     0,     0,     0,     0,     0,     0,\n","            0,     0,     0,     0,     0,     0,     0,     0,     0,     0,\n","            0,     0,     0,     0,     0,     0,     0,     0,     0,     0,\n","            0,     0,     0,     0,     0,     0,     0,     0,     0,     0,\n","            0,     0,     0,     0,     0,     0,     0,     0,     0,     0,\n","            0,     0,     0,     0,     0,     0,     0,     0,     0,     0,\n","            0,     0,     0,     0,     0,     0,     0,     0,     0,     0,\n","            0,     0,     0,     0,     0,     0,     0,     0,     0,     0,\n","            0,     0,     0,     0,     0,     0,     0,     0,     0,     0,\n","            0,     0,     0,     0,     0,     0,     0,     0,     0,     0,\n","            0,     0,     0,     0,     0,     0,     0,     0,     0,     0,\n","            0,     0,     0,     0,     0,     0,     0,     0,     0,     0,\n","            0,     0,     0,     0,     0,     0,     0,     0,     0,     0,\n","            0,     0,     0,     0,     0,     0,     0,     0,     0,     0,\n","            0,     0,     0,     0,     0,     0,     0,     0,     0,     0,\n","            0,     0,     0,     0,     0,     0,     0,     0,     0,     0,\n","            0,     0,     0,     0,     0,     0,     0,     0,     0,     0,\n","            0,     0,     0,     0,     0,     0,     0,     0,     0,     0,\n","            0,     0,     0,     0,     0,     0,     0,     0,     0,     0,\n","            0,     0,     0,     0,     0,     0,     0,     0,     0,     0,\n","            0,     0,     0,     0,     0,     0,     0,     0,     0,     0,\n","            0,     0,     0,     0,     0,     0,     0,     0,     0,     0,\n","            0,     0,     0,     0,     0,     0,     0,     0,     0,     0,\n","            0,     0,     0,     0,     0,     0,     0,     0,     0,     0,\n","            0,     0,     0,     0,     0,     0,     0,     0,     0,     0,\n","            0,     0,     0,     0,     0,     0,     0,     0,     0,     0,\n","            0,     0,     0,     0,     0,     0,     0,     0,     0,     0,\n","            0,     0,     0,     0,     0,     0,     0,     0,     0,     0,\n","            0,     0,     0,     0,     0,     0,     0,     0,     0,     0,\n","            0,     0,     0,     0,     0,     0,     0,     0,     0,     0,\n","            0,     0,     0,     0,     0,     0,     0,     0,     0,     0,\n","            0,     0,     0,     0,     0,     0,     0,     0,     0,     0,\n","            0,     0,     0,     0,     0,     0,     0,     0,     0,     0,\n","            0,     0,     0,     0,     0,     0,     0,     0,     0,     0,\n","            0,     0,     0,     0,     0,     0,     0,     0,     0,     0,\n","            0,     0,     0,     0,     0,     0,     0,     0,   101,  2449,\n","          724,   973,   165,  1617, 50021,   379,   249, 21717,   119,   215,\n","          136,   104,   104, 34380,   141,   147,   141,   760,   104,   104,\n","          137,   248,   136,   104,   104, 33887,   104,   104,   137,   194,\n","         9570,   293,  4940,  2990,   145,   709,   112,   724,   676,  2533,\n","         1544,   343,  1978,   165,  3135,   112,   106,   801,   130,   632,\n","        50058,   119,  5889,   640,  7754,   112,   972,  2775,   117,   170,\n","          249,   117,   170,   107,   189,  2186,  1385,   141,  2911,   119,\n","         2498,   150,  8774,   145,   189,   724,   165,  9891,  1784,   112,\n","          724, 44001,   112, 10711,   119,   133, 50057,   136,   104,   104,\n","          423,   141,   598,   104,   104,   137, 21737,   248,  2137,  2186,\n","          112,   102])\n","8,640 training samples\n","  960 validation samples\n","Exiting function get_loader_cache - took 68.98 seconds\n","Entered function define_dataloaders\n","Exiting function define_dataloaders - took 0.00 seconds\n","Entered function define_model\n"]},{"name":"stderr","output_type":"stream","text":["You are using a model of type megatron-bert to instantiate a model of type bert. This is not supported for all configurations of models and can yield errors.\n"]},{"data":{"application/vnd.jupyter.widget-view+json":{"model_id":"ef333f9fc5c8449fab52bdc008b58126","version_major":2,"version_minor":0},"text/plain":["pytorch_model.bin:   0%|          | 0.00/713M [00:00\u003c?, ?B/s]"]},"metadata":{},"output_type":"display_data"},{"name":"stderr","output_type":"stream","text":["Some weights of BertForSequenceClassification were not initialized from the model checkpoint at UFNLP/gatortron-base and are newly initialized: ['bert.embeddings.LayerNorm.bias', 'bert.embeddings.LayerNorm.weight', 'bert.encoder.layer.0.attention.output.LayerNorm.bias', 'bert.encoder.layer.0.attention.output.LayerNorm.weight', 'bert.encoder.layer.0.output.LayerNorm.bias', 'bert.encoder.layer.0.output.LayerNorm.weight', 'bert.encoder.layer.1.attention.output.LayerNorm.bias', 'bert.encoder.layer.1.attention.output.LayerNorm.weight', 'bert.encoder.layer.1.output.LayerNorm.bias', 'bert.encoder.layer.1.output.LayerNorm.weight', 'bert.encoder.layer.10.attention.output.LayerNorm.bias', 'bert.encoder.layer.10.attention.output.LayerNorm.weight', 'bert.encoder.layer.10.output.LayerNorm.bias', 'bert.encoder.layer.10.output.LayerNorm.weight', 'bert.encoder.layer.11.attention.output.LayerNorm.bias', 'bert.encoder.layer.11.attention.output.LayerNorm.weight', 'bert.encoder.layer.11.output.LayerNorm.bias', 'bert.encoder.layer.11.output.LayerNorm.weight', 'bert.encoder.layer.12.attention.output.LayerNorm.bias', 'bert.encoder.layer.12.attention.output.LayerNorm.weight', 'bert.encoder.layer.12.output.LayerNorm.bias', 'bert.encoder.layer.12.output.LayerNorm.weight', 'bert.encoder.layer.13.attention.output.LayerNorm.bias', 'bert.encoder.layer.13.attention.output.LayerNorm.weight', 'bert.encoder.layer.13.output.LayerNorm.bias', 'bert.encoder.layer.13.output.LayerNorm.weight', 'bert.encoder.layer.14.attention.output.LayerNorm.bias', 'bert.encoder.layer.14.attention.output.LayerNorm.weight', 'bert.encoder.layer.14.output.LayerNorm.bias', 'bert.encoder.layer.14.output.LayerNorm.weight', 'bert.encoder.layer.15.attention.output.LayerNorm.bias', 'bert.encoder.layer.15.attention.output.LayerNorm.weight', 'bert.encoder.layer.15.output.LayerNorm.bias', 'bert.encoder.layer.15.output.LayerNorm.weight', 'bert.encoder.layer.16.attention.output.LayerNorm.bias', 'bert.encoder.layer.16.attention.output.LayerNorm.weight', 'bert.encoder.layer.16.output.LayerNorm.bias', 'bert.encoder.layer.16.output.LayerNorm.weight', 'bert.encoder.layer.17.attention.output.LayerNorm.bias', 'bert.encoder.layer.17.attention.output.LayerNorm.weight', 'bert.encoder.layer.17.output.LayerNorm.bias', 'bert.encoder.layer.17.output.LayerNorm.weight', 'bert.encoder.layer.18.attention.output.LayerNorm.bias', 'bert.encoder.layer.18.attention.output.LayerNorm.weight', 'bert.encoder.layer.18.output.LayerNorm.bias', 'bert.encoder.layer.18.output.LayerNorm.weight', 'bert.encoder.layer.19.attention.output.LayerNorm.bias', 'bert.encoder.layer.19.attention.output.LayerNorm.weight', 'bert.encoder.layer.19.output.LayerNorm.bias', 'bert.encoder.layer.19.output.LayerNorm.weight', 'bert.encoder.layer.2.attention.output.LayerNorm.bias', 'bert.encoder.layer.2.attention.output.LayerNorm.weight', 'bert.encoder.layer.2.output.LayerNorm.bias', 'bert.encoder.layer.2.output.LayerNorm.weight', 'bert.encoder.layer.20.attention.output.LayerNorm.bias', 'bert.encoder.layer.20.attention.output.LayerNorm.weight', 'bert.encoder.layer.20.output.LayerNorm.bias', 'bert.encoder.layer.20.output.LayerNorm.weight', 'bert.encoder.layer.21.attention.output.LayerNorm.bias', 'bert.encoder.layer.21.attention.output.LayerNorm.weight', 'bert.encoder.layer.21.output.LayerNorm.bias', 'bert.encoder.layer.21.output.LayerNorm.weight', 'bert.encoder.layer.22.attention.output.LayerNorm.bias', 'bert.encoder.layer.22.attention.output.LayerNorm.weight', 'bert.encoder.layer.22.output.LayerNorm.bias', 'bert.encoder.layer.22.output.LayerNorm.weight', 'bert.encoder.layer.23.attention.output.LayerNorm.bias', 'bert.encoder.layer.23.attention.output.LayerNorm.weight', 'bert.encoder.layer.23.output.LayerNorm.bias', 'bert.encoder.layer.23.output.LayerNorm.weight', 'bert.encoder.layer.3.attention.output.LayerNorm.bias', 'bert.encoder.layer.3.attention.output.LayerNorm.weight', 'bert.encoder.layer.3.output.LayerNorm.bias', 'bert.encoder.layer.3.output.LayerNorm.weight', 'bert.encoder.layer.4.attention.output.LayerNorm.bias', 'bert.encoder.layer.4.attention.output.LayerNorm.weight', 'bert.encoder.layer.4.output.LayerNorm.bias', 'bert.encoder.layer.4.output.LayerNorm.weight', 'bert.encoder.layer.5.attention.output.LayerNorm.bias', 'bert.encoder.layer.5.attention.output.LayerNorm.weight', 'bert.encoder.layer.5.output.LayerNorm.bias', 'bert.encoder.layer.5.output.LayerNorm.weight', 'bert.encoder.layer.6.attention.output.LayerNorm.bias', 'bert.encoder.layer.6.attention.output.LayerNorm.weight', 'bert.encoder.layer.6.output.LayerNorm.bias', 'bert.encoder.layer.6.output.LayerNorm.weight', 'bert.encoder.layer.7.attention.output.LayerNorm.bias', 'bert.encoder.layer.7.attention.output.LayerNorm.weight', 'bert.encoder.layer.7.output.LayerNorm.bias', 'bert.encoder.layer.7.output.LayerNorm.weight', 'bert.encoder.layer.8.attention.output.LayerNorm.bias', 'bert.encoder.layer.8.attention.output.LayerNorm.weight', 'bert.encoder.layer.8.output.LayerNorm.bias', 'bert.encoder.layer.8.output.LayerNorm.weight', 'bert.encoder.layer.9.attention.output.LayerNorm.bias', 'bert.encoder.layer.9.attention.output.LayerNorm.weight', 'bert.encoder.layer.9.output.LayerNorm.bias', 'bert.encoder.layer.9.output.LayerNorm.weight', 'classifier.bias', 'classifier.weight']\n","You should probably TRAIN this model on a down-stream task to be able to use it for predictions and inference.\n"]},{"name":"stdout","output_type":"stream","text":["Exiting function define_model - took 4.94 seconds\n","Entered function train\n","\n","======== Epoch 1 / 4 ========\n","Training...\n"]},{"name":"stderr","output_type":"stream","text":["/usr/local/lib/python3.10/dist-packages/transformers/optimization.py:429: FutureWarning: This implementation of AdamW is deprecated and will be removed in a future version. Use the PyTorch implementation torch.optim.AdamW instead, or set `no_deprecation_warning=True` to disable this warning\n","  warnings.warn(\n"]},{"name":"stdout","output_type":"stream","text":["  Batch    40  of    540.    Elapsed: 44.300710916519165.\n","  Batch    80  of    540.    Elapsed: 88.51730108261108.\n","  Batch   120  of    540.    Elapsed: 132.72234654426575.\n","  Batch   160  of    540.    Elapsed: 176.91475081443787.\n","  Batch   200  of    540.    Elapsed: 221.1029908657074.\n","  Batch   240  of    540.    Elapsed: 265.30495953559875.\n","  Batch   280  of    540.    Elapsed: 309.49387860298157.\n","  Batch   320  of    540.    Elapsed: 353.68317103385925.\n","  Batch   360  of    540.    Elapsed: 397.87222027778625.\n","  Batch   400  of    540.    Elapsed: 442.06385493278503.\n","  Batch   440  of    540.    Elapsed: 486.25200176239014.\n","  Batch   480  of    540.    Elapsed: 530.437397480011.\n","  Batch   520  of    540.    Elapsed: 574.6225373744965.\n","\n","  Average training loss: 0.72\n","  Training epcoh took: 596.7072694301605\n","\n","Running Validation...\n","  Accuracy: 0.48\n","  Validation Loss: 0.72\n","  Validation took: 21.49752950668335\n","\n","======== Epoch 2 / 4 ========\n","Training...\n","  Batch    40  of    540.    Elapsed: 44.18484163284302.\n","  Batch    80  of    540.    Elapsed: 88.36378979682922.\n","  Batch   120  of    540.    Elapsed: 132.54567742347717.\n","  Batch   160  of    540.    Elapsed: 176.73779845237732.\n","  Batch   200  of    540.    Elapsed: 220.92970609664917.\n","  Batch   240  of    540.    Elapsed: 265.1216399669647.\n","  Batch   280  of    540.    Elapsed: 309.31861114501953.\n","  Batch   320  of    540.    Elapsed: 353.5154552459717.\n","  Batch   360  of    540.    Elapsed: 397.7147796154022.\n","  Batch   400  of    540.    Elapsed: 441.91091990470886.\n","  Batch   440  of    540.    Elapsed: 486.1092596054077.\n","  Batch   480  of    540.    Elapsed: 530.3049209117889.\n","  Batch   520  of    540.    Elapsed: 574.4960684776306.\n","\n","  Average training loss: 0.70\n","  Training epcoh took: 596.5951373577118\n","\n","Running Validation...\n","  Accuracy: 0.48\n","  Validation Loss: 0.70\n","  Validation took: 21.505932092666626\n","\n","======== Epoch 3 / 4 ========\n","Training...\n","  Batch    40  of    540.    Elapsed: 44.18895506858826.\n","  Batch    80  of    540.    Elapsed: 88.38024973869324.\n","  Batch   120  of    540.    Elapsed: 132.57227087020874.\n","  Batch   160  of    540.    Elapsed: 176.76386094093323.\n","  Batch   200  of    540.    Elapsed: 220.95379781723022.\n","  Batch   240  of    540.    Elapsed: 265.1502161026001.\n","  Batch   280  of    540.    Elapsed: 309.35409140586853.\n","  Batch   320  of    540.    Elapsed: 353.54709029197693.\n","  Batch   360  of    540.    Elapsed: 397.7345471382141.\n","  Batch   400  of    540.    Elapsed: 441.93602752685547.\n","  Batch   440  of    540.    Elapsed: 486.1330406665802.\n","  Batch   480  of    540.    Elapsed: 530.3308682441711.\n","  Batch   520  of    540.    Elapsed: 574.5523655414581.\n","\n","  Average training loss: 0.70\n","  Training epcoh took: 596.6558248996735\n","\n","Running Validation...\n","  Accuracy: 0.52\n","  Validation Loss: 0.69\n","  Validation took: 21.512532234191895\n","\n","======== Epoch 4 / 4 ========\n","Training...\n","  Batch    40  of    540.    Elapsed: 44.201027154922485.\n","  Batch    80  of    540.    Elapsed: 88.39714360237122.\n","  Batch   120  of    540.    Elapsed: 132.59043431282043.\n","  Batch   160  of    540.    Elapsed: 176.79326629638672.\n","  Batch   200  of    540.    Elapsed: 220.99324679374695.\n","  Batch   240  of    540.    Elapsed: 265.1967444419861.\n","  Batch   280  of    540.    Elapsed: 309.4074287414551.\n","  Batch   320  of    540.    Elapsed: 353.6081690788269.\n","  Batch   360  of    540.    Elapsed: 397.81463074684143.\n","  Batch   400  of    540.    Elapsed: 442.0147223472595.\n","  Batch   440  of    540.    Elapsed: 486.2132499217987.\n","  Batch   480  of    540.    Elapsed: 530.4193823337555.\n","  Batch   520  of    540.    Elapsed: 574.6269619464874.\n","\n","  Average training loss: 0.70\n","  Training epcoh took: 596.7339239120483\n","\n","Running Validation...\n","  Accuracy: 0.48\n","  Validation Loss: 0.69\n","  Validation took: 21.508487701416016\n","\n","Training complete!\n","Total training took 2472.7224526405334 (h:mm:ss)\n","Exiting function train - took 2472.73 seconds\n","Entered function get_test_dataset\n","Number of test sentences: 2,400\n","\n","Exiting function get_test_dataset - took 15.44 seconds\n","Entered function evaluate\n","Exiting function evaluate - took 54.61 seconds\n","gatatron\n","[{'accuracy': 0.5, 'f1': 0.6666666865348816, 'tnr': 0.0, 'tpr': 1.0, 'mcc': 0.0}]\n","Entered function get_loader_cache\n"]},{"ename":"KeyboardInterrupt","evalue":"","output_type":"error","traceback":["\u001b[0;31m---------------------------------------------------------------------------\u001b[0m","\u001b[0;31mKeyboardInterrupt\u001b[0m                         Traceback (most recent call last)","\u001b[0;32m\u003cipython-input-23-17924b0b67f5\u003e\u001b[0m in \u001b[0;36m\u003ccell line: 12\u003e\u001b[0;34m()\u001b[0m\n\u001b[1;32m     29\u001b[0m       \u001b[0;32mcontinue\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     30\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---\u003e 31\u001b[0;31m     \u001b[0mtrain_dataset\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mvalidation_dataset\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mget_loader_cache\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mtokenizer\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mtokenizer_str\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mtrain_val_df\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m     32\u001b[0m     \u001b[0mtrain_dataloader\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mvalidation_dataloader\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mdefine_dataloaders\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mtrain_dataset\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mvalidation_dataset\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     33\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n","\u001b[0;32m\u003cipython-input-5-1dece129e1a1\u003e\u001b[0m in \u001b[0;36mcall\u001b[0;34m(*args, **kwargs)\u001b[0m\n\u001b[1;32m      4\u001b[0m             \u001b[0mprint\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m'Entered function %s'\u001b[0m \u001b[0;34m%\u001b[0m \u001b[0mfunc\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m__name__\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m      5\u001b[0m             \u001b[0mstart\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mtime\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mtime\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m----\u003e 6\u001b[0;31m             \u001b[0mresult\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mfunc\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m*\u001b[0m\u001b[0margs\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m**\u001b[0m\u001b[0mkwargs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m      7\u001b[0m             \u001b[0mend\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mtime\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mtime\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m      8\u001b[0m             \u001b[0mfunc_time\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mend\u001b[0m \u001b[0;34m-\u001b[0m \u001b[0mstart\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n","\u001b[0;32m\u003cipython-input-13-ebc5e211320f\u003e\u001b[0m in \u001b[0;36mget_loader_cache\u001b[0;34m(tokenizer, tokenizer_str, data)\u001b[0m\n\u001b[1;32m     47\u001b[0m       \u001b[0;31m#   (6) Create attention masks for [PAD] tokens.\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     48\u001b[0m       \u001b[0;32mif\u001b[0m \u001b[0msent\u001b[0m \u001b[0;32mnot\u001b[0m \u001b[0;32min\u001b[0m \u001b[0mCACHE\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---\u003e 49\u001b[0;31m         CACHE[encoder_params_str]['sentences'][sent] = tokenizer.encode_plus(\n\u001b[0m\u001b[1;32m     50\u001b[0m                           \u001b[0msent\u001b[0m\u001b[0;34m,\u001b[0m                      \u001b[0;31m# Sentence to encode.\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     51\u001b[0m                   \u001b[0;34m**\u001b[0m\u001b[0mencoder_params\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n","\u001b[0;32m/usr/local/lib/python3.10/dist-packages/transformers/tokenization_utils_base.py\u001b[0m in \u001b[0;36mencode_plus\u001b[0;34m(self, text, text_pair, add_special_tokens, padding, truncation, max_length, stride, is_split_into_words, pad_to_multiple_of, return_tensors, return_token_type_ids, return_attention_mask, return_overflowing_tokens, return_special_tokens_mask, return_offsets_mapping, return_length, verbose, **kwargs)\u001b[0m\n\u001b[1;32m   3006\u001b[0m         )\n\u001b[1;32m   3007\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m-\u003e 3008\u001b[0;31m         return self._encode_plus(\n\u001b[0m\u001b[1;32m   3009\u001b[0m             \u001b[0mtext\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mtext\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   3010\u001b[0m             \u001b[0mtext_pair\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mtext_pair\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n","\u001b[0;32m/usr/local/lib/python3.10/dist-packages/transformers/tokenization_utils.py\u001b[0m in \u001b[0;36m_encode_plus\u001b[0;34m(self, text, text_pair, add_special_tokens, padding_strategy, truncation_strategy, max_length, stride, is_split_into_words, pad_to_multiple_of, return_tensors, return_token_type_ids, return_attention_mask, return_overflowing_tokens, return_special_tokens_mask, return_offsets_mapping, return_length, verbose, **kwargs)\u001b[0m\n\u001b[1;32m    717\u001b[0m             )\n\u001b[1;32m    718\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--\u003e 719\u001b[0;31m         \u001b[0mfirst_ids\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mget_input_ids\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mtext\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    720\u001b[0m         \u001b[0msecond_ids\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mget_input_ids\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mtext_pair\u001b[0m\u001b[0;34m)\u001b[0m \u001b[0;32mif\u001b[0m \u001b[0mtext_pair\u001b[0m \u001b[0;32mis\u001b[0m \u001b[0;32mnot\u001b[0m \u001b[0;32mNone\u001b[0m \u001b[0;32melse\u001b[0m \u001b[0;32mNone\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    721\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n","\u001b[0;31mKeyboardInterrupt\u001b[0m: "]}],"source":["full_data_for_classfier = get_data_for_classifier()\n","\n","results_file = os.path.join(sample_size_dir, 'results_d10df678-9164-4b3f-b6ce-d77c3331b552.json')\n","\n","if not os.path.exists(results_file):\n","  with open(results_file, 'w') as f:\n","    f.write('{}')\n","\n","with open(results_file, 'r') as f:\n","  results = json.load(f)\n","\n","for model_name, model_str in [('BioBert', PRE_EXISTING_MODELS.BIO_BERT), ('sap-bert', PRE_EXISTING_MODELS.SAP_BERT), ('gatatron', PRE_EXISTING_MODELS.GATATRON), ]:\n","\n","  tokenizer_dict = {\n","      'pretrained_string': model_str,\n","      'padding_side':'left',\n","      'truncate_side':'left',\n","      'do_lower_case':True\n","  }\n","  tokenizer = get_tokenizer(**tokenizer_dict)\n","  tokenizer_str = json.dumps(tokenizer_dict, sort_keys=True)\n","\n","  if model_name not in results:\n","    results[model_name] = []\n","  for cross_fold_index, (train_val_df, test_df) in enumerate(get_balanced_cross_fold_train_test_split(\n","      full_data_for_classfier, 5)):\n","    if len(results[model_name]) \u003e cross_fold_index: # we already have results\n","      print(\"Skipping %s cross fold index %d because we have results\" % (model_name, cross_fold_index))\n","      continue\n","\n","    train_dataset, validation_dataset = get_loader_cache(tokenizer, tokenizer_str, train_val_df)\n","    train_dataloader, validation_dataloader = define_dataloaders(train_dataset, validation_dataset)\n","\n","    epochs = 4\n","    total_steps = len(train_dataloader) * epochs\n","    model = define_model(model_str)\n","    optimiser, scheduler = define_optimiser_and_scheduler(model)\n","    train(model, train_dataloader, validation_dataloader, optimiser, scheduler)\n","    test_loader = get_test_dataset(test_df)\n","    results[model_name].append(evaluate(model, test_loader))\n","\n","    print(model_name)\n","    print(results[model_name])\n","    with open(results_file, 'w') as f:\n","      json.dump(results, f)\n","  print(results)"]}],"metadata":{"accelerator":"GPU","colab":{"collapsed_sections":["KRDJE4FLLCcO","qJPJNXTJuVsS","SSGW-nh5-GfE","ZkuwRxmcjQhr","klq3iD_y-K43","FSW39FULGy2P","rK8VKDBPXoUu","2P7R1QDFX6r_","lLkXZWX08z8S"],"gpuType":"V100","name":"","version":""},"kernelspec":{"display_name":"Python 3","name":"python3"},"language_info":{"name":"python"},"widgets":{"application/vnd.jupyter.widget-state+json":{"0499364c0e674d368f395bbb1f84c8a4":{"model_module":"@jupyter-widgets/base","model_module_version":"1.2.0","model_name":"LayoutModel","state":{"_model_module":"@jupyter-widgets/base","_model_module_version":"1.2.0","_model_name":"LayoutModel","_view_count":null,"_view_module":"@jupyter-widgets/base","_view_module_version":"1.2.0","_view_name":"LayoutView","align_content":null,"align_items":null,"align_self":null,"border":null,"bottom":null,"display":null,"flex":null,"flex_flow":null,"grid_area":null,"grid_auto_columns":null,"grid_auto_flow":null,"grid_auto_rows":null,"grid_column":null,"grid_gap":null,"grid_row":null,"grid_template_areas":null,"grid_template_columns":null,"grid_template_rows":null,"height":null,"justify_content":null,"justify_items":null,"left":null,"margin":null,"max_height":null,"max_width":null,"min_height":null,"min_width":null,"object_fit":null,"object_position":null,"order":null,"overflow":null,"overflow_x":null,"overflow_y":null,"padding":null,"right":null,"top":null,"visibility":null,"width":null}},"0c62faa4a9f6414cbdcb0201b174f6fd":{"model_module":"@jupyter-widgets/base","model_module_version":"1.2.0","model_name":"LayoutModel","state":{"_model_module":"@jupyter-widgets/base","_model_module_version":"1.2.0","_model_name":"LayoutModel","_view_count":null,"_view_module":"@jupyter-widgets/base","_view_module_version":"1.2.0","_view_name":"LayoutView","align_content":null,"align_items":null,"align_self":null,"border":null,"bottom":null,"display":null,"flex":null,"flex_flow":null,"grid_area":null,"grid_auto_columns":null,"grid_auto_flow":null,"grid_auto_rows":null,"grid_column":null,"grid_gap":null,"grid_row":null,"grid_template_areas":null,"grid_template_columns":null,"grid_template_rows":null,"height":null,"justify_content":null,"justify_items":null,"left":null,"margin":null,"max_height":null,"max_width":null,"min_height":null,"min_width":null,"object_fit":null,"object_position":null,"order":null,"overflow":null,"overflow_x":null,"overflow_y":null,"padding":null,"right":null,"top":null,"visibility":null,"width":null}},"2db54f33cf3a4d8b8ddbf6042501c077":{"model_module":"@jupyter-widgets/controls","model_module_version":"1.5.0","model_name":"HTMLModel","state":{"_dom_classes":[],"_model_module":"@jupyter-widgets/controls","_model_module_version":"1.5.0","_model_name":"HTMLModel","_view_count":null,"_view_module":"@jupyter-widgets/controls","_view_module_version":"1.5.0","_view_name":"HTMLView","description":"","description_tooltip":null,"layout":"IPY_MODEL_56c227ab6ec2471a9a8bbf12fa30db81","placeholder":"​","style":"IPY_MODEL_7a44485f85c14513a0bc362452714a5c","value":"pytorch_model.bin: 100%"}},"56c227ab6ec2471a9a8bbf12fa30db81":{"model_module":"@jupyter-widgets/base","model_module_version":"1.2.0","model_name":"LayoutModel","state":{"_model_module":"@jupyter-widgets/base","_model_module_version":"1.2.0","_model_name":"LayoutModel","_view_count":null,"_view_module":"@jupyter-widgets/base","_view_module_version":"1.2.0","_view_name":"LayoutView","align_content":null,"align_items":null,"align_self":null,"border":null,"bottom":null,"display":null,"flex":null,"flex_flow":null,"grid_area":null,"grid_auto_columns":null,"grid_auto_flow":null,"grid_auto_rows":null,"grid_column":null,"grid_gap":null,"grid_row":null,"grid_template_areas":null,"grid_template_columns":null,"grid_template_rows":null,"height":null,"justify_content":null,"justify_items":null,"left":null,"margin":null,"max_height":null,"max_width":null,"min_height":null,"min_width":null,"object_fit":null,"object_position":null,"order":null,"overflow":null,"overflow_x":null,"overflow_y":null,"padding":null,"right":null,"top":null,"visibility":null,"width":null}},"62f2bcd722ab4663845d394797b7406a":{"model_module":"@jupyter-widgets/controls","model_module_version":"1.5.0","model_name":"ProgressStyleModel","state":{"_model_module":"@jupyter-widgets/controls","_model_module_version":"1.5.0","_model_name":"ProgressStyleModel","_view_count":null,"_view_module":"@jupyter-widgets/base","_view_module_version":"1.2.0","_view_name":"StyleView","bar_color":null,"description_width":""}},"7897f217604d4a519e97c011854da903":{"model_module":"@jupyter-widgets/controls","model_module_version":"1.5.0","model_name":"HTMLModel","state":{"_dom_classes":[],"_model_module":"@jupyter-widgets/controls","_model_module_version":"1.5.0","_model_name":"HTMLModel","_view_count":null,"_view_module":"@jupyter-widgets/controls","_view_module_version":"1.5.0","_view_name":"HTMLView","description":"","description_tooltip":null,"layout":"IPY_MODEL_88a88ea51447450685a86d85046da0ce","placeholder":"​","style":"IPY_MODEL_a3d995ab6acd4cca897b7c88d7d006ce","value":" 713M/713M [00:01\u0026lt;00:00, 465MB/s]"}},"7a44485f85c14513a0bc362452714a5c":{"model_module":"@jupyter-widgets/controls","model_module_version":"1.5.0","model_name":"DescriptionStyleModel","state":{"_model_module":"@jupyter-widgets/controls","_model_module_version":"1.5.0","_model_name":"DescriptionStyleModel","_view_count":null,"_view_module":"@jupyter-widgets/base","_view_module_version":"1.2.0","_view_name":"StyleView","description_width":""}},"88a88ea51447450685a86d85046da0ce":{"model_module":"@jupyter-widgets/base","model_module_version":"1.2.0","model_name":"LayoutModel","state":{"_model_module":"@jupyter-widgets/base","_model_module_version":"1.2.0","_model_name":"LayoutModel","_view_count":null,"_view_module":"@jupyter-widgets/base","_view_module_version":"1.2.0","_view_name":"LayoutView","align_content":null,"align_items":null,"align_self":null,"border":null,"bottom":null,"display":null,"flex":null,"flex_flow":null,"grid_area":null,"grid_auto_columns":null,"grid_auto_flow":null,"grid_auto_rows":null,"grid_column":null,"grid_gap":null,"grid_row":null,"grid_template_areas":null,"grid_template_columns":null,"grid_template_rows":null,"height":null,"justify_content":null,"justify_items":null,"left":null,"margin":null,"max_height":null,"max_width":null,"min_height":null,"min_width":null,"object_fit":null,"object_position":null,"order":null,"overflow":null,"overflow_x":null,"overflow_y":null,"padding":null,"right":null,"top":null,"visibility":null,"width":null}},"8cc5acf638da4ca4bd903b2ffed8582e":{"model_module":"@jupyter-widgets/controls","model_module_version":"1.5.0","model_name":"FloatProgressModel","state":{"_dom_classes":[],"_model_module":"@jupyter-widgets/controls","_model_module_version":"1.5.0","_model_name":"FloatProgressModel","_view_count":null,"_view_module":"@jupyter-widgets/controls","_view_module_version":"1.5.0","_view_name":"ProgressView","bar_style":"success","description":"","description_tooltip":null,"layout":"IPY_MODEL_0499364c0e674d368f395bbb1f84c8a4","max":712850277,"min":0,"orientation":"horizontal","style":"IPY_MODEL_62f2bcd722ab4663845d394797b7406a","value":712850277}},"a3d995ab6acd4cca897b7c88d7d006ce":{"model_module":"@jupyter-widgets/controls","model_module_version":"1.5.0","model_name":"DescriptionStyleModel","state":{"_model_module":"@jupyter-widgets/controls","_model_module_version":"1.5.0","_model_name":"DescriptionStyleModel","_view_count":null,"_view_module":"@jupyter-widgets/base","_view_module_version":"1.2.0","_view_name":"StyleView","description_width":""}},"ef333f9fc5c8449fab52bdc008b58126":{"model_module":"@jupyter-widgets/controls","model_module_version":"1.5.0","model_name":"HBoxModel","state":{"_dom_classes":[],"_model_module":"@jupyter-widgets/controls","_model_module_version":"1.5.0","_model_name":"HBoxModel","_view_count":null,"_view_module":"@jupyter-widgets/controls","_view_module_version":"1.5.0","_view_name":"HBoxView","box_style":"","children":["IPY_MODEL_2db54f33cf3a4d8b8ddbf6042501c077","IPY_MODEL_8cc5acf638da4ca4bd903b2ffed8582e","IPY_MODEL_7897f217604d4a519e97c011854da903"],"layout":"IPY_MODEL_0c62faa4a9f6414cbdcb0201b174f6fd"}}}}},"nbformat":4,"nbformat_minor":0}