Performances+statistical_test.ipynb

{
 "cells": [
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Performances and Statistical Test\n",
    "Perform t-tests to compute statistical significance between performances values."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "from Utils import load_data, model_performances\n",
    "import pandas as pd\n",
    "import glob\n",
    "import numpy as np\n",
    "from scipy.stats import ttest_rel\n",
    "from sklearn.metrics import classification_report, roc_auc_score\n",
    "import collections\n",
    "import json"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "# ________________________________________Utils _______________________________________________________\n",
    "MODELNAMES_M_MTC = ['multimodal_text_captions_BO_v{}'.format(i) for i in range(10)]\n",
    "MODELNAMES_M_MTT = ['multimodal_text_tags_BO_v{}'.format(i) for i in range(10)]\n",
    "MODELNAMES_MTC = ['multimodal_text_captions_v{}'.format(i) for i in range(10)]\n",
    "MODELNAMES_MTT = ['multimodal_text_tags_v{}'.format(i) for i in range(10)]\n",
    "MODELNAMES_VISUAL_BERT = ['model_VisualBERT_v{}'.format(i) for i in range(10)]\n",
    "MODELNAMES_BERT = ['model_BERT_v{}'.format(i) for i in range(10)]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "MODELFAMILIES =  [MODELNAMES_M_MTC] + [MODELNAMES_MTC] + [MODELNAMES_MTT]+ [MODELNAMES_M_MTT]+ [MODELNAMES_VISUAL_BERT] + [MODELNAMES_BERT]\n",
    "\n",
    "models = []\n",
    "for x in MODELFAMILIES:\n",
    "    models.append(model_performances.model_family_name(x).split('_v')[0])\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "label_column = 'misogynous'\n",
    "identity_terms = load_data.read_clear_identity_terms()\n",
    "identity_tags = load_data.read_clear_identity_tags()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Functions to Compute metrics"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "def flatten(x):\n",
    "    if isinstance(x, collections.Iterable):\n",
    "        return [a for i in x for a in flatten(i)]\n",
    "    else:\n",
    "        return [x]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "def performances_report_10Fold(data, model, threshold):\n",
    "    aucs = [roc_auc_score(data['label_'+str(0)].values, data[model + '_v'+str(0)])]\n",
    "    dict_mis = classification_report(data['label_'+str(0)].values, (data[model + '_v'+str(0)]>threshold).astype(int).values, target_names=['not_mis','mis'], output_dict=True)['mis']\n",
    "    dict_not = classification_report(data['label_'+str(0)].values, (data[model + '_v'+str(0)]>threshold).astype(int).values, target_names=['not_mis','mis'], output_dict=True)['not_mis']\n",
    "\n",
    "    for i in range(1,10):\n",
    "        a = classification_report(data['label_'+str(i)].values, (data[model + '_v'+str(i)]>threshold).astype(int).values, target_names=['not_mis','mis'], output_dict=True)\n",
    "        dict_mis = {key:[dict_mis[key], a['mis'][key]] for key in a['mis']}\n",
    "        dict_not = {key:[dict_not[key], a['not_mis'][key]] for key in a['not_mis']}\n",
    "        \n",
    "        aucs = [aucs, roc_auc_score(data['label_'+str(i)].values, data[model + '_v'+str(i)])]\n",
    "        i=i+1\n",
    "\n",
    "    for key in dict_mis:\n",
    "        dict_mis[key] = flatten(dict_mis[key])\n",
    "        dict_not[key] = flatten(dict_not[key])\n",
    "        aucs = flatten(aucs)\n",
    "    return dict_mis, dict_not, aucs\n",
    "\n",
    "def mean_dict(dict):\n",
    "    mean_dict={}\n",
    "    for key in dict:\n",
    "        mean_dict[key] =np.mean(dict[key])\n",
    "    return mean_dict\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [],
   "source": [
    "def compute_bias_metrics_on_syn_10(modelnames, syn_10_df, test_df, identity_terms, identity_tags, label_column):\n",
    "    final_multimodal_scores = {}\n",
    "    bias_metrics_text = {}\n",
    "    bias_metrics_image = {}\n",
    "    bias_value_multimodal_metrics = {}\n",
    "    overall_auc_metrics = {}\n",
    "    syn_data=syn_10_df.copy()\n",
    "    text={}\n",
    "    image={}\n",
    "    overall_auc_metrics_syn = {}\n",
    "    for i in range(len(modelnames)): #range(len([MODELNAMES[1]])): #range(len(MODELNAMES)):\n",
    "        syn_10_df=syn_data\n",
    "        syn_10_df['file_name'] = syn_10_df['file_name_'+str(i)]\n",
    "        syn_10_df=syn_10_df.merge(load_data.load_syn_identity_data().drop(columns=['misogynous', 'Text Transcription']),\n",
    "                        how='inner', on='file_name')\n",
    "\n",
    "        syn_10_df['misogynous'] = syn_10_df['label_'+str(i)]\n",
    "        \n",
    "        identity_terms_present = model_performances.identity_element_presence(syn_10_df, identity_terms, 'misogynous')\n",
    "        tags_present = model_performances.identity_element_presence_OR(syn_10_df, identity_tags, label_column)\n",
    "\n",
    "        bias_metrics_text[i] = model_performances.compute_bias_metrics_for_model(syn_10_df, identity_terms_present, modelnames[i],\n",
    "                                                            label_column)\n",
    "        bias_metrics_image[i] = model_performances.compute_bias_metrics_for_model(syn_10_df, tags_present, modelnames[i],\n",
    "                                                            label_column)\n",
    "        overall_auc_metrics[i] = model_performances.calculate_overall_auc(test_df, modelnames[i])\n",
    "        \n",
    "        \n",
    "        overall_auc_metrics_syn[i] = model_performances.calculate_overall_auc(syn_10_df, modelnames[i])\n",
    "\n",
    "        final_multimodal_scores[i] = model_performances.get_final_multimodal_metric_nan(bias_metrics_text[i],\n",
    "                                                                bias_metrics_image[i],\n",
    "                                                                overall_auc_metrics[i],\n",
    "                                                                modelnames[i])\n",
    "\n",
    "        bias_value_multimodal_metrics[i] = np.nanmean([\n",
    "            np.nanmean([\n",
    "                bias_metrics_text[i][modelnames[i] + '_subgroup_auc'],\n",
    "                bias_metrics_text[i][modelnames[i] + '_bpsn_auc'],\n",
    "                bias_metrics_text[i][modelnames[i] + '_bnsp_auc']\n",
    "            ]),\n",
    "            np.nanmean([\n",
    "                bias_metrics_image[i][modelnames[i] + '_subgroup_auc'],\n",
    "                bias_metrics_image[i][modelnames[i] + '_bpsn_auc'],\n",
    "                bias_metrics_image[i][modelnames[i] + '_bnsp_auc']\n",
    "            ])\n",
    "        ])\n",
    "        \n",
    "        text[i] = np.nanmean([\n",
    "                bias_metrics_text[i][modelnames[i] + '_subgroup_auc'],\n",
    "                bias_metrics_text[i][modelnames[i] + '_bpsn_auc'],\n",
    "                bias_metrics_text[i][modelnames[i] + '_bnsp_auc']\n",
    "            ])\n",
    "        image[i] = np.nanmean([\n",
    "                bias_metrics_image[i][modelnames[i] + '_subgroup_auc'],\n",
    "                bias_metrics_image[i][modelnames[i] + '_bpsn_auc'],\n",
    "                bias_metrics_image[i][modelnames[i] + '_bnsp_auc']\n",
    "            ])\n",
    "    return final_multimodal_scores, overall_auc_metrics, bias_value_multimodal_metrics, text, image, overall_auc_metrics_syn"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Load Unimodal Models"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [],
   "source": [
    "# _______________________________________ Load Data__________________________________________________\n",
    "train_df = load_data.load_training_data()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [],
   "source": [
    "def get_names_from_ids(iteration, data_type='train'):\n",
    "        if data_type == 'train':\n",
    "                annotation_folder = './Annotations/10Fold_train/'\n",
    "        else:\n",
    "                annotation_folder = './Annotations/train/'\n",
    "        f = open(annotation_folder+'test_'+str(iteration)+'.jsonl')\n",
    "        data = []\n",
    "        names = []\n",
    "        for line in f:\n",
    "                data.append(json.loads(line))\n",
    "        #data = json.load(f)\n",
    "        for dict in data:\n",
    "                names.append(dict['img'].split('.')[0]+'.jpg')\n",
    "        return names"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Load Visual Bert Data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [],
   "source": [
    "path_VisualBERT_Train = \"./VisualBert/Predictions/training/\"\n",
    "path_VisualBERT_Test = \"./VisualBert/Predictions/Test/\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [],
   "source": [
    "train_df=pd.DataFrame()\n",
    "i=1\n",
    "for MODELNAME in MODELNAMES_VISUAL_BERT:\n",
    "    file_name = ''\n",
    "    # read the file corresponding to the iteration (we can't no the time-token a priori)\n",
    "    for file in os.listdir(path_VisualBERT_Train):\n",
    "        if file.endswith(\".csv\") and file.startswith(\"pred_Train_\" + str(i)+'.'):\n",
    "            file_name = file\n",
    "            break\n",
    "    if file_name == '':\n",
    "        raise Exception('Predictions for iteration {i} on synthetic data not found.'.format(i=i))\n",
    "    pred = pd.read_csv(path_VisualBERT_Train + file_name, usecols = ['id','proba'])\n",
    "    pred.columns = ['file_name', MODELNAME]\n",
    "    \n",
    "    train_df[MODELNAME]= pred[MODELNAME]\n",
    "    train_df['file_name']=get_names_from_ids(i)\n",
    "    train_df=train_df.merge(load_data.load_training_data()[['misogynous', 'file_name']],\n",
    "                        how='inner', on='file_name')\n",
    "    train_df.rename(columns={'misogynous':'label_'+str(i-1)}, inplace=True)#real label associated to the split\n",
    "    train_df.rename(columns={'file_name':'file_name_'+str(i-1)}, inplace=True)\n",
    "\n",
    "    i=i+1"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>model_VisualBERT_v0</th>\n",
       "      <th>file_name_0</th>\n",
       "      <th>label_0</th>\n",
       "      <th>model_VisualBERT_v1</th>\n",
       "      <th>file_name_1</th>\n",
       "      <th>label_1</th>\n",
       "      <th>model_VisualBERT_v2</th>\n",
       "      <th>file_name_2</th>\n",
       "      <th>label_2</th>\n",
       "      <th>model_VisualBERT_v3</th>\n",
       "      <th>...</th>\n",
       "      <th>label_6</th>\n",
       "      <th>model_VisualBERT_v7</th>\n",
       "      <th>file_name_7</th>\n",
       "      <th>label_7</th>\n",
       "      <th>model_VisualBERT_v8</th>\n",
       "      <th>file_name_8</th>\n",
       "      <th>label_8</th>\n",
       "      <th>model_VisualBERT_v9</th>\n",
       "      <th>file_name_9</th>\n",
       "      <th>label_9</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>0.206465</td>\n",
       "      <td>5827.jpg</td>\n",
       "      <td>0</td>\n",
       "      <td>0.962831</td>\n",
       "      <td>9764.jpg</td>\n",
       "      <td>1</td>\n",
       "      <td>0.999915</td>\n",
       "      <td>11621.jpg</td>\n",
       "      <td>1</td>\n",
       "      <td>0.995756</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0.469791</td>\n",
       "      <td>9531.jpg</td>\n",
       "      <td>0</td>\n",
       "      <td>0.008137</td>\n",
       "      <td>1772.jpg</td>\n",
       "      <td>0</td>\n",
       "      <td>0.988881</td>\n",
       "      <td>2310.jpg</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>0.900002</td>\n",
       "      <td>2454.jpg</td>\n",
       "      <td>0</td>\n",
       "      <td>0.041153</td>\n",
       "      <td>8222.jpg</td>\n",
       "      <td>0</td>\n",
       "      <td>0.000031</td>\n",
       "      <td>7751.jpg</td>\n",
       "      <td>0</td>\n",
       "      <td>0.329732</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0.047154</td>\n",
       "      <td>11416.jpg</td>\n",
       "      <td>0</td>\n",
       "      <td>0.379920</td>\n",
       "      <td>8581.jpg</td>\n",
       "      <td>1</td>\n",
       "      <td>0.663906</td>\n",
       "      <td>11086.jpg</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>0.999964</td>\n",
       "      <td>6492.jpg</td>\n",
       "      <td>1</td>\n",
       "      <td>0.611585</td>\n",
       "      <td>4152.jpg</td>\n",
       "      <td>1</td>\n",
       "      <td>0.999986</td>\n",
       "      <td>8915.jpg</td>\n",
       "      <td>1</td>\n",
       "      <td>0.998707</td>\n",
       "      <td>...</td>\n",
       "      <td>1</td>\n",
       "      <td>0.362885</td>\n",
       "      <td>7957.jpg</td>\n",
       "      <td>0</td>\n",
       "      <td>0.520290</td>\n",
       "      <td>4366.jpg</td>\n",
       "      <td>1</td>\n",
       "      <td>0.246651</td>\n",
       "      <td>9239.jpg</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>0.999963</td>\n",
       "      <td>2054.jpg</td>\n",
       "      <td>1</td>\n",
       "      <td>0.973622</td>\n",
       "      <td>4656.jpg</td>\n",
       "      <td>1</td>\n",
       "      <td>0.999980</td>\n",
       "      <td>10632.jpg</td>\n",
       "      <td>1</td>\n",
       "      <td>0.999943</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0.065626</td>\n",
       "      <td>2063.jpg</td>\n",
       "      <td>0</td>\n",
       "      <td>0.917175</td>\n",
       "      <td>14105.jpg</td>\n",
       "      <td>1</td>\n",
       "      <td>0.023393</td>\n",
       "      <td>2914.jpg</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>0.000028</td>\n",
       "      <td>5388.jpg</td>\n",
       "      <td>0</td>\n",
       "      <td>0.039521</td>\n",
       "      <td>5653.jpg</td>\n",
       "      <td>1</td>\n",
       "      <td>0.998182</td>\n",
       "      <td>7885.jpg</td>\n",
       "      <td>1</td>\n",
       "      <td>0.999903</td>\n",
       "      <td>...</td>\n",
       "      <td>1</td>\n",
       "      <td>0.408895</td>\n",
       "      <td>5387.jpg</td>\n",
       "      <td>0</td>\n",
       "      <td>0.972901</td>\n",
       "      <td>11269.jpg</td>\n",
       "      <td>1</td>\n",
       "      <td>0.065919</td>\n",
       "      <td>9142.jpg</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>995</th>\n",
       "      <td>0.999964</td>\n",
       "      <td>17.jpg</td>\n",
       "      <td>1</td>\n",
       "      <td>0.063730</td>\n",
       "      <td>1033.jpg</td>\n",
       "      <td>1</td>\n",
       "      <td>0.000070</td>\n",
       "      <td>6206.jpg</td>\n",
       "      <td>0</td>\n",
       "      <td>0.000741</td>\n",
       "      <td>...</td>\n",
       "      <td>1</td>\n",
       "      <td>0.520263</td>\n",
       "      <td>6665.jpg</td>\n",
       "      <td>1</td>\n",
       "      <td>0.021589</td>\n",
       "      <td>7846.jpg</td>\n",
       "      <td>0</td>\n",
       "      <td>0.624310</td>\n",
       "      <td>6179.jpg</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>996</th>\n",
       "      <td>0.000027</td>\n",
       "      <td>3869.jpg</td>\n",
       "      <td>0</td>\n",
       "      <td>0.967645</td>\n",
       "      <td>8874.jpg</td>\n",
       "      <td>1</td>\n",
       "      <td>0.000261</td>\n",
       "      <td>5822.jpg</td>\n",
       "      <td>0</td>\n",
       "      <td>0.000214</td>\n",
       "      <td>...</td>\n",
       "      <td>1</td>\n",
       "      <td>0.092523</td>\n",
       "      <td>9708.jpg</td>\n",
       "      <td>0</td>\n",
       "      <td>0.012388</td>\n",
       "      <td>3288.jpg</td>\n",
       "      <td>0</td>\n",
       "      <td>0.837757</td>\n",
       "      <td>11354.jpg</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>997</th>\n",
       "      <td>0.999964</td>\n",
       "      <td>11351.jpg</td>\n",
       "      <td>1</td>\n",
       "      <td>0.870447</td>\n",
       "      <td>3912.jpg</td>\n",
       "      <td>1</td>\n",
       "      <td>0.999834</td>\n",
       "      <td>9669.jpg</td>\n",
       "      <td>1</td>\n",
       "      <td>0.997703</td>\n",
       "      <td>...</td>\n",
       "      <td>1</td>\n",
       "      <td>0.104544</td>\n",
       "      <td>1733.jpg</td>\n",
       "      <td>0</td>\n",
       "      <td>0.986157</td>\n",
       "      <td>2908.jpg</td>\n",
       "      <td>1</td>\n",
       "      <td>0.988568</td>\n",
       "      <td>3073.jpg</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>998</th>\n",
       "      <td>0.999894</td>\n",
       "      <td>1722.jpg</td>\n",
       "      <td>0</td>\n",
       "      <td>0.786689</td>\n",
       "      <td>13456.jpg</td>\n",
       "      <td>1</td>\n",
       "      <td>0.000036</td>\n",
       "      <td>434.jpg</td>\n",
       "      <td>0</td>\n",
       "      <td>0.000046</td>\n",
       "      <td>...</td>\n",
       "      <td>1</td>\n",
       "      <td>0.342351</td>\n",
       "      <td>5876.jpg</td>\n",
       "      <td>1</td>\n",
       "      <td>0.008553</td>\n",
       "      <td>5025.jpg</td>\n",
       "      <td>0</td>\n",
       "      <td>0.759225</td>\n",
       "      <td>13096.jpg</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>999</th>\n",
       "      <td>0.999963</td>\n",
       "      <td>8422.jpg</td>\n",
       "      <td>1</td>\n",
       "      <td>0.072662</td>\n",
       "      <td>5864.jpg</td>\n",
       "      <td>0</td>\n",
       "      <td>0.999982</td>\n",
       "      <td>7349.jpg</td>\n",
       "      <td>1</td>\n",
       "      <td>0.000336</td>\n",
       "      <td>...</td>\n",
       "      <td>1</td>\n",
       "      <td>0.368123</td>\n",
       "      <td>539.jpg</td>\n",
       "      <td>0</td>\n",
       "      <td>0.851513</td>\n",
       "      <td>11323.jpg</td>\n",
       "      <td>1</td>\n",
       "      <td>0.559425</td>\n",
       "      <td>4974.jpg</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>1000 rows × 30 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "     model_VisualBERT_v0 file_name_0  label_0  model_VisualBERT_v1  \\\n",
       "0               0.206465    5827.jpg        0             0.962831   \n",
       "1               0.900002    2454.jpg        0             0.041153   \n",
       "2               0.999964    6492.jpg        1             0.611585   \n",
       "3               0.999963    2054.jpg        1             0.973622   \n",
       "4               0.000028    5388.jpg        0             0.039521   \n",
       "..                   ...         ...      ...                  ...   \n",
       "995             0.999964      17.jpg        1             0.063730   \n",
       "996             0.000027    3869.jpg        0             0.967645   \n",
       "997             0.999964   11351.jpg        1             0.870447   \n",
       "998             0.999894    1722.jpg        0             0.786689   \n",
       "999             0.999963    8422.jpg        1             0.072662   \n",
       "\n",
       "    file_name_1  label_1  model_VisualBERT_v2 file_name_2  label_2  \\\n",
       "0      9764.jpg        1             0.999915   11621.jpg        1   \n",
       "1      8222.jpg        0             0.000031    7751.jpg        0   \n",
       "2      4152.jpg        1             0.999986    8915.jpg        1   \n",
       "3      4656.jpg        1             0.999980   10632.jpg        1   \n",
       "4      5653.jpg        1             0.998182    7885.jpg        1   \n",
       "..          ...      ...                  ...         ...      ...   \n",
       "995    1033.jpg        1             0.000070    6206.jpg        0   \n",
       "996    8874.jpg        1             0.000261    5822.jpg        0   \n",
       "997    3912.jpg        1             0.999834    9669.jpg        1   \n",
       "998   13456.jpg        1             0.000036     434.jpg        0   \n",
       "999    5864.jpg        0             0.999982    7349.jpg        1   \n",
       "\n",
       "     model_VisualBERT_v3  ... label_6  model_VisualBERT_v7  file_name_7  \\\n",
       "0               0.995756  ...       0             0.469791     9531.jpg   \n",
       "1               0.329732  ...       0             0.047154    11416.jpg   \n",
       "2               0.998707  ...       1             0.362885     7957.jpg   \n",
       "3               0.999943  ...       0             0.065626     2063.jpg   \n",
       "4               0.999903  ...       1             0.408895     5387.jpg   \n",
       "..                   ...  ...     ...                  ...          ...   \n",
       "995             0.000741  ...       1             0.520263     6665.jpg   \n",
       "996             0.000214  ...       1             0.092523     9708.jpg   \n",
       "997             0.997703  ...       1             0.104544     1733.jpg   \n",
       "998             0.000046  ...       1             0.342351     5876.jpg   \n",
       "999             0.000336  ...       1             0.368123      539.jpg   \n",
       "\n",
       "    label_7  model_VisualBERT_v8  file_name_8 label_8  model_VisualBERT_v9  \\\n",
       "0         0             0.008137     1772.jpg       0             0.988881   \n",
       "1         0             0.379920     8581.jpg       1             0.663906   \n",
       "2         0             0.520290     4366.jpg       1             0.246651   \n",
       "3         0             0.917175    14105.jpg       1             0.023393   \n",
       "4         0             0.972901    11269.jpg       1             0.065919   \n",
       "..      ...                  ...          ...     ...                  ...   \n",
       "995       1             0.021589     7846.jpg       0             0.624310   \n",
       "996       0             0.012388     3288.jpg       0             0.837757   \n",
       "997       0             0.986157     2908.jpg       1             0.988568   \n",
       "998       1             0.008553     5025.jpg       0             0.759225   \n",
       "999       0             0.851513    11323.jpg       1             0.559425   \n",
       "\n",
       "     file_name_9 label_9  \n",
       "0       2310.jpg       1  \n",
       "1      11086.jpg       1  \n",
       "2       9239.jpg       0  \n",
       "3       2914.jpg       0  \n",
       "4       9142.jpg       0  \n",
       "..           ...     ...  \n",
       "995     6179.jpg       1  \n",
       "996    11354.jpg       1  \n",
       "997     3073.jpg       1  \n",
       "998    13096.jpg       1  \n",
       "999     4974.jpg       1  \n",
       "\n",
       "[1000 rows x 30 columns]"
      ]
     },
     "execution_count": 15,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "train_df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "Index(['model_VisualBERT_v0', 'file_name_0', 'label_0', 'model_VisualBERT_v1',\n",
       "       'file_name_1', 'label_1', 'model_VisualBERT_v2', 'file_name_2',\n",
       "       'label_2', 'model_VisualBERT_v3', 'file_name_3', 'label_3',\n",
       "       'model_VisualBERT_v4', 'file_name_4', 'label_4', 'model_VisualBERT_v5',\n",
       "       'file_name_5', 'label_5', 'model_VisualBERT_v6', 'file_name_6',\n",
       "       'label_6', 'model_VisualBERT_v7', 'file_name_7', 'label_7',\n",
       "       'model_VisualBERT_v8', 'file_name_8', 'label_8', 'model_VisualBERT_v9',\n",
       "       'file_name_9', 'label_9'],\n",
       "      dtype='object')"
      ]
     },
     "execution_count": 16,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "train_df.columns"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [],
   "source": [
    "test_df = load_data.load_test_data()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [],
   "source": [
    "for i in range(1, 11):\n",
    "    file_name = ''\n",
    "    # read the file corresponding to the iteration (we can't no the time-token a priori)\n",
    "    for file in os.listdir(path_VisualBERT_Test):\n",
    "        if file.endswith(\".csv\") and file.startswith(\"pred_test_model\" + str(i)+'.'):\n",
    "            file_name = file\n",
    "            break\n",
    "    if file_name == '':\n",
    "        print()\n",
    "        raise Exception('Predictions for iteration {i} on test data not found.'.format(i=i))\n",
    "\n",
    "    pred = pd.read_csv(path_VisualBERT_Test + file_name, usecols=['id', 'proba'])\n",
    "    pred.columns = ['file_name', MODELNAMES_VISUAL_BERT[i - 1]]\n",
    "    pred['file_name'] = pred['file_name'].astype(str) + '.jpg'\n",
    "    #pred[MODELNAMES_BERT[i - 1] + '_label'] = pred[MODELNAMES_BERT[i - 1] + '_label'].astype(int)\n",
    "    test_df = pd.merge(test_df, pred, on=\"file_name\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>file_name</th>\n",
       "      <th>Text Transcription</th>\n",
       "      <th>misogynous</th>\n",
       "      <th>shaming</th>\n",
       "      <th>stereotype</th>\n",
       "      <th>objectification</th>\n",
       "      <th>violence</th>\n",
       "      <th>model_VisualBERT_v0</th>\n",
       "      <th>model_VisualBERT_v1</th>\n",
       "      <th>model_VisualBERT_v2</th>\n",
       "      <th>model_VisualBERT_v3</th>\n",
       "      <th>model_VisualBERT_v4</th>\n",
       "      <th>model_VisualBERT_v5</th>\n",
       "      <th>model_VisualBERT_v6</th>\n",
       "      <th>model_VisualBERT_v7</th>\n",
       "      <th>model_VisualBERT_v8</th>\n",
       "      <th>model_VisualBERT_v9</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>15190.jpg</td>\n",
       "      <td>ASIAN GIRLS GMPIREHELENAHARPER.DEVIANTART.COM ...</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0.999905</td>\n",
       "      <td>0.996712</td>\n",
       "      <td>0.999973</td>\n",
       "      <td>0.996469</td>\n",
       "      <td>0.999954</td>\n",
       "      <td>0.992020</td>\n",
       "      <td>0.953584</td>\n",
       "      <td>0.994752</td>\n",
       "      <td>0.998717</td>\n",
       "      <td>0.992619</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>15986.jpg</td>\n",
       "      <td>That Guy Really Likes This Show. VERY DEMOTIVA...</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0.999914</td>\n",
       "      <td>0.996296</td>\n",
       "      <td>0.999963</td>\n",
       "      <td>0.998470</td>\n",
       "      <td>0.999954</td>\n",
       "      <td>0.989565</td>\n",
       "      <td>0.943142</td>\n",
       "      <td>0.994618</td>\n",
       "      <td>0.999232</td>\n",
       "      <td>0.990376</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>15005.jpg</td>\n",
       "      <td>A MAN WITH DREAMS... NEEDS A WOMAN WITH VISION</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0.263468</td>\n",
       "      <td>0.280405</td>\n",
       "      <td>0.687476</td>\n",
       "      <td>0.035794</td>\n",
       "      <td>0.979758</td>\n",
       "      <td>0.709324</td>\n",
       "      <td>0.826826</td>\n",
       "      <td>0.918317</td>\n",
       "      <td>0.466603</td>\n",
       "      <td>0.507890</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>15923.jpg</td>\n",
       "      <td>CLAIMS TO LOVE CLEANING DOESN'T EVEN CLEAN DOT...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0.007080</td>\n",
       "      <td>0.007222</td>\n",
       "      <td>0.003045</td>\n",
       "      <td>0.001416</td>\n",
       "      <td>0.007882</td>\n",
       "      <td>0.060396</td>\n",
       "      <td>0.206591</td>\n",
       "      <td>0.008641</td>\n",
       "      <td>0.042311</td>\n",
       "      <td>0.100411</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>15971.jpg</td>\n",
       "      <td>Women Women Women 0 Respect</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0.905814</td>\n",
       "      <td>0.996184</td>\n",
       "      <td>0.999928</td>\n",
       "      <td>0.735428</td>\n",
       "      <td>0.999781</td>\n",
       "      <td>0.863464</td>\n",
       "      <td>0.888396</td>\n",
       "      <td>0.990268</td>\n",
       "      <td>0.970614</td>\n",
       "      <td>0.853977</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>995</th>\n",
       "      <td>15964.jpg</td>\n",
       "      <td>YEAH, I HAVE A GIRLFRIEND. BUT SHE'S BEING A B...</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0.567974</td>\n",
       "      <td>0.881868</td>\n",
       "      <td>0.079093</td>\n",
       "      <td>0.115195</td>\n",
       "      <td>0.981669</td>\n",
       "      <td>0.855231</td>\n",
       "      <td>0.891095</td>\n",
       "      <td>0.954878</td>\n",
       "      <td>0.989614</td>\n",
       "      <td>0.813421</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>996</th>\n",
       "      <td>15990.jpg</td>\n",
       "      <td>PRAY FOR MY GIRLFRIEND SHE'S USED THE DOG SNAP...</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>0.992131</td>\n",
       "      <td>0.994005</td>\n",
       "      <td>0.999963</td>\n",
       "      <td>0.996151</td>\n",
       "      <td>0.999960</td>\n",
       "      <td>0.943783</td>\n",
       "      <td>0.892845</td>\n",
       "      <td>0.791448</td>\n",
       "      <td>0.998287</td>\n",
       "      <td>0.921779</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>997</th>\n",
       "      <td>15461.jpg</td>\n",
       "      <td>HOW TO HAVE FUN WITH A MINOR Fung Step 3-&gt; Ste...</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>0.999908</td>\n",
       "      <td>0.996512</td>\n",
       "      <td>0.999978</td>\n",
       "      <td>0.996837</td>\n",
       "      <td>0.999949</td>\n",
       "      <td>0.991129</td>\n",
       "      <td>0.940909</td>\n",
       "      <td>0.993650</td>\n",
       "      <td>0.999069</td>\n",
       "      <td>0.993262</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>998</th>\n",
       "      <td>16018.jpg</td>\n",
       "      <td>this could be us but you ain't here Calvin Kle...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0.853722</td>\n",
       "      <td>0.071663</td>\n",
       "      <td>0.121594</td>\n",
       "      <td>0.026180</td>\n",
       "      <td>0.941637</td>\n",
       "      <td>0.290404</td>\n",
       "      <td>0.493553</td>\n",
       "      <td>0.414918</td>\n",
       "      <td>0.151530</td>\n",
       "      <td>0.600766</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>999</th>\n",
       "      <td>15228.jpg</td>\n",
       "      <td>LADIES THIS IS HOW TO TAKE A SEXY SELFIE</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0.998254</td>\n",
       "      <td>0.996356</td>\n",
       "      <td>0.999972</td>\n",
       "      <td>0.994612</td>\n",
       "      <td>0.999960</td>\n",
       "      <td>0.990102</td>\n",
       "      <td>0.943646</td>\n",
       "      <td>0.990956</td>\n",
       "      <td>0.998443</td>\n",
       "      <td>0.988510</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>1000 rows × 17 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "     file_name                                 Text Transcription  misogynous  \\\n",
       "0    15190.jpg  ASIAN GIRLS GMPIREHELENAHARPER.DEVIANTART.COM ...           1   \n",
       "1    15986.jpg  That Guy Really Likes This Show. VERY DEMOTIVA...           1   \n",
       "2    15005.jpg     A MAN WITH DREAMS... NEEDS A WOMAN WITH VISION           0   \n",
       "3    15923.jpg  CLAIMS TO LOVE CLEANING DOESN'T EVEN CLEAN DOT...           0   \n",
       "4    15971.jpg                        Women Women Women 0 Respect           1   \n",
       "..         ...                                                ...         ...   \n",
       "995  15964.jpg  YEAH, I HAVE A GIRLFRIEND. BUT SHE'S BEING A B...           1   \n",
       "996  15990.jpg  PRAY FOR MY GIRLFRIEND SHE'S USED THE DOG SNAP...           1   \n",
       "997  15461.jpg  HOW TO HAVE FUN WITH A MINOR Fung Step 3-> Ste...           1   \n",
       "998  16018.jpg  this could be us but you ain't here Calvin Kle...           0   \n",
       "999  15228.jpg           LADIES THIS IS HOW TO TAKE A SEXY SELFIE           0   \n",
       "\n",
       "     shaming  stereotype  objectification  violence  model_VisualBERT_v0  \\\n",
       "0          0           1                1         0             0.999905   \n",
       "1          0           1                1         0             0.999914   \n",
       "2          0           0                0         0             0.263468   \n",
       "3          0           0                0         0             0.007080   \n",
       "4          0           1                0         1             0.905814   \n",
       "..       ...         ...              ...       ...                  ...   \n",
       "995        0           1                1         0             0.567974   \n",
       "996        1           1                1         1             0.992131   \n",
       "997        0           0                1         1             0.999908   \n",
       "998        0           0                0         0             0.853722   \n",
       "999        0           0                0         0             0.998254   \n",
       "\n",
       "     model_VisualBERT_v1  model_VisualBERT_v2  model_VisualBERT_v3  \\\n",
       "0               0.996712             0.999973             0.996469   \n",
       "1               0.996296             0.999963             0.998470   \n",
       "2               0.280405             0.687476             0.035794   \n",
       "3               0.007222             0.003045             0.001416   \n",
       "4               0.996184             0.999928             0.735428   \n",
       "..                   ...                  ...                  ...   \n",
       "995             0.881868             0.079093             0.115195   \n",
       "996             0.994005             0.999963             0.996151   \n",
       "997             0.996512             0.999978             0.996837   \n",
       "998             0.071663             0.121594             0.026180   \n",
       "999             0.996356             0.999972             0.994612   \n",
       "\n",
       "     model_VisualBERT_v4  model_VisualBERT_v5  model_VisualBERT_v6  \\\n",
       "0               0.999954             0.992020             0.953584   \n",
       "1               0.999954             0.989565             0.943142   \n",
       "2               0.979758             0.709324             0.826826   \n",
       "3               0.007882             0.060396             0.206591   \n",
       "4               0.999781             0.863464             0.888396   \n",
       "..                   ...                  ...                  ...   \n",
       "995             0.981669             0.855231             0.891095   \n",
       "996             0.999960             0.943783             0.892845   \n",
       "997             0.999949             0.991129             0.940909   \n",
       "998             0.941637             0.290404             0.493553   \n",
       "999             0.999960             0.990102             0.943646   \n",
       "\n",
       "     model_VisualBERT_v7  model_VisualBERT_v8  model_VisualBERT_v9  \n",
       "0               0.994752             0.998717             0.992619  \n",
       "1               0.994618             0.999232             0.990376  \n",
       "2               0.918317             0.466603             0.507890  \n",
       "3               0.008641             0.042311             0.100411  \n",
       "4               0.990268             0.970614             0.853977  \n",
       "..                   ...                  ...                  ...  \n",
       "995             0.954878             0.989614             0.813421  \n",
       "996             0.791448             0.998287             0.921779  \n",
       "997             0.993650             0.999069             0.993262  \n",
       "998             0.414918             0.151530             0.600766  \n",
       "999             0.990956             0.998443             0.988510  \n",
       "\n",
       "[1000 rows x 17 columns]"
      ]
     },
     "execution_count": 18,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "test_df"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### BERT"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [],
   "source": [
    "path_BERT_Train = \"./Bert/predictions/Train/\"\n",
    "path_BERT_Test = \"./Bert/predictions/Test/\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "pred_Train_1_1668432545.csv\n",
      "pred_Train_2_1668432573.csv\n",
      "pred_Train_3_1668432602.csv\n",
      "pred_Train_4_1668432632.csv\n",
      "pred_Train_5_1668432662.csv\n",
      "pred_Train_6_1668432691.csv\n",
      "pred_Train_7_1668432722.csv\n",
      "pred_Train_8_1668432750.csv\n",
      "pred_Train_9_1668432780.csv\n",
      "pred_Train_10_1668432809.csv\n"
     ]
    }
   ],
   "source": [
    "i=1\n",
    "for MODELNAME in MODELNAMES_BERT:\n",
    "    file_name = ''\n",
    "    # read the file corresponding to the iteration (we can't no the time-token a priori)\n",
    "    for file in os.listdir(path_BERT_Train):\n",
    "        if file.endswith(\".csv\") and file.startswith(\"pred_Train_\" + str(i)+'_'):\n",
    "            file_name = file\n",
    "            break\n",
    "    if file_name == '':\n",
    "        raise Exception('Predictions for iteration {i} on synthetic data not found.'.format(i=i))\n",
    "    print(file_name)\n",
    "    pred = pd.read_csv(path_BERT_Train + file_name, usecols = ['id','proba'])\n",
    "    pred.columns = ['file_name_'+str(i-1), MODELNAME]\n",
    "    pred['file_name_'+str(i-1)] =  pred['file_name_'+str(i-1)].astype(str)+'.jpg'\n",
    "    train_df = pd.merge(train_df, pred, on='file_name_'+str(i-1))\n",
    "    i=i+1"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>model_VisualBERT_v0</th>\n",
       "      <th>file_name_0</th>\n",
       "      <th>label_0</th>\n",
       "      <th>model_VisualBERT_v1</th>\n",
       "      <th>file_name_1</th>\n",
       "      <th>label_1</th>\n",
       "      <th>model_VisualBERT_v2</th>\n",
       "      <th>file_name_2</th>\n",
       "      <th>label_2</th>\n",
       "      <th>model_VisualBERT_v3</th>\n",
       "      <th>...</th>\n",
       "      <th>model_BERT_v0</th>\n",
       "      <th>model_BERT_v1</th>\n",
       "      <th>model_BERT_v2</th>\n",
       "      <th>model_BERT_v3</th>\n",
       "      <th>model_BERT_v4</th>\n",
       "      <th>model_BERT_v5</th>\n",
       "      <th>model_BERT_v6</th>\n",
       "      <th>model_BERT_v7</th>\n",
       "      <th>model_BERT_v8</th>\n",
       "      <th>model_BERT_v9</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>0.206465</td>\n",
       "      <td>5827.jpg</td>\n",
       "      <td>0</td>\n",
       "      <td>0.962831</td>\n",
       "      <td>9764.jpg</td>\n",
       "      <td>1</td>\n",
       "      <td>0.999915</td>\n",
       "      <td>11621.jpg</td>\n",
       "      <td>1</td>\n",
       "      <td>0.995756</td>\n",
       "      <td>...</td>\n",
       "      <td>0.786190</td>\n",
       "      <td>0.992523</td>\n",
       "      <td>0.940009</td>\n",
       "      <td>0.987400</td>\n",
       "      <td>0.041032</td>\n",
       "      <td>0.005473</td>\n",
       "      <td>0.010425</td>\n",
       "      <td>0.519786</td>\n",
       "      <td>0.422194</td>\n",
       "      <td>0.997236</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>0.900002</td>\n",
       "      <td>2454.jpg</td>\n",
       "      <td>0</td>\n",
       "      <td>0.041153</td>\n",
       "      <td>8222.jpg</td>\n",
       "      <td>0</td>\n",
       "      <td>0.000031</td>\n",
       "      <td>7751.jpg</td>\n",
       "      <td>0</td>\n",
       "      <td>0.329732</td>\n",
       "      <td>...</td>\n",
       "      <td>0.491323</td>\n",
       "      <td>0.002396</td>\n",
       "      <td>0.000404</td>\n",
       "      <td>0.655005</td>\n",
       "      <td>0.037980</td>\n",
       "      <td>0.004949</td>\n",
       "      <td>0.620156</td>\n",
       "      <td>0.752777</td>\n",
       "      <td>0.111845</td>\n",
       "      <td>0.938273</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>0.999964</td>\n",
       "      <td>6492.jpg</td>\n",
       "      <td>1</td>\n",
       "      <td>0.611585</td>\n",
       "      <td>4152.jpg</td>\n",
       "      <td>1</td>\n",
       "      <td>0.999986</td>\n",
       "      <td>8915.jpg</td>\n",
       "      <td>1</td>\n",
       "      <td>0.998707</td>\n",
       "      <td>...</td>\n",
       "      <td>0.749784</td>\n",
       "      <td>0.150281</td>\n",
       "      <td>0.986370</td>\n",
       "      <td>0.828453</td>\n",
       "      <td>0.105714</td>\n",
       "      <td>0.996371</td>\n",
       "      <td>0.952132</td>\n",
       "      <td>0.759242</td>\n",
       "      <td>0.868543</td>\n",
       "      <td>0.025829</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>0.999963</td>\n",
       "      <td>2054.jpg</td>\n",
       "      <td>1</td>\n",
       "      <td>0.973622</td>\n",
       "      <td>4656.jpg</td>\n",
       "      <td>1</td>\n",
       "      <td>0.999980</td>\n",
       "      <td>10632.jpg</td>\n",
       "      <td>1</td>\n",
       "      <td>0.999943</td>\n",
       "      <td>...</td>\n",
       "      <td>0.136777</td>\n",
       "      <td>0.711936</td>\n",
       "      <td>0.996937</td>\n",
       "      <td>0.957651</td>\n",
       "      <td>0.082578</td>\n",
       "      <td>0.993810</td>\n",
       "      <td>0.019216</td>\n",
       "      <td>0.025852</td>\n",
       "      <td>0.923950</td>\n",
       "      <td>0.003006</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>0.000028</td>\n",
       "      <td>5388.jpg</td>\n",
       "      <td>0</td>\n",
       "      <td>0.039521</td>\n",
       "      <td>5653.jpg</td>\n",
       "      <td>1</td>\n",
       "      <td>0.998182</td>\n",
       "      <td>7885.jpg</td>\n",
       "      <td>1</td>\n",
       "      <td>0.999903</td>\n",
       "      <td>...</td>\n",
       "      <td>0.002827</td>\n",
       "      <td>0.014650</td>\n",
       "      <td>0.533931</td>\n",
       "      <td>0.998570</td>\n",
       "      <td>0.926967</td>\n",
       "      <td>0.479097</td>\n",
       "      <td>0.920404</td>\n",
       "      <td>0.794859</td>\n",
       "      <td>0.951807</td>\n",
       "      <td>0.001913</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>995</th>\n",
       "      <td>0.999964</td>\n",
       "      <td>17.jpg</td>\n",
       "      <td>1</td>\n",
       "      <td>0.063730</td>\n",
       "      <td>1033.jpg</td>\n",
       "      <td>1</td>\n",
       "      <td>0.000070</td>\n",
       "      <td>6206.jpg</td>\n",
       "      <td>0</td>\n",
       "      <td>0.000741</td>\n",
       "      <td>...</td>\n",
       "      <td>0.918494</td>\n",
       "      <td>0.351506</td>\n",
       "      <td>0.000668</td>\n",
       "      <td>0.005629</td>\n",
       "      <td>0.433275</td>\n",
       "      <td>0.988119</td>\n",
       "      <td>0.615927</td>\n",
       "      <td>0.890731</td>\n",
       "      <td>0.337252</td>\n",
       "      <td>0.974663</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>996</th>\n",
       "      <td>0.000027</td>\n",
       "      <td>3869.jpg</td>\n",
       "      <td>0</td>\n",
       "      <td>0.967645</td>\n",
       "      <td>8874.jpg</td>\n",
       "      <td>1</td>\n",
       "      <td>0.000261</td>\n",
       "      <td>5822.jpg</td>\n",
       "      <td>0</td>\n",
       "      <td>0.000214</td>\n",
       "      <td>...</td>\n",
       "      <td>0.112705</td>\n",
       "      <td>0.823033</td>\n",
       "      <td>0.004043</td>\n",
       "      <td>0.000374</td>\n",
       "      <td>0.091171</td>\n",
       "      <td>0.751037</td>\n",
       "      <td>0.385388</td>\n",
       "      <td>0.683702</td>\n",
       "      <td>0.190710</td>\n",
       "      <td>0.998095</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>997</th>\n",
       "      <td>0.999964</td>\n",
       "      <td>11351.jpg</td>\n",
       "      <td>1</td>\n",
       "      <td>0.870447</td>\n",
       "      <td>3912.jpg</td>\n",
       "      <td>1</td>\n",
       "      <td>0.999834</td>\n",
       "      <td>9669.jpg</td>\n",
       "      <td>1</td>\n",
       "      <td>0.997703</td>\n",
       "      <td>...</td>\n",
       "      <td>0.668825</td>\n",
       "      <td>0.101254</td>\n",
       "      <td>0.975470</td>\n",
       "      <td>0.990921</td>\n",
       "      <td>0.139503</td>\n",
       "      <td>0.998141</td>\n",
       "      <td>0.957960</td>\n",
       "      <td>0.385615</td>\n",
       "      <td>0.960522</td>\n",
       "      <td>0.995111</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>998</th>\n",
       "      <td>0.999894</td>\n",
       "      <td>1722.jpg</td>\n",
       "      <td>0</td>\n",
       "      <td>0.786689</td>\n",
       "      <td>13456.jpg</td>\n",
       "      <td>1</td>\n",
       "      <td>0.000036</td>\n",
       "      <td>434.jpg</td>\n",
       "      <td>0</td>\n",
       "      <td>0.000046</td>\n",
       "      <td>...</td>\n",
       "      <td>0.725438</td>\n",
       "      <td>0.997154</td>\n",
       "      <td>0.000181</td>\n",
       "      <td>0.000499</td>\n",
       "      <td>0.686592</td>\n",
       "      <td>0.002186</td>\n",
       "      <td>0.630951</td>\n",
       "      <td>0.878994</td>\n",
       "      <td>0.007703</td>\n",
       "      <td>0.990759</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>999</th>\n",
       "      <td>0.999963</td>\n",
       "      <td>8422.jpg</td>\n",
       "      <td>1</td>\n",
       "      <td>0.072662</td>\n",
       "      <td>5864.jpg</td>\n",
       "      <td>0</td>\n",
       "      <td>0.999982</td>\n",
       "      <td>7349.jpg</td>\n",
       "      <td>1</td>\n",
       "      <td>0.000336</td>\n",
       "      <td>...</td>\n",
       "      <td>0.809119</td>\n",
       "      <td>0.006663</td>\n",
       "      <td>0.996088</td>\n",
       "      <td>0.456234</td>\n",
       "      <td>0.905142</td>\n",
       "      <td>0.868954</td>\n",
       "      <td>0.035162</td>\n",
       "      <td>0.879268</td>\n",
       "      <td>0.890072</td>\n",
       "      <td>0.026812</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>1000 rows × 40 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "     model_VisualBERT_v0 file_name_0  label_0  model_VisualBERT_v1  \\\n",
       "0               0.206465    5827.jpg        0             0.962831   \n",
       "1               0.900002    2454.jpg        0             0.041153   \n",
       "2               0.999964    6492.jpg        1             0.611585   \n",
       "3               0.999963    2054.jpg        1             0.973622   \n",
       "4               0.000028    5388.jpg        0             0.039521   \n",
       "..                   ...         ...      ...                  ...   \n",
       "995             0.999964      17.jpg        1             0.063730   \n",
       "996             0.000027    3869.jpg        0             0.967645   \n",
       "997             0.999964   11351.jpg        1             0.870447   \n",
       "998             0.999894    1722.jpg        0             0.786689   \n",
       "999             0.999963    8422.jpg        1             0.072662   \n",
       "\n",
       "    file_name_1  label_1  model_VisualBERT_v2 file_name_2  label_2  \\\n",
       "0      9764.jpg        1             0.999915   11621.jpg        1   \n",
       "1      8222.jpg        0             0.000031    7751.jpg        0   \n",
       "2      4152.jpg        1             0.999986    8915.jpg        1   \n",
       "3      4656.jpg        1             0.999980   10632.jpg        1   \n",
       "4      5653.jpg        1             0.998182    7885.jpg        1   \n",
       "..          ...      ...                  ...         ...      ...   \n",
       "995    1033.jpg        1             0.000070    6206.jpg        0   \n",
       "996    8874.jpg        1             0.000261    5822.jpg        0   \n",
       "997    3912.jpg        1             0.999834    9669.jpg        1   \n",
       "998   13456.jpg        1             0.000036     434.jpg        0   \n",
       "999    5864.jpg        0             0.999982    7349.jpg        1   \n",
       "\n",
       "     model_VisualBERT_v3  ... model_BERT_v0  model_BERT_v1  model_BERT_v2  \\\n",
       "0               0.995756  ...      0.786190       0.992523       0.940009   \n",
       "1               0.329732  ...      0.491323       0.002396       0.000404   \n",
       "2               0.998707  ...      0.749784       0.150281       0.986370   \n",
       "3               0.999943  ...      0.136777       0.711936       0.996937   \n",
       "4               0.999903  ...      0.002827       0.014650       0.533931   \n",
       "..                   ...  ...           ...            ...            ...   \n",
       "995             0.000741  ...      0.918494       0.351506       0.000668   \n",
       "996             0.000214  ...      0.112705       0.823033       0.004043   \n",
       "997             0.997703  ...      0.668825       0.101254       0.975470   \n",
       "998             0.000046  ...      0.725438       0.997154       0.000181   \n",
       "999             0.000336  ...      0.809119       0.006663       0.996088   \n",
       "\n",
       "    model_BERT_v3  model_BERT_v4  model_BERT_v5 model_BERT_v6  model_BERT_v7  \\\n",
       "0        0.987400       0.041032       0.005473      0.010425       0.519786   \n",
       "1        0.655005       0.037980       0.004949      0.620156       0.752777   \n",
       "2        0.828453       0.105714       0.996371      0.952132       0.759242   \n",
       "3        0.957651       0.082578       0.993810      0.019216       0.025852   \n",
       "4        0.998570       0.926967       0.479097      0.920404       0.794859   \n",
       "..            ...            ...            ...           ...            ...   \n",
       "995      0.005629       0.433275       0.988119      0.615927       0.890731   \n",
       "996      0.000374       0.091171       0.751037      0.385388       0.683702   \n",
       "997      0.990921       0.139503       0.998141      0.957960       0.385615   \n",
       "998      0.000499       0.686592       0.002186      0.630951       0.878994   \n",
       "999      0.456234       0.905142       0.868954      0.035162       0.879268   \n",
       "\n",
       "     model_BERT_v8 model_BERT_v9  \n",
       "0         0.422194      0.997236  \n",
       "1         0.111845      0.938273  \n",
       "2         0.868543      0.025829  \n",
       "3         0.923950      0.003006  \n",
       "4         0.951807      0.001913  \n",
       "..             ...           ...  \n",
       "995       0.337252      0.974663  \n",
       "996       0.190710      0.998095  \n",
       "997       0.960522      0.995111  \n",
       "998       0.007703      0.990759  \n",
       "999       0.890072      0.026812  \n",
       "\n",
       "[1000 rows x 40 columns]"
      ]
     },
     "execution_count": 22,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "train_df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [],
   "source": [
    "for i in range(1, 11):\n",
    "    file_name = ''\n",
    "    # read the file corresponding to the iteration (we can't no the time-token a priori)\n",
    "    for file in os.listdir(path_BERT_Test):\n",
    "        if file.endswith(\".csv\") and file.startswith(\"pred_test_\" + str(i)+\"_\"):\n",
    "            file_name = file\n",
    "            break\n",
    "    if file_name == '':\n",
    "        print()\n",
    "        raise Exception('Predictions for iteration {i} on test data not found.'.format(i=i))\n",
    "\n",
    "    pred = pd.read_csv(path_BERT_Test + file_name, usecols=['id', 'proba'])\n",
    "    pred.columns = ['file_name', MODELNAMES_BERT[i - 1]]\n",
    "    pred['file_name'] = pred['file_name'].astype(str) + '.jpg'\n",
    "    #pred[MODELNAMES_BERT[i - 1] + '_label'] = pred[MODELNAMES_BERT[i - 1] + '_label'].astype(int)\n",
    "    test_df = pd.merge(test_df, pred, on=\"file_name\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "Index(['file_name', 'Text Transcription', 'misogynous', 'shaming',\n",
       "       'stereotype', 'objectification', 'violence', 'model_VisualBERT_v0',\n",
       "       'model_VisualBERT_v1', 'model_VisualBERT_v2', 'model_VisualBERT_v3',\n",
       "       'model_VisualBERT_v4', 'model_VisualBERT_v5', 'model_VisualBERT_v6',\n",
       "       'model_VisualBERT_v7', 'model_VisualBERT_v8', 'model_VisualBERT_v9',\n",
       "       'model_BERT_v0', 'model_BERT_v1', 'model_BERT_v2', 'model_BERT_v3',\n",
       "       'model_BERT_v4', 'model_BERT_v5', 'model_BERT_v6', 'model_BERT_v7',\n",
       "       'model_BERT_v8', 'model_BERT_v9'],\n",
       "      dtype='object')"
      ]
     },
     "execution_count": 22,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "test_df.columns"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Unimodal Text"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {},
   "outputs": [],
   "source": [
    "unimodal_train_path = './Unimodal/predictions/Text_pred_10Fold.csv'\n",
    "unimodal_test_path = './Unimodal/predictionsTest/Text_pred_10Fold.csv'\n",
    "MODELNAMES_TEXT = ['unimodal_text_v{}'.format(i) for i in range(10)]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {},
   "outputs": [],
   "source": [
    "pred = pd.read_csv(unimodal_train_path, sep='\\t', usecols=['id', 'pred'])\n",
    "i = 0\n",
    "for fold in np.array_split(pred, 10):\n",
    "    fold.columns = ['file_name_'+str(i), MODELNAMES_TEXT[i]]\n",
    "    train_df = pd.merge(train_df, fold, on='file_name_'+str(i))\n",
    "    i=i+1"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>model_VisualBERT_v0</th>\n",
       "      <th>file_name_0</th>\n",
       "      <th>label_0</th>\n",
       "      <th>model_VisualBERT_v1</th>\n",
       "      <th>file_name_1</th>\n",
       "      <th>label_1</th>\n",
       "      <th>model_VisualBERT_v2</th>\n",
       "      <th>file_name_2</th>\n",
       "      <th>label_2</th>\n",
       "      <th>model_VisualBERT_v3</th>\n",
       "      <th>...</th>\n",
       "      <th>unimodal_text_v0</th>\n",
       "      <th>unimodal_text_v1</th>\n",
       "      <th>unimodal_text_v2</th>\n",
       "      <th>unimodal_text_v3</th>\n",
       "      <th>unimodal_text_v4</th>\n",
       "      <th>unimodal_text_v5</th>\n",
       "      <th>unimodal_text_v6</th>\n",
       "      <th>unimodal_text_v7</th>\n",
       "      <th>unimodal_text_v8</th>\n",
       "      <th>unimodal_text_v9</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>0.206465</td>\n",
       "      <td>5827.jpg</td>\n",
       "      <td>0</td>\n",
       "      <td>0.962831</td>\n",
       "      <td>9764.jpg</td>\n",
       "      <td>1</td>\n",
       "      <td>0.999915</td>\n",
       "      <td>11621.jpg</td>\n",
       "      <td>1</td>\n",
       "      <td>0.995756</td>\n",
       "      <td>...</td>\n",
       "      <td>0.716009</td>\n",
       "      <td>0.659856</td>\n",
       "      <td>0.801545</td>\n",
       "      <td>0.343322</td>\n",
       "      <td>0.541126</td>\n",
       "      <td>0.523656</td>\n",
       "      <td>0.232028</td>\n",
       "      <td>0.097245</td>\n",
       "      <td>0.164340</td>\n",
       "      <td>0.888618</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>0.900002</td>\n",
       "      <td>2454.jpg</td>\n",
       "      <td>0</td>\n",
       "      <td>0.041153</td>\n",
       "      <td>8222.jpg</td>\n",
       "      <td>0</td>\n",
       "      <td>0.000031</td>\n",
       "      <td>7751.jpg</td>\n",
       "      <td>0</td>\n",
       "      <td>0.329732</td>\n",
       "      <td>...</td>\n",
       "      <td>0.165136</td>\n",
       "      <td>0.502955</td>\n",
       "      <td>0.055388</td>\n",
       "      <td>0.645190</td>\n",
       "      <td>0.120404</td>\n",
       "      <td>0.140960</td>\n",
       "      <td>0.579082</td>\n",
       "      <td>0.150768</td>\n",
       "      <td>0.684266</td>\n",
       "      <td>0.859328</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>0.999964</td>\n",
       "      <td>6492.jpg</td>\n",
       "      <td>1</td>\n",
       "      <td>0.611585</td>\n",
       "      <td>4152.jpg</td>\n",
       "      <td>1</td>\n",
       "      <td>0.999986</td>\n",
       "      <td>8915.jpg</td>\n",
       "      <td>1</td>\n",
       "      <td>0.998707</td>\n",
       "      <td>...</td>\n",
       "      <td>0.860050</td>\n",
       "      <td>0.807957</td>\n",
       "      <td>0.740347</td>\n",
       "      <td>0.496732</td>\n",
       "      <td>0.068227</td>\n",
       "      <td>0.763816</td>\n",
       "      <td>0.117275</td>\n",
       "      <td>0.105782</td>\n",
       "      <td>0.911819</td>\n",
       "      <td>0.061357</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>0.999963</td>\n",
       "      <td>2054.jpg</td>\n",
       "      <td>1</td>\n",
       "      <td>0.973622</td>\n",
       "      <td>4656.jpg</td>\n",
       "      <td>1</td>\n",
       "      <td>0.999980</td>\n",
       "      <td>10632.jpg</td>\n",
       "      <td>1</td>\n",
       "      <td>0.999943</td>\n",
       "      <td>...</td>\n",
       "      <td>0.663105</td>\n",
       "      <td>0.637365</td>\n",
       "      <td>0.941983</td>\n",
       "      <td>0.540022</td>\n",
       "      <td>0.093168</td>\n",
       "      <td>0.252664</td>\n",
       "      <td>0.574554</td>\n",
       "      <td>0.165804</td>\n",
       "      <td>0.923068</td>\n",
       "      <td>0.138661</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>0.000028</td>\n",
       "      <td>5388.jpg</td>\n",
       "      <td>0</td>\n",
       "      <td>0.039521</td>\n",
       "      <td>5653.jpg</td>\n",
       "      <td>1</td>\n",
       "      <td>0.998182</td>\n",
       "      <td>7885.jpg</td>\n",
       "      <td>1</td>\n",
       "      <td>0.999903</td>\n",
       "      <td>...</td>\n",
       "      <td>0.030513</td>\n",
       "      <td>0.650166</td>\n",
       "      <td>0.960102</td>\n",
       "      <td>0.660472</td>\n",
       "      <td>0.791665</td>\n",
       "      <td>0.581676</td>\n",
       "      <td>0.775880</td>\n",
       "      <td>0.178797</td>\n",
       "      <td>0.810652</td>\n",
       "      <td>0.034840</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>995</th>\n",
       "      <td>0.999964</td>\n",
       "      <td>17.jpg</td>\n",
       "      <td>1</td>\n",
       "      <td>0.063730</td>\n",
       "      <td>1033.jpg</td>\n",
       "      <td>1</td>\n",
       "      <td>0.000070</td>\n",
       "      <td>6206.jpg</td>\n",
       "      <td>0</td>\n",
       "      <td>0.000741</td>\n",
       "      <td>...</td>\n",
       "      <td>0.938401</td>\n",
       "      <td>0.494559</td>\n",
       "      <td>0.451599</td>\n",
       "      <td>0.828844</td>\n",
       "      <td>0.330451</td>\n",
       "      <td>0.896399</td>\n",
       "      <td>0.232947</td>\n",
       "      <td>0.579574</td>\n",
       "      <td>0.343696</td>\n",
       "      <td>0.679110</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>996</th>\n",
       "      <td>0.000027</td>\n",
       "      <td>3869.jpg</td>\n",
       "      <td>0</td>\n",
       "      <td>0.967645</td>\n",
       "      <td>8874.jpg</td>\n",
       "      <td>1</td>\n",
       "      <td>0.000261</td>\n",
       "      <td>5822.jpg</td>\n",
       "      <td>0</td>\n",
       "      <td>0.000214</td>\n",
       "      <td>...</td>\n",
       "      <td>0.522837</td>\n",
       "      <td>0.731896</td>\n",
       "      <td>0.147648</td>\n",
       "      <td>0.386790</td>\n",
       "      <td>0.051374</td>\n",
       "      <td>0.778548</td>\n",
       "      <td>0.395377</td>\n",
       "      <td>0.129621</td>\n",
       "      <td>0.112330</td>\n",
       "      <td>0.625302</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>997</th>\n",
       "      <td>0.999964</td>\n",
       "      <td>11351.jpg</td>\n",
       "      <td>1</td>\n",
       "      <td>0.870447</td>\n",
       "      <td>3912.jpg</td>\n",
       "      <td>1</td>\n",
       "      <td>0.999834</td>\n",
       "      <td>9669.jpg</td>\n",
       "      <td>1</td>\n",
       "      <td>0.997703</td>\n",
       "      <td>...</td>\n",
       "      <td>0.847437</td>\n",
       "      <td>0.770199</td>\n",
       "      <td>0.885918</td>\n",
       "      <td>0.759013</td>\n",
       "      <td>0.330060</td>\n",
       "      <td>0.717825</td>\n",
       "      <td>0.959996</td>\n",
       "      <td>0.245845</td>\n",
       "      <td>0.970577</td>\n",
       "      <td>0.476467</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>998</th>\n",
       "      <td>0.999894</td>\n",
       "      <td>1722.jpg</td>\n",
       "      <td>0</td>\n",
       "      <td>0.786689</td>\n",
       "      <td>13456.jpg</td>\n",
       "      <td>1</td>\n",
       "      <td>0.000036</td>\n",
       "      <td>434.jpg</td>\n",
       "      <td>0</td>\n",
       "      <td>0.000046</td>\n",
       "      <td>...</td>\n",
       "      <td>0.472933</td>\n",
       "      <td>0.775551</td>\n",
       "      <td>0.150926</td>\n",
       "      <td>0.111194</td>\n",
       "      <td>0.709597</td>\n",
       "      <td>0.706944</td>\n",
       "      <td>0.577444</td>\n",
       "      <td>0.246927</td>\n",
       "      <td>0.276034</td>\n",
       "      <td>0.593935</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>999</th>\n",
       "      <td>0.999963</td>\n",
       "      <td>8422.jpg</td>\n",
       "      <td>1</td>\n",
       "      <td>0.072662</td>\n",
       "      <td>5864.jpg</td>\n",
       "      <td>0</td>\n",
       "      <td>0.999982</td>\n",
       "      <td>7349.jpg</td>\n",
       "      <td>1</td>\n",
       "      <td>0.000336</td>\n",
       "      <td>...</td>\n",
       "      <td>0.432436</td>\n",
       "      <td>0.183602</td>\n",
       "      <td>0.842173</td>\n",
       "      <td>0.398861</td>\n",
       "      <td>0.808799</td>\n",
       "      <td>0.302027</td>\n",
       "      <td>0.752706</td>\n",
       "      <td>0.168201</td>\n",
       "      <td>0.864932</td>\n",
       "      <td>0.719422</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>1000 rows × 50 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "     model_VisualBERT_v0 file_name_0  label_0  model_VisualBERT_v1  \\\n",
       "0               0.206465    5827.jpg        0             0.962831   \n",
       "1               0.900002    2454.jpg        0             0.041153   \n",
       "2               0.999964    6492.jpg        1             0.611585   \n",
       "3               0.999963    2054.jpg        1             0.973622   \n",
       "4               0.000028    5388.jpg        0             0.039521   \n",
       "..                   ...         ...      ...                  ...   \n",
       "995             0.999964      17.jpg        1             0.063730   \n",
       "996             0.000027    3869.jpg        0             0.967645   \n",
       "997             0.999964   11351.jpg        1             0.870447   \n",
       "998             0.999894    1722.jpg        0             0.786689   \n",
       "999             0.999963    8422.jpg        1             0.072662   \n",
       "\n",
       "    file_name_1  label_1  model_VisualBERT_v2 file_name_2  label_2  \\\n",
       "0      9764.jpg        1             0.999915   11621.jpg        1   \n",
       "1      8222.jpg        0             0.000031    7751.jpg        0   \n",
       "2      4152.jpg        1             0.999986    8915.jpg        1   \n",
       "3      4656.jpg        1             0.999980   10632.jpg        1   \n",
       "4      5653.jpg        1             0.998182    7885.jpg        1   \n",
       "..          ...      ...                  ...         ...      ...   \n",
       "995    1033.jpg        1             0.000070    6206.jpg        0   \n",
       "996    8874.jpg        1             0.000261    5822.jpg        0   \n",
       "997    3912.jpg        1             0.999834    9669.jpg        1   \n",
       "998   13456.jpg        1             0.000036     434.jpg        0   \n",
       "999    5864.jpg        0             0.999982    7349.jpg        1   \n",
       "\n",
       "     model_VisualBERT_v3  ... unimodal_text_v0  unimodal_text_v1  \\\n",
       "0               0.995756  ...         0.716009          0.659856   \n",
       "1               0.329732  ...         0.165136          0.502955   \n",
       "2               0.998707  ...         0.860050          0.807957   \n",
       "3               0.999943  ...         0.663105          0.637365   \n",
       "4               0.999903  ...         0.030513          0.650166   \n",
       "..                   ...  ...              ...               ...   \n",
       "995             0.000741  ...         0.938401          0.494559   \n",
       "996             0.000214  ...         0.522837          0.731896   \n",
       "997             0.997703  ...         0.847437          0.770199   \n",
       "998             0.000046  ...         0.472933          0.775551   \n",
       "999             0.000336  ...         0.432436          0.183602   \n",
       "\n",
       "     unimodal_text_v2 unimodal_text_v3  unimodal_text_v4  unimodal_text_v5  \\\n",
       "0            0.801545         0.343322          0.541126          0.523656   \n",
       "1            0.055388         0.645190          0.120404          0.140960   \n",
       "2            0.740347         0.496732          0.068227          0.763816   \n",
       "3            0.941983         0.540022          0.093168          0.252664   \n",
       "4            0.960102         0.660472          0.791665          0.581676   \n",
       "..                ...              ...               ...               ...   \n",
       "995          0.451599         0.828844          0.330451          0.896399   \n",
       "996          0.147648         0.386790          0.051374          0.778548   \n",
       "997          0.885918         0.759013          0.330060          0.717825   \n",
       "998          0.150926         0.111194          0.709597          0.706944   \n",
       "999          0.842173         0.398861          0.808799          0.302027   \n",
       "\n",
       "    unimodal_text_v6  unimodal_text_v7  unimodal_text_v8 unimodal_text_v9  \n",
       "0           0.232028          0.097245          0.164340         0.888618  \n",
       "1           0.579082          0.150768          0.684266         0.859328  \n",
       "2           0.117275          0.105782          0.911819         0.061357  \n",
       "3           0.574554          0.165804          0.923068         0.138661  \n",
       "4           0.775880          0.178797          0.810652         0.034840  \n",
       "..               ...               ...               ...              ...  \n",
       "995         0.232947          0.579574          0.343696         0.679110  \n",
       "996         0.395377          0.129621          0.112330         0.625302  \n",
       "997         0.959996          0.245845          0.970577         0.476467  \n",
       "998         0.577444          0.246927          0.276034         0.593935  \n",
       "999         0.752706          0.168201          0.864932         0.719422  \n",
       "\n",
       "[1000 rows x 50 columns]"
      ]
     },
     "execution_count": 25,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "train_df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {},
   "outputs": [],
   "source": [
    "pred = pd.read_csv(unimodal_test_path, sep='\\t', usecols=['id', 'pred'])\n",
    "i = 0\n",
    "for fold in np.array_split(pred, 10):\n",
    "    fold.columns = ['file_name', MODELNAMES_TEXT[i]]\n",
    "    test_df = pd.merge(test_df, fold, on='file_name')    \n",
    "    i=i+1"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>file_name</th>\n",
       "      <th>Text Transcription</th>\n",
       "      <th>misogynous</th>\n",
       "      <th>shaming</th>\n",
       "      <th>stereotype</th>\n",
       "      <th>objectification</th>\n",
       "      <th>violence</th>\n",
       "      <th>model_VisualBERT_v0</th>\n",
       "      <th>model_VisualBERT_v1</th>\n",
       "      <th>model_VisualBERT_v2</th>\n",
       "      <th>...</th>\n",
       "      <th>unimodal_text_v0</th>\n",
       "      <th>unimodal_text_v1</th>\n",
       "      <th>unimodal_text_v2</th>\n",
       "      <th>unimodal_text_v3</th>\n",
       "      <th>unimodal_text_v4</th>\n",
       "      <th>unimodal_text_v5</th>\n",
       "      <th>unimodal_text_v6</th>\n",
       "      <th>unimodal_text_v7</th>\n",
       "      <th>unimodal_text_v8</th>\n",
       "      <th>unimodal_text_v9</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>15190.jpg</td>\n",
       "      <td>ASIAN GIRLS GMPIREHELENAHARPER.DEVIANTART.COM ...</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0.999905</td>\n",
       "      <td>0.996712</td>\n",
       "      <td>0.999973</td>\n",
       "      <td>...</td>\n",
       "      <td>0.891632</td>\n",
       "      <td>0.894904</td>\n",
       "      <td>0.894893</td>\n",
       "      <td>0.901804</td>\n",
       "      <td>0.897775</td>\n",
       "      <td>0.904913</td>\n",
       "      <td>0.904312</td>\n",
       "      <td>0.892371</td>\n",
       "      <td>0.899648</td>\n",
       "      <td>0.888035</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>15986.jpg</td>\n",
       "      <td>That Guy Really Likes This Show. VERY DEMOTIVA...</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0.999914</td>\n",
       "      <td>0.996296</td>\n",
       "      <td>0.999963</td>\n",
       "      <td>...</td>\n",
       "      <td>0.602306</td>\n",
       "      <td>0.574069</td>\n",
       "      <td>0.563183</td>\n",
       "      <td>0.581283</td>\n",
       "      <td>0.565852</td>\n",
       "      <td>0.581378</td>\n",
       "      <td>0.594737</td>\n",
       "      <td>0.628536</td>\n",
       "      <td>0.585531</td>\n",
       "      <td>0.615766</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>15005.jpg</td>\n",
       "      <td>A MAN WITH DREAMS... NEEDS A WOMAN WITH VISION</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0.263468</td>\n",
       "      <td>0.280405</td>\n",
       "      <td>0.687476</td>\n",
       "      <td>...</td>\n",
       "      <td>0.802283</td>\n",
       "      <td>0.799277</td>\n",
       "      <td>0.806764</td>\n",
       "      <td>0.820269</td>\n",
       "      <td>0.829410</td>\n",
       "      <td>0.815493</td>\n",
       "      <td>0.810987</td>\n",
       "      <td>0.795741</td>\n",
       "      <td>0.803643</td>\n",
       "      <td>0.796717</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>15923.jpg</td>\n",
       "      <td>CLAIMS TO LOVE CLEANING DOESN'T EVEN CLEAN DOT...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0.007080</td>\n",
       "      <td>0.007222</td>\n",
       "      <td>0.003045</td>\n",
       "      <td>...</td>\n",
       "      <td>0.384444</td>\n",
       "      <td>0.370022</td>\n",
       "      <td>0.363571</td>\n",
       "      <td>0.377545</td>\n",
       "      <td>0.393123</td>\n",
       "      <td>0.400722</td>\n",
       "      <td>0.379677</td>\n",
       "      <td>0.355487</td>\n",
       "      <td>0.394114</td>\n",
       "      <td>0.412966</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>15971.jpg</td>\n",
       "      <td>Women Women Women 0 Respect</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0.905814</td>\n",
       "      <td>0.996184</td>\n",
       "      <td>0.999928</td>\n",
       "      <td>...</td>\n",
       "      <td>0.941116</td>\n",
       "      <td>0.936386</td>\n",
       "      <td>0.945409</td>\n",
       "      <td>0.945651</td>\n",
       "      <td>0.939623</td>\n",
       "      <td>0.933454</td>\n",
       "      <td>0.946942</td>\n",
       "      <td>0.939384</td>\n",
       "      <td>0.939326</td>\n",
       "      <td>0.937820</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>995</th>\n",
       "      <td>15964.jpg</td>\n",
       "      <td>YEAH, I HAVE A GIRLFRIEND. BUT SHE'S BEING A B...</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0.567974</td>\n",
       "      <td>0.881868</td>\n",
       "      <td>0.079093</td>\n",
       "      <td>...</td>\n",
       "      <td>0.487062</td>\n",
       "      <td>0.517036</td>\n",
       "      <td>0.477081</td>\n",
       "      <td>0.486946</td>\n",
       "      <td>0.472799</td>\n",
       "      <td>0.479309</td>\n",
       "      <td>0.497491</td>\n",
       "      <td>0.481165</td>\n",
       "      <td>0.483816</td>\n",
       "      <td>0.485478</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>996</th>\n",
       "      <td>15990.jpg</td>\n",
       "      <td>PRAY FOR MY GIRLFRIEND SHE'S USED THE DOG SNAP...</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>0.992131</td>\n",
       "      <td>0.994005</td>\n",
       "      <td>0.999963</td>\n",
       "      <td>...</td>\n",
       "      <td>0.405238</td>\n",
       "      <td>0.430124</td>\n",
       "      <td>0.403387</td>\n",
       "      <td>0.377663</td>\n",
       "      <td>0.395504</td>\n",
       "      <td>0.383512</td>\n",
       "      <td>0.375734</td>\n",
       "      <td>0.372640</td>\n",
       "      <td>0.427392</td>\n",
       "      <td>0.393909</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>997</th>\n",
       "      <td>15461.jpg</td>\n",
       "      <td>HOW TO HAVE FUN WITH A MINOR Fung Step 3-&gt; Ste...</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>0.999908</td>\n",
       "      <td>0.996512</td>\n",
       "      <td>0.999978</td>\n",
       "      <td>...</td>\n",
       "      <td>0.357203</td>\n",
       "      <td>0.371048</td>\n",
       "      <td>0.362940</td>\n",
       "      <td>0.365109</td>\n",
       "      <td>0.414402</td>\n",
       "      <td>0.415614</td>\n",
       "      <td>0.396358</td>\n",
       "      <td>0.385421</td>\n",
       "      <td>0.353028</td>\n",
       "      <td>0.393908</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>998</th>\n",
       "      <td>16018.jpg</td>\n",
       "      <td>this could be us but you ain't here Calvin Kle...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0.853722</td>\n",
       "      <td>0.071663</td>\n",
       "      <td>0.121594</td>\n",
       "      <td>...</td>\n",
       "      <td>0.263323</td>\n",
       "      <td>0.231879</td>\n",
       "      <td>0.241135</td>\n",
       "      <td>0.277481</td>\n",
       "      <td>0.265675</td>\n",
       "      <td>0.290668</td>\n",
       "      <td>0.259727</td>\n",
       "      <td>0.256042</td>\n",
       "      <td>0.281247</td>\n",
       "      <td>0.228946</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>999</th>\n",
       "      <td>15228.jpg</td>\n",
       "      <td>LADIES THIS IS HOW TO TAKE A SEXY SELFIE</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0.998254</td>\n",
       "      <td>0.996356</td>\n",
       "      <td>0.999972</td>\n",
       "      <td>...</td>\n",
       "      <td>0.846859</td>\n",
       "      <td>0.860702</td>\n",
       "      <td>0.841280</td>\n",
       "      <td>0.850942</td>\n",
       "      <td>0.857928</td>\n",
       "      <td>0.858570</td>\n",
       "      <td>0.838859</td>\n",
       "      <td>0.852339</td>\n",
       "      <td>0.841615</td>\n",
       "      <td>0.855658</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>1000 rows × 37 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "     file_name                                 Text Transcription  misogynous  \\\n",
       "0    15190.jpg  ASIAN GIRLS GMPIREHELENAHARPER.DEVIANTART.COM ...           1   \n",
       "1    15986.jpg  That Guy Really Likes This Show. VERY DEMOTIVA...           1   \n",
       "2    15005.jpg     A MAN WITH DREAMS... NEEDS A WOMAN WITH VISION           0   \n",
       "3    15923.jpg  CLAIMS TO LOVE CLEANING DOESN'T EVEN CLEAN DOT...           0   \n",
       "4    15971.jpg                        Women Women Women 0 Respect           1   \n",
       "..         ...                                                ...         ...   \n",
       "995  15964.jpg  YEAH, I HAVE A GIRLFRIEND. BUT SHE'S BEING A B...           1   \n",
       "996  15990.jpg  PRAY FOR MY GIRLFRIEND SHE'S USED THE DOG SNAP...           1   \n",
       "997  15461.jpg  HOW TO HAVE FUN WITH A MINOR Fung Step 3-> Ste...           1   \n",
       "998  16018.jpg  this could be us but you ain't here Calvin Kle...           0   \n",
       "999  15228.jpg           LADIES THIS IS HOW TO TAKE A SEXY SELFIE           0   \n",
       "\n",
       "     shaming  stereotype  objectification  violence  model_VisualBERT_v0  \\\n",
       "0          0           1                1         0             0.999905   \n",
       "1          0           1                1         0             0.999914   \n",
       "2          0           0                0         0             0.263468   \n",
       "3          0           0                0         0             0.007080   \n",
       "4          0           1                0         1             0.905814   \n",
       "..       ...         ...              ...       ...                  ...   \n",
       "995        0           1                1         0             0.567974   \n",
       "996        1           1                1         1             0.992131   \n",
       "997        0           0                1         1             0.999908   \n",
       "998        0           0                0         0             0.853722   \n",
       "999        0           0                0         0             0.998254   \n",
       "\n",
       "     model_VisualBERT_v1  model_VisualBERT_v2  ...  unimodal_text_v0  \\\n",
       "0               0.996712             0.999973  ...          0.891632   \n",
       "1               0.996296             0.999963  ...          0.602306   \n",
       "2               0.280405             0.687476  ...          0.802283   \n",
       "3               0.007222             0.003045  ...          0.384444   \n",
       "4               0.996184             0.999928  ...          0.941116   \n",
       "..                   ...                  ...  ...               ...   \n",
       "995             0.881868             0.079093  ...          0.487062   \n",
       "996             0.994005             0.999963  ...          0.405238   \n",
       "997             0.996512             0.999978  ...          0.357203   \n",
       "998             0.071663             0.121594  ...          0.263323   \n",
       "999             0.996356             0.999972  ...          0.846859   \n",
       "\n",
       "     unimodal_text_v1  unimodal_text_v2  unimodal_text_v3  unimodal_text_v4  \\\n",
       "0            0.894904          0.894893          0.901804          0.897775   \n",
       "1            0.574069          0.563183          0.581283          0.565852   \n",
       "2            0.799277          0.806764          0.820269          0.829410   \n",
       "3            0.370022          0.363571          0.377545          0.393123   \n",
       "4            0.936386          0.945409          0.945651          0.939623   \n",
       "..                ...               ...               ...               ...   \n",
       "995          0.517036          0.477081          0.486946          0.472799   \n",
       "996          0.430124          0.403387          0.377663          0.395504   \n",
       "997          0.371048          0.362940          0.365109          0.414402   \n",
       "998          0.231879          0.241135          0.277481          0.265675   \n",
       "999          0.860702          0.841280          0.850942          0.857928   \n",
       "\n",
       "     unimodal_text_v5  unimodal_text_v6  unimodal_text_v7  unimodal_text_v8  \\\n",
       "0            0.904913          0.904312          0.892371          0.899648   \n",
       "1            0.581378          0.594737          0.628536          0.585531   \n",
       "2            0.815493          0.810987          0.795741          0.803643   \n",
       "3            0.400722          0.379677          0.355487          0.394114   \n",
       "4            0.933454          0.946942          0.939384          0.939326   \n",
       "..                ...               ...               ...               ...   \n",
       "995          0.479309          0.497491          0.481165          0.483816   \n",
       "996          0.383512          0.375734          0.372640          0.427392   \n",
       "997          0.415614          0.396358          0.385421          0.353028   \n",
       "998          0.290668          0.259727          0.256042          0.281247   \n",
       "999          0.858570          0.838859          0.852339          0.841615   \n",
       "\n",
       "     unimodal_text_v9  \n",
       "0            0.888035  \n",
       "1            0.615766  \n",
       "2            0.796717  \n",
       "3            0.412966  \n",
       "4            0.937820  \n",
       "..                ...  \n",
       "995          0.485478  \n",
       "996          0.393909  \n",
       "997          0.393908  \n",
       "998          0.228946  \n",
       "999          0.855658  \n",
       "\n",
       "[1000 rows x 37 columns]"
      ]
     },
     "execution_count": 27,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "test_df"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Unimodal tags"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {},
   "outputs": [],
   "source": [
    "unimodal_train_path = './Unimodal/predictions/Clarifai_pred_10Fold.csv'\n",
    "unimodal_test_path = './Unimodal/predictionsTest/Clarifai_pred_10Fold.csv'\n",
    "MODELNAMES_TAGS = ['unimodal_tags_v{}'.format(i) for i in range(10)]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "metadata": {},
   "outputs": [],
   "source": [
    "pred = pd.read_csv(unimodal_train_path, sep='\\t', usecols=['id', 'pred'])\n",
    "i = 0\n",
    "for fold in np.array_split(pred, 10):\n",
    "    fold.columns = ['file_name_'+str(i), MODELNAMES_TAGS[i]]\n",
    "    train_df = pd.merge(train_df, fold, on='file_name_'+str(i))\n",
    "    i=i+1"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "Index(['model_VisualBERT_v0', 'file_name_0', 'label_0', 'model_VisualBERT_v1',\n",
       "       'file_name_1', 'label_1', 'model_VisualBERT_v2', 'file_name_2',\n",
       "       'label_2', 'model_VisualBERT_v3', 'file_name_3', 'label_3',\n",
       "       'model_VisualBERT_v4', 'file_name_4', 'label_4', 'model_VisualBERT_v5',\n",
       "       'file_name_5', 'label_5', 'model_VisualBERT_v6', 'file_name_6',\n",
       "       'label_6', 'model_VisualBERT_v7', 'file_name_7', 'label_7',\n",
       "       'model_VisualBERT_v8', 'file_name_8', 'label_8', 'model_VisualBERT_v9',\n",
       "       'file_name_9', 'label_9', 'model_BERT_v0', 'model_BERT_v1',\n",
       "       'model_BERT_v2', 'model_BERT_v3', 'model_BERT_v4', 'model_BERT_v5',\n",
       "       'model_BERT_v6', 'model_BERT_v7', 'model_BERT_v8', 'model_BERT_v9',\n",
       "       'unimodal_text_v0', 'unimodal_text_v1', 'unimodal_text_v2',\n",
       "       'unimodal_text_v3', 'unimodal_text_v4', 'unimodal_text_v5',\n",
       "       'unimodal_text_v6', 'unimodal_text_v7', 'unimodal_text_v8',\n",
       "       'unimodal_text_v9', 'unimodal_tags_v0', 'unimodal_tags_v1',\n",
       "       'unimodal_tags_v2', 'unimodal_tags_v3', 'unimodal_tags_v4',\n",
       "       'unimodal_tags_v5', 'unimodal_tags_v6', 'unimodal_tags_v7',\n",
       "       'unimodal_tags_v8', 'unimodal_tags_v9'],\n",
       "      dtype='object')"
      ]
     },
     "execution_count": 30,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "train_df.columns"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "metadata": {},
   "outputs": [],
   "source": [
    "pred = pd.read_csv(unimodal_test_path, sep='\\t', usecols=['id', 'pred'])\n",
    "i = 0\n",
    "for fold in np.array_split(pred, 10):\n",
    "    fold.columns = ['file_name', MODELNAMES_TAGS[i]]\n",
    "    test_df = pd.merge(test_df, fold, on='file_name')    \n",
    "    i=i+1"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "Index(['file_name', 'Text Transcription', 'misogynous', 'shaming',\n",
       "       'stereotype', 'objectification', 'violence', 'model_VisualBERT_v0',\n",
       "       'model_VisualBERT_v1', 'model_VisualBERT_v2', 'model_VisualBERT_v3',\n",
       "       'model_VisualBERT_v4', 'model_VisualBERT_v5', 'model_VisualBERT_v6',\n",
       "       'model_VisualBERT_v7', 'model_VisualBERT_v8', 'model_VisualBERT_v9',\n",
       "       'model_BERT_v0', 'model_BERT_v1', 'model_BERT_v2', 'model_BERT_v3',\n",
       "       'model_BERT_v4', 'model_BERT_v5', 'model_BERT_v6', 'model_BERT_v7',\n",
       "       'model_BERT_v8', 'model_BERT_v9', 'unimodal_text_v0',\n",
       "       'unimodal_text_v1', 'unimodal_text_v2', 'unimodal_text_v3',\n",
       "       'unimodal_text_v4', 'unimodal_text_v5', 'unimodal_text_v6',\n",
       "       'unimodal_text_v7', 'unimodal_text_v8', 'unimodal_text_v9',\n",
       "       'unimodal_tags_v0', 'unimodal_tags_v1', 'unimodal_tags_v2',\n",
       "       'unimodal_tags_v3', 'unimodal_tags_v4', 'unimodal_tags_v5',\n",
       "       'unimodal_tags_v6', 'unimodal_tags_v7', 'unimodal_tags_v8',\n",
       "       'unimodal_tags_v9'],\n",
       "      dtype='object')"
      ]
     },
     "execution_count": 32,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "test_df.columns"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 35,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>file_name</th>\n",
       "      <th>Text Transcription</th>\n",
       "      <th>misogynous</th>\n",
       "      <th>shaming</th>\n",
       "      <th>stereotype</th>\n",
       "      <th>objectification</th>\n",
       "      <th>violence</th>\n",
       "      <th>model_VisualBERT_v0</th>\n",
       "      <th>model_VisualBERT_v1</th>\n",
       "      <th>model_VisualBERT_v2</th>\n",
       "      <th>...</th>\n",
       "      <th>unimodal_tags_v0</th>\n",
       "      <th>unimodal_tags_v1</th>\n",
       "      <th>unimodal_tags_v2</th>\n",
       "      <th>unimodal_tags_v3</th>\n",
       "      <th>unimodal_tags_v4</th>\n",
       "      <th>unimodal_tags_v5</th>\n",
       "      <th>unimodal_tags_v6</th>\n",
       "      <th>unimodal_tags_v7</th>\n",
       "      <th>unimodal_tags_v8</th>\n",
       "      <th>unimodal_tags_v9</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>15190.jpg</td>\n",
       "      <td>ASIAN GIRLS GMPIREHELENAHARPER.DEVIANTART.COM ...</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0.999905</td>\n",
       "      <td>0.996712</td>\n",
       "      <td>0.999973</td>\n",
       "      <td>...</td>\n",
       "      <td>0.841095</td>\n",
       "      <td>0.844879</td>\n",
       "      <td>0.841648</td>\n",
       "      <td>0.850045</td>\n",
       "      <td>0.849826</td>\n",
       "      <td>0.852751</td>\n",
       "      <td>0.847754</td>\n",
       "      <td>0.848688</td>\n",
       "      <td>0.847019</td>\n",
       "      <td>0.849012</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>15986.jpg</td>\n",
       "      <td>That Guy Really Likes This Show. VERY DEMOTIVA...</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0.999914</td>\n",
       "      <td>0.996296</td>\n",
       "      <td>0.999963</td>\n",
       "      <td>...</td>\n",
       "      <td>0.520474</td>\n",
       "      <td>0.517265</td>\n",
       "      <td>0.521734</td>\n",
       "      <td>0.517112</td>\n",
       "      <td>0.526077</td>\n",
       "      <td>0.518164</td>\n",
       "      <td>0.512942</td>\n",
       "      <td>0.521300</td>\n",
       "      <td>0.520155</td>\n",
       "      <td>0.517832</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>15005.jpg</td>\n",
       "      <td>A MAN WITH DREAMS... NEEDS A WOMAN WITH VISION</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0.263468</td>\n",
       "      <td>0.280405</td>\n",
       "      <td>0.687476</td>\n",
       "      <td>...</td>\n",
       "      <td>0.594372</td>\n",
       "      <td>0.589146</td>\n",
       "      <td>0.598615</td>\n",
       "      <td>0.592580</td>\n",
       "      <td>0.596606</td>\n",
       "      <td>0.592334</td>\n",
       "      <td>0.583083</td>\n",
       "      <td>0.588754</td>\n",
       "      <td>0.589312</td>\n",
       "      <td>0.592981</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>15923.jpg</td>\n",
       "      <td>CLAIMS TO LOVE CLEANING DOESN'T EVEN CLEAN DOT...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0.007080</td>\n",
       "      <td>0.007222</td>\n",
       "      <td>0.003045</td>\n",
       "      <td>...</td>\n",
       "      <td>0.365064</td>\n",
       "      <td>0.364095</td>\n",
       "      <td>0.366484</td>\n",
       "      <td>0.364221</td>\n",
       "      <td>0.351718</td>\n",
       "      <td>0.368988</td>\n",
       "      <td>0.372602</td>\n",
       "      <td>0.363364</td>\n",
       "      <td>0.356766</td>\n",
       "      <td>0.372120</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>15971.jpg</td>\n",
       "      <td>Women Women Women 0 Respect</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0.905814</td>\n",
       "      <td>0.996184</td>\n",
       "      <td>0.999928</td>\n",
       "      <td>...</td>\n",
       "      <td>0.594372</td>\n",
       "      <td>0.589146</td>\n",
       "      <td>0.598615</td>\n",
       "      <td>0.592580</td>\n",
       "      <td>0.596606</td>\n",
       "      <td>0.592334</td>\n",
       "      <td>0.583083</td>\n",
       "      <td>0.588754</td>\n",
       "      <td>0.589312</td>\n",
       "      <td>0.592981</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>995</th>\n",
       "      <td>15964.jpg</td>\n",
       "      <td>YEAH, I HAVE A GIRLFRIEND. BUT SHE'S BEING A B...</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0.567974</td>\n",
       "      <td>0.881868</td>\n",
       "      <td>0.079093</td>\n",
       "      <td>...</td>\n",
       "      <td>0.340599</td>\n",
       "      <td>0.353420</td>\n",
       "      <td>0.353570</td>\n",
       "      <td>0.347510</td>\n",
       "      <td>0.351462</td>\n",
       "      <td>0.355004</td>\n",
       "      <td>0.352048</td>\n",
       "      <td>0.353859</td>\n",
       "      <td>0.351936</td>\n",
       "      <td>0.351282</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>996</th>\n",
       "      <td>15990.jpg</td>\n",
       "      <td>PRAY FOR MY GIRLFRIEND SHE'S USED THE DOG SNAP...</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>0.992131</td>\n",
       "      <td>0.994005</td>\n",
       "      <td>0.999963</td>\n",
       "      <td>...</td>\n",
       "      <td>0.841095</td>\n",
       "      <td>0.844879</td>\n",
       "      <td>0.841648</td>\n",
       "      <td>0.850045</td>\n",
       "      <td>0.849826</td>\n",
       "      <td>0.852751</td>\n",
       "      <td>0.847754</td>\n",
       "      <td>0.848688</td>\n",
       "      <td>0.847019</td>\n",
       "      <td>0.849012</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>997</th>\n",
       "      <td>15461.jpg</td>\n",
       "      <td>HOW TO HAVE FUN WITH A MINOR Fung Step 3-&gt; Ste...</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>0.999908</td>\n",
       "      <td>0.996512</td>\n",
       "      <td>0.999978</td>\n",
       "      <td>...</td>\n",
       "      <td>0.638507</td>\n",
       "      <td>0.641410</td>\n",
       "      <td>0.635588</td>\n",
       "      <td>0.642491</td>\n",
       "      <td>0.649024</td>\n",
       "      <td>0.644026</td>\n",
       "      <td>0.640937</td>\n",
       "      <td>0.648135</td>\n",
       "      <td>0.645202</td>\n",
       "      <td>0.640154</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>998</th>\n",
       "      <td>16018.jpg</td>\n",
       "      <td>this could be us but you ain't here Calvin Kle...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0.853722</td>\n",
       "      <td>0.071663</td>\n",
       "      <td>0.121594</td>\n",
       "      <td>...</td>\n",
       "      <td>0.594372</td>\n",
       "      <td>0.589146</td>\n",
       "      <td>0.598615</td>\n",
       "      <td>0.592580</td>\n",
       "      <td>0.596606</td>\n",
       "      <td>0.592334</td>\n",
       "      <td>0.583083</td>\n",
       "      <td>0.588754</td>\n",
       "      <td>0.589312</td>\n",
       "      <td>0.592981</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>999</th>\n",
       "      <td>15228.jpg</td>\n",
       "      <td>LADIES THIS IS HOW TO TAKE A SEXY SELFIE</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0.998254</td>\n",
       "      <td>0.996356</td>\n",
       "      <td>0.999972</td>\n",
       "      <td>...</td>\n",
       "      <td>0.841095</td>\n",
       "      <td>0.844879</td>\n",
       "      <td>0.841648</td>\n",
       "      <td>0.850045</td>\n",
       "      <td>0.849826</td>\n",
       "      <td>0.852751</td>\n",
       "      <td>0.847754</td>\n",
       "      <td>0.848688</td>\n",
       "      <td>0.847019</td>\n",
       "      <td>0.849012</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>1000 rows × 47 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "     file_name                                 Text Transcription  misogynous  \\\n",
       "0    15190.jpg  ASIAN GIRLS GMPIREHELENAHARPER.DEVIANTART.COM ...           1   \n",
       "1    15986.jpg  That Guy Really Likes This Show. VERY DEMOTIVA...           1   \n",
       "2    15005.jpg     A MAN WITH DREAMS... NEEDS A WOMAN WITH VISION           0   \n",
       "3    15923.jpg  CLAIMS TO LOVE CLEANING DOESN'T EVEN CLEAN DOT...           0   \n",
       "4    15971.jpg                        Women Women Women 0 Respect           1   \n",
       "..         ...                                                ...         ...   \n",
       "995  15964.jpg  YEAH, I HAVE A GIRLFRIEND. BUT SHE'S BEING A B...           1   \n",
       "996  15990.jpg  PRAY FOR MY GIRLFRIEND SHE'S USED THE DOG SNAP...           1   \n",
       "997  15461.jpg  HOW TO HAVE FUN WITH A MINOR Fung Step 3-> Ste...           1   \n",
       "998  16018.jpg  this could be us but you ain't here Calvin Kle...           0   \n",
       "999  15228.jpg           LADIES THIS IS HOW TO TAKE A SEXY SELFIE           0   \n",
       "\n",
       "     shaming  stereotype  objectification  violence  model_VisualBERT_v0  \\\n",
       "0          0           1                1         0             0.999905   \n",
       "1          0           1                1         0             0.999914   \n",
       "2          0           0                0         0             0.263468   \n",
       "3          0           0                0         0             0.007080   \n",
       "4          0           1                0         1             0.905814   \n",
       "..       ...         ...              ...       ...                  ...   \n",
       "995        0           1                1         0             0.567974   \n",
       "996        1           1                1         1             0.992131   \n",
       "997        0           0                1         1             0.999908   \n",
       "998        0           0                0         0             0.853722   \n",
       "999        0           0                0         0             0.998254   \n",
       "\n",
       "     model_VisualBERT_v1  model_VisualBERT_v2  ...  unimodal_tags_v0  \\\n",
       "0               0.996712             0.999973  ...          0.841095   \n",
       "1               0.996296             0.999963  ...          0.520474   \n",
       "2               0.280405             0.687476  ...          0.594372   \n",
       "3               0.007222             0.003045  ...          0.365064   \n",
       "4               0.996184             0.999928  ...          0.594372   \n",
       "..                   ...                  ...  ...               ...   \n",
       "995             0.881868             0.079093  ...          0.340599   \n",
       "996             0.994005             0.999963  ...          0.841095   \n",
       "997             0.996512             0.999978  ...          0.638507   \n",
       "998             0.071663             0.121594  ...          0.594372   \n",
       "999             0.996356             0.999972  ...          0.841095   \n",
       "\n",
       "     unimodal_tags_v1  unimodal_tags_v2  unimodal_tags_v3  unimodal_tags_v4  \\\n",
       "0            0.844879          0.841648          0.850045          0.849826   \n",
       "1            0.517265          0.521734          0.517112          0.526077   \n",
       "2            0.589146          0.598615          0.592580          0.596606   \n",
       "3            0.364095          0.366484          0.364221          0.351718   \n",
       "4            0.589146          0.598615          0.592580          0.596606   \n",
       "..                ...               ...               ...               ...   \n",
       "995          0.353420          0.353570          0.347510          0.351462   \n",
       "996          0.844879          0.841648          0.850045          0.849826   \n",
       "997          0.641410          0.635588          0.642491          0.649024   \n",
       "998          0.589146          0.598615          0.592580          0.596606   \n",
       "999          0.844879          0.841648          0.850045          0.849826   \n",
       "\n",
       "     unimodal_tags_v5  unimodal_tags_v6  unimodal_tags_v7  unimodal_tags_v8  \\\n",
       "0            0.852751          0.847754          0.848688          0.847019   \n",
       "1            0.518164          0.512942          0.521300          0.520155   \n",
       "2            0.592334          0.583083          0.588754          0.589312   \n",
       "3            0.368988          0.372602          0.363364          0.356766   \n",
       "4            0.592334          0.583083          0.588754          0.589312   \n",
       "..                ...               ...               ...               ...   \n",
       "995          0.355004          0.352048          0.353859          0.351936   \n",
       "996          0.852751          0.847754          0.848688          0.847019   \n",
       "997          0.644026          0.640937          0.648135          0.645202   \n",
       "998          0.592334          0.583083          0.588754          0.589312   \n",
       "999          0.852751          0.847754          0.848688          0.847019   \n",
       "\n",
       "     unimodal_tags_v9  \n",
       "0            0.849012  \n",
       "1            0.517832  \n",
       "2            0.592981  \n",
       "3            0.372120  \n",
       "4            0.592981  \n",
       "..                ...  \n",
       "995          0.351282  \n",
       "996          0.849012  \n",
       "997          0.640154  \n",
       "998          0.592981  \n",
       "999          0.849012  \n",
       "\n",
       "[1000 rows x 47 columns]"
      ]
     },
     "execution_count": 35,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "test_df"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Unimodal Captions"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "metadata": {},
   "outputs": [],
   "source": [
    "unimodal_capt_train_path = './Unimodal/predictions/Azure_pred_10Fold.csv'\n",
    "unimodal_capt_test_path = './Unimodal/predictionsTest/Azure_pred_10Fold.csv'\n",
    "MODELNAMES_CAPTIONS = ['unimodal_captions_v{}'.format(i) for i in range(10)]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 34,
   "metadata": {},
   "outputs": [],
   "source": [
    "pred = pd.read_csv(unimodal_capt_train_path, sep='\\t', usecols=['id', 'pred'])\n",
    "i = 0\n",
    "for fold in np.array_split(pred, 10):\n",
    "    pred.columns = ['file_name_'+str(i), MODELNAMES_CAPTIONS[i]]\n",
    "    train_df = pd.merge(train_df, pred, on='file_name_'+str(i))\n",
    "    i=i+1"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 38,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>model_VisualBERT_v0</th>\n",
       "      <th>file_name_0</th>\n",
       "      <th>label_0</th>\n",
       "      <th>model_VisualBERT_v1</th>\n",
       "      <th>file_name_1</th>\n",
       "      <th>label_1</th>\n",
       "      <th>model_VisualBERT_v2</th>\n",
       "      <th>file_name_2</th>\n",
       "      <th>label_2</th>\n",
       "      <th>model_VisualBERT_v3</th>\n",
       "      <th>...</th>\n",
       "      <th>unimodal_captions_v0</th>\n",
       "      <th>unimodal_captions_v1</th>\n",
       "      <th>unimodal_captions_v2</th>\n",
       "      <th>unimodal_captions_v3</th>\n",
       "      <th>unimodal_captions_v4</th>\n",
       "      <th>unimodal_captions_v5</th>\n",
       "      <th>unimodal_captions_v6</th>\n",
       "      <th>unimodal_captions_v7</th>\n",
       "      <th>unimodal_captions_v8</th>\n",
       "      <th>unimodal_captions_v9</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>0.206465</td>\n",
       "      <td>5827.jpg</td>\n",
       "      <td>0</td>\n",
       "      <td>0.962831</td>\n",
       "      <td>9764.jpg</td>\n",
       "      <td>1</td>\n",
       "      <td>0.999915</td>\n",
       "      <td>11621.jpg</td>\n",
       "      <td>1</td>\n",
       "      <td>0.995756</td>\n",
       "      <td>...</td>\n",
       "      <td>0.477079</td>\n",
       "      <td>0.715125</td>\n",
       "      <td>0.697329</td>\n",
       "      <td>0.485558</td>\n",
       "      <td>0.316651</td>\n",
       "      <td>0.669374</td>\n",
       "      <td>0.515227</td>\n",
       "      <td>0.664286</td>\n",
       "      <td>0.386923</td>\n",
       "      <td>0.810820</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>0.900002</td>\n",
       "      <td>2454.jpg</td>\n",
       "      <td>0</td>\n",
       "      <td>0.041153</td>\n",
       "      <td>8222.jpg</td>\n",
       "      <td>0</td>\n",
       "      <td>0.000031</td>\n",
       "      <td>7751.jpg</td>\n",
       "      <td>0</td>\n",
       "      <td>0.329732</td>\n",
       "      <td>...</td>\n",
       "      <td>0.489165</td>\n",
       "      <td>0.345605</td>\n",
       "      <td>0.587590</td>\n",
       "      <td>0.451608</td>\n",
       "      <td>0.478497</td>\n",
       "      <td>0.485314</td>\n",
       "      <td>0.326760</td>\n",
       "      <td>0.440053</td>\n",
       "      <td>0.539308</td>\n",
       "      <td>0.490972</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>0.999964</td>\n",
       "      <td>6492.jpg</td>\n",
       "      <td>1</td>\n",
       "      <td>0.611585</td>\n",
       "      <td>4152.jpg</td>\n",
       "      <td>1</td>\n",
       "      <td>0.999986</td>\n",
       "      <td>8915.jpg</td>\n",
       "      <td>1</td>\n",
       "      <td>0.998707</td>\n",
       "      <td>...</td>\n",
       "      <td>0.461764</td>\n",
       "      <td>0.476466</td>\n",
       "      <td>0.832976</td>\n",
       "      <td>0.433552</td>\n",
       "      <td>0.464484</td>\n",
       "      <td>0.365924</td>\n",
       "      <td>0.474564</td>\n",
       "      <td>0.424157</td>\n",
       "      <td>0.261337</td>\n",
       "      <td>0.680890</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>0.999963</td>\n",
       "      <td>2054.jpg</td>\n",
       "      <td>1</td>\n",
       "      <td>0.973622</td>\n",
       "      <td>4656.jpg</td>\n",
       "      <td>1</td>\n",
       "      <td>0.999980</td>\n",
       "      <td>10632.jpg</td>\n",
       "      <td>1</td>\n",
       "      <td>0.999943</td>\n",
       "      <td>...</td>\n",
       "      <td>0.294667</td>\n",
       "      <td>0.756665</td>\n",
       "      <td>0.899203</td>\n",
       "      <td>0.880476</td>\n",
       "      <td>0.297992</td>\n",
       "      <td>0.504349</td>\n",
       "      <td>0.404067</td>\n",
       "      <td>0.298292</td>\n",
       "      <td>0.518742</td>\n",
       "      <td>0.402626</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>0.000028</td>\n",
       "      <td>5388.jpg</td>\n",
       "      <td>0</td>\n",
       "      <td>0.039521</td>\n",
       "      <td>5653.jpg</td>\n",
       "      <td>1</td>\n",
       "      <td>0.998182</td>\n",
       "      <td>7885.jpg</td>\n",
       "      <td>1</td>\n",
       "      <td>0.999903</td>\n",
       "      <td>...</td>\n",
       "      <td>0.328054</td>\n",
       "      <td>0.385999</td>\n",
       "      <td>0.382364</td>\n",
       "      <td>0.401464</td>\n",
       "      <td>0.629326</td>\n",
       "      <td>0.473833</td>\n",
       "      <td>0.474806</td>\n",
       "      <td>0.424157</td>\n",
       "      <td>0.480279</td>\n",
       "      <td>0.498594</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>995</th>\n",
       "      <td>0.999964</td>\n",
       "      <td>17.jpg</td>\n",
       "      <td>1</td>\n",
       "      <td>0.063730</td>\n",
       "      <td>1033.jpg</td>\n",
       "      <td>1</td>\n",
       "      <td>0.000070</td>\n",
       "      <td>6206.jpg</td>\n",
       "      <td>0</td>\n",
       "      <td>0.000741</td>\n",
       "      <td>...</td>\n",
       "      <td>0.394368</td>\n",
       "      <td>0.350044</td>\n",
       "      <td>0.309058</td>\n",
       "      <td>0.350996</td>\n",
       "      <td>0.425227</td>\n",
       "      <td>0.389445</td>\n",
       "      <td>0.482310</td>\n",
       "      <td>0.590947</td>\n",
       "      <td>0.480279</td>\n",
       "      <td>0.717600</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>996</th>\n",
       "      <td>0.000027</td>\n",
       "      <td>3869.jpg</td>\n",
       "      <td>0</td>\n",
       "      <td>0.967645</td>\n",
       "      <td>8874.jpg</td>\n",
       "      <td>1</td>\n",
       "      <td>0.000261</td>\n",
       "      <td>5822.jpg</td>\n",
       "      <td>0</td>\n",
       "      <td>0.000214</td>\n",
       "      <td>...</td>\n",
       "      <td>0.489165</td>\n",
       "      <td>0.889961</td>\n",
       "      <td>0.469140</td>\n",
       "      <td>0.397322</td>\n",
       "      <td>0.464484</td>\n",
       "      <td>0.486940</td>\n",
       "      <td>0.482310</td>\n",
       "      <td>0.481572</td>\n",
       "      <td>0.397240</td>\n",
       "      <td>0.870599</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>997</th>\n",
       "      <td>0.999964</td>\n",
       "      <td>11351.jpg</td>\n",
       "      <td>1</td>\n",
       "      <td>0.870447</td>\n",
       "      <td>3912.jpg</td>\n",
       "      <td>1</td>\n",
       "      <td>0.999834</td>\n",
       "      <td>9669.jpg</td>\n",
       "      <td>1</td>\n",
       "      <td>0.997703</td>\n",
       "      <td>...</td>\n",
       "      <td>0.731646</td>\n",
       "      <td>0.808139</td>\n",
       "      <td>0.578389</td>\n",
       "      <td>0.484625</td>\n",
       "      <td>0.594209</td>\n",
       "      <td>0.454806</td>\n",
       "      <td>0.840676</td>\n",
       "      <td>0.468185</td>\n",
       "      <td>0.817135</td>\n",
       "      <td>0.576451</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>998</th>\n",
       "      <td>0.999894</td>\n",
       "      <td>1722.jpg</td>\n",
       "      <td>0</td>\n",
       "      <td>0.786689</td>\n",
       "      <td>13456.jpg</td>\n",
       "      <td>1</td>\n",
       "      <td>0.000036</td>\n",
       "      <td>434.jpg</td>\n",
       "      <td>0</td>\n",
       "      <td>0.000046</td>\n",
       "      <td>...</td>\n",
       "      <td>0.489165</td>\n",
       "      <td>0.702788</td>\n",
       "      <td>0.389878</td>\n",
       "      <td>0.382472</td>\n",
       "      <td>0.695113</td>\n",
       "      <td>0.673907</td>\n",
       "      <td>0.565717</td>\n",
       "      <td>0.329490</td>\n",
       "      <td>0.483741</td>\n",
       "      <td>0.630071</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>999</th>\n",
       "      <td>0.999963</td>\n",
       "      <td>8422.jpg</td>\n",
       "      <td>1</td>\n",
       "      <td>0.072662</td>\n",
       "      <td>5864.jpg</td>\n",
       "      <td>0</td>\n",
       "      <td>0.999982</td>\n",
       "      <td>7349.jpg</td>\n",
       "      <td>1</td>\n",
       "      <td>0.000336</td>\n",
       "      <td>...</td>\n",
       "      <td>0.509997</td>\n",
       "      <td>0.518582</td>\n",
       "      <td>0.474040</td>\n",
       "      <td>0.467842</td>\n",
       "      <td>0.464484</td>\n",
       "      <td>0.610989</td>\n",
       "      <td>0.773022</td>\n",
       "      <td>0.450477</td>\n",
       "      <td>0.354898</td>\n",
       "      <td>0.423635</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>1000 rows × 70 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "     model_VisualBERT_v0 file_name_0  label_0  model_VisualBERT_v1  \\\n",
       "0               0.206465    5827.jpg        0             0.962831   \n",
       "1               0.900002    2454.jpg        0             0.041153   \n",
       "2               0.999964    6492.jpg        1             0.611585   \n",
       "3               0.999963    2054.jpg        1             0.973622   \n",
       "4               0.000028    5388.jpg        0             0.039521   \n",
       "..                   ...         ...      ...                  ...   \n",
       "995             0.999964      17.jpg        1             0.063730   \n",
       "996             0.000027    3869.jpg        0             0.967645   \n",
       "997             0.999964   11351.jpg        1             0.870447   \n",
       "998             0.999894    1722.jpg        0             0.786689   \n",
       "999             0.999963    8422.jpg        1             0.072662   \n",
       "\n",
       "    file_name_1  label_1  model_VisualBERT_v2 file_name_2  label_2  \\\n",
       "0      9764.jpg        1             0.999915   11621.jpg        1   \n",
       "1      8222.jpg        0             0.000031    7751.jpg        0   \n",
       "2      4152.jpg        1             0.999986    8915.jpg        1   \n",
       "3      4656.jpg        1             0.999980   10632.jpg        1   \n",
       "4      5653.jpg        1             0.998182    7885.jpg        1   \n",
       "..          ...      ...                  ...         ...      ...   \n",
       "995    1033.jpg        1             0.000070    6206.jpg        0   \n",
       "996    8874.jpg        1             0.000261    5822.jpg        0   \n",
       "997    3912.jpg        1             0.999834    9669.jpg        1   \n",
       "998   13456.jpg        1             0.000036     434.jpg        0   \n",
       "999    5864.jpg        0             0.999982    7349.jpg        1   \n",
       "\n",
       "     model_VisualBERT_v3  ... unimodal_captions_v0  unimodal_captions_v1  \\\n",
       "0               0.995756  ...             0.477079              0.715125   \n",
       "1               0.329732  ...             0.489165              0.345605   \n",
       "2               0.998707  ...             0.461764              0.476466   \n",
       "3               0.999943  ...             0.294667              0.756665   \n",
       "4               0.999903  ...             0.328054              0.385999   \n",
       "..                   ...  ...                  ...                   ...   \n",
       "995             0.000741  ...             0.394368              0.350044   \n",
       "996             0.000214  ...             0.489165              0.889961   \n",
       "997             0.997703  ...             0.731646              0.808139   \n",
       "998             0.000046  ...             0.489165              0.702788   \n",
       "999             0.000336  ...             0.509997              0.518582   \n",
       "\n",
       "     unimodal_captions_v2 unimodal_captions_v3  unimodal_captions_v4  \\\n",
       "0                0.697329             0.485558              0.316651   \n",
       "1                0.587590             0.451608              0.478497   \n",
       "2                0.832976             0.433552              0.464484   \n",
       "3                0.899203             0.880476              0.297992   \n",
       "4                0.382364             0.401464              0.629326   \n",
       "..                    ...                  ...                   ...   \n",
       "995              0.309058             0.350996              0.425227   \n",
       "996              0.469140             0.397322              0.464484   \n",
       "997              0.578389             0.484625              0.594209   \n",
       "998              0.389878             0.382472              0.695113   \n",
       "999              0.474040             0.467842              0.464484   \n",
       "\n",
       "     unimodal_captions_v5 unimodal_captions_v6  unimodal_captions_v7  \\\n",
       "0                0.669374             0.515227              0.664286   \n",
       "1                0.485314             0.326760              0.440053   \n",
       "2                0.365924             0.474564              0.424157   \n",
       "3                0.504349             0.404067              0.298292   \n",
       "4                0.473833             0.474806              0.424157   \n",
       "..                    ...                  ...                   ...   \n",
       "995              0.389445             0.482310              0.590947   \n",
       "996              0.486940             0.482310              0.481572   \n",
       "997              0.454806             0.840676              0.468185   \n",
       "998              0.673907             0.565717              0.329490   \n",
       "999              0.610989             0.773022              0.450477   \n",
       "\n",
       "     unimodal_captions_v8 unimodal_captions_v9  \n",
       "0                0.386923             0.810820  \n",
       "1                0.539308             0.490972  \n",
       "2                0.261337             0.680890  \n",
       "3                0.518742             0.402626  \n",
       "4                0.480279             0.498594  \n",
       "..                    ...                  ...  \n",
       "995              0.480279             0.717600  \n",
       "996              0.397240             0.870599  \n",
       "997              0.817135             0.576451  \n",
       "998              0.483741             0.630071  \n",
       "999              0.354898             0.423635  \n",
       "\n",
       "[1000 rows x 70 columns]"
      ]
     },
     "execution_count": 38,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "train_df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 35,
   "metadata": {},
   "outputs": [],
   "source": [
    "pred = pd.read_csv(unimodal_capt_test_path, sep='\\t', usecols=['id', 'pred'])\n",
    "i = 0\n",
    "for fold in np.array_split(pred, 10):\n",
    "    fold.columns = ['file_name', MODELNAMES_CAPTIONS[i]]\n",
    "    test_df = pd.merge(test_df, fold, on='file_name')    \n",
    "    i=i+1"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 40,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>file_name</th>\n",
       "      <th>Text Transcription</th>\n",
       "      <th>misogynous</th>\n",
       "      <th>shaming</th>\n",
       "      <th>stereotype</th>\n",
       "      <th>objectification</th>\n",
       "      <th>violence</th>\n",
       "      <th>model_VisualBERT_v0</th>\n",
       "      <th>model_VisualBERT_v1</th>\n",
       "      <th>model_VisualBERT_v2</th>\n",
       "      <th>...</th>\n",
       "      <th>unimodal_captions_v0</th>\n",
       "      <th>unimodal_captions_v1</th>\n",
       "      <th>unimodal_captions_v2</th>\n",
       "      <th>unimodal_captions_v3</th>\n",
       "      <th>unimodal_captions_v4</th>\n",
       "      <th>unimodal_captions_v5</th>\n",
       "      <th>unimodal_captions_v6</th>\n",
       "      <th>unimodal_captions_v7</th>\n",
       "      <th>unimodal_captions_v8</th>\n",
       "      <th>unimodal_captions_v9</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>15190.jpg</td>\n",
       "      <td>ASIAN GIRLS GMPIREHELENAHARPER.DEVIANTART.COM ...</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0.999905</td>\n",
       "      <td>0.996712</td>\n",
       "      <td>0.999973</td>\n",
       "      <td>...</td>\n",
       "      <td>0.889933</td>\n",
       "      <td>0.886331</td>\n",
       "      <td>0.889263</td>\n",
       "      <td>0.883864</td>\n",
       "      <td>0.891418</td>\n",
       "      <td>0.891415</td>\n",
       "      <td>0.887573</td>\n",
       "      <td>0.876316</td>\n",
       "      <td>0.888052</td>\n",
       "      <td>0.883788</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>15986.jpg</td>\n",
       "      <td>That Guy Really Likes This Show. VERY DEMOTIVA...</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0.999914</td>\n",
       "      <td>0.996296</td>\n",
       "      <td>0.999963</td>\n",
       "      <td>...</td>\n",
       "      <td>0.667075</td>\n",
       "      <td>0.658943</td>\n",
       "      <td>0.665472</td>\n",
       "      <td>0.654119</td>\n",
       "      <td>0.656501</td>\n",
       "      <td>0.663121</td>\n",
       "      <td>0.643521</td>\n",
       "      <td>0.644103</td>\n",
       "      <td>0.638976</td>\n",
       "      <td>0.658781</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>15005.jpg</td>\n",
       "      <td>A MAN WITH DREAMS... NEEDS A WOMAN WITH VISION</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0.263468</td>\n",
       "      <td>0.280405</td>\n",
       "      <td>0.687476</td>\n",
       "      <td>...</td>\n",
       "      <td>0.708920</td>\n",
       "      <td>0.736682</td>\n",
       "      <td>0.700530</td>\n",
       "      <td>0.725499</td>\n",
       "      <td>0.726722</td>\n",
       "      <td>0.707675</td>\n",
       "      <td>0.712013</td>\n",
       "      <td>0.711984</td>\n",
       "      <td>0.698909</td>\n",
       "      <td>0.730864</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>15923.jpg</td>\n",
       "      <td>CLAIMS TO LOVE CLEANING DOESN'T EVEN CLEAN DOT...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0.007080</td>\n",
       "      <td>0.007222</td>\n",
       "      <td>0.003045</td>\n",
       "      <td>...</td>\n",
       "      <td>0.383447</td>\n",
       "      <td>0.391585</td>\n",
       "      <td>0.374796</td>\n",
       "      <td>0.396488</td>\n",
       "      <td>0.394887</td>\n",
       "      <td>0.409477</td>\n",
       "      <td>0.379483</td>\n",
       "      <td>0.392727</td>\n",
       "      <td>0.369027</td>\n",
       "      <td>0.392225</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>15971.jpg</td>\n",
       "      <td>Women Women Women 0 Respect</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0.905814</td>\n",
       "      <td>0.996184</td>\n",
       "      <td>0.999928</td>\n",
       "      <td>...</td>\n",
       "      <td>0.626783</td>\n",
       "      <td>0.635847</td>\n",
       "      <td>0.651030</td>\n",
       "      <td>0.629952</td>\n",
       "      <td>0.647840</td>\n",
       "      <td>0.643548</td>\n",
       "      <td>0.625514</td>\n",
       "      <td>0.636158</td>\n",
       "      <td>0.644416</td>\n",
       "      <td>0.644575</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>995</th>\n",
       "      <td>15964.jpg</td>\n",
       "      <td>YEAH, I HAVE A GIRLFRIEND. BUT SHE'S BEING A B...</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0.567974</td>\n",
       "      <td>0.881868</td>\n",
       "      <td>0.079093</td>\n",
       "      <td>...</td>\n",
       "      <td>0.440025</td>\n",
       "      <td>0.416866</td>\n",
       "      <td>0.438744</td>\n",
       "      <td>0.433386</td>\n",
       "      <td>0.430405</td>\n",
       "      <td>0.429349</td>\n",
       "      <td>0.431764</td>\n",
       "      <td>0.427540</td>\n",
       "      <td>0.412526</td>\n",
       "      <td>0.447267</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>996</th>\n",
       "      <td>15990.jpg</td>\n",
       "      <td>PRAY FOR MY GIRLFRIEND SHE'S USED THE DOG SNAP...</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>0.992131</td>\n",
       "      <td>0.994005</td>\n",
       "      <td>0.999963</td>\n",
       "      <td>...</td>\n",
       "      <td>0.759221</td>\n",
       "      <td>0.760225</td>\n",
       "      <td>0.759019</td>\n",
       "      <td>0.749041</td>\n",
       "      <td>0.750318</td>\n",
       "      <td>0.748817</td>\n",
       "      <td>0.751680</td>\n",
       "      <td>0.755464</td>\n",
       "      <td>0.745404</td>\n",
       "      <td>0.767064</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>997</th>\n",
       "      <td>15461.jpg</td>\n",
       "      <td>HOW TO HAVE FUN WITH A MINOR Fung Step 3-&gt; Ste...</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>0.999908</td>\n",
       "      <td>0.996512</td>\n",
       "      <td>0.999978</td>\n",
       "      <td>...</td>\n",
       "      <td>0.623214</td>\n",
       "      <td>0.606287</td>\n",
       "      <td>0.612410</td>\n",
       "      <td>0.638706</td>\n",
       "      <td>0.611408</td>\n",
       "      <td>0.626956</td>\n",
       "      <td>0.604579</td>\n",
       "      <td>0.642349</td>\n",
       "      <td>0.594409</td>\n",
       "      <td>0.631315</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>998</th>\n",
       "      <td>16018.jpg</td>\n",
       "      <td>this could be us but you ain't here Calvin Kle...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0.853722</td>\n",
       "      <td>0.071663</td>\n",
       "      <td>0.121594</td>\n",
       "      <td>...</td>\n",
       "      <td>0.385833</td>\n",
       "      <td>0.421846</td>\n",
       "      <td>0.373666</td>\n",
       "      <td>0.408240</td>\n",
       "      <td>0.377094</td>\n",
       "      <td>0.392957</td>\n",
       "      <td>0.392972</td>\n",
       "      <td>0.400472</td>\n",
       "      <td>0.383878</td>\n",
       "      <td>0.406116</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>999</th>\n",
       "      <td>15228.jpg</td>\n",
       "      <td>LADIES THIS IS HOW TO TAKE A SEXY SELFIE</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0.998254</td>\n",
       "      <td>0.996356</td>\n",
       "      <td>0.999972</td>\n",
       "      <td>...</td>\n",
       "      <td>0.684108</td>\n",
       "      <td>0.714689</td>\n",
       "      <td>0.695018</td>\n",
       "      <td>0.696541</td>\n",
       "      <td>0.717915</td>\n",
       "      <td>0.691033</td>\n",
       "      <td>0.675491</td>\n",
       "      <td>0.695614</td>\n",
       "      <td>0.713560</td>\n",
       "      <td>0.678641</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>1000 rows × 57 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "     file_name                                 Text Transcription  misogynous  \\\n",
       "0    15190.jpg  ASIAN GIRLS GMPIREHELENAHARPER.DEVIANTART.COM ...           1   \n",
       "1    15986.jpg  That Guy Really Likes This Show. VERY DEMOTIVA...           1   \n",
       "2    15005.jpg     A MAN WITH DREAMS... NEEDS A WOMAN WITH VISION           0   \n",
       "3    15923.jpg  CLAIMS TO LOVE CLEANING DOESN'T EVEN CLEAN DOT...           0   \n",
       "4    15971.jpg                        Women Women Women 0 Respect           1   \n",
       "..         ...                                                ...         ...   \n",
       "995  15964.jpg  YEAH, I HAVE A GIRLFRIEND. BUT SHE'S BEING A B...           1   \n",
       "996  15990.jpg  PRAY FOR MY GIRLFRIEND SHE'S USED THE DOG SNAP...           1   \n",
       "997  15461.jpg  HOW TO HAVE FUN WITH A MINOR Fung Step 3-> Ste...           1   \n",
       "998  16018.jpg  this could be us but you ain't here Calvin Kle...           0   \n",
       "999  15228.jpg           LADIES THIS IS HOW TO TAKE A SEXY SELFIE           0   \n",
       "\n",
       "     shaming  stereotype  objectification  violence  model_VisualBERT_v0  \\\n",
       "0          0           1                1         0             0.999905   \n",
       "1          0           1                1         0             0.999914   \n",
       "2          0           0                0         0             0.263468   \n",
       "3          0           0                0         0             0.007080   \n",
       "4          0           1                0         1             0.905814   \n",
       "..       ...         ...              ...       ...                  ...   \n",
       "995        0           1                1         0             0.567974   \n",
       "996        1           1                1         1             0.992131   \n",
       "997        0           0                1         1             0.999908   \n",
       "998        0           0                0         0             0.853722   \n",
       "999        0           0                0         0             0.998254   \n",
       "\n",
       "     model_VisualBERT_v1  model_VisualBERT_v2  ...  unimodal_captions_v0  \\\n",
       "0               0.996712             0.999973  ...              0.889933   \n",
       "1               0.996296             0.999963  ...              0.667075   \n",
       "2               0.280405             0.687476  ...              0.708920   \n",
       "3               0.007222             0.003045  ...              0.383447   \n",
       "4               0.996184             0.999928  ...              0.626783   \n",
       "..                   ...                  ...  ...                   ...   \n",
       "995             0.881868             0.079093  ...              0.440025   \n",
       "996             0.994005             0.999963  ...              0.759221   \n",
       "997             0.996512             0.999978  ...              0.623214   \n",
       "998             0.071663             0.121594  ...              0.385833   \n",
       "999             0.996356             0.999972  ...              0.684108   \n",
       "\n",
       "     unimodal_captions_v1  unimodal_captions_v2  unimodal_captions_v3  \\\n",
       "0                0.886331              0.889263              0.883864   \n",
       "1                0.658943              0.665472              0.654119   \n",
       "2                0.736682              0.700530              0.725499   \n",
       "3                0.391585              0.374796              0.396488   \n",
       "4                0.635847              0.651030              0.629952   \n",
       "..                    ...                   ...                   ...   \n",
       "995              0.416866              0.438744              0.433386   \n",
       "996              0.760225              0.759019              0.749041   \n",
       "997              0.606287              0.612410              0.638706   \n",
       "998              0.421846              0.373666              0.408240   \n",
       "999              0.714689              0.695018              0.696541   \n",
       "\n",
       "     unimodal_captions_v4  unimodal_captions_v5  unimodal_captions_v6  \\\n",
       "0                0.891418              0.891415              0.887573   \n",
       "1                0.656501              0.663121              0.643521   \n",
       "2                0.726722              0.707675              0.712013   \n",
       "3                0.394887              0.409477              0.379483   \n",
       "4                0.647840              0.643548              0.625514   \n",
       "..                    ...                   ...                   ...   \n",
       "995              0.430405              0.429349              0.431764   \n",
       "996              0.750318              0.748817              0.751680   \n",
       "997              0.611408              0.626956              0.604579   \n",
       "998              0.377094              0.392957              0.392972   \n",
       "999              0.717915              0.691033              0.675491   \n",
       "\n",
       "     unimodal_captions_v7  unimodal_captions_v8  unimodal_captions_v9  \n",
       "0                0.876316              0.888052              0.883788  \n",
       "1                0.644103              0.638976              0.658781  \n",
       "2                0.711984              0.698909              0.730864  \n",
       "3                0.392727              0.369027              0.392225  \n",
       "4                0.636158              0.644416              0.644575  \n",
       "..                    ...                   ...                   ...  \n",
       "995              0.427540              0.412526              0.447267  \n",
       "996              0.755464              0.745404              0.767064  \n",
       "997              0.642349              0.594409              0.631315  \n",
       "998              0.400472              0.383878              0.406116  \n",
       "999              0.695614              0.713560              0.678641  \n",
       "\n",
       "[1000 rows x 57 columns]"
      ]
     },
     "execution_count": 40,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "test_df"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Unimodal Models - Performances"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 36,
   "metadata": {},
   "outputs": [],
   "source": [
    "for i in range(0,11):\n",
    "    test_df['label_'+str(i)]=test_df['misogynous']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "dict_mis, dict_not, aucs_text = performances_report_10Fold(train_df, 'unimodal_text', 0.5)\n",
    "print(mean_dict(dict_mis))\n",
    "print(mean_dict(dict_not))\n",
    "print(aucs_text)\n",
    "print(np.mean(aucs_text))\n",
    "\n",
    "print('___________ ON TEST _________________')\n",
    "dict_mis, dict_not, aucs_text_test = performances_report_10Fold(test_df, 'unimodal_text', 0.5)\n",
    "print(mean_dict(dict_mis))\n",
    "print(mean_dict(dict_not))\n",
    "print(aucs_text_test)\n",
    "print(np.mean(aucs_text_test))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 38,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'precision': 0.6837402165707819, 'recall': 0.6069947151466487, 'f1-score': 0.6425675951709374, 'support': 500.0}\n",
      "{'precision': 0.6466513439677882, 'recall': 0.7184182633999957, 'f1-score': 0.6803108464744185, 'support': 500.0}\n",
      "[0.6993711773623851, 0.7295285757181463, 0.7283905410218546, 0.6929193873631649, 0.7010744297719089, 0.7025039999999999, 0.7187818781878187, 0.718491462770639, 0.7065528262113049, 0.6947450974115614]\n",
      "0.7092359375818784\n",
      "___________ ON TEST _________________\n",
      "{'precision': 0.5971018463371055, 'recall': 0.6887999999999999, 'f1-score': 0.6396752767527675, 'support': 500.0}\n",
      "{'precision': 0.6323144796380091, 'recall': 0.5352, 'f1-score': 0.5797077776466423, 'support': 500.0}\n",
      "[0.654462, 0.650622, 0.65305, 0.650618, 0.6511739999999999, 0.65231, 0.6511619999999999, 0.6513180000000001, 0.650414, 0.6522660000000001]\n",
      "0.6517396000000001\n"
     ]
    }
   ],
   "source": [
    "dict_mis, dict_not, aucs_tags = performances_report_10Fold(train_df, 'unimodal_tags', 0.5)\n",
    "print(mean_dict(dict_mis))\n",
    "print(mean_dict(dict_not))\n",
    "print(aucs_tags)\n",
    "print(np.mean(aucs_tags))\n",
    "\n",
    "print('___________ ON TEST _________________')\n",
    "dict_mis, dict_not, aucs_tags_test = performances_report_10Fold(test_df, 'unimodal_tags', 0.5)\n",
    "print(mean_dict(dict_mis))\n",
    "print(mean_dict(dict_not))\n",
    "print(aucs_tags_test)\n",
    "print(np.mean(aucs_tags_test))\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 39,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'precision': 0.6779615715715052, 'recall': 0.4646880974567732, 'f1-score': 0.551022668654052, 'support': 500.0}\n",
      "{'precision': 0.5925526862286177, 'recall': 0.7786265811586195, 'f1-score': 0.6727819337996012, 'support': 500.0}\n",
      "[0.6621678380421695, 0.6717485737163448, 0.6961249244634403, 0.6751508386146853, 0.6590956382553022, 0.686302, 0.6858545854585458, 0.6671360675937336, 0.6555846223384894, 0.687134631620645]\n",
      "0.6746299720103356\n",
      "___________ ON TEST _________________\n",
      "{'precision': 0.5753913241920594, 'recall': 0.4812, 'f1-score': 0.5238139694946471, 'support': 500.0}\n",
      "{'precision': 0.5544322776597956, 'recall': 0.6450000000000001, 'f1-score': 0.5961084184216908, 'support': 500.0}\n",
      "[0.6071719999999999, 0.6093999999999999, 0.612504, 0.6074280000000001, 0.6085559999999999, 0.6051200000000001, 0.610456, 0.6065559999999999, 0.611296, 0.6096799999999999]\n",
      "0.6088167999999999\n"
     ]
    }
   ],
   "source": [
    "dict_mis, dict_not, aucs_capt = performances_report_10Fold(train_df, 'unimodal_captions', 0.5)\n",
    "print(mean_dict(dict_mis))\n",
    "print(mean_dict(dict_not))\n",
    "print(aucs_capt)\n",
    "print(np.mean(aucs_capt))\n",
    "\n",
    "print('___________ ON TEST _________________')\n",
    "dict_mis, dict_not, aucs_capt_tags = performances_report_10Fold(test_df, 'unimodal_captions', 0.5)\n",
    "print(mean_dict(dict_mis))\n",
    "print(mean_dict(dict_not))\n",
    "print(aucs_capt_tags)\n",
    "print(np.mean(aucs_capt_tags))\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 40,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'precision': 0.7980052490672436, 'recall': 0.8083822871176279, 'f1-score': 0.7984640654916466, 'support': 500.0}\n",
      "{'precision': 0.8114640485353879, 'recall': 0.7824559161298417, 'f1-score': 0.7879607760515374, 'support': 500.0}\n",
      "[0.8846158461704621, 0.9005384846361726, 0.8753596740822557, 0.878066785096985, 0.8736814725890355, 0.8882119999999999, 0.873187318731873, 0.8718655486390039, 0.8941315765263061, 0.8972467079333705]\n",
      "0.8836905414405465\n",
      "___________ ON TEST _________________\n",
      "{'precision': 0.6201033970417956, 'recall': 0.8033999999999999, 'f1-score': 0.6981329122169008, 'support': 500.0}\n",
      "{'precision': 0.7228968203502883, 'recall': 0.5029999999999999, 'f1-score': 0.5884439842887901, 'support': 500.0}\n",
      "[0.755618, 0.747358, 0.71622, 0.673264, 0.7353940000000001, 0.7223539999999999, 0.709852, 0.6665160000000001, 0.726496, 0.726712]\n",
      "0.7179784\n"
     ]
    }
   ],
   "source": [
    "dict_mis, dict_not, aucs_bert = performances_report_10Fold(train_df, 'model_BERT', 0.5)\n",
    "print(mean_dict(dict_mis))\n",
    "print(mean_dict(dict_not))\n",
    "print(aucs_bert)\n",
    "print(np.mean(aucs_bert))\n",
    "\n",
    "print('___________ ON TEST _________________')\n",
    "dict_mis, dict_not, aucs_bert_test = performances_report_10Fold(test_df, 'model_BERT', 0.5)\n",
    "print(mean_dict(dict_mis))\n",
    "print(mean_dict(dict_not))\n",
    "print(aucs_bert_test)\n",
    "print(np.mean(aucs_bert_test))"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Load Multimodal\n",
    "### Load Data MTC"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 43,
   "metadata": {},
   "outputs": [],
   "source": [
    "path_azure_train = './Multimodal/predictions/text_captions_pred_10Fold.csv'\n",
    "path_azure_test = './Multimodal/predictions/predictions_test/baseline_multimodal_text_captions_scores.tsv'\n",
    "path_azure_BO = './Mitigated/predictions/predictions'\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 44,
   "metadata": {},
   "outputs": [],
   "source": [
    "pred = pd.read_csv(path_azure_train, sep='\\t', usecols=['id', 'pred'])\n",
    "i = 0\n",
    "for fold in np.array_split(pred, 10):\n",
    "    pred.columns = ['file_name_'+str(i), MODELNAMES_MTC[i]]\n",
    "    train_df = pd.merge(train_df, pred, on='file_name_'+str(i))\n",
    "    i=i+1"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 45,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Load azure predictions on Test\n",
    "pred = pd.read_csv(path_azure_test, sep='\\t').drop(columns=label_column)\n",
    "test_df = pd.merge(test_df, pred, on=\"file_name\")"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### M_MTC "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 46,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Load azure BO predictions on Test\n",
    "for fname in glob.glob(path_azure_BO + '/BO_Final_Test_*.csv'):\n",
    "    pred = pd.read_csv(fname, sep='\\t').drop(columns=['misogynous'])\n",
    "\n",
    "    test_df = pd.merge(test_df, pred, on=\"file_name\")\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 45,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "Index(['file_name', 'Text Transcription', 'misogynous', 'shaming',\n",
       "       'stereotype', 'objectification', 'violence', 'model_VisualBERT_v0',\n",
       "       'model_VisualBERT_v1', 'model_VisualBERT_v2', 'model_VisualBERT_v3',\n",
       "       'model_VisualBERT_v4', 'model_VisualBERT_v5', 'model_VisualBERT_v6',\n",
       "       'model_VisualBERT_v7', 'model_VisualBERT_v8', 'model_VisualBERT_v9',\n",
       "       'model_BERT_v0', 'model_BERT_v1', 'model_BERT_v2', 'model_BERT_v3',\n",
       "       'model_BERT_v4', 'model_BERT_v5', 'model_BERT_v6', 'model_BERT_v7',\n",
       "       'model_BERT_v8', 'model_BERT_v9', 'unimodal_text_v0',\n",
       "       'unimodal_text_v1', 'unimodal_text_v2', 'unimodal_text_v3',\n",
       "       'unimodal_text_v4', 'unimodal_text_v5', 'unimodal_text_v6',\n",
       "       'unimodal_text_v7', 'unimodal_text_v8', 'unimodal_text_v9',\n",
       "       'unimodal_tags_v0', 'unimodal_tags_v1', 'unimodal_tags_v2',\n",
       "       'unimodal_tags_v3', 'unimodal_tags_v4', 'unimodal_tags_v5',\n",
       "       'unimodal_tags_v6', 'unimodal_tags_v7', 'unimodal_tags_v8',\n",
       "       'unimodal_tags_v9', 'unimodal_captions_v0', 'unimodal_captions_v1',\n",
       "       'unimodal_captions_v2', 'unimodal_captions_v3', 'unimodal_captions_v4',\n",
       "       'unimodal_captions_v5', 'unimodal_captions_v6', 'unimodal_captions_v7',\n",
       "       'unimodal_captions_v8', 'unimodal_captions_v9',\n",
       "       'multimodal_text_captions_v0', 'multimodal_text_captions_v1',\n",
       "       'multimodal_text_captions_v2', 'multimodal_text_captions_v3',\n",
       "       'multimodal_text_captions_v4', 'multimodal_text_captions_v5',\n",
       "       'multimodal_text_captions_v6', 'multimodal_text_captions_v7',\n",
       "       'multimodal_text_captions_v8', 'multimodal_text_captions_v9',\n",
       "       'multimodal_text_captions_BO_v0', 'multimodal_text_captions_BO_v1',\n",
       "       'multimodal_text_captions_BO_v2', 'multimodal_text_captions_BO_v3',\n",
       "       'multimodal_text_captions_BO_v4', 'multimodal_text_captions_BO_v5',\n",
       "       'multimodal_text_captions_BO_v6', 'multimodal_text_captions_BO_v7',\n",
       "       'multimodal_text_captions_BO_v8', 'multimodal_text_captions_BO_v9'],\n",
       "      dtype='object')"
      ]
     },
     "execution_count": 45,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "test_df.columns"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Load Data MTT"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 47,
   "metadata": {},
   "outputs": [],
   "source": [
    "path_clarifai_train = './Multimodal/predictions/text_tags_pred_10Fold.csv'\n",
    "path_clarifai_test = './Multimodal/predictions/predictions_test/baseline_multimodal_text_tags_scores.tsv'\n",
    "path_clarifai_BO = './Mitigated/predictions/predictions'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 48,
   "metadata": {},
   "outputs": [],
   "source": [
    "pred = pd.read_csv(path_clarifai_train, sep='\\t', usecols=['id', 'pred'])\n",
    "i = 0\n",
    "for fold in np.array_split(pred, 10):\n",
    "    pred.columns = ['file_name_'+str(i), MODELNAMES_MTT[i]]\n",
    "    train_df = pd.merge(train_df, pred, on='file_name_'+str(i))\n",
    "    i=i+1"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 49,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Load clarifai predictions on Test\n",
    "pred = pd.read_csv(path_clarifai_test, sep='\\t').drop(columns=label_column)\n",
    "test_df = pd.merge(test_df, pred, on=\"file_name\")"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### M_MTT"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Load clarifai BO predictions on Test\n",
    "for fname in glob.glob(path_clarifai_BO + '/BO_Final_Test_*.csv'):\n",
    "    print(fname)\n",
    "    pred = pd.read_csv(fname, sep='\\t').drop(columns=['misogynous'])\n",
    "\n",
    "    test_df = pd.merge(test_df, pred, on=\"file_name\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 57,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>file_name</th>\n",
       "      <th>Text Transcription</th>\n",
       "      <th>misogynous</th>\n",
       "      <th>shaming</th>\n",
       "      <th>stereotype</th>\n",
       "      <th>objectification</th>\n",
       "      <th>violence</th>\n",
       "      <th>model_VisualBERT_v0</th>\n",
       "      <th>model_VisualBERT_v1</th>\n",
       "      <th>model_VisualBERT_v2</th>\n",
       "      <th>...</th>\n",
       "      <th>multimodal_text_tags_BO_v0</th>\n",
       "      <th>multimodal_text_tags_BO_v1</th>\n",
       "      <th>multimodal_text_tags_BO_v2</th>\n",
       "      <th>multimodal_text_tags_BO_v3</th>\n",
       "      <th>multimodal_text_tags_BO_v4</th>\n",
       "      <th>multimodal_text_tags_BO_v5</th>\n",
       "      <th>multimodal_text_tags_BO_v6</th>\n",
       "      <th>multimodal_text_tags_BO_v7</th>\n",
       "      <th>multimodal_text_tags_BO_v8</th>\n",
       "      <th>multimodal_text_tags_BO_v9</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>15190.jpg</td>\n",
       "      <td>ASIAN GIRLS GMPIREHELENAHARPER.DEVIANTART.COM ...</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0.999905</td>\n",
       "      <td>0.996712</td>\n",
       "      <td>0.999973</td>\n",
       "      <td>...</td>\n",
       "      <td>0.985998</td>\n",
       "      <td>0.994967</td>\n",
       "      <td>0.989864</td>\n",
       "      <td>0.992748</td>\n",
       "      <td>0.996309</td>\n",
       "      <td>0.978714</td>\n",
       "      <td>0.986465</td>\n",
       "      <td>0.997046</td>\n",
       "      <td>0.982466</td>\n",
       "      <td>0.987795</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>15986.jpg</td>\n",
       "      <td>That Guy Really Likes This Show. VERY DEMOTIVA...</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0.999914</td>\n",
       "      <td>0.996296</td>\n",
       "      <td>0.999963</td>\n",
       "      <td>...</td>\n",
       "      <td>0.564997</td>\n",
       "      <td>0.708139</td>\n",
       "      <td>0.571399</td>\n",
       "      <td>0.504168</td>\n",
       "      <td>0.761264</td>\n",
       "      <td>0.740978</td>\n",
       "      <td>0.448315</td>\n",
       "      <td>0.730896</td>\n",
       "      <td>0.646367</td>\n",
       "      <td>0.326329</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>15005.jpg</td>\n",
       "      <td>A MAN WITH DREAMS... NEEDS A WOMAN WITH VISION</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0.263468</td>\n",
       "      <td>0.280405</td>\n",
       "      <td>0.687476</td>\n",
       "      <td>...</td>\n",
       "      <td>0.558833</td>\n",
       "      <td>0.633289</td>\n",
       "      <td>0.567085</td>\n",
       "      <td>0.444068</td>\n",
       "      <td>0.651713</td>\n",
       "      <td>0.610815</td>\n",
       "      <td>0.528171</td>\n",
       "      <td>0.598300</td>\n",
       "      <td>0.674171</td>\n",
       "      <td>0.329684</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>15923.jpg</td>\n",
       "      <td>CLAIMS TO LOVE CLEANING DOESN'T EVEN CLEAN DOT...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0.007080</td>\n",
       "      <td>0.007222</td>\n",
       "      <td>0.003045</td>\n",
       "      <td>...</td>\n",
       "      <td>0.073385</td>\n",
       "      <td>0.096423</td>\n",
       "      <td>0.007824</td>\n",
       "      <td>0.057170</td>\n",
       "      <td>0.069283</td>\n",
       "      <td>0.127752</td>\n",
       "      <td>0.122120</td>\n",
       "      <td>0.047632</td>\n",
       "      <td>0.156576</td>\n",
       "      <td>0.311017</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>15971.jpg</td>\n",
       "      <td>Women Women Women 0 Respect</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0.905814</td>\n",
       "      <td>0.996184</td>\n",
       "      <td>0.999928</td>\n",
       "      <td>...</td>\n",
       "      <td>0.953364</td>\n",
       "      <td>0.936189</td>\n",
       "      <td>0.984050</td>\n",
       "      <td>0.894438</td>\n",
       "      <td>0.884030</td>\n",
       "      <td>0.890672</td>\n",
       "      <td>0.928771</td>\n",
       "      <td>0.962340</td>\n",
       "      <td>0.927887</td>\n",
       "      <td>0.955298</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>995</th>\n",
       "      <td>15964.jpg</td>\n",
       "      <td>YEAH, I HAVE A GIRLFRIEND. BUT SHE'S BEING A B...</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0.567974</td>\n",
       "      <td>0.881868</td>\n",
       "      <td>0.079093</td>\n",
       "      <td>...</td>\n",
       "      <td>0.248196</td>\n",
       "      <td>0.416932</td>\n",
       "      <td>0.655247</td>\n",
       "      <td>0.200556</td>\n",
       "      <td>0.520801</td>\n",
       "      <td>0.399858</td>\n",
       "      <td>0.184549</td>\n",
       "      <td>0.351404</td>\n",
       "      <td>0.346384</td>\n",
       "      <td>0.322597</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>996</th>\n",
       "      <td>15990.jpg</td>\n",
       "      <td>PRAY FOR MY GIRLFRIEND SHE'S USED THE DOG SNAP...</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>0.992131</td>\n",
       "      <td>0.994005</td>\n",
       "      <td>0.999963</td>\n",
       "      <td>...</td>\n",
       "      <td>0.786092</td>\n",
       "      <td>0.963578</td>\n",
       "      <td>0.932848</td>\n",
       "      <td>0.867914</td>\n",
       "      <td>0.979822</td>\n",
       "      <td>0.960921</td>\n",
       "      <td>0.870016</td>\n",
       "      <td>0.948037</td>\n",
       "      <td>0.876826</td>\n",
       "      <td>0.930156</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>997</th>\n",
       "      <td>15461.jpg</td>\n",
       "      <td>HOW TO HAVE FUN WITH A MINOR Fung Step 3-&gt; Ste...</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>0.999908</td>\n",
       "      <td>0.996512</td>\n",
       "      <td>0.999978</td>\n",
       "      <td>...</td>\n",
       "      <td>0.619472</td>\n",
       "      <td>0.829204</td>\n",
       "      <td>0.753908</td>\n",
       "      <td>0.689598</td>\n",
       "      <td>0.899371</td>\n",
       "      <td>0.762517</td>\n",
       "      <td>0.757881</td>\n",
       "      <td>0.860606</td>\n",
       "      <td>0.686878</td>\n",
       "      <td>0.528091</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>998</th>\n",
       "      <td>16018.jpg</td>\n",
       "      <td>this could be us but you ain't here Calvin Kle...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0.853722</td>\n",
       "      <td>0.071663</td>\n",
       "      <td>0.121594</td>\n",
       "      <td>...</td>\n",
       "      <td>0.295897</td>\n",
       "      <td>0.345588</td>\n",
       "      <td>0.173995</td>\n",
       "      <td>0.155120</td>\n",
       "      <td>0.424296</td>\n",
       "      <td>0.275864</td>\n",
       "      <td>0.121296</td>\n",
       "      <td>0.176684</td>\n",
       "      <td>0.223003</td>\n",
       "      <td>0.076604</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>999</th>\n",
       "      <td>15228.jpg</td>\n",
       "      <td>LADIES THIS IS HOW TO TAKE A SEXY SELFIE</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0.998254</td>\n",
       "      <td>0.996356</td>\n",
       "      <td>0.999972</td>\n",
       "      <td>...</td>\n",
       "      <td>0.969435</td>\n",
       "      <td>0.991138</td>\n",
       "      <td>0.974067</td>\n",
       "      <td>0.980991</td>\n",
       "      <td>0.988289</td>\n",
       "      <td>0.962163</td>\n",
       "      <td>0.975330</td>\n",
       "      <td>0.996338</td>\n",
       "      <td>0.970535</td>\n",
       "      <td>0.987811</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>1000 rows × 108 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "     file_name                                 Text Transcription  misogynous  \\\n",
       "0    15190.jpg  ASIAN GIRLS GMPIREHELENAHARPER.DEVIANTART.COM ...           1   \n",
       "1    15986.jpg  That Guy Really Likes This Show. VERY DEMOTIVA...           1   \n",
       "2    15005.jpg     A MAN WITH DREAMS... NEEDS A WOMAN WITH VISION           0   \n",
       "3    15923.jpg  CLAIMS TO LOVE CLEANING DOESN'T EVEN CLEAN DOT...           0   \n",
       "4    15971.jpg                        Women Women Women 0 Respect           1   \n",
       "..         ...                                                ...         ...   \n",
       "995  15964.jpg  YEAH, I HAVE A GIRLFRIEND. BUT SHE'S BEING A B...           1   \n",
       "996  15990.jpg  PRAY FOR MY GIRLFRIEND SHE'S USED THE DOG SNAP...           1   \n",
       "997  15461.jpg  HOW TO HAVE FUN WITH A MINOR Fung Step 3-> Ste...           1   \n",
       "998  16018.jpg  this could be us but you ain't here Calvin Kle...           0   \n",
       "999  15228.jpg           LADIES THIS IS HOW TO TAKE A SEXY SELFIE           0   \n",
       "\n",
       "     shaming  stereotype  objectification  violence  model_VisualBERT_v0  \\\n",
       "0          0           1                1         0             0.999905   \n",
       "1          0           1                1         0             0.999914   \n",
       "2          0           0                0         0             0.263468   \n",
       "3          0           0                0         0             0.007080   \n",
       "4          0           1                0         1             0.905814   \n",
       "..       ...         ...              ...       ...                  ...   \n",
       "995        0           1                1         0             0.567974   \n",
       "996        1           1                1         1             0.992131   \n",
       "997        0           0                1         1             0.999908   \n",
       "998        0           0                0         0             0.853722   \n",
       "999        0           0                0         0             0.998254   \n",
       "\n",
       "     model_VisualBERT_v1  model_VisualBERT_v2  ...  \\\n",
       "0               0.996712             0.999973  ...   \n",
       "1               0.996296             0.999963  ...   \n",
       "2               0.280405             0.687476  ...   \n",
       "3               0.007222             0.003045  ...   \n",
       "4               0.996184             0.999928  ...   \n",
       "..                   ...                  ...  ...   \n",
       "995             0.881868             0.079093  ...   \n",
       "996             0.994005             0.999963  ...   \n",
       "997             0.996512             0.999978  ...   \n",
       "998             0.071663             0.121594  ...   \n",
       "999             0.996356             0.999972  ...   \n",
       "\n",
       "     multimodal_text_tags_BO_v0  multimodal_text_tags_BO_v1  \\\n",
       "0                      0.985998                    0.994967   \n",
       "1                      0.564997                    0.708139   \n",
       "2                      0.558833                    0.633289   \n",
       "3                      0.073385                    0.096423   \n",
       "4                      0.953364                    0.936189   \n",
       "..                          ...                         ...   \n",
       "995                    0.248196                    0.416932   \n",
       "996                    0.786092                    0.963578   \n",
       "997                    0.619472                    0.829204   \n",
       "998                    0.295897                    0.345588   \n",
       "999                    0.969435                    0.991138   \n",
       "\n",
       "     multimodal_text_tags_BO_v2  multimodal_text_tags_BO_v3  \\\n",
       "0                      0.989864                    0.992748   \n",
       "1                      0.571399                    0.504168   \n",
       "2                      0.567085                    0.444068   \n",
       "3                      0.007824                    0.057170   \n",
       "4                      0.984050                    0.894438   \n",
       "..                          ...                         ...   \n",
       "995                    0.655247                    0.200556   \n",
       "996                    0.932848                    0.867914   \n",
       "997                    0.753908                    0.689598   \n",
       "998                    0.173995                    0.155120   \n",
       "999                    0.974067                    0.980991   \n",
       "\n",
       "     multimodal_text_tags_BO_v4  multimodal_text_tags_BO_v5  \\\n",
       "0                      0.996309                    0.978714   \n",
       "1                      0.761264                    0.740978   \n",
       "2                      0.651713                    0.610815   \n",
       "3                      0.069283                    0.127752   \n",
       "4                      0.884030                    0.890672   \n",
       "..                          ...                         ...   \n",
       "995                    0.520801                    0.399858   \n",
       "996                    0.979822                    0.960921   \n",
       "997                    0.899371                    0.762517   \n",
       "998                    0.424296                    0.275864   \n",
       "999                    0.988289                    0.962163   \n",
       "\n",
       "     multimodal_text_tags_BO_v6  multimodal_text_tags_BO_v7  \\\n",
       "0                      0.986465                    0.997046   \n",
       "1                      0.448315                    0.730896   \n",
       "2                      0.528171                    0.598300   \n",
       "3                      0.122120                    0.047632   \n",
       "4                      0.928771                    0.962340   \n",
       "..                          ...                         ...   \n",
       "995                    0.184549                    0.351404   \n",
       "996                    0.870016                    0.948037   \n",
       "997                    0.757881                    0.860606   \n",
       "998                    0.121296                    0.176684   \n",
       "999                    0.975330                    0.996338   \n",
       "\n",
       "     multimodal_text_tags_BO_v8  multimodal_text_tags_BO_v9  \n",
       "0                      0.982466                    0.987795  \n",
       "1                      0.646367                    0.326329  \n",
       "2                      0.674171                    0.329684  \n",
       "3                      0.156576                    0.311017  \n",
       "4                      0.927887                    0.955298  \n",
       "..                          ...                         ...  \n",
       "995                    0.346384                    0.322597  \n",
       "996                    0.876826                    0.930156  \n",
       "997                    0.686878                    0.528091  \n",
       "998                    0.223003                    0.076604  \n",
       "999                    0.970535                    0.987811  \n",
       "\n",
       "[1000 rows x 108 columns]"
      ]
     },
     "execution_count": 57,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "test_df"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Multimodal Models Performances"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 51,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'precision': 0.7924292127226086, 'recall': 0.8050081474476366, 'f1-score': 0.7985970961113835, 'support': 500.0}\n",
      "{'precision': 0.8020151640176302, 'recall': 0.7892140886588701, 'f1-score': 0.7954875394761741, 'support': 500.0}\n",
      "[0.8755955214387718, 0.8792713442097888, 0.8597961413324049, 0.8544307342679726, 0.8580072028811524, 0.8829400000000001, 0.8777517751775177, 0.8745779392232482, 0.8950355801423207, 0.9005637826655836]\n",
      "0.875797002133876\n",
      "___________ ON TEST _________________\n",
      "{'precision': 0.6437151316854697, 'recall': 0.8130000000000001, 'f1-score': 0.7185151330423144, 'support': 500.0}\n",
      "{'precision': 0.7462800920688386, 'recall': 0.55, 'f1-score': 0.6332640340980895, 'support': 500.0}\n",
      "[0.7573920000000001, 0.756988, 0.760612, 0.755506, 0.757568, 0.756124, 0.7565999999999999, 0.759064, 0.7564759999999999, 0.7573359999999999]\n",
      "0.7573665999999999\n"
     ]
    }
   ],
   "source": [
    "dict_mis, dict_not, aucs_MTC = performances_report_10Fold(train_df, 'multimodal_text_captions', 0.5)\n",
    "print(mean_dict(dict_mis))\n",
    "print(mean_dict(dict_not))\n",
    "print(aucs_MTC)\n",
    "print(np.mean(aucs_MTC))\n",
    "\n",
    "print('___________ ON TEST _________________')\n",
    "dict_mis, dict_not, aucs_MTC_test = performances_report_10Fold(test_df, 'multimodal_text_captions', 0.5)\n",
    "print(mean_dict(dict_mis))\n",
    "print(mean_dict(dict_not))\n",
    "print(aucs_MTC_test)\n",
    "print(np.mean(aucs_MTC_test))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 52,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'precision': 0.8087227998245329, 'recall': 0.8179368909920657, 'f1-score': 0.8130977969070823, 'support': 500.0}\n",
      "{'precision': 0.8162932659680882, 'recall': 0.8064946867383267, 'f1-score': 0.8111487787697753, 'support': 500.0}\n",
      "[0.8869119288294378, 0.9037133420078071, 0.8755837825507545, 0.8705948722873056, 0.8678031212484995, 0.895912, 0.8937653765376538, 0.8862236162007329, 0.910935643742575, 0.9037608185051957]\n",
      "0.8895204501909962\n",
      "___________ ON TEST _________________\n",
      "{'precision': 0.6652883808284787, 'recall': 0.8601999999999999, 'f1-score': 0.7502815011995432, 'support': 500.0}\n",
      "{'precision': 0.8023076526515881, 'recall': 0.5672, 'f1-score': 0.6645414912859864, 'support': 500.0}\n",
      "[0.786016, 0.7865, 0.789364, 0.7844520000000001, 0.786752, 0.7861199999999999, 0.785284, 0.7863, 0.785856, 0.7852720000000001]\n",
      "0.7861916\n"
     ]
    }
   ],
   "source": [
    "dict_mis, dict_not, aucs_MTT = performances_report_10Fold(train_df, 'multimodal_text_tags', 0.5)\n",
    "print(mean_dict(dict_mis))\n",
    "print(mean_dict(dict_not))\n",
    "print(aucs_MTT)\n",
    "print(np.mean(aucs_MTT))\n",
    "\n",
    "print('___________ ON TEST _________________')\n",
    "dict_mis, dict_not, aucs_MTT_test = performances_report_10Fold(test_df, 'multimodal_text_tags', 0.5)\n",
    "print(mean_dict(dict_mis))\n",
    "print(mean_dict(dict_not))\n",
    "print(aucs_MTT_test)\n",
    "print(np.mean(aucs_MTT_test))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 41,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[0.766952, 0.7635419999999999, 0.782264, 0.776804, 0.77888, 0.764616, 0.761944, 0.791212, 0.777908, 0.764644]\n",
      "0.7728765999999999\n"
     ]
    }
   ],
   "source": [
    "aucs=[]\n",
    "for i in range(0,10):\n",
    " aucs.append(model_performances.calculate_overall_auc(test_df, MODELNAMES_VISUAL_BERT[i]))\n",
    "print(aucs)\n",
    "print(np.mean(aucs))\n",
    " "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 42,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'precision': 0.83602141849214, 'recall': 0.8515582014328711, 'f1-score': 0.8429938638787066, 'support': 500.0}\n",
      "{'precision': 0.849807312059599, 'recall': 0.830975666122783, 'f1-score': 0.8394333954232345, 'support': 500.0}\n",
      "[0.9294434599645587, 0.9291642478230407, 0.9078313903929502, 0.9003604922860253, 0.9096478591436574, 0.9202739999999999, 0.9257125712571257, 0.9132555087932663, 0.926191704766819, 0.9272764375657713]\n",
      "0.9189157671993214\n",
      "___________ ON TEST _________________\n",
      "{'precision': 0.6153716346246292, 'recall': 0.8926000000000001, 'f1-score': 0.72741779142959, 'support': 500.0}\n",
      "{'precision': 0.8071036672553221, 'recall': 0.438, 'f1-score': 0.5633462531122417, 'support': 500.0}\n",
      "[0.766952, 0.7635419999999999, 0.782264, 0.776804, 0.77888, 0.764616, 0.761944, 0.791212, 0.777908, 0.764644]\n",
      "0.7728765999999999\n"
     ]
    }
   ],
   "source": [
    "dict_mis, dict_not, aucs_vbert = performances_report_10Fold(train_df, 'model_VisualBERT', 0.5)\n",
    "print(mean_dict(dict_mis))\n",
    "print(mean_dict(dict_not))\n",
    "print(aucs_vbert)\n",
    "print(np.mean(aucs_vbert))\n",
    "\n",
    "print('___________ ON TEST _________________')\n",
    "dict_mis, dict_not, aucs_vbert_test = performances_report_10Fold(test_df, 'model_VisualBERT', 0.5)\n",
    "print(mean_dict(dict_mis))\n",
    "print(mean_dict(dict_not))\n",
    "print(aucs_vbert_test)\n",
    "print(np.mean(aucs_vbert_test))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 60,
   "metadata": {},
   "outputs": [],
   "source": [
    "test_df.to_csv(\"test_pred_perc.csv\", index=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 61,
   "metadata": {},
   "outputs": [],
   "source": [
    "train_df.to_csv(\"train_pred_perc.csv\", index=False)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## PLOT"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 53,
   "metadata": {},
   "outputs": [],
   "source": [
    "import matplotlib.pyplot as plt"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 54,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAiwAAAIjCAYAAAAgIUA8AAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjYuMCwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy89olMNAAAACXBIWXMAAA9hAAAPYQGoP6dpAABXmklEQVR4nO3deVyVZf7/8fc5KJsCpii44w6au0kuNWWWLWOZTZmKGqmplS1kpeWeqWO/HKc0bdGsdNKmbHc0o9E0twksrQA3GEwFt0FkF879+8Ovp06Aghy4b+D1fDx4jNzL5eecMXif674Wm2EYhgAAACzMbnYBAAAAl0NgAQAAlkdgAQAAlkdgAQAAlkdgAQAAlkdgAQAAlkdgAQAAllfD7ALcweFw6NixY/Lz85PNZjO7HAAAUAKGYejcuXNq1KiR7PZL96FUicBy7NgxNW3a1OwyAADAFThy5IiaNGlyyWuqRGDx8/OTdOEF+/v7m1wNAAAoifT0dDVt2tT5e/xSqkRgufgYyN/fn8ACAEAlU5LhHAy6BQAAlkdgAQAAlkdgAQAAlkdgAQAAlkdgAQAAlkdgAQAAlkdgAQAAlkdgAQAAlkdgAQAAlkdgAQAAlkdgAQAAlkdgAQAAlkdgAQAAllcldmsGAABSVlaW4uPjL3lNdna2kpKSFBISIh8fn0teGxoaKl9fX3eWeMUILAAAVBHx8fHq3r2729qLiYlRt27d3NZeWRBYAACoIkJDQxUTE3PJa+Li4hQREaFVq1YpLCzssu1ZBYEFAIBK4sCBAzp37lyF/X2Xe7zk5+enNm3aVEgtBBYAACqBAwcOqG3btm5rLyIiwi3t7N+/v0JCC4EFAIBK4GLPSkke5VxKaQbdXsrFR0sV1eNDYAEAoBIJCwsr80DYPn36uKmaisM6LAAAwPLoYQEAoBKw5eeoa7BdPmn7pWPm9zf4pO1X12C7bPk5FfL3EVgAAKgEvDOSFTuutvTtOOlbs6uRwiTFjqutuIxkSb3L/e8jsAAAUAnk1G6mbq9naPXq1QqzwPoocfHxGj58uJbf3qxC/j4CCwAAlYBRw1t7UhzKrtNWatTF7HKUneLQnhSHjBreFfL3mf8QDAAA4DLoYQEAoJKoEVBDSdlJ8j79W6+Gv6e/mvg1UW5Brg6lHSp0T/t67SVJiWcTlZ2f7XKuce3GCvAK0JmcM0rJTHE5V6tmLTX3b64CR4ES/pdQqN18I98dL6nECCwAAFQSdW+sqxkHZ0gHfzt2R8s7NP+6+UrNTNWQL4YUumffqH2SpKnfTdXek3tdzs3tO1cDWw3UxqSNmrtrrsu53o166/WbX1d2fnaR7b4a9qobXlHJEVgAAKgkzvz7jJZOWuqyKaG/p78kKahWkNb+eW2x987pM6fIHhZJGhAyQJ3rd3Y5V6tmLUmSTw2fItvNSMq4shdxhQgsAABUEvln8xXiE+J8zPN7Xh5eRR6/qEVAi2LP1fWuq7redYs852H3KLLd2P/GlqBi92HQLQAAsDwCCwAAsDwCCwAAsDwCCwAAsDwCCwAAsDwCCwAAsDymNQOABWVlZSk+Pv6S12RnZyspKUkhISHy8fG55LWhoaHy9fV1Z4moYFlZWZKk2NiyTScuzb+bS4mLiytTHaVFYAEAC4qPj1f37t3d1l5MTIy6devmtvZQ8S4G2LFjx5pciSs/P78K+XsILABgQaGhoYqJibnkNXFxcYqIiNCqVasUFhZ22fZQuQ0aNEhS2XvLSvPv5nL8/PzUpk2bMrVRUgQWALAgX1/fEveIhIWF0XtSDQQGBmrMmDFua6+y/bth0C0AALA8AgsAALA8AgsAALA8AgsAALA8AgsAALA8AgsAALC8KwosS5YsUUhIiLy9vRUeHq7du3cXe+358+c1e/ZstWrVSt7e3urcubM2bNhQpjYBAED1Uup1WNauXauoqCgtW7ZM4eHhWrRokQYMGKCEhAQ1aNCg0PVTp07VqlWr9Oabbyo0NFQbN27U3Xffre3bt6tr165X1CYAVAUHDhzQuXPnrvj+i0uju2OJ9IpcAAy4EjbDMIzS3BAeHq5rrrlGixcvliQ5HA41bdpUEydO1OTJkwtd36hRIz3//PN65JFHnMfuuece+fj4aNWqVVfU5h+lp6crICBAZ8+elb+/f2leDgCTXW7PnKq6X86BAwfUtm1bs8twsX///koRWthnqWxiY2PVvXt3S2zXUJrf36XqYcnLy1NMTIymTJniPGa329W/f3/t2LGjyHtyc3Pl7e3tcszHx0fbtm0rU5u5ubnO79PT00vzMgBYiDv3zLHCD+CSutizUpbl0d25iV1ERESZensqEvssFa8kYa40PXNWCnOlCiynTp1SQUGBgoKCXI4HBQUV+wYNGDBACxcu1PXXX69WrVopOjpa69atU0FBwRW3OW/ePM2aNas0pQOwqMvtmVPV98sp6/Loffr0cWM1lQP7LBWvNGEuIiListdYKcyV+15Cf//73zV27FiFhobKZrOpVatWioyM1IoVK664zSlTpigqKsr5fXp6upo2beqOcgFUsJLumVPZ9j1B+WGfpeKVJMyV9nGZVZQqsAQGBsrDw0Opqakux1NTUxUcHFzkPfXr19cnn3yinJwcnT59Wo0aNdLkyZPVsmXLK27Ty8tLXl5epSkdAIAqr6RhrjL2zJVqWrOnp6e6d++u6Oho5zGHw6Ho6Gj16tXrkvd6e3urcePGys/P10cffaS77rqrzG0CAIDqodSPhKKiojRq1Cj16NFDPXv21KJFi5SZmanIyEhJ0siRI9W4cWPNmzdPkrRr1y4dPXpUXbp00dGjRzVz5kw5HA4988wzJW4TAABUb6UOLEOGDNHJkyc1ffp0paSkqEuXLtqwYYNz0GxycrLs9t86bnJycjR16lQdPnxYtWvX1u2336733ntPderUKXGbAACgeruiQbePPvqoHn300SLPbd682eX7P/3pT/rll1/K1CYAAKje2EsIAABYXrlPawZQvbH8PAB3ILAAKDfuXH6+JItclURlWX4egCsCC4Byw/LzANyFwAKg3LH8fGG2/Bx1DbbLJ22/dMzc4YQ+afvVNdguW36OqXUAl0JgAQATeGcka+PD/jq581H9svO34/4Oh5rkFyjXJh2qWbPQfe3zzkuSEmvWULbN5nKucX6BAhwOnbHblVLDw+VcLYeh5vn5KpCU4Fm43V3jautgRrKk3mV+bUB5ILAAgAlyajdT//1ecvSv73L8juDemt9xglKzUjXku0mF7tt383uSpKm7Z2nv2YMu5+ZePV4DG/bRxiObNDf+XZdzvet11OvdnlF2fraG/PuhQu3WeCFBK25vVtaXBZQbAgsAmMCo4a1fPj+tteNfc9lgzt/TX/JroqCCXK29am3hG+u1lyTNuXGhsvOzXU41rt1Y8grQgLrN1Ln1HS7natWsJfk3l4+jQGv/7NpufHy87km6R0YNbze9OsD9CCwAYJL8s/kK8QlR+/8LIb/n5eFV5PGLWgS0KPZcXe+6qutdt8hzHnaPQu3m+ORIBSUsGjAJC8cBAADLI7AAAADLI7AAAADLYwwLgHJVI6CGkrKT5H36twGd/p7+auLXRLkFuTqUdqjQPRfHWCSeTSxyYGmAV4DO5JxRSmaKy7laNWupuX9zFTgKlPC/BJdzSdlJkutMXwCVCIEFQLmqe2NdzTg4Q/rdDNw7Wt6h+dfNV2pmqoZ8MaTQPftG7ZMkTf1uqvae3Otybm7fuRrYaqA2Jm3U3F1zXc71btRbr9/8+oWpu0W06+FLYgEqKwILgHJ15t9ntHTS0sJTdyUF1QoqNMX29+b0mVP01F1JA0IGqHP9zi7natWsJUnyqeFT9NTdrHuu/IWgwpR1w0yJTTOrIgILgHJjy89RRx+HwnLzFJab99uJ3FPSuVPyklTkxN1jP0iSipy4m5soSar7f1+u5/KkjP/Jo4h2bbl56lqf5eetzp0bZkpsmlmVEFgAlBvvjGTFjqstfTtO+tbcWsIkxY6rrTiWn7c0d2yYKbFpZlVEYCmlrKwsxcfHX/Ka0vyHEhoaKl9fX3eWCFhGTu1m6vZ6hlavXq2w3z0SMkNcfLyGDx+u5Sw/XymUdcNMqWpumlmdEVhKKT4+Xt27d3dbezExMWX+jxKwKqOGt/akOJRdp63UqIuptWSnOLQnxcHy80AlRWAppdDQUMXExFzymotdiCXp0gw1+VMnAHNkZWVJkmJjY6+4DXc+9gCsjsBSSr6+viXuEXFHlyaAqunio+WxY8eaXMlv/Pz8zC4BKBaBBQBMMGjQIEllG8dWmt7cy7HK1F1bfo66Btvlk7ZfOmb+Yuw+afvVNZjZZVZAYAEAEwQGBmrMmDFuaasq9eZ6ZyRr48P+OrnzUf2y87fj/g6HmuQXKNcmHapZs9B97fPOS5ISa9ZQts3mcq5xfoECHA6dsduVUsN18cBaDkPN8/NVICnBs3C7bfLOM7vMIggsAADLyKndTP33e8nRv77L8TuCe2t+xwlKzUrVkO8mFbpv383vSZKm7p6lvWcPupybe/V4DWzYRxuPbNLc+HddzvWu11Gvd3vmwurI/36oULtvNXpCE0Y+xOwyCyCwACg3DCxFaRk1vPXL56e1dvxrhVdH9muioIJcrb2qiNWR/2//qTk3Lix6dWSvAA2o20ydW9/hcq5WzVqSf3P5OAqKXHU5IymD2WUWQWABUG4YWIorkX82XyE+Ic5NMH/Py8OryOMXtQgocn1kSVJd77qq611ofWRJkofdo8h2Y/975WEb7kVgAVBuSjKw9GIPSnESExM1bdo0vfDCC2rRovhfRpIu2wtjlYGlAEqPwAKg3JRkYGlsbGyJ9nuZNm3aZa9hIUag6iKwADDV5RZjLO1WFwCqJgILAFOVZDFG9oQBYP6qPAAAAJdBDwsAWFBJdoa/OFW7JFO22RkelR2BBQAsqDQ7w5dk0DIDklHZEVj+4MCBAzp37lyZ2ijNp57LYRomUD2VZGd4BiSjOiGw/M6BAwfUtm1bt7VXkk89JbF//35CC1DNlHRneAYko7ogsPzOxZ6Vsu586s6lxCMiIsrc4wMAQGVHYCmCO3Y+5VMPAADuQ2ABAFiGOzbMlNg0syoisAAALMOKG2ZKbJppBQQWAIBllGTDzJK4OAawrGMSJWZrWgWBBQBgGSXZMLM03DEmEdbA0vwAAMDyCCwAAMDyCCwAAMDyCCwAAMDyCCwAAMDyCCwAAMDyCCwAAMDyWIcFAFCpZGVlOVfELc7FJfVLsrR+WRepQ8UgsPxBjYAaSspOkvdpb+cxf09/NfFrotyCXB1KO1Tonvb12kuSEs8mKjs/2+Vc49qNFeAVoDM5Z5SSmeJyrlbNWmru31wFjgIl/C+hULv5Rr47XhIAVCnx8fHq3r17ia6NiIi47DUxMTEsLlcJEFj+oO6NdTXj4Azp4G/H7mh5h+ZfN1+pmaka8sWQQvfsG7VPkjT1u6nae3Kvy7m5fedqYKuB2pi0UXN3zXU517tRb71+8+vKzs8ust1Xw151wysCgKolNDRUMTExl7ymNJsfhoaGurM8lBObYRhGaW9asmSJXnrpJaWkpKhz58569dVX1bNnz2KvX7RokZYuXark5GQFBgbqL3/5i+bNmydv7wu9GDNnztSsWbNc7mnXrt1lu/wuSk9PV0BAgM6ePSt/f//SvhynPbu368GIGzXnlflqEdLCedy/pq+a+DRQbkGeDmUeK3Rfe/8QSVJi5nFlF+S6nGvsE6iAmrV1Ji9dKTlnXM7V8vBW81rBKjAcSjiXXKjdgqOZGhUxUss/3aquPXtf8esCAMCKSvP7u9Q9LGvXrlVUVJSWLVum8PBwLVq0SAMGDFBCQoIaNGhQ6Pp//OMfmjx5slasWKHevXtr//79euCBB2Sz2bRw4ULndR06dNDXX3/9W2E1Kr7zxzsjWXuGeUs7Z0o7C5/3ktT+Eve3uMS5uv/3VRSPS7QbO6624jKSJRFYAADVV6lTwcKFCzV27FhFRkZKkpYtW6Yvv/xSK1as0OTJkwtdv337dvXp00fDhg2TJIWEhGjo0KHatWuXayE1aig4OLhENeTm5io397eejPT09NK+jCLl1G6mbq9naPXq1QqzQBdhXHy8hg8fruW3NzO7FAAATFWqwJKXl6eYmBhNmTLFecxut6t///7asWNHkff07t1bq1at0u7du9WzZ08dPnxY69ev14gRI1yuO3DggBo1aiRvb2/16tVL8+bNU7NmRf+injdvXqFHSO5g1PDWnhSHsuu0lRp1cXv7pZWd4tCeFIeMGt6XvxgAgCqsVOuwnDp1SgUFBQoKCnI5HhQUpJSUlCLvGTZsmGbPnq2+ffuqZs2aatWqlW644QY999xzzmvCw8O1cuVKbdiwQUuXLlViYqKuu+46nTt3rsg2p0yZorNnzzq/jhw5UpqXAQAAKplyXzhu8+bNmjt3rl577TXFxsZq3bp1+vLLL/XCCy84r7ntttt07733qlOnThowYIDWr1+vtLQ0ffDBB0W26eXlJX9/f5cvAABQdZXqkVBgYKA8PDyUmprqcjw1NbXY8SfTpk3TiBEjNGbMGElSx44dlZmZqYceekjPP/+87PbCmalOnTpq27atDh48WOgcAACofkrVw+Lp6anu3bsrOjraeczhcCg6Olq9evUq8p6srKxCocTDw0OSVNyM6oyMDB06dEgNGzYsTXkAAKCKKvUsoaioKI0aNUo9evRQz549tWjRImVmZjpnDY0cOVKNGzfWvHnzJEkDBw7UwoUL1bVrV4WHh+vgwYOaNm2aBg4c6AwukyZN0sCBA9W8eXMdO3ZMM2bMkIeHh4YOHerGlwoAACqrUgeWIUOG6OTJk5o+fbpSUlLUpUsXbdiwwTkQNzk52aVHZerUqbLZbJo6daqOHj2q+vXra+DAgXrxxRed1/z6668aOnSoTp8+rfr166tv377auXOn6tev74aXCAAAKrsrWunWaty10m1sbKy6d+9umX0lrFYPAADuVJrf3+U+SwgAAKCsCCwAAMDyCCwAAMDyKn6HQQvLysqSdGHsSFmUZlvzS4mLiytTHQAAVBUElt+Jj4+XJI0dO9bkSlz5+fmZXQIAAKYisPzOoEGDJEmhoaHy9fW94nbi4uIUERGhVatWKSwsrEw1+fn5qU2bNmVqAwCAyo7A8juBgYHOLQTcISwsjOnIAAC4AYNuAQCA5RFYAACA5RFYAACA5RFYAACA5RFYAACA5RFYAACA5RFYAACA5RFYAACA5RFYAACA5bHSbSllZWU59xwqzsVNC0uyeWFZtwEAAKA6ILCUUnx8vLp3716iayMiIi57TUxMDMv3AwBwGQSWUgoNDVVMTMwlr8nOzlZSUpJCQkLk4+Nz2fYAAMCl2QzDMMwuoqzS09MVEBCgs2fPyt/f3+xyAABACZTm9zeDbgEAgOURWAAAgOURWAAAgOURWAAAgOURWAAAgOURWAAAgOURWAAAgOURWAAAgOURWAAAgOURWAAAgOURWAAAgOURWAAAgOURWAAAgOURWAAAgOURWAAAgOURWAAAgOURWAAAgOURWAAAgOURWAAAgOURWAAAgOURWAAAgOURWAAAgOURWAAAgOURWAAAgOURWAAAgOURWAAAgOURWAAAgOURWAAAgOURWAAAgOURWAAAgOVdUWBZsmSJQkJC5O3trfDwcO3evfuS1y9atEjt2rWTj4+PmjZtqieffFI5OTllahMAAFQfpQ4sa9euVVRUlGbMmKHY2Fh17txZAwYM0IkTJ4q8/h//+IcmT56sGTNmKC4uTsuXL9fatWv13HPPXXGbAACgerEZhmGU5obw8HBdc801Wrx4sSTJ4XCoadOmmjhxoiZPnlzo+kcffVRxcXGKjo52Hnvqqae0a9cubdu27Yra/KP09HQFBATo7Nmz8vf3L83LAQAAJinN7+9S9bDk5eUpJiZG/fv3/60Bu139+/fXjh07irynd+/eiomJcT7iOXz4sNavX6/bb7/9itvMzc1Venq6yxcAAKi6apTm4lOnTqmgoEBBQUEux4OCghQfH1/kPcOGDdOpU6fUt29fGYah/Px8jR8/3vlI6EranDdvnmbNmlWa0gEAQCVW7rOENm/erLlz5+q1115TbGys1q1bpy+//FIvvPDCFbc5ZcoUnT171vl15MgRN1YMAACsplQ9LIGBgfLw8FBqaqrL8dTUVAUHBxd5z7Rp0zRixAiNGTNGktSxY0dlZmbqoYce0vPPP39FbXp5ecnLy6s0pQMAgEqsVD0snp6e6t69u8sAWofDoejoaPXq1avIe7KysmS3u/41Hh4ekiTDMK6oTQAAUL2UqodFkqKiojRq1Cj16NFDPXv21KJFi5SZmanIyEhJ0siRI9W4cWPNmzdPkjRw4EAtXLhQXbt2VXh4uA4ePKhp06Zp4MCBzuByuTYBAED1VurAMmTIEJ08eVLTp09XSkqKunTpog0bNjgHzSYnJ7v0qEydOlU2m01Tp07V0aNHVb9+fQ0cOFAvvvhiidsEAADVW6nXYbEi1mEBAKDyKbd1WAAAAMxAYAEAAJZHYAEAAJZHYAEAAJZHYAEAAJZHYAEAAJZHYAEAAJZHYAEAAJZHYAEAAJZHYAEAAJZHYAEAAJZHYAEAAJZHYAEAAJZHYAEAAJZHYAEAAJZHYAEAAJZHYAEAAJZHYAEAAJZHYAEAAJZHYAEAAJZHYAEAAJZHYAEAAJZHYAEAAJZHYAEAAJZHYAEAAJZHYAEAAJZHYAEAAJZHYAEAAJZHYAEAAJZHYAEAAJZHYAEAAJZHYAEAAJZHYAEAAJZHYAEAAJZHYAEAAJZHYAEAAJZHYAEAAJZHYAEAAJZHYAEAAJZHYAEAAJZHYAEAAJZHYAEAAJZHYAEAAJZHYAEAAJZHYAEAAJZHYAEAAJZHYAEAAJZHYAEAAJZHYAEAAJZHYAEAAJZ3RYFlyZIlCgkJkbe3t8LDw7V79+5ir73hhhtks9kKfd1xxx3Oax544IFC52+99dYrKQ0AAFRBNUp7w9q1axUVFaVly5YpPDxcixYt0oABA5SQkKAGDRoUun7dunXKy8tzfn/69Gl17txZ9957r8t1t956q95++23n915eXqUtDQAAVFGl7mFZuHChxo4dq8jISLVv317Lli2Tr6+vVqxYUeT1devWVXBwsPNr06ZN8vX1LRRYvLy8XK676qqrruwVAQCAKqdUgSUvL08xMTHq37//bw3Y7erfv7927NhRojaWL1+u+++/X7Vq1XI5vnnzZjVo0EDt2rXThAkTdPr06WLbyM3NVXp6ussXAACoukoVWE6dOqWCggIFBQW5HA8KClJKSspl79+9e7d++uknjRkzxuX4rbfeqnfffVfR0dH661//qi1btui2225TQUFBke3MmzdPAQEBzq+mTZuW5mUAAIBKptRjWMpi+fLl6tixo3r27Oly/P7773f+uWPHjurUqZNatWqlzZs366abbirUzpQpUxQVFeX8Pj09ndACAEAVVqoelsDAQHl4eCg1NdXleGpqqoKDgy95b2ZmptasWaPRo0df9u9p2bKlAgMDdfDgwSLPe3l5yd/f3+ULAABUXaUKLJ6enurevbuio6OdxxwOh6Kjo9WrV69L3vvPf/5Tubm5ioiIuOzf8+uvv+r06dNq2LBhacoDAABVVKlnCUVFRenNN9/UO++8o7i4OE2YMEGZmZmKjIyUJI0cOVJTpkwpdN/y5cs1aNAg1atXz+V4RkaGnn76ae3cuVNJSUmKjo7WXXfdpdatW2vAgAFX+LIAAEBVUuoxLEOGDNHJkyc1ffp0paSkqEuXLtqwYYNzIG5ycrLsdtcclJCQoG3btumrr74q1J6Hh4f27t2rd955R2lpaWrUqJFuueUWvfDCC6zFAgAAJEk2wzAMs4soq/T0dAUEBOjs2bOMZwEAoJIoze9v9hICAACWR2ABAACWR2ABAACWR2ABAACWR2ABAACWR2ABAACWR2ABAACWR2ABAACWR2ABAACWR2ABAACWR2ABAACWR2ABAACWR2ABAACWR2ABAACWR2ABAACWR2ABAACWR2ABAACWV8PsAlB1ZGVlKT4+/pLXZGdnKykpSSEhIfLx8bnktaGhofL19XVniQCASorAAreJj49X9+7d3dZeTEyMunXr5rb2AACVF4EFbhMaGqqYmJhLXhMXF6eIiAitWrVKYWFhl20PAACJwAI38vX1LXGPSFhYGL0nAIASY9AtAACwPAILAACwPAILAACwPAILAACwPAILAACwPAILAACwPAILAACwPAILAACwPAILAACwPFa6RYkdOHBA586dK1MbcXFxLv97pfz8/NSmTZsytQEAqDwILCiRAwcOqG3btm5rLyIiosxt7N+/n9ACANUEgQUlcrFnpSSbFl5Kdna2kpKSFBISIh8fnytq4+IGimXt7QEAVB4EFpSKOzYt7NOnj5uqAQBUFwy6BQAAlkdgAQAAlkdgAQAAlkdgAQAAlkdgAQAAlkdgAQAAlkdgAQAAlkdgAQAAlsfCcSixGgE1lJSdJO/T3s5j/p7+auLXRLkFuTqUdqjQPe3rtZckJZ5NVHZ+tsu5xrUbK8ArQGdyziglM8XlXK2atdTcv7kKHAVK+F+Cy7mk7CTJw00vCgBQKRBYUGJ1b6yrGQdnSAd/O3ZHyzs0/7r5Ss1M1ZAvhhS6Z9+ofZKkqd9N1d6Te13Oze07VwNbDdTGpI2au2uuy7nejXrr9ZtfV3Z+dpHteviSWACgOiGwoMTO/PuMlk5aqtDQUOcxf09/SVJQrSCt/fPaYu+d02dOkT0skjQgZIA61+/scq5WzVqSJJ8aPoXajY+P1z1Z91z5CwEAVDoEFpRY/tl8hfiEOB/z/J6Xh1eRxy9qEdCi2HN1veuqrnfdIs952D0KtZvjkyMVlLBoAECVwKBbAABgeQQWAABgeTwSQonY8nPUNdgun7T90jFzc65P2n51DbbLlp9jah0AgIpDYEGJeGckK3ZcbenbcdK35tYSJil2XG3FZSRL6m1uMQCACkFgQYnk1G6mbq9naPXq1Qr73SwhM8TFx2v48OFafnszU+sAAFScKwosS5Ys0UsvvaSUlBR17txZr776qnr27FnktTfccIO2bNlS6Pjtt9+uL7/8UpJkGIZmzJihN998U2lpaerTp4+WLl2qNm3aXEl5KAdGDW/tSXEou05bqVEXU2vJTnFoT4pDRg3vy18MAKgSSj0YYe3atYqKitKMGTMUGxurzp07a8CAATpx4kSR169bt07Hjx93fv3000/y8PDQvffe67xmwYIFeuWVV7Rs2TLt2rVLtWrV0oABA5STwxgFAABwBYFl4cKFGjt2rCIjI9W+fXstW7ZMvr6+WrFiRZHX161bV8HBwc6vTZs2ydfX1xlYDMPQokWLNHXqVN11113q1KmT3n33XR07dkyffPJJkW3m5uYqPT3d5QsAAFRdpXoklJeXp5iYGE2ZMsV5zG63q3///tqxY0eJ2li+fLnuv/9+1ap1YSXTxMREpaSkqH///s5rAgICFB4erh07duj+++8v1Ma8efM0a9as0pSOMsrKypIkxcbGlqmd7OxsJSUlKSQkRD4+PlfURlxcXJlqAABUPqUKLKdOnVJBQYGCgoJcjgcFBSk+Pv6y9+/evVs//fSTli9f7jyWkpLibOOPbV4890dTpkxRVFSU8/v09HQ1bdq0xK8DpXfx/9+xY8eaXMlv/Pz8zC4BAFBBKnSW0PLly9WxY8diB+iWlJeXl7y8vNxUFUpi0KBBkqTQ0FD5+vpecTtxcXGKiIjQqlWrFBYWdsXt+Pn5MSgbAKqRUgWWwMBAeXh4KDU11eV4amqqgoODL3lvZmam1qxZo9mzZ7scv3hfamqqGjZs6NJmly5dSlMeylFgYKDGjBnjtvbCwsLUrVs3t7UHAKjaShVYPD091b17d0VHRzs/cTscDkVHR+vRRx+95L3//Oc/lZubq4iICJfjLVq0UHBwsKKjo50BJT09Xbt27dKECRNKUx5gWVlZWZd9bFrS8T1l7eUCgMqo1I+EoqKiNGrUKPXo0UM9e/bUokWLlJmZqcjISEnSyJEj1bhxY82bN8/lvuXLl2vQoEGqV6+ey3GbzaYnnnhCc+bMUZs2bdSiRQtNmzZNjRo1coYioLKLj49X9+7d3dJWTEwMvVMAqp1SB5YhQ4bo5MmTmj59ulJSUtSlSxdt2LDBOWg2OTlZdrvrbOmEhARt27ZNX331VZFtPvPMM8rMzNRDDz2ktLQ09e3bVxs2bJC3NwuDoWoIDQ1VTEzMJa8p6fieUJNXGgYAM9gMwzDMLqKs0tPTFRAQoLNnz8rf39/scnAJsbGx6t69O70EReC9AVDdlOb3N3sJwW1KMk7j4hoqJVlLhbEaAICLCCxwm9KM0/jj4Oui0NMAALiIwAK3Kck4jdKsdMtYDQDARQQWuI2vr2+JekT69OlTAdUAAKqSUm9+CAAAUNEILAAAwPIILAAAwPIILAAAwPIYdAu4wYEDB3Tu3LkytVGaNWouhZ2sAVRFBBagjA4cOKC2bdu6rb2SrFFzOfv37ye0AKhSCCxAGV3sWbncHkCXU5o1aopzcT+isvb2AIDVEFgANwkLCyvzyrysUQMARWPQLQAAsDwCCwAAsDwCCwAAsDwCCwAAsDwCCwAAsDwCCwAAsDwCCwAAsDwCCwAAsDwCCwAAsDwCCwAAsDyW5gfcoEZADSVlJ8n7tLfzmL+nv5r4NVFuQa4OpR0qdE/7eu0lSYlnE5Wdn+1yrnHtxgrwCtCZnDNKyUxxOVerZi0192+uAkeBEv6X4HIuKTtJ8nDTiwIACyGwAG5Q98a6mnFwhnTwt2N3tLxD86+br9TMVA35Ykihe/aN2idJmvrdVO09udfl3Ny+czWw1UBtTNqoubvmupzr3ai3Xr/5dWXnZxfZrocviQVA1UNgAcrIlp+jRj+nac64yWoR0sJ53L+mr3TsBwUV5Glt+AuFbzz2gyRpTpsIZbfMdTnVuMZV0rEfNMC7sTr/4d5aHt7SsR/kYzgKtZuYlKipfpNky89xz4sDAIsgsABl5J2RrD3DvKWdM6Wdhc97SWp/iftbXOJc3f/7KopHEe22l3TH2NqKy0iW1PsSLQNA5UJgAcoop3YzdXs9Q6tXr1ZYaKiptcTFx2v48OFafnszU+sAAHcjsABlZNTw1p4Uh7LrtJUadTG1luwUh/akOGTU8L78xQBQiTCtGQAAWB6BBQAAWB6BBQAAWB6BBQAAWB6BBQAAWB6BBQAAWB6BBQAAWB6BBQAAWB4LxwFllJWVJUmKjY0tUzvZ2dlKSkpSSEiIfHx8rqiNuLi4MtUAAFZFYAHKKD4+XpI0duxYkyv5jZ+fn9klAIBbEViAMho0aJAkKTQ0VL6+vlfcTlxcnCIiIrRq1SqFhYVdcTt+fn5q06bNFd8PAFZEYAHKKDAwUGPGjHFbe2FhYerWrZvb2gOAqoBBtwAAwPIILAAAwPIILAAAwPIILAAAwPIILAAAwPIILAAAwPKY1gxUgKysLOcCc8W5uErt5VarLet6LwBQGRFYgAoQHx+v7t27l+jaiIiIS56PiYlhnRYA1Q6BBagAoaGhiomJueQ1Jd1LKDQ01N3lAYDl2QzDMMwuoqzS09MVEBCgs2fPyt/f3+xyAABACZTm9/cVDbpdsmSJQkJC5O3trfDwcO3evfuS16elpemRRx5Rw4YN5eXlpbZt22r9+vXO8zNnzpTNZnP54lMkAAC4qNSPhNauXauoqCgtW7ZM4eHhWrRokQYMGKCEhAQ1aNCg0PV5eXm6+eab1aBBA3344Ydq3Lix/vvf/6pOnTou13Xo0EFff/31b4XV4GkVAAC4oNSpYOHChRo7dqwiIyMlScuWLdOXX36pFStWaPLkyYWuX7Fihc6cOaPt27erZs2akqSQkJDChdSooeDg4NKWAwAAqoFSPRLKy8tTTEyM+vfv/1sDdrv69++vHTt2FHnPZ599pl69eumRRx5RUFCQrr76as2dO1cFBQUu1x04cECNGjVSy5YtNXz4cCUnJxdbR25urtLT012+AABA1VWqwHLq1CkVFBQoKCjI5XhQUJBSUlKKvOfw4cP68MMPVVBQoPXr12vatGl6+eWXNWfOHOc14eHhWrlypTZs2KClS5cqMTFR1113nc6dO1dkm/PmzVNAQIDzq2nTpqV5GQAAoJIp94EiDodDDRo00BtvvCEPDw91795dR48e1UsvvaQZM2ZIkm677Tbn9Z06dVJ4eLiaN2+uDz74QKNHjy7U5pQpUxQVFeX8Pj09ndACAEAVVqrAEhgYKA8PD6WmprocT01NLXb8ScOGDVWzZk15eHg4j4WFhSklJUV5eXny9PQsdE+dOnXUtm1bHTx4sMg2vby85OXlVZrSAQBAJVaqR0Kenp7q3r27oqOjncccDoeio6PVq1evIu/p06ePDh48KIfD4Ty2f/9+NWzYsMiwIkkZGRk6dOiQGjZsWJryAABAFVXqdViioqL05ptv6p133lFcXJwmTJigzMxM56yhkSNHasqUKc7rJ0yYoDNnzujxxx/X/v379eWXX2ru3Ll65JFHnNdMmjRJW7ZsUVJSkrZv3667775bHh4eGjp0qBteIgAAqOxKPYZlyJAhOnnypKZPn66UlBR16dJFGzZscA7ETU5Olt3+Ww5q2rSpNm7cqCeffFKdOnVS48aN9fjjj+vZZ591XvPrr79q6NChOn36tOrXr6++fftq586dql+/vhteIgAAqOxYmh8AAJii3JfmBwAAqEgEFgAAYHlVYsOei0+1WPEWAIDK4+Lv7ZKMTqkSgeXiirgsHgcAQOVz7tw5BQQEXPKaKjHo1uFw6NixY/Lz85PNZjO7HOfKu0eOHGEQ8B/w3hSP96ZovC/F470pHu9N8az03hiGoXPnzqlRo0YuM4yLUiV6WOx2u5o0aWJ2GYX4+/ub/o/Bqnhvisd7UzTel+Lx3hSP96Z4VnlvLtezchGDbgEAgOURWAAAgOURWMqBl5eXZsyYwQaNReC9KR7vTdF4X4rHe1M83pviVdb3pkoMugUAAFUbPSwAAMDyCCwAAMDyCCwAAMDyCCwAAMDyCCwALMHDw0MnTpwwuwwAFlUlVro1w+DBg0t87bp168qxEuvZu3dvia/t1KlTOVZiPZ999lmJr73zzjvLsRLrYcIi4F4eHh46fvy4GjRoYHYpbkFguUK/X0rYMAx9/PHHCggIUI8ePSRJMTExSktLK1WwqSq6dOkim80mwzAuu7dTQUFBBVVlDYMGDXL5/uL79PvvL6pu7w0ur1+/flq3bp3q1KljdimoBKrahwACyxV6++23nX9+9tlndd9992nZsmXy8PCQdOGXzcMPP2yJfRoqWmJiovPPe/bs0aRJk/T000+rV69ekqQdO3bo5Zdf1oIFC8wq0TQOh8P556+//lrPPvus5s6d6/LeTJ06VXPnzjWrRFO99dZbql279iWveeyxxyqoGuvZvHmz8vLyzC7Dkg4cOKDp06fr9ddfL/Rz9+zZs5owYYLmzJmjli1bmlQhyoqF49ygfv362rZtm9q1a+dyPCEhQb1799bp06dNqsx8PXv21MyZM3X77be7HF+/fr2mTZummJgYkyoz39VXX61ly5apb9++Lse3bt2qhx56SHFxcSZVZo6Lm5heDP1FsdlsOnz4cAVWZS12u10pKSlVpovfnR566CHVqVOn2A9Czz77rNLT07V06dIKrsw8drtdc+bMqTIfAuhhcYP8/HzFx8cXCizx8fEun6iro3379qlFixaFjrdo0UK//PKLCRVZx6FDh4rs2g8ICFBSUlKF12MF33//Pb+ML+OXX35RSkrKJa+pbmPDJGnLli1atWpVsefvu+8+DRs2rAIrsobf9/wXxWazEViqk8jISI0ePVqHDh1Sz549JUm7du3S/PnzFRkZaXJ15goLC9O8efP01ltvydPTU5KUl5enefPmKSwszOTqzHXNNdcoKipK7733noKCgiRJqampevrpp53/jqqTy413wgU33XRTkWMTfj9urDqOf0pOTr5k2A0MDNSRI0cqsCJrqEofAggsbvD//t//U3BwsF5++WUdP35cktSwYUM9/fTTeuqpp0yuzlzLli3TwIED1aRJE+envr1798pms+nzzz83uTpzrVixQnfffbeaNWumpk2bSpKOHDmiNm3a6JNPPjG3OBPwdLpkdu3apfr165tdhuUEBATo0KFDat68eZHnDx48WO3GFFa1DwGMYXGz9PR0Sap2/2FcSmZmplavXq34+HhJF3pdhg0bplq1aplcmfkMw9CmTZtc3pv+/ftXuR80JTFr1iw9/fTT8vX1LfL8unXrNHPmzFJNm69qGMNSvPvuu0/nz5/Xxx9/XOT5u+66S56envrnP/9ZwZWZp8r9ezHgFufPnzc2bdpkLFu2zEhPTzcMwzCOHj1qnDt3zuTKUBlkZ2cbDofD7DJMt2zZMuOee+4xhg4dauzcudMwDMOIjo42unTpYvj6+hrjx483uUJz2Ww2IzU11ewyLCk2Ntbw8vIy7rnnHmPXrl1GWlqakZaWZuzcudMYPHiw4eXlZcTExJhdZoWaOXOmkZmZWez5jz76yOjYsWMFVlQ2BBY3SEpKMkJDQw1fX1/Dw8PDOHTokGEYhvHYY48Z48aNM7k687377rtGnz59jIYNGxpJSUmGYRjGwoULjU8++cTkysxVUFBgzJ4922jUqJHLv5upU6cab731lsnVVbx58+YZNWvWNLp3727UqlXL8PX1NV588UUjODjYmDdvnnHmzBmzSzTdDTfcYBw5csTsMizr888/N+rXr2/Y7XaXr/r16xuffvqp2eWZoip9CCCwuMFdd91lREREGLm5uUbt2rWdv3j+/e9/G61btza5OnO99tprRmBgoDFnzhzD29vb+d68/fbbxg033GBydeaaNWuW0bJlS2PVqlWGj4+P871Zs2aNce2115pcXcVr27atsXLlSsMwDOPbb781bDabcccddxgZGRkmV2YddrudHpZizJo1y8jMzDSysrKMdevWGQsWLDD++te/Gh9//PElexmqsqr2IYDA4gZ169Y14uPjDcMwXAJLYmKi4ePjY2ZppgsLCzM+/vhjwzBc35t9+/YZ9erVM7Ey87Vq1cr4+uuvDcNwfW/i4uKMOnXqmFmaKby9vY3k5GTn956ensb3339vYkXWwyOh4hHmCqtqHwLY/NANHA5HkdMIf/31V/n5+ZlQkXUkJiaqa9euhY57eXkpMzPThIqs4+jRo2rdunWh4w6HQ+fPnzehInPl5ubK29vb+b2np6fq1q1rYkXWVB0HZJeEwfyRQpKTk9WvXz9J0nXXXaeaNWtq1qxZlXbCA9Oa3eCWW27RokWL9MYbb0i68AMlIyNDM2bMKLTCa3XTokUL/fDDD4WmGm7YsKHar8PSvn17bd26tdB78+GHHxYZ8qqDadOmOWcJ5eXlac6cOS77dknSwoULzSjNMtq2bXvZ0HLmzJkKqsZaCHOuqtqHAAKLG7z88ssaMGCA2rdvr5ycHA0bNkwHDhxQYGCg3n//fbPLM1VUVJQeeeQR5eTkyDAM7d69W++//75zMbnqbPr06Ro1apSOHj0qh8OhdevWKSEhQe+++66++OILs8urcNdff70SEhKc3/fu3bvQMvz8Qrow/fuPIQ4XEOYKq0ofAliHxU3y8/O1du1a/fjjj8rIyFC3bt00fPhw+fj4mF2a6VavXq2ZM2fq0KFDkqRGjRpp1qxZGj16tMmVmW/r1q2aPXu2y7+b6dOn65ZbbjG7NFhQlVtXw43sdrsWLVp02TA3atSoCqrIfDfccMNlA5zNZtM333xTQRWVDYEFFSYrK0sZGRn8sAWukIeHh44fP85/Q0UgzFV9DLp1g3feeUdffvml8/tnnnlGderUUe/evfXf//7XxMrMl52draysLEmSr6+vsrOztWjRIn311VcmV2a+I0eO6Ndff3V+v3v3bj3xxBPOsVDVTfv27V266x9++GGdOnXK+f2JEyeKXQW3ujD+b68gFMb7UvXRw+IG7dq109KlS9WvXz/t2LFDN910kxYtWqQvvvhCNWrU0Lp168wu0TS33HKLBg8erPHjxystLU3t2rWTp6enTp06pYULF2rChAlml2ia6667Tg899JBGjBihlJQUtW3bVldffbUOHDigiRMnavr06WaXWKH++AnZ399fP/zwg1q2bCnpwsaQDRs2rNY7oEdGRpboF/OKFSsqoBprsdvtSk1NZZ+l32nfvr22bdvmHGj78MMPa/bs2QoMDJR04UNASEiI80Ol1RFY3MDX11fx8fFq1qyZnn32WR0/flzvvvuufv75Z91www06efKk2SWaJjAwUFu2bFGHDh301ltv6dVXX9WePXv00Ucfafr06YqLizO7RNNcddVV2rlzp9q1a6dXXnlFa9eu1XfffaevvvpK48ePLzTgtKr7Y2Dx8/PTjz/+6BJYGjVqVC13Ir7IbrerefPm6tq16yWn8Ra3n05VRpgrrKp9CGCWkBvUrl1bp0+fVrNmzfTVV18pKipKkuTt7a3s7GyTqzNXVlaWcy2ar776SoMHD5bdbte1115b7R+XnT9/Xl5eXpKkr7/+WnfeeackKTQ01LnrN/B7EyZM0Pvvv6/ExERFRkYqIiKiUk9Tdad33nmnRGGuOivqfalMj9IYw+IGN998s8aMGaMxY8Zo//79zrVXfv75Z4WEhJhbnMlat26tTz75REeOHNHGjRuds19OnDhR7Xe07tChg5YtW6atW7dq06ZNuvXWWyVJx44dU7169UyuruLZbLZCPzwr0w/TirBkyRIdP35czzzzjD7//HM1bdpU9913nzZu3Fjtf0lPmDBBZ8+eVWJiom688UYtX75cH3/8caEvVF4EFjdYsmSJevXqpZMnT+qjjz5y/rKJiYnR0KFDTa7OXNOnT9ekSZMUEhKi8PBw9erVS9KF3pbqujjaRX/961/1+uuv64YbbtDQoUPVuXNnSdJnn32mnj17mlxdxTMMQzfddJO6deumbt26KTs7WwMHDnR+f/PNN5tdoiV4eXlp6NCh2rRpk3755Rd16NBBDz/8sEJCQpSRkWF2eaYhzBVW1T4EMIYF5S4lJUXHjx9X586dZbdfyMi7d++Wv7+/QkNDTa7OXAUFBUpPT9dVV13lPJaUlCRfX99qNz1z1qxZJbpuxowZ5VxJ5XHkyBG9/fbbWrlypfLy8hQfH6/atWubXZYl/Pe//9XKlSv17rvvKj8/Xz///HO1e2/sdruuvvpq1ahxYfTH3r17FRoaKk9PT0lyvi+VZVwYgcWNsrKylJycrLy8PJfjnTp1MqkiAFVNbm6u1q1bpxUrVmjbtm3685//rMjISN16663ODwQgzElV70MAgcUNTp48qQceeEAbNmwo8nxlSa/l5fvvv9cHH3xQZJirzlO+pQv7BhX33sTGxppUlbXk5eUpLy+v2v2yKcrDDz+sNWvWqGnTpnrwwQc1fPhw5xRVEOaqOv4fdIMnnnhCZ8+e1a5du+Tj46MNGzbonXfeUZs2bfTZZ5+ZXZ6p1qxZo969eysuLk4ff/yxzp8/r59//lnffPNNtd8P5ZVXXlFkZKSCgoK0Z88e9ezZU/Xq1dPhw4d12223mV2eKd5++21NnDhRq1evliRNmTJFfn5+CggI0M0336zTp0+bXKG5li1bJn9/f7Vs2VJbtmzRQw89pMGDBxf6qo4efvhhNWzYUPPnz9ef//xnHTlyRP/85z91++23E1b+IC8vr1KOd6KHxQ0aNmyoTz/9VD179pS/v7++//57tW3bVp999pkWLFigbdu2mV2iaTp16qRx48bpkUceca6r0aJFC40bN04NGzYscZdlVRQaGqoZM2Zo6NChLmuOTJ8+XWfOnNHixYvNLrFCvfjii3rxxRfVp08fxcbG6r777tMnn3yiJ554Qna7Xa+88or+/Oc/a+nSpWaXapoHHnigRIMm33777QqoxlrsdruaNWumrl27XvI9qm69um+//bZiY2N17bXXavjw4ZoyZYoWLlyo/Px89evXT2vWrKk0sxIJLG7g7++vvXv3KiQkRM2bN9c//vEP9enTR4mJierQoUOlWUWwPNSqVcs5vbtevXravHmzOnbsqLi4OPXr169arzfi6+uruLg4NW/eXA0aNNCmTZvUuXNnHThwQNdee221601o06aNZs+eraFDh+r7779XeHi4PvjgA91zzz2SpH/9618aP358tV+/B0UjzBVW1T4EsHCcG7Rr104JCQkKCQlR586d9frrryskJETLli1Tw4YNzS7PVFdddZXOnTsnSWrcuLF++ukndezYUWlpadU6yElScHCwzpw5o+bNm6tZs2bauXOnOnfurMTExGo5DTM5OVl9+/aVJPXo0UM1atTQ1Vdf7TzfqVOnah1wcWkrV640uwTLWblypZYvX17sh4Crr75a48ePN7nKkiOwuMHjjz/u/EE6Y8YM3XrrrVq9erU8PT2r/X9E119/vTZt2qSOHTvq3nvv1eOPP65vvvlGmzZt0k033WR2eabq16+fPvvsM3Xt2lWRkZF68skn9eGHH+r777+vluMQfr/yryR5enqqZs2azu9r1KhR7QewA6VR1T4E8EioHGRlZTn3FqruI/jPnDmjnJwcNWrUSA6HQwsWLND27dvVpk0bTZ061WX9kerG4XDI4XA410hYs2aN870ZN26cc62E6sJut+ubb75xLjXfu3dvffDBB2rSpIkk6dSpU7r55psJLUAJVbX9uQgsbnbx7azMqwkCZrDb7bLZbMXud2IYhmw2W6X54QqYrap9CCCwuMny5cv1t7/9TQcOHJB0YQDhE088oTFjxphcmfkKCgr08ccfO3dmbt++ve666y5nz0J19r///U/Lly93eW8iIyOr5YZ2JR1M27x583KuBKgaqtqHAAKLG0yfPl0LFy7UxIkTnXvl7NixQ4sXL9aTTz6p2bNnm1yheX7++WfdeeedSklJUbt27SRJ+/fvV/369fX555+7PE+tbr799lvdeeed8vf3V48ePSRd2H8qLS1Nn3/+ua6//nqTKzTf+++/rzvvvFO1atUyuxSg0qlqHwIILG5Qv359vfLKK4U2Onz//fc1ceJEnTp1yqTKzNerVy/Vr19f77zzjnO8yv/+9z898MADOnnypLZv325yhebp2LGjevXqpaVLl8rDw0PShd6ohx9+WNu3b9e+fftMrtB8/v7++uGHH5zP3AFUXwQWN6hTp47+85//qE2bNi7H9+/fr549eyotLc2cwizAx8dH33//vTp06OBy/KefftI111yj7Oxskyozn4+Pj3744Qdnz9NFCQkJ6tKlS7V+by764yBBACWzd+/eEl9bWfa7YxCBG4wYMUJLly7VwoULXY6/8cYbGj58uElVWUPbtm2VmppaKLCcOHFCrVu3Nqkqa+jWrZvi4uIKBZa4uDh17tzZpKoAVAVdunQpdvzK71WmMSwElisUFRXl/LPNZtNbb72lr776Stdee60kadeuXUpOTtbIkSPNKtE06enpzj/PmzdPjz32mGbOnOl8b3bu3KnZs2frr3/9q1klmub3n3oee+wxPf744zp48KDLe7NkyRLNnz/frBIt5V//+pcaN25sdhlApZOYmGh2CW7HI6ErdOONN5boOpvNpm+++aacq7GWiyPTL/rjVO/ff19Zkr27XGrU/u9Vx/fmon79+mndunWqU6eOy/H09HQNGjSo2v33BOACAgvcbsuWLSW+9k9/+lM5VmI9pdkHp7KM3He3Py52ddGJEyfUuHFjnT9/3qTKgMrtl19+UXJysvLy8lyO33nnnSZVVDo8EnIzpmFWvxBSGtU1hJTE7x+X/fLLL0pJSXF+X1BQoA0bNvB4CLgChw8f1t133619+/a59PBe7PWuLL259LC4GdMwi9exY0etX79eTZs2NbsUy+HfjeujxKJ+LPn4+OjVV1/Vgw8+WNGlAZXawIED5eHhobfeekstWrTQ7t27dfr0aT311FP6f//v/+m6664zu8QSoYfFzch/xUtKSqI7vxj8u5Fzl+qWLVtq9+7dql+/vvOcp6enGjRo4FyvBkDJ7dixQ998840CAwNlt9tlt9vVt29f56SIPXv2mF1iiRBYAFjCxcdlDoej2GsuLiUOoOQKCgrk5+cnSQoMDNSxY8fUrl07NW/eXAkJCSZXV3J2swuoapiGWbzrrrtOPj4+ZpdhSREREfL39ze7DEt44IEHlJmZWeh4UlIS2xUAV+Dqq6/Wjz/+KEkKDw/XggUL9N1332n27NmV6jE0gcUN+vXr51zNtm/fvvLy8pJ0YRpmv379TKzMWtavX6+GDRuaXYYlLV26VIGBgWaXYQk//vijOnXqpB07djiPvfPOO+rcuTPvEXAFpk6d6uy5nD17thITE3Xddddp/fr1euWVV0yuruQYdOsGTMMs3meffVbkcZvNJm9vb7Vu3VotWrSo4KqsIzo6WtHR0Tpx4kShRyErVqwwqSpznT9/Xs8995xeeeUVPfXUUzp48KD+9a9/aeHChRo7dqzZ5QFVwpkzZ3TVVVdVqkesjGEpA6ZhXt6gQYOKXCjt91ub9+3bV5988olzc8TqYtasWZo9e7Z69Oihhg0bVqofHOWpZs2aeumll+Tr66sXXnhBNWrU0JYtW5w7oQMou7p165pdQqnRw1IGTMO8vOjoaD3//PN68cUX1bNnT0nS7t27NW3aNE2dOlUBAQEaN26cwsPDtXz5cpOrrVgNGzbUggULNGLECLNLsZTz589r8uTJWrJkiZ566ilt27ZN+/fv1/Lly3X77bebXR5Q6dx4442X/EBUWVaPpoelDJiGeXmPP/643njjDfXu3dt57KabbpK3t7ceeugh/fzzz1q0aFG1DHV5eXku7wsu6NGjh7KysrR582Zde+21MgxDCxYs0ODBg/Xggw/qtddeM7tEoFLp0qWLy/fnz5/XDz/8oJ9++kmjRo0yp6grQGApA6ZhXt6hQ4eKnP3i7++vw4cPS5LatGmjU6dOVXRpphszZoz+8Y9/aNq0aWaXYik9evTQK6+84lwt2maz6dlnn9Utt9xCbxRwBf72t78VeXzmzJnKyMio4GquHI+E3OCBBx7QkiVLCi3Hn5SUpBEjRmjr1q0mVWa+vn37ys/PT++++66zB+rkyZMaOXKkMjMz9e233+rrr7/WI488UqnWA3CHxx9/XO+++646deqkTp06qWbNmi7nFy5caFJl1pWbm+uchQegbA4ePKiePXvqzJkzZpdSIkxrdgOmYRZv+fLlSkxMVJMmTdS6dWu1bt1aTZo0UVJSkt566y1JUkZGhqZOnWpypRVv79696tKli+x2u3766Sft2bPH+fXDDz+YXZ6p3nvvPfXp00eNGjVybhi5aNEibdiwweTKgKpjx44d8vb2NruMEuORkBvs3r1bzz33nG644QamYf5Bu3bt9Msvv+irr77S/v37ncduvvlm2e0X8vKgQYNMrNA8//73v80uwZKWLl2q6dOn64knntCLL77o3JitTp06WrRoke666y6TKwQql8GDB7t8bxiGjh8/ru+//75SPZLmkZAbzZgxg2mYuCK//vqrJKlJkyYmV2K+9u3ba+7cuRo0aJD8/Pz0448/qmXLlvrpp590ww03VMvxTkBZREZGunxvt9tVv3599evXT7fccotJVZUePSxu8PtpmFOmTNG2bds0ePBgpmH+n8zMTG3ZskXJycnKy8tzOffYY4+ZVJX5HA6H5syZo5dfftk58M3Pz09PPfWUnn/+eWcPVHWTmJiorl27Fjru5eVV5JL9AC7t7bffNrsEtyCwuAHTMIu3Z88e3X777crKylJmZqbq1q2rU6dOydfXVw0aNKjWgeX555/X8uXLNX/+fPXp00eStG3bNs2cOVM5OTl68cUXTa7QHC1atNAPP/zgnIV30YYNGxQWFmZSVUDldeTIEdlsNmcP7u7du/WPf/xD7du310MPPWRydaVgoMwefPBBIyMjo9Dx2NhYo0OHDiZUZB1/+tOfjLFjxxoFBQVG7dq1jUOHDhnJycnG9ddfb3z00Udml2eqhg0bGp9++mmh45988onRqFEjEyoy16xZs4zMzEzjzTffNBo3bmysWbPGqFWrlvH+++8bc+bMcf4ZQOn07dvXePfddw3DMIzjx48bfn5+Rq9evYzAwEBj1qxZJldXcgSWcpaTk2N2CaYKCAgw4uPjnX/+5ZdfDMMwjJ07dxrt2rUzszTTeXl5GQkJCYWOx8fHG97e3iZUZC673W6kpqYahmEYq1atMlq3bm3YbDbDZrMZjRs3Nt566y2TKwQqpzp16jh/Dv/97383evfubRiGYWzcuNFo0aKFmaWVSvV8SF4OmIZZtJo1azrHYjRo0EDJycmSpICAAB05csTM0kzXuXNnLV68uNDxxYsXq3PnziZUZC7jd+P/hw8frgMHDigjI0MpKSn69ddfNXr0aBOrAyqv8+fPO9cv+vrrr3XnnXdKkkJDQ3X8+HEzSysVAosbLF26VFFRUbr99tuVlpZWaBpmdfTuu+8qJydHXbt21X/+8x9J0p/+9CdNnz5dq1ev1hNPPKGrr77a5CrNtWDBAq1YsULt27fX6NGjNXr0aLVv314rV67USy+9ZHZ5pvjjqtAXxzoBuHIdOnTQsmXLtHXrVm3atEm33nqrJOnYsWOqV6+eydWVHNOa3YBpmIV5eHjo+PHjSk5O1rlz53TjjTfqxIkTGjlypLZv3642bdpoxYoV1bIn4feOHTumJUuWKD4+XpIUFhamhx9+WI0aNTK5sopnt9sVEBBw2a0sKsuqnIBVbN68WXfffbfS09M1atQorVixQpL03HPPKT4+XuvWrTO5wpIhsLiBj4+P4uPj1bx5c5fAcuDAAXXq1EnZ2dlml1jh7Ha7UlJS+HSMErPb7Vq0aJECAgIueV1l2qwNsIqCggKlp6frqquuch5LSkqqVL2YTGt2A6ZhFq06b/pYnL179+rqq6+W3W7X3r17L3ltp06dKqgq67j//vsrzQ9PoDLx8PBwCSuSFBISYk4xV4jAUgazZ8/WpEmTFBUVpUceeUQ5OTkyDEO7d+/W+++/r3nz5jn3y6mObrrpJtWocel/YrGxsRVUjTV06dLF2fPUpUsX2Ww2FdXJabPZnGOhqgsCLuA+3bp1U3R0tK666ip17dr1kv99VZafwwSWMpg1a5bGjx+vMWPGyMfHR1OnTlVWVpaGDRumRo0a6e9//7vuv/9+s8s0zYABA1S7dm2zy7CUxMRE567ViYmJJldjLTydBtznrrvu0rFjx3TVVVdVmf3aGMNSBkWN08jKylJGRka179ZmDMvlffvtt+rdu3ehXqj8/Hxt375d119/vUmVAagK7Ha7rrnmGo0ePVpDhw6Vn5+f2SWVCdOay4hpmEWje//ybrzxxiJnvJw9e1Y33nijCRUBqEq2bNmiDh06aNKkSWrYsKEeeOABbd261eyyrhg9LGXANMzi0cNyeXa7Xampqc5HRBft379fPXr0UHp6ukmVAahKMjMz9cEHH2jlypXaunWrWrdurdGjR2vUqFEKDg42u7wSI7CUAdMwi/ff//5XzZo1o6elCIMHD5Ykffrpp7r11ludK1BKF6Ye7t27V+3atav2qyQDcL+DBw/q7bff1nvvvaeUlBTdeuut+uyzz8wuq0QYdFtGTMMs2h+neOM3FwOuYRjy8/OTj4+P85ynp6euvfZajR071qzyAFRhrVu31nPPPafmzZtrypQp+vLLL80uqcQILGVA7wGuxNtvvy3pwhoIkyZNUq1atUyuCEB18O2332rFihX66KOPZLfbdd9991WqPbp4JFQGjNOAO5w4cUIJCQmSpHbt2vHvCYDbHDt2TCtXrtTKlSt18OBB9e7dW6NHj9Z9991X6T4sEVgAk5w7d04PP/yw1qxZ41wkzsPDQ0OGDNGSJUsuOzYKAC7ltttu09dff63AwECNHDlSDz74oNq1a2d2WVeMac0od1u3blVERIR69eqlo0ePSpLee+89bdu2zeTKzDVmzBjt2rVLX3zxhdLS0pSWlqYvvvhC33//vcaNG2d2eQAquZo1a+rDDz/Ur7/+qr/+9a+VOqxI9LCgnH300UcaMWKEhg8frvfee0+//PKLWrZsqcWLF2v9+vVav3692SWaplatWtq4caP69u3rcnzr1q269dZblZmZaVJlAGA99LCgXM2ZM0fLli3Tm2++qZo1azqP9+nTp9LsX1Fe6tWrV+Rjn4CAgEKblAFAdUdgQblKSEgocon5gIAApaWlVXxBFjJ16lRFRUUpJSXFeSwlJUVPP/20pk2bZmJlAGA9TGtGuQoODtbBgwcLbWO+bds2tWzZ0pyiLGLp0qU6ePCgmjVrpmbNmkmSkpOT5eXlpZMnT+r11193Xlvde6MAgMCCcjV27Fg9/vjjWrFihWw2m44dO6YdO3Zo0qRJ1b4XoarsoAoAFYFBtyhXhmFo7ty5mjdvnrKysiRJXl5emjRpkl544QWTqwMAVBYEFlSIvLw8HTx4UBkZGWrfvr1q165tdkkAgEqEwAKYpKCgQH/729/0wQcfKDk5WXl5eS7nq+Mu3wBQHMawoFzdfffdRe65ZLPZ5O3trdatW2vYsGGVfkGjKzFr1iy99dZbeuqppzR16lQ9//zzSkpK0ieffKLp06ebXR4AWArTmlGuAgIC9M033yg2NlY2m002m0179uzRN998o/z8fK1du1adO3fWd999Z3apFW716tV688039dRTT6lGjRoaOnSo3nrrLU2fPl07d+40uzwAsBQCC8pVcHCwhg0bpsOHD+ujjz7SRx99pEOHDikiIkKtWrVSXFycRo0apWeffdbsUitcSkqKOnbsKEmqXbu2zp49K0n685//XKm2fAeAikBgQblavny5nnjiCdntv/1Ts9vtmjhxot544w3ZbDY9+uij+umnn0ys0hxNmjTR8ePHJUmtWrXSV199JUn6z3/+Iy8vLzNLAwDLIbCgXOXn5ys+Pr7Q8fj4eOcOxd7e3kWOc6nq7r77bkVHR0uSJk6cqGnTpqlNmzbOXVUBAL9h0C3K1YgRIzR69Gg999xzuuaaayRd6EGYO3euRo4cKUnasmWLOnToYGaZppg/f77zz0OGDFHz5s21fft2tWnTRgMHDjSxMgCwHqY1o1wVFBRo/vz5Wrx4sVJTUyVJQUFBmjhxop599ll5eHgoOTlZdrtdTZo0MblaAIBVEVhQYdLT0yVJ/v7+JldirpiYGE2aNEmffvppoffi7NmzGjRokBYtWqTOnTubVCEAWA9jWFBh/P39q31YkaSXX35Z/fr1K/K9CAgIUP/+/fXSSy+ZUBkAWBc9LCh3H374YbGruVbHXYhbtWqljz/+WJ06dSry/L59+3TXXXfp8OHDFVwZAFgXPSwoV6+88ooiIyMVFBSkPXv2qGfPnqpXr54OHz6s2267zezyTHH06FH5+fkVe7527drO6c4AgAsILChXr732mt544w29+uqr8vT01DPPPKNNmzbpsccecy6UVt3Ur19fCQkJxZ6Pj49XYGBgBVYEANZHYEG5Sk5OVu/evSVJPj4+OnfunKQL053ff/99M0szTf/+/fXiiy8Wec4wDL344ovq379/BVcFANZGYEG5Cg4Odu463KxZM+ceOYmJiaquw6emTp2qffv2KTw8XB988IF+/PFH/fjjj1q7dq3Cw8P1008/6fnnnze7TACwFBaOQ7nq16+fPvvsM3Xt2lWRkZF68skn9eGHH+r777/X4MGDzS7PFK1atdLXX3+tBx54QPfff79zlV/DMNS+fXtt2rRJrVu3NrlKALAWZgmhXDkcDjkcDtWocSEbr1mzxrma67hx4+Tp6Wlyheb64YcfdODAARmGobZt26pLly5mlwQAlkRgAQAAlscjIZS7nJwc7d27VydOnJDD4XA5d+edd5pUFQCgMiGwoFxt2LBBI0eO1KlTpwqds9lszh2bAQC4FGYJoVxNnDhR9957r44fP+4cz3Lxi7ACACgpxrCgXPn7+2vPnj1q1aqV2aUAACoxelhQrv7yl79o8+bNZpdhWVu3blVERIR69eqlo0ePSpLee+89bdu2zeTKAMBaGMOCcrV48WLde++92rp1qzp27KiaNWu6nH/sscdMqsx8H330kUaMGKHhw4drz549ys3NlSSdPXtWc+fO1fr1602uEACsg0dCKFfLly/X+PHj5e3trXr16jkXSZMuDLqtzjsSd+3aVU8++aRGjhwpPz8//fjjj2rZsqX27Nmj2267TSkpKWaXCACWQQ8LytXzzz+vWbNmafLkybLbeQL5ewkJCbr++usLHQ8ICFBaWlrFFwQAFsZvEJSrvLw8DRkyhLBShODgYB08eLDQ8W3btqlly5YmVAQA1sVvEZSrUaNGae3atWaXYUljx47V448/rl27dslms+nYsWNavXq1Jk2apAkTJphdHgBYCo+EUK4KCgq0YMECbdy4UZ06dSo06HbhwoUmVWa+yZMny+Fw6KabblJWVpauv/56eXl5adKkSZo4caLZ5QGApTDoFuXqxhtvLPaczWbTN998U4HVWFNeXp4OHjyojIwMtW/fXrVr1za7JACwHAILAACwPB4JASa5++67XaZ5X2Sz2eTt7a3WrVtr2LBhateunQnVAYC10MOCcjF48OASXbdu3bpyrsS6HnjgAX3yySeqU6eOunfvLkmKjY1VWlqabrnlFv34449KSkpSdHS0+vTpY3K1AGAuelhQLgICAswuwfKCg4M1bNgwLV682Dnt2+Fw6PHHH5efn5/WrFmj8ePH69lnn2WpfgDVHj0sgEnq16+v7777Tm3btnU5vn//fvXu3VunTp3Svn37dN1117GQHIBqj3VYAJPk5+crPj6+0PH4+HgVFBRIkry9vYsc5wIA1Q2PhACTjBgxQqNHj9Zzzz2na665RpL0n//8R3PnztXIkSMlSVu2bFGHDh3MLBMALIFHQoBJCgoKNH/+fC1evFipqamSpKCgIE2cOFHPPvusPDw8lJycLLvdriZNmphcLQCYi8ACWEB6erokyd/f3+RKAMCaCCwAAMDyGMMCmOjDDz/UBx98oOTkZOXl5bmci42NNakqALAeZgkBJnnllVcUGRmpoKAg7dmzRz179lS9evV0+PBh3XbbbWaXBwCWwiMhwCShoaGaMWOGhg4dKj8/P/34449q2bKlpk+frjNnzmjx4sVmlwgAlkEPC2CS5ORk9e7dW5Lk4+Ojc+fOSbow3fn99983szQAsBwCC2CS4OBgnTlzRpLUrFkz7dy5U5KUmJgoOj4BwBWBBTBJv3799Nlnn0mSIiMj9eSTT+rmm2/WkCFDdPfdd5tcHQBYC2NYAJM4HA45HA7VqHFhst6aNWu0fft2tWnTRuPGjZOnp6fJFQKAdRBYAACA5bEOC2CinJwc7d27VydOnJDD4XA5d+edd5pUFQBYD4EFMMmGDRs0cuRInTp1qtA5m83m3LEZAMCgW8A0EydO1L333qvjx487x7Nc/CKsAIArxrAAJvH399eePXvUqlUrs0sBAMujhwUwyV/+8hdt3rzZ7DIAoFKghwUwSVZWlu69917Vr19fHTt2VM2aNV3OP/bYYyZVBgDWQ2ABTLJ8+XKNHz9e3t7eqlevnmw2m/OczWbT4cOHTawOAKyFwAKYJDg4WI899pgmT54su52nswBwKfyUBEySl5enIUOGEFYAoAT4SQmYZNSoUVq7dq3ZZQBApcDCcYBJCgoKtGDBAm3cuFGdOnUqNOh24cKFJlUGANbDGBbAJDfeeGOx52w2m7755psKrAYArI3AAgAALI8xLAAAwPIYwwJUsMGDB5founXr1pVzJQBQeRBYgAoWEBBgdgkAUOkwhgUAAFgeY1gAAIDlEVgAAIDlEVgAAIDlEVgAAIDlEVgAAIDlEVgAAIDlEVgAAIDlEVgAAIDl/X9l5r3i0MCPYAAAAABJRU5ErkJggg==",
      "text/plain": [
       "<Figure size 640x480 with 1 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "fig, ax = plt.subplots()\n",
    "ax.boxplot([aucs_text, aucs_tags, aucs_capt, aucs_bert, aucs_MTT, aucs_MTC, aucs_vbert], meanline=True, showmeans=True)\n",
    "plt.xticks([1, 2, 3, 4, 5, 6, 7], [\"Text-based\", \"Image Tag-based\", \"Image Caption-based\", \"Text-BERT\", \"MTT\", \"MTC\", \"VisualBERT\"], rotation=90)\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 55,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAjUAAAIoCAYAAACRXRrdAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjYuMCwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy89olMNAAAACXBIWXMAAA9hAAAPYQGoP6dpAABpSklEQVR4nO3deVhUZf8G8HuGHQVc2BEFQwUTUDERXHKhtEVF1CwljNwjNdBSzCVNwVfTfE2TFrRFU4vQTHtdwjRUlATNKFYVMWUQNUDZnTm/P/wxNQ0og8oZDvfnuubKec5zznxnLnPuec5zniMTBEEAERERURMnF7sAIiIiooeBoYaIiIgkgaGGiIiIJIGhhoiIiCSBoYaIiIgkgaGGiIiIJIGhhoiIiCSBoYaIiIgkgaGGiIiIJIGhhoiIiCShQaFm48aNcHFxgampKXx9fZGcnHzP/uvWrUOXLl1gZmYGZ2dnhIeHo6KiQqdjVlRUICwsDG3btkXLli0xevRoFBQUNKR8IiIikiCdQ83OnTsRERGBJUuWIDU1Fd7e3hg6dCiuXbtWa/+vvvoK8+fPx5IlS5Ceno7Y2Fjs3LkTCxYs0OmY4eHh+P777/HNN9/g6NGjuHr1KoKCghrwlomIiEiKZLre0NLX1xdPPPEENmzYAABQqVRwdnbGzJkzMX/+fK3+r7/+OtLT05GQkKBumzNnDk6dOoVjx47V65jFxcWwsbHBV199hTFjxgAAMjIy4OHhgaSkJPTp0+e+datUKly9ehUWFhaQyWS6vGUiIiISiSAIuHXrFhwdHSGX32csRtBBZWWlYGBgIOzatUujPSQkRBgxYkSt+2zbtk2wsrISTp06JQiCIJw/f15wd3cXVqxYUe9jJiQkCACEv/76S6NP+/bthbVr19b6uhUVFUJxcbH68ccffwgA+OCDDz744IOPJvi4fPnyfXOKIXRw/fp1KJVK2NnZabTb2dkhIyOj1n3Gjx+P69evo1+/fhAEAXfu3MH06dPVp5/qc0yFQgFjY2O0atVKq49Coaj1daOjo7F06VKt9suXL8PS0rJe75eIiIjEVVJSAmdnZ1hYWNy3r06hpiGOHDmCqKgofPjhh/D19UVOTg5mz56Nd999F4sWLXpkrxsZGYmIiAj185oPxdLSkqGGiIioianP1BGdQo21tTUMDAy0rjoqKCiAvb19rfssWrQIL7/8MiZPngwA8PT0RGlpKaZOnYq33367Xse0t7dHVVUVioqKNEZr7vW6JiYmMDEx0eXtERERUROm09VPxsbG8PHx0Zj0q1KpkJCQAD8/v1r3KSsr05rYY2BgAAAQBKFex/Tx8YGRkZFGn8zMTOTl5dX5ukRERNS86Hz6KSIiAhMnTkSvXr3Qu3dvrFu3DqWlpQgNDQUAhISEwMnJCdHR0QCA4cOHY+3atejRo4f69NOiRYswfPhwdbi53zGtrKwwadIkREREoE2bNrC0tMTMmTPh5+dXryufiIiISPp0DjXjxo1DYWEhFi9eDIVCge7du2P//v3qib55eXkaIzMLFy6ETCbDwoULceXKFdjY2GD48OFYsWJFvY8JAO+//z7kcjlGjx6NyspKDB06FB9++OGDvHciIiKSEJ3XqWmqSkpKYGVlheLiYk4UJiIiaiJ0+f7mvZ+IiIhIEhhqiIiISBIYaoiIiEgSGGqIiIhIEhhqiIiISBIYaoiIiEgSGGqIiIhIEh75DS2JiEgcSqUSiYmJyM/Ph4ODA/r3769eyZ1IihhqiIgkKD4+HnPmzEFubq66zcXFBWvWrEFQUJB4hZHoysrKkJGRcc8+5eXlyM3NhYuLC8zMzO7Z193dHebm5g+zxAZjqCEikpj4+HiMGTMGzz//PLZv345u3bohLS0NUVFRGDNmDOLi4hhsmrGMjAz4+Pg8tOOlpKSgZ8+eD+14D4K3SSAikhClUgk3Nzd4enpi9+7dGvfiU6lUCAwMRFpaGrKzs3kqqpmqz0hNeno6goODsXXrVnh4eNyz76MeqdHl+5sjNUREEpKYmIjc3Fxs375dI9AAgFwuR2RkJPz9/ZGYmIiBAweKUySJytzcvN4jKx4eHnozClMfvPqJiEhC8vPzAQDdunWrdXtNe00/IilhqCEikhAHBwcAQFpaWq3ba9pr+hFJCUMNEZGE9O/fHy4uLoiKioJKpdLYplKpEB0dDVdXV/Tv31+kCokeHYYaIiIJMTAwwJo1a7B3714EBgYiKSkJt27dQlJSEgIDA7F371689957nCRMksSJwkREeio7Oxu3bt2qc3vNWiK1mTVrFr766iv4+/ur22xsbDBr1iyUl5dj27ZtWvvcb00SCwsLdOrUqf5vgKiRMdQQEemhnPTf8MLIXoDFv/6ZLldC9lc1BEMZYGuitZ/sagUAQLA2hpOzHE7O/7jU9q+/8PPOD3B0rwFgZaS5Y6UKshtVEGQAHEy1C1JUQKYCvj58Fm4eng/47ogeDYYaIiI9dOtiCqa8ZodNra002p+7XYqVhTeQZ2iI55wdtfb77WIeAGCCgx3OmWqGnqhr1zG8tAzbLVoiyrqNxjb/snJ8VFCI2zIZ/FyctY579NKfaKNSIT3/d4ChhvQUQw0RkR46+2cZPvmwAIYWNzTaD5QrcfD/R2oMbSu09uupHqmpgqGx5rTJxX9VYUm5CkKLchhaFWlsS65Uoef/j9QYOlzQOu6QmpGacY8/4DsjenQYaoiI9NDwoBeglBvfc7XWe82pqXHx4kUsWrQI7777LlxdXe/Ztz5zatw4p4b0GEMNEZEesra2xuTJk+/ZJzU1FcHBwfU63qJFi+7bR5/u4UPUEAw1RERNlLu7O1JSUu7ZR9e7LRM1ZQw1RERNVH3v4dO3b99GqIZIfFx8j4iIiCSBoYaIiIgkgaGGiIiIJIGhhoiIiCSBoYaIiIgkgaGGiIiIJIGhhoiIiCSB69QQERFJSHZ2Nm7duvVAx0hPT9f474OwsLBAp0a6vQZDDRERkURkZ2ejc+fOD+149b0Nx/1kZWU1SrBhqCEiIpKImhGarVu3wsPDo8HH0eX2GveSnp6O4ODgBx45qq8GhZqNGzdi9erVUCgU8Pb2xgcffIDevXvX2nfgwIE4evSoVvuzzz6Lffv2AQBkMlmt+65atQpvvvkmgLt3j7106ZLG9ujoaMyfP78hb4GIiEiyPDw8HvjmpE3x9ho6h5qdO3ciIiICMTEx8PX1xbp16zB06FBkZmbC1tZWq398fDyqqqrUz2/cuAFvb2+MHTtW3Zafn6+xz//+9z9MmjQJo0eP1mhftmwZpkyZon5uYWGha/lEREQkUTqHmrVr12LKlCkIDQ0FAMTExGDfvn3YvHlzraMmbdq00Xi+Y8cOmJuba4Qae3t7jT7fffcdBg0ahI4dO2q0W1hYaPUlIiIiAnS8pLuqqgopKSkICAj4+wByOQICApCUlFSvY8TGxuLFF19EixYtat1eUFCAffv2YdKkSVrbVq5cibZt26JHjx5YvXo17ty5U+frVFZWoqSkRONBRERE0qXTSM3169ehVCphZ2en0W5nZ4eMjIz77p+cnIy0tDTExsbW2efzzz+HhYUFgoKCNNpnzZqFnj17ok2bNjhx4gQiIyORn5+PtWvX1nqc6OhoLF26tB7vioiIiKSgUa9+io2NhaenZ52TigFg8+bNmDBhAkxNTTXaIyIi1H/28vKCsbExpk2bhujoaJiYmGgdJzIyUmOfkpISODs7P4R3QURERPpIp9NP1tbWMDAwQEFBgUZ7QUHBfee6lJaWYseOHbWeVqqRmJiIzMxMTJ48+b61+Pr64s6dO8jNza11u4mJCSwtLTUeREREJF06jdQYGxvDx8cHCQkJCAwMBACoVCokJCTg9ddfv+e+33zzDSorK++5kE9sbCx8fHzg7e1931rOnj0LuVxe6xVXREREzZWhlSFyy3NheuPvMx6WxpZoZ9EOlcpKnC86r7VP17ZdAQAXiy+i/E65xjanlk6wMrHCzYqbUJQqNLa1MGqBDpYdoFQpkflXptZx7wh1z319FHQ+/RQREYGJEyeiV69e6N27N9atW4fS0lL11VAhISFwcnJCdHS0xn6xsbEIDAxE27Ztaz1uSUkJvvnmG6xZs0ZrW1JSEk6dOoVBgwbBwsICSUlJCA8PR3BwMFq3bq3rWyAiIpKsNoPaYEnOEiDn77bnOj6Hlf1XoqC0AOP2jtPa57eJvwEAFh5fiHOF5zS2RfWLwvDHhuNA7gFEnYrS2Obv6I+PnvoI5XfKaz3uBx4fPIR3VH86h5px48ahsLAQixcvhkKhQPfu3bF//3715OG8vDzI5ZpntTIzM3Hs2DEcPHiwzuPu2LEDgiDgpZde0tpmYmKCHTt24J133kFlZSVcXV0RHh6uMWeGiIiouZPdqYDj70VYPm0+XF1c1e2WRubA1bOwU1Zhp++72jtePQsAWN4pGOUdKzU2ORm2Bq6exVBTJ3j/a98WBqbA1bMwE1S1Hld55Qp62Mshu1Px4G+uHmSCIAiN8koiKykpgZWVFYqLizm/hoiIJCn98A54/DxN7DK0pA/4CB6DX2zQvrp8f/PeT0RERBJR0bI9en50G9u2bYOHu7vY5SA9IwMTJkxA7LPtG+X1GGqIiIgkQjA0xRmFCuWtOgOO3cUuB+UKFc4oVBAMTe/f+SHQ6ZJuIiIiIn3FUENERESSwFBDREREksBQQ0RERJLAUENERESSwFBDREREksBQQ0RERJLAUENERESSwFBDREREksAVhYmIiCSirKwMAJCamvpAxykvL0dubi5cXFxgZmbW4OOkp6c/UB26YqghIiKSiIyMDADAlClTRK5Ek4WFRaO8DkMNERGRRAQGBgIA3N3dYW5u3uDjpKenIzg4GFu3boWHh8cD1WRhYYFOnTo90DHqi6GGiIiaHaVSicTEROTn58PBwQH9+/eHgYGB2GU9MGtra0yePPmhHc/DwwM9e/Z8aMd71DhRmIiImpX4+Hi4ublh0KBBGD9+PAYNGgQ3NzfEx8eLXRo9II7UEBFRsxEfH48xY8bg+eefx/bt29GtWzekpaUhKioKY8aMQVxcHIKCgsQu85EqKytTz72pS80E3/pM9H3QU10Pk0wQBEHsIhpDSUkJrKysUFxcDEtLS7HLISKiRqZUKuHm5gZPT098++23OH78uPr0U9++fTF69GikpaUhOztbEqei6pKamgofH5+HdryUlJRHeopKl+9vjtQQEVGzkJiYiNzcXEybNg2dO3dGbm6uepuLiwumTp2K77//HomJiRg4cKBodT5q7u7uSElJuWcfXS7pdnd3f5jlPRCGGiIiahby8/MBAAsWLKj19NPbb7+t0U+qzM3N6zWy0rdv30ao5uFiqCEiombB1tYWwN0v6927d0Muv3utTJ8+fbB79248+eSTOHbsmLofNT28+omIiAhAM5liKmkMNURE1Cxcu3YNAHDs2DEEBgYiKSkJt27dQlJSEgIDA3H8+HGNftT0MNQQEVGz4ODgAACIjo7Gb7/9Bn9/f1haWsLf3189r+af/ajp4ZwaIiJqWqrKcPaXXbhyS3NEpaXcDLaGrVAl3MHF0qsoLCzU2O4otMZgj7b4JelLzFswCop8BcrKSmFu3gIe9o/hu+1bMbCnDbJy9iL7/D71fiYwhJejO4yMjZBbXaBVTnsjW1i1tET7HkMAY/1Yr6W54jo1RETUpOSd+h57k8KwqbWVRvtzt0uxsvAG8gwN8Zyzo9Z+v13MAwBMcLDDOVMTjW1R165jeGkZtlu0RJR1G41t/mXl+KigELdlMvi5OGsd9+ilP9FGpULeM1vR3nf4g749+heuU0NERJL1Z4UZPvmwAMPH+cLJ6e/wYiY3xi7bFqiGEpNKbqC4uFhjvw+V/gAAq9QcWP7xK8rKytXbVlWbING7D+w6OmFMeaXGfkYqA+xs1Q6GxkaYXl6kVc+26jJs+XQzYp9ti/YP8X2S7hhqiEiv1WdJd10XCtOXJd2pYf7IzsXZ7CqcXR73EI9ahmO/HXygI7RsbfOQaqGGYqghIr2WkZHRpJZ0p0cvMDAQwL0Danp6OoKDgx/aa27duhUeHh51brewsECnTp0e2utRw3BODRHptfrefC84OPi+XzwAR2qaC47wSQfn1BCRZNR3SXcA8PDw4CgMAZD2rQCoblynhoiIiCSBoYaIiIgkgaGGiIiIJKFBoWbjxo1wcXGBqakpfH19kZycXGffgQMHQiaTaT2ee+45dZ9XXnlFa/uwYcM0jnPz5k1MmDABlpaWaNWqFSZNmoTbt283pHwiIiKSIJ1Dzc6dOxEREYElS5YgNTUV3t7eGDp0aJ03AIuPj0d+fr76kZaWBgMDA4wdO1aj37BhwzT6bd++XWP7hAkT8Pvvv+PQoUPYu3cvfv75Z0ydOlXX8omIiEiidA41a9euxZQpUxAaGoquXbsiJiYG5ubm2Lx5c63927RpA3t7e/Xj0KFDMDc31wo1JiYmGv1at26t3paeno79+/fj008/ha+vL/r164cPPvgAO3bswNWrV3V9C0RERCRBOoWaqqoqpKSkICAg4O8DyOUICAhAUlJSvY4RGxuLF198ES1atNBoP3LkCGxtbdGlSxfMmDEDN27cUG9LSkpCq1at0KtXL3VbQEAA5HI5Tp06VevrVFZWoqSkRONBRERE0qVTqLl+/TqUSiXs7Ow02u3s7KBQKO67f3JyMtLS0jB58mSN9mHDhuGLL75AQkIC/vOf/+Do0aN45plnoFQqAQAKhQK2trYa+xgaGqJNmzZ1vm50dDSsrKzUD2dn7ZuQERERkXQ06uJ7sbGx8PT0RO/evTXaX3zxRfWfPT094eXlhcceewxHjhzBkCFDGvRakZGRiIiIUD8vKSlhsCEiIpIwnUZqrK2tYWBggIKCAo32goIC2Nvb33Pf0tJS7NixA5MmTbrv63Ts2BHW1tbIyckBANjb22tNRL5z5w5u3rxZ5+uamJjA0tJS40FERETSpVOoMTY2ho+PDxISEtRtKpUKCQkJ8PPzu+e+33zzDSorK+t1g7E///wTN27cgIODAwDAz88PRUVFSElJUfc5fPgwVCoVfH19dXkLREREJFE6X/0UERGBTz75BJ9//jnS09MxY8YMlJaWIjQ0FAAQEhKCyMhIrf1iY2MRGBiItm3barTfvn0bb775Jk6ePInc3FwkJCRg5MiRcHNzw9ChQwHcvZ/LsGHDMGXKFCQnJ+P48eN4/fXX8eKLL8LR0bEh75uIiIgkRuc5NePGjUNhYSEWL14MhUKB7t27Y//+/erJw3l5eZDLNbNSZmYmjh07hoMHD2odz8DAAOfOncPnn3+OoqIiODo64umnn8a7774LExMTdb9t27bh9ddfx5AhQyCXyzF69GisX79e1/KJiIhIomSCIAhiF9EYdLl1ORE1LampqfDx8UFKSgrv0k0kMbp8f/PeT0RERCQJDDVEREQkCQw1REREJAkMNURERCQJDDVEREQkCQw1REREJAkMNURERCQJDDVEREQkCQw1REREJAkMNURERCQJDDVEREQkCQw1REREJAkMNURERCQJDDVEREQkCQw1REREJAkMNURERCQJDDVEREQkCQw1REREJAkMNURERCQJDDVEREQkCQw1REREJAkMNURERCQJDDVEREQkCYZiF0BElJ2djVu3bjV4//T0dI3/PggLCwt06tTpgY9DRI2PoYaIRJWdnY3OnTs/lGMFBwc/lONkZWUx2BA1QQw1RCSqmhGarVu3wsPDo0HHKC8vR25uLlxcXGBmZtbgWtLT0xEcHPxAo0ZEJB6GGhEolUokJiYiPz8fDg4O6N+/PwwMDMQui0hUHh4e6NmzZ4P379u370OshoiaIk4UbmTx8fFwc3PDoEGDMH78eAwaNAhubm6Ij48XuzQiIqImjaGmEcXHx2PMmDHw9PREUlISbt26haSkJHh6emLMmDEMNkRERA+AoaaRKJVKzJkzB88//zx2796NPn36oGXLlujTpw92796N559/HnPnzoVSqRS7VCIioiaJoaaRJCYmIjc3FwsWLIBcrvmxy+VyREZG4uLFi0hMTBSpQiIioqaNoaaR5OfnAwC6detW6/aa9pp+REREpBuGmkbi4OAAAEhLS6t1e017TT8iIiLSDUNNI+nfvz9cXFwQFRUFlUqlsU2lUiE6Ohqurq7o37+/SBUSERE1bQ0KNRs3boSLiwtMTU3h6+uL5OTkOvsOHDgQMplM6/Hcc88BAKqrqzFv3jx4enqiRYsWcHR0REhICK5evapxHBcXF61jrFy5siHli8LAwABr1qzB3r17ERgYqHH1U2BgIPbu3Yv33nuP69UQERE1kM6L7+3cuRMRERGIiYmBr68v1q1bh6FDhyIzMxO2trZa/ePj41FVVaV+fuPGDXh7e2Ps2LEAgLKyMqSmpmLRokXw9vbGX3/9hdmzZ2PEiBE4ffq0xrGWLVuGKVOmqJ9bWFjoWv5Dcb/71NSsblqbWbNm4auvvoK/v7+6zcbGBrNmzUJ5eTm2bdumtc+9VknlfWqIiIju0jnUrF27FlOmTEFoaCgAICYmBvv27cPmzZsxf/58rf5t2rTReL5jxw6Ym5urQ42VlRUOHTqk0WfDhg3o3bs38vLy0L59e3W7hYUF7O3tdS35ocpJ/w0vjOwFWPzroytXQvZXNQRDGWBrorWf7GoFAECwNoaTsxxOzuZ/b/zrL/y88wMc3WsAWBlp7lipguxGFQQZAAdT7YIUFfjmx7Nw8/B8wHdGRETUtOkUaqqqqpCSkoLIyEh1m1wuR0BAAJKSkup1jNjYWLz44oto0aJFnX2Ki4shk8nQqlUrjfaVK1fi3XffRfv27TF+/HiEh4fD0LD2t1BZWYnKykr185KSknrVdz+3LqZgymt22NTaSqP9udulWFl4A3mGhnjO2VFrv98u5gEAJjjY4ZypZuiJunYdw0vLsN2iJaKsNUOgf1k5PiooxG2ZDH4uzlrHPXrpTxTk/w4w1BARUTOnU6i5fv06lEol7OzsNNrt7OyQkZFx3/2Tk5ORlpaG2NjYOvtUVFRg3rx5eOmll2BpaalunzVrFnr27Ik2bdrgxIkTiIyMRH5+PtauXVvrcaKjo7F06dJ6vrP6O/tnGT75sACGFjc02g+UK3Hw/0dqDG0rtPbrqR6pqYKhseZUpsV/VWFJuQpCi3IYWhVpbEuuVKHn/4/UGDpc0DruEEUFvhn7+AO+KyLxyO5UoIe9HGZFWcBVca9dMCvKQg97OWR3tP8fJiL916g3tIyNjYWnpyd69+5d6/bq6mq88MILEAQBmzZt0tgWERGh/rOXlxeMjY0xbdo0REdHw8RE+3RPZGSkxj4lJSVwdtYe6dDV8KAXoJQbw93dHebm5rX2qW1OjUqlwpw5c9DOqB3C3wjHpUuXsGjRIrz77rvo0KED3n//fVz580+sWbJGa3G++82pceOcGmrCTG/nIXVaS+DnacDP4tbiASB1Wkuk384D4H+/7kSkZ3QKNdbW1jAwMEBBQYFGe0FBwX3nupSWlmLHjh1YtmxZrdtrAs2lS5dw+PBhjVGa2vj6+uLOnTvIzc1Fly5dtLabmJjUGnYelLW1NSZPnnzPPqmpqQgODq51W2FhIUJCQtTPFy1apLH9n9tqpKSkPNDdi4n0WUXL9uj50W1s27YNHu7uotaSnpGBCRMmIPbZ9vfvTER6R6dQY2xsDB8fHyQkJCAwMBDA3RGIhIQEvP766/fc95tvvkFlZWWtX/Y1gSY7Oxs//fQT2rZte99azp49C7lcXusVV2Jzd3dHSkqKRtv+/fvx9ttvIzExEebm5urRnJpRmNLSUgwYMAArVqzAsGHDtI5HJFWCoSnOKFQob9UZcOwuai3lChXOKFQQDGuZlE9Eek/n008RERGYOHEievXqhd69e2PdunUoLS1VXw0VEhICJycnREdHa+wXGxuLwMBArcBSXV2NMWPGIDU1FXv37oVSqYRCoQBw98opY2NjJCUl4dSpUxg0aBAsLCyQlJSE8PBwBAcHo3Xr1g1974+Mubm51shKzURlQ0ND9ba+ffuqt9dMtPb39+eoDBERUQPoHGrGjRuHwsJCLF68GAqFAt27d8f+/fvVk4fz8vK05oRkZmbi2LFjOHjwoNbxrly5gj179gAAunfvrrHtp59+wsCBA2FiYoIdO3bgnXfeQWVlJVxdXREeHq4xZ0bf/XNF4d27d2t8RlxRmIiI6ME1aKLw66+/XufppiNHjmi1denSBYIg1NrfxcWlzm01evbsiZMnT+pcpz6pWVF4zJgxCAwMRGRkJLp164a0tDRER0dj7969iIuL44rCREREDdSoVz81d0FBQYiLi8OcOXM0VhR2dXVFXFwcgoKCRKyOiIioaWOoaWRBQUEYOXIkEhMTkZ+fDwcHB/Tv358jNERERA+IoUYEBgYGGDhwoNhlEBERSYq4y3cSERERPSQMNURERCQJPP1ERKIztDJEbnkuTG/8veidpbEl2lm0Q6WyEueLzmvt07VtVwDAxeKLKL9TrrHNqaUTrEyscLPiJhSlCo1tLYxaoINlByhVSmT+lamxLbc8F+D0NqImi6GGiETXZlAbLMlZAuT83fZcx+ewsv9KFJQWYNzecVr7/DbxNwDAwuMLca7wnMa2qH5RGP7YcBzIPYCoU1Ea2/wd/fHRUx+h/E55rcc1MGeqIWqqGGqISHQ3f7qJTXM3adwSxNL47v3f7FrYYefzO+vcd3nf5bWO1ADAUJeh8Lbx1tjWwqgFAMDM0EzruBkZGRhdNrrhb4SIRMVQQ0Siu1N8By5mLupTSv9kYmBSa3sNVyvXOre1MW2DNqZtat1mIDfQOm6FWQWgrGfRRKR3OFGYiIiIJIGhhoiIiCSBoYaIiIgkgaGGiIiIJIGhhoiIiCSBoYaIiIgkgaGGiIiIJIGhhoiIiCSBi+8RkajKysoAAKmpqQ0+Rnl5OXJzc+Hi4gIzM7MGHyc9Pb3B+xKR+BhqiEhUGRkZAIApU6aIXMnfLCwsxC6BiBqAoYaIRBUYGAgAcHd3h7m5eYOOkZ6ejuDgYGzduhUeHh4PVI+FhQU6der0QMcgInEw1BCRqKytrTF58uQ6t5eVlalHcx6GBwlPRKTfGGqISK9lZGTAx8enXn2Dg4Pv2yclJQU9e/Z80LKISA8x1BCRXnN3d0dKSso9++gyUdjd3f1hlkdEekQmCIIgdhGNoaSkBFZWViguLoalpaXY5RAREVE96PL9zXVqiIiISBIYaoiIiEgSGGqIiIhIEhhqiIiISBIYaoiIiEgSGGqIiIhIEhhqiIiISBIYaoiIiEgSGGqIiIhIEhhqiIiISBIYaoiIiEgSGhRqNm7cCBcXF5iamsLX1xfJycl19h04cCBkMpnW47nnnlP3EQQBixcvhoODA8zMzBAQEIDs7GyN49y8eRMTJkyApaUlWrVqhUmTJuH27dsNKZ+IiIgkSOdQs3PnTkRERGDJkiVITU2Ft7c3hg4dimvXrtXaPz4+Hvn5+epHWloaDAwMMHbsWHWfVatWYf369YiJicGpU6fQokULDB06FBUVFeo+EyZMwO+//45Dhw5h7969+PnnnzF16tQGvGUiIiKSJEFHvXv3FsLCwtTPlUql4OjoKERHR9dr//fff1+wsLAQbt++LQiCIKhUKsHe3l5YvXq1uk9RUZFgYmIibN++XRAEQfjjjz8EAMIvv/yi7vO///1PkMlkwpUrV+r1usXFxQIAobi4uF79iYiISHy6fH/rNFJTVVWFlJQUBAQEqNvkcjkCAgKQlJRUr2PExsbixRdfRIsWLQAAFy9ehEKh0DimlZUVfH191cdMSkpCq1at0KtXL3WfgIAAyOVynDp1qtbXqaysRElJicaDiIiIpEunUHP9+nUolUrY2dlptNvZ2UGhUNx3/+TkZKSlpWHy5Mnqtpr97nVMhUIBW1tbje2GhoZo06ZNna8bHR0NKysr9cPZ2fn+b5CIiIiarEa9+ik2Nhaenp7o3bv3I3+tyMhIFBcXqx+XL19+5K9JRERE4tEp1FhbW8PAwAAFBQUa7QUFBbC3t7/nvqWlpdixYwcmTZqk0V6z372OaW9vrzUR+c6dO7h582adr2tiYgJLS0uNBxEREUmXTqHG2NgYPj4+SEhIULepVCokJCTAz8/vnvt+8803qKysRHBwsEa7q6sr7O3tNY5ZUlKCU6dOqY/p5+eHoqIipKSkqPscPnwYKpUKvr6+urwFIiIikihDXXeIiIjAxIkT0atXL/Tu3Rvr1q1DaWkpQkNDAQAhISFwcnJCdHS0xn6xsbEIDAxE27ZtNdplMhneeOMNLF++HJ06dYKrqysWLVoER0dHBAYGAgA8PDwwbNgwTJkyBTExMaiursbrr7+OF198EY6Ojg1860RERCQlOoeacePGobCwEIsXL4ZCoUD37t2xf/9+9UTfvLw8yOWaA0CZmZk4duwYDh48WOsx33rrLZSWlmLq1KkoKipCv379sH//fpiamqr7bNu2Da+//jqGDBkCuVyO0aNHY/369bqWT0RERBIlEwRBELuIxlBSUgIrKysUFxdzfg0REVETocv3N+/9RERERJLAUENERESSwFBDREREksBQQ0RERJLAUENERESSwFBDREREksBQQ0RERJLAUENERESSwFBDREREksBQQ0RERJLAUENERESSwFBDREREksBQQ0RERJLAUENERESSwFBDREREksBQQ0RERJLAUENERESSwFBDREREksBQQ0RERJLAUENERESSwFBDREREksBQQ0RERJLAUENERESSwFBDREREksBQQ0RERJLAUENERESSwFBDREREksBQQ0RERJLAUENERESSwFBDREREksBQQ0RERJLAUENERESSwFBDREREksBQQ0RERJLQoFCzceNGuLi4wNTUFL6+vkhOTr5n/6KiIoSFhcHBwQEmJibo3LkzfvjhB/V2FxcXyGQyrUdYWJi6z8CBA7W2T58+vSHlExERkQQZ6rrDzp07ERERgZiYGPj6+mLdunUYOnQoMjMzYWtrq9W/qqoKTz31FGxtbREXFwcnJydcunQJrVq1Uvf55ZdfoFQq1c/T0tLw1FNPYezYsRrHmjJlCpYtW6Z+bm5urmv5REREJFE6h5q1a9diypQpCA0NBQDExMRg37592Lx5M+bPn6/Vf/Pmzbh58yZOnDgBIyMjAHdHZv7JxsZG4/nKlSvx2GOP4cknn9RoNzc3h729va4lE5GEKZVKJCYmIj8/Hw4ODujfvz8MDAzELouIRKDT6aeqqiqkpKQgICDg7wPI5QgICEBSUlKt++zZswd+fn4ICwuDnZ0dunXrhqioKI2RmX+/xtatW/Hqq69CJpNpbNu2bRusra3RrVs3REZGoqysrM5aKysrUVJSovEgImmJj4+Hm5sbBg0ahPHjx2PQoEFwc3NDfHy82KURkQh0CjXXr1+HUqmEnZ2dRrudnR0UCkWt+1y4cAFxcXFQKpX44YcfsGjRIqxZswbLly+vtf/u3btRVFSEV155RaN9/Pjx2Lp1K3766SdERkbiyy+/RHBwcJ21RkdHw8rKSv1wdnbW5a0SkZ6Lj4/HmDFj4OnpiaSkJNy6dQtJSUnw9PTEmDFjGGyImiGZIAhCfTtfvXoVTk5OOHHiBPz8/NTtb731Fo4ePYpTp05p7dO5c2dUVFTg4sWL6iHhtWvXYvXq1cjPz9fqP3ToUBgbG+P777+/Zy2HDx/GkCFDkJOTg8cee0xre2VlJSorK9XPS0pK4OzsjOLiYlhaWtb3LRORHlIqlXBzc4Onpyd2794Nufzv32cqlQqBgYFIS0tDdnY2T0URNXElJSWwsrKq1/e3TnNqrK2tYWBggIKCAo32goKCOue6ODg4wMjISOMfFg8PDygUClRVVcHY2FjdfunSJfz444/1+oXl6+sLAHWGGhMTE5iYmNTrfRFR05KYmIjc3Fxs375dI9AAd0+JR0ZGwt/fH4mJiRg4cKA4RRJRo9Pp9JOxsTF8fHyQkJCgblOpVEhISNAYufmnvn37IicnByqVSt2WlZUFBwcHjUADAFu2bIGtrS2ee+65+9Zy9uxZAHdDExE1LzWjvN26dat1e017baPBRCRdOq9TExERgU8++QSff/450tPTMWPGDJSWlqqvhgoJCUFkZKS6/4wZM3Dz5k3Mnj0bWVlZ2LdvH6KiojTWoAHuhqMtW7Zg4sSJMDTUHEA6f/483n33XaSkpCA3Nxd79uxBSEgIBgwYAC8vr4a8byJqwmp+zKSlpdW6vaadP3qImhedL+keN24cCgsLsXjxYigUCnTv3h379+9XTx7Oy8vTGA52dnbGgQMHEB4eDi8vLzg5OWH27NmYN2+exnF//PFH5OXl4dVXX9V6TWNjY/z4449Yt24dSktL4ezsjNGjR2PhwoW6lk9EEtC/f3+4uLggKiqq1jk10dHRcHV1Rf/+/UWskogam04ThZsyXSYaEZH+q7n66fnnn0dkZCS6deuGtLQ0REdHY+/evYiLi0NQUJDYZRLRA3pkE4WJiPRFUFAQ4uLiMGfOHPj7+6vbXV1dGWiImimO1BBRk8YVhYmkjSM1RNRsGBgY8LJtIgLAUENETRxHaoiohs6XdBMR6Qve+4mI/omhhoiaJN77iYj+jROFiajJ4b2fiJoPXb6/OVJDRE1Ozb2fFixYUOe9ny5evIjExESRKiQiMTDUEFGTw3s/EVFtGGqIqMnhvZ+IqDYMNUTU5Pzz3k8qlUpjG+/9RNR8MdQQUZNjYGCANWvWYO/evQgMDNS4+ikwMBB79+7Fe++9x0nCRM0MF98joiaJ934ion/jJd1E1KRxRWEiaeO9n4io2eC9n4ioBufUEBERkSQw1BAREZEkMNQQERGRJDDUEBERkSQw1BAREZEkMNQQERGRJDDUEBERkSQw1BAREZEkMNQQERGRJDDUEBERkSQw1BAREZEkMNQQERGRJDDUEBERkSQw1BAREZEkGIpdADUvZWVlyMjIuGef8vJy5ObmwsXFBWZmZnX2c3d3h7m5+cMukYiImiiGGmpUGRkZ8PHxeSjHSklJQc+ePR/KsYiIqOljqKFG5e7ujpSUlHv2SU9PR3BwMLZu3QoPD497HouIiKgGQw01KnNz83qPrnh4eHAkhoiI6o2hhh6q7Oxs3Lp164GOkZ6ervHfhrKwsECnTp0e6BhERNR0yARBEHTdaePGjVi9ejUUCgW8vb3xwQcfoHfv3nX2Lyoqwttvv434+HjcvHkTHTp0wLp16/Dss88CAN555x0sXbpUY58uXbpoTCitqKjAnDlzsGPHDlRWVmLo0KH48MMPYWdnV6+aS0pKYGVlheLiYlhaWur6lqkesrOz0fWJrjBspZmVlaVKVF+vhsxIBhNHE639Ki5VAACM7Y0hN9G8IK/6ejWUpUoYWBjAqI2RxjZVhQpVBVWADDBtb6p93D8rkJWexWBDRNSE6fL9rfNIzc6dOxEREYGYmBj4+vpi3bp1GDp0KDIzM2Fra6vVv6qqCk899RRsbW0RFxcHJycnXLp0Ca1atdLo9/jjj+PHH3/8uzBDzdLCw8Oxb98+fPPNN7CyssLrr7+OoKAgHD9+XNe3QI9I4dVL6Dq8LVQBNhrtXnecMaayN27IbuO/5ge09ltWOhoA8LHpT/jT4KbGttEVT8Bb2R6nDM9jn8lZjW2P3bHFxMr+qEA1olrs0Tqu4buZuP1XIQCGGiKi5kDnULN27VpMmTIFoaGhAICYmBjs27cPmzdvxvz587X6b968GTdv3sSJEydgZHT3l7aLi4t2IYaGsLe3r/U1i4uLERsbi6+++gqDBw8GAGzZsgUeHh44efIk+vTpo+vboEeg4Pdj+LFzJQqv5Gu0W6quoN2d46iUAf5GRlr7da16HwDQ3cgQ5TKZxjanO7tgpVLhSbkcIYYGGttaqK6iw53TUALwNtY+bqdgM+QLNx7wXRERUVOhU6ipqqpCSkoKIiMj1W1yuRwBAQFISkqqdZ89e/bAz88PYWFh+O6772BjY4Px48dj3rx5MDD4+0sqOzsbjo6OMDU1hZ+fH6Kjo9G+fXsAdy/dra6uRkBAgLq/u7s72rdvj6SkpFpDTWVlJSorK9XPS0pKdHmr1AD9R01C4q67odXU9O/TQbcA1MyOyb14EQsXLnwor7d8+XKUuboCAGS1bM9v0QLtewx5KK9FRET6T6dQc/36dSiVSq15LHZ2dnUuqHbhwgUcPnwYEyZMwA8//ICcnBy89tprqK6uxpIlSwAAvr6++Oyzz9ClSxfk5+dj6dKl6N+/P9LS0mBhYQGFQgFjY2OtU1Z2dnZQKBS1vm50dLTWPB16tKwdnDHqtXfu2adDWRk+7TH0nn24+B4RETXEI7/6SaVSwdbWFh9//DEMDAzg4+ODK1euYPXq1epQ88wzz6j7e3l5wdfXFx06dMDXX3+NSZMmNeh1IyMjERERoX5eUlICZ2fnB3sz9MDqe0l33759G6EaIiKSEp1CjbW1NQwMDFBQUKDRXlBQUOd8GAcHBxgZGWmcavLw8IBCoUBVVRWMjY219mnVqhU6d+6MnJwcAIC9vT2qqqpQVFSkMVpzr9c1MTGBiYn2lTZEREQkTTrd0NLY2Bg+Pj5ISEhQt6lUKiQkJMDPz6/Wffr27YucnByoVCp1W1ZWFhwcHGoNNABw+/ZtnD9/Hg4ODgAAHx8fGBkZabxuZmYm8vLy6nxdIiIial50vkt3REQEPvnkE3z++edIT0/HjBkzUFpaqr4aKiQkRGMi8YwZM3Dz5k3Mnj0bWVlZ2LdvH6KiohAWFqbuM3fuXBw9ehS5ubk4ceIERo0aBQMDA7z00ksAACsrK0yaNAkRERH46aefkJKSgtDQUPj5+fHKJyIiIgLQgDk148aNQ2FhIRYvXgyFQoHu3btj//796snDeXl5kMv/zkrOzs44cOAAwsPD4eXlBScnJ8yePRvz5s1T9/nzzz/x0ksv4caNG7CxsUG/fv1w8uRJ2Nj8vd7J+++/D7lcjtGjR2ssvkdEREQENHBF4aaIKwoTERE1Pbp8f+t8+omIiIhIHzHUEBERkSQw1BAREZEkMNQQERGRJDDUEBERkSQw1BAREZEkMNQQERGRJDDUEBERkSQw1BAREZEkMNQQERGRJDDUEBERkSQw1BAREZEkMNQQERGRJDDUEBERkSQw1BAREZEkMNQQERGRJDDUEBERkSQw1BAREZEkMNQQERGRJDDUEBERkSQw1BAREZEkMNQQERGRJDDUEBERkSQw1BAREZEkMNQQERGRJDDUEBERkSQw1BAREZEkMNQQERGRJDDUEBERkSQw1BAREZEkMNQQERGRJDDUEBERkSQw1BAREZEkMNQQERGRJDQo1GzcuBEuLi4wNTWFr68vkpOT79m/qKgIYWFhcHBwgImJCTp37owffvhBvT06OhpPPPEELCwsYGtri8DAQGRmZmocY+DAgZDJZBqP6dOnN6R8IiIikiCdQ83OnTsRERGBJUuWIDU1Fd7e3hg6dCiuXbtWa/+qqio89dRTyM3NRVxcHDIzM/HJJ5/AyclJ3efo0aMICwvDyZMncejQIVRXV+Ppp59GaWmpxrGmTJmC/Px89WPVqlW6lk9EREQSJRMEQdBlB19fXzzxxBPYsGEDAEClUsHZ2RkzZ87E/PnztfrHxMRg9erVyMjIgJGRUb1eo7CwELa2tjh69CgGDBgA4O5ITffu3bFu3TpdylUrKSmBlZUViouLYWlp2aBjEBERUePS5ftbp5GaqqoqpKSkICAg4O8DyOUICAhAUlJSrfvs2bMHfn5+CAsLg52dHbp164aoqCgolco6X6e4uBgA0KZNG432bdu2wdraGt26dUNkZCTKysrqPEZlZSVKSko0HkRERCRdhrp0vn79OpRKJezs7DTa7ezskJGRUes+Fy5cwOHDhzFhwgT88MMPyMnJwWuvvYbq6mosWbJEq79KpcIbb7yBvn37olu3bur28ePHo0OHDnB0dMS5c+cwb948ZGZmIj4+vtbXjY6OxtKlS3V5e0RERNSE6RRqGkKlUsHW1hYff/wxDAwM4OPjgytXrmD16tW1hpqwsDCkpaXh2LFjGu1Tp05V/9nT0xMODg4YMmQIzp8/j8cee0zrOJGRkYiIiFA/LykpgbOz80N8Z0RERKRPdAo11tbWMDAwQEFBgUZ7QUEB7O3ta93HwcEBRkZGMDAwULd5eHhAoVCgqqoKxsbG6vbXX38de/fuxc8//4x27drdsxZfX18AQE5OTq2hxsTEBCYmJvV+b0RERNS06TSnxtjYGD4+PkhISFC3qVQqJCQkwM/Pr9Z9+vbti5ycHKhUKnVbVlYWHBwc1IFGEAS8/vrr2LVrFw4fPgxXV9f71nL27FkAd0MTERERkc6XdEdEROCTTz7B559/jvT0dMyYMQOlpaUIDQ0FAISEhCAyMlLdf8aMGbh58yZmz56NrKws7Nu3D1FRUQgLC1P3CQsLw9atW/HVV1/BwsICCoUCCoUC5eXlAIDz58/j3XffRUpKCnJzc7Fnzx6EhIRgwIAB8PLyetDPgIiIiCRA5zk148aNQ2FhIRYvXgyFQoHu3btj//796snDeXl5kMv/zkrOzs44cOAAwsPD4eXlBScnJ8yePRvz5s1T99m0aROAu5dt/9OWLVvwyiuvwNjYGD/++CPWrVuH0tJSODs7Y/To0Vi4cGFD3jMRERFJkM7r1DRVXKeGiIio6Xlk69QQERER6SuGGiIiIpIEhhoiIiKSBIYaIiIikgSGGiIiIpIEhhoiIiKSBIYaIiIikgSGGiIiIpIEhhoiIiKSBIYaIiIikgSGGiIiIpIEhhoiIiKSBIYaIiIikgSGGiIiIpIEhhoiIiKSBEOxCyCiu8rKypCRkVHn9vLycuTm5sLFxQVmZmb3PJa7uzvMzc0fdolERHqNoYZIT2RkZMDHx+ehHCslJQU9e/Z8KMciImoqGGqI9IS7uztSUlLq3J6eno7g4GBs3boVHh4e9z0WEVFzw1BDpCfMzc3rNbri4eHBURgiolow1BA1kuzsbNy6davB+6enp2v890FYWFigU6dOD3wcIiJ9wlBD1Aiys7PRuXPnh3Ks4ODgh3KcrKwsBhsikhSGGqJGcPuvQnTvZIzX5s2Co5Ojur2l3Ay2hq1QJdzBn9WFWvt1NHYAAFytvoFblaUoLCyEjY0NjIyNYGvQCi0NzFCiLMV1ZYnGfmYyEzgYtYFKUCG3ukBj29UrV/Hh4rW4/VchAIYaIpIOhhqiRmB6Ow9TXrPDesP9wD8yxnO3S7Gy8AbyDA0R7Oyotd9vF/MAAMsd7HDO1OTu/7F/3d0Wde06hpeWYbtFS0RZt9HYz7+sHB8VFOK2TIYXXZw1D2oIHJ1miYLbeQD8H+K7JCISF0MNUSOoaNken3xYgJXrw+Hq4qputzQyB8xsYaesws7Sq9o7PuUCAFhemo9yZaXGJicza8CoJYZWlcC74qbGthYGpkALe5gJKuy8laex7WLuRQxZMBebh7V/OG+OiEhPMNQQNYLSKhXOZlch/08LONj+HSYqAFwrVuHu/4raISO1TPX/f7LTWnzvfBkAqAC0/P/H3yoA3LhVs6/mcYtulOLsVRUEQ9OH8daIiPQGQw1RI6hZKXjKlCkiV/I3CwsLsUsgInqoGGqIGkFgYCCAB7t9gS6L790PL+kmIiliqCFqBNbW1pg8efJDORYX3yMiqh1DDZGeuN8NLXVZfI83tCSi5oihhkhP1PeGlvVZfI83tCSi5oihhkhP3O+Glv+++ul+xyIiam5kgiAIYhfRGEpKSmBlZYXi4mJYWlqKXQ4RERHVgy7f3/JGqomIiIjokWKoISIiIkloUKjZuHEjXFxcYGpqCl9fXyQnJ9+zf1FREcLCwuDg4AATExN07twZP/zwg07HrKioQFhYGNq2bYuWLVti9OjRKCjQvFEfERERNV86h5qdO3ciIiICS5YsQWpqKry9vTF06FBcu3at1v5VVVV46qmnkJubi7i4OGRmZuKTTz6Bk5OTTscMDw/H999/j2+++QZHjx7F1atXERQU1IC3TERERFKk80RhX19fPPHEE9iwYQMAQKVSwdnZGTNnzsT8+fO1+sfExGD16tXIyMiAkZFRg45ZXFwMGxsbfPXVVxgzZgyAu5e/enh4ICkpCX369Llv3ZwoTERE1PQ8sonCVVVVSElJQUBAwN8HkMsREBCApKSkWvfZs2cP/Pz8EBYWBjs7O3Tr1g1RUVFQKpX1PmZKSgqqq6s1+ri7u6N9+/Z1vm5lZSVKSko0HkRERCRdOoWa69evQ6lUws7OTqPdzs4OCoWi1n0uXLiAuLg4KJVK/PDDD1i0aBHWrFmD5cuX1/uYCoUCxsbGaNWqVb1fNzo6GlZWVuqHs7OzLm+ViIiImphHfvWTSqWCra0tPv74Y/j4+GDcuHF4++23ERMT80hfNzIyEsXFxerH5cuXH+nrERERkbh0WlHY2toaBgYGWlcdFRQUwN7evtZ9HBwcYGRkBAMDA3Wbh4cHFAoFqqqq6nVMe3t7VFVVoaioSGO05l6va2JiAhMTE13eHhERETVhOo3UGBsbw8fHBwkJCeo2lUqFhIQE+Pn51bpP3759kZOTA5VKpW7LysqCg4MDjI2N63VMHx8fGBkZafTJzMxEXl5ena9LREREzYvOp58iIiLwySef4PPPP0d6ejpmzJiB0tJShIaGAgBCQkIQGRmp7j9jxgzcvHkTs2fPRlZWFvbt24eoqCiEhYXV+5hWVlaYNGkSIiIi8NNPPyElJQWhoaHw8/Or15VPREREJH0639By3LhxKCwsxOLFi6FQKNC9e3fs379fPdE3Ly8PcvnfWcnZ2RkHDhxAeHg4vLy84OTkhNmzZ2PevHn1PiYAvP/++5DL5Rg9ejQqKysxdOhQfPjhhw/y3omIiEhCeENLIiIi0lu6fH/rPFLTVNVkN65XQ0RE1HTUfG/XZwym2YSaW7duAQDXqyEiImqCbt26BSsrq3v2aTann1QqFa5evQoLCwvIZDKxy0FJSQmcnZ1x+fJlng77F342tePnUjd+NnXjZ1M3fjZ106fPRhAE3Lp1C46OjhpzdmvTbEZq5HI52rVrJ3YZWiwtLUX/C6Ov+NnUjp9L3fjZ1I2fTd342dRNXz6b+43Q1HjkKwoTERERNQaGGiIiIpIEhhqRmJiYYMmSJbyVQy342dSOn0vd+NnUjZ9N3fjZ1K2pfjbNZqIwERERSRtHaoiIiEgSGGqIiIhIEhhqiIiISBIYaoiIiEgSms3ie6Rfzp07V+++Xl5ej7ASakoMDAyQn58PW1tbsUshIj3EUPMIBQUF1btvfHz8I6xE/3Tv3h0ymQyCINz3thVKpbKRqhLfnj176t13xIgRj7AS/cSLNYkeLqn9UGCoeYT+uayzIAjYtWsXrKys0KtXLwBASkoKioqKdAo/UnHx4kX1n8+cOYO5c+fizTffhJ+fHwAgKSkJa9aswapVq8QqURSBgYEaz2uC3z+f12hOYY/qZ/DgwYiPj0erVq3ELkXvZGdnY/Hixfjoo4+0lv0vLi7GjBkzsHz5cnTs2FGkCsUhtR8KDDWP0JYtW9R/njdvHl544QXExMTAwMAAwN0vpddee00v7qvR2Dp06KD+89ixY7F+/Xo8++yz6jYvLy84Oztj0aJFWl/0UqZSqdR//vHHHzFv3jxERUVphL2FCxciKipKrBJF9+mnn6Jly5b37DNr1qxGqka/HDlyBFVVVWKXoZdWr14NZ2fnWv+9tbKygrOzM1avXo1NmzaJUB09LFx8r5HY2Njg2LFj6NKli0Z7ZmYm/P39cePGDZEqE5+ZmRlSU1Ph4eGh0Z6eno6ePXuivLxcpMrE1a1bN8TExKBfv34a7YmJiZg6dSrS09NFqkw8NTemrflhUBuZTIYLFy40YlX6Qy6XQ6FQSOZUwsPUpUsXbN26FU888USt21NSUjB+/HhkZmY2cmXiksvlWL58uWR+KHCkppHcuXMHGRkZWqEmIyND49d5c+Th4YHo6Gh8+umnMDY2BgBUVVUhOjpaK+g0J+fPn6/1NIKVlRVyc3MbvR59cfr0aX5p38Mff/wBhUJxzz7NcfJ9Xl7ePf/eWFtb4/Lly41Ykf745xmE2shkMoYa0hQaGopJkybh/Pnz6N27NwDg1KlTWLlyJUJDQ0WuTlwxMTEYPnw42rVrp/7H9ty5c5DJZPj+++9Frk48TzzxBCIiIvDll1/Czs4OAFBQUIA333xT/XeoubnfpHIChgwZUus8iX9OzG+O87GsrKxw/vx5jVPf/5STk9MspwIA0vqhwNNPjUSlUuG9997Df//7X+Tn5wMAHBwcMHv2bMyZM+eeKbk5KC0txbZt25CRkQHg7ujN+PHj0aJFC5ErE09OTg5GjRqFrKwsODs7AwAuX76MTp06Yffu3XBzcxO5wsbH0yv3JpfLkZycDBsbm3v2q+uLXcpeeOEFVFdXY9euXbVuHzlyJIyNjfHNN980cmXiktrVTww1IigpKQGAZvurgOpPEAQcOnRII+wFBAQ02xGLpUuX4s0334S5uXmt2+Pj4/HOO+/otA6SlDD01e3MmTPw8/PD888/j7feeks9FSAjIwOrVq3Cvn37cOLECfTs2VPkShuX5P7OCNRoqqurhUOHDgkxMTFCSUmJIAiCcOXKFeHWrVsiVya+L774Qujbt6/g4OAg5ObmCoIgCGvXrhV2794tcmX6oby8XFCpVGKXoRdiYmKE0aNHCy+99JJw8uRJQRAEISEhQejevbtgbm4uTJ8+XeQKxSOTyYSCggKxy9Bb33//vWBjYyPI5XKNh42NjfDdd9+JXZ4o3nnnHaG0tLTO7d9++63g6enZiBU9GIaaRpKbmyu4u7sL5ubmgoGBgXD+/HlBEARh1qxZwrRp00SuTlwffvihYG1tLSxfvlwwNTVVfzZbtmwRBg4cKHJ14lEqlcKyZcsER0dHjb8zCxcuFD799FORqxNHdHS0YGRkJPj4+AgtWrQQzM3NhRUrVgj29vZCdHS0cPPmTbFLFNXAgQOFy5cvi12GXlq6dKlQWloqlJWVCfHx8cKqVauE//znP8KuXbvu+aXeHEjphwJDTSMZOXKkEBwcLFRWVgotW7ZUf0H99NNPgpubm8jVicvDw0PYtWuXIAiCxmfz22+/CW3bthWxMnEtXbpU6Nixo7B161bBzMxM/bns2LFD6NOnj8jViaNz587CZ599JgiCIPz888+CTCYTnnvuOeH27dsiV6Yf5HI5R2rqwM+mdlL7ocCrnxpJYmIiTpw4ob5kuYaLiwuuXLkiUlX64eLFi+jRo4dWu4mJCUpLS0WoSD988cUX+PjjjzFkyBBMnz5d3e7t7a2eY9Pc5OXlYfDgwQCA/v37w8jICEuXLm3WE8r/SeAUyTrxs6ndli1b8Mknn2DixIlITEzEk08+iRMnTiAnJ6dJ/n/Fu3Q3EpVKVetllH/++ScsLCxEqEh/uLq64uzZs1rt+/fvb9br1Fy5cqXWK5xUKhWqq6tFqEh8lZWVMDU1VT83NjZGmzZtRKxI/zTXSeT1wc9Gm9R+KHCkppE8/fTTWLduHT7++GMAd//nun37NpYsWaJxe4DmKCIiAmFhYaioqIAgCEhOTsb27dvVC/I1V127dkViYqLW5bdxcXG1jmw1F4sWLVJf/VRVVYXly5dr3GcNANauXStGaXqhc+fO9/3yvnnzZiNVo1/42WiT2g8FhppGsmbNGgwdOhRdu3ZFRUUFxo8fj+zsbFhbW2P79u1ilyeqyZMnw8zMDAsXLkRZWRnGjx8PR0dH/Pe//8WLL74odnmiWbx4MSZOnIgrV65ApVIhPj4emZmZ+OKLL7B3716xyxPFgAEDNJax9/f317olQnP/Nb506VKtkEd38bOpnZR+KHCdmkZ0584d7Ny5E7/++itu376Nnj17YsKECTAzMxO7NL1RVlaG27dvS2fNhAeUmJiIZcuWafydWbx4MZ5++mmxSyM9JLk1Rx4ifja1Gzhw4H1/CMhkMhw+fLiRKnowDDUkuvLycgiCoP6lcOnSJezatQtdu3bllzeRDqS2OuzDxM+meeBE4Uby+eefY9++fernb731Flq1agV/f39cunRJxMrEN3LkSHzxxRcAgKKiIvTu3Rtr1qzByJEjsWnTJpGrE8/ly5fx559/qp8nJyfjjTfeUM/Lao66du2qMefhtddew/Xr19XPr127Vudqw82B8P/3diJt/GyaB47UNJIuXbpg06ZNGDx4MJKSkjBkyBCsW7cOe/fuhaGhIeLj48UuUTTW1tY4evQoHn/8cXz66af44IMPcObMGXz77bdYvHgx0tPTxS5RFP3798fUqVPx8ssvQ6FQoHPnzujWrRuys7Mxc+ZMLF68WOwSG92/TyFYWlri7Nmz6NixI4C7N/x0cHCASqUSs0zRhIaG1uuLe/PmzY1QjX7hZ1O7rl274tixY+rJwa+99hqWLVsGa2trAHd/KLi4uKCsrEzMMuuNE4UbyeXLl9WX5+7evRtjxozB1KlT0bdvXwwcOFDc4kRWVlamvqz94MGDCAoKglwuR58+fZr1KFZaWpr6btxff/01PD09cfz4cRw8eBDTp09vlqHm32r7Tdacf41//vnn6NChA3r06MF1Wf6Fn03tMjIycOfOHfXzrVu3Yu7cuepQIwgCKioqxCpPZww1jaRly5a4ceMG2rdvj4MHDyIiIgIAYGpqivLycpGrE5ebmxt2796NUaNG4cCBAwgPDwdw9xdCc77pZ3V1NUxMTAAAP/74I0aMGAEAcHd3V9/pneifZsyYge3bt+PixYsIDQ1FcHBwk74892HiZ1M/Tf2HAufUNJKnnnoKkydPxuTJk5GVlaVem+b333+Hi4uLuMWJbPHixZg7dy5cXFzg6+sLPz8/AHdHbZrzeiyPP/44YmJikJiYiEOHDmHYsGEAgKtXr6Jt27YiVycOmUym9Q9sU/oH91HbuHEj8vPz8dZbb+H777+Hs7MzXnjhBRw4cKDZj07ws2keOKemkRQVFWHhwoW4fPkyZsyYof6CWrJkCYyNjfH222+LXKG4FAoF8vPz4e3tDbn8btZOTk6GpaUl3N3dRa5OHEeOHMGoUaNQUlKCiRMnqs/1L1iwABkZGc1yHpZcLke3bt1gaHh3kPncuXNwd3dX337kzp07+P3332tdvbs5unTpEj777DN88cUX6s+mZcuWYpelF/jZ3GVgYACFQgEbGxsAgIWFBc6dOwdXV1cAd+epOTo6Npn/p3j6qZG0atUKGzZs0GpfunSpCNXoH3t7e9jb22u01cwnaa4GDhyI69evo6SkBK1bt1a3T506tdle4bNkyRKN5yNHjtTqM3r06MYqR+/J5XLIZDIIgtBkvpQaCz+buwRBwJAhQ9Q/FMrLyzF8+HCNHwpNCUdqGllZWRny8vJQVVWl0e7l5SVSRfrh9OnT+Prrr2v9bJrjiARRQ1VWViI+Ph6bN2/GsWPH8PzzzyM0NBTDhg1Tj4I2V/xstNX3h/W/f1DoK4aaRlJYWIhXXnkF+/fvr3V7c/6lsGPHDoSEhGDo0KE4ePAgnn76aWRlZaGgoACjRo3Cli1bxC5RNHFxcXWGvdTUVJGq0j9VVVWoqqpqlqcP/um1117Djh074OzsjFdffRUTJkxQX8XS3PGzaR4YahrJhAkTcOnSJaxbtw4DBw7Erl27UFBQgOXLl2PNmjV47rnnxC5RNF5eXpg2bRrCwsJgYWGBX3/9Fa6urpg2bRocHBya7Sm69evX4+2338Yrr7yCjz/+GKGhoTh//jx++eUXhIWFYcWKFWKXKIotW7YgNTUVffr0wYQJExAZGYm1a9fizp07GDx4MHbs2NFsJ1LL5XK0b98ePXr0uOcE6uY4+snPRjdN9YcCQ00jcXBwwHfffYfevXvD0tISp0+fRufOnbFnzx6sWrUKx44dE7tE0bRo0UJ9FVjbtm1x5MgReHp6Ij09HYMHD262ly+7u7tjyZIleOmll9Rhr2PHjli8eDFu3rxZ6xwtqVuxYgVWrFiBvn37IjU1FS+88AJ2796NN954A3K5HOvXr8fzzz/fbFeifuWVV+p1NVhzHP3kZ1M3Sf1QEKhRWFhYCBcvXhQEQRDat28vHDt2TBAEQbhw4YJgZmYmYmXic3JyEs6dOycIgiB4enoKX331lSAIgnDixAnB0tJSzNJEZWZmJuTm5gqCIAg2NjbC2bNnBUEQhKysLKFNmzZiliYaNzc39d+PX375RZDL5UJcXJx6+w8//CC0b99erPKImpzly5cLZmZmQkBAgNCmTRth+vTpgr29vbBy5Uph1apVQrt27YTp06eLXWa98eqnRtKlSxdkZmbCxcUF3t7e+Oijj+Di4oKYmBg4ODiIXZ6oBgwYgEOHDsHT0xNjx47F7NmzcfjwYRw6dAhDhgwRuzzR2Nvb4+bNm+jQoQPat2+PkydPwtvbGxcvXmy262rk5eWhX79+AIBevXrB0NAQ3bp1U2/38vJqtiN7RA3x2WefITY2Fi+99BJOnz4NX19ffP311+qrCLt164bp06eLXGX9MdQ0ktmzZ6v/sV2yZAmGDRuGbdu2wdjYGJ999pm4xYlsw4YN6mW43377bRgZGeHEiRMYPXo0Fi5cKHJ14hk8eDD27NmDHj16IDQ0FOHh4YiLi8Pp06cRFBQkdnmi+OcqywBgbGwMIyMj9XNDQ8NmPemeSFdS+6HAOTUiKSsrQ0ZGBtq3b88Z+FQrlUoFlUqlXj9ix44dOHHiBDp16oRp06ap15FoTuRyOQ4fPqxe3t7f3x9ff/012rVrBwC4fv06nnrqKQYbonr6901i/zl/D2h6i+8x1Iig5iPn8u5/UyqV2LVrl/qO3F27dsXIkSPVX+hEgOaCaf9W0y6TyZrMP8BEYpPaDwWGmkYUGxuL999/H9nZ2QCATp064Y033sDkyZNFrkxcv//+O0aMGAGFQoEuXboAALKysmBjY4Pvv/9eYyi0ufnrr78QGxurEfZCQ0Ob7Y346nvX9g4dOjziSoikQWo/FBhqGsnixYuxdu1azJw5U33DxqSkJGzYsAHh4eFYtmyZyBWKx8/PDzY2Nvj888/VtwP466+/8Morr6CwsBAnTpwQuUJx/PzzzxgxYgQsLS3Rq1cvAEBKSgqKiorw/fffY8CAASJXqB+2b9+OESNGoEWLFmKXQtTkSO2HAkNNI7GxscH69evx0ksvabRv374dM2fOxPXr10WqTHxmZmY4ffo0Hn/8cY32tLQ0PPHEEygvLxepMnF5enrCz88PmzZtgoGBAYC7p+lee+01nDhxAr/99pvIFeoHS0tLnD17Vj0HgIiaL05YaCTV1dXqX9v/5OPj0+RuGPawde7cGQUFBVqh5tq1a3BzcxOpKvHl5OQgLi5OHWiAu3fUjYiIwBdffCFiZfqFv8uIGubcuXP17ttU7k/IUNNIXn75ZWzatAlr167VaP/4448xYcIEkaoST0lJifrP0dHRmDVrFt555x306dMHAHDy5EksW7YM//nPf8QqUXQ9e/ZEenq6ep5RjfT0dHh7e4tUFRFJRffu3eucT/NPTWlODUPNIxQREaH+s0wmw6effoqDBw+qv7hPnTqFvLw8hISEiFWiaFq1aqVx9ZcgCHjhhRfUbTX/kw0fPrzJ/M/0MPzzl9OsWbMwe/Zs5OTkaIS9jRs3YuXKlWKVqHf+97//wcnJSewyiJqcixcvil3CQ8c5NY/QoEGD6tVPJpPh8OHDj7ga/XL06NF6933yyScfYSX65V5XIvxTU/rl9CgMHjwY8fHxaNWqlUZ7SUkJAgMDm93/T0R0F0MNkR6p75UIQNO5GuFR+PeCYTWuXbsGJycnVFdXi1QZUdP2xx9/IC8vD1VVVRrtI0aMEKki3fD0kwh4CWrdPD098cMPP8DZ2VnsUkTRnINKffzz9Nwff/wBhUKhfq5UKrF//36eiiJqgAsXLmDUqFH47bffNEaLa6YENJWRYY7UiICXoNbt30t00138O3NXzek5oParnszMzPDBBx/g1VdfbezSiJq04cOHw8DAAJ9++ilcXV2RnJyMGzduYM6cOXjvvffQv39/sUusF47UiIA5knTFvzN31dyhvGPHjkhOToaNjY16m7GxMWxtbTUugSei+klKSsLhw4dhbW0NuVwOuVyOfv36qa9OPXPmjNgl1gtDDemV/v37w8zMTOwySE/VnJ5TqVR19qlZ1p2I6k+pVMLCwgIAYG1tjatXr6JLly7o0KEDMjMzRa6u/uRiF9Ac8RLUuv3www9wcHAQuwy9ExwcDEtLS7HL0BuvvPIKSktLtdpzc3N5+wiiBujWrRt+/fVXAICvry9WrVqF48ePY9myZU3qtDfn1DQSXoJatz179tTaLpPJYGpqCjc3N7i6ujZyVaTPevTogZKSEmzdulV9L7XPP/8cs2bNwuDBg7Fr1y6RKyRqWg4cOIDS0lIEBQUhJycHzz//PLKystC2bVvs3LkTgwcPFrvEemGoaSS8BLVuda3N8s87xPbr1w+7d+9W3/CyuUhISEBCQgKuXbumdcpl8+bNIlUlvurqaixYsADr16/HnDlzkJOTg//9739Yu3YtpkyZInZ5RJJw8+ZNtG7dukmdzuWcmkeMl6De36FDh/D2229jxYoV6N27NwAgOTkZixYtwsKFC2FlZYVp06Zh7ty5iI2NFbnaxrN06VIsW7YMvXr1goODQ5P6h+VRMzIywurVq2Fubo53330XhoaGOHr0qHrUhogeXJs2bcQuQWccqXnEeAnq/XXr1g0ff/wx/P39NdqPHz+OqVOn4vfff8ePP/6IV199FXl5eSJV2fgcHBywatUqvPzyy2KXoneqq6sxf/58bNy4EXPmzMGxY8eQlZWF2NhYPPvss2KXR9TkDBo06J4/nJrKFAmO1DxivAT1/s6fP1/rJFhLS0tcuHABANCpUydcv369sUsTVVVVlVbQo7t69eqFsrIyHDlyBH369IEgCFi1ahWCgoLw6quv4sMPPxS7RKImpXv37hrPq6urcfbsWaSlpWHixIniFNUAHKnRA839EtR+/frBwsICX3zxhTr0FRYWIiQkBKWlpfj555/x448/IiwsrEldWvig5s2bh5YtW2LRokVil6J3Jk2ahPXr12utyn3mzBm8/PLLSEtLE6kyIml55513cPv2bbz33ntil1IvDDWN5JVXXsHGjRu1/hHOzc3Fyy+/jMTERJEqE19mZiZGjhyJixcvqm+PcPnyZXTs2BHfffcdOnfujN27d+PWrVvN6lTM7Nmz8cUXX8DLywteXl4wMjLS2L527VqRKtNvlZWVMDExEbsMIknIyclB7969cfPmTbFLqReefmokv/76K7y8vOq8BLU569KlC/744w8cPHgQWVlZ6rannnoKcvndpZQCAwNFrFAc586dUw8J/3vkoTmP7NX48ssvERMTg4sXLyIpKQkdOnTAunXr4OrqipEjR4pdHpEkJCUlwdTUVOwy6o2hppEkJydjwYIFGDhwIC9BrYVcLsewYcMwbNgwsUvRGz/99JPYJeitTZs2YfHixXjjjTewYsUK9c32WrVqhXXr1jHUEOkoKChI47kgCMjPz8fp06eb1Clwnn5qZEuWLOElqLUoLS3F0aNHa73l/axZs0SqSn/8+eefAIB27dqJXIl+6Nq1K6KiohAYGKhxE9S0tDQMHDiw2U0qJ3pQoaGhGs/lcjlsbGwwePBgPP300yJVpTuGmkbCS1DrdubMGTz77LMoKytDaWkp2rRpg+vXr8Pc3By2trbqK6CaG5VKheXLl2PNmjW4ffs2gLt3MZ8zZw7efvtt9am55sjMzAwZGRno0KGDRqjJzs6Gl5cXysvLxS6RiETA00+NhJeg1i08PBzDhw9HTEwMrKyscPLkSRgZGSE4OBizZ88WuzzRvP3224iNjcXKlSvRt29fAMCxY8fwzjvvoKKiAitWrBC5QvG4urri7Nmz6htc1ti/fz88PDxEqoqo6bp8+TJkMpl6NDg5ORlfffUVunbtiqlTp4pcnQ4EahSvvvqqcPv2ba321NRU4fHHHxehIv1hZWUlZGRkqP/8xx9/CIIgCCdPnhS6dOkiZmmicnBwEL777jut9t27dwuOjo4iVCS+pUuXCqWlpcInn3wiODk5CTt27BBatGghbN++XVi+fLn6z0Skm379+glffPGFIAiCkJ+fL1hYWAh+fn6CtbW1sHTpUpGrqz+GGj1QUVEhdgmisra2FrKysgRBEIROnToJ+/fvFwRBENLT0wVzc3MxSxOViYmJkJmZqdWekZEhmJqailCR+ORyuVBQUCAIgiBs3bpVcHNzE2QymSCTyQQnJyfh008/FblCoqapVatW6h+X//3vfwV/f39BEAThwIEDgqurq5il6aT5npQXwZdffom+ffvC0dERly5dAgCsW7cO+/fvF7kycXzxxReoqKhAjx498MsvvwAAnnzySSxevBjbtm3DG2+8gW7duolcpXi8vb2xYcMGrfYNGzbA29tbhIrEJ/xjCuCECROQnZ2N27dvQ6FQ4M8//8SkSZNErI6o6aqurlav7/Tjjz9ixIgRAAB3d3fk5+eLWZpOGGoayaZNmxAREYFnn30WRUVFWpegNkehoaEoKSlBVFQUHBwcAAArVqxA69atMWPGDBQWFuLjjz8WuUrxrFq1Cps3b0bXrl0xadIkTJo0CV27dsVnn32G1atXi12eaP69Rk/NhHIiarjHH38cMTExSExMxKFDh9TLa1y9ehVt27YVubr649VPjYSXoGqTy+VQKBT8QrqHq1evYuPGjcjIyAAAeHh44LXXXoOjo6PIlYlDLpfDysrqvosPNpXVT4n0xZEjRzBq1CiUlJRg4sSJ2Lx5MwBgwYIFyMjIQHx8vMgV1g+vfmokFy9eRI8ePbTaTUxMUFpaKkJF+oEr496bo6Njs77KqTZLly6FlZWV2GUQSUrNj+uSkhK0bt1a3T516lSYm5uLWJluGGoaCS9Brd2QIUNgaHjvv4apqamNVI34zp07h27dukEul+PcuXP37Ovl5dVIVemXF198kaN7RI+AgYGBRqABABcXF3GKaSCGmkds2bJlmDt3LiIiIhAWFoaKigoIgoDk5GRs374d0dHR+PTTT8UuUzRDhw5Fy5YtxS5Db3Tv3l19Sq579+6QyWSo7QyxTCZTz8tqTjiyR/Tw9OzZEwkJCWjdujV69Ohxz/+/msqPS4aaR2zp0qWYPn06Jk+eDDMzMyxcuBBlZWUYP348HB0d8d///hcvvvii2GWK5s033+Sv7n+4ePEibGxs1H8mTZwCSPTwjBw5ElevXkXr1q0lc9NgThR+xGqbDFtWVobbt283+y9zAwMD5OfnN/vPoS4///wz/P39tU7P3blzBydOnMCAAQNEqoyIpEIul+OJJ57ApEmT8NJLL8HCwkLskh4IL+luBLwEtXbM0/c2aNCgWq/iKS4uxqBBg0SoiIik5ujRo3j88ccxd+5cODg44JVXXkFiYqLYZTUYR2oeMV6CWrdLly6hffv2nCdRB7lcjoKCAvXpqBpZWVno1asXSkpKRKqMiKSmtLQUX3/9NT777DMkJibCzc0NkyZNwsSJE2Fvby92efXGUPOIyeVyrFu37r6XoE6cOLGRKiJ9FxQUBAD47rvvMGzYMPUqnwCgVCpx7tw5dOnSpdmuRE1Ej1ZOTg62bNmCL7/8EgqFAsOGDcOePXvELqteOFG4EfASVNJFTQAWBAEWFhYwMzNTbzM2NkafPn0wZcoUscojIolzc3PDggUL0KFDB0RGRmLfvn1il1RvDDWPGE+tkK62bNkC4O76EHPnzkWLFi1EroiImouff/4Zmzdvxrfffgu5XI4XXnihSd1TjaefHjHeCoAe1LVr15CZmQkA6NKlC/8uEdFDdfXqVXz22Wf47LPPkJOTA39/f0yaNAkvvPBCk/tRxaufHjGVSsUvoXpITExEcHAw/Pz8cOXKFQB372p+7NgxkSsTz61bt/Dyyy/DyckJTz75JJ588kk4OTkhODgYxcXFYpdHRBLwzDPPoEOHDvjggw8watQopKen49ixYwgNDW1ygQZgqCE98O2332Lo0KEwMzPDmTNnUFlZCeDupctRUVEiVyeeyZMn49SpU9i7dy+KiopQVFSEvXv34vTp05g2bZrY5RGRBBgZGSEuLg5//vkn/vOf/6BLly5il/RAePqJRNejRw+Eh4cjJCRE4w7mZ86cwTPPPAOFQiF2iaJo0aIFDhw4gH79+mm0JyYmYtiwYc36RqhERLXhSA2JLjMzs9bVca2srFBUVNT4BemJtm3b1roUgJWVldZN54iIiKGG9IC9vT1ycnK02o8dO4aOHTuKUJF+WLhwISIiIjRGqhQKBd58800sWrRIxMqIiPQTL+km0U2ZMgWzZ8/G5s2bIZPJcPXqVSQlJWHu3LnN+st706ZNyMnJQfv27dG+fXsAQF5eHkxMTFBYWIiPPvpI3bep3EGXiOhRYqgh0c2fPx8qlQpDhgxBWVkZBgwYABMTE8ydOxczZ84UuzzRSOWuuUREjYUThUlvVFVVIScnB7dv30bXrl3RsmVLsUsiIqImhKGGiIiIJIGnn0h0o0aNqvV2EjKZDKampnBzc8P48eOb/PoJulIqlXj//ffx9ddfIy8vD1VVVRrbm+Od3YmI7oVXP5HorKyscPjwYaSmpkImk0Emk+HMmTM4fPgw7ty5g507d8Lb2xvHjx8Xu9RGtXTpUqxduxbjxo1DcXExIiIiEBQUBLlcjnfeeUfs8oiI9A5PP5Ho5s+fj5KSEmzYsAFy+d2crVKpMHv2bFhYWGDFihWYPn06fv/992Z124THHnsM69evx3PPPQcLCwucPXtW3Xby5El89dVXYpdIRKRXGGpIdDY2Njh+/Dg6d+6s0Z6VlQV/f39cv34dv/32G/r379+sFuNr0aIF0tPT0b59ezg4OGDfvn3o2bMnLly4gB49evD+T0RE/8LTTyS6O3fuICMjQ6s9IyMDSqUSAGBqalrrvBspa9euHfLz8wHcHbU5ePAgAOCXX36BiYmJmKUREeklThQm0b388suYNGkSFixYgCeeeALA3S/uqKgohISEAACOHj2Kxx9/XMwyG92oUaOQkJAAX19fzJw5E8HBwYiNjUVeXh7Cw8PFLo+ISO/w9BOJTqlUYuXKldiwYQMKCgoAAHZ2dpg5cybmzZsHAwMD5OXlQS6Xo127diJXK56TJ0/ixIkT6NSpE4YPHy52OUREeoehhvRKSUkJAMDS0lLkSoiIqKnhnBrSK5aWls0+0KSkpGDQoEHqgPdPxcXFGDRoEH799VcRKiMi0m+cU0N6IS4urs5F5prbzRrXrFmDwYMH1xrurKysEBAQgNWrV2Pr1q0iVEdEpL84UkOiW79+PUJDQ2FnZ4czZ86gd+/eaNu2LS5cuIBnnnlG7PIa3alTpzBy5Mg6t48YMQInTpxoxIqIiJoGhhoS3YcffoiPP/4YH3zwAYyNjfHWW2/h0KFDmDVrVrNci+XKlSuwsLCoc3vLli3Vl3oTEdHfGGpIdHl5efD39wcAmJmZ4datWwDuXuq9fft2MUsThY2NDTIzM+vcnpGRAWtr60asiIioaWCoIdHZ29urb87Yvn17nDx5EgBw8eJFNMeL8wICArBixYpatwmCgBUrViAgIKCRqyIi0n+cKEyiGzx4MPbs2YMePXogNDQU4eHhiIuLw+nTpxEUFCR2eY1u4cKF8PHxga+vL+bMmaO+O3lGRgbWrFmDrKwsfPbZZ+IWSUSkh7hODYlOpVJBpVLB0PBuxt6xY4d6kblp06bB2NhY5Aob3+nTp/HKK6/gjz/+UN8eQhAEdO3aFVu2bFGvvExERH9jqCHSY2fPnkV2djYEQUDnzp3RvXt3sUsiItJbDDWkFyoqKnDu3Dlcu3YNKpVKY9uIESNEqoqIiJoSzqkh0e3fvx8hISG4fv261jaZTKa+UzcREdG98OonEt3MmTMxduxY5Ofnq+fX1DwYaIiIqL54+olEZ2lpiTNnzuCxxx4TuxQiImrCOFJDohszZgyOHDkidhlERNTEcaSGRFdWVoaxY8fCxsYGnp6eMDIy0tg+a9YskSoTX2JiIj766COcP38ecXFxcHJywpdffglXV1f069dP7PKIiPQKJwqT6LZv346DBw/C1NQUR44cUa/LAtydKNxcQ823336Ll19+GRMmTMCZM2dQWVkJACguLkZUVBR++OEHkSskItIvHKkh0dnb22PWrFmYP38+5HKeEa3Ro0cPhIeHIyQkBBYWFvj111/RsWNHnDlzBs888wwUCoXYJRIR6RV+g5DoqqqqMG7cOAaaf8nMzMSAAQO02q2srFBUVNT4BRER6Tl+i5DoJk6ciJ07d4pdht6xt7dHTk6OVvuxY8fQsWNHESoiItJvnFNDolMqlVi1ahUOHDgALy8vrYnCa9euFakycU2ZMgWzZ8/G5s2bIZPJcPXqVSQlJWHu3LlYtGiR2OUREekdhhoS3W+//YYePXoAANLS0jS2/XPScHMzf/58qFQqDBkyBGVlZRgwYABMTEwwd+5czJw5U+zyiIj0DicKE+m5qqoq5OTk4Pbt2+jatStatmwpdklERHqJoYaIiIgkgaefSDRBQUH16hcfH/+IK9FPo0aNqvX0m0wmg6mpKdzc3DB+/Hh06dJFhOqIiPQPr34i0VhZWdXr0VxZWVnh8OHDSE1NhUwmg0wmw5kzZ3D48GHcuXMHO3fuhLe3N44fPy52qUREeoGnn4j01Pz581FSUoINGzao1/BRqVSYPXs2LCwssGLFCkyfPh2///47jh07JnK1RETiY6gh0lM2NjY4fvw4OnfurNGelZUFf39/XL9+Hb/99hv69+/PxfiIiMDTT0R6686dO8jIyNBqz8jIgFKpBACYmpo268veiYj+iROFifTUyy+/jEmTJmHBggV44oknAAC//PILoqKiEBISAgA4evQoHn/8cTHLJCLSGzz9RKSnlEolVq5ciQ0bNqCgoAAAYGdnh5kzZ2LevHkwMDBAXl4e5HI52rVrJ3K1RETiY6ghagJKSkoAAJaWliJXQkSkvxhqiIiISBI4p4ZIj8XFxeHrr79GXl4eqqqqNLalpqaKVBURkX7i1U9Eemr9+vUIDQ2FnZ0dzpw5g969e6Nt27a4cOECnnnmGbHLIyLSOzz9RKSn3N3dsWTJErz00kuwsLDAr7/+io4dO2Lx4sW4efMmNmzYIHaJRER6hSM1RHoqLy8P/v7+AAAzMzPcunULwN1Lvbdv3y5maUREeomhhkhP2dvb4+bNmwCA9u3b4+TJkwCAixcvggOsRETaGGqI9NTgwYOxZ88eAEBoaCjCw8Px1FNPYdy4cRg1apTI1RER6R/OqSHSUyqVCiqVCoaGdy9S3LFjB06cOIFOnTph2rRpMDY2FrlCIiL9wlBDREREksB1aoj0WEVFBc6dO4dr165BpVJpbBsxYoRIVRER6SeGGiI9tX//foSEhOD69eta22QymfpO3UREdBcnChPpqZkzZ2Ls2LHIz89Xz6+peTDQEBFp45waIj1laWmJM2fO4LHHHhO7FCKiJoEjNUR6asyYMThy5IjYZRARNRkcqSHSU2VlZRg7dixsbGzg6ekJIyMjje2zZs0SqTIiIv3EUEOkp2JjYzF9+nSYmpqibdu2kMlk6m0ymQwXLlwQsToiIv3DUEOkp+zt7TFr1izMnz8fcjnPFBMR3Q//pSTSU1VVVRg3bhwDDRFRPfFfSyI9NXHiROzcuVPsMoiImgwuvkekp5RKJVatWoUDBw7Ay8tLa6Lw2rVrRaqMiEg/cU4NkZ4aNGhQndtkMhkOHz7ciNUQEek/hhoiIiKSBM6pISIiIkngnBoiPRMUFFSvfvHx8Y+4EiKipoWhhkjPWFlZiV0CEVGTxDk1REREJAmcU0NERESSwFBDREREksBQQ0RERJLAUENERESSwFBDREREksBQQ0RERJLAUENERESSwFBDREREkvB/xL7eyLnEHRkAAAAASUVORK5CYII=",
      "text/plain": [
       "<Figure size 640x480 with 1 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "\n",
    "fig, ax = plt.subplots()\n",
    "ax.boxplot([aucs_text_test, aucs_tags_test, aucs_capt_tags, aucs_bert_test, aucs_MTT_test, aucs_MTC_test, aucs_vbert_test], meanline=True, showmeans=True)\n",
    "plt.xticks([1, 2, 3, 4, 5, 6, 7], [\"Text-based\", \"Image Tag-based\", \"Image Caption-based\", \"Text-BERT\", \"MTT\", \"MTC\", \"VisualBERT\"], rotation=90)\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 56,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[0.8183228518518519,\n",
       " 0.7067193580246913,\n",
       " 0.811590888888889,\n",
       " 0.7874842129629629,\n",
       " 0.8365212539682539,\n",
       " 0.7621710841750842,\n",
       " 0.7467604814814814,\n",
       " 0.8475135238095238,\n",
       " 0.7410681534391533,\n",
       " 0.8056298293650794]"
      ]
     },
     "execution_count": 56,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "aucs_vbert_bias =list({0: 0.8183228518518519, 1: 0.7067193580246913, 2: 0.811590888888889, 3: 0.7874842129629629, 4: 0.8365212539682539, 5: 0.7621710841750842, 6: 0.7467604814814814, 7: 0.8475135238095238, 8: 0.7410681534391533, 9: 0.8056298293650794}.values())\n",
    "aucs_vbert_bias"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Load Synthetic"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 57,
   "metadata": {},
   "outputs": [],
   "source": [
    "syn_df = load_data.load_syn_identity_data()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 58,
   "metadata": {},
   "outputs": [],
   "source": [
    "annotation_folder = './Annotations/synthetic/'\n",
    "def get_names_from_ids(iteration):\n",
    "        f = open(annotation_folder+'test_'+str(iteration)+'.jsonl')\n",
    "        data = []\n",
    "        names = []\n",
    "        for line in f:\n",
    "                data.append(json.loads(line))\n",
    "        #data = json.load(f)\n",
    "        for dict in data:\n",
    "                names.append(dict['img'].split('.')[0]+'.jpg')\n",
    "        return names"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 59,
   "metadata": {},
   "outputs": [],
   "source": [
    "path_pred_syn = \"./VisualBert/Predictions/Synthetic/\"\n",
    "pred_syn=[]\n",
    "syn_10_df=pd.DataFrame()\n",
    "i=1\n",
    "for MODELNAME in MODELNAMES_VISUAL_BERT:\n",
    "    file_name = ''\n",
    "    # read the file corresponding to the iteration (we can't no the time-token a priori)\n",
    "    for file in os.listdir(path_pred_syn):\n",
    "        if file.endswith(\".csv\") and file.startswith(\"syn_\" + str(i)+'.'):\n",
    "            file_name = file\n",
    "            break\n",
    "    if file_name == '':\n",
    "        raise Exception('Predictions for iteration {i} on synthetic data not found.'.format(i=i))\n",
    "    pred = pd.read_csv(path_pred_syn + file_name, usecols = ['id','proba'])\n",
    "    pred.columns = ['file_name', MODELNAME]\n",
    "    \n",
    "    #syn_10_df['label_'+MODELNAME]=pred[MODELNAME + '_label']\n",
    "    syn_10_df[MODELNAME]= pred[MODELNAME]\n",
    "    syn_10_df['file_name']=get_names_from_ids(i)\n",
    "    syn_10_df=syn_10_df.merge(load_data.load_syn_identity_data()[['misogynous', 'file_name']],\n",
    "                        how='inner', on='file_name')\n",
    "    syn_10_df.rename(columns={'misogynous':'label_'+str(i-1)}, inplace=True)#real label associated to the split\n",
    "    syn_10_df.rename(columns={'file_name':'file_name_'+str(i-1)}, inplace=True)\n",
    "\n",
    "    i=i+1"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 60,
   "metadata": {},
   "outputs": [],
   "source": [
    "path_azure_syn = './Multimodal/predictions/predictions_syn/baseline_multimodal_text_captions_SYN_scores.tsv'\n",
    "path_azure_BO_syn = './Mitigated/predictions/MTC_syn_10_df.csv'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 61,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "Index(['model_VisualBERT_v0', 'file_name_0', 'label_0', 'model_VisualBERT_v1',\n",
       "       'file_name_1', 'label_1', 'model_VisualBERT_v2', 'file_name_2',\n",
       "       'label_2', 'model_VisualBERT_v3', 'file_name_3', 'label_3',\n",
       "       'model_VisualBERT_v4', 'file_name_4', 'label_4', 'model_VisualBERT_v5',\n",
       "       'file_name_5', 'label_5', 'model_VisualBERT_v6', 'file_name_6',\n",
       "       'label_6', 'model_VisualBERT_v7', 'file_name_7', 'label_7',\n",
       "       'model_VisualBERT_v8', 'file_name_8', 'label_8', 'model_VisualBERT_v9',\n",
       "       'file_name_9', 'label_9', 'multimodal_text_captions_v0',\n",
       "       'multimodal_text_captions_v1', 'multimodal_text_captions_v2',\n",
       "       'multimodal_text_captions_v3', 'multimodal_text_captions_v4',\n",
       "       'multimodal_text_captions_v5', 'multimodal_text_captions_v6',\n",
       "       'multimodal_text_captions_v7', 'multimodal_text_captions_v8',\n",
       "       'multimodal_text_captions_v9'],\n",
       "      dtype='object')"
      ]
     },
     "execution_count": 61,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Load azure predictions on SYN\n",
    "pred = pd.read_csv(path_azure_syn, sep='\\t').drop(columns=label_column)\n",
    "\n",
    "a = int(len(pred)/10 )\n",
    "for i in range(0,10):\n",
    "    beg = a*i\n",
    "    end = a*(i+1)\n",
    "    syn_10_df['multimodal_text_captions_v'+str(i)] = list(pred['multimodal_text_captions_v'+str(i)][beg:end])\n",
    "syn_10_df.columns"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 62,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>model_VisualBERT_v0</th>\n",
       "      <th>file_name_0</th>\n",
       "      <th>label_0</th>\n",
       "      <th>model_VisualBERT_v1</th>\n",
       "      <th>file_name_1</th>\n",
       "      <th>label_1</th>\n",
       "      <th>model_VisualBERT_v2</th>\n",
       "      <th>file_name_2</th>\n",
       "      <th>label_2</th>\n",
       "      <th>model_VisualBERT_v3</th>\n",
       "      <th>...</th>\n",
       "      <th>multimodal_text_captions_BO_v0</th>\n",
       "      <th>multimodal_text_captions_BO_v1</th>\n",
       "      <th>multimodal_text_captions_BO_v2</th>\n",
       "      <th>multimodal_text_captions_BO_v3</th>\n",
       "      <th>multimodal_text_captions_BO_v4</th>\n",
       "      <th>multimodal_text_captions_BO_v5</th>\n",
       "      <th>multimodal_text_captions_BO_v6</th>\n",
       "      <th>multimodal_text_captions_BO_v7</th>\n",
       "      <th>multimodal_text_captions_BO_v8</th>\n",
       "      <th>multimodal_text_captions_BO_v9</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>0.012071</td>\n",
       "      <td>SYN_49.jpg</td>\n",
       "      <td>0</td>\n",
       "      <td>0.996255</td>\n",
       "      <td>SYN_20.jpg</td>\n",
       "      <td>1</td>\n",
       "      <td>0.998068</td>\n",
       "      <td>SYN_40.jpg</td>\n",
       "      <td>0</td>\n",
       "      <td>0.846515</td>\n",
       "      <td>...</td>\n",
       "      <td>0.089251</td>\n",
       "      <td>0.891331</td>\n",
       "      <td>0.407616</td>\n",
       "      <td>0.780110</td>\n",
       "      <td>0.556986</td>\n",
       "      <td>0.501726</td>\n",
       "      <td>0.859859</td>\n",
       "      <td>0.860225</td>\n",
       "      <td>0.969511</td>\n",
       "      <td>0.941125</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>0.026192</td>\n",
       "      <td>SIN_51.jpg</td>\n",
       "      <td>0</td>\n",
       "      <td>0.001039</td>\n",
       "      <td>SIN_170.jpg</td>\n",
       "      <td>1</td>\n",
       "      <td>0.000045</td>\n",
       "      <td>SIN_104.jpg</td>\n",
       "      <td>0</td>\n",
       "      <td>0.991903</td>\n",
       "      <td>...</td>\n",
       "      <td>0.647151</td>\n",
       "      <td>0.664549</td>\n",
       "      <td>0.168989</td>\n",
       "      <td>0.839904</td>\n",
       "      <td>0.871047</td>\n",
       "      <td>0.028915</td>\n",
       "      <td>0.581634</td>\n",
       "      <td>0.940612</td>\n",
       "      <td>0.320404</td>\n",
       "      <td>0.799684</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>0.997714</td>\n",
       "      <td>SYN_69.jpg</td>\n",
       "      <td>0</td>\n",
       "      <td>0.992926</td>\n",
       "      <td>SIN_105.jpg</td>\n",
       "      <td>1</td>\n",
       "      <td>0.981816</td>\n",
       "      <td>SYN_44.jpg</td>\n",
       "      <td>1</td>\n",
       "      <td>0.005895</td>\n",
       "      <td>...</td>\n",
       "      <td>0.220379</td>\n",
       "      <td>0.014468</td>\n",
       "      <td>0.740198</td>\n",
       "      <td>0.009989</td>\n",
       "      <td>0.095283</td>\n",
       "      <td>0.729122</td>\n",
       "      <td>0.998462</td>\n",
       "      <td>0.025532</td>\n",
       "      <td>0.823892</td>\n",
       "      <td>0.793365</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>0.717852</td>\n",
       "      <td>SYN_114.jpg</td>\n",
       "      <td>1</td>\n",
       "      <td>0.996592</td>\n",
       "      <td>SIN_113.jpg</td>\n",
       "      <td>1</td>\n",
       "      <td>0.736905</td>\n",
       "      <td>SIN_41.jpg</td>\n",
       "      <td>1</td>\n",
       "      <td>0.845425</td>\n",
       "      <td>...</td>\n",
       "      <td>0.952061</td>\n",
       "      <td>0.990917</td>\n",
       "      <td>0.971473</td>\n",
       "      <td>0.542988</td>\n",
       "      <td>0.580493</td>\n",
       "      <td>0.603140</td>\n",
       "      <td>0.989025</td>\n",
       "      <td>0.535257</td>\n",
       "      <td>0.003780</td>\n",
       "      <td>0.907331</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>0.959301</td>\n",
       "      <td>SYN_89.jpg</td>\n",
       "      <td>1</td>\n",
       "      <td>0.996598</td>\n",
       "      <td>SYN_14.jpg</td>\n",
       "      <td>0</td>\n",
       "      <td>0.000080</td>\n",
       "      <td>SYN_13.jpg</td>\n",
       "      <td>1</td>\n",
       "      <td>0.916593</td>\n",
       "      <td>...</td>\n",
       "      <td>0.964493</td>\n",
       "      <td>0.828906</td>\n",
       "      <td>0.766104</td>\n",
       "      <td>0.061438</td>\n",
       "      <td>0.378935</td>\n",
       "      <td>0.877839</td>\n",
       "      <td>0.453970</td>\n",
       "      <td>0.519785</td>\n",
       "      <td>0.054727</td>\n",
       "      <td>0.974454</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>0.878191</td>\n",
       "      <td>SYN_121.jpg</td>\n",
       "      <td>0</td>\n",
       "      <td>0.980748</td>\n",
       "      <td>SIN_114.jpg</td>\n",
       "      <td>1</td>\n",
       "      <td>0.010145</td>\n",
       "      <td>SYN_90.jpg</td>\n",
       "      <td>1</td>\n",
       "      <td>0.000382</td>\n",
       "      <td>...</td>\n",
       "      <td>0.119377</td>\n",
       "      <td>0.102780</td>\n",
       "      <td>0.127394</td>\n",
       "      <td>0.027388</td>\n",
       "      <td>0.864355</td>\n",
       "      <td>0.022900</td>\n",
       "      <td>0.947713</td>\n",
       "      <td>0.856871</td>\n",
       "      <td>0.393374</td>\n",
       "      <td>0.436365</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6</th>\n",
       "      <td>0.999637</td>\n",
       "      <td>SYN_1.jpg</td>\n",
       "      <td>0</td>\n",
       "      <td>0.751023</td>\n",
       "      <td>SYN_21.jpg</td>\n",
       "      <td>0</td>\n",
       "      <td>0.977857</td>\n",
       "      <td>SYN_4.jpg</td>\n",
       "      <td>0</td>\n",
       "      <td>0.870523</td>\n",
       "      <td>...</td>\n",
       "      <td>0.536092</td>\n",
       "      <td>0.031560</td>\n",
       "      <td>0.603495</td>\n",
       "      <td>0.687838</td>\n",
       "      <td>0.878386</td>\n",
       "      <td>0.007514</td>\n",
       "      <td>0.930981</td>\n",
       "      <td>0.811757</td>\n",
       "      <td>0.807037</td>\n",
       "      <td>0.550162</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7</th>\n",
       "      <td>0.004084</td>\n",
       "      <td>SYN_23.jpg</td>\n",
       "      <td>1</td>\n",
       "      <td>0.995678</td>\n",
       "      <td>SYN_80.jpg</td>\n",
       "      <td>1</td>\n",
       "      <td>0.999970</td>\n",
       "      <td>SYN_108.jpg</td>\n",
       "      <td>1</td>\n",
       "      <td>0.001423</td>\n",
       "      <td>...</td>\n",
       "      <td>0.092685</td>\n",
       "      <td>0.647242</td>\n",
       "      <td>0.986107</td>\n",
       "      <td>0.003285</td>\n",
       "      <td>0.210502</td>\n",
       "      <td>0.327350</td>\n",
       "      <td>0.037642</td>\n",
       "      <td>0.872981</td>\n",
       "      <td>0.978534</td>\n",
       "      <td>0.711779</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8</th>\n",
       "      <td>0.999221</td>\n",
       "      <td>SYN_10.jpg</td>\n",
       "      <td>1</td>\n",
       "      <td>0.004510</td>\n",
       "      <td>SYN_85.jpg</td>\n",
       "      <td>1</td>\n",
       "      <td>0.000049</td>\n",
       "      <td>SYN_75.jpg</td>\n",
       "      <td>0</td>\n",
       "      <td>0.751444</td>\n",
       "      <td>...</td>\n",
       "      <td>0.921204</td>\n",
       "      <td>0.008743</td>\n",
       "      <td>0.037978</td>\n",
       "      <td>0.753439</td>\n",
       "      <td>0.935790</td>\n",
       "      <td>0.180489</td>\n",
       "      <td>0.820333</td>\n",
       "      <td>0.108910</td>\n",
       "      <td>0.968572</td>\n",
       "      <td>0.875829</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9</th>\n",
       "      <td>0.957423</td>\n",
       "      <td>SIN_64.jpg</td>\n",
       "      <td>0</td>\n",
       "      <td>0.993563</td>\n",
       "      <td>SYN_59.jpg</td>\n",
       "      <td>1</td>\n",
       "      <td>0.980057</td>\n",
       "      <td>SIN_11.jpg</td>\n",
       "      <td>0</td>\n",
       "      <td>0.844976</td>\n",
       "      <td>...</td>\n",
       "      <td>0.446291</td>\n",
       "      <td>0.973507</td>\n",
       "      <td>0.560534</td>\n",
       "      <td>0.898820</td>\n",
       "      <td>0.981116</td>\n",
       "      <td>0.012856</td>\n",
       "      <td>0.340941</td>\n",
       "      <td>0.688720</td>\n",
       "      <td>0.909187</td>\n",
       "      <td>0.861869</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>10</th>\n",
       "      <td>0.028079</td>\n",
       "      <td>SIN_2.jpg</td>\n",
       "      <td>0</td>\n",
       "      <td>0.974318</td>\n",
       "      <td>SYN_38.jpg</td>\n",
       "      <td>1</td>\n",
       "      <td>0.000057</td>\n",
       "      <td>SIN_32.jpg</td>\n",
       "      <td>0</td>\n",
       "      <td>0.000159</td>\n",
       "      <td>...</td>\n",
       "      <td>0.326331</td>\n",
       "      <td>0.997803</td>\n",
       "      <td>0.118442</td>\n",
       "      <td>0.136822</td>\n",
       "      <td>0.981144</td>\n",
       "      <td>0.297222</td>\n",
       "      <td>0.094026</td>\n",
       "      <td>0.021488</td>\n",
       "      <td>0.993289</td>\n",
       "      <td>0.832169</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>11</th>\n",
       "      <td>0.004144</td>\n",
       "      <td>SYN_104.jpg</td>\n",
       "      <td>0</td>\n",
       "      <td>0.987357</td>\n",
       "      <td>SYN_8.jpg</td>\n",
       "      <td>1</td>\n",
       "      <td>0.000031</td>\n",
       "      <td>SYN_123.jpg</td>\n",
       "      <td>1</td>\n",
       "      <td>0.978204</td>\n",
       "      <td>...</td>\n",
       "      <td>0.006374</td>\n",
       "      <td>0.777985</td>\n",
       "      <td>0.179125</td>\n",
       "      <td>0.977471</td>\n",
       "      <td>0.681183</td>\n",
       "      <td>0.258521</td>\n",
       "      <td>0.949793</td>\n",
       "      <td>0.752548</td>\n",
       "      <td>0.630425</td>\n",
       "      <td>0.831105</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>12</th>\n",
       "      <td>0.410500</td>\n",
       "      <td>SIN_1.jpg</td>\n",
       "      <td>0</td>\n",
       "      <td>0.996465</td>\n",
       "      <td>SIN_101.jpg</td>\n",
       "      <td>0</td>\n",
       "      <td>0.998618</td>\n",
       "      <td>SYN_57.jpg</td>\n",
       "      <td>1</td>\n",
       "      <td>0.003838</td>\n",
       "      <td>...</td>\n",
       "      <td>0.540545</td>\n",
       "      <td>0.520195</td>\n",
       "      <td>0.386572</td>\n",
       "      <td>0.532256</td>\n",
       "      <td>0.962692</td>\n",
       "      <td>0.248363</td>\n",
       "      <td>0.534641</td>\n",
       "      <td>0.181502</td>\n",
       "      <td>0.796783</td>\n",
       "      <td>0.948646</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>13</th>\n",
       "      <td>0.010282</td>\n",
       "      <td>SYN_50.jpg</td>\n",
       "      <td>0</td>\n",
       "      <td>0.994869</td>\n",
       "      <td>SIN_60.jpg</td>\n",
       "      <td>0</td>\n",
       "      <td>0.999872</td>\n",
       "      <td>SYN_39.jpg</td>\n",
       "      <td>0</td>\n",
       "      <td>0.164859</td>\n",
       "      <td>...</td>\n",
       "      <td>0.097761</td>\n",
       "      <td>0.985648</td>\n",
       "      <td>0.130455</td>\n",
       "      <td>0.085414</td>\n",
       "      <td>0.058129</td>\n",
       "      <td>0.100489</td>\n",
       "      <td>0.868502</td>\n",
       "      <td>0.233986</td>\n",
       "      <td>0.705600</td>\n",
       "      <td>0.932851</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>14 rows × 50 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "    model_VisualBERT_v0  file_name_0  label_0  model_VisualBERT_v1  \\\n",
       "0              0.012071   SYN_49.jpg        0             0.996255   \n",
       "1              0.026192   SIN_51.jpg        0             0.001039   \n",
       "2              0.997714   SYN_69.jpg        0             0.992926   \n",
       "3              0.717852  SYN_114.jpg        1             0.996592   \n",
       "4              0.959301   SYN_89.jpg        1             0.996598   \n",
       "5              0.878191  SYN_121.jpg        0             0.980748   \n",
       "6              0.999637    SYN_1.jpg        0             0.751023   \n",
       "7              0.004084   SYN_23.jpg        1             0.995678   \n",
       "8              0.999221   SYN_10.jpg        1             0.004510   \n",
       "9              0.957423   SIN_64.jpg        0             0.993563   \n",
       "10             0.028079    SIN_2.jpg        0             0.974318   \n",
       "11             0.004144  SYN_104.jpg        0             0.987357   \n",
       "12             0.410500    SIN_1.jpg        0             0.996465   \n",
       "13             0.010282   SYN_50.jpg        0             0.994869   \n",
       "\n",
       "    file_name_1  label_1  model_VisualBERT_v2  file_name_2  label_2  \\\n",
       "0    SYN_20.jpg        1             0.998068   SYN_40.jpg        0   \n",
       "1   SIN_170.jpg        1             0.000045  SIN_104.jpg        0   \n",
       "2   SIN_105.jpg        1             0.981816   SYN_44.jpg        1   \n",
       "3   SIN_113.jpg        1             0.736905   SIN_41.jpg        1   \n",
       "4    SYN_14.jpg        0             0.000080   SYN_13.jpg        1   \n",
       "5   SIN_114.jpg        1             0.010145   SYN_90.jpg        1   \n",
       "6    SYN_21.jpg        0             0.977857    SYN_4.jpg        0   \n",
       "7    SYN_80.jpg        1             0.999970  SYN_108.jpg        1   \n",
       "8    SYN_85.jpg        1             0.000049   SYN_75.jpg        0   \n",
       "9    SYN_59.jpg        1             0.980057   SIN_11.jpg        0   \n",
       "10   SYN_38.jpg        1             0.000057   SIN_32.jpg        0   \n",
       "11    SYN_8.jpg        1             0.000031  SYN_123.jpg        1   \n",
       "12  SIN_101.jpg        0             0.998618   SYN_57.jpg        1   \n",
       "13   SIN_60.jpg        0             0.999872   SYN_39.jpg        0   \n",
       "\n",
       "    model_VisualBERT_v3  ... multimodal_text_captions_BO_v0  \\\n",
       "0              0.846515  ...                       0.089251   \n",
       "1              0.991903  ...                       0.647151   \n",
       "2              0.005895  ...                       0.220379   \n",
       "3              0.845425  ...                       0.952061   \n",
       "4              0.916593  ...                       0.964493   \n",
       "5              0.000382  ...                       0.119377   \n",
       "6              0.870523  ...                       0.536092   \n",
       "7              0.001423  ...                       0.092685   \n",
       "8              0.751444  ...                       0.921204   \n",
       "9              0.844976  ...                       0.446291   \n",
       "10             0.000159  ...                       0.326331   \n",
       "11             0.978204  ...                       0.006374   \n",
       "12             0.003838  ...                       0.540545   \n",
       "13             0.164859  ...                       0.097761   \n",
       "\n",
       "    multimodal_text_captions_BO_v1  multimodal_text_captions_BO_v2  \\\n",
       "0                         0.891331                        0.407616   \n",
       "1                         0.664549                        0.168989   \n",
       "2                         0.014468                        0.740198   \n",
       "3                         0.990917                        0.971473   \n",
       "4                         0.828906                        0.766104   \n",
       "5                         0.102780                        0.127394   \n",
       "6                         0.031560                        0.603495   \n",
       "7                         0.647242                        0.986107   \n",
       "8                         0.008743                        0.037978   \n",
       "9                         0.973507                        0.560534   \n",
       "10                        0.997803                        0.118442   \n",
       "11                        0.777985                        0.179125   \n",
       "12                        0.520195                        0.386572   \n",
       "13                        0.985648                        0.130455   \n",
       "\n",
       "   multimodal_text_captions_BO_v3  multimodal_text_captions_BO_v4  \\\n",
       "0                        0.780110                        0.556986   \n",
       "1                        0.839904                        0.871047   \n",
       "2                        0.009989                        0.095283   \n",
       "3                        0.542988                        0.580493   \n",
       "4                        0.061438                        0.378935   \n",
       "5                        0.027388                        0.864355   \n",
       "6                        0.687838                        0.878386   \n",
       "7                        0.003285                        0.210502   \n",
       "8                        0.753439                        0.935790   \n",
       "9                        0.898820                        0.981116   \n",
       "10                       0.136822                        0.981144   \n",
       "11                       0.977471                        0.681183   \n",
       "12                       0.532256                        0.962692   \n",
       "13                       0.085414                        0.058129   \n",
       "\n",
       "    multimodal_text_captions_BO_v5 multimodal_text_captions_BO_v6  \\\n",
       "0                         0.501726                       0.859859   \n",
       "1                         0.028915                       0.581634   \n",
       "2                         0.729122                       0.998462   \n",
       "3                         0.603140                       0.989025   \n",
       "4                         0.877839                       0.453970   \n",
       "5                         0.022900                       0.947713   \n",
       "6                         0.007514                       0.930981   \n",
       "7                         0.327350                       0.037642   \n",
       "8                         0.180489                       0.820333   \n",
       "9                         0.012856                       0.340941   \n",
       "10                        0.297222                       0.094026   \n",
       "11                        0.258521                       0.949793   \n",
       "12                        0.248363                       0.534641   \n",
       "13                        0.100489                       0.868502   \n",
       "\n",
       "    multimodal_text_captions_BO_v7  multimodal_text_captions_BO_v8  \\\n",
       "0                         0.860225                        0.969511   \n",
       "1                         0.940612                        0.320404   \n",
       "2                         0.025532                        0.823892   \n",
       "3                         0.535257                        0.003780   \n",
       "4                         0.519785                        0.054727   \n",
       "5                         0.856871                        0.393374   \n",
       "6                         0.811757                        0.807037   \n",
       "7                         0.872981                        0.978534   \n",
       "8                         0.108910                        0.968572   \n",
       "9                         0.688720                        0.909187   \n",
       "10                        0.021488                        0.993289   \n",
       "11                        0.752548                        0.630425   \n",
       "12                        0.181502                        0.796783   \n",
       "13                        0.233986                        0.705600   \n",
       "\n",
       "   multimodal_text_captions_BO_v9  \n",
       "0                        0.941125  \n",
       "1                        0.799684  \n",
       "2                        0.793365  \n",
       "3                        0.907331  \n",
       "4                        0.974454  \n",
       "5                        0.436365  \n",
       "6                        0.550162  \n",
       "7                        0.711779  \n",
       "8                        0.875829  \n",
       "9                        0.861869  \n",
       "10                       0.832169  \n",
       "11                       0.831105  \n",
       "12                       0.948646  \n",
       "13                       0.932851  \n",
       "\n",
       "[14 rows x 50 columns]"
      ]
     },
     "execution_count": 62,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Load azure BO predictions on SYN\n",
    "pred = pd.read_csv(path_azure_BO_syn, usecols=MODELNAMES_M_MTC, sep='\\t')\n",
    "for col in pred.columns:\n",
    "    syn_10_df[col]=pred[col]\n",
    "syn_10_df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 63,
   "metadata": {},
   "outputs": [],
   "source": [
    "path_clarifai_syn = './Multimodal/predictions/predictions_syn/baseline_multimodal_text_tags_SYN_scores.tsv'\n",
    "path_clarifai_BO_syn = './Mitigated/predictions/MTT_syn_10_df.csv'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 64,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "Index(['model_VisualBERT_v0', 'file_name_0', 'label_0', 'model_VisualBERT_v1',\n",
       "       'file_name_1', 'label_1', 'model_VisualBERT_v2', 'file_name_2',\n",
       "       'label_2', 'model_VisualBERT_v3', 'file_name_3', 'label_3',\n",
       "       'model_VisualBERT_v4', 'file_name_4', 'label_4', 'model_VisualBERT_v5',\n",
       "       'file_name_5', 'label_5', 'model_VisualBERT_v6', 'file_name_6',\n",
       "       'label_6', 'model_VisualBERT_v7', 'file_name_7', 'label_7',\n",
       "       'model_VisualBERT_v8', 'file_name_8', 'label_8', 'model_VisualBERT_v9',\n",
       "       'file_name_9', 'label_9', 'multimodal_text_captions_v0',\n",
       "       'multimodal_text_captions_v1', 'multimodal_text_captions_v2',\n",
       "       'multimodal_text_captions_v3', 'multimodal_text_captions_v4',\n",
       "       'multimodal_text_captions_v5', 'multimodal_text_captions_v6',\n",
       "       'multimodal_text_captions_v7', 'multimodal_text_captions_v8',\n",
       "       'multimodal_text_captions_v9', 'multimodal_text_captions_BO_v0',\n",
       "       'multimodal_text_captions_BO_v1', 'multimodal_text_captions_BO_v2',\n",
       "       'multimodal_text_captions_BO_v3', 'multimodal_text_captions_BO_v4',\n",
       "       'multimodal_text_captions_BO_v5', 'multimodal_text_captions_BO_v6',\n",
       "       'multimodal_text_captions_BO_v7', 'multimodal_text_captions_BO_v8',\n",
       "       'multimodal_text_captions_BO_v9', 'multimodal_text_tags_v0',\n",
       "       'multimodal_text_tags_v1', 'multimodal_text_tags_v2',\n",
       "       'multimodal_text_tags_v3', 'multimodal_text_tags_v4',\n",
       "       'multimodal_text_tags_v5', 'multimodal_text_tags_v6',\n",
       "       'multimodal_text_tags_v7', 'multimodal_text_tags_v8',\n",
       "       'multimodal_text_tags_v9'],\n",
       "      dtype='object')"
      ]
     },
     "execution_count": 64,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Load azure predictions on SYN\n",
    "pred = pd.read_csv(path_clarifai_syn, sep='\\t').drop(columns=label_column)\n",
    "# reconstruct the syn_10Fold predictions:\n",
    "a = int(len(pred)/10 )\n",
    "for i in range(0,10):\n",
    "    beg = a*i\n",
    "    end = a*(i+1)\n",
    "    syn_10_df['multimodal_text_tags_v'+str(i)] = list(pred['multimodal_text_tags_v'+str(i)][beg:end])\n",
    "syn_10_df.columns"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 65,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>model_VisualBERT_v0</th>\n",
       "      <th>file_name_0</th>\n",
       "      <th>label_0</th>\n",
       "      <th>model_VisualBERT_v1</th>\n",
       "      <th>file_name_1</th>\n",
       "      <th>label_1</th>\n",
       "      <th>model_VisualBERT_v2</th>\n",
       "      <th>file_name_2</th>\n",
       "      <th>label_2</th>\n",
       "      <th>model_VisualBERT_v3</th>\n",
       "      <th>...</th>\n",
       "      <th>multimodal_text_tags_BO_v0</th>\n",
       "      <th>multimodal_text_tags_BO_v1</th>\n",
       "      <th>multimodal_text_tags_BO_v2</th>\n",
       "      <th>multimodal_text_tags_BO_v3</th>\n",
       "      <th>multimodal_text_tags_BO_v4</th>\n",
       "      <th>multimodal_text_tags_BO_v5</th>\n",
       "      <th>multimodal_text_tags_BO_v6</th>\n",
       "      <th>multimodal_text_tags_BO_v7</th>\n",
       "      <th>multimodal_text_tags_BO_v8</th>\n",
       "      <th>multimodal_text_tags_BO_v9</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>0.012071</td>\n",
       "      <td>SYN_49.jpg</td>\n",
       "      <td>0</td>\n",
       "      <td>0.996255</td>\n",
       "      <td>SYN_20.jpg</td>\n",
       "      <td>1</td>\n",
       "      <td>0.998068</td>\n",
       "      <td>SYN_40.jpg</td>\n",
       "      <td>0</td>\n",
       "      <td>0.846515</td>\n",
       "      <td>...</td>\n",
       "      <td>0.022264</td>\n",
       "      <td>0.849302</td>\n",
       "      <td>0.931545</td>\n",
       "      <td>0.690724</td>\n",
       "      <td>0.782228</td>\n",
       "      <td>0.860387</td>\n",
       "      <td>0.850078</td>\n",
       "      <td>0.880739</td>\n",
       "      <td>0.925911</td>\n",
       "      <td>0.649766</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>0.026192</td>\n",
       "      <td>SIN_51.jpg</td>\n",
       "      <td>0</td>\n",
       "      <td>0.001039</td>\n",
       "      <td>SIN_170.jpg</td>\n",
       "      <td>1</td>\n",
       "      <td>0.000045</td>\n",
       "      <td>SIN_104.jpg</td>\n",
       "      <td>0</td>\n",
       "      <td>0.991903</td>\n",
       "      <td>...</td>\n",
       "      <td>0.323579</td>\n",
       "      <td>0.603642</td>\n",
       "      <td>0.009011</td>\n",
       "      <td>0.724738</td>\n",
       "      <td>0.939935</td>\n",
       "      <td>0.014559</td>\n",
       "      <td>0.021923</td>\n",
       "      <td>0.792715</td>\n",
       "      <td>0.783678</td>\n",
       "      <td>0.246295</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>0.997714</td>\n",
       "      <td>SYN_69.jpg</td>\n",
       "      <td>0</td>\n",
       "      <td>0.992926</td>\n",
       "      <td>SIN_105.jpg</td>\n",
       "      <td>1</td>\n",
       "      <td>0.981816</td>\n",
       "      <td>SYN_44.jpg</td>\n",
       "      <td>1</td>\n",
       "      <td>0.005895</td>\n",
       "      <td>...</td>\n",
       "      <td>0.101526</td>\n",
       "      <td>0.068318</td>\n",
       "      <td>0.893430</td>\n",
       "      <td>0.054212</td>\n",
       "      <td>0.041387</td>\n",
       "      <td>0.140243</td>\n",
       "      <td>0.981143</td>\n",
       "      <td>0.010277</td>\n",
       "      <td>0.575979</td>\n",
       "      <td>0.723907</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>0.717852</td>\n",
       "      <td>SYN_114.jpg</td>\n",
       "      <td>1</td>\n",
       "      <td>0.996592</td>\n",
       "      <td>SIN_113.jpg</td>\n",
       "      <td>1</td>\n",
       "      <td>0.736905</td>\n",
       "      <td>SIN_41.jpg</td>\n",
       "      <td>1</td>\n",
       "      <td>0.845425</td>\n",
       "      <td>...</td>\n",
       "      <td>0.726078</td>\n",
       "      <td>0.973216</td>\n",
       "      <td>0.936768</td>\n",
       "      <td>0.505499</td>\n",
       "      <td>0.239624</td>\n",
       "      <td>0.705688</td>\n",
       "      <td>0.847039</td>\n",
       "      <td>0.572566</td>\n",
       "      <td>0.266054</td>\n",
       "      <td>0.904904</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>0.959301</td>\n",
       "      <td>SYN_89.jpg</td>\n",
       "      <td>1</td>\n",
       "      <td>0.996598</td>\n",
       "      <td>SYN_14.jpg</td>\n",
       "      <td>0</td>\n",
       "      <td>0.000080</td>\n",
       "      <td>SYN_13.jpg</td>\n",
       "      <td>1</td>\n",
       "      <td>0.916593</td>\n",
       "      <td>...</td>\n",
       "      <td>0.580646</td>\n",
       "      <td>0.928719</td>\n",
       "      <td>0.789841</td>\n",
       "      <td>0.325893</td>\n",
       "      <td>0.310648</td>\n",
       "      <td>0.485413</td>\n",
       "      <td>0.688810</td>\n",
       "      <td>0.986518</td>\n",
       "      <td>0.178393</td>\n",
       "      <td>0.934139</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>0.878191</td>\n",
       "      <td>SYN_121.jpg</td>\n",
       "      <td>0</td>\n",
       "      <td>0.980748</td>\n",
       "      <td>SIN_114.jpg</td>\n",
       "      <td>1</td>\n",
       "      <td>0.010145</td>\n",
       "      <td>SYN_90.jpg</td>\n",
       "      <td>1</td>\n",
       "      <td>0.000382</td>\n",
       "      <td>...</td>\n",
       "      <td>0.031317</td>\n",
       "      <td>0.627413</td>\n",
       "      <td>0.179640</td>\n",
       "      <td>0.110232</td>\n",
       "      <td>0.587033</td>\n",
       "      <td>0.001156</td>\n",
       "      <td>0.874927</td>\n",
       "      <td>0.967661</td>\n",
       "      <td>0.354084</td>\n",
       "      <td>0.273529</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6</th>\n",
       "      <td>0.999637</td>\n",
       "      <td>SYN_1.jpg</td>\n",
       "      <td>0</td>\n",
       "      <td>0.751023</td>\n",
       "      <td>SYN_21.jpg</td>\n",
       "      <td>0</td>\n",
       "      <td>0.977857</td>\n",
       "      <td>SYN_4.jpg</td>\n",
       "      <td>0</td>\n",
       "      <td>0.870523</td>\n",
       "      <td>...</td>\n",
       "      <td>0.615158</td>\n",
       "      <td>0.041636</td>\n",
       "      <td>0.708007</td>\n",
       "      <td>0.964272</td>\n",
       "      <td>0.968780</td>\n",
       "      <td>0.003297</td>\n",
       "      <td>0.677516</td>\n",
       "      <td>0.071734</td>\n",
       "      <td>0.875146</td>\n",
       "      <td>0.632462</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7</th>\n",
       "      <td>0.004084</td>\n",
       "      <td>SYN_23.jpg</td>\n",
       "      <td>1</td>\n",
       "      <td>0.995678</td>\n",
       "      <td>SYN_80.jpg</td>\n",
       "      <td>1</td>\n",
       "      <td>0.999970</td>\n",
       "      <td>SYN_108.jpg</td>\n",
       "      <td>1</td>\n",
       "      <td>0.001423</td>\n",
       "      <td>...</td>\n",
       "      <td>0.022809</td>\n",
       "      <td>0.740199</td>\n",
       "      <td>0.999283</td>\n",
       "      <td>0.043093</td>\n",
       "      <td>0.182609</td>\n",
       "      <td>0.026354</td>\n",
       "      <td>0.034329</td>\n",
       "      <td>0.789103</td>\n",
       "      <td>0.967846</td>\n",
       "      <td>0.168667</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8</th>\n",
       "      <td>0.999221</td>\n",
       "      <td>SYN_10.jpg</td>\n",
       "      <td>1</td>\n",
       "      <td>0.004510</td>\n",
       "      <td>SYN_85.jpg</td>\n",
       "      <td>1</td>\n",
       "      <td>0.000049</td>\n",
       "      <td>SYN_75.jpg</td>\n",
       "      <td>0</td>\n",
       "      <td>0.751444</td>\n",
       "      <td>...</td>\n",
       "      <td>0.844226</td>\n",
       "      <td>0.041548</td>\n",
       "      <td>0.070360</td>\n",
       "      <td>0.704351</td>\n",
       "      <td>0.828553</td>\n",
       "      <td>0.013666</td>\n",
       "      <td>0.324674</td>\n",
       "      <td>0.079673</td>\n",
       "      <td>0.739040</td>\n",
       "      <td>0.925620</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9</th>\n",
       "      <td>0.957423</td>\n",
       "      <td>SIN_64.jpg</td>\n",
       "      <td>0</td>\n",
       "      <td>0.993563</td>\n",
       "      <td>SYN_59.jpg</td>\n",
       "      <td>1</td>\n",
       "      <td>0.980057</td>\n",
       "      <td>SIN_11.jpg</td>\n",
       "      <td>0</td>\n",
       "      <td>0.844976</td>\n",
       "      <td>...</td>\n",
       "      <td>0.210285</td>\n",
       "      <td>0.954517</td>\n",
       "      <td>0.819898</td>\n",
       "      <td>0.853221</td>\n",
       "      <td>0.964016</td>\n",
       "      <td>0.014656</td>\n",
       "      <td>0.488108</td>\n",
       "      <td>0.768872</td>\n",
       "      <td>0.738601</td>\n",
       "      <td>0.669022</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>10</th>\n",
       "      <td>0.028079</td>\n",
       "      <td>SIN_2.jpg</td>\n",
       "      <td>0</td>\n",
       "      <td>0.974318</td>\n",
       "      <td>SYN_38.jpg</td>\n",
       "      <td>1</td>\n",
       "      <td>0.000057</td>\n",
       "      <td>SIN_32.jpg</td>\n",
       "      <td>0</td>\n",
       "      <td>0.000159</td>\n",
       "      <td>...</td>\n",
       "      <td>0.127784</td>\n",
       "      <td>0.997041</td>\n",
       "      <td>0.045411</td>\n",
       "      <td>0.180792</td>\n",
       "      <td>0.962660</td>\n",
       "      <td>0.605973</td>\n",
       "      <td>0.116612</td>\n",
       "      <td>0.013849</td>\n",
       "      <td>0.977911</td>\n",
       "      <td>0.897006</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>11</th>\n",
       "      <td>0.004144</td>\n",
       "      <td>SYN_104.jpg</td>\n",
       "      <td>0</td>\n",
       "      <td>0.987357</td>\n",
       "      <td>SYN_8.jpg</td>\n",
       "      <td>1</td>\n",
       "      <td>0.000031</td>\n",
       "      <td>SYN_123.jpg</td>\n",
       "      <td>1</td>\n",
       "      <td>0.978204</td>\n",
       "      <td>...</td>\n",
       "      <td>0.013240</td>\n",
       "      <td>0.784759</td>\n",
       "      <td>0.124416</td>\n",
       "      <td>0.984577</td>\n",
       "      <td>0.719125</td>\n",
       "      <td>0.083654</td>\n",
       "      <td>0.717039</td>\n",
       "      <td>0.943318</td>\n",
       "      <td>0.597274</td>\n",
       "      <td>0.441257</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>12</th>\n",
       "      <td>0.410500</td>\n",
       "      <td>SIN_1.jpg</td>\n",
       "      <td>0</td>\n",
       "      <td>0.996465</td>\n",
       "      <td>SIN_101.jpg</td>\n",
       "      <td>0</td>\n",
       "      <td>0.998618</td>\n",
       "      <td>SYN_57.jpg</td>\n",
       "      <td>1</td>\n",
       "      <td>0.003838</td>\n",
       "      <td>...</td>\n",
       "      <td>0.420739</td>\n",
       "      <td>0.950659</td>\n",
       "      <td>0.529683</td>\n",
       "      <td>0.167097</td>\n",
       "      <td>0.956629</td>\n",
       "      <td>0.087640</td>\n",
       "      <td>0.051174</td>\n",
       "      <td>0.565418</td>\n",
       "      <td>0.738958</td>\n",
       "      <td>0.768341</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>13</th>\n",
       "      <td>0.010282</td>\n",
       "      <td>SYN_50.jpg</td>\n",
       "      <td>0</td>\n",
       "      <td>0.994869</td>\n",
       "      <td>SIN_60.jpg</td>\n",
       "      <td>0</td>\n",
       "      <td>0.999872</td>\n",
       "      <td>SYN_39.jpg</td>\n",
       "      <td>0</td>\n",
       "      <td>0.164859</td>\n",
       "      <td>...</td>\n",
       "      <td>0.075407</td>\n",
       "      <td>0.993711</td>\n",
       "      <td>0.580910</td>\n",
       "      <td>0.671007</td>\n",
       "      <td>0.015702</td>\n",
       "      <td>0.066703</td>\n",
       "      <td>0.337283</td>\n",
       "      <td>0.094409</td>\n",
       "      <td>0.203874</td>\n",
       "      <td>0.935862</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>14 rows × 70 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "    model_VisualBERT_v0  file_name_0  label_0  model_VisualBERT_v1  \\\n",
       "0              0.012071   SYN_49.jpg        0             0.996255   \n",
       "1              0.026192   SIN_51.jpg        0             0.001039   \n",
       "2              0.997714   SYN_69.jpg        0             0.992926   \n",
       "3              0.717852  SYN_114.jpg        1             0.996592   \n",
       "4              0.959301   SYN_89.jpg        1             0.996598   \n",
       "5              0.878191  SYN_121.jpg        0             0.980748   \n",
       "6              0.999637    SYN_1.jpg        0             0.751023   \n",
       "7              0.004084   SYN_23.jpg        1             0.995678   \n",
       "8              0.999221   SYN_10.jpg        1             0.004510   \n",
       "9              0.957423   SIN_64.jpg        0             0.993563   \n",
       "10             0.028079    SIN_2.jpg        0             0.974318   \n",
       "11             0.004144  SYN_104.jpg        0             0.987357   \n",
       "12             0.410500    SIN_1.jpg        0             0.996465   \n",
       "13             0.010282   SYN_50.jpg        0             0.994869   \n",
       "\n",
       "    file_name_1  label_1  model_VisualBERT_v2  file_name_2  label_2  \\\n",
       "0    SYN_20.jpg        1             0.998068   SYN_40.jpg        0   \n",
       "1   SIN_170.jpg        1             0.000045  SIN_104.jpg        0   \n",
       "2   SIN_105.jpg        1             0.981816   SYN_44.jpg        1   \n",
       "3   SIN_113.jpg        1             0.736905   SIN_41.jpg        1   \n",
       "4    SYN_14.jpg        0             0.000080   SYN_13.jpg        1   \n",
       "5   SIN_114.jpg        1             0.010145   SYN_90.jpg        1   \n",
       "6    SYN_21.jpg        0             0.977857    SYN_4.jpg        0   \n",
       "7    SYN_80.jpg        1             0.999970  SYN_108.jpg        1   \n",
       "8    SYN_85.jpg        1             0.000049   SYN_75.jpg        0   \n",
       "9    SYN_59.jpg        1             0.980057   SIN_11.jpg        0   \n",
       "10   SYN_38.jpg        1             0.000057   SIN_32.jpg        0   \n",
       "11    SYN_8.jpg        1             0.000031  SYN_123.jpg        1   \n",
       "12  SIN_101.jpg        0             0.998618   SYN_57.jpg        1   \n",
       "13   SIN_60.jpg        0             0.999872   SYN_39.jpg        0   \n",
       "\n",
       "    model_VisualBERT_v3  ... multimodal_text_tags_BO_v0  \\\n",
       "0              0.846515  ...                   0.022264   \n",
       "1              0.991903  ...                   0.323579   \n",
       "2              0.005895  ...                   0.101526   \n",
       "3              0.845425  ...                   0.726078   \n",
       "4              0.916593  ...                   0.580646   \n",
       "5              0.000382  ...                   0.031317   \n",
       "6              0.870523  ...                   0.615158   \n",
       "7              0.001423  ...                   0.022809   \n",
       "8              0.751444  ...                   0.844226   \n",
       "9              0.844976  ...                   0.210285   \n",
       "10             0.000159  ...                   0.127784   \n",
       "11             0.978204  ...                   0.013240   \n",
       "12             0.003838  ...                   0.420739   \n",
       "13             0.164859  ...                   0.075407   \n",
       "\n",
       "    multimodal_text_tags_BO_v1  multimodal_text_tags_BO_v2  \\\n",
       "0                     0.849302                    0.931545   \n",
       "1                     0.603642                    0.009011   \n",
       "2                     0.068318                    0.893430   \n",
       "3                     0.973216                    0.936768   \n",
       "4                     0.928719                    0.789841   \n",
       "5                     0.627413                    0.179640   \n",
       "6                     0.041636                    0.708007   \n",
       "7                     0.740199                    0.999283   \n",
       "8                     0.041548                    0.070360   \n",
       "9                     0.954517                    0.819898   \n",
       "10                    0.997041                    0.045411   \n",
       "11                    0.784759                    0.124416   \n",
       "12                    0.950659                    0.529683   \n",
       "13                    0.993711                    0.580910   \n",
       "\n",
       "   multimodal_text_tags_BO_v3  multimodal_text_tags_BO_v4  \\\n",
       "0                    0.690724                    0.782228   \n",
       "1                    0.724738                    0.939935   \n",
       "2                    0.054212                    0.041387   \n",
       "3                    0.505499                    0.239624   \n",
       "4                    0.325893                    0.310648   \n",
       "5                    0.110232                    0.587033   \n",
       "6                    0.964272                    0.968780   \n",
       "7                    0.043093                    0.182609   \n",
       "8                    0.704351                    0.828553   \n",
       "9                    0.853221                    0.964016   \n",
       "10                   0.180792                    0.962660   \n",
       "11                   0.984577                    0.719125   \n",
       "12                   0.167097                    0.956629   \n",
       "13                   0.671007                    0.015702   \n",
       "\n",
       "    multimodal_text_tags_BO_v5 multimodal_text_tags_BO_v6  \\\n",
       "0                     0.860387                   0.850078   \n",
       "1                     0.014559                   0.021923   \n",
       "2                     0.140243                   0.981143   \n",
       "3                     0.705688                   0.847039   \n",
       "4                     0.485413                   0.688810   \n",
       "5                     0.001156                   0.874927   \n",
       "6                     0.003297                   0.677516   \n",
       "7                     0.026354                   0.034329   \n",
       "8                     0.013666                   0.324674   \n",
       "9                     0.014656                   0.488108   \n",
       "10                    0.605973                   0.116612   \n",
       "11                    0.083654                   0.717039   \n",
       "12                    0.087640                   0.051174   \n",
       "13                    0.066703                   0.337283   \n",
       "\n",
       "    multimodal_text_tags_BO_v7  multimodal_text_tags_BO_v8  \\\n",
       "0                     0.880739                    0.925911   \n",
       "1                     0.792715                    0.783678   \n",
       "2                     0.010277                    0.575979   \n",
       "3                     0.572566                    0.266054   \n",
       "4                     0.986518                    0.178393   \n",
       "5                     0.967661                    0.354084   \n",
       "6                     0.071734                    0.875146   \n",
       "7                     0.789103                    0.967846   \n",
       "8                     0.079673                    0.739040   \n",
       "9                     0.768872                    0.738601   \n",
       "10                    0.013849                    0.977911   \n",
       "11                    0.943318                    0.597274   \n",
       "12                    0.565418                    0.738958   \n",
       "13                    0.094409                    0.203874   \n",
       "\n",
       "   multimodal_text_tags_BO_v9  \n",
       "0                    0.649766  \n",
       "1                    0.246295  \n",
       "2                    0.723907  \n",
       "3                    0.904904  \n",
       "4                    0.934139  \n",
       "5                    0.273529  \n",
       "6                    0.632462  \n",
       "7                    0.168667  \n",
       "8                    0.925620  \n",
       "9                    0.669022  \n",
       "10                   0.897006  \n",
       "11                   0.441257  \n",
       "12                   0.768341  \n",
       "13                   0.935862  \n",
       "\n",
       "[14 rows x 70 columns]"
      ]
     },
     "execution_count": 65,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Load azure BO predictions on SYN\n",
    "pred = pd.read_csv(path_clarifai_BO_syn, usecols=MODELNAMES_M_MTT, sep='\\t')\n",
    "for col in pred.columns:\n",
    "    syn_10_df[col]=pred[col]\n",
    "syn_10_df"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Compute Bias metrics"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 66,
   "metadata": {},
   "outputs": [],
   "source": [
    "dict={}\n",
    "aucs={}\n",
    "bias={}\n",
    "text={}\n",
    "image={}\n",
    "auc_syn={}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 67,
   "metadata": {},
   "outputs": [],
   "source": [
    "MODELFAMILIES_BIAS =  [MODELNAMES_MTT]+ [MODELNAMES_M_MTT] + [MODELNAMES_MTC] + [MODELNAMES_M_MTC] + [MODELNAMES_VISUAL_BERT]\n",
    "\n",
    "models_bias = []\n",
    "for x in MODELFAMILIES_BIAS:\n",
    "    models_bias.append(model_performances.model_family_name(x).split('_v')[0])\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 68,
   "metadata": {},
   "outputs": [],
   "source": [
    "from cgi import test\n",
    "\n",
    "for fam in MODELFAMILIES_BIAS:\n",
    "     dict[fam[0].split('_v')[0]], aucs[fam[0].split('_v')[0]], bias[fam[0].split('_v')[0]], text[fam[0].split('_v')[0]], image[fam[0].split('_v')[0]], auc_syn[fam[0].split('_v')[0]] = compute_bias_metrics_on_syn_10(fam, syn_10_df, test_df, identity_terms, identity_tags, label_column)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 69,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "multimodal_text_tags\n",
      "0.7861916\n",
      "multimodal_text_tags_BO\n",
      "0.8082933999999999\n",
      "multimodal_text_captions\n",
      "0.7573665999999999\n",
      "multimodal_text_captions_BO\n",
      "0.7799546\n",
      "model_VisualBERT\n",
      "0.7728765999999999\n"
     ]
    }
   ],
   "source": [
    "for k in aucs.keys():\n",
    "    print(k)\n",
    "    print(np.mean(list(aucs[k].values())))"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Compute and Plot Bias Metrics"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 70,
   "metadata": {},
   "outputs": [],
   "source": [
    "all_aucs=[]\n",
    "for model in aucs.keys():\n",
    "    all_aucs.append(list(aucs[model].values()))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 71,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAiwAAAHcCAYAAAD1DfFtAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjYuMCwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy89olMNAAAACXBIWXMAAA9hAAAPYQGoP6dpAABBTElEQVR4nO3de1xVdb7/8Td7IzcFClHwgmLHCziSF7xF2WjDZHkZGbJMZTS8NE05Mw3aOWFeRmeSpov5mLJs5qDWsdJ0yBptrIZGw4nyF9QUBYolYQqo0wQICLr3/v1R7toBxkY3a8F+PR+P/XBY3+/6rs+eHfDmu75rLR+Hw+EQAACAiVmMLgAAAOD7EFgAAIDpEVgAAIDpEVgAAIDpEVgAAIDpEVgAAIDpEVgAAIDpEVgAAIDp+RpdwKVit9t1/PhxBQcHy8fHx+hyAABACzgcDlVXV6tnz56yWJqfR+kwgeX48eOKiooyugwAANAKR48eVe/evZtt7zCBJTg4WNJXbzgkJMTgagAAQEtUVVUpKirK+Xu8OR0msJw/DRQSEkJgAQCgnfm+5RwsugUAAKZHYAEAAKZHYAEAAKZHYAEAAKZHYAEAAKZHYAEAAKZHYAEAAKZHYAEAAKbXYW4cB1wqNptNOTk5KisrU48ePTRu3DhZrVajywIAr8YMC/AtWVlZ6t+/vyZMmKBZs2ZpwoQJ6t+/v7KysowuDQC8GoEF+FpWVpamT5+uuLg45ebmqrq6Wrm5uYqLi9P06dMJLQBgIB+Hw+EwuohLoaqqSqGhoaqsrORZQnCbzWZT//79FRcXp507d7o84txutyspKUkFBQUqLi7m9BAAXEIt/f3NDAsgKScnRyUlJVq6dKlLWJEki8Wi9PR0HTlyRDk5OQZVCADejcACSCorK5MkDRkypMn289vP9wMAtC0CCyCpR48ekqSCgoIm289vP98PANC2CCyApHHjxik6Olpr1qyR3W53abPb7crIyFC/fv00btw4gyoEAO9GYAEkWa1WPfLII9q1a5eSkpJcrhJKSkrSrl279PDDD7PgFgAMwo3jgK8lJydrx44dWrx4sRISEpzb+/Xrpx07dig5OdnA6gDAu3FZM/Ad3OkWANpOS39/M8MCfIfVatX48eONLgMA8C2tWsOyfv16RUdHKyAgQGPGjNGBAwcu2H/dunUaNGiQAgMDFRUVpd/85jc6c+aMsz0jI0OjRo1ScHCwunfvrqSkJB08eLA1pQEAgA7I7cCybds2paWlaeXKlcrPz9fQoUM1ceJEnThxosn+zz33nO69916tXLlShYWFyszM1LZt27R06VJnn3379umuu+7S22+/rddff11nz57V9ddfr5qamta/MwAA0GG4vYZlzJgxGjVqlB5//HFJX13yGRUVpV/+8pe69957G/VftGiRCgsLlZ2d7dy2ePFivfPOO9q/f3+Txzh58qS6d++uffv26dprr21RXaxhAQCg/fHIrfkbGhqUl5enxMTEbwawWJSYmKjc3Nwm90lISFBeXp7ztNGnn36qV155RZMmTWr2OJWVlZKksLCwZvvU19erqqrK5QUAADomtxbdnjp1SjabTRERES7bIyIiVFRU1OQ+s2bN0qlTp3TNNdfI4XDo3LlzuuOOO1xOCX2b3W7X3XffrauvvrrZ26RLX617WbVqlTvlAwCAdsrjN47bu3ev1qxZoyeeeEL5+fnKysrS7t279bvf/a7J/nfddZcKCgq0devWC46bnp6uyspK5+vo0aOeKB8AAJiAWzMs4eHhslqtqqiocNleUVGhyMjIJvdZvny5fvazn2nBggWSpLi4ONXU1Oj222/Xfffd5/Jk3EWLFmnXrl1688031bt37wvW4u/vL39/f3fKBwAA7ZRbMyx+fn6Kj493WUBrt9uVnZ2tq666qsl9amtrXUKJJOdNuM6v93U4HFq0aJFefPFFvfHGG+rXr59bbwIAAHRsbt84Li0tTXPnztXIkSM1evRorVu3TjU1NUpNTZUkzZkzR7169VJGRoYkaerUqVq7dq2GDx+uMWPG6PDhw1q+fLmmTp3qDC533XWXnnvuOb300ksKDg5WeXm5JCk0NFSBgYGX6r0CAIB2yu3AMmPGDJ08eVIrVqxQeXm5hg0bpj179jgX4paWlrrMqCxbtkw+Pj5atmyZjh07pm7dumnq1Km6//77nX2efPJJSWp0d9FNmzbptttua8XbAgAAHQnPEgIAAIbxyH1YAAAAjEBgAQAApkdgAQAApkdgAQAApkdgAQAApkdgAQAApkdgAQAApkdgAQAApkdgAQAApkdgAQAApkdgAQAApkdgAQAApkdgAQAApkdgAQAApkdgAQAApkdgAQAApkdgAQAApkdgAQAApkdgAQAApkdgAQAApkdgAQAApkdgAQAApkdgAQAApudrdAFAW6itrVVRUVGL+9fV1amkpETR0dEKDAx061gxMTEKCgpyt0QAwAUQWOAVioqKFB8f3ybHysvL04gRI9rkWADgLQgs8AoxMTHKy8trcf/CwkKlpKRoy5Ytio2NdftYAIBLi8ACrxAUFNSqWY/Y2FhmSwDABFh0CwAATI/AAgAATI/AAgAATI/AAgAATI/AAgAATI/AAgAATI/LmtFuFRcXq7q62iNjFxYWuvzrKcHBwRowYIBHjwEAHQGBBe1ScXGxBg4c6PHjpKSkePwYhw4dIrQAwPcgsKBdOj+z0po70bbExTxLqKXO303XU7NEANCREFjQrnnyTrRXX321R8YFALiPwIJ2yefcGQ2PtCjwy0PS8fa5djzwy0MaHmmRz7kzRpcCAKZHYEG7FHC6VPk/7yK9+XPpTaOraZ1YSfk/76LC06WSEowuBwBMrVWBZf369XrooYdUXl6uoUOH6rHHHtPo0aOb7b9u3To9+eSTKi0tVXh4uKZPn66MjAwFBARIkt5880099NBDysvLU1lZmV588UUlJSW16g3BO5zp0kcjnjqtZ599VrHt9OnIhUVFmj17tjIn9TG6FAAwPbcDy7Zt25SWlqYNGzZozJgxWrdunSZOnKiDBw+qe/fujfo/99xzuvfee7Vx40YlJCTo0KFDuu222+Tj46O1a9dKkmpqajR06FDNmzdPycnJF/+u0OE5fAP0XrlddZcNlHoOM7qcVqkrt+u9crscvgFGlwIApud2YFm7dq0WLlyo1NRUSdKGDRu0e/dubdy4Uffee2+j/m+99ZauvvpqzZo1S5IUHR2tmTNn6p133nH2ufHGG3XjjTe29j0AAIAOzq3Vig0NDcrLy1NiYuI3A1gsSkxMVG5ubpP7JCQkKC8vTwcOHJAkffrpp3rllVc0adKkiyhbqq+vV1VVlcsLAAB0TG7NsJw6dUo2m00REREu2yMiIlRUVNTkPrNmzdKpU6d0zTXXyOFw6Ny5c7rjjju0dOnS1lctKSMjQ6tWrbqoMQAAQPvg8etB9+7dqzVr1uiJJ55Qfn6+srKytHv3bv3ud7+7qHHT09NVWVnpfB09evQSVQwAAMzGrRmW8PBwWa1WVVRUuGyvqKhQZGRkk/ssX75cP/vZz7RgwQJJUlxcnGpqanT77bfrvvvuk8XSuszk7+8vf3//Vu0LAADaF7fSgp+fn+Lj45Wdne3cZrfblZ2drauuuqrJfWpraxuFEqvVKklyOBzu1gsAALyQ21cJpaWlae7cuRo5cqRGjx6tdevWqaamxnnV0Jw5c9SrVy9lZGRIkqZOnaq1a9dq+PDhGjNmjA4fPqzly5dr6tSpzuBy+vRpHT582HmMI0eO6P3331dYWJj69OEeFQAAeDu3A8uMGTN08uRJrVixQuXl5Ro2bJj27NnjXIhbWlrqMqOybNky+fj4aNmyZTp27Ji6deumqVOn6v7773f2effddzVhwgTn12lpaZKkuXPnavPmza19bwAAoIPwcXSQ8zJVVVUKDQ1VZWWlQkJCjC4HHpafn6/4+Hjl5eV57OGHntYR3gMAXKyW/v5un0+NAwAAXoWHH6Ld8g31VUldiQL+/c2t7UP8QtQ7uLfqbfX65MtPGu0zuOtgSdKRyiOqO1fn0tarSy+F+ofqizNfqLym3KWtc6fO6hvSVza7TQf/c7DRuAMuH6BOlk46WnVU1WerXdq6B3VXeGC4Kusrdez0Mef2kroS+YbyLQgALcFPS7RbYRPCtPLwSumb9dqafMVkPTDuAVXUVGjGrhmN9vlw7oeSpGX/XKYPTn7g0rbmmjWa+l9T9WrJq1rzzhqXtoSeCXrqx0+p7lxdk+Pum7FPYQFhevD/Pai9n+91aVsyconm/mCu3i57W0v2LWn0HgAA3481LGiX8vPzNea6Mdq2e5tivvW05vY0w1JUVKQZk2fonTfeYQ0LAK/V0t/fzLCg3TpXeU7RgdHOEPJt/lb/Jref1y+0X7NtYQFhCgtoeubDarFecNyokKhm20L9QxXqH+r8+kzgGZ2rPNdsfwDAN1h0CwAATI/AAgAATI/AAgAATI81LGiXamtrJX21+NYT6urqVFJSoujoaAUGBnrkGIWFhR4ZFwA6IgIL2qWioiJJ0sKFCw2u5OIFBwcbXQIAmB6BBe1SUlKSJCkmJkZBQUGXfPzCwkKlpKRoy5Ytio2NveTjnxccHKwBAwZ4bHwA6CgILGiXwsPDtWDBAo8fJzY2lnukAIAJsOgWAACYHoEFAACYHoEFAACYHoEFAACYHoEFAACYHoEFAACYHoEFAACYHoEFAACYHoEFAACYHne6hVeora11Pn+oJc4/mLA1Dyj01OMCAMCbEVjgFYqKihQfH+/2fikpKW7vk5eXx+38AeASI7DAK8TExCgvL6/F/evq6lRSUqLo6GgFBga6fSwAwKXl43A4HEYXcSlUVVUpNDRUlZWVCgkJMbocAADQAi39/c2iWwAAYHoEFgAAYHoEFgAAYHoEFgAAYHoEFgAAYHoEFgAAYHoEFgAAYHoEFgAAYHoEFgAAYHoEFgAAYHoEFgAAYHoEFgAAYHoEFgAAYHoEFgAAYHoEFgAAYHqtCizr169XdHS0AgICNGbMGB04cOCC/detW6dBgwYpMDBQUVFR+s1vfqMzZ85c1JgAAMB7uB1Ytm3bprS0NK1cuVL5+fkaOnSoJk6cqBMnTjTZ/7nnntO9996rlStXqrCwUJmZmdq2bZuWLl3a6jEBAIB38XE4HA53dhgzZoxGjRqlxx9/XJJkt9sVFRWlX/7yl7r33nsb9V+0aJEKCwuVnZ3t3LZ48WK988472r9/f6vGbEpVVZVCQ0NVWVmpkJAQd94SAAAwSEt/f7s1w9LQ0KC8vDwlJiZ+M4DFosTEROXm5ja5T0JCgvLy8pyneD799FO98sormjRpUqvHlKT6+npVVVW5vAAAQMfk607nU6dOyWazKSIiwmV7RESEioqKmtxn1qxZOnXqlK655ho5HA6dO3dOd9xxh/OUUGvGlKSMjAytWrXKnfIBAEA75fGrhPbu3as1a9boiSeeUH5+vrKysrR792797ne/u6hx09PTVVlZ6XwdPXr0ElUMAADMxq0ZlvDwcFmtVlVUVLhsr6ioUGRkZJP7LF++XD/72c+0YMECSVJcXJxqamp0++2367777mvVmJLk7+8vf39/d8oHAADtlFszLH5+foqPj3dZQGu325Wdna2rrrqqyX1qa2tlsbgexmq1SpIcDkerxgQAAN7FrRkWSUpLS9PcuXM1cuRIjR49WuvWrVNNTY1SU1MlSXPmzFGvXr2UkZEhSZo6darWrl2r4cOHa8yYMTp8+LCWL1+uqVOnOoPL940JAAC8m9uBZcaMGTp58qRWrFih8vJyDRs2THv27HEumi0tLXWZUVm2bJl8fHy0bNkyHTt2TN26ddPUqVN1//33t3hMAADg3dy+D4tZcR8WAADaH4/chwUAAMAIBBYAAGB6BBYAAGB6BBYAAGB6BBYAAGB6BBYAAGB6BBYAAGB6BBYAAGB6BBYAAGB6BBYAAGB6BBYAAGB6BBYAAGB6BBYAAGB6BBYAAGB6BBYAAGB6BBYAAGB6BBYAAGB6BBYAAGB6BBYAAGB6BBYAAGB6BBYAAGB6BBYAAGB6BBYAAGB6BBYAAGB6BBYAAGB6BBYAAGB6BBYAAGB6BBYAAGB6BBYAAGB6BBYAAGB6BBYAAGB6BBYAAGB6BBYAAGB6BBYAAGB6BBYAAGB6BBYAAGB6vkYXAAAApNraWhUVFbW4f11dnUpKShQdHa3AwMAW7xcTE6OgoKDWlGgoAgsAACZQVFSk+Ph4jx8nLy9PI0aM8PhxLjUCCwAAJhATE6O8vLwW9y8sLFRKSoq2bNmi2NhYt47THrUqsKxfv14PPfSQysvLNXToUD322GMaPXp0k33Hjx+vffv2Ndo+adIk7d69W5JUUVGh//mf/9Frr72mL7/8Utdee60ee+wxDRgwoDXlAQDQ7gQFBbVq5iM2NrZdzpi4y+1Ft9u2bVNaWppWrlyp/Px8DR06VBMnTtSJEyea7J+VlaWysjLnq6CgQFarVTfffLMkyeFwKCkpSZ9++qleeuklvffee+rbt68SExNVU1Nzce8OAAB0CG4HlrVr12rhwoVKTU3V4MGDtWHDBgUFBWnjxo1N9g8LC1NkZKTz9frrrysoKMgZWIqLi/X222/rySef1KhRozRo0CA9+eSTqqur0/PPP39x7w4AAHQIbp0SamhoUF5entLT053bLBaLEhMTlZub26IxMjMzdeutt6pz586SpPr6eklSQECAy5j+/v7av3+/FixY0OQ49fX1zn0lqaqqyp23Yjo2m005OTkqKytTjx49NG7cOFmtVqPLAtCBuXtViuR9V6bAPNwKLKdOnZLNZlNERITL9oiIiBb9R3/gwAEVFBQoMzPTuS0mJkZ9+vRRenq6nnrqKXXu3FmPPvqoPv/8c5WVlTU7VkZGhlatWuVO+aaVlZWlxYsXq6SkxLktOjpajzzyiJKTk40rDECH1lZXpUjt98oUmEebXiWUmZmpuLg4lwW6nTp1UlZWlubPn6+wsDBZrVYlJibqxhtvlMPhaHas9PR0paWlOb+uqqpSVFSUR+v3hKysLE2fPl1TpkzR888/ryFDhqigoEBr1qzR9OnTtWPHDkILAI9w96oUyfuuTIF5uBVYwsPDZbVaVVFR4bK9oqJCkZGRF9y3pqZGW7du1erVqxu1xcfH6/3331dlZaUaGhrUrVs3jRkzRiNHjmx2PH9/f/n7+7tTvunYbDYtXrxYU6ZM0c6dO2WxfLWkaOzYsdq5c6eSkpK0ZMkSTZs2jdNDAC651l6VInnPlSkwD7cW3fr5+Sk+Pl7Z2dnObXa7XdnZ2brqqqsuuO/27dtVX1+vlJSUZvuEhoaqW7duKi4u1rvvvqtp06a5U167k5OTo5KSEi1dutQZVs6zWCxKT0/XkSNHlJOTY1CFAACYg9unhNLS0jR37lyNHDlSo0eP1rp161RTU6PU1FRJ0pw5c9SrVy9lZGS47JeZmamkpCR17dq10Zjbt29Xt27d1KdPH3344Yf69a9/raSkJF1//fWtfFvtw/k1OkOGDGmy/fz2C63lAQDAG7gdWGbMmKGTJ09qxYoVKi8v17Bhw7Rnzx7nQtzS0tJGswUHDx7U/v379dprrzU5ZllZmdLS0lRRUaEePXpozpw5Wr58eSveTvvSo0cPSVJBQYHGjh3bqL2goMClHwAA3srHcaGVre1IVVWVQkNDVVlZqZCQEKPLaRGbzab+/fsrLi7OZQ2L9NWptqSkJBUUFKi4uJg1LABMIT8/X/Hx8Vz1YwId5bNo6e9vt28ch0vHarXqkUce0a5du5SUlKTc3FxVV1crNzdXSUlJ2rVrlx5++GHCCgDA6/HwQ4MlJydrx44dWrx4sRISEpzb+/XrxyXNAAB8jcBiAsnJyZo2bRp3ugUAoBkEFpOwWq0aP3680WUAAGBKrGEBAACmR2ABAACmR2ABAACmR2ABAACmR2ABAACmx1VCHlRbW6uioqIW96+rq1NJSYmio6MVGBjo1rFiYmIUFBTkbokAALQLBBY3FBcXq7q6usX9CwsLL/h06ktpy5Ytio2NbVHf4OBgDRgwwMMVAQBw6RBYWqi4uFgDBw40uoxmuRuMDh06RGgBALQbBJYWOv2fkxo2wE93/s+v1LNXT+f2LpZAdfe9TA2Oc/r87MlG+13h99WTlo+f/bfOOBpc2rpbL1MXa6CqbDU6ZavS2YazOnnypLp166YQ/y7q0SlMdoddJWcrGo3bp1N3+fpYVX7uP6q1n3FpC7MG6zJrF5221+nEuS9d2k4cq9Bjyx7S6f+clERgAQC0DwSWFgo4XaqFd0boj757pG/lh8mna/TAyX+r1NdXKVE9G+334ZFSSdLve0TogwB/l7Y1J05pak2tng/uojXhYV9t9JX0Hymhtk5PVZzUaR8f3Rod1WjcfZ99rjC7XU90D9fezq5rV5b8+z+aW1WtV4MCdW9EN5e2WFuD8n/eRYWnSyUlCACA9oDA0kJnuvTRn5+o0AN//I36Rfdzbg/pFCQFdleErUHbao433vHH0ZKk39eUqc5W79LUKzBc6tRFExuqNPTMFy5tna0BUudIBTrs2lZd2mjY4B/1liy++u/aCv3iXJ1LW3f/yyX/UI09W6Ntda6zPsc++1wjVv5amZP6uPP2AQAwFIGlhWoa7Hq/uEFlnwerR/dvftmfkXSi0q6v/q9sHALya+1f/6+IRm2f1EqSXVIXSV1crhI6Exiof1ef37fxuB869+3WqK20ViqVXVJgo31PnKrRe+V2OXwDLvyGAQAwEQJLC52/PHnhwoUGV3JpBAcHG10CAAAtRmBpoaSkJEmevd/J+cug3blEuTW4rBkA0N4QWFooPDxcCxYsaJNjxcbGasSIEW1yLAAA2gNuzQ8AAEyPGRYPcvfW/IWFhS7/uoNb8wMAOjICiwcVFRUpPj7e7f1aczv/vLw8TiMBADosAosHxcTEKC8vr8X9L/bhhwAAdFQEFg8KCgpye9bj6quv9lA1AAC0XwQW4DtsNptycnJUVlamHj16aNy4cbJarUaXBQBejauEgG/JyspS//79NWHCBM2aNUsTJkxQ//79lZWVZXRpAODVCCzA17KysjR9+nTFxcUpNzdX1dXVys3NVVxcnKZPn05oAQADEVgAfXUaaPHixZoyZYp27typsWPHqkuXLho7dqx27typKVOmaMmSJbLZbEaXCgBeicACSMrJyVFJSYmWLl0qi8X128JisSg9PV1HjhxRTk6OQRUCgHcjsACSysrKJElDhgxpsv389vP9AABti6uEAEk9evSQJBUUFGjs2LGN2gsKClz6AWZVXFys6upqj41/MXfkdgcPacV3EVgASePGjVN0dLTWrFmjnTt3upwWstvtysjIUL9+/TRu3DgDqwQurLi4WAMHDmyTY7XmjtzuOnToEKEFTgQWQJLVatUjjzyi6dOnKykpSenp6RoyZIgKCgqUkZGhXbt2aceOHdyPBaZ2fmZly5Ytio2N9cgxLuaO3C1VWFiolJQUj84Uof0hsABfS05O1o4dO7R48WIlJCQ4t/fr1087duxQcnKygdUBLRcbG+vRZ4txR24YgcACfEtycrKmTZvGnW4BwGQILMB3WK1WjR8/3ugyAADfwmXNAADA9AgsAADA9AgsAADA9AgsAADA9FoVWNavX6/o6GgFBARozJgxOnDgQLN9x48fLx8fn0avyZMnO/ucPn1aixYtUu/evRUYGKjBgwdrw4YNrSkNAAB0QG4Hlm3btiktLU0rV65Ufn6+hg4dqokTJ+rEiRNN9s/KylJZWZnzVVBQIKvVqptvvtnZJy0tTXv27NGWLVtUWFiou+++W4sWLdLLL7/c+ncGAAA6DLcDy9q1a7Vw4UKlpqY6Z0KCgoK0cePGJvuHhYUpMjLS+Xr99dcVFBTkEljeeustzZ07V+PHj1d0dLRuv/12DR069IIzNwAAwHu4dR+WhoYG5eXlKT093bnNYrEoMTFRubm5LRojMzNTt956qzp37uzclpCQoJdfflnz5s1Tz549tXfvXh06dEiPPvpos+PU19ervr7e+XVVVZU7bwUAOhyfc2c0bICfyk7s08dFpc7tIZ2C1Duwu+ptDfqk5nij/QaHREuSjtSUqc5W79LWKzBcoZ266IuGKpWf+cKlrbM1QH07R8rmsOtgdam+a0CX3upk8dXR2gpVn6tzaevuf7nC/UNVebZGx+pOurT9++TnGh5pkc+5M269f3RsbgWWU6dOyWazKSIiwmV7RESEioqKvnf/AwcOqKCgQJmZmS7bH3vsMd1+++3q3bu3fH19ZbFY9Oc//1nXXntts2NlZGRo1apV7pQPAB1awOlSLbwzQvdWbJQqvtk++XSNHjj5b1X4+mpGVM9G+3145KuwsaxHhD4I8HdpW3PilKbW1OrV4C5aEx7m0pZQW6enKk6qzsdHM6KjGo2777PPFWa368Hu4drbOcilbcm//6O5VdV6OyhQSyK6ubTF1jco/+ddVHi6VFKCAKmN73SbmZmpuLg4jR492mX7Y489prffflsvv/yy+vbtqzfffFN33XWXevbsqcTExCbHSk9PV1pamvPrqqoqRUU1/oYBAG9xpksf/fmJCj3wx9+oX3Q/5/aQTkFSYHdF2Bq0rYkZFv04WpL0+2ZmWNSpiyY2VGloEzMs6hypQIdd25qYYQn+UW/J4qv/rq3QL5qYYZF/qMaerdG278ywHPvsc41Y+WtlTurjzttHB+dWYAkPD5fValVFRYXL9oqKCkVGRl5w35qaGm3dulWrV6922V5XV6elS5fqxRdfdF45dOWVV+r999/Xww8/3Gxg8ff3l7+/f5NtAOCNHL4Ber+4QT26/1CDYxo//NBf0uAL7N9Pw5ptC/v61RSrpMFq/mGLF/pTMvTr17edqc3Xe+V2OXwDLrAnvI1bi279/PwUHx+v7Oxs5za73a7s7GxdddVVF9x3+/btqq+vV0pKisv2s2fP6uzZs7JYXEuxWq2y2+3ulAcAADoot08JpaWlae7cuRo5cqRGjx6tdevWqaamRqmpqZKkOXPmqFevXsrIyHDZLzMzU0lJSeratavL9pCQEP3whz/UPffco8DAQPXt21f79u3TM888o7Vr117EWwMAAB2F24FlxowZOnnypFasWKHy8nINGzZMe/bscS7ELS0tbTRbcvDgQe3fv1+vvfZak2Nu3bpV6enpmj17tr744gv17dtX999/v+64445WvCUAANDRtGrR7aJFi7Ro0aIm2/bu3dto26BBg+RwOJodLzIyUps2bWpNKQAAmFJxcbGqq6s9Nn5hYaHLv54SHBysAQMGePQYLdGmVwkBAOANiouLNXDgwDY51nfXhnrCoUOHDA8tBBYAAC6x8zMrW7ZsUWxsrEeOUVdXp5KSEkVHRyswMNAjxygsLFRKSopHZ4paisACAICHxMbGasSI5i/5vlhXX321x8Y2m1Y9rRkAAKAtEVgAAIDpEVgAAIDpEVgAAIDpEVgAAIDpEVgAAIDpcVkzAHQQtbW1kqT8/HyPHaOt7v0BfBeBBQA6iKKiIknSwoULDa7k0ggODja6BJgIgQUAOoikpCRJUkxMjIKCgjxyjPN3PvXkHVwl8zy/BuZBYAGADiI8PFwLFixok2N5+g6uwHex6BYAAJgegQUAAJgegQUAAJgegQUAAJgegQUAAJgegQUAAJgegQUAAJgegQUAAJgegQUAAJgegQUAAJgegQUAAJgegQUAAJgegQUAAJgegQUAAJgegQUAAJier9EFAADQEfmG+qqkrkQB/w5wbgvxC1Hv4N6qt9Xrky8/abTP4K6DJUlHKo+o7lydS1uvLr0U6h+qL858ofKacpe2zp06q29IX9nsNh38z8FG4w64fIA6WTrpaNVRVZ+tdmnrHtRd4YHhqqyv1LHTx1zajp857t6b9iACCwAAHhA2IUwrD6+UDn+zbfIVk/XAuAdUUVOhGbtmNNrnw7kfSpKW/XOZPjj5gUvbmmvWaOp/TdWrJa9qzTtrXNoSeiboqR8/pbpzdU2Ou2/GPoUFhOnB//eg9n6+16VtycglmvuDuXq77G0t2bfEpa1vQF933rJHEVgAwEvV1taqqKjIrX0KCwtd/m2pmJgYBQUFubVPe/fFP77Qk0ueVExMjHNbiF+IJCmic4S2TdnW7L6/v/r3Tc6wSNLE6Ika2m2oS1vnTp0lSYG+gU2OG+wXLEn671H/rV8M+4VLW/eg7pKksT3GNtr304Ofard2N/8m2xCBBQC8VFFRkeLj41u1b0pKilv98/LyNGLEiFYdq706V3lO0YHRztM83+Zv9W9y+3n9Qvs12xYWEKawgLAm26wW6wXHjQqJarYt1D9Uof6hLtvOBJxptn9bI7AAgJeKiYlRXl6eW/vU1dWppKRE0dHRCgwMdOtYwMUgsACAlwoKCmrVrMfVV1/tgWo6Fp9zZzQ80qLALw9Jx9vvBbmBXx7S8EiLfM4ZP9NCYAEA4BILOF2q/J93kd78ufSm0dW0Xqyk/J93UeHpUkkJhtZCYAEA4BI706WPRjx1Ws8++6xi2/HpsMKiIs2ePVuZk/oYXQqBBQCAS83hG6D3yu2qu2yg1HOY0eW0Wl25Xe+V2+XwDfj+zh7Wfk+sAQAAr0FgAQAApteqwLJ+/XpFR0crICBAY8aM0YEDB5rtO378ePn4+DR6TZ482dmnqXYfHx899NBDrSkPAAB0MG4Hlm3btiktLU0rV65Ufn6+hg4dqokTJ+rEiRNN9s/KylJZWZnzVVBQIKvVqptvvtnZ59vtZWVl2rhxo3x8fHTTTTe1/p0BAIAOw+3AsnbtWi1cuFCpqakaPHiwNmzYoKCgIG3cuLHJ/mFhYYqMjHS+Xn/9dQUFBbkElm+3R0ZG6qWXXtKECRN0xRVXtP6dAQCADsOtwNLQ0KC8vDwlJiZ+M4DFosTEROXm5rZojMzMTN16663q3Llzk+0VFRXavXu35s+ff8Fx6uvrVVVV5fICAAAdk1uB5dSpU7LZbIqIiHDZHhERofLy8mb2+saBAwdUUFCgBQsWNNvn6aefVnBwsJKTky84VkZGhkJDQ52vqKjmn48AAADatza9SigzM1NxcXEaPXp0s302btyo2bNnKyDgwtd8p6enq7Ky0vk6evTopS4XAACYhFs3jgsPD5fValVFRYXL9oqKCkVGRl5w35qaGm3dulWrV69utk9OTo4OHjyobduaf+T2ef7+/vL3929Z4QAAoF1za4bFz89P8fHxys7Odm6z2+3Kzs7WVVdddcF9t2/frvr6+gs+kjwzM1Px8fEaOnSoO2UBAIAOzu1TQmlpafrzn/+sp59+WoWFhfrFL36hmpoapaamSpLmzJmj9PT0RvtlZmYqKSlJXbt2bXLcqqoqbd++/YLrWwAAgHdy+1lCM2bM0MmTJ7VixQqVl5dr2LBh2rNnj3MhbmlpqSwW1xx08OBB7d+/X6+99lqz427dulUOh0MzZ850tyQAANDB+TgcDofRRVwKVVVVCg0NVWVlpUJCQowuBwDgxfLz8xUfH6+8vDyNGDHC6HJarS3eR0t/f/MsIQAAYHoEFgAAYHoEFgAAYHoEFgAAYHoEFgAAYHoEFgAAYHoEFgAAYHoEFgAAYHoEFgAAYHoEFgAAYHoEFgAAYHoEFgAAYHoEFgAAYHoEFgAAYHoEFgAAYHoEFgAAYHoEFgAAYHoEFgAAYHoEFgAAYHq+RhcAAEBHU1tbK0nKz8/32DHq6upUUlKi6OhoBQYGeuQYhYWFHhm3NQgsAABcYkVFRZKkhQsXGlzJpREcHGx0CQQWAAAutaSkJElSTEyMgoKCPHKMwsJCpaSkaMuWLYqNjfXIMaSvwsqAAQM8Nn5LEVgAALjEwsPDtWDBgjY5VmxsrEaMGNEmxzISi24BAIDpEVgAAIDpEVgAAIDpEVgAAIDpEVgAAIDpEVgAAIDpEVgAAIDpEVgAAIDpEVgAAIDpEVgAAIDpcWt+AABMoLa21vnQxJY4/yRld5+o7MnnG3kSgQUAABMoKipSfHy82/ulpKS41T8vL69dPnuIwAIAgAnExMQoLy+vxf3r6upUUlKi6OhoBQYGunWc9sjH4XA4jC7iUqiqqlJoaKgqKysVEhJidDkALhGbzaacnByVlZWpR48eGjdunKxWq9FlAbhEWvr7m0W3AEwrKytL/fv314QJEzRr1ixNmDBB/fv3V1ZWltGlAWhjBBYAppSVlaXp06crLi5Oubm5qq6uVm5uruLi4jR9+nRCC+BlOCUEwHRsNpv69++vuLg47dy5UxbLN39b2e12JSUlqaCgQMXFxZweAto5j54SWr9+vaKjoxUQEKAxY8bowIEDzfYdP368fHx8Gr0mT57s0q+wsFA/+clPFBoaqs6dO2vUqFEqLS1tTXkA2rmcnByVlJRo6dKlLmFFkiwWi9LT03XkyBHl5OQYVCGAtuZ2YNm2bZvS0tK0cuVK5efna+jQoZo4caJOnDjRZP+srCyVlZU5XwUFBbJarbr55pudfT755BNdc801iomJ0d69e/XBBx9o+fLlCggIaP07A9BulZWVSZKGDBnSZPv57ef7Aej43L6see3atVq4cKFSU1MlSRs2bNDu3bu1ceNG3XvvvY36h4WFuXy9detWBQUFuQSW++67T5MmTdKDDz7o3PZf//Vf7pYGoIPo0aOHJKmgoEBjx45t1F5QUODSD0DH59YMS0NDg/Ly8pSYmPjNABaLEhMTlZub26IxMjMzdeutt6pz586SvjofvXv3bg0cOFATJ05U9+7dNWbMGO3cufOC49TX16uqqsrlBaBjGDdunKKjo7VmzRrZ7XaXNrvdroyMDPXr10/jxo0zqEIAbc2tGZZTp07JZrMpIiLCZXtERESLbid84MABFRQUKDMz07ntxIkTOn36tB544AH9/ve/1x/+8Aft2bNHycnJ+sc//qEf/vCHTY6VkZGhVatWuVM+gHbCarXqkUce0fTp0zVt2jTdcMMNCgwMVF1dnfbs2aPdu3drx44dLLgFvEib3uk2MzNTcXFxGj16tHPb+b+epk2bpt/85jeSpGHDhumtt97Shg0bmg0s6enpSktLc35dVVWlqKgoD1YPoC0lJydryZIlevTRR7Vr1y7ndl9fXy1ZskTJyckGVgegrbkVWMLDw2W1WlVRUeGyvaKiQpGRkRfct6amRlu3btXq1asbjenr66vBgwe7bI+NjdX+/fubHc/f31/+/v7ulA+gHcnKytLDDz+syZMn68Ybb3TOsPztb3/Tww8/rLFjxxJaAC/iVmDx8/NTfHy8srOzlZSUJOmrGZLs7GwtWrTogvtu375d9fX1jR7S5Ofnp1GjRungwYMu2w8dOqS+ffu6Ux6ADsJms2nx4sWaMmVKo/uw3HHHHUpKStKSJUs0bdo0TgsBXsLtU0JpaWmaO3euRo4cqdGjR2vdunWqqalxXjU0Z84c9erVSxkZGS77ZWZmKikpSV27dm005j333KMZM2bo2muv1YQJE7Rnzx799a9/1d69e1v3rgC0a+fvw/L88883ex+WhIQE5eTkaPz48cYUCaBNuR1YZsyYoZMnT2rFihUqLy/XsGHDtGfPHudC3NLS0kY/YA4ePKj9+/frtddea3LMn/70p9qwYYMyMjL0q1/9SoMGDdJf/vIXXXPNNa14SwDaO+7DAuC7uDU/ANPZu3evJkyYoNzc3Cbvw5Kbm6uEhAT94x//YIYFaOd4WjOAdov7sAD4LgILANM5fx+WXbt2KSkpyeVpzUlJSdq1a5cefvhhFtwCXqRN78MCAC2VnJysHTt2aPHixUpISHBu79evn3bs2MElzYCXYQ0LAFOz2WzKyclRWVmZevTooXHjxjGzAnQgLf39zQwLgItWXFys6urqFvWtq6tTSUlJq45z7Ngxbd26tcX9o6OjFRgY2KK+wcHBGjBgQKvqAuB5BBYAF6W4uFiDRw2W72WuP05sNTadPXVWPp185N+z8V2pz3x2RpLkF+kni7/rcrqzp87KVmOTNdiqTmGdXNrsZ+xqqGiQfKSAPgGNx/38jGST/Lr5yRL0nXH/c1a2KpssQRb5dfNzaTv35Tl9/P8+JrQAJkVgAXBRTv/npAZP7Sp7YjeX7dcEDdGvuk5T+dkv9KvyJxvt94dzsyRJ662vqtTn3y5tM2xXaYSjn97yOaSXrO+6tA2wR2qB/Tqd0Vmt9N3eaNzl55LVRQHabNmnQssxl7Y5lyVqSvAY5dYW6tF/Z7m0Wf5+Uqf/c1ISgQUwI9awALgohW9sVfg/f6GT31lXEmK3q/c5m+p9pE86dWq03+CGs5KkI518Vefj49LW65xNoXa7vrBYVO7rOm5nu0N9z52TTdJBv8bjDmg4q06Sjvr6qtriOm73czaF2+2qtPjomK/r32vdbDaduvpJxV53a0vfOoBLgDUsANrEmS59NPGJKj377LOKjYlp1O4vaXDj3Zz6XaAt7OtXU6zfM+6Fnt0e+vXrvMKiIk2cPVuZE/tcYC8ARiKwALgoNQ12vVdu1z8/Pa26y+zfv0MrnF+o684iWncUltn0XrldDt/Ga2IAmAOBBcBFKSoqkiQtXLjQ4EouXnBwsNElAGgGgQXARUlKSpIkxcTEKCgoyCPHKCwsVEpKirZs2aLY2FiPHIPLmgFzI7AAuCjh4eFasGBBmxwrNjZWI0aMaJNjATAXniUEAABMjxkWAG2qtrbWue6lpQoLC13+bSlPnqYC0LYILADaVFFRkeLj41u1b0pKilv98/LyOIUEdBAEFgBtKiYmRnl5eW7t09rLmmOauC8MgPaJO90CAADDtPT3N4tuAQCA6RFYAACA6RFYAACA6RFYAACA6RFYAACA6RFYAACA6RFYAACA6RFYAACA6RFYAACA6RFYAACA6RFYAACA6RFYAACA6RFYAACA6fkaXcClcv6h01VVVQZXAgAAWur87+3zv8eb02ECS3V1tSQpKirK4EoAAIC7qqurFRoa2my7j+P7Ik07Ybfbdfz4cQUHB8vHx8foclqlqqpKUVFROnr0qEJCQowux6vxWZgLn4d58FmYR0f5LBwOh6qrq9WzZ09ZLM2vVOkwMywWi0W9e/c2uoxLIiQkpF3/x9eR8FmYC5+HefBZmEdH+CwuNLNyHotuAQCA6RFYAACA6RFYTMTf318rV66Uv7+/0aV4PT4Lc+HzMA8+C/Pwts+iwyy6BQAAHRczLAAAwPQILAAAwPQILAAAwPQILAAAwPQILAAAwPQILAAAmJjVatWJEyeMLsNwBBYDXXfddfryyy+NLgOS5s2b53yAJgCYCXcf+QqBxUB79+5VQ0OD0WVA0tNPP626ujqjy4Ck4uJizZw50/nI+W+rrKzUrFmz9OmnnxpQmfc5fvy4lixZ0uxncc8996iiosKAyuCNOszDD4GLwV8w5vHQQw8pKiqqyYe5hYaGKioqSg899JCefPJJA6rzLmvXrlVVVVWzn0V1dbXWrl2rP/zhDwZU513+93//V126dLlgn1/96ldtVI0xuNOtgSwWi9544w2FhYVdsN+VV17ZRhV5L4vFouLiYnXr1u2C/dr7E1Hbg0GDBmnLli0aNWpUk+15eXmaNWuWDh482MaVeZ8hQ4Zow4YNuuaaa5psf+utt7Rw4UJ99NFHbVyZd7FYLOrdu7esVmuzfXx8fDr8zCOBxUAWi0U+Pj5N/nV/fruPj49sNpsB1XmX859Fc/gs2k5gYKCKiorUt2/fJts/++wzxcbGqra2to0r8z6dO3dWYWGh+vTp02R7aWmpYmNjVVNT08aVeReLxaLy8nJ1797d6FIMxSkhg73zzjvf+1c92saOHTu+d7YLnhcaGqpPPvmk2cBy+PBhZrraSGBgoEpKSpoNLCUlJQoMDGzjqrzPhf6Y8ibMsBiI1GwefBbmccstt+js2bN68cUXm2yfNm2a/Pz8tH379jauzPtMnjxZPXv21J///Ocm2xcsWKDjx4/rlVdeaePKvAs/n77CVUIATCU9PV1/+9vfNH36dB04cECVlZWqrKzUO++8o5tuukmvvvqq0tPTjS7TKyxZskSbNm3SkiVLXK4Gqqio0OLFi7V582YtWbLEwAq9w8qVKy+44DYrK8s71jo6YJjx48c7jh49anQZcDgc0dHRjs8//9zoMvC1v/71r45u3bo5LBaLy6tbt26Ol156yejyvMqGDRsc/v7+DovF4rjsssscl19+ucNisTj8/f0dTzzxhNHleY0NGzY4brrpJsfMmTMdb7/9tsPhcDiys7Mdw4YNcwQFBTnuuOMOgyv0PE4JGchqtaqsrMzrp/nMgM/CPFavXq0lS5bIx8dHe/bs0eHDh+VwODRw4EBdf/31CgoKMrpEr/HMM89oxowZOnXqlF544QWXz2L69Onq3bu30SV6hQceeEArVqzQlVdeqaKiIjkcDt1333167LHH9Otf/1o///nPdfnllxtdpscRWAzEeUnz4LMwD8KjefBZmMOgQYO0dOlSzZ07Vzk5OfrhD3+oSZMmadu2bercubPR5bUZ1rAYjNXf5sFnYQ78DWUefBbmUFpaquuuu06SNG7cOHXq1EmrVq3yqrAicVmz4QYOHPi9vyi/+OKLNqrGu/3oRz+Sr++FvyXy8/PbqBrvRng0Dz4L49XX1ysgIMD5tZ+fn1fegoHAYrBVq1YpNDTU6DIgaeLEid9762u0DYK8eRDkzWH58uXO9VsNDQ36/e9/3+h3x9q1a40orc0QWAx26623cn7YJO655x4+C5MgyJsHQd541157rcujKBISEhrdht8bZsJYdGsgFrSZB5+FebAA2jz4LGAmLLo1kOPr59PAeOR28+B7wjz4LGAmzLAYKDU1tUU/EDZu3NgG1Xi3zz77TH369OEHtAlYLBZVVFTwjC0TYIbFHAYPHqz9+/c7F9reeeedWr16tcLDwyVJJ06cUHR0dId/ICiBxUAWi0V9+/bV8OHDL/gXfnPPVMGls3r16hb1W7FihYcrAUHePAjy5vDd4BgSEqL3339fV1xxhaSvHpXQo0cP2e12I8v0OBbdGugXv/iFnn/+eR05ckSpqalKSUnxykvVzOC3v/2tevbsqe7duzcbHn18fAgsbeDpp59uUZCH5z399NMt6sf3Rdtq6vvCG0IlMywGq6+vV1ZWljZu3Ki33npLkydP1vz583X99dd7xX+AZjF58mS98cYbmjhxoubNm6cpU6bIYmGJlxHuuusuPf/88+rbty9B3mAWi6VFQZ7Lmj3ruzMswcHB+te//uUyw9KzZ0/ZbDYjy/Q4AouJfPbZZ9q8ebOeeeYZnTt3Th999BGXE7ah48eP6+mnn9bmzZtVVVWlOXPmaN68eRo0aJDRpXkdgrw5EOTNwWq1qry83LmuKzg4WB988IH69esnicACAxw9elSbNm3S5s2b1dDQoKKiIgKLQd58801t2rRJf/nLXxQXF6e///3vCgwMNLosr0SQNxZB3ngWi0VDhgxx3sDvgw8+UExMjPz8/CTJ+X3R0QMLa1gM9u2/JPfv368pU6bo8ccf1w033MBfMgYaNWqUSkpK9PHHH+u9997T2bNnCSwGsVgs8vHxkcPh6PA/kM2oZ8+eSk9PV3p6ujPIjxo1iiDfhlauXOny9bRp0xr1uemmm9qqHMMww2KgO++8U1u3blVUVJTmzZun2bNnOy9TgzFyc3O1ceNGvfDCCxo4cKBSU1M1a9YsXXbZZUaX5lWaCvKpqakEeYPV1dVp+/btWr9+vT788EOVl5crJCTE6LLgJQgsBrJYLOrTp4+GDx9+wfPyWVlZbViVd3rwwQe1efNmnTp1SrNnz1ZqaqquvPJKo8vySgR58yHIm1NDQ4MaGhq85hQpgcVAt912W4sWEG7atKkNqvFu58PjlClTnOeFm9LRHy5mBgR58yDIm8emTZuUn5+vsWPHavbs2UpPT9fatWt17tw5XXfdddq6dau6du1qdJkeRWABJI0fP/57w6OPj4/eeOONNqrIexHkzYMgbw7333+/7r//fl199dXKz8/XLbfcop07d+ruu++WxWLRH//4R02ZMkVPPvmk0aV6FIEFANAkgrw5DBgwQKtXr9bMmTP17rvvasyYMXrhhRecC23/9re/6Y477tBnn31mcKWeRWABmvHPf/5TI0eOlL+/v9GlAPBi/v7+Onz4sKKiopxff/DBB85Ly48dO6Z+/fqpoaHByDI9juX2QDNuvPFGHTt2zOgyAFP55z//qfr6eqPL8Cpnz551+cPJz89PnTp1cn7t6+vrFZf8cx8WoBlMPgKN3XjjjS4P3kPb+Pjjj1VeXi7pq59NRUVFOn36tCTp1KlTRpbWZggsAIAWI8gb40c/+pHL//dTpkyRJOdNFb3hkRUEFqAZTz31lCIiIowuA4CXO3LkiNElmAKLbgFJ8+bNa1G/jRs3ergSwNyee+45TZs2TZ07dza6FHgZAgugr+430bdvXw0fPvyCU94vvvhiG1YFGIsgb7wPPvigxX07+k39CCyApLvuukvPP/+8+vbtq9TUVKWkpCgsLMzosgBDEeSN9+2Hf16Ij49Ph79SiMACfO3bD9x76623NHnyZM2fP1/XX3+9VyxoA76LIG88d24G17dvXw9WYjwCC9CEzz77TJs3b9Yzzzyjc+fO6aOPPvKaB4wB30aQh1lwlRDQhG9Pw3b0aVbgQvz9/TVz5kzNnDnTGeTvvPNOgryBPv74Y5WWlja6s+1PfvITgypqGwQW4Gvf/kty//79mjJlih5//HHdcMMNsli4KTRAkDfWp59+qp/+9Kf68MMPXda1nJ/p6uifCT+FAUl33nmnevTooQceeEBTpkzR0aNHtX37dk2aNImwAq9WX1+v559/Xj/+8Y81cOBAffjhh3r88cdVWlrK7Eob+/Wvf61+/frpxIkTCgoK0kcffaQ333xTI0eO1N69e40uz+NYwwLoq78c+/Tpo+HDh1/wvHxWVlYbVgUY684779TWrVsVFRWlefPmafbs2QoPDze6LK8VHh6uN954Q1deeaVCQ0N14MABDRo0SG+88YYWL16s9957z+gSPYpTQoCkOXPmsIAQ+I4NGzaoT58+uuKKK7Rv3z7t27evyX4E+bZhs9kUHBws6avwcvz4cQ0aNEh9+/bVwYMHDa7O8wgsgKTNmzcbXQJgOgR5cxkyZIj+9a9/qV+/fhozZowefPBB+fn56U9/+pNXPIySU0IAALQDr776qmpqapScnKzDhw9rypQpOnTokLp27apt27bpuuuuM7pEjyKwAADQTn3xxRe6/PLLvWImjMACAABMjzUsAAC0AxMmTLjgTMobb7zRhtW0PQILAADtwLBhw1y+Pnv2rN5//30VFBRo7ty5xhTVhggsAAC0A48++miT23/729/q9OnTbVxN22MNCwAA7djhw4c1evRoffHFF0aX4lHccxwAgHYsNzdXAQEBRpfhcZwSAgCgHUhOTnb52uFwqKysTO+++66WL19uUFVth8ACAEA7EBoa6vK1xWLRoEGDtHr1al1//fUGVdV2WMMCAABMjzUsAAC0A0ePHtXnn3/u/PrAgQO6++679ac//cnAqtoOgQUAgHZg1qxZ+sc//iFJKi8vV2Jiog4cOKD77rtPq1evNrg6zyOwAADQDhQUFGj06NGSpBdeeEFxcXF666239Oyzz3rFE+cJLAAAtANnz56Vv7+/JOnvf/+7fvKTn0iSYmJiVFZWZmRpbYLAAgBAO/CDH/xAGzZsUE5Ojl5//XXdcMMNkqTjx4+ra9euBlfneQQWAADagT/84Q966qmnNH78eM2cOVNDhw6VJL388svOU0UdGZc1AwDQTthsNlVVVenyyy93bispKVFQUJC6d+9uYGWeR2ABAACmx51uAQAwqREjRig7O1uXX365hg8fLh8fn2b75ufnt2FlbY/AAgCASU2bNk3Hjx/X5ZdfrqSkJKPLMRSnhAAAMDGLxaJRo0Zp/vz5mjlzpoKDg40uyRBcJQQAgInt27dPP/jBD7RkyRL16NFDt912m3Jycowuq80xwwIAQDtQU1OjF154QZs3b1ZOTo769++v+fPna+7cuYqMjDS6PI8jsAAA0M4cPnxYmzZt0v/93/+pvLxcN9xwg15++WWjy/IoAgsAAO1QTU2Nnn32WaWnp+vLL7+UzWYzuiSP4iohAADakTfffFMbN27UX/7yF1ksFt1yyy2aP3++0WV5HDMsAACY3PHjx7V582Zt3rxZhw8fVkJCgubPn69bbrlFnTt3Nrq8NsEMCwAAJnbjjTfq73//u8LDwzVnzhzNmzdPgwYNMrqsNkdgAQDAxDp16qQdO3ZoypQpslqtRpdjGE4JAQAA0+PGcQAAwPQILAAAwPQILAAAwPQILAAAwPQILAAAwPQILAAAwPQILAAAwPQILAAAwPT+PxpLOvN4yHorAAAAAElFTkSuQmCC",
      "text/plain": [
       "<Figure size 640x480 with 1 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "fig, ax = plt.subplots()\n",
    "ax.boxplot(all_aucs, meanline=True, showmeans=True)\n",
    "plt.xticks([1, 2, 3, 4, 5], [\"MTT\",\"M-MTT\", \"MTC\", \"M-MTC\", \"VisualBERT\"], rotation=90)\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 72,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "multimodal_text_tags\n",
      "0.7214094173240341\n",
      "multimodal_text_tags_BO\n",
      "0.763443900677409\n",
      "multimodal_text_captions\n",
      "0.759676891670675\n",
      "multimodal_text_captions_BO\n",
      "0.7863781637966971\n",
      "model_VisualBERT\n",
      "0.6948312824033991\n"
     ]
    }
   ],
   "source": [
    "for k in dict.keys():\n",
    "    print(k)\n",
    "    print(np.mean(list(dict[k].values())))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 81,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "multimodal_text_tags\n",
      "0.656627234648068\n",
      "multimodal_text_tags_BO\n",
      "0.718594401354818\n",
      "multimodal_text_captions\n",
      "0.7619871833413501\n",
      "multimodal_text_captions_BO\n",
      "0.7928017275933942\n",
      "model_VisualBERT\n",
      "0.6167859648067981\n"
     ]
    }
   ],
   "source": [
    "for k in bias.keys():\n",
    "    print(k)\n",
    "    print(np.mean(list(bias[k].values())))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 73,
   "metadata": {},
   "outputs": [],
   "source": [
    "all_bias=[]\n",
    "for model in models_bias:\n",
    "    all_bias.append(list(dict[model].values()))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 74,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAiwAAAHcCAYAAAD1DfFtAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjYuMCwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy89olMNAAAACXBIWXMAAA9hAAAPYQGoP6dpAAA890lEQVR4nO3df1RVdb7/8RfnKD+OAmkIopGHxh9Q/sbEUL6jjUUljgw5mUoamVOm6zYh00T5I82gqcnrmpkmqqtp1yYthvF21dG52Wg0Ut6BZowGVFTSFEjzKogICXz/KE+dAOOg5+wN5/lYi+WcvT/7s99nTsCLz/7sz/ZpampqEgAAgIlZjC4AAADg+xBYAACA6RFYAACA6RFYAACA6RFYAACA6RFYAACA6RFYAACA6RFYAACA6XUxuoArobGxUcePH1dgYKB8fHyMLgcAALRBU1OTqqur1adPH1kslx5D6RSB5fjx44qIiDC6DAAA0A5Hjx7VNddcc8k2nSKwBAYGSvrqDQcFBRlcDQAAaIuqqipFREQ4fo9fSqcILBcvAwUFBRFYAADoYNoynYNJtwAAwPQILAAAwPQILAAAwPQILAAAwPQILAAAwPQILAAAwPQILAAAwPQILAAAwPQILAAAwPQILAAAwPQILAAAwPQILAAAwPQ6xcMPge9z7tw5lZSUtLl9bW2tysrKZLfbFRAQ4NK5oqKiZLPZXC0RAHAJBBZ4hZKSEsXExHjkXAUFBRo5cqRHzgUA3oLAAq8QFRWlgoKCNrcvLi5WSkqK1q9fr+joaJfPBQC4sggs8Ao2m61dox7R0dGMlgCACTDpFgAAmB6BBQAAmB6BBQAAmB6BBQAAmB6BBQAAmB6BBQAAmB6BBQAAmB6BBQAAmB6BBQAAmF67AssLL7wgu90uf39/xcbGas+ePZdsv2rVKg0aNEgBAQGKiIjQI488ovPnzzv2P/nkk/Lx8XH6YnlzAABwkctL82/cuFFpaWnKzs5WbGysVq1apYSEBO3bt0+hoaHN2v/hD3/QY489pjVr1iguLk779+/XvffeKx8fH61cudLR7oYbbtA777zzTWFdeGoAAAD4issjLCtXrtTcuXOVmpqq66+/XtnZ2bLZbFqzZk2L7Xfv3q2xY8dqxowZstvtuvXWWzV9+vRmozJdunRR7969HV8hISHte0cAAKDTcSmw1NfXq6CgQBMnTvymA4tFEydOVH5+fovHxMXFqaCgwBFQDh06pK1bt+qOO+5wanfgwAH16dNH1113nWbOnKkjR460WkddXZ2qqqqcvgAAQOfl0nWXkydPqqGhQWFhYU7bw8LCVFJS0uIxM2bM0MmTJzVu3Dg1NTXpwoULevDBB/X444872sTGxmrt2rUaNGiQysvLtWzZMsXHx6uoqEiBgYHN+szKytKyZctcKR0AAHRgbr9LaOfOncrMzNTvf/97FRYWKjc3V1u2bNFTTz3laHP77bfrpz/9qYYOHaqEhARt3bpVp0+f1ptvvtlinxkZGTpz5ozj6+jRo+5+GwAAwEAujbCEhITIarWqsrLSaXtlZaV69+7d4jGLFy/WPffco/vvv1+SNGTIENXU1OhnP/uZnnjiCVkszTPTVVddpYEDB6q0tLTFPv38/OTn5+dK6QCA7zh37lyro+Otqa2tVVlZmex2uwICAtp8XFRUlGw2m6slAg4uBRZfX1/FxMRox44dSkpKkiQ1NjZqx44dWrBgQYvHnDt3rlkosVqtkqSmpqYWjzl79qwOHjyoe+65x5XyAAAuKCkpUUxMjEfOVVBQoJEjR3rkXOicXL53OC0tTbNnz9aoUaM0evRorVq1SjU1NUpNTZUkzZo1S3379lVWVpYkafLkyVq5cqVGjBih2NhYlZaWavHixZo8ebIjuKSnp2vy5Mnq16+fjh8/rqVLl8pqtWr69OlX8K0CAL4tKipKBQUFLh1TXFyslJQUrV+/XtHR0S6dC7gcLgeWadOm6cSJE1qyZIkqKio0fPhwbdu2zTER98iRI04jKosWLZKPj48WLVqkY8eOqVevXpo8ebKefvppR5vPPvtM06dP1xdffKFevXpp3Lhx+uCDD9SrV68r8BYBAC2x2WztHvWIjo5mxAQe5dPU2nWZDqSqqkrBwcE6c+aMgoKCjC4HnUBhYaFiYmIYxga+g+8NXEmu/P7mWUIAAMD0CCwAAMD0CCwAAMD0CCwAAMD0CCwAAMD0CCwAAMD0CCwAAMD0XF44Dm3n6nM6eEYHAAAtI7C4kaee08ECTgCAzo7A4kauPqeDZ3QAANAyAosbtfc5HTyjAwAAZ0y6BQAApkdgAQAApkdgAQAApkdgAQAApkdgAQAApkdgAQAApkdgAQAApkdgAQAApsfCcQBMq6GhQXl5eSovL1d4eLji4+NltVqNLguAARhhAWBKubm56t+/vyZMmKAZM2ZowoQJ6t+/v3Jzc40uDYABCCwATCc3N1dTp07VkCFDlJ+fr+rqauXn52vIkCGaOnUqoQXwQgQWAKbS0NCghQsXKjExUZs2bdKYMWPUvXt3jRkzRps2bVJiYqLS09PV0NBgdKkAPIg5LABMJS8vT2VlZXrjjTdksTj/TWWxWJSRkaG4uDjl5eVp/PjxxhQJXGHnzp1TSUmJS8fU1taqrKxMdrtdAQEBbT4uKipKNpvN1RINR2ABYCrl5eWSpMGDB7e4/+L2i+2AzqCkpEQxMTEeOVdBQYFGjhzpkXNdSQQWAKYSHh4uSSoqKtKYMWOa7S8qKnJqB3QGUVFRKigocOmY4uJipaSkaP369YqOjnbpXB0RgQWAqcTHx8tutyszM1ObNm1yuizU2NiorKwsRUZGKj4+3sAqgSvLZrO1e9QjOjq6Q46YuIpJtwBMxWq16vnnn9fmzZuVlJTkdJdQUlKSNm/erF//+tesxwJ4GUZYAJhOcnKycnJytHDhQsXFxTm2R0ZGKicnR8nJyQZWB8AIBBYAppScnKwpU6aw0i0ASQQWACZmtVq5ddlFBw4cUHV1tdv6Ly4udvrXHQIDAzVgwAC39Y+OicACAJ3EgQMHNHDgQI+cKyUlxa3979+/n9ACJwQWAOgkLo6suHqbqyvau1hZW128Vdedo0TomAgsANDJuPs217Fjx7qtb6A13NYMAABMj8ACAABMj8ACAABMj8ACAABMj8ACAABMj8ACAABMj8ACAABMj8ACAABMj8ACAABMj8ACAABMj8ACAABMj8ACAABMr12B5YUXXpDdbpe/v79iY2O1Z8+eS7ZftWqVBg0apICAAEVEROiRRx7R+fPnL6tPAADgPVwOLBs3blRaWpqWLl2qwsJCDRs2TAkJCfr8889bbP+HP/xBjz32mJYuXari4mKtXr1aGzdu1OOPP97uPgEAgHdxObCsXLlSc+fOVWpqqq6//nplZ2fLZrNpzZo1LbbfvXu3xo4dqxkzZshut+vWW2/V9OnTnUZQXO0TAAB4ly6uNK6vr1dBQYEyMjIc2ywWiyZOnKj8/PwWj4mLi9P69eu1Z88ejR49WocOHdLWrVt1zz33tLvPuro61dXVOV5XVVW58jbQCRw4cEDV1dVu67+4uNjpX3cJDAzUgAED3HoOAOgMXAosJ0+eVENDg8LCwpy2h4WFqaSkpMVjZsyYoZMnT2rcuHFqamrShQsX9OCDDzouCbWnz6ysLC1btsyV0tGJHDhwQAMHDvTIuVJSUtx+jv379xNaAOB7uBRY2mPnzp3KzMzU73//e8XGxqq0tFQPP/ywnnrqKS1evLhdfWZkZCgtLc3xuqqqShEREVeqZJjcxZGV9evXKzo62i3nqK2tVVlZmex2uwICAtxyjuLiYqWkpLh1pAgAOguXAktISIisVqsqKyudtldWVqp3794tHrN48WLdc889uv/++yVJQ4YMUU1NjX72s5/piSeeaFeffn5+8vPzc6V0dELR0dEaOXKk2/ofO3as2/oGALjGpcDi6+urmJgY7dixQ0lJSZKkxsZG7dixQwsWLGjxmHPnzslicZ7ba7VaJUlNTU3t6hMA0JzPhfMa0duigNP7peMdc5mtgNP7NaK3RT4Xzn9/Y3gVly8JpaWlafbs2Ro1apRGjx6tVatWqaamRqmpqZKkWbNmqW/fvsrKypIkTZ48WStXrtSIESMcl4QWL16syZMnO4LL9/UJAPh+/mePqPCB7tJ7D0jvGV1N+0RLKnygu4rPHpEUZ3Q5MBGXA8u0adN04sQJLVmyRBUVFRo+fLi2bdvmmDR75MgRpxGVRYsWycfHR4sWLdKxY8fUq1cvTZ48WU8//XSb+wQAfL/z3a/VyJfO6vXXX1d0VJTR5bRLcUmJZs6cqdV3XGt0KTCZdk26XbBgQauXa3bu3Ol8gi5dtHTpUi1durTdfQIAvl9TF399VNGo2qsGSn2GG11Ou9RWNOqjikY1dfE3uhSYTMe8yAkAALwKgQUAAJgegQUAAJgegQUAAJgegQUAAJie25fmB660zrA4lsQCWQDgCgILOpzOsDiWxAJZAOAKAgs6nM6wOJbEAlkA4AoCCzqczrA4luS9C2SdO3dOJSUlbW5/OU/OjoqKks1mc7XEDq1LcBeV1ZbJ/4tv/rsK8g3SNYHXqK6hTgdPH2x2zPVXXy9JOnzmsGov1Drt69u9r4L9gnXq/ClV1FQ47evWtZv6BfVTQ2OD9v3fvmb9DugxQF0tXXW06qiqv3R+KnmoLVQhASE6U3dGx84ec2wvqy1Tl2B+NaE5/qsA4FElJSWKiYnxyLkKCgrc+kRvM+o5oaeWli6VSr/ZNum6SXom/hlV1lRq2uZpzY75ePbHkqRFf1ukvSf2Ou3LHJepyT+YrO1l25X5YabTvrg+cXrplpdUe6G2xX53Tdulnv499ez/Pqudn+102pc+Kl2zb5itD8o/UPqu9GbvAfguAgsAj4qKilJBQUGb2xcXFyslJUXr169XdHS0y+fyNqf+ekovpr/o9N6DfIMkSWHdwrQxcWOrx64Yu6LFERZJSrAnaFivYU77unXtJkkK6BLQYr+BvoGSpEdvfFTzhs9z2hdqC5UkjQkf43RsSUmJpv11mrT40u8T3ofAAsCjbDZbu0Y9oqOjvW60pD0unLkge4DdcZnn2/ysfi1uvygyOLLVfT39e6qnf8sjH1aL9ZL9RgRFtLov2C9YwX7BjtfnA87rwpkLrbaH9+q494QCAACvQWABAACmR2ABAACmR2ABAACmR2ABAACmR2ABAACmx23NAC7bgQMHVF1d/f0N26G4uNjpX3cJDAzUgAED3HoOAO1HYAFwWQ4cOKCBAwe6/TwpKSluP8f+/fsJLYBJEVgAXJaLIyvtWYm2LS7nWUJtdXE1XXeNEgG4fAQWAFeEO1eiHTt2rFv6BdBxMOkWAACYHoEFAACYHoEFAACYHoEFAACYHoEFAACYHoEFAACYHoEFAACYHoEFAACYHoEFAACYHoEFAACYHoEFAACYHs8SAnBZfC6c14jeFgWc3i8d75h/AwWc3q8RvS3yuXDe6FIAtILAgg6pS3AXldWWyf8Lf8e2IN8gXRN4jeoa6nTw9MFmx1x/9fWSpMNnDqv2Qq3Tvr7d+yrYL1inzp9SRU2F075uXbupX1A/NTQ2aN//7WvW74AeA9TV0lVHq46q+kvnp/2G2kIVEhCiM3VndOzsMad9x88fd+1Nm5T/2SMqfKC79N4D0ntGV9M+0ZIKH+iu4rNHJMUZXQ6AFhBY0CH1nNBTS0uXSqXfbJt03SQ9E/+MKmsqNW3ztGbHfDz7Y0nSor8t0t4Te532ZY7L1OQfTNb2su3K/DDTaV9cnzi9dMtLqr1Q22K/u6btUk//nnr2f5/Vzs92Ou1LH5Wu2TfM1gflHyh9V7rTvn7+/Vx5y6Z1vvu1GvnSWb3++uuKjooyupx2KS4p0cyZM7X6jmuNLgVAKwgs6JBO/fWUXkx/UVHf+gUZ5BskSQrrFqaNiRtbPXbF2BUtjrBIUoI9QcN6DXPa161rN0lSQJeAFvsN9A2UJD1646OaN3ye075QW6gkaUz4mGbHHtp3SFu0pfU32UE0dfHXRxWNqr1qoNRnuNHltEttRaM+qmhUUxf/728MwBAEFnRIF85ckD3A7rjM821+Vr8Wt18UGRzZ6r6e/j3V079ni/usFusl+40Iimh1X7BfsIL9gp22nfdnvgQAtBWBBcBl6+hzispqy9QlmB+HgJnxHQrgsnWGOUU9J7Q8sgbAHAgsAC5bR59TVFJSoml/nSYtvvT7BGAcAguAy9bR5xSdDzivC2cutNoegPE65ipPAADAqxBYAACA6RFYAACA6TGHxQUHDhxQdXX19zdsp+LiYqd/3SUwMFADBgxw6zkAALiS2hVYXnjhBT333HOqqKjQsGHD9Nvf/lajR49use348eO1a9euZtvvuOMObdny1Sqf9957r9atW+e0PyEhQdu2bWtPeW5x4MABDRw40CPnSklJcfs59u/fT2gBAHQYLgeWjRs3Ki0tTdnZ2YqNjdWqVauUkJCgffv2KTQ0tFn73Nxc1dfXO15/8cUXGjZsmH760586tbvtttv06quvOl77+fm5WppbXRxZWb9+vaKjo91yjtraWpWVlclutysgIMAt5yguLlZKSopbR4rc7dy5c5KkwsJCt53DU58FAKBtXA4sK1eu1Ny5c5WamipJys7O1pYtW7RmzRo99thjzdr37Ol8S+KGDRtks9maBRY/Pz/17t3b1XI8Ljo6WiNHjnRb/2PHjnVb351FSUmJJGnu3LkGV3JlBAYGGl0COonOEOYJ8miNS4Glvr5eBQUFysjIcGyzWCyaOHGi8vPz29TH6tWrdffdd6tbt25O23fu3KnQ0FD16NFDN998s1asWKGrr766xT7q6upUV1fneF1VVeXK20AHl5SUJEmKioqSzWZr0zEXf8i21eHDh7V48WI99dRTioxsfZ2Qlrjyg5z5RLiSOlOYJ8jju1wKLCdPnlRDQ4PCwsKctoeFhTm+US5lz549Kioq0urVq52233bbbUpOTlZkZKQOHjyoxx9/XLfffrvy8/NltVqb9ZOVlaVly5a5Ujo6kZCQEN1///0uHVNYWNiuuUGLF7u+9GlBQYFbR+GA1rQnzLvq4mVld14eJ8ijJR69S2j16tUaMmRIswm6d999t+N/DxkyREOHDtUPfvAD7dy5Uz/60Y+a9ZORkaG0tDTH66qqKkVEtL6qJRAVFaWCgoI2t7+cYe9vL08PeFJ7wnx7ufvyOPBdLgWWkJAQWa1WVVZWOm2vrKz83vknNTU12rBhg5YvX/6957nuuusUEhKi0tLSFgOLn5+f6SblwtxsNpvLP1yZTwQA5uFSYPH19VVMTIx27NjhGHpsbGzUjh07tGDBgkse+9Zbb6murq5Nw/KfffaZvvjiC4WHh7tSHgADuHuiJ3dsAZDacUkoLS1Ns2fP1qhRozR69GitWrVKNTU1jruGZs2apb59+yorK8vpuNWrVyspKanZRNqzZ89q2bJluvPOO9W7d28dPHhQjz76qPr376+EhITLeGsAPIGJngA8weXAMm3aNJ04cUJLlixRRUWFhg8frm3btjkm4h45ckQWi/OK//v27dP777+vv/zlL836s1qt2rt3r9atW6fTp0+rT58+uvXWW/XUU09x2QfoANw90dMTkzwlJnriymN19CurXZNuFyxY0OoloJ07dzbbNmjQIDU1NbXYPiAgQNu3b29PGQBMwFMTPZnkiY6E1dGvPJ4lBADAFcbq6FcegQUAADdhdfQrx/L9TQAAAIxFYAEAAKZHYAEAAKZHYAEAAKZHYAEAAKZHYAEAAKZHYAEAAKZHYAEAAKZHYAEAAKZHYAEAAKZHYAEAAKZHYAEAAKZHYAEAAKbH05oBeNS5c+dUUlLS5vbFxcVO/7oiKipKNpvN5eMAmA+BBYBHlZSUKCYmxuXjUlJSXD6moKBAI0eOdPk4AOZDYAHgUVFRUSooKGhz+9raWpWVlclutysgIMDlcwHoHAgsADzKZrO5POoxduxYN1UDoKNg0i0AADA9AgsAADA9AgsAADA9AgsAADA9AgsAADA9AgsAADA9bmt2QZfgLiqrLZP/F/6ObUG+Qbom8BrVNdTp4OmDzY65/urrJUmHzxxW7YVap319u/dVsF+wTp0/pYqaCqd93bp2U7+gfmpobNC+/9vXrN8BPQaoq6WrjlYdVfWX1U77Qm2hCgkI0Zm6Mzp29pjTvuPnj7v2pgEAMAECiwt6TuippaVLpdJvtk26bpKeiX9GlTWVmrZ5WrNjPp79sSRp0d8Wae+JvU77MsdlavIPJmt72XZlfpjptC+uT5xeuuUl1V6obbHfXdN2qad/Tz37v89q52c7nfalj0rX7Btm64PyD5S+K91pXz//fq68ZQAATIHA4oJTfz2lF9NfdFo9M8g3SJIU1i1MGxM3tnrsirErWhxhkaQEe4KG9RrmtK9b126SpIAuAS32G+gbKEl69MZHNW/4PKd9obZQSdKY8DHNjj2075C2aEvrbxIAABMisLjgwpkLsgfYHZd5vs3P6tfi9osigyNb3dfTv6d6+vdscZ/VYr1kvxFBEa3uC/YLVrBfsNO28/7nW20PAIBZMekWAACYHoEFAACYHoEFAACYHoEFAACYHoEFAACYHoEFAACYHoEFAACYHoEFAACYHoEFAACYHoEFAACYHkvzt5HPhfMa0duigNP7peMdN+cFnN6vEb0t8rnAEv0AgI6DwNJG/mePqPCB7tJ7D0jvGV1N+0VLKnygu4rPHpEUZ3Q5AAC0CYGljc53v1YjXzqr119/XdHfelpzR1NcUqKZM2dq9R3XGl0KAIOdO3dOJSUlLh1TXFzs9G9bRUVFyWazuXQM8G0EljZq6uKvjyoaVXvVQKnPcKPLabfaikZ9VNGopi7+RpcCwGAlJSWKiYlp17EpKSkutS8oKNDIkSPbdS5AIrAAgNeKiopSQUGBS8fU1taqrKxMdrtdAQEBLp0LuBwEFgDwUjabrV2jHmPHjnVDNcClddzbXQAAgNdoV2B54YUXZLfb5e/vr9jYWO3Zs6fVtuPHj5ePj0+zr0mTJjnaNDU1acmSJQoPD1dAQIAmTpyoAwcOtKc0AADQCbkcWDZu3Ki0tDQtXbpUhYWFGjZsmBISEvT555+32D43N1fl5eWOr6KiIlmtVv30pz91tHn22Wf1m9/8RtnZ2frwww/VrVs3JSQk6Px51goBAADtmMOycuVKzZ07V6mpqZKk7OxsbdmyRWvWrNFjjz3WrH3Pnj2dXm/YsEE2m80RWJqamrRq1SotWrRIU6ZMkSS99tprCgsL06ZNm3T33Xe7/KYAADASi41eeS4Flvr6ehUUFCgjI8OxzWKxaOLEicrPz29TH6tXr9bdd9+tbt26SZIOHz6siooKTZw40dEmODhYsbGxys/PbzGw1NXVqa6uzvG6qqrKlbcBAIBbsdjoledSYDl58qQaGhoUFhbmtD0sLKxNiw/t2bNHRUVFWr16tWNbRUWFo4/v9nlx33dlZWVp2bJlrpQOAIDHsNjolefR25pXr16tIUOGaPTo0ZfVT0ZGhtLS0hyvq6qqFBERcbnlAQBwRbDY6JXn0oW1kJAQWa1WVVZWOm2vrKxU7969L3lsTU2NNmzYoDlz5jhtv3icK336+fkpKCjI6QsAAHReLgUWX19fxcTEaMeOHY5tjY2N2rFjh2666aZLHvvWW2+prq6u2XLOkZGR6t27t1OfVVVV+vDDD7+3TwAA4B1cviSUlpam2bNna9SoURo9erRWrVqlmpoax11Ds2bNUt++fZWVleV03OrVq5WUlKSrr77aabuPj49+/vOfa8WKFRowYIAiIyO1ePFi9enTR0lJSe1/ZwAAoNNwObBMmzZNJ06c0JIlS1RRUaHhw4dr27ZtjkmzR44ckcXiPHCzb98+vf/++/rLX/7SYp+PPvqoampq9LOf/UynT5/WuHHjtG3bNvn7G3/NDACA9ugS3EVltWXy/+Kb32VBvkG6JvAa1TXU6eDpg82Ouf7q6yVJh88cVu2FWqd9fbv3VbBfsE6dP6WKGuebUrp17aZ+Qf3U0Nigff+3r1m/A3oMUFdLVx2tOqrqL6ud9oXaQhUSEKIzdWd07Owxp33Hzx937U27Ubsm3S5YsEALFixocd/OnTubbRs0aJCamppa7c/Hx0fLly/X8uXL21MOAACm03NCTy0tXSqVfrNt0nWT9Ez8M6qsqdS0zdOaHfPx7I8lSYv+tkh7T+x12pc5LlOTfzBZ28u2K/PDTKd9cX3i9NItL6n2Qm2L/e6atks9/Xvq2f99Vjs/2+m0L31UumbfMFsflH+g9F3pTvv6+fdz5S27FQ8/BADADU799ZReTH/R6UnVQb5f3SQS1i1MGxM3tnrsirErWhxhkaQEe4KG9RrmtK9b16/WNgvoEtBiv4G+gZKkR298VPOGz3PaF2oLlSSNCR/T7NhD+w5pi7a0/iY9iMACAIAbXDhzQfYAu+Myz7f5Wf1a3H5RZHBkq/t6+vdUT/+eLe6zWqyX7DciqPUlQIL9ghXsF+y07by/8SvcXtRx1wsGAABeg8ACAABMj8ACAABMj8ACAABMj8ACAABMj8ACAABMj8ACAABMj3VYAJhWQ0OD8vLyVF5ervDwcMXHx8tqtRpdFgADMMICwJRyc3PVv39/TZgwQTNmzNCECRPUv39/5ebmGl0aAAMQWACYTm5urqZOnaohQ4YoPz9f1dXVys/P15AhQzR16lRCC+CFCCwATKWhoUELFy5UYmKiNm3apDFjxqh79+4aM2aMNm3apMTERKWnp6uhocHoUgF4EHNY2ujcuXOSpMLCQredo7a2VmVlZbLb7QoICHDLOYqLi93SL3Cl5OXlqaysTG+88YYsFue/qSwWizIyMhQXF6e8vDyNHz/emCIBeByBpY1KSkokSXPnzjW4kisjMDDQ6BKAFpWXl0uSBg8e3OL+i9svtgPgHQgsbZSUlCRJioqKks1mc8s5iouLlZKSovXr1ys6Otot55C+CisDBgxwW//A5QgPD5ckFRUVacyYMc32FxUVObUD4B0ILG0UEhKi+++/3yPnio6O1siRIz1yLsBs4uPjZbfblZmZqU2bNjldFmpsbFRWVpYiIyMVHx9vYJUAPI1JtwBMxWq16vnnn9fmzZuVlJTkdJdQUlKSNm/erF//+tesxwJ4GUZYAJhOcnKycnJytHDhQsXFxTm2R0ZGKicnR8nJyQZWB8AIBBYAppScnKwpU6aw0i0ASQQWACZmtVq5dRmAJOawAACADoDAAgAATI/AAgAATI/AAgAATI/AAgAATI/AAgAATI/AAgAATI/AAgAATI/AAgAATI/AAgAATI/AAgAATI/AAgAATI/AAgAATI/AAgAATI/AAgAATK+L0QUAANDZnDt3TpJUWFjotnPU1taqrKxMdrtdAQEBbjlHcXGxW/ptDwIL8B0NDQ3Ky8tTeXm5wsPDFR8fL6vVanRZADqQkpISSdLcuXMNruTKCAwMNLoEAgvwbbm5uVq4cKHKysoc2+x2u55//nklJycbVxiADiUpKUmSFBUVJZvN5pZzFBcXKyUlRevXr1d0dLRbziF9FVYGDBjgtv7bisACfC03N1dTp05VYmKi3njjDQ0ePFhFRUXKzMzU1KlTlZOTQ2gB0CYhISG6//77PXKu6OhojRw50iPnMhKTbgF9dRlo4cKFSkxM1KZNmzRmzBh1795dY8aM0aZNm5SYmKj09HQ1NDQYXSoAeCUCCyApLy9PZWVlevzxx2WxOH9bWCwWZWRk6PDhw8rLyzOoQgDwbgQWQFJ5ebkkafDgwS3uv7j9YjsAgGcRWABJ4eHhkqSioqIW91/cfrEdAMCzCCyApPj4eNntdmVmZqqxsdFpX2Njo7KyshQZGan4+HiDKgQA70ZgASRZrVY9//zz2rx5s5KSkpSfn6/q6mrl5+crKSlJmzdv1q9//WvWYwEAg7QrsLzwwguy2+3y9/dXbGys9uzZc8n2p0+f1vz58xUeHi4/Pz8NHDhQW7dudex/8skn5ePj4/QVFRXVntKAdktOTlZOTo4+/vhjxcXFKSgoSHFxcSoqKuKWZgAwmMvrsGzcuFFpaWnKzs5WbGysVq1apYSEBO3bt0+hoaHN2tfX1+uWW25RaGiocnJy1LdvX3366ae66qqrnNrdcMMNeuedd74prAtLxMDzkpOTNWXKFFa6BQCTcTkVrFy5UnPnzlVqaqokKTs7W1u2bNGaNWv02GOPNWu/Zs0anTp1Srt371bXrl0lfbVyaLNCunRR7969XS0HuOKsVqvGjx9vdBkAgG9x6ZJQfX29CgoKNHHixG86sFg0ceJE5efnt3jM22+/rZtuuknz589XWFiYBg8erMzMzGYLcB04cEB9+vTRddddp5kzZ+rIkSOt1lFXV6eqqiqnLwAA0Hm5FFhOnjyphoYGhYWFOW0PCwtTRUVFi8ccOnRIOTk5amho0NatW7V48WI9//zzWrFihaNNbGys1q5dq23btunFF1/U4cOHFR8fr+rq6hb7zMrKUnBwsOMrIiLClbcBAAA6GLdPFGlsbFRoaKhefvllWa1WxcTE6NixY3ruuee0dOlSSdLtt9/uaD906FDFxsaqX79+evPNNzVnzpxmfWZkZCgtLc3xuqqqitACAEAn5lJgCQkJkdVqVWVlpdP2ysrKVuefhIeHq2vXrk6TFqOjo1VRUaH6+nr5+vo2O+aqq67SwIEDVVpa2mKffn5+8vPzc6V0AADQgbl0ScjX11cxMTHasWOHY1tjY6N27Nihm266qcVjxo4dq9LSUqfFuPbv36/w8PAWw4oknT17VgcPHmRVUQAAIKkd67CkpaXplVde0bp161RcXKx58+appqbGcdfQrFmzlJGR4Wg/b948nTp1Sg8//LD279+vLVu2KDMzU/Pnz3e0SU9P165du1RWVqbdu3frJz/5iaxWq6ZPn34F3iIAAOjoXJ7DMm3aNJ04cUJLlixRRUWFhg8frm3btjkm4h45csTpabcRERHavn27HnnkEQ0dOlR9+/bVww8/rF/+8peONp999pmmT5+uL774Qr169dK4ceP0wQcfqFevXlfgLQIAgI6uXZNuFyxYoAULFrS4b+fOnc223XTTTfrggw9a7W/Dhg3tKQMAAHgJniUEAABMj8ACAABMj8ACAABMj8ACAABMj8ACAABMj8ACAABMj8ACAABMj8ACAABMj8ACAABMj8ACAABMj8ACAABMj8ACAABMj8ACAABMr11PawY6s4aGBuXl5am8vFzh4eGKj4+X1Wo1uiwA8GqMsADfkpubq/79+2vChAmaMWOGJkyYoP79+ys3N9fo0gDAqxFYgK/l5uZq6tSpGjJkiPLz81VdXa38/HwNGTJEU6dOJbQAgIEILIC+ugy0cOFCJSYmatOmTRozZoy6d++uMWPGaNOmTUpMTFR6eroaGhqMLhUAvBKBBZCUl5ensrIyPf7447JYnL8tLBaLMjIydPjwYeXl5RlUIQB4NwILIKm8vFySNHjw4Bb3X9x+sR0AwLMILICk8PBwSVJRUVGL+y9uv9gOAOBZBBZAUnx8vOx2uzIzM9XY2Oi0r7GxUVlZWYqMjFR8fLxBFQKAd2MdFjc6d+6cSkpK2ty+uLjY6d+2ioqKks1mc+kYOLNarXr++ec1depUJSUlKSMjQ4MHD1ZRUZGysrK0efNm5eTksB4LABiEwOJGJSUliomJcfm4lJQUl9oXFBRo5MiRLp8HzpKTk5WTk6OFCxcqLi7OsT0yMlI5OTlKTk42sDoA8G4EFjeKiopSQUFBm9vX1taqrKxMdrtdAQEBLp0HV0ZycrKmTJnCSrcAYDIEFjey2Wwuj3yMHTvWTdWgraxWq8aPH290GQCAb2HSLQAAMD0CCwAAMD0CCwAAMD0CCwAAMD0CCwAAMD0CCwAAMD0CCwAAMD0CCwAAMD0CCwAAMD0CCwAAMD0CCwAAMD0CCwAAMD0CCwAAMD0CCwAAMD0CCwAAMD0CCwAAMD0CCwAAMD0CCwAAMD0CCwAAMD0CCwAAMD0CCwAAML12BZYXXnhBdrtd/v7+io2N1Z49ey7Z/vTp05o/f77Cw8Pl5+engQMHauvWrZfVJwAA8B4uB5aNGzcqLS1NS5cuVWFhoYYNG6aEhAR9/vnnLbavr6/XLbfcorKyMuXk5Gjfvn165ZVX1Ldv33b3CQAAvIvLgWXlypWaO3euUlNTdf311ys7O1s2m01r1qxpsf2aNWt06tQpbdq0SWPHjpXdbtcPf/hDDRs2rN19AgAA7+JSYKmvr1dBQYEmTpz4TQcWiyZOnKj8/PwWj3n77bd10003af78+QoLC9PgwYOVmZmphoaGdvdZV1enqqoqpy8AANB5uRRYTp48qYaGBoWFhTltDwsLU0VFRYvHHDp0SDk5OWpoaNDWrVu1ePFiPf/881qxYkW7+8zKylJwcLDjKyIiwpW3AQAAOhi33yXU2Nio0NBQvfzyy4qJidG0adP0xBNPKDs7u919ZmRk6MyZM46vo0ePXsGKAQCA2XRxpXFISIisVqsqKyudtldWVqp3794tHhMeHq6uXbvKarU6tkVHR6uiokL19fXt6tPPz09+fn6ulA4AADowl0ZYfH19FRMTox07dji2NTY2aseOHbrppptaPGbs2LEqLS1VY2OjY9v+/fsVHh4uX1/fdvUJAAC8i8uXhNLS0vTKK69o3bp1Ki4u1rx581RTU6PU1FRJ0qxZs5SRkeFoP2/ePJ06dUoPP/yw9u/fry1btigzM1Pz589vc58AAMC7uXRJSJKmTZumEydOaMmSJaqoqNDw4cO1bds2x6TZI0eOyGL5JgdFRERo+/bteuSRRzR06FD17dtXDz/8sH75y1+2uU8AAODdfJqampqMLuJyVVVVKTg4WGfOnFFQUJDR5QAA4HaFhYWKiYlRQUGBRo4caXQ57eLK72+eJQQAAEyPwAIAAEyPwAIAAEyPwAIAAEyPwAIAAEyPwAIAAEyPwAIAAEyPwAIAAEyPwAIAAEyPwAIAAEyPwAIAAEyPwAIAAEyPwAIAAEyPwAIAAEyPwAIAAEyPwAIAAEyPwAIAAEyPwAIAAEyPwAIAAEyPwAIAAEyPwAIAAEyPwAIAAEyPwAIAAEyPwAIAAEyPwAIAAEyPwAIAAEyPwAIAAEyvi9EFAADg7c6dO6eSkhKXjikuLnb6t62ioqJks9lcOsYMCCwAABispKREMTEx7To2JSXFpfYFBQUaOXJku85lJAILAAAGi4qKUkFBgUvH1NbWqqysTHa7XQEBAS6dqyPyaWpqajK6iMtVVVWl4OBgnTlzRkFBQUaXAwAA2sCV399MugUAAKZHYAEAAKZHYAEAAKZHYAEAAKZHYAEAAKZHYAEAAKZHYAEAAKZHYAEAAKZHYAEAAKZHYAEAAKZHYAEAAKZHYAEAAKZHYAEAAKbXxegCroSLD5yuqqoyuBIAANBWF39vX/w9fimdIrBUV1dLkiIiIgyuBAAAuKq6ulrBwcGXbOPT1JZYY3KNjY06fvy4AgMD5ePjY3Q57VZVVaWIiAgdPXpUQUFBRpfj1fgszIPPwlz4PMyjM3wWTU1Nqq6uVp8+fWSxXHqWSqcYYbFYLLrmmmuMLuOKCQoK6rD/8XU2fBbmwWdhLnwe5tHRP4vvG1m5iEm3AADA9AgsAADA9AgsJuLn56elS5fKz8/P6FK8Hp+FefBZmAufh3l422fRKSbdAgCAzo0RFgAAYHoEFgAAYHoEFgAAYHoEFgAAYHoEFgAAYHoEFgAATMxqterzzz83ugzDEVgMcvPNN+v06dNGl4Gv3XfffY6HaAKAmbD6yFcILAbZuXOn6uvrjS4DX1u3bp1qa2uNLgOSDhw4oOnTpzseO/9tZ86c0YwZM3To0CEDKvM+x48fV3p6equfxS9+8QtVVlYaUBm8Uad4+CFwufgLxjyee+45RUREtPgwt+DgYEVEROi5557Tiy++aEB13mXlypWqqqpq9bOorq7WypUr9atf/cqA6rzLf/zHf6h79+6XbPNv//ZvHqrGGKx0axCLxaJ3331XPXv2vGS7oUOHeqgi72axWHTgwAH16tXrku068hNRO4pBgwZp/fr1uvHGG1vcX1BQoBkzZmjfvn0ersz7DB48WNnZ2Ro3blyL+3fv3q25c+fqk08+8XBl3sViseiaa66R1WpttY2Pj0+nH3kksBjEYrHIx8enxb/sL2738fFRQ0ODAdV5n4ufR2v4PDwnICBAJSUl6tevX4v7P/30U0VHR+vcuXMersz7dOvWTcXFxbr22mtb3H/kyBFFR0erpqbGw5V5F4vFooqKCoWGhhpdiqG4JGSgDz/88Hv/oofn5OTkfO+IF9wvODhYBw8ebDWwlJaWMtLlIQEBASorK2s1sJSVlSkgIMDDVXmfS/0x5U0YYTEIidlc+DzM46677tKXX36pP/3pTy3unzJlinx9ffXWW295uDLvM2nSJPXp00evvPJKi/vvv/9+HT9+XFu3bvVwZd6Fn09f4S4hAKaSkZGhP//5z5o6dar27NmjM2fO6MyZM/rwww915513avv27crIyDC6TK+Qnp6uV199Venp6U53A1VWVmrhwoVau3at0tPTDazQOyxduvSSE25zc3O9Y75jEwwxfvz4pqNHjxpdBr5mt9ubPvvsM6PLwNf++7//u6lXr15NFovF6atXr15N//Vf/2V0eV4lOzu7yc/Pr8lisTRdddVVTT169GiyWCxNfn5+Tb///e+NLs9rZGdnN915551N06dPb/rggw+ampqamnbs2NE0fPjwJpvN1vTggw8aXKH7cUnIIFarVeXl5V4/xGcWfB7msXz5cqWnp8vHx0fbtm1TaWmpmpqaNHDgQN16662y2WxGl+g1XnvtNU2bNk0nT57Um2++6fRZTJ06Vddcc43RJXqFZ555RkuWLNHQoUNVUlKipqYmPfHEE/rtb3+rhx9+WA888IB69OhhdJluR2AxCNckzYXPwzwIj+bBZ2EOgwYN0uOPP67Zs2crLy9PP/zhD3XHHXdo48aN6tatm9HleQxzWAzEzG9z4fMwB/6GMg8+C3M4cuSIbr75ZklSfHy8unbtqmXLlnlVWJG4rdlQAwcO/N5fkqdOnfJQNfjRj36kLl0u/S1RWFjooWq8G+HRPPgsjFdXVyd/f3/Ha19fX69cgoHAYqBly5YpODjY6DLwtYSEhO9d+hqeQZg3D4K8OSxevNgxf6u+vl4rVqxo9vtj5cqVRpTmMQQWA919991cGzaRX/ziF3weJkGYNw+CvPH+3//7f06PooiLi2u2DL83jIQx6dYgTGYzFz4P82ACtHnwWcBMmHRrkKavn00DcyC3mwffF+bBZwEzYYTFIKmpqW36YbBmzRoPVINPP/1U1157LT+gTcBisaiyspLnbJkAIyzmcP311+v99993TLR96KGHtHz5coWEhEiSPv/8c9nt9k7/QFACi0EsFov69eunESNGXPKv+9aep4Ira/ny5W1qt2TJEjdXAsK8eRDkzeG7wTEoKEj/+Mc/dN1110n66lEJ4eHhamxsNLJMt2PSrUHmzZunN954Q4cPH1ZqaqpSUlK88jY1s3jyySfVp08fhYaGthogfXx8CCwesG7dujaFebjfunXr2tSO7wvPaun7whtCJSMsBqqrq1Nubq7WrFmj3bt3a9KkSZozZ45uvfVWr/iPz0wmTZqkd999VwkJCbrvvvuUmJgoi4UpXkaYP3++3njjDfXr148wbzCLxdKmIM9tze713RGWwMBA/fOf/3QaYenTp48aGhqMLNPtCCwm8emnn2rt2rV67bXXdOHCBX3yySfcSuhhx48f17p167R27VpVVVVp1qxZuu+++zRo0CCjS/M6hHlzIMibg9VqVUVFhWNeV2BgoPbu3avIyEhJBBZ42NGjR/Xqq69q7dq1qq+vV0lJCYHFQO+9955effVV/fGPf9SQIUP0zjvvKCAgwOiyvBJh3lgEeeNZLBYNHjzYsYDf3r17FRUVJV9fX0lyfF909sDCHBYDffuvyPfff1+JiYn63e9+p9tuu42/Ygx24403qqysTP/617/00Ucf6csvvySwGMRiscjHx0dNTU2d/geyGfXp00cZGRnKyMhwBPkbb7yRIO9BS5cudXo9ZcqUZm3uvPNOT5VjGEZYDPLQQw9pw4YNioiI0H333aeZM2c6blGDcfLz87VmzRq9+eabGjhwoFJTUzVjxgxdddVVRpfmVVoK86mpqYR5g9XW1uqtt97SCy+8oI8//lgVFRUKCgoyuix4CQKLQSwWi6699lqNGDHiktfkc3NzPViV93r22We1du1anTx5UjNnzlRqaqqGDh1qdFleiTBvPgR5c6qvr1d9fb3XXCIlsBjk3nvvbdPkwVdffdUD1eBigExMTHRcF25JZ3+4mBkQ5s2DIG8er776qgoLCzVmzBjNnDlTGRkZWrlypS5cuKCbb75ZGzZs0NVXX210mW5FYAEkjR8//nsDpI+Pj959910PVeS9CPPmQZA3h6efflpPP/20xo4dq8LCQt11113atGmTfv7zn8tiseg3v/mNEhMT9eKLLxpdqlsRWAAALSLIm8OAAQO0fPlyTZ8+XX//+98VGxurN9980zHR9s9//rMefPBBffrppwZX6l4EFqAVf/vb3zRq1Cj5+fkZXQoAL+bn56fS0lJFREQ4Xu/du9dxa/mxY8cUGRmp+vp6I8t0O6bbA624/fbbdezYMaPLAEzlb3/7m+rq6owuw6t8+eWXTn84+fr6qmvXro7XXbp08Ypb/lmHBWgFg49Ac7fffrvTg/fgGf/6179UUVEh6aufTSUlJTp79qwk6eTJk0aW5jEEFgBAmxHkjfGjH/3I6f/7xMRESXIsqugNj6wgsACteOmllxQWFmZ0GQC83OHDh40uwRSYdAtIuu+++9rUbs2aNW6uBDC3P/zhD5oyZYq6detmdCnwMgQWQF+tN9GvXz+NGDHikkPef/rTnzxYFWAsgrzx9u7d2+a2nX1RPwILIGn+/Pl644031K9fP6WmpiolJUU9e/Y0uizAUAR543374Z+X4uPj0+nvFCKwAF/79gP3du/erUmTJmnOnDm69dZbvWJCG/BdBHnjubIYXL9+/dxYifEILEALPv30U61du1avvfaaLly4oE8++cRrHjAGfBtBHmbBXUJAC749DNvZh1mBS/Hz89P06dM1ffp0R5B/6KGHCPIG+te//qUjR440W9n2xz/+sUEVeQaBBfjat/+SfP/995WYmKjf/e53uu2222SxsCg0QJA31qFDh/STn/xEH3/8sdO8losjXZ39M+GnMCDpoYceUnh4uJ555hklJibq6NGjeuutt3THHXcQVuDV6urq9MYbb+iWW27RwIED9fHHH+t3v/udjhw5wuiKhz388MOKjIzU559/LpvNpk8++UTvvfeeRo0apZ07dxpdntsxhwXQV385XnvttRoxYsQlr8vn5uZ6sCrAWA899JA2bNigiIgI3XfffZo5c6ZCQkKMLstrhYSE6N1339XQoUMVHBysPXv2aNCgQXr33Xe1cOFCffTRR0aX6FZcEgIkzZo1iwmEwHdkZ2fr2muv1XXXXaddu3Zp165dLbYjyHtGQ0ODAgMDJX0VXo4fP65BgwapX79+2rdvn8HVuR+BBZC0du1ao0sATIcgby6DBw/WP//5T0VGRio2NlbPPvusfH199fLLL3vFwyi5JAQAQAewfft21dTUKDk5WaWlpUpMTNT+/ft19dVXa+PGjbr55puNLtGtCCwAAHRQp06dUo8ePbxiJIzAAgAATI85LAAAdAATJky45EjKu+++68FqPI/AAgBABzB8+HCn119++aX+8Y9/qKioSLNnzzamKA8isAAA0AH8+7//e4vbn3zySZ09e9bD1Xgec1gAAOjASktLNXr0aJ06dcroUtyKNccBAOjA8vPz5e/vb3QZbsclIQAAOoDk5GSn101NTSovL9ff//53LV682KCqPIfAAgBABxAcHOz02mKxaNCgQVq+fLluvfVWg6ryHOawAAAA02MOCwAAHcDRo0f12WefOV7v2bNHP//5z/Xyyy8bWJXnEFgAAOgAZsyYob/+9a+SpIqKCk2cOFF79uzRE088oeXLlxtcnfsRWAAA6ACKioo0evRoSdKbb76pIUOGaPfu3Xr99de94onzBBYAADqAL7/8Un5+fpKkd955Rz/+8Y8lSVFRUSovLzeyNI8gsAAA0AHccMMNys7OVl5env7nf/5Ht912myTp+PHjuvrqqw2uzv0ILAAAdAC/+tWv9NJLL2n8+PGaPn26hg0bJkl6++23HZeKOjNuawYAoINoaGhQVVWVevTo4dhWVlYmm82m0NBQAytzPwILAAAwPVa6BQDApEaOHKkdO3aoR48eGjFihHx8fFptW1hY6MHKPI/AAgCASU2ZMkXHjx9Xjx49lJSUZHQ5huKSEAAAJmaxWHTjjTdqzpw5mj59ugIDA40uyRDcJQQAgInt2rVLN9xwg9LT0xUeHq57771XeXl5RpflcYywAADQAdTU1OjNN9/U2rVrlZeXp/79+2vOnDmaPXu2evfubXR5bkdgAQCggyktLdWrr76q//zP/1RFRYVuu+02vf3220aX5VYEFgAAOqCamhq9/vrrysjI0OnTp9XQ0GB0SW7FXUIAAHQg7733ntasWaM//vGPslgsuuuuuzRnzhyjy3I7RlgAADC548ePa+3atVq7dq1KS0sVFxenOXPm6K677lK3bt2MLs8jGGEBAMDEbr/9dr3zzjsKCQnRrFmzdN9992nQoEFGl+VxBBYAAEysa9euysnJUWJioqxWq9HlGIZLQgAAwPRYOA4AAJgegQUAAJgegQUAAJgegQUAAJgegQUAAJgegQUAAJgegQUAAJgegQUAAJje/wcHjFTKjA+ecQAAAABJRU5ErkJggg==",
      "text/plain": [
       "<Figure size 640x480 with 1 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "fig, ax = plt.subplots()\n",
    "ax.boxplot(all_bias, meanline=True, showmeans=True)\n",
    "plt.xticks([1, 2, 3, 4, 5], [\"MTT\",\"M-MTT\", \"MTC\", \"M-MTC\", \"VisualBERT\"], rotation=90)\n",
    "plt.show()"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## T-Test\n",
    "### T-Test on Multimodal-Mitigated"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 75,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "t-test on multimodal bias metric, alpha: 0.05\n",
      "['multimodal_text_tags', 'multimodal_text_tags_BO', 'multimodal_text_captions', 'multimodal_text_captions_BO', 'model_VisualBERT']\n",
      "[[False False  True  True False]\n",
      " [False False False False  True]\n",
      " [ True False False  True  True]\n",
      " [ True False  True False  True]\n",
      " [False  True  True  True False]]\n",
      "t-test on multimodal bias metric, alpha: 0.1\n",
      "['multimodal_text_tags', 'multimodal_text_tags_BO', 'multimodal_text_captions', 'multimodal_text_captions_BO', 'model_VisualBERT']\n",
      "[[False  True  True  True False]\n",
      " [ True False False False  True]\n",
      " [ True False False  True  True]\n",
      " [ True False  True False  True]\n",
      " [False  True  True  True False]]\n"
     ]
    }
   ],
   "source": [
    "# ___________________________________________t-test on multimodal bias metric (MBE)__________________________________\n",
    "\n",
    "matrix = np.zeros((len(models_bias), len(models_bias)))\n",
    "a = 0\n",
    "for model1 in models_bias:\n",
    "    b = 0\n",
    "    for model2 in models_bias:\n",
    "        # Python paired sample t-test\n",
    "        test = ttest_rel(list(dict[model1].values()), list(dict[model2].values()))\n",
    "        matrix[a, b] = test.pvalue\n",
    "        b = b + 1\n",
    "    a = a + 1\n",
    "\n",
    "alpha = 0.05\n",
    "print('t-test on multimodal bias metric, alpha: {alpha}'.format(alpha=alpha))\n",
    "print(list(dict.keys()))\n",
    "print(matrix < alpha)\n",
    "\n",
    "alpha = 0.1\n",
    "print('t-test on multimodal bias metric, alpha: {alpha}'.format(alpha=alpha))\n",
    "print(list(dict.keys()))\n",
    "print(matrix < alpha)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 108,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([[       nan, 0.08749892, 0.03387146, 0.00890545, 0.20361391],\n",
       "       [0.08749892,        nan, 0.8600624 , 0.21657068, 0.02562294],\n",
       "       [0.03387146, 0.8600624 ,        nan, 0.04776851, 0.01668474],\n",
       "       [0.00890545, 0.21657068, 0.04776851,        nan, 0.00717605],\n",
       "       [0.20361391, 0.02562294, 0.01668474, 0.00717605,        nan]])"
      ]
     },
     "execution_count": 108,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "matrix"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 109,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "t-test on multimodal bias metric, alpha: 0.05\n",
      "['multimodal_text_tags', 'multimodal_text_tags_BO', 'multimodal_text_captions', 'multimodal_text_captions_BO', 'model_VisualBERT']\n",
      "[[False  True  True  True  True]\n",
      " [ True False  True  True  True]\n",
      " [ True  True False  True  True]\n",
      " [ True  True  True False False]\n",
      " [ True  True  True False False]]\n",
      "t-test on multimodal bias metric, alpha: 0.1\n",
      "['multimodal_text_tags', 'multimodal_text_tags_BO', 'multimodal_text_captions', 'multimodal_text_captions_BO', 'model_VisualBERT']\n",
      "[[False  True  True  True  True]\n",
      " [ True False  True  True  True]\n",
      " [ True  True False  True  True]\n",
      " [ True  True  True False  True]\n",
      " [ True  True  True  True False]]\n"
     ]
    }
   ],
   "source": [
    "# ___________________________________________t-test on multimodal aucs__________________________________\n",
    "\n",
    "matrix = np.zeros((len(models_bias), len(models_bias)))\n",
    "a = 0\n",
    "for model1 in list(aucs.keys()):\n",
    "    b = 0\n",
    "    for model2 in list(aucs.keys()):\n",
    "        # Python paired sample t-test\n",
    "        test = ttest_rel(list(aucs[model1].values()), list(aucs[model2].values()))\n",
    "        matrix[a, b] = test.pvalue\n",
    "        b = b + 1\n",
    "    a = a + 1\n",
    "\n",
    "alpha = 0.05\n",
    "print('t-test on multimodal bias metric, alpha: {alpha}'.format(alpha=alpha))\n",
    "print(list(aucs.keys()))\n",
    "print(matrix < alpha)\n",
    "\n",
    "alpha = 0.1\n",
    "print('t-test on multimodal bias metric, alpha: {alpha}'.format(alpha=alpha))\n",
    "print(list(aucs.keys()))\n",
    "print(matrix < alpha)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "fig, ax = plt.subplots()\n",
    "ax.boxplot([aucs_MTT_bias, aucs_MTT_BO_bias, aucs_MTC_bias, aucs_MTC_BO_bias, aucs_vbert_bias ], meanline=True, showmeans=True)\n",
    "plt.xticks([1, 2, 3, 4, 5], [\"MTT\", \"BO_MTT\", \"MTC\", \"BO_MTC\", \"VisualBERT\"], rotation=30)\n",
    "plt.show()"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### T-test on unimodal-multimodal"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 88,
   "metadata": {},
   "outputs": [],
   "source": [
    "MODELFAMILIES =  [MODELNAMES_TEXT] + [MODELNAMES_TAGS] + [MODELNAMES_CAPTIONS] + [MODELNAMES_BERT] + [MODELNAMES_MTT]+ [MODELNAMES_MTC] + [MODELNAMES_VISUAL_BERT] \n",
    "\n",
    "models = []\n",
    "for x in MODELFAMILIES:\n",
    "    models.append(model_performances.model_family_name(x).split('_v')[0])\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 89,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['unimodal_text',\n",
       " 'unimodal_tags',\n",
       " 'unimodal_captions',\n",
       " 'model_BERT',\n",
       " 'multimodal_text_tags',\n",
       " 'multimodal_text_captions',\n",
       " 'model_VisualBERT']"
      ]
     },
     "execution_count": 89,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "models"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 90,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "t-test on multimodal bias metric, alpha: 0.05\n",
      "['unimodal_text', 'unimodal_tags', 'unimodal_captions', 'model_BERT', 'multimodal_text_tags', 'multimodal_text_captions', 'model_VisualBERT']\n",
      "[[False  True  True  True  True  True  True]\n",
      " [ True False  True  True  True  True  True]\n",
      " [ True  True False  True  True  True  True]\n",
      " [ True  True  True False False  True  True]\n",
      " [ True  True  True False False  True  True]\n",
      " [ True  True  True  True  True False  True]\n",
      " [ True  True  True  True  True  True False]]\n",
      "t-test on multimodal bias metric, alpha: 0.1\n",
      "['unimodal_text', 'unimodal_tags', 'unimodal_captions', 'model_BERT', 'multimodal_text_tags', 'multimodal_text_captions', 'model_VisualBERT']\n",
      "[[False  True  True  True  True  True  True]\n",
      " [ True False  True  True  True  True  True]\n",
      " [ True  True False  True  True  True  True]\n",
      " [ True  True  True False  True  True  True]\n",
      " [ True  True  True  True False  True  True]\n",
      " [ True  True  True  True  True False  True]\n",
      " [ True  True  True  True  True  True False]]\n"
     ]
    }
   ],
   "source": [
    "# ___________________________________________t-test on aucs metric TRAIN__________________________________\n",
    "\n",
    "matrix = np.zeros((len(models), len(models)))\n",
    "a = 0\n",
    "for model1 in models:\n",
    "    b = 0\n",
    "    for model2 in models:\n",
    "        # Python paired sample t-test\n",
    "        _, _, aucs1 = performances_report_10Fold(train_df, model1, 0.5)\n",
    "        _, _, aucs2 = performances_report_10Fold(train_df, model2, 0.5)\n",
    "\n",
    "        test = ttest_rel(aucs1, aucs2)\n",
    "        matrix[a, b] = test.pvalue\n",
    "        b = b + 1\n",
    "    a = a + 1\n",
    "\n",
    "alpha = 0.05\n",
    "print('t-test on multimodal bias metric, alpha: {alpha}'.format(alpha=alpha))\n",
    "print(models)\n",
    "print(matrix < alpha)\n",
    "\n",
    "alpha = 0.1\n",
    "print('t-test on multimodal bias metric, alpha: {alpha}'.format(alpha=alpha))\n",
    "print(models)\n",
    "print(matrix < alpha)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 76,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "t-test on multimodal bias metric, alpha: 0.05\n",
      "['multimodal_text_captions_BO', 'multimodal_text_captions', 'multimodal_text_tags', 'multimodal_text_tags_BO', 'model_VisualBERT', 'model_BERT']\n",
      "[[False  True  True  True False  True]\n",
      " [ True False  True  True  True  True]\n",
      " [ True  True False  True  True  True]\n",
      " [ True  True  True False  True  True]\n",
      " [False  True  True  True False  True]\n",
      " [ True  True  True  True  True False]]\n",
      "t-test on multimodal bias metric, alpha: 0.1\n",
      "['multimodal_text_captions_BO', 'multimodal_text_captions', 'multimodal_text_tags', 'multimodal_text_tags_BO', 'model_VisualBERT', 'model_BERT']\n",
      "[[False  True  True  True  True  True]\n",
      " [ True False  True  True  True  True]\n",
      " [ True  True False  True  True  True]\n",
      " [ True  True  True False  True  True]\n",
      " [ True  True  True  True False  True]\n",
      " [ True  True  True  True  True False]]\n"
     ]
    }
   ],
   "source": [
    "# ___________________________________________t-test on aucs metric TEST__________________________________\n",
    "\n",
    "matrix = np.zeros((len(models), len(models)))\n",
    "a = 0\n",
    "for model1 in models:\n",
    "    b = 0\n",
    "    for model2 in models:\n",
    "        # Python paired sample t-test\n",
    "        _, _, aucs1 = performances_report_10Fold(test_df, model1, 0.5)\n",
    "        _, _, aucs2 = performances_report_10Fold(test_df, model2, 0.5)\n",
    "\n",
    "        test = ttest_rel(aucs1, aucs2)\n",
    "        matrix[a, b] = test.pvalue\n",
    "        b = b + 1\n",
    "    a = a + 1\n",
    "\n",
    "alpha = 0.05\n",
    "print('t-test on multimodal bias metric, alpha: {alpha}'.format(alpha=alpha))\n",
    "print(models)\n",
    "print(matrix < alpha)\n",
    "\n",
    "alpha = 0.1\n",
    "print('t-test on multimodal bias metric, alpha: {alpha}'.format(alpha=alpha))\n",
    "print(models)\n",
    "print(matrix < alpha)"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3.8.13 ('ProjEnv')",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.13 (default, Mar 28 2022, 06:59:08) [MSC v.1916 64 bit (AMD64)]"
  },
  "orig_nbformat": 4,
  "vscode": {
   "interpreter": {
    "hash": "036d5e79b2f89acf0b2080b6b489e4897d66fffe68690462bc44e5a56924deac"
   }
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}