notebooks/Demographic Bias Analysis.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "```\n",
    "Copyright 2021 Twitter, Inc.\n",
    "SPDX-License-Identifier: Apache-2.0\n",
    "```\n",
    "\n",
    "## Demographic Bias Analysis\n",
    "\n",
    "This notebook is to produce results from image cropping. We conduct the analysis on the Wikipedia data.\n",
    "\n",
    "Results are recorded in .txt files."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "import math\n",
    "import os\n",
    "import random\n",
    "import shlex\n",
    "import subprocess\n",
    "import sys\n",
    "import time\n",
    "from pathlib import Path\n",
    "\n",
    "import matplotlib.image as mpimg\n",
    "import matplotlib.pyplot as plt\n",
    "import numpy as np\n",
    "import pandas as pd\n",
    "from IPython.display import display\n",
    "from matplotlib.patches import Rectangle\n",
    "from PIL import Image\n",
    "\n",
    "## Add seed for reproducibility\n",
    "random.seed(42)\n",
    "np.random.seed(42)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "# ! rm -rf attached"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "import platform\n",
    "\n",
    "BIN_MAPS = {\"Darwin\": \"mac\", \"Linux\": \"linux\"}\n",
    "\n",
    "HOME_DIR = Path(\"../\").expanduser()\n",
    "\n",
    "try:\n",
    "    import google.colab\n",
    "    ! pip install pandas scikit-learn scikit-image statsmodels requests dash\n",
    "    ! [[ -d image-crop-analysis ]] || git clone https://github.com/twitter-research/image-crop-analysis.git\n",
    "    HOME_DIR = Path(\"./image-crop-analysis\").expanduser()\n",
    "    IN_COLAB = True\n",
    "except:\n",
    "    IN_COLAB = False\n",
    "\n",
    "sys.path.append(str(HOME_DIR / \"src\"))\n",
    "bin_dir = HOME_DIR / Path(\"./bin\")\n",
    "bin_path = bin_dir / BIN_MAPS[platform.system()] / \"candidate_crops\"\n",
    "model_path = bin_dir / \"fastgaze.vxm\"\n",
    "data_dir = HOME_DIR / Path(\"./data/\")\n",
    "data_dir.exists()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Collecting and Cleaning Data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def get_filepaths(directory):\n",
    "    file_paths = []  # List which will store all of the full filepaths.\n",
    "    file_paths = [str(p) for p in Path(directory).glob(\"*\")]\n",
    "    return file_paths"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Read and process Wiki image data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# [Instruction]: out the folder directory which contains all the images here\n",
    "IMG_DIR_PATH = data_dir / \"./images\"\n",
    "print(\"Directory exists:\", Path(IMG_DIR_PATH).exists())\n",
    "\n",
    "# get the list of all files path\n",
    "full_file_paths = get_filepaths(IMG_DIR_PATH)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "IMAGE_MAP_PATH = data_dir / \"./dataset.tsv\"\n",
    "print(\"File exists:\", Path(IMAGE_MAP_PATH).exists())\n",
    "IMAGE_DATA_PATH = data_dir / \"./dataset.json\"\n",
    "print(\"File exists:\", Path(IMAGE_DATA_PATH).exists())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "def read_wiki_img_data():\n",
    "    df = pd.read_csv(IMAGE_MAP_PATH, sep=\"\\t\")\n",
    "    df[\"path\"] = df.local_path.apply(lambda x: IMG_DIR_PATH / x)\n",
    "\n",
    "    print(\n",
    "        \"Reading finished. Cleaning .pdf mistakes:\",\n",
    "        sum(df.local_path.apply(lambda x: x[-3:]) == \"pdf\"),\n",
    "        \"entities filtered.\",\n",
    "    )\n",
    "    df = df[\n",
    "        df.local_path.apply(lambda x: x[-3:]) != \"pdf\"\n",
    "    ]  # clean out one wrong example, if exists\n",
    "    return df\n",
    "\n",
    "\n",
    "wiki_pandas = read_wiki_img_data()\n",
    "wiki_pandas"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "gender_map_dict = {\"Q6581072\": \"female\", \"Q6581097\": \"male\"}\n",
    "wiki_pandas.loc[:, \"gender\"] = wiki_pandas.sex_or_gender.map(gender_map_dict)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "ETHNIC_SIZE_THRESHOLD = 40\n",
    "ethnic_size_pandas = wiki_pandas.groupby(\"ethnic_group\").size()\n",
    "print(\n",
    "    \"top ethnic_group by size:\\n\", ethnic_size_pandas.sort_values(ascending=False)[:20]\n",
    ")\n",
    "ethnic_big_enough = ethnic_size_pandas[\n",
    "    ethnic_size_pandas.values >= ETHNIC_SIZE_THRESHOLD\n",
    "].index.values"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "ethnic_details_map_dict = {\n",
    "    \"Q127885\": \"Serbs: nation and South Slavic ethnic group formed in the Balkans\",\n",
    "    \"Q161652\": \"Japanase\",\n",
    "    \"Q190168\": \"Yoruba people: ethnic group of West Africa\",\n",
    "    \"Q42406\": \"English people: nation and ethnic group native to England\",\n",
    "    \"Q44806\": \"Ukrainians: East Slavic ethnic group native to Ukraine\",\n",
    "    \"Q49085\": \"African Americans: racial or ethnic group in the United States with African ancestry\",\n",
    "    \"Q539051\": \"Greeks: people of southeastern Europe\",\n",
    "    \"Q678551\": \"American Jews\",\n",
    "    \"Q726673\": \"Swedish-speaking population of Finland\",\n",
    "    \"Q7325\": \"Jewish\",\n",
    "    \"Q79797\": \"Armenians: ethnic group native to the Armenian Highland\",\n",
    "    \"Q179248\": \"Albanians\",\n",
    "    \"Q2325516\": \"Armenian American\",\n",
    "}\n",
    "\n",
    "wiki_pandas[\"race_details\"] = wiki_pandas[\"ethnic_group\"].map(ethnic_details_map_dict)\n",
    "\n",
    "# mapping by https://www.census.gov/topics/population/race/about.html\n",
    "ethnic_race_map_dict = {\n",
    "    \"Q127885\": \"white\",\n",
    "    \"Q161652\": \"asian\",\n",
    "    \"Q190168\": \"black\",\n",
    "    \"Q42406\": \"white\",\n",
    "    \"Q44806\": \"white\",\n",
    "    \"Q49085\": \"black\",\n",
    "    \"Q539051\": \"white\",\n",
    "    \"Q678551\": \"white\",\n",
    "    \"Q726673\": \"white\",\n",
    "    \"Q7325\": \"white\",\n",
    "    \"Q79797\": \"white\",\n",
    "    \"Q179248\": \"white\",\n",
    "    \"Q2325516\": \"white\",\n",
    "}\n",
    "\n",
    "wiki_pandas[\"race\"] = wiki_pandas[\"ethnic_group\"].map(ethnic_race_map_dict)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# cut if the group is too small\n",
    "GROUP_THRESHOLD = 100\n",
    "\n",
    "def get_grouped_images_dict_wiki(\n",
    "    wiki_pandas, by_columns=[\"race\"], group_threshold=GROUP_THRESHOLD\n",
    "):\n",
    "    \"\"\"\n",
    "    race_threshold: if the size of the group is less than the threshold, then ignore the group to be in the dictionary\n",
    "    \"\"\"\n",
    "    grouped_images_dict = {}\n",
    "    grouped_images_race = wiki_pandas.groupby(by_columns)\n",
    "\n",
    "    for name, group in grouped_images_race:\n",
    "        print(name)\n",
    "        print(\"Size of the group:\", len(group))\n",
    "        if group_threshold is not None:\n",
    "            if len(group) < group_threshold:\n",
    "                print(\"\\tSize is smaller than\", group_threshold, \". Skip this group.\")\n",
    "                continue\n",
    "        grouped_images_dict[name] = group\n",
    "\n",
    "    return grouped_images_dict"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# choose your option here\n",
    "grouped_images_dict = get_grouped_images_dict_wiki(\n",
    "    wiki_pandas, by_columns=[\"race\", \"gender\"]\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "grouped_images_dict.keys()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Setting up Tools to use"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def img_resize_fixed_aspect(img, fixed_width=None, fixed_height=256):\n",
    "    w, h = img.size\n",
    "    if fixed_height:\n",
    "        return img.resize((int(w * fixed_height / h), fixed_height))\n",
    "    elif fixed_width:\n",
    "        raise Exception(\"Not implemented.\")\n",
    "\n",
    "\n",
    "def attach_img(\n",
    "    images,\n",
    "    fixed_height=None,\n",
    "    display_img_full_size=False,\n",
    "    display_figsize=None,\n",
    "    pixel_size_upperbound=None,\n",
    "):\n",
    "    \"\"\"\n",
    "    Given a list of images (of PIL Image class or image paths), attach them horizontally and return a new image.\n",
    "    \n",
    "    images: list of images or a list of paths to images\n",
    "    fixed_height: all images are resized (while fixing the aspect ratio) to the specified height.\n",
    "    If you don't want to resize, set this to None\n",
    "    display_img_full_size: If true, will display the image at full resolution\n",
    "    display_figsize: if given, as (x,y), will use matplotlib to display images with figsize=(x,y)\n",
    "    pixel_size_upperbound: if given, images with width or height above this number will be resized down \n",
    "    while fixing the aspect ratio until both the width and height is at most the number.\n",
    "    \"\"\"\n",
    "    if isinstance(images[0], str) or isinstance(images[0], Path):\n",
    "        images = [Image.open(str(img_path)) for img_path in images]\n",
    "    if fixed_height is not None:\n",
    "        images = [\n",
    "            img_resize_fixed_aspect(img, fixed_height=fixed_height) for img in images\n",
    "        ]\n",
    "    if pixel_size_upperbound is not None:\n",
    "        for i in range(len(images)):\n",
    "            w, h = images[i].size\n",
    "            if w > pixel_size_upperbound:\n",
    "                images[i] = images[i].resize(\n",
    "                    (pixel_size_upperbound, int(pixel_size_upperbound / w * h))\n",
    "                )\n",
    "            w, h = images[i].size\n",
    "            if h > pixel_size_upperbound:\n",
    "                images[i] = images[i].resize(\n",
    "                    (int(pixel_size_upperbound / h * w), pixel_size_upperbound)\n",
    "                )\n",
    "    widths, heights = zip(*(i.size for i in images))\n",
    "    total_width = sum(widths)\n",
    "    max_height = max(heights)\n",
    "    new_im = Image.new(\"RGB\", (total_width, max_height))\n",
    "    x_offset = 0\n",
    "    for im in images:\n",
    "        new_im.paste(im, (x_offset, 0))\n",
    "        x_offset += im.size[0]\n",
    "    if display_img_full_size:\n",
    "        display(new_im)  # This shows image at full resolution\n",
    "    if display_figsize:\n",
    "        fig = plt.figure(figsize=display_figsize)\n",
    "        plt.imshow(\n",
    "            np.asarray(new_im)\n",
    "        )  # This shows scaled down version in matplotlib plot using figsize\n",
    "    return (new_im, widths, heights)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Run below cell if want to explore the images"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "for ethnic in ethnic_big_enough:\n",
    "    print(\n",
    "        ethnic, ethnic_race_map_dict[ethnic], \" size: \", ethnic_size_pandas.loc[ethnic]\n",
    "    )\n",
    "    display(\n",
    "        attach_img(\n",
    "            wiki_pandas[wiki_pandas.ethnic_group == ethnic]\n",
    "            .sample(n=10, replace=False)\n",
    "            .path.values,\n",
    "            fixed_height=100,\n",
    "        )[0]\n",
    "    )\n",
    "\n",
    "for gender in wiki_pandas.gender.unique():\n",
    "    print(gender)\n",
    "    display(\n",
    "        attach_img(\n",
    "            wiki_pandas[wiki_pandas.gender == gender]\n",
    "            .sample(n=10, replace=False)\n",
    "            .path.values,\n",
    "            fixed_height=100,\n",
    "        )[0]\n",
    "    )"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "(new_im, widths, heights) = attach_img(full_file_paths[0:2], fixed_height=None)\n",
    "display(new_im)\n",
    "print(\"widths:\", widths)\n",
    "print(\"heights:\", heights)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def locate_which_img(location, widths):\n",
    "    \"\"\"\n",
    "    Given a list of widths of the images that are attached (from left to right), and a location,\n",
    "    output what image contains that location.\n",
    "    E.g. widths = 100, 200, 100\n",
    "    location = 50 --> index = 0\n",
    "    location = 350 --> index = 2\n",
    "    location = 100 --> index = 1\n",
    "    location >= 400 --> index = -1\n",
    "    \"\"\"\n",
    "    index = 0\n",
    "    while location >= widths[index]:\n",
    "        location -= widths[index]\n",
    "        index += 1\n",
    "        if index >= len(widths):\n",
    "            return -1\n",
    "    return index"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def parse_output(output):\n",
    "    output = output.splitlines()\n",
    "    final_output = {\"salient_point\": [], \"crops\": [], \"all_salient_points\": []}\n",
    "    key = \"salient_point\"\n",
    "    for i, line in enumerate(output):\n",
    "        line = line.split()\n",
    "        if len(line) in {2, 4}:\n",
    "            line = [int(v) for v in line]\n",
    "            if i != 0:\n",
    "                key = \"crops\"\n",
    "        elif len(line) == 3:\n",
    "            key = \"all_salient_points\"\n",
    "            line = [float(v) for v in line]\n",
    "        else:\n",
    "            raise RuntimeError(f\"Invalid line: {line}\")\n",
    "        final_output[key].append(line)\n",
    "    return final_output"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def get_salient_info(img_path):\n",
    "    \"\"\"\n",
    "    Given a path (as an instance of Path or a string or an image), output the salient information.\n",
    "    \n",
    "    The output is a dictionary with:\n",
    "    'salient_point': a list of pixels, usually only one. E.g. [[507, 328]]\n",
    "    'crops': a list of crops (for each aspect ratio) in the format crop_x crop_y crop_w crop_h\n",
    "    'all_salient_points': the pixel location at the original image with its salient score. It is a list of\n",
    "    [x, y, salient score]\n",
    "    \"\"\"\n",
    "    if isinstance(img_path, str):\n",
    "        img_path = Path(img_path)\n",
    "    try:\n",
    "        cmd = f\"{str(bin_path)} {str(model_path)} '{img_path.absolute()}' show_all_points\"\n",
    "        output = subprocess.check_output(cmd, shell=True)  # Success!\n",
    "        return parse_output(output)\n",
    "    except:\n",
    "        print(\"Running the model to get salient point fails. Returning None.\")\n",
    "        return None"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "salient_scores = [\n",
    "    point[2] for point in get_salient_info(full_file_paths[3])[\"all_salient_points\"]\n",
    "]\n",
    "np.percentile(salient_scores, 100)\n",
    "np.mean(salient_scores)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Higher-level Tools"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "SAVE_DIR_ATTACHED = data_dir / \"./attached\"\n",
    "if not os.path.isdir(SAVE_DIR_ATTACHED):\n",
    "    os.mkdir(SAVE_DIR_ATTACHED)\n",
    "os.path.isdir(SAVE_DIR_ATTACHED)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "RESULT_FILE_PATH = data_dir / \"./results.txt\"\n",
    "if not os.path.isfile(RESULT_FILE_PATH):\n",
    "    # Creates a new file\n",
    "    with open(RESULT_FILE_PATH, \"a\") as fp:\n",
    "        pass\n",
    "os.path.isfile(RESULT_FILE_PATH)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Pairwise comparison"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "SAVE_DIR_ATTACHED = data_dir / \"./attached\"\n",
    "if not os.path.isdir(SAVE_DIR_ATTACHED):\n",
    "    os.mkdir(SAVE_DIR_ATTACHED)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def compute_group_comparison(\n",
    "    df_list,\n",
    "    save_dir=SAVE_DIR_ATTACHED,\n",
    "    num_iterations=200,\n",
    "    log_every=100,\n",
    "    warn_outside_ori_image=True,\n",
    "    fixed_height=None,\n",
    "    pixel_size_upperbound=None,\n",
    "):\n",
    "    \"\"\"\n",
    "    Given a list of dataframe of groups g_1, g_2, ..., g_k, each of which contains the column 'path' to the image file:\n",
    "    - sample uniformly one image from each group\n",
    "    - attach image into one image\n",
    "    - apply saliency mapping\n",
    "    - find the number of times max salient points land in each picture\n",
    "    \"\"\"\n",
    "    num_max_salient_per_group = np.zeros(len(df_list))\n",
    "    for i in range(num_iterations):\n",
    "        salient_info = None\n",
    "        while salient_info is None:  # if cmd in get_salient_info failed, resample\n",
    "            # Sample images as paths\n",
    "            sample_img_paths = [(df.sample())[\"path\"].item() for df in df_list]\n",
    "            attached_img, widths, heights = attach_img(\n",
    "                sample_img_paths,\n",
    "                fixed_height=fixed_height,\n",
    "                pixel_size_upperbound=pixel_size_upperbound,\n",
    "            )\n",
    "            attached_path = str(save_dir) + \"/\" + str(i) + \".jpg\"\n",
    "            attached_img.save(attached_path)\n",
    "            # try to get salient information. Will get None if fails.\n",
    "            salient_info = get_salient_info(attached_path)\n",
    "\n",
    "        # get salient point and where it lands\n",
    "        all_salient_points = salient_info[\"salient_point\"]\n",
    "        if len(all_salient_points) > 1:\n",
    "            print(\n",
    "                \"Warning: there is more than one maximum salient point. Using the first one returned from the algorithms.\"\n",
    "            )\n",
    "        salient_point_x, salient_point_y = all_salient_points[0]\n",
    "        max_salient_pic_index = locate_which_img(salient_point_x, widths)\n",
    "        if warn_outside_ori_image and salient_point_y >= heights[max_salient_pic_index]:\n",
    "            # this means the point is outside of the original image\n",
    "            print(\n",
    "                \"Warning: salient point is located at the background from attaching images, outside of original.\"\n",
    "            )\n",
    "        num_max_salient_per_group[max_salient_pic_index] += 1\n",
    "        if (i + 1) % log_every == 0:\n",
    "            print((i + 1), \"/\", num_iterations, \"iterations of sampling has been done.\")\n",
    "    return num_max_salient_per_group"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# In general, we can give the name of two groups in grouped_images_dict, and return this statistics of how many land in each\n",
    "def compute_comparison_from_group_name(\n",
    "    grouped_images_dict,\n",
    "    group_name_list,\n",
    "    normalize=False,\n",
    "    save_dir=SAVE_DIR_ATTACHED,\n",
    "    num_iterations=200,\n",
    "    fixed_height=None,\n",
    "    pixel_size_upperbound=None,\n",
    "    save_result=RESULT_FILE_PATH,\n",
    "    log_every=100,\n",
    "    print_summary=True,\n",
    "    save_setting=\"default\",\n",
    "):\n",
    "    \"\"\"\n",
    "    Given a list of group names, which are keys of the dictionary grouped_images_dict,\n",
    "    whose values are pandas dataframe with column \"path\",\n",
    "    return the number of times salient points lands in each of the group out of num_iterations iterations of sampling.\n",
    "    \n",
    "    Args:\n",
    "        grouped_images_dict: a dictionary where keys are group names, and values are pandas dataframe which contains 'path'\n",
    "    \"\"\"\n",
    "    start_time = time.time()\n",
    "    stats = compute_group_comparison(\n",
    "        [grouped_images_dict[group_name] for group_name in group_name_list],\n",
    "        save_dir=save_dir,\n",
    "        num_iterations=num_iterations,\n",
    "        fixed_height=fixed_height,\n",
    "        log_every=log_every,\n",
    "        pixel_size_upperbound=pixel_size_upperbound,\n",
    "    )\n",
    "    if normalize:\n",
    "        stats = stats / num_iterations\n",
    "    if print_summary:\n",
    "        print(\"The groups entered are: \" + str(group_name_list))\n",
    "        print(\"The statistic is: \" + str(stats))\n",
    "        print(\"Size of the sample: \" + str(num_iterations))\n",
    "    if save_result:\n",
    "        with open(RESULT_FILE_PATH, \"a\") as fp:\n",
    "            to_write_info_dict = {\n",
    "                \"group_name_list\": group_name_list,\n",
    "                \"stats\": stats,\n",
    "                \"num_iterations\": num_iterations,\n",
    "            }\n",
    "            if save_setting:\n",
    "                to_write_info_dict[\"setting\"] = save_setting\n",
    "            fp.write(str(to_write_info_dict) + \"\\n\")\n",
    "    time_used = time.time() - start_time\n",
    "    print(\"total time used:\", time_used, \"seconds.\")\n",
    "    print(\"time used per comparison:\", time_used / num_iterations, \"seconds.\")\n",
    "    return stats"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Get a list of saliency point statistics\n",
    "\n",
    "Such as max, 95th, 90th, 50th, and 25th percentile"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def compute_saliency_stats(\n",
    "    file_path_list,\n",
    "    sample=None,\n",
    "    percentile_queries=[25, 50, 75, 90, 95, 100],\n",
    "    fixed_height=None,\n",
    "    pixel_size_upperbound=None,\n",
    "    save_dir=SAVE_DIR_ATTACHED,\n",
    "    log_every=100,\n",
    "    print_summary=True,\n",
    "    delete_img=True,\n",
    "):\n",
    "    \"\"\"\n",
    "    Given a list of paths to images, compute saliency statistics of interests of all those images.\n",
    "    \n",
    "    Args:\n",
    "        sample: None if we want to compute all. If a number is given, only do the specified number of samples (without replacement)\n",
    "\n",
    "    Returns:\n",
    "        pandas dataframe with file_path as an index, and the statistics as a column\n",
    "    \"\"\"\n",
    "    start_time = time.time()\n",
    "\n",
    "    if sample is not None:\n",
    "        file_path_list = random.sample(file_path_list, sample)\n",
    "\n",
    "    result_data = {\n",
    "        str(percent) + \"%_tile\": [] for percent in percentile_queries\n",
    "    }  # all columns\n",
    "    result_data[\"path\"] = []  # rows of the data\n",
    "    result_data[\"mean\"] = []\n",
    "\n",
    "    for i, path in enumerate(file_path_list):\n",
    "        attached_img, widths, heights = attach_img(\n",
    "            [path],\n",
    "            fixed_height=fixed_height,\n",
    "            pixel_size_upperbound=pixel_size_upperbound,\n",
    "        )\n",
    "        attached_path = str(save_dir) + \"/\" + str(i) + \".jpg\"\n",
    "        attached_img.save(attached_path)\n",
    "        # try to get salient information. Will get None if fails.\n",
    "        salient_info = get_salient_info(attached_path)\n",
    "        # After getting the info delete the file:\n",
    "        if delete_img:\n",
    "            Path(attached_path).unlink()\n",
    "\n",
    "        salient_scores = [point[2] for point in salient_info[\"all_salient_points\"]]\n",
    "\n",
    "        # put data in\n",
    "        result_data[\"path\"].append(path)\n",
    "        result_data[\"mean\"].append(np.mean(salient_scores))\n",
    "        for percent in percentile_queries:\n",
    "            result_data[str(percent) + \"%_tile\"].append(\n",
    "                np.percentile(salient_scores, percent)\n",
    "            )\n",
    "\n",
    "        if (i + 1) % log_every == 0:\n",
    "            print((i + 1), \"iterations of sampling has been done.\")\n",
    "\n",
    "    time_used = time.time() - start_time\n",
    "    print(\"total time used:\", time_used, \"seconds.\")\n",
    "    print(\"time used per comparison:\", time_used / len(file_path_list), \"seconds.\")\n",
    "\n",
    "    return pd.DataFrame.from_dict(result_data).set_index(\"path\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Analysis\n",
    "\n",
    "Change the setting and setting name below (for saving purpose), and modify `all_pairs_replicate_ori` as needed."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# change the setting here\n",
    "NUM_ITERATION = 10  # 5000\n",
    "FIXED_HEIGHT = (\n",
    "    256\n",
    ")  # if we want to scale each image to the same fixed height, put a number such as 256 here\n",
    "DIM_BOUND = None\n",
    "# if for large image, we want to scale it down so width and height are no more than a fixed number, e.g. put 1024 here.\n",
    "# if not, very few large images are rejected from sampling as the size is too large for the model to accept\n",
    "SETTING_NAME = \"wiki_fixed_height_intersect\"  # for saving results and plots"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "all_race_to_compare = list(grouped_images_dict.keys())\n",
    "all_pairs_replicate_ori = [\n",
    "    (all_race_to_compare[i], all_race_to_compare[j])\n",
    "    for i in range(len(all_race_to_compare))\n",
    "    for j in range(i + 1, len(all_race_to_compare))\n",
    "]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# another way, if we want to flip\n",
    "all_pairs_replicate_ori"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "max_salient_compare_dict = {}\n",
    "\n",
    "for pair in all_pairs_replicate_ori:\n",
    "    max_salient_compare_dict[pair] = compute_comparison_from_group_name(\n",
    "        grouped_images_dict,\n",
    "        pair,\n",
    "        num_iterations=NUM_ITERATION,\n",
    "        save_setting=SETTING_NAME,\n",
    "        fixed_height=FIXED_HEIGHT,\n",
    "        pixel_size_upperbound=DIM_BOUND,\n",
    "    )"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def plot_pairwise_stats(\n",
    "    stat_dict,\n",
    "    figsize=None,\n",
    "    setting_name=\"unknown\",\n",
    "    num_iteration=\"unknown\",\n",
    "    confidence_interval_err=2,\n",
    "    middle_band_width=0.1,\n",
    "    x_label_angle=10,\n",
    "):\n",
    "    \"\"\"\n",
    "  Given a dictionary of pairs of group and comparison statisitcs:\n",
    "  ('group1', 'group2'): [num_group1_is_picked, num_group2_is_picked]\n",
    "  Plot the bar graph on all pairs in this format on the probability p that group1 is picked.\n",
    "  \n",
    "  The std error is assumed to be sqrt(p(1-p)/n), a confidence interval for Bernoulli inference.\n",
    "  The bar graph plot +- 2 std err, giving 95% confidence interval.\n",
    "  \n",
    "  Args:\n",
    "      confidence_interval_err: the width of the confidence interval in the plotsetting_name: the setting of this experiment. Only used for the title of the plot and name of the saved figure\n",
    "      num_iteration: the number of samples used (int or str). Only used for the title of the plot and name of the saved figure\n",
    "      x_label_angle: angle to rotate the x label. May need to increase for lengthy labels.\n",
    "      middle_band_width: add two horizontal lines above and below 0.5 symmetrically to the plot, so creating a band of given width.\n",
    "                    If None, no line is added.\n",
    "  \"\"\"\n",
    "    x_labels = [\n",
    "        \"{}-{}\".format(*pair[0]) + \"\\nhigher than\\n\" + \"{}-{}\".format(*pair[1])\n",
    "        for pair in stat_dict.keys()\n",
    "    ]\n",
    "    prob = [val[0] / (val[0] + val[1]) for val in stat_dict.values()]\n",
    "    total = [(val[0] + val[1]) for val in stat_dict.values()]\n",
    "    y_err = [\n",
    "        confidence_interval_err * math.sqrt(p * (1 - p) / n)\n",
    "        for p, n in zip(prob, total)\n",
    "    ]\n",
    "    fig, ax = plt.subplots(figsize=figsize)\n",
    "    ax.bar(x_labels, prob, yerr=y_err)\n",
    "\n",
    "    if middle_band_width is not None:\n",
    "        ax.axhline(0.5, color=\"r\", label=f\"Demographic Parity\")\n",
    "    plt.xlim(-0.5, len(x_labels) - 0.5)\n",
    "    plt.xticks(rotation=x_label_angle, fontsize=16)\n",
    "    ax.set_ylabel(\"Probability $\\pm$ 2 * error\", fontsize=20)\n",
    "    plt.ylim(0.0, 1.0)\n",
    "    plt.yticks(fontsize=16)\n",
    "    ax.yaxis.grid(True)\n",
    "    plt.legend(fontsize=16)\n",
    "    plt.title(f\"Probabilities with {num_iteration} samples\", fontsize=20)\n",
    "    plt.tight_layout()\n",
    "    plt.savefig(setting_name + \"_n=\" + str(num_iteration) + \".jpg\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "max_salient_compare_dict"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "plot_pairwise_stats(\n",
    "    max_salient_compare_dict,\n",
    "    setting_name=SETTING_NAME,\n",
    "    num_iteration=NUM_ITERATION,\n",
    "    figsize=(12, 4),\n",
    "    x_label_angle=0,\n",
    ")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Pairing across all groups at once"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# change the setting here\n",
    "NUM_ITERATION = 10  # 5000\n",
    "FIXED_HEIGHT = (\n",
    "    None\n",
    ")  # if we want to scale each image to the same fixed height, put a number such as 256 here\n",
    "DIM_BOUND = None\n",
    "# if for large image, we want to scale it down so width and height are no more than a fixed number, e.g. put 1024 here.\n",
    "# if not, very few large images are rejected from sampling as the size is too large for the model to accept\n",
    "SETTING_NAME = \"wiki_no_scaling_intersect_together\"  # for saving results and plots"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "max_salient_all_groups_stats = compute_comparison_from_group_name(\n",
    "    grouped_images_dict,\n",
    "    grouped_images_dict.keys(),\n",
    "    num_iterations=NUM_ITERATION,\n",
    "    save_setting=SETTING_NAME,\n",
    "    fixed_height=FIXED_HEIGHT,\n",
    "    pixel_size_upperbound=DIM_BOUND,\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "max_salient_all_groups_dict = {\n",
    "    group: stat\n",
    "    for group, stat in zip(grouped_images_dict.keys(), max_salient_all_groups_stats)\n",
    "}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def plot_dict_values(\n",
    "    stat_dict,\n",
    "    figsize=None,\n",
    "    setting_name=\"unknown\",\n",
    "    num_iteration=\"unknown\",\n",
    "    confidence_interval_err=2,\n",
    "    middle_band_width=0.1,\n",
    "    x_label_angle=10,\n",
    "):\n",
    "    x_labels = [\"{}-{}\".format(*group_name) for group_name in stat_dict.keys()]\n",
    "    print(x_labels)\n",
    "    total = sum(stat_dict.values())\n",
    "    print(type(total), type(list(stat_dict.values())))\n",
    "    prob = list(stat_dict.values()) / total\n",
    "    y_err = [confidence_interval_err * math.sqrt(p * (1 - p) / total) for p in prob]\n",
    "\n",
    "    fig, ax = plt.subplots(figsize=figsize)\n",
    "    ax.bar(x_labels, prob, yerr=y_err)\n",
    "\n",
    "    ax.plot(\n",
    "        [-0.5, len(x_labels) - 0.5],\n",
    "        np.full(2, 1 / len(x_labels)),\n",
    "        \"r\",\n",
    "        label=f\"average\",\n",
    "    )\n",
    "\n",
    "    plt.xticks(rotation=x_label_angle, fontsize=16)\n",
    "    ax.set_ylabel(\"Probability $\\pm$ 2 * error\", fontsize=20)\n",
    "    plt.ylim(0.0, 1.0)\n",
    "    ax.yaxis.grid(True)\n",
    "    plt.yticks(fontsize=16)\n",
    "    plt.legend(fontsize=16)\n",
    "    plt.title(f\"Probabilities with {num_iteration} samples\", fontsize=20)\n",
    "    plt.tight_layout()\n",
    "    plt.savefig(setting_name + \"_n=\" + str(num_iteration) + \".jpg\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "max_salient_all_groups_stats"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "max_salient_all_groups_dict"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "plot_dict_values(\n",
    "    max_salient_all_groups_dict,\n",
    "    setting_name=SETTING_NAME,\n",
    "    num_iteration=NUM_ITERATION,\n",
    "    figsize=(12, 4),\n",
    "    x_label_angle=0,\n",
    ")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Stats of saliency scores"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# change the setting here\n",
    "NUM_ITERATION = 10  # None # None if want to do whole thing\n",
    "FIXED_HEIGHT = (\n",
    "    None\n",
    ")  # if we want to scale each image to the same fixed height, put a number such as 256 here\n",
    "DIM_BOUND = None\n",
    "# if for large image, we want to scale it down so width and height are no more than a fixed number, e.g. put 1024 here.\n",
    "# if not, very few large images are rejected from sampling as the size is too large for the model to accept\n",
    "SETTING_NAME = \"wiki_no_scaling_intersect_stat\"  # for saving results and plots"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "all_stats_dict = {}\n",
    "for key in grouped_images_dict.keys():\n",
    "    print(\"Computing stats for:\", key)\n",
    "    file_path_list = grouped_images_dict[key].path.values.tolist()\n",
    "    all_stats_dict[key] = compute_saliency_stats(\n",
    "        file_path_list,\n",
    "        sample=NUM_ITERATION,\n",
    "        percentile_queries=[50, 95, 100],\n",
    "        fixed_height=None,\n",
    "        pixel_size_upperbound=None,\n",
    "        save_dir=SAVE_DIR_ATTACHED,\n",
    "        log_every=100,\n",
    "        print_summary=True,\n",
    "    )"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "group_to_plot = [(\"white\", \"female\"), (\"white\", \"male\")]  # all_stats_dict.keys()\n",
    "for col_name in next(\n",
    "    iter(all_stats_dict.values())\n",
    ").columns:  # columns from any pandas dataframe\n",
    "    for key in group_to_plot:\n",
    "        pandas_data = all_stats_dict[key]\n",
    "        plt.hist(\n",
    "            pandas_data[col_name],\n",
    "            alpha=0.35,\n",
    "            density=True,\n",
    "            label=str(key) + \" on \" + str(col_name),\n",
    "            bins=50,\n",
    "        )\n",
    "    plt.ylabel(\"frequency\")\n",
    "    plt.title(\"Hist of \" + str(group_to_plot) + \" on saliency \" + str(col_name))\n",
    "    plt.legend()\n",
    "    plt.tight_layout()\n",
    "    plt.show()\n",
    "    plt.savefig(SETTING_NAME + \"_n=\" + str(NUM_ITERATION) + \".jpg\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def prob_a_less_than_b(a, b):\n",
    "    \"\"\"\n",
    "    Given two lists a,b, calculate the probability that\n",
    "    random samples x,y from a,b will satisfies a < b\n",
    "    \n",
    "    Clculating this exactly for lists of length n, m takes runtime O(n log n + m log m) (for sorting),\n",
    "    then O(n + m) in addition (without sorting)\n",
    "    \"\"\"\n",
    "    a = sorted(a)\n",
    "    b = sorted(b)\n",
    "\n",
    "    prob = 0\n",
    "    j = 0  # index of b that keeps moving till x > y\n",
    "    for i, x in enumerate(a):\n",
    "        while (j < len(b)) and (x >= b[j]):\n",
    "            j += 1\n",
    "        prob += (len(b) - j) / (len(a) * len(b))\n",
    "\n",
    "    return prob"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "group_list = list(all_stats_dict.keys())\n",
    "for i in range(len(group_list)):\n",
    "    for j in range(i + 1, len(group_list)):\n",
    "        print(\n",
    "            \"Probability that group\",\n",
    "            group_list[i],\n",
    "            \">=\",\n",
    "            group_list[j],\n",
    "            \"is\",\n",
    "            1\n",
    "            - prob_a_less_than_b(\n",
    "                all_stats_dict[group_list[i]][\"100%_tile\"].values,\n",
    "                all_stats_dict[group_list[j]][\"100%_tile\"].values,\n",
    "            ),\n",
    "        )"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "group_list = list(all_stats_dict.keys())\n",
    "for i in range(len(group_list)):\n",
    "    for j in range(i + 1, len(group_list)):\n",
    "        print(\n",
    "            \"Probability that group\",\n",
    "            group_list[i],\n",
    "            \">=\",\n",
    "            group_list[j],\n",
    "            \"is\",\n",
    "            1\n",
    "            - prob_a_less_than_b(\n",
    "                all_stats_dict[group_list[i]][\"95%_tile\"].values,\n",
    "                all_stats_dict[group_list[j]][\"95%_tile\"].values,\n",
    "            ),\n",
    "        )"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "for (g1, g2), df_t in all_stats_dict.items():\n",
    "    out_path = f\"./all_stats_dict_{g1}_{g2}.tsv\"\n",
    "    print(out_path)\n",
    "    df_t.to_csv(out_path, sep=\"\\t\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "print(f\"Setting = {SETTING_NAME}\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def plot_group_saliency_histogram(\n",
    "    group_to_plot, col_names=None, col_name_title_map=None, file_id=None, **hist_kwargs\n",
    "):\n",
    "    if col_names is None:\n",
    "        col_names = [\"50%_tile\", \"95%_tile\", \"100%_tile\", \"mean\"]\n",
    "    if col_name_title_map is None:\n",
    "        col_name_title_map = {\"50%_tile\": \"median\", \"100%_tile\": \"max\"}\n",
    "    for col_name in col_names:  # columns from any pandas dataframe\n",
    "        plt_title = col_name_title_map.get(col_name, col_name)\n",
    "        fig = plt.figure(figsize=(5, 2))\n",
    "        for key in group_to_plot:\n",
    "            g1, g2 = key\n",
    "            out_path = f\"./all_stats_dict_{g1}_{g2}.tsv\"\n",
    "            pandas_data = pd.read_csv(out_path, sep=\"\\t\", index_col=0)\n",
    "            plt.hist(pandas_data[col_name], label=f\"{g1}-{g2}\", **hist_kwargs)\n",
    "        plt.ylabel(\"frequency\")\n",
    "        plt.title(f\"Distribution of {plt_title} saliency\")\n",
    "        plt.legend(loc=\"upper left\")\n",
    "        plt.tight_layout()\n",
    "        plt.savefig(f\"saliency_dist_gender_dif_on_{file_id}_{plt_title}.png\")\n",
    "        plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "group_to_plot = [(\"black\", \"female\"), (\"black\", \"male\")]  # all_stats_dict.keys()\n",
    "plot_group_saliency_histogram(\n",
    "    group_to_plot,\n",
    "    file_id=\"black\",\n",
    "    bins=50,\n",
    "    alpha=0.5,\n",
    "    density=True,\n",
    "    lw=5,\n",
    "    cumulative=True,\n",
    "    histtype=\"step\",\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "group_to_plot = [(\"white\", \"female\"), (\"white\", \"male\")]  # all_stats_dict.keys()\n",
    "plot_group_saliency_histogram(\n",
    "    group_to_plot,\n",
    "    file_id=\"white\",\n",
    "    bins=50,\n",
    "    alpha=0.5,\n",
    "    density=True,\n",
    "    lw=5,\n",
    "    cumulative=True,\n",
    "    histtype=\"step\",\n",
    ")"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python [conda env:image-crop]",
   "language": "python",
   "name": "conda-env-image-crop-py"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.9"
  },
  "mldash_entity": {
   "created_at_millis": 1620093916544,
   "hash": "1ff3a46963607b08a592e3062b9a0249f75a94ef",
   "inferred_pdp_safe": false,
   "is_vfs_dir": false,
   "marked_pdp_safe": false,
   "owner": "smishra",
   "shared_to_everyone": false,
   "shared_to_ldap_groups": [],
   "shared_to_ldap_users": [],
   "size": 73367,
   "tags": [],
   "uuid": "1389400705069326339",
   "vfs_path": "/user/smishra/notebooks/ImageCrop/Demographic Bias Analysis.ipynb"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}