notebooks/Image Crop Analysis.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "```\n",
    "Copyright 2021 Twitter, Inc.\n",
    "SPDX-License-Identifier: Apache-2.0\n",
    "```\n",
    "\n",
    "## Image Crop Analysis\n",
    "\n",
    "[![Open In Colab](https://colab.research.google.com/assets/colab-badge.svg)](https://colab.research.google.com/github/twitter-research/image-crop-analysis/blob/master/notebooks/Image%20Crop%20Analysis.ipynb)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "import logging\n",
    "import shlex\n",
    "import subprocess\n",
    "import sys\n",
    "from collections import namedtuple\n",
    "from pathlib import Path\n",
    "\n",
    "import matplotlib.image as mpimg\n",
    "import matplotlib.pyplot as plt\n",
    "import numpy as np\n",
    "from matplotlib.collections import PatchCollection\n",
    "from matplotlib.patches import Rectangle\n",
    "\n",
    "logging.basicConfig(level=logging.ERROR)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "import platform\n",
    "\n",
    "BIN_MAPS = {\"Darwin\": \"mac\", \"Linux\": \"linux\"}\n",
    "\n",
    "HOME_DIR = Path(\"../\").expanduser()\n",
    "\n",
    "try:\n",
    "    import google.colab\n",
    "    ! pip install pandas scikit-learn scikit-image statsmodels requests dash\n",
    "    ! [[ -d image-crop-analysis ]] || git clone https://github.com/twitter-research/image-crop-analysis.git\n",
    "    HOME_DIR = Path(\"./image-crop-analysis\").expanduser()\n",
    "    IN_COLAB = True\n",
    "except:\n",
    "    IN_COLAB = False\n",
    "\n",
    "sys.path.append(str(HOME_DIR / \"src\"))\n",
    "bin_dir = HOME_DIR / Path(\"./bin\")\n",
    "bin_path = bin_dir / BIN_MAPS[platform.system()] / \"candidate_crops\"\n",
    "model_path = bin_dir / \"fastgaze.vxm\"\n",
    "data_dir = HOME_DIR / Path(\"./data/\")\n",
    "data_dir.exists()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "img_path = next(data_dir.glob(\"./*.jpeg\"))\n",
    "img_path"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "img = mpimg.imread(img_path)\n",
    "plt.imshow(img)\n",
    "plt.gca().add_patch(\n",
    "    Rectangle((0, 0), 200, 112, linewidth=1, edgecolor=\"r\", facecolor=\"none\")\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "str(img_path.absolute())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "cmd = f\"{str(bin_path)} {str(model_path)} '{img_path.absolute()}' show_all_points\"\n",
    "cmd"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "output = subprocess.check_output(cmd, shell=True)  # Success!\n",
    "print(output.splitlines())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "! {str(bin_path)} {str(model_path)} '{img_path.absolute()}' show_all_points | head"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "from crop_api import ImageSaliencyModel, is_symmetric, parse_output, reservoir_sampling"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "parse_output(output).keys()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "model = ImageSaliencyModel(crop_binary_path=bin_path, crop_model_path=model_path)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "plt.matplotlib.__version__"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "list(data_dir.glob(\"./*.jpeg\"))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "for img_path in data_dir.glob(\"*.jpeg\"):\n",
    "    print(img_path)\n",
    "    model.plot_img_crops(img_path)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "for img_path in data_dir.glob(\"*.jpeg\"):\n",
    "    print(img_path)\n",
    "    model.plot_img_crops(img_path)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "for img_path in reservoir_sampling(data_dir.glob(\"./*.jpeg\"), K=5):\n",
    "    model.plot_img_crops(img_path)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "model.plot_img_crops(data_dir / Path(\"./dummy.jpeg\"), topK=2)\n",
    "plt.savefig(\"dummy.jpg\", bbox_inches=\"tight\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Sample crops based on saliency scores\n",
    "\n",
    "\n",
    "* First, we show the top 3 crops based sorted saliency scores (highest first)\n",
    "* Next, we show the top 3 crops sampled based on saliency scores converted into probs using the following formula:\n",
    "\n",
    "$$\n",
    "\\begin{equation}\n",
    "p_i = \\frac{exp(s_i)}{Z}\\\\\n",
    "Z = \\sum_{j=0}^{j=N} exp(s_j)\n",
    "\\end{equation}\n",
    "$$"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "model.plot_img_crops(data_dir / Path(\"./dummy.jpeg\"), topK=3)\n",
    "plt.savefig(\"dummy_top3.jpeg\", bbox_inches=\"tight\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "model.plot_img_crops(data_dir / Path(\"./dummy.jpeg\"), topK=3, sample=True)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Crop an image generated using combination of images"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "from PIL import Image\n",
    "\n",
    "from image_manipulation import join_images"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "images = [Image.open(x) for x in data_dir.glob(\"./*.jpeg\")]\n",
    "img = join_images(images, col_wrap=2, img_size=(128, -1))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "img"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "images = [\n",
    "    Image.open(data_dir / Path(\"./dummy.jpeg\")),\n",
    "    Image.open(data_dir / Path(\"./dummy.jpeg\")),\n",
    "]\n",
    "img = join_images(images, col_wrap=2, img_size=(128, 128), padding=0)\n",
    "model.plot_img_crops_using_img(img, topK=5)\n",
    "plt.savefig(\"dummy_dummy.jpeg\", bbox_inches=\"tight\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "images = [\n",
    "    Image.open(data_dir / Path(\"./dummy.jpeg\")),\n",
    "    Image.open(data_dir / Path(\"./dummy.jpeg\")),\n",
    "]\n",
    "img = join_images(images, col_wrap=1, img_size=(128, 128), padding=100)\n",
    "model.plot_img_crops_using_img(img, topK=1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "img"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "img.save(\"dummy_dummy_stiched.jpeg\", \"JPEG\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "img_path = Path(\"dummy_dummy_stiched.jpeg\")\n",
    "model.plot_img_crops(img_path, topK=1)\n",
    "plt.savefig(\"dummy_dummy.jpeg\", bbox_inches=\"tight\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "model.plot_img_crops(data_dir / Path(\"./dummy.jpeg\"), topK=2, aspectRatios=[0.56])\n",
    "plt.savefig(\"dummy.jpeg\", bbox_inches=\"tight\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "img_path = Path(\"dummy_dummy_stiched.jpeg\")\n",
    "model.plot_img_crops(img_path, topK=1, add_saliency_line=False, col_wrap=3)\n",
    "plt.savefig(\"dummy_dummy.jpeg\", bbox_inches=\"tight\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python [conda env:image-crop-analysis]",
   "language": "python",
   "name": "conda-env-image-crop-analysis-py"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.9.4"
  },
  "mldash_entity": {
   "created_at_millis": 1620095915717,
   "hash": "2fcd96a2e80eeefd7a908fa89ef475b56bffe759",
   "inferred_pdp_safe": false,
   "is_vfs_dir": false,
   "marked_pdp_safe": false,
   "owner": "smishra",
   "shared_to_everyone": false,
   "shared_to_ldap_groups": [],
   "shared_to_ldap_users": [],
   "size": 6942268,
   "tags": [],
   "uuid": "1389409090200166402",
   "vfs_path": "/user/smishra/notebooks/ImageCrop/Image Crop Analysis.ipynb"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}