database_prep.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Before run the .ipynb file, please follow the link and download AVA image database first.\n",
    "#### Link to download AVA: http://academictorrents.com/details/71631f83b11d3d79d8f84efe0a7e12f0ac001460\n",
    "#### Note: Need a Bittorrent client to download all the files\n",
    "#### After finishing downloading, we have an folder called \"AVA_dataset\" which contains all the 7z files of images\n",
    "#### Unzip all the files to the folder \"AVA_dataset\"\n",
    "#### Move the AVA_dataset which contains all AVA images to the folder \"database\" contained in our project"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Program Start ..."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 69,
   "metadata": {},
   "outputs": [],
   "source": [
    "import os\n",
    "import tarfile\n",
    "import shutil\n",
    "import glob\n",
    "import requests\n",
    "import zipfile\n",
    "import io\n",
    "import urllib\n",
    "import cv2\n",
    "import pandas as pd\n",
    "import sys\n",
    "from sys import stdout"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 70,
   "metadata": {},
   "outputs": [],
   "source": [
    "if sys.version_info[0] < 3:\n",
    "    raise Exception(\"Must be using Python 3\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 71,
   "metadata": {},
   "outputs": [],
   "source": [
    "from urllib.request import urlretrieve"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Download VOC ..."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 87,
   "metadata": {},
   "outputs": [],
   "source": [
    "url_voc = \"http://host.robots.ox.ac.uk/pascal/VOC/voc2012/VOCtrainval_11-May-2012.tar\"\n",
    "voc, headers = urllib.request.urlretrieve(url_voc, 'voc.tar')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 91,
   "metadata": {},
   "outputs": [],
   "source": [
    "tar = tarfile.open('voc.tar')\n",
    "tar.extractall('database/voc_emotic_ava')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 92,
   "metadata": {},
   "outputs": [],
   "source": [
    "files = \"database/voc_emotic_ava/VOCdevkit/VOC2012/JPEGImages/*\"\n",
    "for path in glob.glob(files):\n",
    "    name = path.split(\"/\")[-1]\n",
    "    newp1 = path.replace(name,\"\")+\"JPEGImages__\"+name\n",
    "    newp2 = path.replace(name, \"\")+\"VOC\"+name\n",
    "    os.rename(path, newp1)\n",
    "    shutil.copy(newp1, 'database/voc_emotic_ava/')\n",
    "    os.rename(newp1, newp2)\n",
    "    shutil.copy(newp2, 'database/voc_emotic_ava/')\n",
    "    os.remove(newp2)\n",
    "shutil.rmtree('database/voc_emotic_ava/VOCdevkit')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Download Emotic..."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 93,
   "metadata": {},
   "outputs": [],
   "source": [
    "dest = os.path.dirname(\"database/voc_emotic_ava\")\n",
    "session = requests.Session()\n",
    "response = session.get('https://docs.google.com/uc?export=download',params={'id': '0B7sjGeF4f3FYQUVlZ3ZOai1ieEU'}, stream=True)\n",
    "token = None\n",
    "for key, value in response.cookies.items():\n",
    "    if key.startswith('download_warning'):\n",
    "        token = value\n",
    "if token:\n",
    "    params = {'id': '0B7sjGeF4f3FYQUVlZ3ZOai1ieEU', 'confirm': token}\n",
    "    response = session.get('https://docs.google.com/uc?export=download', params=params, stream=True)\n",
    "current_download_size = [0]\n",
    "with open('emotic.zip', 'wb') as f:\n",
    "    for chunk in response.iter_content(32768):\n",
    "        f.write(chunk)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 94,
   "metadata": {},
   "outputs": [],
   "source": [
    "zipfile.ZipFile('emotic.zip', 'r').extractall('database/voc_emotic_ava')\n",
    "folders = os.listdir(\"database/voc_emotic_ava/emotic/\")\n",
    "for folder in folders: \n",
    "    files = \"database/voc_emotic_ava/emotic/\"+folder + \"/images/*\"\n",
    "    for path in glob.glob(files):\n",
    "        name = path.split(\"/\")[-1]\n",
    "        newp = path.replace(name,\"\")+\"EMOTIC__\"+name\n",
    "        os.rename(path, newp)\n",
    "        shutil.move(newp, 'database/voc_emotic_ava/')\n",
    "shutil.rmtree('database/voc_emotic_ava/emotic')\n",
    "shutil.rmtree('database/voc_emotic_ava/__MACOSX')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Download Blur Detection Dataset ..."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 95,
   "metadata": {},
   "outputs": [],
   "source": [
    "url_blur = \"http://www.cse.cuhk.edu.hk/~leojia/projects/dblurdetect/data/BlurDatasetImage.zip\"\n",
    "blur, headers = urllib.request.urlretrieve(url_blur, 'blur.zip')\n",
    "zipfile.ZipFile('blur.zip','r').extractall('database/blur_dataset')\n",
    "for path in glob.glob('database/blur_dataset/image/*'):\n",
    "    shutil.move(path, 'database/blur_dataset/')\n",
    "shutil.rmtree('database/blur_dataset/image')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Rename and Move AVA Images"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 96,
   "metadata": {},
   "outputs": [],
   "source": [
    "if os.path.isdir('database/AVA_dataset') == True:\n",
    "    files = 'database/AVA_dataset/images/*'\n",
    "    for path in glob.glob(files):\n",
    "        name = path.split(\"/\")[-1]\n",
    "        newp = path.replace(name, \"\")+\"AVA__\"+name\n",
    "        os.rename(path, newp)\n",
    "        shutil.move(newp, 'database/voc_emotic_ava/')\n",
    "    shutil.rmtree('database/voc_emotic_ava/AVA_dataset')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Clean Image Not Been Used ..."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 97,
   "metadata": {},
   "outputs": [],
   "source": [
    "df_ims = pd.read_csv('image_attri.csv')\n",
    "ims = df_ims['name']\n",
    "ims = ims.values.tolist()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 98,
   "metadata": {},
   "outputs": [],
   "source": [
    "folders = os.listdir('database')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 99,
   "metadata": {},
   "outputs": [],
   "source": [
    "for folder in folders:\n",
    "    files = 'database/'+folder+\"/*\"\n",
    "    for path in glob.glob(files):\n",
    "        name = path.replace(\"database/\",\"\")\n",
    "        if name not in ims:\n",
    "            os.remove(path)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Sampling Patches from Images"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 101,
   "metadata": {},
   "outputs": [],
   "source": [
    "try:\n",
    "    if os.path.isdir('database/patches') == False:\n",
    "        os.mkdir('database/patches')\n",
    "except OSError:\n",
    "    print (\"Creation of the directory %s failed\" % 'patch')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 102,
   "metadata": {},
   "outputs": [],
   "source": [
    "def patch_sampling(im_path, patch_index, x, y, width, height):\n",
    "    img=cv2.imread(im_path)\n",
    "    patch = img[x:x+width, y:y+height]\n",
    "    name = im_path.split('/')[-1]\n",
    "    p_path = 'database/patches/'+name+\"_\"+\"patch_\"+str(patch_index)+\".jpg\"\n",
    "    cv2.imwrite(p_path,patch)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 103,
   "metadata": {},
   "outputs": [],
   "source": [
    "df_coor = pd.read_csv('all_patches.csv')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 104,
   "metadata": {},
   "outputs": [],
   "source": [
    "for folder in folders:\n",
    "    if folder != 'patches':\n",
    "        im_paths = 'database/'+folder+\"/*\"\n",
    "        for im_path in glob.glob(im_paths):\n",
    "            im_name = im_path.split('/')[-1]\n",
    "            patch_name = \"patches/\"+im_name\n",
    "            row = df_coor.loc[df_coor['name_patch']==patch_name]\n",
    "            if len(row) == 1:\n",
    "                x1 = int(row['top_patch_1'].values.tolist()[0])\n",
    "                y1 = int(row['left_patch_1'].values.tolist()[0])\n",
    "                x2 = int(row['top_patch_2'].values.tolist()[0])\n",
    "                y2 = int(row['left_patch_2'].values.tolist()[0])\n",
    "                x3 = int(row['top_patch_3'].values.tolist()[0])\n",
    "                y3 = int(row['left_patch_3'].values.tolist()[0])\n",
    "                h1 = int(row['height_patch_1'].values.tolist()[0])\n",
    "                w1 = int(row['width_patch_1'].values.tolist()[0])\n",
    "                h2 = int(row['height_patch_2'].values.tolist()[0])\n",
    "                w2 = int(row['width_patch_2'].values.tolist()[0])\n",
    "                h3 = int(row['height_patch_3'].values.tolist()[0])\n",
    "                w3 = int(row['width_patch_3'].values.tolist()[0])\n",
    "                patch_sampling(im_path, 1, x1, y1, w1, h1)\n",
    "                patch_sampling(im_path, 2, x2, y2, w2, h2)\n",
    "                patch_sampling(im_path, 3, x3, y3, w3, h3)"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.8"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}