image_extraction_from_tarfiles(1).ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "ApzxrjJkH9HC"
   },
   "source": [
    "### Image extraction from tar files\n",
    "\n",
    "1. Extract images from the tar files\n",
    "2. Rename the images to their respective \"Roco Names\"\n",
    "3. Transfer the images to the \"images\" folder\n",
    "4. Delete all temporary directories used for the image extractions\n",
    "\n",
    "\n",
    "\n",
    "\n",
    "**- Note: The tar files were downloaded via comand line.**\n",
    "\n",
    "- Also, `extract_images.py` was ran in command line to perform the above mentioned functions"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "63-J0h7iImDu"
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "executionInfo": {
     "elapsed": 385,
     "status": "ok",
     "timestamp": 1595212881627,
     "user": {
      "displayName": "adeniyi adeboye",
      "photoUrl": "",
      "userId": "02625876087283522783"
     },
     "user_tz": 240
    },
    "id": "PXtLSoixCW9F"
   },
   "outputs": [],
   "source": [
    "import os\n",
    "import pandas as pd\n",
    "import numpy as np\n",
    "import re\n",
    "from os import walk\n",
    "import math\n",
    "from collections import Counter\n",
    "\n",
    "import shutil\n",
    "import sys\n",
    "import tarfile"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'/home/ec2-user/SageMaker'"
      ]
     },
     "execution_count": 2,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "###know the current directory\n",
    "os.getcwd()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 34
    },
    "colab_type": "code",
    "executionInfo": {
     "elapsed": 350,
     "status": "ok",
     "timestamp": 1595212884494,
     "user": {
      "displayName": "adeniyi adeboye",
      "photoUrl": "",
      "userId": "02625876087283522783"
     },
     "user_tz": 240
    },
    "id": "zX0zu465CvEm",
    "outputId": "ca7b0ad9-0cbd-478a-d7a5-a1296a00fa52"
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['.ipynb_checkpoints',\n",
       " 'roco_data',\n",
       " 'captions.txt',\n",
       " 'ext_list.txt',\n",
       " 'semtypes.txt',\n",
       " 'dlinks.txt',\n",
       " 'licences.txt',\n",
       " 'images',\n",
       " 'url_list.txt',\n",
       " 'cuis.txt',\n",
       " 'keywords.txt']"
      ]
     },
     "execution_count": 3,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "os.listdir('roco-dataset/data/train/radiology')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "executionInfo": {
     "elapsed": 3987,
     "status": "ok",
     "timestamp": 1595233282201,
     "user": {
      "displayName": "adeniyi adeboye",
      "photoUrl": "",
      "userId": "02625876087283522783"
     },
     "user_tz": 240
    },
    "id": "Ax8movAXCvcK"
   },
   "outputs": [],
   "source": [
    "train_download_links = 'roco-dataset/data/train/radiology/dlinks.txt'\n",
    "test_download_links = 'roco-dataset/data/test/radiology/dlinks.txt'\n",
    "val_download_links = 'roco-dataset/data/validation/radiology/dlinks.txt'"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "oDnnjbEnUESb"
   },
   "source": [
    "### Train Dataset --- get the url_links for the training dataset"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "executionInfo": {
     "elapsed": 1174,
     "status": "ok",
     "timestamp": 1595234026641,
     "user": {
      "displayName": "adeniyi adeboye",
      "photoUrl": "",
      "userId": "02625876087283522783"
     },
     "user_tz": 240
    },
    "id": "raIcjiqwUPbI"
   },
   "outputs": [],
   "source": [
    "df_train = pd.read_csv(train_download_links, delimiter = '\\t', header = None)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 195
    },
    "colab_type": "code",
    "executionInfo": {
     "elapsed": 250,
     "status": "ok",
     "timestamp": 1595234112012,
     "user": {
      "displayName": "adeniyi adeboye",
      "photoUrl": "",
      "userId": "02625876087283522783"
     },
     "user_tz": 240
    },
    "id": "p8JRFjNgUPg9",
    "outputId": "29ac4259-d173-4d4f-f596-a004d5f9aa55"
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>0</th>\n",
       "      <th>1</th>\n",
       "      <th>2</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>ROCO_00002</td>\n",
       "      <td>wget -r ftp://ftp.ncbi.nlm.nih.gov/pub/pmc/oa_...</td>\n",
       "      <td>AMHSR-4-14-g002.jpg</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>ROCO_00003</td>\n",
       "      <td>wget -r ftp://ftp.ncbi.nlm.nih.gov/pub/pmc/oa_...</td>\n",
       "      <td>IJD2009-150251.001.jpg</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>ROCO_00004</td>\n",
       "      <td>wget -r ftp://ftp.ncbi.nlm.nih.gov/pub/pmc/oa_...</td>\n",
       "      <td>11999_2007_30_Fig6_HTML.jpg</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>ROCO_00005</td>\n",
       "      <td>wget -r ftp://ftp.ncbi.nlm.nih.gov/pub/pmc/oa_...</td>\n",
       "      <td>IJD2013-683423.005.jpg</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>ROCO_00007</td>\n",
       "      <td>wget -r ftp://ftp.ncbi.nlm.nih.gov/pub/pmc/oa_...</td>\n",
       "      <td>amjcaserep-17-301-g001.jpg</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "            0                                                  1  \\\n",
       "0  ROCO_00002  wget -r ftp://ftp.ncbi.nlm.nih.gov/pub/pmc/oa_...   \n",
       "1  ROCO_00003  wget -r ftp://ftp.ncbi.nlm.nih.gov/pub/pmc/oa_...   \n",
       "2  ROCO_00004  wget -r ftp://ftp.ncbi.nlm.nih.gov/pub/pmc/oa_...   \n",
       "3  ROCO_00005  wget -r ftp://ftp.ncbi.nlm.nih.gov/pub/pmc/oa_...   \n",
       "4  ROCO_00007  wget -r ftp://ftp.ncbi.nlm.nih.gov/pub/pmc/oa_...   \n",
       "\n",
       "                             2  \n",
       "0          AMHSR-4-14-g002.jpg  \n",
       "1       IJD2009-150251.001.jpg  \n",
       "2  11999_2007_30_Fig6_HTML.jpg  \n",
       "3       IJD2013-683423.005.jpg  \n",
       "4   amjcaserep-17-301-g001.jpg  "
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_train.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 34
    },
    "colab_type": "code",
    "executionInfo": {
     "elapsed": 270,
     "status": "ok",
     "timestamp": 1595234114956,
     "user": {
      "displayName": "adeniyi adeboye",
      "photoUrl": "",
      "userId": "02625876087283522783"
     },
     "user_tz": 240
    },
    "id": "032B-u6tUleu",
    "outputId": "a3f7023e-2e11-41b5-e723-26d5a603760b"
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(65450, 3)"
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_train.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "executionInfo": {
     "elapsed": 1321,
     "status": "ok",
     "timestamp": 1595233427499,
     "user": {
      "displayName": "adeniyi adeboye",
      "photoUrl": "",
      "userId": "02625876087283522783"
     },
     "user_tz": 240
    },
    "id": "Ka0GBV_FUljs"
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0    0\n",
       "1    0\n",
       "2    0\n",
       "dtype: int64"
      ]
     },
     "execution_count": 8,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_train.isnull().sum()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "executionInfo": {
     "elapsed": 1309,
     "status": "ok",
     "timestamp": 1595233483181,
     "user": {
      "displayName": "adeniyi adeboye",
      "photoUrl": "",
      "userId": "02625876087283522783"
     },
     "user_tz": 240
    },
    "id": "p4SM0al0Ulne"
   },
   "outputs": [],
   "source": [
    "df_train['url_list'] = df_train[1].apply(lambda x: x.split()[2])\n",
    "df_train['pmc_id'] = df_train['url_list'].apply(lambda x: x.split('/')[-1][:-7])\n",
    "df_train['img'] = df_train['pmc_id'] + '/' + df_train[2]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "executionInfo": {
     "elapsed": 1220,
     "status": "ok",
     "timestamp": 1595233483399,
     "user": {
      "displayName": "adeniyi adeboye",
      "photoUrl": "",
      "userId": "02625876087283522783"
     },
     "user_tz": 240
    },
    "id": "XhFqK3pTUzQo"
   },
   "outputs": [],
   "source": [
    "df_train.sort_values(by = 'pmc_id', inplace = True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "8E54uupDUzWC"
   },
   "outputs": [],
   "source": [
    "df_train.reset_index(inplace = True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "executionInfo": {
     "elapsed": 1081,
     "status": "ok",
     "timestamp": 1595233517849,
     "user": {
      "displayName": "adeniyi adeboye",
      "photoUrl": "",
      "userId": "02625876087283522783"
     },
     "user_tz": 240
    },
    "id": "8bobcN39UlsL"
   },
   "outputs": [],
   "source": [
    "df_train.drop(['index'], axis = 1, inplace = True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "executionInfo": {
     "elapsed": 847,
     "status": "ok",
     "timestamp": 1595233517850,
     "user": {
      "displayName": "adeniyi adeboye",
      "photoUrl": "",
      "userId": "02625876087283522783"
     },
     "user_tz": 240
    },
    "id": "IKVbUzwRU7wF"
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>0</th>\n",
       "      <th>1</th>\n",
       "      <th>2</th>\n",
       "      <th>url_list</th>\n",
       "      <th>pmc_id</th>\n",
       "      <th>img</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>ROCO_35947</td>\n",
       "      <td>wget -r ftp://ftp.ncbi.nlm.nih.gov/pub/pmc/oa_...</td>\n",
       "      <td>1471-2482-2-1-4.jpg</td>\n",
       "      <td>ftp://ftp.ncbi.nlm.nih.gov/pub/pmc/oa_package/...</td>\n",
       "      <td>PMC101389</td>\n",
       "      <td>PMC101389/1471-2482-2-1-4.jpg</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>ROCO_38384</td>\n",
       "      <td>wget -r ftp://ftp.ncbi.nlm.nih.gov/pub/pmc/oa_...</td>\n",
       "      <td>neh055f1.jpg</td>\n",
       "      <td>ftp://ftp.ncbi.nlm.nih.gov/pub/pmc/oa_package/...</td>\n",
       "      <td>PMC1062148</td>\n",
       "      <td>PMC1062148/neh055f1.jpg</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>ROCO_56680</td>\n",
       "      <td>wget -r ftp://ftp.ncbi.nlm.nih.gov/pub/pmc/oa_...</td>\n",
       "      <td>cc2926-3.jpg</td>\n",
       "      <td>ftp://ftp.ncbi.nlm.nih.gov/pub/pmc/oa_package/...</td>\n",
       "      <td>PMC1065025</td>\n",
       "      <td>PMC1065025/cc2926-3.jpg</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>ROCO_03822</td>\n",
       "      <td>wget -r ftp://ftp.ncbi.nlm.nih.gov/pub/pmc/oa_...</td>\n",
       "      <td>cc2926-1.jpg</td>\n",
       "      <td>ftp://ftp.ncbi.nlm.nih.gov/pub/pmc/oa_package/...</td>\n",
       "      <td>PMC1065025</td>\n",
       "      <td>PMC1065025/cc2926-1.jpg</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>ROCO_09480</td>\n",
       "      <td>wget -r ftp://ftp.ncbi.nlm.nih.gov/pub/pmc/oa_...</td>\n",
       "      <td>cc2940-1.jpg</td>\n",
       "      <td>ftp://ftp.ncbi.nlm.nih.gov/pub/pmc/oa_package/...</td>\n",
       "      <td>PMC1065094</td>\n",
       "      <td>PMC1065094/cc2940-1.jpg</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "            0                                                  1  \\\n",
       "0  ROCO_35947  wget -r ftp://ftp.ncbi.nlm.nih.gov/pub/pmc/oa_...   \n",
       "1  ROCO_38384  wget -r ftp://ftp.ncbi.nlm.nih.gov/pub/pmc/oa_...   \n",
       "2  ROCO_56680  wget -r ftp://ftp.ncbi.nlm.nih.gov/pub/pmc/oa_...   \n",
       "3  ROCO_03822  wget -r ftp://ftp.ncbi.nlm.nih.gov/pub/pmc/oa_...   \n",
       "4  ROCO_09480  wget -r ftp://ftp.ncbi.nlm.nih.gov/pub/pmc/oa_...   \n",
       "\n",
       "                     2                                           url_list  \\\n",
       "0  1471-2482-2-1-4.jpg  ftp://ftp.ncbi.nlm.nih.gov/pub/pmc/oa_package/...   \n",
       "1         neh055f1.jpg  ftp://ftp.ncbi.nlm.nih.gov/pub/pmc/oa_package/...   \n",
       "2         cc2926-3.jpg  ftp://ftp.ncbi.nlm.nih.gov/pub/pmc/oa_package/...   \n",
       "3         cc2926-1.jpg  ftp://ftp.ncbi.nlm.nih.gov/pub/pmc/oa_package/...   \n",
       "4         cc2940-1.jpg  ftp://ftp.ncbi.nlm.nih.gov/pub/pmc/oa_package/...   \n",
       "\n",
       "       pmc_id                            img  \n",
       "0   PMC101389  PMC101389/1471-2482-2-1-4.jpg  \n",
       "1  PMC1062148        PMC1062148/neh055f1.jpg  \n",
       "2  PMC1065025        PMC1065025/cc2926-3.jpg  \n",
       "3  PMC1065025        PMC1065025/cc2926-1.jpg  \n",
       "4  PMC1065094        PMC1065094/cc2940-1.jpg  "
      ]
     },
     "execution_count": 13,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_train.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 215,
   "metadata": {},
   "outputs": [],
   "source": [
    "###os.chdir('/home/ec2-user/SageMaker')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 244,
   "metadata": {},
   "outputs": [],
   "source": [
    "###Get your url_list for downloads\n",
    "with open('ext_list.txt', 'w') as file_handler:\n",
    "    for item in df_train['url_list'].values.tolist():\n",
    "        #print(item)\n",
    "        file_handler.write(\"{}\\n\".format(item))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 120,
   "metadata": {},
   "outputs": [],
   "source": [
    "####THIS FUNCTION IS == extract_images.py ---- IT IS RECOMMENDED TO BE RUN THROUGH COMMAND LINE\n",
    "def extract_images(dir_links):\n",
    "    \n",
    "    \"\"\"\n",
    "    This function perform the following tasks:\n",
    "    -- Extract images from the tar files\n",
    "    -- Rename the images to their respective \"Roco Names\"\n",
    "    -- Transfer the images to the \"images\" folder\n",
    "    -- Delete all temporary directories used for the image extractions\n",
    "    \"\"\"\n",
    "    \n",
    "    ##prepare the dataframe that contains the pmc_id column to be used to tar extraction\n",
    "    df = pd.read_csv(dir_links, delimiter = '\\t', header = None)\n",
    "    df['url_list'] = df[1].apply(lambda x: x.split()[2])\n",
    "    df['pmc_id'] = df['url_list'].apply(lambda x: x.split('/')[-1][:-7])\n",
    "    df['img'] = df['pmc_id'] + '/' + df[2]\n",
    "    df.sort_values(by = 'pmc_id', inplace = True)\n",
    "    df.reset_index(inplace = True)\n",
    "    df.drop(['index'], axis = 1, inplace = True)\n",
    "    \n",
    "    ##get the roco_data -- tar files and sort it\n",
    "    roco_data = os.listdir('roco_data')\n",
    "    roco_data.sort()\n",
    "    \n",
    "    \n",
    "    ##function used to remove non-downloadable  or non-extracted files from the df\n",
    "    def remove_missing_data(df, column, roco_list, id):\n",
    "        \n",
    "        pmc_id = df[column].values.tolist()\n",
    "        \n",
    "        def missing_index(pmc_id, roco_list, id):\n",
    "            idx_missing = []\n",
    "            for idx, x in enumerate(pmc_id[:-id]):\n",
    "                if x != roco_list[idx]:\n",
    "                    idx_missing.append(idx)\n",
    "            return idx_missing, pmc_id\n",
    "        \n",
    "        for x in range(id):\n",
    "            miss_id, pmc_id = missing_index(pmc_id, roco_list, id)\n",
    "            id_miss = miss_id[0]\n",
    "            del pmc_id[id_miss]\n",
    "            df.drop([id_miss], inplace = True)\n",
    "            df.reset_index(inplace = True)\n",
    "            df.drop(['index'], axis = 1, inplace = True)\n",
    "            id -= 1\n",
    "        return df\n",
    "    \n",
    "    ##check for missing ids and remove them from the dataframe\n",
    "    roco_list = [re.sub(r'\\.tar.*', '', x) for x in roco_data]\n",
    "    id = df.shape[0] - len(roco_data)\n",
    "    df = remove_missing_data(df, 'pmc_id', roco_list, id)\n",
    "    \n",
    "    \n",
    "    ##extract_tarfiles dir created\n",
    "    os.mkdir('extract_tarfiles')\n",
    "    \n",
    "    ##Extract files with the specific images needed from the downloaded tar files\n",
    "    idx_list = [] ##index of the tar files that do not contain images to be extracted\n",
    "    ##extract images from the downloaded tar files\n",
    "    for idx, x in enumerate(roco_data):\n",
    "        image = df['img'][idx]\n",
    "        archive_tarfile = tarfile.open(os.path.join('./roco_data', x))\n",
    "        try:\n",
    "            archive_tarfile.extract(image, 'extract_tarfiles')\n",
    "            print(image, x)\n",
    "        except:\n",
    "            print('Could not extract images from {0}...'.format(x))\n",
    "            idx_list.append(idx)\n",
    "            continue\n",
    "    \n",
    "    ##drop the tar files in df that do not contain images to be extracted\n",
    "    df.drop(idx_list, inplace = True)\n",
    "    df.reset_index(inplace = True)\n",
    "    df.drop(['index'], axis = 1, inplace = True)\n",
    "    \n",
    "    ##delete tar files in roco_data dir that do not contain images to be extracted\n",
    "    for idx in idx_list:\n",
    "        print('tar file to delete: {0}'.format(roco_data[idx]))\n",
    "        os.unlink('roco_data/' + roco_data[idx])\n",
    "        \n",
    "    print('\\n Amount of files, where images were successfully extracted from {0}'.format(df.shape[0]))\n",
    "    \n",
    "    ##load the extracted images into a specific folder -- \"images_folder2\"\n",
    "    os.mkdir('images_folder2')\n",
    "    for ext_file in os.listdir('./extract_tarfiles'):\n",
    "        for images in os.listdir('./extract_tarfiles/' + ext_file):\n",
    "            imgs = os.path.join('./extract_tarfiles/' + ext_file, images)\n",
    "            shutil.copy(imgs, 'images_folder2')\n",
    "            #print(imgs)\n",
    "    print('\\n Finished loading images into images_folder2 \\n')\n",
    "    \n",
    "    img_list = os.listdir('images_folder2')\n",
    "    print('\\n Amount of images successfully extracted == {0} \\n'.format(len(img_list)))\n",
    "    \n",
    "    ##amount of missing images after extraction \n",
    "    id_img = df.shape[0] - len(img_list)\n",
    "    \n",
    "    ###lets remove missing images from the df_tr\n",
    "    df = remove_missing_data(df, 2, img_list, id_img)\n",
    "    \n",
    "    roco_names = df[0].values.tolist()\n",
    "    ##rename the images to their \"ROCO\" names\n",
    "    for idx, images in enumerate(img_list):\n",
    "        print(images)\n",
    "        os.rename('./images_folder2/' + images, './images_folder2/' + roco_names[idx] + '.jpg')\n",
    "        \n",
    "    ##check if the images folder already exist...if not create a new one\n",
    "    if not os.path.exists('images'):\n",
    "        os.mkdir('images')\n",
    "        \n",
    "    ###Finally,  lets copy the images to the original \"images\" folder ----\n",
    "    print('\\n Copying the new extracted images into the \"images\" folder \\n')\n",
    "    for images in os.listdir('images_folder2'):\n",
    "        imgs = os.path.join('images_folder2/', images)\n",
    "        shutil.copy(imgs, 'images')\n",
    "        print(imgs)\n",
    "        \n",
    "    img_list = os.listdir('images')\n",
    "    print('\\n Amount of images in the images folder: {0} \\n'.format(len(img_list)))\n",
    "    \n",
    "    ###delete the previous/initial files and folders created  --- they consume a large space\n",
    "    folder_list = ['extract_tarfiles', 'images_folder2']\n",
    "    for item in folder_list:\n",
    "        try:\n",
    "            shutil.rmtree(item)\n",
    "        except:\n",
    "            print('{0} already deleted'.format(item))\n",
    "            continue\n",
    "        else:\n",
    "            print('Deleted {0}'.format(item))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [],
   "source": [
    "##Get the validation dataset url_links\n",
    "df_val = pd.read_csv(val_download_links, delimiter = '\\t', header = None)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>0</th>\n",
       "      <th>1</th>\n",
       "      <th>2</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>ROCO_00020</td>\n",
       "      <td>wget -r ftp://ftp.ncbi.nlm.nih.gov/pub/pmc/oa_...</td>\n",
       "      <td>CRIONM2014-931546.003.jpg</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>ROCO_00027</td>\n",
       "      <td>wget -r ftp://ftp.ncbi.nlm.nih.gov/pub/pmc/oa_...</td>\n",
       "      <td>cios-1-176-g005.jpg</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>ROCO_00059</td>\n",
       "      <td>wget -r ftp://ftp.ncbi.nlm.nih.gov/pub/pmc/oa_...</td>\n",
       "      <td>poljradiol-78-3-35-g001.jpg</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>ROCO_00062</td>\n",
       "      <td>wget -r ftp://ftp.ncbi.nlm.nih.gov/pub/pmc/oa_...</td>\n",
       "      <td>p147_fig4a.jpg</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>ROCO_00068</td>\n",
       "      <td>wget -r ftp://ftp.ncbi.nlm.nih.gov/pub/pmc/oa_...</td>\n",
       "      <td>CRIGM2017-1710501.002.jpg</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "            0                                                  1  \\\n",
       "0  ROCO_00020  wget -r ftp://ftp.ncbi.nlm.nih.gov/pub/pmc/oa_...   \n",
       "1  ROCO_00027  wget -r ftp://ftp.ncbi.nlm.nih.gov/pub/pmc/oa_...   \n",
       "2  ROCO_00059  wget -r ftp://ftp.ncbi.nlm.nih.gov/pub/pmc/oa_...   \n",
       "3  ROCO_00062  wget -r ftp://ftp.ncbi.nlm.nih.gov/pub/pmc/oa_...   \n",
       "4  ROCO_00068  wget -r ftp://ftp.ncbi.nlm.nih.gov/pub/pmc/oa_...   \n",
       "\n",
       "                             2  \n",
       "0    CRIONM2014-931546.003.jpg  \n",
       "1          cios-1-176-g005.jpg  \n",
       "2  poljradiol-78-3-35-g001.jpg  \n",
       "3               p147_fig4a.jpg  \n",
       "4    CRIGM2017-1710501.002.jpg  "
      ]
     },
     "execution_count": 15,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_val.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(8180, 3)"
      ]
     },
     "execution_count": 16,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_val.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0    0\n",
       "1    0\n",
       "2    0\n",
       "dtype: int64"
      ]
     },
     "execution_count": 17,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_val.isnull().sum()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "executionInfo": {
     "elapsed": 385,
     "status": "ok",
     "timestamp": 1595213105632,
     "user": {
      "displayName": "adeniyi adeboye",
      "photoUrl": "",
      "userId": "02625876087283522783"
     },
     "user_tz": 240
    },
    "id": "SiCa9-3KFKS5"
   },
   "outputs": [],
   "source": [
    "df_val['url_list'] = df_val[1].apply(lambda x: x.split()[2])\n",
    "df_val['pmc_id'] = df_val['url_list'].apply(lambda x: x.split('/')[-1][:-7])\n",
    "df_val['img'] = df_val['pmc_id'] + '/' + df_val[2]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "executionInfo": {
     "elapsed": 385,
     "status": "ok",
     "timestamp": 1595213147775,
     "user": {
      "displayName": "adeniyi adeboye",
      "photoUrl": "",
      "userId": "02625876087283522783"
     },
     "user_tz": 240
    },
    "id": "Rin1Uv5bHEf8"
   },
   "outputs": [],
   "source": [
    "df_val.sort_values(by = 'pmc_id', inplace = True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "executionInfo": {
     "elapsed": 259,
     "status": "ok",
     "timestamp": 1595213186553,
     "user": {
      "displayName": "adeniyi adeboye",
      "photoUrl": "",
      "userId": "02625876087283522783"
     },
     "user_tz": 240
    },
    "id": "MoKs0Fr-HOyB"
   },
   "outputs": [],
   "source": [
    "df_val.reset_index(inplace = True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "executionInfo": {
     "elapsed": 337,
     "status": "ok",
     "timestamp": 1595213219381,
     "user": {
      "displayName": "adeniyi adeboye",
      "photoUrl": "",
      "userId": "02625876087283522783"
     },
     "user_tz": 240
    },
    "id": "I5aSUOyNHYRT"
   },
   "outputs": [],
   "source": [
    "df_val.drop(['index'], axis = 1, inplace = True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "WoCz4YJBoYQM"
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>0</th>\n",
       "      <th>1</th>\n",
       "      <th>2</th>\n",
       "      <th>url_list</th>\n",
       "      <th>pmc_id</th>\n",
       "      <th>img</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>ROCO_70277</td>\n",
       "      <td>wget -r ftp://ftp.ncbi.nlm.nih.gov/pub/pmc/oa_...</td>\n",
       "      <td>cc2926-2.jpg</td>\n",
       "      <td>ftp://ftp.ncbi.nlm.nih.gov/pub/pmc/oa_package/...</td>\n",
       "      <td>PMC1065025</td>\n",
       "      <td>PMC1065025/cc2926-2.jpg</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>ROCO_39258</td>\n",
       "      <td>wget -r ftp://ftp.ncbi.nlm.nih.gov/pub/pmc/oa_...</td>\n",
       "      <td>1471-2296-3-6-2.jpg</td>\n",
       "      <td>ftp://ftp.ncbi.nlm.nih.gov/pub/pmc/oa_package/...</td>\n",
       "      <td>PMC107839</td>\n",
       "      <td>PMC107839/1471-2296-3-6-2.jpg</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>ROCO_49332</td>\n",
       "      <td>wget -r ftp://ftp.ncbi.nlm.nih.gov/pub/pmc/oa_...</td>\n",
       "      <td>pmed.0020079.g002.jpg</td>\n",
       "      <td>ftp://ftp.ncbi.nlm.nih.gov/pub/pmc/oa_package/...</td>\n",
       "      <td>PMC1087206</td>\n",
       "      <td>PMC1087206/pmed.0020079.g002.jpg</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>ROCO_65330</td>\n",
       "      <td>wget -r ftp://ftp.ncbi.nlm.nih.gov/pub/pmc/oa_...</td>\n",
       "      <td>pmed.0020154.g005.jpg</td>\n",
       "      <td>ftp://ftp.ncbi.nlm.nih.gov/pub/pmc/oa_package/...</td>\n",
       "      <td>PMC1160569</td>\n",
       "      <td>PMC1160569/pmed.0020154.g005.jpg</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>ROCO_50186</td>\n",
       "      <td>wget -r ftp://ftp.ncbi.nlm.nih.gov/pub/pmc/oa_...</td>\n",
       "      <td>1471-2334-5-42-3.jpg</td>\n",
       "      <td>ftp://ftp.ncbi.nlm.nih.gov/pub/pmc/oa_package/...</td>\n",
       "      <td>PMC1174869</td>\n",
       "      <td>PMC1174869/1471-2334-5-42-3.jpg</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "            0                                                  1  \\\n",
       "0  ROCO_70277  wget -r ftp://ftp.ncbi.nlm.nih.gov/pub/pmc/oa_...   \n",
       "1  ROCO_39258  wget -r ftp://ftp.ncbi.nlm.nih.gov/pub/pmc/oa_...   \n",
       "2  ROCO_49332  wget -r ftp://ftp.ncbi.nlm.nih.gov/pub/pmc/oa_...   \n",
       "3  ROCO_65330  wget -r ftp://ftp.ncbi.nlm.nih.gov/pub/pmc/oa_...   \n",
       "4  ROCO_50186  wget -r ftp://ftp.ncbi.nlm.nih.gov/pub/pmc/oa_...   \n",
       "\n",
       "                       2                                           url_list  \\\n",
       "0           cc2926-2.jpg  ftp://ftp.ncbi.nlm.nih.gov/pub/pmc/oa_package/...   \n",
       "1    1471-2296-3-6-2.jpg  ftp://ftp.ncbi.nlm.nih.gov/pub/pmc/oa_package/...   \n",
       "2  pmed.0020079.g002.jpg  ftp://ftp.ncbi.nlm.nih.gov/pub/pmc/oa_package/...   \n",
       "3  pmed.0020154.g005.jpg  ftp://ftp.ncbi.nlm.nih.gov/pub/pmc/oa_package/...   \n",
       "4   1471-2334-5-42-3.jpg  ftp://ftp.ncbi.nlm.nih.gov/pub/pmc/oa_package/...   \n",
       "\n",
       "       pmc_id                               img  \n",
       "0  PMC1065025           PMC1065025/cc2926-2.jpg  \n",
       "1   PMC107839     PMC107839/1471-2296-3-6-2.jpg  \n",
       "2  PMC1087206  PMC1087206/pmed.0020079.g002.jpg  \n",
       "3  PMC1160569  PMC1160569/pmed.0020154.g005.jpg  \n",
       "4  PMC1174869   PMC1174869/1471-2334-5-42-3.jpg  "
      ]
     },
     "execution_count": 22,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_val.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 135,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [],
   "source": [
    "with open('url_list.txt', 'w') as file_handler:\n",
    "    for item in df_val['url_list'].values.tolist():\n",
    "        #print(item)\n",
    "        file_handler.write(\"{}\\n\".format(item))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "ovULRzlPp3Ch"
   },
   "source": [
    "### --- Get the url_links for the test dataset"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 137,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "PSxUs2y7qAeg"
   },
   "outputs": [],
   "source": [
    "###os.chdir('/home/ec2-user/SageMaker')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "executionInfo": {
     "elapsed": 483,
     "status": "ok",
     "timestamp": 1595221972112,
     "user": {
      "displayName": "adeniyi adeboye",
      "photoUrl": "",
      "userId": "02625876087283522783"
     },
     "user_tz": 240
    },
    "id": "CYhhBZ9ZCvvA"
   },
   "outputs": [],
   "source": [
    "df_test = pd.read_csv(test_download_links, delimiter = '\\t', header = None)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 195
    },
    "colab_type": "code",
    "executionInfo": {
     "elapsed": 781,
     "status": "ok",
     "timestamp": 1595222035417,
     "user": {
      "displayName": "adeniyi adeboye",
      "photoUrl": "",
      "userId": "02625876087283522783"
     },
     "user_tz": 240
    },
    "id": "kn7LW6cZCvxp",
    "outputId": "b5328734-3be3-48bc-eb11-8f50c933f06e"
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>0</th>\n",
       "      <th>1</th>\n",
       "      <th>2</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>ROCO_00001</td>\n",
       "      <td>wget -r ftp://ftp.ncbi.nlm.nih.gov/pub/pmc/oa_...</td>\n",
       "      <td>cro-0008-0385-g01.jpg</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>ROCO_00006</td>\n",
       "      <td>wget -r ftp://ftp.ncbi.nlm.nih.gov/pub/pmc/oa_...</td>\n",
       "      <td>ol-11-05-3298-g02.jpg</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>ROCO_00016</td>\n",
       "      <td>wget -r ftp://ftp.ncbi.nlm.nih.gov/pub/pmc/oa_...</td>\n",
       "      <td>cureus-0009-00000001639-i01.jpg</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>ROCO_00025</td>\n",
       "      <td>wget -r ftp://ftp.ncbi.nlm.nih.gov/pub/pmc/oa_...</td>\n",
       "      <td>EJD-10-188-g001.jpg</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>ROCO_00031</td>\n",
       "      <td>wget -r ftp://ftp.ncbi.nlm.nih.gov/pub/pmc/oa_...</td>\n",
       "      <td>PWKI-10-23517-g001.jpg</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "            0                                                  1  \\\n",
       "0  ROCO_00001  wget -r ftp://ftp.ncbi.nlm.nih.gov/pub/pmc/oa_...   \n",
       "1  ROCO_00006  wget -r ftp://ftp.ncbi.nlm.nih.gov/pub/pmc/oa_...   \n",
       "2  ROCO_00016  wget -r ftp://ftp.ncbi.nlm.nih.gov/pub/pmc/oa_...   \n",
       "3  ROCO_00025  wget -r ftp://ftp.ncbi.nlm.nih.gov/pub/pmc/oa_...   \n",
       "4  ROCO_00031  wget -r ftp://ftp.ncbi.nlm.nih.gov/pub/pmc/oa_...   \n",
       "\n",
       "                                 2  \n",
       "0            cro-0008-0385-g01.jpg  \n",
       "1            ol-11-05-3298-g02.jpg  \n",
       "2  cureus-0009-00000001639-i01.jpg  \n",
       "3              EJD-10-188-g001.jpg  \n",
       "4           PWKI-10-23517-g001.jpg  "
      ]
     },
     "execution_count": 24,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_test.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 34
    },
    "colab_type": "code",
    "executionInfo": {
     "elapsed": 297,
     "status": "ok",
     "timestamp": 1595222048148,
     "user": {
      "displayName": "adeniyi adeboye",
      "photoUrl": "",
      "userId": "02625876087283522783"
     },
     "user_tz": 240
    },
    "id": "-bYUEmeUpIh-",
    "outputId": "3fc488c4-8cb3-4f4e-95e7-b323b2022e72"
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(8179, 3)"
      ]
     },
     "execution_count": 25,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_test.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 84
    },
    "colab_type": "code",
    "executionInfo": {
     "elapsed": 248,
     "status": "ok",
     "timestamp": 1595222067031,
     "user": {
      "displayName": "adeniyi adeboye",
      "photoUrl": "",
      "userId": "02625876087283522783"
     },
     "user_tz": 240
    },
    "id": "A78fi5jXpJZy",
    "outputId": "f81abbda-d8a9-43ce-9bc8-0c6af10f63f9"
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0    0\n",
       "1    0\n",
       "2    0\n",
       "dtype: int64"
      ]
     },
     "execution_count": 26,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "##check for null values\n",
    "df_test.isnull().sum()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "executionInfo": {
     "elapsed": 290,
     "status": "ok",
     "timestamp": 1595222192861,
     "user": {
      "displayName": "adeniyi adeboye",
      "photoUrl": "",
      "userId": "02625876087283522783"
     },
     "user_tz": 240
    },
    "id": "Nbajv_38pJeF"
   },
   "outputs": [],
   "source": [
    "df_test['url_list'] = df_test[1].apply(lambda x: x.split()[2])\n",
    "df_test['pmc_id'] = df_test['url_list'].apply(lambda x: x.split('/')[-1][:-7])\n",
    "df_test['img'] = df_test['pmc_id'] + '/' + df_test[2]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "executionInfo": {
     "elapsed": 293,
     "status": "ok",
     "timestamp": 1595222304081,
     "user": {
      "displayName": "adeniyi adeboye",
      "photoUrl": "",
      "userId": "02625876087283522783"
     },
     "user_tz": 240
    },
    "id": "kWf0aZhdpiZW"
   },
   "outputs": [],
   "source": [
    "df_test.sort_values(by = 'pmc_id', inplace = True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "executionInfo": {
     "elapsed": 420,
     "status": "ok",
     "timestamp": 1595222322358,
     "user": {
      "displayName": "adeniyi adeboye",
      "photoUrl": "",
      "userId": "02625876087283522783"
     },
     "user_tz": 240
    },
    "id": "zVrvozZSpicK"
   },
   "outputs": [],
   "source": [
    "df_test.reset_index(inplace = True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "executionInfo": {
     "elapsed": 1192,
     "status": "ok",
     "timestamp": 1595222357216,
     "user": {
      "displayName": "adeniyi adeboye",
      "photoUrl": "",
      "userId": "02625876087283522783"
     },
     "user_tz": 240
    },
    "id": "lire9uFiqOrJ"
   },
   "outputs": [],
   "source": [
    "df_test.drop(['index'], axis = 1, inplace = True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "executionInfo": {
     "elapsed": 299,
     "status": "ok",
     "timestamp": 1595222524370,
     "user": {
      "displayName": "adeniyi adeboye",
      "photoUrl": "",
      "userId": "02625876087283522783"
     },
     "user_tz": 240
    },
    "id": "YY_B8yMgqW9x"
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>0</th>\n",
       "      <th>1</th>\n",
       "      <th>2</th>\n",
       "      <th>url_list</th>\n",
       "      <th>pmc_id</th>\n",
       "      <th>img</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>ROCO_78739</td>\n",
       "      <td>wget -r ftp://ftp.ncbi.nlm.nih.gov/pub/pmc/oa_...</td>\n",
       "      <td>1471-2296-3-6-3.jpg</td>\n",
       "      <td>ftp://ftp.ncbi.nlm.nih.gov/pub/pmc/oa_package/...</td>\n",
       "      <td>PMC107839</td>\n",
       "      <td>PMC107839/1471-2296-3-6-3.jpg</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>ROCO_66766</td>\n",
       "      <td>wget -r ftp://ftp.ncbi.nlm.nih.gov/pub/pmc/oa_...</td>\n",
       "      <td>1477-7819-3-19-2.jpg</td>\n",
       "      <td>ftp://ftp.ncbi.nlm.nih.gov/pub/pmc/oa_package/...</td>\n",
       "      <td>PMC1087895</td>\n",
       "      <td>PMC1087895/1477-7819-3-19-2.jpg</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>ROCO_23290</td>\n",
       "      <td>wget -r ftp://ftp.ncbi.nlm.nih.gov/pub/pmc/oa_...</td>\n",
       "      <td>1477-7819-3-29-2.jpg</td>\n",
       "      <td>ftp://ftp.ncbi.nlm.nih.gov/pub/pmc/oa_package/...</td>\n",
       "      <td>PMC1156959</td>\n",
       "      <td>PMC1156959/1477-7819-3-29-2.jpg</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>ROCO_59505</td>\n",
       "      <td>wget -r ftp://ftp.ncbi.nlm.nih.gov/pub/pmc/oa_...</td>\n",
       "      <td>1471-2334-5-42-2.jpg</td>\n",
       "      <td>ftp://ftp.ncbi.nlm.nih.gov/pub/pmc/oa_package/...</td>\n",
       "      <td>PMC1174869</td>\n",
       "      <td>PMC1174869/1471-2334-5-42-2.jpg</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>ROCO_81227</td>\n",
       "      <td>wget -r ftp://ftp.ncbi.nlm.nih.gov/pub/pmc/oa_...</td>\n",
       "      <td>1472-6815-5-4-3.jpg</td>\n",
       "      <td>ftp://ftp.ncbi.nlm.nih.gov/pub/pmc/oa_package/...</td>\n",
       "      <td>PMC1180430</td>\n",
       "      <td>PMC1180430/1472-6815-5-4-3.jpg</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "            0                                                  1  \\\n",
       "0  ROCO_78739  wget -r ftp://ftp.ncbi.nlm.nih.gov/pub/pmc/oa_...   \n",
       "1  ROCO_66766  wget -r ftp://ftp.ncbi.nlm.nih.gov/pub/pmc/oa_...   \n",
       "2  ROCO_23290  wget -r ftp://ftp.ncbi.nlm.nih.gov/pub/pmc/oa_...   \n",
       "3  ROCO_59505  wget -r ftp://ftp.ncbi.nlm.nih.gov/pub/pmc/oa_...   \n",
       "4  ROCO_81227  wget -r ftp://ftp.ncbi.nlm.nih.gov/pub/pmc/oa_...   \n",
       "\n",
       "                      2                                           url_list  \\\n",
       "0   1471-2296-3-6-3.jpg  ftp://ftp.ncbi.nlm.nih.gov/pub/pmc/oa_package/...   \n",
       "1  1477-7819-3-19-2.jpg  ftp://ftp.ncbi.nlm.nih.gov/pub/pmc/oa_package/...   \n",
       "2  1477-7819-3-29-2.jpg  ftp://ftp.ncbi.nlm.nih.gov/pub/pmc/oa_package/...   \n",
       "3  1471-2334-5-42-2.jpg  ftp://ftp.ncbi.nlm.nih.gov/pub/pmc/oa_package/...   \n",
       "4   1472-6815-5-4-3.jpg  ftp://ftp.ncbi.nlm.nih.gov/pub/pmc/oa_package/...   \n",
       "\n",
       "       pmc_id                              img  \n",
       "0   PMC107839    PMC107839/1471-2296-3-6-3.jpg  \n",
       "1  PMC1087895  PMC1087895/1477-7819-3-19-2.jpg  \n",
       "2  PMC1156959  PMC1156959/1477-7819-3-29-2.jpg  \n",
       "3  PMC1174869  PMC1174869/1471-2334-5-42-2.jpg  \n",
       "4  PMC1180430   PMC1180430/1472-6815-5-4-3.jpg  "
      ]
     },
     "execution_count": 31,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_test.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 87,
   "metadata": {},
   "outputs": [],
   "source": [
    "with open('url_list.txt', 'w') as file_handler:\n",
    "    for item in df_test['url_list'].values.tolist():\n",
    "        #print(item)\n",
    "        file_handler.write(\"{}\\n\".format(item))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "NpjroRrKpInN"
   },
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "colab": {
   "authorship_tag": "ABX9TyMGTT+D5QjlSWc9fcTUAPKr",
   "collapsed_sections": [],
   "name": "image_extraction_from_tarfiles.ipynb",
   "provenance": []
  },
  "kernelspec": {
   "display_name": "conda_python3",
   "language": "python",
   "name": "conda_python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.10"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}