From 6e6b1ee910b18950306ce0b0bea2ff88d0978c4e Mon Sep 17 00:00:00 2001 From: fatih Date: Wed, 19 Aug 2020 11:17:09 +0300 Subject: [PATCH] format codebase with black --- midv500/convert_dataset.py | 65 ++++++++++++------- midv500/download_dataset.py | 122 +++++++++++++++++++----------------- midv500/utils.py | 32 +++++++--- setup.cfg | 2 +- setup.py | 12 ++-- 5 files changed, 134 insertions(+), 99 deletions(-) diff --git a/midv500/convert_dataset.py b/midv500/convert_dataset.py index b049567..43adaa3 100644 --- a/midv500/convert_dataset.py +++ b/midv500/convert_dataset.py @@ -4,7 +4,11 @@ import argparse import numpy as np from tqdm import tqdm -from midv500.utils import list_annotation_paths_recursively, get_bbox_inside_image, create_dir +from midv500.utils import ( + list_annotation_paths_recursively, + get_bbox_inside_image, + create_dir, +) def convert(root_dir: str, export_dir: str): @@ -19,7 +23,7 @@ def convert(root_dir: str, export_dir: str): # raise error if export_dir is given as a json file path if "json" in export_dir: - raise ValueError('export_dir should be a directory, not a file path!') + raise ValueError("export_dir should be a directory, not a file path!") # create export_dir if not present create_dir(export_dir) @@ -43,9 +47,9 @@ def convert(root_dir: str, export_dir: str): # prepare image info image_dict = dict() image_dict["file_name"] = rel_image_path - image_dict['height'] = image.shape[0] - image_dict['width'] = image.shape[1] - image_dict['id'] = ind + image_dict["height"] = image.shape[0] + image_dict["width"] = image.shape[1] + image_dict["id"] = ind # add image info images.append(image_dict) @@ -58,8 +62,8 @@ def convert(root_dir: str, export_dir: str): # load mask coords abs_annotation_path = os.path.join(root_dir, rel_annotation_path) - quad = json.load(open(abs_annotation_path, 'r')) - mask_coords = quad['quad'] + quad = json.load(open(abs_annotation_path, "r")) + mask_coords = quad["quad"] # create mask from poly coords mask = np.zeros(image.shape, dtype=np.uint8) @@ -77,20 +81,29 @@ def convert(root_dir: str, export_dir: str): label_bbox = get_bbox_inside_image(label_bbox, image_bbox) # calculate coco style bbox coords [minx, miny, width, height] and area - label_area = int((label_bbox[2]-label_bbox[0]) * (label_bbox[3]-label_bbox[1])) - label_bbox = [label_bbox[0], label_bbox[1], label_bbox[2]-label_bbox[0], label_bbox[3]-label_bbox[1]] + label_area = int( + (label_bbox[2] - label_bbox[0]) * (label_bbox[3] - label_bbox[1]) + ) + label_bbox = [ + label_bbox[0], + label_bbox[1], + label_bbox[2] - label_bbox[0], + label_bbox[3] - label_bbox[1], + ] # prepare annotation info annotation_dict = dict() annotation_dict["iscrowd"] = 0 - annotation_dict["image_id"] = image_dict['id'] - annotation_dict['category_id'] = 1 # id card - annotation_dict['ignore'] = 0 - annotation_dict['id'] = ind - - annotation_dict['bbox'] = label_bbox - annotation_dict['area'] = label_area - annotation_dict['segmentation'] = [[single_coord for coord_pair in mask_coords for single_coord in coord_pair]] + annotation_dict["image_id"] = image_dict["id"] + annotation_dict["category_id"] = 1 # id card + annotation_dict["ignore"] = 0 + annotation_dict["id"] = ind + + annotation_dict["bbox"] = label_bbox + annotation_dict["area"] = label_area + annotation_dict["segmentation"] = [ + [single_coord for coord_pair in mask_coords for single_coord in coord_pair] + ] # add annotation info annotations.append(annotation_dict) @@ -98,22 +111,28 @@ def convert(root_dir: str, export_dir: str): coco_dict = dict() coco_dict["images"] = images coco_dict["annotations"] = annotations - coco_dict["categories"] = [{'name': 'id_card', 'id': 1}] + coco_dict["categories"] = [{"name": "id_card", "id": 1}] # export coco dict export_path = os.path.join(export_dir, "midv500_coco.json") - with open(export_path, 'w') as f: + with open(export_path, "w") as f: json.dump(coco_dict, f) -if __name__ == '__main__': +if __name__ == "__main__": # construct the argument parser ap = argparse.ArgumentParser() # add the arguments to the parser - ap.add_argument("root_dir", default="data/", help="Directory of the downloaded MIDV-500 dataset.") - ap.add_argument("export_dir", default="coco/", help="Directory for coco file to be exported.") + ap.add_argument( + "root_dir", + default="data/", + help="Directory of the downloaded MIDV-500 dataset.", + ) + ap.add_argument( + "export_dir", default="coco/", help="Directory for coco file to be exported." + ) args = vars(ap.parse_args()) # convert dataset - convert(args['root_dir'], args['export_dir']) + convert(args["root_dir"], args["export_dir"]) diff --git a/midv500/download_dataset.py b/midv500/download_dataset.py index 1dab8e5..f348677 100644 --- a/midv500/download_dataset.py +++ b/midv500/download_dataset.py @@ -2,56 +2,58 @@ import argparse from midv500.utils import download, unzip -all_links = ['ftp://smartengines.com/midv-500/dataset/01_alb_id.zip', - 'ftp://smartengines.com/midv-500/dataset/02_aut_drvlic_new.zip', - 'ftp://smartengines.com/midv-500/dataset/03_aut_id_old.zip', - 'ftp://smartengines.com/midv-500/dataset/04_aut_id.zip', - 'ftp://smartengines.com/midv-500/dataset/05_aze_passport.zip', - 'ftp://smartengines.com/midv-500/dataset/06_bra_passport.zip', - 'ftp://smartengines.com/midv-500/dataset/07_chl_id.zip', - 'ftp://smartengines.com/midv-500/dataset/08_chn_homereturn.zip', - 'ftp://smartengines.com/midv-500/dataset/09_chn_id.zip', - 'ftp://smartengines.com/midv-500/dataset/10_cze_id.zip', - 'ftp://smartengines.com/midv-500/dataset/11_cze_passport.zip', - 'ftp://smartengines.com/midv-500/dataset/12_deu_drvlic_new.zip', - 'ftp://smartengines.com/midv-500/dataset/13_deu_drvlic_old.zip', - 'ftp://smartengines.com/midv-500/dataset/14_deu_id_new.zip', - 'ftp://smartengines.com/midv-500/dataset/15_deu_id_old.zip', - 'ftp://smartengines.com/midv-500/dataset/16_deu_passport_new.zip', - 'ftp://smartengines.com/midv-500/dataset/17_deu_passport_old.zip', - 'ftp://smartengines.com/midv-500/dataset/18_dza_passport.zip', - 'ftp://smartengines.com/midv-500/dataset/19_esp_drvlic.zip', - 'ftp://smartengines.com/midv-500/dataset/20_esp_id_new.zip', - 'ftp://smartengines.com/midv-500/dataset/21_esp_id_old.zip', - 'ftp://smartengines.com/midv-500/dataset/22_est_id.zip', - 'ftp://smartengines.com/midv-500/dataset/23_fin_drvlic.zip', - 'ftp://smartengines.com/midv-500/dataset/24_fin_id.zip', - 'ftp://smartengines.com/midv-500/dataset/25_grc_passport.zip', - 'ftp://smartengines.com/midv-500/dataset/26_hrv_drvlic.zip', - 'ftp://smartengines.com/midv-500/dataset/27_hrv_passport.zip', - 'ftp://smartengines.com/midv-500/dataset/28_hun_passport.zip', - 'ftp://smartengines.com/midv-500/dataset/29_irn_drvlic.zip', - 'ftp://smartengines.com/midv-500/dataset/30_ita_drvlic.zip', - 'ftp://smartengines.com/midv-500/dataset/31_jpn_drvlic.zip', - 'ftp://smartengines.com/midv-500/dataset/32_lva_passport.zip', - 'ftp://smartengines.com/midv-500/dataset/33_mac_id.zip', - 'ftp://smartengines.com/midv-500/dataset/34_mda_passport.zip', - 'ftp://smartengines.com/midv-500/dataset/35_nor_drvlic.zip', - 'ftp://smartengines.com/midv-500/dataset/36_pol_drvlic.zip', - 'ftp://smartengines.com/midv-500/dataset/37_prt_id.zip', - 'ftp://smartengines.com/midv-500/dataset/38_rou_drvlic.zip', - 'ftp://smartengines.com/midv-500/dataset/39_rus_internalpassport.zip', - 'ftp://smartengines.com/midv-500/dataset/40_srb_id.zip', - 'ftp://smartengines.com/midv-500/dataset/41_srb_passport.zip', - 'ftp://smartengines.com/midv-500/dataset/42_svk_id.zip', - 'ftp://smartengines.com/midv-500/dataset/43_tur_id.zip', - 'ftp://smartengines.com/midv-500/dataset/44_ukr_id.zip', - 'ftp://smartengines.com/midv-500/dataset/45_ukr_passport.zip', - 'ftp://smartengines.com/midv-500/dataset/46_ury_passport.zip', - 'ftp://smartengines.com/midv-500/dataset/47_usa_bordercrossing.zip', - 'ftp://smartengines.com/midv-500/dataset/48_usa_passportcard.zip', - 'ftp://smartengines.com/midv-500/dataset/49_usa_ssn82.zip', - 'ftp://smartengines.com/midv-500/dataset/50_xpo_id.zip'] +all_links = [ + "ftp://smartengines.com/midv-500/dataset/01_alb_id.zip", + "ftp://smartengines.com/midv-500/dataset/02_aut_drvlic_new.zip", + "ftp://smartengines.com/midv-500/dataset/03_aut_id_old.zip", + "ftp://smartengines.com/midv-500/dataset/04_aut_id.zip", + "ftp://smartengines.com/midv-500/dataset/05_aze_passport.zip", + "ftp://smartengines.com/midv-500/dataset/06_bra_passport.zip", + "ftp://smartengines.com/midv-500/dataset/07_chl_id.zip", + "ftp://smartengines.com/midv-500/dataset/08_chn_homereturn.zip", + "ftp://smartengines.com/midv-500/dataset/09_chn_id.zip", + "ftp://smartengines.com/midv-500/dataset/10_cze_id.zip", + "ftp://smartengines.com/midv-500/dataset/11_cze_passport.zip", + "ftp://smartengines.com/midv-500/dataset/12_deu_drvlic_new.zip", + "ftp://smartengines.com/midv-500/dataset/13_deu_drvlic_old.zip", + "ftp://smartengines.com/midv-500/dataset/14_deu_id_new.zip", + "ftp://smartengines.com/midv-500/dataset/15_deu_id_old.zip", + "ftp://smartengines.com/midv-500/dataset/16_deu_passport_new.zip", + "ftp://smartengines.com/midv-500/dataset/17_deu_passport_old.zip", + "ftp://smartengines.com/midv-500/dataset/18_dza_passport.zip", + "ftp://smartengines.com/midv-500/dataset/19_esp_drvlic.zip", + "ftp://smartengines.com/midv-500/dataset/20_esp_id_new.zip", + "ftp://smartengines.com/midv-500/dataset/21_esp_id_old.zip", + "ftp://smartengines.com/midv-500/dataset/22_est_id.zip", + "ftp://smartengines.com/midv-500/dataset/23_fin_drvlic.zip", + "ftp://smartengines.com/midv-500/dataset/24_fin_id.zip", + "ftp://smartengines.com/midv-500/dataset/25_grc_passport.zip", + "ftp://smartengines.com/midv-500/dataset/26_hrv_drvlic.zip", + "ftp://smartengines.com/midv-500/dataset/27_hrv_passport.zip", + "ftp://smartengines.com/midv-500/dataset/28_hun_passport.zip", + "ftp://smartengines.com/midv-500/dataset/29_irn_drvlic.zip", + "ftp://smartengines.com/midv-500/dataset/30_ita_drvlic.zip", + "ftp://smartengines.com/midv-500/dataset/31_jpn_drvlic.zip", + "ftp://smartengines.com/midv-500/dataset/32_lva_passport.zip", + "ftp://smartengines.com/midv-500/dataset/33_mac_id.zip", + "ftp://smartengines.com/midv-500/dataset/34_mda_passport.zip", + "ftp://smartengines.com/midv-500/dataset/35_nor_drvlic.zip", + "ftp://smartengines.com/midv-500/dataset/36_pol_drvlic.zip", + "ftp://smartengines.com/midv-500/dataset/37_prt_id.zip", + "ftp://smartengines.com/midv-500/dataset/38_rou_drvlic.zip", + "ftp://smartengines.com/midv-500/dataset/39_rus_internalpassport.zip", + "ftp://smartengines.com/midv-500/dataset/40_srb_id.zip", + "ftp://smartengines.com/midv-500/dataset/41_srb_passport.zip", + "ftp://smartengines.com/midv-500/dataset/42_svk_id.zip", + "ftp://smartengines.com/midv-500/dataset/43_tur_id.zip", + "ftp://smartengines.com/midv-500/dataset/44_ukr_id.zip", + "ftp://smartengines.com/midv-500/dataset/45_ukr_passport.zip", + "ftp://smartengines.com/midv-500/dataset/46_ury_passport.zip", + "ftp://smartengines.com/midv-500/dataset/47_usa_bordercrossing.zip", + "ftp://smartengines.com/midv-500/dataset/48_usa_passportcard.zip", + "ftp://smartengines.com/midv-500/dataset/49_usa_ssn82.zip", + "ftp://smartengines.com/midv-500/dataset/50_xpo_id.zip", +] def download_dataset(download_dir: str): @@ -60,27 +62,31 @@ def download_dataset(download_dir: str): """ for link in all_links: - print('--------------------------------------------------------------') + print("--------------------------------------------------------------") # doownlaod zip file - print('\nDownloading:', link[40:]) + print("\nDownloading:", link[40:]) download(link, download_dir) - print('Downloaded:', link[40:]) + print("Downloaded:", link[40:]) # unzip zip file - print('Unzipping:', link[40:]) + print("Unzipping:", link[40:]) zip_path = os.path.join(download_dir, link[40:]) unzip(zip_path, download_dir) - print('Unzipped:', link[40:].replace('.zip', '')) + print("Unzipped:", link[40:].replace(".zip", "")) # remove zip file os.remove(zip_path) -if __name__ == '__main__': +if __name__ == "__main__": # construct the argument parser ap = argparse.ArgumentParser() # add the arguments to the parser - ap.add_argument("download_dir", default="data/", help="Directory for MIDV-500 dataset to be downloaded.") + ap.add_argument( + "download_dir", + default="data/", + help="Directory for MIDV-500 dataset to be downloaded.", + ) args = vars(ap.parse_args()) # download dataset - download_dataset(args['download_dir']) + download_dataset(args["download_dir"]) diff --git a/midv500/utils.py b/midv500/utils.py index 2c7a886..4b58314 100644 --- a/midv500/utils.py +++ b/midv500/utils.py @@ -36,7 +36,9 @@ def get_bbox_inside_image(label_bbox: list, image_bbox: list) -> list: return corrected_label_bbox -def list_annotation_paths_recursively(directory: str, ignore_background_only_ones: bool = True) -> list: +def list_annotation_paths_recursively( + directory: str, ignore_background_only_ones: bool = True +) -> list: """ Accepts a folder directory containing image files. Returns a list of image file paths present in given directory. @@ -66,9 +68,9 @@ def list_annotation_paths_recursively(directory: str, ignore_background_only_one else: try: # load poly - with open(abs_filepath, 'r') as json_file: + with open(abs_filepath, "r") as json_file: quad = json.load(json_file) - coords = quad['quad'] + coords = quad["quad"] except: # fix for 29_irn_drvlic.json continue @@ -87,8 +89,14 @@ def list_annotation_paths_recursively(directory: str, ignore_background_only_one continue abs_filepath = abs_filepath.replace("\\", "/") # for windows - relative_filepath = abs_filepath.split(directory)[-1] # get relative path from abs path - relative_filepath = [relative_filepath[1:] if relative_filepath[0] == "/" else relative_filepath][0] + relative_filepath = abs_filepath.split(directory)[ + -1 + ] # get relative path from abs path + relative_filepath = [ + relative_filepath[1:] + if relative_filepath[0] == "/" + else relative_filepath + ][0] relative_filepath_list.append(relative_filepath) number_of_files = len(relative_filepath_list) @@ -111,6 +119,7 @@ class TqdmUpTo(tqdm): Provides `update_to(n)` which uses `tqdm.update(delta_n)`. https://pypi.org/project/tqdm/#hooks-and-callbacks """ + def update_to(self, b=1, bsize=1, tsize=None): """ b : int, optional @@ -137,10 +146,15 @@ def download(url: str, save_dir: str): # create save_dir if not present create_dir(save_dir) # download file - with TqdmUpTo(unit='B', unit_scale=True, miniters=1, - desc=url.split('/')[-1]) as t: # all optional kwargs - urlretrieve(url, filename=os.path.join(save_dir, url.split('/')[-1]), - reporthook=t.update_to, data=None) + with TqdmUpTo( + unit="B", unit_scale=True, miniters=1, desc=url.split("/")[-1] + ) as t: # all optional kwargs + urlretrieve( + url, + filename=os.path.join(save_dir, url.split("/")[-1]), + reporthook=t.update_to, + data=None, + ) def unzip(file_path: str, dest_dir: str): diff --git a/setup.cfg b/setup.cfg index 825916b..bb72dab 100644 --- a/setup.cfg +++ b/setup.cfg @@ -1,6 +1,6 @@ [flake8] max-line-length = 119 -exclude =.git,__pycache__,docs/source/conf.py,build,dist +exclude =.git,.github,.vscode,__pycache__,build,dist ignore = I101,I201,F401,F403,S001,D100,D101,D102,D103,D104,D105,D106,D107,D200,D205,D400,W504,D202,E203,E722,W503,B006 inline-quotes = " diff --git a/setup.py b/setup.py index ffa8440..698aeec 100644 --- a/setup.py +++ b/setup.py @@ -11,19 +11,15 @@ def get_long_description(): def get_requirements(): - with open('requirements.txt') as f: + with open("requirements.txt") as f: return f.read().splitlines() def get_version(): current_dir = os.path.abspath(os.path.dirname(__file__)) - version_file = os.path.join(current_dir, - "midv500", - "__init__.py") + version_file = os.path.join(current_dir, "midv500", "__init__.py") with io.open(version_file, encoding="utf-8") as f: - return re.search(r'^__version__ = [\'"]([^\'"]*)[\'"]', - f.read(), - re.M).group(1) + return re.search(r'^__version__ = [\'"]([^\'"]*)[\'"]', f.read(), re.M).group(1) setuptools.setup( @@ -42,5 +38,5 @@ def get_version(): "Operating System :: OS Independent", ], install_requires=get_requirements(), - python_requires='>=3.5', + python_requires=">=3.5", )