diff --git a/benchmark/benchmark_bbox.py b/benchmark/benchmark_bbox.py index 02d843f7a..c2b517426 100644 --- a/benchmark/benchmark_bbox.py +++ b/benchmark/benchmark_bbox.py @@ -1,9 +1,6 @@ from __future__ import division, print_function import argparse -import math -import os -import sys from abc import ABC from collections import defaultdict from timeit import Timer @@ -12,24 +9,15 @@ import cv2 import numpy as np import pandas as pd -import importlib.metadata from imgaug import augmenters as iaa from imgaug.augmentables.bbs import BoundingBox, BoundingBoxesOnImage -from pytablewriter import MarkdownTableWriter -from pytablewriter.style import Style from tqdm import tqdm import albumentations as A -cv2.setNumThreads(0) -cv2.ocl.setUseOpenCL(False) - -os.environ["OMP_NUM_THREADS"] = "1" -os.environ["OPENBLAS_NUM_THREADS"] = "1" -os.environ["MKL_NUM_THREADS"] = "1" -os.environ["VECLIB_MAXIMUM_THREADS"] = "1" -os.environ["NUMEXPR_NUM_THREADS"] = "1" +from benchmark.utils import set_bench_env_vars, get_package_versions, MarkdownGenerator, get_image, format_results +set_bench_env_vars() DEFAULT_BENCHMARKING_LIBRARIES = [ "imgaug", @@ -65,95 +53,10 @@ def parse_args(): return parser.parse_args() -def get_package_versions(): - packages = [ - "albumentations", - "imgaug", - "numpy", - "opencv-python", - ] - package_versions = {"Python": sys.version} - for package in packages: - try: - package_versions[package] = importlib.metadata.distribution(package).version - except importlib.metadata.PackageNotFoundError: - pass - return package_versions - - -class MarkdownGenerator: - def __init__(self, df, package_versions): - self._df = df - self._package_versions = package_versions - self._libraries_description = {"torchvision": "(Pillow-SIMD backend)"} - - def _highlight_best_result(self, results): - best_result = float("-inf") - for result in results: - try: - result = int(result) - except ValueError: - continue - if result > best_result: - best_result = result - return ["**{}**".format(r) if r == str(best_result) else r for r in results] - - def _make_headers(self): - libraries = self._df.columns.to_list() - columns = [] - for library in libraries: - version = self._package_versions[library] - library_description = self._libraries_description.get(library) - if library_description: - library += " {}".format(library_description) - - columns.append("{library}
{version}".format(library=library, version=version)) - return [""] + columns - - def _make_value_matrix(self): - index = self._df.index.tolist() - values = self._df.values.tolist() - value_matrix = [] - for transform, results in zip(index, values): - row = [transform] + self._highlight_best_result(results) - value_matrix.append(row) - return value_matrix - - def _make_versions_text(self): - libraries = ["Python", "numpy", "opencv-python"] - libraries_with_versions = [ - "{library} {version}".format(library=library, version=self._package_versions[library].replace("\n", "")) - for library in libraries - ] - return "Python and library versions: {}.".format(", ".join(libraries_with_versions)) - - def print(self): - writer = MarkdownTableWriter() - writer.headers = self._make_headers() - writer.value_matrix = self._make_value_matrix() - writer.styles = [Style(align="left")] + [Style(align="center") for _ in range(len(writer.headers) - 1)] - writer.write_table() - print("\n" + self._make_versions_text()) - - -def read_img_cv2(img_size=(512, 512, 3)): - img = np.zeros(shape=img_size, dtype=np.uint8) - return img - - def generate_random_bboxes(bbox_nums: int = 1): return np.sort(np.random.random(size=(bbox_nums, 4))) -def format_results(seconds_per_image_for_aug, show_std=False): - if seconds_per_image_for_aug is None: - return "-" - result = str(np.round(np.mean(seconds_per_image_for_aug), 3)) - if show_std: - result += " ± {}".format(math.ceil(np.std(seconds_per_image_for_aug))) - return result - - class BenchmarkTest(ABC): def __str__(self): return self.__class__.__name__ @@ -410,7 +313,7 @@ def main(): package_versions = get_package_versions() if args.print_package_versions: print(package_versions) - seconds_per_image = defaultdict(dict) + images_per_second = defaultdict(dict) libraries = args.libraries benchmarks = [ @@ -434,7 +337,7 @@ def main(): PiecewiseAffine(), # very slow Sequence(), ] - imgs = [read_img_cv2(img_size=(100, 100, 3)) for _ in range(args.images)] + imgs = [get_image(img_size=(100, 100, 3)) for _ in range(args.images)] bboxes = [generate_random_bboxes(args.bboxes) for _ in range(args.images)] for library in libraries: @@ -443,16 +346,16 @@ def main(): pbar = tqdm(total=len(benchmarks)) for benchmark in benchmarks: pbar.set_description("Current benchmark: {} | {}".format(library, benchmark)) - benchmark_second_per_image = None + benchmark_images_per_second = None if benchmark.is_supported_by(library): timer = Timer(lambda: benchmark.run(library, imgs, bboxes=bboxes, class_ids=class_ids)) run_times = timer.repeat(number=1, repeat=args.runs) - benchmark_second_per_image = [run_time * 1000 / args.images for run_time in run_times] - seconds_per_image[library][str(benchmark)] = benchmark_second_per_image + benchmark_images_per_second = [1 / (run_time / args.images) for run_time in run_times] + images_per_second[library][str(benchmark)] = benchmark_images_per_second pbar.update(1) pbar.close() pd.set_option("display.width", 1000) - df = pd.DataFrame.from_dict(seconds_per_image) + df = pd.DataFrame.from_dict(images_per_second) df = df.applymap(lambda r: format_results(r, args.show_std)) df = df[libraries] augmentations = [str(i) for i in benchmarks] diff --git a/benchmark/benchmark_keypoints.py b/benchmark/benchmark_keypoints.py index d75a2841b..6e8e01dbb 100644 --- a/benchmark/benchmark_keypoints.py +++ b/benchmark/benchmark_keypoints.py @@ -1,10 +1,6 @@ from __future__ import division, print_function import argparse -import math -import os -import random -import sys from abc import ABC from collections import defaultdict from timeit import Timer @@ -13,24 +9,15 @@ import cv2 import numpy as np import pandas as pd -import pkg_resources from imgaug import augmenters as iaa from imgaug.augmentables.kps import Keypoint, KeypointsOnImage -from pytablewriter import MarkdownTableWriter -from pytablewriter.style import Style from tqdm import tqdm import albumentations as A -cv2.setNumThreads(0) -cv2.ocl.setUseOpenCL(False) - -os.environ["OMP_NUM_THREADS"] = "1" -os.environ["OPENBLAS_NUM_THREADS"] = "1" -os.environ["MKL_NUM_THREADS"] = "1" -os.environ["VECLIB_MAXIMUM_THREADS"] = "1" -os.environ["NUMEXPR_NUM_THREADS"] = "1" +from benchmark.utils import set_bench_env_vars, get_package_versions, MarkdownGenerator, get_image, format_results +set_bench_env_vars() DEFAULT_BENCHMARKING_LIBRARIES = [ "imgaug", @@ -66,97 +53,12 @@ def parse_args(): return parser.parse_args() -def get_package_versions(): - packages = [ - "albumentations", - "imgaug", - "numpy", - "opencv-python", - ] - package_versions = {"Python": sys.version} - for package in packages: - try: - package_versions[package] = pkg_resources.get_distribution(package).version - except pkg_resources.DistributionNotFound: - pass - return package_versions - - -class MarkdownGenerator: - def __init__(self, df, package_versions): - self._df = df - self._package_versions = package_versions - self._libraries_description = {"torchvision": "(Pillow-SIMD backend)"} - - def _highlight_best_result(self, results): - best_result = float("-inf") - for result in results: - try: - result = int(result) - except ValueError: - continue - if result > best_result: - best_result = result - return ["**{}**".format(r) if r == str(best_result) else r for r in results] - - def _make_headers(self): - libraries = self._df.columns.to_list() - columns = [] - for library in libraries: - version = self._package_versions[library] - library_description = self._libraries_description.get(library) - if library_description: - library += " {}".format(library_description) - - columns.append("{library}
{version}".format(library=library, version=version)) - return [""] + columns - - def _make_value_matrix(self): - index = self._df.index.tolist() - values = self._df.values.tolist() - value_matrix = [] - for transform, results in zip(index, values): - row = [transform] + self._highlight_best_result(results) - value_matrix.append(row) - return value_matrix - - def _make_versions_text(self): - libraries = ["Python", "numpy", "opencv-python"] - libraries_with_versions = [ - "{library} {version}".format(library=library, version=self._package_versions[library].replace("\n", "")) - for library in libraries - ] - return "Python and library versions: {}.".format(", ".join(libraries_with_versions)) - - def print(self): - writer = MarkdownTableWriter() - writer.headers = self._make_headers() - writer.value_matrix = self._make_value_matrix() - writer.styles = [Style(align="left")] + [Style(align="center") for _ in range(len(writer.headers) - 1)] - writer.write_table() - print("\n" + self._make_versions_text()) - - -def read_img_cv2(img_size=(512, 512, 3)): - img = np.zeros(shape=img_size, dtype=np.uint8) - return img - - def generate_random_keypoints(points_num: int = 1, w: int = 512, h: int = 512): xs = np.random.randint(0, w - 1, points_num) ys = np.random.randint(0, h - 1, points_num) return np.pad(np.stack([xs, ys], axis=1), [(0, 0), (0, 2)]) -def format_results(seconds_per_image_for_aug, show_std=False): - if seconds_per_image_for_aug is None: - return "-" - result = str(np.round(np.mean(seconds_per_image_for_aug), 3)) - if show_std: - result += " ± {}".format(math.ceil(np.std(seconds_per_image_for_aug))) - return result - - class BenchmarkTest(ABC): def __str__(self): return self.__class__.__name__ @@ -399,7 +301,7 @@ def main(): package_versions = get_package_versions() if args.print_package_versions: print(package_versions) - seconds_per_image = defaultdict(dict) + images_per_second = defaultdict(dict) libraries = args.libraries benchmarks = [ @@ -425,7 +327,7 @@ def main(): w, h = 100, 100 - imgs = [read_img_cv2(img_size=(w, h, 3)) for _ in range(args.images)] + imgs = [get_image(img_size=(w, h, 3)) for _ in range(args.images)] batch_keypoints = [generate_random_keypoints(args.keypoints, w=w, h=h) for _ in range(args.images)] for library in libraries: @@ -434,16 +336,16 @@ def main(): pbar = tqdm(total=len(benchmarks)) for benchmark in benchmarks: pbar.set_description("Current benchmark: {} | {}".format(library, benchmark)) - benchmark_second_per_image = None + benchmark_images_per_second = None if benchmark.is_supported_by(library): timer = Timer(lambda: benchmark.run(library, imgs, keypoints=batch_keypoints, class_ids=class_ids)) run_times = timer.repeat(number=1, repeat=args.runs) - benchmark_second_per_image = [run_time * 1000 / args.images for run_time in run_times] - seconds_per_image[library][str(benchmark)] = benchmark_second_per_image + benchmark_second_per_image = [1 / (run_time / args.images) for run_time in run_times] + images_per_second[library][str(benchmark)] = benchmark_images_per_second pbar.update(1) pbar.close() pd.set_option("display.width", 1000) - df = pd.DataFrame.from_dict(seconds_per_image) + df = pd.DataFrame.from_dict(images_per_second) df = df.applymap(lambda r: format_results(r, args.show_std)) df = df[libraries] augmentations = [str(i) for i in benchmarks] diff --git a/benchmark/image_benchmark.py b/benchmark/image_benchmark.py index 16985efd0..0a62a63fd 100644 --- a/benchmark/image_benchmark.py +++ b/benchmark/image_benchmark.py @@ -2,9 +2,7 @@ import copy import os import random -import sys from collections import defaultdict -from contextlib import suppress from pathlib import Path from timeit import Timer from typing import Any, Dict, List, Union @@ -15,7 +13,6 @@ import kornia.augmentation as Kaug import numpy as np import pandas as pd -import pkg_resources import torch from imgaug import augmenters as iaa from PIL import Image @@ -31,18 +28,11 @@ read_img_kornia, read_img_pillow, read_img_torch, + set_bench_env_vars, + get_package_versions, ) -cv2.setNumThreads(0) -cv2.ocl.setUseOpenCL(False) - -torch.set_num_threads(1) - -os.environ["OMP_NUM_THREADS"] = "1" -os.environ["OPENBLAS_NUM_THREADS"] = "1" -os.environ["MKL_NUM_THREADS"] = "1" -os.environ["VECLIB_MAXIMUM_THREADS"] = "1" -os.environ["NUMEXPR_NUM_THREADS"] = "1" +set_bench_env_vars() DEFAULT_BENCHMARKING_LIBRARIES = ["albumentations", "torchvision", "kornia", "augly", "imgaug"] @@ -70,26 +60,6 @@ def parse_args() -> argparse.Namespace: return parser.parse_args() -def get_package_versions() -> Dict[str, str]: - packages = [ - "albumentations", - "imgaug", - "torchvision", - "numpy", - "opencv-python-headless", - "scikit-image", - "scipy", - "pillow", - "kornia", - "augly", - ] - package_versions = {"Python": sys.version} - for package in packages: - with suppress(pkg_resources.DistributionNotFound): - package_versions[package] = pkg_resources.get_distribution(package).version - return package_versions - - class BenchmarkTest: def __str__(self) -> str: return self.__class__.__name__ diff --git a/benchmark/utils.py b/benchmark/utils.py index 4cb3f4ecb..49202b6b9 100644 --- a/benchmark/utils.py +++ b/benchmark/utils.py @@ -1,7 +1,11 @@ import math +import sys +import pkg_resources from pathlib import Path -from typing import Dict, List, Optional +from typing import Dict, List, Optional, Sequence +from contextlib import suppress +import os import cv2 import kornia as K import numpy as np @@ -132,3 +136,40 @@ def get_markdown_table(data: dict[str, str]) -> str: markdown_table += f"| {key} | {value} |\n" return markdown_table + + +def set_bench_env_vars() -> None: + cv2.setNumThreads(0) + cv2.ocl.setUseOpenCL(False) + + torch.set_num_threads(1) + + os.environ["OMP_NUM_THREADS"] = "1" + os.environ["OPENBLAS_NUM_THREADS"] = "1" + os.environ["MKL_NUM_THREADS"] = "1" + os.environ["VECLIB_MAXIMUM_THREADS"] = "1" + os.environ["NUMEXPR_NUM_THREADS"] = "1" + + +def get_package_versions() -> Dict[str, str]: + packages = [ + "albumentations", + "imgaug", + "torchvision", + "numpy", + "opencv-python-headless", + "scikit-image", + "scipy", + "pillow", + "kornia", + "augly", + ] + package_versions = {"Python": sys.version} + for package in packages: + with suppress(pkg_resources.DistributionNotFound): + package_versions[package] = pkg_resources.get_distribution(package).version + return package_versions + + +def get_image(img_size: Sequence[int]) -> np.ndarray: + return np.empty([100, 100, 3], dtype=np.uint8)