Skip to content

Commit

Permalink
Refactor benchmark
Browse files Browse the repository at this point in the history
  • Loading branch information
@Dipet
Dipet committed Mar 24, 2024
1 parent e68a597 commit bec1c54
Show file tree
Hide file tree
Showing 4 changed files with 61 additions and 245 deletions.
113 changes: 8 additions & 105 deletions benchmark/benchmark_bbox.py
View file
Original file line number Diff line number Diff line change
@@ -1,9 +1,6 @@
from __future__ import division, print_function

import argparse
import math
import os
import sys
from abc import ABC
from collections import defaultdict
from timeit import Timer
Expand All @@ -12,24 +9,15 @@
import cv2
import numpy as np
import pandas as pd
import importlib.metadata
from imgaug import augmenters as iaa
from imgaug.augmentables.bbs import BoundingBox, BoundingBoxesOnImage
from pytablewriter import MarkdownTableWriter
from pytablewriter.style import Style
from tqdm import tqdm

import albumentations as A

cv2.setNumThreads(0)
cv2.ocl.setUseOpenCL(False)

os.environ["OMP_NUM_THREADS"] = "1"
os.environ["OPENBLAS_NUM_THREADS"] = "1"
os.environ["MKL_NUM_THREADS"] = "1"
os.environ["VECLIB_MAXIMUM_THREADS"] = "1"
os.environ["NUMEXPR_NUM_THREADS"] = "1"
from benchmark.utils import set_bench_env_vars, get_package_versions, MarkdownGenerator, get_image, format_results

set_bench_env_vars()

DEFAULT_BENCHMARKING_LIBRARIES = [
"imgaug",
Expand Down Expand Up @@ -65,95 +53,10 @@ def parse_args():
return parser.parse_args()


def get_package_versions():
packages = [
"albumentations",
"imgaug",
"numpy",
"opencv-python",
]
package_versions = {"Python": sys.version}
for package in packages:
try:
package_versions[package] = importlib.metadata.distribution(package).version
except importlib.metadata.PackageNotFoundError:
pass
return package_versions


class MarkdownGenerator:
def __init__(self, df, package_versions):
self._df = df
self._package_versions = package_versions
self._libraries_description = {"torchvision": "(Pillow-SIMD backend)"}

def _highlight_best_result(self, results):
best_result = float("-inf")
for result in results:
try:
result = int(result)
except ValueError:
continue
if result > best_result:
best_result = result
return ["**{}**".format(r) if r == str(best_result) else r for r in results]

def _make_headers(self):
libraries = self._df.columns.to_list()
columns = []
for library in libraries:
version = self._package_versions[library]
library_description = self._libraries_description.get(library)
if library_description:
library += " {}".format(library_description)

columns.append("{library}<br><small>{version}</small>".format(library=library, version=version))
return [""] + columns

def _make_value_matrix(self):
index = self._df.index.tolist()
values = self._df.values.tolist()
value_matrix = []
for transform, results in zip(index, values):
row = [transform] + self._highlight_best_result(results)
value_matrix.append(row)
return value_matrix

def _make_versions_text(self):
libraries = ["Python", "numpy", "opencv-python"]
libraries_with_versions = [
"{library} {version}".format(library=library, version=self._package_versions[library].replace("\n", ""))
for library in libraries
]
return "Python and library versions: {}.".format(", ".join(libraries_with_versions))

def print(self):
writer = MarkdownTableWriter()
writer.headers = self._make_headers()
writer.value_matrix = self._make_value_matrix()
writer.styles = [Style(align="left")] + [Style(align="center") for _ in range(len(writer.headers) - 1)]
writer.write_table()
print("\n" + self._make_versions_text())


def read_img_cv2(img_size=(512, 512, 3)):
img = np.zeros(shape=img_size, dtype=np.uint8)
return img


def generate_random_bboxes(bbox_nums: int = 1):
return np.sort(np.random.random(size=(bbox_nums, 4)))


def format_results(seconds_per_image_for_aug, show_std=False):
if seconds_per_image_for_aug is None:
return "-"
result = str(np.round(np.mean(seconds_per_image_for_aug), 3))
if show_std:
result += " ± {}".format(math.ceil(np.std(seconds_per_image_for_aug)))
return result


class BenchmarkTest(ABC):
def __str__(self):
return self.__class__.__name__
Expand Down Expand Up @@ -410,7 +313,7 @@ def main():
package_versions = get_package_versions()
if args.print_package_versions:
print(package_versions)
seconds_per_image = defaultdict(dict)
images_per_second = defaultdict(dict)
libraries = args.libraries

benchmarks = [
Expand All @@ -434,7 +337,7 @@ def main():
PiecewiseAffine(), # very slow
Sequence(),
]
imgs = [read_img_cv2(img_size=(100, 100, 3)) for _ in range(args.images)]
imgs = [get_image(img_size=(100, 100, 3)) for _ in range(args.images)]
bboxes = [generate_random_bboxes(args.bboxes) for _ in range(args.images)]

for library in libraries:
Expand All @@ -443,16 +346,16 @@ def main():
pbar = tqdm(total=len(benchmarks))
for benchmark in benchmarks:
pbar.set_description("Current benchmark: {} | {}".format(library, benchmark))
benchmark_second_per_image = None
benchmark_images_per_second = None
if benchmark.is_supported_by(library):
timer = Timer(lambda: benchmark.run(library, imgs, bboxes=bboxes, class_ids=class_ids))
run_times = timer.repeat(number=1, repeat=args.runs)
benchmark_second_per_image = [run_time * 1000 / args.images for run_time in run_times]
seconds_per_image[library][str(benchmark)] = benchmark_second_per_image
benchmark_images_per_second = [1 / (run_time / args.images) for run_time in run_times]
images_per_second[library][str(benchmark)] = benchmark_images_per_second
pbar.update(1)
pbar.close()
pd.set_option("display.width", 1000)
df = pd.DataFrame.from_dict(seconds_per_image)
df = pd.DataFrame.from_dict(images_per_second)
df = df.applymap(lambda r: format_results(r, args.show_std))
df = df[libraries]
augmentations = [str(i) for i in benchmarks]
Expand Down
114 changes: 8 additions & 106 deletions benchmark/benchmark_keypoints.py
View file
Original file line number Diff line number Diff line change
@@ -1,10 +1,6 @@
from __future__ import division, print_function

import argparse
import math
import os
import random
import sys
from abc import ABC
from collections import defaultdict
from timeit import Timer
Expand All @@ -13,24 +9,15 @@
import cv2
import numpy as np
import pandas as pd
import pkg_resources
from imgaug import augmenters as iaa
from imgaug.augmentables.kps import Keypoint, KeypointsOnImage
from pytablewriter import MarkdownTableWriter
from pytablewriter.style import Style
from tqdm import tqdm

import albumentations as A

cv2.setNumThreads(0)
cv2.ocl.setUseOpenCL(False)

os.environ["OMP_NUM_THREADS"] = "1"
os.environ["OPENBLAS_NUM_THREADS"] = "1"
os.environ["MKL_NUM_THREADS"] = "1"
os.environ["VECLIB_MAXIMUM_THREADS"] = "1"
os.environ["NUMEXPR_NUM_THREADS"] = "1"
from benchmark.utils import set_bench_env_vars, get_package_versions, MarkdownGenerator, get_image, format_results

set_bench_env_vars()

DEFAULT_BENCHMARKING_LIBRARIES = [
"imgaug",
Expand Down Expand Up @@ -66,97 +53,12 @@ def parse_args():
return parser.parse_args()


def get_package_versions():
packages = [
"albumentations",
"imgaug",
"numpy",
"opencv-python",
]
package_versions = {"Python": sys.version}
for package in packages:
try:
package_versions[package] = pkg_resources.get_distribution(package).version
except pkg_resources.DistributionNotFound:
pass
return package_versions


class MarkdownGenerator:
def __init__(self, df, package_versions):
self._df = df
self._package_versions = package_versions
self._libraries_description = {"torchvision": "(Pillow-SIMD backend)"}

def _highlight_best_result(self, results):
best_result = float("-inf")
for result in results:
try:
result = int(result)
except ValueError:
continue
if result > best_result:
best_result = result
return ["**{}**".format(r) if r == str(best_result) else r for r in results]

def _make_headers(self):
libraries = self._df.columns.to_list()
columns = []
for library in libraries:
version = self._package_versions[library]
library_description = self._libraries_description.get(library)
if library_description:
library += " {}".format(library_description)

columns.append("{library}<br><small>{version}</small>".format(library=library, version=version))
return [""] + columns

def _make_value_matrix(self):
index = self._df.index.tolist()
values = self._df.values.tolist()
value_matrix = []
for transform, results in zip(index, values):
row = [transform] + self._highlight_best_result(results)
value_matrix.append(row)
return value_matrix

def _make_versions_text(self):
libraries = ["Python", "numpy", "opencv-python"]
libraries_with_versions = [
"{library} {version}".format(library=library, version=self._package_versions[library].replace("\n", ""))
for library in libraries
]
return "Python and library versions: {}.".format(", ".join(libraries_with_versions))

def print(self):
writer = MarkdownTableWriter()
writer.headers = self._make_headers()
writer.value_matrix = self._make_value_matrix()
writer.styles = [Style(align="left")] + [Style(align="center") for _ in range(len(writer.headers) - 1)]
writer.write_table()
print("\n" + self._make_versions_text())


def read_img_cv2(img_size=(512, 512, 3)):
img = np.zeros(shape=img_size, dtype=np.uint8)
return img


def generate_random_keypoints(points_num: int = 1, w: int = 512, h: int = 512):
xs = np.random.randint(0, w - 1, points_num)
ys = np.random.randint(0, h - 1, points_num)
return np.pad(np.stack([xs, ys], axis=1), [(0, 0), (0, 2)])


def format_results(seconds_per_image_for_aug, show_std=False):
if seconds_per_image_for_aug is None:
return "-"
result = str(np.round(np.mean(seconds_per_image_for_aug), 3))
if show_std:
result += " ± {}".format(math.ceil(np.std(seconds_per_image_for_aug)))
return result


class BenchmarkTest(ABC):
def __str__(self):
return self.__class__.__name__
Expand Down Expand Up @@ -399,7 +301,7 @@ def main():
package_versions = get_package_versions()
if args.print_package_versions:
print(package_versions)
seconds_per_image = defaultdict(dict)
images_per_second = defaultdict(dict)
libraries = args.libraries

benchmarks = [
Expand All @@ -425,7 +327,7 @@ def main():

w, h = 100, 100

imgs = [read_img_cv2(img_size=(w, h, 3)) for _ in range(args.images)]
imgs = [get_image(img_size=(w, h, 3)) for _ in range(args.images)]
batch_keypoints = [generate_random_keypoints(args.keypoints, w=w, h=h) for _ in range(args.images)]

for library in libraries:
Expand All @@ -434,16 +336,16 @@ def main():
pbar = tqdm(total=len(benchmarks))
for benchmark in benchmarks:
pbar.set_description("Current benchmark: {} | {}".format(library, benchmark))
benchmark_second_per_image = None
benchmark_images_per_second = None
if benchmark.is_supported_by(library):
timer = Timer(lambda: benchmark.run(library, imgs, keypoints=batch_keypoints, class_ids=class_ids))
run_times = timer.repeat(number=1, repeat=args.runs)
benchmark_second_per_image = [run_time * 1000 / args.images for run_time in run_times]
seconds_per_image[library][str(benchmark)] = benchmark_second_per_image
benchmark_second_per_image = [1 / (run_time / args.images) for run_time in run_times]
images_per_second[library][str(benchmark)] = benchmark_images_per_second
pbar.update(1)
pbar.close()
pd.set_option("display.width", 1000)
df = pd.DataFrame.from_dict(seconds_per_image)
df = pd.DataFrame.from_dict(images_per_second)
df = df.applymap(lambda r: format_results(r, args.show_std))
df = df[libraries]
augmentations = [str(i) for i in benchmarks]
Expand Down
Loading

0 comments on commit bec1c54

Please sign in to comment.