feat: add db benchmark (#9)

* refactor: separate benchmark command

* add db benchmark command

bench_wizard/__version__.py

@@ -1 +1 @@
-__version__ = "0.4.2"
+__version__ = "0.5.0"

bench_wizard/benchmark.py

@@ -1,53 +1,51 @@
-import json
+from __future__ import annotations
+
 import os
 import subprocess
-from typing import List
+from dataclasses import dataclass
+from typing import List, Optional
 
 from bench_wizard.cargo import Cargo
-from bench_wizard.config import Config
 from bench_wizard.exceptions import BenchmarkCargoException
 from bench_wizard.output import Output
 
-from bench_wizard.parser import BenchmarkParser
 
-# TODO: need as configurable option
-DIFF_MARGIN = 10  # percent
+@dataclass
+class BenchmarksConfig:
+    pallets: [str]
+    dump_results: Optional[str] = None
+    output_dir: Optional[str] = None
+    template: Optional[str] = None
 
 
 class Benchmark:
-    """ Represents single benchmark"""
+    """Represents single benchmark"""
 
-    def __init__(self, pallet: str, command: [str], ref_value: float, extrinsics: list):
+    def __init__(self, pallet: str, command: [str]):
         self._pallet = pallet
         self._stdout = None
         self._command = command
-        self._ref_value = ref_value
-        self._extrinsics = extrinsics
-
-        self._extrinsics_results = []
-
         self._total_time = 0
 
         self._completed = False
         self._acceptable = False
         self._rerun = False
 
+        self._error = False
+        self._error_reason = None
+
     @property
     def pallet(self):
         return self._pallet
 
-    @property
-    def acceptable(self) -> bool:
-        return self._acceptable
-
-    @acceptable.setter
-    def acceptable(self, value: bool):
-        self._acceptable = value
-
     @property
     def completed(self) -> bool:
         return self._completed
 
+    @property
+    def is_error(self) -> bool:
+        return self._error
+
     @property
     def raw(self) -> bytes:
         return self._stdout
@@ -57,19 +55,11 @@ def run(self, rerun: bool = False) -> None:
         result = subprocess.run(self._command, capture_output=True)
 
         if result.returncode != 0:
-            raise BenchmarkCargoException(result.stderr.decode("utf-8"))
+            self._error = True
+            self._error_reason = result.stderr.decode("utf-8")
+            return
 
         self._stdout = result.stdout
-
-        parser = BenchmarkParser(result.stdout)
-
-        self._total_time = parser.total_time(self._extrinsics)
-
-        margin = int(self._ref_value * DIFF_MARGIN / 100)
-
-        diff = int(self._ref_value - self._total_time)
-
-        self.acceptable = diff >= -margin
         self._rerun = rerun
         self._completed = True
 
@@ -78,60 +68,36 @@ def dump(self, dest: str) -> None:
         with open(os.path.join(dest, f"{self._pallet}.results"), "wb") as f:
             f.write(self._stdout)
 
-    @property
-    def ref_value(self):
-        return self._ref_value
-
-    @property
-    def total_time(self):
-        return self._total_time
-
     @property
     def rerun(self):
         return self._rerun
 
-    @property
-    def percentage(self) -> float:
-        diff = int(self._ref_value - self._total_time)
-
-        percentage = (diff / self._ref_value) * 100
-
-        return percentage
 
-
-def _prepare_benchmarks(config: Config, reference_values: dict) -> List[Benchmark]:
+def _prepare_benchmarks(config: BenchmarksConfig) -> List[Benchmark]:
     benchmarks = []
 
     for pallet in config.pallets:
         cargo = Cargo(pallet=pallet, template=config.template)
+
         if config.output_dir:
             output_file = os.path.join(config.output_dir, f"{pallet}.rs")
             cargo.output = output_file
 
-        ref_data = reference_values[pallet]
-        ref_value = sum(list(map(lambda x: float(x), ref_data.values())))
-        benchmarks.append(
-            Benchmark(pallet, cargo.command(), ref_value, ref_data.keys())
-        )
+        benchmarks.append(Benchmark(pallet, cargo.command()))
 
     return benchmarks
 
 
-def _run_benchmarks(benchmarks: List[Benchmark], output: Output, rerun=False) -> None:
-    # Note : this can be simplified into one statement
-
-    if rerun:
-        [bench.run(rerun) for bench in benchmarks if bench.acceptable is False]
-    else:
-        output.track(benchmarks)
-        for bench in benchmarks:
-            # Output updates to easily show progress
-            output.update(bench)
-            bench.run()
-            output.update(bench)
+def _run_benchmarks(benchmarks: List[Benchmark], output: Output) -> None:
+    output.track(benchmarks)
+    for bench in benchmarks:
+        # Output updates to easily show progress
+        output.update(bench)
+        bench.run()
+        output.update(bench)
 
 
-def _build(manifest: str) -> None:
+def _build_with_runtime_features(manifest: str) -> None:
     command = [
         "cargo",
         "build",
@@ -146,38 +112,24 @@ def _build(manifest: str) -> None:
         raise BenchmarkCargoException(result.stderr.decode("utf-8"))
 
 
-def run_pallet_benchmarks(config: Config, to_output: Output) -> None:
-    if not config.do_pallet_bench:
-        return
-
-    to_output.info("Substrate Node Performance check ... ")
-
-    if config.do_pallet_bench:
-        with open(config.reference_values, "r") as f:
-            s = json.load(f)
+def run_pallet_benchmarks(config: BenchmarksConfig, to_output: Output) -> None:
+    benchmarks = _prepare_benchmarks(config)
+    pallets = []
+    for bench in benchmarks:
+        pallets.append(bench.pallet)
 
-        benchmarks = _prepare_benchmarks(config, s)
+    to_output.info(f"Benchmarking: {pallets}")
 
-        to_output.info("Compiling - this may take a while...")
+    to_output.info("Compiling - this may take a while...")
 
-        _build("node/Cargo.toml")
+    _build_with_runtime_features("node/Cargo.toml")
 
-        to_output.info("Running benchmarks - this may take a while...")
+    to_output.info("Running benchmarks - this may take a while...")
 
-        _run_benchmarks(benchmarks, to_output)
+    _run_benchmarks(benchmarks, to_output)
 
-        if [b.acceptable for b in benchmarks].count(False) == 1:
-            # if only one failed - rerun it
-            _run_benchmarks(benchmarks, to_output, True)
-
-        to_output.results(benchmarks)
+    to_output.results(benchmarks)
 
+    if config.dump_results:
         for bench in benchmarks:
-            if not config.performance_check:
-                # TODO: consolidate the check mess here ( there are too many flags )
-                print(bench.raw.decode("utf-8"))
-
-            if config.dump_results:
-                bench.dump(config.dump_results)
-
-        to_output.footnote()
+            bench.dump(config.dump_results)

bench_wizard/db_bench.py

@@ -1,23 +1,24 @@
 import json
 import os
 import subprocess
+from dataclasses import dataclass
 
 from typing import Tuple, Union
 
-from bench_wizard.config import Config
 
+@dataclass
+class DBPerformanceConfig:
+    substrate_dir: str
 
-def db_benchmark(config: Config) -> Union[None, Tuple[dict, dict]]:
-    if not config.do_db_bench:
-        return None
 
-    print("Performing Database benchmark ( this may take a while ) ... ")
+def db_benchmark(config: DBPerformanceConfig) -> Union[None, Tuple[dict, dict]]:
+    print("Performing Database read/write benchmark ( this may take a while ) ... ")
 
     # clone only if dir does not exit
-    if not os.path.isdir(config.substrate_repo_path):
-        print(f"Cloning Substrate repository into {config.substrate_repo_path}")
+    if not os.path.isdir(config.substrate_dir):
+        print(f"Cloning Substrate repository into {config.substrate_dir}")
 
-        command = f"git clone https://github.com/paritytech/substrate.git {config.substrate_repo_path}".split(
+        command = f"git clone https://github.com/paritytech/substrate.git {config.substrate_dir}".split(
             " "
         )
         result = subprocess.run(command)
@@ -34,15 +35,15 @@ def db_benchmark(config: Config) -> Union[None, Tuple[dict, dict]]:
     )
 
     read_result = subprocess.run(
-        read_benchmark_command, capture_output=True, cwd=config.substrate_repo_path
+        read_benchmark_command, capture_output=True, cwd=config.substrate_dir
     )
 
     if read_result.returncode != 0:
         print(f"Failed to run read DB benchmarks: {read_result.stderr}")
         return None
 
     write_result = subprocess.run(
-        write_benchmark_command, capture_output=True, cwd=config.substrate_repo_path
+        write_benchmark_command, capture_output=True, cwd=config.substrate_dir
     )
 
     if write_result.returncode != 0:
@@ -56,20 +57,34 @@ def db_benchmark(config: Config) -> Union[None, Tuple[dict, dict]]:
 
 def display_db_benchmark_results(results: tuple) -> None:
     if not results:
+        print("Failed to run db benchmarks")
         return
 
     print("Database benchmark results:\n")
-    print(f"{'Name':^75}|{'Raw average(ns)':^26}|{'Average(ns)':^21}|")
+    print(f"{'Name':^75}|{'Raw average(ns)':^26}|{'Average(ns)':^21}| Reference value")
 
     for oper in results:
         for result in oper:
-            print(
-                f"{result['name']:<75}| {result['raw_average']:^25}| {result['average']:^20}|"
-            )
-
+            name = result["name"]
+            if "RocksDb" in name and "read" in name:
+                print(
+                    f"{result['name']:<75}| {result['raw_average']:^25}| {result['average']:^20}| 25000"
+                )
+            elif "RocksDb" in name and "write" in name:
+                print(
+                    f"{result['name']:<75}| {result['raw_average']:^25}| {result['average']:^20}| 100000"
+                )
+            elif "read" in name:
+                print(
+                    f"{result['name']:<75}| {result['raw_average']:^25}| {result['average']:^20}| 8000"
+                )
+            else:
+                print(
+                    f"{result['name']:<75}| {result['raw_average']:^25}| {result['average']:^20}| 50000"
+                )
     print("")
 
 
-def run_db_benchmark(config: Config):
+def run_db_benchmark(config: DBPerformanceConfig):
     results = db_benchmark(config)
     display_db_benchmark_results(results)