feat: split build and benchmark run commands

bench_wizard/__version__.py

@@ -1 +1 @@
-__version__ = "0.3.0"
+__version__ = "0.4.0"

bench_wizard/benchmark.py

@@ -3,6 +3,7 @@
 import subprocess
 from typing import List
 
+from bench_wizard.cargo import Cargo
 from bench_wizard.config import Config
 from bench_wizard.exceptions import BenchmarkCargoException
 from bench_wizard.output import Output
@@ -34,7 +35,6 @@ class Benchmark:
     """ Represents single benchmark"""
 
     def __init__(self, pallet: str, command: [str], ref_value: float, extrinsics: list):
-
         self._pallet = pallet
         self._stdout = None
         self._command = command
@@ -120,17 +120,16 @@ def _prepare_benchmarks(config: Config, reference_values: dict) -> List[Benchmar
     benchmarks = []
 
     for pallet in config.pallets:
-        command = COMMAND + [f"--pallet={pallet}"]
+        cargo = Cargo(pallet=pallet, template=config.template)
         if config.output_dir:
             output_file = os.path.join(config.output_dir, f"{pallet}.rs")
-            command += [f"--output={output_file}"]
-
-        if config.template:
-            command += [f"--template={config.template}"]
+            cargo.output = output_file
 
         ref_data = reference_values[pallet]
         ref_value = sum(list(map(lambda x: float(x), ref_data.values())))
-        benchmarks.append(Benchmark(pallet, command, ref_value, ref_data.keys()))
+        benchmarks.append(
+            Benchmark(pallet, cargo.command(), ref_value, ref_data.keys())
+        )
 
     return benchmarks
 
@@ -149,6 +148,21 @@ def _run_benchmarks(benchmarks: List[Benchmark], output: Output, rerun=False) ->
             output.update(bench)
 
 
+def _build(manifest: str) -> None:
+    command = [
+        "cargo",
+        "build",
+        "--release",
+        "--features=runtime-benchmarks",
+        f"--manifest-path={manifest}",
+    ]
+
+    result = subprocess.run(command, capture_output=True)
+
+    if result.returncode != 0:
+        raise BenchmarkCargoException(result.stderr.decode("utf-8"))
+
+
 def run_pallet_benchmarks(config: Config, to_output: Output) -> None:
     if not config.do_pallet_bench:
         return
@@ -161,6 +175,10 @@ def run_pallet_benchmarks(config: Config, to_output: Output) -> None:
 
         benchmarks = _prepare_benchmarks(config, s)
 
+        to_output.info("Compiling - this may take a while...")
+
+        _build("node/Cargo.toml")
+
         to_output.info("Running benchmarks - this may take a while...")
 
         _run_benchmarks(benchmarks, to_output)

bench_wizard/cargo.py

@@ -0,0 +1,43 @@
+from dataclasses import dataclass
+from typing import Optional, List
+
+
+@dataclass
+class Cargo:
+    pallet: str
+    manifest: str = "node/Cargo.toml"
+    chain: str = "dev"
+    steps: int = 5
+    repeat: int = 20
+    extrinsic: str = "*"
+    execution: str = "wasm"
+    wasm_execution: str = "compiled"
+    heap_pages: int = 4096
+    output: Optional[str] = None
+    template: Optional[str] = None
+
+    def command(self) -> List[str]:
+        cmd = [
+            "cargo",
+            "run",
+            "--release",
+            "--features=runtime-benchmarks",
+            f"--manifest-path={self.manifest}",
+            "--",
+            "benchmark",
+            f"--pallet={self.pallet}",
+            f"--chain={self.chain}",
+            f"--steps={self.steps}",
+            f"--repeat={self.repeat}",
+            f"--extrinsic={self.extrinsic}",
+            f"--execution={self.execution}",
+            f"--wasm-execution={self.wasm_execution}",
+            f"--heap-pages={self.heap_pages}",
+        ]
+
+        if self.output:
+            cmd.append(f"--output={self.output}")
+        if self.template:
+            cmd.append(f"--template={self.template}")
+
+        return cmd

bench_wizard/output.py

@@ -30,7 +30,10 @@ def track(self, benchmarks: ["Benchmark"]):
         self._tracker = len(benchmarks)
 
     def update(self, benchmark: "Benchmark"):
-        print(f"Running {self._completed}/{self._tracker}", end="\r")
+        print(
+            f"Running {self._completed}/{self._tracker} (pallet: {benchmark.pallet})",
+            end="\r",
+        )
         self._completed += benchmark.completed
 
     def results(self, benchmarks: ["Benchmark"]):

pyproject.toml

@@ -1,6 +1,6 @@
 [tool.poetry]
 name = "bench-wizard"
-version = "0.3.0"
+version = "0.4.0"
 description = ""
 authors = ["Martin Hloska <[email protected]>"]
 

tests/test_bench_wizard.py

@@ -2,4 +2,4 @@
 
 
 def test_version():
-    assert __version__ == "0.3.0"
+    assert __version__ == "0.4.0"