WIP

performance-benchmarking/.gitignore

@@ -0,0 +1,2 @@
+tmp
+output

performance-benchmarking/Makefile

@@ -0,0 +1,39 @@
+TMP ?= ./tmp
+PANDOC_EXTS ?= +auto_identifiers
+
+HTML_SRC=$(wildcard src/html/*)
+MD_SRC=$(wildcard src/md/*)
+
+HTML_FROM_MD=$(patsubst src/md/%.md,$(TMP)/html_from_md/%.html,$(MD_SRC))
+
+ALL_HTML=$(HTML_SRC) $(HTML_FROM_MD)
+
+TEMPLATES=$(wildcard templates/*)
+
+BOX_WHISKERS=$(shell grep -e '~include tmp/box_whisker' $(MD_SRC) | cut -d' ' -f2)
+
+default: output/index.html
+
+output/index.html: $(TMP)/index.html $(BOX_WHISKERS)
+	mkdir -p $(dir $@)
+	bin/preprocess < $< > $@~
+	mv $@~ $@
+
+$(TMP)/box_whisker/%.html: src/box_whisker/%.yaml bin/mk-graph
+	mkdir -p $(dir $@)
+	bin/mk-graph $@~ < $<
+	mv $@~ $@
+
+$(TMP)/index.html: $(MD_SRC) $(TEMPLATES)
+	mkdir -p $(dir $@)
+	pandoc \
+	  --standalone \
+	  --embed-resources \
+	  --toc \
+	  --metadata title="benchcomp Documentation" \
+	  --template=templates/pandoc-template.html \
+	  --highlight-style=pygments \
+	  --from markdown$(PANDOC_EXTS) \
+	  --to html \
+	  < $< > $@~
+	  mv $@~ $@

performance-benchmarking/bin/mk-graph

@@ -0,0 +1,76 @@
+#!/usr/bin/env python3
+
+import subprocess
+import sys
+
+import jinja2
+import yaml
+
+
+BOXPLOT = """
+$data << EOD
+{% for graph in graphs -%}
+{{ loop.index}} {{ graph["low"] }} {{ graph["bottom"] }} {{ graph["top"] }} {{ graph["high"] }}
+{%- endfor %}
+EOD
+
+set terminal svg noenhanced size 360,120
+
+set border 2 linecolor "#000000"
+set ytics nomirror tc "#000000"
+
+unset key
+
+set ylabel ""
+
+set xtics ({% for graph in graphs %}"{{ graph["title"] }}" {{ loop.index }}{% if not loop.last %}, {% endif %}{% endfor %})
+
+set xrange [0:2]
+
+plot 0 with lines lc -1, \\
+    '$data' using 1:2:3:4:5 with candlesticks whiskerbars lc -1
+"""
+
+
+def get_gnu_file(graphs):
+    templ = jinja2.Environment(
+        loader=jinja2.BaseLoader).from_string(BOXPLOT)
+    return templ.render(graphs=graphs)
+
+
+def render_gnuplot(gnu_file):
+    with open(f"/tmp/gnu.gnu", "w") as handle:
+        print(gnu_file, file=handle)
+
+    proc = subprocess.Popen(
+        ["gnuplot"], stdout=subprocess.PIPE, stdin=subprocess.PIPE,
+        text=True)
+    out, _ = proc.communicate(input=gnu_file)
+    if proc.returncode:
+        print(gnu_file, file=sys.stderr)
+        sys.exit(1)
+    print(gnu_file)
+    return out
+
+
+
+def main():
+    data = yaml.safe_load(sys.stdin)
+
+    gnu_file = get_gnu_file(data["graphs"])
+    svg = render_gnuplot(gnu_file)
+    data["svg"] = "\n".join(
+        [line.strip() for line in svg.splitlines()][1:])
+
+    env = jinja2.Environment(loader=jinja2.FileSystemLoader("templates"))
+    templ = env.get_template("box-whiskers.jinja.html")
+    out = templ.render(data=data)
+
+    out = "\n".join([line.strip() for line in out.splitlines()])
+
+    with open(sys.argv[1], "w") as handle:
+        print(out, file=handle)
+
+
+if __name__ == "__main__":
+    main()

performance-benchmarking/bin/preprocess

@@ -0,0 +1,21 @@
+#!/usr/bin/env python3
+
+import re
+import sys
+
+pat = re.compile(r".+~include (?P<path>.+)</p>")
+ok = True
+for line in sys.stdin:
+    m = pat.match(line)
+    if not m:
+        print(line, end="")
+        continue
+    print(f"including {m['path']}", file=sys.stderr)
+    try:
+        with open(m["path"]) as included:
+            for line in included:
+                print(line, end="")
+    except FileNotFoundError:
+        ok = False
+        print(f"ERROR: File not found: {m['path']}", file=sys.stderr)
+sys.exit(0 if ok else 1)

performance-benchmarking/src/box_whisker/cbmc.yaml

@@ -0,0 +1,17 @@
+graphs:
+  - median: 0.7
+    high: 0.9
+    low: 0.8
+    top: 1
+    bottom: 0.2
+    title: release-->optimized
+    losers:
+      s2n_foo_bar: 1
+      freertos_list_update: 1
+      s2n_begin_session: 1
+      e_sdk_clear_seed: 1
+    winners:
+      s2n_init: 0.32
+      s2n_ed25519_gulp: 0.29
+      freertos_list_remove: 0.11
+      e_sdk_random_token: 0.10