diff --git a/infra/generate_cfgs.py b/infra/generate_cfgs.py new file mode 100755 index 00000000..c1cfdee6 --- /dev/null +++ b/infra/generate_cfgs.py @@ -0,0 +1,59 @@ +#!/usr/bin/env python3 +import glob +import os + +def make_cfgs(bench, data_dir): + cwd = os.getcwd() + path = f"{data_dir}/{bench}" + runmodes = os.listdir(path) + for mode in runmodes: + os.chdir(f"{path}/{mode}") + + # HACK: check if opt-18 exists + # otherwise use opt + # On Linux, sometimes it's called opt-18, while on mac it seems to be just opt + # Also, on some machines, just running `opt-18` hangs, so we pass the version flag + if os.system("opt-18 --version") == 0: + opt = "opt-18" + else: + opt = "opt" + + # https://llvm.org/docs/Passes.html#dot-cfg-print-cfg-of-function-to-dot-file + cmd = f"{opt} -disable-output -passes=dot-cfg {bench}-{mode}.ll" + os.system(cmd) + + # Delete the -init.ll file (We don't need it for nightly, + # so just reduce the amount of clutter we copy to the nightly machine) + os.system(f"rm {bench}-{mode}-init.ll") + + # Find all the dot files (can't use glob because it doesn't match hidden files) + # There are also a bunch of files that start with ._Z that I don't think we care about? + dots = [f for f in os.listdir(".") if f.endswith(".dot") and not f.startswith("._Z") and not f.startswith("._bril")] + for dot in dots: + name = dot.split(".")[1] + + # Convert to png + cmd = f"dot -Tpng -o {name}.png {dot}" + os.system(cmd) + + pngs = glob.glob("*.png") + print(f"Generated {len(pngs)} CFGs for {bench} {mode}") + with open("png_names.txt", "w") as f: + f.write("\n".join(pngs)) + + # Clean up dot files + os.system("rm .*.dot") + + # Reset dir + os.chdir(cwd) + + +if __name__ == '__main__': + # expect a single argument + if len(os.sys.argv) != 2: + print("Usage: generate_line_counts.py ") + exit(1) + data_dir = os.sys.argv[1] + benchmarks = os.listdir(data_dir) + for bench in benchmarks: + make_cfgs(bench, data_dir) diff --git a/infra/generate_line_counts.py b/infra/generate_line_counts.py index 4a5a9464..c804b293 100755 --- a/infra/generate_line_counts.py +++ b/infra/generate_line_counts.py @@ -98,7 +98,7 @@ def get_rows_for_benchmark(bench, profile_data): def benchmarks_table(): - profile_data = json.load(open(f'{output_path}/data/profile.json')) + profile_data = json.load(open(f'{output_path}/profile.json')) benchmarks = set([x["benchmark"] for x in profile_data]) rows = header() rows += [ @@ -121,15 +121,15 @@ def benchmarks_table(): return "\n".join(rows) def generate_latex(output_path): - with open(f'{output_path}/data/linecount.tex', "w") as f: + with open(f'{output_path}/linecount.tex', "w") as f: f.write(linecount_table()) - with open(f'{output_path}/data/detailed-linecount.tex', "w") as f: + with open(f'{output_path}/detailed-linecount.tex', "w") as f: f.write(detailed_linecount_table()) - with open(f'{output_path}/data/benchmarks.tex', "w") as f: + with open(f'{output_path}/benchmarks.tex', "w") as f: f.write(benchmarks_table()) - tex_files = glob.glob(f"{output_path}/data/*.tex") + tex_files = glob.glob(f"{output_path}/*.tex") for tex in tex_files: - cmd = " ".join(["pdflatex", f"-output-directory {output_path}/data/", tex]) + cmd = " ".join(["pdflatex", f"-output-directory {output_path}/", tex, "> /dev/null 2>&1"]) os.system(cmd) diff --git a/infra/nightly-resources/handlers.js b/infra/nightly-resources/handlers.js index 65667762..fe607a19 100644 --- a/infra/nightly-resources/handlers.js +++ b/infra/nightly-resources/handlers.js @@ -24,15 +24,16 @@ async function load_index() { // Top-level load function for the llvm page async function load_llvm() { - await loadCommonData(); const params = new URLSearchParams(window.location.search); const benchmark = params.get("benchmark"); const runMode = params.get("runmode"); + if (!benchmark || !runMode) { console.error("missing query params, this probably shouldn't happen"); + return; } - const llvm = await fetchText(`./data/llvm/${benchmark}-${runMode}.ll`); - document.getElementById("llvm").innerText = llvm; + showIR(benchmark, runMode); + showCFGs(benchmark, runMode); } async function load_table() { @@ -99,32 +100,6 @@ async function loadBaseline(url) { refreshView(); } -function toggleWarnings() { - const elt = document.getElementById("warnings-toggle"); - elt.classList.toggle("active"); - const content = elt.nextElementSibling; - if (content.style.display === "block") { - elt.innerText = `Show ${GLOBAL_DATA.warnings.size} Warnings`; - content.style.display = "none"; - } else { - elt.innerText = `Hide ${GLOBAL_DATA.warnings.size} Warnings`; - content.style.display = "block"; - } -} - -function toggleChart() { - const elt = document.getElementById("chart-toggle"); - elt.classList.toggle("active"); - const content = elt.nextElementSibling; - if (content.style.display === "block") { - elt.innerText = "Show Chart"; - content.style.display = "none"; - } else { - elt.innerText = "Hide Chart"; - content.style.display = "block"; - } -} - function onRadioClick(elt) { GLOBAL_DATA.chart.mode = elt.value; document.getElementById("speedup-formula").style.visibility = @@ -135,3 +110,44 @@ function onRadioClick(elt) { function copyToClipboard(eltId) { navigator.clipboard.writeText(document.getElementById(eltId).innerText); } + +function expand(elt, labelElt, text) { + elt.classList.add("expanded"); + elt.classList.remove("collapsed"); + labelElt.innerText = text; +} + +function collapse(elt, labelElt, text) { + elt.classList.add("collapsed"); + elt.classList.remove("expanded"); + labelElt.innerText = text; +} + +function toggle(elt, showText, hideText) { + const content = elt.nextElementSibling; + if (content.classList.contains("expanded")) { + collapse(content, elt, showText); + } else { + expand(content, elt, hideText); + } +} + +function toggleAllPngs(elt) { + const btns = Array.from(document.getElementsByClassName("pngToggle")); + + if (elt.innerText == "Expand All") { + elt.innerText = "Collapse All"; + btns.forEach((btn) => { + const txt = btn.innerText.replace("\u25B6 Show", "\u25BC Hide"); + const content = btn.nextElementSibling; + expand(content, btn, txt); + }); + } else { + elt.innerText = "Expand All"; + btns.forEach((btn) => { + const txt = btn.innerText.replace("\u25BC Hide", "\u25B6 Show"); + const content = btn.nextElementSibling; + collapse(content, btn, txt); + }); + } +} diff --git a/infra/nightly-resources/index.html b/infra/nightly-resources/index.html index 41ddc6de..2ebbc037 100644 --- a/infra/nightly-resources/index.html +++ b/infra/nightly-resources/index.html @@ -7,12 +7,11 @@ - + - Nightlies @@ -22,13 +21,15 @@

Nightlies

Profile

- - - - -
-

Speedup = (LLVM-O0 TIME / RUN MODE TIME)

- + +
+ + + +
+

Speedup = (LLVM-O0 TIME / RUN MODE TIME)

+ +
@@ -36,8 +37,13 @@

Profile

- -
+ +
diff --git a/infra/nightly-resources/index.js b/infra/nightly-resources/index.js index 96784780..06af0a1a 100644 --- a/infra/nightly-resources/index.js +++ b/infra/nightly-resources/index.js @@ -52,7 +52,7 @@ function refreshView() { function renderWarnings() { const toggle = document.getElementById("warnings-toggle"); - toggle.innerText = `Show ${GLOBAL_DATA.warnings.size} Warnings`; + toggle.innerText = `\u25B6 Show ${GLOBAL_DATA.warnings.size} Warnings`; const warningContainer = document.getElementById("warnings"); warningContainer.innerHTML = ""; diff --git a/infra/nightly-resources/llvm.html b/infra/nightly-resources/llvm.html index e11c1757..ccc30d74 100644 --- a/infra/nightly-resources/llvm.html +++ b/infra/nightly-resources/llvm.html @@ -3,23 +3,20 @@ - - - - - - - - + + LLVM

LLVM

-
+ +
+ +
diff --git a/infra/nightly-resources/llvm.js b/infra/nightly-resources/llvm.js new file mode 100644 index 00000000..1022491e --- /dev/null +++ b/infra/nightly-resources/llvm.js @@ -0,0 +1,46 @@ +async function showIR(benchmark, runMode) { + const llvm = await fetchText( + `./data/llvm/${benchmark}/${runMode}/${benchmark}-${runMode}.ll`, + ); + document.getElementById("llvm-ir").innerText = llvm; +} + +async function showCFGs(benchmark, runMode) { + let pngs = ( + await fetchText(`./data/llvm/${benchmark}/${runMode}/png_names.txt`) + ).split("\n"); + + // Move main.png and _main.png to top + const _main = "_main.png"; + if (pngs.includes(_main)) { + pngs = pngs.filter((x) => x !== _main); + pngs.unshift(_main); + } + const main = "main.png"; + if (pngs.includes(main)) { + pngs = pngs.filter((x) => x !== main); + pngs.unshift(main); + } + + const pngContainer = document.getElementById("llvm-cfg"); + pngs.forEach((png) => { + const elt = document.createElement("div"); + + const btn = document.createElement("button"); + btn.innerText = `\u25B6 Show ${png}`; + btn.classList.add("collapsible"); + btn.classList.add("pngToggle"); + btn.onclick = (elt) => + toggle(elt.target, `\u25B6 Show ${png}`, `\u25BC Hide ${png}`); + + elt.appendChild(btn); + + const img = document.createElement("img"); + img.classList.add("cfg"); + img.classList.add("collapsed"); + img.src = `data/llvm/${benchmark}/${runMode}/${png}`; + elt.appendChild(img); + + pngContainer.appendChild(elt); + }); +} diff --git a/infra/nightly-resources/stylesheet.css b/infra/nightly-resources/stylesheet.css index 43a11098..5fb8f102 100644 --- a/infra/nightly-resources/stylesheet.css +++ b/infra/nightly-resources/stylesheet.css @@ -19,20 +19,18 @@ body { } .collapsible { - background-color: #777; - color: white; + background-color: white; cursor: pointer; - padding: 18px; - width: 100%; + padding: 10px 0px; border: none; text-align: left; outline: none; font-size: 15px; - margin: 10px 0px; + font-weight: normal; } .active, .collapsible:hover { - background-color: #555; + font-weight: bold; } .content { @@ -58,4 +56,17 @@ body { #speedup-formula { visibility: hidden; +} + +.cfg { + max-height: 600px; + display: block; +} + +.expanded { + display: block; +} + +.collapsed { + display: none; } \ No newline at end of file diff --git a/infra/nightly-resources/table.html b/infra/nightly-resources/table.html index a9b4ef8b..5a6fda98 100644 --- a/infra/nightly-resources/table.html +++ b/infra/nightly-resources/table.html @@ -3,16 +3,10 @@ - - - - - - TABLE diff --git a/infra/nightly.sh b/infra/nightly.sh index e9eb8ac2..dca4362e 100755 --- a/infra/nightly.sh +++ b/infra/nightly.sh @@ -30,6 +30,7 @@ MYDIR="$(cd -P "$(dirname "$src")" && pwd)" TOP_DIR="$MYDIR/.." RESOURCE_DIR="$MYDIR/nightly-resources" NIGHTLY_DIR="$TOP_DIR/nightly" +DATA_DIR="$TOP_DIR/nightly/data" # Make sure we're in the right place cd $MYDIR @@ -46,32 +47,34 @@ mkdir -p "$NIGHTLY_DIR/data" "$NIGHTLY_DIR/data/llvm" "$NIGHTLY_DIR/output" pushd $TOP_DIR # Run profiler. -# create temporary directory structure necessary for bench runs -mkdir -p ./tmp/bench # locally, run on argument if [ "$LOCAL" != "" ]; then - ./infra/profile.py "$@" "$NIGHTLY_DIR" + ./infra/profile.py "$@" "$NIGHTLY_DIR" >> $NIGHTLY_DIR/log.txt 2>&1 else export LLVM_SYS_180_PREFIX="/usr/lib/llvm-18/" make runtime # run on all benchmarks in nightly - ./infra/profile.py benchmarks/passing "$NIGHTLY_DIR" + ./infra/profile.py benchmarks/passing "$DATA_DIR" >> $NIGHTLY_DIR/log.txt 2>&1 fi # Generate latex after running the profiler (depends on profile.json) -./infra/generate_line_counts.py "$NIGHTLY_DIR" +./infra/generate_line_counts.py "$DATA_DIR" >> $NIGHTLY_DIR/log.txt 2>&1 -rm -r ./tmp/ +# Generate CFGs for LLVM after running the profiler +./infra/generate_cfgs.py "$DATA_DIR/llvm" >> $NIGHTLY_DIR/log.txt 2>&1 popd # Update HTML index page. cp "$RESOURCE_DIR"/* "$NIGHTLY_DIR/output" -# Copy json directory to the artifact +# Copy data directory to the artifact cp -r "$NIGHTLY_DIR/data" "$NIGHTLY_DIR/output/data" +# Copy log +cp "$NIGHTLY_DIR/log.txt" "$NIGHTLY_DIR/output" + # gzip all JSON in the nightly dir if [ "$LOCAL" == "" ]; then gzip "$NIGHTLY_DIR/output/data/profile.json" diff --git a/infra/profile.py b/infra/profile.py index 45936e3b..d49584ed 100755 --- a/infra/profile.py +++ b/infra/profile.py @@ -18,51 +18,55 @@ "llvm-O3-eggcc", ] -def get_eggcc_options(name, profile_dir): - match name: +# Where to output files that are needed for nightly report +DATA_DIR = None + +# Where to write intermediate files that should be cleaned up at the end of this script +TMP_DIR = "tmp" + +def get_eggcc_options(run_mode, benchmark): + llvm_out_dir = f"{DATA_DIR}/llvm/{benchmark}/{run_mode}" + match run_mode: case "rvsdg-round-trip-to-executable": - return f'--run-mode rvsdg-round-trip-to-executable --llvm-output-dir {profile_dir}/llvm-{name}' + return f'--run-mode rvsdg-round-trip-to-executable --llvm-output-dir {llvm_out_dir}' case "cranelift-O3": return f'--run-mode cranelift --optimize-egglog false --optimize-brilift true' case "llvm-O0": - return f'--run-mode llvm --optimize-egglog false --optimize-bril-llvm false --llvm-output-dir {profile_dir}/llvm-{name}' + return f'--run-mode llvm --optimize-egglog false --optimize-bril-llvm false --llvm-output-dir {llvm_out_dir}' case "llvm-O3": - return f'--run-mode llvm --optimize-egglog false --optimize-bril-llvm true --llvm-output-dir {profile_dir}/llvm-{name}' + return f'--run-mode llvm --optimize-egglog false --optimize-bril-llvm true --llvm-output-dir {llvm_out_dir}' case "llvm-O0-eggcc": - return f'--run-mode llvm --optimize-egglog true --optimize-bril-llvm false --llvm-output-dir {profile_dir}/llvm-{name}' + return f'--run-mode llvm --optimize-egglog true --optimize-bril-llvm false --llvm-output-dir {llvm_out_dir}' case "llvm-O3-eggcc": - return f'--run-mode llvm --optimize-egglog true --optimize-bril-llvm true --llvm-output-dir {profile_dir}/llvm-{name}' + return f'--run-mode llvm --optimize-egglog true --optimize-bril-llvm true --llvm-output-dir {llvm_out_dir}' case _: - raise Exception("Unexpected run mode: " + name) + raise Exception("Unexpected run mode: " + run_mode) class Benchmark: def __init__(self, path, treatment, index, total): self.path = path + self.name = path.split("/")[-1][:-len(".bril")] self.treatment = treatment # index of this benchmark (for printing) self.index = index # total number of benchmarks being run self.total = total -def benchmark_name(benchmark_path): - return benchmark_path.split("/")[-1][:-len(".bril")] +def benchmark_profile_dir(name): + return f'{TMP_DIR}/{name}' -def benchmark_profile_dir(benchmark_path): - return f'./tmp/bench/{benchmark_name(benchmark_path)}' - -def setup_benchmark(benchmark_path): - # strip off the .bril to get just the profile name - profile_dir = benchmark_profile_dir(benchmark_path) +def setup_benchmark(name): + profile_dir = benchmark_profile_dir(name) try: os.mkdir(profile_dir) except FileExistsError: print(f'{profile_dir} exists, overwriting contents') def optimize(benchmark): - print(f'[{benchmark.index}/{benchmark.total}] Optimizing {benchmark.path} with {benchmark.treatment}') - profile_dir = benchmark_profile_dir(benchmark.path) - cmd = f'cargo run --release {benchmark.path} {get_eggcc_options(benchmark.treatment, profile_dir)} -o {profile_dir}/{benchmark.treatment}' + print(f'[{benchmark.index}/{benchmark.total}] Optimizing {benchmark.name} with {benchmark.treatment}') + profile_dir = benchmark_profile_dir(benchmark.name) + cmd = f'cargo run --release {benchmark.path} {get_eggcc_options(benchmark.treatment, benchmark.name)} -o {profile_dir}/{benchmark.treatment}' print(f'Running: {cmd}') start = time.time() subprocess.call(cmd, shell=True) @@ -72,8 +76,8 @@ def optimize(benchmark): def bench(benchmark): - print(f'[{benchmark.index}/{benchmark.total}] Benchmarking {benchmark.path} with {benchmark.treatment}') - profile_dir = benchmark_profile_dir(benchmark.path) + print(f'[{benchmark.index}/{benchmark.total}] Benchmarking {benchmark.name} with {benchmark.treatment}') + profile_dir = benchmark_profile_dir(benchmark.name) with open(f'{profile_dir}/{benchmark.treatment}-args') as f: args = f.read().rstrip() @@ -100,24 +104,14 @@ def should_have_llvm_ir(runMethod): "llvm-O3-eggcc", ] -def dump_llvm_ir(runMethod, benchmark, output_directory): - from_path = f'./tmp/bench/{benchmark}/llvm-{runMethod}/{benchmark}-{runMethod}.ll' - - to_path = f'{output_directory}/data/llvm/{benchmark}-{runMethod}.ll' - - os.system(f'cp {from_path} {to_path}') - - # aggregate all profile info into a single json array. -def aggregate(compile_times, bench_times, output_directory): +def aggregate(compile_times, bench_times): res = [] for path in sorted(compile_times.keys()): name = path.split("/")[-2] runMethod = path.split("/")[-1] result = {"runMethod": runMethod, "benchmark": name, "hyperfine": bench_times[path], "compileTime": compile_times[path]} - if should_have_llvm_ir(runMethod): - dump_llvm_ir(runMethod, name, output_directory) res.append(result) return res @@ -129,17 +123,20 @@ def aggregate(compile_times, bench_times, output_directory): print("Usage: profile.py ") exit(1) - profile_path, output_path = os.sys.argv[1:] + # Create tmp directory for intermediate files + try: + os.mkdir(TMP_DIR) + except FileExistsError: + print(f"{TMP_DIR} exits, overwriting contents") + + bril_dir, DATA_DIR = os.sys.argv[1:] profiles = [] # if it is a directory get all files - if os.path.isdir(profile_path): - print(f'Running all bril files in {profile_path}') - profiles = glob(f'{profile_path}/**/*.bril', recursive=True) + if os.path.isdir(bril_dir): + print(f'Running all bril files in {bril_dir}') + profiles = glob(f'{bril_dir}/**/*.bril', recursive=True) else: - profiles = [profile_path] - - for benchmark_path in profiles: - setup_benchmark(benchmark_path) + profiles = [bril_dir] to_run = [] index = 0 @@ -148,6 +145,9 @@ def aggregate(compile_times, bench_times, output_directory): for treatment in treatments: to_run.append(Benchmark(benchmark_path, treatment, index, total)) index += 1 + + for benchmark in to_run: + setup_benchmark(benchmark.name) compile_times = {} @@ -181,6 +181,9 @@ def aggregate(compile_times, bench_times, output_directory): (path, _bench_data) = res bench_data[path] = _bench_data - nightly_data = aggregate(compile_times, bench_data, output_path) - with open(f"{output_path}/data/profile.json", "w") as profile: + nightly_data = aggregate(compile_times, bench_data) + with open(f"{DATA_DIR}/profile.json", "w") as profile: json.dump(nightly_data, profile, indent=2) + + # Clean up intermediate files + os.system(f"rm -rf {TMP_DIR}")