Skip to content

Commit

Permalink
add more running mode
Browse files Browse the repository at this point in the history 
Signed-off-by: Su Yihan <[email protected]>
  • Loading branch information
@yviansu
yviansu committed Feb 29, 2024
1 parent 4632dbb commit a0b8879
Show file tree
Hide file tree
Showing 4 changed files with 140 additions and 27 deletions.
4 changes: 3 additions & 1 deletion tests/benchmark/README.md
View file
Original file line number Diff line number Diff line change
Expand Up @@ -39,7 +39,9 @@ These benchmarks are based on some open source efforts to measure performance of
# run specific benchmark
node run_benchmark.js --benchmark binarytrees
# run specific runtime mode
node run_benchmark.js --runtime wamr-aot # (wamr-aot | wamr-interp | qjs)
node run_benchmark.js --runtimes wamr-aot # (wamr-aot | wamr-interp | qjs | node)
# get result after multiple times warm up
node run_benchmark.js --warmup 3
```

## Validate benchmark result
Expand Down
88 changes: 69 additions & 19 deletions tests/benchmark/run_benchmark.js
View file
Original file line number Diff line number Diff line change
Expand Up @@ -54,6 +54,7 @@ const wamr_stack_size = args['--stack-size'] ? parseInt(args['--stack-size']) :
const wamr_gc_heap = args['--gc-heap'] ? parseInt(args['--gc-heap']) : 40960000;
const specifed_benchmarks = args['--benchmarks'] ? args['--benchmarks'].split(',') : null;
const specified_runtimes = args['--runtimes'] ? args['--runtimes'].split(',') : null;
const warm_up_times = args['--warmup'] ? parseInt(args['--warmup']) : 0;

const default_gc_size_option = `--gc-heap-size=${wamr_gc_heap}`
const stack_size_option = `--stack-size=${wamr_stack_size}`
Expand All @@ -80,9 +81,27 @@ try {
}
}

let ts_times = [];
let js_times = [];
let aot_times = [];
let node_cmd;
try {
node_cmd = execSync('which node').toString().trim();
} catch (error) {
if (process.env.NODE_PATH) {
node_cmd = process.env.NODE_PATH;
} else {
const default_node_path = '/usr/local/bin/node';
if (fs.existsSync(default_node_path)) {
node_cmd = default_node_path;
} else {
console.error("Error: NODE_PATH is not defined, and no default node path is provided.");
process.exit(1);
}
}
}

let wamr_interp_times = [];
let qjs_js_times = [];
let wamr_aot_times = [];
let v8_js_times = [];
let prefixs = [];

let benchmark_options = {
Expand Down Expand Up @@ -118,6 +137,7 @@ function collect_benchmark_options(options) {

console.log(`\x1b[33m======================== options ========================\x1b[0m`);
console.log(`QJS_PATH: ${qjs}`);
console.log(`NODE_PATH: ${node_cmd}`);
console.log(`strategy: run ${multirun} times and get average`);
console.log(`clean generated files: ${shouldClean ? 'true' : 'false'}`);
console.log(`\x1b[33m======================== running ========================\x1b[0m`);
Expand All @@ -127,6 +147,9 @@ function run_multiple_times(cmd) {
let elapse_arr = [];

try {
for (let i = 0; i < warm_up_times; i++) {
execSync(cmd);
}
for (let i = 0; i < multirun; i++) {
let start = performance.now();
let ret = execSync(cmd);
Expand Down Expand Up @@ -192,7 +215,7 @@ for (let benchmark of benchmarks) {
else {
process.stdout.write(`WAMR interpreter ... \t`);
elapsed = run_multiple_times(`${iwasm_gc} ${collect_benchmark_options(benchmark_options[prefix]?.wamr_option)} -f main ${prefix}.wasm`);
ts_times.push(elapsed);
wamr_interp_times.push(elapsed);
console.log(`${elapsed.toFixed(2)}ms`);
}

Expand All @@ -202,7 +225,7 @@ for (let benchmark of benchmarks) {
else {
process.stdout.write(`WAMR AoT ... \t\t`);
elapsed = run_multiple_times(`${iwasm_gc} ${collect_benchmark_options(benchmark_options[prefix]?.wamr_option)} -f main ${prefix}.aot`);
aot_times.push(elapsed);
wamr_aot_times.push(elapsed);
console.log(`${elapsed.toFixed(2)}ms`);
}

Expand All @@ -212,7 +235,17 @@ for (let benchmark of benchmarks) {
else {
process.stdout.write(`QuickJS ... \t\t`);
elapsed = run_multiple_times(`${qjs} ${js_file}`);
js_times.push(elapsed);
qjs_js_times.push(elapsed);
console.log(`${elapsed.toFixed(2)}ms`);
}

if (specified_runtimes && !specified_runtimes.includes('node')) {
console.log(`\x1b[33mSkip Node due to argument filter.\x1b[0m`);
}
else {
process.stdout.write(`Node ... \t\t`);
elapsed = run_multiple_times(`${node_cmd} ${js_file}`);
v8_js_times.push(elapsed);
console.log(`${elapsed.toFixed(2)}ms`);
}

Expand All @@ -229,38 +262,55 @@ console.log(`\x1b[32m====================== results ======================\x1b[0
let results = [];

for (let i = 0; i < executed_benchmarks; i++) {
let ts_time = ts_times[i];
let js_time = js_times[i];
let aot_time = aot_times[i];
let wamr_interp_time = wamr_interp_times[i];
let qjs_js_time = qjs_js_times[i];
let wamr_aot_time = wamr_aot_times[i];
let v8_js_time = v8_js_times[i];

let r = {
benchmark: prefixs[i]
}

if (ts_time) {
r['WAMR_interpreter'] = ts_time.toFixed(2) + 'ms';
if (wamr_interp_time) {
r['WAMR_interpreter'] = wamr_interp_time.toFixed(2) + 'ms';
}

if (aot_time) {
r['WAMR_aot'] = aot_time.toFixed(2) + 'ms';
if (wamr_aot_time) {
r['WAMR_aot'] = wamr_aot_time.toFixed(2) + 'ms';
}

if (js_time) {
r['QuickJS'] = js_time.toFixed(2) + 'ms';
if (qjs_js_time) {
r['QuickJS'] = qjs_js_time.toFixed(2) + 'ms';
}

if (ts_time && js_time) {
let ratio = ts_time / js_time;
if (v8_js_time) {
r['Node'] = v8_js_time.toFixed(2) + 'ms';
}

if (wamr_interp_time && qjs_js_time) {
let ratio = wamr_interp_time / qjs_js_time;
let formatted_result = ratio.toFixed(2);
r['WAMR_interpreter/qjs'] = formatted_result;
}

if (aot_time && js_time) {
let ratio_aot = aot_time / js_time;
if (wamr_aot_time && qjs_js_time) {
let ratio_aot = wamr_aot_time / qjs_js_time;
let formatted_result_aot = ratio_aot.toFixed(2);
r['WAMR_aot/qjs'] = formatted_result_aot;
}

if (wamr_interp_time && v8_js_time) {
let ratio = wamr_interp_time / v8_js_time;
let formatted_result = ratio.toFixed(2);
r['WAMR_interpreter/node'] = formatted_result;
}

if (wamr_aot_time && v8_js_time) {
let ratio_aot = wamr_aot_time / v8_js_time;
let formatted_result_aot = ratio_aot.toFixed(2);
r['WAMR_aot/node'] = formatted_result_aot;
}

results.push(r);
}

Expand Down
25 changes: 21 additions & 4 deletions tools/validate/run_module/run_module_on_chrome.html
View file
Original file line number Diff line number Diff line change
Expand Up @@ -14,17 +14,31 @@
<table>
<tr>
<td><label for="pathInput">path:</label></td>
<td><input type="text" id="pathInput" required></td>
<td><input type="text" id="pathInput" required style="width: 500px;"></td>
</tr>
<tr>
<td><label for="funcNameInput">function name:</label></td>
<td><input type="text" id="funcNameInput" required></td>
<td><input type="text" id="funcNameInput" style="width: 500px;"></td>
</tr>
<tr>
<td><label for="argsInput">arguments:</label></td>
<td><input type="text" id="argsInput"></td>
<td><input type="text" id="argsInput" style="width: 500px;"></td>
</tr>
<tr>
<td><label for="warmupTimes">warmup times:</label></td>
<td><input type="text" id="warmupTimes" style="width: 500px;"></td>
</tr>
<tr>
<td><label for="targetSelect">run target:</label></td>
<td>
<select id="targetSelect" style="width: 500px;">
<option value="wasm">WASM</option>
<option value="js">JS</option>
</select>
</td>
</tr>
</table>
<p></p>
<button type="submit">submit</button>
</form>

Expand All @@ -36,10 +50,13 @@
var path = document.getElementById("pathInput").value;
var funcName = document.getElementById("funcNameInput").value;
var args = document.getElementById("argsInput").value.split(",");
var warmupTimes= document.getElementById("warmupTimes").value;
var runTarget = document.getElementById("targetSelect").value;

run_wasm_module(path, funcName, ...args);
run_wasm_module(path, funcName, warmupTimes, runTarget, ...args);
});
</script>
<p id="result"></p>
<p id="time"></p>
</body>
</html>
50 changes: 47 additions & 3 deletions tools/validate/run_module/run_module_on_chrome.js
View file
Original file line number Diff line number Diff line change
Expand Up @@ -5,20 +5,64 @@

import { importObject, setWasmMemory } from './import_object.js';

export function run_wasm_module(wasmFilePath, wasmFuncName, ...funcArgs) {
fetch(wasmFilePath)
export function run_wasm_module(filePath, funcName, warmupTimes, runTarget, ...funcArgs) {
const parts = filePath.split(".");
const extension = parts[parts.length - 1];
if (runTarget === 'js') {
if (extension !== 'js') {
const resultElement = document.getElementById('result');
resultElement.innerHTML = `Error: filePath must end with ".js`;
}
fetch(filePath)
.then(response => response.text())
.then(script => {
if (warmupTimes) {
for (let i = 0; i < parseInt(warmupTimes); i++) {
eval(script);
}
}
const start_time = performance.now();
let res = eval(script);
if (funcName) {
res = window[funcName](...funcArgs);
}
const end_time = performance.now();
if (typeof res !== 'object' || res === null) {
const resultElement = document.getElementById('result');
resultElement.innerHTML = `The result is: ${res}`;
}
const timeElement = document.getElementById('time');
timeElement.innerHTML = `Execution time is: ${end_time - start_time}`;
});
} else if (runTarget === 'wasm') {
if (extension !== 'wasm') {
const resultElement = document.getElementById('result');
resultElement.innerHTML = `Error: filePath must end with ".wasm`;
}
fetch(filePath)
.then((response) => response.arrayBuffer())
.then((bytes) => WebAssembly.instantiate(bytes, importObject))
.then((results) => {
const exports = results.instance.exports;
setWasmMemory(exports.default);
const startFunc = exports._entry;
const exportedFunc = exports[funcName];
if (warmupTimes) {
for (let i = 0; i < parseInt(warmupTimes); i++) {
startFunc();
exportedFunc(...funcArgs);
}
}
const start_time = performance.now();
startFunc();
const exportedFunc = exports[wasmFuncName];
const res = exportedFunc(...funcArgs);
const end_time = performance.now();
if (typeof res !== 'object' || res === null) {
const resultElement = document.getElementById('result');
resultElement.innerHTML = `The result is: ${res}`;
}
const timeElement = document.getElementById('time');
timeElement.innerHTML = `Execution time is: ${end_time - start_time}`;
});
}
}

0 comments on commit a0b8879

Please sign in to comment.