From a78bc23673ef9a450587e86fc2837ef3638ffbd8 Mon Sep 17 00:00:00 2001 From: Mauro Toscano <12560266+MauroToscano@users.noreply.github.com> Date: Mon, 7 Aug 2023 15:23:22 -0300 Subject: [PATCH] Stabilize benches for add and sub - Update tables (#519) * More additions * Add print * Increase add to one million * Add samples to add * base config, triple time and samples for add * Test 100.000 * Test 100.000 * Revert "Test 100.000" This reverts commit 258810a2d094f6378396dae2151f99b803d49fc1. * Revert "Test 100.000" This reverts commit 679c9ff0fc16652e8d197d93c375de6e00ee7ff1. * Reduce sqrt ssize * Example sum * Example sum * One million add sub * Update measurements * Update Readme * Fix function description * Fmt --- README.md | 35 +++++++++++++++++++++++++---------- benches/benches/add.rs | 22 +++++++++++++++------- benches/benches/invert.rs | 2 +- benches/benches/mul.rs | 6 +++--- benches/benches/pow.rs | 6 +++--- benches/benches/sqrt.rs | 10 +++++----- benches/benches/sub.rs | 21 ++++++++++++++------- benches/benches/utils.rs | 6 +++--- 8 files changed, 69 insertions(+), 39 deletions(-) diff --git a/README.md b/README.md index 6b5887714..790bec357 100644 --- a/README.md +++ b/README.md @@ -19,16 +19,31 @@ So, we decided to build our library, focusing on performance, with clear documen Benchmark results are hosted [here](https://lambdaclass.github.io/lambdaworks/bench). -These are the results of execution of the benchmarks for finite field arithmetic using the STARK field prime (p = 3618502788666131213697322783095070105623107215331596699973092056135872020481). Benchmark results were run with AMD Ryzen 7 PRO 4750G with Radeon Graphics (32 GB RAM) using Ubuntu 20.04.6 LTS - -| | arkworks | lambdaworks | -| -------- | --------- | ----------- | -| `add` | 15.170 μs | 13.042 μs | -| `sub` | 15.493 μs | 14.888 μs | -| `mul` | 60.462 μs | 57.014 μs | -| `invert` | 35.475 ms | 35.216 ms | -| `sqrt` | 126.39 ms | 133.74 ms | -| `pow` | 12.139 ms | 12.148 ms | +These are the results of execution of the benchmarks for finite field arithmetic using the STARK field prime (p = 3618502788666131213697322783095070105623107215331596699973092056135872020481). + +Differences of 3% are common for some measurements, so small differences are not statistically relevant. + +ARM - M1 + +| Operation| N | Arkworks | Lambdaworks | +| -------- | --- | --------- | ----------- | +| `mul` | 10k | 115 μs | 117 μs | +| `add` | 1M | 8.6 ms | 7.3 ms | +| `sub` | 1M | 7.57 ms | 7.27 ms | +| `pow` | 10k | 11.5 ms | 12.6 ms | +| `invert` | 10k | 33.3 ms | 30.7 ms | + +x86 - AMD Ryzen 7 PRO + +| Operation | N | Arkworks (ASM)* | Lambdaworks | +| -------- | --- | --------- | ----------- | +| `mul` | 10k | 102.7 us | 94.4 us +| `add` | 1M | 4.9 ms | 5.6 ms | +| `sub` | 1M | 4.5 ms | 5.3 ms +| `pow` | 10k | 10.5 ms | 9.7 ms | +| `invert` | 10k | 33.4 ms | 37.45 ms | + +*assembly feature was enabled manually for that bench, and is not activated by default when running criterion To run them locally, you will need `cargo-criterion` and `cargo-flamegraph`. Install it with: diff --git a/benches/benches/add.rs b/benches/benches/add.rs index 671ee6ed7..cadb43357 100644 --- a/benches/benches/add.rs +++ b/benches/benches/add.rs @@ -1,4 +1,4 @@ -use std::ops::Add; +use std::{ops::Add, time::Duration}; use criterion::{black_box, criterion_group, criterion_main, Criterion}; use utils::generate_random_elements; @@ -10,17 +10,17 @@ pub mod utils; const BENCHMARK_NAME: &str = "add"; pub fn criterion_benchmark(c: &mut Criterion) { - let arkworks_vec = generate_random_elements(); + let arkworks_vec = generate_random_elements(2000000); // arkworks-ff { c.bench_function( - &format!("{} 10000 elements | ark-ff - ef8f758", BENCHMARK_NAME), + &format!("{} 1M elements | ark-ff - ef8f758", BENCHMARK_NAME), |b| { b.iter(|| { let mut iter = arkworks_vec.iter(); - for _i in 0..10000 { + for _i in 0..1000000 { let a = iter.next().unwrap(); let b = iter.next().unwrap(); black_box(black_box(&a).add(black_box(b))); @@ -35,12 +35,12 @@ pub fn criterion_benchmark(c: &mut Criterion) { let lambdaworks_vec = to_lambdaworks_vec(&arkworks_vec); c.bench_function( - &format!("{} 10000 elements | lambdaworks", BENCHMARK_NAME,), + &format!("{} 1M elements | lambdaworks", BENCHMARK_NAME,), |b| { b.iter(|| { let mut iter = lambdaworks_vec.iter(); - for _i in 0..10000 { + for _i in 0..1000000 { let a = iter.next().unwrap(); let b = iter.next().unwrap(); black_box(black_box(&a).add(black_box(b))); @@ -51,5 +51,13 @@ pub fn criterion_benchmark(c: &mut Criterion) { } } -criterion_group!(benches, criterion_benchmark); +criterion_group! { + name = benches; + // This can be any expression that returns a `Criterion` object. + config = Criterion::default() + .significance_level(0.01) + .measurement_time(Duration::from_secs(15)) + .sample_size(300); + targets = criterion_benchmark +} criterion_main!(benches); diff --git a/benches/benches/invert.rs b/benches/benches/invert.rs index 7ed0152ab..8a3c9b512 100644 --- a/benches/benches/invert.rs +++ b/benches/benches/invert.rs @@ -9,7 +9,7 @@ pub mod utils; const BENCHMARK_NAME: &str = "invert"; pub fn criterion_benchmark(c: &mut Criterion) { - let arkworks_vec = generate_random_elements()[0..10000].to_vec(); + let arkworks_vec = generate_random_elements(10000).to_vec(); // arkworks-ff { diff --git a/benches/benches/mul.rs b/benches/benches/mul.rs index c28d22aeb..22c037dde 100644 --- a/benches/benches/mul.rs +++ b/benches/benches/mul.rs @@ -9,13 +9,13 @@ pub mod utils; const BENCHMARK_NAME: &str = "mul"; pub fn criterion_benchmark(c: &mut Criterion) { - let arkworks_vec = generate_random_elements(); + let arkworks_vec = generate_random_elements(20000); // arkworks-ff { c.bench_function( &format!( - "{} 10000 elements | ark-ff - commit: ef8f758 ", + "{} 10K elements | ark-ff - commit: ef8f758 ", BENCHMARK_NAME ), |b| { @@ -37,7 +37,7 @@ pub fn criterion_benchmark(c: &mut Criterion) { let lambdaworks_vec = to_lambdaworks_vec(&arkworks_vec); c.bench_function( - &format!("{} 10000 elements | lambdaworks", BENCHMARK_NAME,), + &format!("{} 10K elements | lambdaworks", BENCHMARK_NAME,), |b| { b.iter(|| { let mut iter = lambdaworks_vec.iter(); diff --git a/benches/benches/pow.rs b/benches/benches/pow.rs index 001cb5634..2dc21e1e6 100644 --- a/benches/benches/pow.rs +++ b/benches/benches/pow.rs @@ -10,7 +10,7 @@ pub mod utils; const BENCHMARK_NAME: &str = "pow"; pub fn criterion_benchmark(c: &mut Criterion) { - let arkworks_vec = generate_random_elements(); + let arkworks_vec = generate_random_elements(20000); let mut rng = rand_chacha::ChaCha20Rng::seed_from_u64(9001); @@ -22,7 +22,7 @@ pub fn criterion_benchmark(c: &mut Criterion) { // arkworks-ff { c.bench_function( - &format!("{} 10000 elements | ark-ff - ef8f758", BENCHMARK_NAME), + &format!("{} 10K elements | ark-ff - ef8f758", BENCHMARK_NAME), |b| { b.iter(|| { let mut iter = arkworks_vec.iter(); @@ -43,7 +43,7 @@ pub fn criterion_benchmark(c: &mut Criterion) { let lambdaworks_vec = to_lambdaworks_vec(&arkworks_vec); c.bench_function( - &format!("{} 10000 elements | lambdaworks", BENCHMARK_NAME,), + &format!("{} 10K elements | lambdaworks", BENCHMARK_NAME,), |b| { b.iter(|| { let mut iter = lambdaworks_vec.iter(); diff --git a/benches/benches/sqrt.rs b/benches/benches/sqrt.rs index 1e2fbc2ba..8b26adffa 100644 --- a/benches/benches/sqrt.rs +++ b/benches/benches/sqrt.rs @@ -13,7 +13,7 @@ pub fn criterion_benchmark(c: &mut Criterion) { let mut rng = ::seed_from_u64(9001); let mut arkworks_vec = Vec::new(); - for _i in 0..1000 { + for _i in 0..100 { let a = F::rand(&mut rng); let square = a * a; arkworks_vec.push(square); @@ -22,12 +22,12 @@ pub fn criterion_benchmark(c: &mut Criterion) { // arkworks-ff { c.bench_function( - &format!("{} 1000 elements | ark-ff - ef8f758", BENCHMARK_NAME), + &format!("{} 100 elements | ark-ff - ef8f758", BENCHMARK_NAME), |b| { b.iter(|| { let mut iter = arkworks_vec.iter(); - for _i in 0..1000 { + for _i in 0..100 { let a = iter.next().unwrap(); black_box(black_box(a).sqrt()); } @@ -41,12 +41,12 @@ pub fn criterion_benchmark(c: &mut Criterion) { let lambdaworks_vec = to_lambdaworks_vec(&arkworks_vec); c.bench_function( - &format!("{} 1000 elements | lambdaworks", BENCHMARK_NAME,), + &format!("{} 100 elements | lambdaworks", BENCHMARK_NAME,), |b| { b.iter(|| { let mut iter = lambdaworks_vec.iter(); - for _i in 0..1000 { + for _i in 0..100 { let a = iter.next().unwrap(); black_box(black_box(a).sqrt()); } diff --git a/benches/benches/sub.rs b/benches/benches/sub.rs index 83be7d182..3bb0a7d30 100644 --- a/benches/benches/sub.rs +++ b/benches/benches/sub.rs @@ -1,5 +1,5 @@ use criterion::{black_box, criterion_group, criterion_main, Criterion}; -use std::ops::Sub; +use std::{ops::Sub, time::Duration}; use utils::{generate_random_elements, to_lambdaworks_vec}; pub mod utils; @@ -7,17 +7,17 @@ pub mod utils; const BENCHMARK_NAME: &str = "sub"; pub fn criterion_benchmark(c: &mut Criterion) { - let arkworks_vec = generate_random_elements(); + let arkworks_vec = generate_random_elements(2000000); // arkworks-ff { c.bench_function( - &format!("{} 10000 elements | ark-ff - ef8f758", BENCHMARK_NAME), + &format!("{} 1M elements | ark-ff - ef8f758", BENCHMARK_NAME), |b| { b.iter(|| { let mut iter = arkworks_vec.iter(); - for _i in 0..10000 { + for _i in 0..1000000 { let a = iter.next().unwrap(); let b = iter.next().unwrap(); black_box(black_box(&a).sub(black_box(b))); @@ -31,12 +31,12 @@ pub fn criterion_benchmark(c: &mut Criterion) { { let lambdaworks_vec = to_lambdaworks_vec(&arkworks_vec); c.bench_function( - &format!("{} 10000 elements | lambdaworks", BENCHMARK_NAME,), + &format!("{} 1M elements | lambdaworks", BENCHMARK_NAME,), |b| { b.iter(|| { let mut iter = lambdaworks_vec.iter(); - for _i in 0..10000 { + for _i in 0..1000000 { let a = iter.next().unwrap(); let b = iter.next().unwrap(); black_box(black_box(&a).sub(black_box(b))); @@ -47,5 +47,12 @@ pub fn criterion_benchmark(c: &mut Criterion) { } } -criterion_group!(benches, criterion_benchmark); +criterion_group! { + name = benches; + config = Criterion::default() + .significance_level(0.01) + .measurement_time(Duration::from_secs(15)) + .sample_size(500); + targets = criterion_benchmark +} criterion_main!(benches); diff --git a/benches/benches/utils.rs b/benches/benches/utils.rs index 0d3dcf32b..c7fad130e 100644 --- a/benches/benches/utils.rs +++ b/benches/benches/utils.rs @@ -9,11 +9,11 @@ use lambdaworks_math::{ }; use rand::SeedableRng; -/// Creates 20000 random elements -pub fn generate_random_elements() -> Vec { +/// Creates `amount` random elements +pub fn generate_random_elements(amount: u64) -> Vec { let mut rng = rand_chacha::ChaCha20Rng::seed_from_u64(9001); let mut arkworks_vec = Vec::new(); - for _i in 0..20000 { + for _i in 0..amount { let a = Fq::rand(&mut rng); arkworks_vec.push(a); }