Stabilize benches for add and sub - Update tables

README.md

@@ -19,16 +19,31 @@ So, we decided to build our library, focusing on performance, with clear documen
 
 Benchmark results are hosted [here](https://lambdaclass.github.io/lambdaworks/bench).
 
-These are the results of execution of the benchmarks for finite field arithmetic using the STARK field prime (p = 3618502788666131213697322783095070105623107215331596699973092056135872020481). Benchmark results were run with AMD Ryzen 7 PRO 4750G with Radeon Graphics (32 GB RAM) using Ubuntu 20.04.6 LTS
-
-|          | arkworks  | lambdaworks |
-| -------- | --------- | ----------- |
-| `add`    | 15.170 μs | 13.042 μs   |
-| `sub`    | 15.493 μs | 14.888 μs   |
-| `mul`    | 60.462 μs | 57.014 μs   |
-| `invert` | 35.475 ms | 35.216 ms   |
-| `sqrt`   | 126.39 ms | 133.74 ms   |
-| `pow`    | 12.139 ms | 12.148 ms   |
+These are the results of execution of the benchmarks for finite field arithmetic using the STARK field prime (p = 3618502788666131213697322783095070105623107215331596699973092056135872020481). 
+
+Differences of 3% are common for some measurements, so small differences are not statistically relevant.
+
+ARM - M1
+
+| Operation| N    | Arkworks  | Lambdaworks |
+| -------- | --- | --------- | ----------- |
+| `mul`    |   10k  | 115 μs | 117 μs   |
+| `add`    |   1M  | 8.6 ms  | 7.3 ms    |
+| `sub`    |   1M  | 7.57 ms   | 7.27 ms     |
+| `pow`    |   10k  | 11.5 ms   | 12.6 ms    |
+| `invert` |  10k   | 33.3 ms  | 30.7 ms   | 
+
+x86 - AMD Ryzen 7 PRO 
+
+| Operation | N    | Arkworks (ASM)*  | Lambdaworks |
+| -------- | --- | --------- | ----------- |
+| `mul`    |   10k  | 102.7 us | 94.4 us   
+| `add`    |   1M  | 4.9 ms  | 5.6 ms    |
+| `sub`    |   1M  |  4.5 ms  |  5.3 ms   
+| `pow`    |   10k  |  10.5 ms   | 9.7 ms    |
+| `invert` |  10k   | 33.4 ms  | 37.45 ms |
+
+*assembly feature was enabled manually for that bench, and is not activated by default when running criterion
 
 To run them locally, you will need `cargo-criterion` and `cargo-flamegraph`. Install it with:
 

benches/benches/add.rs

@@ -1,4 +1,4 @@
-use std::ops::Add;
+use std::{ops::Add, time::Duration};
 
 use criterion::{black_box, criterion_group, criterion_main, Criterion};
 use utils::generate_random_elements;
@@ -10,17 +10,17 @@ pub mod utils;
 const BENCHMARK_NAME: &str = "add";
 
 pub fn criterion_benchmark(c: &mut Criterion) {
-    let arkworks_vec = generate_random_elements();
+    let arkworks_vec = generate_random_elements(2000000);
 
     // arkworks-ff
     {
         c.bench_function(
-            &format!("{} 10000 elements | ark-ff - ef8f758", BENCHMARK_NAME),
+            &format!("{} 1M elements | ark-ff - ef8f758", BENCHMARK_NAME),
             |b| {
                 b.iter(|| {
                     let mut iter = arkworks_vec.iter();
 
-                    for _i in 0..10000 {
+                    for _i in 0..1000000 {
                         let a = iter.next().unwrap();
                         let b = iter.next().unwrap();
                         black_box(black_box(&a).add(black_box(b)));
@@ -35,12 +35,12 @@ pub fn criterion_benchmark(c: &mut Criterion) {
         let lambdaworks_vec = to_lambdaworks_vec(&arkworks_vec);
 
         c.bench_function(
-            &format!("{} 10000 elements | lambdaworks", BENCHMARK_NAME,),
+            &format!("{} 1M elements | lambdaworks", BENCHMARK_NAME,),
             |b| {
                 b.iter(|| {
                     let mut iter = lambdaworks_vec.iter();
 
-                    for _i in 0..10000 {
+                    for _i in 0..1000000 {
                         let a = iter.next().unwrap();
                         let b = iter.next().unwrap();
                         black_box(black_box(&a).add(black_box(b)));
@@ -51,5 +51,13 @@ pub fn criterion_benchmark(c: &mut Criterion) {
     }
 }
 
-criterion_group!(benches, criterion_benchmark);
+criterion_group! {
+    name = benches;
+    // This can be any expression that returns a `Criterion` object.
+    config = Criterion::default()
+        .significance_level(0.01)
+        .measurement_time(Duration::from_secs(15))
+        .sample_size(300);
+    targets = criterion_benchmark
+}
 criterion_main!(benches);

benches/benches/invert.rs

@@ -9,7 +9,7 @@ pub mod utils;
 const BENCHMARK_NAME: &str = "invert";
 
 pub fn criterion_benchmark(c: &mut Criterion) {
-    let arkworks_vec = generate_random_elements()[0..10000].to_vec();
+    let arkworks_vec = generate_random_elements(10000).to_vec();
 
     // arkworks-ff
     {

benches/benches/mul.rs

@@ -9,13 +9,13 @@ pub mod utils;
 const BENCHMARK_NAME: &str = "mul";
 
 pub fn criterion_benchmark(c: &mut Criterion) {
-    let arkworks_vec = generate_random_elements();
+    let arkworks_vec = generate_random_elements(20000);
 
     // arkworks-ff
     {
         c.bench_function(
             &format!(
-                "{} 10000 elements | ark-ff - commit: ef8f758 ",
+                "{} 10K elements | ark-ff - commit: ef8f758 ",
                 BENCHMARK_NAME
             ),
             |b| {
@@ -37,7 +37,7 @@ pub fn criterion_benchmark(c: &mut Criterion) {
         let lambdaworks_vec = to_lambdaworks_vec(&arkworks_vec);
 
         c.bench_function(
-            &format!("{} 10000 elements | lambdaworks", BENCHMARK_NAME,),
+            &format!("{} 10K elements | lambdaworks", BENCHMARK_NAME,),
             |b| {
                 b.iter(|| {
                     let mut iter = lambdaworks_vec.iter();

benches/benches/pow.rs

@@ -10,7 +10,7 @@ pub mod utils;
 const BENCHMARK_NAME: &str = "pow";
 
 pub fn criterion_benchmark(c: &mut Criterion) {
-    let arkworks_vec = generate_random_elements();
+    let arkworks_vec = generate_random_elements(20000);
 
     let mut rng = rand_chacha::ChaCha20Rng::seed_from_u64(9001);
 
@@ -22,7 +22,7 @@ pub fn criterion_benchmark(c: &mut Criterion) {
     // arkworks-ff
     {
         c.bench_function(
-            &format!("{} 10000 elements | ark-ff - ef8f758", BENCHMARK_NAME),
+            &format!("{} 10K elements | ark-ff - ef8f758", BENCHMARK_NAME),
             |b| {
                 b.iter(|| {
                     let mut iter = arkworks_vec.iter();
@@ -43,7 +43,7 @@ pub fn criterion_benchmark(c: &mut Criterion) {
         let lambdaworks_vec = to_lambdaworks_vec(&arkworks_vec);
 
         c.bench_function(
-            &format!("{} 10000 elements | lambdaworks", BENCHMARK_NAME,),
+            &format!("{} 10K elements | lambdaworks", BENCHMARK_NAME,),
             |b| {
                 b.iter(|| {
                     let mut iter = lambdaworks_vec.iter();

benches/benches/sqrt.rs

@@ -13,7 +13,7 @@ pub fn criterion_benchmark(c: &mut Criterion) {
     let mut rng = <rand_chacha::ChaCha20Rng as rand::SeedableRng>::seed_from_u64(9001);
 
     let mut arkworks_vec = Vec::new();
-    for _i in 0..1000 {
+    for _i in 0..100 {
         let a = F::rand(&mut rng);
         let square = a * a;
         arkworks_vec.push(square);
@@ -22,12 +22,12 @@ pub fn criterion_benchmark(c: &mut Criterion) {
     // arkworks-ff
     {
         c.bench_function(
-            &format!("{} 1000 elements | ark-ff - ef8f758", BENCHMARK_NAME),
+            &format!("{} 100 elements | ark-ff - ef8f758", BENCHMARK_NAME),
             |b| {
                 b.iter(|| {
                     let mut iter = arkworks_vec.iter();
 
-                    for _i in 0..1000 {
+                    for _i in 0..100 {
                         let a = iter.next().unwrap();
                         black_box(black_box(a).sqrt());
                     }
@@ -41,12 +41,12 @@ pub fn criterion_benchmark(c: &mut Criterion) {
         let lambdaworks_vec = to_lambdaworks_vec(&arkworks_vec);
 
         c.bench_function(
-            &format!("{} 1000 elements | lambdaworks", BENCHMARK_NAME,),
+            &format!("{} 100 elements | lambdaworks", BENCHMARK_NAME,),
             |b| {
                 b.iter(|| {
                     let mut iter = lambdaworks_vec.iter();
 
-                    for _i in 0..1000 {
+                    for _i in 0..100 {
                         let a = iter.next().unwrap();
                         black_box(black_box(a).sqrt());
                     }

benches/benches/sub.rs

@@ -1,23 +1,23 @@
 use criterion::{black_box, criterion_group, criterion_main, Criterion};
-use std::ops::Sub;
+use std::{ops::Sub, time::Duration};
 use utils::{generate_random_elements, to_lambdaworks_vec};
 
 pub mod utils;
 
 const BENCHMARK_NAME: &str = "sub";
 
 pub fn criterion_benchmark(c: &mut Criterion) {
-    let arkworks_vec = generate_random_elements();
+    let arkworks_vec = generate_random_elements(2000000);
 
     // arkworks-ff
     {
         c.bench_function(
-            &format!("{} 10000 elements | ark-ff - ef8f758", BENCHMARK_NAME),
+            &format!("{} 1M elements | ark-ff - ef8f758", BENCHMARK_NAME),
             |b| {
                 b.iter(|| {
                     let mut iter = arkworks_vec.iter();
 
-                    for _i in 0..10000 {
+                    for _i in 0..1000000 {
                         let a = iter.next().unwrap();
                         let b = iter.next().unwrap();
                         black_box(black_box(&a).sub(black_box(b)));
@@ -31,12 +31,12 @@ pub fn criterion_benchmark(c: &mut Criterion) {
     {
         let lambdaworks_vec = to_lambdaworks_vec(&arkworks_vec);
         c.bench_function(
-            &format!("{} 10000 elements | lambdaworks", BENCHMARK_NAME,),
+            &format!("{} 1M elements | lambdaworks", BENCHMARK_NAME,),
             |b| {
                 b.iter(|| {
                     let mut iter = lambdaworks_vec.iter();
 
-                    for _i in 0..10000 {
+                    for _i in 0..1000000 {
                         let a = iter.next().unwrap();
                         let b = iter.next().unwrap();
                         black_box(black_box(&a).sub(black_box(b)));
@@ -47,5 +47,12 @@ pub fn criterion_benchmark(c: &mut Criterion) {
     }
 }
 
-criterion_group!(benches, criterion_benchmark);
+criterion_group! {
+    name = benches;
+    config = Criterion::default()
+        .significance_level(0.01)
+        .measurement_time(Duration::from_secs(15))
+        .sample_size(500);
+    targets = criterion_benchmark
+}
 criterion_main!(benches);

benches/benches/utils.rs

@@ -9,11 +9,11 @@ use lambdaworks_math::{
 };
 use rand::SeedableRng;
 
-/// Creates 20000 random elements
-pub fn generate_random_elements() -> Vec<Fq> {
+/// Creates `amount` random elements
+pub fn generate_random_elements(amount: u64) -> Vec<Fq> {
     let mut rng = rand_chacha::ChaCha20Rng::seed_from_u64(9001);
     let mut arkworks_vec = Vec::new();
-    for _i in 0..20000 {
+    for _i in 0..amount {
         let a = Fq::rand(&mut rng);
         arkworks_vec.push(a);
     }