some benchmarks over QQ

sumiya11 · Jan 12, 2024 · ea4dc29 · ea4dc29
1 parent 27b8e65
commit ea4dc29
Show file tree

Hide file tree

Showing 21 changed files with 261 additions and 90 deletions.
diff --git a/benchmark/benchmark_systems.jl b/benchmark/benchmark_systems.jl
@@ -128,12 +128,15 @@ function benchmark_set_3()
         dummy_system("dummy", ground_field),
         ("cyclic 7", Groebner.cyclicn(7, k=ground_field)),
         ("cyclic 8", Groebner.cyclicn(8, k=ground_field)),
+        ("cyclic 9", Groebner.cyclicn(9, k=ground_field)),
         ("katsura 9", Groebner.katsuran(9, k=ground_field)),
         ("katsura 10", Groebner.katsuran(10, k=ground_field)),
         ("katsura 11", Groebner.katsuran(11, k=ground_field)),
         ("eco 10", Groebner.eco10(k=ground_field)),
         ("eco 11", Groebner.eco11(k=ground_field)),
         ("eco 12", Groebner.eco12(k=ground_field)),
+        ("eco 13", Groebner.eco13(k=ground_field)),
+        ("noon 7", Groebner.noonn(7, k=ground_field)),
         ("noon 8", Groebner.noonn(8, k=ground_field)),
         ("noon 9", Groebner.noonn(9, k=ground_field)),
         ("henrion 6", Groebner.henrion6(k=ground_field)),

diff --git a/benchmark/generate/msolve/run_in_msolve.jl b/benchmark/generate/msolve/run_in_msolve.jl
@@ -54,13 +54,17 @@ function process_system()
     for iter in 1:NUM_RUNS
         @info "Computing GB.." iter
         problemfile = (@__DIR__) * "/$BENCHMARK_DIR/$PROBLEM_NAME/$(PROBLEM_NAME).in"
+        __io = open(problemfile, "r")
+        CHAR = parse(Int, readlines(__io)[2])
+        close(__io)
         outputfile =
             VALIDATE ? (@__DIR__) * "/$BENCHMARK_DIR/$PROBLEM_NAME/$(output_filename())" :
             "/dev/null"
-        cmd = Cmd(
-            `$BIN_PATH_NORM -g 2 -l 44 -c 0 -f $problemfile -o $outputfile`,
-            detach=false
-        )
+        cmd = if CHAR > 0
+            Cmd(`$BIN_PATH_NORM -g 2 -l 44 -c 0 -f $problemfile -o $outputfile`, detach=false)
+        else
+            Cmd(`$BIN_PATH_NORM -g 2 -c 0 -f $problemfile -o $outputfile`, detach=false)
+        end
         timing = time_ns()
         proc = run(cmd, wait=true)
         # push!(children, proc)

diff --git a/benchmark/one_script_to_run_them_all.jl b/benchmark/one_script_to_run_them_all.jl
@@ -34,6 +34,8 @@ progressbar_enabled() =
 
 const _available_backends = ["groebner", "singular", "maple", "msolve", "openf4"]
 
+const _skip_singular = true
+
 # Parses command-line arguments
 #! format: off
 function parse_commandline()
@@ -880,11 +882,11 @@ function check_args(args)
     if backend == "openf4" && args["suite"] in [3, 7]
         throw("Running benchmarks over the rationals is not possible for openf4")
     end
-    if backend == "msolve" && args["suite"] in [3, 7] && args["validate"] != "no"
-        throw(
-            "Validating results for msolve over the rationals is not possible. Use command line option --validate=no"
-        )
-    end
+    # if backend == "msolve" && args["suite"] in [3, 7] && args["validate"] != "no"
+    #     throw(
+    #         "Validating results for msolve over the rationals is not possible. Use command line option --validate=no"
+    #     )
+    # end
     if backend == "learn_apply" && args in [3, 7]
         throw("Cannot learn & apply over the rationals")
     end
@@ -922,6 +924,9 @@ function main()
         systems = []
         solved_problems = []
         for backend in _available_backends
+            if _skip_singular && backend == "singular"
+                continue
+            end
             args_ = copy(args)
             args_["backend"] = backend
             try
@@ -943,6 +948,7 @@ function main()
     else
         solved_problems = []
         try
+            check_args(args)
             populate_benchmarks(args)
             solved_problems = run_benchmarks(args)
             validate_results(args, solved_problems)

diff --git a/benchmark/results/benchmark_result_3.md b/benchmark/results/benchmark_result_3.md
@@ -1,31 +1,44 @@
 ## Benchmark results
 
-2023-12-31T02:52:27.789
+2024-01-12T13:17:56.773
 
-Benchmarked backends: Any["groebner", "maple", "msolve", "singular"]
+Benchmarked backends: Any["groebner", "maple", "msolve"]
 
 Benchmark suite: The rationals
 
 - Workers: 8
-- Timeout: 100 s
+- Timeout: 2000 s
 - Aggregated over: 1 runs
 
 **All timings in seconds.**
 
-|Model|groebner|maple|msolve|singular|
-|-----|---|---|---|---|
-|cyclic 7|1.23|1.69|2.56| - |
-|cyclic 8|28.93|33.62|67.23| - |
-|dummy|0.00|0.01|0.02|0.00|
-|eco 10|0.67|0.92|0.80|30.60|
-|eco 11|3.55|5.64|6.26| - |
-|eco 12|29.99|52.92|41.68| - |
-|henrion 6|0.72| - |6.34|5.22|
-|katsura 9|4.06|10.30|5.74| - |
-|katsura 10|29.28| - |43.10| - |
-|katsura 11| - | - | - | - |
-|noon 8|5.31|10.19|17.45|13.60|
-|noon 9| - | - | - | - |
+|Model|groebner|maple|msolve|
+|:----|---|---|---|
+|chandra 9|1.92|5.85|WA|
+|chandra 10|4.11|27.81|WA|
+|chandra 11|14.09|151.60|WA|
+|chandra 12|90.53|1166.48|WA|
+|chandra 13|548.26| - |WA|
+|cyclic 7|0.94|1.50|1.07|
+|cyclic 8|25.17|24.92|26.84|
+|cyclic 9| - | - | - |
+|dummy|0.00|0.01|0.03|
+|eco 10|0.50|0.72|0.57|
+|eco 11|3.42|4.72|3.67|
+|eco 12|28.89|39.68|25.40|
+|eco 13|359.91|532.73|443.58|
+|henrion 6|0.66|1.96|3.80|
+|henrion 7|394.70| - | - |
+|ipp|134.31|196.19|WA|
+|katsura 9|4.11|9.07|2.77|
+|katsura 10|23.64|102.40|20.62|
+|katsura 11|319.80|1479.51|244.90|
+|noon 7|0.97|1.02|1.56|
+|noon 8|5.82|9.62|12.76|
+|noon 9|42.68|102.74|102.05|
+|reimer 6|0.54|0.65|0.28|
+|reimer 7|10.80|37.55|7.06|
+|reimer 8|816.37| - |364.48|
 
 *Benchmarking environment:*
 

diff --git a/benchmark/results/benchmark_result_7.md b/benchmark/results/benchmark_result_7.md
@@ -0,0 +1,46 @@
+## Benchmark results
+
+2024-01-12T13:17:57.172
+
+Benchmarked backends: Any["groebner", "maple"]
+
+Benchmark suite: HC
+
+- Workers: 8
+- Timeout: 1200 s
+- Aggregated over: 1 runs
+
+**All timings in seconds.**
+
+|Model|groebner|maple|
+|:----|---|---|
+|boon|0.00|0.01|
+|chandra 5|0.01|0.10|
+|chandra 6|0.02|0.15|
+|chandra 7|0.07|0.46|
+|chandra 8|0.30| - |
+|chandra 9|1.26| - |
+|chandra 10|4.70| - |
+|chandra 11|16.81| - |
+|chandra 12|107.68| - |
+|ipp|195.66| - |
+|rps10| - | - |
+
+*Benchmarking environment:*
+
+* Total RAM (GiB): 188
+* Processor: Intel(R) Xeon(R) Gold 6130 CPU @ 2.10GHz
+* Julia version: 1.9.2
+
+Versions of the dependencies:
+
+* Primes : 0.5.5
+* TimerOutputs : 0.5.23
+* PrecompileTools : 1.2.0
+* MultivariatePolynomials : 0.5.3
+* Combinatorics : 1.0.2
+* AbstractAlgebra : 0.34.7
+* Nemo : 0.38.3
+* Atomix : 0.1.0
+* ExprTools : 0.1.10
+* PrettyTables : 2.3.1
diff --git a/benchmark/results/groebner/benchmark_result_3.md b/benchmark/results/groebner/benchmark_result_3.md
@@ -1,31 +1,44 @@
 ## Benchmark results
 
-2023-12-31T14:39:14.826
+2024-01-12T12:10:34.027
 
 Benchmarked backend: groebner
 
 Benchmark suite: The rationals
 
 - Workers: 8
-- Timeout: 1200 s
+- Timeout: 2000 s
 - Aggregated over: 1 runs
 
 **All timings in seconds.**
 
-|Model|Total|
-|-----|---|
-|cyclic 7|1.19|
-|cyclic 8|27.54|
+|Model|Total, s|
+|:----|---|
+|chandra 9|1.92|
+|chandra 10|4.11|
+|chandra 11|14.09|
+|chandra 12|90.53|
+|chandra 13|548.26|
+|cyclic 7|0.94|
+|cyclic 8|25.17|
+|cyclic 9| - |
 |dummy|0.00|
-|eco 10|0.68|
-|eco 11|3.53|
-|eco 12|31.14|
-|henrion 6|0.76|
-|katsura 9|3.73|
-|katsura 10|27.37|
-|katsura 11|355.52|
-|noon 8|5.29|
-|noon 9|56.26|
+|eco 10|0.50|
+|eco 11|3.42|
+|eco 12|28.89|
+|eco 13|359.91|
+|henrion 6|0.66|
+|henrion 7|394.70|
+|ipp|134.31|
+|katsura 9|4.11|
+|katsura 10|23.64|
+|katsura 11|319.80|
+|noon 7|0.97|
+|noon 8|5.82|
+|noon 9|42.68|
+|reimer 6|0.54|
+|reimer 7|10.80|
+|reimer 8|816.37|
 
 *Benchmarking environment:*
 

diff --git a/benchmark/results/groebner/benchmark_result_7.md b/benchmark/results/groebner/benchmark_result_7.md
@@ -1,29 +1,29 @@
 ## Benchmark results
 
-2024-01-12T09:04:15.474
+2024-01-12T13:17:47.010
 
 Benchmarked backend: groebner
 
 Benchmark suite: HC
 
-- Workers: 4
+- Workers: 8
 - Timeout: 1200 s
 - Aggregated over: 1 runs
 
 **All timings in seconds.**
 
 |Model|Total, s|
 |:----|---|
-|boon|0.01|
+|boon|0.00|
 |chandra 5|0.01|
-|chandra 6|0.04|
-|chandra 7|0.22|
-|chandra 8|0.44|
-|chandra 9|2.52|
-|chandra 10|5.07|
-|chandra 11|17.69|
-|chandra 12|108.19|
-|ipp|185.73|
+|chandra 6|0.02|
+|chandra 7|0.07|
+|chandra 8|0.30|
+|chandra 9|1.26|
+|chandra 10|4.70|
+|chandra 11|16.81|
+|chandra 12|107.68|
+|ipp|195.66|
 |rps10| - |
 
 *Benchmarking environment:*

diff --git a/benchmark/results/maple/benchmark_result_3.md b/benchmark/results/maple/benchmark_result_3.md
@@ -1,31 +1,44 @@
 ## Benchmark results
 
-2023-12-31T14:11:04.046
+2024-01-12T12:45:58.856
 
 Benchmarked backend: maple
 
 Benchmark suite: The rationals
 
 - Workers: 8
-- Timeout: 1200 s
+- Timeout: 2000 s
 - Aggregated over: 1 runs
 
 **All timings in seconds.**
 
-|Model|Total|
-|-----|---|
-|cyclic 7|1.75|
-|cyclic 8|30.09|
-|dummy|0.02|
-|eco 10|0.78|
-|eco 11|5.63|
-|eco 12|52.84|
-|henrion 6| - |
-|katsura 9|9.53|
-|katsura 10|136.96|
-|katsura 11| - |
-|noon 8|11.05|
-|noon 9|122.91|
+|Model|Total, s|
+|:----|---|
+|chandra 9|5.85|
+|chandra 10|27.81|
+|chandra 11|151.60|
+|chandra 12|1166.48|
+|chandra 13| - |
+|cyclic 7|1.50|
+|cyclic 8|24.92|
+|cyclic 9| - |
+|dummy|0.01|
+|eco 10|0.72|
+|eco 11|4.72|
+|eco 12|39.68|
+|eco 13|532.73|
+|henrion 6|1.96|
+|henrion 7| - |
+|ipp|196.19|
+|katsura 9|9.07|
+|katsura 10|102.40|
+|katsura 11|1479.51|
+|noon 7|1.02|
+|noon 8|9.62|
+|noon 9|102.74|
+|reimer 6|0.65|
+|reimer 7|37.55|
+|reimer 8| - |
 
 *Benchmarking environment:*
 

diff --git a/benchmark/results/maple/benchmark_result_7.md b/benchmark/results/maple/benchmark_result_7.md
@@ -1,24 +1,28 @@
 ## Benchmark results
 
-2024-01-12T08:36:25.401
+2024-01-12T13:17:53.095
 
 Benchmarked backend: maple
 
 Benchmark suite: HC
 
-- Workers: 4
-- Timeout: 1000 s
+- Workers: 8
+- Timeout: 1200 s
 - Aggregated over: 1 runs
 
 **All timings in seconds.**
 
 |Model|Total, s|
 |:----|---|
 |boon|0.01|
-|chandra 5|0.07|
-|chandra 6|0.13|
-|chandra 7|0.36|
-|chandra 8|1.40|
+|chandra 5|0.10|
+|chandra 6|0.15|
+|chandra 7|0.46|
+|chandra 8| - |
+|chandra 9| - |
+|chandra 10| - |
+|chandra 11| - |
+|chandra 12| - |
 |ipp| - |
 |rps10| - |