diff --git a/benchmark/benchmark_systems.jl b/benchmark/benchmark_systems.jl index daab4406..e17d0537 100644 --- a/benchmark/benchmark_systems.jl +++ b/benchmark/benchmark_systems.jl @@ -128,12 +128,15 @@ function benchmark_set_3() dummy_system("dummy", ground_field), ("cyclic 7", Groebner.cyclicn(7, k=ground_field)), ("cyclic 8", Groebner.cyclicn(8, k=ground_field)), + ("cyclic 9", Groebner.cyclicn(9, k=ground_field)), ("katsura 9", Groebner.katsuran(9, k=ground_field)), ("katsura 10", Groebner.katsuran(10, k=ground_field)), ("katsura 11", Groebner.katsuran(11, k=ground_field)), ("eco 10", Groebner.eco10(k=ground_field)), ("eco 11", Groebner.eco11(k=ground_field)), ("eco 12", Groebner.eco12(k=ground_field)), + ("eco 13", Groebner.eco13(k=ground_field)), + ("noon 7", Groebner.noonn(7, k=ground_field)), ("noon 8", Groebner.noonn(8, k=ground_field)), ("noon 9", Groebner.noonn(9, k=ground_field)), ("henrion 6", Groebner.henrion6(k=ground_field)), diff --git a/benchmark/generate/msolve/run_in_msolve.jl b/benchmark/generate/msolve/run_in_msolve.jl index 900482c8..7cbbb709 100644 --- a/benchmark/generate/msolve/run_in_msolve.jl +++ b/benchmark/generate/msolve/run_in_msolve.jl @@ -54,13 +54,17 @@ function process_system() for iter in 1:NUM_RUNS @info "Computing GB.." iter problemfile = (@__DIR__) * "/$BENCHMARK_DIR/$PROBLEM_NAME/$(PROBLEM_NAME).in" + __io = open(problemfile, "r") + CHAR = parse(Int, readlines(__io)[2]) + close(__io) outputfile = VALIDATE ? (@__DIR__) * "/$BENCHMARK_DIR/$PROBLEM_NAME/$(output_filename())" : "/dev/null" - cmd = Cmd( - `$BIN_PATH_NORM -g 2 -l 44 -c 0 -f $problemfile -o $outputfile`, - detach=false - ) + cmd = if CHAR > 0 + Cmd(`$BIN_PATH_NORM -g 2 -l 44 -c 0 -f $problemfile -o $outputfile`, detach=false) + else + Cmd(`$BIN_PATH_NORM -g 2 -c 0 -f $problemfile -o $outputfile`, detach=false) + end timing = time_ns() proc = run(cmd, wait=true) # push!(children, proc) diff --git a/benchmark/one_script_to_run_them_all.jl b/benchmark/one_script_to_run_them_all.jl index 0528e961..d3c23d27 100644 --- a/benchmark/one_script_to_run_them_all.jl +++ b/benchmark/one_script_to_run_them_all.jl @@ -34,6 +34,8 @@ progressbar_enabled() = const _available_backends = ["groebner", "singular", "maple", "msolve", "openf4"] +const _skip_singular = true + # Parses command-line arguments #! format: off function parse_commandline() @@ -880,11 +882,11 @@ function check_args(args) if backend == "openf4" && args["suite"] in [3, 7] throw("Running benchmarks over the rationals is not possible for openf4") end - if backend == "msolve" && args["suite"] in [3, 7] && args["validate"] != "no" - throw( - "Validating results for msolve over the rationals is not possible. Use command line option --validate=no" - ) - end + # if backend == "msolve" && args["suite"] in [3, 7] && args["validate"] != "no" + # throw( + # "Validating results for msolve over the rationals is not possible. Use command line option --validate=no" + # ) + # end if backend == "learn_apply" && args in [3, 7] throw("Cannot learn & apply over the rationals") end @@ -922,6 +924,9 @@ function main() systems = [] solved_problems = [] for backend in _available_backends + if _skip_singular && backend == "singular" + continue + end args_ = copy(args) args_["backend"] = backend try @@ -943,6 +948,7 @@ function main() else solved_problems = [] try + check_args(args) populate_benchmarks(args) solved_problems = run_benchmarks(args) validate_results(args, solved_problems) diff --git a/benchmark/results/benchmark_result_3.md b/benchmark/results/benchmark_result_3.md index b26f3997..544e0509 100644 --- a/benchmark/results/benchmark_result_3.md +++ b/benchmark/results/benchmark_result_3.md @@ -1,31 +1,44 @@ ## Benchmark results -2023-12-31T02:52:27.789 +2024-01-12T13:17:56.773 -Benchmarked backends: Any["groebner", "maple", "msolve", "singular"] +Benchmarked backends: Any["groebner", "maple", "msolve"] Benchmark suite: The rationals - Workers: 8 -- Timeout: 100 s +- Timeout: 2000 s - Aggregated over: 1 runs **All timings in seconds.** -|Model|groebner|maple|msolve|singular| -|-----|---|---|---|---| -|cyclic 7|1.23|1.69|2.56| - | -|cyclic 8|28.93|33.62|67.23| - | -|dummy|0.00|0.01|0.02|0.00| -|eco 10|0.67|0.92|0.80|30.60| -|eco 11|3.55|5.64|6.26| - | -|eco 12|29.99|52.92|41.68| - | -|henrion 6|0.72| - |6.34|5.22| -|katsura 9|4.06|10.30|5.74| - | -|katsura 10|29.28| - |43.10| - | -|katsura 11| - | - | - | - | -|noon 8|5.31|10.19|17.45|13.60| -|noon 9| - | - | - | - | +|Model|groebner|maple|msolve| +|:----|---|---|---| +|chandra 9|1.92|5.85|WA| +|chandra 10|4.11|27.81|WA| +|chandra 11|14.09|151.60|WA| +|chandra 12|90.53|1166.48|WA| +|chandra 13|548.26| - |WA| +|cyclic 7|0.94|1.50|1.07| +|cyclic 8|25.17|24.92|26.84| +|cyclic 9| - | - | - | +|dummy|0.00|0.01|0.03| +|eco 10|0.50|0.72|0.57| +|eco 11|3.42|4.72|3.67| +|eco 12|28.89|39.68|25.40| +|eco 13|359.91|532.73|443.58| +|henrion 6|0.66|1.96|3.80| +|henrion 7|394.70| - | - | +|ipp|134.31|196.19|WA| +|katsura 9|4.11|9.07|2.77| +|katsura 10|23.64|102.40|20.62| +|katsura 11|319.80|1479.51|244.90| +|noon 7|0.97|1.02|1.56| +|noon 8|5.82|9.62|12.76| +|noon 9|42.68|102.74|102.05| +|reimer 6|0.54|0.65|0.28| +|reimer 7|10.80|37.55|7.06| +|reimer 8|816.37| - |364.48| *Benchmarking environment:* diff --git a/benchmark/results/benchmark_result_7.md b/benchmark/results/benchmark_result_7.md new file mode 100644 index 00000000..f6426ee8 --- /dev/null +++ b/benchmark/results/benchmark_result_7.md @@ -0,0 +1,46 @@ +## Benchmark results + +2024-01-12T13:17:57.172 + +Benchmarked backends: Any["groebner", "maple"] + +Benchmark suite: HC + +- Workers: 8 +- Timeout: 1200 s +- Aggregated over: 1 runs + +**All timings in seconds.** + +|Model|groebner|maple| +|:----|---|---| +|boon|0.00|0.01| +|chandra 5|0.01|0.10| +|chandra 6|0.02|0.15| +|chandra 7|0.07|0.46| +|chandra 8|0.30| - | +|chandra 9|1.26| - | +|chandra 10|4.70| - | +|chandra 11|16.81| - | +|chandra 12|107.68| - | +|ipp|195.66| - | +|rps10| - | - | + +*Benchmarking environment:* + +* Total RAM (GiB): 188 +* Processor: Intel(R) Xeon(R) Gold 6130 CPU @ 2.10GHz +* Julia version: 1.9.2 + +Versions of the dependencies: + +* Primes : 0.5.5 +* TimerOutputs : 0.5.23 +* PrecompileTools : 1.2.0 +* MultivariatePolynomials : 0.5.3 +* Combinatorics : 1.0.2 +* AbstractAlgebra : 0.34.7 +* Nemo : 0.38.3 +* Atomix : 0.1.0 +* ExprTools : 0.1.10 +* PrettyTables : 2.3.1 diff --git a/benchmark/results/groebner/benchmark_result_3.md b/benchmark/results/groebner/benchmark_result_3.md index ddfac7d0..a29db95c 100644 --- a/benchmark/results/groebner/benchmark_result_3.md +++ b/benchmark/results/groebner/benchmark_result_3.md @@ -1,31 +1,44 @@ ## Benchmark results -2023-12-31T14:39:14.826 +2024-01-12T12:10:34.027 Benchmarked backend: groebner Benchmark suite: The rationals - Workers: 8 -- Timeout: 1200 s +- Timeout: 2000 s - Aggregated over: 1 runs **All timings in seconds.** -|Model|Total| -|-----|---| -|cyclic 7|1.19| -|cyclic 8|27.54| +|Model|Total, s| +|:----|---| +|chandra 9|1.92| +|chandra 10|4.11| +|chandra 11|14.09| +|chandra 12|90.53| +|chandra 13|548.26| +|cyclic 7|0.94| +|cyclic 8|25.17| +|cyclic 9| - | |dummy|0.00| -|eco 10|0.68| -|eco 11|3.53| -|eco 12|31.14| -|henrion 6|0.76| -|katsura 9|3.73| -|katsura 10|27.37| -|katsura 11|355.52| -|noon 8|5.29| -|noon 9|56.26| +|eco 10|0.50| +|eco 11|3.42| +|eco 12|28.89| +|eco 13|359.91| +|henrion 6|0.66| +|henrion 7|394.70| +|ipp|134.31| +|katsura 9|4.11| +|katsura 10|23.64| +|katsura 11|319.80| +|noon 7|0.97| +|noon 8|5.82| +|noon 9|42.68| +|reimer 6|0.54| +|reimer 7|10.80| +|reimer 8|816.37| *Benchmarking environment:* diff --git a/benchmark/results/groebner/benchmark_result_7.md b/benchmark/results/groebner/benchmark_result_7.md index c3faaaa7..9efe3905 100644 --- a/benchmark/results/groebner/benchmark_result_7.md +++ b/benchmark/results/groebner/benchmark_result_7.md @@ -1,12 +1,12 @@ ## Benchmark results -2024-01-12T09:04:15.474 +2024-01-12T13:17:47.010 Benchmarked backend: groebner Benchmark suite: HC -- Workers: 4 +- Workers: 8 - Timeout: 1200 s - Aggregated over: 1 runs @@ -14,16 +14,16 @@ Benchmark suite: HC |Model|Total, s| |:----|---| -|boon|0.01| +|boon|0.00| |chandra 5|0.01| -|chandra 6|0.04| -|chandra 7|0.22| -|chandra 8|0.44| -|chandra 9|2.52| -|chandra 10|5.07| -|chandra 11|17.69| -|chandra 12|108.19| -|ipp|185.73| +|chandra 6|0.02| +|chandra 7|0.07| +|chandra 8|0.30| +|chandra 9|1.26| +|chandra 10|4.70| +|chandra 11|16.81| +|chandra 12|107.68| +|ipp|195.66| |rps10| - | *Benchmarking environment:* diff --git a/benchmark/results/maple/benchmark_result_3.md b/benchmark/results/maple/benchmark_result_3.md index 5c380940..8cb70bb0 100644 --- a/benchmark/results/maple/benchmark_result_3.md +++ b/benchmark/results/maple/benchmark_result_3.md @@ -1,31 +1,44 @@ ## Benchmark results -2023-12-31T14:11:04.046 +2024-01-12T12:45:58.856 Benchmarked backend: maple Benchmark suite: The rationals - Workers: 8 -- Timeout: 1200 s +- Timeout: 2000 s - Aggregated over: 1 runs **All timings in seconds.** -|Model|Total| -|-----|---| -|cyclic 7|1.75| -|cyclic 8|30.09| -|dummy|0.02| -|eco 10|0.78| -|eco 11|5.63| -|eco 12|52.84| -|henrion 6| - | -|katsura 9|9.53| -|katsura 10|136.96| -|katsura 11| - | -|noon 8|11.05| -|noon 9|122.91| +|Model|Total, s| +|:----|---| +|chandra 9|5.85| +|chandra 10|27.81| +|chandra 11|151.60| +|chandra 12|1166.48| +|chandra 13| - | +|cyclic 7|1.50| +|cyclic 8|24.92| +|cyclic 9| - | +|dummy|0.01| +|eco 10|0.72| +|eco 11|4.72| +|eco 12|39.68| +|eco 13|532.73| +|henrion 6|1.96| +|henrion 7| - | +|ipp|196.19| +|katsura 9|9.07| +|katsura 10|102.40| +|katsura 11|1479.51| +|noon 7|1.02| +|noon 8|9.62| +|noon 9|102.74| +|reimer 6|0.65| +|reimer 7|37.55| +|reimer 8| - | *Benchmarking environment:* diff --git a/benchmark/results/maple/benchmark_result_7.md b/benchmark/results/maple/benchmark_result_7.md index a4a8fa5e..09db8f19 100644 --- a/benchmark/results/maple/benchmark_result_7.md +++ b/benchmark/results/maple/benchmark_result_7.md @@ -1,13 +1,13 @@ ## Benchmark results -2024-01-12T08:36:25.401 +2024-01-12T13:17:53.095 Benchmarked backend: maple Benchmark suite: HC -- Workers: 4 -- Timeout: 1000 s +- Workers: 8 +- Timeout: 1200 s - Aggregated over: 1 runs **All timings in seconds.** @@ -15,10 +15,14 @@ Benchmark suite: HC |Model|Total, s| |:----|---| |boon|0.01| -|chandra 5|0.07| -|chandra 6|0.13| -|chandra 7|0.36| -|chandra 8|1.40| +|chandra 5|0.10| +|chandra 6|0.15| +|chandra 7|0.46| +|chandra 8| - | +|chandra 9| - | +|chandra 10| - | +|chandra 11| - | +|chandra 12| - | |ipp| - | |rps10| - | diff --git a/benchmark/results/msolve/benchmark_result_3.md b/benchmark/results/msolve/benchmark_result_3.md index 0b849d59..381f9665 100644 --- a/benchmark/results/msolve/benchmark_result_3.md +++ b/benchmark/results/msolve/benchmark_result_3.md @@ -1,31 +1,44 @@ ## Benchmark results -2023-12-31T14:45:53.543 +2024-01-12T13:17:51.416 Benchmarked backend: msolve Benchmark suite: The rationals - Workers: 8 -- Timeout: 1200 s +- Timeout: 2000 s - Aggregated over: 1 runs **All timings in seconds.** -|Model|Total| -|-----|---| -|cyclic 7|1.20| -|cyclic 8|25.64| +|Model|Total, s| +|:----|---| +|chandra 9|0.03| +|chandra 10|0.03| +|chandra 11|0.02| +|chandra 12|0.03| +|chandra 13|0.03| +|cyclic 7|1.07| +|cyclic 8|26.84| +|cyclic 9| - | |dummy|0.03| -|eco 10|0.62| -|eco 11|3.58| -|eco 12|27.44| -|henrion 6|3.95| -|katsura 9|2.35| -|katsura 10|17.89| -|katsura 11|197.02| -|noon 8|11.92| -|noon 9|126.53| +|eco 10|0.57| +|eco 11|3.67| +|eco 12|25.40| +|eco 13|443.58| +|henrion 6|3.80| +|henrion 7| - | +|ipp|0.06| +|katsura 9|2.77| +|katsura 10|20.62| +|katsura 11|244.90| +|noon 7|1.56| +|noon 8|12.76| +|noon 9|102.05| +|reimer 6|0.28| +|reimer 7|7.06| +|reimer 8|364.48| *Benchmarking environment:* diff --git a/benchmark/results/msolve/benchmark_result_7.md b/benchmark/results/msolve/benchmark_result_7.md new file mode 100644 index 00000000..d5b5638e --- /dev/null +++ b/benchmark/results/msolve/benchmark_result_7.md @@ -0,0 +1,46 @@ +## Benchmark results + +2024-01-12T13:26:27.874 + +Benchmarked backend: msolve + +Benchmark suite: HC + +- Workers: 8 +- Timeout: 200 s +- Aggregated over: 1 runs + +**All timings in seconds.** + +|Model|Total, s| +|:----|---| +|boon|0.04| +|chandra 5|0.03| +|chandra 6|0.04| +|chandra 7|0.04| +|chandra 8|0.04| +|chandra 9|0.04| +|chandra 10|0.04| +|chandra 11|0.05| +|chandra 12|0.04| +|ipp|0.07| +|rps10| - | + +*Benchmarking environment:* + +* Total RAM (GiB): 188 +* Processor: Intel(R) Xeon(R) Gold 6130 CPU @ 2.10GHz +* Julia version: 1.9.2 + +Versions of the dependencies: + +* Primes : 0.5.5 +* TimerOutputs : 0.5.23 +* PrecompileTools : 1.2.0 +* MultivariatePolynomials : 0.5.3 +* Combinatorics : 1.0.2 +* AbstractAlgebra : 0.34.7 +* Nemo : 0.38.3 +* Atomix : 0.1.0 +* ExprTools : 0.1.10 +* PrettyTables : 2.3.1 diff --git a/benchmark/validate/benchmark_7/boon/certificate b/benchmark/validate/benchmark_7/boon/certificate new file mode 100644 index 00000000..ea69542c --- /dev/null +++ b/benchmark/validate/benchmark_7/boon/certificate @@ -0,0 +1 @@ +4231eb9f6e1dc3aa542993c239b9b54638c96ec62d1e82f29f316ff70a798811 diff --git a/benchmark/validate/benchmark_7/chandra 10/certificate b/benchmark/validate/benchmark_7/chandra 10/certificate new file mode 100644 index 00000000..fb7a858c --- /dev/null +++ b/benchmark/validate/benchmark_7/chandra 10/certificate @@ -0,0 +1 @@ +45296bfafb158e76fd0dbe49c232adaf0cb69edba7497c6bfdbf4b5829f6da10 diff --git a/benchmark/validate/benchmark_7/chandra 11/certificate b/benchmark/validate/benchmark_7/chandra 11/certificate new file mode 100644 index 00000000..9da185c1 --- /dev/null +++ b/benchmark/validate/benchmark_7/chandra 11/certificate @@ -0,0 +1 @@ +83304e951db3a095eb4b3919517ba5069d3432b3fe6c8ecd80c40e20f2e9506a diff --git a/benchmark/validate/benchmark_7/chandra 12/certificate b/benchmark/validate/benchmark_7/chandra 12/certificate new file mode 100644 index 00000000..ae9c5b1d --- /dev/null +++ b/benchmark/validate/benchmark_7/chandra 12/certificate @@ -0,0 +1 @@ +acc215429c71a754d4358bf04503fd0ff9593753b13aab95e39b8f1825eadb33 diff --git a/benchmark/validate/benchmark_7/chandra 5/certificate b/benchmark/validate/benchmark_7/chandra 5/certificate new file mode 100644 index 00000000..23ce7ae1 --- /dev/null +++ b/benchmark/validate/benchmark_7/chandra 5/certificate @@ -0,0 +1 @@ +ca7ade5550149a60eb5ec892080a1e566aea5699bd7ee2bc824856c2e4658915 diff --git a/benchmark/validate/benchmark_7/chandra 6/certificate b/benchmark/validate/benchmark_7/chandra 6/certificate new file mode 100644 index 00000000..ce72ce1b --- /dev/null +++ b/benchmark/validate/benchmark_7/chandra 6/certificate @@ -0,0 +1 @@ +ca0ec4a4c8b1293be2415393f6420435162573724ec1b3a87b0b560a120f634a diff --git a/benchmark/validate/benchmark_7/chandra 7/certificate b/benchmark/validate/benchmark_7/chandra 7/certificate new file mode 100644 index 00000000..a1b4c769 --- /dev/null +++ b/benchmark/validate/benchmark_7/chandra 7/certificate @@ -0,0 +1 @@ +3728b148d64f0c878c0a21b3ae2f13de63cfcc7c7d09f4ccb9a6a4d72a5c790c diff --git a/benchmark/validate/benchmark_7/chandra 8/certificate b/benchmark/validate/benchmark_7/chandra 8/certificate new file mode 100644 index 00000000..977351d9 --- /dev/null +++ b/benchmark/validate/benchmark_7/chandra 8/certificate @@ -0,0 +1 @@ +57120ce196f25bb424f6586cf1ab2e270316aaeca112129017abacff26b1f1d0 diff --git a/benchmark/validate/benchmark_7/chandra 9/certificate b/benchmark/validate/benchmark_7/chandra 9/certificate new file mode 100644 index 00000000..34dd74f4 --- /dev/null +++ b/benchmark/validate/benchmark_7/chandra 9/certificate @@ -0,0 +1 @@ +9c44e127e6bb261c88dec29a3d75f881e69f8879a9bd3a2a593f08056e5d4791 diff --git a/benchmark/validate/benchmark_7/ipp/certificate b/benchmark/validate/benchmark_7/ipp/certificate new file mode 100644 index 00000000..f5fc67b8 --- /dev/null +++ b/benchmark/validate/benchmark_7/ipp/certificate @@ -0,0 +1 @@ +9a19551499a5fa5d4c8379c49b7b770bebfeba221122e98568bbc291d3685b2f