Skip to content

Commit

Permalink
upd benchmarks
Browse files Browse the repository at this point in the history
  • Loading branch information
Sasha Demin committed Jan 27, 2024
1 parent a0baf1c commit c53a3f9
Show file tree
Hide file tree
Showing 32 changed files with 623 additions and 433 deletions.
4 changes: 2 additions & 2 deletions benchmark/Manifest.toml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -153,9 +153,9 @@ version = "6.2.1+2"

[[deps.Groebner]]
deps = ["AbstractAlgebra", "Atomix", "Combinatorics", "ExprTools", "HostCPUFeatures", "Logging", "MultivariatePolynomials", "Nemo", "PrecompileTools", "PrettyTables", "Primes", "Printf", "Random", "TimerOutputs"]
path = ".."
git-tree-sha1 = "5eb13678ae76a60d70090a4c50d1dc3eab0ce180"
uuid = "0b43b601-686d-58a3-8a1c-6623616c7cd4"
version = "0.6.3"
version = "0.6.4"

[[deps.GroupsCore]]
deps = ["Markdown", "Random"]
Expand Down
4 changes: 2 additions & 2 deletions benchmark/README.md
View file
Original file line number Diff line number Diff line change
Expand Up @@ -4,10 +4,10 @@ Benchmarks for `Groebner.jl`, `Maple`, `msolve`, `OpenF4`, `Singular`.

The definitions of benchmark systems can be found in `benchmark_systems.jl`.

Computed bases will be verified against short certificates that are assumed to
Computed bases will be verified against short SHA certificate, which are assumed to
be correct.

For running the benchmarks, you will need a Julia client v1.6+ installed.
For running the benchmarks, you will need a Julia client v1.6+ installed (preferably 1.9).
See the official installation guide at
https://julialang.org/downloads/platform/.

Expand Down
18 changes: 9 additions & 9 deletions benchmark/benchmark_systems.jl
View file
Original file line number Diff line number Diff line change
Expand Up @@ -36,10 +36,10 @@ end
function benchmark_set_0()
(
name="dummy benchmark set",
field=AbstractAlgebra.QQ,
field=AbstractAlgebra.GF(7),
systems=[
dummy_system("dummy 1", AbstractAlgebra.QQ),
dummy_system("dummy 2", AbstractAlgebra.QQ)
dummy_system("dummy 1", AbstractAlgebra.GF(7)),
dummy_system("dummy 2", AbstractAlgebra.GF(7))
]
)
end
Expand All @@ -52,7 +52,7 @@ function benchmark_set_1()
("cyclic 7", Groebner.cyclicn(7, k=ground_field)),
("cyclic 8", Groebner.cyclicn(8, k=ground_field)),
("cyclic 9", Groebner.cyclicn(9, k=ground_field)),
("cyclic 10", Groebner.cyclicn(10, k=ground_field)),
# ("cyclic 10", Groebner.cyclicn(10, k=ground_field)),
("katsura 10", Groebner.katsuran(10, k=ground_field)),
("katsura 11", Groebner.katsuran(11, k=ground_field)),
("katsura 12", Groebner.katsuran(12, k=ground_field)),
Expand All @@ -65,15 +65,15 @@ function benchmark_set_1()
("noon 8", Groebner.noonn(8, k=ground_field)),
("noon 9", Groebner.noonn(9, k=ground_field)),
("noon 10", Groebner.noonn(10, k=ground_field)),
("noon 11", Groebner.noonn(11, k=ground_field)),
# ("noon 11", Groebner.noonn(11, k=ground_field)),
("henrion 5", Groebner.henrion5(k=ground_field)),
("henrion 6", Groebner.henrion6(k=ground_field)),
("henrion 7", Groebner.henrion7(k=ground_field)),
("henrion 8", Groebner.henrion8(k=ground_field)),
("reimer 6", Groebner.reimern(6, k=ground_field)),
("reimer 7", Groebner.reimern(7, k=ground_field)),
("reimer 8", Groebner.reimern(8, k=ground_field)),
("reimer 9", Groebner.reimern(9, k=ground_field)),
# ("reimer 9", Groebner.reimern(9, k=ground_field)),
("chandra 11", Groebner.chandran(11, k=ground_field)),
("chandra 12", Groebner.chandran(12, k=ground_field)),
("chandra 13", Groebner.chandran(13, k=ground_field)),
Expand All @@ -91,7 +91,7 @@ function benchmark_set_2()
("cyclic 7", Groebner.cyclicn(7, k=ground_field)),
("cyclic 8", Groebner.cyclicn(8, k=ground_field)),
("cyclic 9", Groebner.cyclicn(9, k=ground_field)),
("cyclic 10", Groebner.cyclicn(10, k=ground_field)),
# ("cyclic 10", Groebner.cyclicn(10, k=ground_field)),
("katsura 10", Groebner.katsuran(10, k=ground_field)),
("katsura 11", Groebner.katsuran(11, k=ground_field)),
("katsura 12", Groebner.katsuran(12, k=ground_field)),
Expand All @@ -104,15 +104,15 @@ function benchmark_set_2()
("noon 8", Groebner.noonn(8, k=ground_field)),
("noon 9", Groebner.noonn(9, k=ground_field)),
("noon 10", Groebner.noonn(10, k=ground_field)),
("noon 11", Groebner.noonn(11, k=ground_field)),
# ("noon 11", Groebner.noonn(11, k=ground_field)),
("henrion 5", Groebner.henrion5(k=ground_field)),
("henrion 6", Groebner.henrion6(k=ground_field)),
("henrion 7", Groebner.henrion7(k=ground_field)),
("henrion 8", Groebner.henrion8(k=ground_field)),
("reimer 6", Groebner.reimern(6, k=ground_field)),
("reimer 7", Groebner.reimern(7, k=ground_field)),
("reimer 8", Groebner.reimern(8, k=ground_field)),
("reimer 9", Groebner.reimern(9, k=ground_field)),
# ("reimer 9", Groebner.reimern(9, k=ground_field)),
("chandra 11", Groebner.chandran(11, k=ground_field)),
("chandra 12", Groebner.chandran(12, k=ground_field)),
("chandra 13", Groebner.chandran(13, k=ground_field)),
Expand Down
8 changes: 4 additions & 4 deletions benchmark/generate/benchmark_systems/biomodels/parser.jl
View file
Original file line number Diff line number Diff line change
Expand Up @@ -4,19 +4,19 @@ import AbstractAlgebra
function read_BIOMDs(nspecies)
@info "Reading biomodels"
systems = []
for smth in readdir(abspath("benchmark/biomodels"))
for smth in readdir((@__DIR__))
system = []

@info "" smth
!(occursin("BIOMD", smth)) && continue

io = open(abspath("benchmark/biomodels/$smth/species_map.txt"), "r")
io = open((@__DIR__) * "/$smth/species_map.txt", "r")
vs = map(strip ∘ first, map(split, readlines(io)))
close(io)
# @info "variables" vs
!(length(vs) in nspecies) && continue

io = open(abspath("benchmark/biomodels/$smth/parameters.txt"), "r")
io = open((@__DIR__) * "/$smth/parameters.txt", "r")
params = readlines(io)
params = map(p -> replace(p, "/" => "//"), params)
params = map(params) do p
Expand All @@ -30,7 +30,7 @@ function read_BIOMDs(nspecies)
end
close(io)

io = open(abspath("benchmark/biomodels/$smth/odes.txt"), "r")
io = open((@__DIR__) * "/$smth/odes.txt", "r")
odes = readlines(io)
close(io)

Expand Down
4 changes: 2 additions & 2 deletions benchmark/generate/groebner/Manifest.toml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -153,9 +153,9 @@ version = "6.2.1+2"

[[deps.Groebner]]
deps = ["AbstractAlgebra", "Atomix", "Combinatorics", "ExprTools", "HostCPUFeatures", "Logging", "MultivariatePolynomials", "Nemo", "PrecompileTools", "PrettyTables", "Primes", "Printf", "Random", "TimerOutputs"]
path = "../../.."
git-tree-sha1 = "5eb13678ae76a60d70090a4c50d1dc3eab0ce180"
uuid = "0b43b601-686d-58a3-8a1c-6623616c7cd4"
version = "0.6.3"
version = "0.6.4"

[[deps.GroupsCore]]
deps = ["Markdown", "Random"]
Expand Down
29 changes: 26 additions & 3 deletions benchmark/generate/learn_apply/Manifest.toml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -50,6 +50,12 @@ git-tree-sha1 = "f1f03a9fa24271160ed7e73051fba3c1a759b53f"
uuid = "6e4b80f9-dd63-53aa-95a3-0cdb28fa8baf"
version = "1.4.0"

[[deps.BitTwiddlingConvenienceFunctions]]
deps = ["Static"]
git-tree-sha1 = "0c5f81f47bbbcf4aea7b2959135713459170798b"
uuid = "62783981-4cbd-42fc-bca8-16325de8dc4b"
version = "0.1.5"

[[deps.Calcium_jll]]
deps = ["Antic_jll", "Arb_jll", "Artifacts", "FLINT_jll", "GMP_jll", "JLLWrappers", "Libdl", "MPFR_jll", "Pkg"]
git-tree-sha1 = "37fd335ecca9bd6ab25a2a4d26b7b2f6fe64c246"
Expand Down Expand Up @@ -146,17 +152,28 @@ uuid = "781609d7-10c4-51f6-84f2-b8444358ff6d"
version = "6.2.1+2"

[[deps.Groebner]]
deps = ["AbstractAlgebra", "Atomix", "Combinatorics", "ExprTools", "Logging", "MultivariatePolynomials", "Nemo", "PrecompileTools", "PrettyTables", "Primes", "Printf", "Random", "TimerOutputs"]
path = "../../.."
deps = ["AbstractAlgebra", "Atomix", "Combinatorics", "ExprTools", "HostCPUFeatures", "Logging", "MultivariatePolynomials", "Nemo", "PrecompileTools", "PrettyTables", "Primes", "Printf", "Random", "TimerOutputs"]
git-tree-sha1 = "5eb13678ae76a60d70090a4c50d1dc3eab0ce180"
uuid = "0b43b601-686d-58a3-8a1c-6623616c7cd4"
version = "0.6.3"
version = "0.6.4"

[[deps.GroupsCore]]
deps = ["Markdown", "Random"]
git-tree-sha1 = "6df9cd6ee79fc59feab33f63a1b3c9e95e2461d5"
uuid = "d5909c97-4eac-4ecc-a3dc-fdd0858a4120"
version = "0.4.2"

[[deps.HostCPUFeatures]]
deps = ["BitTwiddlingConvenienceFunctions", "IfElse", "Libdl", "Static"]
git-tree-sha1 = "eb8fed28f4994600e29beef49744639d985a04b2"
uuid = "3e5b6fbb-0976-4d2c-9146-d79de83f2fb0"
version = "0.1.16"

[[deps.IfElse]]
git-tree-sha1 = "debdd00ffef04665ccbb3e150747a77560e8fad1"
uuid = "615f187c-cbe4-4ef1-ba3b-2fcf58d6d173"
version = "0.1.1"

[[deps.IntegerMathUtils]]
git-tree-sha1 = "b8ffb903da9f7b8cf695a8bead8e01814aa24b30"
uuid = "18e54dd8-cb9d-406c-a71d-865a43cbb235"
Expand Down Expand Up @@ -364,6 +381,12 @@ uuid = "6462fe0b-24de-5631-8697-dd941f90decc"
deps = ["Libdl", "LinearAlgebra", "Random", "Serialization", "SuiteSparse_jll"]
uuid = "2f01184e-e22b-5df5-ae63-d93ebab69eaf"

[[deps.Static]]
deps = ["IfElse"]
git-tree-sha1 = "f295e0a1da4ca425659c57441bcb59abb035a4bc"
uuid = "aedffcd0-7271-4cad-89d0-dc628f76c6d3"
version = "0.8.8"

[[deps.Statistics]]
deps = ["LinearAlgebra", "SparseArrays"]
uuid = "10745b16-79ce-11e8-11f9-7d13ad32a3b2"
Expand Down
17 changes: 15 additions & 2 deletions benchmark/generate/learn_apply/run_in_learn_apply.jl
View file
Original file line number Diff line number Diff line change
Expand Up @@ -60,6 +60,12 @@ function process_system()
timing3 = @timed flag, _ =
groebner_apply!(trace, (system, system, system, system), threaded=:no)
@assert flag
timing4 = @timed flag, _ = groebner_apply!(
trace,
(system, system, system, system, system, system, system, system),
threaded=:no
)
@assert flag
@debug "Result is" gb
if VALIDATE
output_fn = (@__DIR__) * "/$BENCHMARK_DIR/$PROBLEM_NAME/$(output_filename())"
Expand Down Expand Up @@ -92,6 +98,8 @@ function process_system()
min(runtime[PROBLEM_NAME][:total_time_apply], timing2.time)
runtime[PROBLEM_NAME][:total_time_apply_4x] =
min(runtime[PROBLEM_NAME][:total_time_apply_4x], timing3.time)
runtime[PROBLEM_NAME][:total_time_apply_8x] =
min(runtime[PROBLEM_NAME][:total_time_apply_8x], timing4.time)
end
# for cat in ID_TIME_CATEGORIES
# if haskey(runtime[PROBLEM_NAME], cat)
Expand All @@ -104,8 +112,13 @@ function dump_timings()
timings = ""
timings *= "$PROBLEM_NAME\n"
for (key, model_runtime) in runtime
for c in
[:total_time_F4, :total_time_learn, :total_time_apply, :total_time_apply_4x]
for c in [
:total_time_F4,
:total_time_learn,
:total_time_apply,
:total_time_apply_4x,
:total_time_apply_8x
]
timings *= "$c, "
timings *= string(model_runtime[c]) * "\n"
end
Expand Down
80 changes: 1 addition & 79 deletions benchmark/generate/openf4/Manifest.toml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -2,7 +2,7 @@

julia_version = "1.9.2"
manifest_format = "2.0"
project_hash = "eb2490cccda0f14cdfe8e2f4733f8e311a7e9319"
project_hash = "623d5d4f83b54da08b7ad95e49fa725c6b1f466a"

[[deps.AbstractAlgebra]]
deps = ["GroupsCore", "InteractiveUtils", "LinearAlgebra", "MacroTools", "Preferences", "Random", "RandomExtensions", "SparseArrays", "Test"]
Expand Down Expand Up @@ -32,27 +32,6 @@ git-tree-sha1 = "d9a9701b899b30332bbcb3e1679c41cce81fb0e8"
uuid = "6e4b80f9-dd63-53aa-95a3-0cdb28fa8baf"
version = "1.3.2"

[[deps.ChainRulesCore]]
deps = ["Compat", "LinearAlgebra", "SparseArrays"]
git-tree-sha1 = "e30f2f4e20f7f186dc36529910beaedc60cfa644"
uuid = "d360d2e6-b24c-11e9-a2a3-2a2ae2dbcce4"
version = "1.16.0"

[[deps.Combinatorics]]
git-tree-sha1 = "08c8b6831dc00bfea825826be0bc8336fc369860"
uuid = "861a8166-3701-5b0c-9a16-15d98fcdc6aa"
version = "1.0.2"

[[deps.Compat]]
deps = ["UUIDs"]
git-tree-sha1 = "e460f044ca8b99be31d35fe54fc33a5c33dd8ed7"
uuid = "34da2185-b29b-5c13-b0c7-acf172513d20"
version = "4.9.0"
weakdeps = ["Dates", "LinearAlgebra"]

[deps.Compat.extensions]
CompatLinearAlgebraExt = "LinearAlgebra"

[[deps.CompilerSupportLibraries_jll]]
deps = ["Artifacts", "Libdl"]
uuid = "e66e0078-7015-5450-92f7-15fbd957f2ae"
Expand All @@ -64,12 +43,6 @@ git-tree-sha1 = "fcbb72b032692610bfbdb15018ac16a36cf2e406"
uuid = "adafc99b-e345-5852-983c-f28acb93d879"
version = "0.3.1"

[[deps.DataStructures]]
deps = ["Compat", "InteractiveUtils", "OrderedCollections"]
git-tree-sha1 = "3dbd312d370723b6bb43ba9d02fc36abade4518d"
uuid = "864edb3b-99cc-5e75-8d2d-829cb0a9cfe8"
version = "0.18.15"

[[deps.Dates]]
deps = ["Printf"]
uuid = "ade2ca70-3891-5945-98fb-dc099432e06a"
Expand All @@ -83,31 +56,15 @@ deps = ["ArgTools", "FileWatching", "LibCURL", "NetworkOptions"]
uuid = "f43a241f-c20a-4ad4-852c-f6b1247861c6"
version = "1.6.0"

[[deps.ExprTools]]
git-tree-sha1 = "27415f162e6028e81c72b82ef756bf321213b6ec"
uuid = "e2ba6199-217a-4e67-a87a-7c52f15ade04"
version = "0.1.10"

[[deps.FileWatching]]
uuid = "7b1f6079-737a-58dc-b8bc-7a2ca5c1b5ee"

[[deps.Groebner]]
deps = ["AbstractAlgebra", "Combinatorics", "ExprTools", "Logging", "MultivariatePolynomials", "Primes", "Random", "SIMD", "SnoopPrecompile"]
git-tree-sha1 = "44f595de4f6485ab5ba71fe257b5eadaa3cf161e"
uuid = "0b43b601-686d-58a3-8a1c-6623616c7cd4"
version = "0.4.4"

[[deps.GroupsCore]]
deps = ["Markdown", "Random"]
git-tree-sha1 = "9e1a5e9f3b81ad6a5c613d181664a0efc6fe6dd7"
uuid = "d5909c97-4eac-4ecc-a3dc-fdd0858a4120"
version = "0.4.0"

[[deps.IntegerMathUtils]]
git-tree-sha1 = "b8ffb903da9f7b8cf695a8bead8e01814aa24b30"
uuid = "18e54dd8-cb9d-406c-a71d-865a43cbb235"
version = "0.1.2"

[[deps.InteractiveUtils]]
deps = ["Markdown"]
uuid = "b77e0a4c-d291-57a0-90e8-8db25a27a240"
Expand Down Expand Up @@ -169,18 +126,6 @@ uuid = "a63ad114-7e13-5084-954f-fe012c677804"
uuid = "14a3606d-f60d-562e-9121-12d972cd8159"
version = "2022.10.11"

[[deps.MultivariatePolynomials]]
deps = ["ChainRulesCore", "DataStructures", "LinearAlgebra", "MutableArithmetics"]
git-tree-sha1 = "6c2e970692b6f4fed2508865c43a0f67f3820cf4"
uuid = "102ac46a-7ee4-5c85-9060-abc95bfdeaa3"
version = "0.5.2"

[[deps.MutableArithmetics]]
deps = ["LinearAlgebra", "SparseArrays", "Test"]
git-tree-sha1 = "6985021d02ab8c509c841bb8b2becd3145a7b490"
uuid = "d8a4904e-b15c-11e9-3269-09a3773c0cb0"
version = "1.3.3"

[[deps.NetworkOptions]]
uuid = "ca575930-c2e3-43a9-ace4-1e988b2c1908"
version = "1.2.0"
Expand All @@ -190,11 +135,6 @@ deps = ["Artifacts", "CompilerSupportLibraries_jll", "Libdl"]
uuid = "4536629a-c528-5b80-bd46-f80d51c5b363"
version = "0.3.21+4"

[[deps.OrderedCollections]]
git-tree-sha1 = "2e73fe17cac3c62ad1aebe70d44c963c3cfdc3e3"
uuid = "bac558e1-5e72-5ebc-8fee-abe8a469f55d"
version = "1.6.2"

[[deps.Parsers]]
deps = ["Dates", "PrecompileTools", "UUIDs"]
git-tree-sha1 = "716e24b21538abc91f6205fd1d8363f39b442851"
Expand All @@ -218,12 +158,6 @@ git-tree-sha1 = "00805cd429dcb4870060ff49ef443486c262e38e"
uuid = "21216c6a-2e73-6563-6e65-726566657250"
version = "1.4.1"

[[deps.Primes]]
deps = ["IntegerMathUtils"]
git-tree-sha1 = "4c9f306e5d6603ae203c2000dd460d81a5251489"
uuid = "27ebfcd6-29c5-5fa9-bf4b-fb8fc14df3ae"
version = "0.5.4"

[[deps.Printf]]
deps = ["Unicode"]
uuid = "de0858da-6303-5e67-8744-51eddeeeb8d7"
Expand Down Expand Up @@ -256,21 +190,9 @@ version = "0.4.3"
uuid = "ea8e919c-243c-51af-8825-aaa63cd721ce"
version = "0.7.0"

[[deps.SIMD]]
deps = ["PrecompileTools"]
git-tree-sha1 = "0e270732477b9e551d884e6b07e23bb2ec947790"
uuid = "fdea26ae-647d-5447-a871-4b548cad5224"
version = "3.4.5"

[[deps.Serialization]]
uuid = "9e88b42a-f829-5b0c-bbe9-9e923198166b"

[[deps.SnoopPrecompile]]
deps = ["Preferences"]
git-tree-sha1 = "e760a70afdcd461cf01a575947738d359234665c"
uuid = "66db9d55-30c0-4569-8b51-7e840670fc0c"
version = "1.0.3"

[[deps.Sockets]]
uuid = "6462fe0b-24de-5631-8697-dd941f90decc"

Expand Down
1 change: 0 additions & 1 deletion benchmark/generate/openf4/Project.toml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -5,7 +5,6 @@ BenchmarkTools = "6e4b80f9-dd63-53aa-95a3-0cdb28fa8baf"
CpuId = "adafc99b-e345-5852-983c-f28acb93d879"
Dates = "ade2ca70-3891-5945-98fb-dc099432e06a"
Distributed = "8ba89e20-285c-5b6f-9357-94700520ee1b"
Groebner = "0b43b601-686d-58a3-8a1c-6623616c7cd4"
Logging = "56ddb016-857b-54e1-b83d-db4d58db5568"
Printf = "de0858da-6303-5e67-8744-51eddeeeb8d7"
ProgressMeter = "92933f4c-e287-5a05-a399-4b506db050ca"
Expand Down
Loading

0 comments on commit c53a3f9

Please sign in to comment.