Adding CUTEst.jl to the optimization benchmarks

.buildkite/launch_benchmarks.yml.signature

@@ -1,2 +1,3 @@
-Salted__�c?�ު��DwL�J�/���)��|p��5+c��\~o
�.��	RR�hAZR��
-$'���� 6���!�m�^����S�j�Y�����
+Salted__�c?
+�ު��DwL�J�/���)��|p��5+c��\~o
�.��	RR�hAZR��
+$'���� 6���!�m�^����S�j�Y�����

.buildkite/run_benchmark.yml

@@ -19,8 +19,8 @@ steps:
       - JuliaCI/julia#v1:
           version: "1.10"
       - staticfloat/sandbox:
-          rootfs_url: "https://github.com/thazhemadam/openmodelica-rootfs-image/releases/download/v1.23.0/rootfs-openmodelica-v1.23.0.amd64.tar.gz"
-          rootfs_treehash: "82970243dc4f188e599a976abc20951f4aba2912"
+          rootfs_url: "https://github.com/thazhemadam/ScimlBenchmarks-rootfs-image/releases/download/rootfs-image/rootfs-script.amd64.tar.gz"
+          rootfs_treehash: "943bde824554f0afad5a939341f05e3bdb393051"
           uid: 1000
           gid: 1000
           workspaces:

benchmarks/OptimizationCUTEst/CUTEst_bounded.jmd

@@ -0,0 +1,141 @@
+---
+title: CUTEst Bounded Constrained Nonlinear Optimization Benchmarks
+author: Alonso M. Cisneros
+---
+
+# Introduction
+
+CUTEst, the Constraind and Unconstrained Testing Environment is, as the name suggests is a
+collection of around 1500 problems for general nonlinear optimization used to test
+optimization routines. The wrapper
+[CUTEst.jl](https://github.com/JuliaSmoothOptimizers/CUTEst.jl) provides convenient access
+to the problem collection, which we can leverage to test the optimizers made available by
+Optimization.jl.
+
+This benchmark uses the following packages:
+
+```julia
+using Optimization
+using OptimizationNLPModels
+using CUTEst
+using OptimizationOptimJL
+using Ipopt
+using OptimizationMOI
+using OptimizationMOI: MOI as MOI
+# Analysis and plotting
+using DataFrames
+using Plots
+using StatsPlots
+```
+
+# Benchmarks
+
+We will be testing the [Ipopt]() and the [LBFGS]() optimizers on these classes of
+problems.
+
+```julia
+optimizers = [
+        Optimization.LBFGS(),
+        MOI.OptimizerWithAttributes(Ipopt.Optimizer, "print_level" => 0)
+    ]
+
+function get_stats(sol, ::Optimization.LBFGS)
+    return (length(sol.u), sol.stats.time, "LBFGS", Symbol(sol.retcode))
+end
+
+function get_stats(sol, ::OptimizationMOI.MOI.OptimizerWithAttributes)
+    return (length(sol.u), MOI.get(sol.original.model, MOI.SolveTimeSec()),
+        "Ipopt", Symbol(sol.retcode))
+end
+
+function run_benchmarks(problems, optimizers)
+    problem = String[]
+    n_vars = Int64[]
+    secs = Float64[]
+    solver = String[]
+    retcode = Symbol[]
+
+    optz = length(optimizers)
+    n = length(problems)
+
+    @info "here 1"
+
+    broadcast(c -> sizehint!(c, optz * n), [problem, n_vars, secs, solver, retcode])
+
+    @info "here 2"
+
+    for prob_name in problems
+        @info prob_name
+        nlp_prob = CUTEstModel(prob_name)
+        prob = OptimizationNLPModels.OptimizationProblem(nlp_prob, Optimization.AutoForwardDiff())
+        for optimizer in optimizers
+            sol = solve(prob, optimizer; maxiters = 1e7)
+
+            @info "Solved $(prob_name) with $(optimizer)"
+            vars, time, alg, code = get_stats(sol, optimizer)
+
+            push!(problem, prob_name)
+            push!(n_vars, vars)
+            push!(secs, time)
+            push!(solver, alg)
+            push!(retcode, code)
+
+        end
+        finalize(nlp_prob)
+    end
+
+    return DataFrame(problem = problem, n_vars = n_vars, secs = secs, solver = solver,
+        retcode = retcode)
+end
+```
+
+## Equality/Inequality constrained problems with bounded variables
+
+Now we analyze the subset of problems with equality/inequality constraints and whose
+variables are bounded. There are 666 such problems.
+
+The following figure shows the results of the same benchmarks previously described for the
+problems on this section.
+
+```julia
+@info "before"
+eq_bou_problems = CUTEst.select(min_con=1, only_equ_con=true, only_free_var=false)
+@info "after1"
+
+# Analysis
+eq_bou_results =  run_benchmarks(eq_bou_problems, optimizers)
+@info "after2"
+
+@df eq_bou_results scatter(:n_vars, :secs,
+        group = :solver,
+        xlabel = "n. variables",
+        ylabel = "secs.",
+        title = "Time to solution by optimizer and number of vars",
+    )
+@info "after3"
+```
+
+Next, we examine the same relationship for problems with inequality-constrained problems,
+of which there are 244.
+
+```julia
+@info "after4"
+neq_bou_problems = CUTEst.select(min_con=1, only_ineq_con=true, only_free_var=false)
+@info "after5"
+
+# Analysis
+neq_bou_results =  run_benchmarks(neq_bou_problems, optimizers)
+@info "after6"
+
+@df neq_bou_results scatter(:n_vars, :secs,
+        group = :solver,
+        xlabel = "n. variables",
+        ylabel = "secs.",
+        title = "Time to solution by optimizer and number of vars",
+    )
+```
+
+```julia, echo = false
+using SciMLBenchmarks
+SciMLBenchmarks.bench_footer(WEAVE_ARGS[:folder],WEAVE_ARGS[:file])
+```

benchmarks/OptimizationCUTEst/CUTEst_quadratic.jmd

@@ -0,0 +1,109 @@
+---
+title: CUTEst Quadratic Programming with Linear Constraints Benchmarks 
+author: Alonso M. Cisneros
+---
+
+# Introduction
+
+CUTEst, the Constraind and Unconstrained Testing Environment is, as the name suggests is a
+collection of around 1500 problems for general nonlinear optimization used to test
+optimization routines. The wrapper
+[CUTEst.jl](https://github.com/JuliaSmoothOptimizers/CUTEst.jl) provides convenient access
+to the problem collection, which we can leverage to test the optimizers made available by
+Optimization.jl.
+
+This benchmark uses the following packages:
+
+```julia
+using Optimization
+using OptimizationNLPModels
+using CUTEst
+using OptimizationOptimJL
+using Ipopt
+using OptimizationMOI
+using OptimizationMOI: MOI as MOI
+# Analysis and plotting
+using DataFrames
+using Plots
+using StatsPlots
+```
+
+# Benchmarks
+
+We will be testing the [Ipopt]() and the [LBFGS]() optimizers on these classes of
+problems.
+
+```julia
+optimizers = [
+        Optimization.LBFGS(),
+        MOI.OptimizerWithAttributes(Ipopt.Optimizer, "print_level" => 0)
+    ]
+
+function get_stats(sol, ::Optimization.LBFGS)
+    return (length(sol.u), sol.stats.time, "LBFGS", Symbol(sol.retcode))
+end
+
+function get_stats(sol, ::OptimizationMOI.MOI.OptimizerWithAttributes)
+    return (length(sol.u), MOI.get(sol.original.model, MOI.SolveTimeSec()),
+        "Ipopt", Symbol(sol.retcode))
+end
+
+function run_benchmarks(problems, optimizers)
+    problem = String[]
+    n_vars = Int64[]
+    secs = Float64[]
+    solver = String[]
+    retcode = Symbol[]
+
+    optz = length(optimizers)
+    n = length(problems)
+
+    broadcast(c -> sizehint!(c, optz * n), [problem, n_vars, secs, solver, retcode])
+
+    for prob_name in problems
+        nlp_prob = CUTEstModel(prob_name)
+        prob = OptimizationNLPModels.OptimizationProblem(nlp_prob, Optimization.AutoForwardDiff())
+        for optimizer in optimizers
+            sol = solve(prob, optimizer; maxiters = 1e7)
+
+            @info "Solved $(prob_name) with $(optimizer)"
+            vars, time, alg, code = get_stats(sol, optimizer)
+
+            push!(problem, prob_name)
+            push!(n_vars, vars)
+            push!(secs, time)
+            push!(solver, alg)
+            push!(retcode, code)
+
+        end
+        finalize(nlp_prob)
+    end
+
+    return DataFrame(problem = problem, n_vars = n_vars, secs = secs, solver = solver,
+        retcode = retcode)
+end
+```
+
+# Quadratic programs with linear constraints
+
+Lastly, we examine the problems with a quadratic objective function and only linear
+constraints. There are 252 such problems in the suite.
+
+```julia
+quad_problems = CUTEst.select(objtype="quadratic", contype="linear")
+
+# Analysis
+quad_results =  run_benchmarks(quad_problems, optimizers)
+
+@df neq_bou_results scatter(:n_vars, :secs,
+        group = :solver,
+        xlabel = "n. variables",
+        ylabel = "secs.",
+        title = "Time to solution by optimizer and number of vars",
+    )
+```
+
+```julia, echo = false
+using SciMLBenchmarks
+SciMLBenchmarks.bench_footer(WEAVE_ARGS[:folder],WEAVE_ARGS[:file])
+```