From ed67920137bbfe6d1b0ee070bedc648ed3c5dff9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Fran=C3=A7ois=20Pacaud?= Date: Tue, 23 Jul 2024 21:15:15 +0200 Subject: [PATCH] remove benchmark scripts (#359) --- benchmark/.Project.toml | 8 -- benchmark/README.md | 9 -- benchmark/benchmark-cutest.jl | 112 --------------- benchmark/benchmark-power.jl | 143 ------------------- benchmark/config.jl | 38 ----- benchmark/cutest-quick-names.csv | 229 ------------------------------- benchmark/plot.jl | 84 ------------ benchmark/runbenchmarks.jl | 74 ---------- benchmark/smaller-set.jl | 33 ----- 9 files changed, 730 deletions(-) delete mode 100644 benchmark/.Project.toml delete mode 100644 benchmark/README.md delete mode 100644 benchmark/benchmark-cutest.jl delete mode 100644 benchmark/benchmark-power.jl delete mode 100644 benchmark/config.jl delete mode 100644 benchmark/cutest-quick-names.csv delete mode 100644 benchmark/plot.jl delete mode 100644 benchmark/runbenchmarks.jl delete mode 100644 benchmark/smaller-set.jl diff --git a/benchmark/.Project.toml b/benchmark/.Project.toml deleted file mode 100644 index 5490797b..00000000 --- a/benchmark/.Project.toml +++ /dev/null @@ -1,8 +0,0 @@ -[deps] -CUTEst = "1b53aba6-35b6-5f92-a507-53c67d53f819" -Ipopt = "b6b21f68-93f8-5de0-b562-5493be1d77c9" -MathOptInterface = "b8f27783-ece8-5eb3-8dc8-9495eed66fee" -NLPModelsIpopt = "f4238b75-b362-5c4c-b852-0801c9a21d71" -Plots = "91a5bcdd-55d7-5caf-9e0b-520d859cae80" -PowerModels = "c36e90e8-916a-50a6-bd94-075b64ef4655" -JuMP = "4076af6c-e467-56ae-b986-b466b2749572" \ No newline at end of file diff --git a/benchmark/README.md b/benchmark/README.md deleted file mode 100644 index d7a88628..00000000 --- a/benchmark/README.md +++ /dev/null @@ -1,9 +0,0 @@ -Running benchmark requires pacakge ArgParse. First install ArgParse with: -``` -julia -e 'import Pkg; Pkg.add("ArgParse")' -``` - -To get a help for running benchmark, run: -``` -julia runbenchmark.jl --help -``` diff --git a/benchmark/benchmark-cutest.jl b/benchmark/benchmark-cutest.jl deleted file mode 100644 index 6489ebf3..00000000 --- a/benchmark/benchmark-cutest.jl +++ /dev/null @@ -1,112 +0,0 @@ -include("config.jl") - -@everywhere using CUTEst - -if SOLVER == "master" || SOLVER == "current" - @everywhere begin - using MadNLP, MadNLPHSL - LinSol = @isdefined(MadNLPMa57) ? MadNLPMa57 : MadNLPMa57 # for older version of MadNLP - solver = nlp -> madnlp(nlp,linear_solver=LinSol,max_wall_time=900., print_level=PRINT_LEVEL, tol=1e-6) - function get_status(code::MadNLP.Status) - if code == MadNLP.SOLVE_SUCCEEDED - return 1 - elseif code == MadNLP.SOLVED_TO_ACCEPTABLE_LEVEL - return 2 - else - return 3 - end - end - end -elseif SOLVER == "ipopt" - @everywhere begin - solver = nlp -> ipopt(nlp,linear_solver="ma57",max_cpu_time=900., print_level=PRINT_LEVEL, tol=1e-6) - using NLPModelsIpopt - function get_status(code::Symbol) - if code == :first_order - return 1 - elseif code == :acceptable - return 2 - else - return 3 - end - end - end -elseif SOLVER == "knitro" - # TODO -else - error("Proper SOLVER should be given") -end - - -@everywhere function decodemodel(name) - println("Decoding $name") - finalize(CUTEstModel(name)) -end - -@everywhere function evalmodel(name,solver;gcoff=false) - println("Solving $name") - nlp = CUTEstModel(name; decode=false) - try - gcoff && GC.enable(false); - mem = @allocated begin - t = @elapsed begin - retval = solver(nlp) - end - end - gcoff && GC.enable(true); - finalize(nlp) - return (status=get_status(retval.status),time=t,mem=mem,iter=retval.iter) - catch e - finalize(nlp) - return (status=3,time=0.,mem=0,iter=0) - end - println("Solved $name") -end - -function benchmark(solver,probs;warm_up_probs = [], decode = false) - println("Warming up (forcing JIT compile)") - decode && broadcast(decodemodel,warm_up_probs) - r = [remotecall.(prob->evalmodel(prob,solver;gcoff=GCOFF),i,warm_up_probs) for i in procs() if i!= 1] - fetch.(r) - - println("Decoding problems") - decode && broadcast(decodemodel,probs) - - println("Solving problems") - retvals = pmap(prob->evalmodel(prob,solver;gcoff=GCOFF),probs) - status = [retval.status for retval in retvals] - time = [retval.time for retval in retvals] - mem = [retval.mem for retval in retvals] - iter = [retval.iter for retval in retvals] - status,time,mem,iter -end - -exclude = [ - # MadNLP running into error - # Ipopt running into error - "EG3", # lfact blows up - # Problems that are hopelessly large - "TAX213322","TAXR213322","TAX53322","TAXR53322", - "YATP1LS","YATP2LS","YATP1CLS","YATP2CLS", - "CYCLOOCT","CYCLOOCF", - "LIPPERT1", - "GAUSSELM", - "BA-L52LS","BA-L73LS","BA-L21LS" -] - -if QUICK - probs = readdlm("cutest-quick-names.csv")[:] -else - probs = CUTEst.select() -end - -filter!(e->!(e in exclude),probs) - -status,time,mem,iter = benchmark(solver,probs;warm_up_probs = ["EIGMINA"], decode = DECODE) - -f = !QUICK ? "" : "-quick" -writedlm("name-cutest$f.csv",probs,',') -writedlm("status-cutest-$(SOLVER)$f.csv",status,',') -writedlm("time-cutest-$(SOLVER)$f.csv",time,',') -writedlm("mem-cutest-$(SOLVER)$f.csv",mem,',') -writedlm("iter-cutest-$(SOLVER)$f.csv",iter,',') diff --git a/benchmark/benchmark-power.jl b/benchmark/benchmark-power.jl deleted file mode 100644 index fa8a4aa5..00000000 --- a/benchmark/benchmark-power.jl +++ /dev/null @@ -1,143 +0,0 @@ -include("config.jl") - -@everywhere begin - if haskey(ENV, "PGLIB_PATH") - const PGLIB_PATH = ENV["PGLIB_PATH"] - else - error("Unable to find path to PGLIB benchmark.\n"* - "Please set environment variable `PGLIB_PATH` to run benchmark with PowerModels.jl") - end - - using PowerModels, MathOptInterface, JuMP - const MOI = MathOptInterface - - PowerModels.silence() - - function evalmodel(prob,solver;gcoff=false) - case,type = prob - pm = instantiate_model(joinpath(PGLIB_PATH,case),type,PowerModels.build_opf) - println("Solving $(get_name(pm))") - gcoff && GC.enable(false); - retval = solver(pm) - gcoff && GC.enable(true); - return retval - end - - function get_status(code::MOI.TerminationStatusCode) - if code == MOI.LOCALLY_SOLVED - return 1 - elseif code == MOI.ALMOST_OPTIMAL - return 2 - else - return 3 - end - end - - get_name(pm) = "$(pm.data["name"])-$(typeof(pm))" -end - -if SOLVER == "master" || SOLVER == "current" - @everywhere begin - using MadNLP, MadNLPHSL - LinSol = @isdefined(MadNLPMa57) ? MadNLPMa57 : Ma57Solver - - solver = pm -> begin - set_optimizer( - pm.model,()-> MadNLP.Optimizer( - linear_solver=LinSol, - max_wall_time=900., - tol=1e-6, - print_level=PRINT_LEVEL - ) - ) - mem=@allocated begin - t=@elapsed begin - optimize_model!(pm) - end - end - return get_status(termination_status(pm.model)),t,mem,barrier_iterations(pm.model) - end - end -elseif SOLVER == "ipopt" - @everywhere begin - using Ipopt - - const ITER = [-1] - function ipopt_callback( - alg_mod::Cint,iter_count::Cint,obj_value::Float64, - inf_pr::Float64,inf_du::Float64,mu::Float64,d_norm::Float64, - regularization_size::Float64,alpha_du::Float64,alpha_pr::Float64,ls_trials::Cint) - - ITER[] += 1 - return true - end - - solver = pm -> begin - ITER[] = 0 - set_optimizer(pm.model, Ipopt.Optimizer) - set_optimizer_attributes( - pm.model, - "linear_solver"=>"ma57", - "max_cpu_time"=>900., - "tol"=>1e-6, - "print_level"=>PRINT_LEVEL - ) - MOI.set(pm.model, Ipopt.CallbackFunction(), ipopt_callback) - mem=@allocated begin - t=@elapsed begin - optimize_model!(pm) - end - end - return get_status(termination_status(pm.model)),t,mem,ITER[] - end - end -elseif SOLVER == "knitro" - # TODO -else - error("Proper SOLVER should be given") -end - - -function benchmark(solver,probs;warm_up_probs = []) - println("Warming up (forcing JIT compile)") - warm_up_pms = [ - instantiate_model(joinpath(PGLIB_PATH,case),type,PowerModels.build_opf) - for (case,type) in warm_up_probs] - println(get_name.(warm_up_pms)) - rs = [remotecall.(solver,i,warm_up_pms) for i in procs() if i!= 1] - ws = [wait.(r) for r in rs] - fs= [fetch.(r) for r in rs] - - println("Solving problems") - retvals = pmap(prob->evalmodel(prob,solver;gcoff=GCOFF),probs) - - status = [status for (status,time,mem,iter) in retvals] - time = [time for (status,time,mem,iter) in retvals] - mem = [mem for (status,time,mem,iter) in retvals] - iter = [iter for (status,time,mem,iter) in retvals] - - return status,time,mem,iter -end - -if QUICK - cases = filter!(e->(occursin("pglib_opf_case",e) && occursin("pegase",e)),readdir(PGLIB_PATH)) - types = [ACPPowerModel, ACRPowerModel] -else - cases = filter!(e->occursin("pglib_opf_case",e),readdir(PGLIB_PATH)) - types = [ACPPowerModel, ACRPowerModel, ACTPowerModel, - DCPPowerModel, DCMPPowerModel, NFAPowerModel, - DCPLLPowerModel,LPACCPowerModel, SOCWRPowerModel, - QCRMPowerModel,QCLSPowerModel] -end -probs = [(case,type) for case in cases for type in types] -name = ["$case-$type" for case in cases for type in types] - -status,time,mem,iter = benchmark(solver,probs;warm_up_probs = [ - ("pglib_opf_case1888_rte.m",ACPPowerModel) -]) - -writedlm("name-power.csv",name,',') -writedlm("status-power-$(SOLVER).csv",status) -writedlm("time-power-$(SOLVER).csv",time) -writedlm("mem-power-$(SOLVER).csv",mem) -writedlm("iter-power-$(SOLVER).csv",iter) diff --git a/benchmark/config.jl b/benchmark/config.jl deleted file mode 100644 index 740985cb..00000000 --- a/benchmark/config.jl +++ /dev/null @@ -1,38 +0,0 @@ -using Pkg, Distributed, DelimitedFiles - -const NP = ARGS[1] -const SOLVER = ARGS[2] -const VERBOSE = ARGS[3] == "true" -const QUICK = ARGS[4] == "true" -const GCOFF = ARGS[5] == "true" -const DECODE = ARGS[6] == "true" - -Pkg.instantiate() - -if SOLVER == "master" - Pkg.add(PackageSpec(name="MadNLP",rev="master")) - Pkg.add(PackageSpec(name="MadNLPHSL",rev="master")) - Pkg.build("MadNLPHSL") -elseif SOLVER == "current" - Pkg.develop(path=joinpath(@__DIR__,"..")) - Pkg.develop(path=joinpath(@__DIR__,"..","lib","MadNLPHSL")) - Pkg.build("MadNLPHSL") -elseif SOLVER == "ipopt" -elseif SOLVER == "knitro" -else - Pkg.add(PackageSpec(name="MadNLP",rev="$SOLVER")) - Pkg.add(PackageSpec(name="MadNLPHSL",rev="$SOLVER")) - Pkg.build("MadNLPHSL") -end - -# Set verbose option -if SOLVER == "ipopt" - const PRINT_LEVEL = VERBOSE ? 5 : 0 -elseif SOLVER == "knitro" - const PRINT_LEVEL = VERBOSE ? 3 : 0 -else - using MadNLP - const PRINT_LEVEL = VERBOSE ? MadNLP.INFO : MadNLP.ERROR -end - -addprocs(parse(Int,NP)) diff --git a/benchmark/cutest-quick-names.csv b/benchmark/cutest-quick-names.csv deleted file mode 100644 index 33b05671..00000000 --- a/benchmark/cutest-quick-names.csv +++ /dev/null @@ -1,229 +0,0 @@ -PRIMALC1 -DIXMAANI -HIER13 -LUKVLI7 -GAUSS2 -LUKSAN13LS -CHARDIS1 -A5NSDSIL -QPCBOEI1 -POLAK4 -DUAL2 -EXPFITA -VAREIGVL -MPC2 -BLOWEYA -DECONVB -MSS1 -POWELLBC -ACOPP57 -WALL50 -FBRAIN2 -ACOPP300 -AUG2D -HS106 -GMNCASE2 -LUKVLE8 -READING2 -MAXLIKA -CHEBYQAD -HYDROELM -GULFNE -CLEUVEN4 -HAIFAL -JUDGENE -DITTERT -TRIGON1NE -OBSTCLAE -READING6 -SBRYBND -ARGLINC -CVXQP2 -TABLE8 -NINENEW -STEENBRA -BA-L1SP -EXPFITNE -LUKSAN17 -DUALC5 -STCQP1 -DEGENQP -DEGDIAG -LEUVEN7 -DALLASM -READING8 -HS101 -GENROSEBNE -EIGENALS -READING7 -OET3 -CHANDHEQ -YATP1LS -OSCIPATH -SEMICN2U -MODBEALENE -JANNSON4 -DTOC1NC -KSS -TABLE1 -DRCAV2LQ -MGH17SLS -BRAINPC2 -PROBPENL -MGH17S -DIAGPQB -DEMBO7 -HS119 -PORTSNQP -YATP1CNE -THURBER -VESUVIOU -TAX213322 -NGONE -MRIBASIS -EXPFITC -FBRAINNE -LINVERSENE -HYDCAR6LS -GMNCASE4 -ZAMB2-11 -ALJAZZAF -HIMMELBK -WOODSNE -LUKVLI10 -CHANNEL -ORBIT2 -EIGENA2 -ACOPP118 -CHNRSNBM -LHAIFAM -NASH -CYCLIC3LS -CYCLIC3 -BA-L49LS -HS99 -CATENA -CHWIRUT1 -OPTPRLOC -HYDROELL -BIGBANK -OSORIO -SPINLS -MNISTS0 -ANTWERP -PORTFL4 -PDE1 -CURLY20 -DEGENLPA -LUBRIFC -MANCINONE -DALE -HATFLDC -INTEGREQ -NET1 -LUKSAN12 -UBH5 -AGG -WATSONNE -TAX13322 -PRIMAL1 -10FOLDTR -QPCBLEND -CYCLOOCT -FIVE20B -HUESTIS -TWIRISM1 -DMN37142 -LIARWHDNE -COOLHANS -MSS3 -BDRY2 -TRO5X5 -MSS2 -TAX53322 -CORE1 -LINSPANH -ZAMB2-9 -KSIP -CHAINWOONE -DEGTRIDL -LINCONT -TWIRIBG1 -POWER -DMN37143 -PRIMAL3 -EIGENBCO -TRIMLOSS -SPANHYD -OPTCNTRL -ROSEPETAL -SANTALS -PRIMALC8 -SPECANNE -READING5 -EXPQUAD -ARGLCLE -CHNRSBNE -MODBEALE -EIGENC -ARGTRIG -STATIC3 -CRESC132 -CHANDHEU -KISSING2 -EXPLIN -GILBERT -GPP -LUKVLI9 -RES -LCH -MUONSINE -BA-L73 -TRO21X5 -SSEBNLN -ELATTAR -TWOD -PRIMAL4 -COATING -WALL100 -MSQRTA -PRIMAL2 -ODNAMUR -GENROSENE -TRIGON2NE -CHARDIS0 -SPMSQRT -QING -SMMPSF -NUFFIELD -GOFFIN -ELEC -BA-L16LS -SYNPOP24 -ZIGZAG -SSEBLIN -BA-L1 -FCCU -CONT5-QP -QPNBAND -AIRPORT -FEEDLOC -KISSING -FERRISDC -MAKELA4 -VANDANIUMS -AVION2 -BROWNALE -ROSEPETAL2 -DEGENQPC -DRUGDISE -QINGNE -BA-L52LS -JANNSON3 -NONMSQRTNE -DECONVC -BA-L52 -WALL10 -MODEL -OPTMASS -ORTHREGF diff --git a/benchmark/plot.jl b/benchmark/plot.jl deleted file mode 100644 index 1b24c8e6..00000000 --- a/benchmark/plot.jl +++ /dev/null @@ -1,84 +0,0 @@ -using Plots, DelimitedFiles - -const CLASSES = filter(e-> e in ["cutest","power"], ARGS) -const SOLVERS = filter(e-> e in ["current","master","ipopt","knitro"], ARGS) - -const LABELS = Dict( - "current" => "MadNLP (dev)", - "master" => "MadNLP (master)", - "ipopt" => "Ipopt", - "knitro" => "Knitro", -) - -for class in CLASSES - - time = Dict() - status = Dict() - mem = Dict() - iter = Dict() - - name = readdlm("name-$class.csv")[:] - for solver in SOLVERS - status[solver] = readdlm("status-$class-$solver.csv")[:] - time[solver] = readdlm("time-$class-$solver.csv")[:] - mem[solver] = readdlm("mem-$class-$solver.csv")[:] - iter[solver] = readdlm("iter-$class-$solver.csv")[:] - end - - for (str,metric) in [("time",time),("iterations",iter),("memory",mem)] - relmetric = deepcopy(metric) - - for i=1:length(name) - top = [] - topstatus = 3 - for solver in SOLVERS - if status[solver][i] < topstatus - empty!(top) - push!(top,solver) - topstatus = status[solver][i] - elseif status[solver][i] == topstatus - push!(top,solver) - end - end - - fastest = Inf - for solver in top - fastest = min(fastest,metric[solver][i]) - end - - for solver in SOLVERS - if status[solver][i] == 3 - relmetric[solver][i] = NaN - else - relmetric[solver][i] = log2(relmetric[solver][i]/fastest) - end - end - end - - for solver in SOLVERS - filter!(a->!isnan(a),relmetric[solver]) - end - - p = plot(; - ylim=(0,1), - xlim=( - minimum(minimum(relmetric[solver]) for solver in SOLVERS), - maximum(maximum(relmetric[solver]) for solver in SOLVERS) - ), - xlabel="Not More Than 2ˣ-Times Worse Than Best Solver ($str)", - ylabel="Fraction of Problems Solved", - framestyle=:box, - legend=:bottomright) - - for solver in SOLVERS - y = [0:length(relmetric[solver]);length(relmetric[solver])]/length(name) - push!(relmetric[solver],maximum(maximum(relmetric[s]) for s in SOLVERS)) - push!(relmetric[solver],0) - plot!(p,sort(relmetric[solver]),y; - qqline=:none, - linetype=:steppost, - label=LABELS[solver]) - end - savefig(p,"$str-$class.pdf") - end -end diff --git a/benchmark/runbenchmarks.jl b/benchmark/runbenchmarks.jl deleted file mode 100644 index c41e1101..00000000 --- a/benchmark/runbenchmarks.jl +++ /dev/null @@ -1,74 +0,0 @@ -try - using ArgParse -catch e - println("Package ArgParse is required, but not installed. Install now? [y/n]") - if readline() == "y" - import Pkg - Pkg.add("ArgParse") - using ArgParse - else - exit() - end -end - -function parse_commandline() - s = ArgParseSettings() - - @add_arg_table s begin - "--nprocs", "-p" - help = "number of worker processors" - arg_type = Int - default = 1 - "--verbose", "-v" - help = "print the solver outputs" - action = :store_true - "--gcoff", "-g" - help = "turn of the julia garbage collector" - action = :store_true - "--quick", "-q" - help = "run tests with reduced number of instances" - action = :store_true - "--decode", "-d" - help = "decode the cutest instances" - action = :store_true - "testsets" - help = "testsets for benchmark (separated by comma). possible values: cutest, power" - required = true - "solvers" - help = "solvers to benchmark (separated by comma). possible values: current, master, ipopt" - required = true - end - - return parse_args(s) -end - -function main() - pargs = parse_commandline() - - CLASSES = split(pargs["testsets"],",") - SOLVERS = split(pargs["solvers"],",") - - # sanity check - issubset(CLASSES,["cutest","power"]) || error("argument testsets is incorrect") - issubset(SOLVERS,["current","master","ipopt","knitro"]) || error("argument solvers is incorrect") - - PROJECT_PATH = dirname(@__FILE__) - - cp( - joinpath(PROJECT_PATH, ".Project.toml"), - joinpath(PROJECT_PATH, "Project.toml"), - force=true - ) - - for class in CLASSES - for solver in SOLVERS - launch_script = joinpath(PROJECT_PATH, "benchmark-$class.jl") - run(`julia --project=$PROJECT_PATH $launch_script $(pargs["nprocs"]) $solver $(pargs["verbose"]) $(pargs["quick"]) $(pargs["gcoff"]) $(pargs["decode"])`) - end - end - - run(Cmd([["julia","--project=$PROJECT_PATH","plot.jl"];String.(CLASSES);String.(SOLVERS)])) - -end - -main() diff --git a/benchmark/smaller-set.jl b/benchmark/smaller-set.jl deleted file mode 100644 index 5f7365b2..00000000 --- a/benchmark/smaller-set.jl +++ /dev/null @@ -1,33 +0,0 @@ -using CUTEst, DelimitedFiles - -probs= CUTEst.select() - - -similar(a,b;atol=10,rtol=0.5) = (abs(a-b) < atol) || (abs(a-b)/max(a,b) < rtol) - -function is_similar_exists(bin,nlp) - for (name,nvar,ncon,nnzh,nnzj) in bin - if similar(nvar,nlp.meta.nvar) && similar(ncon,nlp.meta.ncon) && similar(nnzh,nlp.meta.nnzh) && similar(nnzj,nlp.meta.nnzj) - return true - end - end - return false -end - -bin = Tuple{String,Int,Int,Int,Int}[] - - -for i=1:length(probs) - prob = probs[i] - nlp = CUTEstModel(prob) - finalize(nlp) - - if is_similar_exists(bin,nlp) - println("#$cnt Skipping $(nlp.meta.name)") - else - println("#$cnt Putting $(nlp.meta.name) in the bin") - push!(bin, (nlp.meta.name,nlp.meta.nvar,nlp.meta.ncon,nlp.meta.nnzh,nlp.meta.nnzj)) - end -end - -writedlm("cutest-quick-names.csv",[name for (name,nvar,ncon,nnzh,nnzj) in bin],',')