Runner: run multiple versions of each solver

README.md

@@ -57,11 +57,15 @@ Remember to activate the appropriate virtual environment before running the runn
 
 1. **Run Benchmark Runner**
    ```shell
-   python runner/run_benchmarks.py benchmarks/benchmark_config.yaml
+   ./runner/benchmark_all.sh
    ```
-
    The app will save the runtime and memory consumption into a CSV file.
 
+   *Note: If you encounter a "permission denied" error, make sure to set the script as executable by running:*
+   ```shell
+   chmod +x ./runner/benchmark_all.sh
+   ```
+
 1. **Run Website**
    ```shell
    streamlit run website/app.py

results/benchmark_results.csv

@@ -1,19 +1,21 @@
-Benchmark,Solver,Status,Termination Condition,Objective Value,Runtime (s),Memory Usage (MB),Max Integrality Violation,Duality Gap
-pypsa-eur-sec-2-24h,highs,ok,optimal,15337522234.402815,52.85370707511902,431.176,,
-pypsa-eur-sec-2-24h,glpk,TO,Timeout,,900,193.164,,
-pypsa-eur-sec-2-24h,scip,ok,optimal,15337522234.40238,284.5882496833801,761.444,,
-pypsa-eur-elec-trex-3-24h,highs,ok,optimal,7250311849.251861,16.958754539489746,303.32,,
-pypsa-eur-elec-trex-3-24h,glpk,ok,optimal,7250311849.0,47.73776602745056,230.576,,
-pypsa-eur-elec-trex-3-24h,scip,ok,optimal,7250311849.251635,21.06801414489746,427.352,,
-pypsa-eur-elec-op-3-24h,highs,ok,optimal,7346414429.2273445,6.1663877964019775,289.2,,
-pypsa-eur-elec-op-3-24h,glpk,ok,optimal,7346414429.0,29.45422053337097,231.088,,
-pypsa-eur-elec-op-3-24h,scip,ok,optimal,7346414429.227432,24.30247139930725,426.44,,
-pypsa-eur-elec-op-ucconv-3-24h,highs,ok,optimal,10530583212.988428,150.31189489364624,626.752,0,0
-pypsa-eur-elec-op-ucconv-3-24h,glpk,TO,Timeout,,900,182.904,0,0
-pypsa-eur-elec-op-ucconv-3-24h,scip,TO,Timeout,,900,760.456,0,0
-pypsa-gas+wind+sol+ely-1-1h,highs,ok,optimal,31714138111.861046,10.052644968032837,464.252,,
-pypsa-gas+wind+sol+ely-1-1h,glpk,ok,optimal,31714138110.0,137.99040961265564,299.336,,
-pypsa-gas+wind+sol+ely-1-1h,scip,ok,optimal,31714138111.860943,173.0952386856079,755.6,,
-pypsa-gas+wind+sol+ely-ucgas-1-1h,highs,ok,optimal,84667526618.3079,104.27388262748718,526.808,0,0
-pypsa-gas+wind+sol+ely-ucgas-1-1h,glpk,ok,optimal,84667526620.0,307.3840732574463,306.436,0,0
-pypsa-gas+wind+sol+ely-ucgas-1-1h,scip,ok,optimal,84667526618.30772,429.5963785648346,852.612,0,0
+Benchmark,Solver,Status,Termination Condition,Objective Value,Runtime (s),Memory Usage (MB),Max Integrality Violation,Duality Gap,Solver Version,Solver Release Year
+pypsa-eur-sec-2-24h,highs,ok,optimal,15337522234.402815,52.85370707511902,431.176,,,,2024
+pypsa-eur-sec-2-24h,glpk,TO,Timeout,,900,193.164,,,,2024
+pypsa-eur-sec-2-24h,scip,ok,optimal,15337522234.40238,284.5882496833801,761.444,,,,2024
+pypsa-eur-elec-trex-3-24h,highs,ok,optimal,7250311849.251861,16.958754539489746,303.32,,,,2024
+pypsa-eur-elec-trex-3-24h,glpk,ok,optimal,7250311849.0,47.73776602745056,230.576,,,,2024
+pypsa-eur-elec-trex-3-24h,scip,ok,optimal,7250311849.251635,21.06801414489746,427.352,,,,2024
+pypsa-eur-elec-op-3-24h,highs,ok,optimal,7346414429.2273445,6.1663877964019775,289.2,,,,2024
+pypsa-eur-elec-op-3-24h,glpk,ok,optimal,7346414429.0,29.45422053337097,231.088,,,,2024
+pypsa-eur-elec-op-3-24h,scip,ok,optimal,7346414429.227432,24.30247139930725,426.44,,,,2024
+pypsa-eur-elec-op-ucconv-3-24h,highs,ok,optimal,10530583212.988428,150.31189489364624,626.752,0,0,,2024
+pypsa-eur-elec-op-ucconv-3-24h,glpk,TO,Timeout,,900,182.904,0,0,,2024
+pypsa-eur-elec-op-ucconv-3-24h,scip,TO,Timeout,,900,760.456,0,0,,2024
+pypsa-gas+wind+sol+ely-1-1h,highs,ok,optimal,31714138111.861046,10.052644968032837,464.252,,,,2024
+pypsa-gas+wind+sol+ely-1-1h,glpk,ok,optimal,31714138110.0,137.99040961265564,299.336,,,,2024
+pypsa-gas+wind+sol+ely-1-1h,scip,ok,optimal,31714138111.860943,173.0952386856079,755.6,,,,2024
+pypsa-gas+wind+sol+ely-ucgas-1-1h,highs,ok,optimal,84667526618.3079,104.27388262748718,526.808,0,0,5.2.1,2024
+pypsa-gas+wind+sol+ely-ucgas-1-1h,glpk,ok,optimal,84667526620.0,307.3840732574463,306.436,0,0,,2024
+pypsa-gas+wind+sol+ely-ucgas-1-1h,scip,ok,optimal,84667526618.30772,429.5963785648346,852.612,0,0,,2024
+pypsa-gas+wind+sol+ely-ucgas-1-1h,highs,ok,optimal,84667526618.3079,85.89334106445312,502.912,0.49998257402330637,0.0,1.5.3,2023
+pypsa-gas+wind+sol+ely-ucgas-1-1h,highs,ok,optimal,84667526618.3079,87.3093752861023,502.856,0.49998257402330637,0.0,1.5.0.dev0,2022

runner/benchmark_all.sh

@@ -0,0 +1,43 @@
+#!/bin/bash
+
+REQUIREMENTS_PATH="./"
+BENCHMARK_SCRIPT="./runner/run_benchmarks.py"
+BENCHMARK_CONFIG="./benchmarks/benchmark_config.yaml"
+
+idx=0
+
+for req_file in ./runner/requirements-*.txt; do
+    year=$(basename "$req_file" | sed -E 's/requirements-(.*)\.txt/\1/')
+
+    echo "Running benchmarks for the year: $year"
+
+    env_dir="env-$year"
+    python3 -m venv "$env_dir"
+    source "$env_dir/bin/activate"
+
+    if [ -f "$req_file" ]; then
+        pip install --upgrade pip
+        pip install -r "$req_file"
+        pip install git+https://github.com/PyPSA/linopy.git@40a27f9e7f5d33acd1d256334a1b193899b166ad
+    else
+        echo "Error: Requirements file $req_file not found."
+        deactivate
+        rm -rf "$env_dir"
+        continue
+    fi
+
+    if [ "$idx" -eq 0 ]; then
+        python "$BENCHMARK_SCRIPT" "$BENCHMARK_CONFIG" "$year"
+    else
+        python "$BENCHMARK_SCRIPT" "$BENCHMARK_CONFIG" "$year" false
+    fi
+
+    deactivate
+    rm -rf "$env_dir"
+
+    echo "Completed benchmarks for the year: $year"
+
+    idx=$((idx + 1))
+done
+
+echo "All benchmarks completed."

runner/requirements-2022.txt

@@ -0,0 +1,4 @@
+highspy==1.5.0.dev0
+linopy==0.3.14
+pypsa==0.31.1
+requests==2.31.0

runner/requirements-2023.txt

@@ -0,0 +1,4 @@
+highspy==1.5.3
+linopy==0.3.14
+pypsa==0.31.1
+requests==2.31.0

runner/requirements-2024.txt

@@ -0,0 +1,7 @@
+glpk==0.4.7
+gurobipy==11.0.3
+highspy==1.8.0
+linopy==0.3.14
+PySCIPOpt==5.2.1
+pypsa==0.31.1
+requests==2.31.0

runner/run_benchmarks.py

@@ -4,12 +4,26 @@
 import statistics
 import subprocess
 import sys
+from importlib.metadata import PackageNotFoundError, version
 from pathlib import Path
 
 import requests
 import yaml
 
 
+def get_solver_version(solver_name):
+    if solver_name == "highs":
+        return version("highspy")
+    elif solver_name == "scip":
+        return version("PySCIPOpt")
+    elif solver_name == "gurobi":
+        return version("gurobipy")
+    elif solver_name == "glpk":
+        return version("glpk")
+    else:
+        raise NotImplementedError(f"We do not yet support {solver_name}")
+
+
 def download_file_from_google_drive(url, dest_path: Path):
     """Download a file from url and save it locally in the specified folder if it doesn't already exist."""
     # Ensure the destination folder exists
@@ -37,9 +51,10 @@ def parse_memory(output):
     raise ValueError(f"Could not find memory usage in subprocess output:\n{output}")
 
 
-def write_csv_headers(results_csv, mean_stddev_csv):
+def write_csv_headers(results_csv, mean_stddev_csv, override):
+    mode = "w" if override else "a"
     # Initialize CSV files with headers
-    with open(results_csv, mode="w", newline="") as file:
+    with open(results_csv, mode=mode, newline="") as file:
         writer = csv.writer(file)
         writer.writerow(
             [
@@ -52,10 +67,12 @@ def write_csv_headers(results_csv, mean_stddev_csv):
                 "Memory Usage (MB)",
                 "Max Integrality Violation",
                 "Duality Gap",
+                "Solver Version",
+                "Solver Release Year",
             ]
         )
 
-    with open(mean_stddev_csv, mode="w", newline="") as file:
+    with open(mean_stddev_csv, mode=mode, newline="") as file:
         writer = csv.writer(file)
         writer.writerow(
             [
@@ -68,6 +85,8 @@ def write_csv_headers(results_csv, mean_stddev_csv):
                 "Runtime StdDev (s)",
                 "Memory Mean (MB)",
                 "Memory StdDev (MB)",
+                "Solver Version",
+                "Solver Release Year",
             ]
         )
 
@@ -87,6 +106,8 @@ def write_csv_row(results_csv, benchmark_name, solver, metrics):
                 metrics["memory"],
                 metrics["max_integrality_violation"],
                 metrics["duality_gap"],
+                metrics["solver_version"],
+                metrics["solver_release_year"],
             ]
         )
 
@@ -110,7 +131,7 @@ def write_csv_summary_row(mean_stddev_csv, benchmark_name, solver, metrics):
         )
 
 
-def benchmark_solver(input_file, solver_name, timeout):
+def benchmark_solver(input_file, solver_name, timeout, year, solver_version):
     command = [
         "/usr/bin/time",
         "--format",
@@ -158,12 +179,22 @@ def benchmark_solver(input_file, solver_name, timeout):
         }
     else:
         metrics = json.loads(result.stdout.splitlines()[-1])
+
     metrics["memory"] = memory
+    metrics["solver_version"] = solver_version
+    metrics["solver_release_year"] = year
 
     return metrics
 
 
-def main(benchmark_file_path, solvers, iterations=1, timeout=15 * 60):
+def main(
+    benchmark_file_path,
+    solvers,
+    year=None,
+    iterations=1,
+    timeout=15 * 60,
+    override=True,
+):
     results = {}
 
     # Load benchmarks from YAML file
@@ -173,10 +204,13 @@ def main(benchmark_file_path, solvers, iterations=1, timeout=15 * 60):
     # Create results folder `results/` if it doesn't exist
     results_folder = Path(__file__).parent.parent / "results"
     os.makedirs(results_folder, exist_ok=True)
+
     results_csv = results_folder / "benchmark_results.csv"
     mean_stddev_csv = results_folder / "benchmark_results_mean_stddev.csv"
-    write_csv_headers(results_csv, mean_stddev_csv)
 
+    # Write headers if overriding or file doesn't exist
+    if override or not results_csv.exists() or not mean_stddev_csv.exists():
+        write_csv_headers(results_csv, mean_stddev_csv, override)
     # TODO put the benchmarks in a better place; for now storing in `runner/benchmarks/``
     benchmarks_folder = Path(__file__).parent / "benchmarks/"
     os.makedirs(benchmarks_folder, exist_ok=True)
@@ -196,13 +230,28 @@ def main(benchmark_file_path, solvers, iterations=1, timeout=15 * 60):
             raise ValueError("No valid 'path' or 'url' found for benchmark entry.")
 
         for solver in solvers:
+            try:
+                solver_version = get_solver_version(solver)
+            except PackageNotFoundError:
+                print(f"Solver {solver} is not available. Skipping.")
+                continue
+
             metrics = {}
             runtimes = []
             memory_usages = []
 
             for i in range(iterations):
-                print(f"Running solver {solver} on {benchmark_path.name} ({i})...")
-                metrics = benchmark_solver(benchmark_path, solver, timeout)
+                print(
+                    f"Running solver {solver} (version {solver_version}) on {benchmark_path.name} ({i})..."
+                )
+
+                metrics = benchmark_solver(
+                    benchmark_path,
+                    solver,
+                    timeout,
+                    year,
+                    solver_version,
+                )
 
                 runtimes.append(metrics["runtime"])
                 memory_usages.append(metrics["memory"])
@@ -235,17 +284,20 @@ def main(benchmark_file_path, solvers, iterations=1, timeout=15 * 60):
 
 
 if __name__ == "__main__":
-    # Check for benchmark file argument
+    # Check for benchmark file argument and optional year and override arguments
     if len(sys.argv) < 2:
-        print("Usage: python run_benchmarks.py <path_to_benchmarks.yaml>")
+        raise ValueError(
+            "Usage: python run_benchmarks.py <path_to_benchmarks.yaml> [<year>] [<override>]"
+        )
         sys.exit(1)
 
     benchmark_file_path = sys.argv[1]
+    year = sys.argv[2] if len(sys.argv) > 2 else None
+    override = sys.argv[3].lower() == "true" if len(sys.argv) > 3 else True
 
     # solvers = ["highs", "glpk"]  # For dev and testing
     solvers = ["highs", "glpk", "scip"]  # For production
 
-    main(benchmark_file_path, solvers)
-
+    main(benchmark_file_path, solvers, year, override=override)
     # Print a message indicating completion
     print("Benchmarking complete.")

website/app.py

@@ -23,11 +23,9 @@
       align-items: center;
       font-size: 18px;
     }
-
     .appview-container {
       margin-top: 56px;
     }
-
     header.st-emotion-cache-12fmjuu {
       top: 56px;
     }
@@ -38,7 +36,6 @@
   """,
     unsafe_allow_html=True,
 )
-
 pages = [
     st.Page("home.py", title="Home"),
     st.Page("benchmarks.py", title="Benchmark Details"),