Merge pull request #110 from yoctoyotta1024/pathfix

refactor: improve Python examples with pathlib and improve CI build order

.github/workflows/CI.yml

@@ -1,4 +1,4 @@
-name: Python test
+name: CI (pytest and documentation)
 
 on: [push, pull_request]
 
@@ -39,7 +39,7 @@ jobs:
         run: |
           cd docs && make html
       - name: Archive build artifacts (i.e. documentation)
-        uses: actions/upload-artifact@v3
+        uses: actions/upload-artifact@v4
         with:
           name: html-docs
           path: docs/build/html/
@@ -58,7 +58,7 @@ jobs:
           name: html-docs
           path: docs/build/html/
       - name: Publish docs on GitHub pages
-        uses: peaceiris/actions-gh-pages@v3
+        uses: peaceiris/actions-gh-pages@v4
         with:
           github_token: ${{ secrets.GITHUB_TOKEN }}
           publish_dir: docs/build/html

.github/workflows/build.yml

@@ -1,92 +1,133 @@
-name: Build
+name: build
 
 on: pull_request
 
 jobs:
-  build:
+  setup:
     runs-on: ubuntu-latest
     steps:
-    - name: Checkout
-      uses: actions/checkout@v4
-
-    - name: Setup
-      run: |
-        sudo apt-get update
-        sudo apt-get install -y \
-        cmake libnetcdff-dev liblapack-dev python3-dev \
-        python3-numpy python3-mpi4py python3-pip curl \
-        libfyaml-dev libopenmpi-dev openmpi-bin
-
-    - name: Build YAXT
-      run: |
-        curl -s -L https://swprojects.dkrz.de/redmine/attachments/download/534/yaxt-0.11.1.tar.gz | tar xvz
-        cd yaxt-0.11.1
-        ./configure --without-regard-for-quality --without-example-programs --without-perf-programs --with-pic \
-        --prefix=$HOME/yaxt
-        make -j 4
-        make install
-
-    - name: Build YAC
-      run: |
-        curl -s -L https://gitlab.dkrz.de/dkrz-sw/yac/-/archive/v3.2.0/yac-v3.2.0.tar.gz | tar xvz
-        cd yac-v3.2.0
-        ./configure CFLAGS="-fPIC" CC=mpicc FC=mpif90 --disable-mpi-checks --with-yaxt-root=${HOME}/yaxt \
-        --prefix=$HOME/yac
-        make -j 4
-        make install
-
-    - name: Build main
-      run: |
-        mkdir build && cd build
-        cmake \
-        -DCMAKE_C_COMPILER=gcc \
-        -DCMAKE_CXX_COMPILER=g++ \
-        -DCMAKE_CXX_FLAGS="-Werror -Wall -pedantic -O3" \
-        -DKokkos_ENABLE_SERIAL=ON \
-        -DENABLE_YAC_COUPLING=ON \
-        -DYAXT_ROOT=${HOME}/yaxt \
-        -DYAC_ROOT=${HOME}/yac \
-        -DCMAKE_MODULE_PATH=${PWD}/../libs/coupldyn_yac/cmake ..
-        make
-
-    - name: Build example adia0d
-      run: cd build && make adia0d
-
-    - name: Build example golcolls
-      run: cd build && make golcolls
-
-    - name: Build example longcolls
-      run: cd build && make longcolls
-
-    - name: Build example lowlistcolls
-      run: cd build && make lowlistcolls
-
-    - name: Build example szakallurbichcolls
-      run: cd build && make szakallurbichcolls
-
-    - name: Build example testikstraubcolls
-      run: cd build && make testikstraubcolls
-
-    - name: Build example const2d
-      run: cd build && make const2d
-
-    - name: Build example divfree2d
-      run: cd build && make divfree2d
-
-    - name: Build example eurec4a1d
-      run: cd build && make eurec4a1d
-
-    - name: Build example rshaft1d
-      run: cd build && make rshaft1d
-
-    - name: Build example spdtest
-      run: cd build && make spdtest
-
-    - name: Build example bubble3d
-      run: cd build && make bubble3d
-
-    - name: Build example fromfile
-      run: cd build && make fromfile
-
-    - name: Build example fromfile_irreg
-      run: cd build && make fromfile_irreg
+      - name: Checkout
+        uses: actions/checkout@v4
+
+      - name: Cache YAXT
+        id: cache_yaxt
+        uses: actions/cache@v4
+        with:
+          path: ${{ runner.temp }}/yaxt/
+          key: ${{ runner.os }}-yaxt-cache-v0.11.1
+          restore-keys: |
+            ${{ runner.os }}-yaxt-
+
+      - name: Cache YAC
+        id: cache_yac
+        uses: actions/cache@v4
+        with:
+          path: ${{ runner.temp }}/yac/
+          key: ${{ runner.os }}-yac-cache-v3.2.0
+          restore-keys: |
+            ${{ runner.os }}-yac-
+
+      - name: Setup
+        if: steps.cache_yaxt.outputs.cache-hit != 'true' || steps.cache_yac.outputs.cache-hit != 'true'
+        run: |
+          sudo apt-get update
+          sudo apt-get install -y \
+          cmake libnetcdff-dev liblapack-dev python3-dev \
+          python3-numpy python3-mpi4py python3-pip curl \
+          libfyaml-dev libopenmpi-dev openmpi-bin
+
+      - name: Build YAXT
+        if: steps.cache_yaxt.outputs.cache-hit != 'true'
+        run: |
+          curl -s -L https://swprojects.dkrz.de/redmine/attachments/download/534/yaxt-0.11.1.tar.gz | tar xvz
+          cd yaxt-0.11.1
+          ./configure --without-regard-for-quality --without-example-programs \
+            --without-perf-programs --with-pic --prefix=${{ runner.temp }}/yaxt
+          make -j 4
+          make install
+
+      - name: Build YAC
+        if: steps.cache_yac.outputs.cache-hit != 'true'
+        run: |
+          curl -s -L https://gitlab.dkrz.de/dkrz-sw/yac/-/archive/v3.2.0/yac-v3.2.0.tar.gz | tar xvz
+          cd yac-v3.2.0
+          ./configure CFLAGS="-fPIC" CC=mpicc FC=mpif90 --disable-mpi-checks \
+            --with-yaxt-root=${{ runner.temp }}/yaxt --prefix=${{ runner.temp }}/yac
+          make -j 4
+          make install
+
+  build_basic_examples:
+    runs-on: ubuntu-latest
+    needs: setup
+    strategy:
+      matrix:
+        target: [all, adia0d, golcolls, longcolls, lowlistcolls, szakallurbichcolls,
+                testikstraubcolls, const2d, divfree2d, eurec4a1d, rshaft1d, spdtest]
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v4
+
+      - name: Setup
+        run: |
+          sudo apt-get update
+          sudo apt-get install -y \
+          cmake libopenmpi-dev openmpi-bin
+
+      - name: Build Example
+        run: |
+          mkdir build
+          cmake -S ./ -B ./build \
+          -DCMAKE_C_COMPILER=gcc \
+          -DCMAKE_CXX_COMPILER=g++ \
+          -DCMAKE_CXX_FLAGS="-Werror -Wall -pedantic -O3" \
+          -DKokkos_ENABLE_SERIAL=ON \
+          -DENABLE_YAC_COUPLING=OFF
+
+      - name: Compile Example
+        run: cd build && make ${{ matrix.target }}
+
+  build_yac_examples:
+    runs-on: ubuntu-latest
+    needs: setup
+    strategy:
+      matrix:
+        target: [all, bubble3d, fromfile, fromfile_irreg]
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v4
+
+      - name: Setup
+        run: |
+          sudo apt-get update
+          sudo apt-get install -y \
+          cmake libnetcdff-dev liblapack-dev python3-dev \
+          python3-numpy python3-mpi4py python3-pip curl \
+          libfyaml-dev libopenmpi-dev openmpi-bin
+
+      - name: Restore YAXT Cache
+        uses: actions/cache@v4
+        with:
+          path: ${{ runner.temp }}/yaxt/
+          key: ${{ runner.os }}-yaxt-cache-v0.11.1
+
+      - name: Restore YAC Cache
+        uses: actions/cache@v4
+        with:
+          path: ${{ runner.temp }}/yac/
+          key: ${{ runner.os }}-yac-cache-v3.2.0
+
+      - name: Build YAC Example
+        run: |
+          mkdir build
+          cmake -S ./ -B ./build \
+          -DCMAKE_C_COMPILER=gcc \
+          -DCMAKE_CXX_COMPILER=g++ \
+          -DCMAKE_CXX_FLAGS="-Werror -Wall -pedantic -O3" \
+          -DKokkos_ENABLE_SERIAL=ON \
+          -DENABLE_YAC_COUPLING=ON \
+          -DYAXT_ROOT=${{ runner.temp }}/yaxt \
+          -DYAC_ROOT=${{ runner.temp }}/yac \
+          -DCMAKE_MODULE_PATH=${PWD}/libs/coupldyn_yac/cmake
+
+      - name: Compile YAC Example
+        run: cd build && make ${{ matrix.target }}

.github/workflows/pre-commit.yml

@@ -10,7 +10,7 @@ jobs:
       uses: actions/checkout@v4
 
     - name: Python setup
-      uses: actions/setup-python@v3
+      uses: actions/setup-python@v5
 
     - name: Pre-commit run
-      uses: pre-commit/[email protected].0
+      uses: pre-commit/[email protected].1

.pre-commit-config.yaml

@@ -39,7 +39,10 @@ repos:
     rev: 0.8.0
     hooks:
     -   id: cpplint
-        args: [--linelength=100, "--filter=-runtime/references,-readability/braces,-build/include,-build/c++11"]
+        args:
+            - --linelength=100
+            - --filter=-runtime/references,-readability/braces,-whitespace/indent_namespace,
+              -build/include,-build/c++11,-build/c++17
         types_or: [c, c++, cuda]
 -   repo: https://github.com/compilerla/conventional-pre-commit
     rev: v3.4.0

examples/adiabaticparcel/as2017.py

@@ -24,19 +24,21 @@
 """
 
 import os
+import shutil
+import subprocess
 import sys
 import numpy as np
 import matplotlib.pyplot as plt
 from pathlib import Path
 
-path2CLEO = sys.argv[1]
-path2build = sys.argv[2]
-configfile = sys.argv[3]
+path2CLEO = Path(sys.argv[1])
+path2build = Path(sys.argv[2])
+configfile = Path(sys.argv[3])
 
-sys.path.append(path2CLEO)  # for imports from pySD package
+sys.path.append(str(path2CLEO))  # imports from pySD
 sys.path.append(
-    path2CLEO + "/examples/exampleplotting/"
-)  # for imports from example plotting package
+    str(path2CLEO / "examples" / "exampleplotting")
+)  # imports from example plots package
 
 from plotssrc import as2017fig
 from pySD import editconfigfile
@@ -49,11 +51,11 @@
 
 ############### INPUTS ##################
 # path and filenames for creating SD initial conditions and for running model
-constsfile = path2CLEO + "/libs/cleoconstants.hpp"
-binpath = path2build + "/bin/"
-sharepath = path2build + "/share/"
-initSDsfile = sharepath + "as2017_dimlessSDsinit.dat"
-gridfile = sharepath + "as2017_dimlessGBxboundaries.dat"
+constsfile = path2CLEO / "libs" / "cleoconstants.hpp"
+binpath = path2build / "bin"
+sharepath = path2build / "share"
+initSDsfile = sharepath / "as2017_dimlessSDsinit.dat"
+gridfile = sharepath / "as2017_dimlessGBxboundaries.dat"
 
 # booleans for [making, saving] initialisation figures
 isfigures = [True, True]
@@ -116,19 +118,19 @@ def displacement(time, w_avg, thalf):
 ############### RUN EXAMPLE ##################
 ###  delete any existing datasets
 for run_num in range(len(monors) * len(paramslist)):
-    dataset = binpath + "as2017_sol" + str(run_num) + ".zarr"
-    os.system("rm -rf " + dataset)
+    dataset = "as2017_sol" + str(run_num) + ".zarr"
+    shutil.rmtree(binpath / dataset, ignore_errors=True)
 
 ### ensure build, share and bin directories exist
 if path2CLEO == path2build:
     raise ValueError("build directory cannot be CLEO")
 else:
-    Path(path2build).mkdir(exist_ok=True)
-    Path(sharepath).mkdir(exist_ok=True)
-    Path(binpath).mkdir(exist_ok=True)
+    path2build.mkdir(exist_ok=True)
+    sharepath.mkdir(exist_ok=True)
+    binpath.mkdir(exist_ok=True)
 
 ### create file (and plot) for gridbox boundaries
-os.system("rm " + gridfile)
+shutil.rmtree(gridfile, ignore_errors=True)
 cgrid.write_gridboxboundaries_binary(gridfile, zgrid, xgrid, ygrid, constsfile)
 rgrid.print_domain_info(constsfile, gridfile)
 if isfigures[0]:
@@ -148,7 +150,7 @@ def displacement(time, w_avg, thalf):
     initattrsgen = attrsgen.AttrsGenerator(
         radiigen, dryradiigen, xiprobdist, coord3gen, coord1gen, coord2gen
     )
-    os.system("rm " + initSDsfile)
+    shutil.rmtree(initSDsfile, ignore_errors=True)
     csupers.write_initsuperdrops_binary(
         initSDsfile, initattrsgen, configfile, constsfile, gridfile, nsupers, numconc
     )
@@ -163,24 +165,25 @@ def displacement(time, w_avg, thalf):
     fig, axs = plt.subplots(nrows=3, ncols=1, figsize=(5, 16))
     for params, lwdth in zip(paramslist, lwdths):
         ### edit relevant setup file parameters
-        params["zarrbasedir"] = binpath + "as2017_sol" + str(runnum) + ".zarr"
-        params["setup_filename"] = binpath + "as2017_setup.txt"
+        zarrbasedir = "as2017_sol" + str(runnum) + ".zarr"
+        params["zarrbasedir"] = str(binpath / zarrbasedir)
+        params["setup_filename"] = str(binpath / "as2017_setup.txt")
         editconfigfile.edit_config_params(configfile, params)
 
         ### delete any existing dataset
-        os.system("rm -rf " + params["zarrbasedir"])
-        os.system("rm " + params["setup_filename"])
+        shutil.rmtree(params["zarrbasedir"], ignore_errors=True)
+        shutil.rmtree(params["setup_filename"], ignore_errors=True)
 
         ### run model
         os.chdir(path2build)
-        executable = path2build + "/examples/adiabaticparcel/src/adia0d"
-        print("Executable: " + executable)
-        print("Config file: " + configfile)
-        os.system(executable + " " + configfile)
+        executable = path2build / "examples" / "adiabaticparcel" / "src" / "adia0d"
+        print("Executable: " + str(executable))
+        print("Config file: " + str(configfile))
+        subprocess.run([executable, configfile])
 
         ### load results
-        setupfile = binpath + "as2017_setup.txt"
-        dataset = binpath + "as2017_sol" + str(runnum) + ".zarr"
+        setupfile = binpath / "as2017_setup.txt"
+        dataset = binpath / Path("as2017_sol" + str(runnum) + ".zarr")
 
         ### read in constants and intial setup from setup .txt file
         config = pysetuptxt.get_config(setupfile, nattrs=3, isprint=True)
@@ -237,9 +240,7 @@ def displacement(time, w_avg, thalf):
 
     fig.tight_layout()
 
-    savename = "as2017fig_" + str(i) + ".png"
-    fig.savefig(
-        binpath + savename, dpi=400, bbox_inches="tight", facecolor="w", format="png"
-    )
-    print("Figure .png saved as: " + binpath + savename)
+    savename = binpath / Path("as2017fig_" + str(i) + ".png")
+    fig.savefig(savename, dpi=400, bbox_inches="tight", facecolor="w", format="png")
+    print("Figure .png saved as: " + str(savename))
     plt.show()