ci: add shellcheck gh action, lint fixes, parallel acts

.github/workflows/ci.yml

@@ -25,7 +25,7 @@ jobs:
       - uses: actions/checkout@v4
         with:
           submodules: recursive
-      - name: Setup Build Env
+      - name: Install Build Tools
         run: sudo ./scripts/install-build-tools.sh
       - name: Setup Local Dependencies
         run: ./scripts/setup-dependencies.sh
@@ -38,7 +38,7 @@ jobs:
       - uses: actions/checkout@v4
         with:
           submodules: recursive
-      - name: Setup Build Env
+      - name: Install Build Tools
         run: sudo ./scripts/install-build-tools.sh
       - name: Setup Local Dependencies
         run: ./scripts/setup-dependencies.sh
@@ -50,7 +50,7 @@ jobs:
     name: Pylint
     runs-on: ubuntu-22.04
     continue-on-error: true
-    timeout-minutes: 10
+    timeout-minutes: 5
     strategy:
       matrix:
         python-version: ["3.10"]
@@ -62,10 +62,25 @@ jobs:
         uses: actions/setup-python@v5
         with:
           python-version: ${{ matrix.python-version }}
-      - name: Setup Build Env
+      - name: Install Build Tools
         run: sudo ./scripts/install-build-tools.sh
       - name: Lint with Pylint
         run: ./scripts/pylint.sh
+  shellcheck:
+    name: Shellcheck
+    runs-on: ubuntu-22.04
+    continue-on-error: true
+    timeout-minutes: 5
+    steps:
+      - uses: actions/checkout@v4
+        with:
+          submodules: recursive
+      - name: Install shellcheck
+        run: |
+          sudo apt-get update
+          sudo apt-get install -y shellcheck
+      - name: Lint with Shellcheck
+        run: ./scripts/shellcheck.sh
   unit-and-integration-test:
     name: Unit and Integration Tests
     runs-on: ubuntu-22.04
@@ -74,7 +89,7 @@ jobs:
       - uses: actions/checkout@v4
         with:
           submodules: recursive
-      - name: Setup Build Env
+      - name: Install Build Tools
         run: sudo ./scripts/install-build-tools.sh
       - name: Setup Local Dependencies
         run: ./scripts/setup-dependencies.sh
@@ -84,7 +99,7 @@ jobs:
         run: ./scripts/test.sh
       - name: Shorten SHA
         id: vars
-        run: echo "::set-output name=sha_short::$(git rev-parse --short HEAD)"
+        run: echo "sha_short=$(git rev-parse --short HEAD)" >> $GITHUB_ENV
       - uses: actions/upload-artifact@v4
         if: ${{ !env.ACT }}
         name: Archive Test Results
@@ -114,4 +129,3 @@ jobs:
           name: OpenCBDC Transaction Processor docs for ${{ steps.vars.outputs.sha_short }}
           path: ./doxygen_generated/html/*
           retention-days: 7
-

scripts/benchmarks.sh

@@ -64,6 +64,7 @@ done
 # to the location of this script, the user can run this script from any folder.
 SCRIPT_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )"
 REPO_TOP_DIR="${SCRIPT_DIR}/.."
+BUILD_DIR=
 if [[ -z "${BUILD_DIR+x}" ]]
 then
     BUILD_DIR="${REPO_TOP_DIR}/build"
@@ -78,7 +79,8 @@ fi
 # If the build folder is a relative path, convert it to an absolute path
 # to avoid potential relative path errors and to improve readability
 # if the path is written to stdout.
-export BUILD_DIR=$(cd "$BUILD_DIR"; pwd)
+BUILD_DIR=$(cd "$BUILD_DIR"; pwd)
+export BUILD_DIR
 echo "Build folder: '${BUILD_DIR}'"
 echo
 

scripts/build-docker.sh

@@ -12,7 +12,7 @@ DOCKER_IMAGE_TAG_TWOPHASE=${DOCKER_IMAGE_TAG:-opencbdc-tx-twophase}
 git submodule init && git submodule update
 
 # Build docker image
-docker build --target base -t $DOCKER_IMAGE_TAG_BASE -f $SCRIPT_DIR/../Dockerfile $SCRIPT_DIR/..
-docker build --target builder --build-arg BASE_IMAGE=base -t $DOCKER_IMAGE_TAG_BUILDER -f $SCRIPT_DIR/../Dockerfile $SCRIPT_DIR/..
-docker build --target twophase --build-arg BASE_IMAGE=base -t $DOCKER_IMAGE_TAG_TWOPHASE -f $SCRIPT_DIR/../Dockerfile $SCRIPT_DIR/..
-docker build --target atomizer --build-arg BASE_IMAGE=base -t $DOCKER_IMAGE_TAG_ATOMIZER -f $SCRIPT_DIR/../Dockerfile $SCRIPT_DIR/..
+docker build --target base -t "$DOCKER_IMAGE_TAG_BASE" -f "$SCRIPT_DIR"/../Dockerfile "$SCRIPT_DIR"/..
+docker build --target builder --build-arg BASE_IMAGE=base -t "$DOCKER_IMAGE_TAG_BUILDER" -f "$SCRIPT_DIR"/../Dockerfile "$SCRIPT_DIR"/..
+docker build --target twophase --build-arg BASE_IMAGE=base -t "$DOCKER_IMAGE_TAG_TWOPHASE" -f "$SCRIPT_DIR"/../Dockerfile "$SCRIPT_DIR"/..
+docker build --target atomizer --build-arg BASE_IMAGE=base -t "$DOCKER_IMAGE_TAG_ATOMIZER" -f "$SCRIPT_DIR"/../Dockerfile "$SCRIPT_DIR"/..

scripts/build.sh

@@ -27,7 +27,7 @@ if [ $# -gt 0 ]; then
         Profiling|--profiling|-p) CMAKE_BUILD_TYPE="Profiling";;
         Debug|--debug|-d) CMAKE_BUILD_TYPE="Debug";;
         --help|-h) help;;
-        *) help $1;;
+        *) help "$1";;
     esac
 fi
 
@@ -37,11 +37,12 @@ echo "Building..."
 PREFIX="$(cd "$(dirname "$0")"/.. && pwd)/prefix"
 
 if [ -z ${BUILD_DIR+x} ]; then
-    export BUILD_DIR=build
+    BUILD_DIR=build
+    export BUILD_DIR
 fi
 
-mkdir -p $BUILD_DIR
-cd $BUILD_DIR
+mkdir -p "$BUILD_DIR"
+cd "$BUILD_DIR"
 
 CMAKE_FLAGS=-DCMAKE_PREFIX_PATH="${PREFIX}"
 CPUS=1
@@ -60,5 +61,4 @@ fi
 
 echo "Building $CMAKE_BUILD_TYPE"
 eval "cmake -DCMAKE_BUILD_TYPE=${CMAKE_BUILD_TYPE} ${CMAKE_FLAGS} .."
-make -j$CPUS
-
+make -j"$CPUS"

scripts/create-e2e-report.sh

@@ -1,6 +1,6 @@
 #!/usr/bin/env bash
 set -e
-SCRIPT_DIR=$( cd -- "$( dirname -- "${BASH_SOURCE[0]}" )" &> /dev/null && pwd )
+
 TESTRUN_PATH=$1
 
 function readAndFormatLogs() {
@@ -11,15 +11,15 @@ function readAndFormatLogs() {
         return
     fi
 
-    for logfile in $(ls $logdir); do
+    for logfile in "$logdir"/*; do
         logfile_path="$logdir/$logfile"
-        logfile_content=$(cat $logfile_path)
+        logfile_content=$(<"$logfile_path")
         message+="\n<details>\n<summary>$logfile</summary>\n\n\`\`\`\n$logfile_content\n\`\`\`\n</details>\n"
     done
     echo "$message"
 }
 
-testrun_logs="\n<details>\n<summary>View Testrun</summary>\n\n\`\`\`\n$(cat $TESTRUN_PATH/testrun.log)\n\`\`\`\n</details>\n\n"
-container_logs=$(readAndFormatLogs $TESTRUN_PATH/logs)
+testrun_logs="\n<details>\n<summary>View Testrun</summary>\n\n\`\`\`\n$(cat "$TESTRUN_PATH"/testrun.log)\n\`\`\`\n</details>\n\n"
+container_logs=$(readAndFormatLogs "$TESTRUN_PATH"/logs)
 
 printf "# E2E Results\n# TestRun Logs\n%b\n\n# Container Logs\n%b\n" "$testrun_logs" "$container_logs"

scripts/install-build-tools.sh

@@ -17,7 +17,7 @@ fi
 
 # Supporting these versions for buildflow
 PYTHON_VERSIONS=("3.10" "3.11" "3.12")
-echo "Python3 versions supported: ${PYTHON_VERSIONS[@]}"
+echo "Python3 versions supported: ${PYTHON_VERSIONS[*]}"
 
 # check if supported version of python3 is already installed, and save the version
 PY_INSTALLED=''
@@ -60,7 +60,7 @@ create_venv_install_python() {
         if ! $SUDO apt install -y python3-pip; then
             echo "Failed to install python3-pip"
             wget https://bootstrap.pypa.io/get-pip.py
-            $SUDO python${PY_VERSION} get-pip.py
+            $SUDO python"$PY_VERSION" get-pip.py
             rm get-pip.py
         fi
         # add deadsnakes to download the python venv module
@@ -112,7 +112,6 @@ if [[ "$OSTYPE" == "darwin"* ]]; then
         exit 1
     fi
 
-    CPUS=$(sysctl -n hw.ncpu)
     # ensure development environment is set correctly for clang
     $SUDO xcode-select -switch /Library/Developer/CommandLineTools
 
@@ -121,7 +120,8 @@ if [[ "$OSTYPE" == "darwin"* ]]; then
         exit 1
     fi
 
-    brew install llvm@14 googletest google-benchmark lcov make wget cmake bash bc
+    CPUS=$(sysctl -n hw.ncpu)
+    echo llvm@14 googletest google-benchmark lcov make wget cmake bash bc | xargs -n 1 -P "$CPUS" brew install
     brew upgrade bash
 
     BREW_ROOT=$(brew --prefix)
@@ -132,7 +132,7 @@ if [[ "$OSTYPE" == "darwin"* ]]; then
     fi
     GMAKE=/usr/local/bin/gmake
     if [[ ! -L "$GMAKE" ]]; then
-        $SUDO ln -s $(xcode-select -p)/usr/bin/gnumake /usr/local/bin/gmake
+        $SUDO ln -s "$(xcode-select -p)"/usr/bin/gnumake /usr/local/bin/gmake
     fi
 
     # install valid python version if not installed yet
@@ -172,8 +172,8 @@ elif [[ "$OSTYPE" == "linux-gnu"* ]]; then
 
     $SUDO apt update -y
     $SUDO apt install -y clang-format-14 clang-tidy-14
-    $SUDO ln -sf $(which clang-format-14) /usr/local/bin/clang-format
-    $SUDO ln -sf $(which clang-tidy-14) /usr/local/bin/clang-tidy
+    $SUDO ln -sf "$(which clang-format-14)" /usr/local/bin/clang-format
+    $SUDO ln -sf "$(which clang-tidy-14)" /usr/local/bin/clang-tidy
 
     # install valid python version if not installed yet
     if [[ -z "$PY_INSTALLED" ]]; then
@@ -194,9 +194,9 @@ elif [[ "$OSTYPE" == "linux-gnu"* ]]; then
         MAX_RETRIES=2
         while [[ $MAX_RETRIES -gt 0 ]]; do
             # install python3 valid version and venv module
-            if $SUDO apt install -y ${FULL_PY}; then
+            if $SUDO apt install -y "$FULL_PY"; then
                 echo "${FULL_PY} installed successfully"
-                PY_INSTALLED=${PY_VERS}
+                PY_INSTALLED="$PY_VERS"
                 break
             fi
             MAX_RETRIES=$((MAX_RETRIES - 1))
@@ -215,8 +215,8 @@ if ! which "python${PY_INSTALLED}" &> /dev/null; then
     exit 1
 else
     # create virtual environment and install python packages for the valid python version
-    PYTHON_PATH=$(which "python${PY_INSTALLED}")
-    create_venv_install_python "${PYTHON_PATH}" ${PY_INSTALLED}
+    PYTHON_PATH=$(which python"$PY_INSTALLED")
+    create_venv_install_python "$PYTHON_PATH" "$PY_INSTALLED"
 fi
 echo "To activate the virtual env to run python, run 'source ./scripts/activate-venv.sh'"
 

scripts/lint.sh

@@ -14,6 +14,7 @@ if [ -n "$whitespace_files" ]; then
     printf '%s\n' "${whitespace_files[@]}"
 fi
 
+# shellcheck disable=SC2016
 newline_files=$(printf '%s' "${check_files[@]}" | xargs -r -I {} bash -c 'test "$(tail -c 1 "{}" | wc -l)" -eq 0 && echo {}' | cat)
 
 if [ -n "$newline_files" ] ; then
@@ -25,9 +26,12 @@ if [ -n "$whitespace_files" ] || [ -n "$newline_files" ] ; then
     exit 1
 fi
 
-check_format_files=$(git ls-files | grep -E "tools|tests|src|cmake-tests" \
-                     | grep -E "\..*pp")
-clang-format --style=file --Werror --dry-run ${check_format_files[@]}
+check_format_files=$(git ls-files | \
+                     grep -E "tools|tests|src|cmake-tests" | \
+                     grep -E "\..*pp")
+
+echo "${check_format_files}" | \
+    xargs -n1 -I{} clang-format --style=file --Werror --dry-run {}
 
 if ! command -v clang-tidy &>/dev/null; then
     echo "clang-tidy does not appear to be installed"
@@ -46,6 +50,6 @@ fi
 
 # use python from the virtual environment for clang-tidy
 if source "./scripts/activate-venv.sh"; then
-    python /usr/local/bin/run-clang-tidy.py -p ${BUILD_DIR} "tests/.*/.*\.cpp|src/.*/.*\.cpp|tools/.*/.*\.cpp"
+    python /usr/local/bin/run-clang-tidy.py -p "$BUILD_DIR" "tests/.*/.*\.cpp|src/.*/.*\.cpp|tools/.*/.*\.cpp"
     deactivate
 fi

scripts/lua_bench.sh

@@ -29,8 +29,8 @@ for arg in "$@"; do
     fi
 done
 ./build/tools/bench/parsec/lua/lua_bench --component_id=0 \
-    --ticket_machine0_endpoint=$IP:7777 --ticket_machine_count=1 \
-    --shard_count=1 --shard0_count=1 --shard00_endpoint=$IP:5556 \
-    --agent_count=1 --agent0_endpoint=$IP:$PORT \
-    --loglevel=$LOGLEVEL scripts/gen_bytecode.lua $N_WALLETS
-echo done
+    --ticket_machine0_endpoint="$IP":7777 --ticket_machine_count=1 \
+    --shard_count=1 --shard0_count=1 --shard00_endpoint="$IP":5556 \
+    --agent_count=1 --agent0_endpoint="$IP":"$PORT" \
+    --loglevel="$LOGLEVEL" scripts/gen_bytecode.lua $N_WALLETS
+echo "done"; echo

scripts/native-system-benchmark.sh

@@ -107,7 +107,8 @@ case "$DURATION" in
 esac
 
 if [[ -n "$_help" ]]; then
-    printf "$usage" "$(basename $0)"
+    # shellcheck disable=SC2059
+    printf "$usage" "$(basename "$0")"
     exit "$_err"
 fi
 
@@ -151,7 +152,7 @@ on_int() {
     printf 'Interrupting all components\n'
     trap '' SIGINT # avoid interrupting ourself
     for i in $PIDS; do # intentionally unquoted
-        if [[ -n "RECORD" ]]; then
+        if [[ -n "$RECORD" ]]; then
             kill -SIGINT -- "-$i"
         else
             kill -SIGINT -- "$i"
@@ -178,7 +179,7 @@ on_int() {
     if [[ -x "$(which flamegraph.pl)" && -x "$(which stackcollapse-perf.pl)" && -n "$(find "$TESTDIR" -maxdepth 1 -name '*.perf' -print -quit)" ]]; then
         printf 'Generating Flamegraphs\n'
         for i in "$TESTDIR"/*.perf; do
-            waitpid -t 5 -e $(lsof -Qt "$i") &>/dev/null
+            waitpid -t 5 -e "$(lsof -Qt "$i")" &>/dev/null
             perf script -i "$i" | stackcollapse-perf.pl > "${i/.perf/.folded}"
             flamegraph.pl "${i/.perf/.folded}" > "${i/.perf/.svg}"
             rm -- "${i/.perf/.folded}"
@@ -194,7 +195,7 @@ on_int() {
 
     printf 'Terminating any remaining processes\n'
     for i in $PIDS; do # intentionally unquoted
-        if [[ -n "RECORD" ]]; then
+        if [[ -n "$RECORD" ]]; then
             kill -SIGTERM -- "-$i"
         else
             kill -SIGTERM -- "$i"
@@ -253,15 +254,15 @@ run() {
     COMP=
     case "$RECORD" in
         perf)
-            $@ &> "$PROC_LOG" &
+            "$@" &> "$PROC_LOG" &
             COMP="$!"
             perf record -F 99 -a -g -o "$PNAME".perf -p "$COMP" &> "$PERF_LOG" &
             PERFS="$PERFS $!";;
         debug)
             ${DBG} "$@" &> "$PROC_LOG" &
             COMP="$!";;
         *)
-            $@ &> "$PROC_LOG" &
+            "$@" &> "$PROC_LOG" &
             COMP="$!";;
     esac
 
@@ -287,9 +288,9 @@ seed() {
     if test ! -e "$SEEDDIR"/"$preseed_id"; then
         printf 'Creating %s\n' "$preseed_id"
         mkdir -p -- "$SEEDDIR"/"$preseed_id"
-        pushd "$SEEDDIR"/"$preseed_id" &> /dev/null
+        pushd "$SEEDDIR"/"$preseed_id" &> /dev/null || exit
         PID=$(PNAME=seeder BLOCK=1 run "$(getpath seeder)" "$CFG")
-        popd &> /dev/null
+        popd &> /dev/null || exit
     fi
 
     printf 'Using %s as seed\n' "$preseed_id"
@@ -320,25 +321,27 @@ launch() {
                 for node in $(seq 0 $(( "$raft" - 1 )) ); do
                     export PNAME="$1${id}_$node"
                     PID=$(run "$(getpath "$1")" "$CFG" "$id" "$node")
+                    # shellcheck disable=SC2013
                     for ep in $(awk -F'[":]' "/$PNAME.*endpoint/ { print \$3 }" "$CFG"); do
                         "$RT"/scripts/wait-for-it.sh -q -t 5 -h localhost -p "$ep"
                     done
                     printf 'Launched logical %s %d, replica %d [PID: %d]\n' "$1" "$id" "$node" "$PID"
-                    if [[ -n "RECORD" ]]; then
-                        PIDS="$PIDS $(getpgid $PID)"
+                    if [[ -n "$RECORD" ]]; then
+                        PIDS="$PIDS $(getpgid "$PID")"
                     else
                         PIDS="$PIDS $PID"
                     fi
                 done
             else
                 export PNAME="$1${id}"
                 PID=$(run "$(getpath "$1")" "$CFG" "$id")
+                # shellcheck disable=SC2013
                 for ep in $(awk -F'[":]' "/$PNAME.*endpoint/ { print \$3 }" "$CFG"); do
                     "$RT"/scripts/wait-for-it.sh -q -t 5 -h localhost -p "$ep"
                 done
                 printf 'Launched %s %d [PID: %d]\n' "$1" "$id" "$PID"
-                if [[ -n "RECORD" ]]; then
-                    PIDS="$PIDS $(getpgid $PID)"
+                if [[ -n "$RECORD" ]]; then
+                    PIDS="$PIDS $(getpgid "$PID")"
                 else
                     PIDS="$PIDS $PID"
                 fi