Improve script and download robustness (#73)

* Improve robustness of bash scripts * Remove intel macos and make wget quiet * Removed quotes around patterns * Simplified sudoIfAvailable
mathworks · Aug 8, 2024 · f97b708 · f97b708
1 parent 6c8b6f3
commit f97b708
Show file tree

Hide file tree

Showing 5 changed files with 158 additions and 73 deletions.
diff --git a/.circleci/test-deploy.yml b/.circleci/test-deploy.yml
@@ -13,7 +13,6 @@ integration-tests: &integration-tests
     orb-tools/pack,
     integration-test-install,
     integration-test-install-release,
-    integration-test-install-release-macos-intel-on-arm,
     integration-test-run-command,
     integration-test-run-tests
   ]
@@ -23,10 +22,6 @@ executors:
     machine:
       image: ubuntu-2204:2024.01.1
   macos:
-    macos:
-      xcode: 15.1.0
-    resource_class: macos.x86.medium.gen2
-  macos-arm:
     macos:
       xcode: 15.3.0
     resource_class: macos.m1.medium.gen1
@@ -463,33 +458,28 @@ workflows:
       - integration-test-install:
           matrix:
             parameters:
-              executor: [linux, windows, macos, macos-arm]
+              executor: [linux, windows, macos]
 
       - integration-test-install-release:
           matrix:
             parameters:
-              executor: [linux, windows, macos, macos-arm]
+              executor: [linux, windows, macos]
               release: [R2023bU1]
 
-      - integration-test-install-release:
-          name: integration-test-install-release-macos-intel-on-arm
-          executor: macos-arm
-          release: R2023aU1
-
       - integration-test-run-command:
           matrix:
             parameters:
-              executor: [linux, windows, macos, macos-arm]
+              executor: [linux, windows, macos]
 
       - integration-test-run-tests:
           matrix:
             parameters:
-              executor: [linux, windows, macos, macos-arm]
+              executor: [linux, windows, macos]
 
       - integration-test-run-build:
           matrix:
             parameters:
-              executor: [linux, windows, macos, macos-arm]
+              executor: [linux, windows, macos]
 
       - orb-tools/pack:
           filters: *filters

diff --git a/src/scripts/install.sh b/src/scripts/install.sh
@@ -1,4 +1,4 @@
-#!/usr/bin/env bash
+#!/bin/bash
 
 # Exit script if you try to use an uninitialized variable.
 set -o nounset
@@ -10,23 +10,42 @@ set -o errexit
 set -o pipefail
 
 sudoIfAvailable() {
-    if [[ -x $(command -v sudo) ]]; then
+    if command -v sudo >/dev/null 2>&1; then
         sudo -E bash "$@"
     else
         bash "$@"
     fi
 }
 
-downloadAndRun() {
-    url=$1
-    shift
-    curl -sfL $url | sudoIfAvailable -s -- "$@"
+stream() {
+    local url="$1"
+    if command -v wget >/dev/null 2>&1; then
+        wget --retry-connrefused --waitretry=5 -qO- "$url"
+    elif command -v curl >/dev/null 2>&1; then
+        curl --retry 5 --retry-connrefused --retry-delay 5 -sSL "$url"
+    else
+        echo "Could not find wget or curl command" >&2
+        return 1
+    fi
+}
+
+download() {
+    local url="$1"
+    local filename="$2"
+    if command -v wget >/dev/null 2>&1; then
+        wget --retry-connrefused --waitretry=5 -qO "$filename" "$url" 2>&1
+    elif command -v curl >/dev/null 2>&1; then
+        curl --retry 5 --retry-connrefused --retry-delay 5 -sSLo "$filename" "$url"
+    else
+        echo "Could not find wget or curl command" >&2
+        return 1
+    fi
 }
 
 os=$(uname)
 arch=$(uname -m)
 binext=""
-tmpdir=$(dirname "$(mktemp -u)")
+tmpdir=$(mktemp -d 2>/dev/null || mktemp -d -t 'install')
 rootdir="$tmpdir/matlab_root"
 batchdir="$tmpdir/matlab-batch"
 mpmdir="$tmpdir/mpm"
@@ -35,49 +54,49 @@ mpmbaseurl="https://www.mathworks.com/mpm"
 
 # resolve release
 parsedrelease=$(echo "$PARAM_RELEASE" | tr '[:upper:]' '[:lower:]')
-if [[ $parsedrelease = "latest" ]]; then
-    mpmrelease=$(curl https://ssd.mathworks.com/supportfiles/ci/matlab-release/v0/latest)
+if [[ "$parsedrelease" = "latest" ]]; then
+    mpmrelease=$(stream https://ssd.mathworks.com/supportfiles/ci/matlab-release/v0/latest)
 else
-    mpmrelease="${parsedrelease}"
+    mpmrelease="$parsedrelease"
 fi
 
 # validate release is supported
-if [[ $mpmrelease < "r2020b" ]]; then
+if [[ "$mpmrelease" < "r2020b" ]]; then
     echo "Release '${mpmrelease}' is not supported. Use 'R2020b' or a later release.">&2
     exit 1
 fi
 
 # install system dependencies
-if [[ $os = Linux ]]; then
+if [[ "$os" = "Linux" ]]; then
     # install MATLAB dependencies
     release=$(echo "${mpmrelease}" | grep -ioE "(r[0-9]{4}[a-b])")
-    downloadAndRun https://ssd.mathworks.com/supportfiles/ci/matlab-deps/v0/install.sh "$release"
+    stream https://ssd.mathworks.com/supportfiles/ci/matlab-deps/v0/install.sh | sudoIfAvailable -s -- "$release"
     # install mpm depencencies
     sudoIfAvailable -c "apt-get install --no-install-recommends --no-upgrade --yes \
         wget \
         unzip \
         ca-certificates"
-elif [[ $os = Darwin && $arch = arm64 ]]; then
-    if [[ $mpmrelease < "r2023b" ]]; then
+elif [[ "$os" = "Darwin" && "$arch" = "arm64" ]]; then
+    if [[ "$mpmrelease" < "r2023b" ]]; then
         # install Rosetta 2
         sudoIfAvailable -c "softwareupdate --install-rosetta --agree-to-license"
     else
         # install Java runtime
         jdkpkg="$tmpdir/jdk.pkg"
-        curl -sfL https://corretto.aws/downloads/latest/amazon-corretto-8-aarch64-macos-jdk.pkg -o $jdkpkg
-        sudoIfAvailable -c "installer -pkg $jdkpkg -target /"
+        download https://corretto.aws/downloads/latest/amazon-corretto-8-aarch64-macos-jdk.pkg "$jdkpkg"
+        sudoIfAvailable -c "installer -pkg '$jdkpkg' -target /"
     fi
 fi
 
 # set os specific options
-if [[ $os = CYGWIN* || $os = MINGW* || $os = MSYS* ]]; then
+if [[ "$os" = CYGWIN* || "$os" = MINGW* || "$os" = MSYS* ]]; then
     mwarch="win64"
     binext=".exe"
     rootdir=$(cygpath "$rootdir")
     mpmdir=$(cygpath "$mpmdir")
     batchdir=$(cygpath "$batchdir")
-elif [[ $os = Darwin ]]; then
-    if [[ $arch = arm64 && ! $mpmrelease < "r2023b" ]]; then
+elif [[ "$os" = "Darwin" ]]; then
+    if [[ "$arch" = "arm64" && ! "$mpmrelease" < "r2023b" ]]; then
          mwarch="maca64"
      else
          mwarch="maci64"
@@ -93,18 +112,18 @@ mkdir -p "$batchdir"
 mkdir -p "$mpmdir"
 
 # install mpm
-curl -o "$mpmdir/mpm$binext" -sfL "$mpmbaseurl/$mwarch/mpm"
+download "$mpmbaseurl/$mwarch/mpm" "$mpmdir/mpm$binext"
 chmod +x "$mpmdir/mpm$binext"
 
 # install matlab-batch
-curl -o "$batchdir/matlab-batch$binext" -sfL "$batchbaseurl/$mwarch/matlab-batch$binext"
+download "$batchbaseurl/$mwarch/matlab-batch$binext" "$batchdir/matlab-batch$binext"
 chmod +x "$batchdir/matlab-batch$binext"
 
 # install matlab
 "$mpmdir/mpm$binext" install \
-    --release=$mpmrelease \
+    --release="$mpmrelease" \
     --destination="$rootdir" \
     --products ${PARAM_PRODUCTS} MATLAB
 
 # add MATLAB and matlab-batch to path
-echo 'export PATH="'$rootdir'/bin:'$batchdir':$PATH"' >> $BASH_ENV
+echo 'export PATH="'$rootdir'/bin:'$batchdir':$PATH"' >> $BASH_ENV
diff --git a/src/scripts/run-build.sh b/src/scripts/run-build.sh
@@ -1,23 +1,44 @@
-downloadAndRun() {
-    url=$1
-    shift
-    if [[ -x $(command -v sudo) ]]; then
-    curl -sfL $url | sudo -E bash -s -- "$@"
+#!/bin/bash
+
+# Exit script if you try to use an uninitialized variable.
+set -o nounset
+
+# Exit script if a statement returns a non-true return value.
+set -o errexit
+
+# Use the error status of the first failure, rather than that of the last item in a pipeline.
+set -o pipefail
+
+sudoIfAvailable() {
+    if command -v sudo >/dev/null 2>&1; then
+        sudo -E bash "$@"
+    else
+        bash "$@"
+    fi
+}
+
+stream() {
+    local url="$1"
+    if command -v wget >/dev/null 2>&1; then
+        wget --retry-connrefused --waitretry=5 -qO- "$url"
+    elif command -v curl >/dev/null 2>&1; then
+        curl --retry 5 --retry-connrefused --retry-delay 5 -sSL "$url"
     else
-    curl -sfL $url | bash -s -- "$@"
+        echo "Could not find wget or curl command" >&2
+        return 1
     fi
 }
 
-tmpdir=$(mktemp -d 2>/dev/null || mktemp -d -t 'run-command')
+tmpdir=$(mktemp -d 2>/dev/null || mktemp -d -t 'run-build')
 
 # install run-matlab-command
-downloadAndRun https://ssd.mathworks.com/supportfiles/ci/run-matlab-command/v2/install.sh "${tmpdir}/bin"
+stream https://ssd.mathworks.com/supportfiles/ci/run-matlab-command/v2/install.sh | sudoIfAvailable -s -- "${tmpdir}/bin"
 
 # form OS appropriate paths for MATLAB
 os=$(uname)
-scriptdir=$tmpdir
+scriptdir="$tmpdir"
 binext=""
-if [[ $os = CYGWIN* || $os = MINGW* || $os = MSYS* ]]; then
+if [[ "$os" = CYGWIN* || "$os" = MINGW* || "$os" = MSYS* ]]; then
     scriptdir=$(cygpath -w "$scriptdir")
     binext=".exe"
 fi
@@ -33,12 +54,12 @@ if [ -n "$PARAM_BUILD_OPTIONS" ]; then
 fi
 
 # create script to execute
-script=command_${RANDOM}
-scriptpath=${tmpdir}/${script}.m
+script="command_${RANDOM}"
+scriptpath="${tmpdir}/${script}.m"
 echo "cd(getenv('MW_ORIG_WORKING_FOLDER'));" > "$scriptpath"
 cat << EOF >> "$scriptpath"
 $buildCommand
 EOF
 
 # run MATLAB command
-"${tmpdir}/bin/run-matlab-command$binext" "setenv('MW_ORIG_WORKING_FOLDER', cd('${scriptdir//\'/\'\'}'));$script" $PARAM_STARTUP_OPTIONS
+"${tmpdir}/bin/run-matlab-command$binext" "setenv('MW_ORIG_WORKING_FOLDER', cd('${scriptdir//\'/\'\'}'));$script" $PARAM_STARTUP_OPTIONS
diff --git a/src/scripts/run-command.sh b/src/scripts/run-command.sh
@@ -1,30 +1,51 @@
-downloadAndRun() {
-    url=$1
-    shift
-    if [[ -x $(command -v sudo) ]]; then
-    curl -sfL $url | sudo -E bash -s -- "$@"
+#!/bin/bash
+
+# Exit script if you try to use an uninitialized variable.
+set -o nounset
+
+# Exit script if a statement returns a non-true return value.
+set -o errexit
+
+# Use the error status of the first failure, rather than that of the last item in a pipeline.
+set -o pipefail
+
+sudoIfAvailable() {
+    if command -v sudo >/dev/null 2>&1; then
+        sudo -E bash "$@"
+    else
+        bash "$@"
+    fi
+}
+
+stream() {
+    local url="$1"
+    if command -v wget >/dev/null 2>&1; then
+        wget --retry-connrefused --waitretry=5 -qO- "$url"
+    elif command -v curl >/dev/null 2>&1; then
+        curl --retry 5 --retry-connrefused --retry-delay 5 -sSL "$url"
     else
-    curl -sfL $url | bash -s -- "$@"
+        echo "Could not find wget or curl command" >&2
+        return 1
     fi
 }
 
 tmpdir=$(mktemp -d 2>/dev/null || mktemp -d -t 'run-command')
 
 # install run-matlab-command
-downloadAndRun https://ssd.mathworks.com/supportfiles/ci/run-matlab-command/v2/install.sh "${tmpdir}/bin"
+stream https://ssd.mathworks.com/supportfiles/ci/run-matlab-command/v2/install.sh | sudoIfAvailable -s -- "${tmpdir}/bin"
 
 # form OS appropriate paths for MATLAB
 os=$(uname)
-scriptdir=$tmpdir
+scriptdir="$tmpdir"
 binext=""
-if [[ $os = CYGWIN* || $os = MINGW* || $os = MSYS* ]]; then
+if [[ "$os" = CYGWIN* || "$os" = MINGW* || "$os" = MSYS* ]]; then
     scriptdir=$(cygpath -w "$scriptdir")
     binext=".exe"
 fi
 
 # create script to execute
-script=command_${RANDOM}
-scriptpath=${tmpdir}/${script}.m
+script="command_${RANDOM}"
+scriptpath="${tmpdir}/${script}.m"
 echo "cd(getenv('MW_ORIG_WORKING_FOLDER'));" > "$scriptpath"
 cat << EOF >> "$scriptpath"
 ${PARAM_COMMAND}