From e594d24401a6c828c9b8fee3536fa1f689e5834c Mon Sep 17 00:00:00 2001 From: Long Vu Date: Tue, 14 May 2024 15:37:10 -0400 Subject: [PATCH 01/11] downloadrepos: allow to be used as library as well --- downloadrepos | 29 ++++++++++++++++++----------- 1 file changed, 18 insertions(+), 11 deletions(-) diff --git a/downloadrepos b/downloadrepos index 9702c26..497e0f6 100755 --- a/downloadrepos +++ b/downloadrepos @@ -1,4 +1,6 @@ #!/bin/sh +# This file can be used both as executable script or library to be sourced. +# To use as library to be sourced, set DOWNLOADREPOS_AS_LIB=1 env var. downloadrepos() { github_repo="$1"; shift @@ -22,16 +24,21 @@ downloadgithubrepos() { set +x } -. ./default_build_params +if [ -z "$DOWNLOADREPOS_AS_LIB" ]; then + # Script mode, otherwise, library mode. + + . ./default_build_params + + if [ -z "$1" ]; then + downloadgithubrepos $PAVICS_SDI_REPO $PAVICS_SDI_BRANCH + downloadgithubrepos $FINCH_REPO $FINCH_BRANCH + downloadgithubrepos $PAVICS_LANDING_REPO $PAVICS_LANDING_BRANCH + downloadgithubrepos $RAVEN_REPO $RAVEN_BRANCH + downloadgithubrepos $RAVENPY_REPO $RAVENPY_BRANCH + downloadgithubrepos $ESGF_COMPUTE_API_REPO $ESGF_COMPUTE_API_BRANCH + else + set -x + downloadrepos "$@" + fi -if [ -z "$1" ]; then - downloadgithubrepos $PAVICS_SDI_REPO $PAVICS_SDI_BRANCH - downloadgithubrepos $FINCH_REPO $FINCH_BRANCH - downloadgithubrepos $PAVICS_LANDING_REPO $PAVICS_LANDING_BRANCH - downloadgithubrepos $RAVEN_REPO $RAVEN_BRANCH - downloadgithubrepos $RAVENPY_REPO $RAVENPY_BRANCH - downloadgithubrepos $ESGF_COMPUTE_API_REPO $ESGF_COMPUTE_API_BRANCH -else - set -x - downloadrepos "$@" fi From 5216bca1fd9f48c1bfb3ad20d6cc2ced3afff49b Mon Sep 17 00:00:00 2001 From: Long Vu Date: Tue, 14 May 2024 16:24:29 -0400 Subject: [PATCH 02/11] downloadrepos: only download repos that will be tested However, old "download all default repos" have to be preserved for back-compat with other external scripts. Fixes https://github.com/Ouranosinc/PAVICS-e2e-workflow-tests/issues/30 --- downloadrepos | 51 +++++++++++++++++++++++++++++++++++++++++++-------- testall | 16 ++++------------ 2 files changed, 47 insertions(+), 20 deletions(-) diff --git a/downloadrepos b/downloadrepos index 497e0f6..c91616c 100755 --- a/downloadrepos +++ b/downloadrepos @@ -24,21 +24,56 @@ downloadgithubrepos() { set +x } -if [ -z "$DOWNLOADREPOS_AS_LIB" ]; then - # Script mode, otherwise, library mode. +lowercase_boolean_build_params() { + TEST_MAGPIE_AUTH="`echo "$TEST_MAGPIE_AUTH" | tr '[:upper:]' '[:lower:]'`" + TEST_PAVICS_SDI_REPO="`echo "$TEST_PAVICS_SDI_REPO" | tr '[:upper:]' '[:lower:]'`" + TEST_PAVICS_SDI_WEAVER="`echo "$TEST_PAVICS_SDI_WEAVER" | tr '[:upper:]' '[:lower:]'`" + TEST_FINCH_REPO="`echo "$TEST_FINCH_REPO" | tr '[:upper:]' '[:lower:]'`" + TEST_PAVICS_LANDING_REPO="`echo "$TEST_PAVICS_LANDING_REPO" | tr '[:upper:]' '[:lower:]'`" + TEST_RAVEN_REPO="`echo "$TEST_RAVEN_REPO" | tr '[:upper:]' '[:lower:]'`" + TEST_RAVENPY_REPO="`echo "$TEST_RAVENPY_REPO" | tr '[:upper:]' '[:lower:]'`" + TEST_ESGF_COMPUTE_API_REPO="`echo "$TEST_ESGF_COMPUTE_API_REPO" | tr '[:upper:]' '[:lower:]'`" + TEST_LOCAL_NOTEBOOKS="`echo "$TEST_LOCAL_NOTEBOOKS" | tr '[:upper:]' '[:lower:]'`" +} +downloadrepos_main() { . ./default_build_params + lowercase_boolean_build_params + + if [ -z "$DOWNLOAD_ALL_DEFAULT_REPOS" ]; then + # Back-compat with old default behavior, used in binder/reorg-notebook + # and other external scripts that autodeploy tutorial notebooks (see + # https://github.com/bird-house/birdhouse-deploy/blob/444a7c35a31aa8ad351e47f659383ba5c2919705/birdhouse/deployment/trigger-deploy-notebook#L64-L75) + DOWNLOAD_ALL_DEFAULT_REPOS=true + fi + if [ -z "$1" ]; then - downloadgithubrepos $PAVICS_SDI_REPO $PAVICS_SDI_BRANCH - downloadgithubrepos $FINCH_REPO $FINCH_BRANCH - downloadgithubrepos $PAVICS_LANDING_REPO $PAVICS_LANDING_BRANCH - downloadgithubrepos $RAVEN_REPO $RAVEN_BRANCH - downloadgithubrepos $RAVENPY_REPO $RAVENPY_BRANCH - downloadgithubrepos $ESGF_COMPUTE_API_REPO $ESGF_COMPUTE_API_BRANCH + if [ x"$DOWNLOAD_ALL_DEFAULT_REPOS" = xtrue -o x"$TEST_PAVICS_SDI_REPO" = xtrue ]; then + downloadgithubrepos $PAVICS_SDI_REPO $PAVICS_SDI_BRANCH + fi + if [ x"$DOWNLOAD_ALL_DEFAULT_REPOS" = xtrue -o x"$TEST_FINCH_REPO" = xtrue ]; then + downloadgithubrepos $FINCH_REPO $FINCH_BRANCH + fi + if [ x"$DOWNLOAD_ALL_DEFAULT_REPOS" = xtrue -o x"$TEST_PAVICS_LANDING_REPO" = xtrue ]; then + downloadgithubrepos $PAVICS_LANDING_REPO $PAVICS_LANDING_BRANCH + fi + if [ x"$DOWNLOAD_ALL_DEFAULT_REPOS" = xtrue -o x"$TEST_RAVEN_REPO" = xtrue ]; then + downloadgithubrepos $RAVEN_REPO $RAVEN_BRANCH + fi + if [ x"$DOWNLOAD_ALL_DEFAULT_REPOS" = xtrue -o x"$TEST_RAVENPY_REPO" = xtrue ]; then + downloadgithubrepos $RAVENPY_REPO $RAVENPY_BRANCH + fi + if [ x"$DOWNLOAD_ALL_DEFAULT_REPOS" = xtrue -o x"$TEST_ESGF_COMPUTE_API_REPO" = xtrue ]; then + downloadgithubrepos $ESGF_COMPUTE_API_REPO $ESGF_COMPUTE_API_BRANCH + fi else set -x downloadrepos "$@" fi +} +if [ -z "$DOWNLOADREPOS_AS_LIB" ]; then + # Script mode, not library mode. + downloadrepos_main "$@" fi diff --git a/testall b/testall index 621a915..ec943fb 100755 --- a/testall +++ b/testall @@ -37,7 +37,10 @@ fi git clean -fdx # download all additional repos containing extra notebooks to test -./downloadrepos +DOWNLOADREPOS_AS_LIB=1 +. ./downloadrepos +DOWNLOAD_ALL_DEFAULT_REPOS=false +downloadrepos_main # 'export' useful vars so they can be used by the CONFIG_OVERRIDE_SCRIPT_URL in runtest. @@ -86,17 +89,6 @@ rm -v $RAVENPY_REPO_NAME-$RAVENPY_BRANCH/pyproject.toml rm -v $ESGF_COMPUTE_API_REPO_NAME-$ESGF_COMPUTE_API_BRANCH/setup.cfg rm -v $ESGF_COMPUTE_API_REPO_NAME-$ESGF_COMPUTE_API_BRANCH/tox.ini -# lowercase -TEST_MAGPIE_AUTH="`echo "$TEST_MAGPIE_AUTH" | tr '[:upper:]' '[:lower:]'`" -TEST_PAVICS_SDI_REPO="`echo "$TEST_PAVICS_SDI_REPO" | tr '[:upper:]' '[:lower:]'`" -TEST_PAVICS_SDI_WEAVER="`echo "$TEST_PAVICS_SDI_WEAVER" | tr '[:upper:]' '[:lower:]'`" -TEST_FINCH_REPO="`echo "$TEST_FINCH_REPO" | tr '[:upper:]' '[:lower:]'`" -TEST_PAVICS_LANDING_REPO="`echo "$TEST_PAVICS_LANDING_REPO" | tr '[:upper:]' '[:lower:]'`" -TEST_RAVEN_REPO="`echo "$TEST_RAVEN_REPO" | tr '[:upper:]' '[:lower:]'`" -TEST_RAVENPY_REPO="`echo "$TEST_RAVENPY_REPO" | tr '[:upper:]' '[:lower:]'`" -TEST_ESGF_COMPUTE_API_REPO="`echo "$TEST_ESGF_COMPUTE_API_REPO" | tr '[:upper:]' '[:lower:]'`" -TEST_LOCAL_NOTEBOOKS="`echo "$TEST_LOCAL_NOTEBOOKS" | tr '[:upper:]' '[:lower:]'`" - NOTEBOOKS_TO_TEST="" if [ x"$TEST_MAGPIE_AUTH" = xtrue ]; then NOTEBOOKS_TO_TEST="$NOTEBOOKS_TO_TEST notebooks-auth/*.ipynb" From 57936f573532193080c6ceb75771c500f34c5619 Mon Sep 17 00:00:00 2001 From: Long Vu Date: Tue, 14 May 2024 17:25:56 -0400 Subject: [PATCH 03/11] testall: make processing here available to other repos via downloadrepos --- downloadrepos | 65 +++++++++++++++++++++++++++++++++++++++------------ testall | 58 +++++++++++++++++++-------------------------- 2 files changed, 74 insertions(+), 49 deletions(-) diff --git a/downloadrepos b/downloadrepos index c91616c..f7f1b20 100755 --- a/downloadrepos +++ b/downloadrepos @@ -24,16 +24,51 @@ downloadgithubrepos() { set +x } +# USAGE: VAR_TO_LOWER="$(lowercase "$VAR_TO_LOWER")" +lowercase() { + echo "$1" | tr '[:upper:]' '[:lower:]' +} + lowercase_boolean_build_params() { - TEST_MAGPIE_AUTH="`echo "$TEST_MAGPIE_AUTH" | tr '[:upper:]' '[:lower:]'`" - TEST_PAVICS_SDI_REPO="`echo "$TEST_PAVICS_SDI_REPO" | tr '[:upper:]' '[:lower:]'`" - TEST_PAVICS_SDI_WEAVER="`echo "$TEST_PAVICS_SDI_WEAVER" | tr '[:upper:]' '[:lower:]'`" - TEST_FINCH_REPO="`echo "$TEST_FINCH_REPO" | tr '[:upper:]' '[:lower:]'`" - TEST_PAVICS_LANDING_REPO="`echo "$TEST_PAVICS_LANDING_REPO" | tr '[:upper:]' '[:lower:]'`" - TEST_RAVEN_REPO="`echo "$TEST_RAVEN_REPO" | tr '[:upper:]' '[:lower:]'`" - TEST_RAVENPY_REPO="`echo "$TEST_RAVENPY_REPO" | tr '[:upper:]' '[:lower:]'`" - TEST_ESGF_COMPUTE_API_REPO="`echo "$TEST_ESGF_COMPUTE_API_REPO" | tr '[:upper:]' '[:lower:]'`" - TEST_LOCAL_NOTEBOOKS="`echo "$TEST_LOCAL_NOTEBOOKS" | tr '[:upper:]' '[:lower:]'`" + TEST_MAGPIE_AUTH="$(lowercase "$TEST_MAGPIE_AUTH")" + TEST_PAVICS_SDI_REPO="$(lowercase "$TEST_PAVICS_SDI_REPO")" + TEST_PAVICS_SDI_WEAVER="$(lowercase "$TEST_PAVICS_SDI_WEAVER")" + TEST_FINCH_REPO="$(lowercase "$TEST_FINCH_REPO")" + TEST_PAVICS_LANDING_REPO="$(lowercase "$TEST_PAVICS_LANDING_REPO")" + TEST_RAVEN_REPO="$(lowercase "$TEST_RAVEN_REPO")" + TEST_RAVENPY_REPO="$(lowercase "$TEST_RAVENPY_REPO")" + TEST_ESGF_COMPUTE_API_REPO="$(lowercase "$TEST_ESGF_COMPUTE_API_REPO")" + TEST_LOCAL_NOTEBOOKS="$(lowercase "$TEST_LOCAL_NOTEBOOKS")" +} + +# Replace all slash (/) by dash (-) because (/) is illegal in folder name +# for branch name of the format "feature/my_wizbang-feature". +# Github does the same when downloading repo archive by downloadrepos above. +# USAGE: export BRANCH_NAME="$(sanitize_branch_name "$BRANCH_NAME")" +sanitize_branch_name() { + echo "$1" | sed "s@/@-@g" +} + +# Ex: extract 'pavics-sdi' from 'Ouranosinc/pavics-sdi'. +# USAGE: REPO_NAME_ONLY="$(extract_repo_name "$REPO_NAME")" +extract_repo_name() { + echo "$1" | sed "s@^.*/@@g" +} + +# Branches that have allowed characters such as '+' other than alphanum, '-', '_' and '.' are converted to '-' in archives. +# USAGE: FOLDER_NAME="$(sanitize_extracted_folder_name "$FOLDER_NAME")" +sanitize_extracted_folder_name() { + echo "$1" | sed "s@[^a-zA-Z0-9_\-\.]@-@g" +} + +# Presence of setup.cfg, tox.ini, pyproject.toml files confuse py.test execution rootdir discovery. +# USAGE: delete_files_confusing_pytest "$CHECKOUT_DIR" +delete_files_confusing_pytest() { + for afile in setup.cfg tox.ini pyproject.toml; do + if [ -f "$1/$afile" ]; then + rm -v "$1/$afile" + fi + done } downloadrepos_main() { @@ -49,22 +84,22 @@ downloadrepos_main() { fi if [ -z "$1" ]; then - if [ x"$DOWNLOAD_ALL_DEFAULT_REPOS" = xtrue -o x"$TEST_PAVICS_SDI_REPO" = xtrue ]; then + if [ x"$DOWNLOAD_ALL_DEFAULT_REPOS" = xtrue ] || [ x"$TEST_PAVICS_SDI_REPO" = xtrue ]; then downloadgithubrepos $PAVICS_SDI_REPO $PAVICS_SDI_BRANCH fi - if [ x"$DOWNLOAD_ALL_DEFAULT_REPOS" = xtrue -o x"$TEST_FINCH_REPO" = xtrue ]; then + if [ x"$DOWNLOAD_ALL_DEFAULT_REPOS" = xtrue ] || [ x"$TEST_FINCH_REPO" = xtrue ]; then downloadgithubrepos $FINCH_REPO $FINCH_BRANCH fi - if [ x"$DOWNLOAD_ALL_DEFAULT_REPOS" = xtrue -o x"$TEST_PAVICS_LANDING_REPO" = xtrue ]; then + if [ x"$DOWNLOAD_ALL_DEFAULT_REPOS" = xtrue ] || [ x"$TEST_PAVICS_LANDING_REPO" = xtrue ]; then downloadgithubrepos $PAVICS_LANDING_REPO $PAVICS_LANDING_BRANCH fi - if [ x"$DOWNLOAD_ALL_DEFAULT_REPOS" = xtrue -o x"$TEST_RAVEN_REPO" = xtrue ]; then + if [ x"$DOWNLOAD_ALL_DEFAULT_REPOS" = xtrue ] || [ x"$TEST_RAVEN_REPO" = xtrue ]; then downloadgithubrepos $RAVEN_REPO $RAVEN_BRANCH fi - if [ x"$DOWNLOAD_ALL_DEFAULT_REPOS" = xtrue -o x"$TEST_RAVENPY_REPO" = xtrue ]; then + if [ x"$DOWNLOAD_ALL_DEFAULT_REPOS" = xtrue ] || [ x"$TEST_RAVENPY_REPO" = xtrue ]; then downloadgithubrepos $RAVENPY_REPO $RAVENPY_BRANCH fi - if [ x"$DOWNLOAD_ALL_DEFAULT_REPOS" = xtrue -o x"$TEST_ESGF_COMPUTE_API_REPO" = xtrue ]; then + if [ x"$DOWNLOAD_ALL_DEFAULT_REPOS" = xtrue ] || [ x"$TEST_ESGF_COMPUTE_API_REPO" = xtrue ]; then downloadgithubrepos $ESGF_COMPUTE_API_REPO $ESGF_COMPUTE_API_BRANCH fi else diff --git a/testall b/testall index ec943fb..89d5c18 100755 --- a/testall +++ b/testall @@ -43,33 +43,28 @@ DOWNLOAD_ALL_DEFAULT_REPOS=false downloadrepos_main # 'export' useful vars so they can be used by the CONFIG_OVERRIDE_SCRIPT_URL in runtest. - -# replace all slash (/) by dash (-) because (/) is illegal in folder name -# for branch name of the format "feature/my_wizbang-feature" -# github does the same when downloading repo archive by downloadrepos above -export PAVICS_SDI_BRANCH="`echo "$PAVICS_SDI_BRANCH" | sed "s@/@-@g"`" -export PAVICS_SDI_REPO_NAME="`echo "$PAVICS_SDI_REPO" | sed "s@^.*/@@g"`" -export FINCH_BRANCH="`echo "$FINCH_BRANCH" | sed "s@/@-@g"`" -export FINCH_REPO_NAME="`echo "$FINCH_REPO" | sed "s@^.*/@@g"`" -export PAVICS_LANDING_BRANCH="`echo "$PAVICS_LANDING_BRANCH" | sed "s@/@-@g"`" -export PAVICS_LANDING_REPO_NAME="`echo "$PAVICS_LANDING_REPO" | sed "s@^.*/@@g"`" -export RAVEN_BRANCH="`echo "$RAVEN_BRANCH" | sed "s@/@-@g"`" -export RAVEN_REPO_NAME="`echo "$RAVEN_REPO" | sed "s@^.*/@@g"`" -export RAVENPY_BRANCH="`echo "$RAVENPY_BRANCH" | sed "s@/@-@g"`" -export RAVENPY_REPO_NAME="`echo "$RAVENPY_REPO" | sed "s@^.*/@@g"`" -export ESGF_COMPUTE_API_BRANCH="`echo "$ESGF_COMPUTE_API_BRANCH" | sed "s@/@-@g"`" -export ESGF_COMPUTE_API_REPO_NAME="`echo "$ESGF_COMPUTE_API_REPO" | sed "s@^.*/@@g"`" - -# branches that have allowed characters such as '+' other than alphanum, '-' and '_' are converted to '-' in archives -export PAVICS_SDI_DIR=`echo "${PAVICS_SDI_REPO_NAME}-${PAVICS_SDI_BRANCH}" | sed "s@[^a-zA-Z0-9_\-\.]@-@g"` -export FINCH_DIR=`echo "${FINCH_REPO_NAME}-${FINCH_BRANCH}" | sed "s@[^a-zA-Z0-9_\-\.]@-@g"` -export PAVICS_LANDING_DIR=`echo "${PAVICS_LANDING_REPO_NAME}-${PAVICS_LANDING_BRANCH}" | sed "s@[^a-zA-Z0-9_\-\.]@-@g"` -export RAVEN_DIR=`echo "${RAVEN_REPO_NAME}-${RAVEN_BRANCH}" | sed "s@[^a-zA-Z0-9_\-\.]@-@g"` -export RAVENPY_DIR=`echo "${RAVENPY_REPO_NAME}-${RAVENPY_BRANCH}" | sed "s@[^a-zA-Z0-9_\-\.]@-@g"` -export ESGF_COMPUTE_API_DIR=`echo "${ESGF_COMPUTE_API_REPO_NAME}-${ESGF_COMPUTE_API_BRANCH}" | sed "s@[^a-zA-Z0-9_\-\.]@-@g"` +export PAVICS_SDI_BRANCH="$(sanitize_branch_name "$PAVICS_SDI_BRANCH")" +export PAVICS_SDI_REPO_NAME="$(extract_repo_name "$PAVICS_SDI_REPO")" +export FINCH_BRANCH="$(sanitize_branch_name "$FINCH_BRANCH")" +export FINCH_REPO_NAME="$(extract_repo_name "$FINCH_REPO")" +export PAVICS_LANDING_BRANCH="$(sanitize_branch_name "$PAVICS_LANDING_BRANCH")" +export PAVICS_LANDING_REPO_NAME="$(extract_repo_name "$PAVICS_LANDING_REPO")" +export RAVEN_BRANCH="$(sanitize_branch_name "$RAVEN_BRANCH")" +export RAVEN_REPO_NAME="$(extract_repo_name "$RAVEN_REPO")" +export RAVENPY_BRANCH="$(sanitize_branch_name "$RAVENPY_BRANCH")" +export RAVENPY_REPO_NAME="$(extract_repo_name "$RAVENPY_REPO")" +export ESGF_COMPUTE_API_BRANCH="$(sanitize_branch_name "$ESGF_COMPUTE_API_BRANCH")" +export ESGF_COMPUTE_API_REPO_NAME="$(extract_repo_name "$ESGF_COMPUTE_API_REPO")" + +export PAVICS_SDI_DIR="$(sanitize_extracted_folder_name "${PAVICS_SDI_REPO_NAME}-${PAVICS_SDI_BRANCH}")" +export FINCH_DIR="$(sanitize_extracted_folder_name "${FINCH_REPO_NAME}-${FINCH_BRANCH}")" +export PAVICS_LANDING_DIR="$(sanitize_extracted_folder_name "${PAVICS_LANDING_REPO_NAME}-${PAVICS_LANDING_BRANCH}")" +export RAVEN_DIR="$(sanitize_extracted_folder_name "${RAVEN_REPO_NAME}-${RAVEN_BRANCH}")" +export RAVENPY_DIR="$(sanitize_extracted_folder_name "${RAVENPY_REPO_NAME}-${RAVENPY_BRANCH}")" +export ESGF_COMPUTE_API_DIR="$(sanitize_extracted_folder_name "${ESGF_COMPUTE_API_REPO_NAME}-${ESGF_COMPUTE_API_BRANCH}")" # lowercase VERIFY_SSL string -VERIFY_SSL="`echo "$VERIFY_SSL" | tr '[:upper:]' '[:lower:]'`" +VERIFY_SSL="$(lowercase "$VERIFY_SSL")" if [ x"$VERIFY_SSL" = xfalse ]; then # if Env var DISABLE_VERIFY_SSL is present, notebook should disable ssl # cert verification @@ -79,15 +74,6 @@ if [ x"$VERIFY_SSL" = xfalse ]; then echo "setting env var DISABLE_VERIFY_SSL for notebooks" fi -# presence of setup.cfg, tox.ini, pyproject.toml files confuse py.test execution rootdir discovery -rm -v $FINCH_REPO_NAME-$FINCH_BRANCH/setup.cfg -rm -v $RAVEN_REPO_NAME-$RAVEN_BRANCH/setup.cfg -rm -v $RAVEN_REPO_NAME-$RAVEN_BRANCH/pyproject.toml -rm -v $RAVENPY_REPO_NAME-$RAVENPY_BRANCH/setup.cfg -rm -v $RAVENPY_REPO_NAME-$RAVENPY_BRANCH/tox.ini -rm -v $RAVENPY_REPO_NAME-$RAVENPY_BRANCH/pyproject.toml -rm -v $ESGF_COMPUTE_API_REPO_NAME-$ESGF_COMPUTE_API_BRANCH/setup.cfg -rm -v $ESGF_COMPUTE_API_REPO_NAME-$ESGF_COMPUTE_API_BRANCH/tox.ini NOTEBOOKS_TO_TEST="" if [ x"$TEST_MAGPIE_AUTH" = xtrue ]; then @@ -100,6 +86,7 @@ if [ x"$TEST_PAVICS_SDI_REPO" = xtrue ]; then fi fi if [ x"$TEST_FINCH_REPO" = xtrue ]; then + delete_files_confusing_pytest "$FINCH_DIR" NOTEBOOKS_TO_TEST="$NOTEBOOKS_TO_TEST ${FINCH_DIR}/docs/source/notebooks/*.ipynb" fi @@ -118,13 +105,16 @@ if [ x"$TEST_PAVICS_LANDING_REPO" = xtrue ]; then fi if [ x"$TEST_RAVEN_REPO" = xtrue ]; then + delete_files_confusing_pytest "$RAVEN_DIR" NOTEBOOKS_TO_TEST="$NOTEBOOKS_TO_TEST ${RAVEN_DIR}/docs/source/notebooks/*.ipynb" fi if [ x"$TEST_RAVENPY_REPO" = xtrue ]; then + delete_files_confusing_pytest "$RAVENPY_DIR" NOTEBOOKS_TO_TEST="$NOTEBOOKS_TO_TEST ${RAVENPY_DIR}/docs/notebooks/*.ipynb" NOTEBOOKS_TO_TEST="$NOTEBOOKS_TO_TEST ${RAVENPY_DIR}/docs/notebooks/paper/*.ipynb" fi if [ x"$TEST_ESGF_COMPUTE_API_REPO" = xtrue ]; then + delete_files_confusing_pytest "$ESGF_COMPUTE_API_DIR" NOTEBOOKS_TO_TEST="$NOTEBOOKS_TO_TEST ${ESGF_COMPUTE_API_DIR}/examples/*.ipynb" fi From cccae703298d8c75c18ce555a0eb1f985d763878 Mon Sep 17 00:00:00 2001 From: Long Vu Date: Tue, 14 May 2024 17:30:37 -0400 Subject: [PATCH 04/11] testall: make available all functions in downloadrepos to CONFIG_PARAMETERS_SCRIPT_URL --- testall | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/testall b/testall index 89d5c18..e928ba5 100755 --- a/testall +++ b/testall @@ -2,6 +2,9 @@ . ./default_build_params +DOWNLOADREPOS_AS_LIB=1 +. ./downloadrepos + set -x # Allow full override of ALL Jenkins params before running test suite. @@ -37,8 +40,6 @@ fi git clean -fdx # download all additional repos containing extra notebooks to test -DOWNLOADREPOS_AS_LIB=1 -. ./downloadrepos DOWNLOAD_ALL_DEFAULT_REPOS=false downloadrepos_main From b84da182ee205e9983229e50fa4fce8b1d5cbcd6 Mon Sep 17 00:00:00 2001 From: Long Vu Date: Tue, 14 May 2024 18:05:26 -0400 Subject: [PATCH 05/11] Jenkins: all artifacts are now under buildout/ so easier to add new nb from new repos No need to update 'archiveArtifacts' in Jenkinsfile anymore. Since we only download enabled repos, we could not hardcode artifact repo list anyways. DEFAULT_PRODUCTION_HOST also overridable for other repos. --- Jenkinsfile | 11 +---------- runtest | 34 +++++++++++++++++++++------------- testall | 1 + 3 files changed, 23 insertions(+), 23 deletions(-) diff --git a/Jenkinsfile b/Jenkinsfile index cadb2e9..7782d36 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -144,16 +144,7 @@ Note this is another run, will double the time and no guaranty to have same erro post { always { - archiveArtifacts(artifacts: 'notebooks/*.ipynb', fingerprint: true) - archiveArtifacts(artifacts: 'pavics-sdi-*/docs/source/notebooks/*.ipynb', fingerprint: true) - archiveArtifacts(artifacts: 'pavics-sdi-*/docs/source/notebook-components/*.ipynb', fingerprint: true) - archiveArtifacts(artifacts: 'finch-*/docs/source/notebooks/*.ipynb', fingerprint: true) - archiveArtifacts(artifacts: 'raven-*/docs/source/notebooks/*.ipynb', fingerprint: true) - archiveArtifacts(artifacts: 'RavenPy-*/docs/notebooks/*.ipynb', fingerprint: true) - archiveArtifacts(artifacts: 'RavenPy-*/docs/notebooks/paper/*.ipynb', fingerprint: true) - archiveArtifacts(artifacts: 'esgf-compute-api-*/examples/*.ipynb', fingerprint: true) - archiveArtifacts(artifacts: 'PAVICS-landing-*/content/notebooks/climate_indicators/*.ipynb', fingerprint: true) - archiveArtifacts(artifacts: 'buildout/*.output.ipynb', fingerprint: true, allowEmptyArchive: true) + archiveArtifacts(artifacts: 'buildout/*.ipynb', fingerprint: true, allowEmptyArchive: true) archiveArtifacts(artifacts: 'buildout/env-dump/', fingerprint: true) } unsuccessful { // Run if the current builds status is "Aborted", "Failure" or "Unstable" diff --git a/runtest b/runtest index 1ab8bb5..eca5a7d 100755 --- a/runtest +++ b/runtest @@ -1,6 +1,10 @@ #!/bin/sh -DEFAULT_PRODUCTION_HOST="pavics.ouranos.ca" +# Load shared functions, make available to CONFIG_OVERRIDE_SCRIPT_URL. +DOWNLOADREPOS_AS_LIB=1 +. ./downloadrepos + +DEFAULT_PRODUCTION_HOST="${DEFAULT_PRODUCTION_HOST:=pavics.ouranos.ca}" NOTEBOOKS="$1" if [ -z "$NOTEBOOKS" ]; then @@ -67,7 +71,7 @@ py.test --nbval $NOTEBOOKS --nbval-sanitize-with notebooks/output-sanitize.cfg $ EXIT_CODE="$?" # lowercase SAVE_RESULTING_NOTEBOOK string -SAVE_RESULTING_NOTEBOOK="`echo "$SAVE_RESULTING_NOTEBOOK" | tr '[:upper:]' '[:lower:]'`" +SAVE_RESULTING_NOTEBOOK="$(lowercase "$SAVE_RESULTING_NOTEBOOK")" # save notebooks resulting from the run @@ -79,15 +83,19 @@ SAVE_RESULTING_NOTEBOOK="`echo "$SAVE_RESULTING_NOTEBOOK" | tr '[:upper:]' '[:lo # work-around as nbval can not save the result of the run # see https://github.com/computationalmodelling/nbval/issues/112 -if [ x"$SAVE_RESULTING_NOTEBOOK" = xtrue ]; then - mkdir -p buildout - for nb in $NOTEBOOKS; do - filename="`basename "$nb"`" - filename="`echo "$filename" | sed "s/.ipynb$//"`" # remove .ipynb ext - if [ -e "buildout/${filename}.output.ipynb" ]; then - # prevent name clash - filename="${filename}_`date '+%s'`" - fi +mkdir -p buildout/ +for nb in $NOTEBOOKS; do + filename="$(basename "$nb")" + filename="$(echo "$filename" | sed "s/.ipynb$//")" # remove .ipynb ext + if [ -e "buildout/${filename}.ipynb" ]; then + # prevent name clash + filename="${filename}_$(date '+%s')" + fi + + # Save original notebooks that we sed replace the PAVICS_HOST. + cp "$nb" "buildout/${filename}.ipynb" + + if [ x"$SAVE_RESULTING_NOTEBOOK" = xtrue ]; then # Timeout must not be more than 240s (4 mins). Default in Jenkinsfile. # Tutorial notebooks should be fast so user do not lose patience waiting # for them to run. If more than 4 mins, in addition to simplifying the @@ -95,8 +103,8 @@ if [ x"$SAVE_RESULTING_NOTEBOOK" = xtrue ]; then jupyter nbconvert --to notebook --execute \ --ExecutePreprocessor.timeout=${SAVE_RESULTING_NOTEBOOK_TIMEOUT:=240} --allow-errors \ --output-dir buildout --output "${filename}.output.ipynb" "$nb" - done -fi + fi +done # exit with return code from py.test exit $EXIT_CODE diff --git a/testall b/testall index e928ba5..1e05305 100755 --- a/testall +++ b/testall @@ -2,6 +2,7 @@ . ./default_build_params +# Load shared functions, make available to CONFIG_PARAMETERS_SCRIPT_URL. DOWNLOADREPOS_AS_LIB=1 . ./downloadrepos From cfe5f95a85a4ca21dd630dd46850ac5252105632 Mon Sep 17 00:00:00 2001 From: Long Vu Date: Tue, 14 May 2024 18:36:13 -0400 Subject: [PATCH 06/11] testall: git clean before CONFIG_PARAMETERS_SCRIPT_URL to avoid wiping files created by the script --- testall | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/testall b/testall index 1e05305..6b254d6 100755 --- a/testall +++ b/testall @@ -8,6 +8,9 @@ DOWNLOADREPOS_AS_LIB=1 set -x +# emulate "clean after checkout" of single branch pipeline +git clean -fdx + # Allow full override of ALL Jenkins params before running test suite. # Intended to overrride all params in Jenkinsfile. # @@ -37,9 +40,6 @@ if [ -n "$CONFIG_PARAMETERS_SCRIPT_URL" ]; then . "$TMP_PARAMS_OVERRIDE" fi -# emulate "clean after checkout" of single branch pipeline -git clean -fdx - # download all additional repos containing extra notebooks to test DOWNLOAD_ALL_DEFAULT_REPOS=false downloadrepos_main From 7253e6aa6d410dcedeaaf1a6a89eed3067103e6e Mon Sep 17 00:00:00 2001 From: Long Vu Date: Tue, 14 May 2024 18:55:12 -0400 Subject: [PATCH 07/11] runtest: allow to override --nbval-sanitize-with from CONFIG_OVERRIDE_SCRIPT_URL --- runtest | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/runtest b/runtest index eca5a7d..e804850 100755 --- a/runtest +++ b/runtest @@ -67,7 +67,8 @@ if [ -n "$CONFIG_OVERRIDE_SCRIPT_URL" ]; then fi fi -py.test --nbval $NOTEBOOKS --nbval-sanitize-with notebooks/output-sanitize.cfg $PYTEST_EXTRA_OPTS +# CONFIG_OVERRIDE_SCRIPT_URL can override NBVAL_SANITIZE_CFG_FILE. +py.test --nbval $NOTEBOOKS --nbval-sanitize-with "${NBVAL_SANITIZE_CFG_FILE:=notebooks/output-sanitize.cfg}" $PYTEST_EXTRA_OPTS EXIT_CODE="$?" # lowercase SAVE_RESULTING_NOTEBOOK string From d93b229d5b679a8033506433b779b717639ad583 Mon Sep 17 00:00:00 2001 From: Long Vu Date: Tue, 14 May 2024 19:11:23 -0400 Subject: [PATCH 08/11] downloadrepos: avoid set +x because it hides all subsequent commands from being logged --- downloadrepos | 2 -- 1 file changed, 2 deletions(-) diff --git a/downloadrepos b/downloadrepos index f7f1b20..a9d8765 100755 --- a/downloadrepos +++ b/downloadrepos @@ -15,13 +15,11 @@ downloadgithubrepos() { repo_owner="`echo "$owner_and_repo_name" | sed "s@/.*\\$@@g"`" repo_name="`echo "$owner_and_repo_name" | sed "s@^.*/@@g"`" repo_branch="$1"; shift - set -x # clean up other previously downloaded branches of the same repo as well rm -rf ${repo_name}-* ls | grep $repo_name downloadrepos https://github.com/$repo_owner/$repo_name "$repo_branch" ls | grep $repo_name - set +x } # USAGE: VAR_TO_LOWER="$(lowercase "$VAR_TO_LOWER")" From 9160053016260a1b8fbeea8a77af76dc49855ba4 Mon Sep 17 00:00:00 2001 From: Long Vu Date: Tue, 14 May 2024 19:40:09 -0400 Subject: [PATCH 09/11] jenkins sample override: demo runnings notebooks from an external repo --- .../jenkins-params-external-repos.include.sh | 47 +++++++++++++++++++ 1 file changed, 47 insertions(+) create mode 100644 test-override/jenkins-params-external-repos.include.sh diff --git a/test-override/jenkins-params-external-repos.include.sh b/test-override/jenkins-params-external-repos.include.sh new file mode 100644 index 0000000..b62858a --- /dev/null +++ b/test-override/jenkins-params-external-repos.include.sh @@ -0,0 +1,47 @@ +#!/bin/sh +# +# Sample Jenkins params override script to demonstrate running new notebooks +# from an external repo and on-the-fly CONFIG_OVERRIDE_SCRIPT_URL file creation. +# +# This script is intended for param CONFIG_PARAMETERS_SCRIPT_URL. + +# Scenario: we want to run notebooks from an external repo, unknown to current Jenkins config. +# https://github.com/roocs/rook/tree/master/notebooks/*.ipynb + +# Disable all existing default repos to avoid downloading them and running them. +TEST_PAVICS_SDI_REPO="false" +TEST_FINCH_REPO="false" +TEST_PAVICS_LANDING_REPO="false" +TEST_LOCAL_NOTEBOOKS="false" + +# Set new external repo vars. Need 'export' so CONFIG_OVERRIDE_SCRIPT_URL can see them. +export ROOK_REPO="roocs/rook" +export ROOK_BRANCH="master" + +# Not checking for expected output, just checking whether the code can run without errors. +PYTEST_EXTRA_OPTS="$PYTEST_EXTRA_OPTS --nbval-lax" + +# Create CONFIG_OVERRIDE_SCRIPT_URL file on-the-fly to run the notebooks from +# our external repo. + +CONFIG_OVERRIDE_SCRIPT_URL="/tmp/custom-repos.include.sh" + +# Populate the content of our CONFIG_OVERRIDE_SCRIPT_URL. +echo ' +#!/bin/sh +# Sample config override script to run new notebooks from new external repo. + +# Replicate processing steps in 'testall' script. + +# Download the external repo. +downloadgithubrepos $ROOK_REPO $ROOK_BRANCH + +# Prep vars for including new nbs in nb list to test. +ROOK_REPO_NAME="$(extract_repo_name "$ROOK_REPO")" +ROOK_DIR="$(sanitize_extracted_folder_name "${ROOK_REPO_NAME}-${ROOK_BRANCH}")" + +delete_files_confusing_pytest "$ROOK_DIR" + +# Set new nbs as nb list to test. +NOTEBOOKS="$ROOK_DIR/notebooks/*.ipynb" +' > "$CONFIG_OVERRIDE_SCRIPT_URL" From 25c0abb438c43b6d8de313e98684dca5846b0231 Mon Sep 17 00:00:00 2001 From: Long Vu Date: Tue, 14 May 2024 19:47:29 -0400 Subject: [PATCH 10/11] runtest: CONFIG_PARAMETERS_SCRIPT_URL can override DEFAULT_PRODUCTION_HOST --- runtest | 1 + 1 file changed, 1 insertion(+) diff --git a/runtest b/runtest index e804850..a4ae259 100755 --- a/runtest +++ b/runtest @@ -4,6 +4,7 @@ DOWNLOADREPOS_AS_LIB=1 . ./downloadrepos +# CONFIG_PARAMETERS_SCRIPT_URL can override DEFAULT_PRODUCTION_HOST. DEFAULT_PRODUCTION_HOST="${DEFAULT_PRODUCTION_HOST:=pavics.ouranos.ca}" NOTEBOOKS="$1" From 01824907bcef64218b071d9a6504334c05780d60 Mon Sep 17 00:00:00 2001 From: Long Vu Date: Thu, 16 May 2024 16:22:05 -0400 Subject: [PATCH 11/11] runtest: add repo and branch name to archived nbs name to id which repo they came from and further avoid name clash --- runtest | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/runtest b/runtest index a4ae259..e7ab37a 100755 --- a/runtest +++ b/runtest @@ -87,7 +87,8 @@ SAVE_RESULTING_NOTEBOOK="$(lowercase "$SAVE_RESULTING_NOTEBOOK")" mkdir -p buildout/ for nb in $NOTEBOOKS; do - filename="$(basename "$nb")" + repo_branch_name="$(extract_repo_name "$nb")" + filename="${repo_branch_name}--$(basename "$nb")" filename="$(echo "$filename" | sed "s/.ipynb$//")" # remove .ipynb ext if [ -e "buildout/${filename}.ipynb" ]; then # prevent name clash