From 3ba08775a5bd618c53ad1253a798099826782087 Mon Sep 17 00:00:00 2001 From: Dan Mick Date: Thu, 26 Oct 2023 16:13:00 -0700 Subject: [PATCH] Changes to allow staging/testing a container image: New $PRERELEASE variable pulls pkgs from prerelease dir on download.ceph.com (requires PRERELEASE_USERNAME/PRERELEASE_PASSWORD in env) Existing $FORCE_BUILD ignores images on quay.io and builds unconditionally Add ability to pass in desired container flavors Hack ceph-release after installation, because ceph-release is built during ceph build, and doesn't know what we need Use tempfile to pass tagname back to parent New $TEST_BUILD_ONLY avoids attempting to push image to quay.io Avoid logging into quay.io if PRERELEASE Note: only x86_64 supported Signed-off-by: Dan Mick --- Makefile | 5 ++++ .../centos/daemon-base/__DOCKERFILE_INSTALL__ | 14 +++++++++- contrib/build-ceph-base.sh | 26 ++++++++++++++----- contrib/ceph-build-config.sh | 10 ++++--- maint-lib/makelib.mk | 3 +++ 5 files changed, 48 insertions(+), 10 deletions(-) diff --git a/Makefile b/Makefile index 9b3a1113e..3cb56ff79 100644 --- a/Makefile +++ b/Makefile @@ -43,6 +43,11 @@ BASEOS_TAG ?= "" CEPH_DEVEL ?= false OSD_FLAVOR ?= "default" +# Default to not doing prerelease (env var must have a value) +PRERELEASE ?= false +PRERELEASE_USERNAME ?= "" +PRERELEASE_PASSWORD ?= "" + # ============================================================================== # Internal definitions diff --git a/ceph-releases/ALL/centos/daemon-base/__DOCKERFILE_INSTALL__ b/ceph-releases/ALL/centos/daemon-base/__DOCKERFILE_INSTALL__ index 1857010ee..239d9d504 100644 --- a/ceph-releases/ALL/centos/daemon-base/__DOCKERFILE_INSTALL__ +++ b/ceph-releases/ALL/centos/daemon-base/__DOCKERFILE_INSTALL__ @@ -93,9 +93,21 @@ bash -c ' \ fi ;\ else \ RELEASE_VER=1 ;\ - REPO_URL="http://download.ceph.com/rpm-${CEPH_VERSION}/el__ENV_[DISTRO_VERSION]__/"; \ + if [[ __ENV_[PRERELEASE]__ = true ]] ; then \ + REPO_URL="https://__ENV_[PRERELEASE_USERNAME]__:__ENV_[PRERELEASE_PASSWORD]__@download.ceph.com/prerelease/rpm-${CEPH_VERSION}/el__ENV_[DISTRO_VERSION]__/"; \ + else \ + REPO_URL="http://download.ceph.com/rpm-${CEPH_VERSION}/el__ENV_[DISTRO_VERSION]__/"; \ + fi \ fi && \ rpm -Uvh "$REPO_URL/noarch/ceph-release-1-${RELEASE_VER}.el__ENV_[DISTRO_VERSION]__.noarch.rpm" && \ + # + # We have to change this after installing because ceph-release is built long + # ago in the ceph build + # + if [[ __ENV_[PRERELEASE]__ = true ]] ; then \ + sed -i "s;http://download.ceph.com/;https://__ENV_[PRERELEASE_USERNAME]__:__ENV_[PRERELEASE_PASSWORD]__@download.ceph.com/prerelease/;" /etc/yum.repos.d/ceph.repo ; \ + dnf clean expire-cache ; \ + fi && \ if [[ __ENV_[DISTRO_VERSION]__ -eq 8 ]]; then \ yum install -y dnf-plugins-core ; \ yum copr enable -y tchaikov/python-scikit-learn ; \ diff --git a/contrib/build-ceph-base.sh b/contrib/build-ceph-base.sh index 888ec3070..f22b276c6 100644 --- a/contrib/build-ceph-base.sh +++ b/contrib/build-ceph-base.sh @@ -17,7 +17,9 @@ source "${SCRIPT_DIR}/ceph-build-config.sh" flavors_to_build="$(get_flavors_to_build "${ARCH}")" install_docker -do_login +if [[ "${PRERELEASE}" != true ]] ; then + do_login +fi do_clean echo '' @@ -73,10 +75,10 @@ for flavor in $flavors_to_build; do FLAVORS="${ceph_codename}-${version_without_build},${ceph_container_distro_dir},${distro_release}" \ IMAGES_TO_BUILD=daemon-base \ TAG_REGISTRY="" \ - DAEMON_BASE_TAG="${full_build_tag}" \ - BASEOS_REGISTRY="${baseos_registry_setting}" \ - BASEOS_REPO="${baseos_repo_setting}" \ - BASEOS_TAG="${baseos_tag_setting}" \ + DAEMON_BASE_TAG=${full_build_tag} \ + BASEOS_REGISTRY=${baseos_registry_setting} \ + BASEOS_REPO=${baseos_repo_setting} \ + BASEOS_TAG=${baseos_tag_setting} \ build" if [ -z "${DRY_RUN:-}" ]; then ${make_cmd} @@ -84,7 +86,19 @@ for flavor in $flavors_to_build; do # Just echo the make command we would've executed if this is a dry run dry_run_info "${make_cmd}" fi - do_push "${full_build_tag}" + if [ "$PRERELEASE" = true ] ; then + if [ -z "$FULL_BUILD_TAG_TMPFILE" ] ; then + echo "FULL_BUILD_TAG_TMPFILE not passed, can't pass back container tag name" + else + echo "${full_build_tag}" > "$FULL_BUILD_TAG_TMPFILE" + fi + continue + fi + if [ "$TEST_BUILD_ONLY" = true ] ; then + echo "would push ${full_build_tag}" + else + do_push "${full_build_tag}" + fi done # for version in ${ceph_version_list} done # for flavor in $flavors_to_build diff --git a/contrib/ceph-build-config.sh b/contrib/ceph-build-config.sh index 580ad5fd7..d32a3761a 100644 --- a/contrib/ceph-build-config.sh +++ b/contrib/ceph-build-config.sh @@ -13,8 +13,8 @@ trap 'exit $?' ERR # These build scripts don't need to have the aarch64 part of the distro specified # I.e., specifying 'luminous,centos-arm64,7' is not necessary for aarch64 builds; these scripts # will do the right build. See configurable CENTOS_AARCH64_FLAVOR_DISTRO below -X86_64_FLAVORS_TO_BUILD="pacific,centos,8 quincy,centos,8 reef,centos,8" -AARCH64_FLAVORS_TO_BUILD="pacific,centos,8 quincy,centos,8 reef,centos,8" +X86_64_FLAVORS_TO_BUILD="${X86_64_FLAVORS_TO_BUILD:-pacific,centos,8 quincy,centos,8 reef,centos,8}" +AARCH64_FLAVORS_TO_BUILD="${AARCH64_FLAVORS_TO_BUILD:-pacific,centos,8 quincy,centos,8 reef,centos,8}" # Allow running this script with the env var ARCH='aarch64' to build arm images # ARCH='x86_64' @@ -226,7 +226,11 @@ function get_ceph_download_url () { *) error "get_ceph_download_url - unknown distro '${distro}''" esac - echo "https://download.ceph.com/${flavor_path}/${arch}/" + if [ "$PRERELEASE" = true ] ; then + echo "https://$PRERELEASE_USERNAME:$PRERELEASE_PASSWORD@download.ceph.com/prerelease/${flavor_path}/${arch}/" + else + echo "https://download.ceph.com/${flavor_path}/${arch}/" + fi } # Return a list of the ceph version strings available on the server as: diff --git a/maint-lib/makelib.mk b/maint-lib/makelib.mk index 16b6a5a3a..0eca35fcb 100644 --- a/maint-lib/makelib.mk +++ b/maint-lib/makelib.mk @@ -37,6 +37,9 @@ $(shell bash -c 'set -eu ; \ set_var IMAGES_TO_BUILD "$(IMAGES_TO_BUILD)" ; \ set_var STAGING_DIR "staging/$$CEPH_VERSION$$CEPH_POINT_RELEASE-$$DISTRO-$$DISTRO_VERSION-$$HOST_ARCH" ; \ set_var RELEASE "$(RELEASE)" ; \ + set_var PRERELEASE "$(PRERELEASE)" ; \ + set_var PRERELEASE_USERNAME "$(PRERELEASE_USERNAME)" ; \ + set_var PRERELEASE_PASSWORD "$(PRERELEASE_PASSWORD)" ; \ \ daemon_base_img="$$(val_or_default "$(DAEMON_BASE_TAG)" \ "daemon-base:$(RELEASE)-$$CEPH_VERSION-$$DISTRO-$$BASEOS_TAG-$$HOST_ARCH")" ; \