From aa3d78b1a2ba46905cef6355d30511ad205995b9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Pawe=C5=82=20Chrz=C4=85szcz?= Date: Wed, 11 Oct 2023 12:53:40 +0200 Subject: [PATCH 1/5] Use arm docker executor instead of running docker manually Previously there were no docker executors for arm, so we had to use the machine ones. Now the same jobs can be reused for both architectures. --- .circleci/template.yml | 89 ++++++++++++++++-------------------------- 1 file changed, 33 insertions(+), 56 deletions(-) diff --git a/.circleci/template.yml b/.circleci/template.yml index edf6d7654b3..e23428ba90b 100644 --- a/.circleci/template.yml +++ b/.circleci/template.yml @@ -203,9 +203,14 @@ executors: - *elasticsearch_container - *cassandra_container - *cassandra_proxy_container - arm64: - machine: - image: ubuntu-2004:current + otp_26_arm64: + docker: + - image: *OTP26 + resource_class: arm.medium + otp_26_redis_arm64: + docker: + - image: *OTP26 + - *redis_container resource_class: arm.medium commands: fetch_build_packages: @@ -213,7 +218,7 @@ commands: - run: name: Install packages necessary for building command: | - $EXEC tools/circle-install-packages.sh \ + tools/circle-install-packages.sh \ 'libssl-dev unixodbc-dev unixodbc tdsodbc rsync zlib1g-dev' maybe_build_deps_and_cache: steps: @@ -274,7 +279,7 @@ commands: when: on_success command: | echo "Success!" - $EXEC ./rebar3 codecov analyze + ./rebar3 codecov analyze tools/circle-upload-codecov.sh run_small_tests: steps: @@ -282,11 +287,11 @@ commands: - run: name: Wait for redis command: | - $EXEC tools/wait-for-it.sh -p 6379 + tools/wait-for-it.sh -p 6379 - run: name: Run Small Tests command: | - $EXEC tools/test.sh -p small_tests -s true -e true + tools/test.sh -p small_tests -s true -e true run_docker_smoke_test: steps: - checkout @@ -394,7 +399,7 @@ commands: steps: - run: name: Prepare for cache - command: $EXEC bash -c 'echo $OTP_VERSION-$ARCH > otp_version' + command: bash -c 'echo $OTP_VERSION-$ARCH > otp_version' jobs: # ######################### @@ -415,16 +420,14 @@ jobs: command: tools/circle-build-and-push-docker.sh docker_smoke_test: - executor: otp_26 + executor: << parameters.executor >> + parameters: + executor: + type: executor steps: - setup_remote_docker - run_docker_smoke_test - docker_smoke_test_arm64: - executor: arm64 - steps: - - run_docker_smoke_test - docs_build_deploy: parallelism: 1 docker: @@ -477,23 +480,6 @@ jobs: pip3 install mike mike deploy $DOCS_TAG --remote git@github.com:esl/MongooseDocs.git --branch gh-pages --push --rebase - build_in_docker_arm64: - executor: arm64 - environment: - DOCKER_IMAGE: *OTP26 - EXEC: docker exec main - steps: - - run: docker run -dit --name main --rm -v `pwd`:`pwd` $DOCKER_IMAGE - - checkout - - prepare_for_cache - - restore_certs - - fetch_build_packages - - run: $EXEC tools/configure with-all - - run: $EXEC tools/build-releases.sh - - run: $EXEC make rel - - cache_prod_build: {arch: arm64} - - persist_workspace - build_in_docker: executor: << parameters.executor >> parameters: @@ -503,6 +489,10 @@ jobs: type: boolean description: When enabled, prod release will be generated besides the dev ones default: false + arch: + type: string + description: Target architecture + default: amd64 parallelism: 1 environment: SKIP_RELEASE: 1 @@ -522,7 +512,7 @@ jobs: condition: << parameters.build_prod >> steps: - run: make rel - - cache_prod_build: {arch: amd64} + - cache_prod_build: {arch: << parameters.arch >>} - run: name: Generate DOAP and Markdown files with supported XEPs command: make xeplist @@ -546,27 +536,6 @@ jobs: - upload_results_to_aws - publish_github_comment - small_tests_in_docker_arm64: - executor: arm64 - environment: - DOCKER_IMAGE: *OTP26 - EXEC: docker exec main - PRESET: small_tests - SKIP_AUTO_COMPILE: true - steps: - - run: - name: Run main docker image - command: | - docker run -dit -e PRESET -e SKIP_AUTO_COMPILE --name main --network host --rm \ - -v `pwd`:`pwd` $DOCKER_IMAGE - - run: - name: Run redis docker image - command: | - docker run -dit --name redis --network host --rm redis:__REDIS_VERSION__ - - run_small_tests - - upload_results_to_aws - - publish_github_comment - big_tests_in_docker: executor: << parameters.executor >> parallelism: 1 @@ -799,9 +768,12 @@ workflows: context: mongooseim-org build_prod: false filters: *all_tags - - build_in_docker_arm64: + - build_in_docker: name: otp_26_docker_arm64 + executor: otp_26_arm64 context: mongooseim-org + build_prod: true + arch: arm64 filters: *all_tags # ============= DIALYZER ============= - dialyzer: @@ -849,8 +821,9 @@ workflows: requires: - otp_26_docker filters: *all_tags - - small_tests_in_docker_arm64: + - small_tests_in_docker: name: small_tests_26_arm64 + executor: otp_26_redis_arm64 context: mongooseim-org requires: - otp_26_docker_arm64 @@ -1004,11 +977,15 @@ workflows: filters: *all_tags - docker_smoke_test: + name: docker_smoke_test + executor: otp_26 context: mongooseim-org requires: - docker_build_and_ship filters: *all_tags - - docker_smoke_test_arm64: + - docker_smoke_test: + name: docker_smoke_test_arm64 + executor: otp_26_arm64 context: mongooseim-org requires: - docker_build_and_ship From d494bb09899c7de46813c4293dd5acc6b8dab1bd Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Pawe=C5=82=20Chrz=C4=85szcz?= Date: Wed, 11 Oct 2023 15:20:08 +0200 Subject: [PATCH 2/5] Bump redis version, because only the recent versions support arm64 The machine executor was apparently able to run x86_64 images as well. --- tools/db-versions.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tools/db-versions.sh b/tools/db-versions.sh index f58d89419d6..7aa405b100a 100644 --- a/tools/db-versions.sh +++ b/tools/db-versions.sh @@ -9,7 +9,7 @@ PGSQL_VERSION_DEFAULT=latest LDAP_VERSION_DEFAULT="1.5.0" -REDIS_VERSION_DEFAULT="6.2.6" +REDIS_VERSION_DEFAULT="7.2.1" RMQ_VERSION_DEFAULT="3.11-alpine" From 8c2e5489875ab8fa5f0fcce8caf1fd1bfc9a77b4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Pawe=C5=82=20Chrz=C4=85szcz?= Date: Thu, 12 Oct 2023 08:38:56 +0200 Subject: [PATCH 3/5] Add missing info about JSON --- .circleci/template.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.circleci/template.yml b/.circleci/template.yml index e23428ba90b..fcb96b2343e 100644 --- a/.circleci/template.yml +++ b/.circleci/template.yml @@ -514,7 +514,7 @@ jobs: - run: make rel - cache_prod_build: {arch: << parameters.arch >>} - run: - name: Generate DOAP and Markdown files with supported XEPs + name: Generate DOAP, JSON and Markdown files with supported XEPs command: make xeplist - run: name: Build Big Tests From 5c8a61c846e834795e7c03d9cb08fa577252fbff Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Pawe=C5=82=20Chrz=C4=85szcz?= Date: Thu, 12 Oct 2023 08:39:29 +0200 Subject: [PATCH 4/5] Download uploaders for aws and codecov for the correct platform The machine arm executor was able to run the x86_64 code, while the docker one is not. --- tools/circle-upload-codecov.sh | 10 +++++++--- tools/circleci-upload-to-s3.sh | 4 +--- 2 files changed, 8 insertions(+), 6 deletions(-) diff --git a/tools/circle-upload-codecov.sh b/tools/circle-upload-codecov.sh index 90cd2fe3876..49ef48e583c 100755 --- a/tools/circle-upload-codecov.sh +++ b/tools/circle-upload-codecov.sh @@ -2,11 +2,15 @@ set -eo pipefail +PLATFORM=`uname -m` +[ $PLATFORM == "x86_64" ] && PLATFORM=linux +BASE_URL="https://uploader.codecov.io/latest/$PLATFORM" + # Source: https://docs.codecov.com/docs/codecov-uploader#integrity-checking-the-uploader curl https://keybase.io/codecovsecurity/pgp_keys.asc | gpg --no-default-keyring --keyring trustedkeys.gpg --import -curl -Os https://uploader.codecov.io/latest/linux/codecov -curl -Os https://uploader.codecov.io/latest/linux/codecov.SHA256SUM -curl -Os https://uploader.codecov.io/latest/linux/codecov.SHA256SUM.sig +curl -Os "$BASE_URL/codecov" +curl -Os "$BASE_URL/codecov.SHA256SUM" +curl -Os "$BASE_URL/codecov.SHA256SUM.sig" gpgv codecov.SHA256SUM.sig codecov.SHA256SUM shasum -a 256 -c codecov.SHA256SUM diff --git a/tools/circleci-upload-to-s3.sh b/tools/circleci-upload-to-s3.sh index 54db7e39890..40f60e7d580 100755 --- a/tools/circleci-upload-to-s3.sh +++ b/tools/circleci-upload-to-s3.sh @@ -14,8 +14,6 @@ fi echo "Uploading test results to s3" echo $(s3_url ${CT_REPORTS}) - - FILE_COUNT=$(find "${CT_REPORTS}" -type f | wc -l) echo "Uploading $FILE_COUNT files" ls $CT_REPORTS @@ -23,7 +21,7 @@ ls $CT_REPORTS if which aws ; then echo "aws tool ready" else - curl "https://awscli.amazonaws.com/awscli-exe-linux-x86_64.zip" -o "awscliv2.zip" + curl "https://awscli.amazonaws.com/awscli-exe-linux-`uname -m`.zip" -o "awscliv2.zip" unzip -q awscliv2.zip sudo ./aws/install fi From c075882efa40dc16c96b64eb10917bf19deb9c48 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Pawe=C5=82=20Chrz=C4=85szcz?= Date: Fri, 13 Oct 2023 16:28:20 +0200 Subject: [PATCH 5/5] Apply code review comments --- tools/circle-upload-codecov.sh | 8 ++++---- tools/circleci-upload-to-s3.sh | 2 +- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/tools/circle-upload-codecov.sh b/tools/circle-upload-codecov.sh index 49ef48e583c..a50f1cf334a 100755 --- a/tools/circle-upload-codecov.sh +++ b/tools/circle-upload-codecov.sh @@ -2,15 +2,15 @@ set -eo pipefail -PLATFORM=`uname -m` +PLATFORM=$(uname -m) [ $PLATFORM == "x86_64" ] && PLATFORM=linux BASE_URL="https://uploader.codecov.io/latest/$PLATFORM" # Source: https://docs.codecov.com/docs/codecov-uploader#integrity-checking-the-uploader curl https://keybase.io/codecovsecurity/pgp_keys.asc | gpg --no-default-keyring --keyring trustedkeys.gpg --import -curl -Os "$BASE_URL/codecov" -curl -Os "$BASE_URL/codecov.SHA256SUM" -curl -Os "$BASE_URL/codecov.SHA256SUM.sig" +curl -Os "${BASE_URL}/codecov" +curl -Os "${BASE_URL}/codecov.SHA256SUM" +curl -Os "${BASE_URL}/codecov.SHA256SUM.sig" gpgv codecov.SHA256SUM.sig codecov.SHA256SUM shasum -a 256 -c codecov.SHA256SUM diff --git a/tools/circleci-upload-to-s3.sh b/tools/circleci-upload-to-s3.sh index 40f60e7d580..db951a9d672 100755 --- a/tools/circleci-upload-to-s3.sh +++ b/tools/circleci-upload-to-s3.sh @@ -21,7 +21,7 @@ ls $CT_REPORTS if which aws ; then echo "aws tool ready" else - curl "https://awscli.amazonaws.com/awscli-exe-linux-`uname -m`.zip" -o "awscliv2.zip" + curl "https://awscli.amazonaws.com/awscli-exe-linux-$(uname -m).zip" -o "awscliv2.zip" unzip -q awscliv2.zip sudo ./aws/install fi