Skip to content

feat(BOUN-1236): implement secret tracking #16780

feat(BOUN-1236): implement secret tracking

feat(BOUN-1236): implement secret tracking #16780

Workflow file for this run

name: CI Main
on:
merge_group:
# Allows you to run this workflow manually from the Actions tab
workflow_dispatch:
push:
branches:
- master
- 'dev-gh-*'
pull_request:
branches-ignore:
- hotfix-* # This is to ensure that this workflow is not triggered twice on ic-private, as it's already triggered from release-testing
# Used as reusable workflow within release-testing workflow
workflow_call:
# runs for the same workflow are cancelled on PRs but not on master
# explanation: on push to master head_ref is not set, so we want it to fall back to run_id so it is not cancelled
concurrency:
group: ${{ github.workflow }}-${{ github.head_ref || github.run_id }}
cancel-in-progress: true
env:
CI_COMMIT_SHA: ${{ github.sha }}
CI_JOB_NAME: ${{ github.job }}
CI_JOB_URL: "${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}"
CI_PIPELINE_SOURCE: ${{ github.event_name }}
CI_PROJECT_DIR: ${{ github.workspace }}
CI_EVENT_NAME: ${{ github.event_name }}
BRANCH_NAME: ${{ github.head_ref || github.ref_name }}
CI_RUN_ID: ${{ github.run_id }}
RUSTFLAGS: "--remap-path-prefix=${CI_PROJECT_DIR}=/ic"
BUILDEVENT_DATASET: "github-ci-dfinity"
jobs:
bazel-test-all:
name: Bazel Test All
container:
image: ghcr.io/dfinity/ic-build@sha256:80e976b63af2b1b352c8c5959cb6c6b02aaa56a4efa327569d8c85c9c81a2cec
options: >-
-e NODE_NAME --privileged --cgroupns host -v /cache:/cache -v /var/sysimage:/var/sysimage -v /var/tmp:/var/tmp -v /ceph-s3-info:/ceph-s3-info
timeout-minutes: 90
runs-on:
group: zh1
labels: dind-large
steps:
- name: Checkout
uses: actions/checkout@v4
with:
fetch-depth: ${{ github.event_name == 'pull_request' && 256 || 0 }}
- name: Login to Dockerhub
shell: bash
run: ./ci/scripts/docker-login.sh
env:
DOCKER_HUB_USER: ${{ vars.DOCKER_HUB_USER }}
DOCKER_HUB_PASSWORD_RO: ${{ secrets.DOCKER_HUB_PASSWORD_RO }}
- name: Set BAZEL_EXTRA_ARGS
shell: bash
run: |
set -xeuo pipefail
# Determine which tests to skip and append 'long_test' for pull requests, merge groups or push on dev-gh-*
EXCLUDED_TEST_TAGS=(
system_test_hourly
system_test_nightly
system_test_nightly_nns
system_test_staging
system_test_hotfix
system_test_benchmark
fuzz_test
fi_tests_nightly
nns_tests_nightly
)
if [[ "$CI_EVENT_NAME" =~ ^(pull_request|merge_group)$ ]]; then
if [[ "$CI_EVENT_NAME" == "merge_group" || "${RUN_ON_DIFF_ONLY:-}" == "true" ]]; then
EXCLUDED_TEST_TAGS+=(long_test)
fi
elif [[ "$CI_EVENT_NAME" == "push" ]] && [[ "$BRANCH_NAME" =~ ^dev-gh-.* ]]; then
EXCLUDED_TEST_TAGS+=(long_test)
fi
# Export excluded tags as environment variable for ci/bazel-scripts/diff.sh
echo "EXCLUDED_TEST_TAGS=${EXCLUDED_TEST_TAGS[*]}" >> $GITHUB_ENV
# Prepend tags with '-' and join them with commas for Bazel
TEST_TAG_FILTERS=$(IFS=,; echo "${EXCLUDED_TEST_TAGS[*]/#/-}")
# Determine BAZEL_EXTRA_ARGS based on event type or branch name
BAZEL_EXTRA_ARGS="--test_tag_filters=$TEST_TAG_FILTERS"
if [[ "${{ github.event_name }}" == 'merge_group' ]]; then
BAZEL_EXTRA_ARGS+=" --test_timeout_filters=short,moderate --flaky_test_attempts=3"
elif [[ $BRANCH_NAME =~ ^hotfix-.* ]]; then
BAZEL_EXTRA_ARGS+=" --test_timeout_filters=short,moderate"
else
BAZEL_EXTRA_ARGS+=" --keep_going"
fi
# Export BAZEL_EXTRA_ARGS to environment
echo "BAZEL_EXTRA_ARGS=$BAZEL_EXTRA_ARGS" >> $GITHUB_ENV
env:
RUN_ON_DIFF_ONLY: ${{ github.event_name == 'pull_request' && !contains(github.event.pull_request.labels.*.name, 'CI_ALL_BAZEL_TARGETS') }}
- name: Run Bazel Test All
id: bazel-test-all
uses: ./.github/actions/bazel-test-all/
env:
AWS_SHARED_CREDENTIALS_CONTENT: ${{ secrets.AWS_SHARED_CREDENTIALS_FILE }}
# Only run ci/bazel-scripts/diff.sh on PRs that are not labeled with "CI_ALL_BAZEL_TARGETS".
OVERRIDE_DIDC_CHECK: ${{ contains(github.event.pull_request.labels.*.name, 'CI_OVERRIDE_DIDC_CHECK') }}
CI_OVERRIDE_BUF_BREAKING: ${{ contains(github.event.pull_request.labels.*.name, 'CI_OVERRIDE_BUF_BREAKING') }}
RUN_ON_DIFF_ONLY: ${{ github.event_name == 'pull_request' && !contains(github.event.pull_request.labels.*.name, 'CI_ALL_BAZEL_TARGETS') }}
with:
BAZEL_COMMAND: "test"
BAZEL_TARGETS: "//..."
BAZEL_CI_CONFIG: "--config=ci --repository_cache=/cache/bazel"
# check if PR title contains release and set timeout filters accordingly
BAZEL_EXTRA_ARGS: ${{ env.BAZEL_EXTRA_ARGS }}
BUILDEVENT_APIKEY: ${{ secrets.HONEYCOMB_API_TOKEN }}
- name: Upload bazel-bep
# runs only if previous step succeeded or failed;
# we avoid collecting artifacts of jobs that were cancelled
if: success() || failure()
uses: actions/upload-artifact@v4
with:
name: ${{ github.job }}-bep
retention-days: 14
if-no-files-found: ignore
compression-level: 9
path: |
bazel-bep.pb
profile.json
- name: Upload bazel-targets
uses: actions/upload-artifact@v4
with:
name: bazel-targets
retention-days: 14
if-no-files-found: error
path: |
bazel-targets
bazel-build-all-config-check:
runs-on:
labels: dind-large
container:
image: ghcr.io/dfinity/ic-build@sha256:80e976b63af2b1b352c8c5959cb6c6b02aaa56a4efa327569d8c85c9c81a2cec
options: >-
-e NODE_NAME --privileged --cgroupns host -v /cache:/cache -v /var/sysimage:/var/sysimage -v /var/tmp:/var/tmp -v /ceph-s3-info:/ceph-s3-info
timeout-minutes: 90
name: Bazel Build All Config Check
steps:
- name: Checkout
uses: actions/checkout@v4
with:
fetch-depth: ${{ github.event_name == 'pull_request' && 256 || 0 }}
- name: Login to Dockerhub
shell: bash
run: ./ci/scripts/docker-login.sh
env:
DOCKER_HUB_USER: ${{ vars.DOCKER_HUB_USER }}
DOCKER_HUB_PASSWORD_RO: ${{ secrets.DOCKER_HUB_PASSWORD_RO }}
- name: Run bazel build --config=check //rs/...
id: bazel-build-config-check
uses: ./.github/actions/bazel-test-all/
with:
BAZEL_COMMAND: "build"
BAZEL_TARGETS: "//rs/..."
BAZEL_CI_CONFIG: "--config=check --config=ci --keep_going"
- name: Upload bazel-bep
# runs only if previous step succeeded or failed;
# we avoid collecting artifacts of jobs that were cancelled
if: success() || failure()
uses: actions/upload-artifact@v4
with:
name: ${{ github.job }}-bep
retention-days: 14
if-no-files-found: ignore
compression-level: 9
path: |
bazel-bep.pb
profile.json
bazel-test-macos-intel:
name: Bazel Test macOS Intel
timeout-minutes: 130
runs-on:
labels: macOS
# Run on protected branches, but only on public repo
if: ${{ github.ref_protected && github.repository == 'dfinity/ic' }}
steps:
- name: Checkout
uses: actions/checkout@v4
with:
fetch-depth: ${{ github.event_name == 'pull_request' && 256 || 0 }}
- name: Set PATH
run: |
echo "/usr/local/bin" >> $GITHUB_PATH
echo "$HOME/.cargo/bin:" >> $GITHUB_PATH
- name: Login to Dockerhub
shell: bash
run: ./ci/scripts/docker-login.sh
env:
DOCKER_HUB_USER: ${{ vars.DOCKER_HUB_USER }}
DOCKER_HUB_PASSWORD_RO: ${{ secrets.DOCKER_HUB_PASSWORD_RO }}
- name: Run Bazel Test Darwin x86-64
id: bazel-test-darwin-x86-64
uses: ./.github/actions/bazel-test-all/
env:
AWS_SHARED_CREDENTIALS_CONTENT: ${{ secrets.AWS_SHARED_CREDENTIALS_FILE }}
with:
BAZEL_CI_CONFIG: "--config=ci --config macos_ci"
BAZEL_COMMAND: test
BAZEL_EXTRA_ARGS: '--test_tag_filters=test_macos'
BAZEL_STARTUP_ARGS: "--output_base /var/tmp/bazel-output/${CI_RUN_ID}"
BAZEL_TARGETS: "//rs/... //publish/binaries/..."
BUILDEVENT_APIKEY: ${{ secrets.HONEYCOMB_API_TOKEN }}
- name: Upload bazel-bep
# runs only if previous step succeeded or failed;
# we avoid collecting artifacts of jobs that were cancelled
if: success() || failure()
uses: actions/upload-artifact@v4
with:
name: ${{ github.job }}-bep
retention-days: 14
if-no-files-found: ignore
compression-level: 9
path: |
bazel-bep.pb
profile.json
- name: Purge Bazel Output
if: always()
shell: bash
run: |
sudo rm -rf /private/var/tmp/bazel-output
bazel-build-fuzzers:
name: Bazel Build Fuzzers
runs-on:
labels: dind-large
container:
image: ghcr.io/dfinity/ic-build@sha256:80e976b63af2b1b352c8c5959cb6c6b02aaa56a4efa327569d8c85c9c81a2cec
options: >-
-e NODE_NAME --privileged --cgroupns host -v /cache:/cache -v /var/sysimage:/var/sysimage -v /var/tmp:/var/tmp -v /ceph-s3-info:/ceph-s3-info
timeout-minutes: 90
steps:
- name: Checkout
uses: actions/checkout@v4
with:
fetch-depth: ${{ github.event_name == 'pull_request' && 256 || 0 }}
- name: Run Bazel Build Fuzzers
id: bazel-build-fuzzers
uses: ./.github/actions/bazel-test-all/
with:
BAZEL_COMMAND: "build"
BAZEL_TARGETS: "//rs/..."
BAZEL_EXTRA_ARGS: "--keep_going --config=fuzzing --build_tag_filters=libfuzzer"
BUILDEVENT_APIKEY: ${{ secrets. HONEYCOMB_API_TOKEN }}
- name: Upload bazel-bep
# runs only if previous step succeeded or failed;
# we avoid collecting artifacts of jobs that were cancelled
if: success() || failure()
uses: actions/upload-artifact@v4
with:
name: ${{ github.job }}-bep
retention-days: 14
if-no-files-found: ignore
compression-level: 9
path: |
bazel-bep.pb
profile.json
bazel-build-fuzzers-afl:
name: Bazel Build Fuzzers AFL
runs-on:
labels: dind-large
container:
image: ghcr.io/dfinity/ic-build@sha256:80e976b63af2b1b352c8c5959cb6c6b02aaa56a4efa327569d8c85c9c81a2cec
options: >-
-e NODE_NAME --privileged --cgroupns host -v /cache:/cache -v /var/sysimage:/var/sysimage -v /var/tmp:/var/tmp -v /ceph-s3-info:/ceph-s3-info
timeout-minutes: 90
steps:
- name: Checkout
uses: actions/checkout@v4
with:
fetch-depth: ${{ github.event_name == 'pull_request' && 256 || 0 }}
- name: Run Bazel Build Fuzzers AFL
id: bazel-build-fuzzers-afl
uses: ./.github/actions/bazel-test-all/
with:
BAZEL_COMMAND: "build"
BAZEL_TARGETS: "//rs/..."
BAZEL_EXTRA_ARGS: "--keep_going --config=afl"
BUILDEVENT_APIKEY: ${{ secrets. HONEYCOMB_API_TOKEN }}
- name: Upload bazel-bep
# runs only if previous step succeeded or failed;
# we avoid collecting artifacts of jobs that were cancelled
if: success() || failure()
uses: actions/upload-artifact@v4
with:
name: ${{ github.job }}-bep
retention-days: 14
if-no-files-found: ignore
compression-level: 9
path: |
bazel-bep.pb
profile.json
python-ci-tests:
name: Python CI Tests
runs-on:
labels: dind-small
container:
image: ghcr.io/dfinity/ic-build@sha256:80e976b63af2b1b352c8c5959cb6c6b02aaa56a4efa327569d8c85c9c81a2cec
timeout-minutes: 30
steps:
- name: Checkout
uses: actions/checkout@v4
with:
fetch-depth: ${{ github.event_name == 'pull_request' && 256 || 0 }}
- name: Set up Python
uses: actions/setup-python@v5
with:
python-version: '3.12'
- name: Run Python CI Tests
id: python-ci-tests
shell: bash
run: |
set -xeuo pipefail
export PYTHONPATH=$PWD/ci/src:$PWD/ci/src/dependencies
pip3 install --ignore-installed -r requirements.txt
cd ci/src
pytest -m "not fails_on_merge_train" -v -o junit_family=xunit1 \
--junitxml=../../test_report.xml --cov=. --cov-report=term \
--cov-report=term-missing --cov-report=html --cov-branch
env:
CI_COMMIT_REF_PROTECTED: ${{ github.ref_protected }}
CI_DEFAULT_BRANCH: ${{ github.event.repository.default_branch }}
REPO_NAME: ${{ github.repository }}
build-ic:
name: Build IC
runs-on:
labels: dind-large
container:
image: ghcr.io/dfinity/ic-build@sha256:80e976b63af2b1b352c8c5959cb6c6b02aaa56a4efa327569d8c85c9c81a2cec
options: >-
-e NODE_NAME --privileged --cgroupns host -v /cache:/cache -v /var/sysimage:/var/sysimage -v /var/tmp:/var/tmp -v /ceph-s3-info:/ceph-s3-info
timeout-minutes: 90
if: ${{ github.event_name != 'merge_group' }}
steps:
- name: Checkout
uses: actions/checkout@v4
with:
fetch-depth: ${{ github.event_name == 'pull_request' && 256 || 0 }}
- name: Login to Dockerhub
shell: bash
run: ./ci/scripts/docker-login.sh
env:
DOCKER_HUB_USER: ${{ vars.DOCKER_HUB_USER }}
DOCKER_HUB_PASSWORD_RO: ${{ secrets.DOCKER_HUB_PASSWORD_RO }}
- name: Run Build IC
id: build-ic
shell: bash
run: |
set -eExuo pipefail
[ -n "${NODE_NAME:-}" ] && echo "Run on node: $NODE_NAME" >>$GITHUB_STEP_SUMMARY
REPO_NAME="${GITHUB_REPOSITORY##*/}"
rm -rf "/cache/job/${CI_JOB_NAME}/${CI_RUN_ID}"
mkdir -p "/cache/job/${CI_JOB_NAME}/${CI_RUN_ID}/artifacts"
ln -s "/cache/job/${CI_JOB_NAME}/${CI_RUN_ID}/artifacts" /__w/$REPO_NAME/$REPO_NAME/artifacts
buildevents cmd "$CI_RUN_ID" "$CI_JOB_NAME" build-command -- \
"$CI_PROJECT_DIR"/ci/scripts/run-build-ic.sh
rm -rf "/cache/job/${CI_JOB_NAME}/${CI_RUN_ID}"
env:
BAZEL_COMMAND: "build"
MERGE_BASE_SHA: ${{ github.event.pull_request.base.sha }}
BUILDEVENT_APIKEY: ${{ secrets.HONEYCOMB_API_TOKEN }}
BRANCH_HEAD_SHA: ${{ github.event.pull_request.head.sha }}
RUN_ON_DIFF_ONLY: ${{ github.event_name == 'pull_request' && !contains(github.event.pull_request.labels.*.name, 'CI_ALL_BAZEL_TARGETS') }}
- name: Upload build-ic.tar
uses: actions/upload-artifact@v4
with:
name: build-ic
retention-days: 1
if-no-files-found: error
path: |
build-ic.tar
build-determinism:
name: Build Determinism
runs-on: ubuntu-latest
timeout-minutes: 30
needs: [build-ic, bazel-test-all]
strategy:
matrix:
include:
- TARGET: "//publish/binaries:upload"
PATH0: "release"
PATH1: "build-ic/release"
SETUPOS_FLAG: "false"
- TARGET: "//publish/canisters:upload"
PATH0: "canisters"
PATH1: "build-ic/canisters"
SETUPOS_FLAG: "false"
- TARGET: "//ic-os/guestos/envs/prod:upload_disk-img"
PATH0: "guest-os/update-img"
PATH1: "build-ic/icos/guestos"
SETUPOS_FLAG: "false"
- TARGET: "//ic-os/hostos/envs/prod:upload_update-img"
PATH0: "host-os/update-img"
PATH1: "build-ic/icos/hostos"
SETUPOS_FLAG: "false"
- TARGET: "//ic-os/setupos/envs/prod:upload_disk-img"
PATH0: "setup-os/disk-img"
PATH1: "build-ic/icos/setupos"
SETUPOS_FLAG: "true"
steps:
- name: Checkout
uses: actions/checkout@v4
with:
fetch-depth: ${{ github.event_name == 'pull_request' && 256 || 0 }}
- name: Download bazel-targets [bazel-test-all]
uses: actions/download-artifact@v4
with:
name: bazel-targets
- name: Download build-ic.tar [build-ic]
uses: actions/download-artifact@v4
with:
name: build-ic
- name: Build Determinism Test
id: build-determinism
shell: bash
run: |
set -eExuo pipefail
sudo apt update && sudo apt install -y curl
"$CI_PROJECT_DIR"/ci/scripts/build-determinism.sh
env:
TARGET: ${{ matrix.TARGET }}
PATH0: ${{ matrix.PATH0 }}
PATH1: ${{ matrix.PATH1 }}
SETUPOS_FLAG: ${{ matrix.SETUPOS_FLAG }}
cargo-clippy-linux:
name: Cargo Clippy Linux
runs-on:
labels: dind-large
container:
image: ghcr.io/dfinity/ic-build@sha256:80e976b63af2b1b352c8c5959cb6c6b02aaa56a4efa327569d8c85c9c81a2cec
options: >-
-e NODE_NAME --privileged --cgroupns host -v /cache:/cache -v /var/sysimage:/var/sysimage -v /var/tmp:/var/tmp -v /ceph-s3-info:/ceph-s3-info
timeout-minutes: 90
steps:
- name: Checkout
uses: actions/checkout@v4
with:
fetch-depth: ${{ github.event_name == 'pull_request' && 256 || 0 }}
- name: Filter Rust Files [*.{rs,toml,lock}]
uses: dorny/paths-filter@de90cc6fb38fc0963ad72b210f1f284cd68cea36 # v3
id: filter
if: |
github.event_name == 'pull_request' ||
github.event_name == 'merge_group'
with:
filters: |
cargo:
- "**/*.rs"
- "**/*.toml"
- "**/*.lock"
- name: Run Cargo Clippy Linux
id: cargo-clippy-linux
if: |
steps.filter.outputs.cargo == 'true' ||
github.event_name == 'schedule' ||
github.event_name == 'workflow_dispatch'
shell: bash
env:
BUILDEVENT_APIKEY: ${{ secrets. HONEYCOMB_API_TOKEN }}
run: |
set -eExuo pipefail
buildevents cmd "$CI_RUN_ID" "$CI_JOB_NAME" build-command -- \
"$CI_PROJECT_DIR"/ci/scripts/rust-lint.sh
cargo-build-release-linux:
name: Cargo Build Release Linux
runs-on:
labels: dind-large
container:
image: ghcr.io/dfinity/ic-build@sha256:80e976b63af2b1b352c8c5959cb6c6b02aaa56a4efa327569d8c85c9c81a2cec
options: >-
-e NODE_NAME --privileged --cgroupns host -v /cache:/cache -v /var/sysimage:/var/sysimage -v /var/tmp:/var/tmp -v /ceph-s3-info:/ceph-s3-info
timeout-minutes: 90
steps:
- name: Checkout
uses: actions/checkout@v4
with:
fetch-depth: ${{ github.event_name == 'pull_request' && 256 || 0 }}
- name: Filter Rust Files [*.{rs,toml,lock}]
uses: dorny/paths-filter@de90cc6fb38fc0963ad72b210f1f284cd68cea36 # v3
id: filter
if: |
github.event_name == 'pull_request' ||
github.event_name == 'merge_group'
with:
filters: |
cargo:
- "**/*.rs"
- "**/*.toml"
- "**/*.lock"
- name: Run Cargo Build Release Linux
id: cargo-build-release-linux
if: |
steps.filter.outputs.cargo == 'true' ||
github.event_name == 'schedule' ||
github.event_name == 'workflow_dispatch'
shell: bash
env:
BUILDEVENT_APIKEY: ${{ secrets. HONEYCOMB_API_TOKEN }}
run: |
set -eExuo pipefail
buildevents cmd "$CI_RUN_ID" "$CI_JOB_NAME" build-command -- \
cargo build --release --locked