Skip to content
Tests

Update airflow_2.7.2_py3.11.txt #8

Sign in to view logs

Update airflow_2.7.2_py3.11.txt

Update airflow_2.7.2_py3.11.txt #8

Workflow file for this run

.github/workflows/ci.yml at b3a9f3b
# Licensed to the Apache Software Foundation (ASF) under one
# or more contributor license agreements. See the NOTICE file
# distributed with this work for additional information
# regarding copyright ownership. The ASF licenses this file
# to you under the Apache License, Version 2.0 (the
# "License"); you may not use this file except in compliance
# with the License. You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing,
# software distributed under the License is distributed on an
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
# KIND, either express or implied. See the License for the
# specific language governing permissions and limitations
# under the License.
#
---
name: Tests
on: # yamllint disable-line rule:truthy
schedule:
- cron: '28 0 * * *'
push:
branches: ['main', 'v[0-9]+-[0-9]+-test']
pull_request:
branches: ['main']
permissions:
# All other permissions are set to none
contents: read
# Technically read access while waiting for images should be more than enough. However,
# there is a bug in GitHub Actions/Packages and in case private repositories are used, you get a permission
# denied error when attempting to just pull private image, changing the token permission to write solves the
# issue. This is not dangerous, because if it is for "apache/airflow", only maintainers can push ci.yml
# changes. If it is for a fork, then the token is read-only anyway.
packages: write
env:
ANSWER: "yes"
DB_RESET: "true"
VERBOSE: "true"
GITHUB_REPOSITORY: ${{ github.repository }}
GITHUB_USERNAME: ${{ github.actor }}
# You can override CONSTRAINTS_GITHUB_REPOSITORY by setting secret in your repo but by default the
# Airflow one is going to be used
CONSTRAINTS_GITHUB_REPOSITORY: >-
${{ secrets.CONSTRAINTS_GITHUB_REPOSITORY != '' &&
secrets.CONSTRAINTS_GITHUB_REPOSITORY || 'apache/airflow' }}
# In builds from forks, this token is read-only. For scheduled/direct push it is WRITE one
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
IMAGE_TAG: "${{ github.event.pull_request.head.sha || github.sha }}"
USE_SUDO: "true"
INCLUDE_SUCCESS_OUTPUTS: "true"
AIRFLOW_ENABLE_AIP_44: "true"
concurrency:
group: ci-${{ github.event.pull_request.number || github.ref }}
cancel-in-progress: true
jobs:
build-info:
name: "Build info"
# The runs-on cannot refer to env. or secrets. context, so we have no
# option but to specify a hard-coded list here. This is "safe", the
# runner checks if the user is an owner or collaborator of the repo
# before running the workflow.
runs-on: "ubuntu-22.04"
env:
GITHUB_CONTEXT: ${{ toJson(github) }}
outputs:
cache-directive: ${{ steps.selective-checks.outputs.cache-directive }}
affected-providers-list-as-string: >-
${{ steps.selective-checks.outputs.affected-providers-list-as-string }}
upgrade-to-newer-dependencies: ${{ steps.selective-checks.outputs.upgrade-to-newer-dependencies }}
python-versions: ${{ steps.selective-checks.outputs.python-versions }}
python-versions-list-as-string: ${{ steps.selective-checks.outputs.python-versions-list-as-string }}
all-python-versions-list-as-string: >-
${{ steps.selective-checks.outputs.all-python-versions-list-as-string }}
default-python-version: ${{ steps.selective-checks.outputs.default-python-version }}
kubernetes-versions-list-as-string: >-
${{ steps.selective-checks.outputs.kubernetes-versions-list-as-string }}
kubernetes-combos-list-as-string: >-
${{ steps.selective-checks.outputs.kubernetes-combos-list-as-string }}
default-kubernetes-version: ${{ steps.selective-checks.outputs.default-kubernetes-version }}
postgres-versions: ${{ steps.selective-checks.outputs.postgres-versions }}
default-postgres-version: ${{ steps.selective-checks.outputs.default-postgres-version }}
mysql-versions: ${{ steps.selective-checks.outputs.mysql-versions }}
mssql-versions: ${{ steps.selective-checks.outputs.mssql-versions }}
default-mysql-version: ${{ steps.selective-checks.outputs.default-mysql-version }}
default-helm-version: ${{ steps.selective-checks.outputs.default-helm-version }}
default-kind-version: ${{ steps.selective-checks.outputs.default-kind-version }}
full-tests-needed: ${{ steps.selective-checks.outputs.full-tests-needed }}
parallel-test-types-list-as-string: >-
${{ steps.selective-checks.outputs.parallel-test-types-list-as-string }}
mssql-parallelism: ${{ steps.selective-checks.outputs.mssql-parallelism }}
postgres-exclude: ${{ steps.selective-checks.outputs.postgres-exclude }}
mysql-exclude: ${{ steps.selective-checks.outputs.mysql-exclude }}
mssql-exclude: ${{ steps.selective-checks.outputs.mssql-exclude }}
sqlite-exclude: ${{ steps.selective-checks.outputs.sqlite-exclude }}
skip-provider-tests: ${{ steps.selective-checks.outputs.skip-provider-tests }}
run-tests: ${{ steps.selective-checks.outputs.run-tests }}
run-amazon-tests: ${{ steps.selective-checks.outputs.run-amazon-tests }}
run-www-tests: ${{ steps.selective-checks.outputs.run-www-tests }}
run-kubernetes-tests: ${{ steps.selective-checks.outputs.run-kubernetes-tests }}
basic-checks-only: ${{ steps.selective-checks.outputs.basic-checks-only }}
ci-image-build: ${{ steps.selective-checks.outputs.ci-image-build }}
prod-image-build: ${{ steps.selective-checks.outputs.prod-image-build }}
docs-build: ${{ steps.selective-checks.outputs.docs-build }}
needs-helm-tests: ${{ steps.selective-checks.outputs.needs-helm-tests }}
needs-api-tests: ${{ steps.selective-checks.outputs.needs-api-tests }}
needs-api-codegen: ${{ steps.selective-checks.outputs.needs-api-codegen }}
default-branch: ${{ steps.selective-checks.outputs.default-branch }}
default-constraints-branch: ${{ steps.selective-checks.outputs.default-constraints-branch }}
docs-list-as-string: ${{ steps.selective-checks.outputs.docs-list-as-string }}
skip-pre-commits: ${{ steps.selective-checks.outputs.skip-pre-commits }}
helm-test-packages: ${{ steps.selective-checks.outputs.helm-test-packages }}
debug-resources: ${{ steps.selective-checks.outputs.debug-resources }}
runs-on: ${{steps.selective-checks.outputs.runs-on}}
is-self-hosted-runner: ${{ steps.selective-checks.outputs.is-self-hosted-runner }}
is-airflow-runner: ${{ steps.selective-checks.outputs.is-airflow-runner }}
is-amd-runner: ${{ steps.selective-checks.outputs.is-amd-runner }}
is-arm-runner: ${{ steps.selective-checks.outputs.is-arm-runner }}
is-vm-runner: ${{ steps.selective-checks.outputs.is-vm-runner }}
is-k8s-runner: ${{ steps.selective-checks.outputs.is-k8s-runner }}
has-migrations: ${{ steps.selective-checks.outputs.has-migrations }}
source-head-repo: ${{ steps.source-run-info.outputs.source-head-repo }}
pull-request-labels: ${{ steps.source-run-info.outputs.pr-labels }}
in-workflow-build: ${{ steps.source-run-info.outputs.in-workflow-build }}
build-job-description: ${{ steps.source-run-info.outputs.build-job-description }}
canary-run: ${{ steps.source-run-info.outputs.canary-run }}
run-coverage: ${{ steps.source-run-info.outputs.run-coverage }}
steps:
- name: Cleanup repo
run: docker run -v "${GITHUB_WORKSPACE}:/workspace" -u 0:0 bash -c "rm -rf /workspace/*"
- name: "Checkout ${{ github.ref }} ( ${{ github.sha }} )"
uses: actions/checkout@v4
with:
persist-credentials: false
- name: Fetch incoming commit ${{ github.sha }} with its parent
uses: actions/checkout@v4
with:
ref: ${{ github.sha }}
fetch-depth: 2
persist-credentials: false
- name: "Install Breeze"
uses: ./.github/actions/breeze
- name: "Retrieve defaults from branch_defaults.py"
id: defaults
# We could retrieve it differently here - by just importing the variables and
# printing them from python code, however we want to have the same code as used in
# the build-images.yml (there we cannot import python code coming from the PR - we need to
# treat the python code as text and extract the variables from there.
run: |
python - <<EOF >> ${GITHUB_ENV}
from pathlib import Path
import re
import sys
DEFAULTS_CONTENT = Path('dev/breeze/src/airflow_breeze/branch_defaults.py').read_text()
BRANCH_PATTERN = r'^AIRFLOW_BRANCH = "(.*)"$'
CONSTRAINTS_BRANCH_PATTERN = r'^DEFAULT_AIRFLOW_CONSTRAINTS_BRANCH = "(.*)"$'
branch = re.search(BRANCH_PATTERN, DEFAULTS_CONTENT, re.MULTILINE).group(1)
constraints_branch = re.search(CONSTRAINTS_BRANCH_PATTERN, DEFAULTS_CONTENT, re.MULTILINE).group(1)
output = f"""
DEFAULT_BRANCH={branch}
DEFAULT_CONSTRAINTS_BRANCH={constraints_branch}
""".strip()
print(output)
# Stdout is redirected to GITHUB_ENV but we also print it to stderr to see it in ci log
print(output, file=sys.stderr)
EOF
- name: "Get information about the Workflow"
id: source-run-info
run: breeze ci get-workflow-info 2>> ${GITHUB_OUTPUT}
- name: Selective checks
id: selective-checks
env:
PR_LABELS: "${{ steps.source-run-info.outputs.pr-labels }}"
COMMIT_REF: "${{ github.sha }}"
VERBOSE: "false"
run: breeze ci selective-check 2>> ${GITHUB_OUTPUT}
- name: env
run: printenv
env:
PR_LABELS: ${{ steps.source-run-info.outputs.pr-labels }}
GITHUB_CONTEXT: ${{ toJson(github) }}
# Push early BuildX cache to GitHub Registry in Apache repository, This cache does not wait for all the
# tests to complete - it is run very early in the build process for "main" merges in order to refresh
# cache using the current constraints. This will speed up cache refresh in cases when setup.py
# changes or in case of Dockerfile changes. Failure in this step is not a problem (at most it will
# delay cache refresh. It does not attempt to upgrade to newer dependencies.
# We only push CI cache as PROD cache usually does not gain as much from fresh cache because
# it uses prepared airflow and provider packages that invalidate the cache anyway most of the time
push-early-buildx-cache-to-github-registry:
timeout-minutes: 50
name: "Push Early Image Cache"
runs-on: ${{fromJSON(needs.build-info.outputs.runs-on)}}
needs:
- build-info
strategy:
fail-fast: false
matrix:
platform: ["linux/amd64", "linux/arm64"]
env:
RUNS_ON: "${{needs.build-info.outputs.runs-on}}"
UPGRADE_TO_NEWER_DEPENDENCIES: false
VERSION_SUFFIX_FOR_PYPI: "dev0"
continue-on-error: true
steps:
- name: Cleanup repo
run: docker run -v "${GITHUB_WORKSPACE}:/workspace" -u 0:0 bash -c "rm -rf /workspace/*"
if: needs.build-info.outputs.canary-run == 'true' && needs.build-info.outputs.default-branch == 'main'
- name: "Checkout ${{ github.ref }} ( ${{ github.sha }} )"
uses: actions/checkout@v4
with:
persist-credentials: false
if: needs.build-info.outputs.canary-run == 'true' && needs.build-info.outputs.default-branch == 'main'
- name: "Install Breeze"
uses: ./.github/actions/breeze
if: needs.build-info.outputs.canary-run == 'true' && needs.build-info.outputs.default-branch == 'main'
- name: "Start ARM instance"
run: ./scripts/ci/images/ci_start_arm_instance_and_connect_to_docker.sh
if: >
matrix.platform == 'linux/arm64' && needs.build-info.outputs.canary-run == 'true'
&& needs.build-info.outputs.default-branch == 'main'
- name: "Push CI cache ${{ matrix.platform }}"
run: >
breeze ci-image build
--builder airflow_cache
--prepare-buildx-cache
--run-in-parallel
--platform ${{ matrix.platform }}
env:
DEBUG_RESOURCES: ${{needs.build-info.outputs.debug-resources}}
COMMIT_SHA: ${{ github.sha }}
if: needs.build-info.outputs.canary-run == 'true' && needs.build-info.outputs.default-branch == 'main'
- name: "Push CI latest image ${{ matrix.platform }}"
run: >
breeze ci-image build
--tag-as-latest --push --run-in-parallel --platform ${{ matrix.platform }}
env:
DEBUG_RESOURCES: ${{needs.build-info.outputs.debug-resources}}
COMMIT_SHA: ${{ github.sha }}
# We only push "amd" image as it is really only needed for any kind of automated builds in CI
# and currently there is not an easy way to make multi-platform image from two separate builds
if: >
matrix.platform == 'linux/amd64' && needs.build-info.outputs.canary-run == 'true'
&& needs.build-info.outputs.default-branch == 'main'
- name: "Stop ARM instance"
run: ./scripts/ci/images/ci_stop_arm_instance.sh
if: >
always() && matrix.platform == 'linux/arm64' && needs.build-info.outputs.canary-run == 'true'
&& needs.build-info.outputs.default-branch == 'main'
- name: "Clean docker cache for ${{ matrix.platform }}"
run: docker system prune --all --force
if: >
matrix.platform == 'linux/amd64' && needs.build-info.outputs.canary-run == 'true'
&& needs.build-info.outputs.default-branch == 'main'
# Check that after earlier cache push, breeze command will build quickly
check-that-image-builds-quickly:
timeout-minutes: 5
name: "Check that image builds quickly"
runs-on: ${{fromJSON(needs.build-info.outputs.runs-on)}}
needs:
- build-info
- push-early-buildx-cache-to-github-registry
env:
RUNS_ON: "${{needs.build-info.outputs.runs-on}}"
UPGRADE_TO_NEWER_DEPENDENCIES: false
PLATFORM: "linux/amd64"
if: >
needs.build-info.outputs.canary-run == 'true'
&& needs.build-info.outputs.default-branch == 'main'
steps:
- name: Cleanup repo
run: docker run -v "${GITHUB_WORKSPACE}:/workspace" -u 0:0 bash -c "rm -rf /workspace/*"
- name: "Checkout ${{ github.ref }} ( ${{ github.sha }} )"
uses: actions/checkout@v4
with:
persist-credentials: false
- name: "Install Breeze"
uses: ./.github/actions/breeze
- name: "Check that image builds quickly"
run: breeze shell --max-time 120
build-ci-images:
timeout-minutes: 80
name: >-
${{needs.build-info.outputs.build-job-description}} CI images
${{needs.build-info.outputs.all-python-versions-list-as-string}}
runs-on: ${{fromJSON(needs.build-info.outputs.runs-on)}}
needs: [build-info]
env:
DEFAULT_BRANCH: ${{ needs.build-info.outputs.default-branch }}
DEFAULT_CONSTRAINTS_BRANCH: ${{ needs.build-info.outputs.default-constraints-branch }}
RUNS_ON: "${{needs.build-info.outputs.runs-on}}"
# Force more parallelism for build even on public images
PARALLELISM: 6
VERSION_SUFFIX_FOR_PYPI: "dev0"
steps:
- name: Cleanup repo
run: docker run -v "${GITHUB_WORKSPACE}:/workspace" -u 0:0 bash -c "rm -rf /workspace/*"
if: needs.build-info.outputs.in-workflow-build == 'true'
- uses: actions/checkout@v4
with:
ref: ${{ needs.build-info.outputs.targetCommitSha }}
persist-credentials: false
if: needs.build-info.outputs.in-workflow-build == 'true'
- name: >
Build CI Images
${{needs.build-info.outputs.all-python-versions-list-as-string}}:${{env.IMAGE_TAG}}
uses: ./.github/actions/build-ci-images
if: needs.build-info.outputs.in-workflow-build == 'true'
env:
UPGRADE_TO_NEWER_DEPENDENCIES: ${{ needs.build-info.outputs.upgrade-to-newer-dependencies }}
DOCKER_CACHE: ${{ needs.build-info.outputs.cache-directive }}
PYTHON_VERSIONS: ${{needs.build-info.outputs.all-python-versions-list-as-string}}
DEBUG_RESOURCES: ${{needs.build-info.outputs.debug-resources}}
BUILD_TIMEOUT_MINUTES: 70
- name: Verify CI images ${{ env.PYTHON_VERSIONS }}:${{ env.IMAGE_TAG }}
run: breeze ci-image verify --run-in-parallel
env:
PYTHON_VERSIONS: ${{needs.build-info.outputs.all-python-versions-list-as-string}}
DEBUG_RESOURCES: ${{needs.build-info.outputs.debug-resources}}
if: needs.build-info.outputs.in-workflow-build == 'true'
run-breeze-tests:
timeout-minutes: 10
name: Breeze unit tests
runs-on: ${{fromJSON(needs.build-info.outputs.runs-on)}}
needs: [build-info]
steps:
- name: Cleanup repo
run: docker run -v "${GITHUB_WORKSPACE}:/workspace" -u 0:0 bash -c "rm -rf /workspace/*"
- uses: actions/checkout@v4
with:
persist-credentials: false
- uses: actions/setup-python@v4
with:
python-version: "${{needs.build-info.outputs.default-python-version}}"
cache: 'pip'
cache-dependency-path: ./dev/breeze/setup*
- run: pip install --editable ./dev/breeze/
- run: python -m pytest -n auto --color=yes
working-directory: ./dev/breeze/
tests-www:
timeout-minutes: 10
name: React WWW tests
runs-on: ${{fromJSON(needs.build-info.outputs.runs-on)}}
needs: [build-info]
if: needs.build-info.outputs.run-www-tests == 'true'
steps:
- name: Cleanup repo
run: docker run -v "${GITHUB_WORKSPACE}:/workspace" -u 0:0 bash -c "rm -rf /workspace/*"
- name: "Checkout ${{ github.ref }} ( ${{ github.sha }} )"
uses: actions/checkout@v4
with:
persist-credentials: false
- name: "Setup node"
uses: actions/setup-node@v3
with:
node-version: 14
- name: "Cache eslint"
uses: actions/cache@v3
with:
path: 'airflow/www/node_modules'
key: ${{ runner.os }}-www-node-modules-${{ hashFiles('airflow/www/**/yarn.lock') }}
- run: yarn --cwd airflow/www/ install --frozen-lockfile --non-interactive
- run: yarn --cwd airflow/www/ run test
env:
FORCE_COLOR: 2
test-openapi-client-generation:
timeout-minutes: 10
name: "Test OpenAPI client generation"
runs-on: ${{fromJSON(needs.build-info.outputs.runs-on)}}
needs: [build-info]
if: needs.build-info.outputs.needs-api-codegen == 'true'
steps:
- name: Cleanup repo
run: docker run -v "${GITHUB_WORKSPACE}:/workspace" -u 0:0 bash -c "rm -rf /workspace/*"
- name: "Checkout ${{ github.ref }} ( ${{ github.sha }} )"
uses: actions/checkout@v4
with:
fetch-depth: 2
persist-credentials: false
- name: "Generate client codegen diff"
run: ./scripts/ci/openapi/client_codegen_diff.sh
test-examples-of-prod-image-building:
timeout-minutes: 60
name: "Test examples of production image building"
runs-on: ${{fromJSON(needs.build-info.outputs.runs-on)}}
needs: [build-info]
if: needs.build-info.outputs.ci-image-build == 'true'
steps:
- name: Cleanup repo
run: docker run -v "${GITHUB_WORKSPACE}:/workspace" -u 0:0 bash -c "rm -rf /workspace/*"
- name: "Checkout ${{ github.ref }} ( ${{ github.sha }} )"
uses: actions/checkout@v4
with:
fetch-depth: 2
persist-credentials: false
- name: "Setup python"
uses: actions/setup-python@v4
with:
python-version: "${{needs.build-info.outputs.default-python-version}}"
cache: 'pip'
cache-dependency-path: ./dev/requirements.txt
- name: "Test examples of PROD image building"
run: >
cd ./docker_tests &&
python -m pip install -r requirements.txt &&
python -m pytest test_examples_of_prod_image_building.py -n auto --color=yes
test-git-clone-on-windows:
timeout-minutes: 5
name: "Test git clone on Windows"
runs-on: "windows-latest"
needs: [build-info]
steps:
- name: "Checkout ${{ github.ref }} ( ${{ github.sha }} )"
uses: actions/checkout@v4
with:
fetch-depth: 2
persist-credentials: false
wait-for-ci-images:
timeout-minutes: 120
name: "Wait for CI images"
runs-on: "ubuntu-22.04"
needs: [build-info, build-ci-images]
if: needs.build-info.outputs.ci-image-build == 'true'
env:
RUNS_ON: "${{needs.build-info.outputs.runs-on}}"
BACKEND: sqlite
# Force more parallelism for pull even on public images
PARALLELISM: 6
steps:
- name: Cleanup repo
run: docker run -v "${GITHUB_WORKSPACE}:/workspace" -u 0:0 bash -c "rm -rf /workspace/*"
if: needs.build-info.outputs.in-workflow-build == 'false'
- name: "Checkout ${{ github.ref }} ( ${{ github.sha }} )"
uses: actions/checkout@v4
with:
persist-credentials: false
if: needs.build-info.outputs.in-workflow-build == 'false'
- name: "Install Breeze"
uses: ./.github/actions/breeze
if: needs.build-info.outputs.in-workflow-build == 'false'
- name: Wait for CI images ${{ env.PYTHON_VERSIONS }}:${{ env.IMAGE_TAG }}
id: wait-for-images
run: breeze ci-image pull --run-in-parallel --wait-for-image --tag-as-latest
env:
PYTHON_VERSIONS: ${{ needs.build-info.outputs.python-versions-list-as-string }}
DEBUG_RESOURCES: ${{needs.build-info.outputs.debug-resources}}
if: needs.build-info.outputs.in-workflow-build == 'false'
generate-constraints:
permissions:
contents: read
timeout-minutes: 70
name: >
Generate constraints with CI image verifying
${{needs.build-info.outputs.all-python-versions-list-as-string}}
runs-on: ${{fromJSON(needs.build-info.outputs.runs-on)}}
needs: [build-info, wait-for-ci-images]
env:
RUNS_ON: "${{ needs.build-info.outputs.runs-on }}"
PYTHON_VERSIONS: ${{needs.build-info.outputs.all-python-versions-list-as-string}}
DEBUG_RESOURCES: ${{needs.build-info.outputs.debug-resources}}
if: needs.build-info.outputs.ci-image-build == 'true'
steps:
- name: Cleanup repo
run: docker run -v "${GITHUB_WORKSPACE}:/workspace" -u 0:0 bash -c "rm -rf /workspace/*"
- name: "Checkout ${{ github.ref }} ( ${{ github.sha }} )"
uses: actions/checkout@v4
with:
persist-credentials: false
- name: "Install Breeze"
uses: ./.github/actions/breeze
- name: Pull CI images ${{ env.PYTHON_VERSIONS }}:${{ env.IMAGE_TAG }}
run: breeze ci-image pull --run-in-parallel --tag-as-latest
- name: Verify CI images ${{ env.PYTHON_VERSIONS }}:${{ env.IMAGE_TAG }}
run: breeze ci-image verify --run-in-parallel
env:
PYTHON_VERSIONS: ${{needs.build-info.outputs.all-python-versions-list-as-string}}
DEBUG_RESOURCES: ${{needs.build-info.outputs.debug-resources}}
- name: "Source constraints"
shell: bash
run: >
breeze release-management generate-constraints --run-in-parallel
--airflow-constraints-mode constraints-source-providers
- name: "No providers constraints"
shell: bash
timeout-minutes: 25
run: >
breeze release-management generate-constraints --run-in-parallel
--airflow-constraints-mode constraints-no-providers
- name: "PyPI constraints"
shell: bash
timeout-minutes: 25
run: >
breeze release-management generate-constraints --run-in-parallel
--airflow-constraints-mode constraints
- name: "Dependency upgrade summary"
shell: bash
run: |
for PYTHON_VERSION in ${{ env.PYTHON_VERSIONS }}; do
echo "Summarizing Python $PYTHON_VERSION"
cat "files/constraints-${PYTHON_VERSION}"/*.md >> $GITHUB_STEP_SUMMARY || true
done
- name: "Upload constraint artifacts"
uses: actions/upload-artifact@v3
with:
name: constraints
path: ./files/constraints-*/constraints-*.txt
retention-days: 7
static-checks:
timeout-minutes: 45
name: "Static checks"
runs-on: ${{fromJSON(needs.build-info.outputs.runs-on)}}
needs: [build-info, wait-for-ci-images]
env:
RUNS_ON: "${{needs.build-info.outputs.runs-on}}"
PYTHON_MAJOR_MINOR_VERSION: "${{needs.build-info.outputs.default-python-version}}"
UPGRADE_TO_NEWER_DEPENDENCIES: "${{ needs.build-info.outputs.upgrade-to-newer-dependencies }}"
if: needs.build-info.outputs.basic-checks-only == 'false'
steps:
- name: Cleanup repo
run: docker run -v "${GITHUB_WORKSPACE}:/workspace" -u 0:0 bash -c "rm -rf /workspace/*"
- name: "Checkout ${{ github.ref }} ( ${{ github.sha }} )"
uses: actions/checkout@v4
with:
persist-credentials: false
- name: >
Prepare breeze & CI image: ${{needs.build-info.outputs.default-python-version}}:${{env.IMAGE_TAG}}
uses: ./.github/actions/prepare_breeze_and_image
id: breeze
- name: Cache pre-commit envs
uses: actions/cache@v3
with:
path: ~/.cache/pre-commit
# yamllint disable-line rule:line-length
key: "pre-commit-${{steps.breeze.outputs.host-python-version}}-${{ hashFiles('.pre-commit-config.yaml') }}"
restore-keys: |
pre-commit-${{steps.breeze.outputs.host-python-version}}-
- name: "Static checks"
run: breeze static-checks --all-files --show-diff-on-failure --color always --initialize-environment
env:
VERBOSE: "false"
SKIP: ${{ needs.build-info.outputs.skip-pre-commits }}
COLUMNS: "250"
SKIP_GROUP_OUTPUT: "true"
DEFAULT_BRANCH: ${{ needs.build-info.outputs.default-branch }}
RUFF_FORMAT: "github"
# Those checks are run if no image needs to be built for checks. This is for simple changes that
# Do not touch any of the python code or any of the important files that might require building
# The CI Docker image and they can be run entirely using the pre-commit virtual environments on host
static-checks-basic-checks-only:
timeout-minutes: 30
name: "Static checks: basic checks only"
runs-on: ${{fromJSON(needs.build-info.outputs.runs-on)}}
needs: [build-info]
env:
RUNS_ON: "${{needs.build-info.outputs.runs-on}}"
if: needs.build-info.outputs.basic-checks-only == 'true'
steps:
- name: Cleanup repo
run: docker run -v "${GITHUB_WORKSPACE}:/workspace" -u 0:0 bash -c "rm -rf /workspace/*"
- name: "Checkout ${{ github.ref }} ( ${{ github.sha }} )"
uses: actions/checkout@v4
with:
persist-credentials: false
- name: "Setup python"
uses: actions/setup-python@v4
with:
python-version: "${{needs.build-info.outputs.default-python-version}}"
cache: 'pip'
cache-dependency-path: ./dev/breeze/setup*
- name: "Install Breeze"
uses: ./.github/actions/breeze
id: breeze
- name: Cache pre-commit envs
uses: actions/cache@v3
with:
path: ~/.cache/pre-commit
# yamllint disable-line rule:line-length
key: "pre-commit-${{steps.breeze.outputs.host-python-version}}-${{ hashFiles('.pre-commit-config.yaml') }}"
restore-keys: "\
pre-commit-${{steps.breeze.outputs.host-python-version}}-\
${{ hashFiles('.pre-commit-config.yaml') }}\n
pre-commit-${{steps.breeze.outputs.host-python-version}}-"
- name: Fetch incoming commit ${{ github.sha }} with its parent
uses: actions/checkout@v4
with:
ref: ${{ github.sha }}
fetch-depth: 2
persist-credentials: false
- name: "Static checks: basic checks only"
run: >
breeze static-checks --show-diff-on-failure --color always --initialize-environment
--skip-image-upgrade-check --commit-ref "${{ github.sha }}"
env:
VERBOSE: "false"
SKIP_BREEZE_PRE_COMMITS: "true"
SKIP: ${{ needs.build-info.outputs.skip-pre-commits }}
COLUMNS: "250"
build-docs:
timeout-minutes: 60
name: "Build docs"
runs-on: ${{fromJSON(needs.build-info.outputs.runs-on)}}
needs: [build-info, wait-for-ci-images]
if: needs.build-info.outputs.docs-build == 'true'
env:
RUNS_ON: "${{needs.build-info.outputs.runs-on}}"
PYTHON_MAJOR_MINOR_VERSION: "${{needs.build-info.outputs.default-python-version}}"
steps:
- name: Cleanup repo
run: docker run -v "${GITHUB_WORKSPACE}:/workspace" -u 0:0 bash -c "rm -rf /workspace/*"
- name: "Checkout ${{ github.ref }} ( ${{ github.sha }} )"
uses: actions/checkout@v4
with:
persist-credentials: false
- name: >
Prepare breeze & CI image: ${{needs.build-info.outputs.default-python-version}}:${{env.IMAGE_TAG}}
uses: ./.github/actions/prepare_breeze_and_image
- uses: actions/cache@v3
id: cache-doc-inventories
with:
path: ./docs/_inventory_cache/
key: docs-inventory-${{ hashFiles('setup.py','setup.cfg','pyproject.toml;') }}
restore-keys: |
docs-inventory-${{ hashFiles('setup.py','setup.cfg','pyproject.toml;') }}
docs-inventory-
- name: "Build docs"
run: >
breeze build-docs ${{ needs.build-info.outputs.docs-list-as-string }} --docs-only
- name: "Clone airflow-site"
run: >
git clone https://github.com/apache/airflow-site.git ${GITHUB_WORKSPACE}/airflow-site &&
echo "AIRFLOW_SITE_DIRECTORY=${GITHUB_WORKSPACE}/airflow-site" >> "$GITHUB_ENV"
if: needs.build-info.outputs.full-tests-needed == 'true'
- name: "Publish docs"
run: >
breeze release-management publish-docs
--override-versioned --run-in-parallel
${{ needs.build-info.outputs.docs-list-as-string }}
if: needs.build-info.outputs.full-tests-needed == 'true'
- name: "Generate back references for providers"
run: breeze release-management add-back-references all-providers
if: needs.build-info.outputs.full-tests-needed == 'true'
- name: "Generate back references for apache-airflow"
run: breeze release-management add-back-references apache-airflow
if: needs.build-info.outputs.full-tests-needed == 'true'
- name: "Generate back references for docker-stack"
run: breeze release-management add-back-references docker-stack
if: needs.build-info.outputs.full-tests-needed == 'true'
- name: "Generate back references for helm-chart"
run: breeze release-management add-back-references helm-chart
if: needs.build-info.outputs.full-tests-needed == 'true'
- name: Configure AWS credentials
uses: aws-actions/configure-aws-credentials@010d0da01d0b5a38af31e9c3470dbfdabdecca3a # v4.0.1
if: needs.build-info.outputs.canary-run == 'true' && needs.build-info.outputs.default-branch == 'main'
with:
aws-access-key-id: ${{ secrets.DOCS_AWS_ACCESS_KEY_ID }}
aws-secret-access-key: ${{ secrets.DOCS_AWS_SECRET_ACCESS_KEY }}
aws-region: eu-central-1
- name: "Upload documentation to AWS S3"
if: needs.build-info.outputs.canary-run == 'true' && needs.build-info.outputs.default-branch == 'main'
run: aws s3 sync --delete ./files/documentation s3://apache-airflow-docs
spellcheck-docs:
timeout-minutes: 60
name: "Spellcheck docs"
runs-on: ${{fromJSON(needs.build-info.outputs.runs-on)}}
needs: [build-info, wait-for-ci-images]
if: needs.build-info.outputs.docs-build == 'true'
env:
RUNS_ON: "${{needs.build-info.outputs.runs-on}}"
PYTHON_MAJOR_MINOR_VERSION: "${{needs.build-info.outputs.default-python-version}}"
steps:
- name: Cleanup repo
run: docker run -v "${GITHUB_WORKSPACE}:/workspace" -u 0:0 bash -c "rm -rf /workspace/*"
- name: "Checkout ${{ github.ref }} ( ${{ github.sha }} )"
uses: actions/checkout@v4
with:
persist-credentials: false
- name: >
Prepare breeze & CI image: ${{needs.build-info.outputs.default-python-version}}:${{env.IMAGE_TAG}}
uses: ./.github/actions/prepare_breeze_and_image
- uses: actions/cache@v3
id: cache-doc-inventories
with:
path: ./docs/_inventory_cache/
key: docs-inventory-${{ hashFiles('setup.py','setup.cfg','pyproject.toml;') }}
restore-keys: |
docs-inventory-${{ hashFiles('setup.py','setup.cfg','pyproject.toml;') }}
docs-inventory-
- name: "Spellcheck docs"
run: >
breeze build-docs ${{ needs.build-info.outputs.docs-list-as-string }} --spellcheck-only
prepare-test-provider-packages-wheel:
timeout-minutes: 80
name: "Provider packages wheel (verify)"
runs-on: ${{fromJSON(needs.build-info.outputs.runs-on)}}
needs: [build-info, wait-for-ci-images]
env:
RUNS_ON: "${{needs.build-info.outputs.runs-on}}"
PYTHON_MAJOR_MINOR_VERSION: "${{needs.build-info.outputs.default-python-version}}"
if: needs.build-info.outputs.skip-provider-tests != 'true'
steps:
- name: Cleanup repo
run: docker run -v "${GITHUB_WORKSPACE}:/workspace" -u 0:0 bash -c "rm -rf /workspace/*"
- name: "Checkout ${{ github.ref }} ( ${{ github.sha }} )"
uses: actions/checkout@v4
with:
persist-credentials: false
- name: >
Prepare breeze & CI image: ${{needs.build-info.outputs.default-python-version}}:${{env.IMAGE_TAG}}
uses: ./.github/actions/prepare_breeze_and_image
- name: "Cleanup dist files"
run: rm -fv ./dist/*
- name: "Prepare provider documentation"
run: >
breeze release-management prepare-provider-documentation --non-interactive
${{ needs.build-info.outputs.affected-providers-list-as-string }}
- name: "Prepare provider packages: wheel"
run: >
breeze release-management prepare-provider-packages --version-suffix-for-pypi dev0
--package-format wheel ${{ needs.build-info.outputs.affected-providers-list-as-string }}
- name: "Prepare airflow package: wheel"
run: breeze release-management prepare-airflow-package --version-suffix-for-pypi dev0
- name: "Verify wheel packages with twine"
run: pipx install twine --force && twine check dist/*.whl
- name: "Test providers issue generation automatically"
run: >
breeze release-management generate-issue-content-providers
--only-available-in-dist --disable-progress
- name: "Install and verify all provider packages and airflow via wheel files"
run: >
breeze release-management verify-provider-packages --use-packages-from-dist
--package-format wheel --use-airflow-version wheel
if: needs.build-info.outputs.affected-providers-list-as-string == ''
env:
SKIP_CONSTRAINTS: "${{ needs.build-info.outputs.upgrade-to-newer-dependencies }}"
- name: "Install affected provider packages and airflow via wheel files"
run: >
breeze release-management install-provider-packages
--package-format wheel --use-airflow-version wheel --run-in-parallel
if: needs.build-info.outputs.affected-providers-list-as-string != ''
env:
SKIP_CONSTRAINTS: "${{ needs.build-info.outputs.upgrade-to-newer-dependencies }}"
- name: "Prepare airflow package: wheel without suffix and skipping the tag check"
run: >
breeze release-management prepare-provider-packages --skip-tag-check
--package-format wheel ${{ needs.build-info.outputs.affected-providers-list-as-string }}
provider-airflow-compatibility-check:
timeout-minutes: 80
name: "Providers Airflow 2.5 compatibility check"
runs-on: ${{fromJSON(needs.build-info.outputs.runs-on)}}
needs: [build-info, wait-for-ci-images]
env:
RUNS_ON: "${{needs.build-info.outputs.runs-on}}"
PYTHON_MAJOR_MINOR_VERSION: "${{needs.build-info.outputs.default-python-version}}"
if: needs.build-info.outputs.skip-provider-tests != 'true'
steps:
- name: Cleanup repo
run: docker run -v "${GITHUB_WORKSPACE}:/workspace" -u 0:0 bash -c "rm -rf /workspace/*"
- name: "Checkout ${{ github.ref }} ( ${{ github.sha }} )"
uses: actions/checkout@v4
with:
persist-credentials: false
- name: >
Prepare breeze & CI image: ${{needs.build-info.outputs.default-python-version}}:${{env.IMAGE_TAG}}
uses: ./.github/actions/prepare_breeze_and_image
- name: "Cleanup dist files"
run: rm -fv ./dist/*
- name: "Prepare provider packages: wheel"
run: >
breeze release-management prepare-provider-packages --version-suffix-for-pypi dev0
--package-format wheel ${{ needs.build-info.outputs.affected-providers-list-as-string }}
- name: "Fix incompatible 2.5 provider packages"
run: |
# This step should remove the provider packages that are not compatible with 2.5
# or replace them with 2.5 compatible versions. Sometimes we have good reasons to bump
# the min airflow versions for some providers and then we need to add exclusions here.
#
# The Removal can be done with:
#
# rm -vf dist/apache_airflow_providers_<PROVIDER>*.whl
#
# Then it can be followed by downloading a compatible version from PyPI in case other
# providers depend on it and fail with import errors (you need to download compatible version):
#
# pip download --no-deps --dest dist apache-airflow-providers-<PROVIDER>==3.1.0
#
rm -vf dist/apache_airflow_providers_openlineage*.whl
rm -rf dist/apache_airflow_providers_common_io*.whl
- name: "Get all provider extras as AIRFLOW_EXTRAS env variable"
# Extras might be different on S3 so rather than relying on "all" we should get the list of
# packages to be installed from the current provider_dependencies.json file
run: |
python -c 'from pathlib import Path; import json
providers = json.loads(Path("generated/provider_dependencies.json").read_text())
provider_keys = ",".join(providers.keys())
print("AIRFLOW_EXTRAS={}".format(provider_keys))' >> $GITHUB_ENV
- name: "Install and verify all provider packages and airflow on Airflow 2.5 files"
run: >
breeze release-management verify-provider-packages --use-airflow-version 2.5.0
--use-packages-from-dist --airflow-constraints-reference constraints-2.5.0
if: needs.build-info.outputs.affected-providers-list-as-string == ''
- name: "Install affected provider packages and airflow on Airflow 2.5 files"
run: >
breeze release-management install-provider-packages --use-airflow-version 2.5.0
--airflow-constraints-reference constraints-2.5.0 --run-in-parallel
# Make sure to skip the run if the only provider to be installed has been removed
# in the previous step
if: >
needs.build-info.outputs.affected-providers-list-as-string != '' &&
needs.build-info.outputs.affected-providers-list-as-string != 'openlineage'
prepare-install-provider-packages-sdist:
timeout-minutes: 80
name: "Provider packages sdist (install)"
runs-on: ${{fromJSON(needs.build-info.outputs.runs-on)}}
needs: [build-info, wait-for-ci-images]
env:
RUNS_ON: "${{needs.build-info.outputs.runs-on}}"
PYTHON_MAJOR_MINOR_VERSION: "${{needs.build-info.outputs.default-python-version}}"
USE_AIRFLOW_VERSION: "sdist"
if: >
needs.build-info.outputs.canary-run == 'true' &&
needs.build-info.outputs.skip-provider-tests != 'true'
steps:
- name: Cleanup repo
run: docker run -v "${GITHUB_WORKSPACE}:/workspace" -u 0:0 bash -c "rm -rf /workspace/*"
- name: "Checkout ${{ github.ref }} ( ${{ github.sha }} )"
uses: actions/checkout@v4
with:
persist-credentials: false
- name: >
Prepare breeze & CI image: ${{needs.build-info.outputs.default-python-version}}:${{env.IMAGE_TAG}}
uses: ./.github/actions/prepare_breeze_and_image
- name: "Cleanup dist files"
run: rm -fv ./dist/*
- name: "Prepare provider packages: sdist"
run: >
breeze release-management prepare-provider-packages
--version-suffix-for-pypi dev0 --package-format sdist
${{ needs.build-info.outputs.affected-providers-list-as-string }}
- name: "Prepare airflow package: sdist"
run: >
breeze release-management prepare-airflow-package
--version-suffix-for-pypi dev0 --package-format sdist
- name: "Verify sdist packages with twine"
run: pipx install twine --force && twine check dist/*.tar.gz
- name: "Install all provider packages and airflow via sdist files"
run: >
breeze release-management install-provider-packages
--package-format sdist --use-airflow-version sdist --run-in-parallel
if: needs.build-info.outputs.affected-providers-list-as-string == ''
- name: "Install affected provider packages and airflow via sdist files"
run: >
breeze release-management install-provider-packages
--package-format sdist --use-airflow-version sdist --run-in-parallel
if: needs.build-info.outputs.affected-providers-list-as-string != ''
test-airflow-release-commands:
timeout-minutes: 80
name: "Test Airflow release commands"
runs-on: ${{fromJSON(needs.build-info.outputs.runs-on)}}
needs: [build-info, wait-for-ci-images]
if: >
needs.build-info.outputs.is-self-hosted-runner == 'true'
&& needs.build-info.outputs.is-airflow-runner == 'false'
env:
RUNS_ON: "${{needs.build-info.outputs.runs-on}}"
PYTHON_MAJOR_MINOR_VERSION: "${{needs.build-info.outputs.default-python-version}}"
steps:
- name: Cleanup repo
run: docker run -v "${GITHUB_WORKSPACE}:/workspace" -u 0:0 bash -c "rm -rf /workspace/*"
- name: "Checkout ${{ github.ref }} ( ${{ github.sha }} )"
uses: actions/checkout@v4
with:
persist-credentials: false
- name: >
Prepare breeze & CI image: ${{needs.build-info.outputs.default-python-version}}:${{env.IMAGE_TAG}}
uses: ./.github/actions/prepare_breeze_and_image
- name: "Cleanup dist files"
run: rm -fv ./dist/*
- name: "Install required packages"
run: python -m pip install --editable ./dev/breeze/
- name: "Check Airflow create minor branch command"
run: breeze release-management create-minor-branch --version-branch 2-4 -a y
- name: "Check Airflow RC process command"
run: |
breeze release-management start-rc-process --version 2.4.3rc1 --previous-version 2.4.2 -a y
- name: "Check Airflow release process command"
run: |
breeze release-management start-release --release-candidate 2.4.3rc1 --previous-release 2.4.2 -a y
tests-helm:
timeout-minutes: 80
name: "Unit tests Helm: ${{matrix.helm-test-package}}"
runs-on: ${{fromJSON(needs.build-info.outputs.runs-on)}}
needs: [build-info, wait-for-ci-images]
strategy:
fail-fast: false
matrix:
helm-test-package: ${{fromJson(needs.build-info.outputs.helm-test-packages)}}
env:
RUNS_ON: "${{needs.build-info.outputs.runs-on}}"
PARALLEL_TEST_TYPES: "Helm"
BACKEND: "none"
DB_RESET: "false"
PYTHON_MAJOR_MINOR_VERSION: "${{needs.build-info.outputs.default-python-version}}"
JOB_ID: "helm-tests"
USE_XDIST: "true"
if: >
needs.build-info.outputs.needs-helm-tests == 'true' && needs.build-info.outputs.default-branch == 'main'
steps:
- name: Cleanup repo
run: docker run -v "${GITHUB_WORKSPACE}:/workspace" -u 0:0 bash -c "rm -rf /workspace/*"
- name: "Checkout ${{ github.ref }} ( ${{ github.sha }} )"
uses: actions/checkout@v4
with:
persist-credentials: false
- name: >
Prepare breeze & CI image: ${{needs.build-info.outputs.default-python-version}}:${{env.IMAGE_TAG}}
uses: ./.github/actions/prepare_breeze_and_image
- name: "Helm Unit Tests: ${{ matrix.helm-test-package }}"
run: breeze testing helm-tests --helm-test-package "${{ matrix.helm-test-package }}"
tests-postgres:
timeout-minutes: 130
name: >
DB:Postgres${{matrix.postgres-version}},Py${{matrix.python-version}}:
${{needs.build-info.outputs.parallel-test-types-list-as-string}}
runs-on: ${{fromJSON(needs.build-info.outputs.runs-on)}}
needs: [build-info, wait-for-ci-images]
strategy:
matrix:
python-version: "${{fromJson(needs.build-info.outputs.python-versions)}}"
postgres-version: "${{fromJson(needs.build-info.outputs.postgres-versions)}}"
exclude: "${{fromJson(needs.build-info.outputs.postgres-exclude)}}"
fail-fast: false
env:
RUNS_ON: "${{needs.build-info.outputs.runs-on}}"
PARALLEL_TEST_TYPES: "${{needs.build-info.outputs.parallel-test-types-list-as-string}}"
PR_LABELS: "${{needs.build-info.outputs.pull-request-labels}}"
FULL_TESTS_NEEDED: "${{needs.build-info.outputs.full-tests-needed}}"
DEBUG_RESOURCES: "${{needs.build-info.outputs.debug-resources}}"
BACKEND: "postgres"
PYTHON_MAJOR_MINOR_VERSION: "${{matrix.python-version}}"
POSTGRES_VERSION: "${{matrix.postgres-version}}"
BACKEND_VERSION: "${{matrix.postgres-version}}"
JOB_ID: "postgres-${{matrix.postgres-version}}-${{matrix.python-version}}"
ENABLE_COVERAGE: "${{needs.build-info.outputs.run-coverage}}"
if: needs.build-info.outputs.run-tests == 'true'
steps:
- name: Cleanup repo
shell: bash
run: docker run -v "${GITHUB_WORKSPACE}:/workspace" -u 0:0 bash -c "rm -rf /workspace/*"
- name: "Checkout ${{ github.ref }} ( ${{ github.sha }} )"
uses: actions/checkout@v4
with:
persist-credentials: false
- name: "Prepare breeze & CI image: ${{matrix.python-version}}:${{env.IMAGE_TAG}}"
uses: ./.github/actions/prepare_breeze_and_image
- name: >
Migration Tests:
${{matrix.python-version}}:${{needs.build-info.outputs.parallel-test-types-list-as-string}}
uses: ./.github/actions/migration_tests
- name: >
Tests: ${{matrix.python-version}}:${{needs.build-info.outputs.parallel-test-types-list-as-string}}
run: >
breeze testing db-tests
--parallel-test-types "${{needs.build-info.outputs.parallel-test-types-list-as-string}}"
- name: "Tests ARM Pytest collection: ${{matrix.python-version}}"
run: breeze testing db-tests --collect-only --remove-arm-packages
if: matrix.postgres-version == needs.build-info.outputs.default-postgres-version
- name: >
Post Tests success: Postgres"
uses: ./.github/actions/post_tests_success
if: success()
- name: >
Post Tests failure: Postgres"
uses: ./.github/actions/post_tests_failure
if: failure()
tests-postgres-boto:
timeout-minutes: 130
name: >
DB:LatestBoto${{needs.build-info.outputs.default-postgres-version}},
Py${{needs.build-info.outputs.default-python-version}}:
${{needs.build-info.outputs.parallel-test-types-list-as-string}}
runs-on: ${{fromJSON(needs.build-info.outputs.runs-on)}}
needs: [build-info, wait-for-ci-images]
env:
RUNS_ON: "${{needs.build-info.outputs.runs-on}}"
PARALLEL_TEST_TYPES: "${{needs.build-info.outputs.parallel-test-types-list-as-string}}"
PR_LABELS: "${{needs.build-info.outputs.pull-request-labels}}"
FULL_TESTS_NEEDED: "${{needs.build-info.outputs.full-tests-needed}}"
DEBUG_RESOURCES: "${{needs.build-info.outputs.debug-resources}}"
BACKEND: "postgres"
ENABLE_COVERAGE: "${{needs.build-info.outputs.run-coverage}}"
PYTHON_MAJOR_MINOR_VERSION: "${{needs.build-info.outputs.default-python-version}}"
PYTHON_VERSION: "${needs.build-info.outputs.default-python-version}}"
POSTGRES_VERSION: "${{needs.build-info.outputs.default-postgres-version}}"
BACKEND_VERSION: "${{needs.build-info.outputs.default-postgres-version}}"
UPGRADE_BOTO: "true"
JOB_ID: >
postgres-boto-${{needs.build-info.outputs.default-python-version}}-
${{needs.build-info.outputs.default-postgres-version}}
if: needs.build-info.outputs.run-tests == 'true' && needs.build-info.outputs.run-amazon-tests == 'true'
steps:
- name: Cleanup repo
shell: bash
run: docker run -v "${GITHUB_WORKSPACE}:/workspace" -u 0:0 bash -c "rm -rf /workspace/*"
- name: "Checkout ${{ github.ref }} ( ${{ github.sha }} )"
uses: actions/checkout@v4
with:
persist-credentials: false
- name: >
Prepare breeze & CI image: ${{needs.build-info.outputs.default-python-version}}:${{env.IMAGE_TAG}}
uses: ./.github/actions/prepare_breeze_and_image
- name: >
Tests: ${{matrix.python-version}}:${{needs.build-info.outputs.parallel-test-types-list-as-string}}
run: >
breeze testing db-tests
--parallel-test-types "${{needs.build-info.outputs.parallel-test-types-list-as-string}}"
- name: >
Post Tests success: ${{needs.build-info.outputs.default-python-version}}:Boto"
uses: ./.github/actions/post_tests_success
if: success()
- name: >
Post Tests failure: ${{needs.build-info.outputs.default-python-version}}:Boto"
uses: ./.github/actions/post_tests_failure
if: failure()
tests-postgres-min-sqlalchemy:
timeout-minutes: 130
name: >
DB:MinSQLAlchemy${{needs.build-info.outputs.default-postgres-version}},
Py${{needs.build-info.outputs.default-python-version}}:
${{needs.build-info.outputs.parallel-test-types-list-as-string}}
runs-on: ${{fromJSON(needs.build-info.outputs.runs-on)}}
needs: [build-info, wait-for-ci-images]
env:
RUNS_ON: "${{needs.build-info.outputs.runs-on}}"
PARALLEL_TEST_TYPES: "${{needs.build-info.outputs.parallel-test-types-list-as-string}}"
PR_LABELS: "${{needs.build-info.outputs.pull-request-labels}}"
FULL_TESTS_NEEDED: "${{needs.build-info.outputs.full-tests-needed}}"
DEBUG_RESOURCES: "${{needs.build-info.outputs.debug-resources}}"
BACKEND: "postgres"
ENABLE_COVERAGE: "${{needs.build-info.outputs.run-coverage}}"
PYTHON_MAJOR_MINOR_VERSION: "${{needs.build-info.outputs.default-python-version}}"
PYTHON_VERSION: "${needs.build-info.outputs.default-python-version}}"
POSTGRES_VERSION: "${{needs.build-info.outputs.default-postgres-version}}"
BACKEND_VERSION: "${{needs.build-info.outputs.default-postgres-version}}"
DOWNGRADE_SQLALCHEMY: "true"
JOB_ID: >
postgres-min-sqlalchemy-${{needs.build-info.outputs.default-python-version}}-
${{needs.build-info.outputs.default-postgres-version}}
if: needs.build-info.outputs.run-tests == 'true'
steps:
- name: Cleanup repo
shell: bash
run: docker run -v "${GITHUB_WORKSPACE}:/workspace" -u 0:0 bash -c "rm -rf /workspace/*"
- name: "Checkout ${{ github.ref }} ( ${{ github.sha }} )"
uses: actions/checkout@v4
with:
persist-credentials: false
- name: >
Prepare breeze & CI image: ${{needs.build-info.outputs.default-python-version}}:${{env.IMAGE_TAG}}
uses: ./.github/actions/prepare_breeze_and_image
- name: >
Tests: ${{matrix.python-version}}:${{needs.build-info.outputs.parallel-test-types-list-as-string}}
run: >
breeze testing db-tests
--parallel-test-types "${{needs.build-info.outputs.parallel-test-types-list-as-string}}"
- name: >
Post Tests success: ${{needs.build-info.outputs.default-python-version}}:Boto"
uses: ./.github/actions/post_tests_success
if: success()
- name: >
Post Tests failure: ${{needs.build-info.outputs.default-python-version}}:Boto"
uses: ./.github/actions/post_tests_failure
if: failure()
tests-postgres-in-progress-features-disabled:
timeout-minutes: 130
name: >
DB:InProgressDisabledPostgres${{needs.build-info.outputs.default-postgres-version}},
Py${{needs.build-info.outputs.default-python-version}}:
${{needs.build-info.outputs.parallel-test-types-list-as-string}}
runs-on: ${{fromJSON(needs.build-info.outputs.runs-on)}}
needs: [build-info, wait-for-ci-images]
env:
RUNS_ON: "${{needs.build-info.outputs.runs-on}}"
PARALLEL_TEST_TYPES: "${{needs.build-info.outputs.parallel-test-types-list-as-string}}"
PR_LABELS: "${{needs.build-info.outputs.pull-request-labels}}"
FULL_TESTS_NEEDED: "${{needs.build-info.outputs.full-tests-needed}}"
DEBUG_RESOURCES: "${{needs.build-info.outputs.debug-resources}}"
BACKEND: "postgres"
ENABLE_COVERAGE: "${{needs.build-info.outputs.run-coverage}}"
PYTHON_MAJOR_MINOR_VERSION: "${{needs.build-info.outputs.default-python-version}}"
PYTHON_VERSION: "${needs.build-info.outputs.default-python-version}}"
POSTGRES_VERSION: "${{needs.build-info.outputs.default-postgres-version}}"
BACKEND_VERSION: "${{needs.build-info.outputs.default-postgres-version}}"
AIRFLOW_ENABLE_AIP_44: "false"
JOB_ID: >
postgres-in-progress-disabled-${{needs.build-info.outputs.default-python-version}}-
${{needs.build-info.outputs.default-postgres-version}}
if: needs.build-info.outputs.run-tests == 'true' && needs.build-info.outputs.run-amazon-tests == 'true'
steps:
- name: Cleanup repo
shell: bash
run: docker run -v "${GITHUB_WORKSPACE}:/workspace" -u 0:0 bash -c "rm -rf /workspace/*"
- name: "Checkout ${{ github.ref }} ( ${{ github.sha }} )"
uses: actions/checkout@v4
with:
persist-credentials: false
- name: >
Prepare breeze & CI image: ${{needs.build-info.outputs.default-python-version}}:${{env.IMAGE_TAG}}
uses: ./.github/actions/prepare_breeze_and_image
- name: >
Tests: ${{matrix.python-version}}:${{needs.build-info.outputs.parallel-test-types-list-as-string}}
run: >
breeze testing db-tests
--parallel-test-types "${{needs.build-info.outputs.parallel-test-types-list-as-string}}"
- name: >
Post Tests success: ${{needs.build-info.outputs.default-python-version}}:FeaturesDisabled"
uses: ./.github/actions/post_tests_success
if: success()
- name: >
Post Tests failure: ${{needs.build-info.outputs.default-python-version}}:FeaturesDisabled"
uses: ./.github/actions/post_tests_failure
if: failure()
tests-mysql:
timeout-minutes: 130
name: >
DB:MySQL${{matrix.mysql-version}}, Py${{matrix.python-version}}:
${{needs.build-info.outputs.parallel-test-types-list-as-string}}
runs-on: ${{fromJSON(needs.build-info.outputs.runs-on)}}
needs: [build-info, wait-for-ci-images]
strategy:
matrix:
python-version: "${{fromJson(needs.build-info.outputs.python-versions)}}"
mysql-version: "${{fromJson(needs.build-info.outputs.mysql-versions)}}"
exclude: "${{fromJson(needs.build-info.outputs.mysql-exclude)}}"
fail-fast: false
env:
RUNS_ON: "${{needs.build-info.outputs.runs-on}}"
PR_LABELS: "${{needs.build-info.outputs.pull-request-labels}}"
FULL_TESTS_NEEDED: "${{needs.build-info.outputs.full-tests-needed}}"
PARALLEL_TEST_TYPES: "${{needs.build-info.outputs.parallel-test-types-list-as-string}}"
DEBUG_RESOURCES: "${{needs.build-info.outputs.debug-resources}}"
BACKEND: "mysql"
ENABLE_COVERAGE: "${{needs.build-info.outputs.run-coverage}}"
PYTHON_MAJOR_MINOR_VERSION: "${{matrix.python-version}}"
MYSQL_VERSION: "${{matrix.mysql-version}}"
BACKEND_VERSION: "${{matrix.mysql-version}}"
JOB_ID: "mysql-${{matrix.mysql-version}}-${{matrix.python-version}}"
if: needs.build-info.outputs.run-tests == 'true'
steps:
- name: Cleanup repo
shell: bash
run: docker run -v "${GITHUB_WORKSPACE}:/workspace" -u 0:0 bash -c "rm -rf /workspace/*"
- name: "Checkout ${{ github.ref }} ( ${{ github.sha }} )"
uses: actions/checkout@v4
with:
persist-credentials: false
- name: "Prepare breeze & CI image: ${{matrix.python-version}}:${{env.IMAGE_TAG}}"
uses: ./.github/actions/prepare_breeze_and_image
- name: >
Migration Tests:
${{matrix.python-version}}:${{needs.build-info.outputs.parallel-test-types-list-as-string}}
uses: ./.github/actions/migration_tests
- name: >
Tests: ${{matrix.python-version}}:${{needs.build-info.outputs.parallel-test-types-list-as-string}}
run: >
breeze testing db-tests
--parallel-test-types "${{needs.build-info.outputs.parallel-test-types-list-as-string}}"
- name: >
Post Tests success: MySQL"
uses: ./.github/actions/post_tests_success
if: success()
- name: >
Post Tests failure: MySQL"
uses: ./.github/actions/post_tests_failure
if: failure()
tests-mssql:
timeout-minutes: 130
name: >
DB:MSSQL${{matrix.mssql-version}}, Py${{matrix.python-version}}:
${{needs.build-info.outputs.parallel-test-types-list-as-string}}
runs-on: ${{fromJSON(needs.build-info.outputs.runs-on)}}
needs: [build-info, wait-for-ci-images]
strategy:
matrix:
python-version: "${{fromJson(needs.build-info.outputs.python-versions)}}"
mssql-version: "${{fromJson(needs.build-info.outputs.mssql-versions)}}"
exclude: "${{fromJson(needs.build-info.outputs.mssql-exclude)}}"
fail-fast: false
env:
RUNS_ON: "${{needs.build-info.outputs.runs-on}}"
PARALLEL_TEST_TYPES: "${{needs.build-info.outputs.parallel-test-types-list-as-string}}"
PR_LABELS: "${{needs.build-info.outputs.pull-request-labels}}"
FULL_TESTS_NEEDED: "${{needs.build-info.outputs.full-tests-needed}}"
DEBUG_RESOURCES: "${{needs.build-info.outputs.debug-resources}}"
BACKEND: "mssql"
ENABLE_COVERAGE: "${{needs.build-info.outputs.run-coverage}}"
PYTHON_MAJOR_MINOR_VERSION: "${{matrix.python-version}}"
MSSQL_VERSION: "${{matrix.mssql-version}}"
BACKEND_VERSION: "${{matrix.mssql-version}}"
JOB_ID: "mssql-${{matrix.mssql-version}}-${{matrix.python-version}}"
# The below (and corresponding selective checks code) can be removed once
# https://github.com/apache/airflow/issues/31575 is fixed.
# This is a temporary workaround for flaky tests that occur in MSSQL tests for public runners
PARALLELISM: "${{needs.build-info.outputs.mssql-parallelism}}"
if: >
needs.build-info.outputs.run-tests == 'true' &&
(needs.build-info.outputs.is-self-hosted-runner == 'true' &&
needs.build-info.outputs.is-airflow-runner == 'true' ||
needs.build-info.outputs.full-tests-needed == 'true' ||
needs.build-info.outputs.has-migrations == 'true')
steps:
- name: Cleanup repo
shell: bash
run: docker run -v "${GITHUB_WORKSPACE}:/workspace" -u 0:0 bash -c "rm -rf /workspace/*"
- name: "Checkout ${{ github.ref }} ( ${{ github.sha }} )"
uses: actions/checkout@v4
with:
persist-credentials: false
- name: "Prepare breeze & CI image: ${{matrix.python-version}}:${{env.IMAGE_TAG}}"
uses: ./.github/actions/prepare_breeze_and_image
- name: >
Migration Tests:
${{matrix.python-version}}:${{needs.build-info.outputs.parallel-test-types-list-as-string}}
uses: ./.github/actions/migration_tests
- name: >
Tests: ${{matrix.python-version}}:${{needs.build-info.outputs.parallel-test-types-list-as-string}}
run: >
breeze testing db-tests
--parallel-test-types "${{needs.build-info.outputs.parallel-test-types-list-as-string}}"
- name: >
Post Tests success: MsSQL"
uses: ./.github/actions/post_tests_success
if: success()
- name: >
Post Tests failure: MsSQL"
uses: ./.github/actions/post_tests_failure
if: failure()
tests-sqlite:
timeout-minutes: 130
name: >
DB:Sqlite Py${{matrix.python-version}}: ${{needs.build-info.outputs.parallel-test-types-list-as-string}}
runs-on: ${{fromJSON(needs.build-info.outputs.runs-on)}}
needs: [build-info, wait-for-ci-images]
strategy:
matrix:
python-version: ${{ fromJson(needs.build-info.outputs.python-versions) }}
exclude: ${{ fromJson(needs.build-info.outputs.sqlite-exclude) }}
fail-fast: false
if: needs.build-info.outputs.run-tests == 'true'
env:
RUNS_ON: "${{needs.build-info.outputs.runs-on}}"
PARALLEL_TEST_TYPES: "${{needs.build-info.outputs.parallel-test-types-list-as-string}}"
PR_LABELS: "${{needs.build-info.outputs.pull-request-labels}}"
PYTHON_MAJOR_MINOR_VERSION: "${{matrix.python-version}}"
FULL_TESTS_NEEDED: "${{needs.build-info.outputs.full-tests-needed}}"
DEBUG_RESOURCES: "${{needs.build-info.outputs.debug-resources}}"
BACKEND: "sqlite"
ENABLE_COVERAGE: "${{needs.build-info.outputs.run-coverage}}"
BACKEND_VERSION: ""
JOB_ID: "sqlite-${{matrix.python-version}}"
steps:
- name: Cleanup repo
shell: bash
run: docker run -v "${GITHUB_WORKSPACE}:/workspace" -u 0:0 bash -c "rm -rf /workspace/*"
- name: "Checkout ${{ github.ref }} ( ${{ github.sha }} )"
uses: actions/checkout@v4
with:
persist-credentials: false
- name: "Prepare breeze & CI image: ${{matrix.python-version}}:${{env.IMAGE_TAG}}"
uses: ./.github/actions/prepare_breeze_and_image
- name: >
Migration Tests:
${{matrix.python-version}}:${{needs.build-info.outputs.parallel-test-types-list-as-string}}
uses: ./.github/actions/migration_tests
- name: >
Tests: ${{matrix.python-version}}:${{needs.build-info.outputs.parallel-test-types-list-as-string}}
run: >
breeze testing db-tests
--parallel-test-types "${{needs.build-info.outputs.parallel-test-types-list-as-string}}"
- name: >
Post Tests success: Sqlite"
uses: ./.github/actions/post_tests_success
if: success()
- name: >
Post Tests failure: Sqlite"
uses: ./.github/actions/post_tests_failure
if: failure()
tests-integration-postgres:
timeout-minutes: 130
name: Integration Tests Postgres
runs-on: ${{fromJSON(needs.build-info.outputs.runs-on)}}
needs: [build-info, wait-for-ci-images]
env:
RUNS_ON: "${{needs.build-info.outputs.runs-on}}"
PARALLEL_TEST_TYPES: "${{needs.build-info.outputs.parallel-test-types-list-as-string}}"
PR_LABELS: "${{needs.build-info.outputs.pull-request-labels}}"
FULL_TESTS_NEEDED: "${{needs.build-info.outputs.full-tests-needed}}"
DEBUG_RESOURCES: "${{needs.build-info.outputs.debug-resources}}"
BACKEND: "postgres"
ENABLE_COVERAGE: "${{needs.build-info.outputs.run-coverage}}"
PYTHON_MAJOR_MINOR_VERSION: "${{needs.build-info.outputs.default-python-version}}"
POSTGRES_VERSION: "${{needs.build-info.outputs.default-postgres-version}}"
BACKEND_VERSION: "${{needs.build-info.outputs.default-postgres-version}}"
JOB_ID: "integration"
SKIP_PROVIDER_TESTS: "${{needs.build-info.outputs.skip-provider-tests}}"
if: needs.build-info.outputs.run-tests == 'true'
steps:
- name: Cleanup repo
shell: bash
run: docker run -v "${GITHUB_WORKSPACE}:/workspace" -u 0:0 bash -c "rm -rf /workspace/*"
- name: "Checkout ${{ github.ref }} ( ${{ github.sha }} )"
uses: actions/checkout@v4
with:
persist-credentials: false
- name: "Prepare breeze & CI image: ${{env.PYTHON_MAJOR_MINOR_VERSION}}:${{env.IMAGE_TAG}}"
uses: ./.github/actions/prepare_breeze_and_image
- name: "Integration Tests Postgres: cassandra"
run: |
breeze testing integration-tests --integration cassandra
breeze down
if: needs.build-info.outputs.is-airflow-runner != 'true'
- name: "Integration Tests Postgres: mongo"
run: |
breeze testing integration-tests --integration mongo
breeze down
if: needs.build-info.outputs.is-airflow-runner != 'true'
- name: "Integration Tests Postgres: pinot"
run: |
breeze testing integration-tests --integration pinot
breeze down
if: needs.build-info.outputs.is-airflow-runner != 'true'
- name: "Integration Tests Postgres: celery"
run: |
breeze testing integration-tests --integration celery
breeze down
if: needs.build-info.outputs.is-airflow-runner != 'true'
- name: "Integration Tests Postgres: trino, kerberos"
run: |
breeze testing integration-tests --integration trino --integration kerberos
breeze down
if: needs.build-info.outputs.is-airflow-runner != 'true'
- name: "Integration Tests Postgres: Kafka"
run: |
breeze testing integration-tests --integration kafka
breeze down
if: needs.build-info.outputs.is-airflow-runner != 'true'
- name: "Integration Tests Postgres: all-testable"
run: breeze testing integration-tests --integration all-testable
if: needs.build-info.outputs.is-airflow-runner == 'true'
- name: >
Post Tests success: Integration Postgres"
uses: ./.github/actions/post_tests_success
if: success()
- name: >
Post Tests failure: Integration Postgres"
uses: ./.github/actions/post_tests_failure
if: failure()
tests-integration-mysql:
timeout-minutes: 130
name: Integration Tests MySQL
runs-on: ${{fromJSON(needs.build-info.outputs.runs-on)}}
needs: [build-info, wait-for-ci-images]
env:
RUNS_ON: "${{needs.build-info.outputs.runs-on}}"
PARALLEL_TEST_TYPES: "${{needs.build-info.outputs.parallel-test-types-list-as-string}}"
PR_LABELS: "${{needs.build-info.outputs.pull-request-labels}}"
FULL_TESTS_NEEDED: "${{needs.build-info.outputs.full-tests-needed}}"
DEBUG_RESOURCES: "${{needs.build-info.outputs.debug-resources}}"
BACKEND: "mysql"
ENABLE_COVERAGE: "${{needs.build-info.outputs.run-coverage}}"
PYTHON_MAJOR_MINOR_VERSION: "${{needs.build-info.outputs.default-python-version}}"
MYSQL_VERSION: "${{needs.build-info.outputs.default-mysql-version}}"
BACKEND_VERSION: "${{needs.build-info.outputs.default-mysql-version}}"
JOB_ID: "integration"
SKIP_PROVIDER_TESTS: "${{needs.build-info.outputs.skip-provider-tests}}"
if: needs.build-info.outputs.run-tests == 'true'
steps:
- name: Cleanup repo
shell: bash
run: docker run -v "${GITHUB_WORKSPACE}:/workspace" -u 0:0 bash -c "rm -rf /workspace/*"
if: needs.build-info.outputs.is-airflow-runner == 'true'
- name: "Checkout ${{ github.ref }} ( ${{ github.sha }} )"
uses: actions/checkout@v4
with:
persist-credentials: false
if: needs.build-info.outputs.is-airflow-runner == 'true'
- name: "Prepare breeze & CI image: ${{env.PYTHON_MAJOR_MINOR_VERSION}}:${{env.IMAGE_TAG}}"
uses: ./.github/actions/prepare_breeze_and_image
if: needs.build-info.outputs.is-airflow-runner == 'true'
- name: "Integration Tests MySQL: all-testable"
run: breeze testing integration-tests --integration all-testable
if: needs.build-info.outputs.is-airflow-runner == 'true'
- name: >
Post Tests success: Integration MySQL"
uses: ./.github/actions/post_tests_success
if: needs.build-info.outputs.is-airflow-runner == 'true'
- name: >
Post Tests failure: Integration MySQL"
uses: ./.github/actions/post_tests_failure
if: failure()
tests-quarantined:
timeout-minutes: 60
name: "Quarantined tests"
runs-on: ${{fromJSON(needs.build-info.outputs.runs-on)}}
continue-on-error: true
needs: [build-info, wait-for-ci-images]
env:
RUNS_ON: "${{needs.build-info.outputs.runs-on}}"
TEST_TYPE: "All-Quarantined"
PR_LABELS: "${{needs.build-info.outputs.pull-request-labels}}"
PYTHON_MAJOR_MINOR_VERSION: "${{needs.build-info.outputs.default-python-version}}"
DEBUG_RESOURCES: "${{needs.build-info.outputs.debug-resources}}"
JOB_ID: "quarantined-${{needs.build-info.outputs.default-python-version}}"
ENABLE_COVERAGE: "${{needs.build-info.outputs.run-coverage}}"
if: needs.build-info.outputs.run-tests == 'true' && needs.build-info.outputs.is-airflow-runner == 'true'
steps:
- name: Cleanup repo
shell: bash
run: docker run -v "${GITHUB_WORKSPACE}:/workspace" -u 0:0 bash -c "rm -rf /workspace/*"
- name: "Checkout ${{ github.ref }} ( ${{ github.sha }} )"
uses: actions/checkout@v4
with:
persist-credentials: false
- name: >
Prepare breeze & CI image: ${{needs.build-info.outputs.default-python-version}}:${{env.IMAGE_TAG}}
uses: ./.github/actions/prepare_breeze_and_image
- name: >
Tests: postgres:${{needs.build-info.outputs.default-python-version}}:Quarantined
run: breeze testing tests || true
env:
BACKEND: "postgres"
BACKEND_VERSION: "11"
POSTGRES_VERSION: "11"
- name: >
Tests: mysql:${{needs.build-info.outputs.default-python-version}}:Quarantined
run: breeze testing tests || true
env:
BACKEND: "mysql"
BACKEND_VERSION: ${{needs.build-info.outputs.default-mysql-version}}
MYSQL_VERSION: ${{needs.build-info.outputs.default-mysql-version}}
- name: >
Tests: mssql:${{needs.build-info.outputs.default-python-version}}:Quarantined
run: breeze testing tests || true
env:
BACKEND: "mssql"
BACKEND_VERSION: ${{needs.build-info.outputs.default-mssql-version}}
MSSQL_VERSION: ${{needs.build-info.outputs.default-mssql-version}}
- name: >
Tests: sqlite:${{needs.build-info.outputs.default-python-version}}:Quarantined
run: breeze testing tests || true
env:
BACKEND: "sqlite"
BACKEND_VERSION: ""
MSSQL_VERSION: ""
- name: >
Post Tests success: Quarantined"
uses: ./.github/actions/post_tests_success
if: success()
- name: >
Post Tests failure: Quarantined"
uses: ./.github/actions/post_tests_failure
if: failure()
tests-no-db:
timeout-minutes: 60
name: >
Non-DB: Py${{matrix.python-version}}:${{needs.build-info.outputs.parallel-test-types-list-as-string}}
runs-on: ${{fromJSON(needs.build-info.outputs.runs-on)}}
needs: [build-info, wait-for-ci-images]
strategy:
fail-fast: false
matrix:
python-version: ${{ fromJson(needs.build-info.outputs.python-versions) }}
env:
RUNS_ON: "${{needs.build-info.outputs.runs-on}}"
PR_LABELS: "${{needs.build-info.outputs.pull-request-labels}}"
PYTHON_MAJOR_MINOR_VERSION: "${{needs.build-info.outputs.default-python-version}}"
DEBUG_RESOURCES: "${{needs.build-info.outputs.debug-resources}}"
JOB_ID: "quarantined-${{needs.build-info.outputs.default-python-version}}"
ENABLE_COVERAGE: "${{needs.build-info.outputs.run-coverage}}"
if: needs.build-info.outputs.run-tests == 'true'
steps:
- name: Cleanup repo
shell: bash
run: docker run -v "${GITHUB_WORKSPACE}:/workspace" -u 0:0 bash -c "rm -rf /workspace/*"
- name: "Checkout ${{ github.ref }} ( ${{ github.sha }} )"
uses: actions/checkout@v4
with:
persist-credentials: false
- name: >
Prepare breeze & CI image: ${{needs.build-info.outputs.default-python-version}}:${{env.IMAGE_TAG}}
uses: ./.github/actions/prepare_breeze_and_image
- name: "Tests: ${{matrix.python-version}}:Non-DB"
run: >
breeze testing non-db-tests
--parallel-test-types "${{needs.build-info.outputs.parallel-test-types-list-as-string}}"
- name: "Post Tests success: Non-DB"
uses: ./.github/actions/post_tests_success
if: success()
- name: "Post Tests failure: Non-DB"
uses: ./.github/actions/post_tests_failure
if: failure()
summarize-warnings:
timeout-minutes: 15
name: "Summarize warnings"
runs-on: ${{fromJSON(needs.build-info.outputs.runs-on)}}
needs:
- build-info
- tests-postgres
- tests-sqlite
- tests-mysql
- tests-quarantined
- tests-integration-postgres
- tests-integration-mysql
- tests-no-db
env:
RUNS_ON: "${{needs.build-info.outputs.runs-on}}"
steps:
- name: Cleanup repo
run: docker run -v "${GITHUB_WORKSPACE}:/workspace" -u 0:0 bash -c "rm -rf /workspace/*"
- name: "Checkout ${{ github.ref }} ( ${{ github.sha }} )"
uses: actions/checkout@v4
with:
persist-credentials: false
- name: "Download all artifacts from the current build"
uses: actions/download-artifact@v3
with:
path: ./artifacts
- name: "Summarize all warnings"
run: |
ls -R ./artifacts/
cat ./artifacts/test-warnings*/* | sort | uniq
echo
echo Total number of unique warnings $(cat ./artifacts/test-warnings*/* | sort | uniq | wc -l)
build-prod-images:
timeout-minutes: 80
name: >
${{needs.build-info.outputs.build-job-description}} PROD images
${{needs.build-info.outputs.all-python-versions-list-as-string}}
runs-on: ${{fromJSON(needs.build-info.outputs.runs-on)}}
needs: [build-info, build-ci-images]
env:
DEFAULT_BRANCH: ${{ needs.build-info.outputs.default-branch }}
DEFAULT_CONSTRAINTS_BRANCH: ${{ needs.build-info.outputs.default-constraints-branch }}
RUNS_ON: "${{needs.build-info.outputs.runs-on}}"
BACKEND: sqlite
VERSION_SUFFIX_FOR_PYPI: "dev0"
DEBUG_RESOURCES: ${{needs.build-info.outputs.debug-resources}}
# Force more parallelism for build even on public images
PARALLELISM: 6
steps:
- name: Cleanup repo
run: docker run -v "${GITHUB_WORKSPACE}:/workspace" -u 0:0 bash -c "rm -rf /workspace/*"
if: needs.build-info.outputs.in-workflow-build == 'true'
- uses: actions/checkout@v4
with:
ref: ${{ needs.build-info.outputs.targetCommitSha }}
persist-credentials: false
if: needs.build-info.outputs.in-workflow-build == 'true'
- name: "Install Breeze"
uses: ./.github/actions/breeze
if: needs.build-info.outputs.in-workflow-build == 'true'
- name: >
Build PROD Images
${{needs.build-info.outputs.all-python-versions-list-as-string}}:${{env.IMAGE_TAG}}
uses: ./.github/actions/build-prod-images
if: needs.build-info.outputs.in-workflow-build == 'true'
with:
build-provider-packages: ${{ needs.build-info.outputs.default-branch == 'main' }}
env:
UPGRADE_TO_NEWER_DEPENDENCIES: ${{ needs.build-info.outputs.upgrade-to-newer-dependencies }}
DOCKER_CACHE: ${{ needs.build-info.outputs.cache-directive }}
PYTHON_VERSIONS: ${{needs.build-info.outputs.all-python-versions-list-as-string}}
DEBUG_RESOURCES: ${{ needs.build-info.outputs.debug-resources }}
build-prod-images-bullseye:
timeout-minutes: 80
name: >
Build Bullseye PROD images
${{needs.build-info.outputs.all-python-versions-list-as-string}}
runs-on: ${{fromJSON(needs.build-info.outputs.runs-on)}}
needs: [build-info, build-ci-images]
if: needs.build-info.outputs.canary-run == 'true'
env:
DEFAULT_BRANCH: ${{ needs.build-info.outputs.default-branch }}
DEFAULT_CONSTRAINTS_BRANCH: ${{ needs.build-info.outputs.default-constraints-branch }}
RUNS_ON: "${{needs.build-info.outputs.runs-on}}"
BACKEND: sqlite
VERSION_SUFFIX_FOR_PYPI: "dev0"
DEBUG_RESOURCES: ${{needs.build-info.outputs.debug-resources}}
# Force more parallelism for build even on public images
PARALLELISM: 6
steps:
- name: Cleanup repo
run: docker run -v "${GITHUB_WORKSPACE}:/workspace" -u 0:0 bash -c "rm -rf /workspace/*"
- uses: actions/checkout@v3
with:
ref: ${{ needs.build-info.outputs.targetCommitSha }}
persist-credentials: false
submodules: recursive
- name: "Install Breeze"
uses: ./.github/actions/breeze
- name: >
Build Bullseye PROD Images
${{needs.build-info.outputs.all-python-versions-list-as-string}}:${{env.IMAGE_TAG}}
uses: ./.github/actions/build-prod-images
with:
build-provider-packages: ${{ needs.build-info.outputs.default-branch == 'main' }}
env:
UPGRADE_TO_NEWER_DEPENDENCIES: ${{ needs.build-info.outputs.upgrade-to-newer-dependencies }}
DOCKER_CACHE: ${{ needs.build-info.outputs.cache-directive }}
PYTHON_VERSIONS: ${{needs.build-info.outputs.all-python-versions-list-as-string}}
DEBUG_RESOURCES: ${{ needs.build-info.outputs.debug-resources }}
DEBIAN_VERSION: "bullseye"
# Do not override the "bookworm" image - just push a new bullseye image
# TODO: improve caching for that build
IMAGE_TAG: "bullseye-${{ github.event.pull_request.head.sha || github.sha }}"
wait-for-prod-images:
timeout-minutes: 80
name: "Wait for PROD images"
runs-on: ${{fromJSON(needs.build-info.outputs.runs-on)}}
needs: [build-info, wait-for-ci-images, build-prod-images]
if: needs.build-info.outputs.prod-image-build == 'true'
env:
RUNS_ON: "${{needs.build-info.outputs.runs-on}}"
BACKEND: sqlite
PYTHON_MAJOR_MINOR_VERSION: "${{needs.build-info.outputs.default-python-version}}"
# Force more parallelism for pull even on public images
PARALLELISM: 6
steps:
- name: Cleanup repo
run: docker run -v "${GITHUB_WORKSPACE}:/workspace" -u 0:0 bash -c "rm -rf /workspace/*"
if: needs.build-info.outputs.in-workflow-build == 'false'
- name: "Checkout ${{ github.ref }} ( ${{ github.sha }} )"
uses: actions/checkout@v4
with:
persist-credentials: false
if: needs.build-info.outputs.in-workflow-build == 'false'
- name: "Install Breeze"
uses: ./.github/actions/breeze
if: needs.build-info.outputs.in-workflow-build == 'false'
- name: Wait for PROD images ${{ env.PYTHON_VERSIONS }}:${{ env.IMAGE_TAG }}
# We wait for the images to be available either from "build-images.yml' run as pull_request_target
# or from build-prod-images above.
# We are utilising single job to wait for all images because this job merely waits
# For the images to be available.
run: breeze prod-image pull --wait-for-image --run-in-parallel
env:
PYTHON_VERSIONS: ${{ needs.build-info.outputs.python-versions-list-as-string }}
DEBUG_RESOURCES: ${{needs.build-info.outputs.debug-resources}}
if: needs.build-info.outputs.in-workflow-build == 'false'
test-docker-compose-quick-start:
timeout-minutes: 60
name: "Docker-compose quick start with PROD image verifying"
runs-on: ${{fromJSON(needs.build-info.outputs.runs-on)}}
needs: [build-info, wait-for-prod-images]
if: needs.build-info.outputs.prod-image-build == 'true'
env:
PYTHON_MAJOR_MINOR_VERSION: "${{needs.build-info.outputs.default-python-version}}"
steps:
- name: Cleanup repo
run: docker run -v "${GITHUB_WORKSPACE}:/workspace" -u 0:0 bash -c "rm -rf /workspace/*"
- name: "Checkout ${{ github.ref }} ( ${{ github.sha }} )"
uses: actions/checkout@v4
with:
fetch-depth: 2
persist-credentials: false
- name: "Install Breeze"
uses: ./.github/actions/breeze
- name: Pull PROD images ${{ env.PYTHON_VERSIONS }}:${{ env.IMAGE_TAG }}
run: breeze prod-image pull --run-in-parallel --tag-as-latest
env:
PYTHON_VERSIONS: ${{ needs.build-info.outputs.python-versions-list-as-string }}
# Force more parallelism for pull even on public images
PARALLELISM: 6
- name: Verify PROD images ${{ env.PYTHON_VERSIONS }}:${{ env.IMAGE_TAG }}
run: breeze prod-image verify --run-in-parallel
env:
PYTHON_VERSIONS: ${{needs.build-info.outputs.all-python-versions-list-as-string}}
DEBUG_RESOURCES: ${{needs.build-info.outputs.debug-resources}}
- name: "Test docker-compose quick start"
run: breeze testing docker-compose-tests
tests-kubernetes:
timeout-minutes: 240
name: "\
K8S System:${{matrix.executor}} - ${{matrix.use-standard-naming}} - \
${{needs.build-info.outputs.kubernetes-versions-list-as-string}}"
runs-on: ${{fromJSON(needs.build-info.outputs.runs-on)}}
needs: [build-info, wait-for-prod-images]
strategy:
matrix:
executor: [KubernetesExecutor, CeleryExecutor, LocalExecutor]
use-standard-naming: [true, false]
fail-fast: false
env:
RUNS_ON: "${{needs.build-info.outputs.runs-on}}"
DEBUG_RESOURCES: ${{needs.build-info.outputs.debug-resources}}
if: >
( needs.build-info.outputs.run-kubernetes-tests == 'true' ||
needs.build-info.outputs.needs-helm-tests == 'true')
steps:
- name: Cleanup repo
run: docker run -v "${GITHUB_WORKSPACE}:/workspace" -u 0:0 bash -c "rm -rf /workspace/*"
- name: "Checkout ${{ github.ref }} ( ${{ github.sha }} )"
uses: actions/checkout@v4
with:
persist-credentials: false
- name: "Install Breeze"
uses: ./.github/actions/breeze
id: breeze
- name: Pull PROD images ${{ env.PYTHON_VERSIONS }}:${{ env.IMAGE_TAG }}
run: breeze prod-image pull --run-in-parallel --tag-as-latest
env:
PYTHON_VERSIONS: ${{ needs.build-info.outputs.python-versions-list-as-string }}
# Force more parallelism for pull even on public images
PARALLELISM: 6
- name: "Cache bin folder with tools for kubernetes testing"
uses: actions/cache@v3
with:
path: ".build/.k8s-env"
key: "\
k8s-env-${{steps.breeze.outputs.host-python-version}}-\
${{ hashFiles('scripts/ci/kubernetes/k8s_requirements.txt','setup.cfg',\
'setup.py','pyproject.toml','generated/provider_dependencies.json') }}"
- name: Run complete K8S tests ${{needs.build-info.outputs.kubernetes-combos-list-as-string}}
run: breeze k8s run-complete-tests --run-in-parallel --upgrade
env:
PYTHON_VERSIONS: ${{ needs.build-info.outputs.python-versions-list-as-string }}
KUBERNETES_VERSIONS: ${{needs.build-info.outputs.kubernetes-versions-list-as-string}}
EXECUTOR: ${{matrix.executor}}
USE_STANDARD_NAMING: ${{matrix.use-standard-naming}}
VERBOSE: "false"
- name: Upload KinD logs on failure ${{needs.build-info.outputs.kubernetes-combos-list-as-string}}
uses: actions/upload-artifact@v3
if: failure() || cancelled()
with:
name: kind-logs-${{matrix.executor}}-${{matrix.use-standard-naming}}
path: /tmp/kind_logs_*
retention-days: 7
- name: >
Upload test resource logs on failure ${{needs.build-info.outputs.kubernetes-combos-list-as-string}}
uses: actions/upload-artifact@v3
if: failure() || cancelled()
with:
name: k8s-test-resources-${{matrix.executor}}-${{matrix.use-standard-naming}}
path: /tmp/k8s_test_resources_*
retention-days: 7
- name: "Delete clusters just in case they are left"
run: breeze k8s delete-cluster --all
if: always()
update-constraints:
permissions:
contents: write
packages: write
timeout-minutes: 80
name: "Update constraints"
runs-on: ${{fromJSON(needs.build-info.outputs.runs-on)}}
needs:
- build-info
- build-docs
- spellcheck-docs
- wait-for-ci-images
- wait-for-prod-images
- static-checks
- tests-sqlite
- tests-mysql
- tests-postgres
- tests-no-db
- tests-integration-postgres
- tests-integration-mysql
- generate-constraints
env:
RUNS_ON: "${{needs.build-info.outputs.runs-on}}"
DEBUG_RESOURCES: ${{needs.build-info.outputs.debug-resources}}
PYTHON_VERSIONS: ${{ needs.build-info.outputs.python-versions-list-as-string }}
if: needs.build-info.outputs.upgrade-to-newer-dependencies != 'false'
steps:
- name: Cleanup repo
run: docker run -v "${GITHUB_WORKSPACE}:/workspace" -u 0:0 bash -c "rm -rf /workspace/*"
- name: "Checkout ${{ github.ref }} ( ${{ github.sha }} )"
uses: actions/checkout@v4
with:
# Needed to perform push action
persist-credentials: false
- name: "Set constraints branch name"
id: constraints-branch
run: ./scripts/ci/constraints/ci_branch_constraints.sh >> ${GITHUB_OUTPUT}
- name: Checkout ${{ steps.constraints-branch.outputs.branch }}
uses: actions/checkout@v4
with:
path: "constraints"
ref: ${{ steps.constraints-branch.outputs.branch }}
persist-credentials: true
fetch-depth: 0
- name: "Download constraints from the constraints preview"
uses: actions/download-artifact@v3
with:
name: constraints
path: ./files
- name: "Diff in constraints for ${{needs.build-info.outputs.python-versions}}"
run: ./scripts/ci/constraints/ci_diff_constraints.sh
# only commit and push constraints in canary runs (main)
- name: "Commit changed constraint files for ${{needs.build-info.outputs.python-versions}}"
run: ./scripts/ci/constraints/ci_commit_constraints.sh
if: needs.build-info.outputs.canary-run == 'true'
- name: "Push changes"
if: needs.build-info.outputs.canary-run == 'true'
working-directory: "constraints"
run:
git push
# Push BuildX cache to GitHub Registry in Apache repository, if all tests are successful and build
# is executed as result of direct push to "main" or one of the "vX-Y-test" branches
# It rebuilds all images using just-pushed constraints using buildx and pushes them to registry
# It will automatically check if a new python image was released and will pull the latest one if needed
push-buildx-cache-to-github-registry:
timeout-minutes: 50
name: "Push Image Cache"
runs-on: ${{fromJSON(needs.build-info.outputs.runs-on)}}
needs:
- build-info
- update-constraints
if: needs.build-info.outputs.canary-run == 'true'
strategy:
fail-fast: false
matrix:
platform: ["linux/amd64", "linux/arm64"]
env:
RUNS_ON: "${{needs.build-info.outputs.runs-on}}"
VERSION_SUFFIX_FOR_PYPI: "dev0"
steps:
- name: Cleanup repo
run: docker run -v "${GITHUB_WORKSPACE}:/workspace" -u 0:0 bash -c "rm -rf /workspace/*"
- name: "Checkout ${{ github.ref }} ( ${{ github.sha }} )"
uses: actions/checkout@v4
with:
persist-credentials: false
- name: >
Prepare breeze & CI image: ${{needs.build-info.outputs.default-python-version}}:${{env.IMAGE_TAG}}
uses: ./.github/actions/prepare_breeze_and_image
env:
# Always use default Python version of CI image for preparing packages
PYTHON_MAJOR_MINOR_VERSION: "${{needs.build-info.outputs.default-python-version}}"
- name: "Cleanup dist and context file"
run: rm -fv ./dist/* ./docker-context-files/*
- name: "Prepare airflow package for PROD build"
run: breeze release-management prepare-airflow-package --package-format wheel
env:
VERSION_SUFFIX_FOR_PYPI: "dev0"
- name: "Prepare providers packages for PROD build"
run: >
breeze release-management prepare-provider-packages
--package-list-file ./airflow/providers/installed_providers.txt
--package-format wheel
env:
VERSION_SUFFIX_FOR_PYPI: "dev0"
if: needs.build-info.outputs.default-branch == 'main'
- name: "Start ARM instance"
run: ./scripts/ci/images/ci_start_arm_instance_and_connect_to_docker.sh
if: matrix.platform == 'linux/arm64'
- name: "Push CI cache ${{ matrix.platform }}"
run: >
breeze ci-image build
--builder airflow_cache
--prepare-buildx-cache
--run-in-parallel
--platform ${{ matrix.platform }}
env:
DEBUG_RESOURCES: ${{needs.build-info.outputs.debug-resources}}
COMMIT_SHA: ${{ github.sha }}
- name: "Push CI latest image ${{ matrix.platform }}"
run: >
breeze ci-image build
--tag-as-latest --push --run-in-parallel --platform ${{ matrix.platform }}
env:
DEBUG_RESOURCES: ${{needs.build-info.outputs.debug-resources}}
COMMIT_SHA: ${{ github.sha }}
if: matrix.platform == 'linux/amd64'
- name: "Copy dist packages to docker-context files"
run: cp -v --no-preserve=mode,ownership ./dist/*.whl ./docker-context-files
- name: "Push PROD cache ${{ matrix.python-version }} ${{ matrix.platform }}"
run: >
breeze prod-image build
--builder airflow_cache
--install-packages-from-context
--run-in-parallel
--airflow-constraints-mode constraints-source-providers
--prepare-buildx-cache
--platform ${{ matrix.platform }}
env:
COMMIT_SHA: ${{ github.sha }}
- name: "Push PROD latest image ${{ matrix.platform }}"
run: >
breeze prod-image build --tag-as-latest --install-packages-from-context
--push --run-in-parallel --platform ${{ matrix.platform }}
env:
DEBUG_RESOURCES: ${{needs.build-info.outputs.debug-resources}}
COMMIT_SHA: ${{ github.sha }}
if: matrix.platform == 'linux/amd64'
- name: "Stop ARM instance"
run: ./scripts/ci/images/ci_stop_arm_instance.sh
if: always() && matrix.platform == 'linux/arm64'
# This is only a check if ARM images are successfully building when committer runs PR from
# Apache repository. This is needed in case you want to fix failing cache job in "canary" run
# There is no point in running this one in "canary" run, because the above step is doing the
# same build anyway.
build-ci-arm-images:
timeout-minutes: 50
name: >
Build CI ARM images
${{needs.build-info.outputs.all-python-versions-list-as-string}}
runs-on: ${{fromJSON(needs.build-info.outputs.runs-on)}}
needs:
- build-info
- build-docs
- spellcheck-docs
- static-checks
- tests-sqlite
- tests-mysql
- tests-postgres
- tests-no-db
- tests-integration-postgres
- tests-integration-mysql
env:
DEFAULT_BRANCH: ${{ needs.build-info.outputs.default-branch }}
DEFAULT_CONSTRAINTS_BRANCH: ${{ needs.build-info.outputs.default-constraints-branch }}
RUNS_ON: "${{needs.build-info.outputs.runs-on}}"
# Force more parallelism for build even on small instances
PARALLELISM: 6
if: >
needs.build-info.outputs.in-workflow-build == 'true' &&
needs.build-info.outputs.canary-run != 'true'
steps:
- name: Cleanup repo
run: docker run -v "${GITHUB_WORKSPACE}:/workspace" -u 0:0 bash -c "rm -rf /workspace/*"
- uses: actions/checkout@v4
with:
ref: ${{ needs.build-info.outputs.targetCommitSha }}
persist-credentials: false
- name: "Install Breeze"
uses: ./.github/actions/breeze
- name: "Start ARM instance"
run: ./scripts/ci/images/ci_start_arm_instance_and_connect_to_docker.sh
- name: >
Build CI ARM images ${{ env.IMAGE_TAG }}
${{needs.build-info.outputs.all-python-versions-list-as-string}}:${{env.IMAGE_TAG}}
run: >
breeze ci-image build --run-in-parallel --builder airflow_cache --platform "linux/arm64"
env:
UPGRADE_TO_NEWER_DEPENDENCIES: ${{ needs.build-info.outputs.upgrade-to-newer-dependencies }}
DOCKER_CACHE: ${{ needs.build-info.outputs.cache-directive }}
PYTHON_VERSIONS: ${{needs.build-info.outputs.all-python-versions-list-as-string}}
DEBUG_RESOURCES: ${{needs.build-info.outputs.debug-resources}}
COMMIT_SHA: ${{ github.sha }}
- name: "Stop ARM instance"
run: ./scripts/ci/images/ci_stop_arm_instance.sh
if: always()