Merge pull request #65553 from s0undt3ch/hotfix/merge-forward

[master] Merge 3006.x into master
saltstack · Dec 7, 2023 · c9afe46 · c9afe46
2 parents fc65161 + 0401d58
commit c9afe46
Show file tree

Hide file tree

Showing 375 changed files with 21,988 additions and 10,737 deletions.
diff --git a/.github/actionlint.yaml b/.github/actionlint.yaml
@@ -11,3 +11,4 @@ self-hosted-runner:
     - repo-release
     - medium
     - large
+    - macos-13-xlarge
diff --git a/.github/actions/build-onedir-deps/action.yml b/.github/actions/build-onedir-deps/action.yml
@@ -39,7 +39,7 @@ runs:
 
     - name: Cache Deps Onedir Package Directory
       id: onedir-pkg-cache
-      uses: actions/cache@v3
+      uses: actions/cache@v3.3.1
       with:
         path: artifacts/${{ inputs.package-name }}
         key: >

diff --git a/.github/actions/build-onedir-salt/action.yml b/.github/actions/build-onedir-salt/action.yml
@@ -43,7 +43,7 @@ runs:
 
     - name: Download Cached Deps Onedir Package Directory
       id: onedir-bare-cache
-      uses: actions/cache@v3
+      uses: actions/cache@v3.3.1
       with:
         path: artifacts/${{ inputs.package-name }}
         key: >

diff --git a/.github/actions/cached-virtualenv/action.yml b/.github/actions/cached-virtualenv/action.yml
@@ -42,30 +42,52 @@ runs:
       run: |
         echo "cache-key=${{ inputs.cache-seed }}|${{ runner.os }}|${{ runner.arch }}|cached-venv|${{ steps.get-python-version.outputs.version }}|${{ inputs.name }}" >> "${GITHUB_OUTPUT}"
 
+    - name: Define VirtualEnv path
+      shell: bash
+      id: virtualenv-path
+      run: |
+        cd ${{ github.workspace }} > /dev/null 2>&1 || true
+        VENVS_PATH=$(echo ".venvs/py${{ steps.get-python-version.outputs.version }}" | python3 -c 'import sys, pathlib; sys.stdout.write(pathlib.Path.cwd().joinpath(sys.stdin.read()).as_posix())')
+        echo "venvs-path=$VENVS_PATH" | tee -a "$GITHUB_OUTPUT"
+        VENV_PATH=$(echo ".venvs/py${{ steps.get-python-version.outputs.version }}/${{ inputs.name }}" | python3 -c 'import sys, pathlib; sys.stdout.write(pathlib.Path.cwd().joinpath(sys.stdin.read()).as_posix())')
+        echo "venv-path=$VENV_PATH" | tee -a "$GITHUB_OUTPUT"
+
     - name: Cache VirtualEnv
       id: cache-virtualenv
-      uses: actions/cache@v3
+      uses: actions/cache@v3.3.1
       with:
         key: ${{ steps.setup-cache-key.outputs.cache-key }}
-        path: ${{ github.workspace }}/.venvs/py${{ steps.get-python-version.outputs.version }}/${{ inputs.name }}
+        path: ${{ steps.virtualenv-path.outputs.venv-path }}
 
     - name: Create Virtualenv
       shell: bash
       if: ${{ steps.cache-virtualenv.outputs.cache-hit != 'true' }}
       run: |
-        mkdir -p ${{ github.workspace }}/.venvs/py${{ steps.get-python-version.outputs.version }}
-        python3 -m venv --upgrade ${{ github.workspace }}/.venvs/py${{ steps.get-python-version.outputs.version }}/${{ inputs.name }}
+        mkdir -p ${{ steps.virtualenv-path.outputs.venvs-path }}
+        python3 -m venv --upgrade ${{ steps.virtualenv-path.outputs.venv-path }}
 
     - name: Define python executable output
       shell: bash
       id: define-python-executable
       run: |
         shopt -s nocasematch
         if [[ "${{ runner.os }}" =~ "win" ]]; then
-          BIN_DIR="${{ github.workspace }}/.venvs/py${{ steps.get-python-version.outputs.version }}/${{ inputs.name }}/Scripts"
+          BIN_DIR="${{ steps.virtualenv-path.outputs.venv-path }}/Scripts"
+          PY_EXE="$BIN_DIR/python.exe"
         else
-          BIN_DIR="${{ github.workspace }}/.venvs/py${{ steps.get-python-version.outputs.version }}/${{ inputs.name }}/bin"
+          BIN_DIR="${{ steps.virtualenv-path.outputs.venv-path }}/bin"
+          PY_EXE="$BIN_DIR/python3"
+          if [ ! -f "$PY_EXE" ]; then
+            echo "The '${PY_EXE}' binary does not exist. Setting it to '$BIN_DIR/python' ..."
+            PY_EXE="$BIN_DIR/python"
+          fi
+          if [ ! -f "$PY_EXE" ]; then
+            echo "The '${PY_EXE}' binary does not exist. Showing the tree output for '${BIN_DIR}' ..."
+            tree -a "$BIN_DIR"
+            exit 1
+          fi
         fi
         shopt -u nocasematch
-        echo "python-executable=$BIN_DIR/python" >> "${GITHUB_OUTPUT}"
-        echo "${BIN_DIR}" >> "${GITHUB_PATH}"
+        $PY_EXE --version
+        echo "python-executable=$PY_EXE" | tee -a "${GITHUB_OUTPUT}"
+        echo "${BIN_DIR}" | tee -a "${GITHUB_PATH}"
diff --git a/.github/actions/get-python-version/action.yml b/.github/actions/get-python-version/action.yml
@@ -13,19 +13,33 @@ outputs:
     value: ${{ steps.get-python-version.outputs.version }}
   full-version:
     value: ${{ steps.get-python-version.outputs.full-version }}
+  version-sha256sum:
+    value: ${{ steps.get-python-version.outputs.version-sha256sum }}
 
 
 runs:
   using: composite
 
   steps:
 
+    - name: Install System Packages
+      if: ${{ runner.os == 'macOS' }}
+      shell: bash
+      run: |
+        brew install coreutils
+
     - name: Get Python Version
       id: get-python-version
       shell: bash
       run: |
+        echo "Python Binary: ${{ inputs.python-binary }}"
         echo "binary=${{ inputs.python-binary }}" >> "$GITHUB_OUTPUT"
         PY_VERSION=$(${{ inputs.python-binary }} -c "import sys; sys.stdout.write('{}.{}'.format(*sys.version_info))")
+        echo "PY_VERSION=$PY_VERSION"
         echo "version=$PY_VERSION" >> "$GITHUB_OUTPUT"
         PY_FULL_VERSION=$(${{ inputs.python-binary }} -c "import sys; sys.stdout.write('{}.{}.{}'.format(*sys.version_info))")
+        echo "PY_FULL_VERSION=$PY_FULL_VERSION"
         echo "full-version=$PY_FULL_VERSION" >> "$GITHUB_OUTPUT"
+        VERSION_SHA256SUM=$(${{ inputs.python-binary }} --version --version | sha256sum | cut -d ' ' -f 1)
+        echo "VERSION_SHA256SUM=$VERSION_SHA256SUM"
+        echo "version-sha256sum=$VERSION_SHA256SUM" >> "$GITHUB_OUTPUT"
diff --git a/.github/actions/setup-actionlint/action.yml b/.github/actions/setup-actionlint/action.yml
@@ -15,7 +15,7 @@ runs:
   steps:
 
     - name: Cache actionlint Binary
-      uses: actions/cache@v3
+      uses: actions/cache@v3.3.1
       with:
         path: /usr/local/bin/actionlint
         key: ${{ inputs.cache-seed }}|${{ runner.os }}|${{ runner.arch }}|actionlint|${{ inputs.version }}

diff --git a/.github/actions/setup-pre-commit/action.yml b/.github/actions/setup-pre-commit/action.yml
@@ -36,7 +36,7 @@ runs:
         ${{ steps.pre-commit-virtualenv.outputs.python-executable }} -m pip install pre-commit==${{ inputs.version }}
 
     - name: Cache Pre-Commit Hooks
-      uses: actions/cache@v3
+      uses: actions/cache@v3.3.1
       id: pre-commit-hooks-cache
       with:
         key: ${{ steps.pre-commit-virtualenv.outputs.cache-key }}|${{ inputs.version }}|${{ hashFiles('.pre-commit-config.yaml') }}

diff --git a/.github/actions/setup-python-tools-scripts/action.yml b/.github/actions/setup-python-tools-scripts/action.yml
@@ -3,6 +3,10 @@ name: setup-python-tools-scripts
 description: Setup 'python-tools-scripts'
 
 inputs:
+  cache-prefix:
+    required: true
+    type: string
+    description: Seed used to invalidate caches
   cwd:
     type: string
     description: The directory the salt checkout is located in
@@ -29,21 +33,46 @@ runs:
       with:
         python-binary: python3
 
+    - name: Define Cache Hash
+      id: venv-hash
+      shell: bash
+      run: |
+        VENV_NAME_HASH=$(echo "${{ inputs.cache-prefix }}|${{ github.workflow }}|${{
+          steps.get-python-version.outputs.version-sha256sum }}|${{
+          hashFiles('requirements/**/*.txt', 'tools/**/*.py') }}"  | sha256sum | cut -d ' ' -f 1)
+        echo "TOOLS_VIRTUALENV_CACHE_SEED=$VENV_NAME_HASH" | tee -a "${GITHUB_ENV}"
+        echo "venv-hash=$VENV_NAME_HASH" | tee -a "${GITHUB_OUTPUT}"
+
+    - uses: ./.github/actions/cached-virtualenv
+      id: tools-virtualenv
+      with:
+        name: tools.${{ steps.venv-hash.outputs.venv-hash }}
+        cache-seed: tools|${{ steps.venv-hash.outputs.venv-hash }}
+
+    - name: Restore Python Tools Virtualenvs Cache
+      uses: actions/[email protected]
+      with:
+        path: ${{ inputs.cwd }}/.tools-venvs
+        key: ${{ inputs.cache-prefix }}|${{ steps.venv-hash.outputs.venv-hash }}
+
     - name: Install 'python-tools-scripts'
       shell: bash
       working-directory: ${{ inputs.cwd }}
       run: |
-        (python3 -m pip install --help | grep break-system-packages > /dev/null 2>&1) && exitcode=0 || exitcode=1
+        PYTHON_EXE=${{ steps.tools-virtualenv.outputs.python-executable }}
+        (${PYTHON_EXE} -m pip install --help | grep break-system-packages > /dev/null 2>&1) && exitcode=0 || exitcode=1
         if [ $exitcode -eq 0 ]; then
-          python3 -m pip install --break-system-packages -r requirements/static/ci/py${{ steps.get-python-version.outputs.version }}/tools.txt
+          ${PYTHON_EXE} -m pip install --break-system-packages -r requirements/static/ci/py${{ steps.get-python-version.outputs.version }}/tools.txt
         else
-          python3 -m pip install -r requirements/static/ci/py${{ steps.get-python-version.outputs.version }}/tools.txt
+          ${PYTHON_EXE} -m pip install -r requirements/static/ci/py${{ steps.get-python-version.outputs.version }}/tools.txt
         fi
 
     - name: Get 'python-tools-scripts' Version
       id: get-version
       shell: bash
       working-directory: ${{ inputs.cwd }}
       run: |
-        VERSION=$(tools --version)
+        # The first time `tools` runs with newer virtual enviroments we need to disregard the output
+        tools --debug --version
+        VERSION=$(tools --version | tail -n 1)
         echo "version=$VERSION" >> "${GITHUB_OUTPUT}"
diff --git a/.github/actions/setup-relenv/action.yml b/.github/actions/setup-relenv/action.yml
@@ -45,7 +45,7 @@ runs:
         python3 -m pip install relenv==${{ inputs.version }}
 
     - name: Cache Relenv Data Directory
-      uses: actions/cache@v3
+      uses: actions/cache@v3.3.1
       with:
         path: ${{ github.workspace }}/.relenv
         key: ${{ inputs.cache-seed }}|relenv|${{ inputs.version }}|${{ inputs.python-version }}|${{ inputs.platform }}|${{ inputs.arch }}

diff --git a/.github/actions/setup-shellcheck/action.yml b/.github/actions/setup-shellcheck/action.yml
@@ -15,7 +15,7 @@ runs:
   steps:
 
     - name: Cache shellcheck Binary
-      uses: actions/cache@v3
+      uses: actions/cache@v3.3.1
       with:
         path: /usr/local/bin/shellcheck
         key: ${{ inputs.cache-seed }}|${{ runner.os }}|${{ runner.arch }}|shellcheck|${{ inputs.version }}

diff --git a/.github/workflows/build-deb-packages.yml b/.github/workflows/build-deb-packages.yml
@@ -20,6 +20,10 @@ on:
         required: true
         type: string
         description: The backend to build the packages with
+      cache-prefix:
+        required: true
+        type: string
+        description: Seed used to invalidate caches
 
 env:
   COLUMNS: 190
@@ -75,6 +79,7 @@ jobs:
         uses: ./.github/actions/setup-python-tools-scripts
         with:
           cwd: pkgs/checkout/
+          cache-prefix: ${{ inputs.cache-prefix }}
 
       - name: Setup Salt Version
         id: setup-salt-version

diff --git a/.github/workflows/build-deps-ci-action-macos.yml b/.github/workflows/build-deps-ci-action-macos.yml
@@ -68,7 +68,7 @@ jobs:
 
       - name: Cache nox.${{ inputs.distro-slug }}.tar.* for session ${{ inputs.nox-session }}
         id: nox-dependencies-cache
-        uses: actions/cache@v3
+        uses: actions/cache@v3.3.1
         with:
           path: nox.${{ inputs.distro-slug }}.tar.*
           key: ${{ inputs.cache-prefix }}|testrun-deps|${{ inputs.arch }}|${{ inputs.distro-slug }}|${{ inputs.nox-session }}|${{ inputs.python-version }}|${{

diff --git a/.github/workflows/build-deps-ci-action.yml b/.github/workflows/build-deps-ci-action.yml
@@ -72,7 +72,7 @@ jobs:
 
       - name: Cache nox.${{ inputs.distro-slug }}.tar.* for session ${{ inputs.nox-session }}
         id: nox-dependencies-cache
-        uses: actions/cache@v3
+        uses: actions/cache@v3.3.1
         with:
           path: nox.${{ inputs.distro-slug }}.tar.*
           key: ${{ inputs.cache-prefix }}|testrun-deps|${{ inputs.arch }}|${{ inputs.distro-slug }}|${{ inputs.nox-session }}|${{ inputs.python-version }}|${{
@@ -102,6 +102,8 @@ jobs:
       - name: Setup Python Tools Scripts
         if: steps.nox-dependencies-cache.outputs.cache-hit != 'true'
         uses: ./.github/actions/setup-python-tools-scripts
+        with:
+          cache-prefix: ${{ inputs.cache-prefix }}-build-deps-ci
 
       - name: Get Salt Project GitHub Actions Bot Environment
         if: steps.nox-dependencies-cache.outputs.cache-hit != 'true'

diff --git a/.github/workflows/build-deps-onedir-linux.yml b/.github/workflows/build-deps-onedir-linux.yml
@@ -59,6 +59,11 @@ jobs:
 
       - uses: actions/checkout@v4
 
+      - name: Setup Python Tools Scripts
+        uses: ./.github/actions/setup-python-tools-scripts
+        with:
+          cache-prefix: ${{ inputs.cache-seed }}-build-deps-linux-${{ matrix.arch }}
+
       - name: Setup Relenv
         id: setup-relenv
         uses: ./.github/actions/setup-relenv
@@ -69,9 +74,6 @@ jobs:
           cache-seed: ${{ inputs.cache-seed }}
           python-version: ${{ inputs.python-version }}
 
-      - name: Setup Python Tools Scripts
-        uses: ./.github/actions/setup-python-tools-scripts
-
       - name: Install Salt Packaging Dependencies into Relenv Onedir
         uses: ./.github/actions/build-onedir-deps
         with:

diff --git a/.github/workflows/build-deps-onedir-macos.yml b/.github/workflows/build-deps-onedir-macos.yml
@@ -46,7 +46,10 @@ jobs:
       matrix:
         arch:
           - x86_64
-    runs-on: macos-12
+          - aarch64
+    runs-on:
+      - ${{ matrix.arch == 'aarch64' && 'macos-13-xlarge' || 'macos-12' }}
+
     steps:
 
       - name: "Throttle Builds"
@@ -61,19 +64,21 @@ jobs:
         with:
           python-version: "3.10"
 
+      - name: Setup Python Tools Scripts
+        uses: ./.github/actions/setup-python-tools-scripts
+        with:
+          cache-prefix: ${{ inputs.cache-seed }}-build-deps-macos
+
       - name: Setup Relenv
         id: setup-relenv
         uses: ./.github/actions/setup-relenv
         with:
           platform: darwin
-          arch: ${{ matrix.arch }}
+          arch: ${{ matrix.arch == 'aarch64' && 'arm64' || matrix.arch }}
           version: ${{ inputs.relenv-version }}
           cache-seed: ${{ inputs.cache-seed }}
           python-version: ${{ inputs.python-version }}
 
-      - name: Setup Python Tools Scripts
-        uses: ./.github/actions/setup-python-tools-scripts
-
       - name: Install Salt Packaging Dependencies into Relenv Onedir
         uses: ./.github/actions/build-onedir-deps
         with:

diff --git a/.github/workflows/build-deps-onedir-windows.yml b/.github/workflows/build-deps-onedir-windows.yml
@@ -62,6 +62,11 @@ jobs:
         with:
           python-version: "3.10"
 
+      - name: Setup Python Tools Scripts
+        uses: ./.github/actions/setup-python-tools-scripts
+        with:
+          cache-prefix: ${{ inputs.cache-seed }}-build-deps-windows-${{ matrix.arch }}
+
       - name: Setup Relenv
         id: setup-relenv
         uses: ./.github/actions/setup-relenv
@@ -72,9 +77,6 @@ jobs:
           cache-seed: ${{ inputs.cache-seed }}
           python-version: ${{ inputs.python-version }}
 
-      - name: Setup Python Tools Scripts
-        uses: ./.github/actions/setup-python-tools-scripts
-
       - name: Install Salt Packaging Dependencies into Relenv Onedir
         uses: ./.github/actions/build-onedir-deps
         with:

diff --git a/.github/workflows/build-docs.yml b/.github/workflows/build-docs.yml
@@ -56,6 +56,8 @@ jobs:
       - name: Setup Python Tools Scripts
         id: python-tools-scripts
         uses: ./.github/actions/setup-python-tools-scripts
+        with:
+          cache-prefix: ${{ inputs.cache-seed }}
 
       - name: Configure Git
         if: ${{ startsWith(github.event.ref, 'refs/tags') == false }}
@@ -69,7 +71,7 @@ jobs:
 
       - name: Cache Python Tools Docs Virtualenv
         id: tools-venvs-dependencies-cache
-        uses: actions/cache@v3
+        uses: actions/cache@v3.3.1
         with:
           path: .tools-venvs/docs
           key: ${{ inputs.cache-seed }}|${{ github.workflow }}|${{ github.job }}|tools-venvs|${{ steps.python-tools-scripts.outputs.version }}|docs|${{ steps.get-python-version.outputs.version }}|${{ hashFiles('requirements/**/docs.txt') }}