From f0a15012ea72bc618d6a37146611807fead5e825 Mon Sep 17 00:00:00 2001 From: pulpbot Date: Sun, 18 Dec 2022 02:40:51 +0000 Subject: [PATCH] Update CI files [noissue] --- .ci/ansible/Containerfile.j2 | 6 +++- .ci/ansible/settings.py.j2 | 2 +- .ci/scripts/calc_deps_lowerbounds.py | 40 +++++++++++++++++++++ .github/template_gitref | 2 +- .github/workflows/ci.yml | 6 +++- .github/workflows/nightly.yml | 1 + .github/workflows/release.yml | 5 +-- .github/workflows/scripts/before_install.sh | 5 +++ .github/workflows/scripts/before_script.sh | 2 +- .github/workflows/scripts/install.sh | 1 + .github/workflows/scripts/release.py | 6 ++-- .github/workflows/scripts/script.sh | 3 +- .github/workflows/update_ci.yml | 2 +- lint_requirements.txt | 7 ++++ 14 files changed, 76 insertions(+), 12 deletions(-) create mode 100755 .ci/scripts/calc_deps_lowerbounds.py diff --git a/.ci/ansible/Containerfile.j2 b/.ci/ansible/Containerfile.j2 index d61722b..47c3d02 100644 --- a/.ci/ansible/Containerfile.j2 +++ b/.ci/ansible/Containerfile.j2 @@ -27,7 +27,11 @@ RUN pip3 install \ {{ " " }}"{{ item.source }}" {%- endfor %} -RUN mkdir -p /etc/nginx/pulp/ +USER pulp:pulp +RUN PULP_STATIC_ROOT=/var/lib/operator/static/ PULP_CONTENT_ORIGIN=localhost \ + /usr/local/bin/pulpcore-manager collectstatic --clear --noinput --link +USER root:root + {% for item in plugins %} RUN export plugin_path="$(pip3 show {{ item.name }} | sed -n -e 's/Location: //p')/{{ item.name }}" && \ ln $plugin_path/app/webserver_snippets/nginx.conf /etc/nginx/pulp/{{ item.name }}.conf || true diff --git a/.ci/ansible/settings.py.j2 b/.ci/ansible/settings.py.j2 index 16d3a3a..2dad671 100644 --- a/.ci/ansible/settings.py.j2 +++ b/.ci/ansible/settings.py.j2 @@ -8,7 +8,7 @@ TOKEN_SIGNATURE_ALGORITHM = "ES256" CACHE_ENABLED = True REDIS_HOST = "localhost" REDIS_PORT = 6379 -TELEMETRY = False +ANALYTICS = False {% if api_root is defined %} API_ROOT = {{ api_root | repr }} diff --git a/.ci/scripts/calc_deps_lowerbounds.py b/.ci/scripts/calc_deps_lowerbounds.py new file mode 100755 index 0000000..204eac5 --- /dev/null +++ b/.ci/scripts/calc_deps_lowerbounds.py @@ -0,0 +1,40 @@ +# WARNING: DO NOT EDIT! +# +# This file was generated by plugin_template, and is managed by it. Please use +# './plugin-template --github pulp_cookbook' to update this file. +# +# For more info visit https://github.com/pulp/plugin_template + +from packaging.requirements import Requirement + + +# We install these from source, but it would be really handy if we could test pulpcore +# compatibility that way too. +EXCEPTIONS = [ + "pulpcore", +] + + +def main(): + """Calculate the lower bound of dependencies where possible.""" + with open("requirements.txt") as req_file: + for line in req_file: + try: + requirement = Requirement(line) + except ValueError: + print(line.strip()) + else: + if requirement.name.replace("-", "_") in EXCEPTIONS: + print(line.strip()) + else: + for spec in requirement.specifier: + if spec.operator == ">=": + min_version = str(spec)[2:] + print(f"{requirement.name}=={min_version}") + break + else: + print(line.strip()) + + +if __name__ == "__main__": + main() diff --git a/.github/template_gitref b/.github/template_gitref index e184d58..67043e0 100644 --- a/.github/template_gitref +++ b/.github/template_gitref @@ -1 +1 @@ -2021.08.26-179-g8af2847 +2021.08.26-190-ge8465b5 diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index a557ed1..050e565 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -86,10 +86,12 @@ jobs: - TEST: docs - TEST: azure - TEST: s3 + - TEST: lowerbounds outputs: deprecations-pulp: ${{ steps.deprecations.outputs.deprecations-pulp }} deprecations-azure: ${{ steps.deprecations.outputs.deprecations-azure }} deprecations-s3: ${{ steps.deprecations.outputs.deprecations-s3 }} + deprecations-lowerbounds: ${{ steps.deprecations.outputs.deprecations-lowerbounds }} steps: - uses: actions/checkout@v3 @@ -183,7 +185,7 @@ jobs: - name: Extract Deprecations from Logs id: deprecations - run: echo "::set-output name=deprecations-${{ matrix.env.TEST }}::$(docker logs pulp 2>&1 | grep -i pulpcore.deprecation | base64 -w 0)" + run: echo deprecations-${{ matrix.env.TEST }}=$(docker logs pulp 2>&1 | grep -i pulpcore.deprecation | base64 -w 0) >> $GITHUB_OUTPUT - name: Logs if: always() @@ -206,11 +208,13 @@ jobs: test -z "${{ needs.test.outputs.deprecations-pulp }}" test -z "${{ needs.test.outputs.deprecations-azure }}" test -z "${{ needs.test.outputs.deprecations-s3 }}" + test -z "${{ needs.test.outputs.deprecations-lowerbounds }}" - name: Print deprecations if: failure() run: | echo "${{ needs.test.outputs.deprecations-pulp }}" | base64 -d echo "${{ needs.test.outputs.deprecations-azure }}" | base64 -d echo "${{ needs.test.outputs.deprecations-s3 }}" | base64 -d + echo "${{ needs.test.outputs.deprecations-lowerbounds }}" | base64 -d diff --git a/.github/workflows/nightly.yml b/.github/workflows/nightly.yml index 58aec21..6093610 100644 --- a/.github/workflows/nightly.yml +++ b/.github/workflows/nightly.yml @@ -27,6 +27,7 @@ jobs: - TEST: azure - TEST: s3 - TEST: generate-bindings + - TEST: lowerbounds steps: - uses: actions/checkout@v3 diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 54fa26e..71bf09f 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -43,7 +43,7 @@ jobs: - name: Install python dependencies run: | echo ::group::PYDEPS - pip install bandersnatch bump2version gitpython python-redmine towncrier==19.9.0 wheel + pip install packaging~=21.3 bandersnatch bump2version gitpython python-redmine towncrier==19.9.0 wheel echo ::endgroup:: - name: Configure Git with pulpbot name and email @@ -85,6 +85,7 @@ jobs: - TEST: azure - TEST: s3 - TEST: generate-bindings + - TEST: lowerbounds steps: - uses: actions/download-artifact@v3 @@ -292,7 +293,7 @@ jobs: - name: Install python dependencies run: | echo ::group::PYDEPS - pip install gitpython python-redmine requests packaging tweepy + pip install gitpython python-redmine requests packaging~=21.3 tweepy echo ::endgroup:: - name: Push branch and tag to GitHub diff --git a/.github/workflows/scripts/before_install.sh b/.github/workflows/scripts/before_install.sh index 136d38a..3639594 100755 --- a/.github/workflows/scripts/before_install.sh +++ b/.github/workflows/scripts/before_install.sh @@ -120,6 +120,11 @@ fi cd pulp_cookbook +if [[ "$TEST" = "lowerbounds" ]]; then + python3 .ci/scripts/calc_deps_lowerbounds.py > lowerbounds_requirements.txt + mv lowerbounds_requirements.txt requirements.txt +fi + if [ -f $POST_BEFORE_INSTALL ]; then source $POST_BEFORE_INSTALL fi diff --git a/.github/workflows/scripts/before_script.sh b/.github/workflows/scripts/before_script.sh index 3e2828f..6a99b3a 100755 --- a/.github/workflows/scripts/before_script.sh +++ b/.github/workflows/scripts/before_script.sh @@ -33,7 +33,7 @@ tail -v -n +1 .ci/ansible/settings/settings.* ~/.config/pulp_smash/settings.json cmd_prefix bash -c "echo '%wheel ALL=(ALL) NOPASSWD: ALL' > /etc/sudoers.d/nopasswd" cmd_prefix bash -c "usermod -a -G wheel pulp" -SCENARIOS=("pulp" "performance" "azure" "s3" "stream" "plugin-from-pypi" "generate-bindings") +SCENARIOS=("pulp" "performance" "azure" "s3" "stream" "plugin-from-pypi" "generate-bindings" "lowerbounds") if [[ " ${SCENARIOS[*]} " =~ " ${TEST} " ]]; then # Many functional tests require these cmd_prefix dnf install -yq lsof which dnf-plugins-core diff --git a/.github/workflows/scripts/install.sh b/.github/workflows/scripts/install.sh index 2f0a328..f1cdc55 100755 --- a/.github/workflows/scripts/install.sh +++ b/.github/workflows/scripts/install.sh @@ -134,6 +134,7 @@ ansible-playbook start_container.yaml chmod 777 ~/.config/pulp_smash/ chmod 666 ~/.config/pulp_smash/settings.json sudo chown -R 700:700 ~runner/.config + echo ::group::SSL # Copy pulp CA sudo docker cp pulp:/etc/pulp/certs/pulp_webserver.crt /usr/local/share/ca-certificates/pulp_webserver.crt diff --git a/.github/workflows/scripts/release.py b/.github/workflows/scripts/release.py index 5fee545..ed9c77f 100755 --- a/.github/workflows/scripts/release.py +++ b/.github/workflows/scripts/release.py @@ -122,8 +122,8 @@ def create_tag_and_build_package(repo, desired_tag, commit_sha, plugin_path): tag = existing_tag else: raise RuntimeError( - "The '{desired_tag}' tag already exists, but the commit sha does not match " - "'{commit_sha}'." + f"The '{desired_tag}' tag already exists, but the commit sha does not match " + f"'{commit_sha}'." ) # Create a tag if one does not exist @@ -138,7 +138,7 @@ def create_tag_and_build_package(repo, desired_tag, commit_sha, plugin_path): loop = asyncio.get_event_loop() # noqa # fmt: off package_found = asyncio.run( - get_package_from_pypi("pulp-cookbook=={tag.name}", plugin_path) + get_package_from_pypi(f"pulp-cookbook=={tag.name}", plugin_path) ) # noqa # fmt: on if not package_found: diff --git a/.github/workflows/scripts/script.sh b/.github/workflows/scripts/script.sh index 5d879c2..10aedc7 100755 --- a/.github/workflows/scripts/script.sh +++ b/.github/workflows/scripts/script.sh @@ -65,7 +65,8 @@ echo "machine pulp login admin password password " | cmd_user_stdin_prefix bash -c "cat >> ~pulp/.netrc" -cmd_user_stdin_prefix bash -c "chmod og-rw ~pulp/.netrc" +# Some commands like ansible-galaxy specifically require 600 +cmd_user_stdin_prefix bash -c "chmod 600 ~pulp/.netrc" cat unittest_requirements.txt | cmd_stdin_prefix bash -c "cat > /tmp/unittest_requirements.txt" cat functest_requirements.txt | cmd_stdin_prefix bash -c "cat > /tmp/functest_requirements.txt" diff --git a/.github/workflows/update_ci.yml b/.github/workflows/update_ci.yml index c452695..729433a 100644 --- a/.github/workflows/update_ci.yml +++ b/.github/workflows/update_ci.yml @@ -52,7 +52,7 @@ jobs: - name: Set short_ref id: vars - run: echo ::set-output name=short_ref::${GITHUB_REF#refs/*/} + run: echo short_ref=${GITHUB_REF#refs/*/} >> $GITHUB_OUTPUT - name: Run update run: | diff --git a/lint_requirements.txt b/lint_requirements.txt index 2ed6227..d1908ac 100644 --- a/lint_requirements.txt +++ b/lint_requirements.txt @@ -1,3 +1,10 @@ +# WARNING: DO NOT EDIT! +# +# This file was generated by plugin_template, and is managed by it. Please use +# './plugin-template --github pulp_cookbook' to update this file. +# +# For more info visit https://github.com/pulp/plugin_template + # python packages handy for developers, but not required by pulp black check-manifest