Update CI files

[noissue]

.ci/ansible/Containerfile.j2

@@ -2,15 +2,15 @@ FROM {{ ci_base | default("ghcr.io/pulp/pulp-ci-centos:" + pulp_container_tag) }
 
 # Add source directories to container
 {% for item in plugins %}
-{% if item.source.startswith("./") %}
+{% if item.source.startswith("./") or item.ci_requirements | default(false) %}
 ADD {{ item.source }} {{ item.source }}
 {% endif %}
 {% endfor %}
 
 # Install python packages
 # Hacking botocore (https://github.com/boto/botocore/pull/1990)
 
-RUN pip3 install \
+RUN pip3 install
 {%- if stream_test | default(false) -%}
 {{ " " }}django-storages[sftp]
 {%- endif -%}
@@ -20,14 +20,24 @@ RUN pip3 install \
 {%- if azure_test | default(false) -%}
 {{ " " }}django-storages[azure]>=1.12.2
 {%- endif -%}
+{%- if gcp_test | default(false) -%}
+{{ " " }}django-storages[google]>=1.13.2
+{%- endif -%}
 {%- for item in plugins -%}
 {%- if item.name == "pulp-certguard" -%}
 {{ " " }}python-dateutil rhsm
 {%- endif -%}
-{{ " " }}"{{ item.source }}"
+{{ " " }}{{ item.source }}
+{%- if item.ci_requirements | default(false) -%}
+{{ " " }}-r ./{{ item.name }}/ci_requirements.txt
+{%- endif -%}
 {%- endfor %}
 
-RUN mkdir -p /etc/nginx/pulp/
+USER pulp:pulp
+RUN PULP_STATIC_ROOT=/var/lib/operator/static/ PULP_CONTENT_ORIGIN=localhost \
+       /usr/local/bin/pulpcore-manager collectstatic --clear --noinput --link
+USER root:root
+
 {% for item in plugins %}
 RUN export plugin_path="$(pip3 show {{ item.name }} | sed -n -e 's/Location: //p')/{{ item.name }}" && \
     ln $plugin_path/app/webserver_snippets/nginx.conf /etc/nginx/pulp/{{ item.name }}.conf || true

.ci/ansible/settings.py.j2

@@ -8,7 +8,7 @@ TOKEN_SIGNATURE_ALGORITHM = "ES256"
 CACHE_ENABLED = True
 REDIS_HOST = "localhost"
 REDIS_PORT = 6379
-TELEMETRY = False
+ANALYTICS = False
 
 {% if api_root is defined %}
 API_ROOT = {{ api_root | repr }}
@@ -63,3 +63,10 @@ AZURE_OVERWRITE_FILES = True
 AZURE_URL_EXPIRATION_SECS = 120
 AZURE_CONNECTION_STRING = 'DefaultEndpointsProtocol={{ pulp_scheme }};AccountName=devstoreaccount1;AccountKey=Eby8vdM02xNOcqFlqUwJPLlmEtlCDXJ1OUzFT50uSRZ6IFsuFq2UVErCz4I6tq/K1SZFPTOtr/KBHBeksoGMGw==;BlobEndpoint={{ pulp_scheme }}://ci-azurite:10000/devstoreaccount1;'
 {% endif %}
+
+{% if gcp_test | default(false) %}
+DEFAULT_FILE_STORAGE = "storages.backends.gcloud.GoogleCloudStorage"
+MEDIA_ROOT = ""
+GS_BUCKET_NAME = "gcppulp"
+GS_CUSTOM_ENDPOINT = "http://ci-gcp:4443"
+{% endif %}

.ci/assets/release_requirements.txt

@@ -1,4 +1,3 @@
 bump2version
 gitpython
-python-redmine
 towncrier

.ci/scripts/calc_deps_lowerbounds.py

@@ -0,0 +1,30 @@
+# WARNING: DO NOT EDIT!
+#
+# This file was generated by plugin_template, and is managed by it. Please use
+# './plugin-template --github pulp_cookbook' to update this file.
+#
+# For more info visit https://github.com/pulp/plugin_template
+
+from packaging.requirements import Requirement
+
+
+def main():
+    """Calculate the lower bound of dependencies where possible."""
+    with open("requirements.txt") as req_file:
+        for line in req_file:
+            try:
+                requirement = Requirement(line)
+            except ValueError:
+                print(line.strip())
+            else:
+                for spec in requirement.specifier:
+                    if spec.operator == ">=":
+                        min_version = str(spec)[2:]
+                        print(f"{requirement.name}=={min_version}")
+                        break
+                else:
+                    print(line.strip())
+
+
+if __name__ == "__main__":
+    main()

.ci/scripts/check_gettext.sh

@@ -12,7 +12,6 @@ cd "$(dirname "$(realpath -e "$0")")"/../..
 
 set -uv
 
-# check for imports not from pulpcore.plugin. exclude tests
 MATCHES=$(grep -n -r --include \*.py "_(f")
 
 if [ $? -ne 1 ]; then

.ci/scripts/check_requirements.py

@@ -0,0 +1,62 @@
+# WARNING: DO NOT EDIT!
+#
+# This file was generated by plugin_template, and is managed by it. Please use
+# './plugin-template --github pulp_cookbook' to update this file.
+#
+# For more info visit https://github.com/pulp/plugin_template
+
+import warnings
+from pkg_resources import Requirement
+
+
+CHECK_MATRIX = [
+    ("requirements.txt", True, True, True),
+    ("dev_requirements.txt", False, True, False),
+    ("ci_requirements.txt", False, True, True),
+    ("doc_requirements.txt", False, True, False),
+    ("lint_requirements.txt", False, True, True),
+    ("unittest_requirements.txt", False, True, True),
+    ("functest_requirements.txt", False, True, True),
+    ("clitest_requirements.txt", False, True, True),
+]
+
+errors = []
+
+for filename, check_upperbound, check_prereleases, check_r in CHECK_MATRIX:
+    try:
+        with open(filename, "r") as fd:
+            for nr, line in enumerate(fd.readlines()):
+                line = line.strip()
+                if not line or line.startswith("#"):
+                    continue
+                try:
+                    req = Requirement.parse(line)
+                except ValueError:
+                    if line.startswith("git+"):
+                        # The single exception...
+                        if "pulp-smash" not in line:
+                            errors.append(f"{filename}:{nr}: Invalid source requirement: {line}")
+                    elif line.startswith("-r "):
+                        if check_r:
+                            errors.append(f"{filename}:{nr}: Invalid deferred requirement: {line}")
+                    else:
+                        errors.append(f"{filename}:{nr}: Unreadable requirement {line}")
+                else:
+                    if check_prereleases and req.specifier.prereleases:
+                        if req.name != "pulp-cookbook-client":
+                            errors.append(f"{filename}:{nr}: Prerelease versions found in {line}.")
+                    ops = [op for op, ver in req.specs]
+                    spec = str(req.specs)
+                    if "~=" in ops:
+                        warnings.warn(f"{filename}:{nr}: Please avoid using ~= on {req.name}!")
+                    elif "<" not in ops and "<=" not in ops and "==" not in ops:
+                        if check_upperbound:
+                            errors.append(f"{filename}:{nr}: Upper bound missing in {line}.")
+    except FileNotFoundError:
+        # skip this test for plugins that don't use this requirements.txt
+        pass
+
+if errors:
+    print("Dependency issues found:")
+    print("\n".join(errors))
+    exit(1)

.github/template_gitref

@@ -1 +1 @@
-2021.08.26-179-g8af2847
+2021.08.26-202-g6f2c165

.github/workflows/changelog.yml

@@ -26,9 +26,7 @@ jobs:
     steps:
       - uses: actions/checkout@v3
         with:
-          # by default, it uses a depth of 1
-          # this fetches all history so that we can read each commit
-          fetch-depth: 0
+          fetch-depth: 1
 
       - uses: actions/setup-python@v3
         with:

.github/workflows/ci.yml

@@ -10,44 +10,47 @@ name: Cookbook CI
 on: {pull_request: {branches: ['*']}}
 jobs:
 
-
+  ready-to-ship:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v3
+        with:
+          fetch-depth: 0
+      - uses: actions/setup-python@v3
+        with:
+          python-version: "3.8"
+
+      - name: Verify requirements files
+        run: python .ci/scripts/check_requirements.py
   single_commit:
+    runs-on: ubuntu-latest
     name: Assert single commit
     if: github.base_ref == 'main'
     steps:
-    - name: Checkout
-      uses: actions/checkout@v3
-      with:
-        fetch-depth: 30
-    - name: Checkout main
-      run: git fetch origin main
-    - name: create local main branch
-      run: git branch main origin/main
-    - name: Commit Count Check
-      run: test `git log  --oneline --no-merges HEAD ^main  | wc -l ` = 1
-    runs-on: ubuntu-latest
+      - uses: actions/checkout@v3
+        with:
+          fetch-depth: 0
+      - name: Checkout main
+        run: git fetch origin main
+      - name: create local main branch
+        run: git branch main origin/main
+      - name: Commit Count Check
+        run: test `git log  --oneline --no-merges HEAD ^main  | wc -l ` = 1
 
   lint:
     runs-on: ubuntu-latest
-
 
     steps:
       - uses: actions/checkout@v3
         with:
-          # by default, it uses a depth of 1
-          # this fetches all history so that we can read each commit
-          fetch-depth: 0
-
+          fetch-depth: 1
       - uses: actions/setup-python@v3
         with:
           python-version: "3.8"
-
-        # lint_requirements contains tools needed for flake8, etc.
+      # lint_requirements contains tools needed for flake8, etc.
       - name: Install requirements
         run: pip3 install -r lint_requirements.txt
 
-
-
       # run black separately from flake8 to get a diff
       - name: Run black
         run: |
@@ -71,9 +74,6 @@ jobs:
       - name: Check for gettext problems
         run: sh .ci/scripts/check_gettext.sh
 
-      - name: Verify upper bound requirements
-        run: python .ci/scripts/upper_bound.py
-
   test:
     runs-on: ubuntu-latest
     # run only after lint finishes
@@ -86,17 +86,17 @@ jobs:
           - TEST: docs
           - TEST: azure
           - TEST: s3
+          - TEST: lowerbounds
     outputs:
       deprecations-pulp: ${{ steps.deprecations.outputs.deprecations-pulp }}
       deprecations-azure: ${{ steps.deprecations.outputs.deprecations-azure }}
       deprecations-s3: ${{ steps.deprecations.outputs.deprecations-s3 }}
+      deprecations-lowerbounds: ${{ steps.deprecations.outputs.deprecations-lowerbounds }}
 
     steps:
       - uses: actions/checkout@v3
         with:
-          # by default, it uses a depth of 1
-          # this fetches all history so that we can read each commit
-          fetch-depth: 0
+          fetch-depth: 1
 
       - uses: actions/setup-python@v3
         with:
@@ -114,7 +114,6 @@ jobs:
           echo "TEST=${{ matrix.env.TEST }}" >> $GITHUB_ENV
 
       - name: Before Install
-
         run: .github/workflows/scripts/before_install.sh
         shell: bash
         env:
@@ -128,7 +127,6 @@ jobs:
           GITHUB_CONTEXT: ${{ github.event.pull_request.commits_url }}
 
       - name: Install
-
         run: .github/workflows/scripts/install.sh
         shell: bash
         env:
@@ -141,13 +139,7 @@ jobs:
           GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
           GITHUB_CONTEXT: ${{ github.event.pull_request.commits_url }}
 
-      - name: Install Python client
-
-        run: .github/workflows/scripts/install_python_client.sh
-        shell: bash
-
       - name: Before Script
-
         run: .github/workflows/scripts/before_script.sh
         shell: bash
         env:
@@ -168,7 +160,6 @@ jobs:
           SECRETS_CONTEXT: ${{ toJson(secrets) }}
 
       - name: Script
-
         run: .github/workflows/scripts/script.sh
         shell: bash
         env:
@@ -183,7 +174,7 @@ jobs:
 
       - name: Extract Deprecations from Logs
         id: deprecations
-        run: echo "::set-output name=deprecations-${{ matrix.env.TEST }}::$(docker logs pulp 2>&1 | grep -i pulpcore.deprecation | base64 -w 0)"
+        run: echo deprecations-${{ matrix.env.TEST }}=$(docker logs pulp 2>&1 | grep -i pulpcore.deprecation | base64 -w 0) >> $GITHUB_OUTPUT
 
       - name: Logs
         if: always()
@@ -206,11 +197,13 @@ jobs:
           test -z "${{ needs.test.outputs.deprecations-pulp }}"
           test -z "${{ needs.test.outputs.deprecations-azure }}"
           test -z "${{ needs.test.outputs.deprecations-s3 }}"
+          test -z "${{ needs.test.outputs.deprecations-lowerbounds }}"
       - name: Print deprecations
         if: failure()
         run: |
           echo "${{ needs.test.outputs.deprecations-pulp }}" | base64 -d
           echo "${{ needs.test.outputs.deprecations-azure }}" | base64 -d
           echo "${{ needs.test.outputs.deprecations-s3 }}" | base64 -d
+          echo "${{ needs.test.outputs.deprecations-lowerbounds }}" | base64 -d
 
 

.github/workflows/create-branch.yml

@@ -27,9 +27,7 @@ jobs:
     steps:
       - uses: actions/checkout@v3
         with:
-          # by default, it uses a depth of 1
-          # this fetches all history so that we can read each commit
-          fetch-depth: 0
+          fetch-depth: 1
 
       - uses: actions/setup-python@v3
         with:
@@ -42,7 +40,6 @@ jobs:
           echo ::endgroup::
 
       - name: Setting secrets
-
         run: python3 .github/workflows/scripts/secrets.py "$SECRETS_CONTEXT"
         env:
           SECRETS_CONTEXT: ${{ toJson(secrets) }}