From 0e78c38a2c5939a6704cd1c8e441885eec44ecda Mon Sep 17 00:00:00 2001 From: pulpbot Date: Sun, 13 Oct 2024 08:34:41 +0000 Subject: [PATCH] Update CI files --- .ci/ansible/Containerfile.j2 | 4 +- .ci/assets/ci_constraints.txt | 5 +++ .ci/scripts/check_release.py | 11 +++-- .ci/scripts/collect_changes.py | 2 +- .ci/scripts/pr_labels.py | 60 ++++++++++++++++++++++++++ .ci/scripts/validate_commit_message.py | 11 ----- .github/template_gitref | 2 +- .github/workflows/create-branch.yml | 2 - .github/workflows/pr_checks.yml | 55 ++++++++++++----------- .github/workflows/scripts/install.sh | 2 +- .github/workflows/scripts/script.sh | 10 +++-- .github/workflows/update_ci.yml | 31 +------------ MANIFEST.in | 1 + releasing.md | 27 ++++++++++++ template_config.yml | 3 +- 15 files changed, 145 insertions(+), 81 deletions(-) create mode 100755 .ci/scripts/pr_labels.py create mode 100644 releasing.md diff --git a/.ci/ansible/Containerfile.j2 b/.ci/ansible/Containerfile.j2 index afe0b118..e4b86333 100644 --- a/.ci/ansible/Containerfile.j2 +++ b/.ci/ansible/Containerfile.j2 @@ -9,6 +9,7 @@ ADD ./{{ item.name }} ./{{ item.name }} # S3 botocore needs to be patched to handle responses from minio during 0-byte uploads # Hacking botocore (https://github.com/boto/botocore/pull/1990) +# This MUST be the ONLY call to pip install in inside the container. RUN pip3 install --upgrade pip setuptools wheel && \ rm -rf /root/.cache/pip && \ pip3 install @@ -27,7 +28,8 @@ RUN pip3 install --upgrade pip setuptools wheel && \ {{ " " }}-r ./{{ item.name }}/ci_requirements.txt {%- endif -%} {%- endfor %} -{{ " " }}-c ./{{ plugins[0].name }}/.ci/assets/ci_constraints.txt && \ +{{ " " }}-c ./{{ plugins[0].name }}/.ci/assets/ci_constraints.txt \ + pipdeptree && \ rm -rf /root/.cache/pip {% if pulp_env is defined and pulp_env %} diff --git a/.ci/assets/ci_constraints.txt b/.ci/assets/ci_constraints.txt index 2617a408..14b51596 100644 --- a/.ci/assets/ci_constraints.txt +++ b/.ci/assets/ci_constraints.txt @@ -5,3 +5,8 @@ pulpcore>=3.21.30,!=3.23.*,!=3.24.*,!=3.25.*,!=3.26.*,!=3.27.*,!=3.29.*,!=3.30.* tablib!=3.6.0 # 3.6.0: This release introduced a regression removing the "html" optional dependency. + + + +# Newer version seem to have a conflict around packaging, that pip fails to resolve in time. Remove this when this starts to impose an issue. +pipdeptree<=3.23.1 diff --git a/.ci/scripts/check_release.py b/.ci/scripts/check_release.py index 0872feb4..fbe135d2 100755 --- a/.ci/scripts/check_release.py +++ b/.ci/scripts/check_release.py @@ -64,6 +64,7 @@ def main(): for branch in branches: if branch != DEFAULT_BRANCH: # Check if a Z release is needed + reasons = [] changes = repo.git.ls_tree("-r", "--name-only", f"origin/{branch}", "CHANGES/") z_changelog = False for change in changes.split("\n"): @@ -76,23 +77,27 @@ def main(): ) elif ext in Z_CHANGELOG_EXTS: z_changelog = True + if z_changelog: + reasons.append("Backports") last_tag = repo.git.describe("--tags", "--abbrev=0", f"origin/{branch}") req_txt_diff = repo.git.diff( f"{last_tag}", f"origin/{branch}", "--name-only", "--", "requirements.txt" ) - if z_changelog or req_txt_diff: + if req_txt_diff: + reasons.append("requirements.txt") + + if reasons: curr_version = Version(last_tag) assert curr_version.base_version.startswith( branch ), "Current-version has to belong to the current branch!" next_version = Version(f"{branch}.{curr_version.micro + 1}") - reason = "CHANGES" if z_changelog else "requirements.txt" print( f"A Z-release is needed for {branch}, " f"Prev: {last_tag}, " f"Next: {next_version.base_version}, " - f"Reason: {reason}" + f"Reason: {','.join(reasons)}" ) releases.append(next_version) else: diff --git a/.ci/scripts/collect_changes.py b/.ci/scripts/collect_changes.py index d39642c7..f3f03322 100755 --- a/.ci/scripts/collect_changes.py +++ b/.ci/scripts/collect_changes.py @@ -103,7 +103,7 @@ def main(): for change in main_changes: fp.write(change[1]) - repo.git.commit("-m", "Update Changelog", "-m" "[noissue]", CHANGELOG_FILE) + repo.git.commit("-m", "Update Changelog", CHANGELOG_FILE) if __name__ == "__main__": diff --git a/.ci/scripts/pr_labels.py b/.ci/scripts/pr_labels.py new file mode 100755 index 00000000..0c478a21 --- /dev/null +++ b/.ci/scripts/pr_labels.py @@ -0,0 +1,60 @@ +#!/bin/env python3 + +# This script is running with elevated privileges from the main branch against pull requests. + +import re +import sys +import tomllib +from pathlib import Path + +from git import Repo + + +def main(): + assert len(sys.argv) == 3 + + with open("pyproject.toml", "rb") as fp: + PYPROJECT_TOML = tomllib.load(fp) + BLOCKING_REGEX = re.compile(r"DRAFT|WIP|NO\s*MERGE|DO\s*NOT\s*MERGE|EXPERIMENT") + ISSUE_REGEX = re.compile(r"(?:fixes|closes)[\s:]+#(\d+)") + CHERRY_PICK_REGEX = re.compile(r"^\s*\(cherry picked from commit [0-9a-f]*\)\s*$") + try: + CHANGELOG_EXTS = { + f".{item['directory']}" for item in PYPROJECT_TOML["tool"]["towncrier"]["type"] + } + except KeyError: + CHANGELOG_EXTS = {".feature", ".bugfix", ".doc", ".removal", ".misc"} + + repo = Repo(".") + + base_commit = repo.commit(sys.argv[1]) + head_commit = repo.commit(sys.argv[2]) + + pr_commits = list(repo.iter_commits(f"{base_commit}..{head_commit}")) + + labels = { + "multi-commit": len(pr_commits) > 1, + "cherry-pick": False, + "no-issue": False, + "no-changelog": False, + "wip": False, + } + for commit in pr_commits: + labels["wip"] |= BLOCKING_REGEX.search(commit.summary) is not None + no_issue = ISSUE_REGEX.search(commit.message, re.IGNORECASE) is None + labels["no-issue"] |= no_issue + cherry_pick = CHERRY_PICK_REGEX.search(commit.message) is not None + labels["cherry-pick"] |= cherry_pick + changelog_snippets = [ + k + for k in commit.stats.files + if k.startswith("CHANGES/") and Path(k).suffix in CHANGELOG_EXTS + ] + labels["no-changelog"] |= not changelog_snippets + + print("ADD_LABELS=" + ",".join((k for k, v in labels.items() if v))) + print("REMOVE_LABELS=" + ",".join((k for k, v in labels.items() if not v))) + + +if __name__ == "__main__": + main() diff --git a/.ci/scripts/validate_commit_message.py b/.ci/scripts/validate_commit_message.py index 885ee8c9..4b7b30bf 100755 --- a/.ci/scripts/validate_commit_message.py +++ b/.ci/scripts/validate_commit_message.py @@ -13,7 +13,6 @@ import warnings from github import Github -NO_ISSUE = "[noissue]" CHANGELOG_EXTS = [".feature", ".bugfix", ".doc", ".removal", ".misc", ".deprecation"] KEYWORDS = ["fixes", "closes"] @@ -60,15 +59,5 @@ def __check_changelog(issue): for issue in pattern.findall(message): __check_status(issue) __check_changelog(issue) -else: - if NO_ISSUE in message: - print("Commit {sha} has no issues but is tagged {tag}.".format(sha=sha[0:7], tag=NO_ISSUE)) - elif "Merge" in message and "cherry picked from commit" in message: - pass - else: - sys.exit( - "Error: no attached issues found for {sha}. If this was intentional, add " - " '{tag}' to the commit message.".format(sha=sha[0:7], tag=NO_ISSUE) - ) print("Commit message for {sha} passed.".format(sha=sha[0:7])) diff --git a/.github/template_gitref b/.github/template_gitref index 3e378489..4fb36e96 100644 --- a/.github/template_gitref +++ b/.github/template_gitref @@ -1 +1 @@ -2021.08.26-379-gece49c9 +2021.08.26-387-ge627e91 diff --git a/.github/workflows/create-branch.yml b/.github/workflows/create-branch.yml index f6185ab3..dc70bb17 100644 --- a/.github/workflows/create-branch.yml +++ b/.github/workflows/create-branch.yml @@ -93,10 +93,8 @@ jobs: branch: minor-version-bump base: main title: Bump minor version - body: '[noissue]' commit-message: | Bump minor version - [noissue] delete-branch: true - name: Push release branch diff --git a/.github/workflows/pr_checks.yml b/.github/workflows/pr_checks.yml index 846ab131..2a901608 100644 --- a/.github/workflows/pr_checks.yml +++ b/.github/workflows/pr_checks.yml @@ -6,57 +6,62 @@ # For more info visit https://github.com/pulp/plugin_template --- -name: File PR static checks +name: "File PR static checks" on: pull_request_target: - types: [opened, synchronize, reopened] + types: ["opened", "synchronize", "reopened"] # This workflow runs with elevated permissions. # Do not even think about running a single bit of code from the PR. # Static analysis should be fine however. concurrency: - group: ${{ github.event.pull_request.number }}-${{ github.workflow }} + group: "${{ github.event.pull_request.number }}-${{ github.workflow }}" cancel-in-progress: true jobs: - single_commit: - runs-on: ubuntu-latest - name: Label multiple commit PR + apply_labels: + runs-on: "ubuntu-latest" + name: "Label PR" permissions: - pull-requests: write + pull-requests: "write" steps: - uses: "actions/checkout@v4" with: fetch-depth: 0 - - name: Commit Count Check + - uses: "actions/setup-python@v5" + with: + python-version: "3.11" + - name: "Determine PR labels" run: | + pip install GitPython==3.1.42 git fetch origin ${{ github.event.pull_request.head.sha }} - echo "COMMIT_COUNT=$(git log --oneline --no-merges origin/${{ github.base_ref }}..${{ github.event.pull_request.head.sha }} | wc -l)" >> "$GITHUB_ENV" - - uses: actions/github-script@v7 + python .ci/scripts/pr_labels.py "origin/${{ github.base_ref }}" "${{ github.event.pull_request.head.sha }}" >> "$GITHUB_ENV" + - uses: "actions/github-script@v7" + name: "Apply PR Labels" with: script: | - const labelName = "multi-commit"; - const { COMMIT_COUNT } = process.env; + const { ADD_LABELS, REMOVE_LABELS } = process.env; - if (COMMIT_COUNT == 1) - { - try { - await github.rest.issues.removeLabel({ - issue_number: context.issue.number, - owner: context.repo.owner, - repo: context.repo.repo, - name: labelName, - }); - } catch(err) { + if (REMOVE_LABELS.length) { + for await (const labelName of REMOVE_LABELS.split(",")) { + try { + await github.rest.issues.removeLabel({ + issue_number: context.issue.number, + owner: context.repo.owner, + repo: context.repo.repo, + name: labelName, + }); + } catch(err) { + } } } - else - { + if (ADD_LABELS.length) { await github.rest.issues.addLabels({ issue_number: context.issue.number, owner: context.repo.owner, repo: context.repo.repo, - labels: [labelName], + labels: ADD_LABELS.split(","), }); } +... diff --git a/.github/workflows/scripts/install.sh b/.github/workflows/scripts/install.sh index 8f1803df..c55e2593 100755 --- a/.github/workflows/scripts/install.sh +++ b/.github/workflows/scripts/install.sh @@ -166,5 +166,5 @@ if [[ "$TEST" = "azure" ]]; then fi echo ::group::PIP_LIST -cmd_prefix bash -c "pip3 list && pip3 install pipdeptree && pipdeptree" +cmd_prefix bash -c "pip3 list && pipdeptree" echo ::endgroup:: diff --git a/.github/workflows/scripts/script.sh b/.github/workflows/scripts/script.sh index 2d240d25..cb0af947 100755 --- a/.github/workflows/scripts/script.sh +++ b/.github/workflows/scripts/script.sh @@ -46,12 +46,14 @@ pushd ../pulp-openapi-generator # Workaround: Domains are not supported by the published bindings. # Sadly: Different pulpcore-versions aren't either... - # So we exclude the prebuilt ones only for domains disabled. - if [ "$(jq -r '.domain_enabled' <<<"${REPORTED_STATUS}")" = "true" ] || [ "$(jq -r '.online_workers[0].pulp_href|startswith("/pulp/api/v3/")' <<< "${REPORTED_STATUS}")" = "false" ] + # * In the 'pulp' scenario we use the published/prebuilt bindings, so we can test it. + # * In other scenarios we generate new bindings from server spec, so we have a more + # reliable client. + if [ "$TEST" = "pulp" ] then - BUILT_CLIENTS="" - else BUILT_CLIENTS=" file " + else + BUILT_CLIENTS="" fi for ITEM in $(jq -r '.versions[] | tojson' <<<"${REPORTED_STATUS}") diff --git a/.github/workflows/update_ci.yml b/.github/workflows/update_ci.yml index f32a0ef8..e24ec3ef 100644 --- a/.github/workflows/update_ci.yml +++ b/.github/workflows/update_ci.yml @@ -62,13 +62,8 @@ jobs: committer: "pulpbot " author: "pulpbot " title: "Update CI files for branch main" - body: "" branch: "update-ci/main" base: "main" - commit-message: | - Update CI files - - [noissue] delete-branch: true - uses: "actions/checkout@v4" with: @@ -89,13 +84,8 @@ jobs: committer: "pulpbot " author: "pulpbot " title: "Update CI files for branch 1.10" - body: "" branch: "update-ci/1.10" base: "1.10" - commit-message: | - Update CI files - - [noissue] delete-branch: true - uses: "actions/checkout@v4" with: @@ -116,13 +106,8 @@ jobs: committer: "pulpbot " author: "pulpbot " title: "Update CI files for branch 1.11" - body: "" branch: "update-ci/1.11" base: "1.11" - commit-message: | - Update CI files - - [noissue] delete-branch: true - uses: "actions/checkout@v4" with: @@ -143,13 +128,8 @@ jobs: committer: "pulpbot " author: "pulpbot " title: "Update CI files for branch 1.12" - body: "" branch: "update-ci/1.12" base: "1.12" - commit-message: | - Update CI files - - [noissue] delete-branch: true - uses: "actions/checkout@v4" with: @@ -170,13 +150,8 @@ jobs: committer: "pulpbot " author: "pulpbot " title: "Update CI files for branch 1.13" - body: "" branch: "update-ci/1.13" base: "1.13" - commit-message: | - Update CI files - - [noissue] delete-branch: true - uses: "actions/checkout@v4" with: @@ -197,11 +172,7 @@ jobs: committer: "pulpbot " author: "pulpbot " title: "Update CI files for branch 1.14" - body: "" branch: "update-ci/1.14" base: "1.14" - commit-message: | - Update CI files - - [noissue] delete-branch: true +... diff --git a/MANIFEST.in b/MANIFEST.in index 69515046..53db324a 100644 --- a/MANIFEST.in +++ b/MANIFEST.in @@ -9,3 +9,4 @@ include functest_requirements.txt include test_requirements.txt include unittest_requirements.txt recursive-exclude pulp_file/tests/functional * +exclude releasing.md diff --git a/releasing.md b/releasing.md new file mode 100644 index 00000000..3a718e86 --- /dev/null +++ b/releasing.md @@ -0,0 +1,27 @@ +[//]: # "WARNING: DO NOT EDIT!" +[//]: # "" +[//]: # "This file was generated by plugin_template, and is managed by it. Please use" +[//]: # "'./plugin-template --github pulp_file' to update this file." +[//]: # "" +[//]: # "For more info visit https://github.com/pulp/plugin_template" +# Releasing (For Internal Use) + +This document outlines the steps to perform a release. + +### Determine if a Release is Required +- Run the release checker script: + ``` + python3 .ci/scripts/check_release.py + ``` + +### Create a New Y-branch (e.g., 3.23) +- If a new minor version (Y) is needed, trigger a [Create New Release Branch](https://github.com/pulp/pulp_file/actions/workflows/create-branch.yml) job via the GitHub Actions. +- Look for the "Bump minor version" pull request and merge it. + +### Release a New Z-version (Patch Release) (e.g., 3.23.1, 3.22.12) +- Trigger a [Release Pipeline](https://github.com/pulp/pulp_file/actions/workflows/release.yml) job by specifying the release branch (X.Y) via the GitHub Actions. + +### Final Steps (Optional but Recommended) +- Ensure the new version appears on PyPI. +- Verify that the changelog has been updated by looking for the "Update Changelog" pull request. +- Post a brief announcement about the new release on the [Pulp Discourse](https://discourse.pulpproject.org/). diff --git a/template_config.yml b/template_config.yml index 56584be6..a492525d 100644 --- a/template_config.yml +++ b/template_config.yml @@ -1,7 +1,7 @@ # This config represents the latest values used when running the plugin-template. Any settings that # were not present before running plugin-template have been added with their default values. -# generated with plugin_template@2021.08.26-377-g1ba8f46 +# generated with plugin_template@2021.08.26-387-ge627e91 api_root: /pulp/ black: true @@ -26,7 +26,6 @@ flake8_ignore: [] github_org: pulp latest_release_branch: null lint_requirements: true -noissue_marker: '[noissue]' os_required_packages: [] parallel_test_workers: 8 plugin_app_label: file