Skip to content

Commit

Permalink
Update CI files
Browse files Browse the repository at this point in the history
  • Loading branch information
pulpbot committed Oct 6, 2024
1 parent 120d90e commit 936386f
Show file tree
Hide file tree
Showing 12 changed files with 111 additions and 77 deletions.
4 changes: 3 additions & 1 deletion .ci/ansible/Containerfile.j2
Original file line number Diff line number Diff line change
Expand Up @@ -9,6 +9,7 @@ ADD ./{{ item.name }} ./{{ item.name }}
# S3 botocore needs to be patched to handle responses from minio during 0-byte uploads
# Hacking botocore (https://github.com/boto/botocore/pull/1990)

# This MUST be the ONLY call to pip install in inside the container.
RUN pip3 install --upgrade pip setuptools wheel && \
rm -rf /root/.cache/pip && \
pip3 install
Expand All @@ -27,7 +28,8 @@ RUN pip3 install --upgrade pip setuptools wheel && \
{{ " " }}-r ./{{ item.name }}/ci_requirements.txt
{%- endif -%}
{%- endfor %}
{{ " " }}-c ./{{ plugins[0].name }}/.ci/assets/ci_constraints.txt && \
{{ " " }}-c ./{{ plugins[0].name }}/.ci/assets/ci_constraints.txt \
pipdeptree && \
rm -rf /root/.cache/pip

{% if pulp_env is defined and pulp_env %}
Expand Down
5 changes: 5 additions & 0 deletions .ci/assets/ci_constraints.txt
Original file line number Diff line number Diff line change
Expand Up @@ -5,3 +5,8 @@ pulpcore>=3.21.30,!=3.23.*,!=3.24.*,!=3.25.*,!=3.26.*,!=3.27.*,!=3.29.*,!=3.30.*

tablib!=3.6.0
# 3.6.0: This release introduced a regression removing the "html" optional dependency.



# Newer version seem to have a conflict around packaging, that pip fails to resolve in time. Remove this when this starts to impose an issue.
pipdeptree<=3.23.1
11 changes: 8 additions & 3 deletions .ci/scripts/check_release.py
Original file line number Diff line number Diff line change
Expand Up @@ -64,6 +64,7 @@ def main():
for branch in branches:
if branch != DEFAULT_BRANCH:
# Check if a Z release is needed
reasons = []
changes = repo.git.ls_tree("-r", "--name-only", f"origin/{branch}", "CHANGES/")
z_changelog = False
for change in changes.split("\n"):
Expand All @@ -76,23 +77,27 @@ def main():
)
elif ext in Z_CHANGELOG_EXTS:
z_changelog = True
if z_changelog:
reasons.append("Backports")

last_tag = repo.git.describe("--tags", "--abbrev=0", f"origin/{branch}")
req_txt_diff = repo.git.diff(
f"{last_tag}", f"origin/{branch}", "--name-only", "--", "requirements.txt"
)
if z_changelog or req_txt_diff:
if req_txt_diff:
reasons.append("requirements.txt")

if reasons:
curr_version = Version(last_tag)
assert curr_version.base_version.startswith(
branch
), "Current-version has to belong to the current branch!"
next_version = Version(f"{branch}.{curr_version.micro + 1}")
reason = "CHANGES" if z_changelog else "requirements.txt"
print(
f"A Z-release is needed for {branch}, "
f"Prev: {last_tag}, "
f"Next: {next_version.base_version}, "
f"Reason: {reason}"
f"Reason: {','.join(reasons)}"
)
releases.append(next_version)
else:
Expand Down
2 changes: 1 addition & 1 deletion .ci/scripts/collect_changes.py
Original file line number Diff line number Diff line change
Expand Up @@ -103,7 +103,7 @@ def main():
for change in main_changes:
fp.write(change[1])

repo.git.commit("-m", "Update Changelog", "-m" "[noissue]", CHANGELOG_FILE)
repo.git.commit("-m", "Update Changelog", CHANGELOG_FILE)


if __name__ == "__main__":
Expand Down
60 changes: 60 additions & 0 deletions .ci/scripts/pr_labels.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,60 @@
#!/bin/env python3

# This script is running with elevated privileges from the main branch against pull requests.

import re
import sys
import tomllib
from pathlib import Path

from git import Repo


def main():
assert len(sys.argv) == 3

with open("pyproject.toml", "rb") as fp:
PYPROJECT_TOML = tomllib.load(fp)
BLOCKING_REGEX = re.compile(r"DRAFT|WIP|NO\s*MERGE|DO\s*NOT\s*MERGE|EXPERIMENT")
ISSUE_REGEX = re.compile(r"(?:fixes|closes)[\s:]+#(\d+)")
CHERRY_PICK_REGEX = re.compile(r"^\s*\(cherry picked from commit [0-9a-f]*\)\s*$")
try:
CHANGELOG_EXTS = {
f".{item['directory']}" for item in PYPROJECT_TOML["tool"]["towncrier"]["type"]
}
except KeyError:
CHANGELOG_EXTS = {".feature", ".bugfix", ".doc", ".removal", ".misc"}

repo = Repo(".")

base_commit = repo.commit(sys.argv[1])
head_commit = repo.commit(sys.argv[2])

pr_commits = list(repo.iter_commits(f"{base_commit}..{head_commit}"))

labels = {
"multi-commit": len(pr_commits) > 1,
"cherry-pick": False,
"no-issue": False,
"no-changelog": False,
"wip": False,
}
for commit in pr_commits:
labels["wip"] |= BLOCKING_REGEX.search(commit.summary) is not None
no_issue = ISSUE_REGEX.search(commit.message, re.IGNORECASE) is None
labels["no-issue"] |= no_issue
cherry_pick = CHERRY_PICK_REGEX.search(commit.message) is not None
labels["cherry-pick"] |= cherry_pick
changelog_snippets = [
k
for k in commit.stats.files
if k.startswith("CHANGES/") and Path(k).suffix in CHANGELOG_EXTS
]
labels["no-changelog"] |= not changelog_snippets

print("ADD_LABELS=" + ",".join((k for k, v in labels.items() if v)))
print("REMOVE_LABELS=" + ",".join((k for k, v in labels.items() if not v)))


if __name__ == "__main__":
main()
11 changes: 0 additions & 11 deletions .ci/scripts/validate_commit_message.py
Original file line number Diff line number Diff line change
Expand Up @@ -13,7 +13,6 @@
import warnings
from github import Github

NO_ISSUE = "[noissue]"
CHANGELOG_EXTS = [".feature", ".bugfix", ".doc", ".removal", ".misc", ".deprecation"]
KEYWORDS = ["fixes", "closes"]

Expand Down Expand Up @@ -60,15 +59,5 @@ def __check_changelog(issue):
for issue in pattern.findall(message):
__check_status(issue)
__check_changelog(issue)
else:
if NO_ISSUE in message:
print("Commit {sha} has no issues but is tagged {tag}.".format(sha=sha[0:7], tag=NO_ISSUE))
elif "Merge" in message and "cherry picked from commit" in message:
pass
else:
sys.exit(
"Error: no attached issues found for {sha}. If this was intentional, add "
" '{tag}' to the commit message.".format(sha=sha[0:7], tag=NO_ISSUE)
)

print("Commit message for {sha} passed.".format(sha=sha[0:7]))
2 changes: 1 addition & 1 deletion .github/template_gitref
Original file line number Diff line number Diff line change
@@ -1 +1 @@
2021.08.26-379-gece49c9
2021.08.26-385-g17472a1
2 changes: 0 additions & 2 deletions .github/workflows/create-branch.yml
Original file line number Diff line number Diff line change
Expand Up @@ -93,10 +93,8 @@ jobs:
branch: minor-version-bump
base: main
title: Bump minor version
body: '[noissue]'
commit-message: |
Bump minor version
[noissue]
delete-branch: true

- name: Push release branch
Expand Down
55 changes: 30 additions & 25 deletions .github/workflows/pr_checks.yml
Original file line number Diff line number Diff line change
Expand Up @@ -6,57 +6,62 @@
# For more info visit https://github.com/pulp/plugin_template

---
name: File PR static checks
name: "File PR static checks"
on:
pull_request_target:
types: [opened, synchronize, reopened]
types: ["opened", "synchronize", "reopened"]

# This workflow runs with elevated permissions.
# Do not even think about running a single bit of code from the PR.
# Static analysis should be fine however.

concurrency:
group: ${{ github.event.pull_request.number }}-${{ github.workflow }}
group: "${{ github.event.pull_request.number }}-${{ github.workflow }}"
cancel-in-progress: true

jobs:
single_commit:
runs-on: ubuntu-latest
name: Label multiple commit PR
apply_labels:
runs-on: "ubuntu-latest"
name: "Label PR"
permissions:
pull-requests: write
pull-requests: "write"
steps:
- uses: "actions/checkout@v4"
with:
fetch-depth: 0
- name: Commit Count Check
- uses: "actions/setup-python@v5"
with:
python-version: "3.11"
- name: "Determine PR labels"
run: |
pip install GitPython==3.1.42
git fetch origin ${{ github.event.pull_request.head.sha }}
echo "COMMIT_COUNT=$(git log --oneline --no-merges origin/${{ github.base_ref }}..${{ github.event.pull_request.head.sha }} | wc -l)" >> "$GITHUB_ENV"
- uses: actions/github-script@v7
python .ci/scripts/pr_labels.py "origin/${{ github.base_ref }}" "${{ github.event.pull_request.head.sha }}" >> "$GITHUB_ENV"
- uses: "actions/github-script@v7"
name: "Apply PR Labels"
with:
script: |
const labelName = "multi-commit";
const { COMMIT_COUNT } = process.env;
const { ADD_LABELS, REMOVE_LABELS } = process.env;
if (COMMIT_COUNT == 1)
{
try {
await github.rest.issues.removeLabel({
issue_number: context.issue.number,
owner: context.repo.owner,
repo: context.repo.repo,
name: labelName,
});
} catch(err) {
if (REMOVE_LABELS.length) {
for await (const labelName of REMOVE_LABELS.split(",")) {
try {
await github.rest.issues.removeLabel({
issue_number: context.issue.number,
owner: context.repo.owner,
repo: context.repo.repo,
name: labelName,
});
} catch(err) {
}
}
}
else
{
if (ADD_LABELS.length) {
await github.rest.issues.addLabels({
issue_number: context.issue.number,
owner: context.repo.owner,
repo: context.repo.repo,
labels: [labelName],
labels: ADD_LABELS.split(","),
});
}
...
2 changes: 1 addition & 1 deletion .github/workflows/scripts/install.sh
Original file line number Diff line number Diff line change
Expand Up @@ -166,5 +166,5 @@ if [[ "$TEST" = "azure" ]]; then
fi

echo ::group::PIP_LIST
cmd_prefix bash -c "pip3 list && pip3 install pipdeptree && pipdeptree"
cmd_prefix bash -c "pip3 list && pipdeptree"
echo ::endgroup::
31 changes: 1 addition & 30 deletions .github/workflows/update_ci.yml
Original file line number Diff line number Diff line change
Expand Up @@ -62,13 +62,8 @@ jobs:
committer: "pulpbot <[email protected]>"
author: "pulpbot <[email protected]>"
title: "Update CI files for branch main"
body: ""
branch: "update-ci/main"
base: "main"
commit-message: |
Update CI files
[noissue]
delete-branch: true
- uses: "actions/checkout@v4"
with:
Expand All @@ -89,13 +84,8 @@ jobs:
committer: "pulpbot <[email protected]>"
author: "pulpbot <[email protected]>"
title: "Update CI files for branch 1.10"
body: ""
branch: "update-ci/1.10"
base: "1.10"
commit-message: |
Update CI files
[noissue]
delete-branch: true
- uses: "actions/checkout@v4"
with:
Expand All @@ -116,13 +106,8 @@ jobs:
committer: "pulpbot <[email protected]>"
author: "pulpbot <[email protected]>"
title: "Update CI files for branch 1.11"
body: ""
branch: "update-ci/1.11"
base: "1.11"
commit-message: |
Update CI files
[noissue]
delete-branch: true
- uses: "actions/checkout@v4"
with:
Expand All @@ -143,13 +128,8 @@ jobs:
committer: "pulpbot <[email protected]>"
author: "pulpbot <[email protected]>"
title: "Update CI files for branch 1.12"
body: ""
branch: "update-ci/1.12"
base: "1.12"
commit-message: |
Update CI files
[noissue]
delete-branch: true
- uses: "actions/checkout@v4"
with:
Expand All @@ -170,13 +150,8 @@ jobs:
committer: "pulpbot <[email protected]>"
author: "pulpbot <[email protected]>"
title: "Update CI files for branch 1.13"
body: ""
branch: "update-ci/1.13"
base: "1.13"
commit-message: |
Update CI files
[noissue]
delete-branch: true
- uses: "actions/checkout@v4"
with:
Expand All @@ -197,11 +172,7 @@ jobs:
committer: "pulpbot <[email protected]>"
author: "pulpbot <[email protected]>"
title: "Update CI files for branch 1.14"
body: ""
branch: "update-ci/1.14"
base: "1.14"
commit-message: |
Update CI files
[noissue]
delete-branch: true
...
3 changes: 1 addition & 2 deletions template_config.yml
Original file line number Diff line number Diff line change
@@ -1,7 +1,7 @@
# This config represents the latest values used when running the plugin-template. Any settings that
# were not present before running plugin-template have been added with their default values.

# generated with [email protected]377-g1ba8f46
# generated with [email protected]385-g17472a1

api_root: /pulp/
black: true
Expand All @@ -26,7 +26,6 @@ flake8_ignore: []
github_org: pulp
latest_release_branch: null
lint_requirements: true
noissue_marker: '[noissue]'
os_required_packages: []
parallel_test_workers: 8
plugin_app_label: file
Expand Down

0 comments on commit 936386f

Please sign in to comment.