From 1f4df9a1e4df3e636732a7da62d4cfcefa970a58 Mon Sep 17 00:00:00 2001 From: Luka Skugor Date: Thu, 22 Aug 2024 16:40:20 +0200 Subject: [PATCH] Service to annotate commits with bazel build information (#761) --- .bazelrc | 10 +- WORKSPACE.bazel | 6 +- poetry.lock | 36 +++++- pyproject.toml | 1 + release-controller/BUILD.bazel | 49 ++++++++- release-controller/commit_annotator.py | 116 ++++++++++++++++++++ release-controller/git_repo.py | 87 ++++++++++++++- release-controller/pytest.py | 5 +- release-controller/release_notes.py | 8 +- release-controller/test_commit_annotator.py | 30 +++++ release-controller/test_release_notes.py | 1 + release-controller/util.py | 11 ++ requirements.txt | 6 + tools/python/py_oci_image.bzl | 2 +- 14 files changed, 346 insertions(+), 22 deletions(-) create mode 100644 release-controller/commit_annotator.py create mode 100644 release-controller/test_commit_annotator.py diff --git a/.bazelrc b/.bazelrc index 232baa250..325f7477c 100644 --- a/.bazelrc +++ b/.bazelrc @@ -35,7 +35,7 @@ build --@rules_rust//:extra_rustc_flags=-Cdebug-assertions=on build --@rules_rust//:extra_rustc_flag=-Dbindings_with_variant_name build --strip=never -# build:dfinity --remote_cache=bazel-remote.idx.dfinity.network +build:dfinity --remote_cache=bazel-remote.idx.dfinity.network # build --remote_cache=grpc://localhost:9092 build --remote_instance_name=default build --google_default_credentials=false @@ -46,14 +46,14 @@ build --remote_timeout=30s # Default is 60s. build:ci --remote_timeout=5m # Default is 60s. build:ci --remote_upload_local_results=true -# build:dfinity --experimental_remote_downloader=bazel-remote.idx.dfinity.network --experimental_remote_downloader_local_fallback +build:dfinity --experimental_remote_downloader=bazel-remote.idx.dfinity.network --experimental_remote_downloader_local_fallback build:local --experimental_remote_downloader= # Does not produce valid JSON. See https://github.com/bazelbuild/bazel/issues/14209 build --execution_log_json_file=bazel-build-log.json -# build:dfinity --bes_results_url=https://dash.idx.dfinity.network/invocation/ -# build:dfinity --bes_backend=bes.idx.dfinity.network +build:dfinity --bes_results_url=https://dash.idx.dfinity.network/invocation/ +build:dfinity --bes_backend=bes.idx.dfinity.network build --bes_timeout=30s # Default is no timeout. build:ci --bes_timeout=180s # Default is no timeout. build:ci --bes_upload_mode=fully_async @@ -72,7 +72,7 @@ build:fmt --aspects=@rules_rust//rust:defs.bzl%rustfmt_aspect build:fmt --output_groups=+rustfmt_checks build --@rules_rust//:rustfmt.toml=//:rustfmt.toml -test --test_output=errors +test --test_output=streamed test --test_env=RUST_BACKTRACE=full test:precommit --build_tests_only diff --git a/WORKSPACE.bazel b/WORKSPACE.bazel index bde3debc9..843c6c425 100644 --- a/WORKSPACE.bazel +++ b/WORKSPACE.bazel @@ -25,9 +25,9 @@ http_archive( ) http_file( - name = "bazelisk_darwin", - sha256 = "9a4b169038a63ebf60a9b4f367b449ab9b484c4ec7d1ef9f6b7a4196dfd50f33", - url = "https://github.com/bazelbuild/bazelisk/releases/download/v1.20.0/bazelisk-darwin", + name = "target_determinator", + sha256 = "65000bba3a5eb1713d93b1e08e33b6fbe5787535664bbc1ba2f4166b0d26d0a1", + url = "https://github.com/bazel-contrib/target-determinator/releases/download/v0.27.0/target-determinator.linux.amd64", ) http_file( diff --git a/poetry.lock b/poetry.lock index d37f90de3..fbc1387d3 100644 --- a/poetry.lock +++ b/poetry.lock @@ -1482,6 +1482,20 @@ files = [ [package.extras] test = ["pytest (>=6)"] +[[package]] +name = "execnet" +version = "2.1.1" +description = "execnet: rapid multi-Python deployment" +optional = false +python-versions = ">=3.8" +files = [ + {file = "execnet-2.1.1-py3-none-any.whl", hash = "sha256:26dee51f1b80cebd6d0ca8e74dd8745419761d3bef34163928cbebbdc4749fdc"}, + {file = "execnet-2.1.1.tar.gz", hash = "sha256:5189b52c6121c24feae288166ab41b32549c7e2348652736540b9e6e7d4e72e3"}, +] + +[package.extras] +testing = ["hatch", "pre-commit", "pytest", "tox"] + [[package]] name = "executing" version = "2.0.1" @@ -4663,6 +4677,26 @@ pytest = ">=6.2.5" [package.extras] dev = ["pre-commit", "pytest-asyncio", "tox"] +[[package]] +name = "pytest-xdist" +version = "3.6.1" +description = "pytest xdist plugin for distributed testing, most importantly across multiple CPUs" +optional = false +python-versions = ">=3.8" +files = [ + {file = "pytest_xdist-3.6.1-py3-none-any.whl", hash = "sha256:9ed4adfb68a016610848639bb7e02c9352d5d9f03d04809919e2dafc3be4cca7"}, + {file = "pytest_xdist-3.6.1.tar.gz", hash = "sha256:ead156a4db231eec769737f57668ef58a2084a34b2e55c4a8fa20d861107300d"}, +] + +[package.dependencies] +execnet = ">=2.1" +pytest = ">=7.0.0" + +[package.extras] +psutil = ["psutil (>=3.0)"] +setproctitle = ["setproctitle"] +testing = ["filelock"] + [[package]] name = "python-dateutil" version = "2.9.0.post0" @@ -6495,4 +6529,4 @@ cffi = ["cffi (>=1.11)"] [metadata] lock-version = "2.0" python-versions = ">=3.10.0,<4" -content-hash = "de5efe1b66539a41f3c0144baffb98cd9f8c452132bb20b9a08226c8fda60032" +content-hash = "5b9272c3bd66db67fd9b47343e4a0ba8cc82c6a72ee3a082329a524cf9acdde3" diff --git a/pyproject.toml b/pyproject.toml index 9ebd3a843..f9d906f29 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -57,6 +57,7 @@ slackblocks = "^1.0.10" aiohttp = "^3.10.3" quart = "^0.19.6" pytest-asyncio = "^0.23.8" +pytest-xdist = "^3.6.1" [tool.poetry.group.dev.dependencies] black = "^24" diff --git a/release-controller/BUILD.bazel b/release-controller/BUILD.bazel index 923a89a42..3de3f8d68 100644 --- a/release-controller/BUILD.bazel +++ b/release-controller/BUILD.bazel @@ -23,6 +23,7 @@ deps = [ dev_deps = [ requirement("httpretty"), + requirement("pytest-xdist"), ] env = { @@ -43,29 +44,73 @@ py_binary( deps = deps, ) +py_binary( + name = "commit_annotator", + main = "commit_annotator.py", + srcs = glob( + ["*.py"], + exclude = [ + "test_*.py", + "pytest.py", + ], + ), + data = [":bazelisk", ":target_determinator"], + env = env, + deps = deps, +) + native_binary( name = "bazelisk", src = select({ "@platforms//os:linux": "@bazelisk_linux//file", - "@platforms//os:macos": "@bazelisk_darwin//file", }), out = "bazelisk", ) +native_binary( + name = "target_determinator", + src = select({ + "@platforms//os:linux": "@target_determinator//file", + }), + out = "target-determinator", +) + +long_tests = [ + "test_commit_annotator.py", +] + py_test( name = "pytest", srcs = ["pytest.py"], - data = glob(["*.py"]) + [":bazelisk"], + data = glob(["*.py"], exclude = long_tests) + [":bazelisk", ":target_determinator"], env = env, tags = ["no-sandbox"], deps = deps + dev_deps, env_inherit = ["HOME"], ) +py_test( + name = "pytest_enormous", + srcs = ["pytest.py"], + main = "pytest.py", + data = long_tests + glob(["*.py"], exclude = ["test*.py"]) + [":bazelisk", ":target_determinator"], + env = env, + tags = ["no-sandbox"], + deps = deps + dev_deps, + env_inherit = ["HOME"], + size = "enormous", +) + py_oci_image( name = "oci_image", base = "@bazel_image_6_5_0", binary = ":release_controller", ) +py_oci_image( + name = "oci_image_commit_annotator", + base = "@bazel_image_6_5_0", + binary = ":commit_annotator", +) + exports_files(["README.md"]) diff --git a/release-controller/commit_annotator.py b/release-controller/commit_annotator.py new file mode 100644 index 000000000..a7c07074a --- /dev/null +++ b/release-controller/commit_annotator.py @@ -0,0 +1,116 @@ +import logging +import os +import subprocess +import sys +import re +from git_repo import GitRepo +from datetime import datetime +from tenacity import retry, stop_after_attempt +from util import bazel_binary + +sys.path.append(os.path.join(os.path.dirname(__file__))) + +GUESTOS_CHANGED_NOTES_NAMESPACE = "guestos-changed" +GUESTOS_TARGETS_NOTES_NAMESPACE = "guestos-targets" +GUESTOS_BAZEL_TARGETS = "//ic-os/guestos/envs/prod:update-img.tar.zst union //ic-os/setupos/envs/prod:disk-img.tar.zst" +CUTOFF_COMMIT = "8646665552677436c8a889ce970857e531fee49b" + + +def release_branch_date(branch: str) -> datetime: + branch_search = re.search(r"rc--(\d{4}-\d{2}-\d{2})", branch, re.IGNORECASE) + if branch_search: + branch_date = branch_search.group(1) + else: + raise Exception(f"branch '{branch}' does not match RC branch format") + return datetime.strptime(branch_date, "%Y-%m-%d") + + +# target-determinator sometimes fails on first few tries +@retry(stop=stop_after_attempt(10)) +def target_determinator(ic_repo: GitRepo, object: str) -> bool: + ic_repo.checkout(object) + target_determinator_binary = "target-determinator" + target_determinator_binary_local = os.path.abspath(os.curdir) + "/release-controller/target-determinator" + if os.path.exists(target_determinator_binary_local): + target_determinator_binary = target_determinator_binary_local + + p = subprocess.run( + [ + target_determinator_binary, + "-before-query-error-behavior=fatal", + f"-bazel={bazel_binary()}", + "--targets", + GUESTOS_BAZEL_TARGETS, + ic_repo.parent(object), + ], + cwd=ic_repo.dir, + check=True, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + ) + output = p.stdout.decode().strip() + logging.info(f"stdout of target determinator for {object}: '{output}'") + logging.info(f"stderr of target determinator for {object}: '{p.stderr.decode().strip()}'") + return output != "" + + +def annotate_object(ic_repo: GitRepo, object: str): + logging.info("annotating {}".format(object)) + ic_repo.checkout(object) + ic_repo.add_note( + GUESTOS_TARGETS_NOTES_NAMESPACE, + object=object, + content="\n".join( + [ + l + for l in subprocess.check_output(["bazel", "query", f"deps({GUESTOS_BAZEL_TARGETS})"], cwd=ic_repo.dir) + .decode() + .splitlines() + if not l.startswith("@") + ] + ), + ) + ic_repo.add_note( + namespace=GUESTOS_CHANGED_NOTES_NAMESPACE, + object=object, + content=str(target_determinator(ic_repo=ic_repo, object=object)), + ) + + +def annotate_branch(ic_repo: GitRepo, branch: str): + commits = [] + current_commit = branch + ic_repo.checkout(branch) + while True: + if current_commit == CUTOFF_COMMIT: + break + if ic_repo.get_note(namespace=GUESTOS_CHANGED_NOTES_NAMESPACE, object=current_commit): + break + + logging.info("will annotate {}".format(current_commit)) + commits.append(current_commit) + current_commit = ic_repo.parent(current_commit) + + # reverse to annotate oldest objects first so that loop can be easily restarted if it breaks + for c in reversed(commits): + annotate_object(ic_repo=ic_repo, object=c) + + +def main(): + ic_repo = GitRepo( + f"https://oauth2:{os.environ['GITHUB_TOKEN']}@github.com/{os.environ['GITHUB_ORG']}/ic.git", + main_branch="master", + ) + while True: + ic_repo.fetch() + for b in ic_repo.branch_list("rc--*"): + if (datetime.now() - release_branch_date(b)).days > 20: + logging.info("skipping branch {}".format(b)) + continue + logging.info("annotating branch {}".format(b)) + annotate_branch(ic_repo, branch=b) + + +if __name__ == "__main__": + logging.basicConfig(stream=sys.stdout, level=logging.INFO) + main() diff --git a/release-controller/git_repo.py b/release-controller/git_repo.py index 197c5301b..15ec7d45f 100644 --- a/release-controller/git_repo.py +++ b/release-controller/git_repo.py @@ -1,6 +1,7 @@ import logging import os import pathlib +import re import subprocess import tempfile import typing @@ -282,7 +283,7 @@ def file_changes_for_commit(self, commit_hash) -> list[FileChange]: def checkout(self, ref: str): """Checkout the given ref.""" subprocess.check_call( - ["git", "reset", "--hard", f"origin/{self.main_branch}"], + ["git", "reset", "--hard"], cwd=self.dir, stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL, @@ -293,6 +294,89 @@ def checkout(self, ref: str): stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL, ) + if ( + subprocess.check_output( + ["git", "branch", "--show-current"], + cwd=self.dir, + ) + .decode() + .strip() + ): + subprocess.check_call( + ["git", "reset", "--hard", f"origin/{ref}"], + cwd=self.dir, + stdout=subprocess.DEVNULL, + stderr=subprocess.DEVNULL, + ) + + def parent(self, object: str) -> str: + return ( + subprocess.check_output( + ["git", "log", "--pretty=%P", "-n", "1", object], + cwd=self.dir, + ) + .decode() + .strip() + ) + + def branch_list(self, pattern) -> typing.List[str]: + return [ + b.strip().removeprefix("origin/") + for b in subprocess.check_output( + ["git", "branch", "-r", "--list", f"origin/{pattern}"], + cwd=self.dir, + ) + .decode() + .splitlines() + ] + + def _fetch_notes(self): + ref = f"refs/notes/*" + subprocess.check_call( + ["git", "fetch", "origin", f"{ref}:{ref}", "-f", "--prune"], + cwd=self.dir, + stdout=subprocess.DEVNULL, + stderr=subprocess.DEVNULL, + ) + + def _push_notes(self, namespace: str): + subprocess.check_call( + ["git", "push", "origin", f"refs/notes/{namespace}", "-f"], + cwd=self.dir, + stdout=subprocess.DEVNULL, + stderr=subprocess.DEVNULL, + ) + + def _notes(self, namespace: str, *args) -> str: + return subprocess.check_output( + ["git", "notes", f"--ref={namespace}", *args], + cwd=self.dir, + ).decode() + + def add_note(self, namespace: str, object: str, content: str): + self._fetch_notes() + with tempfile.TemporaryDirectory() as td: + f = os.path.join(td, "content") + with open(f, "w") as fh: + fh.write(content) + + self._notes(namespace, "add", f"--file={f}", object, "-f") + + self._push_notes(namespace=namespace) + + def get_note(self, namespace: str, object: str) -> typing.Optional[str]: + self._fetch_notes() + if ( + subprocess.check_output( + ["git", "rev-parse", object], + cwd=self.dir, + ) + .decode() + .strip() + not in self._notes(namespace, "list").strip() + ): + return None + return self._notes(namespace, "show", object) # TODO: test @@ -336,6 +420,7 @@ def push_release_tags(repo: GitRepo, release: Release): .decode("utf-8") .strip() .split(" ")[0] + != v.version ) if tag_version == v.version: logging.info("RC %s: tag %s already exists on origin", release.rc_name, tag) diff --git a/release-controller/pytest.py b/release-controller/pytest.py index e78a9e283..45bfc5e86 100644 --- a/release-controller/pytest.py +++ b/release-controller/pytest.py @@ -1,8 +1,7 @@ import os - if __name__ == "__main__": import pytest - dir_path = os.path.dirname(os.path.realpath(__file__)) - raise SystemExit(pytest.main([dir_path])) + dir_path = os.path.dirname(__file__) + raise SystemExit(pytest.main(args=[dir_path, "-vv", "-n=8"], plugins=["xdist"])) diff --git a/release-controller/release_notes.py b/release-controller/release_notes.py index 65e0d4d05..54f64f471 100755 --- a/release-controller/release_notes.py +++ b/release-controller/release_notes.py @@ -10,6 +10,7 @@ import typing from dataclasses import dataclass from git_repo import GitRepo +from util import bazel_binary import markdown @@ -490,13 +491,8 @@ def format_change(change: Change): def bazel_query(ic_repo: GitRepo, query): """Bazel query package for GuestOS.""" - bazel_binary = "bazel" - bazel_binary_local = os.path.abspath(os.curdir) + "/release-controller/bazelisk" - if os.path.exists(bazel_binary_local): - bazel_binary = bazel_binary_local - bazel_query = [ - bazel_binary, + bazel_binary(), "query", query, ] diff --git a/release-controller/test_commit_annotator.py b/release-controller/test_commit_annotator.py new file mode 100644 index 000000000..1351deafb --- /dev/null +++ b/release-controller/test_commit_annotator.py @@ -0,0 +1,30 @@ +import pathlib +import tempfile +from commit_annotator import target_determinator +from git_repo import GitRepo + + +def _test_guestos_changed(object: str, changed: bool): + with tempfile.TemporaryDirectory() as d: + ic_repo = GitRepo("https://github.com/dfinity/ic.git", main_branch="master", repo_cache_dir=pathlib.Path(d)) + assert target_determinator(object=object, ic_repo=ic_repo) == changed + + +def test_guestos_changed__not_guestos_change(): + _test_guestos_changed(object="00dc67f8d", changed=False) + + +def test_guestos_changed__bumped_dependencies(): + _test_guestos_changed(object="2d0835bba", changed=True) + + +def test_guestos_changed__github_dir_changed(): + _test_guestos_changed(object="94fd38099", changed=False) + + +def test_guestos_changed__replica_changed(): + _test_guestos_changed(object="951e895c7", changed=True) + + +def test_guestos_changed__cargo_lock_paths_only(): + _test_guestos_changed(object="5a250cb34", changed=False) diff --git a/release-controller/test_release_notes.py b/release-controller/test_release_notes.py index 3b6e41912..2570db382 100644 --- a/release-controller/test_release_notes.py +++ b/release-controller/test_release_notes.py @@ -3,6 +3,7 @@ import pytest +@pytest.mark.skip(reason="expensive, will be removed") def test_get_change_description_for_commit(): ic_repo = GitRepo("https://github.com/dfinity/ic.git", main_branch="master") # not a guestos change diff --git a/release-controller/util.py b/release-controller/util.py index d9d088c22..da90c0083 100644 --- a/release-controller/util.py +++ b/release-controller/util.py @@ -1,3 +1,14 @@ +import os + + def version_name(rc_name: str, name: str): date = rc_name.removeprefix("rc--") return f"release-{date}-{name}" + + +def bazel_binary(): + bazel_binary = "bazel" + bazel_binary_local = os.path.abspath(os.curdir) + "/release-controller/bazelisk" + if os.path.exists(bazel_binary_local): + bazel_binary = bazel_binary_local + return bazel_binary diff --git a/requirements.txt b/requirements.txt index 3a5c1ab74..f5a277df9 100644 --- a/requirements.txt +++ b/requirements.txt @@ -715,6 +715,9 @@ email-validator==2.2.0 ; python_version >= "3.10" and python_version < "4.0" \ exceptiongroup==1.2.2 ; python_version >= "3.10" and python_version < "3.11" \ --hash=sha256:3111b9d131c238bec2f8f516e123e14ba243563fb135d3fe885990585aa7795b \ --hash=sha256:47c2edf7c6738fafb49fd34290706d1a1a2f4d1c6df275526b62cbb4aa5393cc +execnet==2.1.1 ; python_full_version >= "3.10.0" and python_version < "4" \ + --hash=sha256:26dee51f1b80cebd6d0ca8e74dd8745419761d3bef34163928cbebbdc4749fdc \ + --hash=sha256:5189b52c6121c24feae288166ab41b32549c7e2348652736540b9e6e7d4e72e3 executing==2.0.1 ; python_version >= "3.10" and python_version < "4" \ --hash=sha256:35afe2ce3affba8ee97f2d69927fa823b08b472b7b994e36a52a964b93d16147 \ --hash=sha256:eac49ca94516ccc753f9fb5ce82603156e590b27525a8bc32cce8ae302eb61bc @@ -1908,6 +1911,9 @@ pytest-asyncio==0.23.8 ; python_full_version >= "3.10.0" and python_version < "4 pytest-mock==3.14.0 ; python_full_version >= "3.10.0" and python_version < "4" \ --hash=sha256:0b72c38033392a5f4621342fe11e9219ac11ec9d375f8e2a0c164539e0d70f6f \ --hash=sha256:2719255a1efeceadbc056d6bf3df3d1c5015530fb40cf347c0f9afac88410bd0 +pytest-xdist==3.6.1 ; python_full_version >= "3.10.0" and python_version < "4" \ + --hash=sha256:9ed4adfb68a016610848639bb7e02c9352d5d9f03d04809919e2dafc3be4cca7 \ + --hash=sha256:ead156a4db231eec769737f57668ef58a2084a34b2e55c4a8fa20d861107300d pytest==8.3.2 ; python_full_version >= "3.10.0" and python_version < "4" \ --hash=sha256:4ba08f9ae7dcf84ded419494d229b48d0903ea6407b030eaec46df5e6a73bba5 \ --hash=sha256:c132345d12ce551242c87269de812483f5bcc87cdbb4722e48487ba194f9fdce diff --git a/tools/python/py_oci_image.bzl b/tools/python/py_oci_image.bzl index 7169c20fe..62d3c2b45 100644 --- a/tools/python/py_oci_image.bzl +++ b/tools/python/py_oci_image.bzl @@ -78,7 +78,7 @@ def py_oci_image(name, binary, tars = [], **kwargs): binary_label = native.package_relative_label(binary) oci_push( - name = "push_image", + name = "{}_push".format(name), image = name, repository = "ghcr.io/dfinity/dre/{}".format(binary_label.name), )