diff --git a/.github/workflows/cd.yaml b/.github/workflows/cd.yaml index 786220f3..bdcd34b3 100644 --- a/.github/workflows/cd.yaml +++ b/.github/workflows/cd.yaml @@ -2,7 +2,8 @@ name: CD on: release: - types: [published] + types: + [published] # trigger only on new release jobs: @@ -44,7 +45,7 @@ jobs: name: Verify package version vs tag version # package version must be same with tag version run: | - PKG_VER="$(python setup.py --version)" + PKG_VER="$(grep -oP 'version = "\K[^"]+' pyproject.toml)" echo "Package version is $PKG_VER" >&2 echo "Tag version is ${{ steps.get_version_tag.outputs.version }}" >&2 if [ "$PKG_VER" != "${{ steps.get_version_tag.outputs.version }}" ]; then diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index ab5aca06..bf4b0ddb 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -13,6 +13,7 @@ repos: name: "ensure hooks are up to date" language: python additional_dependencies: + - "packaging" - "typer" fail_fast: true always_run: true diff --git a/.static_files b/.static_files index 2d923bb5..d8a011c5 100644 --- a/.static_files +++ b/.static_files @@ -25,6 +25,7 @@ scripts/update_openapi_docs.py scripts/update_readme.py scripts/update_lock.py scripts/update_hook_revs.py +scripts/list_outdated_dependencies.py scripts/README.md .github/workflows/check_config_docs.yaml diff --git a/scripts/list_outdated_dependencies.py b/scripts/list_outdated_dependencies.py new file mode 100755 index 00000000..1aa15a13 --- /dev/null +++ b/scripts/list_outdated_dependencies.py @@ -0,0 +1,210 @@ +#!/usr/bin/env python3 + +# Copyright 2021 - 2023 Universität Tübingen, DKFZ, EMBL, and Universität zu Köln +# for the German Human Genome-Phenome Archive (GHGA) +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +"""Check capped dependencies for newer versions.""" +import sys +from collections.abc import Sequence +from pathlib import Path +from typing import Any, NamedTuple + +import httpx +from packaging.requirements import Requirement + +from script_utils import cli, deps, lock_deps + +REPO_ROOT_DIR = Path(__file__).parent.parent.resolve() +PYPROJECT_TOML_PATH = REPO_ROOT_DIR / "pyproject.toml" +DEV_DEPS_PATH = REPO_ROOT_DIR / "requirements-dev.in" +LOCK_FILE_PATH = REPO_ROOT_DIR / "requirements-dev.txt" + + +class OutdatedDep(NamedTuple): + """Encapsulates data of an outdated dependency""" + + name: str + specified_version: str + pypi_version: str + + +def get_main_deps_pyproject(modified_pyproject: dict[str, Any]) -> list[Requirement]: + """Get a list of the dependencies from pyproject.toml""" + + dependencies: list[str] = [] + if "dependencies" in modified_pyproject["project"]: + dependencies = modified_pyproject["project"]["dependencies"] + + return [Requirement(dependency) for dependency in dependencies] + + +def get_optional_deps_pyproject( + modified_pyproject: dict[str, Any] +) -> list[Requirement]: + """Get a list of the optional dependencies from pyproject.toml""" + + dependencies: list[str] = [] + + if "optional-dependencies" in modified_pyproject["project"]: + for optional_dependency_list in modified_pyproject["project"][ + "optional-dependencies" + ]: + dependencies.extend( + modified_pyproject["project"]["optional-dependencies"][ + optional_dependency_list + ] + ) + + return [Requirement(dependency) for dependency in dependencies] + + +def get_deps_dev() -> list[Requirement]: + """Get a list of raw dependency strings from requirements-dev.in""" + with open(DEV_DEPS_PATH, encoding="utf-8") as dev_deps: + dependencies = [ + line + for line in (line.strip() for line in dev_deps) + if line # skip empty lines + and not line.startswith("#") # skip comments + and "requirements-dev-common.in" not in line # skip inclusion line + ] + + return [Requirement(dependency) for dependency in dependencies] + + +def get_version_from_pypi(package_name: str, client: httpx.Client) -> str: + """Make a call to PyPI to get the version information about `package_name`.""" + try: + response = client.get(f"https://pypi.org/pypi/{package_name}/json") + body = response.json() + version = body["info"]["version"] + except (httpx.RequestError, KeyError): + cli.echo_failure(f"Unable to retrieve information for package '{package_name}'") + sys.exit(1) + + return version + + +def get_outdated_deps( + requirements: list[Requirement], strip: bool = False +) -> list[OutdatedDep]: + """Determine which packages have updates available outside of pinned ranges.""" + outdated: list[OutdatedDep] = [] + with httpx.Client(timeout=10) as client: + for requirement in requirements: + pypi_version = get_version_from_pypi(requirement.name, client) + + specified = str(requirement.specifier) + + # Strip the specifier symbols from the front of the string if desired + if strip: + specified = specified.lstrip("<=>!~") + + # append package name, specified version, and latest available version + if not requirement.specifier.contains(pypi_version): + outdated.append(OutdatedDep(requirement.name, specified, pypi_version)) + outdated.sort() + return outdated + + +def print_table( + rows: Sequence[tuple[str, ...]], + headers: tuple[str, ...], + delimiter: str = " | ", +): + """ + List outdated dependencies in a formatted table. + + Args: + `outdated`: A sequence of tuples containing strings. + `headers`: A tuple containing the header strings for the table columns. + """ + if rows and len(rows[0]) != len(headers): + raise RuntimeError("Number of headers doesn't match number of columns") + + header_lengths = [len(header) for header in headers] + + # Find the maximum length of each column + col_widths = [max(len(str(cell)) for cell in col) for col in zip(*rows)] + + # Create a row format based on the maximum column widths + row_format = delimiter.join( + f"{{:<{max(width, header_len)}}}" + for width, header_len in zip(col_widths, header_lengths) + ) + + print(" " + row_format.format(*headers)) + for dependency in rows: + print(" " + row_format.format(*dependency)) + + +def main(transitive: bool = False): + """Check capped dependencies for newer versions. + + Examine `pyproject.toml` and `requirements-dev.in` for capped dependencies. + Make a call to PyPI to see if any newer versions exist. + + Use `transitive` to show outdated transitive dependencies. + """ + modified_pyproject: dict[str, Any] = deps.get_modified_pyproject( + PYPROJECT_TOML_PATH + ) + main_dependencies = get_main_deps_pyproject(modified_pyproject) + optional_dependencies = get_optional_deps_pyproject(modified_pyproject) + dev_dependencies = get_deps_dev() + + outdated_main = get_outdated_deps(main_dependencies) + outdated_optional = get_outdated_deps(optional_dependencies) + outdated_dev = get_outdated_deps(dev_dependencies) + + found_outdated = any([outdated_main, outdated_optional, outdated_dev]) + transitive_headers = ("PACKAGE", "SPECIFIED", "AVAILABLE") + if outdated_main: + location = PYPROJECT_TOML_PATH.name + " - dependencies" + cli.echo_failure(f"Outdated dependencies from {location}:") + print_table(outdated_main, transitive_headers) + if outdated_optional: + location = PYPROJECT_TOML_PATH.name + " - optional-dependencies" + cli.echo_failure(f"Outdated dependencies from {location}:") + print_table(outdated_optional, transitive_headers) + if outdated_dev: + cli.echo_failure(f"Outdated dependencies from {DEV_DEPS_PATH.name}:") + print_table(outdated_dev, transitive_headers) + + if not found_outdated: + cli.echo_success("All top-level dependencies up to date.") + + if transitive: + top_level: set[str] = { + item.name for item in outdated_main + outdated_optional + outdated_dev + } + + print("\nRetrieving transitive dependency information...") + transitive_dependencies = lock_deps.get_lock_file_deps( + LOCK_FILE_PATH, exclude=top_level + ) + outdated_transitive = get_outdated_deps(transitive_dependencies, strip=True) + + if outdated_transitive: + transitive_headers = ("PACKAGE", "PINNED", "AVAILABLE") + + cli.echo_failure("Outdated transitive dependencies:") + print_table(outdated_transitive, transitive_headers) + else: + cli.echo_success("All transitive dependencies up to date.") + + +if __name__ == "__main__": + cli.run(main) diff --git a/scripts/script_utils/deps.py b/scripts/script_utils/deps.py new file mode 100644 index 00000000..5d65cf77 --- /dev/null +++ b/scripts/script_utils/deps.py @@ -0,0 +1,74 @@ +# Copyright 2021 - 2023 Universität Tübingen, DKFZ, EMBL, and Universität zu Köln +# for the German Human Genome-Phenome Archive (GHGA) +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +"""Contains utils for working with dependencies, lock files, etc.""" +from copy import deepcopy +from pathlib import Path +from typing import Any + +import stringcase +import tomli + + +def exclude_from_dependency_list(*, package_name: str, dependencies: list) -> list: + """Exclude the specified package from the provided dependency list.""" + + return [ + dependency + for dependency in dependencies + if not dependency.startswith(package_name) + ] + + +def remove_self_dependencies(pyproject: dict) -> dict: + """Filter out self dependencies (dependencies of the package on it self) from the + dependencies and optional-dependencies in the provided pyproject metadata.""" + + if "project" not in pyproject: + return pyproject + + modified_pyproject = deepcopy(pyproject) + + project_metadata = modified_pyproject["project"] + + package_name = stringcase.spinalcase(project_metadata.get("name")) + + if not package_name: + raise ValueError("The provided project metadata does not contain a name.") + + if "dependencies" in project_metadata: + project_metadata["dependencies"] = exclude_from_dependency_list( + package_name=package_name, dependencies=project_metadata["dependencies"] + ) + + if "optional-dependencies" in project_metadata: + for group in project_metadata["optional-dependencies"]: + project_metadata["optional-dependencies"][ + group + ] = exclude_from_dependency_list( + package_name=package_name, + dependencies=project_metadata["optional-dependencies"][group], + ) + + return modified_pyproject + + +def get_modified_pyproject(pyproject_toml_path: Path) -> dict[str, Any]: + """Get a copy of pyproject.toml with any self-referencing dependencies removed.""" + with open(pyproject_toml_path, "rb") as pyproject_toml: + pyproject = tomli.load(pyproject_toml) + + modified_pyproject = remove_self_dependencies(pyproject) + return modified_pyproject diff --git a/scripts/script_utils/lock_deps.py b/scripts/script_utils/lock_deps.py new file mode 100644 index 00000000..60d857e6 --- /dev/null +++ b/scripts/script_utils/lock_deps.py @@ -0,0 +1,46 @@ +# Copyright 2021 - 2023 Universität Tübingen, DKFZ, EMBL, and Universität zu Köln +# for the German Human Genome-Phenome Archive (GHGA) +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +"""Provides a function to get all dependencies from the lock file""" +import re +from pathlib import Path +from typing import Optional + +from packaging.requirements import Requirement + + +def get_lock_file_deps( + lock_file_path: Path, + exclude: Optional[set[str]] = None, +) -> list[Requirement]: + """Inspect the lock file to get the dependencies. + + Return a list of Requirements objects that contain the dependency info. + """ + dependency_pattern = re.compile(r"([^=\s]+==[^\s]*?)\s") + + # Get the set of dependencies from the provided lock file + with open(lock_file_path, encoding="utf-8") as lock_file: + lines = lock_file.readlines() + + dependencies: list[Requirement] = [] + for line in lines: + if match := re.match(dependency_pattern, line): + dependency_string = match.group(1) + requirement = Requirement(dependency_string) + if not exclude or requirement.name not in exclude: + dependencies.append(requirement) + + return dependencies diff --git a/scripts/update_hook_revs.py b/scripts/update_hook_revs.py index 931d0ce5..3d653f85 100755 --- a/scripts/update_hook_revs.py +++ b/scripts/update_hook_revs.py @@ -21,29 +21,22 @@ from functools import partial from pathlib import Path -from script_utils import cli +from packaging.requirements import Requirement + +from script_utils import cli, lock_deps REPO_ROOT_DIR = Path(__file__).parent.parent.resolve() PRE_COMMIT_CFG_PATH = REPO_ROOT_DIR / ".pre-commit-config.yaml" LOCK_FILE_PATH = REPO_ROOT_DIR / "requirements-dev.txt" -def inspect_lock() -> dict[str, str]: - """Inspect the lock file to get the resolved dependencies""" - dependency_pattern = re.compile(r"([^=\s]+)==([^\s]*?)\s") - dependencies = {} - - # Get the set of dependencies from the requirements-dev.txt lock file - with open(LOCK_FILE_PATH, encoding="utf-8") as lock_file: - lines = lock_file.readlines() - - for line in lines: - match = re.match(dependency_pattern, line) - if match: - package, version = match.groups() - dependencies[package] = version +def make_dependency_dict(requirements: list[Requirement]) -> dict[str, str]: + """Accept a list of Requirement objects and convert to dict""" + processed = { + req.name: str(req.specifier).removeprefix("==") for req in requirements + } - return dependencies + return processed def get_repl_value(match, dependencies: dict[str, str], outdated_hooks: list[str]): @@ -121,9 +114,10 @@ def main(check: bool = False): outdated. If running without `--check`, update `.pre-commit-config.yaml` as needed. """ - dependencies = inspect_lock() + dependencies: list[Requirement] = lock_deps.get_lock_file_deps(LOCK_FILE_PATH) + dependency_dict: dict[str, str] = make_dependency_dict(dependencies) config = get_config() - new_config, outdated_hooks = process_config(dependencies, config) + new_config, outdated_hooks = process_config(dependency_dict, config) if config != new_config: if check: diff --git a/scripts/update_lock.py b/scripts/update_lock.py index 8206cb62..825aade0 100755 --- a/scripts/update_lock.py +++ b/scripts/update_lock.py @@ -23,15 +23,12 @@ import os import re import subprocess -from copy import deepcopy from pathlib import Path from tempfile import TemporaryDirectory -import stringcase -import tomli import tomli_w -from script_utils import cli +from script_utils import cli, deps REPO_ROOT_DIR = Path(__file__).parent.parent.resolve() @@ -41,49 +38,6 @@ OUTPUT_DEV_LOCK_PATH = REPO_ROOT_DIR / "requirements-dev.txt" -def exclude_from_dependency_list(*, package_name: str, dependencies: list) -> list: - """Exclude the specified package from the provided dependency list.""" - - return [ - dependency - for dependency in dependencies - if not dependency.startswith(package_name) - ] - - -def remove_self_dependencies(pyproject: dict) -> dict: - """Filter out self dependencies (dependencies of the package on it self) from the - dependencies and optional-dependencies in the provided pyproject metadata.""" - - if "project" not in pyproject: - return pyproject - - modified_pyproject = deepcopy(pyproject) - - project_metadata = modified_pyproject["project"] - - package_name = stringcase.spinalcase(project_metadata.get("name")) - - if not package_name: - raise ValueError("The provided project metadata does not contain a name.") - - if "dependencies" in project_metadata: - project_metadata["dependencies"] = exclude_from_dependency_list( - package_name=package_name, dependencies=project_metadata["dependencies"] - ) - - if "optional-dependencies" in project_metadata: - for group in project_metadata["optional-dependencies"]: - project_metadata["optional-dependencies"][ - group - ] = exclude_from_dependency_list( - package_name=package_name, - dependencies=project_metadata["optional-dependencies"][group], - ) - - return modified_pyproject - - def fix_temp_dir_comments(file_path: Path): """Fix the temp_dir comments so they don't cause a noisy diff @@ -203,10 +157,7 @@ def main(upgrade: bool = False, check: bool = False): if check: ensure_lock_files_exist() - with open(PYPROJECT_TOML_PATH, "rb") as pyproject_toml: - pyproject = tomli.load(pyproject_toml) - - modified_pyproject = remove_self_dependencies(pyproject) + modified_pyproject = deps.get_modified_pyproject(PYPROJECT_TOML_PATH) extras = ( "optional-dependencies" in modified_pyproject["project"]